From c22d80b69ab5943a8c401253dd061fb5c3698626 Mon Sep 17 00:00:00 2001 From: Max Date: Thu, 8 Oct 2026 14:07:24 +0200 Subject: [PATCH 01/12] Benchmark script for VlasovAmpereOneSpecies and VlasovMaxwellOneSpecies profiling/benchmarks/vlasov_benchmark.py runs a fixed number of time steps of a standard problem on the NumPy or CuPy backend and reports setup time, time per step and per propagator, cost per marker and step, and the host/device transfers of the time loop (optionally as JSON). - Cases: weak Landau damping (both models, the verification tests' setup) and the Weibel instability (Vlasov-Maxwell, Kraus et al. 2017: anisotropic Maxwellian, B3 seed). - Sizes smoke / standard / large (8 cells x 50 ppc, 32 x 1000, 128 x 20000), all overridable. - Every timed region ends with xp.synchronize(); times are the maximum over MPI ranks. - On the fake CuPy the CUDA launches are emulated, as a no-GPU check of the device code path. Co-Authored-By: Claude Opus 5.5 --- profiling/benchmarks/vlasov_benchmark.py | 421 +++++++++++++++++++++++ 1 file changed, 421 insertions(+) create mode 100644 profiling/benchmarks/vlasov_benchmark.py diff --git a/profiling/benchmarks/vlasov_benchmark.py b/profiling/benchmarks/vlasov_benchmark.py new file mode 100644 index 000000000..19319927d --- /dev/null +++ b/profiling/benchmarks/vlasov_benchmark.py @@ -0,0 +1,421 @@ +"""Benchmark of the Vlasov-Ampère and Vlasov-Maxwell models on standard test problems. + +Runs a fixed number of time steps of :class:`~struphy.models.VlasovAmpereOneSpecies` or +:class:`~struphy.models.VlasovMaxwellOneSpecies` on the NumPy (CPU) or CuPy (GPU) backend and reports the setup +time, the time per step, the time per propagator, the cost per marker and the host/device transfers in the time loop. + +Test problems (the physics follows the verification tests in ``src/struphy/models/tests/verification``): + +``landau`` + Weak Landau damping (both models). Periodic box of length 4 pi (k = 0.5), Maxwellian with a density + perturbation of 1e-3 cos(k x), control-variate weights, cubic splines in x. Damping rate -0.1533. +``weibel`` + Weibel instability (Vlasov-Maxwell only), the setup of Kraus et al., J. Plasma Phys. 83 (2017), Sec. 6.2: + box of length 2 pi / 1.25, anisotropic Maxwellian (vth1 = vth3 = 0.02 / sqrt(2), vth2 = sqrt(12) vth1), + seed field B3 = -1e-4 cos(k x), no density perturbation. Growth rate 0.02784. + +All runs use ``alpha = 1`` and ``epsilon = -1`` (electrons, time in units of the inverse plasma frequency, +velocities in units of c), as the verification tests do. + +Sizes (``--size``; each value can be overridden on the command line): + +======== ================= ===== ===== ================================================== +size cells (x, y, z) ppc steps purpose +======== ================= ===== ===== ================================================== +smoke 8, 1, 1 50 3 check that the benchmark runs (seconds) +standard 32, 1, 1 1000 20 the verification tests' resolution (32k markers) +large 128, 1, 1 20000 20 GPU-sized (2.56M markers) +======== ================= ===== ===== ================================================== + +Examples:: + + python profiling/benchmarks/vlasov_benchmark.py --model VlasovAmpereOneSpecies --case landau --size standard + CUNUMPY_BACKEND=cupy python profiling/benchmarks/vlasov_benchmark.py --model VlasovMaxwellOneSpecies \\ + --case weibel --size large --json weibel_gpu.json + mpirun -n 2 python profiling/benchmarks/vlasov_benchmark.py --size standard + +The backend is cunumpy's active backend (``CUNUMPY_BACKEND=numpy|cupy``, or ``--backend``). Every timed region +ends with ``xp.synchronize()``, so GPU times include the kernels and not only their launch. Per-propagator timing +adds one synchronization per propagator and step; ``--no-propagator-timing`` measures the step without them. +Times are the maximum over MPI ranks. Sorting boxes are off by default, since ``SortingBoxes`` does not work on +the CuPy backend yet (struphy-hub/struphy#726); ``--sorting-boxes N`` turns them on. + +Without a GPU, ``CUNUMPY_FAKE_CUPY=1 ... --backend cupy --size smoke`` runs the CuPy code path on cunumpy's fake +CuPy with every CUDA launch emulated on the CPU: a check that the models run on the device backend, not a timing. + +CPU reference (NumPy backend, ``--size standard``, 1 rank, Apple M-series laptop, 8 October 2026): VlasovAmpere +Landau 67 ms per step, VlasovMaxwell Landau 82 ms, VlasovMaxwell Weibel 86 ms (2.1 to 2.7 microseconds per marker +and step; ``VlasovAmpereCoupling`` takes 68 to 86 % of a step). +""" + +import argparse +import contextlib +import json +import math +import os +import platform +import statistics +import tempfile +import time +from dataclasses import asdict, dataclass, field + +import cunumpy as xp +from cunumpy.kernel_testing import fake_cupy_active +from maybempi import MPI + +from struphy.models.tests.gpu_e2e import device_context + +SIZES = { + "smoke": {"num_elements": (8, 1, 1), "ppc": 50, "steps": 3}, + "standard": {"num_elements": (32, 1, 1), "ppc": 1000, "steps": 20}, + "large": {"num_elements": (128, 1, 1), "ppc": 20000, "steps": 20}, +} + +CASES = { + # box length, time step, expected rate (damping < 0, growth > 0) of the electric (landau) or magnetic (weibel) + # field energy amplitude + "landau": { + "r1": 4 * math.pi, + "dt": 0.05, + "rate": -0.1533, + "models": ("VlasovAmpereOneSpecies", "VlasovMaxwellOneSpecies"), + }, + "weibel": {"r1": 2 * math.pi / 1.25, "dt": 0.05, "rate": 0.02784, "models": ("VlasovMaxwellOneSpecies",)}, +} + + +@dataclass +class Result: + """What one benchmark run measured. Times in seconds, maximum over MPI ranks.""" + + model: str + case: str + backend: str + mpi_ranks: int + num_elements: tuple + degree: tuple + ppc: int + n_markers: int + steps: int + dt: float + setup: float = 0.0 # Simulation.allocate (FEEC operators, markers, initial Poisson solve) + compile_cuda: float = 0.0 # Simulation.compile_cuda_kernels (0 on NumPy) + step_times: list = field(default_factory=list) # model.integrate, one per step + diagnostics: float = 0.0 # scalars, saved markers, binned distribution functions (all steps) + output: float = 0.0 # DataContainer.save_data (all steps) + total_run: float = 0.0 # Simulation.run + propagators: dict = field(default_factory=dict) # class name -> total time over all steps + transfers_in_loop: dict = field(default_factory=dict) # cunumpy transfer counts during the time loop + scalars_end: dict = field(default_factory=dict) # name -> value after the last step + host: str = field(default_factory=platform.node) + + @property + def step_median(self) -> float: + # the first step includes one-time work (CUDA module loads, allocations of work arrays) + steady = self.step_times[1:] or self.step_times + return statistics.median(steady) + + @property + def ns_per_marker_step(self) -> float: + return 1e9 * self.step_median / max(self.n_markers, 1) + + +def parse_args(): + parser = argparse.ArgumentParser( + description=__doc__.split("\n\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument("--model", default="VlasovAmpereOneSpecies", choices=CASES["landau"]["models"]) + parser.add_argument("--case", default="landau", choices=tuple(CASES)) + parser.add_argument("--size", default="standard", choices=tuple(SIZES)) + parser.add_argument( + "--num-elements", type=int, nargs=3, metavar=("NX", "NY", "NZ"), help="override the size's grid" + ) + parser.add_argument("--degree", type=int, nargs=3, default=(3, 1, 1), metavar=("PX", "PY", "PZ")) + parser.add_argument("--ppc", type=int, help="markers per cell (overrides the size)") + parser.add_argument("--steps", type=int, help="number of time steps (overrides the size)") + parser.add_argument("--dt", type=float, help="time step (overrides the case)") + parser.add_argument("--backend", choices=("numpy", "cupy"), help="cunumpy backend (default: the active one)") + parser.add_argument("--sorting-boxes", type=int, default=0, metavar="N", help="sorting boxes in x (default: off)") + parser.add_argument("--save-step", type=int, default=1, help="compute diagnostics and save every N steps") + parser.add_argument("--no-propagator-timing", action="store_true", help="time only whole steps") + parser.add_argument("--out", help="output folder (default: a temporary folder, removed afterwards)") + parser.add_argument("--json", help="also write the result to this JSON file (rank 0)") + args = parser.parse_args() + if args.model not in CASES[args.case]["models"]: + parser.error(f"case {args.case!r} is only set up for {', '.join(CASES[args.case]['models'])}") + return args + + +def make_simulation(args, out_folders: str): + """The model, markers, initial condition and options of the chosen case and size.""" + from struphy import ( + BoundaryParameters, + DerhamOptions, + EnvironmentOptions, + LoadingParameters, + SavingParameters, + Simulation, + SortingParameters, + Time, + WeightsParameters, + domains, + grids, + maxwellians, + perturbations, + ) + from struphy.models import VlasovAmpereOneSpecies, VlasovMaxwellOneSpecies + + case = CASES[args.case] + size = SIZES[args.size] + num_elements = tuple(args.num_elements or size["num_elements"]) + ppc = args.ppc or size["ppc"] + steps = args.steps or size["steps"] + dt = args.dt or case["dt"] + + if args.model == "VlasovAmpereOneSpecies": + model = VlasovAmpereOneSpecies(alpha=1.0, epsilon=-1.0, with_B0=False) + else: + model = VlasovMaxwellOneSpecies(alpha=1.0, epsilon=-1.0) + + env = EnvironmentOptions( + out_folders=out_folders, + sim_folder=f"{args.model}_{args.case}", + save_step=args.save_step, + save_restart=False, # restart data would dominate the run time for many markers + ) + time_opts = Time(dt=dt, Tend=round(steps * dt, 14)) + + if args.sorting_boxes: + sorting_params = SortingParameters(boxes_per_dim=(args.sorting_boxes, 1, 1), do_sort=True) + else: + sorting_params = SortingParameters() + + model.kinetic_ions.set_markers( + loading_params=LoadingParameters(ppc=ppc, seed=1234), + weights_params=WeightsParameters(control_variate=True), + boundary_params=BoundaryParameters(), + sorting_params=sorting_params, + saving_params=SavingParameters(n_markers=0), # benchmark the physics, not the marker output + bufsize=0.4, + ) + + for name, prop in model.propagators.__dict__.items(): + prop.options = prop.Options() + model.initial_poisson.options = model.initial_poisson.Options(stab_mat="M0") + + background = maxwellians.Maxwellian3D(n=(1.0, None)) + if args.case == "landau": + model.kinetic_ions.var.add_background(background) + perturbation = perturbations.ModesCos(ls=(1,), amps=(1e-3,)) + model.kinetic_ions.var.add_initial_condition(maxwellians.Maxwellian3D(n=(1.0, perturbation))) + else: + vth1 = 0.02 / math.sqrt(2.0) + anisotropic = maxwellians.Maxwellian3D( + n=(1.0, None), vth1=(vth1, None), vth2=(math.sqrt(12.0) * vth1, None), vth3=(vth1, None) + ) + model.kinetic_ions.var.add_background(anisotropic) + # B3 = -1e-4 cos(k x) in physical coordinates: kx = 2 pi l / Lx, so Lx must be the box length + seed = perturbations.ModesCos(ls=(1,), amps=(-1e-4,), Lx=case["r1"], given_in_basis="physical", comp=2) + model.em_fields.b_field.add_perturbation(seed) + + sim = Simulation( + model=model, + env=env, + time_opts=time_opts, + domain=domains.Cuboid(r1=case["r1"]), + grid=grids.TensorProductGrid(num_elements=num_elements), + derham_opts=DerhamOptions(degree=tuple(args.degree)), + ) + result = Result( + model=args.model, + case=args.case, + backend=xp.get_backend(), + mpi_ranks=MPI.COMM_WORLD.Get_size(), + num_elements=num_elements, + degree=tuple(args.degree), + ppc=ppc, + n_markers=ppc * math.prod(num_elements), + steps=steps, + dt=dt, + ) + return sim, result + + +class TimedPropagator: + """Stands in for a propagator in ``model.prop_list`` and adds its synchronized run time to ``totals``.""" + + def __init__(self, propagator, totals: dict): + self._propagator = propagator + self._totals = totals + self._name = type(propagator).__name__ + + def __call__(self, dt): + xp.synchronize() + t0 = time.perf_counter() + self._propagator(dt) + xp.synchronize() + self._totals[self._name] = self._totals.get(self._name, 0.0) + time.perf_counter() - t0 + + def __getattr__(self, name): + return getattr(self._propagator, name) + + +def instrument(sim, result: Result, timed_propagators: bool): + """Wrap the phases of ``sim.run()`` with synchronized timers and count the transfers of the time loop.""" + from cunumpy.profiling import count_transfers + + from struphy.io.output_handling import DataContainer + + model = sim.model + in_loop = [False] + # the transfer counter of the time loop: count_transfers() is entered when the loop starts + loop = {"context": count_transfers(), "counter": None} + + def timed(fn, record): + def wrapper(*args, **kwargs): + xp.synchronize() + t0 = time.perf_counter() + out = fn(*args, **kwargs) + xp.synchronize() + record(time.perf_counter() - t0) + return out + + return wrapper + + def add(name): + def record(t): + if in_loop[0]: + setattr(result, name, getattr(result, name) + t) + + return record + + def after_allocate(t): + result.setup = t + if timed_propagators: + model._prop_list = [TimedPropagator(p, result.propagators) for p in model.prop_list] + + sim.allocate = timed(sim.allocate, after_allocate) + sim.compile_cuda_kernels = timed(sim.compile_cuda_kernels, lambda t: setattr(result, "compile_cuda", t)) + + original_init_datasets = sim._initialize_hdf5_datasets + + def init_datasets(*args, **kwargs): + out = original_init_datasets(*args, **kwargs) + in_loop[0] = True # the time loop starts after this, the last step of the setup + loop["counter"] = loop["context"].__enter__() + return out + + sim._initialize_hdf5_datasets = init_datasets + model.integrate = timed(model.integrate, lambda t: result.step_times.append(t) if in_loop[0] else None) + for name in ("update_scalar_quantities", "update_markers_to_be_saved", "update_distr_functions"): + setattr(model, name, timed(getattr(model, name), add("diagnostics"))) + + original_save_data = DataContainer.save_data + record_output = add("output") + + def save_data(self, *args, **kwargs): + xp.synchronize() + t0 = time.perf_counter() + out = original_save_data(self, *args, **kwargs) + record_output(time.perf_counter() - t0) + return out + + DataContainer.save_data = save_data + return loop, lambda: setattr(DataContainer, "save_data", original_save_data) + + +def reduce_max(value: float) -> float: + comm = MPI.COMM_WORLD + return comm.allreduce(value, op=MPI.MAX) if comm.Get_size() > 1 else value + + +def run(args) -> Result: + comm = MPI.COMM_WORLD + out = args.out + tmp = None + if out is None: + tmp = tempfile.TemporaryDirectory(prefix="struphy_vlasov_benchmark_") if comm.Get_rank() == 0 else None + out = comm.bcast(tmp.name if tmp else None, root=0) + + sim, result = make_simulation(args, out) + loop, restore = instrument(sim, result, timed_propagators=not args.no_propagator_timing) + try: + # on cunumpy's fake CuPy (no GPU) the CUDA launches are emulated on the CPU: a smoke test, not a timing + with device_context() if xp.get_backend() == "cupy" else contextlib.nullcontext(): + xp.synchronize() + t0 = time.perf_counter() + sim.run() + xp.synchronize() + result.total_run = time.perf_counter() - t0 + finally: + if loop["counter"] is not None: + loop["context"].__exit__(None, None, None) + restore() + + counter = loop["counter"] + kinds = ("to_host", "to_device", "kernel_conversion", "fallback", "device_copy", "sync") + result.transfers_in_loop = {kind: counter.count(kind) for kind in kinds} + result.transfers_in_loop["bytes_to_host"] = counter.bytes_to_host + for name, scalar in sim.model.scalars.dct.items(): + value = float(xp.to_numpy(scalar.value)[0]) if hasattr(scalar, "value") else float("nan") + result.scalars_end[name] = value + + for name in ("setup", "compile_cuda", "diagnostics", "output", "total_run"): + setattr(result, name, reduce_max(getattr(result, name))) + result.step_times = [reduce_max(t) for t in result.step_times] + result.propagators = {k: reduce_max(v) for k, v in sorted(result.propagators.items())} + + comm.Barrier() + if tmp is not None: + tmp.cleanup() + return result + + +def report(result: Result) -> str: + warning = ["!! cunumpy's fake CuPy: CUDA launches emulated on the CPU, the times below are not GPU timings", ""] + lines = (warning if result.backend == "cupy" and fake_cupy_active() else []) + [ + f"{result.model} | {result.case} | backend {result.backend} | {result.mpi_ranks} MPI rank(s) | host {result.host}", + f"grid {result.num_elements}, degree {result.degree}, {result.ppc} markers per cell = {result.n_markers:,} markers, " + f"{result.steps} steps of dt = {result.dt}", + "", + ] + lines += [ + f" setup (allocate + initial Poisson) {result.setup:10.3f} s", + f" CUDA kernel compilation {result.compile_cuda:10.3f} s", + f" time loop: model.integrate {sum(result.step_times):10.3f} s", + f" diagnostics {result.diagnostics:10.3f} s", + f" output {result.output:10.3f} s", + f" Simulation.run total {result.total_run:10.3f} s", + "", + f" step: median {1e3 * result.step_median:.3f} ms (first step {1e3 * result.step_times[0]:.3f} ms), " + f"{result.ns_per_marker_step:.1f} ns per marker and step", + ] + if result.propagators: + lines.append(" per propagator (all steps):") + total = sum(result.propagators.values()) or 1.0 + for name, t in result.propagators.items(): + lines.append(f" {name:<28} {t:10.3f} s ({100 * t / total:5.1f} %)") + lines.append( + " host/device transfers in the time loop (through cunumpy): " + + ", ".join(f"{k}={v}" for k, v in result.transfers_in_loop.items()) + ) + return "\n".join(lines) + + +def main(): + args = parse_args() + if args.backend: + xp.set_backend(args.backend) + result = run(args) + if MPI.COMM_WORLD.Get_rank() == 0: + print(report(result)) + if args.json: + data = asdict(result) + data.update(step_median=result.step_median, ns_per_marker_step=result.ns_per_marker_step) + with open(args.json, "w") as f: + json.dump(data, f, indent=2) + print(f"\nwrote {os.path.abspath(args.json)}") + + +if __name__ == "__main__": + main() From 665cca21febe511974397dfa51f4c029ff88aaf4 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 13 Oct 2026 09:55:07 +0200 Subject: [PATCH 02/12] Rewrote vlasov_benchmark.py using scope-profiler --- profiling/benchmarks/vlasov_benchmark.py | 263 +++++++++++------------ 1 file changed, 128 insertions(+), 135 deletions(-) diff --git a/profiling/benchmarks/vlasov_benchmark.py b/profiling/benchmarks/vlasov_benchmark.py index 19319927d..3762e5713 100644 --- a/profiling/benchmarks/vlasov_benchmark.py +++ b/profiling/benchmarks/vlasov_benchmark.py @@ -2,7 +2,17 @@ Runs a fixed number of time steps of :class:`~struphy.models.VlasovAmpereOneSpecies` or :class:`~struphy.models.VlasovMaxwellOneSpecies` on the NumPy (CPU) or CuPy (GPU) backend and reports the setup -time, the time per step, the time per propagator, the cost per marker and the host/device transfers in the time loop. +time, the time per step, the time per propagator and per kernel, and the cost per marker. + +All timing comes from scope-profiler: the run activates the profiling session of :meth:`Simulation.run`, so the +regions that struphy already defines (``setup: allocate``, ``model.integrate``, ``prop: ``, +``kernel: ``, ...) are recorded, and the benchmark only reads them back. On the CuPy backend the session +records CUDA-event device time for every region in addition to the CPU time (``use_gpu_timing``, backend +``cupy``), and by default synchronizes the device when a region ends (``gpu_sync_on_exit``), so the CPU time of a +region includes its kernels and not only their launch; ``--no-gpu-sync`` records the enqueue-side CPU time instead. +The profile is written to an HDF5 file (``--profile``) for the full scope-profiler tool chain:: + + scope-profiler inspect vlasov_benchmark_VlasovAmpereOneSpecies_landau_cupy.h5 Test problems (the physics follows the verification tests in ``src/struphy/models/tests/verification``): @@ -34,14 +44,13 @@ --case weibel --size large --json weibel_gpu.json mpirun -n 2 python profiling/benchmarks/vlasov_benchmark.py --size standard -The backend is cunumpy's active backend (``CUNUMPY_BACKEND=numpy|cupy``, or ``--backend``). Every timed region -ends with ``xp.synchronize()``, so GPU times include the kernels and not only their launch. Per-propagator timing -adds one synchronization per propagator and step; ``--no-propagator-timing`` measures the step without them. -Times are the maximum over MPI ranks. Sorting boxes are off by default, since ``SortingBoxes`` does not work on -the CuPy backend yet (struphy-hub/struphy#726); ``--sorting-boxes N`` turns them on. +The backend is cunumpy's active backend (``CUNUMPY_BACKEND=numpy|cupy``, or ``--backend``). Times are the maximum +over MPI ranks. Sorting boxes are off by default, since ``SortingBoxes`` does not work on the CuPy backend yet +(struphy-hub/struphy#726); ``--sorting-boxes N`` turns them on. Without a GPU, ``CUNUMPY_FAKE_CUPY=1 ... --backend cupy --size smoke`` runs the CuPy code path on cunumpy's fake CuPy with every CUDA launch emulated on the CPU: a check that the models run on the device backend, not a timing. +The fake CuPy has no CUDA events, so GPU timing is off in that mode. CPU reference (NumPy backend, ``--size standard``, 1 rank, Apple M-series laptop, 8 October 2026): VlasovAmpere Landau 67 ms per step, VlasovMaxwell Landau 82 ms, VlasovMaxwell Weibel 86 ms (2.1 to 2.7 microseconds per marker @@ -56,12 +65,13 @@ import platform import statistics import tempfile -import time from dataclasses import asdict, dataclass, field import cunumpy as xp +import numpy as np from cunumpy.kernel_testing import fake_cupy_active from maybempi import MPI +from scope_profiler import ProfilingOptions, ProfilingResults, read_h5 from struphy.models.tests.gpu_e2e import device_context @@ -83,6 +93,14 @@ "weibel": {"r1": 2 * math.pi / 1.25, "dt": 0.05, "rate": 0.02784, "models": ("VlasovMaxwellOneSpecies",)}, } +# scope-profiler regions of Simulation.run that the report summarizes +SESSION_REGION = "scope_profiler.session" +SETUP_REGION = "setup: allocate" +COMPILE_REGION = "setup: compile cuda kernels" +STEP_REGION = "model.integrate" +PROPAGATOR_PREFIX = "prop: " +KERNEL_PREFIX = "kernel: " + @dataclass class Result: @@ -98,14 +116,15 @@ class Result: n_markers: int steps: int dt: float + profile: str # the scope-profiler HDF5 file of the run + gpu_timing: bool # CUDA-event device times recorded per region + gpu_sync_on_exit: bool # device synchronized at every region end (CPU times include the kernels) setup: float = 0.0 # Simulation.allocate (FEEC operators, markers, initial Poisson solve) compile_cuda: float = 0.0 # Simulation.compile_cuda_kernels (0 on NumPy) step_times: list = field(default_factory=list) # model.integrate, one per step - diagnostics: float = 0.0 # scalars, saved markers, binned distribution functions (all steps) - output: float = 0.0 # DataContainer.save_data (all steps) - total_run: float = 0.0 # Simulation.run - propagators: dict = field(default_factory=dict) # class name -> total time over all steps - transfers_in_loop: dict = field(default_factory=dict) # cunumpy transfer counts during the time loop + total_run: float = 0.0 # the scope-profiler session, i.e. Simulation.run + propagators: dict = field(default_factory=dict) # class name -> {"cpu": total, "gpu": total or None} + kernels: dict = field(default_factory=dict) # kernel name -> {"cpu": total, "gpu": total or None} scalars_end: dict = field(default_factory=dict) # name -> value after the last step host: str = field(default_factory=platform.node) @@ -137,7 +156,15 @@ def parse_args(): parser.add_argument("--backend", choices=("numpy", "cupy"), help="cunumpy backend (default: the active one)") parser.add_argument("--sorting-boxes", type=int, default=0, metavar="N", help="sorting boxes in x (default: off)") parser.add_argument("--save-step", type=int, default=1, help="compute diagnostics and save every N steps") - parser.add_argument("--no-propagator-timing", action="store_true", help="time only whole steps") + parser.add_argument( + "--no-gpu-sync", + action="store_true", + help="do not synchronize the device at region ends (CPU times are then launch times; GPU times unaffected)", + ) + parser.add_argument( + "--profile", + help="scope-profiler HDF5 file of the run (default: vlasov_benchmark___.h5)", + ) parser.add_argument("--out", help="output folder (default: a temporary folder, removed afterwards)") parser.add_argument("--json", help="also write the result to this JSON file (rank 0)") args = parser.parse_args() @@ -146,7 +173,20 @@ def parse_args(): return args -def make_simulation(args, out_folders: str): +def profiling_options(args, profile_path: str) -> ProfilingOptions: + """The scope-profiler settings of the run: GPU timing through CuPy's CUDA events on the CuPy backend.""" + # the fake CuPy (no GPU) has CUDA events without elapsed time, so device timing is only possible on a GPU + gpu_timing = xp.get_backend() == "cupy" and not fake_cupy_active() + return ProfilingOptions( + file_path=profile_path, + label=f"{args.model} {args.case} {xp.get_backend()}", + use_gpu_timing=gpu_timing, + gpu_timing_backend="cupy", + gpu_sync_on_exit=gpu_timing and not args.no_gpu_sync, + ) + + +def make_simulation(args, out_folders: str, profile_path: str): """The model, markers, initial condition and options of the chosen case and size.""" from struphy import ( BoundaryParameters, @@ -218,6 +258,7 @@ def make_simulation(args, out_folders: str): seed = perturbations.ModesCos(ls=(1,), amps=(-1e-4,), Lx=case["r1"], given_in_basis="physical", comp=2) model.em_fields.b_field.add_perturbation(seed) + opts = profiling_options(args, profile_path) sim = Simulation( model=model, env=env, @@ -225,6 +266,7 @@ def make_simulation(args, out_folders: str): domain=domains.Cuboid(r1=case["r1"]), grid=grids.TensorProductGrid(num_elements=num_elements), derham_opts=DerhamOptions(degree=tuple(args.degree)), + profiling_opts=opts, ) result = Result( model=args.model, @@ -237,96 +279,47 @@ def make_simulation(args, out_folders: str): n_markers=ppc * math.prod(num_elements), steps=steps, dt=dt, + profile=os.path.abspath(profile_path), + gpu_timing=opts.use_gpu_timing, + gpu_sync_on_exit=opts.gpu_sync_on_exit, ) return sim, result -class TimedPropagator: - """Stands in for a propagator in ``model.prop_list`` and adds its synchronized run time to ``totals``.""" - - def __init__(self, propagator, totals: dict): - self._propagator = propagator - self._totals = totals - self._name = type(propagator).__name__ - - def __call__(self, dt): - xp.synchronize() - t0 = time.perf_counter() - self._propagator(dt) - xp.synchronize() - self._totals[self._name] = self._totals.get(self._name, 0.0) + time.perf_counter() - t0 - - def __getattr__(self, name): - return getattr(self._propagator, name) - - -def instrument(sim, result: Result, timed_propagators: bool): - """Wrap the phases of ``sim.run()`` with synchronized timers and count the transfers of the time loop.""" - from cunumpy.profiling import count_transfers - - from struphy.io.output_handling import DataContainer - - model = sim.model - in_loop = [False] - # the transfer counter of the time loop: count_transfers() is entered when the loop starts - loop = {"context": count_transfers(), "counter": None} - - def timed(fn, record): - def wrapper(*args, **kwargs): - xp.synchronize() - t0 = time.perf_counter() - out = fn(*args, **kwargs) - xp.synchronize() - record(time.perf_counter() - t0) - return out - - return wrapper - - def add(name): - def record(t): - if in_loop[0]: - setattr(result, name, getattr(result, name) + t) - - return record - - def after_allocate(t): - result.setup = t - if timed_propagators: - model._prop_list = [TimedPropagator(p, result.propagators) for p in model.prop_list] - - sim.allocate = timed(sim.allocate, after_allocate) - sim.compile_cuda_kernels = timed(sim.compile_cuda_kernels, lambda t: setattr(result, "compile_cuda", t)) - - original_init_datasets = sim._initialize_hdf5_datasets - - def init_datasets(*args, **kwargs): - out = original_init_datasets(*args, **kwargs) - in_loop[0] = True # the time loop starts after this, the last step of the setup - loop["counter"] = loop["context"].__enter__() - return out - - sim._initialize_hdf5_datasets = init_datasets - model.integrate = timed(model.integrate, lambda t: result.step_times.append(t) if in_loop[0] else None) - for name in ("update_scalar_quantities", "update_markers_to_be_saved", "update_distr_functions"): - setattr(model, name, timed(getattr(model, name), add("diagnostics"))) - - original_save_data = DataContainer.save_data - record_output = add("output") - - def save_data(self, *args, **kwargs): - xp.synchronize() - t0 = time.perf_counter() - out = original_save_data(self, *args, **kwargs) - record_output(time.perf_counter() - t0) - return out - - DataContainer.save_data = save_data - return loop, lambda: setattr(DataContainer, "save_data", original_save_data) - - -def reduce_max(value: float) -> float: - comm = MPI.COMM_WORLD - return comm.allreduce(value, op=MPI.MAX) if comm.Get_size() > 1 else value +def max_over_ranks_per_call(region) -> list: + """The duration of each call of a region, the maximum over the ranks (every rank makes the same calls).""" + per_rank = [region[rank].durations for rank in region.ranks] + n_calls = min(len(durations) for durations in per_rank) + return np.max([durations[:n_calls] for durations in per_rank], axis=0).tolist() + + +def totals_over_ranks(region) -> dict: + """The total CPU time of a region and, with GPU timing, its total device time; the maximum over the ranks.""" + cpu = max(region.total_durations().values()) + gpu = None + if region.has_gpu_timing: + gpu = max(region[rank].gpu_total_duration for rank in region.ranks if region[rank].has_gpu_timing) + return {"cpu": float(cpu), "gpu": None if gpu is None else float(gpu)} + + +def collect(results: ProfilingResults, result: Result) -> None: + """Fill ``result`` from the scope-profiler regions of the run.""" + if SETUP_REGION in results: + result.setup = results[SETUP_REGION].max_duration + if COMPILE_REGION in results: + result.compile_cuda = results[COMPILE_REGION].max_duration + if SESSION_REGION in results: + result.total_run = results[SESSION_REGION].max_duration + elif results.total_time is not None: + result.total_run = results.total_time + result.step_times = max_over_ranks_per_call(results[STEP_REGION]) + for region in results: + if region.name.startswith(PROPAGATOR_PREFIX): + result.propagators[region.name[len(PROPAGATOR_PREFIX) :]] = totals_over_ranks(region) + elif region.name.startswith(KERNEL_PREFIX): + result.kernels[region.name[len(KERNEL_PREFIX) :]] = totals_over_ranks(region) + result.propagators = dict(sorted(result.propagators.items(), key=lambda item: -item[1]["cpu"])) + result.kernels = dict(sorted(result.kernels.items(), key=lambda item: -item[1]["cpu"])) def run(args) -> Result: @@ -336,34 +329,20 @@ def run(args) -> Result: if out is None: tmp = tempfile.TemporaryDirectory(prefix="struphy_vlasov_benchmark_") if comm.Get_rank() == 0 else None out = comm.bcast(tmp.name if tmp else None, root=0) + profile_path = args.profile or f"vlasov_benchmark_{args.model}_{args.case}_{xp.get_backend()}.h5" + + sim, result = make_simulation(args, out, profile_path) + # on cunumpy's fake CuPy (no GPU) the CUDA launches are emulated on the CPU: a smoke test, not a timing + with device_context() if xp.get_backend() == "cupy" else contextlib.nullcontext(): + sim.run(profiling_activated=True) - sim, result = make_simulation(args, out) - loop, restore = instrument(sim, result, timed_propagators=not args.no_propagator_timing) - try: - # on cunumpy's fake CuPy (no GPU) the CUDA launches are emulated on the CPU: a smoke test, not a timing - with device_context() if xp.get_backend() == "cupy" else contextlib.nullcontext(): - xp.synchronize() - t0 = time.perf_counter() - sim.run() - xp.synchronize() - result.total_run = time.perf_counter() - t0 - finally: - if loop["counter"] is not None: - loop["context"].__exit__(None, None, None) - restore() - - counter = loop["counter"] - kinds = ("to_host", "to_device", "kernel_conversion", "fallback", "device_copy", "sync") - result.transfers_in_loop = {kind: counter.count(kind) for kind in kinds} - result.transfers_in_loop["bytes_to_host"] = counter.bytes_to_host for name, scalar in sim.model.scalars.dct.items(): value = float(xp.to_numpy(scalar.value)[0]) if hasattr(scalar, "value") else float("nan") result.scalars_end[name] = value - for name in ("setup", "compile_cuda", "diagnostics", "output", "total_run"): - setattr(result, name, reduce_max(getattr(result, name))) - result.step_times = [reduce_max(t) for t in result.step_times] - result.propagators = {k: reduce_max(v) for k, v in sorted(result.propagators.items())} + # scope-profiler merges the ranks into one file at the end of the session + if comm.Get_rank() == 0: + collect(read_h5(profile_path), result) comm.Barrier() if tmp is not None: @@ -371,34 +350,48 @@ def run(args) -> Result: return result +def _time_table(title: str, totals: dict, gpu_timing: bool) -> list: + """Lines of a table with the CPU and (if recorded) GPU total of every entry and its share of the CPU sum.""" + if not totals: + return [] + lines = [f" {title}, maximum over ranks):"] + width = max(len(name) for name in totals) + cpu_sum = sum(entry["cpu"] for entry in totals.values()) or 1.0 + header = f" {'':<{width}} {'CPU [s]':>10} {'share':>7}" + if gpu_timing: + header += f" {'GPU [s]':>10}" + lines.append(header) + for name, entry in totals.items(): + line = f" {name:<{width}} {entry['cpu']:>10.3f} {100 * entry['cpu'] / cpu_sum:>6.1f}%" + if gpu_timing: + line += f" {entry['gpu']:>10.3f}" if entry["gpu"] is not None else f" {'-':>10}" + lines.append(line) + return lines + + def report(result: Result) -> str: warning = ["!! cunumpy's fake CuPy: CUDA launches emulated on the CPU, the times below are not GPU timings", ""] lines = (warning if result.backend == "cupy" and fake_cupy_active() else []) + [ f"{result.model} | {result.case} | backend {result.backend} | {result.mpi_ranks} MPI rank(s) | host {result.host}", f"grid {result.num_elements}, degree {result.degree}, {result.ppc} markers per cell = {result.n_markers:,} markers, " f"{result.steps} steps of dt = {result.dt}", - "", ] + if result.gpu_timing: + sync = "device synchronized at region ends" if result.gpu_sync_on_exit else "no synchronization at region ends" + lines.append(f"GPU timing: CUDA events through CuPy, {sync}") lines += [ + "", f" setup (allocate + initial Poisson) {result.setup:10.3f} s", f" CUDA kernel compilation {result.compile_cuda:10.3f} s", f" time loop: model.integrate {sum(result.step_times):10.3f} s", - f" diagnostics {result.diagnostics:10.3f} s", - f" output {result.output:10.3f} s", f" Simulation.run total {result.total_run:10.3f} s", "", f" step: median {1e3 * result.step_median:.3f} ms (first step {1e3 * result.step_times[0]:.3f} ms), " f"{result.ns_per_marker_step:.1f} ns per marker and step", ] - if result.propagators: - lines.append(" per propagator (all steps):") - total = sum(result.propagators.values()) or 1.0 - for name, t in result.propagators.items(): - lines.append(f" {name:<28} {t:10.3f} s ({100 * t / total:5.1f} %)") - lines.append( - " host/device transfers in the time loop (through cunumpy): " - + ", ".join(f"{k}={v}" for k, v in result.transfers_in_loop.items()) - ) + lines += _time_table("per propagator (all steps", result.propagators, result.gpu_timing) + lines += _time_table("per kernel (whole run, including the setup", result.kernels, result.gpu_timing) + lines += ["", f" profile: {result.profile} (scope-profiler inspect {os.path.basename(result.profile)})"] return "\n".join(lines) From 032a642e158e902f80be3caa910ff904395f2f4b Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 13 Oct 2026 09:57:01 +0200 Subject: [PATCH 03/12] Import ProfilingOptions from struphy --- profiling/benchmarks/vlasov_benchmark.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/profiling/benchmarks/vlasov_benchmark.py b/profiling/benchmarks/vlasov_benchmark.py index 3762e5713..c014503cb 100644 --- a/profiling/benchmarks/vlasov_benchmark.py +++ b/profiling/benchmarks/vlasov_benchmark.py @@ -71,8 +71,9 @@ import numpy as np from cunumpy.kernel_testing import fake_cupy_active from maybempi import MPI -from scope_profiler import ProfilingOptions, ProfilingResults, read_h5 +from scope_profiler import ProfilingResults, read_h5 +from struphy import ProfilingOptions from struphy.models.tests.gpu_e2e import device_context SIZES = { From 0db6d8a4b234e57692485cf137871a7e8f2bff8a Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 13 Oct 2026 10:02:51 +0200 Subject: [PATCH 04/12] Write the report --- profiling/benchmarks/vlasov_benchmark.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/profiling/benchmarks/vlasov_benchmark.py b/profiling/benchmarks/vlasov_benchmark.py index c014503cb..c00342770 100644 --- a/profiling/benchmarks/vlasov_benchmark.py +++ b/profiling/benchmarks/vlasov_benchmark.py @@ -10,9 +10,11 @@ records CUDA-event device time for every region in addition to the CPU time (``use_gpu_timing``, backend ``cupy``), and by default synchronizes the device when a region ends (``gpu_sync_on_exit``), so the CPU time of a region includes its kernels and not only their launch; ``--no-gpu-sync`` records the enqueue-side CPU time instead. -The profile is written to an HDF5 file (``--profile``) for the full scope-profiler tool chain:: +The profile is written to an HDF5 file (``--profile``) for the full scope-profiler tool chain, and its HTML report +(call tree, charts, per-rank statistics) next to it (``--report``, ``--no-report``):: scope-profiler inspect vlasov_benchmark_VlasovAmpereOneSpecies_landau_cupy.h5 + open vlasov_benchmark_VlasovAmpereOneSpecies_landau_cupy.html Test problems (the physics follows the verification tests in ``src/struphy/models/tests/verification``): @@ -118,6 +120,7 @@ class Result: steps: int dt: float profile: str # the scope-profiler HDF5 file of the run + report: str | None # the scope-profiler HTML report of the run, if written gpu_timing: bool # CUDA-event device times recorded per region gpu_sync_on_exit: bool # device synchronized at every region end (CPU times include the kernels) setup: float = 0.0 # Simulation.allocate (FEEC operators, markers, initial Poisson solve) @@ -166,6 +169,10 @@ def parse_args(): "--profile", help="scope-profiler HDF5 file of the run (default: vlasov_benchmark___.h5)", ) + parser.add_argument( + "--report", help="scope-profiler HTML report of the run (default: the profile's name with .html)" + ) + parser.add_argument("--no-report", action="store_true", help="do not write the HTML report") parser.add_argument("--out", help="output folder (default: a temporary folder, removed afterwards)") parser.add_argument("--json", help="also write the result to this JSON file (rank 0)") args = parser.parse_args() @@ -281,6 +288,7 @@ def make_simulation(args, out_folders: str, profile_path: str): steps=steps, dt=dt, profile=os.path.abspath(profile_path), + report=None, gpu_timing=opts.use_gpu_timing, gpu_sync_on_exit=opts.gpu_sync_on_exit, ) @@ -343,7 +351,13 @@ def run(args) -> Result: # scope-profiler merges the ranks into one file at the end of the session if comm.Get_rank() == 0: - collect(read_h5(profile_path), result) + results = read_h5(profile_path) + collect(results, result) + if not args.no_report: + from scope_profiler import create_html_report + + report_path = args.report or os.path.splitext(profile_path)[0] + ".html" + result.report = os.path.abspath(create_html_report(results, report_path)) comm.Barrier() if tmp is not None: @@ -393,6 +407,8 @@ def report(result: Result) -> str: lines += _time_table("per propagator (all steps", result.propagators, result.gpu_timing) lines += _time_table("per kernel (whole run, including the setup", result.kernels, result.gpu_timing) lines += ["", f" profile: {result.profile} (scope-profiler inspect {os.path.basename(result.profile)})"] + if result.report: + lines.append(f" report: {result.report}") return "\n".join(lines) From f95ca720307549922cfbb923a96495907280eaf8 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 13 Oct 2026 10:35:50 +0200 Subject: [PATCH 05/12] Added landau damping and weibel profiling examples --- profiling/benchmarks/vlasov_benchmark.py | 431 ------------------ .../landau_damping/params_vlasov_ampere.py | 233 ++++++++++ .../submit_vlasov_ampere_landau_damping.py | 95 ++++ .../landau_damping/params_vlasov_maxwell.py | 234 ++++++++++ .../submit_vlasov_maxwell_landau_damping.py | 95 ++++ ...ubmit_vlasov_maxwell_weibel_instability.py | 95 ++++ .../params_vlasov_maxwell.py | 239 ++++++++++ profiling/package_profiling_results.py | 5 +- 8 files changed, 994 insertions(+), 433 deletions(-) delete mode 100644 profiling/benchmarks/vlasov_benchmark.py create mode 100644 profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py create mode 100644 profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py create mode 100644 profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py create mode 100644 profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py create mode 100644 profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py create mode 100644 profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py diff --git a/profiling/benchmarks/vlasov_benchmark.py b/profiling/benchmarks/vlasov_benchmark.py deleted file mode 100644 index c00342770..000000000 --- a/profiling/benchmarks/vlasov_benchmark.py +++ /dev/null @@ -1,431 +0,0 @@ -"""Benchmark of the Vlasov-Ampère and Vlasov-Maxwell models on standard test problems. - -Runs a fixed number of time steps of :class:`~struphy.models.VlasovAmpereOneSpecies` or -:class:`~struphy.models.VlasovMaxwellOneSpecies` on the NumPy (CPU) or CuPy (GPU) backend and reports the setup -time, the time per step, the time per propagator and per kernel, and the cost per marker. - -All timing comes from scope-profiler: the run activates the profiling session of :meth:`Simulation.run`, so the -regions that struphy already defines (``setup: allocate``, ``model.integrate``, ``prop: ``, -``kernel: ``, ...) are recorded, and the benchmark only reads them back. On the CuPy backend the session -records CUDA-event device time for every region in addition to the CPU time (``use_gpu_timing``, backend -``cupy``), and by default synchronizes the device when a region ends (``gpu_sync_on_exit``), so the CPU time of a -region includes its kernels and not only their launch; ``--no-gpu-sync`` records the enqueue-side CPU time instead. -The profile is written to an HDF5 file (``--profile``) for the full scope-profiler tool chain, and its HTML report -(call tree, charts, per-rank statistics) next to it (``--report``, ``--no-report``):: - - scope-profiler inspect vlasov_benchmark_VlasovAmpereOneSpecies_landau_cupy.h5 - open vlasov_benchmark_VlasovAmpereOneSpecies_landau_cupy.html - -Test problems (the physics follows the verification tests in ``src/struphy/models/tests/verification``): - -``landau`` - Weak Landau damping (both models). Periodic box of length 4 pi (k = 0.5), Maxwellian with a density - perturbation of 1e-3 cos(k x), control-variate weights, cubic splines in x. Damping rate -0.1533. -``weibel`` - Weibel instability (Vlasov-Maxwell only), the setup of Kraus et al., J. Plasma Phys. 83 (2017), Sec. 6.2: - box of length 2 pi / 1.25, anisotropic Maxwellian (vth1 = vth3 = 0.02 / sqrt(2), vth2 = sqrt(12) vth1), - seed field B3 = -1e-4 cos(k x), no density perturbation. Growth rate 0.02784. - -All runs use ``alpha = 1`` and ``epsilon = -1`` (electrons, time in units of the inverse plasma frequency, -velocities in units of c), as the verification tests do. - -Sizes (``--size``; each value can be overridden on the command line): - -======== ================= ===== ===== ================================================== -size cells (x, y, z) ppc steps purpose -======== ================= ===== ===== ================================================== -smoke 8, 1, 1 50 3 check that the benchmark runs (seconds) -standard 32, 1, 1 1000 20 the verification tests' resolution (32k markers) -large 128, 1, 1 20000 20 GPU-sized (2.56M markers) -======== ================= ===== ===== ================================================== - -Examples:: - - python profiling/benchmarks/vlasov_benchmark.py --model VlasovAmpereOneSpecies --case landau --size standard - CUNUMPY_BACKEND=cupy python profiling/benchmarks/vlasov_benchmark.py --model VlasovMaxwellOneSpecies \\ - --case weibel --size large --json weibel_gpu.json - mpirun -n 2 python profiling/benchmarks/vlasov_benchmark.py --size standard - -The backend is cunumpy's active backend (``CUNUMPY_BACKEND=numpy|cupy``, or ``--backend``). Times are the maximum -over MPI ranks. Sorting boxes are off by default, since ``SortingBoxes`` does not work on the CuPy backend yet -(struphy-hub/struphy#726); ``--sorting-boxes N`` turns them on. - -Without a GPU, ``CUNUMPY_FAKE_CUPY=1 ... --backend cupy --size smoke`` runs the CuPy code path on cunumpy's fake -CuPy with every CUDA launch emulated on the CPU: a check that the models run on the device backend, not a timing. -The fake CuPy has no CUDA events, so GPU timing is off in that mode. - -CPU reference (NumPy backend, ``--size standard``, 1 rank, Apple M-series laptop, 8 October 2026): VlasovAmpere -Landau 67 ms per step, VlasovMaxwell Landau 82 ms, VlasovMaxwell Weibel 86 ms (2.1 to 2.7 microseconds per marker -and step; ``VlasovAmpereCoupling`` takes 68 to 86 % of a step). -""" - -import argparse -import contextlib -import json -import math -import os -import platform -import statistics -import tempfile -from dataclasses import asdict, dataclass, field - -import cunumpy as xp -import numpy as np -from cunumpy.kernel_testing import fake_cupy_active -from maybempi import MPI -from scope_profiler import ProfilingResults, read_h5 - -from struphy import ProfilingOptions -from struphy.models.tests.gpu_e2e import device_context - -SIZES = { - "smoke": {"num_elements": (8, 1, 1), "ppc": 50, "steps": 3}, - "standard": {"num_elements": (32, 1, 1), "ppc": 1000, "steps": 20}, - "large": {"num_elements": (128, 1, 1), "ppc": 20000, "steps": 20}, -} - -CASES = { - # box length, time step, expected rate (damping < 0, growth > 0) of the electric (landau) or magnetic (weibel) - # field energy amplitude - "landau": { - "r1": 4 * math.pi, - "dt": 0.05, - "rate": -0.1533, - "models": ("VlasovAmpereOneSpecies", "VlasovMaxwellOneSpecies"), - }, - "weibel": {"r1": 2 * math.pi / 1.25, "dt": 0.05, "rate": 0.02784, "models": ("VlasovMaxwellOneSpecies",)}, -} - -# scope-profiler regions of Simulation.run that the report summarizes -SESSION_REGION = "scope_profiler.session" -SETUP_REGION = "setup: allocate" -COMPILE_REGION = "setup: compile cuda kernels" -STEP_REGION = "model.integrate" -PROPAGATOR_PREFIX = "prop: " -KERNEL_PREFIX = "kernel: " - - -@dataclass -class Result: - """What one benchmark run measured. Times in seconds, maximum over MPI ranks.""" - - model: str - case: str - backend: str - mpi_ranks: int - num_elements: tuple - degree: tuple - ppc: int - n_markers: int - steps: int - dt: float - profile: str # the scope-profiler HDF5 file of the run - report: str | None # the scope-profiler HTML report of the run, if written - gpu_timing: bool # CUDA-event device times recorded per region - gpu_sync_on_exit: bool # device synchronized at every region end (CPU times include the kernels) - setup: float = 0.0 # Simulation.allocate (FEEC operators, markers, initial Poisson solve) - compile_cuda: float = 0.0 # Simulation.compile_cuda_kernels (0 on NumPy) - step_times: list = field(default_factory=list) # model.integrate, one per step - total_run: float = 0.0 # the scope-profiler session, i.e. Simulation.run - propagators: dict = field(default_factory=dict) # class name -> {"cpu": total, "gpu": total or None} - kernels: dict = field(default_factory=dict) # kernel name -> {"cpu": total, "gpu": total or None} - scalars_end: dict = field(default_factory=dict) # name -> value after the last step - host: str = field(default_factory=platform.node) - - @property - def step_median(self) -> float: - # the first step includes one-time work (CUDA module loads, allocations of work arrays) - steady = self.step_times[1:] or self.step_times - return statistics.median(steady) - - @property - def ns_per_marker_step(self) -> float: - return 1e9 * self.step_median / max(self.n_markers, 1) - - -def parse_args(): - parser = argparse.ArgumentParser( - description=__doc__.split("\n\n")[0], formatter_class=argparse.RawDescriptionHelpFormatter - ) - parser.add_argument("--model", default="VlasovAmpereOneSpecies", choices=CASES["landau"]["models"]) - parser.add_argument("--case", default="landau", choices=tuple(CASES)) - parser.add_argument("--size", default="standard", choices=tuple(SIZES)) - parser.add_argument( - "--num-elements", type=int, nargs=3, metavar=("NX", "NY", "NZ"), help="override the size's grid" - ) - parser.add_argument("--degree", type=int, nargs=3, default=(3, 1, 1), metavar=("PX", "PY", "PZ")) - parser.add_argument("--ppc", type=int, help="markers per cell (overrides the size)") - parser.add_argument("--steps", type=int, help="number of time steps (overrides the size)") - parser.add_argument("--dt", type=float, help="time step (overrides the case)") - parser.add_argument("--backend", choices=("numpy", "cupy"), help="cunumpy backend (default: the active one)") - parser.add_argument("--sorting-boxes", type=int, default=0, metavar="N", help="sorting boxes in x (default: off)") - parser.add_argument("--save-step", type=int, default=1, help="compute diagnostics and save every N steps") - parser.add_argument( - "--no-gpu-sync", - action="store_true", - help="do not synchronize the device at region ends (CPU times are then launch times; GPU times unaffected)", - ) - parser.add_argument( - "--profile", - help="scope-profiler HDF5 file of the run (default: vlasov_benchmark___.h5)", - ) - parser.add_argument( - "--report", help="scope-profiler HTML report of the run (default: the profile's name with .html)" - ) - parser.add_argument("--no-report", action="store_true", help="do not write the HTML report") - parser.add_argument("--out", help="output folder (default: a temporary folder, removed afterwards)") - parser.add_argument("--json", help="also write the result to this JSON file (rank 0)") - args = parser.parse_args() - if args.model not in CASES[args.case]["models"]: - parser.error(f"case {args.case!r} is only set up for {', '.join(CASES[args.case]['models'])}") - return args - - -def profiling_options(args, profile_path: str) -> ProfilingOptions: - """The scope-profiler settings of the run: GPU timing through CuPy's CUDA events on the CuPy backend.""" - # the fake CuPy (no GPU) has CUDA events without elapsed time, so device timing is only possible on a GPU - gpu_timing = xp.get_backend() == "cupy" and not fake_cupy_active() - return ProfilingOptions( - file_path=profile_path, - label=f"{args.model} {args.case} {xp.get_backend()}", - use_gpu_timing=gpu_timing, - gpu_timing_backend="cupy", - gpu_sync_on_exit=gpu_timing and not args.no_gpu_sync, - ) - - -def make_simulation(args, out_folders: str, profile_path: str): - """The model, markers, initial condition and options of the chosen case and size.""" - from struphy import ( - BoundaryParameters, - DerhamOptions, - EnvironmentOptions, - LoadingParameters, - SavingParameters, - Simulation, - SortingParameters, - Time, - WeightsParameters, - domains, - grids, - maxwellians, - perturbations, - ) - from struphy.models import VlasovAmpereOneSpecies, VlasovMaxwellOneSpecies - - case = CASES[args.case] - size = SIZES[args.size] - num_elements = tuple(args.num_elements or size["num_elements"]) - ppc = args.ppc or size["ppc"] - steps = args.steps or size["steps"] - dt = args.dt or case["dt"] - - if args.model == "VlasovAmpereOneSpecies": - model = VlasovAmpereOneSpecies(alpha=1.0, epsilon=-1.0, with_B0=False) - else: - model = VlasovMaxwellOneSpecies(alpha=1.0, epsilon=-1.0) - - env = EnvironmentOptions( - out_folders=out_folders, - sim_folder=f"{args.model}_{args.case}", - save_step=args.save_step, - save_restart=False, # restart data would dominate the run time for many markers - ) - time_opts = Time(dt=dt, Tend=round(steps * dt, 14)) - - if args.sorting_boxes: - sorting_params = SortingParameters(boxes_per_dim=(args.sorting_boxes, 1, 1), do_sort=True) - else: - sorting_params = SortingParameters() - - model.kinetic_ions.set_markers( - loading_params=LoadingParameters(ppc=ppc, seed=1234), - weights_params=WeightsParameters(control_variate=True), - boundary_params=BoundaryParameters(), - sorting_params=sorting_params, - saving_params=SavingParameters(n_markers=0), # benchmark the physics, not the marker output - bufsize=0.4, - ) - - for name, prop in model.propagators.__dict__.items(): - prop.options = prop.Options() - model.initial_poisson.options = model.initial_poisson.Options(stab_mat="M0") - - background = maxwellians.Maxwellian3D(n=(1.0, None)) - if args.case == "landau": - model.kinetic_ions.var.add_background(background) - perturbation = perturbations.ModesCos(ls=(1,), amps=(1e-3,)) - model.kinetic_ions.var.add_initial_condition(maxwellians.Maxwellian3D(n=(1.0, perturbation))) - else: - vth1 = 0.02 / math.sqrt(2.0) - anisotropic = maxwellians.Maxwellian3D( - n=(1.0, None), vth1=(vth1, None), vth2=(math.sqrt(12.0) * vth1, None), vth3=(vth1, None) - ) - model.kinetic_ions.var.add_background(anisotropic) - # B3 = -1e-4 cos(k x) in physical coordinates: kx = 2 pi l / Lx, so Lx must be the box length - seed = perturbations.ModesCos(ls=(1,), amps=(-1e-4,), Lx=case["r1"], given_in_basis="physical", comp=2) - model.em_fields.b_field.add_perturbation(seed) - - opts = profiling_options(args, profile_path) - sim = Simulation( - model=model, - env=env, - time_opts=time_opts, - domain=domains.Cuboid(r1=case["r1"]), - grid=grids.TensorProductGrid(num_elements=num_elements), - derham_opts=DerhamOptions(degree=tuple(args.degree)), - profiling_opts=opts, - ) - result = Result( - model=args.model, - case=args.case, - backend=xp.get_backend(), - mpi_ranks=MPI.COMM_WORLD.Get_size(), - num_elements=num_elements, - degree=tuple(args.degree), - ppc=ppc, - n_markers=ppc * math.prod(num_elements), - steps=steps, - dt=dt, - profile=os.path.abspath(profile_path), - report=None, - gpu_timing=opts.use_gpu_timing, - gpu_sync_on_exit=opts.gpu_sync_on_exit, - ) - return sim, result - - -def max_over_ranks_per_call(region) -> list: - """The duration of each call of a region, the maximum over the ranks (every rank makes the same calls).""" - per_rank = [region[rank].durations for rank in region.ranks] - n_calls = min(len(durations) for durations in per_rank) - return np.max([durations[:n_calls] for durations in per_rank], axis=0).tolist() - - -def totals_over_ranks(region) -> dict: - """The total CPU time of a region and, with GPU timing, its total device time; the maximum over the ranks.""" - cpu = max(region.total_durations().values()) - gpu = None - if region.has_gpu_timing: - gpu = max(region[rank].gpu_total_duration for rank in region.ranks if region[rank].has_gpu_timing) - return {"cpu": float(cpu), "gpu": None if gpu is None else float(gpu)} - - -def collect(results: ProfilingResults, result: Result) -> None: - """Fill ``result`` from the scope-profiler regions of the run.""" - if SETUP_REGION in results: - result.setup = results[SETUP_REGION].max_duration - if COMPILE_REGION in results: - result.compile_cuda = results[COMPILE_REGION].max_duration - if SESSION_REGION in results: - result.total_run = results[SESSION_REGION].max_duration - elif results.total_time is not None: - result.total_run = results.total_time - result.step_times = max_over_ranks_per_call(results[STEP_REGION]) - for region in results: - if region.name.startswith(PROPAGATOR_PREFIX): - result.propagators[region.name[len(PROPAGATOR_PREFIX) :]] = totals_over_ranks(region) - elif region.name.startswith(KERNEL_PREFIX): - result.kernels[region.name[len(KERNEL_PREFIX) :]] = totals_over_ranks(region) - result.propagators = dict(sorted(result.propagators.items(), key=lambda item: -item[1]["cpu"])) - result.kernels = dict(sorted(result.kernels.items(), key=lambda item: -item[1]["cpu"])) - - -def run(args) -> Result: - comm = MPI.COMM_WORLD - out = args.out - tmp = None - if out is None: - tmp = tempfile.TemporaryDirectory(prefix="struphy_vlasov_benchmark_") if comm.Get_rank() == 0 else None - out = comm.bcast(tmp.name if tmp else None, root=0) - profile_path = args.profile or f"vlasov_benchmark_{args.model}_{args.case}_{xp.get_backend()}.h5" - - sim, result = make_simulation(args, out, profile_path) - # on cunumpy's fake CuPy (no GPU) the CUDA launches are emulated on the CPU: a smoke test, not a timing - with device_context() if xp.get_backend() == "cupy" else contextlib.nullcontext(): - sim.run(profiling_activated=True) - - for name, scalar in sim.model.scalars.dct.items(): - value = float(xp.to_numpy(scalar.value)[0]) if hasattr(scalar, "value") else float("nan") - result.scalars_end[name] = value - - # scope-profiler merges the ranks into one file at the end of the session - if comm.Get_rank() == 0: - results = read_h5(profile_path) - collect(results, result) - if not args.no_report: - from scope_profiler import create_html_report - - report_path = args.report or os.path.splitext(profile_path)[0] + ".html" - result.report = os.path.abspath(create_html_report(results, report_path)) - - comm.Barrier() - if tmp is not None: - tmp.cleanup() - return result - - -def _time_table(title: str, totals: dict, gpu_timing: bool) -> list: - """Lines of a table with the CPU and (if recorded) GPU total of every entry and its share of the CPU sum.""" - if not totals: - return [] - lines = [f" {title}, maximum over ranks):"] - width = max(len(name) for name in totals) - cpu_sum = sum(entry["cpu"] for entry in totals.values()) or 1.0 - header = f" {'':<{width}} {'CPU [s]':>10} {'share':>7}" - if gpu_timing: - header += f" {'GPU [s]':>10}" - lines.append(header) - for name, entry in totals.items(): - line = f" {name:<{width}} {entry['cpu']:>10.3f} {100 * entry['cpu'] / cpu_sum:>6.1f}%" - if gpu_timing: - line += f" {entry['gpu']:>10.3f}" if entry["gpu"] is not None else f" {'-':>10}" - lines.append(line) - return lines - - -def report(result: Result) -> str: - warning = ["!! cunumpy's fake CuPy: CUDA launches emulated on the CPU, the times below are not GPU timings", ""] - lines = (warning if result.backend == "cupy" and fake_cupy_active() else []) + [ - f"{result.model} | {result.case} | backend {result.backend} | {result.mpi_ranks} MPI rank(s) | host {result.host}", - f"grid {result.num_elements}, degree {result.degree}, {result.ppc} markers per cell = {result.n_markers:,} markers, " - f"{result.steps} steps of dt = {result.dt}", - ] - if result.gpu_timing: - sync = "device synchronized at region ends" if result.gpu_sync_on_exit else "no synchronization at region ends" - lines.append(f"GPU timing: CUDA events through CuPy, {sync}") - lines += [ - "", - f" setup (allocate + initial Poisson) {result.setup:10.3f} s", - f" CUDA kernel compilation {result.compile_cuda:10.3f} s", - f" time loop: model.integrate {sum(result.step_times):10.3f} s", - f" Simulation.run total {result.total_run:10.3f} s", - "", - f" step: median {1e3 * result.step_median:.3f} ms (first step {1e3 * result.step_times[0]:.3f} ms), " - f"{result.ns_per_marker_step:.1f} ns per marker and step", - ] - lines += _time_table("per propagator (all steps", result.propagators, result.gpu_timing) - lines += _time_table("per kernel (whole run, including the setup", result.kernels, result.gpu_timing) - lines += ["", f" profile: {result.profile} (scope-profiler inspect {os.path.basename(result.profile)})"] - if result.report: - lines.append(f" report: {result.report}") - return "\n".join(lines) - - -def main(): - args = parse_args() - if args.backend: - xp.set_backend(args.backend) - result = run(args) - if MPI.COMM_WORLD.Get_rank() == 0: - print(report(result)) - if args.json: - data = asdict(result) - data.update(step_median=result.step_median, ns_per_marker_step=result.ns_per_marker_step) - with open(args.json, "w") as f: - json.dump(data, f, indent=2) - print(f"\nwrote {os.path.abspath(args.json)}") - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py new file mode 100644 index 000000000..c5c7322d6 --- /dev/null +++ b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py @@ -0,0 +1,233 @@ +# ----------------------------- +# Description of the simulation +# ----------------------------- +# Please fill in a verbal description of the simulation. +# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. + +name = "Vlasov-Ampere weak Landau damping" +description = """ +Weak Landau damping with the VlasovAmpereOneSpecies model, the setup of the verification test in +src/struphy/models/tests/verification: periodic box of length 4 pi (k = 0.5), Maxwellian electrons with a density +perturbation of 1e-3 cos(k x), control-variate weights, cubic splines in x. The electric field energy decays at the +rate -0.1533. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, velocities in +units of c). + +Runs on the NumPy (CPU) or CuPy (GPU) backend (--backend). All timing comes from scope-profiler: the run activates +the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per +region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its +kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker +into the run's results folder. +""" + +import argparse +import logging + +from struphy import set_logging_level + +set_logging_level(logging.INFO) + +import math + +import cunumpy as xp + +# ------------------ +# Import Struphy API +# ------------------ +from struphy import ( + BoundaryParameters, + DerhamOptions, + EnvironmentOptions, + LoadingParameters, + ProfilingOptions, + SavingParameters, + Simulation, + SortingParameters, + Time, + WeightsParameters, + domains, + grids, + maxwellians, + perturbations, +) + +# -------------------------- +# Command line parameters +# -------------------------- +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument("--backend", choices=("numpy", "cupy"), help="cunumpy backend (default: the active one).") +parser.add_argument("--num-elements", type=int, default=32, help="Cells in x (1 in y and z).") +parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") +parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") +args, _ = parser.parse_known_args() + +# the backend must be chosen before any array of the model is allocated +if args.backend: + xp.set_backend(args.backend) + +# --------------------- +# Instance of the model +# --------------------- +from struphy.models import VlasovAmpereOneSpecies + +model = VlasovAmpereOneSpecies(alpha=1.0, epsilon=-1.0, with_B0=False) + +# List all variables and decide whether to save their data +model.em_fields.e_field.save_data = True +model.kinetic_ions.var.save_data = False + +# -------------------------- +# Instance of the simulation +# -------------------------- + +# Environment options +env = EnvironmentOptions( + sim_folder=f"sim_{args.id:02d}", + restart=False, + save_restart=False, # restart data would dominate the run time for many markers +) + +# Time stepping +dt = 0.05 +time_opts = Time(dt=dt, Tend=round(args.steps * dt, 14)) + +# Geometry +Lx = 4 * math.pi # k = 0.5 +domain = domains.Cuboid(r1=Lx) + +# Fluid equilibrium (can be used as part of initial conditions) +equil = None + +# Grid +grid = grids.TensorProductGrid(num_elements=(args.num_elements, 1, 1)) + +# Derham options +derham_opts = DerhamOptions(degree=(3, 1, 1)) + +# Profiling options: CUDA-event device times per region on the CuPy backend. +# cunumpy's fake CuPy (CUNUMPY_FAKE_CUPY=1, no GPU) has CUDA events without elapsed time, so GPU timing needs a GPU. +from cunumpy.kernel_testing import fake_cupy_active + +gpu_timing = xp.get_backend() == "cupy" and not fake_cupy_active() +profiling_opts = ProfilingOptions( + label=f"{name} ({xp.get_backend()})", + use_gpu_timing=gpu_timing, + gpu_timing_backend="cupy", + gpu_sync_on_exit=gpu_timing, +) + +# Simulation object +sim = Simulation( + model=model, + name=name, + description=description, + params_path=__file__, + env=env, + time_opts=time_opts, + domain=domain, + equil=equil, + grid=grid, + derham_opts=derham_opts, + profiling_opts=profiling_opts, +) + +# ------------------- +# Particle parameters +# ------------------- + +loading_params = LoadingParameters(ppc=args.ppc, seed=1234) +weights_params = WeightsParameters(control_variate=True) +boundary_params = BoundaryParameters() +# sorting boxes are off: SortingBoxes does not work on the CuPy backend yet (struphy-hub/struphy#726) +sorting_params = SortingParameters() +saving_params = SavingParameters(n_markers=0) # profile the physics, not the marker output + +model.kinetic_ions.set_markers( + loading_params=loading_params, + weights_params=weights_params, + boundary_params=boundary_params, + sorting_params=sorting_params, + saving_params=saving_params, + bufsize=0.4, +) + +# ------------------ +# Propagator options +# ------------------ + +for prop in model.propagators.__dict__.values(): + prop.options = prop.Options() +model.initial_poisson.options = model.initial_poisson.Options(stab_mat="M0") + +# ------------------ +# Initial conditions +# ------------------ +# Initial conditions are the sum of the background(s) and the perturbation(s). +# For kinetic species the background is mandatory; the perturbation is added to the density. + +background = maxwellians.Maxwellian3D(n=(1.0, None)) +model.kinetic_ions.var.add_background(background) + +perturbation = perturbations.ModesCos(ls=(1,), amps=(1e-3,)) +model.kinetic_ions.var.add_initial_condition(maxwellians.Maxwellian3D(n=(1.0, perturbation))) + + +if __name__ == "__main__": + import contextlib + + # Without a GPU, CUNUMPY_FAKE_CUPY=1 --backend cupy runs the CuPy code path on cunumpy's fake CuPy with every + # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU + # (or on NumPy) the context does nothing. + from struphy.models.tests.gpu_e2e import device_context + + with device_context() if fake_cupy_active() else contextlib.nullcontext(): + out = sim.run(profiling_activated=True) + + if sim.rank == 0: + # `out.path_out` is the run's (absolute) output folder. The profiling packaging picks the files + # written to `results` up from here and uploads them next to the run's profiling_data.h5. + import json + import statistics + + import numpy as np + from scope_profiler import create_html_report + + results = out.profile.results # the scope-profiler results of the run, all ranks merged + results_dir = out.path_out / "results" + results_dir.mkdir(exist_ok=True) + + # the full scope-profiler report: call tree, charts, per-rank statistics + create_html_report(results, results_dir / "report.html") + + # cost per step and per marker, from the `model.integrate` region (the maximum over the ranks per step); + # the first step includes one-time work (CUDA module loads, allocations of work arrays) + steps = results["model.integrate"] + per_rank = [steps[rank].durations for rank in steps.ranks] + n_calls = min(len(durations) for durations in per_rank) + step_times = np.max([durations[:n_calls] for durations in per_rank], axis=0) + step_median = float(statistics.median(step_times[1:] if len(step_times) > 1 else step_times)) + n_markers = args.ppc * args.num_elements + summary = { + "backend": xp.get_backend(), + "gpu_timing": gpu_timing, + "mpi_ranks": results.num_ranks, + "num_elements": args.num_elements, + "ppc": args.ppc, + "n_markers": n_markers, + "steps": args.steps, + "dt": dt, + "step_times": step_times.tolist(), + "step_median": step_median, + "ns_per_marker_step": 1e9 * step_median / n_markers, + } + (results_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + + print( + f"\nstep: median {1e3 * step_median:.3f} ms (first step {1e3 * step_times[0]:.3f} ms), " + f"{summary['ns_per_marker_step']:.1f} ns per marker and step ({n_markers:,} markers)" + ) + # the propagators and kernels of the run; on the CuPy backend with GPU columns + results.print_summary(include=["^prop: ", "^kernel: "], sort="total", title="propagators and kernels") diff --git a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py new file mode 100644 index 000000000..a1f3b68c9 --- /dev/null +++ b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py @@ -0,0 +1,95 @@ +"""Vlasov-Ampere weak Landau damping profiling case. + +This file defines the Vlasov-Ampere weak Landau damping profiling case (the `ProfilingCase`) and submits +it: for each rank count, `ProfilingCase.launch` builds and submits a SLURM script +(using `clusters.SLURM_PRESETS` by default), or, without a batch system, runs +directly on this machine. `finalize_run` then packages and uploads each run as +soon as its own job finishes. +Each generated script runs the simulation itself by invoking `params_vlasov_ampere.py` +directly (its `__main__` block is the worker). + +The run is profiled with scope-profiler; `--backend cupy` runs it on the GPU with CUDA-event +timing per region. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_ampere.py` +(defaults: 1000 markers per cell on 32 cells, 20 steps). +""" + +import argparse +import sys +from pathlib import Path + +# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from profiling_job import ProfilingCase # noqa: E402 + + +def main() -> None: + + # Parse arguments, do not remove --upload + parser = argparse.ArgumentParser( + description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload the packaged profiling results to the profiling-data repo.", + ) + parser.add_argument( + "--language", + default="fortran", + choices=["fortran", "c"], + help="Pyccel language to compile the Struphy kernels with.", + ) + parser.add_argument( + "--compiler", + default="GNU", + help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', + ) + parser.add_argument( + "--backend", + default="numpy", + choices=["numpy", "cupy"], + help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", + ) + parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") + parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") + parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") + args = parser.parse_args() + + # Paths relative to this script's location, so it can be run from anywhere. + script_dir = Path(__file__).resolve().parent + params_dir = script_dir / "landau_damping" + + profiling_case = ProfilingCase( + label="vlasov_ampere_landau_damping", + name="Vlasov-Ampere weak Landau damping", + description="Strong scaling of the VlasovAmpereOneSpecies model on weak Landau damping (periodic box, k = 0.5, 1e-3 density perturbation), on the NumPy or CuPy backend.", + physics_problem="Landau damping of an electrostatic wave in a collisionless electron plasma.", + struphy_model_used="VlasovAmpereOneSpecies", + params_source=params_dir / "params_vlasov_ampere.py", + language=args.language, + compiler=args.compiler, + upload=args.upload, + ) + + param_flags = [ + "--backend", + args.backend, + "--ppc", + str(args.ppc), + "--num-elements", + str(args.num_elements), + "--steps", + str(args.steps), + ] + + # Launch one run per rank count + for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): + profiling_case.launch(num_tasks, param_flags=param_flags) + + # Package and push each run as its own job finishes. + profiling_case.finalize_run() + + +if __name__ == "__main__": + main() diff --git a/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py new file mode 100644 index 000000000..d68a59e26 --- /dev/null +++ b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py @@ -0,0 +1,234 @@ +# ----------------------------- +# Description of the simulation +# ----------------------------- +# Please fill in a verbal description of the simulation. +# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. + +name = "Vlasov-Maxwell weak Landau damping" +description = """ +Weak Landau damping with the VlasovMaxwellOneSpecies model, the setup of the verification test in +src/struphy/models/tests/verification: periodic box of length 4 pi (k = 0.5), Maxwellian electrons with a density +perturbation of 1e-3 cos(k x), control-variate weights, cubic splines in x. The electric field energy decays at the +rate -0.1533. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, velocities in +units of c). + +Runs on the NumPy (CPU) or CuPy (GPU) backend (--backend). All timing comes from scope-profiler: the run activates +the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per +region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its +kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker +into the run's results folder. +""" + +import argparse +import logging + +from struphy import set_logging_level + +set_logging_level(logging.INFO) + +import math + +import cunumpy as xp + +# ------------------ +# Import Struphy API +# ------------------ +from struphy import ( + BoundaryParameters, + DerhamOptions, + EnvironmentOptions, + LoadingParameters, + ProfilingOptions, + SavingParameters, + Simulation, + SortingParameters, + Time, + WeightsParameters, + domains, + grids, + maxwellians, + perturbations, +) + +# -------------------------- +# Command line parameters +# -------------------------- +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument("--backend", choices=("numpy", "cupy"), help="cunumpy backend (default: the active one).") +parser.add_argument("--num-elements", type=int, default=32, help="Cells in x (1 in y and z).") +parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") +parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") +args, _ = parser.parse_known_args() + +# the backend must be chosen before any array of the model is allocated +if args.backend: + xp.set_backend(args.backend) + +# --------------------- +# Instance of the model +# --------------------- +from struphy.models import VlasovMaxwellOneSpecies + +model = VlasovMaxwellOneSpecies(alpha=1.0, epsilon=-1.0) + +# List all variables and decide whether to save their data +model.em_fields.e_field.save_data = True +model.em_fields.b_field.save_data = True +model.kinetic_ions.var.save_data = False + +# -------------------------- +# Instance of the simulation +# -------------------------- + +# Environment options +env = EnvironmentOptions( + sim_folder=f"sim_{args.id:02d}", + restart=False, + save_restart=False, # restart data would dominate the run time for many markers +) + +# Time stepping +dt = 0.05 +time_opts = Time(dt=dt, Tend=round(args.steps * dt, 14)) + +# Geometry +Lx = 4 * math.pi # k = 0.5 +domain = domains.Cuboid(r1=Lx) + +# Fluid equilibrium (can be used as part of initial conditions) +equil = None + +# Grid +grid = grids.TensorProductGrid(num_elements=(args.num_elements, 1, 1)) + +# Derham options +derham_opts = DerhamOptions(degree=(3, 1, 1)) + +# Profiling options: CUDA-event device times per region on the CuPy backend. +# cunumpy's fake CuPy (CUNUMPY_FAKE_CUPY=1, no GPU) has CUDA events without elapsed time, so GPU timing needs a GPU. +from cunumpy.kernel_testing import fake_cupy_active + +gpu_timing = xp.get_backend() == "cupy" and not fake_cupy_active() +profiling_opts = ProfilingOptions( + label=f"{name} ({xp.get_backend()})", + use_gpu_timing=gpu_timing, + gpu_timing_backend="cupy", + gpu_sync_on_exit=gpu_timing, +) + +# Simulation object +sim = Simulation( + model=model, + name=name, + description=description, + params_path=__file__, + env=env, + time_opts=time_opts, + domain=domain, + equil=equil, + grid=grid, + derham_opts=derham_opts, + profiling_opts=profiling_opts, +) + +# ------------------- +# Particle parameters +# ------------------- + +loading_params = LoadingParameters(ppc=args.ppc, seed=1234) +weights_params = WeightsParameters(control_variate=True) +boundary_params = BoundaryParameters() +# sorting boxes are off: SortingBoxes does not work on the CuPy backend yet (struphy-hub/struphy#726) +sorting_params = SortingParameters() +saving_params = SavingParameters(n_markers=0) # profile the physics, not the marker output + +model.kinetic_ions.set_markers( + loading_params=loading_params, + weights_params=weights_params, + boundary_params=boundary_params, + sorting_params=sorting_params, + saving_params=saving_params, + bufsize=0.4, +) + +# ------------------ +# Propagator options +# ------------------ + +for prop in model.propagators.__dict__.values(): + prop.options = prop.Options() +model.initial_poisson.options = model.initial_poisson.Options(stab_mat="M0") + +# ------------------ +# Initial conditions +# ------------------ +# Initial conditions are the sum of the background(s) and the perturbation(s). +# For kinetic species the background is mandatory; the perturbation is added to the density. + +background = maxwellians.Maxwellian3D(n=(1.0, None)) +model.kinetic_ions.var.add_background(background) + +perturbation = perturbations.ModesCos(ls=(1,), amps=(1e-3,)) +model.kinetic_ions.var.add_initial_condition(maxwellians.Maxwellian3D(n=(1.0, perturbation))) + + +if __name__ == "__main__": + import contextlib + + # Without a GPU, CUNUMPY_FAKE_CUPY=1 --backend cupy runs the CuPy code path on cunumpy's fake CuPy with every + # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU + # (or on NumPy) the context does nothing. + from struphy.models.tests.gpu_e2e import device_context + + with device_context() if fake_cupy_active() else contextlib.nullcontext(): + out = sim.run(profiling_activated=True) + + if sim.rank == 0: + # `out.path_out` is the run's (absolute) output folder. The profiling packaging picks the files + # written to `results` up from here and uploads them next to the run's profiling_data.h5. + import json + import statistics + + import numpy as np + from scope_profiler import create_html_report + + results = out.profile.results # the scope-profiler results of the run, all ranks merged + results_dir = out.path_out / "results" + results_dir.mkdir(exist_ok=True) + + # the full scope-profiler report: call tree, charts, per-rank statistics + create_html_report(results, results_dir / "report.html") + + # cost per step and per marker, from the `model.integrate` region (the maximum over the ranks per step); + # the first step includes one-time work (CUDA module loads, allocations of work arrays) + steps = results["model.integrate"] + per_rank = [steps[rank].durations for rank in steps.ranks] + n_calls = min(len(durations) for durations in per_rank) + step_times = np.max([durations[:n_calls] for durations in per_rank], axis=0) + step_median = float(statistics.median(step_times[1:] if len(step_times) > 1 else step_times)) + n_markers = args.ppc * args.num_elements + summary = { + "backend": xp.get_backend(), + "gpu_timing": gpu_timing, + "mpi_ranks": results.num_ranks, + "num_elements": args.num_elements, + "ppc": args.ppc, + "n_markers": n_markers, + "steps": args.steps, + "dt": dt, + "step_times": step_times.tolist(), + "step_median": step_median, + "ns_per_marker_step": 1e9 * step_median / n_markers, + } + (results_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + + print( + f"\nstep: median {1e3 * step_median:.3f} ms (first step {1e3 * step_times[0]:.3f} ms), " + f"{summary['ns_per_marker_step']:.1f} ns per marker and step ({n_markers:,} markers)" + ) + # the propagators and kernels of the run; on the CuPy backend with GPU columns + results.print_summary(include=["^prop: ", "^kernel: "], sort="total", title="propagators and kernels") diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py new file mode 100644 index 000000000..6258d0c6a --- /dev/null +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py @@ -0,0 +1,95 @@ +"""Vlasov-Maxwell weak Landau damping profiling case. + +This file defines the Vlasov-Maxwell weak Landau damping profiling case (the `ProfilingCase`) and submits +it: for each rank count, `ProfilingCase.launch` builds and submits a SLURM script +(using `clusters.SLURM_PRESETS` by default), or, without a batch system, runs +directly on this machine. `finalize_run` then packages and uploads each run as +soon as its own job finishes. +Each generated script runs the simulation itself by invoking `params_vlasov_maxwell.py` +directly (its `__main__` block is the worker). + +The run is profiled with scope-profiler; `--backend cupy` runs it on the GPU with CUDA-event +timing per region. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` +(defaults: 1000 markers per cell on 32 cells, 20 steps). +""" + +import argparse +import sys +from pathlib import Path + +# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from profiling_job import ProfilingCase # noqa: E402 + + +def main() -> None: + + # Parse arguments, do not remove --upload + parser = argparse.ArgumentParser( + description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload the packaged profiling results to the profiling-data repo.", + ) + parser.add_argument( + "--language", + default="fortran", + choices=["fortran", "c"], + help="Pyccel language to compile the Struphy kernels with.", + ) + parser.add_argument( + "--compiler", + default="GNU", + help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', + ) + parser.add_argument( + "--backend", + default="numpy", + choices=["numpy", "cupy"], + help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", + ) + parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") + parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") + parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") + args = parser.parse_args() + + # Paths relative to this script's location, so it can be run from anywhere. + script_dir = Path(__file__).resolve().parent + params_dir = script_dir / "landau_damping" + + profiling_case = ProfilingCase( + label="vlasov_maxwell_landau_damping", + name="Vlasov-Maxwell weak Landau damping", + description="Strong scaling of the VlasovMaxwellOneSpecies model on weak Landau damping (periodic box, k = 0.5, 1e-3 density perturbation), on the NumPy or CuPy backend.", + physics_problem="Landau damping of an electrostatic wave in a collisionless electron plasma.", + struphy_model_used="VlasovMaxwellOneSpecies", + params_source=params_dir / "params_vlasov_maxwell.py", + language=args.language, + compiler=args.compiler, + upload=args.upload, + ) + + param_flags = [ + "--backend", + args.backend, + "--ppc", + str(args.ppc), + "--num-elements", + str(args.num_elements), + "--steps", + str(args.steps), + ] + + # Launch one run per rank count + for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): + profiling_case.launch(num_tasks, param_flags=param_flags) + + # Package and push each run as its own job finishes. + profiling_case.finalize_run() + + +if __name__ == "__main__": + main() diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py new file mode 100644 index 000000000..9cf7abcda --- /dev/null +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py @@ -0,0 +1,95 @@ +"""Vlasov-Maxwell Weibel instability profiling case. + +This file defines the Vlasov-Maxwell Weibel instability profiling case (the `ProfilingCase`) and submits +it: for each rank count, `ProfilingCase.launch` builds and submits a SLURM script +(using `clusters.SLURM_PRESETS` by default), or, without a batch system, runs +directly on this machine. `finalize_run` then packages and uploads each run as +soon as its own job finishes. +Each generated script runs the simulation itself by invoking `params_vlasov_maxwell.py` +directly (its `__main__` block is the worker). + +The run is profiled with scope-profiler; `--backend cupy` runs it on the GPU with CUDA-event +timing per region. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` +(defaults: 1000 markers per cell on 32 cells, 20 steps). +""" + +import argparse +import sys +from pathlib import Path + +# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from profiling_job import ProfilingCase # noqa: E402 + + +def main() -> None: + + # Parse arguments, do not remove --upload + parser = argparse.ArgumentParser( + description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload the packaged profiling results to the profiling-data repo.", + ) + parser.add_argument( + "--language", + default="fortran", + choices=["fortran", "c"], + help="Pyccel language to compile the Struphy kernels with.", + ) + parser.add_argument( + "--compiler", + default="GNU", + help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', + ) + parser.add_argument( + "--backend", + default="numpy", + choices=["numpy", "cupy"], + help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", + ) + parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") + parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") + parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") + args = parser.parse_args() + + # Paths relative to this script's location, so it can be run from anywhere. + script_dir = Path(__file__).resolve().parent + params_dir = script_dir / "weibel_instability" + + profiling_case = ProfilingCase( + label="vlasov_maxwell_weibel_instability", + name="Vlasov-Maxwell Weibel instability", + description="Strong scaling of the VlasovMaxwellOneSpecies model on the Weibel instability (Kraus et al. 2017, Sec. 6.2: anisotropic Maxwellian, seed field B3 = -1e-4 cos(k x)), on the NumPy or CuPy backend.", + physics_problem="Weibel instability driven by a temperature anisotropy in a collisionless electron plasma.", + struphy_model_used="VlasovMaxwellOneSpecies", + params_source=params_dir / "params_vlasov_maxwell.py", + language=args.language, + compiler=args.compiler, + upload=args.upload, + ) + + param_flags = [ + "--backend", + args.backend, + "--ppc", + str(args.ppc), + "--num-elements", + str(args.num_elements), + "--steps", + str(args.steps), + ] + + # Launch one run per rank count + for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): + profiling_case.launch(num_tasks, param_flags=param_flags) + + # Package and push each run as its own job finishes. + profiling_case.finalize_run() + + +if __name__ == "__main__": + main() diff --git a/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py new file mode 100644 index 000000000..5f67a4aea --- /dev/null +++ b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py @@ -0,0 +1,239 @@ +# ----------------------------- +# Description of the simulation +# ----------------------------- +# Please fill in a verbal description of the simulation. +# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. + +name = "Vlasov-Maxwell Weibel instability" +description = """ +Weibel instability with the VlasovMaxwellOneSpecies model, the setup of Kraus et al., J. Plasma Phys. 83 (2017), +Sec. 6.2, as in the verification test in src/struphy/models/tests/verification: periodic box of length 2 pi / 1.25, +anisotropic Maxwellian electrons (vth1 = vth3 = 0.02 / sqrt(2), vth2 = sqrt(12) vth1), seed field +B3 = -1e-4 cos(k x), no density perturbation, control-variate weights, cubic splines in x. The magnetic field energy +grows at the rate 0.02784. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, +velocities in units of c). + +Runs on the NumPy (CPU) or CuPy (GPU) backend (--backend). All timing comes from scope-profiler: the run activates +the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per +region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its +kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker +into the run's results folder. +""" + +import argparse +import logging + +from struphy import set_logging_level + +set_logging_level(logging.INFO) + +import math + +import cunumpy as xp + +# ------------------ +# Import Struphy API +# ------------------ +from struphy import ( + BoundaryParameters, + DerhamOptions, + EnvironmentOptions, + LoadingParameters, + ProfilingOptions, + SavingParameters, + Simulation, + SortingParameters, + Time, + WeightsParameters, + domains, + grids, + maxwellians, + perturbations, +) + +# -------------------------- +# Command line parameters +# -------------------------- +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument("--backend", choices=("numpy", "cupy"), help="cunumpy backend (default: the active one).") +parser.add_argument("--num-elements", type=int, default=32, help="Cells in x (1 in y and z).") +parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") +parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") +args, _ = parser.parse_known_args() + +# the backend must be chosen before any array of the model is allocated +if args.backend: + xp.set_backend(args.backend) + +# --------------------- +# Instance of the model +# --------------------- +from struphy.models import VlasovMaxwellOneSpecies + +model = VlasovMaxwellOneSpecies(alpha=1.0, epsilon=-1.0) + +# List all variables and decide whether to save their data +model.em_fields.e_field.save_data = True +model.em_fields.b_field.save_data = True +model.kinetic_ions.var.save_data = False + +# -------------------------- +# Instance of the simulation +# -------------------------- + +# Environment options +env = EnvironmentOptions( + sim_folder=f"sim_{args.id:02d}", + restart=False, + save_restart=False, # restart data would dominate the run time for many markers +) + +# Time stepping +dt = 0.05 +time_opts = Time(dt=dt, Tend=round(args.steps * dt, 14)) + +# Geometry +Lx = 2 * math.pi / 1.25 # k = 1.25 +domain = domains.Cuboid(r1=Lx) + +# Fluid equilibrium (can be used as part of initial conditions) +equil = None + +# Grid +grid = grids.TensorProductGrid(num_elements=(args.num_elements, 1, 1)) + +# Derham options +derham_opts = DerhamOptions(degree=(3, 1, 1)) + +# Profiling options: CUDA-event device times per region on the CuPy backend. +# cunumpy's fake CuPy (CUNUMPY_FAKE_CUPY=1, no GPU) has CUDA events without elapsed time, so GPU timing needs a GPU. +from cunumpy.kernel_testing import fake_cupy_active + +gpu_timing = xp.get_backend() == "cupy" and not fake_cupy_active() +profiling_opts = ProfilingOptions( + label=f"{name} ({xp.get_backend()})", + use_gpu_timing=gpu_timing, + gpu_timing_backend="cupy", + gpu_sync_on_exit=gpu_timing, +) + +# Simulation object +sim = Simulation( + model=model, + name=name, + description=description, + params_path=__file__, + env=env, + time_opts=time_opts, + domain=domain, + equil=equil, + grid=grid, + derham_opts=derham_opts, + profiling_opts=profiling_opts, +) + +# ------------------- +# Particle parameters +# ------------------- + +loading_params = LoadingParameters(ppc=args.ppc, seed=1234) +weights_params = WeightsParameters(control_variate=True) +boundary_params = BoundaryParameters() +# sorting boxes are off: SortingBoxes does not work on the CuPy backend yet (struphy-hub/struphy#726) +sorting_params = SortingParameters() +saving_params = SavingParameters(n_markers=0) # profile the physics, not the marker output + +model.kinetic_ions.set_markers( + loading_params=loading_params, + weights_params=weights_params, + boundary_params=boundary_params, + sorting_params=sorting_params, + saving_params=saving_params, + bufsize=0.4, +) + +# ------------------ +# Propagator options +# ------------------ + +for prop in model.propagators.__dict__.values(): + prop.options = prop.Options() +model.initial_poisson.options = model.initial_poisson.Options(stab_mat="M0") + +# ------------------ +# Initial conditions +# ------------------ +# Initial conditions are the sum of the background(s) and the perturbation(s). +# For kinetic species the background is mandatory; without add_initial_condition() it is the initial condition. + +vth1 = 0.02 / math.sqrt(2.0) +background = maxwellians.Maxwellian3D( + n=(1.0, None), vth1=(vth1, None), vth2=(math.sqrt(12.0) * vth1, None), vth3=(vth1, None) +) +model.kinetic_ions.var.add_background(background) + +# B3 = -1e-4 cos(k x) in physical coordinates: kx = 2 pi l / Lx, so Lx must be the box length +seed = perturbations.ModesCos(ls=(1,), amps=(-1e-4,), Lx=Lx, given_in_basis="physical", comp=2) +model.em_fields.b_field.add_perturbation(seed) + + +if __name__ == "__main__": + import contextlib + + # Without a GPU, CUNUMPY_FAKE_CUPY=1 --backend cupy runs the CuPy code path on cunumpy's fake CuPy with every + # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU + # (or on NumPy) the context does nothing. + from struphy.models.tests.gpu_e2e import device_context + + with device_context() if fake_cupy_active() else contextlib.nullcontext(): + out = sim.run(profiling_activated=True) + + if sim.rank == 0: + # `out.path_out` is the run's (absolute) output folder. The profiling packaging picks the files + # written to `results` up from here and uploads them next to the run's profiling_data.h5. + import json + import statistics + + import numpy as np + from scope_profiler import create_html_report + + results = out.profile.results # the scope-profiler results of the run, all ranks merged + results_dir = out.path_out / "results" + results_dir.mkdir(exist_ok=True) + + # the full scope-profiler report: call tree, charts, per-rank statistics + create_html_report(results, results_dir / "report.html") + + # cost per step and per marker, from the `model.integrate` region (the maximum over the ranks per step); + # the first step includes one-time work (CUDA module loads, allocations of work arrays) + steps = results["model.integrate"] + per_rank = [steps[rank].durations for rank in steps.ranks] + n_calls = min(len(durations) for durations in per_rank) + step_times = np.max([durations[:n_calls] for durations in per_rank], axis=0) + step_median = float(statistics.median(step_times[1:] if len(step_times) > 1 else step_times)) + n_markers = args.ppc * args.num_elements + summary = { + "backend": xp.get_backend(), + "gpu_timing": gpu_timing, + "mpi_ranks": results.num_ranks, + "num_elements": args.num_elements, + "ppc": args.ppc, + "n_markers": n_markers, + "steps": args.steps, + "dt": dt, + "step_times": step_times.tolist(), + "step_median": step_median, + "ns_per_marker_step": 1e9 * step_median / n_markers, + } + (results_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + + print( + f"\nstep: median {1e3 * step_median:.3f} ms (first step {1e3 * step_times[0]:.3f} ms), " + f"{summary['ns_per_marker_step']:.1f} ns per marker and step ({n_markers:,} markers)" + ) + # the propagators and kernels of the run; on the CuPy backend with GPU columns + results.print_summary(include=["^prop: ", "^kernel: "], sort="total", title="propagators and kernels") diff --git a/profiling/package_profiling_results.py b/profiling/package_profiling_results.py index 2512f1390..9a158a1c9 100644 --- a/profiling/package_profiling_results.py +++ b/profiling/package_profiling_results.py @@ -13,9 +13,10 @@ RUN_METADATA_FILE = "run_metadata.json" # Written by the parameter file itself (see the Poisson example), one per `sim_` run -# directory: post-processing figures and small arrays, next to the profiling output. +# directory: post-processing figures, small arrays, scope-profiler HTML reports and JSON summaries +# (see the Vlasov examples), next to the profiling output. RESULTS_DIR_NAME = "results" -RESULTS_FILE_SUFFIXES = (".png", ".npy") +RESULTS_FILE_SUFFIXES = (".png", ".npy", ".html", ".json") def _run_name(launch_id: int) -> str: From 797d4970c2d8ea17c00a8f07150b0d6ab7660cb1 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 13 Oct 2026 10:41:36 +0200 Subject: [PATCH 06/12] Add struphy backend to CLI --- doc/sections/install.rst | 19 +++++ .../landau_damping/params_vlasov_ampere.py | 10 +-- .../submit_vlasov_ampere_landau_damping.py | 12 ++-- .../landau_damping/params_vlasov_maxwell.py | 10 +-- .../submit_vlasov_maxwell_landau_damping.py | 12 ++-- ...ubmit_vlasov_maxwell_weibel_instability.py | 12 ++-- .../params_vlasov_maxwell.py | 10 +-- profiling/profiling_job.py | 16 +++++ src/struphy/__init__.py | 27 +++++++ src/struphy/console/backend.py | 72 +++++++++++++++++++ src/struphy/console/main.py | 20 ++++++ src/struphy/console/tests/test_backend.py | 62 ++++++++++++++++ 12 files changed, 243 insertions(+), 39 deletions(-) create mode 100644 src/struphy/console/backend.py create mode 100644 src/struphy/console/tests/test_backend.py diff --git a/doc/sections/install.rst b/doc/sections/install.rst index 86de999e3..7c227a47b 100644 --- a/doc/sections/install.rst +++ b/doc/sections/install.rst @@ -214,6 +214,25 @@ This compiles at most 4 kernels at the same time; kernels that depend on each ot In case you encounter problems during install visit :ref:`trouble_shoot`. +.. _gpu_backend: + +GPU backend +----------- + +Struphy's arrays live in `cunumpy `_, which runs on NumPy (CPU) or CuPy (GPU). cunumpy picks the backend from the environment variable ``CUNUMPY_BACKEND`` when it is imported:: + + CUNUMPY_BACKEND=cupy python params_maxwell.py + +The choice can also be stored with the command line, so it does not have to be repeated:: + + struphy backend set cupy + struphy backend get + struphy backend unset + +``import struphy`` then sets ``CUNUMPY_BACKEND`` from the stored choice, unless the variable is already set in the shell (an explicit variable always wins). This takes effect only when struphy is imported before cunumpy, as the parameter files generated by ``struphy params`` do. The profiling driver in ``profiling/`` exports the variable in every job script it writes, so profiling cases run on the stored backend unless their submit script is given ``--backend``. ``struphy backend get --value`` prints just the stored backend, for shell scripts. + +With CuPy requested on a machine without a working GPU, cunumpy falls back to NumPy; ``struphy backend get`` shows the backend actually in use. + .. _trouble_shoot: Trouble shooting diff --git a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py index c5c7322d6..9143dd44a 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py +++ b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py @@ -12,7 +12,8 @@ rate -0.1533. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, velocities in units of c). -Runs on the NumPy (CPU) or CuPy (GPU) backend (--backend). All timing comes from scope-profiler: the run activates +Runs on the NumPy (CPU) or CuPy (GPU) backend, chosen by the environment variable CUNUMPY_BACKEND (the profiling +driver exports it in the job script; `struphy backend set cupy` stores a default). All timing comes from scope-profiler: the run activates the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker @@ -58,16 +59,11 @@ # Unknown flags are ignored so the driver can forward other parameters as well. parser = argparse.ArgumentParser() parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -parser.add_argument("--backend", choices=("numpy", "cupy"), help="cunumpy backend (default: the active one).") parser.add_argument("--num-elements", type=int, default=32, help="Cells in x (1 in y and z).") parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") args, _ = parser.parse_known_args() -# the backend must be chosen before any array of the model is allocated -if args.backend: - xp.set_backend(args.backend) - # --------------------- # Instance of the model # --------------------- @@ -178,7 +174,7 @@ if __name__ == "__main__": import contextlib - # Without a GPU, CUNUMPY_FAKE_CUPY=1 --backend cupy runs the CuPy code path on cunumpy's fake CuPy with every + # Without a GPU, CUNUMPY_FAKE_CUPY=1 CUNUMPY_BACKEND=cupy runs the CuPy code path on cunumpy's fake CuPy with every # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU # (or on NumPy) the context does nothing. from struphy.models.tests.gpu_e2e import device_context diff --git a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py index a1f3b68c9..00b8e2d06 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py +++ b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py @@ -8,8 +8,9 @@ Each generated script runs the simulation itself by invoking `params_vlasov_ampere.py` directly (its `__main__` block is the worker). -The run is profiled with scope-profiler; `--backend cupy` runs it on the GPU with CUDA-event -timing per region. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_ampere.py` +The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event +timing per region (default: the backend stored with `struphy backend set`); the job script exports +the choice as CUNUMPY_BACKEND. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_ampere.py` (defaults: 1000 markers per cell on 32 cells, 20 steps). """ @@ -47,9 +48,9 @@ def main() -> None: ) parser.add_argument( "--backend", - default="numpy", + default=None, choices=["numpy", "cupy"], - help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", + help="cunumpy backend to run on (default: 'struphy backend get'); cupy records CUDA-event GPU times.", ) parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") @@ -69,12 +70,11 @@ def main() -> None: params_source=params_dir / "params_vlasov_ampere.py", language=args.language, compiler=args.compiler, + backend=args.backend, upload=args.upload, ) param_flags = [ - "--backend", - args.backend, "--ppc", str(args.ppc), "--num-elements", diff --git a/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py index d68a59e26..d1e30fd6d 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py @@ -12,7 +12,8 @@ rate -0.1533. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, velocities in units of c). -Runs on the NumPy (CPU) or CuPy (GPU) backend (--backend). All timing comes from scope-profiler: the run activates +Runs on the NumPy (CPU) or CuPy (GPU) backend, chosen by the environment variable CUNUMPY_BACKEND (the profiling +driver exports it in the job script; `struphy backend set cupy` stores a default). All timing comes from scope-profiler: the run activates the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker @@ -58,16 +59,11 @@ # Unknown flags are ignored so the driver can forward other parameters as well. parser = argparse.ArgumentParser() parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -parser.add_argument("--backend", choices=("numpy", "cupy"), help="cunumpy backend (default: the active one).") parser.add_argument("--num-elements", type=int, default=32, help="Cells in x (1 in y and z).") parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") args, _ = parser.parse_known_args() -# the backend must be chosen before any array of the model is allocated -if args.backend: - xp.set_backend(args.backend) - # --------------------- # Instance of the model # --------------------- @@ -179,7 +175,7 @@ if __name__ == "__main__": import contextlib - # Without a GPU, CUNUMPY_FAKE_CUPY=1 --backend cupy runs the CuPy code path on cunumpy's fake CuPy with every + # Without a GPU, CUNUMPY_FAKE_CUPY=1 CUNUMPY_BACKEND=cupy runs the CuPy code path on cunumpy's fake CuPy with every # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU # (or on NumPy) the context does nothing. from struphy.models.tests.gpu_e2e import device_context diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py index 6258d0c6a..6bf31db67 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py @@ -8,8 +8,9 @@ Each generated script runs the simulation itself by invoking `params_vlasov_maxwell.py` directly (its `__main__` block is the worker). -The run is profiled with scope-profiler; `--backend cupy` runs it on the GPU with CUDA-event -timing per region. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` +The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event +timing per region (default: the backend stored with `struphy backend set`); the job script exports +the choice as CUNUMPY_BACKEND. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` (defaults: 1000 markers per cell on 32 cells, 20 steps). """ @@ -47,9 +48,9 @@ def main() -> None: ) parser.add_argument( "--backend", - default="numpy", + default=None, choices=["numpy", "cupy"], - help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", + help="cunumpy backend to run on (default: 'struphy backend get'); cupy records CUDA-event GPU times.", ) parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") @@ -69,12 +70,11 @@ def main() -> None: params_source=params_dir / "params_vlasov_maxwell.py", language=args.language, compiler=args.compiler, + backend=args.backend, upload=args.upload, ) param_flags = [ - "--backend", - args.backend, "--ppc", str(args.ppc), "--num-elements", diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py index 9cf7abcda..83e906301 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py @@ -8,8 +8,9 @@ Each generated script runs the simulation itself by invoking `params_vlasov_maxwell.py` directly (its `__main__` block is the worker). -The run is profiled with scope-profiler; `--backend cupy` runs it on the GPU with CUDA-event -timing per region. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` +The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event +timing per region (default: the backend stored with `struphy backend set`); the job script exports +the choice as CUNUMPY_BACKEND. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` (defaults: 1000 markers per cell on 32 cells, 20 steps). """ @@ -47,9 +48,9 @@ def main() -> None: ) parser.add_argument( "--backend", - default="numpy", + default=None, choices=["numpy", "cupy"], - help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", + help="cunumpy backend to run on (default: 'struphy backend get'); cupy records CUDA-event GPU times.", ) parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") @@ -69,12 +70,11 @@ def main() -> None: params_source=params_dir / "params_vlasov_maxwell.py", language=args.language, compiler=args.compiler, + backend=args.backend, upload=args.upload, ) param_flags = [ - "--backend", - args.backend, "--ppc", str(args.ppc), "--num-elements", diff --git a/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py index 5f67a4aea..4ea34eccc 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py @@ -13,7 +13,8 @@ grows at the rate 0.02784. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, velocities in units of c). -Runs on the NumPy (CPU) or CuPy (GPU) backend (--backend). All timing comes from scope-profiler: the run activates +Runs on the NumPy (CPU) or CuPy (GPU) backend, chosen by the environment variable CUNUMPY_BACKEND (the profiling +driver exports it in the job script; `struphy backend set cupy` stores a default). All timing comes from scope-profiler: the run activates the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker @@ -59,16 +60,11 @@ # Unknown flags are ignored so the driver can forward other parameters as well. parser = argparse.ArgumentParser() parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -parser.add_argument("--backend", choices=("numpy", "cupy"), help="cunumpy backend (default: the active one).") parser.add_argument("--num-elements", type=int, default=32, help="Cells in x (1 in y and z).") parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") args, _ = parser.parse_known_args() -# the backend must be chosen before any array of the model is allocated -if args.backend: - xp.set_backend(args.backend) - # --------------------- # Instance of the model # --------------------- @@ -184,7 +180,7 @@ if __name__ == "__main__": import contextlib - # Without a GPU, CUNUMPY_FAKE_CUPY=1 --backend cupy runs the CuPy code path on cunumpy's fake CuPy with every + # Without a GPU, CUNUMPY_FAKE_CUPY=1 CUNUMPY_BACKEND=cupy runs the CuPy code path on cunumpy's fake CuPy with every # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU # (or on NumPy) the context does nothing. from struphy.models.tests.gpu_e2e import device_context diff --git a/profiling/profiling_job.py b/profiling/profiling_job.py index 10a7293db..ea42c996c 100644 --- a/profiling/profiling_job.py +++ b/profiling/profiling_job.py @@ -63,6 +63,7 @@ from upload import _clone_profiling_data, _push_profiling_data from struphy import Compiler +from struphy.console.backend import BACKENDS, configured_backend from utils import _git_commit, _git_commit_short, _make_unique_results_root, _slug script_dir = Path(__file__).resolve().parent @@ -101,6 +102,9 @@ class ProfilingCase: params_source: Path language: str = "fortran" # Pyccel language to compile the Struphy kernels with: "fortran" or "c". compiler: str = "GNU" # Pyccel compiler family: "GNU", "intel", "PGI", "nvidia", or "LLVM". + # cunumpy backend the runs use, "numpy" (CPU) or "cupy" (GPU): exported as CUNUMPY_BACKEND in every job + # script. None takes the backend stored with `struphy backend set` (default numpy). + backend: str | None = None # Whether the packaged results are pushed to the profiling-data repo. Set here rather # than at `finalize_run` time because `setup_run` clones that repo up front, so a # missing/unreachable repo fails before any job is submitted rather than after. @@ -171,11 +175,15 @@ def build_commands(self, ntasks: int, param_flags: list[str] | None = None) -> l ), "set -e", f"source {activate_path!s}", + # cunumpy reads its backend from this variable when it is imported, so it is exported before the run. + # Explicit, so the job script does not depend on the state file at the time it runs. + f"export CUNUMPY_BACKEND={self.backend}", 'echo "----------------------------------------"', f'echo "Running profiling case: {self.label} ({ntasks} MPI ranks)"', f'echo "Description: {self.description}"', f'echo "Physics problem: {self.physics_problem}"', f'echo "Struphy model used: {self.struphy_model_used}"', + f'echo "cunumpy backend: {self.backend}"', f'echo "Case directory: {output_root}"', 'echo "----------------------------------------"', f'mkdir -p "{sim_dir}"', @@ -329,6 +337,13 @@ def setup_run(self) -> None: ) self.venv_path = Path(virtual_env) + # The cunumpy backend of the runs: the case's own choice, else the one stored with `struphy backend set`. + if self.backend is None: + self.backend = configured_backend() + if self.backend not in BACKENDS: + raise ValueError(f"backend must be one of {BACKENDS}, got {self.backend!r}") + print(f"cunumpy backend: {self.backend}") + # Compile Struphy kernels with the case's language and compiler self.compiler_instance = Compiler(language=self.language, compiler=self.compiler) if not self.compiler_instance.compiled(language=self.language): @@ -383,6 +398,7 @@ def case_info_dict(self) -> dict: "physics_problem": self.physics_problem, "struphy_model_used": self.struphy_model_used, "struphy_commit": self.run_commit, + "backend": self.backend, "compiler": self.compiler_instance.to_dict(), "scheduler": "slurm" if self.use_slurm else "local", "parameter_file": str(self.params_source), diff --git a/src/struphy/__init__.py b/src/struphy/__init__.py index 92beb53a1..368bca93e 100644 --- a/src/struphy/__init__.py +++ b/src/struphy/__init__.py @@ -8,6 +8,33 @@ from maybempi import launched_under_mpi +def _apply_stored_backend(): + """Set ``CUNUMPY_BACKEND`` from the state file (``struphy backend set``) unless the variable is already set. + + cunumpy reads the variable when it is imported, so this runs at ``import struphy`` and has an effect only when + struphy is imported before cunumpy. The state file is read here directly (not through ``struphy.utils``, which + imports this package). + """ + if "CUNUMPY_BACKEND" in os.environ: + return + try: + import yaml + except ImportError: + return + state_file = os.path.join(os.path.dirname(__file__), "state.yml") + try: + with open(state_file) as f: + state = yaml.load(f, Loader=yaml.FullLoader) or {} + except (OSError, yaml.YAMLError): + return + backend = state.get("backend") if isinstance(state, dict) else None + if backend in ("numpy", "cupy"): + os.environ["CUNUMPY_BACKEND"] = backend + + +_apply_stored_backend() + + class RankZeroFilter(logging.Filter): def __init__(self, rank: int): super().__init__() diff --git a/src/struphy/console/backend.py b/src/struphy/console/backend.py new file mode 100644 index 000000000..66296564b --- /dev/null +++ b/src/struphy/console/backend.py @@ -0,0 +1,72 @@ +"""The ``struphy backend`` command: choose the cunumpy array backend (NumPy on the CPU or CuPy on the GPU). + +cunumpy picks its backend from the environment variable ``CUNUMPY_BACKEND`` when it is imported. ``struphy backend +set cupy`` stores the choice in struphy's state file; ``import struphy`` then sets ``CUNUMPY_BACKEND`` from it +unless the variable is already set, and the profiling driver exports it in every job script it writes. +""" + +import os + +from struphy.utils import utils + +ENV_VAR = "CUNUMPY_BACKEND" +BACKENDS = ("numpy", "cupy") +DEFAULT_BACKEND = "numpy" +STATE_KEY = "backend" + + +def configured_backend(state: dict | None = None) -> str: + """The backend stored with ``struphy backend set``, or the default ``"numpy"``.""" + if state is None: + state = utils.read_state() + backend = state.get(STATE_KEY, DEFAULT_BACKEND) + if backend not in BACKENDS: + raise ValueError(f"invalid backend {backend!r} in the state file; choose one of {BACKENDS}") + return backend + + +def struphy_backend(action: str, backend: str | None = None, value: bool = False): + """Show, store or forget the cunumpy backend of struphy runs. + + Parameters + ---------- + action : {"get", "set", "unset"} + ``get`` prints the stored backend, the environment variable and the backend cunumpy actually uses in this + process. ``set`` stores ``backend`` in the state file. ``unset`` removes it, back to the default ``numpy``. + + backend : {"numpy", "cupy"}, optional + With ``set``, the backend to store. + + value : bool + With ``get``, print only the stored backend, for use in shell scripts: ``export CUNUMPY_BACKEND=$(struphy + backend get --value)``. + """ + state = utils.read_state() + + if action == "get": + stored = configured_backend(state) + if value: + print(stored) + return + import cunumpy as xp + + print(f"stored backend (struphy backend set): {stored}" + ("" if STATE_KEY in state else " (default)")) + print(f"{ENV_VAR}: {os.environ.get(ENV_VAR, '(not set)')}") + print(f"backend in this process: {xp.get_backend()}") + elif action == "set": + if backend not in BACKENDS: + raise ValueError(f"'struphy backend set' needs a backend, one of {BACKENDS}; got {backend!r}") + state[STATE_KEY] = backend + utils.save_state(state) + print(f"stored backend: {backend}") + print(f"'import struphy' now sets {ENV_VAR}={backend} unless the variable is already set,") + print(f"and the profiling driver exports {ENV_VAR}={backend} in its job scripts.") + env = os.environ.get(ENV_VAR) + if env is not None and env != backend: + print(f"Note: {ENV_VAR}={env} is set in this shell and takes precedence; unset it or export {backend}.") + elif action == "unset": + state.pop(STATE_KEY, None) + utils.save_state(state) + print(f"stored backend removed; the default is {DEFAULT_BACKEND}") + else: + raise ValueError(f"Unknown action {action!r}") diff --git a/src/struphy/console/main.py b/src/struphy/console/main.py index c7b45fd96..a053b2e81 100644 --- a/src/struphy/console/main.py +++ b/src/struphy/console/main.py @@ -91,6 +91,9 @@ def struphy(): # 7. output inspection and post-processing add_parser_output(subparsers) + # 8. "backend" sub-command + add_parser_backend(subparsers) + # parse argument argcomplete.autocomplete(parser) args = parser.parse_args() @@ -129,6 +132,7 @@ def struphy(): "profile": ("struphy.console.profile", "struphy_profile"), "test": ("struphy.console.test", "struphy_test"), "output": ("struphy.console.output", "struphy_output"), + "backend": ("struphy.console.backend", "struphy_backend"), } # import struphy.console.MODULE.FUNC_NAME as func @@ -444,6 +448,22 @@ def add_parser_output(subparsers): parser.add_argument("--directory", help="report directory") +def add_parser_backend(subparsers): + """Add the command-line interface for the cunumpy array backend (NumPy or CuPy).""" + parser = subparsers.add_parser( + "backend", + help="show or choose the array backend of struphy runs: numpy (CPU) or cupy (GPU)", + description=( + "cunumpy picks its backend from the environment variable CUNUMPY_BACKEND. 'struphy backend set' stores " + "a choice in the state file; 'import struphy' then sets CUNUMPY_BACKEND from it unless the variable is " + "already set, and the profiling driver exports it in every job script." + ), + ) + parser.add_argument("action", choices=("get", "set", "unset")) + parser.add_argument("backend", nargs="?", choices=("numpy", "cupy"), help="with set: the backend to store") + parser.add_argument("--value", action="store_true", help="with get: print only the stored backend") + + def add_parser_test(subparsers, list_models): try: import pytest_mpi diff --git a/src/struphy/console/tests/test_backend.py b/src/struphy/console/tests/test_backend.py new file mode 100644 index 000000000..339b78eff --- /dev/null +++ b/src/struphy/console/tests/test_backend.py @@ -0,0 +1,62 @@ +"""Tests for ``struphy backend``: the cunumpy backend stored in the state file.""" + +import functools + +import pytest + +from struphy.console import backend as backend_cli +from struphy.utils import utils + + +@pytest.fixture +def state_dir(tmp_path, monkeypatch): + """Redirect the state file to a temporary folder.""" + monkeypatch.setattr(utils, "read_state", functools.partial(utils.read_state, libpath=str(tmp_path))) + monkeypatch.setattr(utils, "save_state", functools.partial(utils.save_state, libpath=str(tmp_path))) + return tmp_path + + +def test_default_backend_is_numpy(state_dir): + assert backend_cli.configured_backend() == "numpy" + + +def test_set_get_unset(state_dir, capsys): + backend_cli.struphy_backend("set", "cupy") + assert utils.read_state() == {"backend": "cupy"} + assert backend_cli.configured_backend() == "cupy" + + capsys.readouterr() # drop the output of set + backend_cli.struphy_backend("get", value=True) + assert capsys.readouterr().out.strip() == "cupy" + + backend_cli.struphy_backend("unset") + assert "backend" not in utils.read_state() + assert backend_cli.configured_backend() == "numpy" + + +def test_set_keeps_other_state(state_dir): + utils.save_state({"i_path": "inp", "kernels": ["a.py"]}) + backend_cli.struphy_backend("set", "numpy") + assert utils.read_state() == {"i_path": "inp", "kernels": ["a.py"], "backend": "numpy"} + + +def test_set_rejects_unknown_backend(state_dir): + with pytest.raises(ValueError, match="one of"): + backend_cli.struphy_backend("set", "torch") + with pytest.raises(ValueError, match="one of"): + backend_cli.struphy_backend("set") + + +def test_get_reports_stored_env_and_effective(state_dir, monkeypatch, capsys): + monkeypatch.delenv("CUNUMPY_BACKEND", raising=False) + backend_cli.struphy_backend("get") + out = capsys.readouterr().out + assert "stored backend (struphy backend set): numpy (default)" in out + assert "CUNUMPY_BACKEND: (not set)" in out + assert "backend in this process: numpy" in out + + +def test_invalid_backend_in_state_file_is_rejected(state_dir): + utils.save_state({"backend": "torch"}) + with pytest.raises(ValueError, match="invalid backend"): + backend_cli.configured_backend() From 7a6b83f6f69ca0099468b74f848f4677886b45b1 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 13 Oct 2026 10:55:49 +0200 Subject: [PATCH 07/12] Always overwrite the environment variable with set --- doc/sections/install.rst | 2 +- .../landau_damping/params_vlasov_ampere.py | 6 +-- .../submit_vlasov_ampere_landau_damping.py | 4 +- .../landau_damping/params_vlasov_maxwell.py | 6 +-- .../submit_vlasov_maxwell_landau_damping.py | 4 +- ...ubmit_vlasov_maxwell_weibel_instability.py | 4 +- .../params_vlasov_maxwell.py | 6 +-- profiling/profiling_job.py | 10 ++-- src/struphy/__init__.py | 11 ++--- src/struphy/console/backend.py | 49 +++++++++++++------ src/struphy/console/main.py | 4 +- src/struphy/console/tests/test_backend.py | 30 ++++++++++-- 12 files changed, 87 insertions(+), 49 deletions(-) diff --git a/doc/sections/install.rst b/doc/sections/install.rst index 7c227a47b..6ab976a04 100644 --- a/doc/sections/install.rst +++ b/doc/sections/install.rst @@ -229,7 +229,7 @@ The choice can also be stored with the command line, so it does not have to be r struphy backend get struphy backend unset -``import struphy`` then sets ``CUNUMPY_BACKEND`` from the stored choice, unless the variable is already set in the shell (an explicit variable always wins). This takes effect only when struphy is imported before cunumpy, as the parameter files generated by ``struphy params`` do. The profiling driver in ``profiling/`` exports the variable in every job script it writes, so profiling cases run on the stored backend unless their submit script is given ``--backend``. ``struphy backend get --value`` prints just the stored backend, for shell scripts. +The stored choice is the single source of truth: ``import struphy`` sets ``CUNUMPY_BACKEND`` to it, overwriting a value already in the shell. This takes effect only when struphy is imported before cunumpy, as the parameter files generated by ``struphy params`` do. Without a stored choice, cunumpy reads the variable from the shell as usual. The profiling driver in ``profiling/`` runs ``struphy backend set`` in every job script it writes, with the backend of the case (its submit script's ``--backend``, or the stored one). Since the state file belongs to the installation, this changes the backend for every run of that environment, including jobs of other cases that start afterwards. ``struphy backend get --value`` prints just the stored backend, for shell scripts. With CuPy requested on a machine without a working GPU, cunumpy falls back to NumPy; ``struphy backend get`` shows the backend actually in use. diff --git a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py index 9143dd44a..06cee7d10 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py +++ b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py @@ -12,8 +12,8 @@ rate -0.1533. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, velocities in units of c). -Runs on the NumPy (CPU) or CuPy (GPU) backend, chosen by the environment variable CUNUMPY_BACKEND (the profiling -driver exports it in the job script; `struphy backend set cupy` stores a default). All timing comes from scope-profiler: the run activates +Runs on the NumPy (CPU) or CuPy (GPU) backend chosen with `struphy backend set` (the profiling driver runs it in +the job script). All timing comes from scope-profiler: the run activates the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker @@ -174,7 +174,7 @@ if __name__ == "__main__": import contextlib - # Without a GPU, CUNUMPY_FAKE_CUPY=1 CUNUMPY_BACKEND=cupy runs the CuPy code path on cunumpy's fake CuPy with every + # Without a GPU, `struphy backend set cupy` and CUNUMPY_FAKE_CUPY=1 run the CuPy code path on cunumpy's fake CuPy with every # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU # (or on NumPy) the context does nothing. from struphy.models.tests.gpu_e2e import device_context diff --git a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py index 00b8e2d06..e66bd1780 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py +++ b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py @@ -9,8 +9,8 @@ directly (its `__main__` block is the worker). The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event -timing per region (default: the backend stored with `struphy backend set`); the job script exports -the choice as CUNUMPY_BACKEND. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_ampere.py` +timing per region (default: the backend stored with `struphy backend set`); the job script runs +`struphy backend set` with the choice before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_ampere.py` (defaults: 1000 markers per cell on 32 cells, 20 steps). """ diff --git a/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py index d1e30fd6d..cbbb78773 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py @@ -12,8 +12,8 @@ rate -0.1533. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, velocities in units of c). -Runs on the NumPy (CPU) or CuPy (GPU) backend, chosen by the environment variable CUNUMPY_BACKEND (the profiling -driver exports it in the job script; `struphy backend set cupy` stores a default). All timing comes from scope-profiler: the run activates +Runs on the NumPy (CPU) or CuPy (GPU) backend chosen with `struphy backend set` (the profiling driver runs it in +the job script). All timing comes from scope-profiler: the run activates the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker @@ -175,7 +175,7 @@ if __name__ == "__main__": import contextlib - # Without a GPU, CUNUMPY_FAKE_CUPY=1 CUNUMPY_BACKEND=cupy runs the CuPy code path on cunumpy's fake CuPy with every + # Without a GPU, `struphy backend set cupy` and CUNUMPY_FAKE_CUPY=1 run the CuPy code path on cunumpy's fake CuPy with every # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU # (or on NumPy) the context does nothing. from struphy.models.tests.gpu_e2e import device_context diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py index 6bf31db67..5aeb40347 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py @@ -9,8 +9,8 @@ directly (its `__main__` block is the worker). The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event -timing per region (default: the backend stored with `struphy backend set`); the job script exports -the choice as CUNUMPY_BACKEND. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` +timing per region (default: the backend stored with `struphy backend set`); the job script runs +`struphy backend set` with the choice before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` (defaults: 1000 markers per cell on 32 cells, 20 steps). """ diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py index 83e906301..cafacd950 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py @@ -9,8 +9,8 @@ directly (its `__main__` block is the worker). The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event -timing per region (default: the backend stored with `struphy backend set`); the job script exports -the choice as CUNUMPY_BACKEND. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` +timing per region (default: the backend stored with `struphy backend set`); the job script runs +`struphy backend set` with the choice before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` (defaults: 1000 markers per cell on 32 cells, 20 steps). """ diff --git a/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py index 4ea34eccc..415578077 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py @@ -13,8 +13,8 @@ grows at the rate 0.02784. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, velocities in units of c). -Runs on the NumPy (CPU) or CuPy (GPU) backend, chosen by the environment variable CUNUMPY_BACKEND (the profiling -driver exports it in the job script; `struphy backend set cupy` stores a default). All timing comes from scope-profiler: the run activates +Runs on the NumPy (CPU) or CuPy (GPU) backend chosen with `struphy backend set` (the profiling driver runs it in +the job script). All timing comes from scope-profiler: the run activates the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker @@ -180,7 +180,7 @@ if __name__ == "__main__": import contextlib - # Without a GPU, CUNUMPY_FAKE_CUPY=1 CUNUMPY_BACKEND=cupy runs the CuPy code path on cunumpy's fake CuPy with every + # Without a GPU, `struphy backend set cupy` and CUNUMPY_FAKE_CUPY=1 run the CuPy code path on cunumpy's fake CuPy with every # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU # (or on NumPy) the context does nothing. from struphy.models.tests.gpu_e2e import device_context diff --git a/profiling/profiling_job.py b/profiling/profiling_job.py index ea42c996c..740f66364 100644 --- a/profiling/profiling_job.py +++ b/profiling/profiling_job.py @@ -102,8 +102,8 @@ class ProfilingCase: params_source: Path language: str = "fortran" # Pyccel language to compile the Struphy kernels with: "fortran" or "c". compiler: str = "GNU" # Pyccel compiler family: "GNU", "intel", "PGI", "nvidia", or "LLVM". - # cunumpy backend the runs use, "numpy" (CPU) or "cupy" (GPU): exported as CUNUMPY_BACKEND in every job - # script. None takes the backend stored with `struphy backend set` (default numpy). + # cunumpy backend the runs use, "numpy" (CPU) or "cupy" (GPU): every job script runs `struphy backend set` + # with it before the simulation. None keeps the backend stored with `struphy backend set` (default numpy). backend: str | None = None # Whether the packaged results are pushed to the profiling-data repo. Set here rather # than at `finalize_run` time because `setup_run` clones that repo up front, so a @@ -175,9 +175,9 @@ def build_commands(self, ntasks: int, param_flags: list[str] | None = None) -> l ), "set -e", f"source {activate_path!s}", - # cunumpy reads its backend from this variable when it is imported, so it is exported before the run. - # Explicit, so the job script does not depend on the state file at the time it runs. - f"export CUNUMPY_BACKEND={self.backend}", + # the stored backend is the single source of truth: `import struphy` sets CUNUMPY_BACKEND from it, + # so the run below picks it up. Note that the state file is shared by every job of this venv. + f"{self.venv_path / 'bin' / 'struphy'} backend set {self.backend}", 'echo "----------------------------------------"', f'echo "Running profiling case: {self.label} ({ntasks} MPI ranks)"', f'echo "Description: {self.description}"', diff --git a/src/struphy/__init__.py b/src/struphy/__init__.py index 368bca93e..6994a14f0 100644 --- a/src/struphy/__init__.py +++ b/src/struphy/__init__.py @@ -9,14 +9,13 @@ def _apply_stored_backend(): - """Set ``CUNUMPY_BACKEND`` from the state file (``struphy backend set``) unless the variable is already set. + """Set ``CUNUMPY_BACKEND`` to the backend stored with ``struphy backend set``, if there is one. - cunumpy reads the variable when it is imported, so this runs at ``import struphy`` and has an effect only when - struphy is imported before cunumpy. The state file is read here directly (not through ``struphy.utils``, which - imports this package). + The stored choice is the single source of truth: it overwrites a ``CUNUMPY_BACKEND`` already in the + environment. Without a stored choice the variable is left alone. cunumpy reads the variable when it is + imported, so this runs at ``import struphy`` and has an effect only when struphy is imported before cunumpy. + The state file is read here directly (not through ``struphy.utils``, which imports this package). """ - if "CUNUMPY_BACKEND" in os.environ: - return try: import yaml except ImportError: diff --git a/src/struphy/console/backend.py b/src/struphy/console/backend.py index 66296564b..ee2d4f495 100644 --- a/src/struphy/console/backend.py +++ b/src/struphy/console/backend.py @@ -1,8 +1,9 @@ """The ``struphy backend`` command: choose the cunumpy array backend (NumPy on the CPU or CuPy on the GPU). cunumpy picks its backend from the environment variable ``CUNUMPY_BACKEND`` when it is imported. ``struphy backend -set cupy`` stores the choice in struphy's state file; ``import struphy`` then sets ``CUNUMPY_BACKEND`` from it -unless the variable is already set, and the profiling driver exports it in every job script it writes. +set cupy`` stores the choice in struphy's state file, and ``import struphy`` then sets ``CUNUMPY_BACKEND`` to it, +overwriting whatever the shell had: the stored choice is the single source of truth. The profiling driver runs +``struphy backend set`` in every job script before the simulation. """ import os @@ -31,15 +32,15 @@ def struphy_backend(action: str, backend: str | None = None, value: bool = False Parameters ---------- action : {"get", "set", "unset"} - ``get`` prints the stored backend, the environment variable and the backend cunumpy actually uses in this - process. ``set`` stores ``backend`` in the state file. ``unset`` removes it, back to the default ``numpy``. + ``get`` prints the stored backend and the backend cunumpy can provide on this machine. ``set`` stores + ``backend`` in the state file. ``unset`` removes it; cunumpy then falls back to ``CUNUMPY_BACKEND`` from the + shell, or NumPy. backend : {"numpy", "cupy"}, optional With ``set``, the backend to store. value : bool - With ``get``, print only the stored backend, for use in shell scripts: ``export CUNUMPY_BACKEND=$(struphy - backend get --value)``. + With ``get``, print only the stored backend, for use in shell scripts. """ state = utils.read_state() @@ -50,23 +51,39 @@ def struphy_backend(action: str, backend: str | None = None, value: bool = False return import cunumpy as xp - print(f"stored backend (struphy backend set): {stored}" + ("" if STATE_KEY in state else " (default)")) - print(f"{ENV_VAR}: {os.environ.get(ENV_VAR, '(not set)')}") - print(f"backend in this process: {xp.get_backend()}") + # what cunumpy could provide here: it falls back to NumPy when CuPy or CUDA is not functional + effective = xp.get_backend() + + if STATE_KEY in state: + print(f"Backend struphy runs use (struphy backend set): {stored}") + else: + env = os.environ.get(ENV_VAR) + if env is None: + print(f"Backend struphy runs use: {DEFAULT_BACKEND} (nothing stored, cunumpy's default)") + elif env in BACKENDS: + print(f"Backend struphy runs use: {env} (nothing stored, from {ENV_VAR} in the shell)") + else: + print( + f"Backend struphy runs use: {DEFAULT_BACKEND} " + f"(nothing stored; {ENV_VAR}={env} in the shell is not a backend, cunumpy ignores it)" + ) + print(f"Backend available on this machine: {effective}") + requested = stored if STATE_KEY in state else os.environ.get(ENV_VAR, DEFAULT_BACKEND) + if requested in BACKENDS and effective != requested: + print() + print( + f"Warning: {requested} is chosen but cunumpy can only provide {effective} here " + f"(CuPy or CUDA is not functional on this machine). Runs on this machine use {effective}." + ) elif action == "set": if backend not in BACKENDS: raise ValueError(f"'struphy backend set' needs a backend, one of {BACKENDS}; got {backend!r}") state[STATE_KEY] = backend utils.save_state(state) - print(f"stored backend: {backend}") - print(f"'import struphy' now sets {ENV_VAR}={backend} unless the variable is already set,") - print(f"and the profiling driver exports {ENV_VAR}={backend} in its job scripts.") - env = os.environ.get(ENV_VAR) - if env is not None and env != backend: - print(f"Note: {ENV_VAR}={env} is set in this shell and takes precedence; unset it or export {backend}.") + print(f"struphy runs now use the {backend} backend ('import struphy' sets {ENV_VAR}={backend}).") elif action == "unset": state.pop(STATE_KEY, None) utils.save_state(state) - print(f"stored backend removed; the default is {DEFAULT_BACKEND}") + print(f"stored backend removed; runs use {ENV_VAR} from the shell, or {DEFAULT_BACKEND}") else: raise ValueError(f"Unknown action {action!r}") diff --git a/src/struphy/console/main.py b/src/struphy/console/main.py index a053b2e81..2a771a108 100644 --- a/src/struphy/console/main.py +++ b/src/struphy/console/main.py @@ -455,8 +455,8 @@ def add_parser_backend(subparsers): help="show or choose the array backend of struphy runs: numpy (CPU) or cupy (GPU)", description=( "cunumpy picks its backend from the environment variable CUNUMPY_BACKEND. 'struphy backend set' stores " - "a choice in the state file; 'import struphy' then sets CUNUMPY_BACKEND from it unless the variable is " - "already set, and the profiling driver exports it in every job script." + "a choice in the state file, and 'import struphy' then sets CUNUMPY_BACKEND to it (overwriting the " + "shell's value). The profiling driver runs 'struphy backend set' in every job script." ), ) parser.add_argument("action", choices=("get", "set", "unset")) diff --git a/src/struphy/console/tests/test_backend.py b/src/struphy/console/tests/test_backend.py index 339b78eff..8eedf5719 100644 --- a/src/struphy/console/tests/test_backend.py +++ b/src/struphy/console/tests/test_backend.py @@ -47,13 +47,35 @@ def test_set_rejects_unknown_backend(state_dir): backend_cli.struphy_backend("set") -def test_get_reports_stored_env_and_effective(state_dir, monkeypatch, capsys): +def test_get_with_nothing_stored(state_dir, monkeypatch, capsys): monkeypatch.delenv("CUNUMPY_BACKEND", raising=False) backend_cli.struphy_backend("get") out = capsys.readouterr().out - assert "stored backend (struphy backend set): numpy (default)" in out - assert "CUNUMPY_BACKEND: (not set)" in out - assert "backend in this process: numpy" in out + assert "Backend struphy runs use: numpy (nothing stored, cunumpy's default)" in out + assert "Backend available on this machine: numpy" in out + assert "Warning" not in out + + +def test_get_with_nothing_stored_reports_shell_variable(state_dir, monkeypatch, capsys): + monkeypatch.setenv("CUNUMPY_BACKEND", "xyz") + backend_cli.struphy_backend("get") + out = capsys.readouterr().out + assert "CUNUMPY_BACKEND=xyz in the shell is not a backend, cunumpy ignores it" in out + assert "Warning" not in out + + +def test_get_warns_when_cupy_is_unavailable(state_dir, capsys): + import cunumpy as xp + + if xp.get_backend() == "cupy": + pytest.skip("a working GPU: no fallback to warn about") + backend_cli.struphy_backend("set", "cupy") + capsys.readouterr() + backend_cli.struphy_backend("get") + out = capsys.readouterr().out + assert "Backend struphy runs use (struphy backend set): cupy" in out + assert "Backend available on this machine: numpy" in out + assert "Warning: cupy is chosen but cunumpy can only provide numpy here" in out def test_invalid_backend_in_state_file_is_rejected(state_dir): From 78217d9efb43a992aefe98125357dfccf24f914b Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 13 Oct 2026 11:17:02 +0200 Subject: [PATCH 08/12] Back to export... --- doc/sections/install.rst | 19 ---- .../landau_damping/params_vlasov_ampere.py | 6 +- .../submit_vlasov_ampere_landau_damping.py | 7 +- .../landau_damping/params_vlasov_maxwell.py | 6 +- .../submit_vlasov_maxwell_landau_damping.py | 7 +- ...ubmit_vlasov_maxwell_weibel_instability.py | 7 +- .../params_vlasov_maxwell.py | 6 +- profiling/profiling_job.py | 20 ++--- src/struphy/__init__.py | 26 ------ src/struphy/console/backend.py | 89 ------------------- src/struphy/console/main.py | 20 ----- src/struphy/console/tests/test_backend.py | 84 ----------------- 12 files changed, 25 insertions(+), 272 deletions(-) delete mode 100644 src/struphy/console/backend.py delete mode 100644 src/struphy/console/tests/test_backend.py diff --git a/doc/sections/install.rst b/doc/sections/install.rst index 6ab976a04..86de999e3 100644 --- a/doc/sections/install.rst +++ b/doc/sections/install.rst @@ -214,25 +214,6 @@ This compiles at most 4 kernels at the same time; kernels that depend on each ot In case you encounter problems during install visit :ref:`trouble_shoot`. -.. _gpu_backend: - -GPU backend ------------ - -Struphy's arrays live in `cunumpy `_, which runs on NumPy (CPU) or CuPy (GPU). cunumpy picks the backend from the environment variable ``CUNUMPY_BACKEND`` when it is imported:: - - CUNUMPY_BACKEND=cupy python params_maxwell.py - -The choice can also be stored with the command line, so it does not have to be repeated:: - - struphy backend set cupy - struphy backend get - struphy backend unset - -The stored choice is the single source of truth: ``import struphy`` sets ``CUNUMPY_BACKEND`` to it, overwriting a value already in the shell. This takes effect only when struphy is imported before cunumpy, as the parameter files generated by ``struphy params`` do. Without a stored choice, cunumpy reads the variable from the shell as usual. The profiling driver in ``profiling/`` runs ``struphy backend set`` in every job script it writes, with the backend of the case (its submit script's ``--backend``, or the stored one). Since the state file belongs to the installation, this changes the backend for every run of that environment, including jobs of other cases that start afterwards. ``struphy backend get --value`` prints just the stored backend, for shell scripts. - -With CuPy requested on a machine without a working GPU, cunumpy falls back to NumPy; ``struphy backend get`` shows the backend actually in use. - .. _trouble_shoot: Trouble shooting diff --git a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py index 06cee7d10..e0d5b67f9 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py +++ b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py @@ -12,8 +12,8 @@ rate -0.1533. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, velocities in units of c). -Runs on the NumPy (CPU) or CuPy (GPU) backend chosen with `struphy backend set` (the profiling driver runs it in -the job script). All timing comes from scope-profiler: the run activates +Runs on the NumPy (CPU) or CuPy (GPU) backend, chosen by the environment variable CUNUMPY_BACKEND (the profiling +driver exports it in the job script). All timing comes from scope-profiler: the run activates the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker @@ -174,7 +174,7 @@ if __name__ == "__main__": import contextlib - # Without a GPU, `struphy backend set cupy` and CUNUMPY_FAKE_CUPY=1 run the CuPy code path on cunumpy's fake CuPy with every + # Without a GPU, CUNUMPY_FAKE_CUPY=1 CUNUMPY_BACKEND=cupy runs the CuPy code path on cunumpy's fake CuPy with every # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU # (or on NumPy) the context does nothing. from struphy.models.tests.gpu_e2e import device_context diff --git a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py index e66bd1780..63d13b99f 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py +++ b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py @@ -9,8 +9,7 @@ directly (its `__main__` block is the worker). The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event -timing per region (default: the backend stored with `struphy backend set`); the job script runs -`struphy backend set` with the choice before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_ampere.py` +timing per region; the job script exports the choice as CUNUMPY_BACKEND before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_ampere.py` (defaults: 1000 markers per cell on 32 cells, 20 steps). """ @@ -48,9 +47,9 @@ def main() -> None: ) parser.add_argument( "--backend", - default=None, + default="numpy", choices=["numpy", "cupy"], - help="cunumpy backend to run on (default: 'struphy backend get'); cupy records CUDA-event GPU times.", + help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", ) parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") diff --git a/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py index cbbb78773..be311ade1 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py @@ -12,8 +12,8 @@ rate -0.1533. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, velocities in units of c). -Runs on the NumPy (CPU) or CuPy (GPU) backend chosen with `struphy backend set` (the profiling driver runs it in -the job script). All timing comes from scope-profiler: the run activates +Runs on the NumPy (CPU) or CuPy (GPU) backend, chosen by the environment variable CUNUMPY_BACKEND (the profiling +driver exports it in the job script). All timing comes from scope-profiler: the run activates the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker @@ -175,7 +175,7 @@ if __name__ == "__main__": import contextlib - # Without a GPU, `struphy backend set cupy` and CUNUMPY_FAKE_CUPY=1 run the CuPy code path on cunumpy's fake CuPy with every + # Without a GPU, CUNUMPY_FAKE_CUPY=1 CUNUMPY_BACKEND=cupy runs the CuPy code path on cunumpy's fake CuPy with every # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU # (or on NumPy) the context does nothing. from struphy.models.tests.gpu_e2e import device_context diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py index 5aeb40347..8b6eecc32 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py @@ -9,8 +9,7 @@ directly (its `__main__` block is the worker). The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event -timing per region (default: the backend stored with `struphy backend set`); the job script runs -`struphy backend set` with the choice before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` +timing per region; the job script exports the choice as CUNUMPY_BACKEND before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` (defaults: 1000 markers per cell on 32 cells, 20 steps). """ @@ -48,9 +47,9 @@ def main() -> None: ) parser.add_argument( "--backend", - default=None, + default="numpy", choices=["numpy", "cupy"], - help="cunumpy backend to run on (default: 'struphy backend get'); cupy records CUDA-event GPU times.", + help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", ) parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py index cafacd950..8517e119a 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py @@ -9,8 +9,7 @@ directly (its `__main__` block is the worker). The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event -timing per region (default: the backend stored with `struphy backend set`); the job script runs -`struphy backend set` with the choice before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` +timing per region; the job script exports the choice as CUNUMPY_BACKEND before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` (defaults: 1000 markers per cell on 32 cells, 20 steps). """ @@ -48,9 +47,9 @@ def main() -> None: ) parser.add_argument( "--backend", - default=None, + default="numpy", choices=["numpy", "cupy"], - help="cunumpy backend to run on (default: 'struphy backend get'); cupy records CUDA-event GPU times.", + help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", ) parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") diff --git a/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py index 415578077..b4a8fcec4 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py @@ -13,8 +13,8 @@ grows at the rate 0.02784. alpha = 1 and epsilon = -1 (electrons, time in units of the inverse plasma frequency, velocities in units of c). -Runs on the NumPy (CPU) or CuPy (GPU) backend chosen with `struphy backend set` (the profiling driver runs it in -the job script). All timing comes from scope-profiler: the run activates +Runs on the NumPy (CPU) or CuPy (GPU) backend, chosen by the environment variable CUNUMPY_BACKEND (the profiling +driver exports it in the job script). All timing comes from scope-profiler: the run activates the profiling session of Simulation.run, and on the CuPy backend the session records CUDA-event device time per region (GPUOptions.timing) and synchronizes the device at region ends, so the CPU time of a region includes its kernels. The post-processing writes the scope-profiler HTML report and a summary of the cost per step and marker @@ -180,7 +180,7 @@ if __name__ == "__main__": import contextlib - # Without a GPU, `struphy backend set cupy` and CUNUMPY_FAKE_CUPY=1 run the CuPy code path on cunumpy's fake CuPy with every + # Without a GPU, CUNUMPY_FAKE_CUPY=1 CUNUMPY_BACKEND=cupy runs the CuPy code path on cunumpy's fake CuPy with every # CUDA launch emulated on the CPU: a check that the model runs on the device backend, not a timing. On a GPU # (or on NumPy) the context does nothing. from struphy.models.tests.gpu_e2e import device_context diff --git a/profiling/profiling_job.py b/profiling/profiling_job.py index 740f66364..bd7c932da 100644 --- a/profiling/profiling_job.py +++ b/profiling/profiling_job.py @@ -63,7 +63,6 @@ from upload import _clone_profiling_data, _push_profiling_data from struphy import Compiler -from struphy.console.backend import BACKENDS, configured_backend from utils import _git_commit, _git_commit_short, _make_unique_results_root, _slug script_dir = Path(__file__).resolve().parent @@ -102,9 +101,9 @@ class ProfilingCase: params_source: Path language: str = "fortran" # Pyccel language to compile the Struphy kernels with: "fortran" or "c". compiler: str = "GNU" # Pyccel compiler family: "GNU", "intel", "PGI", "nvidia", or "LLVM". - # cunumpy backend the runs use, "numpy" (CPU) or "cupy" (GPU): every job script runs `struphy backend set` - # with it before the simulation. None keeps the backend stored with `struphy backend set` (default numpy). - backend: str | None = None + # cunumpy backend the runs use, "numpy" (CPU) or "cupy" (GPU): exported as CUNUMPY_BACKEND in every job + # script before the simulation. + backend: str = "numpy" # Whether the packaged results are pushed to the profiling-data repo. Set here rather # than at `finalize_run` time because `setup_run` clones that repo up front, so a # missing/unreachable repo fails before any job is submitted rather than after. @@ -175,9 +174,8 @@ def build_commands(self, ntasks: int, param_flags: list[str] | None = None) -> l ), "set -e", f"source {activate_path!s}", - # the stored backend is the single source of truth: `import struphy` sets CUNUMPY_BACKEND from it, - # so the run below picks it up. Note that the state file is shared by every job of this venv. - f"{self.venv_path / 'bin' / 'struphy'} backend set {self.backend}", + # cunumpy reads its backend from this variable when it is imported, so it is exported before the run + f"export CUNUMPY_BACKEND={self.backend}", 'echo "----------------------------------------"', f'echo "Running profiling case: {self.label} ({ntasks} MPI ranks)"', f'echo "Description: {self.description}"', @@ -337,12 +335,8 @@ def setup_run(self) -> None: ) self.venv_path = Path(virtual_env) - # The cunumpy backend of the runs: the case's own choice, else the one stored with `struphy backend set`. - if self.backend is None: - self.backend = configured_backend() - if self.backend not in BACKENDS: - raise ValueError(f"backend must be one of {BACKENDS}, got {self.backend!r}") - print(f"cunumpy backend: {self.backend}") + if self.backend not in ("numpy", "cupy"): + raise ValueError(f"backend must be 'numpy' or 'cupy', got {self.backend!r}") # Compile Struphy kernels with the case's language and compiler self.compiler_instance = Compiler(language=self.language, compiler=self.compiler) diff --git a/src/struphy/__init__.py b/src/struphy/__init__.py index 6994a14f0..92beb53a1 100644 --- a/src/struphy/__init__.py +++ b/src/struphy/__init__.py @@ -8,32 +8,6 @@ from maybempi import launched_under_mpi -def _apply_stored_backend(): - """Set ``CUNUMPY_BACKEND`` to the backend stored with ``struphy backend set``, if there is one. - - The stored choice is the single source of truth: it overwrites a ``CUNUMPY_BACKEND`` already in the - environment. Without a stored choice the variable is left alone. cunumpy reads the variable when it is - imported, so this runs at ``import struphy`` and has an effect only when struphy is imported before cunumpy. - The state file is read here directly (not through ``struphy.utils``, which imports this package). - """ - try: - import yaml - except ImportError: - return - state_file = os.path.join(os.path.dirname(__file__), "state.yml") - try: - with open(state_file) as f: - state = yaml.load(f, Loader=yaml.FullLoader) or {} - except (OSError, yaml.YAMLError): - return - backend = state.get("backend") if isinstance(state, dict) else None - if backend in ("numpy", "cupy"): - os.environ["CUNUMPY_BACKEND"] = backend - - -_apply_stored_backend() - - class RankZeroFilter(logging.Filter): def __init__(self, rank: int): super().__init__() diff --git a/src/struphy/console/backend.py b/src/struphy/console/backend.py deleted file mode 100644 index ee2d4f495..000000000 --- a/src/struphy/console/backend.py +++ /dev/null @@ -1,89 +0,0 @@ -"""The ``struphy backend`` command: choose the cunumpy array backend (NumPy on the CPU or CuPy on the GPU). - -cunumpy picks its backend from the environment variable ``CUNUMPY_BACKEND`` when it is imported. ``struphy backend -set cupy`` stores the choice in struphy's state file, and ``import struphy`` then sets ``CUNUMPY_BACKEND`` to it, -overwriting whatever the shell had: the stored choice is the single source of truth. The profiling driver runs -``struphy backend set`` in every job script before the simulation. -""" - -import os - -from struphy.utils import utils - -ENV_VAR = "CUNUMPY_BACKEND" -BACKENDS = ("numpy", "cupy") -DEFAULT_BACKEND = "numpy" -STATE_KEY = "backend" - - -def configured_backend(state: dict | None = None) -> str: - """The backend stored with ``struphy backend set``, or the default ``"numpy"``.""" - if state is None: - state = utils.read_state() - backend = state.get(STATE_KEY, DEFAULT_BACKEND) - if backend not in BACKENDS: - raise ValueError(f"invalid backend {backend!r} in the state file; choose one of {BACKENDS}") - return backend - - -def struphy_backend(action: str, backend: str | None = None, value: bool = False): - """Show, store or forget the cunumpy backend of struphy runs. - - Parameters - ---------- - action : {"get", "set", "unset"} - ``get`` prints the stored backend and the backend cunumpy can provide on this machine. ``set`` stores - ``backend`` in the state file. ``unset`` removes it; cunumpy then falls back to ``CUNUMPY_BACKEND`` from the - shell, or NumPy. - - backend : {"numpy", "cupy"}, optional - With ``set``, the backend to store. - - value : bool - With ``get``, print only the stored backend, for use in shell scripts. - """ - state = utils.read_state() - - if action == "get": - stored = configured_backend(state) - if value: - print(stored) - return - import cunumpy as xp - - # what cunumpy could provide here: it falls back to NumPy when CuPy or CUDA is not functional - effective = xp.get_backend() - - if STATE_KEY in state: - print(f"Backend struphy runs use (struphy backend set): {stored}") - else: - env = os.environ.get(ENV_VAR) - if env is None: - print(f"Backend struphy runs use: {DEFAULT_BACKEND} (nothing stored, cunumpy's default)") - elif env in BACKENDS: - print(f"Backend struphy runs use: {env} (nothing stored, from {ENV_VAR} in the shell)") - else: - print( - f"Backend struphy runs use: {DEFAULT_BACKEND} " - f"(nothing stored; {ENV_VAR}={env} in the shell is not a backend, cunumpy ignores it)" - ) - print(f"Backend available on this machine: {effective}") - requested = stored if STATE_KEY in state else os.environ.get(ENV_VAR, DEFAULT_BACKEND) - if requested in BACKENDS and effective != requested: - print() - print( - f"Warning: {requested} is chosen but cunumpy can only provide {effective} here " - f"(CuPy or CUDA is not functional on this machine). Runs on this machine use {effective}." - ) - elif action == "set": - if backend not in BACKENDS: - raise ValueError(f"'struphy backend set' needs a backend, one of {BACKENDS}; got {backend!r}") - state[STATE_KEY] = backend - utils.save_state(state) - print(f"struphy runs now use the {backend} backend ('import struphy' sets {ENV_VAR}={backend}).") - elif action == "unset": - state.pop(STATE_KEY, None) - utils.save_state(state) - print(f"stored backend removed; runs use {ENV_VAR} from the shell, or {DEFAULT_BACKEND}") - else: - raise ValueError(f"Unknown action {action!r}") diff --git a/src/struphy/console/main.py b/src/struphy/console/main.py index 2a771a108..c7b45fd96 100644 --- a/src/struphy/console/main.py +++ b/src/struphy/console/main.py @@ -91,9 +91,6 @@ def struphy(): # 7. output inspection and post-processing add_parser_output(subparsers) - # 8. "backend" sub-command - add_parser_backend(subparsers) - # parse argument argcomplete.autocomplete(parser) args = parser.parse_args() @@ -132,7 +129,6 @@ def struphy(): "profile": ("struphy.console.profile", "struphy_profile"), "test": ("struphy.console.test", "struphy_test"), "output": ("struphy.console.output", "struphy_output"), - "backend": ("struphy.console.backend", "struphy_backend"), } # import struphy.console.MODULE.FUNC_NAME as func @@ -448,22 +444,6 @@ def add_parser_output(subparsers): parser.add_argument("--directory", help="report directory") -def add_parser_backend(subparsers): - """Add the command-line interface for the cunumpy array backend (NumPy or CuPy).""" - parser = subparsers.add_parser( - "backend", - help="show or choose the array backend of struphy runs: numpy (CPU) or cupy (GPU)", - description=( - "cunumpy picks its backend from the environment variable CUNUMPY_BACKEND. 'struphy backend set' stores " - "a choice in the state file, and 'import struphy' then sets CUNUMPY_BACKEND to it (overwriting the " - "shell's value). The profiling driver runs 'struphy backend set' in every job script." - ), - ) - parser.add_argument("action", choices=("get", "set", "unset")) - parser.add_argument("backend", nargs="?", choices=("numpy", "cupy"), help="with set: the backend to store") - parser.add_argument("--value", action="store_true", help="with get: print only the stored backend") - - def add_parser_test(subparsers, list_models): try: import pytest_mpi diff --git a/src/struphy/console/tests/test_backend.py b/src/struphy/console/tests/test_backend.py deleted file mode 100644 index 8eedf5719..000000000 --- a/src/struphy/console/tests/test_backend.py +++ /dev/null @@ -1,84 +0,0 @@ -"""Tests for ``struphy backend``: the cunumpy backend stored in the state file.""" - -import functools - -import pytest - -from struphy.console import backend as backend_cli -from struphy.utils import utils - - -@pytest.fixture -def state_dir(tmp_path, monkeypatch): - """Redirect the state file to a temporary folder.""" - monkeypatch.setattr(utils, "read_state", functools.partial(utils.read_state, libpath=str(tmp_path))) - monkeypatch.setattr(utils, "save_state", functools.partial(utils.save_state, libpath=str(tmp_path))) - return tmp_path - - -def test_default_backend_is_numpy(state_dir): - assert backend_cli.configured_backend() == "numpy" - - -def test_set_get_unset(state_dir, capsys): - backend_cli.struphy_backend("set", "cupy") - assert utils.read_state() == {"backend": "cupy"} - assert backend_cli.configured_backend() == "cupy" - - capsys.readouterr() # drop the output of set - backend_cli.struphy_backend("get", value=True) - assert capsys.readouterr().out.strip() == "cupy" - - backend_cli.struphy_backend("unset") - assert "backend" not in utils.read_state() - assert backend_cli.configured_backend() == "numpy" - - -def test_set_keeps_other_state(state_dir): - utils.save_state({"i_path": "inp", "kernels": ["a.py"]}) - backend_cli.struphy_backend("set", "numpy") - assert utils.read_state() == {"i_path": "inp", "kernels": ["a.py"], "backend": "numpy"} - - -def test_set_rejects_unknown_backend(state_dir): - with pytest.raises(ValueError, match="one of"): - backend_cli.struphy_backend("set", "torch") - with pytest.raises(ValueError, match="one of"): - backend_cli.struphy_backend("set") - - -def test_get_with_nothing_stored(state_dir, monkeypatch, capsys): - monkeypatch.delenv("CUNUMPY_BACKEND", raising=False) - backend_cli.struphy_backend("get") - out = capsys.readouterr().out - assert "Backend struphy runs use: numpy (nothing stored, cunumpy's default)" in out - assert "Backend available on this machine: numpy" in out - assert "Warning" not in out - - -def test_get_with_nothing_stored_reports_shell_variable(state_dir, monkeypatch, capsys): - monkeypatch.setenv("CUNUMPY_BACKEND", "xyz") - backend_cli.struphy_backend("get") - out = capsys.readouterr().out - assert "CUNUMPY_BACKEND=xyz in the shell is not a backend, cunumpy ignores it" in out - assert "Warning" not in out - - -def test_get_warns_when_cupy_is_unavailable(state_dir, capsys): - import cunumpy as xp - - if xp.get_backend() == "cupy": - pytest.skip("a working GPU: no fallback to warn about") - backend_cli.struphy_backend("set", "cupy") - capsys.readouterr() - backend_cli.struphy_backend("get") - out = capsys.readouterr().out - assert "Backend struphy runs use (struphy backend set): cupy" in out - assert "Backend available on this machine: numpy" in out - assert "Warning: cupy is chosen but cunumpy can only provide numpy here" in out - - -def test_invalid_backend_in_state_file_is_rejected(state_dir): - utils.save_state({"backend": "torch"}) - with pytest.raises(ValueError, match="invalid backend"): - backend_cli.configured_backend() From d26b7c4ee302cb25693914b397d78ada43f11d37 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 13 Oct 2026 11:22:02 +0200 Subject: [PATCH 09/12] Hardcode the params --- .../landau_damping/params_vlasov_ampere.py | 20 +++++++++---------- .../submit_vlasov_ampere_landau_damping.py | 18 +++-------------- .../landau_damping/params_vlasov_maxwell.py | 20 +++++++++---------- .../submit_vlasov_maxwell_landau_damping.py | 18 +++-------------- ...ubmit_vlasov_maxwell_weibel_instability.py | 18 +++-------------- .../params_vlasov_maxwell.py | 20 +++++++++---------- 6 files changed, 39 insertions(+), 75 deletions(-) diff --git a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py index e0d5b67f9..5429158ff 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py +++ b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py @@ -59,9 +59,6 @@ # Unknown flags are ignored so the driver can forward other parameters as well. parser = argparse.ArgumentParser() parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -parser.add_argument("--num-elements", type=int, default=32, help="Cells in x (1 in y and z).") -parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") -parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") args, _ = parser.parse_known_args() # --------------------- @@ -88,7 +85,8 @@ # Time stepping dt = 0.05 -time_opts = Time(dt=dt, Tend=round(args.steps * dt, 14)) +steps = 20 +time_opts = Time(dt=dt, Tend=round(steps * dt, 14)) # Geometry Lx = 4 * math.pi # k = 0.5 @@ -98,7 +96,8 @@ equil = None # Grid -grid = grids.TensorProductGrid(num_elements=(args.num_elements, 1, 1)) +num_elements = 32 # cells in x (1 in y and z) +grid = grids.TensorProductGrid(num_elements=(num_elements, 1, 1)) # Derham options derham_opts = DerhamOptions(degree=(3, 1, 1)) @@ -134,7 +133,8 @@ # Particle parameters # ------------------- -loading_params = LoadingParameters(ppc=args.ppc, seed=1234) +ppc = 1000 +loading_params = LoadingParameters(ppc=ppc, seed=1234) weights_params = WeightsParameters(control_variate=True) boundary_params = BoundaryParameters() # sorting boxes are off: SortingBoxes does not work on the CuPy backend yet (struphy-hub/struphy#726) @@ -205,15 +205,15 @@ n_calls = min(len(durations) for durations in per_rank) step_times = np.max([durations[:n_calls] for durations in per_rank], axis=0) step_median = float(statistics.median(step_times[1:] if len(step_times) > 1 else step_times)) - n_markers = args.ppc * args.num_elements + n_markers = ppc * num_elements summary = { "backend": xp.get_backend(), "gpu_timing": gpu_timing, "mpi_ranks": results.num_ranks, - "num_elements": args.num_elements, - "ppc": args.ppc, + "num_elements": num_elements, + "ppc": ppc, "n_markers": n_markers, - "steps": args.steps, + "steps": steps, "dt": dt, "step_times": step_times.tolist(), "step_median": step_median, diff --git a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py index 63d13b99f..e28546144 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py +++ b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py @@ -9,8 +9,8 @@ directly (its `__main__` block is the worker). The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event -timing per region; the job script exports the choice as CUNUMPY_BACKEND before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_ampere.py` -(defaults: 1000 markers per cell on 32 cells, 20 steps). +timing per region; the job script exports the choice as CUNUMPY_BACKEND before the simulation. +The resolution (32 cells, 1000 markers per cell, 20 steps) is set in `params_vlasov_ampere.py`. """ import argparse @@ -51,9 +51,6 @@ def main() -> None: choices=["numpy", "cupy"], help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", ) - parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") - parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") - parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") args = parser.parse_args() # Paths relative to this script's location, so it can be run from anywhere. @@ -73,18 +70,9 @@ def main() -> None: upload=args.upload, ) - param_flags = [ - "--ppc", - str(args.ppc), - "--num-elements", - str(args.num_elements), - "--steps", - str(args.steps), - ] - # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks, param_flags=param_flags) + profiling_case.launch(num_tasks) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py index be311ade1..f81b1f10e 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py @@ -59,9 +59,6 @@ # Unknown flags are ignored so the driver can forward other parameters as well. parser = argparse.ArgumentParser() parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -parser.add_argument("--num-elements", type=int, default=32, help="Cells in x (1 in y and z).") -parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") -parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") args, _ = parser.parse_known_args() # --------------------- @@ -89,7 +86,8 @@ # Time stepping dt = 0.05 -time_opts = Time(dt=dt, Tend=round(args.steps * dt, 14)) +steps = 20 +time_opts = Time(dt=dt, Tend=round(steps * dt, 14)) # Geometry Lx = 4 * math.pi # k = 0.5 @@ -99,7 +97,8 @@ equil = None # Grid -grid = grids.TensorProductGrid(num_elements=(args.num_elements, 1, 1)) +num_elements = 32 # cells in x (1 in y and z) +grid = grids.TensorProductGrid(num_elements=(num_elements, 1, 1)) # Derham options derham_opts = DerhamOptions(degree=(3, 1, 1)) @@ -135,7 +134,8 @@ # Particle parameters # ------------------- -loading_params = LoadingParameters(ppc=args.ppc, seed=1234) +ppc = 1000 +loading_params = LoadingParameters(ppc=ppc, seed=1234) weights_params = WeightsParameters(control_variate=True) boundary_params = BoundaryParameters() # sorting boxes are off: SortingBoxes does not work on the CuPy backend yet (struphy-hub/struphy#726) @@ -206,15 +206,15 @@ n_calls = min(len(durations) for durations in per_rank) step_times = np.max([durations[:n_calls] for durations in per_rank], axis=0) step_median = float(statistics.median(step_times[1:] if len(step_times) > 1 else step_times)) - n_markers = args.ppc * args.num_elements + n_markers = ppc * num_elements summary = { "backend": xp.get_backend(), "gpu_timing": gpu_timing, "mpi_ranks": results.num_ranks, - "num_elements": args.num_elements, - "ppc": args.ppc, + "num_elements": num_elements, + "ppc": ppc, "n_markers": n_markers, - "steps": args.steps, + "steps": steps, "dt": dt, "step_times": step_times.tolist(), "step_median": step_median, diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py index 8b6eecc32..0dcb758fc 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py @@ -9,8 +9,8 @@ directly (its `__main__` block is the worker). The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event -timing per region; the job script exports the choice as CUNUMPY_BACKEND before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` -(defaults: 1000 markers per cell on 32 cells, 20 steps). +timing per region; the job script exports the choice as CUNUMPY_BACKEND before the simulation. +The resolution (32 cells, 1000 markers per cell, 20 steps) is set in `params_vlasov_maxwell.py`. """ import argparse @@ -51,9 +51,6 @@ def main() -> None: choices=["numpy", "cupy"], help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", ) - parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") - parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") - parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") args = parser.parse_args() # Paths relative to this script's location, so it can be run from anywhere. @@ -73,18 +70,9 @@ def main() -> None: upload=args.upload, ) - param_flags = [ - "--ppc", - str(args.ppc), - "--num-elements", - str(args.num_elements), - "--steps", - str(args.steps), - ] - # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks, param_flags=param_flags) + profiling_case.launch(num_tasks) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py index 8517e119a..12f2475b0 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py @@ -9,8 +9,8 @@ directly (its `__main__` block is the worker). The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event -timing per region; the job script exports the choice as CUNUMPY_BACKEND before the simulation. `--ppc`, `--num-elements` and `--steps` are forwarded to `params_vlasov_maxwell.py` -(defaults: 1000 markers per cell on 32 cells, 20 steps). +timing per region; the job script exports the choice as CUNUMPY_BACKEND before the simulation. +The resolution (32 cells, 1000 markers per cell, 20 steps) is set in `params_vlasov_maxwell.py`. """ import argparse @@ -51,9 +51,6 @@ def main() -> None: choices=["numpy", "cupy"], help="cunumpy backend to run on; cupy records CUDA-event GPU times per region.", ) - parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") - parser.add_argument("--num-elements", type=int, default=32, help="Cells in x.") - parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") args = parser.parse_args() # Paths relative to this script's location, so it can be run from anywhere. @@ -73,18 +70,9 @@ def main() -> None: upload=args.upload, ) - param_flags = [ - "--ppc", - str(args.ppc), - "--num-elements", - str(args.num_elements), - "--steps", - str(args.steps), - ] - # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks, param_flags=param_flags) + profiling_case.launch(num_tasks) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py index b4a8fcec4..0c06e0b9d 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py @@ -60,9 +60,6 @@ # Unknown flags are ignored so the driver can forward other parameters as well. parser = argparse.ArgumentParser() parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -parser.add_argument("--num-elements", type=int, default=32, help="Cells in x (1 in y and z).") -parser.add_argument("--ppc", type=int, default=1000, help="Markers per cell.") -parser.add_argument("--steps", type=int, default=20, help="Number of time steps.") args, _ = parser.parse_known_args() # --------------------- @@ -90,7 +87,8 @@ # Time stepping dt = 0.05 -time_opts = Time(dt=dt, Tend=round(args.steps * dt, 14)) +steps = 20 +time_opts = Time(dt=dt, Tend=round(steps * dt, 14)) # Geometry Lx = 2 * math.pi / 1.25 # k = 1.25 @@ -100,7 +98,8 @@ equil = None # Grid -grid = grids.TensorProductGrid(num_elements=(args.num_elements, 1, 1)) +num_elements = 32 # cells in x (1 in y and z) +grid = grids.TensorProductGrid(num_elements=(num_elements, 1, 1)) # Derham options derham_opts = DerhamOptions(degree=(3, 1, 1)) @@ -136,7 +135,8 @@ # Particle parameters # ------------------- -loading_params = LoadingParameters(ppc=args.ppc, seed=1234) +ppc = 1000 +loading_params = LoadingParameters(ppc=ppc, seed=1234) weights_params = WeightsParameters(control_variate=True) boundary_params = BoundaryParameters() # sorting boxes are off: SortingBoxes does not work on the CuPy backend yet (struphy-hub/struphy#726) @@ -211,15 +211,15 @@ n_calls = min(len(durations) for durations in per_rank) step_times = np.max([durations[:n_calls] for durations in per_rank], axis=0) step_median = float(statistics.median(step_times[1:] if len(step_times) > 1 else step_times)) - n_markers = args.ppc * args.num_elements + n_markers = ppc * num_elements summary = { "backend": xp.get_backend(), "gpu_timing": gpu_timing, "mpi_ranks": results.num_ranks, - "num_elements": args.num_elements, - "ppc": args.ppc, + "num_elements": num_elements, + "ppc": ppc, "n_markers": n_markers, - "steps": args.steps, + "steps": steps, "dt": dt, "step_times": step_times.tolist(), "step_median": step_median, From 208eed660e7a0c6a33644f4d7674beca81743935 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 13 Oct 2026 13:29:29 +0200 Subject: [PATCH 10/12] Moved backend to profiling_case.launch --- .../submit_vlasov_ampere_landau_damping.py | 3 +- .../submit_vlasov_maxwell_landau_damping.py | 3 +- ...ubmit_vlasov_maxwell_weibel_instability.py | 3 +- profiling/package_profiling_results.py | 2 ++ profiling/profiling_job.py | 36 +++++++++++-------- 5 files changed, 27 insertions(+), 20 deletions(-) diff --git a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py index e28546144..79b104777 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py +++ b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py @@ -66,13 +66,12 @@ def main() -> None: params_source=params_dir / "params_vlasov_ampere.py", language=args.language, compiler=args.compiler, - backend=args.backend, upload=args.upload, ) # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) + profiling_case.launch(num_tasks, backend=args.backend) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py index 0dcb758fc..131aac72d 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_landau_damping.py @@ -66,13 +66,12 @@ def main() -> None: params_source=params_dir / "params_vlasov_maxwell.py", language=args.language, compiler=args.compiler, - backend=args.backend, upload=args.upload, ) # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) + profiling_case.launch(num_tasks, backend=args.backend) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py index 12f2475b0..2da4c7315 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/submit_vlasov_maxwell_weibel_instability.py @@ -66,13 +66,12 @@ def main() -> None: params_source=params_dir / "params_vlasov_maxwell.py", language=args.language, compiler=args.compiler, - backend=args.backend, upload=args.upload, ) # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) + profiling_case.launch(num_tasks, backend=args.backend) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/package_profiling_results.py b/profiling/package_profiling_results.py index 9a158a1c9..180bc17b3 100644 --- a/profiling/package_profiling_results.py +++ b/profiling/package_profiling_results.py @@ -86,6 +86,8 @@ def _write_run_metadata( # Struphy records the rank count it actually ran with; fall back to the requested one # for a run that wrote no metadata of its own. metadata.setdefault("mpi_ranks", job_info.get("ranks")) + # The cunumpy backend is chosen per launch, so it is recorded per run. + metadata["backend"] = job_info.get("backend") metadata["slurm_script"] = job_info.get("slurm_dict") metadata["slurm_script_str"] = job_info.get("job_script") metadata["packaged_files"] = { diff --git a/profiling/profiling_job.py b/profiling/profiling_job.py index bd7c932da..915d80069 100644 --- a/profiling/profiling_job.py +++ b/profiling/profiling_job.py @@ -7,7 +7,7 @@ results directories. It stores everything it computes as attributes on the case itself (`venv_path`, `use_slurm`, `launcher`, `case_output_root`, ...), so there's a single object to thread through the rest of the run. -- For each rank count, the caller calls `ProfilingCase.launch(ntasks, param_flags=None)`, which builds +- For each rank count, the caller calls `ProfilingCase.launch(ntasks, param_flags=None, backend="numpy")`, which builds the per-rank shell commands and either submits a `SlurmScript` (under SLURM) or writes and runs a plain bash script to completion (otherwise) — the caller doesn't need to know which. Pass `param_flags` to override the default commands (e.g. to add a @@ -101,9 +101,6 @@ class ProfilingCase: params_source: Path language: str = "fortran" # Pyccel language to compile the Struphy kernels with: "fortran" or "c". compiler: str = "GNU" # Pyccel compiler family: "GNU", "intel", "PGI", "nvidia", or "LLVM". - # cunumpy backend the runs use, "numpy" (CPU) or "cupy" (GPU): exported as CUNUMPY_BACKEND in every job - # script before the simulation. - backend: str = "numpy" # Whether the packaged results are pushed to the profiling-data repo. Set here rather # than at `finalize_run` time because `setup_run` clones that repo up front, so a # missing/unreachable repo fails before any job is submitted rather than after. @@ -135,7 +132,12 @@ class ProfilingCase: def __post_init__(self) -> None: self.setup_run() - def build_commands(self, ntasks: int, param_flags: list[str] | None = None) -> list[str]: + def build_commands( + self, + ntasks: int, + param_flags: list[str] | None = None, + backend: str = "numpy", + ) -> list[str]: """Build the shell commands that run this case with a single MPI rank count. Reads the setup computed by `setup_run` (`case_output_root`, `venv_path`, @@ -147,6 +149,8 @@ def build_commands(self, ntasks: int, param_flags: list[str] | None = None) -> l param_flags: Extra CLI flags appended to the `params_source` invocation, e.g. `["--ppc", "10"]`. Omit for none. `--id` is always passed on top of these. + backend: cunumpy backend of the run, "numpy" (CPU) or "cupy" (GPU), + exported as CUNUMPY_BACKEND before the simulation. Returns: The shell commands to run, in order, as one script. @@ -175,13 +179,13 @@ def build_commands(self, ntasks: int, param_flags: list[str] | None = None) -> l "set -e", f"source {activate_path!s}", # cunumpy reads its backend from this variable when it is imported, so it is exported before the run - f"export CUNUMPY_BACKEND={self.backend}", + f"export CUNUMPY_BACKEND={backend}", 'echo "----------------------------------------"', f'echo "Running profiling case: {self.label} ({ntasks} MPI ranks)"', f'echo "Description: {self.description}"', f'echo "Physics problem: {self.physics_problem}"', f'echo "Struphy model used: {self.struphy_model_used}"', - f'echo "cunumpy backend: {self.backend}"', + f'echo "cunumpy backend: {backend}"', f'echo "Case directory: {output_root}"', 'echo "----------------------------------------"', f'mkdir -p "{sim_dir}"', @@ -209,6 +213,7 @@ def launch( num_nodes: int | None = None, param_flags: list[str] | None = None, slurm_presets: dict[str, dict] | None = None, + backend: str = "numpy", ) -> None: """Build, submit/run, and record the run for a single rank count. @@ -232,11 +237,17 @@ def launch( slurm_presets: Candidate SLURM presets, keyed by cluster name; one is picked via `detect_machine_name` on every call. Defaults to `clusters.SLURM_PRESETS`. Ignored outside SLURM. + backend: cunumpy backend of this run, "numpy" (CPU) or "cupy" (GPU): + exported as CUNUMPY_BACKEND in the job script and recorded in the + run's metadata, so one case can mix runs on both backends. Raises: - ValueError: If `num_tasks` is not evenly divisible by `num_nodes` - (SLURM only). + ValueError: If `backend` is neither "numpy" nor "cupy", or if + `num_tasks` is not evenly divisible by `num_nodes` (SLURM only). """ + if backend not in ("numpy", "cupy"): + raise ValueError(f"backend must be 'numpy' or 'cupy', got {backend!r}") + if not self.use_slurm: available = os.cpu_count() or 1 if num_tasks > available: @@ -249,7 +260,7 @@ def launch( script_path = profiling_results_base / f"job_profile_{self.label}_{self.launch_count:02d}.sh" # Build the commands to run this case with the given rank count, and write/submit - case_commands = self.build_commands(num_tasks, param_flags) + case_commands = self.build_commands(num_tasks, param_flags, backend) # Submit a SLURM script or run a local bash script, depending on whether SLURM is available. if self.use_slurm: @@ -305,6 +316,7 @@ def launch( job_info = { "launch_id": self.launch_count, "ranks": num_tasks, + "backend": backend, "job_id": job_id, "job_script_path": str(script_path), "job_script": script_text, @@ -335,9 +347,6 @@ def setup_run(self) -> None: ) self.venv_path = Path(virtual_env) - if self.backend not in ("numpy", "cupy"): - raise ValueError(f"backend must be 'numpy' or 'cupy', got {self.backend!r}") - # Compile Struphy kernels with the case's language and compiler self.compiler_instance = Compiler(language=self.language, compiler=self.compiler) if not self.compiler_instance.compiled(language=self.language): @@ -392,7 +401,6 @@ def case_info_dict(self) -> dict: "physics_problem": self.physics_problem, "struphy_model_used": self.struphy_model_used, "struphy_commit": self.run_commit, - "backend": self.backend, "compiler": self.compiler_instance.to_dict(), "scheduler": "slurm" if self.use_slurm else "local", "parameter_file": str(self.params_source), From 0aac0813bd8a4c3e80a386bf581e6cfd48729172 Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 13 Oct 2026 13:32:27 +0200 Subject: [PATCH 11/12] Label the Vlasov profiling runs by backend and rank count --- .../landau_damping/params_vlasov_ampere.py | 10 +++++++++- .../landau_damping/params_vlasov_maxwell.py | 10 +++++++++- .../weibel_instability/params_vlasov_maxwell.py | 10 +++++++++- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py index 5429158ff..d399ead15 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py +++ b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py @@ -107,8 +107,16 @@ from cunumpy.kernel_testing import fake_cupy_active gpu_timing = xp.get_backend() == "cupy" and not fake_cupy_active() + +# scope-profiler run label, the name of the run in the comparison plots of the profiling-data page: the runs of +# one case differ in backend and rank count only, so the label names those two (e.g. "cupy, 4 ranks"). +from maybempi import MPI, SerialMPI + +num_ranks = 1 if isinstance(MPI, SerialMPI) else MPI.COMM_WORLD.Get_size() +profiling_label = f"{xp.get_backend()}, {num_ranks} rank" + ("s" if num_ranks != 1 else "") + profiling_opts = ProfilingOptions( - label=f"{name} ({xp.get_backend()})", + label=profiling_label, use_gpu_timing=gpu_timing, gpu_timing_backend="cupy", gpu_sync_on_exit=gpu_timing, diff --git a/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py index f81b1f10e..fa307566d 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/landau_damping/params_vlasov_maxwell.py @@ -108,8 +108,16 @@ from cunumpy.kernel_testing import fake_cupy_active gpu_timing = xp.get_backend() == "cupy" and not fake_cupy_active() + +# scope-profiler run label, the name of the run in the comparison plots of the profiling-data page: the runs of +# one case differ in backend and rank count only, so the label names those two (e.g. "cupy, 4 ranks"). +from maybempi import MPI, SerialMPI + +num_ranks = 1 if isinstance(MPI, SerialMPI) else MPI.COMM_WORLD.Get_size() +profiling_label = f"{xp.get_backend()}, {num_ranks} rank" + ("s" if num_ranks != 1 else "") + profiling_opts = ProfilingOptions( - label=f"{name} ({xp.get_backend()})", + label=profiling_label, use_gpu_timing=gpu_timing, gpu_timing_backend="cupy", gpu_sync_on_exit=gpu_timing, diff --git a/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py index 0c06e0b9d..e325b3c92 100644 --- a/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py +++ b/profiling/examples/VlasovMaxwellOneSpecies/weibel_instability/params_vlasov_maxwell.py @@ -109,8 +109,16 @@ from cunumpy.kernel_testing import fake_cupy_active gpu_timing = xp.get_backend() == "cupy" and not fake_cupy_active() + +# scope-profiler run label, the name of the run in the comparison plots of the profiling-data page: the runs of +# one case differ in backend and rank count only, so the label names those two (e.g. "cupy, 4 ranks"). +from maybempi import MPI, SerialMPI + +num_ranks = 1 if isinstance(MPI, SerialMPI) else MPI.COMM_WORLD.Get_size() +profiling_label = f"{xp.get_backend()}, {num_ranks} rank" + ("s" if num_ranks != 1 else "") + profiling_opts = ProfilingOptions( - label=f"{name} ({xp.get_backend()})", + label=profiling_label, use_gpu_timing=gpu_timing, gpu_timing_backend="cupy", gpu_sync_on_exit=gpu_timing, From 98c73f8c93a4d812517998ebab29e0dd4f79c83e Mon Sep 17 00:00:00 2001 From: Max Date: Tue, 13 Oct 2026 13:51:52 +0200 Subject: [PATCH 12/12] Add mixed GPU/CPU case --- profiling/clusters.py | 13 ++++ .../landau_damping/params_vlasov_ampere.py | 6 +- .../submit_vlasov_ampere_landau_damping.py | 2 +- ...ere_landau_damping_cpu_node_vs_gpu_node.py | 77 +++++++++++++++++++ profiling/profiling_job.py | 30 ++++---- 5 files changed, 111 insertions(+), 17 deletions(-) create mode 100644 profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping_cpu_node_vs_gpu_node.py diff --git a/profiling/clusters.py b/profiling/clusters.py index d64c9a1f6..483ff83b4 100644 --- a/profiling/clusters.py +++ b/profiling/clusters.py @@ -69,6 +69,19 @@ def detect_machine_name() -> str | None: "mail_type": "none", "time": "00:15:00", }, + # Pitagora's GPU (booster) partition, one node with its 4 GPUs. Not detected by `detect_machine_name` + # (the login nodes are shared with the DCGP partition), so a run picks it with `launch(slurm_preset=...)`. + "pitagora_booster": { + "cpus_per_task": 1, + "mem": "480GB", + "gres": "gpu:4,tmpfs:10g", + "partition": "boost_fua_dbg", + "account": "FUSIO_HLST_6", + "output": "./%x.%j.out", + "error": "./%x.%j.err", + "mail_type": "none", + "time": "00:15:00", + }, "tok": { "cpus_per_task": 1, "mem_per_cpu": "1GB", diff --git a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py index d399ead15..76d3ae9d0 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py +++ b/profiling/examples/VlasovAmpereOneSpecies/landau_damping/params_vlasov_ampere.py @@ -96,8 +96,10 @@ equil = None # Grid -num_elements = 32 # cells in x (1 in y and z) -grid = grids.TensorProductGrid(num_elements=(num_elements, 1, 1)) +# cells in x (1 in y and z). Only x is decomposed, and every rank needs at least `degree` = 3 cells of its own, +# so 1024 cells run on up to 256 ranks (4 cells per rank), one full CPU node. +num_elements = 1024 +grid = grids.TensorProductGrid(num_elements=(num_elements, 1, 1), mpi_dims_mask=(True, False, False)) # Derham options derham_opts = DerhamOptions(degree=(3, 1, 1)) diff --git a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py index 79b104777..86a3cb378 100644 --- a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py +++ b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping.py @@ -10,7 +10,7 @@ The run is profiled with scope-profiler. `--backend cupy` runs it on the GPU with CUDA-event timing per region; the job script exports the choice as CUNUMPY_BACKEND before the simulation. -The resolution (32 cells, 1000 markers per cell, 20 steps) is set in `params_vlasov_ampere.py`. +The resolution (1024 cells, 1000 markers per cell, 20 steps) is set in `params_vlasov_ampere.py`. """ import argparse diff --git a/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping_cpu_node_vs_gpu_node.py b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping_cpu_node_vs_gpu_node.py new file mode 100644 index 000000000..3cd626e76 --- /dev/null +++ b/profiling/examples/VlasovAmpereOneSpecies/submit_vlasov_ampere_landau_damping_cpu_node_vs_gpu_node.py @@ -0,0 +1,77 @@ +"""Vlasov-Ampere weak Landau damping profiling case: one CPU node vs one GPU node. + +This file defines the profiling case (the `ProfilingCase`) and submits its two runs of the same +problem: one full CPU node (256 MPI ranks, NumPy backend) and one full GPU node (4 MPI ranks, one per +GPU, CuPy backend). The two runs go to different SLURM partitions, so each `ProfilingCase.launch` +names its own preset from `clusters.SLURM_PRESETS`. Without a batch system the runs execute directly +on this machine, where the 256-rank run is skipped unless there are that many cores. +`finalize_run` then packages and uploads each run as soon as its own job finishes. +Each generated script runs the simulation itself by invoking `params_vlasov_ampere.py` +directly (its `__main__` block is the worker). + +The runs are profiled with scope-profiler, the CuPy run with CUDA-event timing per region. The +resolution (1024 cells, 1000 markers per cell, 20 steps) is set in `params_vlasov_ampere.py`, shared +with the strong-scaling case of `submit_vlasov_ampere_landau_damping.py`. +""" + +import argparse +import sys +from pathlib import Path + +# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from profiling_job import ProfilingCase # noqa: E402 + + +def main() -> None: + + # Parse arguments, do not remove --upload + parser = argparse.ArgumentParser( + description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload the packaged profiling results to the profiling-data repo.", + ) + parser.add_argument( + "--language", + default="fortran", + choices=["fortran", "c"], + help="Pyccel language to compile the Struphy kernels with.", + ) + parser.add_argument( + "--compiler", + default="GNU", + help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', + ) + args = parser.parse_args() + + # Paths relative to this script's location, so it can be run from anywhere. + script_dir = Path(__file__).resolve().parent + params_dir = script_dir / "landau_damping" + + profiling_case = ProfilingCase( + label="vlasov_ampere_landau_damping_cpu_node_vs_gpu_node", + name="Vlasov-Ampere weak Landau damping: CPU node vs GPU node", + description="The VlasovAmpereOneSpecies model on weak Landau damping (periodic box, k = 0.5, 1e-3 density perturbation, 1024 cells, about 1M markers) on one full CPU node (256 MPI ranks, NumPy) vs one full GPU node (4 MPI ranks on 4 GPUs, CuPy).", + physics_problem="Landau damping of an electrostatic wave in a collisionless electron plasma.", + struphy_model_used="VlasovAmpereOneSpecies", + params_source=params_dir / "params_vlasov_ampere.py", + language=args.language, + compiler=args.compiler, + upload=args.upload, + ) + + # One CPU node: 256 ranks on Pitagora's DCGP partition + profiling_case.launch(256, num_nodes=1, backend="numpy", slurm_preset="pitagora_dcgp") + # One GPU node: one rank per GPU on Pitagora's booster partition + profiling_case.launch(4, num_nodes=1, backend="cupy", slurm_preset="pitagora_booster") + + # Package and push each run as its own job finishes. + profiling_case.finalize_run() + + +if __name__ == "__main__": + main() diff --git a/profiling/profiling_job.py b/profiling/profiling_job.py index 915d80069..d2a37b8f4 100644 --- a/profiling/profiling_job.py +++ b/profiling/profiling_job.py @@ -12,8 +12,9 @@ writes and runs a plain bash script to completion (otherwise) — the caller doesn't need to know which. Pass `param_flags` to override the default commands (e.g. to add a case-specific flag such as an arbitrary `ppc`) before they're wrapped in a script. - Under SLURM, the cluster preset is picked on each call from - `clusters.SLURM_PRESETS` unless a `slurm_presets` argument overrides it. + Under SLURM, the preset of the detected cluster is taken from `clusters.SLURM_PRESETS` + on each call, unless `slurm_preset` names another one (e.g. a GPU partition); + `slurm_options` overrides single pragmas of it. - `ProfilingCase.finalize_run` packages and pushes the case-level metadata straight away, then waits on the runs one by one, packaging and pushing each run's results into that same folder as soon as its own job finishes. With `upload=True`, the @@ -212,8 +213,9 @@ def launch( num_tasks: int = 1, num_nodes: int | None = None, param_flags: list[str] | None = None, - slurm_presets: dict[str, dict] | None = None, backend: str = "numpy", + slurm_preset: str | None = None, + slurm_options: dict | None = None, ) -> None: """Build, submit/run, and record the run for a single rank count. @@ -234,12 +236,16 @@ def launch( outside SLURM, where every rank runs in a single local process group. param_flags: Extra CLI flags forwarded to `build_commands` and appended to the `params_source` invocation, e.g. `["--ppc", "10"]`. - slurm_presets: Candidate SLURM presets, keyed by cluster name; one is - picked via `detect_machine_name` on every call. Defaults to - `clusters.SLURM_PRESETS`. Ignored outside SLURM. backend: cunumpy backend of this run, "numpy" (CPU) or "cupy" (GPU): exported as CUNUMPY_BACKEND in the job script and recorded in the run's metadata, so one case can mix runs on both backends. + slurm_preset: Key of the `clusters.SLURM_PRESETS` entry the job is + submitted with, e.g. "pitagora_booster" for a GPU partition. + Defaults to the cluster found by `detect_machine_name`. Its + `HARDWARE_INFO` entry is only needed when `num_nodes` is not given. + Ignored outside SLURM. + slurm_options: SLURM pragmas that override those of the preset, e.g. + `{"time": "00:30:00"}`. Ignored outside SLURM. Raises: ValueError: If `backend` is neither "numpy" nor "cupy", or if @@ -264,19 +270,15 @@ def launch( # Submit a SLURM script or run a local bash script, depending on whether SLURM is available. if self.use_slurm: - cluster_name = detect_machine_name() + # The preset of this run: the caller's choice, or the one of the detected cluster. + preset_name = slurm_preset or detect_machine_name() if num_nodes is None: # Compute the number of nodes needed to run `num_tasks` ranks, given the cluster's hardware. - cpus_per_node = HARDWARE_INFO[cluster_name]["cpus_per_node"] + cpus_per_node = HARDWARE_INFO[preset_name]["cpus_per_node"] num_nodes = (num_tasks + cpus_per_node - 1) // cpus_per_node if num_tasks % num_nodes != 0: raise ValueError(f"num_tasks ({num_tasks}) is not evenly divisible by num_nodes ({num_nodes}).") - - # Pick the cluster preset for this run, either from the caller's override or the default. - if slurm_presets is not None: - cluster_preset = slurm_presets[cluster_name] - else: - cluster_preset = SLURM_PRESETS[cluster_name] + cluster_preset = {**SLURM_PRESETS[preset_name], **(slurm_options or {})} # Build the slurm script script = SlurmScript(