From ab6595ac3a08041ee694f6c84d4a1a039ddbb06b Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 11:24:42 +0000 Subject: [PATCH 01/21] Delete unused native fft, multiply_phase and Adams entry points fft() and multiply_phase() were reached only by their own smoke tests; GeneralState::adams_source and adams_magnitude (and their workspace definitions) had no caller since the SEP 52 seam removal. Co-Authored-By: Claude Opus 5.5 --- native/include/solprop_native/core.hpp | 8 --- native/src/core.cpp | 27 ---------- native/src/general/state.cpp | 7 --- native/src/general/trial.cpp | 72 ++------------------------ native/src/general/workspace.hpp | 3 -- native/tests/native/test_core.cpp | 49 ------------------ 6 files changed, 3 insertions(+), 163 deletions(-) diff --git a/native/include/solprop_native/core.hpp b/native/include/solprop_native/core.hpp index 42821422d..96fab051a 100644 --- a/native/include/solprop_native/core.hpp +++ b/native/include/solprop_native/core.hpp @@ -1148,9 +1148,6 @@ class GeneralState { void set_initial(std::span field, std::span noise); [[nodiscard]] std::vector trial_step(double dz, const TrialScalars& scalars); void prepare_trial(double dz, const TrialScalars& scalars); - /** solprop's Adams source at an anchor-frame field (SEP 14 phase 2). */ - [[nodiscard]] std::vector adams_source(const std::vector& input, - const TrialScalars& scalars); /** Adopt an Adams step's end state through the shared finish. */ void adopt(const std::vector& field, double dz); /** @@ -1167,7 +1164,6 @@ class GeneralState { void adams_allocate(std::size_t slots, bool compensated, bool interpolate = true); void adams_compensation_start(std::size_t slot, const std::vector& rate); void adams_start(std::size_t slot, const TrialScalars& scalars); - [[nodiscard]] double adams_magnitude(); void adams_scale(double rtol, double atol, const std::vector& weights); [[nodiscard]] std::vector adams_fused_trial(double dz, const AdamsWeights& weights, @@ -1441,10 +1437,6 @@ class GeneralState { std::unique_ptr workspace_; }; -void fft(const ExecutionSpace& execution_space, ComplexView input, ComplexView output); -void multiply_phase(const ExecutionSpace& execution_space, ConstComplexView values, - ConstRealView phase, ComplexView output); - } // namespace solprop_native template <> struct Kokkos::reduction_identity { diff --git a/native/src/core.cpp b/native/src/core.cpp index 4674e1aa9..e3d000db9 100644 --- a/native/src/core.cpp +++ b/native/src/core.cpp @@ -517,31 +517,4 @@ std::vector reference_stream_copies(const std::size_t elements, return seconds; } -void fft(const ExecutionSpace& execution_space, const ComplexView input, - const ComplexView output) { - const std::size_t size = input.extent(0); - if (size == 0) - throw std::invalid_argument("fft input must not be empty"); - if (output.extent(0) != size) { - throw std::invalid_argument("fft input and output must have the same length"); - } - KokkosFFT::fft(execution_space, input, output, KokkosFFT::Normalization::none); -} - -void multiply_phase(const ExecutionSpace& execution_space, - const ConstComplexView values, const ConstRealView phase, - const ComplexView output) { - const std::size_t size = values.extent(0); - if (phase.extent(0) != size || output.extent(0) != size) { - throw std::invalid_argument("values, phase, and output must have the same length"); - } - launch::parallel_for( - "solprop_native.multiply_phase", - Kokkos::RangePolicy(execution_space, 0, size), - KOKKOS_LAMBDA(const std::size_t index) { - output(index) = values(index) * - Complex(Kokkos::cos(phase(index)), Kokkos::sin(phase(index))); - }); -} - } // namespace solprop_native diff --git a/native/src/general/state.cpp b/native/src/general/state.cpp index 4bb6510a1..9b8ef57e6 100644 --- a/native/src/general/state.cpp +++ b/native/src/general/state.cpp @@ -128,10 +128,6 @@ std::vector GeneralState::trial_step(const double dz, void GeneralState::prepare_trial(const double dz, const TrialScalars& scalars) { with_team(&GeneralWorkspace::prepare_trial, dz, scalars); } -std::vector GeneralState::adams_source(const std::vector& input, - const TrialScalars& scalars) { - return with_team(&GeneralWorkspace::adams_source, input, scalars); -} void GeneralState::adopt(const std::vector& field, const double dz) { with_team(&GeneralWorkspace::adopt, field, dz); } @@ -150,9 +146,6 @@ void GeneralState::adams_compensation_start(const std::size_t slot, void GeneralState::adams_start(const std::size_t slot, const TrialScalars& scalars) { with_team(&GeneralWorkspace::adams_start, slot, scalars); } -double GeneralState::adams_magnitude() { - return with_team(&GeneralWorkspace::adams_magnitude); -} void GeneralState::adams_scale(const double rtol, const double atol, const std::vector& weights) { with_team(&GeneralWorkspace::adams_scale, rtol, atol, weights); diff --git a/native/src/general/trial.cpp b/native/src/general/trial.cpp index 1d035df7a..46546e05f 100644 --- a/native/src/general/trial.cpp +++ b/native/src/general/trial.cpp @@ -720,54 +720,6 @@ std::vector GeneralWorkspace::trial_step(const double dz, return collect_trial(); } -// solprop's low-call Adams march (SEP 13, SEP 14 phase 2): one source evaluation at a -// field given in the step's anchor frame, `U(anchor, position) N(U(position, anchor) -// v)`. It is stage zero of a trial whose stage-zero scalars are the evaluation -// position's: the combination with no earlier stage carries the input to the position -// and the finish brings the source back, so the shared stage pipeline, its responses -// and its in-stage absorber are the march's too. The accepted state is held in -// `advanced_`, which only the Runge-Kutta accept writes, while the input takes its -// place. -std::vector GeneralWorkspace::adams_source(const std::vector& input, - const TrialScalars& scalars) { - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - // Every route's stage source serves the Adams march (solprop SEP 16 phase 3): the - // separable stage takes its flow from the stage's own Phi increment, as the modal one - // does. Beside a coupled flow the sample is the material source alone (solprop SEP - // 16 phase 5): the exponential formulation carries the residual in its compensation - // and the flow in its moments, so the stage runs without the flow's pipeline. - require_extent(input.size(), problem_.n_columns() * problem_.dims.n_omega, - "Adams input"); - // The flow is set aside for the evaluation and put back whatever happens. - auto held_flow = std::exchange(flow_, nullptr); - // An aggregate, so it declares no copy or move; it is never copied or moved. - // NOLINTNEXTLINE(cppcoreguidelines-special-member-functions): a local scope guard - struct RestoreFlow { - std::unique_ptr& slot; - std::unique_ptr& held; - // NOLINTNEXTLINE(modernize-use-equals-default): restores the slot (see FlowAside). - ~RestoreFlow() { slot = std::move(held); } - } restore{.slot = flow_, .held = held_flow}; - require_trial_scalars(scalars); - upload_trial(scalars); - if (general_) - general_->set_weights(scalars.transform_weights); - graph::deep_copy(execution_space_, advanced_, current_); - copy_vector(execution_space_, current_, input); - if (problem_.raman.enabled) - prepare_raman_stage(0, scalars.raman_rates.data(), - problem_.raman.n_components > 1 ? scalars.raman_densities.data() - : nullptr); - evaluate_stage_source(0, 0.0); - graph::deep_copy(execution_space_, current_, advanced_); - stages_ready_ = true; - // No Runge-Kutta stage of this state exists to recycle. - pending_fsal_ = false; - has_recycled_ = false; - return stage_to_host(0); -} - // Adopt an Adams step's end state, given on the host before its boundary treatments: // the shared finish masks it, or applies the split-step absorber, and makes it current. void GeneralWorkspace::adopt(const std::vector& field, const double dz) { @@ -1682,8 +1634,9 @@ void GeneralWorkspace::adams_phase() { }); } -// The stage-zero source at an anchor-frame field held in current_, as adams_source -// evaluates it, into stages_(0). +// The Adams march's source at an anchor-frame field held in current_, into stages_(0): +// stage zero of a trial whose stage-zero scalars are the evaluation position's, so the +// shared stage pipeline, its responses and its in-stage absorber are the march's too. void GeneralWorkspace::adams_evaluate_source(const TrialScalars& scalars) { if (source_fault_) std::exchange(source_fault_, {})(); @@ -1797,25 +1750,6 @@ void GeneralWorkspace::adams_upload_divisor(const std::vector& divisor) }); } -double GeneralWorkspace::adams_magnitude() { - require_adams(); - const auto current = current_; - const auto in_norm = norm_mask_; - const auto n = problem_.dims.n_omega; - double total = 0.0; - launch::parallel_reduce( - "solprop_native.adams.magnitude", - Kokkos::RangePolicy(execution_space_, 0, - problem_.n_columns() * n), - KOKKOS_LAMBDA(const std::size_t item, double& update) { - const auto index = item % n; - if (in_norm(index) != 0) - update += squared_abs(current(item / n, index)); - }, - total); - return Kokkos::sqrt(total); -} - namespace { /** The squared scaled norms of up to three defects, reduced in one traversal. */ diff --git a/native/src/general/workspace.hpp b/native/src/general/workspace.hpp index b6d74594e..f6bb5b6a9 100644 --- a/native/src/general/workspace.hpp +++ b/native/src/general/workspace.hpp @@ -35,8 +35,6 @@ class GeneralWorkspace { void set_initial(std::span field, std::span noise); std::vector trial_step(double dz, const TrialScalars& scalars); void prepare_trial(double dz, const TrialScalars& scalars); - std::vector adams_source(const std::vector& input, - const TrialScalars& scalars); void adopt(const std::vector& field, double dz); double low_storage_step(double dz, const TrialScalars& scalars, bool rebuild_table); void low_storage_flows_formed(bool rebuild_table); @@ -48,7 +46,6 @@ class GeneralWorkspace { void adams_allocate(std::size_t slots, bool compensated, bool interpolate); void adams_compensation_start(std::size_t slot, const std::vector& rate); void adams_start(std::size_t slot, const TrialScalars& scalars); - double adams_magnitude(); // The controller's norm scale on the device: rtol * w_i * |u| + atol over the // in-band magnitude, formed each trial without the magnitude crossing to the host. void adams_scale(double rtol, double atol, const std::vector& weights); diff --git a/native/tests/native/test_core.cpp b/native/tests/native/test_core.cpp index 9382fae59..d359d5034 100644 --- a/native/tests/native/test_core.cpp +++ b/native/tests/native/test_core.cpp @@ -360,55 +360,6 @@ void expect_complex_near(const Complex actual, const Complex expected, EXPECT_NEAR(actual.imag(), expected.imag(), tolerance); } -TEST(Fft, ImpulseProducesConstantSpectrum) { - constexpr std::size_t size = 32; - native::ComplexView input("native_test.fft.input", size); - native::ComplexView output("native_test.fft.output", size); - Kokkos::deep_copy(input, native::Complex{}); - const auto host_input = Kokkos::create_mirror_view(input); - host_input(0) = native::Complex(2.0, -3.0); - Kokkos::deep_copy(input, host_input); - - const native::ExecutionSpace execution_space; - native::fft(execution_space, input, output); - execution_space.fence("native FFT test boundary"); - const auto host_output = - Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, output); - - const double tolerance = 64.0 * kEpsilon * std::log2(size) * std::hypot(2.0, 3.0); - for (std::size_t index = 0; index < size; ++index) { - expect_complex_near(host_output(index), {2.0, -3.0}, tolerance); - } -} - -TEST(Fft, RejectsInvalidExtents) { - const native::ExecutionSpace execution_space; - const native::ComplexView empty("native_test.fft.empty", 0); - EXPECT_THROW(native::fft(execution_space, empty, empty), std::invalid_argument); - const native::ComplexView input("native_test.fft.input", 4); - const native::ComplexView output("native_test.fft.output", 3); - EXPECT_THROW(native::fft(execution_space, input, output), std::invalid_argument); -} - -TEST(MultiplyPhase, AppliesQuarterTurnAndPreservesMagnitude) { - constexpr std::size_t size = 4; - native::ComplexView values("native_test.phase.values", size); - native::RealView phase("native_test.phase.angles", size); - native::ComplexView output("native_test.phase.output", size); - Kokkos::deep_copy(values, native::Complex(3.0, 4.0)); - Kokkos::deep_copy(phase, std::numbers::pi_v / 2.0); - const native::ExecutionSpace execution_space; - native::multiply_phase(execution_space, values, phase, output); - execution_space.fence("native phase test boundary"); - const auto host_output = - Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, output); - const double tolerance = 32.0 * kEpsilon; - for (std::size_t index = 0; index < size; ++index) { - expect_complex_near(host_output(index), {-4.0, 3.0}, tolerance); - EXPECT_NEAR(Kokkos::abs(host_output(index)), 5.0, tolerance); - } -} - TEST(GeneralState, MasksAndAppliesCombinedWholeStepPropagator) { auto problem = general_problem(native::FieldFamily::envelope); problem.linear.n_phi_terms = 2; From c59a6d4bbfe0dfdb0d3863f8ad13682820ba9924 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 11:24:47 +0000 Subject: [PATCH 02/21] Include launch.hpp where it is used, not through core.hpp core.hpp uses nothing from launch.hpp; plasma.hpp, z_table.hpp and core.cpp, which relied on it transitively, now include it themselves. dense_output.hpp gets adams_march.hpp through dense.hpp. Co-Authored-By: Claude Opus 5.5 --- native/include/solprop_native/core.hpp | 1 - native/include/solprop_native/dense_output.hpp | 1 - native/include/solprop_native/plasma.hpp | 1 + native/include/solprop_native/z_table.hpp | 1 + native/src/core.cpp | 2 ++ 5 files changed, 4 insertions(+), 2 deletions(-) diff --git a/native/include/solprop_native/core.hpp b/native/include/solprop_native/core.hpp index 96fab051a..e85ebd115 100644 --- a/native/include/solprop_native/core.hpp +++ b/native/include/solprop_native/core.hpp @@ -6,7 +6,6 @@ #include "solprop_native/fields.hpp" #include "solprop_native/host_source.hpp" -#include "solprop_native/launch.hpp" #include #include diff --git a/native/include/solprop_native/dense_output.hpp b/native/include/solprop_native/dense_output.hpp index 14abd1271..2a28b59c7 100644 --- a/native/include/solprop_native/dense_output.hpp +++ b/native/include/solprop_native/dense_output.hpp @@ -4,7 +4,6 @@ #pragma once -#include "solprop_native/adams_march.hpp" #include "solprop_native/core.hpp" #include "solprop_native/dense.hpp" #include "solprop_native/stage_tables.hpp" diff --git a/native/include/solprop_native/plasma.hpp b/native/include/solprop_native/plasma.hpp index 55fb46541..3fc11f287 100644 --- a/native/include/solprop_native/plasma.hpp +++ b/native/include/solprop_native/plasma.hpp @@ -6,6 +6,7 @@ #include "solprop_native/core.hpp" #include "solprop_native/dispatch.hpp" +#include "solprop_native/launch.hpp" #include diff --git a/native/include/solprop_native/z_table.hpp b/native/include/solprop_native/z_table.hpp index e429e09c0..d433ae756 100644 --- a/native/include/solprop_native/z_table.hpp +++ b/native/include/solprop_native/z_table.hpp @@ -6,6 +6,7 @@ #include "solprop_native/chebyshev.hpp" #include "solprop_native/core.hpp" +#include "solprop_native/launch.hpp" #include "solprop_native/refuse.hpp" #include diff --git a/native/src/core.cpp b/native/src/core.cpp index e3d000db9..02fcafa23 100644 --- a/native/src/core.cpp +++ b/native/src/core.cpp @@ -4,6 +4,8 @@ #include "solprop_native/core.hpp" +#include "solprop_native/launch.hpp" + #include #include From 0c81a73bd51e11d366920ab8a928c24e0e1f39aa Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 11:26:20 +0000 Subject: [PATCH 03/21] Drop dead adapter constants, stale _records citations, a timing script backend.py's statistic flag tables had no reader; the deleted solprop_native._records is cited as the C++ converter or solprop's functions instead; tests/general_modal_benchmark.py was a timing script in tests/ that nothing ran. Co-Authored-By: Claude Opus 5.5 --- native/src/solprop_native/_transverse.py | 10 +-- native/src/solprop_native/_watch.py | 8 +- native/src/solprop_native/backend.py | 51 ----------- native/src/solprop_native/session.py | 3 +- native/src/workspace.cpp | 7 +- native/tests/general_modal_benchmark.py | 107 ----------------------- 6 files changed, 15 insertions(+), 171 deletions(-) delete mode 100644 native/tests/general_modal_benchmark.py diff --git a/native/src/solprop_native/_transverse.py b/native/src/solprop_native/_transverse.py index fbc6bbeaa..914e33e36 100644 --- a/native/src/solprop_native/_transverse.py +++ b/native/src/solprop_native/_transverse.py @@ -9,11 +9,11 @@ decision and record (``transverse::Edge``, ``native/include/solprop_native/transverse.hpp``). The edge's configuration and a resume's continuation are solprop's run records (:mod:`solprop.backends.records`, SEP -52), which :mod:`solprop_native._records` converts. This module only translates the -rest: it forms the input noise's readings a launch carries, formats the native record -as :meth:`~solprop.transverse_monitor.TransverseEdge.summary` lays it out, and formats -a native verdict as the edge's warning or exception with the edge's own texts. It -decides nothing. +52), which the extension's record converter (``native/src/records.hpp``) reads. This +module only translates the rest: it forms the input noise's readings a launch carries, +formats the native record as :meth:`~solprop.transverse_monitor.TransverseEdge.summary` +lays it out, and formats a native verdict as the edge's warning or exception with the +edge's own texts. It decides nothing. """ from __future__ import annotations diff --git a/native/src/solprop_native/_watch.py b/native/src/solprop_native/_watch.py index d7877e4d6..c953671dc 100644 --- a/native/src/solprop_native/_watch.py +++ b/native/src/solprop_native/_watch.py @@ -8,10 +8,10 @@ native session owns the boundary monitor's every decision and record (``watch::Monitor``, ``native/include/solprop_native/watch.hpp``). The monitor's configuration and a resume's continuation are solprop's run records -(:mod:`solprop.backends.records`, SEP 52), which :mod:`solprop_native._records` -converts. This module only translates what comes back: it formats the native record -as :meth:`~solprop.monitor.BoundaryMonitor.summary` lays it out, and a native verdict -as solprop's warning or exception. It decides nothing. +(:mod:`solprop.backends.records`, SEP 52), which the extension's record converter +(``native/src/records.hpp``) reads. This module only translates what comes back: it +formats the native record as :meth:`~solprop.monitor.BoundaryMonitor.summary` lays it +out, and a native verdict as solprop's warning or exception. It decides nothing. """ from __future__ import annotations diff --git a/native/src/solprop_native/backend.py b/native/src/solprop_native/backend.py index 9f0f2e3f1..6cbf51e1b 100644 --- a/native/src/solprop_native/backend.py +++ b/native/src/solprop_native/backend.py @@ -198,57 +198,6 @@ def __exit__( and a device build's passes take a thread per (bin, channel) and hold none. """ -_SPATIAL_STAT_FLAGS = { - "energy": 1, - "on_axis_intensity": 2, - "peak_intensity": 4, - "on_axis_spectrum": 8, - "beam_radius": 16, - "fwhm_r": 16, -} - -_FAR_FIELD_FLAGS = { - "beam_spectrum": 1 << 5, - "absorbed_power": 1 << 7, - "region_spectrum": 1 << 8, - "aperture_spectrum": 1 << 9, - "aperture_on_axis_spectrum": 1 << 10, - "on_axis_spectrum": 1 << 11, -} -"""The native far-field pass's flags (solprop SEP 23 section 6), from 32. - -``on_axis_spectrum`` is the pass's ``W = 1`` slot on a Cartesian or line grid; on the -other spatial kinds it stays ``spatial_statistics``' flag 8. -""" - -_FAR_FIELD_ORDER = ( - "beam_spectrum", - "absorbed_power", - "region_spectrum", - "aperture_spectrum", - "aperture_on_axis_spectrum", - "on_axis_spectrum", -) -"""The order in which the native pass packs its statistics.""" - -_STAT_FLAGS = { - "omega0": 1 << 0, - "energy": 1 << 1, - "energy_window": 1 << 2, - "peak_power": 1 << 3, - "peak_power_window": 1 << 4, - "peak_power_total": 1 << 5, - "fwhm_t": 1 << 6, - "fwhm_t_outer": 1 << 6, - "electron_density": 1 << 7, - "peak_ionisation_rate": 1 << 8, - "peak_intensity": 1 << 9, - "quadrature_error": 1 << 10, -} - -_STATISTICS_PLANS = 16 -"""The most native statistics plans a backend keeps, one per spec and names.""" - class NativeBackend: """Propagate modal, radial or Cartesian fields with persistent Kokkos state. diff --git a/native/src/solprop_native/session.py b/native/src/solprop_native/session.py index 196485986..511a0f30d 100644 --- a/native/src/solprop_native/session.py +++ b/native/src/solprop_native/session.py @@ -49,7 +49,8 @@ and a stop a terminal failure, the edge's readings and a collar's current are taken from the resident state with no return to Python, and a monitored resume continues both records natively from solprop's run records - (:mod:`solprop.backends.records`, converted by :mod:`solprop_native._records`); + (:mod:`solprop.backends.records`, read by the extension's record converter, + ``native/src/records.hpp``); - dense output (``interpolate_planes``), for Adams and the embedded Runge-Kutta tableau alike: a plane the march steps across is interpolated natively inside the step that crossed it, the committed state held aside, and observed in order before diff --git a/native/src/workspace.cpp b/native/src/workspace.cpp index 7d58accea..261e084e6 100644 --- a/native/src/workspace.cpp +++ b/native/src/workspace.cpp @@ -69,8 +69,8 @@ rk::Policy to_native(const RungeKuttaPolicyRecord& record) { policy.max_factor = controller.max_factor; policy.lag = static_cast(controller.lag); policy.reject_free = controller.reject_free; - // `_records.rk_policy`: the equivalent norm's power and its reciprocal, as solprop - // forms them from the estimator's order. + // The equivalent norm's power and its reciprocal, as solprop's controller forms them + // from the estimator's order (`solprop.stepper.controller`). const auto power = static_cast(controller.estimator_order + 1); policy.power = power; policy.exponent = 1.0 / power; @@ -146,7 +146,8 @@ watch::Resumption to_native(const ResumptionRecord& record) { transverse::Resumption to_native(const EdgeResumptionRecord& record) { transverse::Resumption out; out.record = record.record; - // `_records.edge_resumption`: a record that is none carries nothing else. + // As `solprop.backends.records.edge_resumption` reads it: a record that is none + // carries nothing else. if (!record.record) return out; const std::array*>, 15> known{{ diff --git a/native/tests/general_modal_benchmark.py b/native/tests/general_modal_benchmark.py deleted file mode 100644 index c6f467b62..000000000 --- a/native/tests/general_modal_benchmark.py +++ /dev/null @@ -1,107 +0,0 @@ -# SPDX-FileCopyrightText: 2026 John C. Travers and contributors -# -# SPDX-License-Identifier: GPL-3.0-or-later - -"""Ten steps of solprop's general modal route on ``numpy_ref`` and Kokkos. - -Each case of ``tests/test_general_modal.py`` propagates the same ten fixed steps on -both backends in this process; the script prints one JSON object per case with the -derived parity bound, the measured departure, the window's nonlinear change and the -propagation times (every repeat, the run's session from its start to its end, -construction excluded). -Run it with the thread environment fixed, for example ``SOLPROP_NATIVE_THREADS=1 -OMP_PROC_BIND=false python tests/general_modal_benchmark.py``. -It lives beside the tests, as the thread probes do, because it propagates their cases. -""" - -# ruff: noqa: I001 -- Kokkos loads first, before any other OpenMP runtime on macOS. - -from __future__ import annotations - -import argparse -import json -import time - -import solprop_native - -import numpy as np -from solprop.backends.numpy_ref import NumpyRefBackend -from solprop.backends.session import Start -from session_runs import run - -from test_general_modal import ( - CASES, - Z_END, - Case, - _linear, # pyright: ignore[reportPrivateUsage] - _parity_bound, # pyright: ignore[reportPrivateUsage] -) - -from solprop_native import NativeBackend - - -def _timed( - backend_class: type[NumpyRefBackend] | type[NativeBackend], case: Case, repeats: int -) -> tuple[np.ndarray, list[float], int]: - seconds: list[float] = [] - field = np.empty(0, dtype=np.complex128) - ledger = 0 - for _ in range(repeats): - backend = backend_class( - case.problem, case.run_request, grid=case.simulation.grid - ) - try: - start = Start(field=case.state, z=0.0) - started = time.perf_counter() - result = run(backend, case.problem, case.run_request, start) - seconds.append(time.perf_counter() - started) - assert result.all_planes()[-1].z == Z_END - field = result.fields[-1] - ledger = sum( - value - for name, value in backend.memory_ledger().items() - if name.startswith("general_") - ) - finally: - backend.close() - return field, seconds, ledger - - -def main() -> None: - """Print one JSON record per case.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("cases", nargs="*", default=sorted(CASES)) - parser.add_argument("--repeats", type=int, default=5) - arguments = parser.parse_args() - info = solprop_native.build_info() - for name in arguments.cases: - case = CASES[name]() - expected, reference_seconds, reference_bytes = _timed( - NumpyRefBackend, case, arguments.repeats - ) - actual, native_seconds, native_bytes = _timed( - NativeBackend, case, arguments.repeats - ) - linear = _linear(case) - scale = float(np.max(np.abs(case.state))) - tables = case.problem.transform.modal_tables - assert tables is not None - record = { - "case": name, - "shape": list(case.problem.shape), - "n_to": case.problem.grids.n_to, - "nodes": int(tables.region_index.size), - "threads": info.thread_count, - "nonlinear_change": float(np.max(np.abs(expected - linear))) / scale, - "bound_relative": _parity_bound(case, expected, linear) / scale, - "departure_relative": float(np.max(np.abs(actual - expected))) / scale, - "numpy_ref_seconds": reference_seconds, - "native_seconds": native_seconds, - "numpy_ref_general_bytes": reference_bytes, - "native_general_bytes": native_bytes, - } - print(json.dumps(record), flush=True) - - -if __name__ == "__main__": - main() From c6e980cb9cf995171ada909bcecc13203a122436 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 11:55:57 +0000 Subject: [PATCH 04/21] Split core.hpp and core.cpp into concept-named headers core.hpp held the execution aliases, the monitor and edge readings, the CPU execution policy, the SEP 28 timing references, the Problem record and GeneralState in one 1450-line header. Each block moves, verbatim and in its old order, to its own file: - execution.hpp: ExecutionSpace, Layout, Range2, Complex, the view aliases and general_vector_lanes (from general_modal.hpp, so that slab.hpp no longer pulls KokkosFFT through general_modal.hpp) - readings.hpp: the reduction sums with their reduction identities, EdgeLines with its field table (from workspace.hpp) - cpu_execution.hpp/.cpp: ThreadPolicy, CpuExecution, FftPlanning, CpuExecutionPool, CpuTeamScope, the spin and the test hooks - reference_timing.hpp/.cpp: reference_fft3_pairs and reference_stream_copies, with the KokkosFFT include - problem.hpp/.cpp: FieldFamily, StatisticsFlag, Problem with its field tables, Problem::validate and compact_problem - general_state.hpp: the Adams weights, TrialScalars, TableKind with its names (from workspace.hpp) and GeneralState, which forward- declares the host source types it only passes on - refuse.hpp: EnvelopeConvergenceError and BackendRefusal Every former includer now includes the headers it uses. The orphaned adams_max_samples comment rejoins its constant. No code changes. Co-Authored-By: Claude Opus 5.5 --- docs/dev/backends.md | 6 +- docs/dev/benchmarks.md | 2 +- native/CMakeLists.txt | 7 +- native/include/solprop_native/adams_phi.hpp | 3 +- native/include/solprop_native/core.hpp | 1449 ----------------- .../include/solprop_native/coupled_flow.hpp | 3 +- .../include/solprop_native/cpu_execution.hpp | 272 ++++ .../include/solprop_native/dense_output.hpp | 2 +- native/include/solprop_native/dispersive.hpp | 4 +- native/include/solprop_native/engines.hpp | 3 +- native/include/solprop_native/execution.hpp | 89 + native/include/solprop_native/far_field.hpp | 3 +- native/include/solprop_native/fft_plan.hpp | 3 +- native/include/solprop_native/fields.hpp | 2 +- .../include/solprop_native/flow_functions.hpp | 3 +- .../include/solprop_native/general_modal.hpp | 14 +- .../include/solprop_native/general_state.hpp | 439 +++++ native/include/solprop_native/low_storage.hpp | 3 +- native/include/solprop_native/plasma.hpp | 3 +- native/include/solprop_native/problem.hpp | 601 +++++++ native/include/solprop_native/readings.hpp | 128 ++ .../solprop_native/reference_timing.hpp | 29 + native/include/solprop_native/refuse.hpp | 13 + native/include/solprop_native/slab.hpp | 4 +- .../include/solprop_native/stage_tables.hpp | 3 +- native/include/solprop_native/workspace.hpp | 29 +- native/include/solprop_native/z_table.hpp | 2 +- native/src/adams_march.cpp | 2 +- native/src/bindings.cpp | 8 +- native/src/{core.cpp => cpu_execution.cpp} | 157 +- native/src/general/kernels.hpp | 3 +- native/src/general/trial.cpp | 2 +- native/src/general/workspace.hpp | 1 + native/src/problem.cpp | 82 + native/src/reference_timing.cpp | 96 ++ native/src/rk_march.cpp | 2 +- native/src/session.cpp | 1 - native/src/session_statistics.cpp | 6 +- native/src/session_widths.cpp | 2 +- native/tests/native/execution.hpp | 2 +- native/tests/native/test_aperture.cpp | 3 +- native/tests/native/test_core.cpp | 5 +- native/tests/native/test_coupled_flow.cpp | 4 +- native/tests/native/test_dispersive.cpp | 4 +- native/tests/native/test_envelope_plasma.cpp | 2 +- native/tests/native/test_fft.cpp | 3 +- native/tests/native/test_fp.cpp | 2 +- native/tests/native/test_general_modal.cpp | 6 +- native/tests/native/test_z_table.cpp | 2 +- 49 files changed, 1840 insertions(+), 1674 deletions(-) delete mode 100644 native/include/solprop_native/core.hpp create mode 100644 native/include/solprop_native/cpu_execution.hpp create mode 100644 native/include/solprop_native/execution.hpp create mode 100644 native/include/solprop_native/general_state.hpp create mode 100644 native/include/solprop_native/problem.hpp create mode 100644 native/include/solprop_native/readings.hpp create mode 100644 native/include/solprop_native/reference_timing.hpp rename native/src/{core.cpp => cpu_execution.cpp} (67%) create mode 100644 native/src/problem.cpp create mode 100644 native/src/reference_timing.cpp diff --git a/docs/dev/backends.md b/docs/dev/backends.md index 678a89474..1accdf808 100644 --- a/docs/dev/backends.md +++ b/docs/dev/backends.md @@ -643,7 +643,7 @@ The adapter `native/src/solprop_native/backend.py`, its session native backend, with no fallback (§1, ADR-029). `numpy_ref` stays selectable by name. -**Three response families** (`FieldFamily`, `include/solprop_native/core.hpp`), which is +**Three response families** (`FieldFamily`, `include/solprop_native/problem.hpp`), which is all `kerr_coefficients` compiles to, each its own compiled column: | family | grid | `thg` | the column works on | plasma | @@ -839,7 +839,7 @@ build (ADR-076, §4). | compact problem | one column on one node, fewer than 65 536 fine samples | kernels on a team of at most eight threads, transforms on that team, products on one BLAS thread | independent scans: Local, or spawned processes through Pool | | sized problem | the rest | each operation's width chosen from its own element count at construction: stage, response, dense product, FFT, Raman, copy, flow, modal and Adams | one process per problem | -The widths are constants of `include/solprop_native/core.hpp` (`operation_threads`, with +The widths are constants of `include/solprop_native/cpu_execution.hpp` (`operation_threads`, with their calibration in [benchmarks.md](benchmarks.md#kokkos-cpu-operation-size-threading)), and `SOLPROP_NATIVE_*_THREADS` overrides a pool ([Performance tuning](../guide/performance/tuning.md)). The stage boundary @@ -1777,7 +1777,7 @@ at most: three and four threads and radix 8 with 32-point leaves at others), which moves ten realistic steps by at most 8 u: within the transform's rounding bound, so it constrains nothing. This threshold and the stage threshold switch only on `__APPLE__` - (`include/solprop_native/core.hpp`): Windows and linux-aarch64 builds take the values + (`include/solprop_native/cpu_execution.hpp`): Windows and linux-aarch64 builds take the values measured on x86-64 Linux (Cascade Lake and Zen 3), which have not been measured on those platforms ([#328](https://github.com/LupoLab/solprop/issues/328)). - **One stage phase.** `sum_t dPhi_t Phi_t` is needed three times per stage (the diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index d34ec2fc1..0d6ae2bfc 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -310,7 +310,7 @@ Every medium width serial instead lost 2.8 times on averaged helium (EPYC geomet 1.08); every medium width full, the FFTs included, gained on resolved cases but lost on the averaged ones (EPYC 0.94, Xeon 0.90 with averaged helium 1.22). -The promotion applies everywhere but macOS (`stable_kernel_team` in `core.hpp`). On an +The promotion applies everywhere but macOS (`stable_kernel_team` in `cpu_execution.hpp`). On an 18-core Apple M5 Max, whose cores are of two kinds, the 11207-element Adams passes of `argon_blueshift` at 18 threads instead of four took the window from 3.0 to 3.2 s, and `h2_raman_averaged` at 18 threads ran 0.86 of `main`'s time without the promotion diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index b8bbc0cc6..615f8f94b 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -810,12 +810,13 @@ if(SOLPROP_NATIVE_ENABLE_COVERAGE) install(TARGETS solprop_native_labels LIBRARY DESTINATION solprop_native) endif() -add_library(solprop_native_core STATIC src/core.cpp src/crossing.cpp +add_library(solprop_native_core STATIC src/cpu_execution.cpp src/crossing.cpp src/general/checks.cpp src/general/setup.cpp src/general/trial.cpp src/general/statistics.cpp src/general/state.cpp src/general_modal.cpp src/coupled_flow.cpp src/dispersive.cpp src/far_field.cpp src/adams_march.cpp src/watch.cpp src/rk_march.cpp src/engines.cpp src/session.cpp src/session_statistics.cpp src/session_widths.cpp src/low_storage.cpp src/transverse.cpp - src/session_edge.cpp src/dense.cpp src/workspace.cpp) + src/session_edge.cpp src/dense.cpp src/workspace.cpp src/problem.cpp + src/reference_timing.cpp) # solprop SEP 39, SEP 45 and SEP 47: the Adams march's decisions, the boundary watch's # predictions, the Runge-Kutta march's positions and norm, the low-storage replay's # positions and table keys, the native session's scheduling, export, statistics @@ -860,7 +861,7 @@ if(SOLPROP_NATIVE_BLAS_BACKEND STREQUAL "OpenBLAS") ) endif() # Compact problems' transforms take a scoped team through FFTW's loop callback -# (core.cpp, `fftw_team_loops`), which only the OpenMP FFTW runs on the OpenMP team. +# (cpu_execution.cpp, `fftw_team_loops`), which only the OpenMP FFTW runs on the OpenMP team. if(SOLPROP_NATIVE_HOST_FFT_BACKEND STREQUAL "FFTW_OPENMP") target_compile_definitions(solprop_native_core PUBLIC SOLPROP_NATIVE_FFTW_OPENMP=1) endif() diff --git a/native/include/solprop_native/adams_phi.hpp b/native/include/solprop_native/adams_phi.hpp index d00f7ce07..ff8af3cd0 100644 --- a/native/include/solprop_native/adams_phi.hpp +++ b/native/include/solprop_native/adams_phi.hpp @@ -2,8 +2,9 @@ // // SPDX-License-Identifier: GPL-3.0-or-later #pragma once -#include "solprop_native/core.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/fp.hpp" +#include "solprop_native/general_state.hpp" #include diff --git a/native/include/solprop_native/core.hpp b/native/include/solprop_native/core.hpp deleted file mode 100644 index e85ebd115..000000000 --- a/native/include/solprop_native/core.hpp +++ /dev/null @@ -1,1449 +0,0 @@ -// SPDX-FileCopyrightText: 2026 John C. Travers and contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#pragma once - -#include "solprop_native/fields.hpp" -#include "solprop_native/host_source.hpp" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace solprop_native { - -using ExecutionSpace = Kokkos::DefaultExecutionSpace; -using MemorySpace = ExecutionSpace::memory_space; -using Layout = Kokkos::LayoutRight; -/** A two-dimensional range whose last index runs fastest, as `Layout` stores it. - - Kokkos's CUDA and HIP default runs the first index fastest, which puts adjacent - threads a whole row apart in a `LayoutRight` view; a host's default is already - this order. */ -using Range2Policy = Kokkos::MDRangePolicy< - ExecutionSpace, Kokkos::Rank<2, Kokkos::Iterate::Right, Kokkos::Iterate::Right>>; - -/** Samples in one host tile of a `Range2`. Swept from 64 to 1024 on the four-mode RDW - at 16 threads (an Apple M5 Max and an AMD EPYC 7763): 64 to 256 were within 2 % - of each other and 1024 the slowest; 64 balances best, a single 2561-bin column - making 41 tiles rather than 11. */ -inline constexpr std::int64_t host_range2_tile = 64; - -/** `Range2Policy` on `space` over `[lower, upper)`. - - On the host a tile holds about `host_range2_tile` samples: part of one row, or as - many whole rows as fit when a row is shorter. Kokkos's host default tiles two rows - by the whole of the last extent, which leaves a kernel over `(columns, n)` only - ceil(columns / 2) tiles to share: one for a single column, two for four modes, - whatever the thread count. A one-thread instance keeps Kokkos's default: it has - nothing to share, and the small tiles cost the four-mode RDW about 1 % at one - thread on an AMD EPYC 7763. Device builds keep Kokkos's default too. A kernel over - a `Range2` that is elementwise changes no value with the tiling; a reduction over - one (the boundary monitor's split removal) may add in another order, within its - rounding (solprop SEP 25). */ -template -[[nodiscard]] Range2Policy Range2(const ExecutionSpace& space, const LT (&lower)[LN], - const UT (&upper)[UN]) { - static_assert(LN == 2 && UN == 2, "Range2 takes two lower and two upper bounds"); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return Range2Policy(space, lower, upper); -#else - if (space.concurrency() == 1) - return Range2Policy(space, lower, upper); - const auto row = std::clamp(static_cast(upper[1]) - - static_cast(lower[1]), - 1, host_range2_tile); - const std::int64_t tile[2] = {host_range2_tile / row, row}; - return Range2Policy(space, lower, upper, tile); -#endif -} -using Complex = Kokkos::complex; - -} // namespace solprop_native - -// NumPy's complex128 is two doubles, real part first, as `Complex` is. -template <> struct solprop_native::fields::ElementDtype { - static constexpr Dtype value = Dtype::complex128; -}; - -namespace solprop_native { - -/** The boundary monitor's five sums of a state (solprop SEP 21), one reduction value: - the energy, then the low guard's, the high guard's, the collar's and the rate's. */ -struct MonitorSums { - Kokkos::Array values{}; - KOKKOS_INLINE_FUNCTION MonitorSums& operator+=(const MonitorSums& other) { - for (std::size_t slot = 0; slot < 5; ++slot) - values[slot] += other.values[slot]; - return *this; - } -}; - -/** A state's whole-state transfers, in bytes, each way (solprop SEP 44): what lets a - test hold a field-free run to no transfer at all, whichever path would have made one. - */ -struct CrossedBytes { - std::size_t to_host{}; - std::size_t from_host{}; -}; - -/** What the transverse edge reads of a state (solprop SEP 44): four numbers per weight - table, and each mode's energy. */ -struct EdgeNumbers { - std::vector sums; - std::vector columns; - /** A requested collar's `(inward, windowed)` per line. */ - std::vector strip; - /** A requested collar's rectified inward current. */ - double current{}; -}; - -/** A requested collar's lines, as Python describes them (solprop SEP 44): for each, its - axis (0 `r`, 1 `x`, 2 `y`), its count of strip nodes and those nodes, its window, - its inward bins `[first, last)`, the factor on its strip's sums and the measure on - its current, and its onset's node, before and after, with the distance between the - last two. An onset whose node is `no_onset` has no slope to read. */ -struct EdgeLines { - static constexpr std::size_t no_onset = static_cast(-1); - std::vector axes, counts, positions, inward, onsets; - std::vector windows, factors, measures, steps, scale; - double k_min{}; - /** The most one batch of a padded transform holds. */ - std::size_t batch_bytes{}; -}; - -/** What a split-step collar's finish reads of the power it finds (solprop SEP 21, - ADR-057), one reduction value: the energy it takes, then the energy, the low - guard's, the high guard's and the collar's, which the monitor judges the - occupancies on. */ -struct SplitSums { - Kokkos::Array values{}; - KOKKOS_INLINE_FUNCTION SplitSums& operator+=(const SplitSums& other) { - for (std::size_t slot = 0; slot < 5; ++slot) - values[slot] += other.values[slot]; - return *this; - } -}; - -/** The boundary monitor's occupancy of a state (solprop #440), one reduction value: its - energy per walk-off bucket. A device's reduction keeps one of these per thread in - team memory, with no atomics, so its size bounds the buckets (`SPLIT_RATE_BUCKETS`). - */ -inline constexpr std::size_t occupancy_bucket_limit = 128; -// The buckets one reduction holds. A reduction's value has to fit the device's shared -// memory per block, which on HIP's smallest block of 64 threads is `(64 + 2) * -// sizeof(value)`, with the kernel's own static memory beside it within 64 KiB. Kokkos -// checks only the first: a value that fills the block passes its check and the dispatch -// then faults on an AMD Instinct MI325X ("Illegal opcode in command stream"). So a -// reading there takes two passes, leaving half the block free. -#ifdef KOKKOS_ENABLE_HIP -inline constexpr std::size_t occupancy_reduction_slots = 64; -#else -inline constexpr std::size_t occupancy_reduction_slots = occupancy_bucket_limit; -#endif -struct OccupancySums { - Kokkos::Array values{}; - KOKKOS_INLINE_FUNCTION OccupancySums& operator+=(const OccupancySums& other) { - for (std::size_t slot = 0; slot < occupancy_reduction_slots; ++slot) - values[slot] += other.values[slot]; - return *this; - } -}; -#ifdef KOKKOS_ENABLE_HIP -static_assert((64 + 2) * sizeof(OccupancySums) <= 48 * 1024); -#endif - -using ComplexView = Kokkos::View; -using ConstComplexView = Kokkos::View; -using RealView = Kokkos::View; -using ConstRealView = Kokkos::View; - -/** Signal exhaustion of the validated angular rule before source assembly. */ -class EnvelopeConvergenceError : public std::runtime_error { -public: - using std::runtime_error::runtime_error; -}; - -/** Refuse a step solprop's reference refuses with `BackendError` (solprop SEP 45 - phase 4): a coupled flow's action past its squarings or its certificate. */ -class BackendRefusal : public std::runtime_error { -public: - using std::runtime_error::runtime_error; -}; - -/** Process-start CPU policy (solprop SEP 46 section 5's process settings): the pool - capacities, fixed when the runtime starts. Execution instances never exceed the - runtime budget. */ -struct ThreadPolicy { - int kernels{}, fft{}, raman_fft{}, raman_tail_fft{}; - // The field plasma column's band form (solprop SEP 25 section 6.7.6): negative takes - // the build's (tiled on a device, the team column on the host), zero the team - // column, and N > 0 the tiled column with host teams of N threads. - int band_tiles{-1}; - // Bit per pool: keep explicit requests distinct from automatic capacities. - unsigned explicit_pools{}; - bool operator==(const ThreadPolicy&) const = default; -}; -void configure_thread_policy(ThreadPolicy requested); - -/** Whether the field plasma column runs its tiled form under `policy`. */ -[[nodiscard]] inline bool band_tiled(const ThreadPolicy& policy) noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return policy.band_tiles != 0; -#else - return policy.band_tiles > 0; -#endif -} - -/** A tiled band column's host team width under `policy`; zero on a device, whose - teams are its response teams. */ -[[nodiscard]] inline int band_tile_threads(const ThreadPolicy& policy) noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - static_cast(policy); - return 0; -#else - return policy.band_tiles > 0 ? policy.band_tiles : 0; -#endif -} -[[nodiscard]] ThreadPolicy thread_policy(); -[[nodiscard]] ExecutionSpace execution_instance(int threads); - -/** Independent CPU operation classes; device dispatch is unchanged. */ -enum class CpuOperation { - stage, - response, - dense, - fft, - raman, - copy, - flow, - modal, - adams -}; -inline constexpr std::size_t cpu_operation_count = - static_cast(CpuOperation::adams) + 1; -/** Process-start pool supplying an operation's capacity and explicit override. */ -enum class CpuPool { kernels, fft, raman_fft, raman_tail_fft }; - -/** One operation class's grain: below `threshold` work it runs on one thread, below - `full` on the medium width, and from `full` on its pool's capacity. Work is - elements for stages and copies, active columns * fine samples for responses, - rows * inner * samples for dense products, active bins * modes^3 for matrix - functions, elements * history rows for the exponential Adams trial's passes, and - actual real-side batch samples for FFT and Raman plans (tails included); the FFT - class's threshold is where a modal or spatial FFTW plan starts to thread. */ -struct OperationGrain { - std::size_t threshold{}, full{}; - bool operator==(const OperationGrain&) const = default; -}; - -/** A workspace's CPU execution settings (solprop SEP 46 section 5's workspace - settings), copied into the workspace when it is built and never read from anywhere - global, so two live workspaces may hold different ones. - - Native code holds no values for them (solprop_native's built-in table, - `_execution.py`, or a machine calibration supplies every field, with the - measurements behind each built-in value) and refuses a record that leaves one - unset (`require_cpu_execution`). - - - `grain`, indexed by `CpuOperation`. - - A response of `response_full_samples` fine samples or more, or of - `response_full_columns` columns or more, takes the full pool at any size above - its threshold. - - `medium_threads`: the medium width. - - A compact problem (`compact_problem`: one column on one node with fewer than - `compact_max_fine_samples` fine samples) runs its kernels on a team of at most - `compact_threads` and its threaded transforms on that whole team. - - `stable_kernel_team`: once a kernel pool holds a full-width instance, a - medium-width kernel takes it too (`CpuExecutionPool::get`). - - A batch of transforms whose row length is known threads by its shape (solprop - issue #587): from the FFT class's threshold, one thread per - `fft_thread_elements` samples and at most one a row -- or `medium_threads`, if - more, when its rows are at least `fft_long_row` samples long -- and its pool's - capacity from the class's full size. FFTW's threaded plans divide the rows - among the threads, and split a row only where it is long. */ -struct CpuExecution { - std::array grain{}; - std::size_t response_full_samples{}, response_full_columns{}; - int medium_threads{}, compact_threads{}; - std::size_t compact_max_fine_samples{}; - bool stable_kernel_team{}; - std::size_t fft_thread_elements{}, fft_long_row{}; - bool operator==(const CpuExecution&) const = default; - [[nodiscard]] const OperationGrain& - operator[](CpuOperation operation) const noexcept { - return grain[static_cast(operation)]; - } -}; - -/** Throw `std::invalid_argument` unless every threshold, bound and width of - `execution` is positive and each class's full size is at least its threshold. */ -void require_cpu_execution(const CpuExecution& execution); - -/** How a workspace plans its host FFTW transforms (solprop SEP 46 phase 3). - - - `legacy`: the plan policy of `fftw_seed_effort`, with the process's wisdom. - - `profile`: a machine calibration's. The workspace forgets every wisdom before it - plans and after; a transform whose exact plan the profile holds imports that - plan's wisdom and is seeded with `FFTW_WISDOM_ONLY` (a plan it no longer builds - is listed in `missing`, and planned at estimate); any other is seeded at its - group's effort, as the policy would, but with no wisdom from elsewhere. - - `capture`: each transform is seeded alone, from no wisdom, at `capture_effort` - within `time_limit` seconds (none when not positive), and its wisdom is kept in - `captured`, for a calibration to time and store. - - A transform's group is `rows` (one thread), `threaded_rows` or `planes`; its key - names its kind, sizes, strides, placement, alignment and threads - (`fft_plan_key`). Efforts are "estimate", "measure" or "patient". - - Plans runs measured before (solprop ADR-076), in the legacy and profile modes: - `stored` holds each transform's own wisdom by key, below the profile's `wisdom`, - which is consulted first. A transform seeded above estimate that `stored` holds, or - any when `keep` is set, is planned alone: the workspace's wisdom is set aside, the - transform's stored wisdom alone imported (or, for one to measure, what the - workspace measured so far, whose sub-plans it may share), and after the seed the - workspace's wisdom comes back over the transform's. A transform seeded twice is - measured once. A plan the stored wisdom built is listed in - `reused`, as is one left at estimate whose stored wisdom was imported for its - estimate plan; one that measured is kept in `measured` when `keep` is set. */ -struct FftPlanning { - enum class Mode { legacy, profile, capture }; - Mode mode{Mode::legacy}; - std::string rows{"measure"}, threaded_rows{"estimate"}, planes{"measure"}; - std::map wisdom; - std::string capture_effort{"measure"}; - double time_limit{-1.0}; - std::map stored; - bool keep{false}; - // What planning found: each transform's group, the wisdom captured, and the - // profile's plans that no longer build; the stored plans reused and the transforms - // measured, each with its own wisdom. - std::map groups, captured, measured; - std::vector missing, reused; - // While one transform plans alone: the workspace's wisdom, set aside. - bool alone{false}; - std::string held; -}; - -/** Choose a width without changing the runtime or a vendor library's pool. */ -[[nodiscard]] int operation_threads(const ThreadPolicy& policy, - const CpuExecution& execution, CpuPool pool, - CpuOperation operation, std::size_t elements, - std::size_t fine_samples = 0); - -using CpuSizing = std::map>; -#if defined(KOKKOS_ENABLE_OPENMP) && !defined(KOKKOS_ENABLE_CUDA) && \ - !defined(KOKKOS_ENABLE_HIP) -inline constexpr bool cpu_size_policy_enabled = true; -#else -inline constexpr bool cpu_size_policy_enabled = false; -#endif - -/** Own execution instances constructed only while a workspace and its plans form. */ -class CpuExecutionPool { -public: - /** A pool of `pool`'s capacity choosing widths by `execution`, which it copies. */ - CpuExecutionPool(CpuPool pool, const CpuExecution& execution) - : pool_(pool), execution_(execution) {} - const ExecutionSpace& get(CpuOperation operation, std::size_t elements, - std::size_t fine_samples = 0); - /** Size a compact problem's instances (`CpuExecution::compact_threads`), before any - `get`; explicit pool requests still apply as given. */ - void compact() noexcept { compact_ = cpu_size_policy_enabled; } - /** The execution settings this pool chooses widths by. */ - [[nodiscard]] const CpuExecution& execution() const noexcept { return execution_; } - /** The widest instance handed out so far; zero before any. */ - [[nodiscard]] int widest() const noexcept { - return instances_.empty() ? 0 : instances_.rbegin()->first; - } - void clear() { instances_.clear(); } - [[nodiscard]] CpuSizing sizing() const { return sizing_; } - -private: - CpuPool pool_; - CpuExecution execution_; - bool compact_{}; - std::map instances_; - CpuSizing sizing_; -}; - -/** A transform batch's row length, as its CPU width and sizing record count it - (`CpuExecutionPool::get`'s `fine_samples` for the FFT and Raman classes): extent 0 - counts the rows and the remaining extents make one row, so a transverse plane is - one row; a rank-1 view is one row. */ -template [[nodiscard]] std::size_t row_length(const View& rows) noexcept { - const std::size_t count = View::rank() == 1 ? 1 : rows.extent(0); - return count == 0 ? 0 : rows.size() / count; -} - -/** Whether the host FFTW's threaded loops take a scoped team (`CpuTeamScope`): the - OpenMP FFTW with its loop callback (FFTW 3.3.9 and later). Without it a compact - problem's transforms stay serial. The first call installs the callback, whose - loops take the runtime's whole team, as FFTW's own do, outside a scope. */ -[[nodiscard]] bool fftw_team_loops() noexcept; - -/** Record that the build's BLAS is an OpenMP OpenBLAS, so that a compact problem's - scope holds it at one thread (`CpuTeamScope`). */ -void register_openmp_blas(bool registered) noexcept; - -/** While it lives, the calling thread's threaded FFTW loops run on a team of - `threads` (at least their own job count), so that a compact problem's transforms - keep its kernels' team instead of waking the whole runtime; zero restores the - runtime's team. A nonzero team also holds an OpenMP OpenBLAS at one thread, through - the calling thread's OpenMP default width: a compact problem's products are a row - or two deep, and an OpenBLAS region at the fitted width between the team's narrower - ones made GNU OpenMP end and restart the surplus threads at every pair (16 cores of - a Neoverse V2: about 100,000 thread creations in a short GNLSE window). The - outermost scope on a thread also sets its OpenMP default to the runtime's width - and restores it at its end. Scopes nest. */ -class CpuTeamScope { -public: - explicit CpuTeamScope(int threads) noexcept; - ~CpuTeamScope(); - CpuTeamScope(const CpuTeamScope&) = delete; - CpuTeamScope& operator=(const CpuTeamScope&) = delete; - CpuTeamScope(CpuTeamScope&&) = delete; - CpuTeamScope& operator=(CpuTeamScope&&) = delete; - -private: - // The enclosing scope's team; unread where FFTW has no loop callback. - [[maybe_unused]] int saved_; - // Whether this scope holds the registered OpenBLAS at one thread. - bool holds_blas_{}; -}; - -/** How long, in milliseconds, LLVM's OpenMP runtime keeps idle threads spinning while - a native call runs, from the outermost `CpuTeamScope` on a thread to its end, where - the caller's block time returns; zero leaves the process's (solprop ADR-071). GNU's - runtime reads its spin count once, at start, from GOMP_SPINCOUNT. */ -void configure_native_spin(int milliseconds); - -/** LLVM's runtime's block time inside a scope, in milliseconds; -1 with another - runtime. A test hook. */ -[[nodiscard]] int scoped_block_time() noexcept; - -/** The calling thread's OpenMP default width inside a scope (`CpuTeamScope`): the - runtime's, on any thread (solprop issue #130); one without OpenMP. A test hook. */ -[[nodiscard]] int scoped_openmp_default() noexcept; - -/** solprop SEP 28 section 4.2's per-evaluation reference, independent of any - workspace: the seconds of each timed forward-then-backward pair of an in-place, - unnormalised rank-3 complex transform of `values` (row-major - `(planes, n_y, n_x)`, copied in once) through Kokkos-FFT on the FFT execution - space, after planning and `warm` untimed pairs. */ -[[nodiscard]] std::vector -reference_fft3_pairs(const std::complex* values, std::size_t planes, - std::size_t n_y, std::size_t n_x, std::size_t warm, - std::size_t pairs); - -/** The same reference's streaming volume-pass: the seconds of each timed scaled copy - `b = s a` between two complex volumes of `elements` first touched in parallel on - the kernels' execution space, after `warm` untimed copies. */ -[[nodiscard]] std::vector -reference_stream_copies(std::size_t elements, std::size_t warm, std::size_t repeats); - -enum class FieldFamily { field_real, field_analytic, envelope }; - -/** The field families by name, for the field tables (`fields.hpp`). */ -[[nodiscard]] constexpr auto names(const FieldFamily* /*unused*/) { - return std::array{std::pair{"field_real", FieldFamily::field_real}, - std::pair{"field_analytic", FieldFamily::field_analytic}, - std::pair{"envelope", FieldFamily::envelope}}; -} - -enum StatisticsFlag : std::uint64_t { - statistics_omega0 = 1ULL << 0U, - statistics_energy = 1ULL << 1U, - statistics_energy_window = 1ULL << 2U, - statistics_peak_power = 1ULL << 3U, - statistics_peak_power_window = 1ULL << 4U, - statistics_peak_power_total = 1ULL << 5U, - statistics_fwhm = 1ULL << 6U, - statistics_electron_density = 1ULL << 7U, - statistics_peak_ionisation_rate = 1ULL << 8U, - statistics_peak_intensity = 1ULL << 9U, - statistics_quadrature_error = 1ULL << 10U, - statistics_populations = 1ULL << 11U, - // solprop SEP 18's run diagnostics: the six maxima over every node of the field - // plasma column's integration (exposure dip and rise, |paid - freed| and freed, the - // free-current integrand in the first 2K samples and anywhere). - statistics_plasma_integration = 1ULL << 12U, -}; - -/** The widest band rule solprop designs (SEP 18 section 5): K <= 8, so 9 taps. */ -inline constexpr std::size_t plasma_max_taps = 9; - -/** The six parts of SEP 18's run diagnostics, in solprop's order. */ -inline constexpr std::size_t plasma_diagnostic_parts = 6; - -struct Setup; - -/** Fixed, Python-free problem data uploaded once by a native state, grouped as - solprop's kernel contract groups it. The adapter fills it field by field (`lower` in - solprop_native's backend); `validate` checks presence and extents before a state - allocates anything. */ -struct Problem { - /** The state's extents and field family. */ - struct Dims { - std::size_t n_omega{}; - std::size_t n_t{}; - std::size_t n_to{}; - std::size_t n_modes{1}; - std::size_t n_pol_state{1}; - std::size_t n_pol_transform{1}; - std::size_t n_nodes{1}; - std::size_t node_chunk{1}; - FieldFamily family{FieldFamily::field_real}; - }; - /** The frequency and time grids, their windows and masks. */ - struct Grid { - std::vector crop_index; - std::vector active_mask; - std::vector norm_mask; - std::vector omega_window; - std::vector omega; - std::vector power_weight; - std::vector time_window; - std::vector time_window_derivative; - std::vector carrier; - double dt_fine{}; - double dt{}; - }; - /** The separable geometries' operator (free space and the hybrid planar kind) and a - radial run's absorber and on-axis rows. */ - struct Spatial { - std::size_t n_x{}, n_y{}; - std::vector k_squared, k_squared_low, numerator, clamped; - std::vector kperp_sq, kperp_sq_low, frame, spectral_rate; - // solprop kernel schema 45 (#231): the user's share of the spectral rate, read only - // by `absorbed_power`; the operator takes the total. - std::vector user_rate; - std::vector absorber_rate; - // solprop kernel schema 31 (SEP 22 phase 2): a transverse collar's power rate per - // column, added to the field attenuation on active bins; zeros leave the operator. - std::vector kperp_rate; - double evanescent_rate{}; - // solprop SEP 23 (free space only): the stored transverse axes k_x (n_x) and k_y - // (n_y) in rad/m, FFT order, column ix + n_x iy, which the far-field statistics - // read; and the slots of the problem's own far-field request, which size the band - // partials at construction (zero for none). Empty and zero elsewhere. - std::vector k_x, k_y; - std::size_t far_field_slots{}; - // solprop's hybrid planar kind (kernel schema 15): n_p slab channels in x on the - // real (n_x, n_p) synthesis and (n_p, n_x) projection, times a raw DFT in y. - // Columns are (p, k_y) with the channel fastest, n_modes = n_p * n_y, and nodes - // (x, y) with x fastest, n_nodes = n_x * n_y; node_chunk holds whole y rows. - // Row-major (n_p, n_omega) wall terms kappa_p^2 enter the separable operator, and - // absorber_rate is then per column. Zero n_p and empty kappa elsewhere. - std::size_t n_p{}; - std::vector kappa_sq; - // A radial run's spatial absorber and its on-axis statistics' rows. - std::vector absorber_matrix; - std::vector spectral_weights; - std::vector on_axis; - }; - /** solprop kernel schema 35 (SEP 34): a non-separable spatial run's plane measure, - the entrance's material squares (n_omega) and transverse squares (n_modes) whose - root is solprop's `longitudinal_root`; `table` when the material square varies - along z and is uploaded as a `TableKind::flux` table of n_omega values. A - separable run's measure is its own operator's (these stay empty). `closure` - allocates the magnetic closure's buffers at construction, for a request with a - local flux statistic. */ - struct Flux { - std::vector k_squared, k_squared_low; - std::vector kperp_sq, kperp_sq_low; - bool table{}; - bool closure{}; - }; - /** The Runge-Kutta tableau and the march's own choices. */ - struct Stepper { - std::size_t n_stages{}; - std::vector tableau_matrix; - std::vector tableau_nodes; - std::vector solution_weights; - std::vector error_weights; - bool first_same_as_last{}; - bool weighted_controller{}; - bool modified_noise{}; - // solprop SEP 24 phase 3 (kernel schema 32): the low-storage RK4IP march. With it - // no Runge-Kutta view is allocated (stages, accumulators, accept buffer, residual, - // recycled stage, whole-step propagator, stage inverse); the accumulator register - // is, the stage register unless it is elided (solprop's - // `low_storage_stage_register`), and `low_storage_flows` flow volumes (0 - // recomputed, 1 a constant operator's table, 2 a varying operator's step pair). - bool low_storage{}; - bool low_storage_stage_register{true}; - std::size_t low_storage_flows{}; - // Whether a separable route's first input crosses to the transverse scratch - // through tiles of team scratch, as a device's does. - bool low_storage_tiled_input{}; - }; - /** The linear operator's fixed terms, row-major (terms, columns, n_omega). solprop - SEP 29 G1: Phi or the prefactor is the general z table when `phi_table` - (`prefactor_table`); its terms are then `n_phi_terms` (`n_prefactor_terms`) slots - that `table_evaluate` fills with the table's values at a trial's positions, and - `phi_terms` (`prefactor_terms`) is empty. */ - /** The shapes the bindings received for the arrays whose layout counts beside them - give, by field name: `validate` holds each to its counts (a vector keeps its - elements alone). Empty for a problem built in C++. */ - using Shapes = std::map>; - struct Linear { - std::size_t n_phi_terms{}; - std::size_t n_prefactor_terms{}; - bool phi_table{}; - bool prefactor_table{}; - std::vector phi_terms; - std::vector prefactor_terms; - Shapes shapes; - }; - /** The temporal absorber, applied in each stage or split around the step. */ - struct Absorber { - bool in_stage{}; - bool split_step{}; - std::vector fine; - std::vector coarse; - }; - /** The modal synthesis and projection onto the nodes. */ - struct Transform { - std::vector synthesis_matrix; - std::vector projection_matrix; - std::vector projection_coarse_matrix; - double synthesis{}; - double projection{}; - }; - /** The instantaneous Kerr response. */ - struct Kerr { - bool nonlinear{}; - double coefficient{}; - double self_factor{}; - double conjugate_factor{}; - double harmonic_factor{}; - bool take_real{}; - }; - /** The field and envelope plasma columns and their rate tables. */ - struct Plasma { - bool enabled{}; - bool envelope{}; - // solprop's private, test-only gated plasma benchmark: the field plasma's current - // enters as w(t) times its running integral, added to the local polarisation, - // instead of the held current's two collared spectra. Never set by a user run. - bool gated{}; - bool stash{}; - // Channel-major 4 x 512 split angular rules and coarse-axis current factors. - std::vector envelope_rule_nodes, envelope_rule_weights, - envelope_drude_factor; - std::vector envelope_loss_factor; - double charge_mass_ratio{}; - double ionisation_potential{}; - double log_field_min{}; - double d_log_field{}; - double field_min{}; - double field_max{}; - double rate_max{}; - std::size_t rate_intervals{}; - // Row-major (4, rate_intervals). - std::vector rate_coefficients; - std::size_t n_channels{1}; - // solprop SEP 18 (kernel schema 25): the field plasma column's running-integral - // rule, one per problem and shared by every channel. Each running integral is the - // cumulative trapezoid of the integrand filtered by the symmetric taps - // (c_0, ..., c_K), centre first, with the window's ends closed by point - // reflection, and the exposure floored at zero where the fraction is formed. The - // trapezoid rule is `band == false` with the one identity tap, and runs the - // version-24 column unchanged. - bool band{}; - std::vector taps{1.0}; - [[nodiscard]] std::size_t half_width() const noexcept { - return taps.empty() ? 0 : taps.size() - 1; - } - // Channel-major rows: log_min, d_log, field_min, field_max, rate_max, - // interval count, coefficient offset, charge/mass ratio, ionisation potential. - std::vector channel_parameters, channel_coefficients; - Shapes shapes; - }; - /** The delayed Raman response. */ - struct Raman { - bool enabled{}; - bool thg{}; - bool tensor_capable{}; - std::size_t n_terms{}; - double fraction{}; - double epsilon0{}; - // Row-major (n_terms, 2 n_to) and (n_terms, 2). - std::vector undamped; - std::vector tensor_weights; - std::size_t n_components{1}; - std::vector row_component, component_fraction; - Shapes shapes; - }; - /** solprop's general modal route. Row-major: synthesis (map terms, nodes, - components, modes, n_omega); projection (map terms, modes, nodes, components, - n_omega); tensor (nodes, C, C, C, C); delayed tensor (responses, terms, C, C, C, - C); delayed spectrum (responses, terms, n_to + 1); node response (nodes), the - owning response plus one and zero for none. Empty otherwise. The map terms are - map_intervals x map_n_deg Chebyshev coefficients along z, one for a constant - guide; a moving guide's are combined per stage over - TrialScalars::transform_weights. */ - struct General { - bool enabled{}; - bool real_form{}; - std::size_t map_intervals{1}, map_n_deg{1}; - std::size_t components{}, nodes{}, node_chunk{}; - std::size_t responses{}, terms{}; - double pad_scale{}, epsilon0{}; - std::vector synthesis, projection, frequency_factor, delayed_spectrum; - std::vector tensor, delayed_tensor; - std::vector node_response; - }; - /** solprop's linear.coupled_flow. `slot` (n_omega) numbers the bins a section's - generator is non-zero at 1, 2, ... in increasing frequency, zero elsewhere. With - an explicit residual (terms > 0), row-major frozen blocks (sections, active, - modes, modes) and residual terms (terms, active, modes, modes); both empty - otherwise. The actions are loaded per step. - - The actions' native formation, all present or all empty: each section's - eigenbasis of its Schur factor, row-major left `W` and right `Y` (sections, - active, modes, modes) and eigenvalues (sections, active, modes); the route - (sections, active), zero where a bin takes its eigenbasis and 1, 2, ... - numbering in row-major order the bins formed by scaling and squaring, whose Schur - factors `Z` and `T` are the rows of the fallback arrays (fallback, modes, - modes). */ - struct Flow { - bool enabled{}; - std::size_t active{}, sections{}, terms{}; - std::vector slot; - std::vector frozen, residual_terms; - std::vector left, right, eigenvalues; - std::vector route; - std::vector fallback_vectors, fallback_triangular; - }; - /** solprop SEP 6's frequency-dependent electronic response - (`responses.dispersive_kerr`, kernel schema 24) on the fused node path, laid out - by the adapter as solprop's `dispersive_terms_layout` lays it out - (`dispersive.hpp`). The form is 0 (a field grid's full cubic), 1 (the - number-conserving sector) or 2 (an envelope's full cubic). Row-major on the - leaf's spectral layout of `bins` (n_to / 2 + 1 on a field grid, n_to on an - envelope): the input multipliers (fields, bins), each output term's multiplier - (terms, bins) with its product row and factor, the static coefficients and the - polynomial's pair constants (channels, 2) per channel, the groups as offsets - (groups + 1) into members (channels), each symmetric mixture's channels' `g^4` - (channels, bins) and the input and output scales (bins). On the general modal - route (SEP 6 Phase 3b) each group's key is its node region (groups), the - polynomial forms one cube per distinct key, and every general node's region - (general nodes) says where a channel acts; both are empty elsewhere, where every - channel acts at every node. Empty otherwise. */ - struct Dispersive { - bool enabled{}; - bool symmetric{}; - int form{}; - std::vector inputs, term_multipliers, coefficients, pair_constants, - quartics, input_scale, output_scale; - std::vector term_rows, term_channels, group_offsets, group_members, - group_keys, node_region; - }; - - Dims dims; - Grid grid; - Spatial spatial; - Flux flux; - Stepper stepper; - Linear linear; - Absorber absorber; - Transform transform; - Kerr kerr; - Plasma plasma; - Raman raman; - General general; - Flow flow; - Dispersive dispersive; - // How the workspace's CPU operations thread (solprop SEP 46): set by the caller, - // refused when unset (`require_cpu_execution`). - CpuExecution cpu_execution{}; - // How the workspace plans its host transforms (solprop SEP 46 phase 3). - FftPlanning fft_planning{}; - /** The set-once data a workspace applies once it has built its state (solprop SEP - 52 section 5.3): the stage, z and flux tables, the flow guard, the controllers, - the low-storage settings, the monitor's weights and the edge's geometry - (`workspace.hpp`). Shared, so that a copy of the problem copies none of it. */ - std::shared_ptr setup; - - /** Refuse, naming the field, data whose presence or extents disagree with the - problem's own flags and extents: the checks that need no execution space. */ - void validate() const; - - /** Free space or the hybrid planar kind: the separable operator and a y FFT. */ - [[nodiscard]] bool separable_geometry() const noexcept { return spatial.n_x != 0; } - [[nodiscard]] bool slab_geometry() const noexcept { return spatial.n_p != 0; } - [[nodiscard]] bool free_geometry() const noexcept { - return spatial.n_x != 0 && spatial.n_p == 0; - } - [[nodiscard]] std::size_t n_columns() const noexcept { - return dims.n_modes * dims.n_pol_state; - } - [[nodiscard]] std::size_t effective_node_pol() const noexcept { - return dims.n_pol_transform * dims.n_pol_state; - } - [[nodiscard]] std::size_t raman_drives() const noexcept { - if (effective_node_pol() == 2) - for (std::size_t index = 1; index < raman.tensor_weights.size(); index += 2) - if (raman.tensor_weights[index] != 0.0) - return 3; - return 1; - } - [[nodiscard]] std::size_t raman_kernels() const noexcept { - return raman_drives() == 3 ? 2 : 1; - } -}; - -/** The most samples an Adams formula of solprop's shipped orders combines. */ -// The problem's groups as the seam fills them, one field table a group (solprop SEP 52 -// section 5.3). The relations a table states within its group are its schema's; the -// extents that a count in another group gives are `Problem::validate`'s. -[[nodiscard]] constexpr auto describe(const Problem::Dims* /*unused*/) { - using G = Problem::Dims; - return fields::table( - "Problem.Dims", fields::value("n_omega", &G::n_omega), - fields::value("n_t", &G::n_t), fields::value("n_to", &G::n_to), - fields::value("n_modes", &G::n_modes), - fields::value("n_pol_state", &G::n_pol_state), - fields::value("n_pol_transform", &G::n_pol_transform), - fields::value("n_nodes", &G::n_nodes), - fields::value("node_chunk", &G::node_chunk), fields::value("family", &G::family)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Grid* /*unused*/) { - using G = Problem::Grid; - return fields::table( - "Problem.Grid", fields::array("crop_index", &G::crop_index), - fields::array("active_mask", &G::active_mask), - fields::array("norm_mask", &G::norm_mask), - fields::array("omega_window", &G::omega_window), - fields::array("omega", &G::omega), - fields::array("power_weight", &G::power_weight), - fields::array("time_window", &G::time_window), - fields::array("time_window_derivative", &G::time_window_derivative), - fields::array("carrier", &G::carrier), fields::value("dt_fine", &G::dt_fine), - fields::value("dt", &G::dt)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Spatial* /*unused*/) { - using G = Problem::Spatial; - return fields::table( - "Problem.Spatial", fields::value("n_x", &G::n_x), fields::value("n_y", &G::n_y), - fields::array("k_squared", &G::k_squared), - fields::array("k_squared_low", &G::k_squared_low), - fields::array("numerator", &G::numerator), fields::array("clamped", &G::clamped), - fields::array("kperp_sq", &G::kperp_sq), - fields::array("kperp_sq_low", &G::kperp_sq_low), - fields::array("frame", &G::frame), - fields::array("spectral_rate", &G::spectral_rate), - fields::array("user_rate", &G::user_rate), - fields::array("absorber_rate", &G::absorber_rate), - fields::array("kperp_rate", &G::kperp_rate), - fields::value("evanescent_rate", &G::evanescent_rate), - fields::array("k_x", &G::k_x), fields::array("k_y", &G::k_y), - fields::value("far_field_slots", &G::far_field_slots), - fields::value("n_p", &G::n_p), fields::array("kappa_sq", &G::kappa_sq), - fields::array("absorber_matrix", &G::absorber_matrix), - fields::array("spectral_weights", &G::spectral_weights), - fields::array("on_axis", &G::on_axis)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Flux* /*unused*/) { - using G = Problem::Flux; - return fields::table("Problem.Flux", fields::array("k_squared", &G::k_squared), - fields::array("k_squared_low", &G::k_squared_low), - fields::array("kperp_sq", &G::kperp_sq), - fields::array("kperp_sq_low", &G::kperp_sq_low), - fields::value("table", &G::table), - fields::value("closure", &G::closure)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Stepper* /*unused*/) { - using G = Problem::Stepper; - return fields::table( - "Problem.Stepper", fields::value("n_stages", &G::n_stages), - fields::array("tableau_matrix", &G::tableau_matrix), - fields::array("tableau_nodes", &G::tableau_nodes), - fields::array("solution_weights", &G::solution_weights), - fields::array("error_weights", &G::error_weights), - fields::value("first_same_as_last", &G::first_same_as_last), - fields::value("weighted_controller", &G::weighted_controller), - fields::value("modified_noise", &G::modified_noise), - fields::value("low_storage", &G::low_storage), - fields::value("low_storage_stage_register", &G::low_storage_stage_register), - fields::value("low_storage_flows", &G::low_storage_flows), - fields::value("low_storage_tiled_input", &G::low_storage_tiled_input)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Linear* /*unused*/) { - using G = Problem::Linear; - return fields::table("Problem.Linear", - fields::value("n_phi_terms", &G::n_phi_terms), - fields::value("n_prefactor_terms", &G::n_prefactor_terms), - fields::value("phi_table", &G::phi_table), - fields::value("prefactor_table", &G::prefactor_table), - fields::array("phi_terms", &G::phi_terms), - fields::array("prefactor_terms", &G::prefactor_terms)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Absorber* /*unused*/) { - using G = Problem::Absorber; - return fields::table("Problem.Absorber", fields::value("in_stage", &G::in_stage), - fields::value("split_step", &G::split_step), - fields::array("fine", &G::fine), - fields::array("coarse", &G::coarse)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Transform* /*unused*/) { - using G = Problem::Transform; - return fields::table( - "Problem.Transform", fields::array("synthesis_matrix", &G::synthesis_matrix), - fields::array("projection_matrix", &G::projection_matrix), - fields::array("projection_coarse_matrix", &G::projection_coarse_matrix), - fields::value("synthesis", &G::synthesis), - fields::value("projection", &G::projection)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Kerr* /*unused*/) { - using G = Problem::Kerr; - return fields::table("Problem.Kerr", fields::value("nonlinear", &G::nonlinear), - fields::value("coefficient", &G::coefficient), - fields::value("self_factor", &G::self_factor), - fields::value("conjugate_factor", &G::conjugate_factor), - fields::value("harmonic_factor", &G::harmonic_factor), - fields::value("take_real", &G::take_real)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Plasma* /*unused*/) { - using G = Problem::Plasma; - using E = fields::Extent; - return fields::table( - "Problem.Plasma", fields::value("enabled", &G::enabled), - fields::value("envelope", &G::envelope), fields::value("gated", &G::gated), - fields::value("stash", &G::stash), - fields::array("envelope_rule_nodes", &G::envelope_rule_nodes, E::of("n_channels"), - E::fixed(4), E::fixed(512)), - fields::array("envelope_rule_weights", &G::envelope_rule_weights, - E::of("n_channels"), E::fixed(4), E::fixed(512)), - fields::array("envelope_drude_factor", &G::envelope_drude_factor, E::any()), - fields::array("envelope_loss_factor", &G::envelope_loss_factor, E::any()), - fields::value("charge_mass_ratio", &G::charge_mass_ratio), - fields::value("ionisation_potential", &G::ionisation_potential), - fields::value("log_field_min", &G::log_field_min), - fields::value("d_log_field", &G::d_log_field), - fields::value("field_min", &G::field_min), - fields::value("field_max", &G::field_max), - fields::value("rate_max", &G::rate_max), - fields::value("rate_intervals", &G::rate_intervals), - fields::array("rate_coefficients", &G::rate_coefficients, E::fixed(4), - E::of("rate_intervals")), - fields::value("n_channels", &G::n_channels), fields::value("band", &G::band), - fields::array("taps", &G::taps, E::any()), - fields::array("channel_parameters", &G::channel_parameters), - fields::array("channel_coefficients", &G::channel_coefficients)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Raman* /*unused*/) { - using G = Problem::Raman; - using E = fields::Extent; - return fields::table( - "Problem.Raman", fields::value("enabled", &G::enabled), - fields::value("thg", &G::thg), - fields::value("tensor_capable", &G::tensor_capable), - fields::value("n_terms", &G::n_terms), fields::value("fraction", &G::fraction), - fields::value("epsilon0", &G::epsilon0), - fields::array("undamped", &G::undamped, E::of("n_terms"), E::any()), - fields::array("tensor_weights", &G::tensor_weights, E::of("n_terms"), - E::fixed(2)), - fields::value("n_components", &G::n_components), - fields::array("row_component", &G::row_component), - fields::array("component_fraction", &G::component_fraction)); -} - -[[nodiscard]] constexpr auto describe(const Problem::General* /*unused*/) { - using G = Problem::General; - return fields::table( - "Problem.General", fields::value("enabled", &G::enabled), - fields::value("real_form", &G::real_form), - fields::value("map_intervals", &G::map_intervals), - fields::value("map_n_deg", &G::map_n_deg), - fields::value("components", &G::components), fields::value("nodes", &G::nodes), - fields::value("node_chunk", &G::node_chunk), - fields::value("responses", &G::responses), fields::value("terms", &G::terms), - fields::value("pad_scale", &G::pad_scale), - fields::value("epsilon0", &G::epsilon0), - fields::array("synthesis", &G::synthesis), - fields::array("projection", &G::projection), - fields::array("frequency_factor", &G::frequency_factor), - fields::array("delayed_spectrum", &G::delayed_spectrum), - fields::array("tensor", &G::tensor), - fields::array("delayed_tensor", &G::delayed_tensor), - fields::array("node_response", &G::node_response)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Flow* /*unused*/) { - using G = Problem::Flow; - return fields::table( - "Problem.Flow", fields::value("enabled", &G::enabled), - fields::value("active", &G::active), fields::value("sections", &G::sections), - fields::value("terms", &G::terms), fields::array("slot", &G::slot), - fields::array("frozen", &G::frozen), - fields::array("residual_terms", &G::residual_terms), - fields::array("left", &G::left), fields::array("right", &G::right), - fields::array("eigenvalues", &G::eigenvalues), fields::array("route", &G::route), - fields::array("fallback_vectors", &G::fallback_vectors), - fields::array("fallback_triangular", &G::fallback_triangular)); -} - -[[nodiscard]] constexpr auto describe(const Problem::Dispersive* /*unused*/) { - using G = Problem::Dispersive; - return fields::table("Problem.Dispersive", fields::value("enabled", &G::enabled), - fields::value("symmetric", &G::symmetric), - fields::value("form", &G::form), - fields::array("inputs", &G::inputs), - fields::array("term_multipliers", &G::term_multipliers), - fields::array("coefficients", &G::coefficients), - fields::array("pair_constants", &G::pair_constants), - fields::array("quartics", &G::quartics), - fields::array("input_scale", &G::input_scale), - fields::array("output_scale", &G::output_scale), - fields::array("term_rows", &G::term_rows), - fields::array("term_channels", &G::term_channels), - fields::array("group_offsets", &G::group_offsets), - fields::array("group_members", &G::group_members), - fields::array("group_keys", &G::group_keys), - fields::array("node_region", &G::node_region)); -} - -inline constexpr std::size_t adams_max_samples = 8; - -/** The most defect norms an Adams trial scores: its order's and its two neighbours'. */ -inline constexpr std::size_t adams_max_norms = 3; - -/** - * The most channels of a coupled block whose channel vectors a host task of the - * resident Adams engine holds on its stack (solprop SEP 16 phase 6). A wider block's - * host passes hold them in a scratch row per bin, allocated with the history (solprop - * SEP 47 phase 4). A device build's passes take a thread per (bin, channel) and - * hold no channel vectors, so it steps any width. - */ -inline constexpr std::size_t adams_max_channels = 16; - -/** - * One Adams trial's coefficient tables, passed by value to the resident engine's - * kernels (solprop SEP 16 phase 3b). `slot` names history rows newest first; - * `predictor` weights the first `order` of them; `corrector` and each of `n_rows` - * defect rows weight `(source, slot[0], ...)` over `width` samples. - */ -struct AdamsWeights { - std::size_t slot[adams_max_samples]{}; - double predictor[adams_max_samples]{}; - double corrector[adams_max_samples]{}; - double rows[adams_max_norms][adams_max_samples]{}; - std::size_t order{}; - std::size_t width{}; - std::size_t n_rows{}; -}; - -/** - * One exponential Adams trial's coefficient tables (solprop SEP 16 phase 3b): the - * predictor's Lagrange coefficients `predictor[sample][monomial]` over the `order` - * newest rows, the corrector's over `(source, slot[0], ...)` and, per scored defect, - * the divided-difference coefficients over the same `width` samples followed by the - * node polynomial's `count` monomial coefficients. - */ -struct ExponentialWeights { - std::size_t slot[adams_max_samples]{}; - double predictor[adams_max_samples][adams_max_samples]{}; - double corrector[adams_max_samples][adams_max_samples]{}; - double rows[adams_max_norms][2 * adams_max_samples]{}; - std::size_t order{}; - std::size_t width{}; - std::size_t count{}; - std::size_t n_rows{}; - // A varying operator's transported compensation (solprop SEP 16 section 4.2): the - // carried samples' spans 1 - theta_j, and the predictor's, the corrector's and each - // defect's weights over (endpoint, carried...), still to be multiplied by the step. - bool compensated{}; - double span[adams_max_samples]{}; - double comp_predictor[adams_max_samples]{}; - double comp_corrector[adams_max_samples + 1]{}; - double comp_rows[3][adams_max_samples + 1]{}; - // Dense observation's weights over (endpoint, history...) at the fraction, and the - // samples' normalised positions. - double dense_weight[adams_max_samples + 1]{}; - double dense_theta[adams_max_samples + 1]{}; - // A scalar carrier frame (solprop SEP 35): sigma = i shift. The moments are those - // of x - sigma, and history row t (the t-th of `slot`) is read rotated by - // factor[t] = exp(sigma (1 - theta_t)); the new source's factor is one. - bool shifted{}; - double shift{}; - // exp(-sigma), formed once a trial. - double unshift_re{1.0}; - double unshift_im{}; - double factor_re[adams_max_samples]{}; - double factor_im[adams_max_samples]{}; -}; - -/** Small host bundle for all stages and the whole-step propagator. */ -struct TrialScalars { - std::vector phi_delta; - std::vector phi_at; - std::vector prefactor; - std::vector density; - std::vector taper; - std::vector preionisation; - std::vector raman_rates; - std::vector kerr_density, plasma_densities, plasma_preionisation, - raman_densities; - // A coupled flow's step section and its stages' residual weights (n_stages, terms). - std::size_t flow_section{}; - std::vector flow_weights; - // A moving guide's map weights (n_stages, map terms), zero outside the section each - // stage lies in (solprop kernel schema 14); empty for maps that do not move. - std::vector transform_weights; - // Each stage's channel densities of SEP 6's dispersive response (n_stages, - // channels), m^-3; empty without it. - std::vector dispersive_densities; -}; - -/** Which general z table a call names (solprop SEP 29 G1). */ -enum class TableKind { phi, prefactor, rate, flux }; - -/** Whether `problem` runs on a compact CPU team (`CpuExecution::compact_threads`): one - column on one node, fewer than its record's `compact_max_fine_samples` fine - samples, and neither the general modal route, a coupled flow nor a separable - geometry. */ -[[nodiscard]] bool compact_problem(const Problem& problem) noexcept; - -class GeneralWorkspace; - -/** Multicolumn general state with batched FFTs and chunked node transforms. */ -class GeneralState { -public: - explicit GeneralState(Problem problem); - ~GeneralState(); - GeneralState(const GeneralState&) = delete; - GeneralState& operator=(const GeneralState&) = delete; - GeneralState(GeneralState&&) = delete; - GeneralState& operator=(GeneralState&&) = delete; - - /** Load the field and noise from host memory, one copy into the memory space. */ - void set_initial(std::span field, std::span noise); - [[nodiscard]] std::vector trial_step(double dz, const TrialScalars& scalars); - void prepare_trial(double dz, const TrialScalars& scalars); - /** Adopt an Adams step's end state through the shared finish. */ - void adopt(const std::vector& field, double dz); - /** - * One step of solprop's low-storage RK4IP march (SEP 24 phase 3): four register - * passes accepted in place, returning the raw accepted state's squared norm over the - * norm mask. `scalars` holds one row per position -- 0 the step's start, 1 its - * midpoint, 2 its end, any later rows ignored -- and its `phi_delta` rows 0 and 1 - * the flows' intervals `P` and `Q`, row 2 the table's, read only when - * `rebuild_table` builds a constant operator's table. - */ - [[nodiscard]] double low_storage_step(double dz, const TrialScalars& scalars, - bool rebuild_table); - /** The resident Adams history (solprop SEP 16 phase 3b, solprop ADR-043). */ - void adams_allocate(std::size_t slots, bool compensated, bool interpolate = true); - void adams_compensation_start(std::size_t slot, const std::vector& rate); - void adams_start(std::size_t slot, const TrialScalars& scalars); - void adams_scale(double rtol, double atol, const std::vector& weights); - [[nodiscard]] std::vector - adams_fused_trial(double dz, const AdamsWeights& weights, - const std::vector& divisor, const TrialScalars& scalars); - void adams_fused_accept(double dz, const AdamsWeights& keep, std::size_t free, - const AdamsWeights& dense); - void adams_hold(); - void adams_restore(); - void adams_observe(const AdamsWeights& weights, const TrialScalars& scalars); - [[nodiscard]] std::vector - adams_exponential_trial(double dz, const ExponentialWeights& weights, - const std::vector& divisor, const TrialScalars& step, - const TrialScalars& sample, const std::vector& rate); - void adams_exponential_accept(double dz, std::size_t free, const AdamsWeights& dense); - void adams_exponential_observe(double xi, double dz, - const ExponentialWeights& weights); - /** - * A Runge-Kutta march's dense output in a native session (solprop SEP 47 section - * 5.3), DOPRI5's continuous extension (Hairer, Norsett and Wanner II.6) as solprop's - * `extension_coefficients` and `_observe_inside` form it. `rk_dense_allocate` keeps - * the tableau's dense-output weights, one per stage, and the views the extension - * needs; `rk_dense_hold` contracts the accepted, unfinished trial over `dz` -- the - * state before it and its stages -- into the extension's five coefficients and marks - * the bins the step's flow changes by more than an e-fold; - * `rk_dense_observe` holds the committed state aside and makes the state the - * extension at the fraction `theta`, carried from the anchor by the flow of `carry` - * (a trial's arrays from the anchor to the plane) and masked; `rk_dense_restore` - * makes the held state current again; `rk_dense_release` frees the views. - */ - void rk_dense_allocate(const std::vector& weights); - void rk_dense_hold(double dz); - void rk_dense_observe(double theta, const TrialScalars& carry); - void rk_dense_restore(); - void rk_dense_release() noexcept; - /** - * The resident Adams history's persistent rows, for a resume (solprop SEP 29 G2): - * `"store"`, and `"states"`, `"rates"` and `"applied"` where allocated, each - * `(slots, columns, n_omega)`. Scratch views are not rows: a trial rebuilds them. - */ - [[nodiscard]] std::vector adams_row_names() const; - /** A named row view's extents; refuses an unknown or unallocated name. */ - [[nodiscard]] std::array - adams_row_extents(const std::string& name) const; - /** A lossless host copy of a named row view, in its LayoutRight order. */ - [[nodiscard]] std::vector adams_rows_to_host(const std::string& name) const; - /** The same copy written straight into `destination`, of `size` elements. */ - void adams_rows_to_host(const std::string& name, std::complex* destination, - std::size_t size) const; - /** Replace a named row view with a host copy of its extents; forget any trial. */ - void adams_rows_from_host(const std::string& name, - const std::vector& values); - /** - * The general z tables (solprop SEP 29 G1). `table_upload` makes a table resident: - * Phi's, the prefactor's, or Phi's rate (the derivative series, for the compensated - * Adams march); `table_evaluate` writes Phi's and the prefactor's values at each - * position into the given term slots, and `table_rate` the rate's value into a Phi - * term slot. `table_terms_to_host` copies one term slot to the host, and - * `phase_sum_to_host` evaluates the stage kernels' phase sum over the Phi terms for - * one row of weights; both are diagnostics. - */ - void table_upload(TableKind kind, std::vector edges, double lower, - double upper, const std::complex* coefficients, - std::size_t n_intervals, std::size_t n_terms, std::size_t n_values); - void table_evaluate(const std::vector& slots, - const std::vector& positions); - void table_rate(std::size_t slot, double z, bool before); - [[nodiscard]] std::vector table_terms_to_host(TableKind kind, - std::size_t slot) const; - [[nodiscard]] std::vector - phase_sum_to_host(const std::vector& weights); - [[nodiscard]] std::vector weighted_error(const std::vector& scale); - /** Load a coupled flow's actions; see `CoupledFlowAction::set_actions`. */ - void set_flow_actions(const std::vector& forward, - const std::vector& inverse, - const std::vector& identity); - /** A coupled flow's held actions; see `CoupledFlowAction::actions_to_host`. */ - [[nodiscard]] std::vector flow_actions_to_host(bool inverse) const; - /** The last exponential Adams trial's coupled block functions, row-major - `(slots + 1, n_active, n_modes, n_modes)`, the flow first: an observation for - tests (solprop SEP 47 phase 4). */ - [[nodiscard]] std::vector adams_block_to_host() const; - /** The last Adams trial's exponent `Delta Phi`, row-major `(n_columns, n_omega)`: - an observation for tests. */ - [[nodiscard]] std::vector adams_exponent_to_host() const; - /** Form a coupled flow's actions natively; see `CoupledFlowAction::form_actions`. */ - [[nodiscard]] std::vector - form_flow_actions(std::size_t section, double dz, - const std::vector& fractions); - void prepare_raman(const std::vector& rates, - const std::vector& densities = {}); - /** - * The spatial statistics of the accepted state at the plane `z`, measured by the - * medium's axial flux (solprop SEP 34): flags 1 energy, 2 on-axis peak intensity, 4 - * peak intensity, 8 on-axis spectrum, 16 the widths' fluence block (the signed - * fluence at each node, the axis fluence, which a separable route leaves NaN since - * only a radial width reads it, then the bound on the largest magnitude), 32 a - * Cartesian plane's fluence moments, which hold its scaled profile for - * `plane_fluence_trace` (the lowest value, its node, the bound on the largest - * magnitude, the largest value, the count of values that aren't finite, then the - * scaled sum and its x and y index moments), packed in that order. - */ - [[nodiscard]] std::vector - spatial_statistics(std::uint64_t flags, double intensity_factor, double z); - /** - * The second moment and the azimuthal average of the plane's scaled fluence that a - * `spatial_statistics` call with flag 32 left, on rings solprop's own rule fixes, - * with the centre in metres from the axes' first samples. Returns the second-moment - * sum, then one average per ring. The profile itself stays in the execution space. - */ - [[nodiscard]] std::vector - plane_fluence_trace(double center_x, double center_y, double dx, double dy, - double extent, double spacing, std::size_t n_radii, - const std::vector& cosines, - const std::vector& sines); - /** - * The modal state's spectrum at one physical point: `matrix`, one or two rows of - * `n_modes` columns, contracted with the modes, as `(n_rows * n_pol, n_omega)` - * samples. What an on-axis scan needs of the state. - */ - [[nodiscard]] std::vector axis_spectrum(const std::vector& matrix); - /** - * solprop SEP 19's on-axis plasma populations of the accepted state: `matrix`'s row - * (as `axis_spectrum` takes it, the pad scale and the taper in it) on the fine time - * grid, scanned by the node scan with each channel's preionisation. Returns each - * channel's final ionised fraction and peak rate, `(n_channels, 2)`. - */ - [[nodiscard]] std::vector - axis_populations(const std::vector& matrix, - const std::vector& channel_preionisation); - /** - * solprop SEP 23's far-field statistics of the accepted state on a Cartesian or line - * grid, one banded pass; see `FarFieldPass::run` (`far_field.hpp`) for the tables, - * the flags (from 32) and the packed order. - */ - [[nodiscard]] std::vector - aperture_statistics(std::uint64_t flags, const std::vector& apertures, - int exponent, const std::vector& regions, - double intensity_factor); - [[nodiscard]] std::vector - statistics(std::uint64_t flags, const std::vector& windows, - const std::vector& on_axis, double density, double taper, - double preionisation, double intensity_factor, - const std::vector& channel_preionisation = {}, - const std::vector& power_weight = {}); - /** - * solprop SEP 47: a general modal basis's statistics, functionals of the physical - * flux `a.H W a` (`solprop.stats._whitened`). The state's columns are whitened by - * `root`, `W^(1/2)` at each frequency, `(n_omega, n_modes, n_modes)` row-major - * (empty for the identity), into a statistics scratch, and the temporal `flags` - * (omega0, energy, peak power total and the width's trace) are evaluated on - * `c = W^(1/2) a`. Returns their packed values, then each column's `sum |c|^2`, - * then, with `spectral`, each bin's `sum |c|^2` over the columns. The state is - * unchanged. - */ - [[nodiscard]] std::vector - whitened_statistics(const std::vector& root, std::uint64_t flags, - const std::vector& windows, double intensity_factor, - bool spectral); - void finish(double dz); - void promote_fsal(); - void clear_fsal(); - void set_absorber_quiet(bool quiet); - /** - * Bind solprop's boundary monitor (SEP 21): its four time weights, row-major - * `(4, n_t)`, and its columns' quadrature by mode (empty for unit weights). From - * here on each split-step finish adds what it takes to an accumulator, which - * `monitor_reading` drains, and keeps the energies of the power it found. - */ - void bind_monitor(const std::vector& weights, - const std::vector& column_weights); - void unbind_monitor() noexcept; - /** - * The bound monitor's reading of the accepted state, over every column: `sum_t P`, - * the four weighted sums `sum_t w_k P`, what the split collar took since the last - * reading, and `sum_t P` and the two guards' and the collar's sums of the power the - * last step's split found (zeros before the first such step). - */ - [[nodiscard]] std::vector monitor_reading(); - /** - * The last accepted step's split collar sums, read without draining anything (SEP - * 52 section 5.2's `AcceptRecord`): what its split took, then `sum_t P` and the two - * guards' and the collar's sums of the power it found, from the masked field before - * the split; none without a bound monitor and a split collar. - */ - [[nodiscard]] std::optional> split_sums(); - /** - * Bind the split collar's walk-off bucket of each frequency bin, by mode - * `(n_modes, n_omega)`, `count` buckets (solprop #440): from here on - * `monitor_reading` also returns the state's energy per bucket, the fastest first. - */ - void bind_occupancy(const std::vector& buckets, std::size_t count); - [[nodiscard]] double quadrature_error() const; - [[nodiscard]] double quadrature_peak(); - [[nodiscard]] std::string synthesis_provider() const; - [[nodiscard]] std::string projection_provider() const; - [[nodiscard]] std::string gemm_shape_policy() const; - [[nodiscard]] std::size_t response_team_size() const noexcept; - [[nodiscard]] std::vector field_to_host() const; - /** The field into `destination`, of the field's size: one copy from the space. */ - void field_to_host(std::span destination) const; - /** The committed field where the host can read it in place, `columns * n_omega` - elements in row-major order, after the state's work is fenced; null where the - memory space is not host-accessible. A reader that passes it through the host - adds its bytes with `count_to_host` (solprop SEP 47 section 5.4). */ - [[nodiscard]] const Complex* host_field() const; - /** Count `bytes` read out of the state by a host reader of `host_field`. */ - void count_to_host(std::size_t bytes) const noexcept; - /** What the transverse edge's reading sums with (solprop SEP 44). */ - void bind_edge(const std::vector& faces, const std::vector& clip, - const std::vector& totals, - const std::vector& synthesis, std::size_t first_row); - /** The bytes through this state's crossings since it was built, each way. */ - [[nodiscard]] CrossedBytes crossed_bytes() const noexcept; - /** The edge's region sums and each mode's energy, where the state lives. */ - /** A requested collar's lines, whose strip and current a reading then forms. */ - void bind_edge_lines(const EdgeLines& lines); - EdgeNumbers edge_reading(bool faces, bool whole, bool totals, bool strip, - bool current); - /** The threads a large crossing takes (`crossing.hpp`), from solprop's budget. */ - void set_crossing_workers(std::size_t workers) noexcept; - [[nodiscard]] std::vector stage_to_host(std::size_t stage) const; - void stage_to_host(std::size_t stage, std::span destination) const; - /** - * The last evaluation's projected polarisation spectrum on the fine rule, or on the - * embedded one (finishing SEP 6's dispersive products there first), a diagnostic. - */ - [[nodiscard]] std::vector projected_to_host(bool coarse); - /** How SEP 6's dispersive response runs: empty without it. */ - [[nodiscard]] std::map dispersive_path() const; - [[nodiscard]] std::map memory_ledger() const; - [[nodiscard]] CpuSizing execution_sizing() const; - /** The CPU execution record the workspace's pools choose widths by. */ - [[nodiscard]] const CpuExecution& cpu_execution() const; - /** The workspace's FFT planning, with what its planning found. */ - [[nodiscard]] const FftPlanning& fft_planning() const; - [[nodiscard]] double planning_seconds() const noexcept; - [[nodiscard]] double raman_planning_seconds() const noexcept; - [[nodiscard]] std::string raman_fft_provider() const; - [[nodiscard]] std::map raman_fft_tasks() const; - [[nodiscard]] std::size_t raman_kernel_transforms() const noexcept; - [[nodiscard]] bool fsal_ready() const noexcept; - [[nodiscard]] std::size_t rk_source_evaluations() const noexcept; - /** On-axis row scans made so far (`axis_populations`), cumulative. */ - [[nodiscard]] std::size_t axis_scans() const noexcept; - [[nodiscard]] bool closed() const noexcept; - void close() noexcept; - /** - * A host source (solprop SEP 49 section 6.2): from here on each stage source is - * `source`'s evaluation of the lab-frame field, the state's own response passes not - * run, and the stage's finish its collar, absorber and propagator alone. Its two - * host rows are allocated here, once, and listed in the memory ledger. Refused on a - * device build and for anything but one real column off the general and separable - * routes; null unbinds. - */ - void bind_host_source(std::shared_ptr source); - /** The bound host source, or null. */ - [[nodiscard]] HostSource* host_source() const noexcept; - /** The next host evaluation's position, m, and purpose, which the engine sets. */ - void host_evaluation(double z, HostPurpose purpose) noexcept; - /** A test hook (solprop issue #129): `fault` runs, once, as the next Adams source - evaluation starts, a trial's with its prediction in the state; empty disarms. */ - void arm_source_fault(std::function fault) noexcept; - -private: - /** `function(workspace, arguments...)` inside the workspace's CPU team scope - (`CpuTeamScope`); defined and used in src/general/state.cpp alone, so that no unit - outside src/general/ sees the workspace class. */ - template - decltype(auto) with_team(Function function, Arguments&&... arguments) const; - - std::unique_ptr workspace_; -}; - -} // namespace solprop_native - -template <> struct Kokkos::reduction_identity { - KOKKOS_FUNCTION static solprop_native::MonitorSums sum() { return {}; } -}; -template <> struct Kokkos::reduction_identity { - KOKKOS_FUNCTION static solprop_native::OccupancySums sum() { return {}; } -}; -template <> struct Kokkos::reduction_identity { - KOKKOS_FUNCTION static solprop_native::SplitSums sum() { return {}; } -}; diff --git a/native/include/solprop_native/coupled_flow.hpp b/native/include/solprop_native/coupled_flow.hpp index 02a38973f..c8d96dff8 100644 --- a/native/include/solprop_native/coupled_flow.hpp +++ b/native/include/solprop_native/coupled_flow.hpp @@ -4,7 +4,8 @@ #pragma once -#include "solprop_native/core.hpp" +#include "solprop_native/execution.hpp" +#include "solprop_native/problem.hpp" #include #include diff --git a/native/include/solprop_native/cpu_execution.hpp b/native/include/solprop_native/cpu_execution.hpp new file mode 100644 index 000000000..cb1527048 --- /dev/null +++ b/native/include/solprop_native/cpu_execution.hpp @@ -0,0 +1,272 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "solprop_native/execution.hpp" + +#include +#include +#include +#include +#include + +namespace solprop_native { + +/** Process-start CPU policy (solprop SEP 46 section 5's process settings): the pool + capacities, fixed when the runtime starts. Execution instances never exceed the + runtime budget. */ +struct ThreadPolicy { + int kernels{}, fft{}, raman_fft{}, raman_tail_fft{}; + // The field plasma column's band form (solprop SEP 25 section 6.7.6): negative takes + // the build's (tiled on a device, the team column on the host), zero the team + // column, and N > 0 the tiled column with host teams of N threads. + int band_tiles{-1}; + // Bit per pool: keep explicit requests distinct from automatic capacities. + unsigned explicit_pools{}; + bool operator==(const ThreadPolicy&) const = default; +}; +void configure_thread_policy(ThreadPolicy requested); + +/** Whether the field plasma column runs its tiled form under `policy`. */ +[[nodiscard]] inline bool band_tiled(const ThreadPolicy& policy) noexcept { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + return policy.band_tiles != 0; +#else + return policy.band_tiles > 0; +#endif +} + +/** A tiled band column's host team width under `policy`; zero on a device, whose + teams are its response teams. */ +[[nodiscard]] inline int band_tile_threads(const ThreadPolicy& policy) noexcept { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + static_cast(policy); + return 0; +#else + return policy.band_tiles > 0 ? policy.band_tiles : 0; +#endif +} +[[nodiscard]] ThreadPolicy thread_policy(); +[[nodiscard]] ExecutionSpace execution_instance(int threads); + +/** Independent CPU operation classes; device dispatch is unchanged. */ +enum class CpuOperation { + stage, + response, + dense, + fft, + raman, + copy, + flow, + modal, + adams +}; +inline constexpr std::size_t cpu_operation_count = + static_cast(CpuOperation::adams) + 1; +/** Process-start pool supplying an operation's capacity and explicit override. */ +enum class CpuPool { kernels, fft, raman_fft, raman_tail_fft }; + +/** One operation class's grain: below `threshold` work it runs on one thread, below + `full` on the medium width, and from `full` on its pool's capacity. Work is + elements for stages and copies, active columns * fine samples for responses, + rows * inner * samples for dense products, active bins * modes^3 for matrix + functions, elements * history rows for the exponential Adams trial's passes, and + actual real-side batch samples for FFT and Raman plans (tails included); the FFT + class's threshold is where a modal or spatial FFTW plan starts to thread. */ +struct OperationGrain { + std::size_t threshold{}, full{}; + bool operator==(const OperationGrain&) const = default; +}; + +/** A workspace's CPU execution settings (solprop SEP 46 section 5's workspace + settings), copied into the workspace when it is built and never read from anywhere + global, so two live workspaces may hold different ones. + + Native code holds no values for them (solprop_native's built-in table, + `_execution.py`, or a machine calibration supplies every field, with the + measurements behind each built-in value) and refuses a record that leaves one + unset (`require_cpu_execution`). + + - `grain`, indexed by `CpuOperation`. + - A response of `response_full_samples` fine samples or more, or of + `response_full_columns` columns or more, takes the full pool at any size above + its threshold. + - `medium_threads`: the medium width. + - A compact problem (`compact_problem`: one column on one node with fewer than + `compact_max_fine_samples` fine samples) runs its kernels on a team of at most + `compact_threads` and its threaded transforms on that whole team. + - `stable_kernel_team`: once a kernel pool holds a full-width instance, a + medium-width kernel takes it too (`CpuExecutionPool::get`). + - A batch of transforms whose row length is known threads by its shape (solprop + issue #587): from the FFT class's threshold, one thread per + `fft_thread_elements` samples and at most one a row -- or `medium_threads`, if + more, when its rows are at least `fft_long_row` samples long -- and its pool's + capacity from the class's full size. FFTW's threaded plans divide the rows + among the threads, and split a row only where it is long. */ +struct CpuExecution { + std::array grain{}; + std::size_t response_full_samples{}, response_full_columns{}; + int medium_threads{}, compact_threads{}; + std::size_t compact_max_fine_samples{}; + bool stable_kernel_team{}; + std::size_t fft_thread_elements{}, fft_long_row{}; + bool operator==(const CpuExecution&) const = default; + [[nodiscard]] const OperationGrain& + operator[](CpuOperation operation) const noexcept { + return grain[static_cast(operation)]; + } +}; + +/** Throw `std::invalid_argument` unless every threshold, bound and width of + `execution` is positive and each class's full size is at least its threshold. */ +void require_cpu_execution(const CpuExecution& execution); + +/** How a workspace plans its host FFTW transforms (solprop SEP 46 phase 3). + + - `legacy`: the plan policy of `fftw_seed_effort`, with the process's wisdom. + - `profile`: a machine calibration's. The workspace forgets every wisdom before it + plans and after; a transform whose exact plan the profile holds imports that + plan's wisdom and is seeded with `FFTW_WISDOM_ONLY` (a plan it no longer builds + is listed in `missing`, and planned at estimate); any other is seeded at its + group's effort, as the policy would, but with no wisdom from elsewhere. + - `capture`: each transform is seeded alone, from no wisdom, at `capture_effort` + within `time_limit` seconds (none when not positive), and its wisdom is kept in + `captured`, for a calibration to time and store. + + A transform's group is `rows` (one thread), `threaded_rows` or `planes`; its key + names its kind, sizes, strides, placement, alignment and threads + (`fft_plan_key`). Efforts are "estimate", "measure" or "patient". + + Plans runs measured before (solprop ADR-076), in the legacy and profile modes: + `stored` holds each transform's own wisdom by key, below the profile's `wisdom`, + which is consulted first. A transform seeded above estimate that `stored` holds, or + any when `keep` is set, is planned alone: the workspace's wisdom is set aside, the + transform's stored wisdom alone imported (or, for one to measure, what the + workspace measured so far, whose sub-plans it may share), and after the seed the + workspace's wisdom comes back over the transform's. A transform seeded twice is + measured once. A plan the stored wisdom built is listed in + `reused`, as is one left at estimate whose stored wisdom was imported for its + estimate plan; one that measured is kept in `measured` when `keep` is set. */ +struct FftPlanning { + enum class Mode { legacy, profile, capture }; + Mode mode{Mode::legacy}; + std::string rows{"measure"}, threaded_rows{"estimate"}, planes{"measure"}; + std::map wisdom; + std::string capture_effort{"measure"}; + double time_limit{-1.0}; + std::map stored; + bool keep{false}; + // What planning found: each transform's group, the wisdom captured, and the + // profile's plans that no longer build; the stored plans reused and the transforms + // measured, each with its own wisdom. + std::map groups, captured, measured; + std::vector missing, reused; + // While one transform plans alone: the workspace's wisdom, set aside. + bool alone{false}; + std::string held; +}; + +/** Choose a width without changing the runtime or a vendor library's pool. */ +[[nodiscard]] int operation_threads(const ThreadPolicy& policy, + const CpuExecution& execution, CpuPool pool, + CpuOperation operation, std::size_t elements, + std::size_t fine_samples = 0); + +using CpuSizing = std::map>; +#if defined(KOKKOS_ENABLE_OPENMP) && !defined(KOKKOS_ENABLE_CUDA) && \ + !defined(KOKKOS_ENABLE_HIP) +inline constexpr bool cpu_size_policy_enabled = true; +#else +inline constexpr bool cpu_size_policy_enabled = false; +#endif + +/** Own execution instances constructed only while a workspace and its plans form. */ +class CpuExecutionPool { +public: + /** A pool of `pool`'s capacity choosing widths by `execution`, which it copies. */ + CpuExecutionPool(CpuPool pool, const CpuExecution& execution) + : pool_(pool), execution_(execution) {} + const ExecutionSpace& get(CpuOperation operation, std::size_t elements, + std::size_t fine_samples = 0); + /** Size a compact problem's instances (`CpuExecution::compact_threads`), before any + `get`; explicit pool requests still apply as given. */ + void compact() noexcept { compact_ = cpu_size_policy_enabled; } + /** The execution settings this pool chooses widths by. */ + [[nodiscard]] const CpuExecution& execution() const noexcept { return execution_; } + /** The widest instance handed out so far; zero before any. */ + [[nodiscard]] int widest() const noexcept { + return instances_.empty() ? 0 : instances_.rbegin()->first; + } + void clear() { instances_.clear(); } + [[nodiscard]] CpuSizing sizing() const { return sizing_; } + +private: + CpuPool pool_; + CpuExecution execution_; + bool compact_{}; + std::map instances_; + CpuSizing sizing_; +}; + +/** A transform batch's row length, as its CPU width and sizing record count it + (`CpuExecutionPool::get`'s `fine_samples` for the FFT and Raman classes): extent 0 + counts the rows and the remaining extents make one row, so a transverse plane is + one row; a rank-1 view is one row. */ +template [[nodiscard]] std::size_t row_length(const View& rows) noexcept { + const std::size_t count = View::rank() == 1 ? 1 : rows.extent(0); + return count == 0 ? 0 : rows.size() / count; +} + +/** Whether the host FFTW's threaded loops take a scoped team (`CpuTeamScope`): the + OpenMP FFTW with its loop callback (FFTW 3.3.9 and later). Without it a compact + problem's transforms stay serial. The first call installs the callback, whose + loops take the runtime's whole team, as FFTW's own do, outside a scope. */ +[[nodiscard]] bool fftw_team_loops() noexcept; + +/** Record that the build's BLAS is an OpenMP OpenBLAS, so that a compact problem's + scope holds it at one thread (`CpuTeamScope`). */ +void register_openmp_blas(bool registered) noexcept; + +/** While it lives, the calling thread's threaded FFTW loops run on a team of + `threads` (at least their own job count), so that a compact problem's transforms + keep its kernels' team instead of waking the whole runtime; zero restores the + runtime's team. A nonzero team also holds an OpenMP OpenBLAS at one thread, through + the calling thread's OpenMP default width: a compact problem's products are a row + or two deep, and an OpenBLAS region at the fitted width between the team's narrower + ones made GNU OpenMP end and restart the surplus threads at every pair (16 cores of + a Neoverse V2: about 100,000 thread creations in a short GNLSE window). The + outermost scope on a thread also sets its OpenMP default to the runtime's width + and restores it at its end. Scopes nest. */ +class CpuTeamScope { +public: + explicit CpuTeamScope(int threads) noexcept; + ~CpuTeamScope(); + CpuTeamScope(const CpuTeamScope&) = delete; + CpuTeamScope& operator=(const CpuTeamScope&) = delete; + CpuTeamScope(CpuTeamScope&&) = delete; + CpuTeamScope& operator=(CpuTeamScope&&) = delete; + +private: + // The enclosing scope's team; unread where FFTW has no loop callback. + [[maybe_unused]] int saved_; + // Whether this scope holds the registered OpenBLAS at one thread. + bool holds_blas_{}; +}; + +/** How long, in milliseconds, LLVM's OpenMP runtime keeps idle threads spinning while + a native call runs, from the outermost `CpuTeamScope` on a thread to its end, where + the caller's block time returns; zero leaves the process's (solprop ADR-071). GNU's + runtime reads its spin count once, at start, from GOMP_SPINCOUNT. */ +void configure_native_spin(int milliseconds); + +/** LLVM's runtime's block time inside a scope, in milliseconds; -1 with another + runtime. A test hook. */ +[[nodiscard]] int scoped_block_time() noexcept; + +/** The calling thread's OpenMP default width inside a scope (`CpuTeamScope`): the + runtime's, on any thread (solprop issue #130); one without OpenMP. A test hook. */ +[[nodiscard]] int scoped_openmp_default() noexcept; + +} // namespace solprop_native diff --git a/native/include/solprop_native/dense_output.hpp b/native/include/solprop_native/dense_output.hpp index 2a28b59c7..a8e9fd0ae 100644 --- a/native/include/solprop_native/dense_output.hpp +++ b/native/include/solprop_native/dense_output.hpp @@ -4,8 +4,8 @@ #pragma once -#include "solprop_native/core.hpp" #include "solprop_native/dense.hpp" +#include "solprop_native/general_state.hpp" #include "solprop_native/stage_tables.hpp" #include diff --git a/native/include/solprop_native/dispersive.hpp b/native/include/solprop_native/dispersive.hpp index d900541d7..d9417f776 100644 --- a/native/include/solprop_native/dispersive.hpp +++ b/native/include/solprop_native/dispersive.hpp @@ -4,8 +4,10 @@ #pragma once -#include "solprop_native/core.hpp" +#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/fft_plan.hpp" +#include "solprop_native/problem.hpp" #include diff --git a/native/include/solprop_native/engines.hpp b/native/include/solprop_native/engines.hpp index 1c940a942..7d1015f5c 100644 --- a/native/include/solprop_native/engines.hpp +++ b/native/include/solprop_native/engines.hpp @@ -5,7 +5,8 @@ #pragma once #include "solprop_native/adams_march.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/general_state.hpp" +#include "solprop_native/host_source.hpp" #include "solprop_native/rk_march.hpp" #include "solprop_native/session.hpp" #include "solprop_native/stage_tables.hpp" diff --git a/native/include/solprop_native/execution.hpp b/native/include/solprop_native/execution.hpp new file mode 100644 index 000000000..e29a34dc2 --- /dev/null +++ b/native/include/solprop_native/execution.hpp @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "solprop_native/fields.hpp" + +#include +#include + +#include +#include +#include + +namespace solprop_native { + +using ExecutionSpace = Kokkos::DefaultExecutionSpace; +using MemorySpace = ExecutionSpace::memory_space; +using Layout = Kokkos::LayoutRight; +/** A two-dimensional range whose last index runs fastest, as `Layout` stores it. + + Kokkos's CUDA and HIP default runs the first index fastest, which puts adjacent + threads a whole row apart in a `LayoutRight` view; a host's default is already + this order. */ +using Range2Policy = Kokkos::MDRangePolicy< + ExecutionSpace, Kokkos::Rank<2, Kokkos::Iterate::Right, Kokkos::Iterate::Right>>; + +/** Samples in one host tile of a `Range2`. Swept from 64 to 1024 on the four-mode RDW + at 16 threads (an Apple M5 Max and an AMD EPYC 7763): 64 to 256 were within 2 % + of each other and 1024 the slowest; 64 balances best, a single 2561-bin column + making 41 tiles rather than 11. */ +inline constexpr std::int64_t host_range2_tile = 64; + +/** `Range2Policy` on `space` over `[lower, upper)`. + + On the host a tile holds about `host_range2_tile` samples: part of one row, or as + many whole rows as fit when a row is shorter. Kokkos's host default tiles two rows + by the whole of the last extent, which leaves a kernel over `(columns, n)` only + ceil(columns / 2) tiles to share: one for a single column, two for four modes, + whatever the thread count. A one-thread instance keeps Kokkos's default: it has + nothing to share, and the small tiles cost the four-mode RDW about 1 % at one + thread on an AMD EPYC 7763. Device builds keep Kokkos's default too. A kernel over + a `Range2` that is elementwise changes no value with the tiling; a reduction over + one (the boundary monitor's split removal) may add in another order, within its + rounding (solprop SEP 25). */ +template +[[nodiscard]] Range2Policy Range2(const ExecutionSpace& space, const LT (&lower)[LN], + const UT (&upper)[UN]) { + static_assert(LN == 2 && UN == 2, "Range2 takes two lower and two upper bounds"); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + return Range2Policy(space, lower, upper); +#else + if (space.concurrency() == 1) + return Range2Policy(space, lower, upper); + const auto row = std::clamp(static_cast(upper[1]) - + static_cast(lower[1]), + 1, host_range2_tile); + const std::int64_t tile[2] = {host_range2_tile / row, row}; + return Range2Policy(space, lower, upper, tile); +#endif +} +using Complex = Kokkos::complex; + +} // namespace solprop_native + +// NumPy's complex128 is two doubles, real part first, as `Complex` is. +template <> struct solprop_native::fields::ElementDtype { + static constexpr Dtype value = Dtype::complex128; +}; + +namespace solprop_native { + +using ComplexView = Kokkos::View; +using ConstComplexView = Kokkos::View; +using RealView = Kokkos::View; +using ConstRealView = Kokkos::View; + +/** + * The vector length of the general source's teams: the adjacent output bins one + * projection team accumulates, and the lanes a device gives a node's samples. + * + * The host runs a team's vector range as one vectorised loop and ignores the + * length; for the projection it is the block of bins (measured on the general + * route's source on native CPU before the first public release). + */ +inline constexpr std::size_t general_vector_lanes = 8; + +} // namespace solprop_native diff --git a/native/include/solprop_native/far_field.hpp b/native/include/solprop_native/far_field.hpp index 5367927f6..7a3e8bf0b 100644 --- a/native/include/solprop_native/far_field.hpp +++ b/native/include/solprop_native/far_field.hpp @@ -4,8 +4,9 @@ #pragma once -#include "solprop_native/core.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/fp.hpp" +#include "solprop_native/problem.hpp" #include diff --git a/native/include/solprop_native/fft_plan.hpp b/native/include/solprop_native/fft_plan.hpp index 8f9617209..0d05eedf5 100644 --- a/native/include/solprop_native/fft_plan.hpp +++ b/native/include/solprop_native/fft_plan.hpp @@ -4,7 +4,8 @@ #pragma once -#include "solprop_native/core.hpp" +#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/execution.hpp" #include diff --git a/native/include/solprop_native/fields.hpp b/native/include/solprop_native/fields.hpp index 3175d4526..787f6b8e0 100644 --- a/native/include/solprop_native/fields.hpp +++ b/native/include/solprop_native/fields.hpp @@ -61,7 +61,7 @@ enum class Dtype : std::uint8_t { float64, complex128, uint64, uint8, boolean }; } /** The dtype an element type takes by default, beyond the numbers and the bools: - `core.hpp` adds the complex one. */ + `execution.hpp` adds the complex one. */ template struct ElementDtype; template [[nodiscard]] constexpr Dtype element_dtype() noexcept { diff --git a/native/include/solprop_native/flow_functions.hpp b/native/include/solprop_native/flow_functions.hpp index d12f768b7..54144f0d5 100644 --- a/native/include/solprop_native/flow_functions.hpp +++ b/native/include/solprop_native/flow_functions.hpp @@ -4,7 +4,8 @@ #pragma once -#include "solprop_native/core.hpp" +#include "solprop_native/execution.hpp" +#include "solprop_native/general_state.hpp" #include diff --git a/native/include/solprop_native/general_modal.hpp b/native/include/solprop_native/general_modal.hpp index 611bb40ae..bf40db619 100644 --- a/native/include/solprop_native/general_modal.hpp +++ b/native/include/solprop_native/general_modal.hpp @@ -5,8 +5,10 @@ #pragma once #include "solprop_native/build.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/cpu_execution.hpp" #include "solprop_native/dispersive.hpp" +#include "solprop_native/execution.hpp" +#include "solprop_native/problem.hpp" #include "solprop_native/raman_fft.hpp" #include @@ -22,16 +24,6 @@ namespace solprop_native { */ inline constexpr std::size_t general_pair_histories = 6; -/** - * The vector length of the general source's teams: the adjacent output bins one - * projection team accumulates, and the lanes a device gives a node's samples. - * - * The host runs a team's vector range as one vectorised loop and ignores the - * length; for the projection it is the block of bins (measured on the general - * route's source on native CPU before the first public release). - */ -inline constexpr std::size_t general_vector_lanes = 8; - /** The bins a host synthesis team sums at a time in local arrays. */ inline constexpr std::size_t general_synthesis_block = 256; diff --git a/native/include/solprop_native/general_state.hpp b/native/include/solprop_native/general_state.hpp new file mode 100644 index 000000000..eb7b35a44 --- /dev/null +++ b/native/include/solprop_native/general_state.hpp @@ -0,0 +1,439 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/execution.hpp" +#include "solprop_native/problem.hpp" +#include "solprop_native/readings.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace solprop_native { + +// host_source.hpp's types, which the state only passes on. +class HostSource; +enum class HostPurpose : int; + +/** The most samples an Adams formula of solprop's shipped orders combines. */ +inline constexpr std::size_t adams_max_samples = 8; + +/** The most defect norms an Adams trial scores: its order's and its two neighbours'. */ +inline constexpr std::size_t adams_max_norms = 3; + +/** + * The most channels of a coupled block whose channel vectors a host task of the + * resident Adams engine holds on its stack (solprop SEP 16 phase 6). A wider block's + * host passes hold them in a scratch row per bin, allocated with the history (solprop + * SEP 47 phase 4). A device build's passes take a thread per (bin, channel) and + * hold no channel vectors, so it steps any width. + */ +inline constexpr std::size_t adams_max_channels = 16; + +/** + * One Adams trial's coefficient tables, passed by value to the resident engine's + * kernels (solprop SEP 16 phase 3b). `slot` names history rows newest first; + * `predictor` weights the first `order` of them; `corrector` and each of `n_rows` + * defect rows weight `(source, slot[0], ...)` over `width` samples. + */ +struct AdamsWeights { + std::size_t slot[adams_max_samples]{}; + double predictor[adams_max_samples]{}; + double corrector[adams_max_samples]{}; + double rows[adams_max_norms][adams_max_samples]{}; + std::size_t order{}; + std::size_t width{}; + std::size_t n_rows{}; +}; + +/** + * One exponential Adams trial's coefficient tables (solprop SEP 16 phase 3b): the + * predictor's Lagrange coefficients `predictor[sample][monomial]` over the `order` + * newest rows, the corrector's over `(source, slot[0], ...)` and, per scored defect, + * the divided-difference coefficients over the same `width` samples followed by the + * node polynomial's `count` monomial coefficients. + */ +struct ExponentialWeights { + std::size_t slot[adams_max_samples]{}; + double predictor[adams_max_samples][adams_max_samples]{}; + double corrector[adams_max_samples][adams_max_samples]{}; + double rows[adams_max_norms][2 * adams_max_samples]{}; + std::size_t order{}; + std::size_t width{}; + std::size_t count{}; + std::size_t n_rows{}; + // A varying operator's transported compensation (solprop SEP 16 section 4.2): the + // carried samples' spans 1 - theta_j, and the predictor's, the corrector's and each + // defect's weights over (endpoint, carried...), still to be multiplied by the step. + bool compensated{}; + double span[adams_max_samples]{}; + double comp_predictor[adams_max_samples]{}; + double comp_corrector[adams_max_samples + 1]{}; + double comp_rows[3][adams_max_samples + 1]{}; + // Dense observation's weights over (endpoint, history...) at the fraction, and the + // samples' normalised positions. + double dense_weight[adams_max_samples + 1]{}; + double dense_theta[adams_max_samples + 1]{}; + // A scalar carrier frame (solprop SEP 35): sigma = i shift. The moments are those + // of x - sigma, and history row t (the t-th of `slot`) is read rotated by + // factor[t] = exp(sigma (1 - theta_t)); the new source's factor is one. + bool shifted{}; + double shift{}; + // exp(-sigma), formed once a trial. + double unshift_re{1.0}; + double unshift_im{}; + double factor_re[adams_max_samples]{}; + double factor_im[adams_max_samples]{}; +}; + +/** Small host bundle for all stages and the whole-step propagator. */ +struct TrialScalars { + std::vector phi_delta; + std::vector phi_at; + std::vector prefactor; + std::vector density; + std::vector taper; + std::vector preionisation; + std::vector raman_rates; + std::vector kerr_density, plasma_densities, plasma_preionisation, + raman_densities; + // A coupled flow's step section and its stages' residual weights (n_stages, terms). + std::size_t flow_section{}; + std::vector flow_weights; + // A moving guide's map weights (n_stages, map terms), zero outside the section each + // stage lies in (solprop kernel schema 14); empty for maps that do not move. + std::vector transform_weights; + // Each stage's channel densities of SEP 6's dispersive response (n_stages, + // channels), m^-3; empty without it. + std::vector dispersive_densities; +}; + +/** Which general z table a call names (solprop SEP 29 G1). */ +enum class TableKind { phi, prefactor, rate, flux }; + +/** The z tables by name (`TableKind`). */ +[[nodiscard]] constexpr auto names(const TableKind* /*unused*/) { + return std::array{ + std::pair{"phi", TableKind::phi}, std::pair{"prefactor", TableKind::prefactor}, + std::pair{"rate", TableKind::rate}, std::pair{"flux", TableKind::flux}}; +} + +class GeneralWorkspace; + +/** Multicolumn general state with batched FFTs and chunked node transforms. */ +class GeneralState { +public: + explicit GeneralState(Problem problem); + ~GeneralState(); + GeneralState(const GeneralState&) = delete; + GeneralState& operator=(const GeneralState&) = delete; + GeneralState(GeneralState&&) = delete; + GeneralState& operator=(GeneralState&&) = delete; + + /** Load the field and noise from host memory, one copy into the memory space. */ + void set_initial(std::span field, std::span noise); + [[nodiscard]] std::vector trial_step(double dz, const TrialScalars& scalars); + void prepare_trial(double dz, const TrialScalars& scalars); + /** Adopt an Adams step's end state through the shared finish. */ + void adopt(const std::vector& field, double dz); + /** + * One step of solprop's low-storage RK4IP march (SEP 24 phase 3): four register + * passes accepted in place, returning the raw accepted state's squared norm over the + * norm mask. `scalars` holds one row per position -- 0 the step's start, 1 its + * midpoint, 2 its end, any later rows ignored -- and its `phi_delta` rows 0 and 1 + * the flows' intervals `P` and `Q`, row 2 the table's, read only when + * `rebuild_table` builds a constant operator's table. + */ + [[nodiscard]] double low_storage_step(double dz, const TrialScalars& scalars, + bool rebuild_table); + /** The resident Adams history (solprop SEP 16 phase 3b, solprop ADR-043). */ + void adams_allocate(std::size_t slots, bool compensated, bool interpolate = true); + void adams_compensation_start(std::size_t slot, const std::vector& rate); + void adams_start(std::size_t slot, const TrialScalars& scalars); + void adams_scale(double rtol, double atol, const std::vector& weights); + [[nodiscard]] std::vector + adams_fused_trial(double dz, const AdamsWeights& weights, + const std::vector& divisor, const TrialScalars& scalars); + void adams_fused_accept(double dz, const AdamsWeights& keep, std::size_t free, + const AdamsWeights& dense); + void adams_hold(); + void adams_restore(); + void adams_observe(const AdamsWeights& weights, const TrialScalars& scalars); + [[nodiscard]] std::vector + adams_exponential_trial(double dz, const ExponentialWeights& weights, + const std::vector& divisor, const TrialScalars& step, + const TrialScalars& sample, const std::vector& rate); + void adams_exponential_accept(double dz, std::size_t free, const AdamsWeights& dense); + void adams_exponential_observe(double xi, double dz, + const ExponentialWeights& weights); + /** + * A Runge-Kutta march's dense output in a native session (solprop SEP 47 section + * 5.3), DOPRI5's continuous extension (Hairer, Norsett and Wanner II.6) as solprop's + * `extension_coefficients` and `_observe_inside` form it. `rk_dense_allocate` keeps + * the tableau's dense-output weights, one per stage, and the views the extension + * needs; `rk_dense_hold` contracts the accepted, unfinished trial over `dz` -- the + * state before it and its stages -- into the extension's five coefficients and marks + * the bins the step's flow changes by more than an e-fold; + * `rk_dense_observe` holds the committed state aside and makes the state the + * extension at the fraction `theta`, carried from the anchor by the flow of `carry` + * (a trial's arrays from the anchor to the plane) and masked; `rk_dense_restore` + * makes the held state current again; `rk_dense_release` frees the views. + */ + void rk_dense_allocate(const std::vector& weights); + void rk_dense_hold(double dz); + void rk_dense_observe(double theta, const TrialScalars& carry); + void rk_dense_restore(); + void rk_dense_release() noexcept; + /** + * The resident Adams history's persistent rows, for a resume (solprop SEP 29 G2): + * `"store"`, and `"states"`, `"rates"` and `"applied"` where allocated, each + * `(slots, columns, n_omega)`. Scratch views are not rows: a trial rebuilds them. + */ + [[nodiscard]] std::vector adams_row_names() const; + /** A named row view's extents; refuses an unknown or unallocated name. */ + [[nodiscard]] std::array + adams_row_extents(const std::string& name) const; + /** A lossless host copy of a named row view, in its LayoutRight order. */ + [[nodiscard]] std::vector adams_rows_to_host(const std::string& name) const; + /** The same copy written straight into `destination`, of `size` elements. */ + void adams_rows_to_host(const std::string& name, std::complex* destination, + std::size_t size) const; + /** Replace a named row view with a host copy of its extents; forget any trial. */ + void adams_rows_from_host(const std::string& name, + const std::vector& values); + /** + * The general z tables (solprop SEP 29 G1). `table_upload` makes a table resident: + * Phi's, the prefactor's, or Phi's rate (the derivative series, for the compensated + * Adams march); `table_evaluate` writes Phi's and the prefactor's values at each + * position into the given term slots, and `table_rate` the rate's value into a Phi + * term slot. `table_terms_to_host` copies one term slot to the host, and + * `phase_sum_to_host` evaluates the stage kernels' phase sum over the Phi terms for + * one row of weights; both are diagnostics. + */ + void table_upload(TableKind kind, std::vector edges, double lower, + double upper, const std::complex* coefficients, + std::size_t n_intervals, std::size_t n_terms, std::size_t n_values); + void table_evaluate(const std::vector& slots, + const std::vector& positions); + void table_rate(std::size_t slot, double z, bool before); + [[nodiscard]] std::vector table_terms_to_host(TableKind kind, + std::size_t slot) const; + [[nodiscard]] std::vector + phase_sum_to_host(const std::vector& weights); + [[nodiscard]] std::vector weighted_error(const std::vector& scale); + /** Load a coupled flow's actions; see `CoupledFlowAction::set_actions`. */ + void set_flow_actions(const std::vector& forward, + const std::vector& inverse, + const std::vector& identity); + /** A coupled flow's held actions; see `CoupledFlowAction::actions_to_host`. */ + [[nodiscard]] std::vector flow_actions_to_host(bool inverse) const; + /** The last exponential Adams trial's coupled block functions, row-major + `(slots + 1, n_active, n_modes, n_modes)`, the flow first: an observation for + tests (solprop SEP 47 phase 4). */ + [[nodiscard]] std::vector adams_block_to_host() const; + /** The last Adams trial's exponent `Delta Phi`, row-major `(n_columns, n_omega)`: + an observation for tests. */ + [[nodiscard]] std::vector adams_exponent_to_host() const; + /** Form a coupled flow's actions natively; see `CoupledFlowAction::form_actions`. */ + [[nodiscard]] std::vector + form_flow_actions(std::size_t section, double dz, + const std::vector& fractions); + void prepare_raman(const std::vector& rates, + const std::vector& densities = {}); + /** + * The spatial statistics of the accepted state at the plane `z`, measured by the + * medium's axial flux (solprop SEP 34): flags 1 energy, 2 on-axis peak intensity, 4 + * peak intensity, 8 on-axis spectrum, 16 the widths' fluence block (the signed + * fluence at each node, the axis fluence, which a separable route leaves NaN since + * only a radial width reads it, then the bound on the largest magnitude), 32 a + * Cartesian plane's fluence moments, which hold its scaled profile for + * `plane_fluence_trace` (the lowest value, its node, the bound on the largest + * magnitude, the largest value, the count of values that aren't finite, then the + * scaled sum and its x and y index moments), packed in that order. + */ + [[nodiscard]] std::vector + spatial_statistics(std::uint64_t flags, double intensity_factor, double z); + /** + * The second moment and the azimuthal average of the plane's scaled fluence that a + * `spatial_statistics` call with flag 32 left, on rings solprop's own rule fixes, + * with the centre in metres from the axes' first samples. Returns the second-moment + * sum, then one average per ring. The profile itself stays in the execution space. + */ + [[nodiscard]] std::vector + plane_fluence_trace(double center_x, double center_y, double dx, double dy, + double extent, double spacing, std::size_t n_radii, + const std::vector& cosines, + const std::vector& sines); + /** + * The modal state's spectrum at one physical point: `matrix`, one or two rows of + * `n_modes` columns, contracted with the modes, as `(n_rows * n_pol, n_omega)` + * samples. What an on-axis scan needs of the state. + */ + [[nodiscard]] std::vector axis_spectrum(const std::vector& matrix); + /** + * solprop SEP 19's on-axis plasma populations of the accepted state: `matrix`'s row + * (as `axis_spectrum` takes it, the pad scale and the taper in it) on the fine time + * grid, scanned by the node scan with each channel's preionisation. Returns each + * channel's final ionised fraction and peak rate, `(n_channels, 2)`. + */ + [[nodiscard]] std::vector + axis_populations(const std::vector& matrix, + const std::vector& channel_preionisation); + /** + * solprop SEP 23's far-field statistics of the accepted state on a Cartesian or line + * grid, one banded pass; see `FarFieldPass::run` (`far_field.hpp`) for the tables, + * the flags (from 32) and the packed order. + */ + [[nodiscard]] std::vector + aperture_statistics(std::uint64_t flags, const std::vector& apertures, + int exponent, const std::vector& regions, + double intensity_factor); + [[nodiscard]] std::vector + statistics(std::uint64_t flags, const std::vector& windows, + const std::vector& on_axis, double density, double taper, + double preionisation, double intensity_factor, + const std::vector& channel_preionisation = {}, + const std::vector& power_weight = {}); + /** + * solprop SEP 47: a general modal basis's statistics, functionals of the physical + * flux `a.H W a` (`solprop.stats._whitened`). The state's columns are whitened by + * `root`, `W^(1/2)` at each frequency, `(n_omega, n_modes, n_modes)` row-major + * (empty for the identity), into a statistics scratch, and the temporal `flags` + * (omega0, energy, peak power total and the width's trace) are evaluated on + * `c = W^(1/2) a`. Returns their packed values, then each column's `sum |c|^2`, + * then, with `spectral`, each bin's `sum |c|^2` over the columns. The state is + * unchanged. + */ + [[nodiscard]] std::vector + whitened_statistics(const std::vector& root, std::uint64_t flags, + const std::vector& windows, double intensity_factor, + bool spectral); + void finish(double dz); + void promote_fsal(); + void clear_fsal(); + void set_absorber_quiet(bool quiet); + /** + * Bind solprop's boundary monitor (SEP 21): its four time weights, row-major + * `(4, n_t)`, and its columns' quadrature by mode (empty for unit weights). From + * here on each split-step finish adds what it takes to an accumulator, which + * `monitor_reading` drains, and keeps the energies of the power it found. + */ + void bind_monitor(const std::vector& weights, + const std::vector& column_weights); + void unbind_monitor() noexcept; + /** + * The bound monitor's reading of the accepted state, over every column: `sum_t P`, + * the four weighted sums `sum_t w_k P`, what the split collar took since the last + * reading, and `sum_t P` and the two guards' and the collar's sums of the power the + * last step's split found (zeros before the first such step). + */ + [[nodiscard]] std::vector monitor_reading(); + /** + * The last accepted step's split collar sums, read without draining anything (SEP + * 52 section 5.2's `AcceptRecord`): what its split took, then `sum_t P` and the two + * guards' and the collar's sums of the power it found, from the masked field before + * the split; none without a bound monitor and a split collar. + */ + [[nodiscard]] std::optional> split_sums(); + /** + * Bind the split collar's walk-off bucket of each frequency bin, by mode + * `(n_modes, n_omega)`, `count` buckets (solprop #440): from here on + * `monitor_reading` also returns the state's energy per bucket, the fastest first. + */ + void bind_occupancy(const std::vector& buckets, std::size_t count); + [[nodiscard]] double quadrature_error() const; + [[nodiscard]] double quadrature_peak(); + [[nodiscard]] std::string synthesis_provider() const; + [[nodiscard]] std::string projection_provider() const; + [[nodiscard]] std::string gemm_shape_policy() const; + [[nodiscard]] std::size_t response_team_size() const noexcept; + [[nodiscard]] std::vector field_to_host() const; + /** The field into `destination`, of the field's size: one copy from the space. */ + void field_to_host(std::span destination) const; + /** The committed field where the host can read it in place, `columns * n_omega` + elements in row-major order, after the state's work is fenced; null where the + memory space is not host-accessible. A reader that passes it through the host + adds its bytes with `count_to_host` (solprop SEP 47 section 5.4). */ + [[nodiscard]] const Complex* host_field() const; + /** Count `bytes` read out of the state by a host reader of `host_field`. */ + void count_to_host(std::size_t bytes) const noexcept; + /** What the transverse edge's reading sums with (solprop SEP 44). */ + void bind_edge(const std::vector& faces, const std::vector& clip, + const std::vector& totals, + const std::vector& synthesis, std::size_t first_row); + /** The bytes through this state's crossings since it was built, each way. */ + [[nodiscard]] CrossedBytes crossed_bytes() const noexcept; + /** The edge's region sums and each mode's energy, where the state lives. */ + /** A requested collar's lines, whose strip and current a reading then forms. */ + void bind_edge_lines(const EdgeLines& lines); + EdgeNumbers edge_reading(bool faces, bool whole, bool totals, bool strip, + bool current); + /** The threads a large crossing takes (`crossing.hpp`), from solprop's budget. */ + void set_crossing_workers(std::size_t workers) noexcept; + [[nodiscard]] std::vector stage_to_host(std::size_t stage) const; + void stage_to_host(std::size_t stage, std::span destination) const; + /** + * The last evaluation's projected polarisation spectrum on the fine rule, or on the + * embedded one (finishing SEP 6's dispersive products there first), a diagnostic. + */ + [[nodiscard]] std::vector projected_to_host(bool coarse); + /** How SEP 6's dispersive response runs: empty without it. */ + [[nodiscard]] std::map dispersive_path() const; + [[nodiscard]] std::map memory_ledger() const; + [[nodiscard]] CpuSizing execution_sizing() const; + /** The CPU execution record the workspace's pools choose widths by. */ + [[nodiscard]] const CpuExecution& cpu_execution() const; + /** The workspace's FFT planning, with what its planning found. */ + [[nodiscard]] const FftPlanning& fft_planning() const; + [[nodiscard]] double planning_seconds() const noexcept; + [[nodiscard]] double raman_planning_seconds() const noexcept; + [[nodiscard]] std::string raman_fft_provider() const; + [[nodiscard]] std::map raman_fft_tasks() const; + [[nodiscard]] std::size_t raman_kernel_transforms() const noexcept; + [[nodiscard]] bool fsal_ready() const noexcept; + [[nodiscard]] std::size_t rk_source_evaluations() const noexcept; + /** On-axis row scans made so far (`axis_populations`), cumulative. */ + [[nodiscard]] std::size_t axis_scans() const noexcept; + [[nodiscard]] bool closed() const noexcept; + void close() noexcept; + /** + * A host source (solprop SEP 49 section 6.2): from here on each stage source is + * `source`'s evaluation of the lab-frame field, the state's own response passes not + * run, and the stage's finish its collar, absorber and propagator alone. Its two + * host rows are allocated here, once, and listed in the memory ledger. Refused on a + * device build and for anything but one real column off the general and separable + * routes; null unbinds. + */ + void bind_host_source(std::shared_ptr source); + /** The bound host source, or null. */ + [[nodiscard]] HostSource* host_source() const noexcept; + /** The next host evaluation's position, m, and purpose, which the engine sets. */ + void host_evaluation(double z, HostPurpose purpose) noexcept; + /** A test hook (solprop issue #129): `fault` runs, once, as the next Adams source + evaluation starts, a trial's with its prediction in the state; empty disarms. */ + void arm_source_fault(std::function fault) noexcept; + +private: + /** `function(workspace, arguments...)` inside the workspace's CPU team scope + (`CpuTeamScope`); defined and used in src/general/state.cpp alone, so that no unit + outside src/general/ sees the workspace class. */ + template + decltype(auto) with_team(Function function, Arguments&&... arguments) const; + + std::unique_ptr workspace_; +}; + +} // namespace solprop_native diff --git a/native/include/solprop_native/low_storage.hpp b/native/include/solprop_native/low_storage.hpp index 304bf250a..c1fd96a08 100644 --- a/native/include/solprop_native/low_storage.hpp +++ b/native/include/solprop_native/low_storage.hpp @@ -4,8 +4,9 @@ #pragma once -#include "solprop_native/core.hpp" #include "solprop_native/engines.hpp" +#include "solprop_native/fields.hpp" +#include "solprop_native/general_state.hpp" #include "solprop_native/rk_march.hpp" #include "solprop_native/stage_tables.hpp" diff --git a/native/include/solprop_native/plasma.hpp b/native/include/solprop_native/plasma.hpp index 3fc11f287..88ab80609 100644 --- a/native/include/solprop_native/plasma.hpp +++ b/native/include/solprop_native/plasma.hpp @@ -4,9 +4,10 @@ #pragma once -#include "solprop_native/core.hpp" #include "solprop_native/dispatch.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/launch.hpp" +#include "solprop_native/problem.hpp" #include diff --git a/native/include/solprop_native/problem.hpp b/native/include/solprop_native/problem.hpp new file mode 100644 index 000000000..719e82604 --- /dev/null +++ b/native/include/solprop_native/problem.hpp @@ -0,0 +1,601 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/execution.hpp" +#include "solprop_native/fields.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace solprop_native { + +enum class FieldFamily { field_real, field_analytic, envelope }; + +/** The field families by name, for the field tables (`fields.hpp`). */ +[[nodiscard]] constexpr auto names(const FieldFamily* /*unused*/) { + return std::array{std::pair{"field_real", FieldFamily::field_real}, + std::pair{"field_analytic", FieldFamily::field_analytic}, + std::pair{"envelope", FieldFamily::envelope}}; +} + +enum StatisticsFlag : std::uint64_t { + statistics_omega0 = 1ULL << 0U, + statistics_energy = 1ULL << 1U, + statistics_energy_window = 1ULL << 2U, + statistics_peak_power = 1ULL << 3U, + statistics_peak_power_window = 1ULL << 4U, + statistics_peak_power_total = 1ULL << 5U, + statistics_fwhm = 1ULL << 6U, + statistics_electron_density = 1ULL << 7U, + statistics_peak_ionisation_rate = 1ULL << 8U, + statistics_peak_intensity = 1ULL << 9U, + statistics_quadrature_error = 1ULL << 10U, + statistics_populations = 1ULL << 11U, + // solprop SEP 18's run diagnostics: the six maxima over every node of the field + // plasma column's integration (exposure dip and rise, |paid - freed| and freed, the + // free-current integrand in the first 2K samples and anywhere). + statistics_plasma_integration = 1ULL << 12U, +}; + +/** The widest band rule solprop designs (SEP 18 section 5): K <= 8, so 9 taps. */ +inline constexpr std::size_t plasma_max_taps = 9; + +/** The six parts of SEP 18's run diagnostics, in solprop's order. */ +inline constexpr std::size_t plasma_diagnostic_parts = 6; + +struct Setup; + +/** Fixed, Python-free problem data uploaded once by a native state, grouped as + solprop's kernel contract groups it. The adapter fills it field by field (`lower` in + solprop_native's backend); `validate` checks presence and extents before a state + allocates anything. */ +struct Problem { + /** The state's extents and field family. */ + struct Dims { + std::size_t n_omega{}; + std::size_t n_t{}; + std::size_t n_to{}; + std::size_t n_modes{1}; + std::size_t n_pol_state{1}; + std::size_t n_pol_transform{1}; + std::size_t n_nodes{1}; + std::size_t node_chunk{1}; + FieldFamily family{FieldFamily::field_real}; + }; + /** The frequency and time grids, their windows and masks. */ + struct Grid { + std::vector crop_index; + std::vector active_mask; + std::vector norm_mask; + std::vector omega_window; + std::vector omega; + std::vector power_weight; + std::vector time_window; + std::vector time_window_derivative; + std::vector carrier; + double dt_fine{}; + double dt{}; + }; + /** The separable geometries' operator (free space and the hybrid planar kind) and a + radial run's absorber and on-axis rows. */ + struct Spatial { + std::size_t n_x{}, n_y{}; + std::vector k_squared, k_squared_low, numerator, clamped; + std::vector kperp_sq, kperp_sq_low, frame, spectral_rate; + // solprop kernel schema 45 (#231): the user's share of the spectral rate, read only + // by `absorbed_power`; the operator takes the total. + std::vector user_rate; + std::vector absorber_rate; + // solprop kernel schema 31 (SEP 22 phase 2): a transverse collar's power rate per + // column, added to the field attenuation on active bins; zeros leave the operator. + std::vector kperp_rate; + double evanescent_rate{}; + // solprop SEP 23 (free space only): the stored transverse axes k_x (n_x) and k_y + // (n_y) in rad/m, FFT order, column ix + n_x iy, which the far-field statistics + // read; and the slots of the problem's own far-field request, which size the band + // partials at construction (zero for none). Empty and zero elsewhere. + std::vector k_x, k_y; + std::size_t far_field_slots{}; + // solprop's hybrid planar kind (kernel schema 15): n_p slab channels in x on the + // real (n_x, n_p) synthesis and (n_p, n_x) projection, times a raw DFT in y. + // Columns are (p, k_y) with the channel fastest, n_modes = n_p * n_y, and nodes + // (x, y) with x fastest, n_nodes = n_x * n_y; node_chunk holds whole y rows. + // Row-major (n_p, n_omega) wall terms kappa_p^2 enter the separable operator, and + // absorber_rate is then per column. Zero n_p and empty kappa elsewhere. + std::size_t n_p{}; + std::vector kappa_sq; + // A radial run's spatial absorber and its on-axis statistics' rows. + std::vector absorber_matrix; + std::vector spectral_weights; + std::vector on_axis; + }; + /** solprop kernel schema 35 (SEP 34): a non-separable spatial run's plane measure, + the entrance's material squares (n_omega) and transverse squares (n_modes) whose + root is solprop's `longitudinal_root`; `table` when the material square varies + along z and is uploaded as a `TableKind::flux` table of n_omega values. A + separable run's measure is its own operator's (these stay empty). `closure` + allocates the magnetic closure's buffers at construction, for a request with a + local flux statistic. */ + struct Flux { + std::vector k_squared, k_squared_low; + std::vector kperp_sq, kperp_sq_low; + bool table{}; + bool closure{}; + }; + /** The Runge-Kutta tableau and the march's own choices. */ + struct Stepper { + std::size_t n_stages{}; + std::vector tableau_matrix; + std::vector tableau_nodes; + std::vector solution_weights; + std::vector error_weights; + bool first_same_as_last{}; + bool weighted_controller{}; + bool modified_noise{}; + // solprop SEP 24 phase 3 (kernel schema 32): the low-storage RK4IP march. With it + // no Runge-Kutta view is allocated (stages, accumulators, accept buffer, residual, + // recycled stage, whole-step propagator, stage inverse); the accumulator register + // is, the stage register unless it is elided (solprop's + // `low_storage_stage_register`), and `low_storage_flows` flow volumes (0 + // recomputed, 1 a constant operator's table, 2 a varying operator's step pair). + bool low_storage{}; + bool low_storage_stage_register{true}; + std::size_t low_storage_flows{}; + // Whether a separable route's first input crosses to the transverse scratch + // through tiles of team scratch, as a device's does. + bool low_storage_tiled_input{}; + }; + /** The linear operator's fixed terms, row-major (terms, columns, n_omega). solprop + SEP 29 G1: Phi or the prefactor is the general z table when `phi_table` + (`prefactor_table`); its terms are then `n_phi_terms` (`n_prefactor_terms`) slots + that `table_evaluate` fills with the table's values at a trial's positions, and + `phi_terms` (`prefactor_terms`) is empty. */ + /** The shapes the bindings received for the arrays whose layout counts beside them + give, by field name: `validate` holds each to its counts (a vector keeps its + elements alone). Empty for a problem built in C++. */ + using Shapes = std::map>; + struct Linear { + std::size_t n_phi_terms{}; + std::size_t n_prefactor_terms{}; + bool phi_table{}; + bool prefactor_table{}; + std::vector phi_terms; + std::vector prefactor_terms; + Shapes shapes; + }; + /** The temporal absorber, applied in each stage or split around the step. */ + struct Absorber { + bool in_stage{}; + bool split_step{}; + std::vector fine; + std::vector coarse; + }; + /** The modal synthesis and projection onto the nodes. */ + struct Transform { + std::vector synthesis_matrix; + std::vector projection_matrix; + std::vector projection_coarse_matrix; + double synthesis{}; + double projection{}; + }; + /** The instantaneous Kerr response. */ + struct Kerr { + bool nonlinear{}; + double coefficient{}; + double self_factor{}; + double conjugate_factor{}; + double harmonic_factor{}; + bool take_real{}; + }; + /** The field and envelope plasma columns and their rate tables. */ + struct Plasma { + bool enabled{}; + bool envelope{}; + // solprop's private, test-only gated plasma benchmark: the field plasma's current + // enters as w(t) times its running integral, added to the local polarisation, + // instead of the held current's two collared spectra. Never set by a user run. + bool gated{}; + bool stash{}; + // Channel-major 4 x 512 split angular rules and coarse-axis current factors. + std::vector envelope_rule_nodes, envelope_rule_weights, + envelope_drude_factor; + std::vector envelope_loss_factor; + double charge_mass_ratio{}; + double ionisation_potential{}; + double log_field_min{}; + double d_log_field{}; + double field_min{}; + double field_max{}; + double rate_max{}; + std::size_t rate_intervals{}; + // Row-major (4, rate_intervals). + std::vector rate_coefficients; + std::size_t n_channels{1}; + // solprop SEP 18 (kernel schema 25): the field plasma column's running-integral + // rule, one per problem and shared by every channel. Each running integral is the + // cumulative trapezoid of the integrand filtered by the symmetric taps + // (c_0, ..., c_K), centre first, with the window's ends closed by point + // reflection, and the exposure floored at zero where the fraction is formed. The + // trapezoid rule is `band == false` with the one identity tap, and runs the + // version-24 column unchanged. + bool band{}; + std::vector taps{1.0}; + [[nodiscard]] std::size_t half_width() const noexcept { + return taps.empty() ? 0 : taps.size() - 1; + } + // Channel-major rows: log_min, d_log, field_min, field_max, rate_max, + // interval count, coefficient offset, charge/mass ratio, ionisation potential. + std::vector channel_parameters, channel_coefficients; + Shapes shapes; + }; + /** The delayed Raman response. */ + struct Raman { + bool enabled{}; + bool thg{}; + bool tensor_capable{}; + std::size_t n_terms{}; + double fraction{}; + double epsilon0{}; + // Row-major (n_terms, 2 n_to) and (n_terms, 2). + std::vector undamped; + std::vector tensor_weights; + std::size_t n_components{1}; + std::vector row_component, component_fraction; + Shapes shapes; + }; + /** solprop's general modal route. Row-major: synthesis (map terms, nodes, + components, modes, n_omega); projection (map terms, modes, nodes, components, + n_omega); tensor (nodes, C, C, C, C); delayed tensor (responses, terms, C, C, C, + C); delayed spectrum (responses, terms, n_to + 1); node response (nodes), the + owning response plus one and zero for none. Empty otherwise. The map terms are + map_intervals x map_n_deg Chebyshev coefficients along z, one for a constant + guide; a moving guide's are combined per stage over + TrialScalars::transform_weights. */ + struct General { + bool enabled{}; + bool real_form{}; + std::size_t map_intervals{1}, map_n_deg{1}; + std::size_t components{}, nodes{}, node_chunk{}; + std::size_t responses{}, terms{}; + double pad_scale{}, epsilon0{}; + std::vector synthesis, projection, frequency_factor, delayed_spectrum; + std::vector tensor, delayed_tensor; + std::vector node_response; + }; + /** solprop's linear.coupled_flow. `slot` (n_omega) numbers the bins a section's + generator is non-zero at 1, 2, ... in increasing frequency, zero elsewhere. With + an explicit residual (terms > 0), row-major frozen blocks (sections, active, + modes, modes) and residual terms (terms, active, modes, modes); both empty + otherwise. The actions are loaded per step. + + The actions' native formation, all present or all empty: each section's + eigenbasis of its Schur factor, row-major left `W` and right `Y` (sections, + active, modes, modes) and eigenvalues (sections, active, modes); the route + (sections, active), zero where a bin takes its eigenbasis and 1, 2, ... + numbering in row-major order the bins formed by scaling and squaring, whose Schur + factors `Z` and `T` are the rows of the fallback arrays (fallback, modes, + modes). */ + struct Flow { + bool enabled{}; + std::size_t active{}, sections{}, terms{}; + std::vector slot; + std::vector frozen, residual_terms; + std::vector left, right, eigenvalues; + std::vector route; + std::vector fallback_vectors, fallback_triangular; + }; + /** solprop SEP 6's frequency-dependent electronic response + (`responses.dispersive_kerr`, kernel schema 24) on the fused node path, laid out + by the adapter as solprop's `dispersive_terms_layout` lays it out + (`dispersive.hpp`). The form is 0 (a field grid's full cubic), 1 (the + number-conserving sector) or 2 (an envelope's full cubic). Row-major on the + leaf's spectral layout of `bins` (n_to / 2 + 1 on a field grid, n_to on an + envelope): the input multipliers (fields, bins), each output term's multiplier + (terms, bins) with its product row and factor, the static coefficients and the + polynomial's pair constants (channels, 2) per channel, the groups as offsets + (groups + 1) into members (channels), each symmetric mixture's channels' `g^4` + (channels, bins) and the input and output scales (bins). On the general modal + route (SEP 6 Phase 3b) each group's key is its node region (groups), the + polynomial forms one cube per distinct key, and every general node's region + (general nodes) says where a channel acts; both are empty elsewhere, where every + channel acts at every node. Empty otherwise. */ + struct Dispersive { + bool enabled{}; + bool symmetric{}; + int form{}; + std::vector inputs, term_multipliers, coefficients, pair_constants, + quartics, input_scale, output_scale; + std::vector term_rows, term_channels, group_offsets, group_members, + group_keys, node_region; + }; + + Dims dims; + Grid grid; + Spatial spatial; + Flux flux; + Stepper stepper; + Linear linear; + Absorber absorber; + Transform transform; + Kerr kerr; + Plasma plasma; + Raman raman; + General general; + Flow flow; + Dispersive dispersive; + // How the workspace's CPU operations thread (solprop SEP 46): set by the caller, + // refused when unset (`require_cpu_execution`). + CpuExecution cpu_execution{}; + // How the workspace plans its host transforms (solprop SEP 46 phase 3). + FftPlanning fft_planning{}; + /** The set-once data a workspace applies once it has built its state (solprop SEP + 52 section 5.3): the stage, z and flux tables, the flow guard, the controllers, + the low-storage settings, the monitor's weights and the edge's geometry + (`workspace.hpp`). Shared, so that a copy of the problem copies none of it. */ + std::shared_ptr setup; + + /** Refuse, naming the field, data whose presence or extents disagree with the + problem's own flags and extents: the checks that need no execution space. */ + void validate() const; + + /** Free space or the hybrid planar kind: the separable operator and a y FFT. */ + [[nodiscard]] bool separable_geometry() const noexcept { return spatial.n_x != 0; } + [[nodiscard]] bool slab_geometry() const noexcept { return spatial.n_p != 0; } + [[nodiscard]] bool free_geometry() const noexcept { + return spatial.n_x != 0 && spatial.n_p == 0; + } + [[nodiscard]] std::size_t n_columns() const noexcept { + return dims.n_modes * dims.n_pol_state; + } + [[nodiscard]] std::size_t effective_node_pol() const noexcept { + return dims.n_pol_transform * dims.n_pol_state; + } + [[nodiscard]] std::size_t raman_drives() const noexcept { + if (effective_node_pol() == 2) + for (std::size_t index = 1; index < raman.tensor_weights.size(); index += 2) + if (raman.tensor_weights[index] != 0.0) + return 3; + return 1; + } + [[nodiscard]] std::size_t raman_kernels() const noexcept { + return raman_drives() == 3 ? 2 : 1; + } +}; + +// The problem's groups as the seam fills them, one field table a group (solprop SEP 52 +// section 5.3). The relations a table states within its group are its schema's; the +// extents that a count in another group gives are `Problem::validate`'s. +[[nodiscard]] constexpr auto describe(const Problem::Dims* /*unused*/) { + using G = Problem::Dims; + return fields::table( + "Problem.Dims", fields::value("n_omega", &G::n_omega), + fields::value("n_t", &G::n_t), fields::value("n_to", &G::n_to), + fields::value("n_modes", &G::n_modes), + fields::value("n_pol_state", &G::n_pol_state), + fields::value("n_pol_transform", &G::n_pol_transform), + fields::value("n_nodes", &G::n_nodes), + fields::value("node_chunk", &G::node_chunk), fields::value("family", &G::family)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Grid* /*unused*/) { + using G = Problem::Grid; + return fields::table( + "Problem.Grid", fields::array("crop_index", &G::crop_index), + fields::array("active_mask", &G::active_mask), + fields::array("norm_mask", &G::norm_mask), + fields::array("omega_window", &G::omega_window), + fields::array("omega", &G::omega), + fields::array("power_weight", &G::power_weight), + fields::array("time_window", &G::time_window), + fields::array("time_window_derivative", &G::time_window_derivative), + fields::array("carrier", &G::carrier), fields::value("dt_fine", &G::dt_fine), + fields::value("dt", &G::dt)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Spatial* /*unused*/) { + using G = Problem::Spatial; + return fields::table( + "Problem.Spatial", fields::value("n_x", &G::n_x), fields::value("n_y", &G::n_y), + fields::array("k_squared", &G::k_squared), + fields::array("k_squared_low", &G::k_squared_low), + fields::array("numerator", &G::numerator), fields::array("clamped", &G::clamped), + fields::array("kperp_sq", &G::kperp_sq), + fields::array("kperp_sq_low", &G::kperp_sq_low), + fields::array("frame", &G::frame), + fields::array("spectral_rate", &G::spectral_rate), + fields::array("user_rate", &G::user_rate), + fields::array("absorber_rate", &G::absorber_rate), + fields::array("kperp_rate", &G::kperp_rate), + fields::value("evanescent_rate", &G::evanescent_rate), + fields::array("k_x", &G::k_x), fields::array("k_y", &G::k_y), + fields::value("far_field_slots", &G::far_field_slots), + fields::value("n_p", &G::n_p), fields::array("kappa_sq", &G::kappa_sq), + fields::array("absorber_matrix", &G::absorber_matrix), + fields::array("spectral_weights", &G::spectral_weights), + fields::array("on_axis", &G::on_axis)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Flux* /*unused*/) { + using G = Problem::Flux; + return fields::table("Problem.Flux", fields::array("k_squared", &G::k_squared), + fields::array("k_squared_low", &G::k_squared_low), + fields::array("kperp_sq", &G::kperp_sq), + fields::array("kperp_sq_low", &G::kperp_sq_low), + fields::value("table", &G::table), + fields::value("closure", &G::closure)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Stepper* /*unused*/) { + using G = Problem::Stepper; + return fields::table( + "Problem.Stepper", fields::value("n_stages", &G::n_stages), + fields::array("tableau_matrix", &G::tableau_matrix), + fields::array("tableau_nodes", &G::tableau_nodes), + fields::array("solution_weights", &G::solution_weights), + fields::array("error_weights", &G::error_weights), + fields::value("first_same_as_last", &G::first_same_as_last), + fields::value("weighted_controller", &G::weighted_controller), + fields::value("modified_noise", &G::modified_noise), + fields::value("low_storage", &G::low_storage), + fields::value("low_storage_stage_register", &G::low_storage_stage_register), + fields::value("low_storage_flows", &G::low_storage_flows), + fields::value("low_storage_tiled_input", &G::low_storage_tiled_input)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Linear* /*unused*/) { + using G = Problem::Linear; + return fields::table("Problem.Linear", + fields::value("n_phi_terms", &G::n_phi_terms), + fields::value("n_prefactor_terms", &G::n_prefactor_terms), + fields::value("phi_table", &G::phi_table), + fields::value("prefactor_table", &G::prefactor_table), + fields::array("phi_terms", &G::phi_terms), + fields::array("prefactor_terms", &G::prefactor_terms)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Absorber* /*unused*/) { + using G = Problem::Absorber; + return fields::table("Problem.Absorber", fields::value("in_stage", &G::in_stage), + fields::value("split_step", &G::split_step), + fields::array("fine", &G::fine), + fields::array("coarse", &G::coarse)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Transform* /*unused*/) { + using G = Problem::Transform; + return fields::table( + "Problem.Transform", fields::array("synthesis_matrix", &G::synthesis_matrix), + fields::array("projection_matrix", &G::projection_matrix), + fields::array("projection_coarse_matrix", &G::projection_coarse_matrix), + fields::value("synthesis", &G::synthesis), + fields::value("projection", &G::projection)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Kerr* /*unused*/) { + using G = Problem::Kerr; + return fields::table("Problem.Kerr", fields::value("nonlinear", &G::nonlinear), + fields::value("coefficient", &G::coefficient), + fields::value("self_factor", &G::self_factor), + fields::value("conjugate_factor", &G::conjugate_factor), + fields::value("harmonic_factor", &G::harmonic_factor), + fields::value("take_real", &G::take_real)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Plasma* /*unused*/) { + using G = Problem::Plasma; + using E = fields::Extent; + return fields::table( + "Problem.Plasma", fields::value("enabled", &G::enabled), + fields::value("envelope", &G::envelope), fields::value("gated", &G::gated), + fields::value("stash", &G::stash), + fields::array("envelope_rule_nodes", &G::envelope_rule_nodes, E::of("n_channels"), + E::fixed(4), E::fixed(512)), + fields::array("envelope_rule_weights", &G::envelope_rule_weights, + E::of("n_channels"), E::fixed(4), E::fixed(512)), + fields::array("envelope_drude_factor", &G::envelope_drude_factor, E::any()), + fields::array("envelope_loss_factor", &G::envelope_loss_factor, E::any()), + fields::value("charge_mass_ratio", &G::charge_mass_ratio), + fields::value("ionisation_potential", &G::ionisation_potential), + fields::value("log_field_min", &G::log_field_min), + fields::value("d_log_field", &G::d_log_field), + fields::value("field_min", &G::field_min), + fields::value("field_max", &G::field_max), + fields::value("rate_max", &G::rate_max), + fields::value("rate_intervals", &G::rate_intervals), + fields::array("rate_coefficients", &G::rate_coefficients, E::fixed(4), + E::of("rate_intervals")), + fields::value("n_channels", &G::n_channels), fields::value("band", &G::band), + fields::array("taps", &G::taps, E::any()), + fields::array("channel_parameters", &G::channel_parameters), + fields::array("channel_coefficients", &G::channel_coefficients)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Raman* /*unused*/) { + using G = Problem::Raman; + using E = fields::Extent; + return fields::table( + "Problem.Raman", fields::value("enabled", &G::enabled), + fields::value("thg", &G::thg), + fields::value("tensor_capable", &G::tensor_capable), + fields::value("n_terms", &G::n_terms), fields::value("fraction", &G::fraction), + fields::value("epsilon0", &G::epsilon0), + fields::array("undamped", &G::undamped, E::of("n_terms"), E::any()), + fields::array("tensor_weights", &G::tensor_weights, E::of("n_terms"), + E::fixed(2)), + fields::value("n_components", &G::n_components), + fields::array("row_component", &G::row_component), + fields::array("component_fraction", &G::component_fraction)); +} + +[[nodiscard]] constexpr auto describe(const Problem::General* /*unused*/) { + using G = Problem::General; + return fields::table( + "Problem.General", fields::value("enabled", &G::enabled), + fields::value("real_form", &G::real_form), + fields::value("map_intervals", &G::map_intervals), + fields::value("map_n_deg", &G::map_n_deg), + fields::value("components", &G::components), fields::value("nodes", &G::nodes), + fields::value("node_chunk", &G::node_chunk), + fields::value("responses", &G::responses), fields::value("terms", &G::terms), + fields::value("pad_scale", &G::pad_scale), + fields::value("epsilon0", &G::epsilon0), + fields::array("synthesis", &G::synthesis), + fields::array("projection", &G::projection), + fields::array("frequency_factor", &G::frequency_factor), + fields::array("delayed_spectrum", &G::delayed_spectrum), + fields::array("tensor", &G::tensor), + fields::array("delayed_tensor", &G::delayed_tensor), + fields::array("node_response", &G::node_response)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Flow* /*unused*/) { + using G = Problem::Flow; + return fields::table( + "Problem.Flow", fields::value("enabled", &G::enabled), + fields::value("active", &G::active), fields::value("sections", &G::sections), + fields::value("terms", &G::terms), fields::array("slot", &G::slot), + fields::array("frozen", &G::frozen), + fields::array("residual_terms", &G::residual_terms), + fields::array("left", &G::left), fields::array("right", &G::right), + fields::array("eigenvalues", &G::eigenvalues), fields::array("route", &G::route), + fields::array("fallback_vectors", &G::fallback_vectors), + fields::array("fallback_triangular", &G::fallback_triangular)); +} + +[[nodiscard]] constexpr auto describe(const Problem::Dispersive* /*unused*/) { + using G = Problem::Dispersive; + return fields::table("Problem.Dispersive", fields::value("enabled", &G::enabled), + fields::value("symmetric", &G::symmetric), + fields::value("form", &G::form), + fields::array("inputs", &G::inputs), + fields::array("term_multipliers", &G::term_multipliers), + fields::array("coefficients", &G::coefficients), + fields::array("pair_constants", &G::pair_constants), + fields::array("quartics", &G::quartics), + fields::array("input_scale", &G::input_scale), + fields::array("output_scale", &G::output_scale), + fields::array("term_rows", &G::term_rows), + fields::array("term_channels", &G::term_channels), + fields::array("group_offsets", &G::group_offsets), + fields::array("group_members", &G::group_members), + fields::array("group_keys", &G::group_keys), + fields::array("node_region", &G::node_region)); +} + +/** Whether `problem` runs on a compact CPU team (`CpuExecution::compact_threads`): one + column on one node, fewer than its record's `compact_max_fine_samples` fine + samples, and neither the general modal route, a coupled flow nor a separable + geometry. */ +[[nodiscard]] bool compact_problem(const Problem& problem) noexcept; + +} // namespace solprop_native diff --git a/native/include/solprop_native/readings.hpp b/native/include/solprop_native/readings.hpp new file mode 100644 index 000000000..a58689f88 --- /dev/null +++ b/native/include/solprop_native/readings.hpp @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "solprop_native/fields.hpp" + +#include + +#include +#include + +namespace solprop_native { + +/** The boundary monitor's five sums of a state (solprop SEP 21), one reduction value: + the energy, then the low guard's, the high guard's, the collar's and the rate's. */ +struct MonitorSums { + Kokkos::Array values{}; + KOKKOS_INLINE_FUNCTION MonitorSums& operator+=(const MonitorSums& other) { + for (std::size_t slot = 0; slot < 5; ++slot) + values[slot] += other.values[slot]; + return *this; + } +}; + +/** A state's whole-state transfers, in bytes, each way (solprop SEP 44): what lets a + test hold a field-free run to no transfer at all, whichever path would have made one. + */ +struct CrossedBytes { + std::size_t to_host{}; + std::size_t from_host{}; +}; + +/** What the transverse edge reads of a state (solprop SEP 44): four numbers per weight + table, and each mode's energy. */ +struct EdgeNumbers { + std::vector sums; + std::vector columns; + /** A requested collar's `(inward, windowed)` per line. */ + std::vector strip; + /** A requested collar's rectified inward current. */ + double current{}; +}; + +/** A requested collar's lines, as Python describes them (solprop SEP 44): for each, its + axis (0 `r`, 1 `x`, 2 `y`), its count of strip nodes and those nodes, its window, + its inward bins `[first, last)`, the factor on its strip's sums and the measure on + its current, and its onset's node, before and after, with the distance between the + last two. An onset whose node is `no_onset` has no slope to read. */ +struct EdgeLines { + static constexpr std::size_t no_onset = static_cast(-1); + std::vector axes, counts, positions, inward, onsets; + std::vector windows, factors, measures, steps, scale; + double k_min{}; + /** The most one batch of a padded transform holds. */ + std::size_t batch_bytes{}; +}; + +/** What a split-step collar's finish reads of the power it finds (solprop SEP 21, + ADR-057), one reduction value: the energy it takes, then the energy, the low + guard's, the high guard's and the collar's, which the monitor judges the + occupancies on. */ +struct SplitSums { + Kokkos::Array values{}; + KOKKOS_INLINE_FUNCTION SplitSums& operator+=(const SplitSums& other) { + for (std::size_t slot = 0; slot < 5; ++slot) + values[slot] += other.values[slot]; + return *this; + } +}; + +/** The boundary monitor's occupancy of a state (solprop #440), one reduction value: its + energy per walk-off bucket. A device's reduction keeps one of these per thread in + team memory, with no atomics, so its size bounds the buckets (`SPLIT_RATE_BUCKETS`). + */ +inline constexpr std::size_t occupancy_bucket_limit = 128; +// The buckets one reduction holds. A reduction's value has to fit the device's shared +// memory per block, which on HIP's smallest block of 64 threads is `(64 + 2) * +// sizeof(value)`, with the kernel's own static memory beside it within 64 KiB. Kokkos +// checks only the first: a value that fills the block passes its check and the dispatch +// then faults on an AMD Instinct MI325X ("Illegal opcode in command stream"). So a +// reading there takes two passes, leaving half the block free. +#ifdef KOKKOS_ENABLE_HIP +inline constexpr std::size_t occupancy_reduction_slots = 64; +#else +inline constexpr std::size_t occupancy_reduction_slots = occupancy_bucket_limit; +#endif +struct OccupancySums { + Kokkos::Array values{}; + KOKKOS_INLINE_FUNCTION OccupancySums& operator+=(const OccupancySums& other) { + for (std::size_t slot = 0; slot < occupancy_reduction_slots; ++slot) + values[slot] += other.values[slot]; + return *this; + } +}; +#ifdef KOKKOS_ENABLE_HIP +static_assert((64 + 2) * sizeof(OccupancySums) <= 48 * 1024); +#endif + +[[nodiscard]] constexpr auto describe(const EdgeLines* /*unused*/) { + using L = EdgeLines; + using E = fields::Extent; + return fields::table( + "EdgeLines", fields::array("axes", &L::axes, E::of("n_lines")), + fields::array("counts", &L::counts, E::of("n_lines")), + fields::array("positions", &L::positions, E::any()), + fields::array("windows", &L::windows, E::any()), + fields::array("inward", &L::inward, E::any()), + fields::array("factors", &L::factors, E::of("n_lines")), + fields::array("measures", &L::measures, E::of("n_lines")), + fields::array("onsets", &L::onsets, E::any()), + fields::array("steps", &L::steps, E::of("n_lines")), + fields::value("k_min", &L::k_min), fields::array("scale", &L::scale, E::any()), + fields::value("batch_bytes", &L::batch_bytes)); +} + +} // namespace solprop_native + +template <> struct Kokkos::reduction_identity { + KOKKOS_FUNCTION static solprop_native::MonitorSums sum() { return {}; } +}; +template <> struct Kokkos::reduction_identity { + KOKKOS_FUNCTION static solprop_native::OccupancySums sum() { return {}; } +}; +template <> struct Kokkos::reduction_identity { + KOKKOS_FUNCTION static solprop_native::SplitSums sum() { return {}; } +}; diff --git a/native/include/solprop_native/reference_timing.hpp b/native/include/solprop_native/reference_timing.hpp new file mode 100644 index 000000000..60de2716d --- /dev/null +++ b/native/include/solprop_native/reference_timing.hpp @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include + +namespace solprop_native { + +/** solprop SEP 28 section 4.2's per-evaluation reference, independent of any + workspace: the seconds of each timed forward-then-backward pair of an in-place, + unnormalised rank-3 complex transform of `values` (row-major + `(planes, n_y, n_x)`, copied in once) through Kokkos-FFT on the FFT execution + space, after planning and `warm` untimed pairs. */ +[[nodiscard]] std::vector +reference_fft3_pairs(const std::complex* values, std::size_t planes, + std::size_t n_y, std::size_t n_x, std::size_t warm, + std::size_t pairs); + +/** The same reference's streaming volume-pass: the seconds of each timed scaled copy + `b = s a` between two complex volumes of `elements` first touched in parallel on + the kernels' execution space, after `warm` untimed copies. */ +[[nodiscard]] std::vector +reference_stream_copies(std::size_t elements, std::size_t warm, std::size_t repeats); + +} // namespace solprop_native diff --git a/native/include/solprop_native/refuse.hpp b/native/include/solprop_native/refuse.hpp index ab8e85707..94cdc4dcb 100644 --- a/native/include/solprop_native/refuse.hpp +++ b/native/include/solprop_native/refuse.hpp @@ -11,6 +11,19 @@ namespace solprop_native { +/** Signal exhaustion of the validated angular rule before source assembly. */ +class EnvelopeConvergenceError : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + +/** Refuse a step solprop's reference refuses with `BackendError` (solprop SEP 45 + phase 4): a coupled flow's action past its squarings or its certificate. */ +class BackendRefusal : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + namespace detail { template void append_part(std::string& text, const Part& part) { diff --git a/native/include/solprop_native/slab.hpp b/native/include/solprop_native/slab.hpp index a80fbc34b..2a3914210 100644 --- a/native/include/solprop_native/slab.hpp +++ b/native/include/solprop_native/slab.hpp @@ -4,8 +4,8 @@ #pragma once -#include "solprop_native/core.hpp" -#include "solprop_native/general_modal.hpp" +#include "solprop_native/execution.hpp" +#include "solprop_native/launch.hpp" #include diff --git a/native/include/solprop_native/stage_tables.hpp b/native/include/solprop_native/stage_tables.hpp index ffb2adbfd..458e99ad7 100644 --- a/native/include/solprop_native/stage_tables.hpp +++ b/native/include/solprop_native/stage_tables.hpp @@ -5,7 +5,8 @@ #pragma once #include "solprop_native/chebyshev.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/fields.hpp" +#include "solprop_native/general_state.hpp" #include "solprop_native/lagrange_basis.hpp" #include "solprop_native/refuse.hpp" diff --git a/native/include/solprop_native/workspace.hpp b/native/include/solprop_native/workspace.hpp index 9ef2b7047..54dd941c6 100644 --- a/native/include/solprop_native/workspace.hpp +++ b/native/include/solprop_native/workspace.hpp @@ -5,10 +5,13 @@ #pragma once #include "solprop_native/adams_march.hpp" -#include "solprop_native/core.hpp" #include "solprop_native/engines.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/fields.hpp" +#include "solprop_native/general_state.hpp" #include "solprop_native/low_storage.hpp" +#include "solprop_native/problem.hpp" +#include "solprop_native/readings.hpp" #include "solprop_native/rk_march.hpp" #include "solprop_native/session.hpp" #include "solprop_native/stage_tables.hpp" @@ -42,13 +45,6 @@ namespace solprop_native { // ------------------------------------------------------------------ the set-once data -/** The z tables by name (`TableKind`). */ -[[nodiscard]] constexpr auto names(const TableKind* /*unused*/) { - return std::array{ - std::pair{"phi", TableKind::phi}, std::pair{"prefactor", TableKind::prefactor}, - std::pair{"rate", TableKind::rate}, std::pair{"flux", TableKind::flux}}; -} - /** One general z table made resident at construction (solprop SEP 29 G1): its breakpoints, admitted span and coefficients `(n_edges - 1, n_terms, n_values)`. */ struct ZTable { @@ -193,23 +189,6 @@ struct Occupancy { fields::value("count", &O::count)); } -[[nodiscard]] constexpr auto describe(const EdgeLines* /*unused*/) { - using L = EdgeLines; - using E = fields::Extent; - return fields::table( - "EdgeLines", fields::array("axes", &L::axes, E::of("n_lines")), - fields::array("counts", &L::counts, E::of("n_lines")), - fields::array("positions", &L::positions, E::any()), - fields::array("windows", &L::windows, E::any()), - fields::array("inward", &L::inward, E::any()), - fields::array("factors", &L::factors, E::of("n_lines")), - fields::array("measures", &L::measures, E::of("n_lines")), - fields::array("onsets", &L::onsets, E::any()), - fields::array("steps", &L::steps, E::of("n_lines")), - fields::value("k_min", &L::k_min), fields::array("scale", &L::scale, E::any()), - fields::value("batch_bytes", &L::batch_bytes)); -} - /** The transverse edge's tables where the state lives (solprop SEP 44), a run's: the faces', the clip's and the totals' weights, a reduction's synthesis (empty for none), the first row, and a requested collar's lines. */ diff --git a/native/include/solprop_native/z_table.hpp b/native/include/solprop_native/z_table.hpp index d433ae756..823baedfb 100644 --- a/native/include/solprop_native/z_table.hpp +++ b/native/include/solprop_native/z_table.hpp @@ -5,7 +5,7 @@ #pragma once #include "solprop_native/chebyshev.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/launch.hpp" #include "solprop_native/refuse.hpp" diff --git a/native/src/adams_march.cpp b/native/src/adams_march.cpp index 720111b41..c716cd582 100644 --- a/native/src/adams_march.cpp +++ b/native/src/adams_march.cpp @@ -4,7 +4,7 @@ #include "solprop_native/adams_march.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/host_source.hpp" #include "solprop_native/refuse.hpp" #include diff --git a/native/src/bindings.cpp b/native/src/bindings.cpp index 0ab331348..b9cbb354a 100644 --- a/native/src/bindings.cpp +++ b/native/src/bindings.cpp @@ -6,14 +6,20 @@ #include "helpers.hpp" #include "records.hpp" #include "solprop_native/adams_march.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/cpu_execution.hpp" #include "solprop_native/crossing.hpp" #include "solprop_native/dense_output.hpp" #include "solprop_native/engines.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/fft_plan.hpp" +#include "solprop_native/fields.hpp" +#include "solprop_native/general_state.hpp" #include "solprop_native/host_source.hpp" #include "solprop_native/lagrange_basis.hpp" #include "solprop_native/low_storage.hpp" +#include "solprop_native/problem.hpp" +#include "solprop_native/readings.hpp" +#include "solprop_native/reference_timing.hpp" #include "solprop_native/refuse.hpp" #include "solprop_native/rk_march.hpp" #include "solprop_native/session.hpp" diff --git a/native/src/core.cpp b/native/src/cpu_execution.cpp similarity index 67% rename from native/src/core.cpp rename to native/src/cpu_execution.cpp index 02fcafa23..3cb5cfda9 100644 --- a/native/src/core.cpp +++ b/native/src/cpu_execution.cpp @@ -2,25 +2,26 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/core.hpp" +#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/launch.hpp" +#include "solprop_native/execution.hpp" -#include +#include #include #include -#include -#include #include -#include +#include #include -#include -#include +#include #ifdef KOKKOS_ENABLE_OPENMP #include #endif +#if defined(KOKKOS_ENABLE_OPENMP) && defined(SOLPROP_NATIVE_FFTW_OPENMP) && \ + !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +#include +#endif namespace solprop_native { namespace { @@ -325,71 +326,6 @@ int scoped_openmp_default() noexcept { #endif } -void Problem::validate() const { - const auto require = [](const bool holds, const char* message) { - if (!holds) - throw std::invalid_argument(message); - }; - require(cpu_execution != CpuExecution{}, - "a native problem needs its CPU execution record (cpu_execution, solprop " - "SEP 46)"); - // A bound array's recorded shape is the layout its counts give; an array set in C++ - // records none. - const auto shaped = [](const Shapes& shapes, const char* field, - const std::vector& expected) { - const auto found = shapes.find(field); - return found == shapes.end() || found->second == expected; - }; - for (const auto& [field, count, rows] : - {std::tuple{"phi_terms", linear.n_phi_terms, - "phi_terms must hold n_phi_terms rows along its first axis"}, - std::tuple{"prefactor_terms", linear.n_prefactor_terms, - "prefactor_terms must hold n_prefactor_terms rows along its first " - "axis"}}) - if (const auto found = linear.shapes.find(field); found != linear.shapes.end()) { - const auto& shape = found->second; - require(shape.size() == 2 || shape.size() == 3, - "fixed terms must be rank-two or rank-three arrays"); - require(shape[0] == count, rows); - } - if (raman.enabled) { - require(raman.n_terms != 0 && - dims.n_to <= - std::numeric_limits::max() / (2 * raman.n_terms) && - raman.undamped.size() == raman.n_terms * 2 * dims.n_to && - shaped(raman.shapes, "undamped", {raman.n_terms, 2 * dims.n_to}), - "raman_undamped must have shape (n_terms, 2 * n_to)"); - require(raman.tensor_weights.size() == 2 * raman.n_terms && - shaped(raman.shapes, "tensor_weights", {raman.n_terms, 2}), - "raman_tensor_weights must have shape (n_terms, 2)"); - } else { - require(raman.n_terms == 0, - "raman.n_terms counts the terms of an enabled response"); - } - require(plasma.rate_coefficients.size() == 4 * plasma.rate_intervals && - shaped(plasma.shapes, "rate_coefficients", {4, plasma.rate_intervals}), - "rate_coefficients must have shape (4, n_intervals)"); - if (plasma.envelope) { - constexpr std::size_t rule = std::size_t{4} * 512; - const std::vector layout{plasma.n_channels, 4, 512}; - require(plasma.envelope_rule_nodes.size() == plasma.n_channels * rule && - plasma.envelope_rule_weights.size() == plasma.n_channels * rule && - shaped(plasma.shapes, "envelope_rule_nodes", layout) && - shaped(plasma.shapes, "envelope_rule_weights", layout), - "envelope rules require (n_channels, 4, 512)"); - require(plasma.envelope_drude_factor.size() == dims.n_omega && - plasma.envelope_loss_factor.size() == dims.n_omega, - "envelope current factors require (n_omega,)"); - } -} - -bool compact_problem(const Problem& problem) noexcept { - return problem.n_columns() == 1 && problem.dims.n_nodes == 1 && - problem.dims.n_pol_transform == 1 && !problem.general.enabled && - !problem.flow.enabled && !problem.separable_geometry() && - problem.dims.n_to < problem.cpu_execution.compact_max_fine_samples; -} - const ExecutionSpace& CpuExecutionPool::get(const CpuOperation operation, const std::size_t elements, const std::size_t fine_samples) { @@ -444,79 +380,4 @@ const ExecutionSpace& CpuExecutionPool::get(const CpuOperation operation, return found->second; } -std::vector reference_fft3_pairs(const std::complex* values, - const std::size_t planes, - const std::size_t n_y, const std::size_t n_x, - const std::size_t warm, - const std::size_t pairs) { - using Volume = Kokkos::View; - using Plan3 = KokkosFFT::Plan; - const ExecutionSpace space = execution_instance(thread_policy().fft); - Volume volume(Kokkos::view_alloc(Kokkos::WithoutInitializing, - "solprop_native.reference.volume"), - planes, n_y, n_x); - const Kokkos::View> - host(reinterpret_cast(values), planes, n_y, n_x); - auto mirror = Kokkos::create_mirror_view(Kokkos::HostSpace{}, volume); - Kokkos::deep_copy(mirror, host); - Kokkos::deep_copy(volume, mirror); - const KokkosFFT::axis_type<3> axes{-3, -2, -1}; - const Plan3 forward(space, volume, volume, KokkosFFT::Direction::forward, axes); - const Plan3 backward(space, volume, volume, KokkosFFT::Direction::backward, axes); - std::vector seconds; - seconds.reserve(pairs); - for (std::size_t index = 0; index < warm + pairs; ++index) { - space.fence(); - const auto start = std::chrono::steady_clock::now(); - KokkosFFT::execute(forward, volume, volume, KokkosFFT::Normalization::none); - KokkosFFT::execute(backward, volume, volume, KokkosFFT::Normalization::none); - space.fence(); - if (index >= warm) - seconds.push_back( - std::chrono::duration(std::chrono::steady_clock::now() - start) - .count()); - } - return seconds; -} - -std::vector reference_stream_copies(const std::size_t elements, - const std::size_t warm, - const std::size_t repeats) { - const ExecutionSpace space = execution_instance(thread_policy().kernels); - const ComplexView source(Kokkos::view_alloc(Kokkos::WithoutInitializing, - "solprop_native.reference.source"), - elements); - const ComplexView target(Kokkos::view_alloc(Kokkos::WithoutInitializing, - "solprop_native.reference.target"), - elements); - using Policy = Kokkos::RangePolicy; - // First touch in parallel, on the threads that copy, so that each page lands where - // the copy reads or writes it. - launch::parallel_for( - "solprop_native.reference.first_touch", Policy(space, 0, elements), - KOKKOS_LAMBDA(const std::size_t index) { - source(index) = Complex(static_cast(index % 7U), 1.0); - target(index) = Complex{}; - }); - std::vector seconds; - seconds.reserve(repeats); - const double scale = 0.5; - for (std::size_t index = 0; index < warm + repeats; ++index) { - space.fence(); - const auto start = std::chrono::steady_clock::now(); - launch::parallel_for( - "solprop_native.reference.scaled_copy", Policy(space, 0, elements), - KOKKOS_LAMBDA(const std::size_t element) { - target(element) = scale * source(element); - }); - space.fence(); - if (index >= warm) - seconds.push_back( - std::chrono::duration(std::chrono::steady_clock::now() - start) - .count()); - } - return seconds; -} - } // namespace solprop_native diff --git a/native/src/general/kernels.hpp b/native/src/general/kernels.hpp index 7e46e5ea7..d3f4161d4 100644 --- a/native/src/general/kernels.hpp +++ b/native/src/general/kernels.hpp @@ -14,12 +14,13 @@ #include "solprop_native/adams_phi.hpp" #include "solprop_native/build.hpp" -#include "solprop_native/core.hpp" #include "solprop_native/coupled_flow.hpp" +#include "solprop_native/cpu_execution.hpp" #include "solprop_native/crossing.hpp" #include "solprop_native/dispatch.hpp" #include "solprop_native/dispersive.hpp" #include "solprop_native/envelope_plasma.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/far_field.hpp" #include "solprop_native/fft_plan.hpp" #include "solprop_native/flow_functions.hpp" diff --git a/native/src/general/trial.cpp b/native/src/general/trial.cpp index 46546e05f..e40a5ceeb 100644 --- a/native/src/general/trial.cpp +++ b/native/src/general/trial.cpp @@ -332,7 +332,7 @@ std::size_t GeneralWorkspace::response_team_size() const noexcept { } // Size from the plan's real-side batch samples, using the shared complete-window -// thresholds in core.hpp. Explicit FFT-pool requests bypass automatic sizing. +// thresholds in cpu_execution.hpp. Explicit FFT-pool requests bypass automatic sizing. // FFTW's factorisation can change with the width; comparisons use solprop's SEP 25 // transform/propagation rounding budgets rather than bitwise agreement. const ExecutionSpace& GeneralWorkspace::fft_space_for(const std::size_t samples, diff --git a/native/src/general/workspace.hpp b/native/src/general/workspace.hpp index f6bb5b6a9..410d0ee77 100644 --- a/native/src/general/workspace.hpp +++ b/native/src/general/workspace.hpp @@ -12,6 +12,7 @@ #endif #include "kernels.hpp" +#include "solprop_native/host_source.hpp" namespace solprop_native { diff --git a/native/src/problem.cpp b/native/src/problem.cpp new file mode 100644 index 000000000..b241d240b --- /dev/null +++ b/native/src/problem.cpp @@ -0,0 +1,82 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "solprop_native/problem.hpp" + +#include "solprop_native/cpu_execution.hpp" + +#include +#include +#include +#include +#include + +namespace solprop_native { + +void Problem::validate() const { + const auto require = [](const bool holds, const char* message) { + if (!holds) + throw std::invalid_argument(message); + }; + require(cpu_execution != CpuExecution{}, + "a native problem needs its CPU execution record (cpu_execution, solprop " + "SEP 46)"); + // A bound array's recorded shape is the layout its counts give; an array set in C++ + // records none. + const auto shaped = [](const Shapes& shapes, const char* field, + const std::vector& expected) { + const auto found = shapes.find(field); + return found == shapes.end() || found->second == expected; + }; + for (const auto& [field, count, rows] : + {std::tuple{"phi_terms", linear.n_phi_terms, + "phi_terms must hold n_phi_terms rows along its first axis"}, + std::tuple{"prefactor_terms", linear.n_prefactor_terms, + "prefactor_terms must hold n_prefactor_terms rows along its first " + "axis"}}) + if (const auto found = linear.shapes.find(field); found != linear.shapes.end()) { + const auto& shape = found->second; + require(shape.size() == 2 || shape.size() == 3, + "fixed terms must be rank-two or rank-three arrays"); + require(shape[0] == count, rows); + } + if (raman.enabled) { + require(raman.n_terms != 0 && + dims.n_to <= + std::numeric_limits::max() / (2 * raman.n_terms) && + raman.undamped.size() == raman.n_terms * 2 * dims.n_to && + shaped(raman.shapes, "undamped", {raman.n_terms, 2 * dims.n_to}), + "raman_undamped must have shape (n_terms, 2 * n_to)"); + require(raman.tensor_weights.size() == 2 * raman.n_terms && + shaped(raman.shapes, "tensor_weights", {raman.n_terms, 2}), + "raman_tensor_weights must have shape (n_terms, 2)"); + } else { + require(raman.n_terms == 0, + "raman.n_terms counts the terms of an enabled response"); + } + require(plasma.rate_coefficients.size() == 4 * plasma.rate_intervals && + shaped(plasma.shapes, "rate_coefficients", {4, plasma.rate_intervals}), + "rate_coefficients must have shape (4, n_intervals)"); + if (plasma.envelope) { + constexpr std::size_t rule = std::size_t{4} * 512; + const std::vector layout{plasma.n_channels, 4, 512}; + require(plasma.envelope_rule_nodes.size() == plasma.n_channels * rule && + plasma.envelope_rule_weights.size() == plasma.n_channels * rule && + shaped(plasma.shapes, "envelope_rule_nodes", layout) && + shaped(plasma.shapes, "envelope_rule_weights", layout), + "envelope rules require (n_channels, 4, 512)"); + require(plasma.envelope_drude_factor.size() == dims.n_omega && + plasma.envelope_loss_factor.size() == dims.n_omega, + "envelope current factors require (n_omega,)"); + } +} + +bool compact_problem(const Problem& problem) noexcept { + return problem.n_columns() == 1 && problem.dims.n_nodes == 1 && + problem.dims.n_pol_transform == 1 && !problem.general.enabled && + !problem.flow.enabled && !problem.separable_geometry() && + problem.dims.n_to < problem.cpu_execution.compact_max_fine_samples; +} + +} // namespace solprop_native diff --git a/native/src/reference_timing.cpp b/native/src/reference_timing.cpp new file mode 100644 index 000000000..07e75b33d --- /dev/null +++ b/native/src/reference_timing.cpp @@ -0,0 +1,96 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "solprop_native/reference_timing.hpp" + +#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/execution.hpp" +#include "solprop_native/launch.hpp" + +#include +#include + +#include +#include +#include +#include + +namespace solprop_native { + +std::vector reference_fft3_pairs(const std::complex* values, + const std::size_t planes, + const std::size_t n_y, const std::size_t n_x, + const std::size_t warm, + const std::size_t pairs) { + using Volume = Kokkos::View; + using Plan3 = KokkosFFT::Plan; + const ExecutionSpace space = execution_instance(thread_policy().fft); + Volume volume(Kokkos::view_alloc(Kokkos::WithoutInitializing, + "solprop_native.reference.volume"), + planes, n_y, n_x); + const Kokkos::View> + host(reinterpret_cast(values), planes, n_y, n_x); + auto mirror = Kokkos::create_mirror_view(Kokkos::HostSpace{}, volume); + Kokkos::deep_copy(mirror, host); + Kokkos::deep_copy(volume, mirror); + const KokkosFFT::axis_type<3> axes{-3, -2, -1}; + const Plan3 forward(space, volume, volume, KokkosFFT::Direction::forward, axes); + const Plan3 backward(space, volume, volume, KokkosFFT::Direction::backward, axes); + std::vector seconds; + seconds.reserve(pairs); + for (std::size_t index = 0; index < warm + pairs; ++index) { + space.fence(); + const auto start = std::chrono::steady_clock::now(); + KokkosFFT::execute(forward, volume, volume, KokkosFFT::Normalization::none); + KokkosFFT::execute(backward, volume, volume, KokkosFFT::Normalization::none); + space.fence(); + if (index >= warm) + seconds.push_back( + std::chrono::duration(std::chrono::steady_clock::now() - start) + .count()); + } + return seconds; +} + +std::vector reference_stream_copies(const std::size_t elements, + const std::size_t warm, + const std::size_t repeats) { + const ExecutionSpace space = execution_instance(thread_policy().kernels); + const ComplexView source(Kokkos::view_alloc(Kokkos::WithoutInitializing, + "solprop_native.reference.source"), + elements); + const ComplexView target(Kokkos::view_alloc(Kokkos::WithoutInitializing, + "solprop_native.reference.target"), + elements); + using Policy = Kokkos::RangePolicy; + // First touch in parallel, on the threads that copy, so that each page lands where + // the copy reads or writes it. + launch::parallel_for( + "solprop_native.reference.first_touch", Policy(space, 0, elements), + KOKKOS_LAMBDA(const std::size_t index) { + source(index) = Complex(static_cast(index % 7U), 1.0); + target(index) = Complex{}; + }); + std::vector seconds; + seconds.reserve(repeats); + const double scale = 0.5; + for (std::size_t index = 0; index < warm + repeats; ++index) { + space.fence(); + const auto start = std::chrono::steady_clock::now(); + launch::parallel_for( + "solprop_native.reference.scaled_copy", Policy(space, 0, elements), + KOKKOS_LAMBDA(const std::size_t element) { + target(element) = scale * source(element); + }); + space.fence(); + if (index >= warm) + seconds.push_back( + std::chrono::duration(std::chrono::steady_clock::now() - start) + .count()); + } + return seconds; +} + +} // namespace solprop_native diff --git a/native/src/rk_march.cpp b/native/src/rk_march.cpp index 094b89af8..9f71015c5 100644 --- a/native/src/rk_march.cpp +++ b/native/src/rk_march.cpp @@ -4,7 +4,7 @@ #include "solprop_native/rk_march.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/refuse.hpp" #include #include diff --git a/native/src/session.cpp b/native/src/session.cpp index b5af2ce02..afcd55b01 100644 --- a/native/src/session.cpp +++ b/native/src/session.cpp @@ -4,7 +4,6 @@ #include "solprop_native/session.hpp" -#include "solprop_native/core.hpp" #include "solprop_native/refuse.hpp" #include diff --git a/native/src/session_statistics.cpp b/native/src/session_statistics.cpp index 178e02252..4360e2c7d 100644 --- a/native/src/session_statistics.cpp +++ b/native/src/session_statistics.cpp @@ -3,8 +3,10 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/chebyshev.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/far_field.hpp" +#include "solprop_native/general_state.hpp" +#include "solprop_native/problem.hpp" #include "solprop_native/refuse.hpp" #include "solprop_native/statistics_plan.hpp" @@ -35,7 +37,7 @@ namespace { constexpr double nan = std::numeric_limits::quiet_NaN(); -// `GeneralState::statistics`' flags (core.hpp `StatisticsFlag`). +// `GeneralState::statistics`' flags (problem.hpp `StatisticsFlag`). constexpr std::uint64_t temporal_omega0 = statistics_omega0; constexpr std::uint64_t temporal_energy = statistics_energy; constexpr std::uint64_t temporal_energy_window = statistics_energy_window; diff --git a/native/src/session_widths.cpp b/native/src/session_widths.cpp index 80318cf9f..801db2180 100644 --- a/native/src/session_widths.cpp +++ b/native/src/session_widths.cpp @@ -16,7 +16,7 @@ #pragma GCC optimize("fp-contract=off") #endif -#include "solprop_native/core.hpp" +#include "solprop_native/general_state.hpp" #include "solprop_native/refuse.hpp" #include "solprop_native/statistics_plan.hpp" diff --git a/native/tests/native/execution.hpp b/native/tests/native/execution.hpp index cc95a94c9..e525d5435 100644 --- a/native/tests/native/execution.hpp +++ b/native/tests/native/execution.hpp @@ -9,7 +9,7 @@ // `_execution.builtin` returns it without a spin. Native code holds no values of its // own; a workspace refuses a problem without a record. -#include "solprop_native/core.hpp" +#include "solprop_native/cpu_execution.hpp" namespace solprop_native::test { diff --git a/native/tests/native/test_aperture.cpp b/native/tests/native/test_aperture.cpp index e1fcb1097..8d08b4f27 100644 --- a/native/tests/native/test_aperture.cpp +++ b/native/tests/native/test_aperture.cpp @@ -11,8 +11,9 @@ // bits; hard-edge membership is shared. #include "bounds.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/far_field.hpp" +#include "solprop_native/problem.hpp" #include #include diff --git a/native/tests/native/test_core.cpp b/native/tests/native/test_core.cpp index d359d5034..8d426325b 100644 --- a/native/tests/native/test_core.cpp +++ b/native/tests/native/test_core.cpp @@ -4,9 +4,12 @@ #include "bounds.hpp" #include "execution.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/gemm.hpp" +#include "solprop_native/general_state.hpp" #include "solprop_native/plasma.hpp" +#include "solprop_native/problem.hpp" #include #include diff --git a/native/tests/native/test_coupled_flow.cpp b/native/tests/native/test_coupled_flow.cpp index a2b1c9afd..99480b055 100644 --- a/native/tests/native/test_coupled_flow.cpp +++ b/native/tests/native/test_coupled_flow.cpp @@ -3,9 +3,11 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "execution.hpp" -#include "solprop_native/core.hpp" #include "solprop_native/coupled_flow.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/flow_functions.hpp" +#include "solprop_native/general_state.hpp" +#include "solprop_native/problem.hpp" #include diff --git a/native/tests/native/test_dispersive.cpp b/native/tests/native/test_dispersive.cpp index 121d3918a..c5aee5dee 100644 --- a/native/tests/native/test_dispersive.cpp +++ b/native/tests/native/test_dispersive.cpp @@ -3,8 +3,10 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "execution.hpp" -#include "solprop_native/core.hpp" #include "solprop_native/dispersive.hpp" +#include "solprop_native/execution.hpp" +#include "solprop_native/general_state.hpp" +#include "solprop_native/problem.hpp" #include #include diff --git a/native/tests/native/test_envelope_plasma.cpp b/native/tests/native/test_envelope_plasma.cpp index ceebd3acd..c53a7eabd 100644 --- a/native/tests/native/test_envelope_plasma.cpp +++ b/native/tests/native/test_envelope_plasma.cpp @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/core.hpp" #include "solprop_native/envelope_plasma.hpp" +#include "solprop_native/execution.hpp" #include #include diff --git a/native/tests/native/test_fft.cpp b/native/tests/native/test_fft.cpp index a78924958..707e390d9 100644 --- a/native/tests/native/test_fft.cpp +++ b/native/tests/native/test_fft.cpp @@ -2,7 +2,8 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/core.hpp" +#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/fft_plan.hpp" #include "solprop_native/raman_fft.hpp" diff --git a/native/tests/native/test_fp.cpp b/native/tests/native/test_fp.cpp index 70c76be3b..cfaa3819b 100644 --- a/native/tests/native/test_fp.cpp +++ b/native/tests/native/test_fp.cpp @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/core.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/fp.hpp" #include diff --git a/native/tests/native/test_general_modal.cpp b/native/tests/native/test_general_modal.cpp index 46f0e7bfc..b226bb520 100644 --- a/native/tests/native/test_general_modal.cpp +++ b/native/tests/native/test_general_modal.cpp @@ -3,8 +3,12 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "execution.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/general_modal.hpp" +#include "solprop_native/general_state.hpp" +#include "solprop_native/problem.hpp" +#include "solprop_native/readings.hpp" #include #include diff --git a/native/tests/native/test_z_table.cpp b/native/tests/native/test_z_table.cpp index 7bd5ad8d2..7c6d657ed 100644 --- a/native/tests/native/test_z_table.cpp +++ b/native/tests/native/test_z_table.cpp @@ -3,7 +3,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "solprop_native/chebyshev.hpp" -#include "solprop_native/core.hpp" +#include "solprop_native/execution.hpp" #include "solprop_native/stage_tables.hpp" #include "solprop_native/z_table.hpp" From 89a1ed59c9cd7d82ac224caa7fef21e4235d3812 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:00:04 +0000 Subject: [PATCH 05/21] Move the host FFTW policy and wisdom to fftw_wisdom.hpp/.cpp fft_plan.hpp held the plan policy, the workspace's planning and the wisdom calls inline beside the device row plans, so every includer, bindings.cpp among them, compiled KokkosFFT and fftw3.h for them. The host-only bodies (fftw_seed_effort, fftw_effort_flag, the wisdom export and import, FftPlanningScope, the seeds' bookkeeping, fft_plan_key, time_fft_plan) and seed_fftw_wisdom from general/kernels.hpp move verbatim to fftw_wisdom.cpp, declared in fftw_wisdom.hpp, which includes neither. detail::planning and the helpers only the seeds use become file-local. The row seed template stays in fft_plan.hpp, which needs fftw3.h and KokkosFFT for it. bindings.cpp now includes fftw_wisdom.hpp alone, and its wisdom-forget test hook calls the new forget_fftw_wisdom: no binding unit compiles KokkosFFT or fftw3.h. Plan-time code only. Co-Authored-By: Claude Opus 5.5 --- docs/dev/backends.md | 4 +- docs/dev/tooling.md | 2 +- native/CMakeLists.txt | 2 +- native/include/solprop_native/fft_plan.hpp | 431 +----------------- native/include/solprop_native/fftw_wisdom.hpp | 133 ++++++ native/src/bindings.cpp | 7 +- native/src/fftw_wisdom.cpp | 416 +++++++++++++++++ native/src/general/kernels.hpp | 37 -- native/src/solprop_native/_wisdom.py | 5 +- native/tests/native/test_fft.cpp | 1 + native/tests/test_fftw_plan_refusal.py | 2 +- 11 files changed, 561 insertions(+), 479 deletions(-) create mode 100644 native/include/solprop_native/fftw_wisdom.hpp create mode 100644 native/src/fftw_wisdom.cpp diff --git a/docs/dev/backends.md b/docs/dev/backends.md index 1accdf808..e680e034f 100644 --- a/docs/dev/backends.md +++ b/docs/dev/backends.md @@ -1449,7 +1449,7 @@ use separate FFT plans for the final partial chunk, aliased onto existing storag Kokkos-FFT plans at `FFTW_ESTIMATE` and consumes measured wisdom in the linked FFTW library, which the workspace seeds before it makes its first plan: a plan made before its wisdom stays at estimate for the run. One policy (`fftw_seed_effort`, -`include/solprop_native/fft_plan.hpp`) decides every seed by the transform's shape and +`include/solprop_native/fftw_wisdom.hpp`) decides every seed by the transform's shape and width. Unset, row transforms (temporal, modal, Raman and plasma) are measured on one thread and left at estimate when threaded, because measured wisdom can select slower threaded plans; free space's transverse planes are measured at any width. An explicit @@ -1486,7 +1486,7 @@ atomic rename; writers serialise on `.lock` (`flock` on POSIX), merge the plans with their own and replace it only when that adds or changes one. A calibration's own workspaces (given `fft_planning`) and its children (`off`) neither read nor write the store. The C++ side only converts wisdom to and from text (`export_fftw_wisdom`, -`import_fftw_wisdom` in `fft_plan.hpp`), registering FFTW's threaded solvers first, +`import_fftw_wisdom` in `fftw_wisdom.hpp`), registering FFTW's threaded solvers first, since they are part of the planner signature an import must match. Metadata records `wisdom_loaded` (the file held plans) and `wisdom_digest` (of the plans the workspace built from wisdom or measured). diff --git a/docs/dev/tooling.md b/docs/dev/tooling.md index f10f9aac0..04e8a9f43 100644 --- a/docs/dev/tooling.md +++ b/docs/dev/tooling.md @@ -243,7 +243,7 @@ to it. - `SOLPROP_THREADS`, the thread budget the native backend's pools stay within, and `SOLPROP_OVERSUBSCRIBE`, which lets an explicit count exceed the CPUs (ADR-052). - The native backend's plans and neighbours: `SOLPROP_FFTW_PLAN` (`estimate` in CI and - tests, `measure` by default, read by `native/include/solprop_native/fft_plan.hpp`), + tests, `measure` by default, read by `native/src/fftw_wisdom.cpp`), `SOLPROP_FFTW_WISDOM` (a file name, beside which the native backend keeps the plans its runs measure instead of the calibration store, ADR-076) and `SOLPROP_BLAS_THREADS` (the width of the other BLAS libraries while a native diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 615f8f94b..cd7bfcf0a 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -816,7 +816,7 @@ add_library(solprop_native_core STATIC src/cpu_execution.cpp src/crossing.cpp src/adams_march.cpp src/watch.cpp src/rk_march.cpp src/engines.cpp src/session.cpp src/session_statistics.cpp src/session_widths.cpp src/low_storage.cpp src/transverse.cpp src/session_edge.cpp src/dense.cpp src/workspace.cpp src/problem.cpp - src/reference_timing.cpp) + src/reference_timing.cpp src/fftw_wisdom.cpp) # solprop SEP 39, SEP 45 and SEP 47: the Adams march's decisions, the boundary watch's # predictions, the Runge-Kutta march's positions and norm, the low-storage replay's # positions and table keys, the native session's scheduling, export, statistics diff --git a/native/include/solprop_native/fft_plan.hpp b/native/include/solprop_native/fft_plan.hpp index 0d05eedf5..86940008e 100644 --- a/native/include/solprop_native/fft_plan.hpp +++ b/native/include/solprop_native/fft_plan.hpp @@ -6,6 +6,7 @@ #include "solprop_native/cpu_execution.hpp" #include "solprop_native/execution.hpp" +#include "solprop_native/fftw_wisdom.hpp" #include @@ -15,309 +16,11 @@ #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) #include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include #include -#include namespace solprop_native { -// The two kinds of batched transform the plan policy tells apart: one-dimensional rows -// (the temporal, modal, Raman and plasma transforms) and free space's two-dimensional -// transverse planes. -enum class FftShape { rows, planes }; - -// The one plan policy for the host FFTW plans: the effort at which a transform of this -// shape, run on `threads` threads, is seeded. Kokkos-FFT plans at FFTW_ESTIMATE and -// consumes the wisdom a higher-effort temporary plan of the same transform left, so -// FFTW_ESTIMATE here means "do not seed". An explicit SOLPROP_FFTW_PLAN applies to -// every transform at every width. Unset, it is `measure`, except for threaded rows, -// which stay at ESTIMATE: there measured planning chose a slower threaded decomposition -// (solprop's docs/dev/benchmarks.md, "Threaded FFT plan selection"). Transverse planes -// are measured at any width (solprop SEP 28: 3.2-3.4 times faster than ESTIMATE). An -// empty value is unset, as for the thread variables; any other is refused, since a typo -// would otherwise change a run's plans in silence. -inline unsigned fftw_seed_effort(const FftShape shape, const int threads) { - const char* text = std::getenv("SOLPROP_FFTW_PLAN"); - if (text == nullptr || *text == '\0') - return shape == FftShape::rows && threads > 1 ? FFTW_ESTIMATE : FFTW_MEASURE; - const std::string_view effort = text; - if (effort == "estimate") - return FFTW_ESTIMATE; - if (effort == "measure") - return FFTW_MEASURE; - if (effort == "patient") - return FFTW_PATIENT; - throw std::invalid_argument("SOLPROP_FFTW_PLAN must be estimate, measure or patient, " - "and is '" + - std::string(effort) + "'"); -} - -// An effort's FFTW flag. -inline unsigned fftw_effort_flag(const std::string_view effort) { - if (effort == "estimate") - return FFTW_ESTIMATE; - if (effort == "measure") - return FFTW_MEASURE; - if (effort == "patient") - return FFTW_PATIENT; - throw std::invalid_argument("an FFTW effort is estimate, measure or patient, not '" + - std::string(effort) + "'"); -} - -// FFTW's process-wide wisdom as text, for the stored plans (solprop ADR-076) and a -// workspace's planning: the Python adapter owns the file, its key and its lock, and -// calls these with the GIL held, as planning does, so they never race the planner. -// Empty when the provider implements FFTW's API without its wisdom (MKL). Both first -// register the threaded solvers, as every seed does: they are part of the planner's -// configuration, whose signature heads exported wisdom and must match for an import. -inline std::string export_fftw_wisdom() { - fftw_init_threads(); - // FFTW allocates the text with plain malloc and documents free() for it: fftw_free - // pairs with fftw_malloc's aligned allocator, which on Windows is another heap. - const auto release = [](char* pointer) { std::free(pointer); }; - const std::unique_ptr text(fftw_export_wisdom_to_string(), - release); - return text ? std::string(text.get()) : std::string(); -} - -// Merge `text` into the process's wisdom. FFTW refuses, leaving its wisdom as it was, -// text that is not wisdom or that another FFTW version or configuration wrote (its -// preamble carries the planner's signature). Wisdom only chooses among FFTW's exact -// algorithms, so a plan taken from it computes the same transform. -inline bool import_fftw_wisdom(const std::string& text) { - fftw_init_threads(); - return fftw_import_wisdom_from_string(text.c_str()) != 0; -} - -namespace detail { -// The planning of the workspace this thread is building (`FftPlanningScope`). -inline thread_local FftPlanning* planning = nullptr; -} // namespace detail - -/** While it lives, the calling thread's seeds follow `planning` (solprop SEP 46). In - profile and capture modes FFTW's process-wide wisdom is forgotten at the start and - at the end, so that no wisdom from an earlier workspace, a file or another - candidate reaches this one, and none of this one's outlives it. Planning holds the - GIL, which serialises FFTW's planner. */ -class FftPlanningScope { -public: - explicit FftPlanningScope(FftPlanning& planning) - : saved_(detail::planning), forgets_(planning.mode != FftPlanning::Mode::legacy) { - detail::planning = &planning; - if (forgets_) { - fftw_init_threads(); - fftw_forget_wisdom(); - } - } - ~FftPlanningScope() { - if (forgets_) - fftw_forget_wisdom(); - detail::planning = saved_; - } - FftPlanningScope(const FftPlanningScope&) = delete; - FftPlanningScope& operator=(const FftPlanningScope&) = delete; - FftPlanningScope(FftPlanningScope&&) = delete; - FftPlanningScope& operator=(FftPlanningScope&&) = delete; - -private: - FftPlanning* saved_; - bool forgets_; -}; - -// Whether `planned` holds a wisdom entry `stored` lacks: a seed that measured. FFTW -// writes one entry a line, under a preamble line that both share. -inline bool adds_wisdom(const std::string& planned, const std::string& stored) { - const auto lines = [](const std::string& text) { - std::vector found; - for (std::size_t first = 0; first < text.size();) { - const std::size_t last = std::min(text.find('\n', first), text.size()); - found.emplace_back(text.data() + first, last - first); - first = last + 1; - } - std::ranges::sort(found); - return found; - }; - const auto had = lines(stored); - return std::ranges::any_of(lines(planned), [&had](const auto line) { - return !std::ranges::binary_search(had, line); - }); -} - -// Merge `text` beneath the process's wisdom: where both hold an entry, the process's -// stays, so that a profile's plans keep theirs. -inline void import_beneath(const std::string& text) { - const std::string held = export_fftw_wisdom(); - fftw_forget_wisdom(); - static_cast(fftw_import_wisdom_from_string(text.c_str())); - if (!held.empty()) - static_cast(fftw_import_wisdom_from_string(held.c_str())); -} - -// The wisdom already known for a transform: measured earlier in this workspace, or -// stored by an earlier run; none when it must be measured. -inline const std::string* known_wisdom(const FftPlanning& planning, - const std::string& key) { - if (const auto found = planning.measured.find(key); found != planning.measured.end()) - return &found->second; - if (const auto found = planning.stored.find(key); found != planning.stored.end()) - return &found->second; - return nullptr; -} - -// Begin planning a transform alone (`FftPlanning::stored`): set the workspace's wisdom -// aside and import only the transform's own known wisdom; a transform to be measured -// starts from what this workspace measured so far, so that it shares their sub-plans -// as a workspace planning in one wisdom does, and from nothing else. -inline void plan_alone(FftPlanning& planning, const std::string& key) { - planning.held = export_fftw_wisdom(); - fftw_forget_wisdom(); - if (const std::string* known = known_wisdom(planning, key)) - static_cast(fftw_import_wisdom_from_string(known->c_str())); - else - for (const auto& entry : planning.measured) - static_cast(fftw_import_wisdom_from_string(entry.second.c_str())); - planning.alone = true; -} - -// End a transform planned alone: keep its own wisdom if it measured, note a reuse of a -// stored plan if not, then give the workspace back its wisdom, imported over the -// transform's so that a profile's plans keep their entries. Every seed begun alone -// ends here, planned or not. -inline void release_seed(const std::string& key, const bool planned) { - FftPlanning* planning = detail::planning; - if (planning == nullptr || !planning->alone) - return; - planning->alone = false; - std::string own; - if (planned) { - own = export_fftw_wisdom(); - const std::string* known = known_wisdom(*planning, key); - if (known != nullptr && !adds_wisdom(own, *known)) { - if (!planning->measured.contains(key)) - planning->reused.push_back(key); - } else if (planning->keep) { - planning->measured[key] = own; - } - } - fftw_forget_wisdom(); - if (!own.empty()) - static_cast(fftw_import_wisdom_from_string(own.c_str())); - if (!planning->held.empty()) - static_cast(fftw_import_wisdom_from_string(planning->held.c_str())); - planning->held.clear(); -} - -// The flags a transform of `key` and `shape` on `threads` threads is seeded with, or -// none to leave it to Kokkos-FFT's estimate plan: the policy, or the workspace's -// planning (`FftPlanning`), which this records the transform in. Capture forgets the -// wisdom before the seed and sets the planner's time limit; profile imports the -// transform's own wisdom. A seed above estimate that the stored plans hold, or any -// that are kept, plans alone (`plan_alone`); a transform left at estimate that they -// hold is not seeded, its stored plan imported for Kokkos-FFT's estimate plan to take, -// as FFTW takes wisdom of any effort for an estimate plan. -inline std::optional begin_seed(const std::string& key, const FftShape shape, - const int threads) { - FftPlanning* planning = detail::planning; - if (planning == nullptr) { - const unsigned effort = fftw_seed_effort(shape, threads); - if (effort == FFTW_ESTIMATE) - return std::nullopt; - return effort; - } - unsigned effort = FFTW_ESTIMATE; - if (planning->mode == FftPlanning::Mode::legacy) { - effort = fftw_seed_effort(shape, threads); - } else { - const char* group = shape == FftShape::planes ? "planes" - : threads > 1 ? "threaded_rows" - : "rows"; - planning->groups[key] = group; - if (planning->mode == FftPlanning::Mode::capture) { - const unsigned captured = fftw_effort_flag(planning->capture_effort); - if (captured == FFTW_ESTIMATE) - return std::nullopt; - fftw_forget_wisdom(); - fftw_set_timelimit(planning->time_limit > 0.0 ? planning->time_limit - : FFTW_NO_TIMELIMIT); - return captured; - } - const std::string& chosen = shape == FftShape::planes ? planning->planes - : threads > 1 ? planning->threaded_rows - : planning->rows; - effort = fftw_effort_flag(chosen); - if (const auto found = planning->wisdom.find(key); - found != planning->wisdom.end()) { - if (fftw_import_wisdom_from_string(found->second.c_str()) == 0) { - planning->missing.push_back(key); - return std::nullopt; - } - return effort | FFTW_WISDOM_ONLY; - } - } - const auto stored = planning->stored.find(key); - if (effort == FFTW_ESTIMATE) { - if (stored != planning->stored.end()) { - import_beneath(stored->second); - planning->reused.push_back(key); - } - return std::nullopt; - } - if (planning->keep || known_wisdom(*planning, key) != nullptr) - plan_alone(*planning, key); - return effort; -} - -// Finish a seed begun with `flags`: a seed planned alone gives the workspace its -// wisdom back (`release_seed`); a capture keeps the transform's wisdom and lifts the -// time limit; a profile's plan that no longer builds from its wisdom is recorded, and -// left to the estimate plan. Any other failure to plan is an error. -inline void end_seed(const std::string& key, const unsigned flags, const bool planned) { - FftPlanning* planning = detail::planning; - const bool capture = - planning != nullptr && planning->mode == FftPlanning::Mode::capture; - if (capture) - fftw_set_timelimit(FFTW_NO_TIMELIMIT); - release_seed(key, planned); - if (!planned) { - if ((flags & FFTW_WISDOM_ONLY) != 0U && planning != nullptr) { - planning->missing.push_back(key); - return; - } - throw std::runtime_error("FFTW could not plan the transform " + key); - } - if (capture) { - const auto release = [](char* pointer) { std::free(pointer); }; - const std::unique_ptr text(fftw_export_wisdom_to_string(), - release); - planning->captured[key] = text ? std::string(text.get()) : std::string(); - } -} - -// A transform's identity for its wisdom: its kind, the sizes and distances, in- or -// out-of-place, the arrays' alignment and the threads (solprop SEP 46 section 6.7). -inline std::string fft_plan_key(const std::string_view kind, const std::string& sizes, - const int howmany, const int input_distance, - const int output_distance, const void* input, - const void* output, const int threads) { - const auto alignment = [](const void* pointer) { - return std::to_string( - fftw_alignment_of(static_cast(const_cast(pointer)))); - }; - return std::string(kind) + ":" + sizes + ":" + std::to_string(howmany) + ":" + - std::to_string(input_distance) + ":" + std::to_string(output_distance) + ":" + - (input == output ? "in" : "out") + ":" + alignment(input) + ":" + - alignment(output) + ":t" + std::to_string(threads); -} - // Seed only last-axis, contiguous, out-of-place row transforms. Planning may // destroy BOTH buffers: callers must invoke this before filling working data, and // before the transform's own Kokkos-FFT plan exists, which would otherwise keep @@ -377,138 +80,6 @@ void seed_row_fft_wisdom(const Execution& execution, const Input& input, fftw_destroy_plan(seeded); } -/** A plan's execution time alone (solprop issue #595): the seconds of one execution - of the transform a plan key names (`fft_plan_key`), planned from `wisdom` at - `effort` with `FFTW_WISDOM_ONLY` (`plan`; NaN when the wisdom no longer builds - it), and of the same transform at FFTW_ESTIMATE (`estimate`), each the median of - five batches after a warm execution, on the key's threads and on arrays of its - alignment, inside a native scope (`CpuTeamScope`) as a step's transforms run, so - that idle threads wait as they do there. FFTW's wisdom is forgotten before each - plan and after. */ -struct PlanAlone { - double plan{}, estimate{}; -}; - -namespace detail { -inline std::vector split_key(const std::string& text, const char by) { - std::vector parts; - std::size_t start = 0; - for (std::size_t at = text.find(by); at != std::string::npos; - start = at + 1, at = text.find(by, start)) - parts.push_back(text.substr(start, at - start)); - parts.push_back(text.substr(start)); - return parts; -} - -inline double median_execution(fftw_plan plan) { - using Clock = std::chrono::steady_clock; - fftw_execute(plan); - const auto first = Clock::now(); - fftw_execute(plan); - const double once = std::chrono::duration(Clock::now() - first).count(); - const int repeats = std::max(2, static_cast(0.005 / std::max(once, 1e-7))); - std::array batches{}; - for (double& batch : batches) { - const auto start = Clock::now(); - for (int repeat = 0; repeat < repeats; ++repeat) - fftw_execute(plan); - batch = std::chrono::duration(Clock::now() - start).count() / repeats; - } - std::ranges::sort(batches); - return batches[2]; -} -} // namespace detail - -inline PlanAlone time_fft_plan(const std::string& key, const std::string& wisdom, - const std::string& effort) { - const auto parts = detail::split_key(key, ':'); - if (parts.size() != 9 || parts[8].empty() || parts[8][0] != 't') - throw std::invalid_argument("not an FFT plan key: " + key); - const std::string& kind = parts[0]; - std::vector extents; - for (const auto& extent : detail::split_key(parts[1], 'x')) - extents.push_back(std::stoi(extent)); - const int howmany = std::stoi(parts[2]); - const int input_distance = std::stoi(parts[3]); - const int output_distance = std::stoi(parts[4]); - const bool in_place = parts[5] == "in"; - const auto input_offset = static_cast(std::stoi(parts[6])); - const auto output_offset = static_cast(std::stoi(parts[7])); - const int threads = std::stoi(parts[8].substr(1)); - // The step's waiting policy and FFTW team: without the scope LLVM's runtime sleeps - // its idle threads at once (`KMP_BLOCKTIME=0`) and each loop pays their wake-up. - const CpuTeamScope scope(threads); - const bool r2c = kind == "r2c"; - const bool c2r = kind == "c2r"; - const bool planes = kind.starts_with("planes-"); - const int sign = kind.ends_with("backward") ? FFTW_BACKWARD : FFTW_FORWARD; - if (!r2c && !c2r && !planes && !kind.starts_with("c2c-")) - throw std::invalid_argument("not an FFT plan key: " + key); - const auto rank = static_cast(extents.size()); - // Real input or output counts doubles, complex counts double pairs. - const std::size_t input_bytes = static_cast(howmany) * - static_cast(input_distance) * - (r2c ? sizeof(double) : sizeof(fftw_complex)); - const std::size_t output_bytes = static_cast(howmany) * - static_cast(output_distance) * - (c2r ? sizeof(double) : sizeof(fftw_complex)); - fftw_init_threads(); - const auto release = [](void* pointer) { fftw_free(pointer); }; - const std::unique_ptr input_memory( - fftw_malloc(std::max(input_bytes, output_bytes) + 64), release); - const std::unique_ptr output_memory( - in_place ? nullptr : fftw_malloc(output_bytes + 64), release); - char* input_base = static_cast(input_memory.get()) + input_offset; - char* output_base = - in_place ? input_base : static_cast(output_memory.get()) + output_offset; - const auto make = [&](const unsigned flags) -> fftw_plan { - fftw_plan_with_nthreads(threads); - if (r2c) - return fftw_plan_many_dft_r2c( - rank, extents.data(), howmany, reinterpret_cast(input_base), nullptr, - 1, input_distance, reinterpret_cast(output_base), nullptr, 1, - output_distance, flags); - if (c2r) - return fftw_plan_many_dft_c2r( - rank, extents.data(), howmany, reinterpret_cast(input_base), - nullptr, 1, input_distance, reinterpret_cast(output_base), nullptr, - 1, output_distance, flags); - return fftw_plan_many_dft( - rank, extents.data(), howmany, reinterpret_cast(input_base), - nullptr, 1, input_distance, reinterpret_cast(output_base), - nullptr, 1, output_distance, sign, flags); - }; - const auto timed = [&](const unsigned flags) { - fftw_forget_wisdom(); - if ((flags & FFTW_WISDOM_ONLY) != 0U && - fftw_import_wisdom_from_string(wisdom.c_str()) == 0) - return std::numeric_limits::quiet_NaN(); - auto* const plan = make(flags); - if (plan == nullptr) { - fftw_forget_wisdom(); - return std::numeric_limits::quiet_NaN(); - } - // Real input is overwritten by complex-to-real plans and harmless otherwise. - std::memset(input_base, 0, input_bytes); - const double seconds = detail::median_execution(plan); - fftw_destroy_plan(plan); - fftw_forget_wisdom(); - return seconds; - }; - PlanAlone result; - result.plan = timed(fftw_effort_flag(effort) | FFTW_WISDOM_ONLY); - result.estimate = timed(FFTW_ESTIMATE); - return result; -} - -} // namespace solprop_native -#else -namespace solprop_native { -// Device builds plan with cuFFT or rocFFT, which keep no wisdom. -class FftPlanningScope { -public: - explicit FftPlanningScope(FftPlanning& /*planning*/) noexcept {} -}; } // namespace solprop_native #endif diff --git a/native/include/solprop_native/fftw_wisdom.hpp b/native/include/solprop_native/fftw_wisdom.hpp new file mode 100644 index 000000000..46cd93994 --- /dev/null +++ b/native/include/solprop_native/fftw_wisdom.hpp @@ -0,0 +1,133 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/execution.hpp" + +/* + * The host FFTW plan policy and wisdom (solprop SEP 46, ADR-076): the effort a + * transform is seeded at, the workspace's planning and the process's wisdom as text. + * Plan-time code only. fftw3.h stays in fftw_wisdom.cpp and in fft_plan.hpp, which + * seeds row transforms through these. + */ +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +#include +#include +#include +#include + +namespace solprop_native { + +// The two kinds of batched transform the plan policy tells apart: one-dimensional rows +// (the temporal, modal, Raman and plasma transforms) and free space's two-dimensional +// transverse planes. +enum class FftShape { rows, planes }; + +// The one plan policy for the host FFTW plans: the effort at which a transform of this +// shape, run on `threads` threads, is seeded. Kokkos-FFT plans at FFTW_ESTIMATE and +// consumes the wisdom a higher-effort temporary plan of the same transform left, so +// FFTW_ESTIMATE here means "do not seed". An explicit SOLPROP_FFTW_PLAN applies to +// every transform at every width. Unset, it is `measure`, except for threaded rows, +// which stay at ESTIMATE: there measured planning chose a slower threaded decomposition +// (solprop's docs/dev/benchmarks.md, "Threaded FFT plan selection"). Transverse planes +// are measured at any width (solprop SEP 28: 3.2-3.4 times faster than ESTIMATE). An +// empty value is unset, as for the thread variables; any other is refused, since a typo +// would otherwise change a run's plans in silence. +unsigned fftw_seed_effort(FftShape shape, int threads); + +// An effort's FFTW flag. +unsigned fftw_effort_flag(std::string_view effort); + +// FFTW's process-wide wisdom as text, for the stored plans (solprop ADR-076) and a +// workspace's planning: the Python adapter owns the file, its key and its lock, and +// calls these with the GIL held, as planning does, so they never race the planner. +// Empty when the provider implements FFTW's API without its wisdom (MKL). Both first +// register the threaded solvers, as every seed does: they are part of the planner's +// configuration, whose signature heads exported wisdom and must match for an import. +std::string export_fftw_wisdom(); + +// Merge `text` into the process's wisdom. FFTW refuses, leaving its wisdom as it was, +// text that is not wisdom or that another FFTW version or configuration wrote (its +// preamble carries the planner's signature). Wisdom only chooses among FFTW's exact +// algorithms, so a plan taken from it computes the same transform. +bool import_fftw_wisdom(const std::string& text); + +// Forget the process's wisdom, the threaded solvers registered first as above. +void forget_fftw_wisdom(); + +/** While it lives, the calling thread's seeds follow `planning` (solprop SEP 46). In + profile and capture modes FFTW's process-wide wisdom is forgotten at the start and + at the end, so that no wisdom from an earlier workspace, a file or another + candidate reaches this one, and none of this one's outlives it. Planning holds the + GIL, which serialises FFTW's planner. */ +class FftPlanningScope { +public: + explicit FftPlanningScope(FftPlanning& planning); + ~FftPlanningScope(); + FftPlanningScope(const FftPlanningScope&) = delete; + FftPlanningScope& operator=(const FftPlanningScope&) = delete; + FftPlanningScope(FftPlanningScope&&) = delete; + FftPlanningScope& operator=(FftPlanningScope&&) = delete; + +private: + FftPlanning* saved_; + bool forgets_; +}; + +// The flags a transform of `key` and `shape` on `threads` threads is seeded with, or +// none to leave it to Kokkos-FFT's estimate plan: the policy, or the workspace's +// planning (`FftPlanning`), which this records the transform in. Capture forgets the +// wisdom before the seed and sets the planner's time limit; profile imports the +// transform's own wisdom. A seed above estimate that the stored plans hold, or any +// that are kept, plans alone (`plan_alone`); a transform left at estimate that they +// hold is not seeded, its stored plan imported for Kokkos-FFT's estimate plan to take, +// as FFTW takes wisdom of any effort for an estimate plan. +std::optional begin_seed(const std::string& key, FftShape shape, int threads); + +// Finish a seed begun with `flags`: a seed planned alone gives the workspace its +// wisdom back (`release_seed`); a capture keeps the transform's wisdom and lifts the +// time limit; a profile's plan that no longer builds from its wisdom is recorded, and +// left to the estimate plan. Any other failure to plan is an error. +void end_seed(const std::string& key, unsigned flags, bool planned); + +// A transform's identity for its wisdom: its kind, the sizes and distances, in- or +// out-of-place, the arrays' alignment and the threads (solprop SEP 46 section 6.7). +std::string fft_plan_key(std::string_view kind, const std::string& sizes, int howmany, + int input_distance, int output_distance, const void* input, + const void* output, int threads); + +/** A plan's execution time alone (solprop issue #595): the seconds of one execution + of the transform a plan key names (`fft_plan_key`), planned from `wisdom` at + `effort` with `FFTW_WISDOM_ONLY` (`plan`; NaN when the wisdom no longer builds + it), and of the same transform at FFTW_ESTIMATE (`estimate`), each the median of + five batches after a warm execution, on the key's threads and on arrays of its + alignment, inside a native scope (`CpuTeamScope`) as a step's transforms run, so + that idle threads wait as they do there. FFTW's wisdom is forgotten before each + plan and after. */ +struct PlanAlone { + double plan{}, estimate{}; +}; + +PlanAlone time_fft_plan(const std::string& key, const std::string& wisdom, + const std::string& effort); + +// Plan, then discard, a batched complex transform of transverse planes at the plan +// policy's effort, leaving its wisdom. It overwrites both arrays; the caller clears +// them. Row transforms are seeded by `seed_row_fft_wisdom` (fft_plan.hpp). +void seed_fftw_wisdom(const ExecutionSpace& space, Complex* input, Complex* output, + const std::vector& extents, int howmany, int input_distance, + int output_distance, int sign); + +} // namespace solprop_native +#else +namespace solprop_native { +// Device builds plan with cuFFT or rocFFT, which keep no wisdom. +class FftPlanningScope { +public: + explicit FftPlanningScope(FftPlanning& /*planning*/) noexcept {} +}; +} // namespace solprop_native +#endif diff --git a/native/src/bindings.cpp b/native/src/bindings.cpp index b9cbb354a..1420ef669 100644 --- a/native/src/bindings.cpp +++ b/native/src/bindings.cpp @@ -11,7 +11,7 @@ #include "solprop_native/dense_output.hpp" #include "solprop_native/engines.hpp" #include "solprop_native/execution.hpp" -#include "solprop_native/fft_plan.hpp" +#include "solprop_native/fftw_wisdom.hpp" #include "solprop_native/fields.hpp" #include "solprop_native/general_state.hpp" #include "solprop_native/host_source.hpp" @@ -2749,10 +2749,7 @@ void bind_testing(nb::module_& testing) { #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) // solprop SEP 46: forget FFTW's process-wide wisdom, for a test of the planning // transaction. - testing.def("forget_fftw_wisdom", [] { - fftw_init_threads(); - fftw_forget_wisdom(); - }); + testing.def("forget_fftw_wisdom", [] { native::forget_fftw_wisdom(); }); #else testing.def("forget_fftw_wisdom", [] {}); #endif diff --git a/native/src/fftw_wisdom.cpp b/native/src/fftw_wisdom.cpp new file mode 100644 index 000000000..eb2943536 --- /dev/null +++ b/native/src/fftw_wisdom.cpp @@ -0,0 +1,416 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "solprop_native/fftw_wisdom.hpp" + +#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/execution.hpp" + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace solprop_native { + +unsigned fftw_seed_effort(const FftShape shape, const int threads) { + const char* text = std::getenv("SOLPROP_FFTW_PLAN"); + if (text == nullptr || *text == '\0') + return shape == FftShape::rows && threads > 1 ? FFTW_ESTIMATE : FFTW_MEASURE; + const std::string_view effort = text; + if (effort == "estimate") + return FFTW_ESTIMATE; + if (effort == "measure") + return FFTW_MEASURE; + if (effort == "patient") + return FFTW_PATIENT; + throw std::invalid_argument("SOLPROP_FFTW_PLAN must be estimate, measure or patient, " + "and is '" + + std::string(effort) + "'"); +} + +unsigned fftw_effort_flag(const std::string_view effort) { + if (effort == "estimate") + return FFTW_ESTIMATE; + if (effort == "measure") + return FFTW_MEASURE; + if (effort == "patient") + return FFTW_PATIENT; + throw std::invalid_argument("an FFTW effort is estimate, measure or patient, not '" + + std::string(effort) + "'"); +} + +std::string export_fftw_wisdom() { + fftw_init_threads(); + // FFTW allocates the text with plain malloc and documents free() for it: fftw_free + // pairs with fftw_malloc's aligned allocator, which on Windows is another heap. + const auto release = [](char* pointer) { std::free(pointer); }; + const std::unique_ptr text(fftw_export_wisdom_to_string(), + release); + return text ? std::string(text.get()) : std::string(); +} + +bool import_fftw_wisdom(const std::string& text) { + fftw_init_threads(); + return fftw_import_wisdom_from_string(text.c_str()) != 0; +} + +void forget_fftw_wisdom() { + fftw_init_threads(); + fftw_forget_wisdom(); +} + +namespace detail { +namespace { +// The planning of the workspace this thread is building (`FftPlanningScope`). +thread_local FftPlanning* planning = nullptr; +} // namespace +} // namespace detail + +namespace { +// Whether `planned` holds a wisdom entry `stored` lacks: a seed that measured. FFTW +// writes one entry a line, under a preamble line that both share. +bool adds_wisdom(const std::string& planned, const std::string& stored) { + const auto lines = [](const std::string& text) { + std::vector found; + for (std::size_t first = 0; first < text.size();) { + const std::size_t last = std::min(text.find('\n', first), text.size()); + found.emplace_back(text.data() + first, last - first); + first = last + 1; + } + std::ranges::sort(found); + return found; + }; + const auto had = lines(stored); + return std::ranges::any_of(lines(planned), [&had](const auto line) { + return !std::ranges::binary_search(had, line); + }); +} + +// Merge `text` beneath the process's wisdom: where both hold an entry, the process's +// stays, so that a profile's plans keep theirs. +void import_beneath(const std::string& text) { + const std::string held = export_fftw_wisdom(); + fftw_forget_wisdom(); + static_cast(fftw_import_wisdom_from_string(text.c_str())); + if (!held.empty()) + static_cast(fftw_import_wisdom_from_string(held.c_str())); +} + +// The wisdom already known for a transform: measured earlier in this workspace, or +// stored by an earlier run; none when it must be measured. +const std::string* known_wisdom(const FftPlanning& planning, const std::string& key) { + if (const auto found = planning.measured.find(key); found != planning.measured.end()) + return &found->second; + if (const auto found = planning.stored.find(key); found != planning.stored.end()) + return &found->second; + return nullptr; +} + +// Begin planning a transform alone (`FftPlanning::stored`): set the workspace's wisdom +// aside and import only the transform's own known wisdom; a transform to be measured +// starts from what this workspace measured so far, so that it shares their sub-plans +// as a workspace planning in one wisdom does, and from nothing else. +void plan_alone(FftPlanning& planning, const std::string& key) { + planning.held = export_fftw_wisdom(); + fftw_forget_wisdom(); + if (const std::string* known = known_wisdom(planning, key)) + static_cast(fftw_import_wisdom_from_string(known->c_str())); + else + for (const auto& entry : planning.measured) + static_cast(fftw_import_wisdom_from_string(entry.second.c_str())); + planning.alone = true; +} + +// End a transform planned alone: keep its own wisdom if it measured, note a reuse of a +// stored plan if not, then give the workspace back its wisdom, imported over the +// transform's so that a profile's plans keep their entries. Every seed begun alone +// ends here, planned or not. +void release_seed(const std::string& key, const bool planned) { + FftPlanning* planning = detail::planning; + if (planning == nullptr || !planning->alone) + return; + planning->alone = false; + std::string own; + if (planned) { + own = export_fftw_wisdom(); + const std::string* known = known_wisdom(*planning, key); + if (known != nullptr && !adds_wisdom(own, *known)) { + if (!planning->measured.contains(key)) + planning->reused.push_back(key); + } else if (planning->keep) { + planning->measured[key] = own; + } + } + fftw_forget_wisdom(); + if (!own.empty()) + static_cast(fftw_import_wisdom_from_string(own.c_str())); + if (!planning->held.empty()) + static_cast(fftw_import_wisdom_from_string(planning->held.c_str())); + planning->held.clear(); +} + +} // namespace + +FftPlanningScope::FftPlanningScope(FftPlanning& planning) + : saved_(detail::planning), forgets_(planning.mode != FftPlanning::Mode::legacy) { + detail::planning = &planning; + if (forgets_) { + fftw_init_threads(); + fftw_forget_wisdom(); + } +} + +FftPlanningScope::~FftPlanningScope() { + if (forgets_) + fftw_forget_wisdom(); + detail::planning = saved_; +} + +std::optional begin_seed(const std::string& key, const FftShape shape, + const int threads) { + FftPlanning* planning = detail::planning; + if (planning == nullptr) { + const unsigned effort = fftw_seed_effort(shape, threads); + if (effort == FFTW_ESTIMATE) + return std::nullopt; + return effort; + } + unsigned effort = FFTW_ESTIMATE; + if (planning->mode == FftPlanning::Mode::legacy) { + effort = fftw_seed_effort(shape, threads); + } else { + const char* group = shape == FftShape::planes ? "planes" + : threads > 1 ? "threaded_rows" + : "rows"; + planning->groups[key] = group; + if (planning->mode == FftPlanning::Mode::capture) { + const unsigned captured = fftw_effort_flag(planning->capture_effort); + if (captured == FFTW_ESTIMATE) + return std::nullopt; + fftw_forget_wisdom(); + fftw_set_timelimit(planning->time_limit > 0.0 ? planning->time_limit + : FFTW_NO_TIMELIMIT); + return captured; + } + const std::string& chosen = shape == FftShape::planes ? planning->planes + : threads > 1 ? planning->threaded_rows + : planning->rows; + effort = fftw_effort_flag(chosen); + if (const auto found = planning->wisdom.find(key); + found != planning->wisdom.end()) { + if (fftw_import_wisdom_from_string(found->second.c_str()) == 0) { + planning->missing.push_back(key); + return std::nullopt; + } + return effort | FFTW_WISDOM_ONLY; + } + } + const auto stored = planning->stored.find(key); + if (effort == FFTW_ESTIMATE) { + if (stored != planning->stored.end()) { + import_beneath(stored->second); + planning->reused.push_back(key); + } + return std::nullopt; + } + if (planning->keep || known_wisdom(*planning, key) != nullptr) + plan_alone(*planning, key); + return effort; +} + +void end_seed(const std::string& key, const unsigned flags, const bool planned) { + FftPlanning* planning = detail::planning; + const bool capture = + planning != nullptr && planning->mode == FftPlanning::Mode::capture; + if (capture) + fftw_set_timelimit(FFTW_NO_TIMELIMIT); + release_seed(key, planned); + if (!planned) { + if ((flags & FFTW_WISDOM_ONLY) != 0U && planning != nullptr) { + planning->missing.push_back(key); + return; + } + throw std::runtime_error("FFTW could not plan the transform " + key); + } + if (capture) { + const auto release = [](char* pointer) { std::free(pointer); }; + const std::unique_ptr text(fftw_export_wisdom_to_string(), + release); + planning->captured[key] = text ? std::string(text.get()) : std::string(); + } +} + +std::string fft_plan_key(const std::string_view kind, const std::string& sizes, + const int howmany, const int input_distance, + const int output_distance, const void* input, + const void* output, const int threads) { + const auto alignment = [](const void* pointer) { + return std::to_string( + fftw_alignment_of(static_cast(const_cast(pointer)))); + }; + return std::string(kind) + ":" + sizes + ":" + std::to_string(howmany) + ":" + + std::to_string(input_distance) + ":" + std::to_string(output_distance) + ":" + + (input == output ? "in" : "out") + ":" + alignment(input) + ":" + + alignment(output) + ":t" + std::to_string(threads); +} + +namespace { +std::vector split_key(const std::string& text, const char by) { + std::vector parts; + std::size_t start = 0; + for (std::size_t at = text.find(by); at != std::string::npos; + start = at + 1, at = text.find(by, start)) + parts.push_back(text.substr(start, at - start)); + parts.push_back(text.substr(start)); + return parts; +} + +double median_execution(fftw_plan plan) { + using Clock = std::chrono::steady_clock; + fftw_execute(plan); + const auto first = Clock::now(); + fftw_execute(plan); + const double once = std::chrono::duration(Clock::now() - first).count(); + const int repeats = std::max(2, static_cast(0.005 / std::max(once, 1e-7))); + std::array batches{}; + for (double& batch : batches) { + const auto start = Clock::now(); + for (int repeat = 0; repeat < repeats; ++repeat) + fftw_execute(plan); + batch = std::chrono::duration(Clock::now() - start).count() / repeats; + } + std::ranges::sort(batches); + return batches[2]; +} +} // namespace + +PlanAlone time_fft_plan(const std::string& key, const std::string& wisdom, + const std::string& effort) { + const auto parts = split_key(key, ':'); + if (parts.size() != 9 || parts[8].empty() || parts[8][0] != 't') + throw std::invalid_argument("not an FFT plan key: " + key); + const std::string& kind = parts[0]; + std::vector extents; + for (const auto& extent : split_key(parts[1], 'x')) + extents.push_back(std::stoi(extent)); + const int howmany = std::stoi(parts[2]); + const int input_distance = std::stoi(parts[3]); + const int output_distance = std::stoi(parts[4]); + const bool in_place = parts[5] == "in"; + const auto input_offset = static_cast(std::stoi(parts[6])); + const auto output_offset = static_cast(std::stoi(parts[7])); + const int threads = std::stoi(parts[8].substr(1)); + // The step's waiting policy and FFTW team: without the scope LLVM's runtime sleeps + // its idle threads at once (`KMP_BLOCKTIME=0`) and each loop pays their wake-up. + const CpuTeamScope scope(threads); + const bool r2c = kind == "r2c"; + const bool c2r = kind == "c2r"; + const bool planes = kind.starts_with("planes-"); + const int sign = kind.ends_with("backward") ? FFTW_BACKWARD : FFTW_FORWARD; + if (!r2c && !c2r && !planes && !kind.starts_with("c2c-")) + throw std::invalid_argument("not an FFT plan key: " + key); + const auto rank = static_cast(extents.size()); + // Real input or output counts doubles, complex counts double pairs. + const std::size_t input_bytes = static_cast(howmany) * + static_cast(input_distance) * + (r2c ? sizeof(double) : sizeof(fftw_complex)); + const std::size_t output_bytes = static_cast(howmany) * + static_cast(output_distance) * + (c2r ? sizeof(double) : sizeof(fftw_complex)); + fftw_init_threads(); + const auto release = [](void* pointer) { fftw_free(pointer); }; + const std::unique_ptr input_memory( + fftw_malloc(std::max(input_bytes, output_bytes) + 64), release); + const std::unique_ptr output_memory( + in_place ? nullptr : fftw_malloc(output_bytes + 64), release); + char* input_base = static_cast(input_memory.get()) + input_offset; + char* output_base = + in_place ? input_base : static_cast(output_memory.get()) + output_offset; + const auto make = [&](const unsigned flags) -> fftw_plan { + fftw_plan_with_nthreads(threads); + if (r2c) + return fftw_plan_many_dft_r2c( + rank, extents.data(), howmany, reinterpret_cast(input_base), nullptr, + 1, input_distance, reinterpret_cast(output_base), nullptr, 1, + output_distance, flags); + if (c2r) + return fftw_plan_many_dft_c2r( + rank, extents.data(), howmany, reinterpret_cast(input_base), + nullptr, 1, input_distance, reinterpret_cast(output_base), nullptr, + 1, output_distance, flags); + return fftw_plan_many_dft( + rank, extents.data(), howmany, reinterpret_cast(input_base), + nullptr, 1, input_distance, reinterpret_cast(output_base), + nullptr, 1, output_distance, sign, flags); + }; + const auto timed = [&](const unsigned flags) { + fftw_forget_wisdom(); + if ((flags & FFTW_WISDOM_ONLY) != 0U && + fftw_import_wisdom_from_string(wisdom.c_str()) == 0) + return std::numeric_limits::quiet_NaN(); + auto* const plan = make(flags); + if (plan == nullptr) { + fftw_forget_wisdom(); + return std::numeric_limits::quiet_NaN(); + } + // Real input is overwritten by complex-to-real plans and harmless otherwise. + std::memset(input_base, 0, input_bytes); + const double seconds = median_execution(plan); + fftw_destroy_plan(plan); + fftw_forget_wisdom(); + return seconds; + }; + PlanAlone result; + result.plan = timed(fftw_effort_flag(effort) | FFTW_WISDOM_ONLY); + result.estimate = timed(FFTW_ESTIMATE); + return result; +} + +void seed_fftw_wisdom(const ExecutionSpace& space, Complex* input, Complex* output, + const std::vector& extents, const int howmany, + const int input_distance, const int output_distance, + const int sign) { + std::string sizes; + for (const int extent : extents) + sizes += (sizes.empty() ? "" : "x") + std::to_string(extent); + const std::string key = fft_plan_key( + sign == FFTW_FORWARD ? "planes-forward" : "planes-backward", sizes, howmany, + input_distance, output_distance, input, output, space.concurrency()); + fftw_init_threads(); + const auto chosen = begin_seed(key, FftShape::planes, space.concurrency()); + if (!chosen) + return; + fftw_plan_with_nthreads(space.concurrency()); + auto* in = reinterpret_cast(input); + auto* out = reinterpret_cast(output); + auto* const seeded = fftw_plan_many_dft( + static_cast(extents.size()), extents.data(), howmany, in, nullptr, 1, + input_distance, out, nullptr, 1, output_distance, sign, *chosen); + // The policy has always tolerated a plane it cannot seed; a profile records it. + if (seeded == nullptr && (*chosen & FFTW_WISDOM_ONLY) == 0U && + (detail::planning == nullptr || + detail::planning->mode == FftPlanning::Mode::legacy)) { + release_seed(key, false); + return; + } + end_seed(key, *chosen, seeded != nullptr); + if (seeded != nullptr) + fftw_destroy_plan(seeded); +} + +} // namespace solprop_native +#endif diff --git a/native/src/general/kernels.hpp b/native/src/general/kernels.hpp index d3f4161d4..3f2b6326f 100644 --- a/native/src/general/kernels.hpp +++ b/native/src/general/kernels.hpp @@ -639,43 +639,6 @@ void tiled_update(const char* label, const ExecutionSpace& execution, } #endif -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -// Plan, then discard, a batched complex transform of transverse planes at the plan -// policy's effort, leaving its wisdom. It overwrites both arrays; the caller clears -// them. Row transforms are seeded by `seed_row_fft_wisdom` (fft_plan.hpp). -inline void seed_fftw_wisdom(const ExecutionSpace& space, Complex* input, - Complex* output, const std::vector& extents, - const int howmany, const int input_distance, - const int output_distance, const int sign) { - std::string sizes; - for (const int extent : extents) - sizes += (sizes.empty() ? "" : "x") + std::to_string(extent); - const std::string key = fft_plan_key( - sign == FFTW_FORWARD ? "planes-forward" : "planes-backward", sizes, howmany, - input_distance, output_distance, input, output, space.concurrency()); - fftw_init_threads(); - const auto chosen = begin_seed(key, FftShape::planes, space.concurrency()); - if (!chosen) - return; - fftw_plan_with_nthreads(space.concurrency()); - auto* in = reinterpret_cast(input); - auto* out = reinterpret_cast(output); - auto* const seeded = fftw_plan_many_dft( - static_cast(extents.size()), extents.data(), howmany, in, nullptr, 1, - input_distance, out, nullptr, 1, output_distance, sign, *chosen); - // The policy has always tolerated a plane it cannot seed; a profile records it. - if (seeded == nullptr && (*chosen & FFTW_WISDOM_ONLY) == 0U && - (detail::planning == nullptr || - detail::planning->mode == FftPlanning::Mode::legacy)) { - release_seed(key, false); - return; - } - end_seed(key, *chosen, seeded != nullptr); - if (seeded != nullptr) - fftw_destroy_plan(seeded); -} -#endif - // Where a separable geometry's column (or a temporal row, which is the same index) // lives in the transverse scratch. Free space: (.., n_y, n_x), x fastest. The hybrid // planar kind: (.., n_p, n_y) with the y transform trailing, the channel fastest in diff --git a/native/src/solprop_native/_wisdom.py b/native/src/solprop_native/_wisdom.py index 8b0be1c1e..03876cc7d 100644 --- a/native/src/solprop_native/_wisdom.py +++ b/native/src/solprop_native/_wisdom.py @@ -17,8 +17,9 @@ keys its winners) holds the wisdom of that transform alone, measured in place by the run that first planned it. A workspace is offered the plans it may use; a transform the store holds plans from its own wisdom only, and one it lacks is measured alone and -added (``FftPlanning`` in ``fft_plan.hpp``). A calibration profile's winners keep -precedence: a key the profile holds is never read from or written to this store. +added (``FftPlanning``, applied in ``fftw_wisdom.cpp``). A calibration profile's +winners keep precedence: a key the profile holds is never read from or written to this +store. By default the file is ``wisdom.json`` in the calibration store's directory of this machine and build (``/machines//``), so different machines and diff --git a/native/tests/native/test_fft.cpp b/native/tests/native/test_fft.cpp index 707e390d9..ec4f7c5d1 100644 --- a/native/tests/native/test_fft.cpp +++ b/native/tests/native/test_fft.cpp @@ -5,6 +5,7 @@ #include "solprop_native/cpu_execution.hpp" #include "solprop_native/execution.hpp" #include "solprop_native/fft_plan.hpp" +#include "solprop_native/fftw_wisdom.hpp" #include "solprop_native/raman_fft.hpp" #include diff --git a/native/tests/test_fftw_plan_refusal.py b/native/tests/test_fftw_plan_refusal.py index 7d78519ef..a5fde57f7 100644 --- a/native/tests/test_fftw_plan_refusal.py +++ b/native/tests/test_fftw_plan_refusal.py @@ -5,7 +5,7 @@ """A mistyped FFTW planning effort is refused (SEP 43 inventory, item 73; #460). Native takes the effort from ``SOLPROP_FFTW_PLAN`` (``fftw_seed_effort`` in -``include/solprop_native/fft_plan.hpp``). Read as ``measure``, a typo would change a +``src/fftw_wisdom.cpp``). Read as ``measure``, a typo would change a run's plans in silence, so it is refused in a fresh process, as the neighbouring thread variables are. """ From 048689ebeb28117d8e9247d6e3bc4d4139588680 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:02:25 +0000 Subject: [PATCH 06/21] Move the seam's set-once and run records to run_records.hpp workspace.hpp held the Setup structs, the run records that mirror solprop.backends.records, their field tables and to_native beside the Run, Probe and Workspace classes. The records move verbatim, with the file comment's paragraph on the binding's conversion, to run_records.hpp; workspace.hpp includes it and drops the headers only the records used. No code changes. Co-Authored-By: Claude Opus 5.5 --- native/include/solprop_native/problem.hpp | 2 +- native/include/solprop_native/run_records.hpp | 519 ++++++++++++++++++ native/include/solprop_native/workspace.hpp | 488 +--------------- native/src/bindings.cpp | 1 + native/src/workspace.cpp | 1 + native/tests/native/test_workspace.cpp | 1 + 6 files changed, 524 insertions(+), 488 deletions(-) create mode 100644 native/include/solprop_native/run_records.hpp diff --git a/native/include/solprop_native/problem.hpp b/native/include/solprop_native/problem.hpp index 719e82604..99c381e1b 100644 --- a/native/include/solprop_native/problem.hpp +++ b/native/include/solprop_native/problem.hpp @@ -341,7 +341,7 @@ struct Problem { /** The set-once data a workspace applies once it has built its state (solprop SEP 52 section 5.3): the stage, z and flux tables, the flow guard, the controllers, the low-storage settings, the monitor's weights and the edge's geometry - (`workspace.hpp`). Shared, so that a copy of the problem copies none of it. */ + (`run_records.hpp`). Shared, so that a copy of the problem copies none of it. */ std::shared_ptr setup; /** Refuse, naming the field, data whose presence or extents disagree with the diff --git a/native/include/solprop_native/run_records.hpp b/native/include/solprop_native/run_records.hpp new file mode 100644 index 000000000..73420f040 --- /dev/null +++ b/native/include/solprop_native/run_records.hpp @@ -0,0 +1,519 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "solprop_native/adams_march.hpp" +#include "solprop_native/engines.hpp" +#include "solprop_native/execution.hpp" +#include "solprop_native/fields.hpp" +#include "solprop_native/general_state.hpp" +#include "solprop_native/low_storage.hpp" +#include "solprop_native/readings.hpp" +#include "solprop_native/rk_march.hpp" +#include "solprop_native/stage_tables.hpp" +#include "solprop_native/statistics_plan.hpp" +#include "solprop_native/transverse.hpp" +#include "solprop_native/watch.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* + * The native workspace's records (solprop SEP 52 section 5.3): the set-once data of a + * problem's `Setup` and the records one run starts from, with what the marches and the + * session read from them (`workspace.hpp` builds and runs the workspace). + * + * The binding converts Python's records into the structs here (each with its field + * table, `fields.hpp`), releases the interpreter lock and forwards; the logic is here, + * where the native tests reach it. + */ +namespace solprop_native { + +// ------------------------------------------------------------------ the set-once data + +/** One general z table made resident at construction (solprop SEP 29 G1): its + breakpoints, admitted span and coefficients `(n_edges - 1, n_terms, n_values)`. */ +struct ZTable { + TableKind kind = TableKind::phi; + std::vector edges; + double lower = 0.0; + double upper = 0.0; + std::vector coefficients; + std::size_t n_terms = 0; + std::size_t n_values = 0; +}; + +[[nodiscard]] constexpr auto describe(const ZTable* /*unused*/) { + using E = fields::Extent; + return fields::table( + "ZTable", fields::value("kind", &ZTable::kind), + fields::array("edges", &ZTable::edges, E::of("n_edges")), + fields::value("lower", &ZTable::lower), fields::value("upper", &ZTable::upper), + fields::array("coefficients", &ZTable::coefficients, E::of("n_edges", -1), + E::of("n_terms"), E::of("n_values")), + fields::count("n_terms", &ZTable::n_terms), + fields::count("n_values", &ZTable::n_values)); +} + +/** The z tables' position slots (`stage::ZSlots::configure`). */ +struct ZSlotsConfig { + bool phi = false; + bool prefactor = false; + std::size_t slots = 0; + bool rate = false; +}; + +[[nodiscard]] constexpr auto describe(const ZSlotsConfig* /*unused*/) { + using Z = ZSlotsConfig; + return fields::table("ZSlots", fields::value("phi", &Z::phi), + fields::value("prefactor", &Z::prefactor), + fields::value("slots", &Z::slots), + fields::value("rate", &Z::rate)); +} + +/** A coupled flow's certificate and the guard's constants (solprop SEP 45 phase 4): + each section's row-sum norm, each section's conditioning bound per active bin + `(n_sections, n_bins)`, the Taylor scale, the most squarings and the rounding + allowance, as solprop's `_flow_squarings` and `_check_flow_conditioning` read + them. */ +struct FlowGuard { + std::vector row_sum; + std::vector condition_bound; + double taylor_scale = 0.0; + int max_squarings = 0; + double allowance = 0.0; +}; + +[[nodiscard]] constexpr auto describe(const FlowGuard* /*unused*/) { + using F = FlowGuard; + using E = fields::Extent; + return fields::table("FlowGuard", + fields::array("row_sum", &F::row_sum, E::of("n_sections")), + fields::array("condition_bound", &F::condition_bound, + E::of("n_sections"), E::of("n_bins")), + fields::value("taylor_scale", &F::taylor_scale), + fields::value("max_squarings", &F::max_squarings), + fields::value("allowance", &F::allowance)); +} + +/** The Runge-Kutta trial's tableau nodes and error control (solprop SEP 45): the + nodes, the tolerances, a weighted controller's weights (empty for none) and the + tableau's dense-output weights (empty without dense output). */ +struct RungeKuttaConfig { + std::vector nodes; + double rtol = 0.0; + double atol = 0.0; + std::vector weights; + std::vector dense_weights; +}; + +[[nodiscard]] constexpr auto describe(const RungeKuttaConfig* /*unused*/) { + using R = RungeKuttaConfig; + using E = fields::Extent; + return fields::table( + "RungeKuttaConfig", fields::array("nodes", &R::nodes, E::of("n_stages")), + fields::value("rtol", &R::rtol), fields::value("atol", &R::atol), + fields::array("weights", &R::weights, E::any()), + fields::array("dense_weights", &R::dense_weights, E::any())); +} + +/** The resident Adams history (solprop SEP 16, SEP 39): its rows, whether trials carry + the transported compensation, whether the steps are kept for interpolation, the + fused (interaction) formulation, and the controller's scale. */ +struct AdamsConfig { + std::size_t slots = 0; + bool compensated = false; + bool interpolate = false; + bool fused = false; + double rtol = 0.0; + double atol = 0.0; + std::vector weights; +}; + +[[nodiscard]] constexpr auto describe(const AdamsConfig* /*unused*/) { + using A = AdamsConfig; + using E = fields::Extent; + return fields::table( + "AdamsConfig", fields::value("slots", &A::slots), + fields::value("compensated", &A::compensated), + fields::value("interpolate", &A::interpolate), fields::value("fused", &A::fused), + fields::value("rtol", &A::rtol), fields::value("atol", &A::atol), + fields::array("weights", &A::weights, E::any())); +} + +/** The boundary monitor's weights where the state lives (solprop SEP 21), a run's: each + run's monitor is its own (`RunRequest.for_run`). `split` says the monitor reads the + split collar's crossing, whose walk-off buckets are the problem's (`Occupancy`). */ +struct MonitorBinding { + std::vector weights; + std::vector column_weights; + bool split = false; +}; + +[[nodiscard]] constexpr auto describe(const MonitorBinding* /*unused*/) { + using M = MonitorBinding; + using E = fields::Extent; + return fields::table("MonitorBinding", + fields::array("weights", &M::weights, E::any()), + fields::array("column_weights", &M::column_weights, E::any()), + fields::value("split", &M::split)); +} + +/** The split collar's walk-off bucket of each bin (solprop #440), the problem's: set + once, and bound to the state by the first run whose monitor reads it. */ +struct Occupancy { + std::vector buckets; + std::size_t count = 0; +}; + +[[nodiscard]] constexpr auto describe(const Occupancy* /*unused*/) { + using O = Occupancy; + using E = fields::Extent; + return fields::table( + "Occupancy", + fields::array("buckets", &O::buckets, fields::Dtype::uint8, E::any()), + fields::value("count", &O::count)); +} + +/** The transverse edge's tables where the state lives (solprop SEP 44), a run's: the + faces', the clip's and the totals' weights, a reduction's synthesis (empty for + none), the first row, and a requested collar's lines. */ +struct EdgeBinding { + std::vector faces; + std::vector clip; + std::vector totals; + std::vector synthesis; + std::size_t first_row = 0; + std::optional lines; +}; + +[[nodiscard]] constexpr auto describe(const EdgeBinding* /*unused*/) { + using B = EdgeBinding; + using E = fields::Extent; + return fields::table("EdgeBinding", fields::array("faces", &B::faces, E::any()), + fields::array("clip", &B::clip, E::any()), + fields::array("totals", &B::totals, E::any()), + fields::array("synthesis", &B::synthesis, E::any()), + fields::value("first_row", &B::first_row), + fields::value("lines", &B::lines)); +} + +/** Everything fixed for a workspace beside the problem's groups (SEP 52 section 5.3). + Each part is absent where the problem has none; `crossing_workers` is the threads a + crossing's copy may use (zero keeps the state's default). */ +struct Setup { + std::optional stage_tables; + std::optional z_slots; + std::vector z_tables; + std::optional flow_guard; + std::optional runge_kutta; + std::optional adams; + std::optional low_storage; + std::optional occupancy; + std::size_t crossing_workers = 0; +}; + +[[nodiscard]] constexpr auto describe(const Setup* /*unused*/) { + using S = Setup; + return fields::table( + "Setup", fields::value("stage_tables", &S::stage_tables), + fields::value("z_slots", &S::z_slots), fields::value("z_tables", &S::z_tables), + fields::value("flow_guard", &S::flow_guard), + fields::value("runge_kutta", &S::runge_kutta), fields::value("adams", &S::adams), + fields::value("low_storage", &S::low_storage), + fields::value("occupancy", &S::occupancy), + fields::value("crossing_workers", &S::crossing_workers)); +} + +// -------------------------------------------------------------------- the run records +// The structs below mirror `solprop.backends.records`, field for field, so that one +// table holds each to its Python dataclass; `to_native` forms what the marches and the +// session read from them. + +/** `solprop.stepper.controller.PIController`. */ +struct ControllerRecord { + double rtol = 0.0; + double atol = 0.0; + double safety = 0.0; + double max_factor = 0.0; + double min_factor = 0.0; + double beta1 = 0.0; + double beta2 = 0.0; + std::int64_t lag = 0; + bool reject_free = false; + std::optional> weights; + std::int64_t estimator_order = 0; + std::string region; +}; + +[[nodiscard]] constexpr auto describe(const ControllerRecord* /*unused*/) { + using C = ControllerRecord; + using E = fields::Extent; + return fields::table( + "PIController", fields::value("rtol", &C::rtol), fields::value("atol", &C::atol), + fields::value("safety", &C::safety), fields::value("max_factor", &C::max_factor), + fields::value("min_factor", &C::min_factor), fields::value("beta1", &C::beta1), + fields::value("beta2", &C::beta2), fields::value("lag", &C::lag), + fields::value("reject_free", &C::reject_free), + fields::array("weights", &C::weights, E::of("n_omega")), + fields::value("estimator_order", &C::estimator_order), + fields::value("region", &C::region)); +} + +/** `solprop.backends.records.RungeKuttaPolicy`. */ +struct RungeKuttaPolicyRecord { + ControllerRecord controller; + double dz_min = 0.0; + double dz_max = 0.0; + bool auto_init = false; + double stretch = 0.0; + double z_end = 0.0; + std::vector save_planes; + bool interpolate = false; + std::vector hard_kinks; + std::vector breakpoints; + int landing_ulps = 0; + double start_growth = 0.0; + std::optional quiet_interval; +}; + +[[nodiscard]] constexpr auto describe(const RungeKuttaPolicyRecord* /*unused*/) { + using R = RungeKuttaPolicyRecord; + return fields::table( + "RungeKuttaPolicy", fields::value("controller", &R::controller), + fields::value("dz_min", &R::dz_min), fields::value("dz_max", &R::dz_max), + fields::value("auto_init", &R::auto_init), fields::value("stretch", &R::stretch), + fields::value("z_end", &R::z_end), fields::value("save_planes", &R::save_planes), + fields::value("interpolate", &R::interpolate), + fields::value("hard_kinks", &R::hard_kinks), + fields::value("breakpoints", &R::breakpoints), + fields::value("landing_ulps", &R::landing_ulps), + fields::value("start_growth", &R::start_growth), + fields::value("quiet_interval", &R::quiet_interval)); +} + +/** `solprop.stepper.schedule.FixedSchedule`: its steps and start; the edges are + accumulated in step order (`FixedSchedule.edges`). */ +struct ScheduleRecord { + std::vector steps; + double z_start = 0.0; +}; + +[[nodiscard]] constexpr auto describe(const ScheduleRecord* /*unused*/) { + using E = fields::Extent; + return fields::table( + "FixedSchedule", fields::array("steps", &ScheduleRecord::steps, E::of("n_steps")), + fields::value("z_start", &ScheduleRecord::z_start)); +} + +/** `solprop.backends.records.PlanePlan`. */ +struct PlanePlanRecord { + double z = 0.0; + std::vector planes; + std::size_t next_plane = 0; + bool launch_pending = true; + std::uint64_t step_planes = 0; + bool dense = false; + std::uint64_t period = 0; + std::size_t replay_steps = 0; + double replay_length = 0.0; + std::size_t attempt_capacity = 0; + std::size_t row_capacity = 0; + std::size_t call_attempts = 0; + double call_seconds = 0.0; +}; + +[[nodiscard]] constexpr auto describe(const PlanePlanRecord* /*unused*/) { + using P = PlanePlanRecord; + using E = fields::Extent; + return fields::table

("PlanePlan", fields::value("z", &P::z), - fields::array("planes", &P::planes, E::of("n_z")), - fields::value("next_plane", &P::next_plane), - fields::value("launch_pending", &P::launch_pending), - fields::value("step_planes", &P::step_planes), - fields::value("dense", &P::dense), - fields::value("period", &P::period), - fields::value("replay_steps", &P::replay_steps), - fields::value("replay_length", &P::replay_length), - fields::value("attempt_capacity", &P::attempt_capacity), - fields::value("row_capacity", &P::row_capacity), - fields::value("call_attempts", &P::call_attempts), - fields::value("call_seconds", &P::call_seconds)); -} - -/** `solprop.backends.records.Resumption`: the numbers a resume reads by name. */ -struct ResumptionRecord { - bool record = false; - bool collar_quiet = false; - std::map numbers; - std::optional>> unwatched; - std::optional> resumed_at; - std::optional>> planes; -}; - -[[nodiscard]] constexpr auto describe(const ResumptionRecord* /*unused*/) { - using R = ResumptionRecord; - return fields::table( - "Resumption", fields::value("record", &R::record), - fields::value("collar_quiet", &R::collar_quiet), - fields::value("numbers", &R::numbers), fields::value("unwatched", &R::unwatched), - fields::value("resumed_at", &R::resumed_at), fields::value("planes", &R::planes)); -} - -/** `solprop.backends.records.EdgeResumption`. */ -struct EdgeResumptionRecord { - bool record = false; - std::map numbers; - std::optional> edge_baseline; - std::optional> edge_max; - std::optional> edge_max_raw; - std::optional reads; - std::optional transverse_status; -}; - -[[nodiscard]] constexpr auto describe(const EdgeResumptionRecord* /*unused*/) { - using R = EdgeResumptionRecord; - return fields::table("EdgeResumption", fields::value("record", &R::record), - fields::value("numbers", &R::numbers), - fields::value("edge_baseline", &R::edge_baseline), - fields::value("edge_max", &R::edge_max), - fields::value("edge_max_raw", &R::edge_max_raw), - fields::value("reads", &R::reads), - fields::value("transverse_status", &R::transverse_status)); -} - -/** `solprop.backends.records.MonitorContinuation`. */ -struct ContinuationRecord { - ResumptionRecord record; - EdgeResumptionRecord edge; - bool plane = false; - std::vector> unwatched; -}; - -[[nodiscard]] constexpr auto describe(const ContinuationRecord* /*unused*/) { - using C = ContinuationRecord; - return fields::table("MonitorContinuation", fields::value("record", &C::record), - fields::value("edge", &C::edge), - fields::value("plane", &C::plane), - fields::value("unwatched", &C::unwatched)); -} - -/** `solprop.backends.records.MonitorRecords`. */ -struct MonitorRecordsRecord { - watch::Config config; - std::optional edge; - std::optional continuation; -}; - -[[nodiscard]] constexpr auto describe(const MonitorRecordsRecord* /*unused*/) { - using M = MonitorRecordsRecord; - return fields::table("MonitorRecords", fields::value("config", &M::config), - fields::value("edge", &M::edge), - fields::value("continuation", &M::continuation)); -} - -/** `solprop.backends.records.RunRecords`: the method's policy is the Adams march's or - the Runge-Kutta march's, told apart by their fields. */ -struct RunRecordsRecord { - std::variant policy; - std::optional low_storage; - std::optional schedule; - PlanePlanRecord planes; - std::optional monitor; -}; - -[[nodiscard]] constexpr auto describe(const RunRecordsRecord* /*unused*/) { - using R = RunRecordsRecord; - return fields::table("RunRecords", fields::value("policy", &R::policy), - fields::value("low_storage", &R::low_storage), - fields::value("schedule", &R::schedule), - fields::value("planes", &R::planes), - fields::value("monitor", &R::monitor)); -} - -/** One persistent row of a resumed Adams history, `(n_slots, rows, n_omega)`. */ -struct HistoryRow { - std::string name; - std::vector values; - std::size_t n_slots = 0; - std::size_t n_rows = 0; - std::size_t n_omega = 0; -}; - -[[nodiscard]] constexpr auto describe(const HistoryRow* /*unused*/) { - using H = HistoryRow; - using E = fields::Extent; - return fields::table("HistoryRow", fields::value("name", &H::name), - fields::array("values", &H::values, E::of("n_slots"), - E::of("n_rows"), E::of("n_omega")), - fields::count("n_slots", &H::n_slots), - fields::count("n_rows", &H::n_rows), - fields::count("n_omega", &H::n_omega)); -} - -/** What one run starts from (`Workspace::start`): its records, the initial field and - noise as the state takes them (`set_initial`), borrowed from the caller for the - call, the march's state at the start (the Adams march's or the Runge-Kutta march's, - none for a fresh state of its own), a resumed history's rows, the statistics plans, - the monitor's launch readings of the input noise (its floor: energy, then four - regions' shares; and the edge's), and the run's monitor weights and edge tables where - the state lives. */ -struct RunStart { - RunRecordsRecord records; - std::span field; - std::span noise; - std::optional adams; - std::optional runge_kutta; - std::vector rows; - std::shared_ptr plane_statistics; - std::shared_ptr row_statistics; - std::optional> floor; - std::optional edge_noise; - std::optional monitor; - std::optional edge; -}; - -[[nodiscard]] constexpr auto describe(const RunStart* /*unused*/) { - using R = RunStart; - using E = fields::Extent; - return fields::table( - "RunStart", fields::value("records", &R::records), - fields::array("field", &R::field, E::any()), - fields::array("noise", &R::noise, E::any()), fields::value("adams", &R::adams), - fields::value("runge_kutta", &R::runge_kutta), fields::value("rows", &R::rows), - fields::value("plane_statistics", &R::plane_statistics), - fields::value("row_statistics", &R::row_statistics), - fields::value("floor", &R::floor), fields::value("edge_noise", &R::edge_noise), - fields::value("monitor", &R::monitor), fields::value("edge", &R::edge)); -} - -/** The Runge-Kutta march's policy from its record (kernel contract section 12): the - controller's gains, factors, lag and acceptance rule flattened in, and the - equivalent norm's power `estimator_order + 1` and its reciprocal. */ -[[nodiscard]] rk::Policy to_native(const RungeKuttaPolicyRecord& record); -/** A fixed schedule's steps and edges, accumulated in step order. */ -[[nodiscard]] march::Schedule to_native(const ScheduleRecord& record); -/** A monitor record's resumption, each number by name; refuses an unknown name. */ -[[nodiscard]] watch::Resumption to_native(const ResumptionRecord& record); -/** An edge record's resumption, likewise. */ -[[nodiscard]] transverse::Resumption to_native(const EdgeResumptionRecord& record); - -/** A test's injected failure by name: the stage (`startup`, `trial`, `trial_source` - or `accept`) and the category (`convergence`, `value`, `backend`, `sigint` or - `sigterm`). */ -[[nodiscard]] engines::MarchFault -fault_from(const std::string& stage, std::size_t after, const std::string& category); - -/** The stage tables' consistency (the checks `adams_stage_tables` makes). */ -void validate(const stage::Tables& tables); - // ------------------------------------------------------------------------ the // workspace diff --git a/native/src/bindings.cpp b/native/src/bindings.cpp index 1420ef669..3c5db0af4 100644 --- a/native/src/bindings.cpp +++ b/native/src/bindings.cpp @@ -22,6 +22,7 @@ #include "solprop_native/reference_timing.hpp" #include "solprop_native/refuse.hpp" #include "solprop_native/rk_march.hpp" +#include "solprop_native/run_records.hpp" #include "solprop_native/session.hpp" #include "solprop_native/stage_tables.hpp" #include "solprop_native/vmath.hpp" diff --git a/native/src/workspace.cpp b/native/src/workspace.cpp index 261e084e6..20a7ae740 100644 --- a/native/src/workspace.cpp +++ b/native/src/workspace.cpp @@ -6,6 +6,7 @@ #include "solprop_native/dense_output.hpp" #include "solprop_native/refuse.hpp" +#include "solprop_native/run_records.hpp" #include #include diff --git a/native/tests/native/test_workspace.cpp b/native/tests/native/test_workspace.cpp index 93f8473b4..d57e14311 100644 --- a/native/tests/native/test_workspace.cpp +++ b/native/tests/native/test_workspace.cpp @@ -7,6 +7,7 @@ // timing probe, each moved out of the binding unit and tested here. #include "execution.hpp" +#include "solprop_native/run_records.hpp" #include "solprop_native/workspace.hpp" #include From a79d3f7c532813f23152ab85aa1f917e3648de1d Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:05:05 +0000 Subject: [PATCH 07/21] Give session::Resident its own header for engines.hpp engines.hpp included all of session.hpp (the session, its plans and methods, dense output, the monitor) for the one interface its GeneralResident implements. session::Resident and the StatisticsRecipe it takes move verbatim to session_resident.hpp, which session.hpp and engines.hpp include. No code changes. Co-Authored-By: Claude Opus 5.5 --- native/include/solprop_native/engines.hpp | 3 +- native/include/solprop_native/session.hpp | 57 +------------ .../solprop_native/session_resident.hpp | 79 +++++++++++++++++++ 3 files changed, 82 insertions(+), 57 deletions(-) create mode 100644 native/include/solprop_native/session_resident.hpp diff --git a/native/include/solprop_native/engines.hpp b/native/include/solprop_native/engines.hpp index 7d1015f5c..464f17c14 100644 --- a/native/include/solprop_native/engines.hpp +++ b/native/include/solprop_native/engines.hpp @@ -8,8 +8,9 @@ #include "solprop_native/general_state.hpp" #include "solprop_native/host_source.hpp" #include "solprop_native/rk_march.hpp" -#include "solprop_native/session.hpp" +#include "solprop_native/session_resident.hpp" #include "solprop_native/stage_tables.hpp" +#include "solprop_native/transverse.hpp" #include #include diff --git a/native/include/solprop_native/session.hpp b/native/include/solprop_native/session.hpp index f34573ebc..543adaa86 100644 --- a/native/include/solprop_native/session.hpp +++ b/native/include/solprop_native/session.hpp @@ -7,6 +7,7 @@ #include "solprop_native/adams_march.hpp" #include "solprop_native/dense.hpp" #include "solprop_native/rk_march.hpp" +#include "solprop_native/session_resident.hpp" #include "solprop_native/statistics_plan.hpp" #include "solprop_native/transverse.hpp" #include "solprop_native/watch.hpp" @@ -86,62 +87,6 @@ enum class Observation : int { launch = 0, plane = 1, step = 2, natural = 3 }; /** How a session ended (`solprop.backends.events.Outcome`). */ enum class Outcome : int { none = 0, completed = 1, stopped = 2, failed = 3 }; -/** A statistics evaluation the resident state makes, with every argument fixed at - preparation (`statistics::Plan`, solprop SEP 47 section 6.4); none without a - plan. */ -struct StatisticsRecipe { - std::shared_ptr plan; - [[nodiscard]] bool empty() const noexcept { return !plan || plan->empty(); } -}; - -/** What a session reads from and writes through the state it leases: the field and - the statistics, where they live. The resident engines' implementation is - `engines::GeneralResident`. */ -class Resident { -public: - Resident() = default; - Resident(const Resident&) = delete; - Resident& operator=(const Resident&) = delete; - Resident(Resident&&) = delete; - Resident& operator=(Resident&&) = delete; - virtual ~Resident() = default; - /** The field's complex element count, `columns * n_omega`. */ - [[nodiscard]] virtual std::size_t elements() const = 0; - /** The field's last axis, the frequency bins, along which `(-1)^k` alternates. */ - [[nodiscard]] virtual std::size_t n_omega() const = 0; - /** The committed field as host memory, interleaved real and imaginary parts, or - null where it is not host-accessible. */ - virtual const double* host_field() = 0; - /** Copy the committed field into `out`, `2 elements()` doubles. */ - virtual void field_to_host(double* out) = 0; - /** Count bytes a host reader of `host_field` took out of the state. */ - virtual void count_to_host(std::size_t bytes) = 0; - /** Evaluate `recipe` on the committed state at `z`. */ - virtual std::vector statistics(const StatisticsRecipe& recipe, double z) = 0; - /** The committed state's boundary reading (`monitor_reading`, `watch::reading_head` - doubles and the occupancy buckets), for the monitor's launch and plane checks. */ - virtual std::vector boundary_reading() { - throw std::logic_error("this state takes no boundary readings"); - } - /** Leave the in-stage temporal collar out of the stages, or put it back, dropping - what was evaluated under the other setting (`_collar_changed`). */ - virtual void set_collar_quiet(bool /*quiet*/) { - throw std::logic_error("this state cannot rest its collar"); - } - /** The largest accepted step's transverse quadrature score since the last read, - clearing it (`quadrature_peak`); NaN where none was scored. */ - virtual double quadrature_peak() { return std::numeric_limits::quiet_NaN(); } - /** The committed state's transverse edge reading (`state.edge_reading`, SEP 44): the - faces' sums and the columns when `faces`, with the clip and totals of every node - when `whole`, the totals when `totals`, a collar's strip when `strip` and its - current when `current`. */ - virtual transverse::Reading edge_reading(bool /*faces*/, bool /*whole*/, - bool /*totals*/, bool /*strip*/, - bool /*current*/) { - throw std::logic_error("this state takes no transverse edge readings"); - } -}; - /** The boundary monitor a session owns (SEP 47 section 6.4): its configuration and, for a resumed run, the record it continues. */ struct MonitorPlan { diff --git a/native/include/solprop_native/session_resident.hpp b/native/include/solprop_native/session_resident.hpp new file mode 100644 index 000000000..8d321dfff --- /dev/null +++ b/native/include/solprop_native/session_resident.hpp @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "solprop_native/statistics_plan.hpp" +#include "solprop_native/transverse.hpp" + +#include +#include +#include +#include +#include + +/* + * What a native session (`session.hpp`) reads from and writes through the state it + * leases, behind one interface the resident engines implement (`engines.hpp`), so that + * they need not see the session. Host code only. + */ +namespace solprop_native::session { + +/** A statistics evaluation the resident state makes, with every argument fixed at + preparation (`statistics::Plan`, solprop SEP 47 section 6.4); none without a + plan. */ +struct StatisticsRecipe { + std::shared_ptr plan; + [[nodiscard]] bool empty() const noexcept { return !plan || plan->empty(); } +}; + +/** What a session reads from and writes through the state it leases: the field and + the statistics, where they live. The resident engines' implementation is + `engines::GeneralResident`. */ +class Resident { +public: + Resident() = default; + Resident(const Resident&) = delete; + Resident& operator=(const Resident&) = delete; + Resident(Resident&&) = delete; + Resident& operator=(Resident&&) = delete; + virtual ~Resident() = default; + /** The field's complex element count, `columns * n_omega`. */ + [[nodiscard]] virtual std::size_t elements() const = 0; + /** The field's last axis, the frequency bins, along which `(-1)^k` alternates. */ + [[nodiscard]] virtual std::size_t n_omega() const = 0; + /** The committed field as host memory, interleaved real and imaginary parts, or + null where it is not host-accessible. */ + virtual const double* host_field() = 0; + /** Copy the committed field into `out`, `2 elements()` doubles. */ + virtual void field_to_host(double* out) = 0; + /** Count bytes a host reader of `host_field` took out of the state. */ + virtual void count_to_host(std::size_t bytes) = 0; + /** Evaluate `recipe` on the committed state at `z`. */ + virtual std::vector statistics(const StatisticsRecipe& recipe, double z) = 0; + /** The committed state's boundary reading (`monitor_reading`, `watch::reading_head` + doubles and the occupancy buckets), for the monitor's launch and plane checks. */ + virtual std::vector boundary_reading() { + throw std::logic_error("this state takes no boundary readings"); + } + /** Leave the in-stage temporal collar out of the stages, or put it back, dropping + what was evaluated under the other setting (`_collar_changed`). */ + virtual void set_collar_quiet(bool /*quiet*/) { + throw std::logic_error("this state cannot rest its collar"); + } + /** The largest accepted step's transverse quadrature score since the last read, + clearing it (`quadrature_peak`); NaN where none was scored. */ + virtual double quadrature_peak() { return std::numeric_limits::quiet_NaN(); } + /** The committed state's transverse edge reading (`state.edge_reading`, SEP 44): the + faces' sums and the columns when `faces`, with the clip and totals of every node + when `whole`, the totals when `totals`, a collar's strip when `strip` and its + current when `current`. */ + virtual transverse::Reading edge_reading(bool /*faces*/, bool /*whole*/, + bool /*totals*/, bool /*strip*/, + bool /*current*/) { + throw std::logic_error("this state takes no transverse edge readings"); + } +}; + +} // namespace solprop_native::session From b37e813b7c17ced2a175b221144ad4b2fdba9a78 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:05:05 +0000 Subject: [PATCH 08/21] Move the ledger's allocator queries out of general/kernels.hpp heap_bytes and device_free_bytes are host code run once around planning, but lived inline in the header every general/ unit and its kernels include, with the allocator headers they need. They move verbatim to general/memory.cpp, declared in general/memory.hpp, which setup.cpp and statistics.cpp, their callers, include. No code changes. Co-Authored-By: Claude Opus 5.5 --- native/CMakeLists.txt | 2 +- native/src/general/kernels.hpp | 48 ---------------------- native/src/general/memory.cpp | 66 +++++++++++++++++++++++++++++++ native/src/general/memory.hpp | 19 +++++++++ native/src/general/setup.cpp | 1 + native/src/general/statistics.cpp | 1 + 6 files changed, 88 insertions(+), 49 deletions(-) create mode 100644 native/src/general/memory.cpp create mode 100644 native/src/general/memory.hpp diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index cd7bfcf0a..5bcefb4a3 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -812,7 +812,7 @@ endif() add_library(solprop_native_core STATIC src/cpu_execution.cpp src/crossing.cpp src/general/checks.cpp src/general/setup.cpp src/general/trial.cpp - src/general/statistics.cpp src/general/state.cpp src/general_modal.cpp src/coupled_flow.cpp src/dispersive.cpp src/far_field.cpp + src/general/statistics.cpp src/general/state.cpp src/general/memory.cpp src/general_modal.cpp src/coupled_flow.cpp src/dispersive.cpp src/far_field.cpp src/adams_march.cpp src/watch.cpp src/rk_march.cpp src/engines.cpp src/session.cpp src/session_statistics.cpp src/session_widths.cpp src/low_storage.cpp src/transverse.cpp src/session_edge.cpp src/dense.cpp src/workspace.cpp src/problem.cpp diff --git a/native/src/general/kernels.hpp b/native/src/general/kernels.hpp index 3f2b6326f..e61a31148 100644 --- a/native/src/general/kernels.hpp +++ b/native/src/general/kernels.hpp @@ -61,11 +61,6 @@ #include #include #include -#ifdef __APPLE__ -#include -#elif defined(__GLIBC__) -#include -#endif namespace solprop_native { namespace general {} // The helpers are found as an unnamed namespace's would be, under their own name so @@ -221,49 +216,6 @@ void apply_generator(const ExecutionSpace& space, const std::size_t n, template KOKKOS_INLINE_FUNCTION void capture_anchor(const Ts&... /*unused*/) {} -// Net process allocator growth includes opaque vendor FFT plan/workspace -// allocations. It is measured separately from Kokkos Views and is not a peak bound. -inline std::size_t heap_bytes() { -#ifdef __APPLE__ - malloc_statistics_t statistics{}; - malloc_zone_statistics(nullptr, &statistics); - return statistics.size_in_use; -#elif defined(__GLIBC__) - // mallinfo2 is glibc 2.33 or later; a wheel built against an older glibc (the - // manylinux_2_28 baseline) finds it at run time when the system provides it. Its - // layout is ten size_t counters, in glibc's declared order. - struct Mallinfo2 { - std::size_t arena, ordblks, smblks, hblks, hblkhd, usmblks, fsmblks, uordblks, - fordblks, keepcost; - }; - using Query = Mallinfo2 (*)(); - static const auto query = reinterpret_cast(dlsym(RTLD_DEFAULT, "mallinfo2")); - if (query == nullptr) - return 0; - const auto statistics = query(); - return statistics.uordblks + statistics.hblkhd; -#else - // Neither allocator interface exists (musl, Windows): the ledger row then measures - // nothing, and construction must still succeed. - return 0; -#endif -} -inline std::size_t device_free_bytes() { -#ifdef KOKKOS_ENABLE_CUDA - std::size_t available{}, total{}; - if (cudaMemGetInfo(&available, &total) != cudaSuccess) - throw std::runtime_error("cudaMemGetInfo failed during FFT planning"); - return available; -#elif defined(KOKKOS_ENABLE_HIP) - std::size_t available{}, total{}; - if (hipMemGetInfo(&available, &total) != hipSuccess) - throw std::runtime_error("hipMemGetInfo failed during FFT planning"); - return available; -#else - return 0; -#endif -} - using ComplexView2 = Kokkos::View; using ComplexView3 = Kokkos::View; using RealView2 = Kokkos::View; diff --git a/native/src/general/memory.cpp b/native/src/general/memory.cpp new file mode 100644 index 000000000..8ea3dab7b --- /dev/null +++ b/native/src/general/memory.cpp @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#include "memory.hpp" + +#include + +#include +#include + +#ifdef KOKKOS_ENABLE_CUDA +#include +#elif defined(KOKKOS_ENABLE_HIP) +#include +#endif +#ifdef __APPLE__ +#include +#elif defined(__GLIBC__) +#include +#endif + +namespace solprop_native::general { + +std::size_t heap_bytes() { +#ifdef __APPLE__ + malloc_statistics_t statistics{}; + malloc_zone_statistics(nullptr, &statistics); + return statistics.size_in_use; +#elif defined(__GLIBC__) + // mallinfo2 is glibc 2.33 or later; a wheel built against an older glibc (the + // manylinux_2_28 baseline) finds it at run time when the system provides it. Its + // layout is ten size_t counters, in glibc's declared order. + struct Mallinfo2 { + std::size_t arena, ordblks, smblks, hblks, hblkhd, usmblks, fsmblks, uordblks, + fordblks, keepcost; + }; + using Query = Mallinfo2 (*)(); + static const auto query = reinterpret_cast(dlsym(RTLD_DEFAULT, "mallinfo2")); + if (query == nullptr) + return 0; + const auto statistics = query(); + return statistics.uordblks + statistics.hblkhd; +#else + // Neither allocator interface exists (musl, Windows): the ledger row then measures + // nothing, and construction must still succeed. + return 0; +#endif +} +std::size_t device_free_bytes() { +#ifdef KOKKOS_ENABLE_CUDA + std::size_t available{}, total{}; + if (cudaMemGetInfo(&available, &total) != cudaSuccess) + throw std::runtime_error("cudaMemGetInfo failed during FFT planning"); + return available; +#elif defined(KOKKOS_ENABLE_HIP) + std::size_t available{}, total{}; + if (hipMemGetInfo(&available, &total) != hipSuccess) + throw std::runtime_error("hipMemGetInfo failed during FFT planning"); + return available; +#else + return 0; +#endif +} + +} // namespace solprop_native::general diff --git a/native/src/general/memory.hpp b/native/src/general/memory.hpp new file mode 100644 index 000000000..dd65d7515 --- /dev/null +++ b/native/src/general/memory.hpp @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +// The host allocator queries of the general workspace's memory ledger: host code, run +// once around planning. +namespace solprop_native::general { + +// Net process allocator growth includes opaque vendor FFT plan/workspace +// allocations. It is measured separately from Kokkos Views and is not a peak bound. +std::size_t heap_bytes(); +/** The device's free memory in bytes; zero on a host build. */ +std::size_t device_free_bytes(); + +} // namespace solprop_native::general diff --git a/native/src/general/setup.cpp b/native/src/general/setup.cpp index 05e425bf0..0c8a12431 100644 --- a/native/src/general/setup.cpp +++ b/native/src/general/setup.cpp @@ -5,6 +5,7 @@ // The general workspace's construction: its allocations, uploads, FFT plans and // memory ledger. +#include "memory.hpp" #include "workspace.hpp" namespace solprop_native { diff --git a/native/src/general/statistics.cpp b/native/src/general/statistics.cpp index ed4b33402..cfe026d09 100644 --- a/native/src/general/statistics.cpp +++ b/native/src/general/statistics.cpp @@ -4,6 +4,7 @@ // The general workspace's statistics and its boundary monitor's readings. +#include "memory.hpp" #include "workspace.hpp" namespace solprop_native { From 8ca19597b540ec8182f583061de7683bee54d2f1 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:06:53 +0000 Subject: [PATCH 09/21] Share the setup-time extent checks and uploads in setup_data.hpp general/kernels.hpp's checked_product, checked_sum, require_extent and require_allocation, general_modal.cpp's finite and the host-vector upload that general_modal.cpp and coupled_flow.cpp each defined move to setup_data.hpp. dispersive.cpp's product_of, the same check with the same message as checked_product, gives way to it. The helpers whose messages name their unit (coupled flow's and general modal's product and require_size, dispersive's require_size and its typed uploads) stay where they are. Setup-time code only; no behaviour changes. Co-Authored-By: Claude Opus 5.5 --- native/include/solprop_native/setup_data.hpp | 70 ++++++++++++++++++++ native/src/coupled_flow.cpp | 17 +---- native/src/dispersive.cpp | 42 ++++++------ native/src/general/kernels.hpp | 26 +------- native/src/general_modal.cpp | 20 +----- 5 files changed, 93 insertions(+), 82 deletions(-) create mode 100644 native/include/solprop_native/setup_data.hpp diff --git a/native/include/solprop_native/setup_data.hpp b/native/include/solprop_native/setup_data.hpp new file mode 100644 index 000000000..0f244363a --- /dev/null +++ b/native/include/solprop_native/setup_data.hpp @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include "solprop_native/execution.hpp" +#include "solprop_native/graph.hpp" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// Setup-time checks and copies of a problem's host data, shared by the units that build +// a workspace's views: extents and their size arithmetic, finiteness, and one copy of a +// host vector into a view. Host code, run before any step. +namespace solprop_native { + +inline void require_extent(const std::size_t actual, const std::size_t expected, + const char* name) { + if (actual != expected) + throw std::invalid_argument(std::string(name) + " has the wrong extent"); +} + +inline std::size_t checked_product(const std::size_t left, const std::size_t right, + const char* name) { + if (left != 0 && right > std::numeric_limits::max() / left) + throw std::length_error(std::string(name) + " extent overflows size_t"); + return left * right; +} + +inline std::size_t checked_sum(const std::size_t left, const std::size_t right, + const char* name) { + if (right > std::numeric_limits::max() - left) + throw std::length_error(std::string(name) + " extent overflows size_t"); + return left + right; +} + +inline void require_allocation(const std::size_t elements, + const std::size_t element_bytes, const char* name) { + static_cast(checked_product(elements, element_bytes, name)); +} + +template bool finite(const std::vector& values) { + return std::all_of(values.begin(), values.end(), [](const Value& value) { + if constexpr (std::is_same_v) + return std::isfinite(value.real()) && std::isfinite(value.imag()); + else + return std::isfinite(value); + }); +} + +template +void upload(const ExecutionSpace& execution, const View& destination, + const std::vector& source) { + auto mirror = Kokkos::create_mirror_view(destination); + auto* data = mirror.data(); + for (std::size_t index = 0; index < source.size(); ++index) + data[index] = source[index]; + graph::deep_copy(execution, destination, mirror); +} + +} // namespace solprop_native diff --git a/native/src/coupled_flow.cpp b/native/src/coupled_flow.cpp index b43bb591d..997c585b1 100644 --- a/native/src/coupled_flow.cpp +++ b/native/src/coupled_flow.cpp @@ -6,6 +6,7 @@ #include "solprop_native/launch.hpp" #include "solprop_native/refuse.hpp" +#include "solprop_native/setup_data.hpp" #include #include @@ -31,22 +32,6 @@ void require_size(const std::size_t actual, const std::size_t expected, refuse("coupled flow ", name, " has ", actual, " values, expected ", expected); } -bool finite(const std::vector& values) { - return std::ranges::all_of(values, [](const Complex& value) { - return std::isfinite(value.real()) && std::isfinite(value.imag()); - }); -} - -template -void upload(const ExecutionSpace& execution, const View& destination, - const std::vector& source) { - auto mirror = Kokkos::create_mirror_view(destination); - auto* data = mirror.data(); - for (std::size_t index = 0; index < source.size(); ++index) - data[index] = source[index]; - graph::deep_copy(execution, destination, mirror); -} - KOKKOS_INLINE_FUNCTION Complex flow_exponential(const Complex value) { const double scale = Kokkos::exp(value.real()); return {scale * Kokkos::cos(value.imag()), scale * Kokkos::sin(value.imag())}; diff --git a/native/src/dispersive.cpp b/native/src/dispersive.cpp index b3281544f..72f2ac5e4 100644 --- a/native/src/dispersive.cpp +++ b/native/src/dispersive.cpp @@ -6,6 +6,7 @@ #include "solprop_native/gemm.hpp" #include "solprop_native/launch.hpp" +#include "solprop_native/setup_data.hpp" #include "solprop_native/slab.hpp" #include @@ -23,13 +24,6 @@ namespace { using Matrix = Kokkos::View>; -std::size_t product_of(const std::size_t left, const std::size_t right, - const char* name) { - if (left != 0 && right > std::numeric_limits::max() / left) - throw std::length_error(std::string(name) + " extent overflows size_t"); - return left * right; -} - void require_size(const std::size_t actual, const std::size_t expected, const char* name) { if (actual != expected) @@ -615,7 +609,7 @@ void require_dispersive(const Problem& problem) { require_size(problem.dispersive.input_scale.size(), bins, "input_scale"); require_size(problem.dispersive.output_scale.size(), bins, "output_scale"); require_size(problem.dispersive.pair_constants.size(), - product_of(2, channels, "pair constants"), "pair_constants"); + checked_product(2, channels, "pair constants"), "pair_constants"); const auto& offsets = problem.dispersive.group_offsets; const auto& members = problem.dispersive.group_members; if (offsets.size() < 2 || offsets.front() != 0 || offsets.back() != members.size() || @@ -656,33 +650,37 @@ void require_dispersive(const Problem& problem) { const std::size_t fields = problem.dispersive.symmetric ? groups : 2 * groups; const std::size_t products = problem.dispersive.symmetric ? groups : distinct.size() + 2 * groups; - require_size(problem.dispersive.inputs.size(), product_of(fields, bins, "inputs"), - "inputs"); + require_size(problem.dispersive.inputs.size(), + checked_product(fields, bins, "inputs"), "inputs"); const std::size_t terms = problem.dispersive.term_rows.size(); if (terms == 0) throw std::invalid_argument("the dispersive response needs an output term"); require_size(problem.dispersive.term_channels.size(), terms, "term_channels"); require_size(problem.dispersive.term_multipliers.size(), - product_of(terms, bins, "term multipliers"), "term_multipliers"); + checked_product(terms, bins, "term multipliers"), "term_multipliers"); for (std::size_t term = 0; term < terms; ++term) if (problem.dispersive.term_rows[term] >= products || problem.dispersive.term_channels[term] > channels + mixtures) throw std::invalid_argument("a dispersive term names no product or factor"); require_size(problem.dispersive.quartics.size(), - mixtures == 0 ? 0 : product_of(channels, bins, "quartics"), "quartics"); + mixtures == 0 ? 0 : checked_product(channels, bins, "quartics"), + "quartics"); // The products and their spectra, and the node slots: the general route's chunk, or // at most the modal columns. - const auto rows = general ? product_of(problem.general.node_chunk, - problem.general.components, "dispersive rows") - : problem.n_columns(); + const auto rows = general + ? checked_product(problem.general.node_chunk, + problem.general.components, "dispersive rows") + : problem.n_columns(); const auto field = - product_of(product_of(products + fields, rows, "dispersive products"), - problem.dims.n_to, "dispersive products"); - static_cast(product_of(field, sizeof(Complex), "dispersive product bytes")); - static_cast(product_of( - product_of(2 * problem.dims.node_chunk, problem.effective_node_pol(), "slots"), - product_of(problem.dims.n_to, sizeof(Complex), "slot bytes"), - "dispersive slots")); + checked_product(checked_product(products + fields, rows, "dispersive products"), + problem.dims.n_to, "dispersive products"); + static_cast( + checked_product(field, sizeof(Complex), "dispersive product bytes")); + static_cast( + checked_product(checked_product(2 * problem.dims.node_chunk, + problem.effective_node_pol(), "slots"), + checked_product(problem.dims.n_to, sizeof(Complex), "slot bytes"), + "dispersive slots")); } void dispersive_mixture_rows( diff --git a/native/src/general/kernels.hpp b/native/src/general/kernels.hpp index e61a31148..5dce67da1 100644 --- a/native/src/general/kernels.hpp +++ b/native/src/general/kernels.hpp @@ -30,6 +30,7 @@ #include "solprop_native/plasma.hpp" #include "solprop_native/raman_fft.hpp" #include "solprop_native/refuse.hpp" +#include "solprop_native/setup_data.hpp" #include "solprop_native/slab.hpp" #include "solprop_native/vmath.hpp" #include "solprop_native/z_table.hpp" @@ -1014,31 +1015,6 @@ void copy_vector(const ExecutionSpace& execution_space, const View& destination, graph::deep_copy(execution_space, destination, mirror); } -inline void require_extent(const std::size_t actual, const std::size_t expected, - const char* name) { - if (actual != expected) - throw std::invalid_argument(std::string(name) + " has the wrong extent"); -} - -inline std::size_t checked_product(const std::size_t left, const std::size_t right, - const char* name) { - if (left != 0 && right > std::numeric_limits::max() / left) - throw std::length_error(std::string(name) + " extent overflows size_t"); - return left * right; -} - -inline std::size_t checked_sum(const std::size_t left, const std::size_t right, - const char* name) { - if (right > std::numeric_limits::max() - left) - throw std::length_error(std::string(name) + " extent overflows size_t"); - return left + right; -} - -inline void require_allocation(const std::size_t elements, - const std::size_t element_bytes, const char* name) { - static_cast(checked_product(elements, element_bytes, name)); -} - // The samples of one tile of the device trapezoid plasma's two-level scans (unused on // a host build). [[maybe_unused]] inline constexpr std::size_t plasma_tile_samples = 1024; diff --git a/native/src/general_modal.cpp b/native/src/general_modal.cpp index 43547069b..0e24887c6 100644 --- a/native/src/general_modal.cpp +++ b/native/src/general_modal.cpp @@ -8,6 +8,7 @@ #include "solprop_native/fft_plan.hpp" #include "solprop_native/launch.hpp" #include "solprop_native/refuse.hpp" +#include "solprop_native/setup_data.hpp" #include #include @@ -71,25 +72,6 @@ void require_size(const std::size_t actual, const std::size_t expected, refuse("general modal ", name, " has ", actual, " values, expected ", expected); } -template bool finite(const std::vector& values) { - return std::all_of(values.begin(), values.end(), [](const Value& value) { - if constexpr (std::is_same_v) - return std::isfinite(value.real()) && std::isfinite(value.imag()); - else - return std::isfinite(value); - }); -} - -template -void upload(const ExecutionSpace& execution, const View& destination, - const std::vector& source) { - auto mirror = Kokkos::create_mirror_view(destination); - auto* data = mirror.data(); - for (std::size_t index = 0; index < source.size(); ++index) - data[index] = source[index]; - graph::deep_copy(execution, destination, mirror); -} - constexpr std::size_t fourth_power(const std::size_t value) { return value * value * value * value; } From 65b8efbcc368affc092ce94a225985b2fde9acb7 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:09:12 +0000 Subject: [PATCH 10/21] Name the spatial statistics flags beside StatisticsFlag session_statistics.cpp named spatial_statistics' flag bits while general/statistics.cpp wrote them as literals, and its plane_moments repeated general/workspace.hpp's plane_moment_slots. The named flags, a mask of them all and plane_moment_slots now sit beside StatisticsFlag in problem.hpp, and both units and statistics_plan.hpp's comments use them. Constants only. Co-Authored-By: Claude Opus 5.5 --- native/include/solprop_native/problem.hpp | 17 ++++++++++++++ .../solprop_native/statistics_plan.hpp | 9 ++++---- native/src/general/statistics.cpp | 23 +++++++++++-------- native/src/general/workspace.hpp | 3 --- native/src/session_statistics.cpp | 14 +---------- 5 files changed, 36 insertions(+), 30 deletions(-) diff --git a/native/include/solprop_native/problem.hpp b/native/include/solprop_native/problem.hpp index 99c381e1b..c7ca287c5 100644 --- a/native/include/solprop_native/problem.hpp +++ b/native/include/solprop_native/problem.hpp @@ -47,6 +47,23 @@ enum StatisticsFlag : std::uint64_t { statistics_plasma_integration = 1ULL << 12U, }; +// `GeneralState::spatial_statistics`' scalar flags. +inline constexpr std::uint64_t spatial_energy = 1; +inline constexpr std::uint64_t spatial_axis = 2; +inline constexpr std::uint64_t spatial_peak = 4; +inline constexpr std::uint64_t spatial_spectrum = 8; +/** The widths' fluence block: the signed fluence at each node, the axis fluence and + the largest magnitude. */ +inline constexpr std::uint64_t spatial_fluence = 16; +/** A Cartesian plane's fluence moments, which hold its scaled profile for the trace. */ +inline constexpr std::uint64_t spatial_moments = 32; +/** Every spatial flag. */ +inline constexpr std::uint64_t spatial_flags = spatial_energy | spatial_axis | + spatial_peak | spatial_spectrum | + spatial_fluence | spatial_moments; +// What a Cartesian plane's widths read of its fluence (`plane_fluence_moments`). +inline constexpr std::size_t plane_moment_slots = 8; + /** The widest band rule solprop designs (SEP 18 section 5): K <= 8, so 9 taps. */ inline constexpr std::size_t plasma_max_taps = 9; diff --git a/native/include/solprop_native/statistics_plan.hpp b/native/include/solprop_native/statistics_plan.hpp index 045f8513a..c6bb4e012 100644 --- a/native/include/solprop_native/statistics_plan.hpp +++ b/native/include/solprop_native/statistics_plan.hpp @@ -268,13 +268,14 @@ namespace widths { const std::vector& samples, double e2_fraction); -/** The widths of a separable or radial fluence block, `spatial_statistics` flag 16: - the signed fluence at each node, the axis fluence and the largest magnitude. */ +/** The widths of a separable or radial fluence block, `spatial_statistics` flag + `spatial_fluence`: the signed fluence at each node, the axis fluence and the + largest magnitude. */ [[nodiscard]] std::array of_block(const Plan& plan, const std::vector& block); -/** A Cartesian plane's widths from its moments, flag 32, and the trace `state` forms - of the scaled fluence those moments left. */ +/** A Cartesian plane's widths from its moments, flag `spatial_moments`, and the + trace `state` forms of the scaled fluence those moments left. */ [[nodiscard]] std::array of_plane(GeneralState& state, const Plan& plan, const std::vector& moments); diff --git a/native/src/general/statistics.cpp b/native/src/general/statistics.cpp index cfe026d09..66d282f90 100644 --- a/native/src/general/statistics.cpp +++ b/native/src/general/statistics.cpp @@ -1545,12 +1545,13 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla "spatial statistics need an initial field and spatial geometry"); if (flux_.square.empty()) throw std::logic_error("spatial statistics need the run's plane measure"); - if ((flags & ~std::uint64_t{63}) != 0 || !std::isfinite(intensity_factor) || + if ((flags & ~spatial_flags) != 0 || !std::isfinite(intensity_factor) || intensity_factor <= 0.0 || !std::isfinite(z)) throw std::invalid_argument( "invalid spatial statistic flags, intensity factor or position"); - if ((flags & 32U) != 0 && (!problem_.free_geometry() || problem_.slab_geometry() || - problem_.spatial.n_y < 2)) + if ((flags & spatial_moments) != 0 && + (!problem_.free_geometry() || problem_.slab_geometry() || + problem_.spatial.n_y < 2)) throw std::invalid_argument("a plane's fluence moments need a Cartesian plane"); constexpr double lowest = std::numeric_limits::lowest(); std::size_t packed_size = 0; @@ -1562,17 +1563,18 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla packed_size = checked_sum(packed_size, count, "spatial statistics output"); return offset; }; - const auto energy_offset = reserve(1, 1); - const auto axis_peak_offset = reserve(2, 1); - const auto peak_offset = reserve(4, 1); - const auto spectrum_offset = reserve(8, problem_.dims.n_omega); + const auto energy_offset = reserve(spatial_energy, 1); + const auto axis_peak_offset = reserve(spatial_axis, 1); + const auto peak_offset = reserve(spatial_peak, 1); + const auto spectrum_offset = reserve(spatial_spectrum, problem_.dims.n_omega); // Widths read the signed fluence at the nodes, the separately reconstructed axis, // which only a radial width reads, and the largest magnitude, which fixes the // fluence's rounding budget (solprop SEP 34). const auto fluence_offset = - reserve(16, checked_sum(problem_.dims.n_nodes, 2, "spatial statistics output")); + reserve(spatial_fluence, + checked_sum(problem_.dims.n_nodes, 2, "spatial statistics output")); // On a Cartesian plane the widths reduce here instead, and the moments cross. - const auto moments_offset = reserve(32, plane_moment_slots); + const auto moments_offset = reserve(spatial_moments, plane_moment_slots); if (packed_size == 0) return {}; if (statistics_packed_.size() < packed_size) { @@ -1581,7 +1583,8 @@ std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t fla } // Every field statistic is the axial flux of the medium at the plane z. flux_at(z); - const bool local = (flags & (2U | 4U | 16U | 32U)) != 0; + const bool local = + (flags & (spatial_axis | spatial_peak | spatial_fluence | spatial_moments)) != 0; if (local) require_flux_closure(); const auto packed = statistics_packed_; diff --git a/native/src/general/workspace.hpp b/native/src/general/workspace.hpp index 410d0ee77..e01a1e87a 100644 --- a/native/src/general/workspace.hpp +++ b/native/src/general/workspace.hpp @@ -19,9 +19,6 @@ namespace solprop_native { // Refuse a problem the workspace cannot run, before it allocates (checks.cpp). void require_general_problem(const Problem& problem); -// What a Cartesian plane's widths read of its fluence (`plane_fluence_moments`). -inline constexpr std::size_t plane_moment_slots = 8; - class GeneralWorkspace { public: explicit GeneralWorkspace(Problem problem); diff --git a/native/src/session_statistics.cpp b/native/src/session_statistics.cpp index 4360e2c7d..a5beb0c98 100644 --- a/native/src/session_statistics.cpp +++ b/native/src/session_statistics.cpp @@ -54,18 +54,6 @@ constexpr std::uint64_t temporal_integration = statistics_plasma_integration; /** The parts of the plasma integration diagnostics, packed last. */ constexpr std::size_t integration_parts = 6; -// `GeneralState::spatial_statistics`' scalar flags. -constexpr std::uint64_t spatial_energy = 1; -constexpr std::uint64_t spatial_axis = 2; -constexpr std::uint64_t spatial_peak = 4; -constexpr std::uint64_t spatial_spectrum = 8; -/** The widths' fluence block: the signed fluence at each node, the axis fluence and - the largest magnitude. */ -constexpr std::uint64_t spatial_fluence = 16; -/** A Cartesian plane's fluence moments, which hold its scaled profile for the trace. */ -constexpr std::uint64_t spatial_moments = 32; -constexpr std::size_t plane_moments = 8; - // `GeneralState::aperture_statistics`' flags (far_field.hpp), packed in this order. constexpr std::uint64_t far_beam = 1ULL << 5U; constexpr std::uint64_t far_absorbed = 1ULL << 7U; @@ -509,7 +497,7 @@ void spatial(GeneralState& state, const Plan& plan, const double z, const bool plane = plan.widths == Widths::plane; const auto packed = state.spatial_statistics( plane ? spatial_moments : spatial_fluence, plan.intensity_factor, z); - if (packed.size() != (plane ? plane_moments : plan.n_nodes + 2)) + if (packed.size() != (plane ? plane_moment_slots : plan.n_nodes + 2)) throw std::logic_error("a fluence pass packed the wrong number of values"); const auto values = plane ? widths::of_plane(state, plan, packed) : widths::of_block(plan, packed); From b7a7568fb2919b81af7cdfa48a51fef3e54fa511 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:10:00 +0000 Subject: [PATCH 11/21] Rename src/general/workspace.hpp to general_workspace.hpp Its class is GeneralWorkspace, and the name no longer reads as the public include/solprop_native/workspace.hpp. Includes and its strict-TU refusal follow. Co-Authored-By: Claude Opus 5.5 --- native/src/general/checks.cpp | 2 +- native/src/general/{workspace.hpp => general_workspace.hpp} | 2 +- native/src/general/setup.cpp | 2 +- native/src/general/state.cpp | 2 +- native/src/general/statistics.cpp | 2 +- native/src/general/trial.cpp | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) rename native/src/general/{workspace.hpp => general_workspace.hpp} (99%) diff --git a/native/src/general/checks.cpp b/native/src/general/checks.cpp index 0d18d3f2b..6b3ef59a6 100644 --- a/native/src/general/checks.cpp +++ b/native/src/general/checks.cpp @@ -4,7 +4,7 @@ // What the general workspace refuses before it allocates, and a trial's scalars. -#include "workspace.hpp" +#include "general_workspace.hpp" #include diff --git a/native/src/general/workspace.hpp b/native/src/general/general_workspace.hpp similarity index 99% rename from native/src/general/workspace.hpp rename to native/src/general/general_workspace.hpp index e01a1e87a..fb083f67e 100644 --- a/native/src/general/workspace.hpp +++ b/native/src/general/general_workspace.hpp @@ -8,7 +8,7 @@ // src/general/: checks.cpp, setup.cpp, trial.cpp (the hot set, kept in one unit so // that its inlining is the compiler's whole view), statistics.cpp and state.cpp. #ifdef SOLPROP_NATIVE_STRICT_TU -#error "general/workspace.hpp contracts: a strict translation unit must not include it" +#error "general_workspace.hpp contracts: a strict translation unit must not include it" #endif #include "kernels.hpp" diff --git a/native/src/general/setup.cpp b/native/src/general/setup.cpp index 0c8a12431..dcf8dd840 100644 --- a/native/src/general/setup.cpp +++ b/native/src/general/setup.cpp @@ -5,8 +5,8 @@ // The general workspace's construction: its allocations, uploads, FFT plans and // memory ledger. +#include "general_workspace.hpp" #include "memory.hpp" -#include "workspace.hpp" namespace solprop_native { diff --git a/native/src/general/state.cpp b/native/src/general/state.cpp index 9b8ef57e6..6fbf194b2 100644 --- a/native/src/general/state.cpp +++ b/native/src/general/state.cpp @@ -4,7 +4,7 @@ // GeneralState over its workspace: host transfers, closing and the forwarders. -#include "workspace.hpp" +#include "general_workspace.hpp" namespace solprop_native { diff --git a/native/src/general/statistics.cpp b/native/src/general/statistics.cpp index 66d282f90..632c4d334 100644 --- a/native/src/general/statistics.cpp +++ b/native/src/general/statistics.cpp @@ -4,8 +4,8 @@ // The general workspace's statistics and its boundary monitor's readings. +#include "general_workspace.hpp" #include "memory.hpp" -#include "workspace.hpp" namespace solprop_native { diff --git a/native/src/general/trial.cpp b/native/src/general/trial.cpp index e40a5ceeb..35cac1b5f 100644 --- a/native/src/general/trial.cpp +++ b/native/src/general/trial.cpp @@ -6,7 +6,7 @@ // Adams and low-storage marches and the split-step absorber, in one unit so that the // compiler sees every call between them. -#include "workspace.hpp" +#include "general_workspace.hpp" namespace solprop_native { From 9c2a86227c6d5007cfb899759670960e9b102562 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:10:30 +0000 Subject: [PATCH 12/21] Rename dense.hpp, the session::Dense interface, to session_dense.hpp It declares the session's dense-output interface; the name frees dense.hpp for the dense:: weights and engines. Co-Authored-By: Claude Opus 5.5 --- native/include/solprop_native/dense_output.hpp | 2 +- native/include/solprop_native/session.hpp | 2 +- native/include/solprop_native/{dense.hpp => session_dense.hpp} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename native/include/solprop_native/{dense.hpp => session_dense.hpp} (100%) diff --git a/native/include/solprop_native/dense_output.hpp b/native/include/solprop_native/dense_output.hpp index a8e9fd0ae..73a5c8e2a 100644 --- a/native/include/solprop_native/dense_output.hpp +++ b/native/include/solprop_native/dense_output.hpp @@ -4,8 +4,8 @@ #pragma once -#include "solprop_native/dense.hpp" #include "solprop_native/general_state.hpp" +#include "solprop_native/session_dense.hpp" #include "solprop_native/stage_tables.hpp" #include diff --git a/native/include/solprop_native/session.hpp b/native/include/solprop_native/session.hpp index 543adaa86..fb4e782ce 100644 --- a/native/include/solprop_native/session.hpp +++ b/native/include/solprop_native/session.hpp @@ -5,8 +5,8 @@ #pragma once #include "solprop_native/adams_march.hpp" -#include "solprop_native/dense.hpp" #include "solprop_native/rk_march.hpp" +#include "solprop_native/session_dense.hpp" #include "solprop_native/session_resident.hpp" #include "solprop_native/statistics_plan.hpp" #include "solprop_native/transverse.hpp" diff --git a/native/include/solprop_native/dense.hpp b/native/include/solprop_native/session_dense.hpp similarity index 100% rename from native/include/solprop_native/dense.hpp rename to native/include/solprop_native/session_dense.hpp From dd4f8dd62c9982ab9846905ef88161eeec3350ab Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:11:10 +0000 Subject: [PATCH 13/21] Rename dense_output.hpp to dense.hpp, the header dense.cpp implements It declares the dense:: weights and the AdamsDense and RungeKuttaDense engines that dense.cpp defines. Co-Authored-By: Claude Opus 5.5 --- native/include/solprop_native/{dense_output.hpp => dense.hpp} | 0 native/include/solprop_native/session_dense.hpp | 2 +- native/src/bindings.cpp | 2 +- native/src/dense.cpp | 3 ++- native/src/workspace.cpp | 2 +- 5 files changed, 5 insertions(+), 4 deletions(-) rename native/include/solprop_native/{dense_output.hpp => dense.hpp} (100%) diff --git a/native/include/solprop_native/dense_output.hpp b/native/include/solprop_native/dense.hpp similarity index 100% rename from native/include/solprop_native/dense_output.hpp rename to native/include/solprop_native/dense.hpp diff --git a/native/include/solprop_native/session_dense.hpp b/native/include/solprop_native/session_dense.hpp index 96e6b0b63..ea30cf2d0 100644 --- a/native/include/solprop_native/session_dense.hpp +++ b/native/include/solprop_native/session_dense.hpp @@ -16,7 +16,7 @@ * observation neither rewinds the controller nor advances the history. * * The implementations (`engines::AdamsDense`, `engines::RungeKuttaDense`, - * dense_output.hpp) are solprop's own observations transcribed: the Adams march's + * dense.hpp) are solprop's own observations transcribed: the Adams march's * interpolant over the last accepted step (`HostHistory.observe`), and DOPRI5's * continuous extension of Hairer, Norsett and Wanner II.6 carried to the plane by the * linear flow (`RungeKuttaDense.observe`). diff --git a/native/src/bindings.cpp b/native/src/bindings.cpp index 3c5db0af4..bd094c7cf 100644 --- a/native/src/bindings.cpp +++ b/native/src/bindings.cpp @@ -8,7 +8,7 @@ #include "solprop_native/adams_march.hpp" #include "solprop_native/cpu_execution.hpp" #include "solprop_native/crossing.hpp" -#include "solprop_native/dense_output.hpp" +#include "solprop_native/dense.hpp" #include "solprop_native/engines.hpp" #include "solprop_native/execution.hpp" #include "solprop_native/fftw_wisdom.hpp" diff --git a/native/src/dense.cpp b/native/src/dense.cpp index fa2013310..bc7d2b330 100644 --- a/native/src/dense.cpp +++ b/native/src/dense.cpp @@ -2,7 +2,8 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/dense_output.hpp" +#include "solprop_native/dense.hpp" + #include "solprop_native/engines.hpp" #include "solprop_native/lagrange_basis.hpp" diff --git a/native/src/workspace.cpp b/native/src/workspace.cpp index 20a7ae740..fda3b31b8 100644 --- a/native/src/workspace.cpp +++ b/native/src/workspace.cpp @@ -4,7 +4,7 @@ #include "solprop_native/workspace.hpp" -#include "solprop_native/dense_output.hpp" +#include "solprop_native/dense.hpp" #include "solprop_native/refuse.hpp" #include "solprop_native/run_records.hpp" From 3d6038f7ee85d0d85c9c018754c17828ea5ce389 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:12:20 +0000 Subject: [PATCH 14/21] Rename session_statistics.cpp to statistics_plan.cpp It implements statistics_plan.hpp (namespace statistics::). It stays a strict file: SOLPROP_NATIVE_STRICT_SOURCES, native/AGENTS.md's list and the licence test's copy of it follow; the namespace, which ci/check_strict_fma.py matches, is unchanged. Co-Authored-By: Claude Opus 5.5 --- native/AGENTS.md | 2 +- native/CMakeLists.txt | 4 ++-- native/include/solprop_native/statistics_plan.hpp | 2 +- native/src/{session_statistics.cpp => statistics_plan.cpp} | 0 tests/backends/test_native_floating_point_licence.py | 2 +- tests/backends/test_native_widths.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) rename native/src/{session_statistics.cpp => statistics_plan.cpp} (100%) diff --git a/native/AGENTS.md b/native/AGENTS.md index 96a009d1d..f78f30aa5 100644 --- a/native/AGENTS.md +++ b/native/AGENTS.md @@ -156,7 +156,7 @@ the gate. the loops that need it. No umbrella fast-math, `nnan`/`ninf`, `nsz`, approximate functions or reciprocals, or flush-to-zero; the `__FAST_MATH__` refusal stays. The marches' and the session's strict files (`adams_march.cpp`, `watch.cpp`, - `rk_march.cpp`, `session.cpp`, `session_statistics.cpp`, `session_widths.cpp`, + `rk_march.cpp`, `session.cpp`, `statistics_plan.cpp`, `session_widths.cpp`, `low_storage.cpp`, `transverse.cpp`, `session_edge.cpp`, `dense.cpp`) are compiled with `-ffp-contract=off` and carry `#pragma GCC optimize("fp-contract=off")`, so that GCC never inlines them into a contracting caller. Their probes are checked in the diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 5bcefb4a3..75a094cd4 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -814,7 +814,7 @@ add_library(solprop_native_core STATIC src/cpu_execution.cpp src/crossing.cpp src/general/checks.cpp src/general/setup.cpp src/general/trial.cpp src/general/statistics.cpp src/general/state.cpp src/general/memory.cpp src/general_modal.cpp src/coupled_flow.cpp src/dispersive.cpp src/far_field.cpp src/adams_march.cpp src/watch.cpp src/rk_march.cpp src/engines.cpp src/session.cpp - src/session_statistics.cpp src/session_widths.cpp src/low_storage.cpp src/transverse.cpp + src/statistics_plan.cpp src/session_widths.cpp src/low_storage.cpp src/transverse.cpp src/session_edge.cpp src/dense.cpp src/workspace.cpp src/problem.cpp src/reference_timing.cpp src/fftw_wisdom.cpp) # solprop SEP 39, SEP 45 and SEP 47: the Adams march's decisions, the boundary watch's @@ -827,7 +827,7 @@ add_library(solprop_native_core STATIC src/cpu_execution.cpp src/crossing.cpp # SOLPROP_NATIVE_STRICT_TU marks them, so that the general workspace's contracting # headers (src/general/) refuse to be included there. set(SOLPROP_NATIVE_STRICT_SOURCES src/adams_march.cpp src/watch.cpp src/rk_march.cpp - src/session.cpp src/session_statistics.cpp src/session_widths.cpp src/low_storage.cpp + src/session.cpp src/statistics_plan.cpp src/session_widths.cpp src/low_storage.cpp src/transverse.cpp src/session_edge.cpp src/dense.cpp) set_source_files_properties(${SOLPROP_NATIVE_STRICT_SOURCES} PROPERTIES COMPILE_DEFINITIONS SOLPROP_NATIVE_STRICT_TU) diff --git a/native/include/solprop_native/statistics_plan.hpp b/native/include/solprop_native/statistics_plan.hpp index c6bb4e012..17ce56da9 100644 --- a/native/include/solprop_native/statistics_plan.hpp +++ b/native/include/solprop_native/statistics_plan.hpp @@ -238,7 +238,7 @@ struct Plan { [[nodiscard]] double half_width(const double* t, const double* power, std::size_t n, bool outer); -/** `a b + c` as session_statistics.cpp rounds it (solprop #135's witness); never +/** `a b + c` as statistics_plan.cpp rounds it (solprop #135's witness); never inlined, as `session::strict_probe`. */ [[nodiscard, gnu::noinline]] double strict_probe(double a, double b, double c); diff --git a/native/src/session_statistics.cpp b/native/src/statistics_plan.cpp similarity index 100% rename from native/src/session_statistics.cpp rename to native/src/statistics_plan.cpp diff --git a/tests/backends/test_native_floating_point_licence.py b/tests/backends/test_native_floating_point_licence.py index 1b6cb1a85..ab4e5ba4c 100644 --- a/tests/backends/test_native_floating_point_licence.py +++ b/tests/backends/test_native_floating_point_licence.py @@ -148,7 +148,7 @@ def test_a_protected_marker_still_does_something(): "src/watch.cpp", "src/rk_march.cpp", "src/session.cpp", - "src/session_statistics.cpp", + "src/statistics_plan.cpp", "src/session_widths.cpp", "src/low_storage.cpp", "src/transverse.cpp", diff --git a/tests/backends/test_native_widths.py b/tests/backends/test_native_widths.py index 65104a726..b0d0dc5e6 100644 --- a/tests/backends/test_native_widths.py +++ b/tests/backends/test_native_widths.py @@ -412,7 +412,7 @@ def test_a_plane_s_widths_bring_only_numbers_across(n_pol, monkeypatch): """The session's plane statistics cross as one packed row, and no field does. The native session reduces the widths in its own pass - (``session_statistics.cpp``); what crosses is the packed evaluation its plan + (``statistics_plan.cpp``); what crosses is the packed evaluation its plan unpacks, counted here. The session counts every field it exports, and the native state every whole-state transfer to the host by any path (its diagnostics' ``crossed_bytes[0]``, solprop SEP 44): both stay at zero over the run. From 51857e2935d94381f6bed074abb7e8ede7b01997 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:13:09 +0000 Subject: [PATCH 15/21] Rename src/helpers.cpp/.hpp to bindings_helpers.cpp/.hpp They are nanobind bindings, part of the _core module target, beside bindings.cpp and bindings_transverse.cpp. Co-Authored-By: Claude Opus 5.5 --- native/CMakeLists.txt | 2 +- native/include/solprop_native/chebyshev.hpp | 2 +- native/src/bindings.cpp | 2 +- native/src/{helpers.cpp => bindings_helpers.cpp} | 2 +- native/src/{helpers.hpp => bindings_helpers.hpp} | 4 ++-- 5 files changed, 6 insertions(+), 6 deletions(-) rename native/src/{helpers.cpp => bindings_helpers.cpp} (99%) rename native/src/{helpers.hpp => bindings_helpers.hpp} (77%) diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 75a094cd4..4cea4518e 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -902,7 +902,7 @@ add_custom_command( ) nanobind_add_module(_core STABLE_ABI NOMINSIZE NB_SUPPRESS_WARNINGS src/bindings.cpp - src/bindings_transverse.cpp src/helpers.cpp src/records.cpp + src/bindings_transverse.cpp src/bindings_helpers.cpp src/records.cpp "${SOLPROP_NATIVE_SOURCE_DIGEST_CPP}") target_link_libraries(_core PRIVATE solprop_native_core ${CMAKE_DL_LIBS}) solprop_native_mark_dependency_headers_system(_core) diff --git a/native/include/solprop_native/chebyshev.hpp b/native/include/solprop_native/chebyshev.hpp index 08f9d3de7..4f6b3e6de 100644 --- a/native/include/solprop_native/chebyshev.hpp +++ b/native/include/solprop_native/chebyshev.hpp @@ -29,7 +29,7 @@ * weights whose difference is a stage's phase increment (solprop SEP 25 section 4.2). * * The functions are callable from host and device code; the host bindings are in - * src/helpers.cpp. + * src/bindings_helpers.cpp. */ namespace solprop_native::chebyshev { diff --git a/native/src/bindings.cpp b/native/src/bindings.cpp index bd094c7cf..45f189c29 100644 --- a/native/src/bindings.cpp +++ b/native/src/bindings.cpp @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: GPL-3.0-or-later +#include "bindings_helpers.hpp" #include "bindings_transverse.hpp" -#include "helpers.hpp" #include "records.hpp" #include "solprop_native/adams_march.hpp" #include "solprop_native/cpu_execution.hpp" diff --git a/native/src/helpers.cpp b/native/src/bindings_helpers.cpp similarity index 99% rename from native/src/helpers.cpp rename to native/src/bindings_helpers.cpp index fc682565c..bace21e4e 100644 --- a/native/src/helpers.cpp +++ b/native/src/bindings_helpers.cpp @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "helpers.hpp" +#include "bindings_helpers.hpp" #include "solprop_native/chebyshev.hpp" #include "solprop_native/lagrange_basis.hpp" diff --git a/native/src/helpers.hpp b/native/src/bindings_helpers.hpp similarity index 77% rename from native/src/helpers.hpp rename to native/src/bindings_helpers.hpp index 059597289..129745190 100644 --- a/native/src/helpers.hpp +++ b/native/src/bindings_helpers.hpp @@ -11,8 +11,8 @@ namespace solprop_native::bindings { /** * Add the host helpers solprop's per-stage and per-step evaluations take from this * backend to the extension module: `clenshaw_points`, `basis_rows` and - * `trial_tables` (helpers.cpp); and the vector-maths layer's batch functions, for its - * tests, to the private `testing` module. + * `trial_tables` (bindings_helpers.cpp); and the vector-maths layer's batch functions, + * for its tests, to the private `testing` module. */ void bind_helpers(nanobind::module_& module_, nanobind::module_& testing); From 7db2e0ecfef10630233534255416c8d9bd1324cf Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:16:56 +0000 Subject: [PATCH 16/21] refactor(native): split the lowering and march protocol from backend backend.py keeps NativeBackend, admission, refusals, the coupled flow's host factors and FFT planning. The code moves verbatim: - _lowering.py: _LowerPlan, _lower and its _lower_* helpers, the table packers and _ZTables. - _march.py: the march protocol (reasons, error categories, FRESH_WATCH, _envelope_errors, _raise_native) and the Adams record _NativeAdamsMarch. - _device.py: the build facts, _on_device, the node-chunk lowering and the resident flow volumes, which both backend and the lowering read. _flow_guard stays in backend beside the coupled flow's guard constants. Each module lists what its siblings import in __all__. The session takes the march protocol from _march; tests that read, import or patch a moved name now target the module where it is looked up. Co-Authored-By: Claude Opus 5.5 --- docs/dev/backends.md | 6 +- docs/dev/kernel-contract.md | 2 +- native/AGENTS.md | 2 +- native/src/solprop_native/_device.py | 182 ++ native/src/solprop_native/_lowering.py | 970 +++++++++++ native/src/solprop_native/_march.py | 410 +++++ native/src/solprop_native/backend.py | 1512 +---------------- native/src/solprop_native/session.py | 20 +- native/tests/test_backend.py | 15 +- native/tests/test_band_plasma.py | 2 +- native/tests/test_coupled_flow.py | 4 +- native/tests/test_dispersive.py | 2 +- native/tests/test_envelope_plasma.py | 3 +- .../tests/test_envelope_statistics_refusal.py | 3 +- native/tests/test_execution.py | 6 +- native/tests/test_free.py | 2 +- native/tests/test_mixtures.py | 2 +- native/tests/test_session_failure.py | 3 +- native/tests/test_workspace.py | 6 +- src/solprop/backends/node_chunk.py | 2 +- tests/backends/native_trace_marches.py | 21 +- tests/backends/test_native_phi_moments.py | 5 +- tests/backends/test_node_chunk.py | 40 +- tests/driver/test_native_march_lifecycle.py | 2 +- tests/driver/test_native_rk_march.py | 2 +- tests/sessions/runs.py | 6 +- tests/test_benchmark_sessions.py | 5 +- 27 files changed, 1695 insertions(+), 1540 deletions(-) create mode 100644 native/src/solprop_native/_device.py create mode 100644 native/src/solprop_native/_lowering.py create mode 100644 native/src/solprop_native/_march.py diff --git a/docs/dev/backends.md b/docs/dev/backends.md index 678a89474..243204528 100644 --- a/docs/dev/backends.md +++ b/docs/dev/backends.md @@ -638,8 +638,10 @@ once-per-run FFTW planning and the output's planes. ### 3.1a What exists -The adapter `native/src/solprop_native/backend.py`, its session -`native/src/solprop_native/session.py` and the engine under `native/src/general/`. The public default is `"auto"`: the +The adapter `native/src/solprop_native/backend.py` (with the lowering +`_lowering.py`, the march protocol `_march.py` and the build's facts `_device.py`), its +session `native/src/solprop_native/session.py` and the engine under +`native/src/general/`. The public default is `"auto"`: the native backend, with no fallback (§1, ADR-029). `numpy_ref` stays selectable by name. diff --git a/docs/dev/kernel-contract.md b/docs/dev/kernel-contract.md index 57c5fe17a..124891014 100644 --- a/docs/dev/kernel-contract.md +++ b/docs/dev/kernel-contract.md @@ -619,7 +619,7 @@ it is prepared, from the problem and its `ExecutionOptions` -- an explicit `node which is authoritative, or the automatic choice (`solprop.backends.node_chunk.plan_node_chunk`) -- records it in its execution metadata, and keeps it out of every identity and resume key. The native backend lowers the plan -further (`solprop_native.backend.native_node_chunk`: whole y rows on the planar kind, and +further (`solprop_native._device.native_node_chunk`: whole y rows on the planar kind, and `device_node_chunk` for a device build's automatic plan: the host rule at a quarter of the accelerator's free bytes, never fewer nodes than the host plan, solprop issue #361). diff --git a/native/AGENTS.md b/native/AGENTS.md index 96a009d1d..d439278c7 100644 --- a/native/AGENTS.md +++ b/native/AGENTS.md @@ -185,7 +185,7 @@ the gate. - The contract with solprop is its kernel contract, whose fields solprop's frozen tables hold. The backend inherits no solprop driver: a run is its native session (`session.py`), and the load, the Adams record and the timing probe are its own - (`_load.py`, `backend.py`, `_probe.py`). A backend holds one `_core.Workspace`, + (`_load.py`, `_march.py`, `_probe.py`). A backend holds one `_core.Workspace`, built from one `_core.Problem` with every set-once table; a run starts from one run record and the probe from the run's start (solprop SEP 52 section 5.3), and nothing configures the workspace after it is built. `check_build()` refuses, in an editable diff --git a/native/src/solprop_native/_device.py b/native/src/solprop_native/_device.py new file mode 100644 index 000000000..babe6bae6 --- /dev/null +++ b/native/src/solprop_native/_device.py @@ -0,0 +1,182 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""What the installed build's execution space decides for a backend. + +The build facts a backend reads once, whether the build runs on an accelerator, the +node chunk the native problem is built with (solprop SEP 47 section 6.5, issue #361) +and the low-storage march's resident flow volumes. +""" + +from __future__ import annotations + +import dataclasses +import functools +import types +from collections.abc import Mapping +from importlib.metadata import version + +from solprop.backends.memory import low_storage_flows +from solprop.backends.node_chunk import NodeChunkPlan, plan_node_chunk +from solprop.backends.session import ExecutionOptions +from solprop.exceptions import BackendError +from solprop.stepper.low_storage import LowStorageRungeKutta +from solprop.translate.problem import PropagationProblem, StepperSettings + +from . import _core, build_info + +__all__ = [ + "DEVICE_BUDGET_SHARE", + "_build_facts", + "_low_storage_flows", + "_on_device", + "_resident_flows", + "device_node_chunk", + "native_node_chunk", +] + + +@dataclasses.dataclass(frozen=True) +class _BuildFacts: + """The parts of a backend's metadata that are fixed for the process.""" + + info: Mapping[str, object] + solprop_version: str + device_team_size: int + device_reduction_size: int + + +@functools.cache +def _build_facts() -> _BuildFacts: + """Return the build record and the installed solprop version, read once. + + Reading them costs about a millisecond (the package metadata lookup and the native + record), which a per-plane metadata record paid at every saved plane. + """ + native_record = { + key: value for key, value in _core.build_record().items() if value is not None + } + return _BuildFacts( + info=types.MappingProxyType(dataclasses.asdict(build_info())), + solprop_version=version("solprop"), + device_team_size=int(native_record["device_team_size"]), + device_reduction_size=int(native_record["device_reduction_size"]), + ) + + +def _on_device() -> bool: + """Return whether the installed build executes on an accelerator.""" + return _build_facts().info["device_backend"] != "none" + + +DEVICE_BUDGET_SHARE = 4 +"""The share of the accelerator's free bytes an automatic plan budgets: one quarter. + +The budget is spent on the host plan's conservative inventory (twenty column buffers a +Cartesian column, about three times the live buffers, solprop issue #349), so the live +space-domain buffers take near a twelfth of the free bytes, leaving the rest for the +state and the march's registers.""" + + +def device_node_chunk( + problem: PropagationProblem, planned: int, free: int | None +) -> int: + """Return the node chunk a device build takes when no budget or chunk is given. + + solprop issue #361: on a device a node chunk is a kernel launch, and fewer, larger + launches are faster: one chunk took 0.74-0.92 of the default's time on AMD Instinct + MI300X spatial routes (SEP 47 Part II), at the cost of space-domain buffers for + every node at once. So an automatic plan budgets :data:`DEVICE_BUDGET_SHARE`'s share + of the free device bytes, through the host plan's own rule and response admissions + (:func:`solprop.backends.node_chunk.plan_node_chunk`): one chunk wherever that fits, + otherwise the largest that does, and never fewer nodes than the host plan's chunk + at the route's default budget. It is consulted only for an automatic plan: an + explicit budget or chunk always wins. It runs at backend construction, after the + translation, so no device runtime starts before a scan forks. + + Parameters + ---------- + problem : solprop.translate.problem.PropagationProblem + The validated problem. + planned : int + The host plan's chunk at the default budget + (:func:`solprop.backends.node_chunk.plan_node_chunk`). + free : int or None + The accelerator's free bytes, or ``None`` where none is known. + + Returns + ------- + int + The device's chunk, between 1 and the node count. + """ + if free is None: + return planned + budget = max(1, free // DEVICE_BUDGET_SHARE) + try: + roomy = plan_node_chunk(problem, ExecutionOptions(node_budget=budget)) + except BackendError: + # A quarter of the free bytes cannot hold one node: the default plan stands, + # and its own allocation reports what does not fit. + return planned + return max(planned, roomy.chunk) + + +def native_node_chunk( + problem: PropagationProblem, plan: NodeChunkPlan, *, device: bool +) -> int: + """Lower the host node-chunk plan to the chunk this backend executes. + + A device build's automatic plan goes through :func:`device_node_chunk`. The chunk + is then bounded by the node count, and on the hybrid planar kind made whole y rows, + as many as it holds and at least one, which is solprop's reference's own blocking. + + Parameters + ---------- + problem : solprop.translate.problem.PropagationProblem + The validated problem. + plan : solprop.backends.node_chunk.NodeChunkPlan + The host plan. + device : bool + Whether this build executes on an accelerator. + + Returns + ------- + int + The chunk the native problem is built with. + """ + chunk = plan.chunk + if device and plan.automatic: + chunk = device_node_chunk(problem, chunk, _core.available_device_memory()) + chunk = min(chunk, problem.transform.n_nodes) + geometry = problem.transform.geometry + if geometry is not None and geometry.kind == "slab": + n_x, n_y = geometry.shape + chunk = min(n_y, max(1, chunk // n_x)) * n_x + return chunk + + +def _low_storage_flows( + problem: PropagationProblem, stepper: StepperSettings, *, device: bool +) -> int | None: + """Return the low-storage march's flow volumes, or None for another march. + + The flows solprop's estimate counts (solprop kernel schema 32) -- except that a + device build recomputes under ``"auto"``, where the estimate's CPU count + over-states it by the table. A device's input crosses the state's layout through + tiles (``docs/dev/benchmarks.md``); a CPU's has its own. + """ + if not isinstance(stepper.integrator, LowStorageRungeKutta): + return None + return _resident_flows( + low_storage_flows(problem, stepper, "native"), stepper, device=device + ) + + +def _resident_flows(flows: int, stepper: StepperSettings, *, device: bool) -> int: + """Return the flow volumes the native state holds for a CPU's count `flows`. + + A device build recomputes under ``"auto"`` and holds none + (:class:`~solprop.backends.records.LowStorageSettings` counts a CPU's). + """ + return 0 if device and stepper.propagator_mode == "auto" else flows diff --git a/native/src/solprop_native/_lowering.py b/native/src/solprop_native/_lowering.py new file mode 100644 index 000000000..149be1f17 --- /dev/null +++ b/native/src/solprop_native/_lowering.py @@ -0,0 +1,970 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""The lowering of solprop's problem and request into the native problem. + +``_lower`` fills ``_core.Problem``'s structs once, where a backend is built; the table +packers form the set-once tables the workspace makes resident (z tables, sections, the +carrier frame's chord, a collar's lines), and ``_ZTables`` sizes the general z tables' +slots (solprop SEP 29 section 6.6). +""" + +from __future__ import annotations + +import dataclasses +from collections.abc import Mapping +from typing import TYPE_CHECKING, Any + +import numpy as np +import numpy.typing as npt +from solprop import transverse_monitor +from solprop.backends.exponential import CarrierReference +from solprop.backends.ledger import source_absent +from solprop.backends.memory import ( + LOCAL_FLUX_STATISTICS, + low_storage_stage_register, + z_table_slots, +) +from solprop.backends.node_chunk import general_planned_chunk, plan_node_chunk +from solprop.backends.spatial import far_field_partials, radial_absorber_matrix +from solprop.constants import EPS0 +from solprop.exceptions import BackendError +from solprop.grid import RadialGrid +from solprop.stats import StatsSpec +from solprop.stepper.multistep import Multistep +from solprop.translate.chebyshev import ( + _EDGE_SLACK_FRACTION, # pyright: ignore[reportPrivateUsage] - the tables' own admitted span + PiecewiseChebyshev, +) +from solprop.translate.coupled import flow_edge_slack +from solprop.translate.general import ( + node_delayed_maps, + node_kerr_form, + node_kerr_tensor, +) +from solprop.translate.linop import power_weight_at +from solprop.translate.plasma_source import ( + _BENCHMARK_GATED, # pyright: ignore[reportPrivateUsage] - the private gated benchmark +) +from solprop.translate.problem import PropagationProblem, StepperSettings +from solprop.translate.routes import flux_of +from solprop.translate.run_request import RunRequest +from solprop.translate.stage import StageTerms +from solprop.transverse_monitor import TransverseEdge + +# The CPU execution record is read from the package at each use: a first-use +# calibration in this process refreshes it (solprop SEP 46 section 5). +import solprop_native as _package + +from . import _core +from ._device import _low_storage_flows, _on_device, native_node_chunk +from ._execution import ExecutionRecord + +if TYPE_CHECKING: + from .backend import ( + _HostFlow, # pyright: ignore[reportPrivateUsage] - the flow's host factors + ) + +__all__ = [ + "_LowerPlan", + "_ZTables", + "_chord_tables", + "_edge_lines", + "_flux_tables", + "_lower", + "_section_tables", +] + +type RealArray = npt.NDArray[np.float64] + + +def _edge_lines(edge: TransverseEdge) -> dict[str, object]: + """Return a requested collar's lines as the workspace takes them (SEP 44).""" + lines = edge.reduction.lines + axes = {"r": 0, "x": 1, "y": 2} + none = 2**64 - 1 + strip = [np.asarray(line.strip, dtype=np.uint64) for line in lines] + window = [np.asarray(line.window, dtype=np.float64) for line in lines] + onsets = [line.onset if line.onset is not None else (none, 0, 0) for line in lines] + space = edge.space + return { + "axes": np.array([axes[line.axis] for line in lines], dtype=np.uint64), + "counts": np.array([array.size for array in strip], dtype=np.uint64), + "positions": np.concatenate(strip) if strip else np.empty(0, dtype=np.uint64), + "windows": np.concatenate(window) if window else np.empty(0, dtype=np.float64), + "inward": np.array([line.inward for line in lines], dtype=np.uint64).reshape( + -1 + ), + "factors": np.array( + [ + line.spacing / (2 * array.size) * line.measure if array.size else 0.0 + for line, array in zip(lines, strip, strict=True) + ], + dtype=np.float64, + ), + "measures": np.array([line.measure for line in lines], dtype=np.float64), + "onsets": np.array(onsets, dtype=np.uint64).reshape(-1), + "steps": np.array([line.onset_step for line in lines], dtype=np.float64), + "k_min": float(edge.k_min), + "scale": np.sqrt(np.asarray(space.r, dtype=np.float64)) + if lines and isinstance(space, RadialGrid) + else np.empty(0, dtype=np.float64), + "batch_bytes": transverse_monitor._STRIP_BATCH_BYTES, # pyright: ignore[reportPrivateUsage] + } + + +def _z_table(kind: str, table: PiecewiseChebyshev) -> dict[str, object]: + """Return one z table as the workspace makes it resident: its breakpoints, the + admitted span (``_EDGE_SLACK_FRACTION`` past each end) and its coefficients.""" + low, high = table.z_span + slack = _EDGE_SLACK_FRACTION * (high - low) + n_intervals, n_terms = table.coefficients.shape[:2] + return { + "kind": kind, + "edges": np.ascontiguousarray(table.breakpoints, dtype=np.float64), + "lower": low - slack, + "upper": high + slack, + "coefficients": np.ascontiguousarray( + table.coefficients, dtype=np.complex128 + ).reshape(n_intervals, n_terms, -1), + } + + +def _flux_tables(problem: PropagationProblem) -> list[dict[str, object]]: + """Return a radial plane measure's material z table, or none (SEP 34).""" + flux = flux_of(problem.route) + if flux is None or flux.material is None: + return [] + return [_z_table("flux", flux.material)] + + +_EMPTY = np.empty(0, dtype=np.complex128) +"""No modified-noise realisation: the native state adds none.""" + + +def _section_tables( + problem: PropagationProblem, *, moving: bool, residual: bool +) -> dict[str, object]: + """Return a coupled flow's and a moving guide's sections as the stage tables take + them: their edges, degrees and the edges' slack (`flow_edge_slack`, + `transform_section`), and whether the flow carries a frozen residual.""" + flow = problem.linear.coupled_flow + maps = problem.transform.modal_tables if moving else None + none: dict[str, object] = {"edges": [], "degrees": 0, "slack": 0.0} + out: dict[str, object] = { + "flow": none, + "flow_residual": False, + "transform": none, + } + if flow is not None: + out |= { + "flow": { + "edges": np.asarray(flow.z_edges, dtype=np.float64).tolist(), + "degrees": int(flow.n_deg), + "slack": float(flow_edge_slack(flow)), + }, + "flow_residual": residual, + } + if maps is not None and maps.synthesis is not None: + edges = np.asarray(maps.z_edges, dtype=np.float64) + out["transform"] = { + "edges": edges.tolist(), + "degrees": int(maps.synthesis.shape[1]), + "slack": _EDGE_SLACK_FRACTION * float(edges[-1] - edges[0]), + } + return out + + +def _chord_tables(carrier: CarrierReference | None) -> dict[str, object]: + """Return the carrier frame's chord as the native stage tables take it. + + The imaginary part of the reference element's operator (`CarrierChord`): a + constant's rate, an expansion's terms over the ``Phi`` weights + the tables already hold, or a z table's reference-element column as a scalar table + of its own (solprop issue #255), packed as its batch pass reads it. None without an + active frame. + """ + chord = None if carrier is None or not carrier.active else carrier.chord + if chord is not None and chord.constant is not None: + return { + "chord": "constant", + "chord_rate": float(np.imag(chord.constant)), + "chord_terms": [], + "chord_table": None, + } + if chord is not None and chord.expansion is not None: + terms = np.imag(np.asarray(chord.expansion.terms)) + return { + "chord": "expansion", + "chord_rate": 0.0, + "chord_terms": terms.tolist(), + "chord_table": None, + } + if chord is not None and chord.table is not None: + edges, lower, upper, coefficients = chord.table.packed() + # The imaginary part a complex Clenshaw pass forms at a real position, by the + # same operations (`stage::chord`). + column = np.ascontiguousarray(np.imag(coefficients)) + return { + "chord": "table", + "chord_rate": 0.0, + "chord_terms": [], + "chord_table": (edges, lower, upper, column), + } + return {"chord": "none", "chord_rate": 0.0, "chord_terms": [], "chord_table": None} + + +_TABLE_POINTS = 16 +"""The most positions one native table evaluation takes (``z_table::max_points``).""" + + +class _ZTables: + """The general z tables of one workspace (solprop SEP 29 section 6.6, G1). + + solprop falls back to a Chebyshev table of the state's size for ``Phi`` + (``linear.phi_coefficients``) or the response prefactor (``responses.prefactor``) + when neither is a weighted sum of fixed arrays. The coefficients cross once, with + the workspace's setup (:meth:`records`), and stay resident; a trial's positions are + then evaluated natively into the state's term slots, ``slots`` of them, and the + stage weights select them: ``Phi(z1) - Phi(z0)`` is the row ``e_1 - e_0`` and + ``Phi(z1)`` the row ``e_1``, so every native consumer of the terms (the Runge-Kutta + stages, the low-storage flows, the Adams phase and the noise) reads the table + unchanged, as one rounded subtraction of two values, as solprop's reference forms + it. The prefactor's slots are its values at the same positions, one-hot weighted. + + The slots are kept as a least-recently-used set of positions in the workspace + (``stage::ZSlots``, solprop SEP 39 phase 4): each call's positions are evaluated + only where no slot holds them yet, so an accepted step's end, which is the next + step's anchor, is never evaluated twice. A call's positions number at most + ``slots`` (the tableau's distinct stage fractions, five for the low-storage march), + so none of one call's positions is evicted by another of its own. A compensated + Adams march reads ``D = dPhi/dz`` from the derivative table (``linear.rate``) into + one more ``Phi`` slot, ``rate_slot``. + + Attributes + ---------- + phi, prefactor : bool + Which quantities are z tables. + slots : int + The position slots, each one value of each table. + rate_slot : int or None + The ``Phi`` term that holds the rate, or ``None`` when no march reads it. + """ + + def __init__(self, problem: PropagationProblem, stepper: StepperSettings) -> None: + """Size the slots from the problem and its method; nothing crosses yet.""" + self._problem = problem + self.phi = problem.linear.phi_representation == "table" + self.prefactor = problem.responses.prefactor_representation == "table" + integrator = stepper.integrator + self.slots = z_table_slots(stepper) + if self.slots > _TABLE_POINTS: + raise BackendError( + f"native evaluates a z table at {_TABLE_POINTS} positions a call, and " + f"this tableau's stages need {self.slots}" + ) + compensated = ( + self.phi + and isinstance(integrator, Multistep) + and integrator.formulation == "exponential" + and not source_absent(problem) + ) + self.rate_slot = self.slots if compensated else None + + @property + def n_phi(self) -> int: + """The native ``Phi`` term count: the slots, and the rate's when it is read.""" + return self.slots + (self.rate_slot is not None) + + @property + def nbytes(self) -> int: + """The bytes of the coefficients that cross to the workspace.""" + coefficients = 0 + if self.phi: + phi = self._problem.linear.phi_coefficients + assert phi is not None + coefficients += phi.nbytes + if self.rate_slot is not None: + coefficients += self._problem.linear.rate.coefficients.nbytes + if self.prefactor: + table = self._problem.responses.prefactor + assert table is not None + coefficients += table.coefficients.nbytes + return int(coefficients) + + def slots_record(self) -> dict[str, object]: + """Return the slots as the workspace configures them (``ZSlots``).""" + return { + "phi": self.phi, + "prefactor": self.prefactor, + "slots": self.slots, + "rate": self.rate_slot is not None, + } + + def records(self) -> list[dict[str, object]]: + """Return the tables the workspace makes resident (``ZTable``).""" + out: list[dict[str, object]] = [] + if self.phi: + out.append(_z_table("phi", self._problem.linear.polynomial)) + if self.rate_slot is not None: + out.append(_z_table("rate", self._problem.linear.rate)) + if self.prefactor: + table = self._problem.responses.prefactor + assert table is not None + out.append(_z_table("prefactor", table)) + return out + + +@dataclasses.dataclass(frozen=True) +class _LowerPlan: + """The backend's choices a lowering takes beside the problem and its request. + + `terms` are the fixed stage terms, `flow` a coupled flow's host factors and + `tables` the z tables; the chunks are the backend's lowered execution choices + (solprop SEP 47 section 6.5), the automatic plan's when absent. `cpu_execution` + is the workspace's CPU record (the process's when absent) and `fft_planning` its + FFT planning record (none on a device); `device` says the state is a device's, and + `setup` holds the workspace's set-once data by part (`NativeBackend._setup`). + """ + + terms: StageTerms + flow: _HostFlow | None = None + tables: _ZTables | None = None + node_chunk: int | None = None + general_chunk: int | None = None + cpu_execution: ExecutionRecord | None = None + fft_planning: Mapping[str, object] | None = None + device: bool = False + setup: Mapping[str, object] | None = None + + +def _lower( + problem: PropagationProblem, request: RunRequest, plan: _LowerPlan +) -> _core.Problem: + """Fill the native problem's structs from solprop's problem, request and `plan`.""" + stepper = request.stepper + node_chunk = plan.node_chunk + if node_chunk is None: + node_chunk = native_node_chunk( + problem, plan_node_chunk(problem), device=_on_device() + ) + general_chunk = plan.general_chunk + if general_chunk is None and problem.transform.modal_tables is not None: + general_chunk = general_planned_chunk(problem, plan_node_chunk(problem)) + native = _core.Problem() + native.cpu_execution = (plan.cpu_execution or _package.EXECUTION).native() + if plan.fft_planning is not None: + native.fft_planning = dict(plan.fft_planning) + kerr = problem.responses.kerr_coefficients + dispersive = problem.responses.dispersive_kerr + absorber = problem.responses.temporal_absorber + absorber_fine = problem.responses.temporal_absorber_fine + spatial_absorber = radial_absorber_matrix( + problem.transform, problem.grids.pad_scale + ) + temporal_in_stage = ( + absorber_fine is not None and problem.responses.absorber_scheme == "nonlinear" + ) + geometry = problem.transform.geometry + # Free space and schema 15's hybrid planar kind share the separable operator and + # the transverse FFT engine; the planar kind adds its slab rule in x. + free = geometry is not None and geometry.kind in ("free", "slab") + spatial_absorption = bool(spatial_absorber.size) or ( + free and geometry is not None and bool(np.any(geometry.absorber_rate)) + ) + + dims = native.dims + dims.n_omega = problem.shape[-1] + dims.n_t = problem.grids.n_t + dims.n_to = problem.grids.n_to + dims.n_modes = problem.shape[0] + dims.n_pol_state = problem.shape[1] + dims.n_pol_transform = problem.transform.n_pol + dims.n_nodes = problem.transform.n_nodes + # solprop SEP 47 section 6.5: the backend's execution choice, lowered by + # `native_node_chunk`. + dims.node_chunk = node_chunk + # solprop SEP 6's number-conserving sector is formed on analytic signals, of the + # filtered fields and, for the polynomial, of the bare one; a Raman drive beside + # it is the cycle-averaged one, which reads the analytic signal too. The general + # modal route synthesises its own analytic signal on the real family. + dims.family = ( + _core.FieldFamily.envelope + if problem.grid_kind == "envelope" + else _core.FieldFamily.field_analytic + if (kerr is not None and kerr.analytic_signal) + or ( + dispersive is not None + and dispersive.sector == "conserving" + and problem.transform.modal_tables is None + ) + else _core.FieldFamily.field_real + ) + + grids = problem.grids + carrier = grids.carrier_2w0 + grid = native.grid + grid.crop_index = np.ascontiguousarray(grids.crop_index, dtype=np.uint64) + grid.active_mask = np.ascontiguousarray(grids.active_mask) + grid.norm_mask = np.ascontiguousarray(grids.norm_mask) + grid.omega_window = np.ascontiguousarray(grids.omega_window) + grid.omega = np.ascontiguousarray(grids.omega) + grid.time_window = np.ascontiguousarray(grids.t_window_fine) + if problem.grid_kind == "field" and problem.responses.plasma_channels: + grid.time_window_derivative = np.ascontiguousarray( + grids.t_window_fine_derivative + ) + if geometry is None: + grid.power_weight = np.ascontiguousarray(power_weight_at(problem, 0.0)) + grid.carrier = np.ascontiguousarray( + np.ones(grids.n_to, dtype=np.complex128) if carrier is None else carrier + ) + grid.dt_fine = grids.dt_fine + grid.dt = grids.dt + + _lower_linear(native, problem, plan) + coarse_matrix = _lower_spatial(native, problem, request) + _lower_stepper(native, problem, stepper, device=plan.device) + + native.absorber.in_stage = temporal_in_stage or spatial_absorption + native.absorber.split_step = ( + absorber is not None and problem.responses.absorber_scheme == "split_step" + ) + if temporal_in_stage: + native.absorber.fine = np.ascontiguousarray(absorber_fine, dtype=np.float64) + elif spatial_absorption: + native.absorber.fine = np.zeros(grids.n_to) + if absorber is not None: + native.absorber.coarse = np.ascontiguousarray(absorber, dtype=np.float64) + + transform = native.transform + transform.synthesis_matrix = np.ascontiguousarray(problem.transform.synthesis) + transform.projection_matrix = np.ascontiguousarray(problem.transform.projection) + transform.projection_coarse_matrix = np.ascontiguousarray(coarse_matrix) + transform.synthesis = 0.0 if free else float(problem.transform.synthesis[0, 0]) + transform.projection = 0.0 if free else float(problem.transform.projection[0, 0]) + + if kerr is not None: + native.kerr.nonlinear = True + native.kerr.coefficient = kerr.coefficient + native.kerr.self_factor = kerr.self_factor + native.kerr.conjugate_factor = kerr.conjugate_factor + native.kerr.harmonic_factor = kerr.harmonic_factor + native.kerr.take_real = kerr.take_real + + _lower_plasma(native, problem, request) + _lower_raman(native, problem) + _lower_general(native, problem, general_chunk) + _lower_flow(native, plan.flow) + _lower_dispersive(native, problem) + _lower_flux(native, problem, request.observation.stats) + if plan.setup is not None: + # The set-once data, each part read through its record's table (SEP 52). + held: Any = native + for name, value in plan.setup.items(): + setattr(held, name, value) + return native + + +def _lower_linear( + native: _core.Problem, problem: PropagationProblem, plan: _LowerPlan +) -> None: + """Fill the fixed terms, (n_terms, rows, n_omega): one empty term on a separable + route, a z table's slots.""" + del problem + tables = plan.tables + linear = native.linear + n_omega = native.dims.n_omega + linear.phi_table = tables is not None and tables.phi + linear.prefactor_table = tables is not None and tables.prefactor + if plan.terms.separable is not None: + linear.n_phi_terms = 1 + linear.n_prefactor_terms = 1 + return + # A z table's terms are slots the native evaluation fills (solprop SEP 29 G1): + # their count crosses in their place. + if tables is not None and tables.phi: + linear.n_phi_terms = tables.n_phi + elif plan.terms.phi is None: + raise BackendError("Kokkos requires a device-ready fixed-term problem") + else: + linear.n_phi_terms = plan.terms.phi.shape[0] + linear.phi_terms = np.ascontiguousarray( + plan.terms.phi.reshape(plan.terms.phi.shape[0], -1, n_omega) + ) + if tables is not None and tables.prefactor: + linear.n_prefactor_terms = tables.slots + elif plan.terms.prefactor is None: + raise BackendError("Kokkos requires a device-ready fixed-term problem") + else: + linear.n_prefactor_terms = plan.terms.prefactor.shape[0] + linear.prefactor_terms = np.ascontiguousarray( + plan.terms.prefactor.reshape(plan.terms.prefactor.shape[0], -1, n_omega) + ) + + +def _lower_spatial( + native: _core.Problem, problem: PropagationProblem, request: RunRequest +) -> np.ndarray: + """Fill the separable operator and a radial run's rows; return the coarse matrix. + + The quadrature alarm's embedded Kronrod projection is the transform's own unless + it is the plain projection or the route has one node, and the hybrid planar kind + packs none: the reference scores no alarm on it. + """ + transform = problem.transform + coarse = ( + np.empty(0, dtype=np.float64) + if transform.n_nodes == 1 + or np.array_equal(transform.projection_coarse, transform.projection) + else transform.projection_coarse + ) + geometry = transform.geometry + spatial = native.spatial + spatial.absorber_matrix = radial_absorber_matrix(transform, problem.grids.pad_scale) + if geometry is not None: + spatial.spectral_weights = geometry.spectral_weights + spatial.on_axis = geometry.on_axis + separable = problem.linear.separable + if geometry is None or geometry.kind not in ("free", "slab") or separable is None: + return coarse + spatial.n_x, spatial.n_y = geometry.shape + spatial.k_squared = separable.k_squared + spatial.k_squared_low = separable.k_squared_low + spatial.kperp_sq = separable.kperp_sq + spatial.kperp_sq_low = separable.kperp_sq_low + spatial.frame = separable.frame + spatial.spectral_rate = separable.spectral_rate + assert separable.user_rate is not None + assert separable.kperp_rate is not None + spatial.user_rate = separable.user_rate + spatial.numerator = separable.prefactor_numerator + spatial.clamped = separable.prefactor_clamped + spatial.evanescent_rate = separable.evanescent_rate + spatial.absorber_rate = geometry.absorber_rate + # A transverse collar's rate per column (solprop kernel schema 31). + spatial.kperp_rate = separable.kperp_rate + if geometry.kind == "free": + # solprop SEP 23: the stored axes and the slots of the problem's own + # far-field request, which size the native band partials. + k_x, k_y = geometry.spectral_axes + slots, _ = far_field_partials(problem, request.observation.stats) + spatial.k_x = np.ascontiguousarray(k_x, dtype=np.float64) + spatial.k_y = np.ascontiguousarray(k_y, dtype=np.float64) + spatial.far_field_slots = slots + return coarse + # Schema 15's `SeparableOperator.kappa_sq` is (n_channels, n_omega), column `c` + # belonging to channel `c % n_channels`; the geometry's collar rate depends on y + # alone (solprop checks it), so it is packed per column `p + n_p * iy`, the index + # of the temporal row it acts on. + assert separable.kappa_sq is not None + n_x, n_y = geometry.shape + n_p = separable.n_channels + spatial.n_p = n_p + spatial.kappa_sq = np.ascontiguousarray(separable.kappa_sq) + spatial.absorber_rate = np.ascontiguousarray( + np.repeat(geometry.absorber_rate.reshape(n_y, n_x)[:, 0], n_p) + ) + return np.empty(0, dtype=np.float64) + + +def _lower_stepper( + native: _core.Problem, + problem: PropagationProblem, + stepper: StepperSettings, + *, + device: bool, +) -> None: + """Fill the tableau and the low-storage march's rows (solprop kernel schema 32).""" + tableau = stepper.tableau + native_stepper = native.stepper + native_stepper.n_stages = tableau.n_stages + native_stepper.tableau_matrix = np.ascontiguousarray(tableau.matrix) + native_stepper.tableau_nodes = np.ascontiguousarray(tableau.nodes) + native_stepper.solution_weights = np.ascontiguousarray(tableau.solution_weights) + native_stepper.error_weights = np.ascontiguousarray(tableau.error_weights) + native_stepper.first_same_as_last = tableau.first_same_as_last + native_stepper.weighted_controller = stepper.controller.weights is not None + native_stepper.modified_noise = problem.initial.noise_model == "modified" + flows = _low_storage_flows(problem, stepper, device=device) + if flows is None: + return + native_stepper.low_storage = True + native_stepper.low_storage_stage_register = low_storage_stage_register( + problem, "native" + ) + native_stepper.low_storage_flows = flows + native_stepper.low_storage_tiled_input = device + + +def _lower_plasma( + native: _core.Problem, problem: PropagationProblem, request: RunRequest +) -> None: + """Fill the plasma columns: the first channel's rate, every channel's rows.""" + channels = problem.responses.population_channels + channel = channels[0] if channels else None + plasma = native.plasma + # solprop SEP 18: the running-integral rule, one per problem. + plasma.band = problem.responses.plasma_integral.policy == "band" + plasma.taps = problem.responses.plasma_integral.taps + if channel is not None: + rate = channel.rate + plasma.enabled = True + # The private gated benchmark (solprop's tests/gated_source.py), read where + # the backend is built, as the reference reads it. + plasma.gated = _BENCHMARK_GATED.get() + plasma.charge_mass_ratio = channel.charge_mass_ratio + plasma.ionisation_potential = channel.ionisation_potential + plasma.log_field_min = rate.log_field_min + plasma.d_log_field = rate.d_log_field + plasma.field_min = rate.field_min + plasma.field_max = rate.field_max + plasma.rate_max = rate.rate_max + plasma.rate_intervals = rate.coefficients.shape[1] + plasma.rate_coefficients = np.ascontiguousarray(rate.coefficients) + plasma.stash = _stash_plasma(problem, request) + _lower_channels(native, problem) + _lower_envelope(native, problem) + + +def _stash_plasma(problem: PropagationProblem, request: RunRequest) -> bool: + """Whether the accepted step's last stage leaves the plasma statistics behind. + + Only when that stage's input is the committed field: a first-same-as-last tableau, + no split-step absorber (applied to the field after the last stage) and no coupled + flow; and not under the modified noise model, where that input carries the noise + draw while solprop defines the statistics on the committed field its reference + scans. Otherwise they are scanned afresh. + """ + absorber = problem.responses.temporal_absorber + return ( + bool(problem.responses.population_channels) + and problem.responses.envelope_plasma is None + and problem.initial.noise_model != "modified" + and request.stepper.tableau.first_same_as_last + and not ( + absorber is not None and problem.responses.absorber_scheme == "split_step" + ) + and problem.linear.coupled_flow is None + and bool( + { + "electron_density", + "peak_ionisation_rate", + "species_ionisation", + "species_electron_density", + } + & set(request.observation.stats.periodic) + ) + ) + + +def _lower_raman(native: _core.Problem, problem: PropagationProblem) -> None: + """Fill the Raman terms, row-major (n_terms, 2 n_to) and (n_terms, 2).""" + terms = problem.responses.raman_terms + raman = native.raman + raman.n_components = len(terms) + raman.row_component = np.asarray( + [component for component, term in enumerate(terms) for _ in term.undamped], + dtype=np.float64, + ) + raman.component_fraction = np.asarray([term.fraction for term in terms]) + if not terms: + return + first = terms[0] + undamped = np.concatenate([term.undamped for term in terms], axis=0) + raman.enabled = True + raman.thg = bool(first.thg) and problem.grid_kind == "field" + raman.tensor_capable = all(term.tensor_capable for term in terms) + raman.n_terms = undamped.shape[0] + raman.fraction = float(first.fraction) + raman.epsilon0 = EPS0 + raman.undamped = np.ascontiguousarray(undamped) + raman.tensor_weights = np.ascontiguousarray( + np.concatenate([term.tensor_weights for term in terms], axis=0) + ) + + +def _lower_dispersive(native: _core.Problem, problem: PropagationProblem) -> None: + """Lay out solprop SEP 6's dispersive response for the native engine. + + The structure the native response takes on every route: the channels grouped as the + leaf groups them (``DispersiveKerrTerms.groups``) -- the symmetric evaluator's one + mixture per node region, the polynomial's species of equal leg weight and region -- + with one filtered field and one product per symmetric group, and for the + polynomial one ``E^3`` per distinct region followed by two fields and two products + per group; each output term's multiplier, product and factor, every multiplier + carrying the sector's ``Pi_+`` and factor (one on a field grid's full cubic, + ``Pi_+ / 8`` for its conserving sector, ``Pi_+ / 4`` on an envelope grid), and the + input multipliers ``Pi_+`` on a field grid's analytic family. A mixture's rows are + rebuilt natively at every stage whose densities change, from its channels' ``g^4`` + (``mixture_filter``'s arithmetic); the first species' stand until then. On the + general modal route each group's key is its region and every node's region is + packed beside them; the route's grid is a field grid. + """ + leaf = problem.responses.dispersive_kerr + if leaf is None: + return + tables = problem.transform.modal_tables + general = tables is not None + kind = "field" if general else problem.grid_kind + form = ( + ("real" if leaf.sector == "full" else "conserving") + if kind == "field" + else ("harmonic" if leaf.sector == "full" else "conserving") + ) + analytic = form == "conserving" and kind == "field" + symmetric = leaf.evaluator == "symmetric" + n_channels = leaf.n_channels + positive = leaf.positive.astype(np.float64) + if kind == "envelope": + scale = positive / 4.0 + elif analytic: + scale = positive / 8.0 + else: + scale = np.ones(positive.shape) + input_scale = positive if analytic else np.ones(positive.shape) + groups = leaf.groups() + # A group's key is its general modal region; every other route has one, so one + # cube is shared by every polynomial group there. + regions = leaf.regions if general else None + keys = [0 if regions is None else int(regions[group[0]]) for group in groups] + distinct = list(dict.fromkeys(keys)) + cubes = [distinct.index(key) for key in keys] + n_cubes = 0 if symmetric else len(distinct) + mixtures = tuple( + index for index, group in enumerate(groups) if symmetric and len(group) > 1 + ) + fields: list[RealArray] = [] + rows: list[int] = [] + channels: list[int] = [] + multipliers: list[RealArray] = [] + for index, group in enumerate(groups): + weight = leaf.filters[group[0]] + if symmetric: + fields.append(weight) + rows.append(index) + channels.append( + n_channels + 1 + mixtures.index(index) + if index in mixtures + else group[0] + ) + multipliers.append(weight * scale) + continue + assert leaf.outputs is not None + fields.extend([weight, weight * weight]) + first = n_cubes + 2 * index + for channel in group: + output = leaf.outputs[channel] + rows.extend([cubes[index], first]) + channels.extend([channel, channel]) + multipliers.extend([output[0] * scale, output[1] * scale]) + rows.append(first + 1) + channels.append(n_channels) + multipliers.append(scale.copy()) + stacked = np.array(fields, dtype=np.float64) + if analytic: + stacked = stacked * leaf.positive + constants = ( + np.zeros((n_channels, 2)) + if leaf.products is None + else np.asarray(leaf.products, dtype=np.float64) + ) + if mixtures: + squares = leaf.filters * leaf.filters + quartics = np.ascontiguousarray(squares * squares, dtype=np.float64) + else: + quartics = np.empty(0, dtype=np.float64) + offsets = np.cumsum([0, *(len(group) for group in groups)]) + lowered = native.dispersive + if tables is not None: + lowered.group_keys = np.asarray(keys, dtype=np.uint64) + lowered.node_region = np.ascontiguousarray(tables.region_index, dtype=np.uint64) + lowered.enabled = True + lowered.symmetric = symmetric + lowered.form = {"real": 0, "conserving": 1, "harmonic": 2}[form] + lowered.inputs = np.ascontiguousarray(stacked) + lowered.term_multipliers = np.ascontiguousarray( + np.array(multipliers, dtype=np.float64) + ) + lowered.term_rows = np.asarray(rows, dtype=np.uint64) + lowered.term_channels = np.asarray(channels, dtype=np.uint64) + lowered.coefficients = np.asarray(leaf.coefficients, dtype=np.float64) + lowered.pair_constants = np.ascontiguousarray(constants) + lowered.group_offsets = np.asarray(offsets, dtype=np.uint64) + lowered.group_members = np.asarray( + [channel for group in groups for channel in group], dtype=np.uint64 + ) + lowered.quartics = quartics + lowered.input_scale = np.ascontiguousarray(input_scale) + lowered.output_scale = np.ascontiguousarray(scale) + + +def _lower_flux( + native: _core.Problem, problem: PropagationProblem, stats: StatsSpec +) -> None: + """Fill the plane measure of solprop SEP 34 (kernel schema 35). + + A separable run (Cartesian, line, planar) is measured by its own operator's root, + which the native state already holds: nothing more is packed, and a plan that is + not that operator is refused. A radial run packs the entrance's material and + transverse squares and whether a material z table follows + (:func:`_flux_tables`); the admission record is not read. ``closure`` + sizes the magnetic closure's buffers for the run's own local statistics, + `stats`. + """ + flux = flux_of(problem.route) + if flux is None: + return + native.flux.closure = bool(set(stats.names) & LOCAL_FLUX_STATISTICS) + if problem.linear.separable is not None: + if flux.base is not problem.linear.separable or flux.material is not None: + raise BackendError( + "solprop-native measures a separable run by its own operator's root, " + "and this plane measure is another" + ) + return + base = flux.base + native.flux.k_squared = np.ascontiguousarray(base.k_squared, dtype=np.complex128) + native.flux.k_squared_low = np.ascontiguousarray( + base.k_squared_low, dtype=np.complex128 + ) + native.flux.kperp_sq = np.ascontiguousarray(base.kperp_sq, dtype=np.float64) + native.flux.kperp_sq_low = np.ascontiguousarray(base.kperp_sq_low, dtype=np.float64) + native.flux.table = flux.material is not None + + +def _lower_general( + native: _core.Problem, problem: PropagationProblem, chunk: int | None +) -> None: + """Fill the general modal route's maps, derived as the reference derives them. + + The per-node tensor, the form and the delayed maps come from solprop's own + derivations (``solprop.translate.general``), so both backends execute the same + arrays; the chunk is the route's own at the same budget + (:func:`solprop.backends.node_chunk.general_planned_chunk`), given as `chunk`. + Nothing is filled for a problem without ``transform.modal_tables``. + """ + tables = problem.transform.modal_tables + if tables is None: + return + assert tables.synthesis is not None + assert tables.projection is not None + assert chunk is not None + regional = problem.responses.regional_responses + owner, delayed_tensor, delayed_spectrum = node_delayed_maps(tables, regional) + terms = 0 if delayed_tensor is None else int(delayed_tensor.shape[1]) + general = native.general + general.enabled = True + general.real_form = node_kerr_form(regional) == "real" + # A moving guide's maps are every Chebyshev coefficient along z, flattened over + # (n_intervals, n_deg) in the order of the stage bundle's transform_weights; a + # constant guide's are the one coefficient. + general.map_intervals, general.map_n_deg = ( + int(one) for one in tables.synthesis.shape[:2] + ) + general.components = int(tables.electric.shape[2]) + general.nodes = int(tables.region_index.size) + general.node_chunk = chunk + general.responses = 0 if delayed_tensor is None else delayed_tensor.shape[0] + general.terms = terms + general.pad_scale = float(problem.grids.pad_scale) + general.epsilon0 = EPS0 + general.synthesis = np.ascontiguousarray(tables.synthesis) + general.projection = np.ascontiguousarray(tables.projection) + general.tensor = np.ascontiguousarray(node_kerr_tensor(tables, regional)) + general.frequency_factor = np.ascontiguousarray( + tables.source_frequency_factor, dtype=np.complex128 + ) + if delayed_tensor is not None: + general.delayed_tensor = np.ascontiguousarray(delayed_tensor) + if delayed_spectrum is not None: + general.delayed_spectrum = np.ascontiguousarray(delayed_spectrum) + if terms: + general.node_response = np.ascontiguousarray( + owner.astype(np.int64) + 1, dtype=np.uint64 + ) + + +def _lower_envelope(native: _core.Problem, problem: PropagationProblem) -> None: + """Fill the accepted cycle rules, (n_channels, 4, 512), and masked factors.""" + assembly = problem.responses.envelope_plasma + if assembly is None: + return + nodes = np.zeros((len(assembly.channels), 4, 512)) + weights = np.zeros_like(nodes) + for channel_index, channel in enumerate(assembly.channels): + for level, rule in enumerate(channel.cycle.rules): + nodes[channel_index, level, : len(rule.nodes)] = rule.nodes + weights[channel_index, level, : len(rule.weights)] = rule.weights + native.plasma.envelope = True + native.plasma.envelope_rule_nodes = nodes + native.plasma.envelope_rule_weights = weights + native.plasma.envelope_drude_factor = assembly.factors.drude + native.plasma.envelope_loss_factor = assembly.factors.loss + + +def _lower_channels(native: _core.Problem, problem: PropagationProblem) -> None: + """Fill independent rate tables without resampling their coefficients.""" + parameters: list[list[float]] = [] + coefficients: list[np.ndarray] = [] + offset = 0 + for channel in problem.responses.population_channels: + rate = channel.rate + parameters.append( + [ + rate.log_field_min, + rate.d_log_field, + rate.field_min, + rate.field_max, + rate.rate_max, + rate.coefficients.shape[1], + offset, + channel.charge_mass_ratio, + channel.ionisation_potential, + ] + ) + coefficients.append(rate.coefficients.ravel()) + offset += rate.coefficients.size + native.plasma.n_channels = len(parameters) + native.plasma.channel_parameters = np.asarray(parameters, dtype=np.float64).reshape( + -1, 9 + ) + if coefficients: + native.plasma.channel_coefficients = np.concatenate(coefficients) + + +def _lower_flow(native: _core.Problem, flow: _HostFlow | None) -> None: + """Fill a coupled flow's bins and residual data; its actions follow per step.""" + if flow is None: + return + slot = np.zeros(flow.active.size, dtype=np.uint64) + slot[flow.active] = np.arange(1, int(np.count_nonzero(flow.active)) + 1) + residual = flow.residual_terms + # The bins formed by scaling and squaring are numbered 1, 2, ... in row-major + # (section, bin) order and carry their Schur factors; the rest take the + # eigenbasis, whose `W` and `Y` are zero where it is not used. + fallback = ~flow.route + route = np.zeros(flow.route.shape, dtype=np.uint64) + route[fallback] = np.arange(1, int(np.count_nonzero(fallback)) + 1) + lowered = native.flow + lowered.enabled = True + lowered.active = int(np.count_nonzero(flow.active)) + lowered.sections = int(flow.frozen.shape[0]) + lowered.terms = 0 if residual is None else int(residual.shape[0]) + lowered.slot = slot + if residual is not None: + # The residual's data are zero on the bins the error norm does not score, so + # the engine's residual there is an exact zero and the frozen flow alone + # carries them, as in solprop's reference. Every term is finite, so each + # product with a zero is a signed zero and the running sums stay +0: the + # source is what adding nothing leaves, but for the sign of a zero. + scored = flow.scored[None, :, None, None] + lowered.frozen = np.ascontiguousarray(np.where(scored, flow.frozen, 0.0)) + lowered.residual_terms = np.ascontiguousarray(np.where(scored, residual, 0.0)) + lowered.left = flow.left + lowered.right = flow.right + lowered.eigenvalues = flow.eigenvalues + lowered.route = route + lowered.fallback_vectors = np.ascontiguousarray(flow.vectors[fallback]) + lowered.fallback_triangular = np.ascontiguousarray(flow.triangular[fallback]) diff --git a/native/src/solprop_native/_march.py b/native/src/solprop_native/_march.py new file mode 100644 index 000000000..1e564c5a0 --- /dev/null +++ b/native/src/solprop_native/_march.py @@ -0,0 +1,410 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""The native march protocol: its returns, its errors and the Adams record. + +Why a native march call returned and which error a failure maps to (``march::Reason``, +``march::Category``), the boundary watch's fresh state, the translation of a native +failure into solprop's exceptions, and the Adams march's record that a run starts +from (solprop SEP 39, SEP 47 section 5.2). +""" + +from __future__ import annotations + +import math +import types +from collections.abc import Mapping, Sequence +from types import TracebackType +from typing import Any + +import numpy as np +from solprop.backends.exponential import carrier_reference +from solprop.backends.ledger import CallLedger, Derivative +from solprop.backends.records import adams_policy +from solprop.exceptions import BackendError, ConvergenceError +from solprop.output.records import MarchSnapshot +from solprop.stepper.multistep import MarchPolicy, Multistep + +from . import _core + +__all__ = [ + "FRESH_WATCH", + "_BUDGET", + "_DIVERGED", + "_ERROR", + "_FLOOR", + "_NativeAdamsMarch", + "_envelope_errors", + "_raise_native", +] + + +class _envelope_errors: + """Translate refinement failures without importing solprop in the extension. + + A class rather than a generator: it wraps every native step call. + """ + + __slots__ = () + + def __enter__(self) -> None: + """Enter the native call.""" + + def __exit__( + self, + kind: type[BaseException] | None, + error: BaseException | None, + traceback: TracebackType | None, + ) -> None: + """Raise a refinement failure leaving the call as solprop's error.""" + del kind, traceback + if isinstance(error, _core.EnvelopeConvergenceError): + raise ConvergenceError(str(error)) from error + if isinstance(error, _core.BackendRefusal): + raise BackendError(str(error)) from error + + +FRESH_WATCH: Mapping[str, object] = types.MappingProxyType( + { + "has_last": False, + "last_z": 0.0, + "since": 0, + "busy": False, + "quiet_run": 0, + "quiet": False, + "warned": False, + "cap": math.inf, + } +) +"""The boundary watch's state at a run's start (``watch::State``): no check yet; a +session's monitor holds its own from there (solprop SEP 45 section 4.8).""" + +_MARCH_COUNTERS = ( + "startup", + "predictor", + "rejected", + "accepted_steps", + "rejected_steps", + "restarts", + "failure_restarts", + "dense_observations", +) +"""The call ledger's counters the native march keeps.""" + +_REASONS = ( + "reached", + "host_step", + "monitor", + "bound", + "budget", + "floor", + "error", + "diverged", + "watch", + "capacity", + "dense", + "stopped", + "halted", +) +"""Why a native march call returned (``march::Reason``), by name, in its order; the +benchmark counters (:mod:`solprop_native._counters`) report these names.""" +( + _REACHED, + _HOST_STEP, + _MONITOR, + _BOUND, + _BUDGET, + _FLOOR, + _ERROR, + _DIVERGED, + _WATCH, + _CAPACITY, + _DENSE, + _STOPPED, + _HALTED, +) = range(len(_REASONS)) + +_CONVERGENCE, _BACKEND, _VALUE, _REFUSED = 1, 2, 3, 4 +"""Which error a native failure maps to (``march::Category``).""" + + +class _NativeAdamsMarch: + """The native Adams march's record: its policy, ledger and history (SEP 39, 47). + + The native workspace steps solprop's exponential Adams march (solprop SEP 39 phase + 2): the controller, the history, the landings and the call ledger's counts advance + natively, from one policy record (:func:`~solprop.backends.records.adams_policy`). + This record is what the run starts from: the policy, the history's entries + (position, identity and engine row), the order, the rejection flags and the + :class:`~solprop.backends.ledger.CallLedger`, each fresh at a load (:meth:`reset`, + :meth:`begin_run`) or restored from a resume's snapshot and ledger (:meth:`resume`, + :meth:`restore_ledger`). A native session hands it to the workspace once, with the + run's start (:meth:`start_state`), and holds the march's state there from then on + (solprop SEP 47 section 5.2). + + `rows` are the names of the resident history's persistent rows + (``Workspace.diagnostics``); `engine` is a scripted harness's own history. + """ + + def __init__( + self, + backend: Any, + method: Multistep, + rows: Sequence[str] = (), + *, + engine: object | None = None, + ) -> None: + """Build the record over the backend's resident history, with no history.""" + self._backend = backend + self._method = method + self._rows = tuple(str(name) for name in rows) + self._engine: Any = engine + stepper = backend._stepper + problem = backend._problem + self._history: list[Derivative] = [] + self._rejections = 0 + self._last_rejected = False + self._order = 1 if method.variable_order else method.order + self.ledger = CallLedger() + # A scripted harness's history holds its own frame. + carrier = ( + carrier_reference(problem, method) + if engine is None + else getattr(engine, "carrier", None) + ) + # Every number the native controller and landings read (SEP 39 section 4.2), + # formed as the reference forms it. + self._policy: MarchPolicy = adams_policy( + problem, stepper, method, carrier=carrier is not None and carrier.active + ) + # Whether the stages left a quiet collar out at the last attempt. + self._collar_was_quiet = False + # The admission refuses a carrier frame on an operator with no chord + # (``refuse_carrier_frame``). + assert not ( + self._policy.carrier and carrier is not None and carrier.chord is None + ) + + @property + def n_slots(self) -> int: + """The history's rows: the method's order and two more.""" + return self._method.order + 2 + + def snapshot_nbytes(self) -> int: + """Return the bytes a snapshot of the history's rows copies, copying nothing.""" + shape = self._backend._problem.shape + return len(self._rows) * self.n_slots * math.prod(shape) * 2 * 8 + + # -- the record ------------------------------------------------------------------ + + def reset(self) -> None: + """Forget the history: the state was loaded outside the march.""" + self._history = [] + self._rejections = 0 + + def begin_run(self) -> None: + """Start a run's accounting afresh: the ledger and the last attempt's flags. + + A backend loaded for another run counts that run's calls alone and chooses its + first order and step as a fresh backend would (SEP 22 section 4.7, SEP 39 R9). + A resume restores its persisted ledger and flags afterwards. + """ + self.ledger = CallLedger() + self._last_rejected = False + self._collar_was_quiet = False + + def restore_ledger( + self, logical: Mapping[str, object], atomic: Mapping[str, object] | None = None + ) -> None: + """Restore a resumed run's logical call counts, not its history. + + - ``logical``: the persisted logical counts; empty installs nothing. + - ``atomic``: persisted atomic counts, which only the TDSE reference keeps. + """ + if atomic: + raise BackendError("native does not restore atomic response call ledgers") + if logical: + self.ledger = CallLedger.from_record(logical) + + def resume( + self, snapshot: MarchSnapshot, z: float + ) -> tuple[str | None, tuple[dict[str, object], ...]]: + """Take a snapshot's history where the run resumes, at `z`. + + Its entries, order and flags go to this record, and its rows are returned as + the workspace installs them with the run's start; the snapshot's state and + proposal are the caller's. A snapshot that cannot continue the march changes + nothing. + + - ``z``: the loaded position, m. + - Returns: ``None`` and the rows when the history continues, otherwise why the + march starts afresh, as a resume did before snapshots were kept, and no rows. + """ + reason = self._refusal(snapshot, z) + if reason is not None: + return reason, () + try: + rows = self._rows_record(snapshot.rows, self._backend._problem.shape) + except BackendError as error: + return str(error), () + self._history = [ + Derivative(float(z), None, identity, int(slot)) + for z, identity, slot in zip( + snapshot.positions, snapshot.identities, snapshot.slots, strict=True + ) + ] + self._order = snapshot.order + self._rejections = snapshot.rejections + self._last_rejected = snapshot.last_rejected + self._collar_was_quiet = snapshot.collar_was_quiet + return None, rows + + def _rows_record( + self, rows: Mapping[str, np.ndarray], shape: tuple[int, ...] + ) -> tuple[dict[str, object], ...]: + """Return a snapshot's rows as the workspace takes them, for a state of `shape`. + + The rows keep ``HostHistory``'s names and layout, ``(n_slots, n_modes, + n_pol, n_omega)``: ``"store"`` and, compensated, ``"states"`` and + ``"rates"``. A frozen-residual flow's history holds each sample's generator + applied to its state, ``"applied"``, where the host engine holds the sample's + weights (``"weights_"``); the two are not interchangeable, and each engine + refuses the other's. Everything is checked before any row is copied. + """ + names = list(self._rows) + weights = sorted(name for name in rows if name.startswith("weights_")) + if weights: + listed = ", ".join(weights) + raise BackendError( + "the native backend cannot continue this Adams history: the snapshot " + f"holds the host engine's frozen-residual weights ({listed}), and the " + "native engine keeps each sample's applied generator ('applied') " + "instead" + ) + foreign = sorted(set(rows) - set(names)) + missing = [name for name in names if name not in rows] + if foreign or missing: + raise BackendError( + "the native backend cannot continue this Adams history: its engine " + f"keeps the rows {names}, and the snapshot " + + (f"lacks {missing}" if missing else f"also holds {foreign}") + ) + expected = (self.n_slots, *shape) + for name in names: + row = rows[name] + if row.shape != expected or not np.iscomplexobj(row): + raise BackendError( + f"the snapshot's {name!r} rows ({row.dtype}, shape {row.shape}) do " + f"not fit the native backend's complex store of shape {expected}" + ) + return tuple( + { + "name": name, + "values": np.ascontiguousarray(rows[name], dtype=np.complex128).reshape( + self.n_slots, -1, shape[-1] + ), + } + for name in names + ) + + def _refusal(self, snapshot: MarchSnapshot, z: float) -> str | None: + """Return why a snapshot cannot continue this march at `z`, or ``None``.""" + shape = tuple(self._backend._problem.shape) + checks = ( + ( + snapshot.history_key != self._method.history_key, + f"the snapshot's history was kept by {snapshot.history_key}, and " + f"this run steps with {self._method.history_key}", + ), + ( + snapshot.z != z, + f"the snapshot is at z = {snapshot.z!r} m and the run resumes at " + f"z = {z!r} m", + ), + ( + snapshot.state.shape != shape, + f"the snapshot's state has shape {snapshot.state.shape}", + ), + ( + # A landing snaps to its plane, a rounding from the step's z + dz. + not math.isclose( + float(snapshot.positions[-1]), snapshot.z, rel_tol=1e-12 + ), + "the snapshot's newest sample is not at its position", + ), + ( + bool(np.any((snapshot.slots < 0) | (snapshot.slots >= self.n_slots))), + "the snapshot's history names no row of this engine", + ), + ) + return next((reason for failed, reason in checks if failed), None) + + def start_state(self, z: float, proposal: float, cursor: int) -> dict[str, object]: + """Return the march's state at a start, as the native call takes it. + + - ``z``: the loaded position, m. + - ``proposal``: the next step's proposal, m: ``dz_init``, or a resumed + snapshot's. + - Returns: the position, the proposal, the order, the rejection flags, the + cursor, the history's entries and the ledger's counts. + """ + ledger = self.ledger + return { + "z": float(z), + "proposal": float(proposal), + "order": self._order, + "rejections": self._rejections, + "last_rejected": self._last_rejected, + "collar_was_quiet": self._collar_was_quiet, + "cursor": int(cursor), + "history": [ + (entry.z, entry.identity, entry.slot) for entry in self._history + ], + "ledger": {name: getattr(ledger, name) for name in _MARCH_COUNTERS}, + "watch": dict(FRESH_WATCH), + } + + def _raise(self, reason: int, result: dict[str, Any]) -> None: + """Raise the error a native call returned with, if any. + + A session forms the call budget's own error (``NativeSession._budget_error``) + before it asks here. + """ + if reason == _FLOOR: + proposed = float(result["floor_proposal"]) + msg = ( + f"step control asked for {proposed:.3e} m, below the floor dz_min =" + f" {self._policy.dz_min:.3e} m; the tolerance cannot be met here" + ) + raise ConvergenceError(msg) + if reason == _DIVERGED: + dz, end = float(result["replay_dz"]), float(result["replay_end"]) + norm = float(result["replay_norm"]) + msg = ( + f"{self._backend.name}: a replayed {self._method.name} step of " + f"{dz:.4g} m ending at z = {end:.6g} m has a non-finite error norm " + f"({norm!r}): the field has diverged. A replayed schedule takes " + "its steps as given and cannot reject one, so a step beyond an " + "explicit stability limit -- the in-stage temporal collar's or a " + "coupled flow's advisory step -- grows until it overflows; the " + "translation refuses a declared step past the collar's limit and " + "warns on one past the flow's. Take shorter steps, or let the " + "controller choose them" + ) + raise ConvergenceError(msg) + if reason == _ERROR: + _raise_native(result) + + +def _raise_native(result: Mapping[str, Any]) -> None: + """Raise a native march's error as the exception class its category names.""" + message = str(result["message"]) + category = int(result["category"]) + if category == _CONVERGENCE: + raise ConvergenceError(message) + if category == _VALUE: + raise ValueError(message) + if category == _REFUSED: + raise BackendError(message) + raise RuntimeError(message) diff --git a/native/src/solprop_native/backend.py b/native/src/solprop_native/backend.py index 6cbf51e1b..cd19daaf7 100644 --- a/native/src/solprop_native/backend.py +++ b/native/src/solprop_native/backend.py @@ -7,35 +7,23 @@ from __future__ import annotations import dataclasses -import functools import logging import math -import types import weakref from collections.abc import Mapping, Sequence -from importlib.metadata import version from pathlib import Path -from types import TracebackType from typing import Any, ClassVar, Literal, cast import numpy as np import numpy.typing as npt -from solprop import transverse_monitor from solprop._process import acquire_host_blas, thread_budget from solprop.backends import tdse_source from solprop.backends.exponential import ( - CarrierReference, carrier_reference, refuse_carrier_frame, refuse_formulation, ) -from solprop.backends.ledger import CallLedger, Derivative, source_absent -from solprop.backends.memory import ( - LOCAL_FLUX_STATISTICS, - low_storage_flows, - low_storage_stage_register, - z_table_slots, -) +from solprop.backends.ledger import source_absent from solprop.backends.node_chunk import ( NodeChunkPlan, general_planned_chunk, @@ -44,7 +32,6 @@ from solprop.backends.protocol import TimingProbe from solprop.backends.records import ( RunRecords, - adams_policy, low_storage_settings, run_records, ) @@ -53,51 +40,27 @@ PreparedExecution, Start, ) -from solprop.backends.spatial import ( - FarFieldTables, - far_field_partials, - radial_absorber_matrix, -) +from solprop.backends.spatial import FarFieldTables from solprop.boundaries import SPLIT_RATE_BUCKETS -from solprop.constants import EPS0 -from solprop.exceptions import ( - BackendError, - ConvergenceError, -) -from solprop.grid import RadialGrid, TimeGrid +from solprop.exceptions import BackendError +from solprop.grid import TimeGrid from solprop.modes.protocol import Mode from solprop.monitor import BoundaryMonitor from solprop.noise import NoiseRealisation -from solprop.output.records import MarchSnapshot from solprop.profiles import ZProfile -from solprop.spatial import ( - FAR_FIELD_STATISTICS, -) -from solprop.stats import ( - StatsSpec, -) +from solprop.spatial import FAR_FIELD_STATISTICS +from solprop.stats import StatsSpec from solprop.stepper.low_storage import LowStorageRungeKutta -from solprop.stepper.multistep import MarchPolicy, Multistep +from solprop.stepper.multistep import Multistep from solprop.stepper.tableau import continuous_extension -from solprop.translate.chebyshev import ( - _EDGE_SLACK_FRACTION, # pyright: ignore[reportPrivateUsage] - the tables' own admitted span - PiecewiseChebyshev, -) from solprop.translate.coupled import ( CoupledFlowFactors, CoupledFlowTables, coupled_flow_factors, - flow_edge_slack, schur_eigenbasis, ) -from solprop.translate.general import ( - node_delayed_maps, - node_kerr_form, - node_kerr_tensor, -) from solprop.translate.linop import ( delta_phi, - power_weight_at, unitary_phase, ) from solprop.translate.plasma_source import ( @@ -114,14 +77,12 @@ GeneralRoute, ModalRoute, SpatialRoute, - flux_of, tdse_descriptor, transverse_of, ) from solprop.translate.run_request import RunRequest from solprop.translate.stage import ( StageScalars, - StageTerms, combine, require_device_ready, terms, @@ -148,10 +109,21 @@ build_info, check_build, ) +from ._device import _build_facts, _on_device, _resident_flows, native_node_chunk from ._execution import ExecutionRecord from ._helpers import NATIVE from ._host_source import TDSEHostSource from ._load import Loaded, StageCache, load, propagator_mode +from ._lowering import ( + _chord_tables, + _edge_lines, + _flux_tables, + _lower, + _LowerPlan, + _section_tables, + _ZTables, +) +from ._march import _envelope_errors, _NativeAdamsMarch from ._probe import NativeProbe from ._statistics import ( StatisticsLowering, @@ -164,31 +136,6 @@ logger = logging.getLogger(__name__) -class _envelope_errors: - """Translate refinement failures without importing solprop in the extension. - - A class rather than a generator: it wraps every native step call. - """ - - __slots__ = () - - def __enter__(self) -> None: - """Enter the native call.""" - - def __exit__( - self, - kind: type[BaseException] | None, - error: BaseException | None, - traceback: TracebackType | None, - ) -> None: - """Raise a refinement failure leaving the call as solprop's error.""" - del kind, traceback - if isinstance(error, _core.EnvelopeConvergenceError): - raise ConvergenceError(str(error)) from error - if isinstance(error, _core.BackendRefusal): - raise BackendError(str(error)) from error - - ADAMS_MAX_CHANNELS = 16 """The most channels of a coupled block whose channel vectors a CPU build's resident Adams engine holds on the stack. @@ -1111,34 +1058,6 @@ def close(self) -> None: self._closed = True -@dataclasses.dataclass(frozen=True) -class _BuildFacts: - """The parts of a backend's metadata that are fixed for the process.""" - - info: Mapping[str, object] - solprop_version: str - device_team_size: int - device_reduction_size: int - - -@functools.cache -def _build_facts() -> _BuildFacts: - """Return the build record and the installed solprop version, read once. - - Reading them costs about a millisecond (the package metadata lookup and the native - record), which a per-plane metadata record paid at every saved plane. - """ - native_record = { - key: value for key, value in _core.build_record().items() if value is not None - } - return _BuildFacts( - info=types.MappingProxyType(dataclasses.asdict(build_info())), - solprop_version=version("solprop"), - device_team_size=int(native_record["device_team_size"]), - device_reduction_size=int(native_record["device_reduction_size"]), - ) - - def _require_initial_array(field: object, shape: tuple[int, ...]) -> None: if not isinstance(field, np.ndarray): raise TypeError("field must be a NumPy ndarray") @@ -1193,344 +1112,6 @@ def _probe_start(records: RunRecords, loaded: Loaded) -> dict[str, object]: } -def _edge_lines(edge: TransverseEdge) -> dict[str, object]: - """Return a requested collar's lines as the workspace takes them (SEP 44).""" - lines = edge.reduction.lines - axes = {"r": 0, "x": 1, "y": 2} - none = 2**64 - 1 - strip = [np.asarray(line.strip, dtype=np.uint64) for line in lines] - window = [np.asarray(line.window, dtype=np.float64) for line in lines] - onsets = [line.onset if line.onset is not None else (none, 0, 0) for line in lines] - space = edge.space - return { - "axes": np.array([axes[line.axis] for line in lines], dtype=np.uint64), - "counts": np.array([array.size for array in strip], dtype=np.uint64), - "positions": np.concatenate(strip) if strip else np.empty(0, dtype=np.uint64), - "windows": np.concatenate(window) if window else np.empty(0, dtype=np.float64), - "inward": np.array([line.inward for line in lines], dtype=np.uint64).reshape( - -1 - ), - "factors": np.array( - [ - line.spacing / (2 * array.size) * line.measure if array.size else 0.0 - for line, array in zip(lines, strip, strict=True) - ], - dtype=np.float64, - ), - "measures": np.array([line.measure for line in lines], dtype=np.float64), - "onsets": np.array(onsets, dtype=np.uint64).reshape(-1), - "steps": np.array([line.onset_step for line in lines], dtype=np.float64), - "k_min": float(edge.k_min), - "scale": np.sqrt(np.asarray(space.r, dtype=np.float64)) - if lines and isinstance(space, RadialGrid) - else np.empty(0, dtype=np.float64), - "batch_bytes": transverse_monitor._STRIP_BATCH_BYTES, # pyright: ignore[reportPrivateUsage] - } - - -def _z_table(kind: str, table: PiecewiseChebyshev) -> dict[str, object]: - """Return one z table as the workspace makes it resident: its breakpoints, the - admitted span (``_EDGE_SLACK_FRACTION`` past each end) and its coefficients.""" - low, high = table.z_span - slack = _EDGE_SLACK_FRACTION * (high - low) - n_intervals, n_terms = table.coefficients.shape[:2] - return { - "kind": kind, - "edges": np.ascontiguousarray(table.breakpoints, dtype=np.float64), - "lower": low - slack, - "upper": high + slack, - "coefficients": np.ascontiguousarray( - table.coefficients, dtype=np.complex128 - ).reshape(n_intervals, n_terms, -1), - } - - -def _flux_tables(problem: PropagationProblem) -> list[dict[str, object]]: - """Return a radial plane measure's material z table, or none (SEP 34).""" - flux = flux_of(problem.route) - if flux is None or flux.material is None: - return [] - return [_z_table("flux", flux.material)] - - -def _flow_guard(flow: _HostFlow | None) -> dict[str, object] | None: - """Return a coupled flow's certificate and the guard's constants, or ``None``. - - Each section's row-sum norm and conditioning bound per active bin, with the scale - solprop's reference reduces an exponent below, the most squarings and the - rounding allowance on two spectral norms (solprop SEP 45 phase 4). - """ - if flow is None: - return None - row_sum = np.ascontiguousarray(flow.row_sum, dtype=np.float64) - return { - "row_sum": row_sum, - "condition_bound": np.ascontiguousarray( - flow.condition_bound, dtype=np.float64 - ).reshape(row_sum.size, -1), - "taylor_scale": _TAYLOR_SCALE, - "max_squarings": _MAX_FLOW_SQUARINGS, - "allowance": _flow_allowance(int(flow.vectors.shape[-1])), - } - - -_EMPTY = np.empty(0, dtype=np.complex128) -"""No modified-noise realisation: the native state adds none.""" - - -def _section_tables( - problem: PropagationProblem, *, moving: bool, residual: bool -) -> dict[str, object]: - """Return a coupled flow's and a moving guide's sections as the stage tables take - them: their edges, degrees and the edges' slack (`flow_edge_slack`, - `transform_section`), and whether the flow carries a frozen residual.""" - flow = problem.linear.coupled_flow - maps = problem.transform.modal_tables if moving else None - none: dict[str, object] = {"edges": [], "degrees": 0, "slack": 0.0} - out: dict[str, object] = { - "flow": none, - "flow_residual": False, - "transform": none, - } - if flow is not None: - out |= { - "flow": { - "edges": np.asarray(flow.z_edges, dtype=np.float64).tolist(), - "degrees": int(flow.n_deg), - "slack": float(flow_edge_slack(flow)), - }, - "flow_residual": residual, - } - if maps is not None and maps.synthesis is not None: - edges = np.asarray(maps.z_edges, dtype=np.float64) - out["transform"] = { - "edges": edges.tolist(), - "degrees": int(maps.synthesis.shape[1]), - "slack": _EDGE_SLACK_FRACTION * float(edges[-1] - edges[0]), - } - return out - - -def _chord_tables(carrier: CarrierReference | None) -> dict[str, object]: - """Return the carrier frame's chord as the native stage tables take it. - - The imaginary part of the reference element's operator (`CarrierChord`): a - constant's rate, an expansion's terms over the ``Phi`` weights - the tables already hold, or a z table's reference-element column as a scalar table - of its own (solprop issue #255), packed as its batch pass reads it. None without an - active frame. - """ - chord = None if carrier is None or not carrier.active else carrier.chord - if chord is not None and chord.constant is not None: - return { - "chord": "constant", - "chord_rate": float(np.imag(chord.constant)), - "chord_terms": [], - "chord_table": None, - } - if chord is not None and chord.expansion is not None: - terms = np.imag(np.asarray(chord.expansion.terms)) - return { - "chord": "expansion", - "chord_rate": 0.0, - "chord_terms": terms.tolist(), - "chord_table": None, - } - if chord is not None and chord.table is not None: - edges, lower, upper, coefficients = chord.table.packed() - # The imaginary part a complex Clenshaw pass forms at a real position, by the - # same operations (`stage::chord`). - column = np.ascontiguousarray(np.imag(coefficients)) - return { - "chord": "table", - "chord_rate": 0.0, - "chord_terms": [], - "chord_table": (edges, lower, upper, column), - } - return {"chord": "none", "chord_rate": 0.0, "chord_terms": [], "chord_table": None} - - -def _on_device() -> bool: - """Return whether the installed build executes on an accelerator.""" - return _build_facts().info["device_backend"] != "none" - - -DEVICE_BUDGET_SHARE = 4 -"""The share of the accelerator's free bytes an automatic plan budgets: one quarter. - -The budget is spent on the host plan's conservative inventory (twenty column buffers a -Cartesian column, about three times the live buffers, solprop issue #349), so the live -space-domain buffers take near a twelfth of the free bytes, leaving the rest for the -state and the march's registers.""" - - -def device_node_chunk( - problem: PropagationProblem, planned: int, free: int | None -) -> int: - """Return the node chunk a device build takes when no budget or chunk is given. - - solprop issue #361: on a device a node chunk is a kernel launch, and fewer, larger - launches are faster: one chunk took 0.74-0.92 of the default's time on AMD Instinct - MI300X spatial routes (SEP 47 Part II), at the cost of space-domain buffers for - every node at once. So an automatic plan budgets :data:`DEVICE_BUDGET_SHARE`'s share - of the free device bytes, through the host plan's own rule and response admissions - (:func:`solprop.backends.node_chunk.plan_node_chunk`): one chunk wherever that fits, - otherwise the largest that does, and never fewer nodes than the host plan's chunk - at the route's default budget. It is consulted only for an automatic plan: an - explicit budget or chunk always wins. It runs at backend construction, after the - translation, so no device runtime starts before a scan forks. - - Parameters - ---------- - problem : solprop.translate.problem.PropagationProblem - The validated problem. - planned : int - The host plan's chunk at the default budget - (:func:`solprop.backends.node_chunk.plan_node_chunk`). - free : int or None - The accelerator's free bytes, or ``None`` where none is known. - - Returns - ------- - int - The device's chunk, between 1 and the node count. - """ - if free is None: - return planned - budget = max(1, free // DEVICE_BUDGET_SHARE) - try: - roomy = plan_node_chunk(problem, ExecutionOptions(node_budget=budget)) - except BackendError: - # A quarter of the free bytes cannot hold one node: the default plan stands, - # and its own allocation reports what does not fit. - return planned - return max(planned, roomy.chunk) - - -def native_node_chunk( - problem: PropagationProblem, plan: NodeChunkPlan, *, device: bool -) -> int: - """Lower the host node-chunk plan to the chunk this backend executes. - - A device build's automatic plan goes through :func:`device_node_chunk`. The chunk - is then bounded by the node count, and on the hybrid planar kind made whole y rows, - as many as it holds and at least one, which is solprop's reference's own blocking. - - Parameters - ---------- - problem : solprop.translate.problem.PropagationProblem - The validated problem. - plan : solprop.backends.node_chunk.NodeChunkPlan - The host plan. - device : bool - Whether this build executes on an accelerator. - - Returns - ------- - int - The chunk the native problem is built with. - """ - chunk = plan.chunk - if device and plan.automatic: - chunk = device_node_chunk(problem, chunk, _core.available_device_memory()) - chunk = min(chunk, problem.transform.n_nodes) - geometry = problem.transform.geometry - if geometry is not None and geometry.kind == "slab": - n_x, n_y = geometry.shape - chunk = min(n_y, max(1, chunk // n_x)) * n_x - return chunk - - -def _low_storage_flows( - problem: PropagationProblem, stepper: StepperSettings, *, device: bool -) -> int | None: - """Return the low-storage march's flow volumes, or None for another march. - - The flows solprop's estimate counts (solprop kernel schema 32) -- except that a - device build recomputes under ``"auto"``, where the estimate's CPU count - over-states it by the table. A device's input crosses the state's layout through - tiles (``docs/dev/benchmarks.md``); a CPU's has its own. - """ - if not isinstance(stepper.integrator, LowStorageRungeKutta): - return None - return _resident_flows( - low_storage_flows(problem, stepper, "native"), stepper, device=device - ) - - -def _resident_flows(flows: int, stepper: StepperSettings, *, device: bool) -> int: - """Return the flow volumes the native state holds for a CPU's count `flows`. - - A device build recomputes under ``"auto"`` and holds none - (:class:`~solprop.backends.records.LowStorageSettings` counts a CPU's). - """ - return 0 if device and stepper.propagator_mode == "auto" else flows - - -FRESH_WATCH: Mapping[str, object] = types.MappingProxyType( - { - "has_last": False, - "last_z": 0.0, - "since": 0, - "busy": False, - "quiet_run": 0, - "quiet": False, - "warned": False, - "cap": math.inf, - } -) -"""The boundary watch's state at a run's start (``watch::State``): no check yet; a -session's monitor holds its own from there (solprop SEP 45 section 4.8).""" - -_MARCH_COUNTERS = ( - "startup", - "predictor", - "rejected", - "accepted_steps", - "rejected_steps", - "restarts", - "failure_restarts", - "dense_observations", -) -"""The call ledger's counters the native march keeps.""" - -_REASONS = ( - "reached", - "host_step", - "monitor", - "bound", - "budget", - "floor", - "error", - "diverged", - "watch", - "capacity", - "dense", - "stopped", - "halted", -) -"""Why a native march call returned (``march::Reason``), by name, in its order; the -benchmark counters (:mod:`solprop_native._counters`) report these names.""" -( - _REACHED, - _HOST_STEP, - _MONITOR, - _BOUND, - _BUDGET, - _FLOOR, - _ERROR, - _DIVERGED, - _WATCH, - _CAPACITY, - _DENSE, - _STOPPED, - _HALTED, -) = range(len(_REASONS)) - -_CONVERGENCE, _BACKEND, _VALUE, _REFUSED = 1, 2, 3, 4 -"""Which error a native failure maps to (``march::Category``).""" - - def stage_tables_available(problem: PropagationProblem) -> bool: """Return whether the native state reads a problem's stage rows from tables. @@ -1601,388 +1182,6 @@ def host_rows_refusal(problem: PropagationProblem) -> str | None: ) -class _NativeAdamsMarch: - """The native Adams march's record: its policy, ledger and history (SEP 39, 47). - - The native workspace steps solprop's exponential Adams march (solprop SEP 39 phase - 2): the controller, the history, the landings and the call ledger's counts advance - natively, from one policy record (:func:`~solprop.backends.records.adams_policy`). - This record is what the run starts from: the policy, the history's entries - (position, identity and engine row), the order, the rejection flags and the - :class:`~solprop.backends.ledger.CallLedger`, each fresh at a load (:meth:`reset`, - :meth:`begin_run`) or restored from a resume's snapshot and ledger (:meth:`resume`, - :meth:`restore_ledger`). A native session hands it to the workspace once, with the - run's start (:meth:`start_state`), and holds the march's state there from then on - (solprop SEP 47 section 5.2). - - `rows` are the names of the resident history's persistent rows - (``Workspace.diagnostics``); `engine` is a scripted harness's own history. - """ - - def __init__( - self, - backend: Any, - method: Multistep, - rows: Sequence[str] = (), - *, - engine: object | None = None, - ) -> None: - """Build the record over the backend's resident history, with no history.""" - self._backend = backend - self._method = method - self._rows = tuple(str(name) for name in rows) - self._engine: Any = engine - stepper = backend._stepper - problem = backend._problem - self._history: list[Derivative] = [] - self._rejections = 0 - self._last_rejected = False - self._order = 1 if method.variable_order else method.order - self.ledger = CallLedger() - # A scripted harness's history holds its own frame. - carrier = ( - carrier_reference(problem, method) - if engine is None - else getattr(engine, "carrier", None) - ) - # Every number the native controller and landings read (SEP 39 section 4.2), - # formed as the reference forms it. - self._policy: MarchPolicy = adams_policy( - problem, stepper, method, carrier=carrier is not None and carrier.active - ) - # Whether the stages left a quiet collar out at the last attempt. - self._collar_was_quiet = False - # The admission refuses a carrier frame on an operator with no chord - # (``refuse_carrier_frame``). - assert not ( - self._policy.carrier and carrier is not None and carrier.chord is None - ) - - @property - def n_slots(self) -> int: - """The history's rows: the method's order and two more.""" - return self._method.order + 2 - - def snapshot_nbytes(self) -> int: - """Return the bytes a snapshot of the history's rows copies, copying nothing.""" - shape = self._backend._problem.shape - return len(self._rows) * self.n_slots * math.prod(shape) * 2 * 8 - - # -- the record ------------------------------------------------------------------ - - def reset(self) -> None: - """Forget the history: the state was loaded outside the march.""" - self._history = [] - self._rejections = 0 - - def begin_run(self) -> None: - """Start a run's accounting afresh: the ledger and the last attempt's flags. - - A backend loaded for another run counts that run's calls alone and chooses its - first order and step as a fresh backend would (SEP 22 section 4.7, SEP 39 R9). - A resume restores its persisted ledger and flags afterwards. - """ - self.ledger = CallLedger() - self._last_rejected = False - self._collar_was_quiet = False - - def restore_ledger( - self, logical: Mapping[str, object], atomic: Mapping[str, object] | None = None - ) -> None: - """Restore a resumed run's logical call counts, not its history. - - - ``logical``: the persisted logical counts; empty installs nothing. - - ``atomic``: persisted atomic counts, which only the TDSE reference keeps. - """ - if atomic: - raise BackendError("native does not restore atomic response call ledgers") - if logical: - self.ledger = CallLedger.from_record(logical) - - def resume( - self, snapshot: MarchSnapshot, z: float - ) -> tuple[str | None, tuple[dict[str, object], ...]]: - """Take a snapshot's history where the run resumes, at `z`. - - Its entries, order and flags go to this record, and its rows are returned as - the workspace installs them with the run's start; the snapshot's state and - proposal are the caller's. A snapshot that cannot continue the march changes - nothing. - - - ``z``: the loaded position, m. - - Returns: ``None`` and the rows when the history continues, otherwise why the - march starts afresh, as a resume did before snapshots were kept, and no rows. - """ - reason = self._refusal(snapshot, z) - if reason is not None: - return reason, () - try: - rows = self._rows_record(snapshot.rows, self._backend._problem.shape) - except BackendError as error: - return str(error), () - self._history = [ - Derivative(float(z), None, identity, int(slot)) - for z, identity, slot in zip( - snapshot.positions, snapshot.identities, snapshot.slots, strict=True - ) - ] - self._order = snapshot.order - self._rejections = snapshot.rejections - self._last_rejected = snapshot.last_rejected - self._collar_was_quiet = snapshot.collar_was_quiet - return None, rows - - def _rows_record( - self, rows: Mapping[str, np.ndarray], shape: tuple[int, ...] - ) -> tuple[dict[str, object], ...]: - """Return a snapshot's rows as the workspace takes them, for a state of `shape`. - - The rows keep ``HostHistory``'s names and layout, ``(n_slots, n_modes, - n_pol, n_omega)``: ``"store"`` and, compensated, ``"states"`` and - ``"rates"``. A frozen-residual flow's history holds each sample's generator - applied to its state, ``"applied"``, where the host engine holds the sample's - weights (``"weights_"``); the two are not interchangeable, and each engine - refuses the other's. Everything is checked before any row is copied. - """ - names = list(self._rows) - weights = sorted(name for name in rows if name.startswith("weights_")) - if weights: - listed = ", ".join(weights) - raise BackendError( - "the native backend cannot continue this Adams history: the snapshot " - f"holds the host engine's frozen-residual weights ({listed}), and the " - "native engine keeps each sample's applied generator ('applied') " - "instead" - ) - foreign = sorted(set(rows) - set(names)) - missing = [name for name in names if name not in rows] - if foreign or missing: - raise BackendError( - "the native backend cannot continue this Adams history: its engine " - f"keeps the rows {names}, and the snapshot " - + (f"lacks {missing}" if missing else f"also holds {foreign}") - ) - expected = (self.n_slots, *shape) - for name in names: - row = rows[name] - if row.shape != expected or not np.iscomplexobj(row): - raise BackendError( - f"the snapshot's {name!r} rows ({row.dtype}, shape {row.shape}) do " - f"not fit the native backend's complex store of shape {expected}" - ) - return tuple( - { - "name": name, - "values": np.ascontiguousarray(rows[name], dtype=np.complex128).reshape( - self.n_slots, -1, shape[-1] - ), - } - for name in names - ) - - def _refusal(self, snapshot: MarchSnapshot, z: float) -> str | None: - """Return why a snapshot cannot continue this march at `z`, or ``None``.""" - shape = tuple(self._backend._problem.shape) - checks = ( - ( - snapshot.history_key != self._method.history_key, - f"the snapshot's history was kept by {snapshot.history_key}, and " - f"this run steps with {self._method.history_key}", - ), - ( - snapshot.z != z, - f"the snapshot is at z = {snapshot.z!r} m and the run resumes at " - f"z = {z!r} m", - ), - ( - snapshot.state.shape != shape, - f"the snapshot's state has shape {snapshot.state.shape}", - ), - ( - # A landing snaps to its plane, a rounding from the step's z + dz. - not math.isclose( - float(snapshot.positions[-1]), snapshot.z, rel_tol=1e-12 - ), - "the snapshot's newest sample is not at its position", - ), - ( - bool(np.any((snapshot.slots < 0) | (snapshot.slots >= self.n_slots))), - "the snapshot's history names no row of this engine", - ), - ) - return next((reason for failed, reason in checks if failed), None) - - def start_state(self, z: float, proposal: float, cursor: int) -> dict[str, object]: - """Return the march's state at a start, as the native call takes it. - - - ``z``: the loaded position, m. - - ``proposal``: the next step's proposal, m: ``dz_init``, or a resumed - snapshot's. - - Returns: the position, the proposal, the order, the rejection flags, the - cursor, the history's entries and the ledger's counts. - """ - ledger = self.ledger - return { - "z": float(z), - "proposal": float(proposal), - "order": self._order, - "rejections": self._rejections, - "last_rejected": self._last_rejected, - "collar_was_quiet": self._collar_was_quiet, - "cursor": int(cursor), - "history": [ - (entry.z, entry.identity, entry.slot) for entry in self._history - ], - "ledger": {name: getattr(ledger, name) for name in _MARCH_COUNTERS}, - "watch": dict(FRESH_WATCH), - } - - def _raise(self, reason: int, result: dict[str, Any]) -> None: - """Raise the error a native call returned with, if any. - - A session forms the call budget's own error (``NativeSession._budget_error``) - before it asks here. - """ - if reason == _FLOOR: - proposed = float(result["floor_proposal"]) - msg = ( - f"step control asked for {proposed:.3e} m, below the floor dz_min =" - f" {self._policy.dz_min:.3e} m; the tolerance cannot be met here" - ) - raise ConvergenceError(msg) - if reason == _DIVERGED: - dz, end = float(result["replay_dz"]), float(result["replay_end"]) - norm = float(result["replay_norm"]) - msg = ( - f"{self._backend.name}: a replayed {self._method.name} step of " - f"{dz:.4g} m ending at z = {end:.6g} m has a non-finite error norm " - f"({norm!r}): the field has diverged. A replayed schedule takes " - "its steps as given and cannot reject one, so a step beyond an " - "explicit stability limit -- the in-stage temporal collar's or a " - "coupled flow's advisory step -- grows until it overflows; the " - "translation refuses a declared step past the collar's limit and " - "warns on one past the flow's. Take shorter steps, or let the " - "controller choose them" - ) - raise ConvergenceError(msg) - if reason == _ERROR: - _raise_native(result) - - -def _raise_native(result: Mapping[str, Any]) -> None: - """Raise a native march's error as the exception class its category names.""" - message = str(result["message"]) - category = int(result["category"]) - if category == _CONVERGENCE: - raise ConvergenceError(message) - if category == _VALUE: - raise ValueError(message) - if category == _REFUSED: - raise BackendError(message) - raise RuntimeError(message) - - -_TABLE_POINTS = 16 -"""The most positions one native table evaluation takes (``z_table::max_points``).""" - - -class _ZTables: - """The general z tables of one workspace (solprop SEP 29 section 6.6, G1). - - solprop falls back to a Chebyshev table of the state's size for ``Phi`` - (``linear.phi_coefficients``) or the response prefactor (``responses.prefactor``) - when neither is a weighted sum of fixed arrays. The coefficients cross once, with - the workspace's setup (:meth:`records`), and stay resident; a trial's positions are - then evaluated natively into the state's term slots, ``slots`` of them, and the - stage weights select them: ``Phi(z1) - Phi(z0)`` is the row ``e_1 - e_0`` and - ``Phi(z1)`` the row ``e_1``, so every native consumer of the terms (the Runge-Kutta - stages, the low-storage flows, the Adams phase and the noise) reads the table - unchanged, as one rounded subtraction of two values, as solprop's reference forms - it. The prefactor's slots are its values at the same positions, one-hot weighted. - - The slots are kept as a least-recently-used set of positions in the workspace - (``stage::ZSlots``, solprop SEP 39 phase 4): each call's positions are evaluated - only where no slot holds them yet, so an accepted step's end, which is the next - step's anchor, is never evaluated twice. A call's positions number at most - ``slots`` (the tableau's distinct stage fractions, five for the low-storage march), - so none of one call's positions is evicted by another of its own. A compensated - Adams march reads ``D = dPhi/dz`` from the derivative table (``linear.rate``) into - one more ``Phi`` slot, ``rate_slot``. - - Attributes - ---------- - phi, prefactor : bool - Which quantities are z tables. - slots : int - The position slots, each one value of each table. - rate_slot : int or None - The ``Phi`` term that holds the rate, or ``None`` when no march reads it. - """ - - def __init__(self, problem: PropagationProblem, stepper: StepperSettings) -> None: - """Size the slots from the problem and its method; nothing crosses yet.""" - self._problem = problem - self.phi = problem.linear.phi_representation == "table" - self.prefactor = problem.responses.prefactor_representation == "table" - integrator = stepper.integrator - self.slots = z_table_slots(stepper) - if self.slots > _TABLE_POINTS: - raise BackendError( - f"native evaluates a z table at {_TABLE_POINTS} positions a call, and " - f"this tableau's stages need {self.slots}" - ) - compensated = ( - self.phi - and isinstance(integrator, Multistep) - and integrator.formulation == "exponential" - and not source_absent(problem) - ) - self.rate_slot = self.slots if compensated else None - - @property - def n_phi(self) -> int: - """The native ``Phi`` term count: the slots, and the rate's when it is read.""" - return self.slots + (self.rate_slot is not None) - - @property - def nbytes(self) -> int: - """The bytes of the coefficients that cross to the workspace.""" - coefficients = 0 - if self.phi: - phi = self._problem.linear.phi_coefficients - assert phi is not None - coefficients += phi.nbytes - if self.rate_slot is not None: - coefficients += self._problem.linear.rate.coefficients.nbytes - if self.prefactor: - table = self._problem.responses.prefactor - assert table is not None - coefficients += table.coefficients.nbytes - return int(coefficients) - - def slots_record(self) -> dict[str, object]: - """Return the slots as the workspace configures them (``ZSlots``).""" - return { - "phi": self.phi, - "prefactor": self.prefactor, - "slots": self.slots, - "rate": self.rate_slot is not None, - } - - def records(self) -> list[dict[str, object]]: - """Return the tables the workspace makes resident (``ZTable``).""" - out: list[dict[str, object]] = [] - if self.phi: - out.append(_z_table("phi", self._problem.linear.polynomial)) - if self.rate_slot is not None: - out.append(_z_table("rate", self._problem.linear.rate)) - if self.prefactor: - table = self._problem.responses.prefactor - assert table is not None - out.append(_z_table("prefactor", table)) - return out - - def _refuse_tdse( problem: PropagationProblem, request: RunRequest, binding: TDSEBinding | None ) -> None: @@ -2207,6 +1406,27 @@ def _flow_allowance(modes: int) -> float: return 1.0 + 64.0 * modes * float(np.finfo(float).eps) +def _flow_guard(flow: _HostFlow | None) -> dict[str, object] | None: + """Return a coupled flow's certificate and the guard's constants, or ``None``. + + Each section's row-sum norm and conditioning bound per active bin, with the scale + solprop's reference reduces an exponent below, the most squarings and the + rounding allowance on two spectral norms (solprop SEP 45 phase 4). + """ + if flow is None: + return None + row_sum = np.ascontiguousarray(flow.row_sum, dtype=np.float64) + return { + "row_sum": row_sum, + "condition_bound": np.ascontiguousarray( + flow.condition_bound, dtype=np.float64 + ).reshape(row_sum.size, -1), + "taylor_scale": _TAYLOR_SCALE, + "max_squarings": _MAX_FLOW_SQUARINGS, + "allowance": _flow_allowance(int(flow.vectors.shape[-1])), + } + + def _refined_record( problem: PropagationProblem, stepper: StepperSettings, @@ -2320,657 +1540,3 @@ def _planning_record(diagnostics: Mapping[str, Any]) -> dict[str, object]: "missing": list(cast(list[str], found["missing"])), } } - - -@dataclasses.dataclass(frozen=True) -class _LowerPlan: - """The backend's choices a lowering takes beside the problem and its request. - - `terms` are the fixed stage terms, `flow` a coupled flow's host factors and - `tables` the z tables; the chunks are the backend's lowered execution choices - (solprop SEP 47 section 6.5), the automatic plan's when absent. `cpu_execution` - is the workspace's CPU record (the process's when absent) and `fft_planning` its - FFT planning record (none on a device); `device` says the state is a device's, and - `setup` holds the workspace's set-once data by part (`NativeBackend._setup`). - """ - - terms: StageTerms - flow: _HostFlow | None = None - tables: _ZTables | None = None - node_chunk: int | None = None - general_chunk: int | None = None - cpu_execution: ExecutionRecord | None = None - fft_planning: Mapping[str, object] | None = None - device: bool = False - setup: Mapping[str, object] | None = None - - -def _lower( - problem: PropagationProblem, request: RunRequest, plan: _LowerPlan -) -> _core.Problem: - """Fill the native problem's structs from solprop's problem, request and `plan`.""" - stepper = request.stepper - node_chunk = plan.node_chunk - if node_chunk is None: - node_chunk = native_node_chunk( - problem, plan_node_chunk(problem), device=_on_device() - ) - general_chunk = plan.general_chunk - if general_chunk is None and problem.transform.modal_tables is not None: - general_chunk = general_planned_chunk(problem, plan_node_chunk(problem)) - native = _core.Problem() - native.cpu_execution = (plan.cpu_execution or _package.EXECUTION).native() - if plan.fft_planning is not None: - native.fft_planning = dict(plan.fft_planning) - kerr = problem.responses.kerr_coefficients - dispersive = problem.responses.dispersive_kerr - absorber = problem.responses.temporal_absorber - absorber_fine = problem.responses.temporal_absorber_fine - spatial_absorber = radial_absorber_matrix( - problem.transform, problem.grids.pad_scale - ) - temporal_in_stage = ( - absorber_fine is not None and problem.responses.absorber_scheme == "nonlinear" - ) - geometry = problem.transform.geometry - # Free space and schema 15's hybrid planar kind share the separable operator and - # the transverse FFT engine; the planar kind adds its slab rule in x. - free = geometry is not None and geometry.kind in ("free", "slab") - spatial_absorption = bool(spatial_absorber.size) or ( - free and geometry is not None and bool(np.any(geometry.absorber_rate)) - ) - - dims = native.dims - dims.n_omega = problem.shape[-1] - dims.n_t = problem.grids.n_t - dims.n_to = problem.grids.n_to - dims.n_modes = problem.shape[0] - dims.n_pol_state = problem.shape[1] - dims.n_pol_transform = problem.transform.n_pol - dims.n_nodes = problem.transform.n_nodes - # solprop SEP 47 section 6.5: the backend's execution choice, lowered by - # `native_node_chunk`. - dims.node_chunk = node_chunk - # solprop SEP 6's number-conserving sector is formed on analytic signals, of the - # filtered fields and, for the polynomial, of the bare one; a Raman drive beside - # it is the cycle-averaged one, which reads the analytic signal too. The general - # modal route synthesises its own analytic signal on the real family. - dims.family = ( - _core.FieldFamily.envelope - if problem.grid_kind == "envelope" - else _core.FieldFamily.field_analytic - if (kerr is not None and kerr.analytic_signal) - or ( - dispersive is not None - and dispersive.sector == "conserving" - and problem.transform.modal_tables is None - ) - else _core.FieldFamily.field_real - ) - - grids = problem.grids - carrier = grids.carrier_2w0 - grid = native.grid - grid.crop_index = np.ascontiguousarray(grids.crop_index, dtype=np.uint64) - grid.active_mask = np.ascontiguousarray(grids.active_mask) - grid.norm_mask = np.ascontiguousarray(grids.norm_mask) - grid.omega_window = np.ascontiguousarray(grids.omega_window) - grid.omega = np.ascontiguousarray(grids.omega) - grid.time_window = np.ascontiguousarray(grids.t_window_fine) - if problem.grid_kind == "field" and problem.responses.plasma_channels: - grid.time_window_derivative = np.ascontiguousarray( - grids.t_window_fine_derivative - ) - if geometry is None: - grid.power_weight = np.ascontiguousarray(power_weight_at(problem, 0.0)) - grid.carrier = np.ascontiguousarray( - np.ones(grids.n_to, dtype=np.complex128) if carrier is None else carrier - ) - grid.dt_fine = grids.dt_fine - grid.dt = grids.dt - - _lower_linear(native, problem, plan) - coarse_matrix = _lower_spatial(native, problem, request) - _lower_stepper(native, problem, stepper, device=plan.device) - - native.absorber.in_stage = temporal_in_stage or spatial_absorption - native.absorber.split_step = ( - absorber is not None and problem.responses.absorber_scheme == "split_step" - ) - if temporal_in_stage: - native.absorber.fine = np.ascontiguousarray(absorber_fine, dtype=np.float64) - elif spatial_absorption: - native.absorber.fine = np.zeros(grids.n_to) - if absorber is not None: - native.absorber.coarse = np.ascontiguousarray(absorber, dtype=np.float64) - - transform = native.transform - transform.synthesis_matrix = np.ascontiguousarray(problem.transform.synthesis) - transform.projection_matrix = np.ascontiguousarray(problem.transform.projection) - transform.projection_coarse_matrix = np.ascontiguousarray(coarse_matrix) - transform.synthesis = 0.0 if free else float(problem.transform.synthesis[0, 0]) - transform.projection = 0.0 if free else float(problem.transform.projection[0, 0]) - - if kerr is not None: - native.kerr.nonlinear = True - native.kerr.coefficient = kerr.coefficient - native.kerr.self_factor = kerr.self_factor - native.kerr.conjugate_factor = kerr.conjugate_factor - native.kerr.harmonic_factor = kerr.harmonic_factor - native.kerr.take_real = kerr.take_real - - _lower_plasma(native, problem, request) - _lower_raman(native, problem) - _lower_general(native, problem, general_chunk) - _lower_flow(native, plan.flow) - _lower_dispersive(native, problem) - _lower_flux(native, problem, request.observation.stats) - if plan.setup is not None: - # The set-once data, each part read through its record's table (SEP 52). - held: Any = native - for name, value in plan.setup.items(): - setattr(held, name, value) - return native - - -def _lower_linear( - native: _core.Problem, problem: PropagationProblem, plan: _LowerPlan -) -> None: - """Fill the fixed terms, (n_terms, rows, n_omega): one empty term on a separable - route, a z table's slots.""" - del problem - tables = plan.tables - linear = native.linear - n_omega = native.dims.n_omega - linear.phi_table = tables is not None and tables.phi - linear.prefactor_table = tables is not None and tables.prefactor - if plan.terms.separable is not None: - linear.n_phi_terms = 1 - linear.n_prefactor_terms = 1 - return - # A z table's terms are slots the native evaluation fills (solprop SEP 29 G1): - # their count crosses in their place. - if tables is not None and tables.phi: - linear.n_phi_terms = tables.n_phi - elif plan.terms.phi is None: - raise BackendError("Kokkos requires a device-ready fixed-term problem") - else: - linear.n_phi_terms = plan.terms.phi.shape[0] - linear.phi_terms = np.ascontiguousarray( - plan.terms.phi.reshape(plan.terms.phi.shape[0], -1, n_omega) - ) - if tables is not None and tables.prefactor: - linear.n_prefactor_terms = tables.slots - elif plan.terms.prefactor is None: - raise BackendError("Kokkos requires a device-ready fixed-term problem") - else: - linear.n_prefactor_terms = plan.terms.prefactor.shape[0] - linear.prefactor_terms = np.ascontiguousarray( - plan.terms.prefactor.reshape(plan.terms.prefactor.shape[0], -1, n_omega) - ) - - -def _lower_spatial( - native: _core.Problem, problem: PropagationProblem, request: RunRequest -) -> np.ndarray: - """Fill the separable operator and a radial run's rows; return the coarse matrix. - - The quadrature alarm's embedded Kronrod projection is the transform's own unless - it is the plain projection or the route has one node, and the hybrid planar kind - packs none: the reference scores no alarm on it. - """ - transform = problem.transform - coarse = ( - np.empty(0, dtype=np.float64) - if transform.n_nodes == 1 - or np.array_equal(transform.projection_coarse, transform.projection) - else transform.projection_coarse - ) - geometry = transform.geometry - spatial = native.spatial - spatial.absorber_matrix = radial_absorber_matrix(transform, problem.grids.pad_scale) - if geometry is not None: - spatial.spectral_weights = geometry.spectral_weights - spatial.on_axis = geometry.on_axis - separable = problem.linear.separable - if geometry is None or geometry.kind not in ("free", "slab") or separable is None: - return coarse - spatial.n_x, spatial.n_y = geometry.shape - spatial.k_squared = separable.k_squared - spatial.k_squared_low = separable.k_squared_low - spatial.kperp_sq = separable.kperp_sq - spatial.kperp_sq_low = separable.kperp_sq_low - spatial.frame = separable.frame - spatial.spectral_rate = separable.spectral_rate - assert separable.user_rate is not None - assert separable.kperp_rate is not None - spatial.user_rate = separable.user_rate - spatial.numerator = separable.prefactor_numerator - spatial.clamped = separable.prefactor_clamped - spatial.evanescent_rate = separable.evanescent_rate - spatial.absorber_rate = geometry.absorber_rate - # A transverse collar's rate per column (solprop kernel schema 31). - spatial.kperp_rate = separable.kperp_rate - if geometry.kind == "free": - # solprop SEP 23: the stored axes and the slots of the problem's own - # far-field request, which size the native band partials. - k_x, k_y = geometry.spectral_axes - slots, _ = far_field_partials(problem, request.observation.stats) - spatial.k_x = np.ascontiguousarray(k_x, dtype=np.float64) - spatial.k_y = np.ascontiguousarray(k_y, dtype=np.float64) - spatial.far_field_slots = slots - return coarse - # Schema 15's `SeparableOperator.kappa_sq` is (n_channels, n_omega), column `c` - # belonging to channel `c % n_channels`; the geometry's collar rate depends on y - # alone (solprop checks it), so it is packed per column `p + n_p * iy`, the index - # of the temporal row it acts on. - assert separable.kappa_sq is not None - n_x, n_y = geometry.shape - n_p = separable.n_channels - spatial.n_p = n_p - spatial.kappa_sq = np.ascontiguousarray(separable.kappa_sq) - spatial.absorber_rate = np.ascontiguousarray( - np.repeat(geometry.absorber_rate.reshape(n_y, n_x)[:, 0], n_p) - ) - return np.empty(0, dtype=np.float64) - - -def _lower_stepper( - native: _core.Problem, - problem: PropagationProblem, - stepper: StepperSettings, - *, - device: bool, -) -> None: - """Fill the tableau and the low-storage march's rows (solprop kernel schema 32).""" - tableau = stepper.tableau - native_stepper = native.stepper - native_stepper.n_stages = tableau.n_stages - native_stepper.tableau_matrix = np.ascontiguousarray(tableau.matrix) - native_stepper.tableau_nodes = np.ascontiguousarray(tableau.nodes) - native_stepper.solution_weights = np.ascontiguousarray(tableau.solution_weights) - native_stepper.error_weights = np.ascontiguousarray(tableau.error_weights) - native_stepper.first_same_as_last = tableau.first_same_as_last - native_stepper.weighted_controller = stepper.controller.weights is not None - native_stepper.modified_noise = problem.initial.noise_model == "modified" - flows = _low_storage_flows(problem, stepper, device=device) - if flows is None: - return - native_stepper.low_storage = True - native_stepper.low_storage_stage_register = low_storage_stage_register( - problem, "native" - ) - native_stepper.low_storage_flows = flows - native_stepper.low_storage_tiled_input = device - - -def _lower_plasma( - native: _core.Problem, problem: PropagationProblem, request: RunRequest -) -> None: - """Fill the plasma columns: the first channel's rate, every channel's rows.""" - channels = problem.responses.population_channels - channel = channels[0] if channels else None - plasma = native.plasma - # solprop SEP 18: the running-integral rule, one per problem. - plasma.band = problem.responses.plasma_integral.policy == "band" - plasma.taps = problem.responses.plasma_integral.taps - if channel is not None: - rate = channel.rate - plasma.enabled = True - # The private gated benchmark (solprop's tests/gated_source.py), read where - # the backend is built, as the reference reads it. - plasma.gated = _BENCHMARK_GATED.get() - plasma.charge_mass_ratio = channel.charge_mass_ratio - plasma.ionisation_potential = channel.ionisation_potential - plasma.log_field_min = rate.log_field_min - plasma.d_log_field = rate.d_log_field - plasma.field_min = rate.field_min - plasma.field_max = rate.field_max - plasma.rate_max = rate.rate_max - plasma.rate_intervals = rate.coefficients.shape[1] - plasma.rate_coefficients = np.ascontiguousarray(rate.coefficients) - plasma.stash = _stash_plasma(problem, request) - _lower_channels(native, problem) - _lower_envelope(native, problem) - - -def _stash_plasma(problem: PropagationProblem, request: RunRequest) -> bool: - """Whether the accepted step's last stage leaves the plasma statistics behind. - - Only when that stage's input is the committed field: a first-same-as-last tableau, - no split-step absorber (applied to the field after the last stage) and no coupled - flow; and not under the modified noise model, where that input carries the noise - draw while solprop defines the statistics on the committed field its reference - scans. Otherwise they are scanned afresh. - """ - absorber = problem.responses.temporal_absorber - return ( - bool(problem.responses.population_channels) - and problem.responses.envelope_plasma is None - and problem.initial.noise_model != "modified" - and request.stepper.tableau.first_same_as_last - and not ( - absorber is not None and problem.responses.absorber_scheme == "split_step" - ) - and problem.linear.coupled_flow is None - and bool( - { - "electron_density", - "peak_ionisation_rate", - "species_ionisation", - "species_electron_density", - } - & set(request.observation.stats.periodic) - ) - ) - - -def _lower_raman(native: _core.Problem, problem: PropagationProblem) -> None: - """Fill the Raman terms, row-major (n_terms, 2 n_to) and (n_terms, 2).""" - terms = problem.responses.raman_terms - raman = native.raman - raman.n_components = len(terms) - raman.row_component = np.asarray( - [component for component, term in enumerate(terms) for _ in term.undamped], - dtype=np.float64, - ) - raman.component_fraction = np.asarray([term.fraction for term in terms]) - if not terms: - return - first = terms[0] - undamped = np.concatenate([term.undamped for term in terms], axis=0) - raman.enabled = True - raman.thg = bool(first.thg) and problem.grid_kind == "field" - raman.tensor_capable = all(term.tensor_capable for term in terms) - raman.n_terms = undamped.shape[0] - raman.fraction = float(first.fraction) - raman.epsilon0 = EPS0 - raman.undamped = np.ascontiguousarray(undamped) - raman.tensor_weights = np.ascontiguousarray( - np.concatenate([term.tensor_weights for term in terms], axis=0) - ) - - -def _lower_dispersive(native: _core.Problem, problem: PropagationProblem) -> None: - """Lay out solprop SEP 6's dispersive response for the native engine. - - The structure the native response takes on every route: the channels grouped as the - leaf groups them (``DispersiveKerrTerms.groups``) -- the symmetric evaluator's one - mixture per node region, the polynomial's species of equal leg weight and region -- - with one filtered field and one product per symmetric group, and for the - polynomial one ``E^3`` per distinct region followed by two fields and two products - per group; each output term's multiplier, product and factor, every multiplier - carrying the sector's ``Pi_+`` and factor (one on a field grid's full cubic, - ``Pi_+ / 8`` for its conserving sector, ``Pi_+ / 4`` on an envelope grid), and the - input multipliers ``Pi_+`` on a field grid's analytic family. A mixture's rows are - rebuilt natively at every stage whose densities change, from its channels' ``g^4`` - (``mixture_filter``'s arithmetic); the first species' stand until then. On the - general modal route each group's key is its region and every node's region is - packed beside them; the route's grid is a field grid. - """ - leaf = problem.responses.dispersive_kerr - if leaf is None: - return - tables = problem.transform.modal_tables - general = tables is not None - kind = "field" if general else problem.grid_kind - form = ( - ("real" if leaf.sector == "full" else "conserving") - if kind == "field" - else ("harmonic" if leaf.sector == "full" else "conserving") - ) - analytic = form == "conserving" and kind == "field" - symmetric = leaf.evaluator == "symmetric" - n_channels = leaf.n_channels - positive = leaf.positive.astype(np.float64) - if kind == "envelope": - scale = positive / 4.0 - elif analytic: - scale = positive / 8.0 - else: - scale = np.ones(positive.shape) - input_scale = positive if analytic else np.ones(positive.shape) - groups = leaf.groups() - # A group's key is its general modal region; every other route has one, so one - # cube is shared by every polynomial group there. - regions = leaf.regions if general else None - keys = [0 if regions is None else int(regions[group[0]]) for group in groups] - distinct = list(dict.fromkeys(keys)) - cubes = [distinct.index(key) for key in keys] - n_cubes = 0 if symmetric else len(distinct) - mixtures = tuple( - index for index, group in enumerate(groups) if symmetric and len(group) > 1 - ) - fields: list[RealArray] = [] - rows: list[int] = [] - channels: list[int] = [] - multipliers: list[RealArray] = [] - for index, group in enumerate(groups): - weight = leaf.filters[group[0]] - if symmetric: - fields.append(weight) - rows.append(index) - channels.append( - n_channels + 1 + mixtures.index(index) - if index in mixtures - else group[0] - ) - multipliers.append(weight * scale) - continue - assert leaf.outputs is not None - fields.extend([weight, weight * weight]) - first = n_cubes + 2 * index - for channel in group: - output = leaf.outputs[channel] - rows.extend([cubes[index], first]) - channels.extend([channel, channel]) - multipliers.extend([output[0] * scale, output[1] * scale]) - rows.append(first + 1) - channels.append(n_channels) - multipliers.append(scale.copy()) - stacked = np.array(fields, dtype=np.float64) - if analytic: - stacked = stacked * leaf.positive - constants = ( - np.zeros((n_channels, 2)) - if leaf.products is None - else np.asarray(leaf.products, dtype=np.float64) - ) - if mixtures: - squares = leaf.filters * leaf.filters - quartics = np.ascontiguousarray(squares * squares, dtype=np.float64) - else: - quartics = np.empty(0, dtype=np.float64) - offsets = np.cumsum([0, *(len(group) for group in groups)]) - lowered = native.dispersive - if tables is not None: - lowered.group_keys = np.asarray(keys, dtype=np.uint64) - lowered.node_region = np.ascontiguousarray(tables.region_index, dtype=np.uint64) - lowered.enabled = True - lowered.symmetric = symmetric - lowered.form = {"real": 0, "conserving": 1, "harmonic": 2}[form] - lowered.inputs = np.ascontiguousarray(stacked) - lowered.term_multipliers = np.ascontiguousarray( - np.array(multipliers, dtype=np.float64) - ) - lowered.term_rows = np.asarray(rows, dtype=np.uint64) - lowered.term_channels = np.asarray(channels, dtype=np.uint64) - lowered.coefficients = np.asarray(leaf.coefficients, dtype=np.float64) - lowered.pair_constants = np.ascontiguousarray(constants) - lowered.group_offsets = np.asarray(offsets, dtype=np.uint64) - lowered.group_members = np.asarray( - [channel for group in groups for channel in group], dtype=np.uint64 - ) - lowered.quartics = quartics - lowered.input_scale = np.ascontiguousarray(input_scale) - lowered.output_scale = np.ascontiguousarray(scale) - - -def _lower_flux( - native: _core.Problem, problem: PropagationProblem, stats: StatsSpec -) -> None: - """Fill the plane measure of solprop SEP 34 (kernel schema 35). - - A separable run (Cartesian, line, planar) is measured by its own operator's root, - which the native state already holds: nothing more is packed, and a plan that is - not that operator is refused. A radial run packs the entrance's material and - transverse squares and whether a material z table follows - (:func:`_flux_tables`); the admission record is not read. ``closure`` - sizes the magnetic closure's buffers for the run's own local statistics, - `stats`. - """ - flux = flux_of(problem.route) - if flux is None: - return - native.flux.closure = bool(set(stats.names) & LOCAL_FLUX_STATISTICS) - if problem.linear.separable is not None: - if flux.base is not problem.linear.separable or flux.material is not None: - raise BackendError( - "solprop-native measures a separable run by its own operator's root, " - "and this plane measure is another" - ) - return - base = flux.base - native.flux.k_squared = np.ascontiguousarray(base.k_squared, dtype=np.complex128) - native.flux.k_squared_low = np.ascontiguousarray( - base.k_squared_low, dtype=np.complex128 - ) - native.flux.kperp_sq = np.ascontiguousarray(base.kperp_sq, dtype=np.float64) - native.flux.kperp_sq_low = np.ascontiguousarray(base.kperp_sq_low, dtype=np.float64) - native.flux.table = flux.material is not None - - -def _lower_general( - native: _core.Problem, problem: PropagationProblem, chunk: int | None -) -> None: - """Fill the general modal route's maps, derived as the reference derives them. - - The per-node tensor, the form and the delayed maps come from solprop's own - derivations (``solprop.translate.general``), so both backends execute the same - arrays; the chunk is the route's own at the same budget - (:func:`solprop.backends.node_chunk.general_planned_chunk`), given as `chunk`. - Nothing is filled for a problem without ``transform.modal_tables``. - """ - tables = problem.transform.modal_tables - if tables is None: - return - assert tables.synthesis is not None - assert tables.projection is not None - assert chunk is not None - regional = problem.responses.regional_responses - owner, delayed_tensor, delayed_spectrum = node_delayed_maps(tables, regional) - terms = 0 if delayed_tensor is None else int(delayed_tensor.shape[1]) - general = native.general - general.enabled = True - general.real_form = node_kerr_form(regional) == "real" - # A moving guide's maps are every Chebyshev coefficient along z, flattened over - # (n_intervals, n_deg) in the order of the stage bundle's transform_weights; a - # constant guide's are the one coefficient. - general.map_intervals, general.map_n_deg = ( - int(one) for one in tables.synthesis.shape[:2] - ) - general.components = int(tables.electric.shape[2]) - general.nodes = int(tables.region_index.size) - general.node_chunk = chunk - general.responses = 0 if delayed_tensor is None else delayed_tensor.shape[0] - general.terms = terms - general.pad_scale = float(problem.grids.pad_scale) - general.epsilon0 = EPS0 - general.synthesis = np.ascontiguousarray(tables.synthesis) - general.projection = np.ascontiguousarray(tables.projection) - general.tensor = np.ascontiguousarray(node_kerr_tensor(tables, regional)) - general.frequency_factor = np.ascontiguousarray( - tables.source_frequency_factor, dtype=np.complex128 - ) - if delayed_tensor is not None: - general.delayed_tensor = np.ascontiguousarray(delayed_tensor) - if delayed_spectrum is not None: - general.delayed_spectrum = np.ascontiguousarray(delayed_spectrum) - if terms: - general.node_response = np.ascontiguousarray( - owner.astype(np.int64) + 1, dtype=np.uint64 - ) - - -def _lower_envelope(native: _core.Problem, problem: PropagationProblem) -> None: - """Fill the accepted cycle rules, (n_channels, 4, 512), and masked factors.""" - assembly = problem.responses.envelope_plasma - if assembly is None: - return - nodes = np.zeros((len(assembly.channels), 4, 512)) - weights = np.zeros_like(nodes) - for channel_index, channel in enumerate(assembly.channels): - for level, rule in enumerate(channel.cycle.rules): - nodes[channel_index, level, : len(rule.nodes)] = rule.nodes - weights[channel_index, level, : len(rule.weights)] = rule.weights - native.plasma.envelope = True - native.plasma.envelope_rule_nodes = nodes - native.plasma.envelope_rule_weights = weights - native.plasma.envelope_drude_factor = assembly.factors.drude - native.plasma.envelope_loss_factor = assembly.factors.loss - - -def _lower_channels(native: _core.Problem, problem: PropagationProblem) -> None: - """Fill independent rate tables without resampling their coefficients.""" - parameters: list[list[float]] = [] - coefficients: list[np.ndarray] = [] - offset = 0 - for channel in problem.responses.population_channels: - rate = channel.rate - parameters.append( - [ - rate.log_field_min, - rate.d_log_field, - rate.field_min, - rate.field_max, - rate.rate_max, - rate.coefficients.shape[1], - offset, - channel.charge_mass_ratio, - channel.ionisation_potential, - ] - ) - coefficients.append(rate.coefficients.ravel()) - offset += rate.coefficients.size - native.plasma.n_channels = len(parameters) - native.plasma.channel_parameters = np.asarray(parameters, dtype=np.float64).reshape( - -1, 9 - ) - if coefficients: - native.plasma.channel_coefficients = np.concatenate(coefficients) - - -def _lower_flow(native: _core.Problem, flow: _HostFlow | None) -> None: - """Fill a coupled flow's bins and residual data; its actions follow per step.""" - if flow is None: - return - slot = np.zeros(flow.active.size, dtype=np.uint64) - slot[flow.active] = np.arange(1, int(np.count_nonzero(flow.active)) + 1) - residual = flow.residual_terms - # The bins formed by scaling and squaring are numbered 1, 2, ... in row-major - # (section, bin) order and carry their Schur factors; the rest take the - # eigenbasis, whose `W` and `Y` are zero where it is not used. - fallback = ~flow.route - route = np.zeros(flow.route.shape, dtype=np.uint64) - route[fallback] = np.arange(1, int(np.count_nonzero(fallback)) + 1) - lowered = native.flow - lowered.enabled = True - lowered.active = int(np.count_nonzero(flow.active)) - lowered.sections = int(flow.frozen.shape[0]) - lowered.terms = 0 if residual is None else int(residual.shape[0]) - lowered.slot = slot - if residual is not None: - # The residual's data are zero on the bins the error norm does not score, so - # the engine's residual there is an exact zero and the frozen flow alone - # carries them, as in solprop's reference. Every term is finite, so each - # product with a zero is a signed zero and the running sums stay +0: the - # source is what adding nothing leaves, but for the sign of a zero. - scored = flow.scored[None, :, None, None] - lowered.frozen = np.ascontiguousarray(np.where(scored, flow.frozen, 0.0)) - lowered.residual_terms = np.ascontiguousarray(np.where(scored, residual, 0.0)) - lowered.left = flow.left - lowered.right = flow.right - lowered.eigenvalues = flow.eigenvalues - lowered.route = route - lowered.fallback_vectors = np.ascontiguousarray(flow.vectors[fallback]) - lowered.fallback_triangular = np.ascontiguousarray(flow.triangular[fallback]) diff --git a/native/src/solprop_native/session.py b/native/src/solprop_native/session.py index 511a0f30d..4a84d68fb 100644 --- a/native/src/solprop_native/session.py +++ b/native/src/solprop_native/session.py @@ -143,6 +143,15 @@ from . import _core, _counters, _transverse, _watch from ._load import Loaded +from ._march import ( + _BUDGET, + _DIVERGED, + _ERROR, + _FLOOR, + FRESH_WATCH, + _envelope_errors, + _raise_native, +) from ._statistics import ( ADMITTED_GENERAL_STATISTICS, ADMITTED_SPATIAL_STATISTICS, @@ -152,16 +161,7 @@ metric_refusal, translated, ) -from .backend import ( - _BUDGET, # pyright: ignore[reportPrivateUsage] - _DIVERGED, # pyright: ignore[reportPrivateUsage] - _ERROR, # pyright: ignore[reportPrivateUsage] - _FLOOR, # pyright: ignore[reportPrivateUsage] - FRESH_WATCH, - NativeBackend, - _envelope_errors, # pyright: ignore[reportPrivateUsage] - _raise_native, # pyright: ignore[reportPrivateUsage] -) +from .backend import NativeBackend _LOG = logging.getLogger(__name__) # A march failure a host source raised (`march::Category::host`, solprop SEP 49). diff --git a/native/tests/test_backend.py b/native/tests/test_backend.py index e4c5a9fb1..5fb0a2083 100644 --- a/native/tests/test_backend.py +++ b/native/tests/test_backend.py @@ -44,9 +44,16 @@ from solprop.translate.stage import terms import solprop_native -from solprop_native import NativeBackend, _core, build_info, check_build +import solprop_native.backend +from solprop_native import ( + NativeBackend, + _core, + _device, # pyright: ignore[reportPrivateUsage] + build_info, + check_build, +) from solprop_native._core import _testing # pyright: ignore[reportPrivateUsage] -from solprop_native.backend import ( +from solprop_native._lowering import ( _lower, # pyright: ignore[reportPrivateUsage] _LowerPlan, # pyright: ignore[reportPrivateUsage] ) @@ -203,7 +210,9 @@ def test_the_solprop_version_is_recorded_and_never_compared( def elsewhere(name: str) -> str: return f"{name}-elsewhere" - monkeypatch.setattr(solprop_native.backend, "version", elsewhere) + # The version is looked up where the build facts are formed (`_device`), and the + # metadata reads the facts through the backend's own name. + monkeypatch.setattr(_device, "version", elsewhere) # The build facts are read once per process; read them afresh under the patch, # through a fresh cache the patch removes again. facts = solprop_native.backend._build_facts # pyright: ignore[reportPrivateUsage] diff --git a/native/tests/test_band_plasma.py b/native/tests/test_band_plasma.py index a0916ea05..8a5d73f01 100644 --- a/native/tests/test_band_plasma.py +++ b/native/tests/test_band_plasma.py @@ -62,7 +62,7 @@ from conftest import PhaseACase from plasma_bounds import assert_regression_guard, plasma_stage_bound, rate_roundoff from solprop_native import NativeBackend, _core -from solprop_native.backend import ( +from solprop_native._lowering import ( _lower, # pyright: ignore[reportPrivateUsage] - corrupt native payloads _LowerPlan, # pyright: ignore[reportPrivateUsage] - corrupt native payloads ) diff --git a/native/tests/test_coupled_flow.py b/native/tests/test_coupled_flow.py index 6f219d170..042b0d8fc 100644 --- a/native/tests/test_coupled_flow.py +++ b/native/tests/test_coupled_flow.py @@ -48,9 +48,11 @@ from solprop_native import NativeBackend, _core from solprop_native._core import _testing # pyright: ignore[reportPrivateUsage] +from solprop_native._lowering import ( + _lower_flow, # pyright: ignore[reportPrivateUsage] +) from solprop_native.backend import ( _host_flow, # pyright: ignore[reportPrivateUsage] - _lower_flow, # pyright: ignore[reportPrivateUsage] ) type ComplexArray = npt.NDArray[np.complex128] diff --git a/native/tests/test_dispersive.py b/native/tests/test_dispersive.py index 534f622d3..0b799a059 100644 --- a/native/tests/test_dispersive.py +++ b/native/tests/test_dispersive.py @@ -43,7 +43,7 @@ from solprop_native import NativeBackend, _core from solprop_native._core import _testing # pyright: ignore[reportPrivateUsage] -from solprop_native.backend import ( +from solprop_native._lowering import ( _lower, # pyright: ignore[reportPrivateUsage] _lower_dispersive, # pyright: ignore[reportPrivateUsage] _LowerPlan, # pyright: ignore[reportPrivateUsage] diff --git a/native/tests/test_envelope_plasma.py b/native/tests/test_envelope_plasma.py index 588422ab9..24f83fcf3 100644 --- a/native/tests/test_envelope_plasma.py +++ b/native/tests/test_envelope_plasma.py @@ -40,7 +40,7 @@ from solprop_native import NativeBackend, _core, build_info from solprop_native._core import _testing # pyright: ignore[reportPrivateUsage] -from solprop_native.backend import ( +from solprop_native._lowering import ( _lower, # pyright: ignore[reportPrivateUsage] - validate the native boundary _LowerPlan, # pyright: ignore[reportPrivateUsage] - validate the native boundary ) @@ -454,6 +454,7 @@ def corrupt_payload(*args: object, **keywords: object) -> _core.Problem: built.plasma.envelope_rule_nodes = nodes return built + # Patched where the backend looks it up: backend.py imports it from _lowering. monkeypatch.setattr(adapter, "_lower", corrupt_payload) owner = NativeBackend( problem, run_request, grid=grid, options=envelope_options(geometry) diff --git a/native/tests/test_envelope_statistics_refusal.py b/native/tests/test_envelope_statistics_refusal.py index 9be0e921e..838380288 100644 --- a/native/tests/test_envelope_statistics_refusal.py +++ b/native/tests/test_envelope_statistics_refusal.py @@ -22,7 +22,7 @@ import solprop_native.backend as adapter from solprop_native import NativeBackend, _core -from solprop_native.backend import ( +from solprop_native._lowering import ( _lower, # pyright: ignore[reportPrivateUsage] _LowerPlan, # pyright: ignore[reportPrivateUsage] ) @@ -56,6 +56,7 @@ def corrupt_payload(*args: object, **keywords: object) -> _core.Problem: built.plasma.envelope_rule_nodes = nodes return built + # Patched where the backend looks it up: backend.py imports it from _lowering. monkeypatch.setattr(adapter, "_lower", corrupt_payload) # The launch plane's statistics meet the bad rule before any step. run_request = run_request.for_run(stats=SPEC) diff --git a/native/tests/test_execution.py b/native/tests/test_execution.py index 9e510de94..1b7e40b6b 100644 --- a/native/tests/test_execution.py +++ b/native/tests/test_execution.py @@ -21,12 +21,12 @@ ExecutionRecord, builtin, ) -from solprop_native._wait import SPIN_MILLISECONDS -from solprop_native.backend import ( - NativeBackend, +from solprop_native._lowering import ( _lower, # pyright: ignore[reportPrivateUsage] _LowerPlan, # pyright: ignore[reportPrivateUsage] ) +from solprop_native._wait import SPIN_MILLISECONDS +from solprop_native.backend import NativeBackend # The constants of core.hpp before SEP 46 moved them here (solprop PR #576), Linux and # Windows, with the transforms' shape rule of solprop issue #587: every value of the diff --git a/native/tests/test_free.py b/native/tests/test_free.py index 2dd054e2f..03cb40a3b 100644 --- a/native/tests/test_free.py +++ b/native/tests/test_free.py @@ -27,7 +27,7 @@ from solprop_native import NativeBackend, _core from solprop_native._core import _testing # pyright: ignore[reportPrivateUsage] -from solprop_native.backend import ( +from solprop_native._lowering import ( _lower, # pyright: ignore[reportPrivateUsage] _LowerPlan, # pyright: ignore[reportPrivateUsage] ) diff --git a/native/tests/test_mixtures.py b/native/tests/test_mixtures.py index 814799ec1..de116ba9f 100644 --- a/native/tests/test_mixtures.py +++ b/native/tests/test_mixtures.py @@ -23,7 +23,7 @@ from solprop_native import NativeBackend, _core from solprop_native._core import _testing # pyright: ignore[reportPrivateUsage] -from solprop_native.backend import ( +from solprop_native._lowering import ( _lower, # pyright: ignore[reportPrivateUsage] - corrupt native payloads _LowerPlan, # pyright: ignore[reportPrivateUsage] - corrupt native payloads ) diff --git a/native/tests/test_session_failure.py b/native/tests/test_session_failure.py index 87ad3fc1b..c2bd68668 100644 --- a/native/tests/test_session_failure.py +++ b/native/tests/test_session_failure.py @@ -22,7 +22,7 @@ import solprop_native.backend as adapter from solprop_native import NativeBackend, _core -from solprop_native.backend import ( +from solprop_native._lowering import ( _lower, # pyright: ignore[reportPrivateUsage] _LowerPlan, # pyright: ignore[reportPrivateUsage] ) @@ -51,6 +51,7 @@ def corrupt_payload(*args: object, **keywords: object) -> _core.Problem: built.plasma.envelope_rule_nodes = nodes return built + # Patched where the backend looks it up: backend.py imports it from _lowering. monkeypatch.setattr(adapter, "_lower", corrupt_payload) owner = NativeBackend( problem, run_request, grid=grid, options=envelope_options(geometry) diff --git a/native/tests/test_workspace.py b/native/tests/test_workspace.py index 6c7993350..43cbe4cef 100644 --- a/native/tests/test_workspace.py +++ b/native/tests/test_workspace.py @@ -38,12 +38,12 @@ from solprop_native import _core from solprop_native._load import load -from solprop_native.backend import ( - FRESH_WATCH, - NativeBackend, +from solprop_native._lowering import ( _lower, # pyright: ignore[reportPrivateUsage] _LowerPlan, # pyright: ignore[reportPrivateUsage] ) +from solprop_native._march import FRESH_WATCH +from solprop_native.backend import NativeBackend _testing: Any = cast(Any, _core)._testing EPS = float(np.finfo(np.float64).eps) diff --git a/src/solprop/backends/node_chunk.py b/src/solprop/backends/node_chunk.py index e8e1abbb1..fee0a25d0 100644 --- a/src/solprop/backends/node_chunk.py +++ b/src/solprop/backends/node_chunk.py @@ -37,7 +37,7 @@ budget, which is the maintainer's decision under #349 and #89, not this move's. A device backend may choose differently when no budget is given (solprop issue #361); -the native backend's rule is ``solprop_native.backend.device_node_chunk``, and an +the native backend's rule is ``solprop_native._device.device_node_chunk``, and an explicit budget or chunk is never overridden by it. """ diff --git a/tests/backends/native_trace_marches.py b/tests/backends/native_trace_marches.py index 87276e099..59e619d90 100644 --- a/tests/backends/native_trace_marches.py +++ b/tests/backends/native_trace_marches.py @@ -36,6 +36,7 @@ from solprop.stepper.schedule import FixedSchedule, reached, replay_tolerance backend_module = pytest.importorskip("solprop_native.backend") +march_module = pytest.importorskip("solprop_native._march") session_module = pytest.importorskip("solprop_native.session") native = pytest.importorskip("solprop_native") _testing = pytest.importorskip("solprop_native._core")._testing @@ -73,15 +74,15 @@ change, compared with solprop's monitor after every drain (SEP 45 section 4.8).""" SPLIT_RATE_BUCKETS = __import__("solprop.boundaries", fromlist=["x"]).SPLIT_RATE_BUCKETS -_REACHED = backend_module._REACHED -_HOST_STEP = backend_module._HOST_STEP -_MONITOR = backend_module._MONITOR -_WATCH = backend_module._WATCH -_FLOOR = backend_module._FLOOR -_BUDGET = backend_module._BUDGET -_DIVERGED = backend_module._DIVERGED -_DENSE = backend_module._DENSE -_envelope_errors = backend_module._envelope_errors +_REACHED = march_module._REACHED +_HOST_STEP = march_module._HOST_STEP +_MONITOR = march_module._MONITOR +_WATCH = march_module._WATCH +_FLOOR = march_module._FLOOR +_BUDGET = march_module._BUDGET +_DIVERGED = march_module._DIVERGED +_DENSE = march_module._DENSE +_envelope_errors = march_module._envelope_errors _raise_rk = session_module._raise_rk logger = logging.getLogger(__name__) @@ -146,7 +147,7 @@ def _verify_watch_of(monitor: BoundaryMonitor, result: dict[str, Any]) -> None: ) -class TraceAdamsMarch(backend_module._NativeAdamsMarch): +class TraceAdamsMarch(march_module._NativeAdamsMarch): """The native Adams record, driven a call at a time over a scripted harness. Parameters diff --git a/tests/backends/test_native_phi_moments.py b/tests/backends/test_native_phi_moments.py index 8da88b3f9..f74f8c8cf 100644 --- a/tests/backends/test_native_phi_moments.py +++ b/tests/backends/test_native_phi_moments.py @@ -89,12 +89,13 @@ def read_moments(module, grid, values): assert active.size >= values.size bins = active[: values.size] backend_module = module.backend + lowering = module._lowering backend = module.NativeBackend(problem, run_request, grid=grid) try: # The payload and the step's resident rows are the backend's own; the state # they open is this test's, so nothing is loaded into the backend's. - payload = backend_module._lower( - problem, run_request, backend_module._LowerPlan(terms=backend._terms) + payload = lowering._lower( + problem, run_request, lowering._LowerPlan(terms=backend._terms) ) arrays, material = native_stage_rows(backend, 0.0, 1e-3) finally: diff --git a/tests/backends/test_node_chunk.py b/tests/backends/test_node_chunk.py index ae0c44d44..baf23cac7 100644 --- a/tests/backends/test_node_chunk.py +++ b/tests/backends/test_node_chunk.py @@ -447,8 +447,18 @@ def native_backend(): return pytest.importorskip("solprop_native.backend") -def test_native_lowering_keeps_the_host_chunk_and_whole_planar_rows(native_backend): - lower = native_backend.native_node_chunk +@pytest.fixture(scope="module") +def native_device(): + return pytest.importorskip("solprop_native._device") + + +@pytest.fixture(scope="module") +def native_lowering(): + return pytest.importorskip("solprop_native._lowering") + + +def test_native_lowering_keeps_the_host_chunk_and_whole_planar_rows(native_device): + lower = native_device.native_node_chunk for name in ("collar-free", "collar-radial", "plasma-line", "capillary-3-modes"): problem = BUILDERS[name]() for budget in (None, 2**21, 2**23): @@ -464,9 +474,7 @@ def test_native_lowering_keeps_the_host_chunk_and_whole_planar_rows(native_backe assert lower(planar, plan, device=False) == rows * n_x -def test_the_device_rule_applies_to_an_automatic_plan_alone( - native_backend, monkeypatch -): +def test_the_device_rule_applies_to_an_automatic_plan_alone(native_device, monkeypatch): """A later qualified device default changes `device_node_chunk` and nothing else.""" problem = BUILDERS["collar-free"]() calls = [] @@ -476,8 +484,8 @@ def larger(problem, planned, free): calls.append(planned) return problem.transform.n_nodes - monkeypatch.setattr(native_backend, "device_node_chunk", larger) - lower = native_backend.native_node_chunk + monkeypatch.setattr(native_device, "device_node_chunk", larger) + lower = native_device.native_node_chunk automatic = plan_node_chunk(problem, ExecutionOptions()) explicit = plan_node_chunk(problem, ExecutionOptions(node_budget=2**18)) forced = plan_node_chunk(problem, ExecutionOptions(node_chunk=3)) @@ -489,14 +497,14 @@ def larger(problem, planned, free): assert calls == [automatic.chunk] # A host build knows no free device bytes: the host plan's chunk. monkeypatch.undo() - assert native_backend.device_node_chunk(problem, 7, None) == 7 + assert native_device.device_node_chunk(problem, 7, None) == 7 -def test_the_device_rule_budgets_a_share_of_the_free_device_bytes(native_backend): +def test_the_device_rule_budgets_a_share_of_the_free_device_bytes(native_device): """One chunk where a quarter of the free bytes holds it, else the largest that does, never below the host plan's chunk (solprop issue #361).""" - rule = native_backend.device_node_chunk - share = native_backend.DEVICE_BUDGET_SHARE + rule = native_device.device_node_chunk + share = native_device.DEVICE_BUDGET_SHARE problem = BUILDERS["collar-free"]() n_nodes = problem.transform.n_nodes narrow = plan_node_chunk(problem, ExecutionOptions(node_budget=2**18)).chunk @@ -532,7 +540,7 @@ def test_the_native_backend_executes_and_records_its_lowered_chunk(native_backen assert record["node_chunk_rule"] in {"host", "device"} -def test_a_general_guide_takes_the_plan_s_budget(native_backend): +def test_a_general_guide_takes_the_plan_s_budget(native_lowering): """The general route's own chunk at the plan's budget, the same on both backends.""" from backends.dispersive_route_cases import ( # noqa: PLC0415 ARGON_FILL, @@ -552,15 +560,15 @@ def test_a_general_guide_takes_the_plan_s_budget(native_backend): assert general_planned_chunk(problem, forced) == 2 whole = _general_maps(problem, automatic) assert whole is not None and whole.node_chunk > 1 - payload = native_backend._lower( + payload = native_lowering._lower( problem, run_request, - native_backend._LowerPlan(terms=terms(problem), general_chunk=2), + native_lowering._LowerPlan(terms=terms(problem), general_chunk=2), ) assert payload.general.node_chunk == 2 assert ( - native_backend._lower( - problem, run_request, native_backend._LowerPlan(terms=terms(problem)) + native_lowering._lower( + problem, run_request, native_lowering._LowerPlan(terms=terms(problem)) ).general.node_chunk == whole.node_chunk ) diff --git a/tests/driver/test_native_march_lifecycle.py b/tests/driver/test_native_march_lifecycle.py index 949f04c39..e35483e31 100644 --- a/tests/driver/test_native_march_lifecycle.py +++ b/tests/driver/test_native_march_lifecycle.py @@ -55,7 +55,7 @@ ] backend_module = pytest.importorskip("solprop_native.backend") -NativeAdamsMarch = backend_module._NativeAdamsMarch +NativeAdamsMarch = pytest.importorskip("solprop_native._march")._NativeAdamsMarch NativeSession = pytest.importorskip("solprop_native.session").NativeSession _testing = pytest.importorskip("solprop_native._core")._testing diff --git a/tests/driver/test_native_rk_march.py b/tests/driver/test_native_rk_march.py index 29d41dd2f..202e39b10 100644 --- a/tests/driver/test_native_rk_march.py +++ b/tests/driver/test_native_rk_march.py @@ -306,6 +306,6 @@ def test_a_check_due_by_distance_beside_the_edge(route): @pytest.mark.parametrize("route", ["gnlse", "capillary (multimode)"]) def test_a_device_build_s_march(route): """On a device the march drives the resident engine from the host (SEP 45).""" - if not backend_module._on_device(): + if not pytest.importorskip("solprop_native._device")._on_device(): pytest.skip("the installed solprop-native build has no accelerator backend") _held(lambda: runge_kutta_route(route)) diff --git a/tests/sessions/runs.py b/tests/sessions/runs.py index 05696897d..901128d26 100644 --- a/tests/sessions/runs.py +++ b/tests/sessions/runs.py @@ -797,7 +797,7 @@ def native_trial( from solprop_native._core import ( # noqa: PLC0415 # pyright: ignore[reportMissingImports, reportMissingModuleSource] _testing, ) - from solprop_native.backend import ( # noqa: PLC0415 # pyright: ignore[reportMissingImports] + from solprop_native._march import ( # noqa: PLC0415 # pyright: ignore[reportMissingImports] _envelope_errors, ) @@ -1071,7 +1071,7 @@ def native_adams_start(backend: Any, z: float, slots: Sequence[int] = (0,)) -> N from solprop_native._core import ( # noqa: PLC0415 # pyright: ignore[reportMissingImports, reportMissingModuleSource] _testing, ) - from solprop_native.backend import ( # noqa: PLC0415 # pyright: ignore[reportMissingImports] + from solprop_native._march import ( # noqa: PLC0415 # pyright: ignore[reportMissingImports] _envelope_errors, ) @@ -1153,7 +1153,7 @@ def native_exponential_trial( from solprop_native._core import ( # noqa: PLC0415 # pyright: ignore[reportMissingImports, reportMissingModuleSource] _testing, ) - from solprop_native.backend import ( # noqa: PLC0415 # pyright: ignore[reportMissingImports] + from solprop_native._march import ( # noqa: PLC0415 # pyright: ignore[reportMissingImports] _envelope_errors, ) diff --git a/tests/test_benchmark_sessions.py b/tests/test_benchmark_sessions.py index 5722606a0..cf3122d1d 100644 --- a/tests/test_benchmark_sessions.py +++ b/tests/test_benchmark_sessions.py @@ -24,6 +24,7 @@ backend = pytest.importorskip("solprop_native.backend") _counters = pytest.importorskip("solprop_native._counters") +_march = pytest.importorskip("solprop_native._march") NativeBackend = backend.NativeBackend @@ -237,8 +238,8 @@ def test_counters_attribute_a_host_step_to_the_work_due_on_it(): assert dict(vars(NativeBackend)) == before assert isinstance(_counters.disable(), _counters.Counters) assert _counters.active is None - # The reason names are the backend's, in march::Reason order. - assert backend._REASONS[backend._HOST_STEP] == "host_step" + # The reason names are the native march's, in march::Reason order. + assert _march._REASONS[_march._HOST_STEP] == "host_step" def test_cases_that_set_their_integrator_refuse_an_override(): From d6445cfb7645fbbd8267847ed93f7881f61ccce7 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:17:15 +0000 Subject: [PATCH 17/21] refactor(native): move the build record to _build.py BuildInfo, build_info, _source_digest, _checkout_digest and check_build move verbatim from the package's __init__ and are re-exported from it, so solprop_native.__all__ and every public name are unchanged. The process start (the processor check, thread budget, profile, wait policy, binding and the _core import) and refresh_profile stay in __init__ in their order, and NumPy still loads before them. _checkout_digest finds the checkout two levels above _build.py, as it did above __init__.py; the digest tests now patch the build module, where check_build looks the names up. Co-Authored-By: Claude Opus 5.5 --- native/AGENTS.md | 2 +- native/cmake/SourceDigest.cmake | 4 +- native/src/solprop_native/__init__.py | 420 +------------------------ native/src/solprop_native/_build.py | 434 ++++++++++++++++++++++++++ native/tests/test_api_contract.py | 32 +- 5 files changed, 464 insertions(+), 428 deletions(-) create mode 100644 native/src/solprop_native/_build.py diff --git a/native/AGENTS.md b/native/AGENTS.md index d439278c7..3bc0a765b 100644 --- a/native/AGENTS.md +++ b/native/AGENTS.md @@ -190,7 +190,7 @@ the gate. record and the probe from the run's start (solprop SEP 52 section 5.3), and nothing configures the workspace after it is built. `check_build()` refuses, in an editable checkout, an extension built from other sources than `native/` - (`build_info().source_digest`; `cmake/SourceDigest.cmake` and `_source_digest` + (`build_info().source_digest`; `cmake/SourceDigest.cmake` and `_build._source_digest` compute it, and stay in step). It records solprop's version without comparing it; never pin solprop's version. - A record that crosses declares its fields once, in a field table beside its C++ diff --git a/native/cmake/SourceDigest.cmake b/native/cmake/SourceDigest.cmake index c94b8cbf4..9e3a5610c 100644 --- a/native/cmake/SourceDigest.cmake +++ b/native/cmake/SourceDigest.cmake @@ -3,8 +3,8 @@ # SPDX-License-Identifier: GPL-3.0-or-later # The digest of what the extension is compiled from, which `check_build()` compares with -# an editable checkout's sources (`solprop_native._source_digest` computes it in Python: -# keep the two in step). The record is one "{path}:{sha256 of its bytes}\n" for +# an editable checkout's sources (`solprop_native._build._source_digest` computes it in +# Python: keep the two in step). The record is one "{path}:{sha256 of its bytes}\n" for # CMakeLists.txt and each file these patterns match, the paths relative to native/ and # sorted bytewise; the digest is the SHA-256 of the record. # diff --git a/native/src/solprop_native/__init__.py b/native/src/solprop_native/__init__.py index 34e811202..13090e2c6 100644 --- a/native/src/solprop_native/__init__.py +++ b/native/src/solprop_native/__init__.py @@ -12,19 +12,18 @@ from __future__ import annotations -import functools -import hashlib import importlib.util import logging import os import platform import sys -from dataclasses import dataclass from importlib.metadata import version from pathlib import Path from typing import TYPE_CHECKING -import numpy as np +# NumPy, and the BLAS it carries, load before the process start below sets the OpenMP +# environment and starts the native runtime. +import numpy # noqa: F401 # pyright: ignore[reportUnusedImport] from ._bind import proc_bind from ._budget import ( @@ -256,210 +255,12 @@ def refresh_profile() -> str: return source -@dataclass(frozen=True) -class BuildInfo: - """Immutable build and execution provenance for this extension. - - Attributes - ---------- - package_version - Installed ``solprop-native`` distribution version. - git_commit - Backend repository commit embedded when CMake configured the extension. - source_digest - SHA-256 of what the extension was compiled from: ``CMakeLists.txt`` and the C++ - sources and headers under ``src`` and ``include`` - (``cmake/SourceDigest.cmake``). :func:`check_build` compares it with an editable - checkout's sources. - solprop_version_built_against - The solprop version beside which CMake configured the extension, or - ``unknown``. Informational: the contract is the seam members - :func:`check_build` looks for, not a version string. - kokkos_version, kokkos_fft_version, kokkos_kernels_version, nanobind_version - Native dependency versions. - build_profile - ``"local"`` (the default: vendor BLAS when one is found, the build machine's - architecture), ``"distribution"`` (``SOLPROP_NATIVE_PROFILE=DISTRIBUTION``: a - CPU baseline and the platform's BLAS, for wheels that run on other machines) - or ``"reference"`` (``SOLPROP_NATIVE_PROFILE=REFERENCE``: no vendor BLAS and no - machine-specific code, for the gate's tidy and sanitizer builds). - cpu_baseline - The oldest processor the build runs on (``-march`` on x86-64, ``-mcpu`` on - Arm), for example ``"x86-64-v3"``; ``"none"`` for the compiler's default or a - native build. - vector_math - The host kernels' transcendental functions: ``"SLEEF 3.9.0 u10, AVX2"`` (its - 1.0 ulp vector functions, in the variant the build selected: ``AVX2``, ``SSE2``, - ``AVX-512F`` on request or ``AdvSIMD``), the default on x86-64, or - ``"scalar libm"``, the system's, on Arm and in device builds - (``SOLPROP_NATIVE_VECTOR_MATH``). - gemm_provider, tpl_blas_enabled, blas_backend, blas_thread_policy - Dense-transform implementation (always ``"HYBRID"``: the vendor GEMM for an - eligible transform, the custom kernels for the rest) and external-library - threading policy. - blas_selection - How configuration chose the BLAS: ``automatic: ...``, ``explicit: ...`` or - ``distribution: ...`` or ``disabled: ...``, including the native-order GEMM - fallback when none was found. - blas_threads, blas_threading - Actual OpenBLAS or MKL BLAS thread limit and threading implementation. - The limit is ``None`` when the provider has no supported runtime query. - execution_space, host_execution_space, device_backend, device_arch - Configured execution-space identity. - device_count - Visible accelerator count; zero for a CPU build. - raman_fft_option - Configured Raman provider; each state reports its actual provider and tasks. - fft_backend, fft_planner, fft_normalization - Transform implementation and planning policy. - fft_library, raman_fft_library, blas_library, openmp_library - Loaded provider paths resolved through this extension; empty if absent. - compiler, compiler_family, compiler_flags, cxx_standard, build_type - Project compilation record. - coverage_instrumented - Whether project sources carry Clang source-based coverage with atomic - counters. These builds are diagnostic and must not supply timing results. - coverage_source_hash - SHA-256 of the ordered project C++ source/header hashes in a coverage - build; empty for ordinary builds. It binds reports to the compiled source. - openmp_enabled - Whether the host build enables OpenMP. - requested_threads, runtime_threads - Requested process cap and the resolved host-runtime concurrency limit. - thread_count - Resolved kernel concurrency within the process cap. - fftw_threads - Resolved modal FFT concurrency; zero for device FFTs. - raman_fft_threads, raman_tail_fft_threads - Resolved complete-chunk and partial-tail Raman FFT concurrency, zero on - devices. Each state also reports its actual provider and task limits. - threaded_fft_min_samples - Real-side samples per execution from which a modal or spatial CPU FFT plan - is threaded; shorter plans run on one FFTW thread. - - This field and the work sizes and medium width below are the built-in - table's for this process (``BUILTIN_EXECUTION``, solprop SEP 46), not - facts of the build: a workspace may apply a machine calibration's instead, - and a run's metadata records the values it applied (``calibration``). - cpu_size_policy_enabled - Whether this build supports operation-specific CPU widths (OpenMP CPU only). - explicit_thread_pools - Explicit pool requests as bits: kernels, FFT, Raman, Raman tail (1, 2, 4, 8). - stage_thread_min_elements, response_thread_min_elements, dense_thread_min_elements - Minimum work sizes for automatic stage, response and native dense threading. - modal_thread_min_elements - Work enabling full-width general-modal synthesis, projection and map updates. - flow_thread_min_elements - Matrix-function work enabling full-width coupled-flow construction. - copy_thread_min_elements - Element count enabling full-width automatic packing and temporal copies. - raman_thread_min_elements - Minimum real-side sample count for automatic Raman threading. - stage_full_thread_elements, response_full_thread_elements - Work sizes enabling full-width stage and response execution. - response_full_thread_samples - Fine samples per response column enabling full-width execution. - response_full_thread_columns - Active response columns enabling full-width execution after the work threshold. - dense_full_thread_elements - Work size enabling full-width native dense execution. - fft_full_thread_elements, raman_full_thread_elements - Real-side sample counts enabling full-width automatic FFT plans. - adams_thread_min_elements, adams_full_thread_elements - Elements times history rows from which the exponential Adams trial's passes - are threaded, and from which they take the full pool. - cpu_medium_threads - Maximum width between an operation's serial and full-width thresholds. - band_column, band_tile_threads - The field plasma column's band form, ``"tiled"`` (a team per tile of each - row, a device's default) or ``"team"`` (a team per node, the host's - default), and a tiled column's host team width (zero on a device or for the - team column); set at start by ``SOLPROP_NATIVE_BAND_TILES``. - native_arch_enabled, aggressive_vectorization_enabled, ipo_enabled, - fast_math_enabled, optimization_policy - Optimization switches and the human-readable compiled policy. - layout - Field layout. - python_version, numpy_version, platform_machine - Host Python environment. - """ - - package_version: str - git_commit: str - source_digest: str - solprop_version_built_against: str - kokkos_version: str - kokkos_fft_version: str - kokkos_kernels_version: str - nanobind_version: str - build_profile: str - cpu_baseline: str - vector_math: str - gemm_provider: str - tpl_blas_enabled: bool - blas_backend: str - blas_thread_policy: str - blas_selection: str - blas_threads: int | None - blas_threading: str - execution_space: str - host_execution_space: str - device_backend: str - device_arch: str - device_count: int - raman_fft_library: str - raman_fft_option: str - fft_backend: str - fft_planner: str - fft_normalization: str - fft_library: str - blas_library: str - openmp_library: str - compiler: str - compiler_family: str - compiler_flags: str - cxx_standard: int - build_type: str - coverage_instrumented: bool - coverage_source_hash: str - openmp_enabled: bool - thread_count: int - requested_threads: int - runtime_threads: int - fftw_threads: int - raman_fft_threads: int - raman_tail_fft_threads: int - threaded_fft_min_samples: int - cpu_size_policy_enabled: bool - explicit_thread_pools: int - stage_thread_min_elements: int - copy_thread_min_elements: int - flow_thread_min_elements: int - modal_thread_min_elements: int - response_thread_min_elements: int - dense_thread_min_elements: int - raman_thread_min_elements: int - stage_full_thread_elements: int - response_full_thread_elements: int - response_full_thread_samples: int - response_full_thread_columns: int - dense_full_thread_elements: int - fft_full_thread_elements: int - raman_full_thread_elements: int - adams_thread_min_elements: int - adams_full_thread_elements: int - cpu_medium_threads: int - band_column: str - band_tile_threads: int - native_arch_enabled: bool - aggressive_vectorization_enabled: bool - ipo_enabled: bool - fast_math_enabled: bool - optimization_policy: str - layout: str - python_version: str - numpy_version: str - platform_machine: str +# The build record, which reads the process's built-in execution table above. +from ._build import ( # noqa: E402 (after BUILTIN_EXECUTION) + BuildInfo, + build_info, + check_build, +) def available_device_memory() -> int | None: @@ -474,209 +275,6 @@ def available_device_memory() -> int | None: return None if free is None else int(free) -def build_info() -> BuildInfo: - """Return the immutable compiled-build record. - - Returns - ------- - BuildInfo - Compiler, dependencies, execution spaces, concurrency, layout, optimization - policy, and source-version provenance. - """ - native_record = _core.build_record() - blas_threads = native_record["blas_threads"] - record = {key: value for key, value in native_record.items() if value is not None} - return BuildInfo( - package_version=__version__, - git_commit=str(record["backend_commit"]), - source_digest=str(record["source_digest"]), - solprop_version_built_against=str(record["solprop_built_against"]), - kokkos_version=str(record["kokkos_version"]), - kokkos_fft_version=str(record["kokkos_fft_version"]), - kokkos_kernels_version=str(record["kokkos_kernels_version"]), - nanobind_version=str(record["nanobind_version"]), - build_profile=str(record["build_profile"]), - cpu_baseline=str(record["cpu_baseline"]), - vector_math=str(record["vector_math"]), - gemm_provider=str(record["gemm_provider"]), - tpl_blas_enabled=bool(record["tpl_blas_enabled"]), - blas_backend=str(record["blas_backend"]), - blas_thread_policy=str(record["blas_thread_policy"]), - blas_selection=str(record["blas_selection"]), - blas_threads=None if blas_threads is None else int(blas_threads), - blas_threading=str(record["blas_threading"]), - execution_space=str(record["execution_space"]), - host_execution_space=str(record["host_execution_space"]), - device_backend=str(record["device_backend"]), - device_arch=str(record["device_arch"]), - device_count=int(record["device_count"]), - raman_fft_library=str(record["raman_fft_library"]), - raman_fft_option=str(record["raman_fft_option"]), - fft_backend=str(record["fft_backend"]), - fft_planner=str(record["fft_planner"]), - fft_normalization=str(record["fft_normalization"]), - fft_library=str(record["fft_library"]), - blas_library=str(record["blas_library"]), - openmp_library=str(record["openmp_library"]), - compiler=str(record["compiler"]), - compiler_family=str(record["compiler_family"]), - compiler_flags=str(record["compiler_flags"]), - cxx_standard=int(record["cxx_standard"]), - build_type=str(record["build_type"]), - coverage_instrumented=bool(record["coverage_instrumented"]), - coverage_source_hash=str(record["coverage_source_hash"]), - openmp_enabled=bool(record["openmp_enabled"]), - thread_count=int(record["thread_count"]), - requested_threads=int(record["requested_threads"]), - runtime_threads=int(record["runtime_threads"]), - fftw_threads=int(record["fftw_threads"]), - raman_fft_threads=int(record["raman_fft_threads"]), - raman_tail_fft_threads=int(record["raman_tail_fft_threads"]), - threaded_fft_min_samples=BUILTIN_EXECUTION.fft_threshold, - cpu_size_policy_enabled=bool(record["cpu_size_policy_enabled"]), - explicit_thread_pools=int(record["explicit_thread_pools"]), - stage_thread_min_elements=BUILTIN_EXECUTION.stage_threshold, - copy_thread_min_elements=BUILTIN_EXECUTION.copy_threshold, - flow_thread_min_elements=BUILTIN_EXECUTION.flow_threshold, - modal_thread_min_elements=BUILTIN_EXECUTION.modal_threshold, - response_thread_min_elements=BUILTIN_EXECUTION.response_threshold, - dense_thread_min_elements=BUILTIN_EXECUTION.dense_threshold, - raman_thread_min_elements=BUILTIN_EXECUTION.raman_threshold, - stage_full_thread_elements=BUILTIN_EXECUTION.stage_full, - response_full_thread_elements=BUILTIN_EXECUTION.response_full, - response_full_thread_samples=BUILTIN_EXECUTION.response_full_samples, - response_full_thread_columns=BUILTIN_EXECUTION.response_full_columns, - dense_full_thread_elements=BUILTIN_EXECUTION.dense_full, - fft_full_thread_elements=BUILTIN_EXECUTION.fft_full, - raman_full_thread_elements=BUILTIN_EXECUTION.raman_full, - adams_thread_min_elements=BUILTIN_EXECUTION.adams_threshold, - adams_full_thread_elements=BUILTIN_EXECUTION.adams_full, - cpu_medium_threads=BUILTIN_EXECUTION.medium_threads, - band_column=str(record["band_column"]), - band_tile_threads=int(record["band_tile_threads"]), - native_arch_enabled=bool(record["native_arch_enabled"]), - aggressive_vectorization_enabled=bool( - record["aggressive_vectorization_enabled"] - ), - ipo_enabled=bool(record["ipo_enabled"]), - fast_math_enabled=bool(record["fast_math_enabled"]), - optimization_policy=str(record["optimization_policy"]), - layout=str(record["layout"]), - python_version=platform.python_version(), - numpy_version=np.__version__, - platform_machine=platform.machine(), - ) - - -def _source_digest(root: Path) -> str: - """Return the digest of the sources an extension is compiled from. - - The computation of ``cmake/SourceDigest.cmake``, which embeds it in the build; the - two must agree. The record is one ``{path}:{sha256 of its bytes}`` line for - ``CMakeLists.txt`` and each C++ source and header under ``src`` and ``include``, the - paths relative to `root` and sorted bytewise, and the digest is the SHA-256 of the - record. - """ - paths = ["CMakeLists.txt"] - for folder in ("src", "include"): - paths.extend( - path.relative_to(root).as_posix() - for path in (root / folder).rglob("*") - if path.suffix in {".cpp", ".hpp", ".h"} and path.is_file() - ) - # Code points sort as their UTF-8 bytes do, so this is CMake's bytewise order. - record = "".join( - f"{path}:{hashlib.sha256((root / path).read_bytes()).hexdigest()}\n" - for path in sorted(paths) - ) - return hashlib.sha256(record.encode()).hexdigest() - - -@functools.cache -def _checkout_digest() -> str | None: - """Return this editable checkout's source digest, computed once per process. - - - Returns: the digest, or ``None`` for an installed wheel, which has no sources - beside it and which solprop's metadata pins to its own version. - """ - root = Path(__file__).parents[2] - return _source_digest(root) if (root / "CMakeLists.txt").is_file() else None - - -def check_build() -> None: - """Validate this build against this checkout's sources and its own metadata. - - Raises - ------ - solprop.exceptions.BackendError - If an editable checkout's ``native`` sources are not the ones the extension - was compiled from, execution-space metadata is inconsistent, a reference build - carries vendor BLAS or machine-specific code, a distribution build carries - native code or no BLAS, fast-math is enabled, or an accelerator build sees no - device. The solprop *version* is recorded, never compared here: the two - packages are released together and pinned to one version. - """ - from solprop.exceptions import BackendError - - info = build_info() - digest = _checkout_digest() - if digest is not None and digest != info.source_digest: - raise BackendError( - "the solprop-native extension was built from other sources than this " - f"checkout's native/ ({Path(__file__).parents[2]}): rebuild it with " - "`uv sync --reinstall-package solprop-native`" - ) - if info.fast_math_enabled: - raise BackendError("solprop-native was built with forbidden fast-math") - if info.openmp_enabled and info.execution_space != "OpenMP": - raise BackendError( - "the build enables OpenMP but its Kokkos execution space is " - f"{info.execution_space!r}" - ) - # A CPU build runs OpenMP. - expected_execution = { - "none": {"OpenMP"}, - "cuda": {"Cuda"}, - "hip": {"HIP"}, - } - if info.execution_space not in expected_execution.get(info.device_backend, set()): - raise BackendError( - f"the {info.device_backend!r} device backend is inconsistent with " - f"execution space {info.execution_space!r}" - ) - if info.device_backend != "none" and info.device_count < 1: - raise BackendError( - f"the {info.device_backend} build has no visible accelerator device" - ) - if info.gemm_provider != "HYBRID": - raise BackendError(f"unknown compiled GEMM provider {info.gemm_provider!r}") - if info.tpl_blas_enabled and info.blas_backend == "none": - raise BackendError("the build enables a BLAS TPL but identifies no provider") - if info.build_profile not in {"local", "distribution", "reference"}: - raise BackendError(f"unknown build profile {info.build_profile!r}") - if info.build_profile == "reference" and ( - info.tpl_blas_enabled or info.native_arch_enabled or info.cpu_baseline != "none" - ): - raise BackendError( - "the reference build links a vendor BLAS or machine-specific code" - ) - if info.build_profile == "distribution" and ( - info.native_arch_enabled or not info.tpl_blas_enabled - ): - raise BackendError( - "the distribution build carries machine-native code or no BLAS" - ) - device_blas = {"cuda": "cuBLAS", "hip": "rocBLAS"} - if ( - info.device_backend != "none" - and info.tpl_blas_enabled - and info.blas_backend != device_blas.get(info.device_backend) - ): - raise BackendError( - "the selected BLAS provider cannot access this accelerator's memory" - ) - - def __getattr__(name: str) -> object: """Load the solprop-dependent backend class only when it is requested.""" if name == "NativeBackend": diff --git a/native/src/solprop_native/_build.py b/native/src/solprop_native/_build.py new file mode 100644 index 000000000..d76c3e41c --- /dev/null +++ b/native/src/solprop_native/_build.py @@ -0,0 +1,434 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""The compiled build's record and its check against this checkout's sources. + +:class:`BuildInfo` and :func:`build_info` read the extension's own record; an editable +checkout's sources are digested as ``cmake/SourceDigest.cmake`` digests them, and +:func:`check_build` refuses an extension built from others or with inconsistent +metadata. The package re-exports the public names. +""" + +from __future__ import annotations + +import functools +import hashlib +import platform +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from . import BUILTIN_EXECUTION, __version__, _core + +__all__ = ["BuildInfo", "build_info", "check_build"] + + +@dataclass(frozen=True) +class BuildInfo: + """Immutable build and execution provenance for this extension. + + Attributes + ---------- + package_version + Installed ``solprop-native`` distribution version. + git_commit + Backend repository commit embedded when CMake configured the extension. + source_digest + SHA-256 of what the extension was compiled from: ``CMakeLists.txt`` and the C++ + sources and headers under ``src`` and ``include`` + (``cmake/SourceDigest.cmake``). :func:`check_build` compares it with an editable + checkout's sources. + solprop_version_built_against + The solprop version beside which CMake configured the extension, or + ``unknown``. Informational: the contract is the seam members + :func:`check_build` looks for, not a version string. + kokkos_version, kokkos_fft_version, kokkos_kernels_version, nanobind_version + Native dependency versions. + build_profile + ``"local"`` (the default: vendor BLAS when one is found, the build machine's + architecture), ``"distribution"`` (``SOLPROP_NATIVE_PROFILE=DISTRIBUTION``: a + CPU baseline and the platform's BLAS, for wheels that run on other machines) + or ``"reference"`` (``SOLPROP_NATIVE_PROFILE=REFERENCE``: no vendor BLAS and no + machine-specific code, for the gate's tidy and sanitizer builds). + cpu_baseline + The oldest processor the build runs on (``-march`` on x86-64, ``-mcpu`` on + Arm), for example ``"x86-64-v3"``; ``"none"`` for the compiler's default or a + native build. + vector_math + The host kernels' transcendental functions: ``"SLEEF 3.9.0 u10, AVX2"`` (its + 1.0 ulp vector functions, in the variant the build selected: ``AVX2``, ``SSE2``, + ``AVX-512F`` on request or ``AdvSIMD``), the default on x86-64, or + ``"scalar libm"``, the system's, on Arm and in device builds + (``SOLPROP_NATIVE_VECTOR_MATH``). + gemm_provider, tpl_blas_enabled, blas_backend, blas_thread_policy + Dense-transform implementation (always ``"HYBRID"``: the vendor GEMM for an + eligible transform, the custom kernels for the rest) and external-library + threading policy. + blas_selection + How configuration chose the BLAS: ``automatic: ...``, ``explicit: ...`` or + ``distribution: ...`` or ``disabled: ...``, including the native-order GEMM + fallback when none was found. + blas_threads, blas_threading + Actual OpenBLAS or MKL BLAS thread limit and threading implementation. + The limit is ``None`` when the provider has no supported runtime query. + execution_space, host_execution_space, device_backend, device_arch + Configured execution-space identity. + device_count + Visible accelerator count; zero for a CPU build. + raman_fft_option + Configured Raman provider; each state reports its actual provider and tasks. + fft_backend, fft_planner, fft_normalization + Transform implementation and planning policy. + fft_library, raman_fft_library, blas_library, openmp_library + Loaded provider paths resolved through this extension; empty if absent. + compiler, compiler_family, compiler_flags, cxx_standard, build_type + Project compilation record. + coverage_instrumented + Whether project sources carry Clang source-based coverage with atomic + counters. These builds are diagnostic and must not supply timing results. + coverage_source_hash + SHA-256 of the ordered project C++ source/header hashes in a coverage + build; empty for ordinary builds. It binds reports to the compiled source. + openmp_enabled + Whether the host build enables OpenMP. + requested_threads, runtime_threads + Requested process cap and the resolved host-runtime concurrency limit. + thread_count + Resolved kernel concurrency within the process cap. + fftw_threads + Resolved modal FFT concurrency; zero for device FFTs. + raman_fft_threads, raman_tail_fft_threads + Resolved complete-chunk and partial-tail Raman FFT concurrency, zero on + devices. Each state also reports its actual provider and task limits. + threaded_fft_min_samples + Real-side samples per execution from which a modal or spatial CPU FFT plan + is threaded; shorter plans run on one FFTW thread. + + This field and the work sizes and medium width below are the built-in + table's for this process (``BUILTIN_EXECUTION``, solprop SEP 46), not + facts of the build: a workspace may apply a machine calibration's instead, + and a run's metadata records the values it applied (``calibration``). + cpu_size_policy_enabled + Whether this build supports operation-specific CPU widths (OpenMP CPU only). + explicit_thread_pools + Explicit pool requests as bits: kernels, FFT, Raman, Raman tail (1, 2, 4, 8). + stage_thread_min_elements, response_thread_min_elements, dense_thread_min_elements + Minimum work sizes for automatic stage, response and native dense threading. + modal_thread_min_elements + Work enabling full-width general-modal synthesis, projection and map updates. + flow_thread_min_elements + Matrix-function work enabling full-width coupled-flow construction. + copy_thread_min_elements + Element count enabling full-width automatic packing and temporal copies. + raman_thread_min_elements + Minimum real-side sample count for automatic Raman threading. + stage_full_thread_elements, response_full_thread_elements + Work sizes enabling full-width stage and response execution. + response_full_thread_samples + Fine samples per response column enabling full-width execution. + response_full_thread_columns + Active response columns enabling full-width execution after the work threshold. + dense_full_thread_elements + Work size enabling full-width native dense execution. + fft_full_thread_elements, raman_full_thread_elements + Real-side sample counts enabling full-width automatic FFT plans. + adams_thread_min_elements, adams_full_thread_elements + Elements times history rows from which the exponential Adams trial's passes + are threaded, and from which they take the full pool. + cpu_medium_threads + Maximum width between an operation's serial and full-width thresholds. + band_column, band_tile_threads + The field plasma column's band form, ``"tiled"`` (a team per tile of each + row, a device's default) or ``"team"`` (a team per node, the host's + default), and a tiled column's host team width (zero on a device or for the + team column); set at start by ``SOLPROP_NATIVE_BAND_TILES``. + native_arch_enabled, aggressive_vectorization_enabled, ipo_enabled, + fast_math_enabled, optimization_policy + Optimization switches and the human-readable compiled policy. + layout + Field layout. + python_version, numpy_version, platform_machine + Host Python environment. + """ + + package_version: str + git_commit: str + source_digest: str + solprop_version_built_against: str + kokkos_version: str + kokkos_fft_version: str + kokkos_kernels_version: str + nanobind_version: str + build_profile: str + cpu_baseline: str + vector_math: str + gemm_provider: str + tpl_blas_enabled: bool + blas_backend: str + blas_thread_policy: str + blas_selection: str + blas_threads: int | None + blas_threading: str + execution_space: str + host_execution_space: str + device_backend: str + device_arch: str + device_count: int + raman_fft_library: str + raman_fft_option: str + fft_backend: str + fft_planner: str + fft_normalization: str + fft_library: str + blas_library: str + openmp_library: str + compiler: str + compiler_family: str + compiler_flags: str + cxx_standard: int + build_type: str + coverage_instrumented: bool + coverage_source_hash: str + openmp_enabled: bool + thread_count: int + requested_threads: int + runtime_threads: int + fftw_threads: int + raman_fft_threads: int + raman_tail_fft_threads: int + threaded_fft_min_samples: int + cpu_size_policy_enabled: bool + explicit_thread_pools: int + stage_thread_min_elements: int + copy_thread_min_elements: int + flow_thread_min_elements: int + modal_thread_min_elements: int + response_thread_min_elements: int + dense_thread_min_elements: int + raman_thread_min_elements: int + stage_full_thread_elements: int + response_full_thread_elements: int + response_full_thread_samples: int + response_full_thread_columns: int + dense_full_thread_elements: int + fft_full_thread_elements: int + raman_full_thread_elements: int + adams_thread_min_elements: int + adams_full_thread_elements: int + cpu_medium_threads: int + band_column: str + band_tile_threads: int + native_arch_enabled: bool + aggressive_vectorization_enabled: bool + ipo_enabled: bool + fast_math_enabled: bool + optimization_policy: str + layout: str + python_version: str + numpy_version: str + platform_machine: str + + +def build_info() -> BuildInfo: + """Return the immutable compiled-build record. + + Returns + ------- + BuildInfo + Compiler, dependencies, execution spaces, concurrency, layout, optimization + policy, and source-version provenance. + """ + native_record = _core.build_record() + blas_threads = native_record["blas_threads"] + record = {key: value for key, value in native_record.items() if value is not None} + return BuildInfo( + package_version=__version__, + git_commit=str(record["backend_commit"]), + source_digest=str(record["source_digest"]), + solprop_version_built_against=str(record["solprop_built_against"]), + kokkos_version=str(record["kokkos_version"]), + kokkos_fft_version=str(record["kokkos_fft_version"]), + kokkos_kernels_version=str(record["kokkos_kernels_version"]), + nanobind_version=str(record["nanobind_version"]), + build_profile=str(record["build_profile"]), + cpu_baseline=str(record["cpu_baseline"]), + vector_math=str(record["vector_math"]), + gemm_provider=str(record["gemm_provider"]), + tpl_blas_enabled=bool(record["tpl_blas_enabled"]), + blas_backend=str(record["blas_backend"]), + blas_thread_policy=str(record["blas_thread_policy"]), + blas_selection=str(record["blas_selection"]), + blas_threads=None if blas_threads is None else int(blas_threads), + blas_threading=str(record["blas_threading"]), + execution_space=str(record["execution_space"]), + host_execution_space=str(record["host_execution_space"]), + device_backend=str(record["device_backend"]), + device_arch=str(record["device_arch"]), + device_count=int(record["device_count"]), + raman_fft_library=str(record["raman_fft_library"]), + raman_fft_option=str(record["raman_fft_option"]), + fft_backend=str(record["fft_backend"]), + fft_planner=str(record["fft_planner"]), + fft_normalization=str(record["fft_normalization"]), + fft_library=str(record["fft_library"]), + blas_library=str(record["blas_library"]), + openmp_library=str(record["openmp_library"]), + compiler=str(record["compiler"]), + compiler_family=str(record["compiler_family"]), + compiler_flags=str(record["compiler_flags"]), + cxx_standard=int(record["cxx_standard"]), + build_type=str(record["build_type"]), + coverage_instrumented=bool(record["coverage_instrumented"]), + coverage_source_hash=str(record["coverage_source_hash"]), + openmp_enabled=bool(record["openmp_enabled"]), + thread_count=int(record["thread_count"]), + requested_threads=int(record["requested_threads"]), + runtime_threads=int(record["runtime_threads"]), + fftw_threads=int(record["fftw_threads"]), + raman_fft_threads=int(record["raman_fft_threads"]), + raman_tail_fft_threads=int(record["raman_tail_fft_threads"]), + threaded_fft_min_samples=BUILTIN_EXECUTION.fft_threshold, + cpu_size_policy_enabled=bool(record["cpu_size_policy_enabled"]), + explicit_thread_pools=int(record["explicit_thread_pools"]), + stage_thread_min_elements=BUILTIN_EXECUTION.stage_threshold, + copy_thread_min_elements=BUILTIN_EXECUTION.copy_threshold, + flow_thread_min_elements=BUILTIN_EXECUTION.flow_threshold, + modal_thread_min_elements=BUILTIN_EXECUTION.modal_threshold, + response_thread_min_elements=BUILTIN_EXECUTION.response_threshold, + dense_thread_min_elements=BUILTIN_EXECUTION.dense_threshold, + raman_thread_min_elements=BUILTIN_EXECUTION.raman_threshold, + stage_full_thread_elements=BUILTIN_EXECUTION.stage_full, + response_full_thread_elements=BUILTIN_EXECUTION.response_full, + response_full_thread_samples=BUILTIN_EXECUTION.response_full_samples, + response_full_thread_columns=BUILTIN_EXECUTION.response_full_columns, + dense_full_thread_elements=BUILTIN_EXECUTION.dense_full, + fft_full_thread_elements=BUILTIN_EXECUTION.fft_full, + raman_full_thread_elements=BUILTIN_EXECUTION.raman_full, + adams_thread_min_elements=BUILTIN_EXECUTION.adams_threshold, + adams_full_thread_elements=BUILTIN_EXECUTION.adams_full, + cpu_medium_threads=BUILTIN_EXECUTION.medium_threads, + band_column=str(record["band_column"]), + band_tile_threads=int(record["band_tile_threads"]), + native_arch_enabled=bool(record["native_arch_enabled"]), + aggressive_vectorization_enabled=bool( + record["aggressive_vectorization_enabled"] + ), + ipo_enabled=bool(record["ipo_enabled"]), + fast_math_enabled=bool(record["fast_math_enabled"]), + optimization_policy=str(record["optimization_policy"]), + layout=str(record["layout"]), + python_version=platform.python_version(), + numpy_version=np.__version__, + platform_machine=platform.machine(), + ) + + +def _source_digest(root: Path) -> str: + """Return the digest of the sources an extension is compiled from. + + The computation of ``cmake/SourceDigest.cmake``, which embeds it in the build; the + two must agree. The record is one ``{path}:{sha256 of its bytes}`` line for + ``CMakeLists.txt`` and each C++ source and header under ``src`` and ``include``, the + paths relative to `root` and sorted bytewise, and the digest is the SHA-256 of the + record. + """ + paths = ["CMakeLists.txt"] + for folder in ("src", "include"): + paths.extend( + path.relative_to(root).as_posix() + for path in (root / folder).rglob("*") + if path.suffix in {".cpp", ".hpp", ".h"} and path.is_file() + ) + # Code points sort as their UTF-8 bytes do, so this is CMake's bytewise order. + record = "".join( + f"{path}:{hashlib.sha256((root / path).read_bytes()).hexdigest()}\n" + for path in sorted(paths) + ) + return hashlib.sha256(record.encode()).hexdigest() + + +@functools.cache +def _checkout_digest() -> str | None: + """Return this editable checkout's source digest, computed once per process. + + - Returns: the digest, or ``None`` for an installed wheel, which has no sources + beside it and which solprop's metadata pins to its own version. + """ + root = Path(__file__).parents[2] + return _source_digest(root) if (root / "CMakeLists.txt").is_file() else None + + +def check_build() -> None: + """Validate this build against this checkout's sources and its own metadata. + + Raises + ------ + solprop.exceptions.BackendError + If an editable checkout's ``native`` sources are not the ones the extension + was compiled from, execution-space metadata is inconsistent, a reference build + carries vendor BLAS or machine-specific code, a distribution build carries + native code or no BLAS, fast-math is enabled, or an accelerator build sees no + device. The solprop *version* is recorded, never compared here: the two + packages are released together and pinned to one version. + """ + from solprop.exceptions import BackendError + + info = build_info() + digest = _checkout_digest() + if digest is not None and digest != info.source_digest: + raise BackendError( + "the solprop-native extension was built from other sources than this " + f"checkout's native/ ({Path(__file__).parents[2]}): rebuild it with " + "`uv sync --reinstall-package solprop-native`" + ) + if info.fast_math_enabled: + raise BackendError("solprop-native was built with forbidden fast-math") + if info.openmp_enabled and info.execution_space != "OpenMP": + raise BackendError( + "the build enables OpenMP but its Kokkos execution space is " + f"{info.execution_space!r}" + ) + # A CPU build runs OpenMP. + expected_execution = { + "none": {"OpenMP"}, + "cuda": {"Cuda"}, + "hip": {"HIP"}, + } + if info.execution_space not in expected_execution.get(info.device_backend, set()): + raise BackendError( + f"the {info.device_backend!r} device backend is inconsistent with " + f"execution space {info.execution_space!r}" + ) + if info.device_backend != "none" and info.device_count < 1: + raise BackendError( + f"the {info.device_backend} build has no visible accelerator device" + ) + if info.gemm_provider != "HYBRID": + raise BackendError(f"unknown compiled GEMM provider {info.gemm_provider!r}") + if info.tpl_blas_enabled and info.blas_backend == "none": + raise BackendError("the build enables a BLAS TPL but identifies no provider") + if info.build_profile not in {"local", "distribution", "reference"}: + raise BackendError(f"unknown build profile {info.build_profile!r}") + if info.build_profile == "reference" and ( + info.tpl_blas_enabled or info.native_arch_enabled or info.cpu_baseline != "none" + ): + raise BackendError( + "the reference build links a vendor BLAS or machine-specific code" + ) + if info.build_profile == "distribution" and ( + info.native_arch_enabled or not info.tpl_blas_enabled + ): + raise BackendError( + "the distribution build carries machine-native code or no BLAS" + ) + device_blas = {"cuda": "cuBLAS", "hip": "rocBLAS"} + if ( + info.device_backend != "none" + and info.tpl_blas_enabled + and info.blas_backend != device_blas.get(info.device_backend) + ): + raise BackendError( + "the selected BLAS provider cannot access this accelerator's memory" + ) diff --git a/native/tests/test_api_contract.py b/native/tests/test_api_contract.py index eaa95f7cc..142d1fc13 100644 --- a/native/tests/test_api_contract.py +++ b/native/tests/test_api_contract.py @@ -35,7 +35,11 @@ from thread_sizing import Sizing, assert_threaded import solprop_native as native -from solprop_native import _bind, _core # pyright: ignore[reportPrivateUsage] +from solprop_native import ( + _bind, # pyright: ignore[reportPrivateUsage] + _build, # pyright: ignore[reportPrivateUsage] + _core, +) from solprop_native._core import _testing # pyright: ignore[reportPrivateUsage] POOL_VARIABLES = ( @@ -258,10 +262,10 @@ def test_the_digest_is_the_sorted_record_of_the_compiled_sources( source_tree: tuple[Path, list[str]], ) -> None: root, names = source_tree - digest = native._source_digest(root) # pyright: ignore[reportPrivateUsage] + digest = _build._source_digest(root) # pyright: ignore[reportPrivateUsage] assert digest == _digest_of(root, names) (root / "src/B.cpp").write_bytes(b"// one byte more\n ") - assert native._source_digest(root) != digest # pyright: ignore[reportPrivateUsage] + assert _build._source_digest(root) != digest # pyright: ignore[reportPrivateUsage] def test_the_build_script_and_python_agree_on_the_digest( @@ -292,15 +296,15 @@ def _editable_root() -> Path: @pytest.fixture def fresh_checkout_digest() -> Iterator[None]: """Read the checkout's digest afresh, and leave no patched value in the cache.""" - native._checkout_digest.cache_clear() # pyright: ignore[reportPrivateUsage] + _build._checkout_digest.cache_clear() # pyright: ignore[reportPrivateUsage] yield - native._checkout_digest.cache_clear() # pyright: ignore[reportPrivateUsage] + _build._checkout_digest.cache_clear() # pyright: ignore[reportPrivateUsage] @pytest.mark.usefixtures("fresh_checkout_digest") def test_an_editable_build_records_its_checkouts_digest() -> None: root = _editable_root() - source_digest = native._source_digest # pyright: ignore[reportPrivateUsage] + source_digest = _build._source_digest # pyright: ignore[reportPrivateUsage] assert native.build_info().source_digest == source_digest(root) assert native.check_build() is None @@ -314,7 +318,7 @@ def test_an_extension_built_from_other_sources_is_refused_naming_uv_sync( def other(root: Path) -> str: return f"not {root}'s" - monkeypatch.setattr(native, "_source_digest", other) + monkeypatch.setattr(_build, "_source_digest", other) with pytest.raises(BackendError, match=r"other sources.*`uv sync "): native.check_build() @@ -325,13 +329,13 @@ def test_the_digest_is_computed_once_per_process( ) -> None: root = _editable_root() calls: list[Path] = [] - source_digest = native._source_digest # pyright: ignore[reportPrivateUsage] + source_digest = _build._source_digest # pyright: ignore[reportPrivateUsage] def counted(where: Path) -> str: calls.append(where) return source_digest(where) - monkeypatch.setattr(native, "_source_digest", counted) + monkeypatch.setattr(_build, "_source_digest", counted) native.check_build() native.check_build() assert calls == [root] @@ -342,16 +346,16 @@ def test_a_wheel_layout_skips_the_comparison( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: """A wheel has no ``CMakeLists.txt`` two levels up: nothing is read or compared.""" - installed = tmp_path / "site-packages" / "solprop_native" / "__init__.py" - monkeypatch.setattr(native, "__file__", str(installed)) + installed = tmp_path / "site-packages" / "solprop_native" / "_build.py" + monkeypatch.setattr(_build, "__file__", str(installed)) def refuse(root: Path) -> str: raise AssertionError(f"a wheel has no sources to hash, asked for {root}") - monkeypatch.setattr(native, "_source_digest", refuse) + monkeypatch.setattr(_build, "_source_digest", refuse) info = dataclasses.replace(native.build_info(), source_digest="a wheel's own") - monkeypatch.setattr(native, "build_info", lambda: info) - assert native._checkout_digest() is None # pyright: ignore[reportPrivateUsage] + monkeypatch.setattr(_build, "build_info", lambda: info) + assert _build._checkout_digest() is None # pyright: ignore[reportPrivateUsage] assert native.check_build() is None From 84d9255c494afbe40b63ec1db3c37ad44155561b Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:20:01 +0000 Subject: [PATCH 18/21] Drop the native lowering's unused empty-noise constant Co-Authored-By: Claude Opus 5.5 --- native/src/solprop_native/_lowering.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/native/src/solprop_native/_lowering.py b/native/src/solprop_native/_lowering.py index 149be1f17..f5febe0b7 100644 --- a/native/src/solprop_native/_lowering.py +++ b/native/src/solprop_native/_lowering.py @@ -139,10 +139,6 @@ def _flux_tables(problem: PropagationProblem) -> list[dict[str, object]]: return [_z_table("flux", flux.material)] -_EMPTY = np.empty(0, dtype=np.complex128) -"""No modified-noise realisation: the native state adds none.""" - - def _section_tables( problem: PropagationProblem, *, moving: bool, residual: bool ) -> dict[str, object]: From f7e78194ef7fac6ce2d3c56e531a75a9fa1c7474 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:20:36 +0000 Subject: [PATCH 19/21] Put statistics_plan.cpp's own header first, as clang-format orders it Since the rename the unit's main header is statistics_plan.hpp. Co-Authored-By: Claude Opus 5.5 --- native/src/statistics_plan.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/native/src/statistics_plan.cpp b/native/src/statistics_plan.cpp index a5beb0c98..f68265d69 100644 --- a/native/src/statistics_plan.cpp +++ b/native/src/statistics_plan.cpp @@ -2,13 +2,14 @@ // // SPDX-License-Identifier: GPL-3.0-or-later +#include "solprop_native/statistics_plan.hpp" + #include "solprop_native/chebyshev.hpp" #include "solprop_native/execution.hpp" #include "solprop_native/far_field.hpp" #include "solprop_native/general_state.hpp" #include "solprop_native/problem.hpp" #include "solprop_native/refuse.hpp" -#include "solprop_native/statistics_plan.hpp" #include #include From 73e43de96e6ff282fe4a31afebaaac58d3b8872e Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:48:24 +0000 Subject: [PATCH 20/21] Move native C++ files into domain directories Every C++ file under native/include/solprop_native/ and native/src/ (the Python package excepted) now lives in one domain directory: runtime/, transforms/, problem/, response/, march/, session/ and general/ for the headers, the same names plus bindings/ for the sources; src/general/ stays where it is. A header include/solprop_native/X.hpp moves to include/solprop_native/

/X.hpp and a source src/X to src//X, with every reference rewritten: the includes, the CMake source lists (the strict list among them, with the licence test's copy), and the path mentions in present-tense docs, comments and docstrings (SEPs and the changelog are left as written). Docstring lines the longer paths pushed past 88 columns are rewrapped, and clang-format re-sorts the includes. Pure moves and reference rewrites: no code changes. Generated by a re-runnable layout script that refuses any C++ file the mapping does not place. Co-Authored-By: Claude Opus 5.5 --- docs/dev/backends.md | 48 +++++++++---------- docs/dev/benchmarks.md | 2 +- docs/dev/decisions.md | 6 +-- docs/dev/dependencies.md | 2 +- docs/dev/kernel-contract.md | 2 +- docs/dev/stepper.md | 8 ++-- docs/dev/tooling.md | 2 +- docs/guide/performance/building-native.md | 2 +- native/AGENTS.md | 12 ++--- native/CMakeLists.txt | 32 ++++++------- native/README.md | 6 +-- native/cmake/VectorMath.cmake | 2 +- .../{ => general}/general_state.hpp | 8 ++-- .../{ => march}/adams_march.hpp | 4 +- .../solprop_native/{ => march}/adams_phi.hpp | 6 +-- .../solprop_native/{ => march}/chebyshev.hpp | 2 +- .../solprop_native/{ => march}/dense.hpp | 6 +-- .../solprop_native/{ => march}/engines.hpp | 14 +++--- .../{ => march}/lagrange_basis.hpp | 2 +- .../{ => march}/low_storage.hpp | 10 ++-- .../solprop_native/{ => march}/rk_march.hpp | 2 +- .../{ => march}/stage_tables.hpp | 10 ++-- .../solprop_native/{ => march}/z_table.hpp | 8 ++-- .../solprop_native/{ => problem}/problem.hpp | 6 +-- .../{ => problem}/run_records.hpp | 24 +++++----- .../{ => problem}/workspace.hpp | 20 ++++---- .../{ => response}/coupled_flow.hpp | 4 +- .../{ => response}/dispersive.hpp | 8 ++-- .../{ => response}/envelope_plasma.hpp | 0 .../{ => response}/far_field.hpp | 6 +-- .../{ => response}/flow_functions.hpp | 4 +- .../{ => response}/general_modal.hpp | 12 ++--- .../solprop_native/{ => response}/plasma.hpp | 8 ++-- .../solprop_native/{ => runtime}/build.hpp | 0 .../{ => runtime}/cpu_execution.hpp | 2 +- .../solprop_native/{ => runtime}/crossing.hpp | 0 .../solprop_native/{ => runtime}/dispatch.hpp | 0 .../{ => runtime}/execution.hpp | 2 +- .../solprop_native/{ => runtime}/fields.hpp | 5 +- .../solprop_native/{ => runtime}/fp.hpp | 0 .../solprop_native/{ => runtime}/graph.hpp | 0 .../{ => runtime}/host_source.hpp | 0 .../solprop_native/{ => runtime}/launch.hpp | 2 +- .../solprop_native/{ => runtime}/refuse.hpp | 0 .../{ => runtime}/setup_data.hpp | 4 +- .../solprop_native/{ => runtime}/vmath.hpp | 0 .../solprop_native/{ => session}/readings.hpp | 2 +- .../solprop_native/{ => session}/session.hpp | 14 +++--- .../{ => session}/session_dense.hpp | 2 +- .../{ => session}/session_resident.hpp | 4 +- .../{ => session}/statistics_plan.hpp | 0 .../{ => session}/transverse.hpp | 2 +- .../solprop_native/{ => session}/watch.hpp | 2 +- .../{ => transforms}/fft_plan.hpp | 10 ++-- .../{ => transforms}/fftw_wisdom.hpp | 4 +- .../solprop_native/{ => transforms}/gemm.hpp | 2 +- .../{ => transforms}/raman_fft.hpp | 4 +- .../{ => transforms}/reference_timing.hpp | 0 .../solprop_native/{ => transforms}/slab.hpp | 4 +- native/src/{ => bindings}/bindings.cpp | 46 +++++++++--------- .../src/{ => bindings}/bindings_helpers.cpp | 8 ++-- .../src/{ => bindings}/bindings_helpers.hpp | 0 .../{ => bindings}/bindings_transverse.cpp | 0 .../{ => bindings}/bindings_transverse.hpp | 2 +- native/src/{ => bindings}/records.cpp | 0 native/src/{ => bindings}/records.hpp | 6 +-- native/src/general/general_workspace.hpp | 9 ++-- native/src/general/kernels.hpp | 44 ++++++++--------- native/src/general/trial.cpp | 10 ++-- native/src/{ => march}/adams_march.cpp | 6 +-- native/src/{ => march}/dense.cpp | 6 +-- native/src/{ => march}/engines.cpp | 2 +- native/src/{ => march}/low_storage.cpp | 4 +- native/src/{ => march}/rk_march.cpp | 4 +- native/src/{ => problem}/problem.cpp | 4 +- native/src/{ => problem}/workspace.cpp | 8 ++-- native/src/{ => response}/coupled_flow.cpp | 8 ++-- native/src/{ => response}/dispersive.cpp | 10 ++-- native/src/{ => response}/far_field.cpp | 4 +- native/src/{ => response}/general_modal.cpp | 12 ++--- native/src/{ => runtime}/coverage_labels.cpp | 0 native/src/{ => runtime}/cpu_execution.cpp | 4 +- native/src/{ => runtime}/crossing.cpp | 4 +- native/src/{ => session}/session.cpp | 4 +- native/src/{ => session}/session_edge.cpp | 2 +- native/src/{ => session}/session_widths.cpp | 6 +-- native/src/{ => session}/statistics_plan.cpp | 16 +++---- native/src/{ => session}/transverse.cpp | 2 +- native/src/{ => session}/watch.cpp | 4 +- native/src/solprop_native/_helpers.py | 4 +- native/src/solprop_native/_transverse.py | 15 +++--- native/src/solprop_native/_watch.py | 8 ++-- native/src/solprop_native/session.py | 10 ++-- native/src/{ => transforms}/fftw_wisdom.cpp | 6 +-- native/src/{ => transforms}/raman_fft.cpp | 2 +- .../src/{ => transforms}/reference_timing.cpp | 8 ++-- native/tests/native/execution.hpp | 2 +- native/tests/native/test_adams_march.cpp | 2 +- native/tests/native/test_adams_phi.cpp | 9 ++-- native/tests/native/test_aperture.cpp | 6 +-- native/tests/native/test_core.cpp | 12 ++--- native/tests/native/test_coupled_flow.cpp | 10 ++-- native/tests/native/test_crossing.cpp | 2 +- native/tests/native/test_dispersive.cpp | 8 ++-- native/tests/native/test_envelope_plasma.cpp | 4 +- native/tests/native/test_fft.cpp | 10 ++-- native/tests/native/test_fp.cpp | 4 +- native/tests/native/test_general_modal.cpp | 12 ++--- native/tests/native/test_helpers.cpp | 4 +- native/tests/native/test_low_storage.cpp | 4 +- native/tests/native/test_rk_march.cpp | 2 +- native/tests/native/test_session.cpp | 2 +- native/tests/native/test_session_monitor.cpp | 2 +- .../tests/native/test_session_transverse.cpp | 2 +- native/tests/native/test_transverse.cpp | 2 +- native/tests/native/test_vmath.cpp | 8 ++-- native/tests/native/test_watch.cpp | 2 +- native/tests/native/test_workspace.cpp | 4 +- native/tests/native/test_z_table.cpp | 8 ++-- native/tests/test_band_plasma.py | 4 +- native/tests/test_band_stress.py | 8 ++-- native/tests/test_coupled_flow.py | 2 +- native/tests/test_fftw_plan_refusal.py | 2 +- native/tests/test_general_modal.py | 4 +- src/solprop/backends/engines.py | 6 +-- src/solprop/backends/numpy_ref/dense.py | 2 +- tests/backends/test_engines.py | 8 ++-- .../test_native_floating_point_licence.py | 42 ++++++++-------- tests/backends/test_native_rk_traces.py | 2 +- tests/backends/test_native_vector_math.py | 6 +-- tests/flow_parity.py | 4 +- 131 files changed, 429 insertions(+), 423 deletions(-) rename native/include/solprop_native/{ => general}/general_state.hpp (99%) rename native/include/solprop_native/{ => march}/adams_march.hpp (99%) rename native/include/solprop_native/{ => march}/adams_phi.hpp (99%) rename native/include/solprop_native/{ => march}/chebyshev.hpp (99%) rename native/include/solprop_native/{ => march}/dense.hpp (96%) rename native/include/solprop_native/{ => march}/engines.hpp (95%) rename native/include/solprop_native/{ => march}/lagrange_basis.hpp (99%) rename native/include/solprop_native/{ => march}/low_storage.hpp (96%) rename native/include/solprop_native/{ => march}/rk_march.hpp (99%) rename native/include/solprop_native/{ => march}/stage_tables.hpp (99%) rename native/include/solprop_native/{ => march}/z_table.hpp (98%) rename native/include/solprop_native/{ => problem}/problem.hpp (99%) rename native/include/solprop_native/{ => problem}/run_records.hpp (97%) rename native/include/solprop_native/{ => problem}/workspace.hpp (96%) rename native/include/solprop_native/{ => response}/coupled_flow.hpp (99%) rename native/include/solprop_native/{ => response}/dispersive.hpp (99%) rename native/include/solprop_native/{ => response}/envelope_plasma.hpp (100%) rename native/include/solprop_native/{ => response}/far_field.hpp (99%) rename native/include/solprop_native/{ => response}/flow_functions.hpp (98%) rename native/include/solprop_native/{ => response}/general_modal.hpp (97%) rename native/include/solprop_native/{ => response}/plasma.hpp (99%) rename native/include/solprop_native/{ => runtime}/build.hpp (100%) rename native/include/solprop_native/{ => runtime}/cpu_execution.hpp (99%) rename native/include/solprop_native/{ => runtime}/crossing.hpp (100%) rename native/include/solprop_native/{ => runtime}/dispatch.hpp (100%) rename native/include/solprop_native/{ => runtime}/execution.hpp (98%) rename native/include/solprop_native/{ => runtime}/fields.hpp (97%) rename native/include/solprop_native/{ => runtime}/fp.hpp (100%) rename native/include/solprop_native/{ => runtime}/graph.hpp (100%) rename native/include/solprop_native/{ => runtime}/host_source.hpp (100%) rename native/include/solprop_native/{ => runtime}/launch.hpp (99%) rename native/include/solprop_native/{ => runtime}/refuse.hpp (100%) rename native/include/solprop_native/{ => runtime}/setup_data.hpp (96%) rename native/include/solprop_native/{ => runtime}/vmath.hpp (100%) rename native/include/solprop_native/{ => session}/readings.hpp (99%) rename native/include/solprop_native/{ => session}/session.hpp (98%) rename native/include/solprop_native/{ => session}/session_dense.hpp (98%) rename native/include/solprop_native/{ => session}/session_resident.hpp (97%) rename native/include/solprop_native/{ => session}/statistics_plan.hpp (100%) rename native/include/solprop_native/{ => session}/transverse.hpp (99%) rename native/include/solprop_native/{ => session}/watch.hpp (99%) rename native/include/solprop_native/{ => transforms}/fft_plan.hpp (98%) rename native/include/solprop_native/{ => transforms}/fftw_wisdom.hpp (98%) rename native/include/solprop_native/{ => transforms}/gemm.hpp (99%) rename native/include/solprop_native/{ => transforms}/raman_fft.hpp (98%) rename native/include/solprop_native/{ => transforms}/reference_timing.hpp (100%) rename native/include/solprop_native/{ => transforms}/slab.hpp (96%) rename native/src/{ => bindings}/bindings.cpp (99%) rename native/src/{ => bindings}/bindings_helpers.cpp (97%) rename native/src/{ => bindings}/bindings_helpers.hpp (100%) rename native/src/{ => bindings}/bindings_transverse.cpp (100%) rename native/src/{ => bindings}/bindings_transverse.hpp (96%) rename native/src/{ => bindings}/records.cpp (100%) rename native/src/{ => bindings}/records.hpp (99%) rename native/src/{ => march}/adams_march.cpp (99%) rename native/src/{ => march}/dense.cpp (97%) rename native/src/{ => march}/engines.cpp (99%) rename native/src/{ => march}/low_storage.cpp (99%) rename native/src/{ => march}/rk_march.cpp (99%) rename native/src/{ => problem}/problem.cpp (97%) rename native/src/{ => problem}/workspace.cpp (99%) rename native/src/{ => response}/coupled_flow.cpp (99%) rename native/src/{ => response}/dispersive.cpp (99%) rename native/src/{ => response}/far_field.cpp (99%) rename native/src/{ => response}/general_modal.cpp (99%) rename native/src/{ => runtime}/coverage_labels.cpp (100%) rename native/src/{ => runtime}/cpu_execution.cpp (99%) rename native/src/{ => runtime}/crossing.cpp (98%) rename native/src/{ => session}/session.cpp (99%) rename native/src/{ => session}/session_edge.cpp (98%) rename native/src/{ => session}/session_widths.cpp (99%) rename native/src/{ => session}/statistics_plan.cpp (99%) rename native/src/{ => session}/transverse.cpp (99%) rename native/src/{ => session}/watch.cpp (99%) rename native/src/{ => transforms}/fftw_wisdom.cpp (99%) rename native/src/{ => transforms}/raman_fft.cpp (99%) rename native/src/{ => transforms}/reference_timing.cpp (95%) diff --git a/docs/dev/backends.md b/docs/dev/backends.md index eba9747ff..9a5788922 100644 --- a/docs/dev/backends.md +++ b/docs/dev/backends.md @@ -149,7 +149,7 @@ the bundle's floats and nothing about the representation reaches the backend split, so `PropagationProblem.device_ready` is false for it. NumPy evaluates the general representation directly. The native CPU build gives the table the split itself (SEP 29 section 6.6): its coefficients cross once, in the workspace's `Problem` (its `z_tables`, uploaded where the workspace is -built; `include/solprop_native/z_table.hpp`), and each trial has the table evaluated at +built; `include/solprop_native/march/z_table.hpp`), and each trial has the table evaluated at its distinct positions into term slots of the state's size (`stage::ZSlots` in `stage_tables.hpp`, a strict Clenshaw pass per position), least recently used first so that a step's end is not evaluated again as the next step's anchor; no position crosses @@ -175,7 +175,7 @@ backend sets the class attribute `helpers` (kernel schema 34), a Adams march and its engines pass it down. `numpy_ref` takes `PLAIN`, the uncompiled Python forms, so the oracle never runs the compiled code it checks; `native` takes its own `NATIVE`, the C++ transcription of the Chebyshev rows -(`native/include/solprop_native/chebyshev.hpp`), which its adapter still evaluates for +(`native/include/solprop_native/march/chebyshev.hpp`), which its adapter still evaluates for the stage scalars at a plane. Both perform the same operations in the same order under strict arithmetic, so the rows a table keeps for its last positions are shared whichever filled them. `NATIVE` names no Adams basis: the compiled Adams march forms its basis and @@ -211,8 +211,8 @@ operation (SEP 52 §5.1). The backend builds a fresh march and engines for each | `HostAdamsMarch.march_to`, in `backends/host/adams.py` | adaptive Adams | an `AdamsEngine`: `HostHistory` | | `HostAdamsMarch._replay_to` | Adams on a fixed schedule | the same engine | | `HostLowStorageMarch.replay_to`, in `backends/host/low_storage.py` | low-storage, always on a schedule | a `RungeKuttaEngine` with a destructive trial: `HostLowStorage` | -| the native session's Adams march, `native/src/adams_march.cpp` | Adams, adaptive and replayed, compiled (ADR-060) | the resident engine (`engines::ResidentEngine`, a `march::Engine`), the march's state from the run record (`_NativeAdamsMarch.start_state`) | -| the native session's Runge-Kutta march, `native/src/rk_march.cpp` | Runge-Kutta, adaptive and replayed, compiled (ADR-069, [SEP 45](seps/archive/sep-0045-native-rk-march.md)) | the staged trial and native norm | +| the native session's Adams march, `native/src/march/adams_march.cpp` | Adams, adaptive and replayed, compiled (ADR-060) | the resident engine (`engines::ResidentEngine`, a `march::Engine`), the march's state from the run record (`_NativeAdamsMarch.start_state`) | +| the native session's Runge-Kutta march, `native/src/march/rk_march.cpp` | Runge-Kutta, adaptive and replayed, compiled (ADR-069, [SEP 45](seps/archive/sep-0045-native-rk-march.md)) | the staged trial and native norm | | the native session's low-storage replay | low-storage, always on a schedule | one in-place native step a schedule step | The reference's Runge-Kutta march is the reference for the compiled Runge-Kutta march, @@ -579,7 +579,7 @@ A backend that does not set it reports `unwatched` and refuses a requested colla **Under the native execution session the judgement is native too** ([SEP 47](seps/archive/sep-0047-run-ownership.md) §6.4). `NativeSession` hands no reading to a Python edge: `transverse::Edge` -(`native/include/solprop_native/transverse.hpp`, a strict translation unit) owns the edge's every decision and record +(`native/include/solprop_native/session/transverse.hpp`, a strict translation unit) owns the edge's every decision and record -- the launch's judgements, the spacing and its ceiling, each read's ledgers and verdict, a collar's current and return, the resume and the record -- and the session reads the resident state for it (`Resident::edge_reading`, the same reduction as `NumpyWorkspace.edge_reading`). The march's `read_at`/`read_spacing` bounds return it to the session, not to Python, where @@ -645,7 +645,7 @@ session `native/src/solprop_native/session.py` and the engine under native backend, with no fallback (§1, ADR-029). `numpy_ref` stays selectable by name. -**Three response families** (`FieldFamily`, `include/solprop_native/problem.hpp`), which is +**Three response families** (`FieldFamily`, `include/solprop_native/problem/problem.hpp`), which is all `kerr_coefficients` compiles to, each its own compiled column: | family | grid | `thg` | the column works on | plasma | @@ -747,7 +747,7 @@ mask and the in-stage absorber preserve it, and the measured saving are `build_info().optimization_policy` records the flag. - **Strict blocks.** A protected function or lambda carries `SOLPROP_NATIVE_STRICT_FUNCTION` and starts its body with `SOLPROP_NATIVE_STRICT_BLOCK` - (`include/solprop_native/fp.hpp`): under Clang a `#pragma clang fp + (`include/solprop_native/runtime/fp.hpp`): under Clang a `#pragma clang fp contract(off)`, which survives inlining; under GCC, which ignores the pragma, `optimize("fp-contract=off")` on the function, so a protected helper is a call there. Both settings are lexical, and `Kokkos::complex`'s operators keep their header's @@ -841,7 +841,7 @@ build (ADR-076, §4). | compact problem | one column on one node, fewer than 65 536 fine samples | kernels on a team of at most eight threads, transforms on that team, products on one BLAS thread | independent scans: Local, or spawned processes through Pool | | sized problem | the rest | each operation's width chosen from its own element count at construction: stage, response, dense product, FFT, Raman, copy, flow, modal and Adams | one process per problem | -The widths are constants of `include/solprop_native/cpu_execution.hpp` (`operation_threads`, with +The widths are constants of `include/solprop_native/runtime/cpu_execution.hpp` (`operation_threads`, with their calibration in [benchmarks.md](benchmarks.md#kokkos-cpu-operation-size-threading)), and `SOLPROP_NATIVE_*_THREADS` overrides a pool ([Performance tuning](../guide/performance/tuning.md)). The stage boundary @@ -913,7 +913,7 @@ wash with the Raman convolution. The native backend's host kernels form these exponentials, the exponential Adams method's flows, phi-function exponentials and compensation factors, and the plasma's rates and fractions in passes over blocks of elements through one vector-maths layer -(`native/include/solprop_native/vmath.hpp`): SLEEF's 1.0 ulp vector functions on x86-64 +(`native/include/solprop_native/runtime/vmath.hpp`): SLEEF's 1.0 ulp vector functions on x86-64 builds, the system libm, with each element's own calls, on Arm and on devices ([ADR-059](decisions.md)). @@ -957,7 +957,7 @@ admits (`solprop.translate.admission.check_general_route`), which it runs where built, so a refusal there is a refusal on both backends, raised once. It adds one limit of its own: a delayed response must be the six channels of the silica tensor Raman model on three components (§4, `_refuse_native_route`). `linear.coupled_flow` is §3.9. The route is a **second execution -path** (`src/general_modal.cpp`, `GeneralModalSource`) beside the fused pipeline and not an +path** (`src/response/general_modal.cpp`, `GeneralModalSource`) beside the fused pipeline and not an extension of it, for the reason §3.1a's three families make plain: those are one real transverse matrix in the time domain, and this route's synthesis is a complex frequency-dependent map of three Cartesian components. @@ -1055,7 +1055,7 @@ name. and on either route (SEP 16 phase 6, SEP 47 phase 4): a bin on the eigenbasis route takes its flow and moments `k! phi_(k+1)` as `W f(Delta Phi + h lambda) Y`, and a bin off it reads them, as the reference does, off the first block row of the exponential of the augmented matrix of - `X = Delta Phi I + h T` (`include/solprop_native/flow_functions.hpp`): the reference's + `X = Delta Phi I + h T` (`include/solprop_native/response/flow_functions.hpp`): the reference's scaling, sixteen-term series and squaring, where the reference takes SciPy's Padé `expm`, a rounding-level difference (SEP 25), with the shift's exponential kept apart in the carry of a history sample. Every block row past the first is a scaled shift, so the passes form the first @@ -1092,7 +1092,7 @@ modified noise draw, and every statistic. unnormalised in-place transform over the y axis. The rows go to time a **block of whole y rows** at a time, as many as the node chunk holds (the reference's block), so `n_p n_y` rows are transformed each way, never `n_x n_y` nodes. The x synthesis and projection are team kernels - (`slab_contract`, `include/solprop_native/slab.hpp`) with each sum in increasing order and the + (`slab_contract`, `include/solprop_native/transforms/slab.hpp`) with each sum in increasing order and the vector loop over contiguous samples, and SEP 6's dispersive response shares them. - **The operator** is evaluated per element and never tabulated under Runge–Kutta (the contract refuses `propagator_mode = "tabulate"` for the kind outside the low-storage march). It subtracts @@ -1120,7 +1120,7 @@ modified noise draw, and every statistic. (mode-averaged, multimode and radial), Cartesian free space and the line, the hybrid planar route and the general modal route -- on field and envelope grids, with both evaluators, both sectors, one, two or three components, mixtures, beside Raman, plasma -and envelope plasma (`src/dispersive.cpp`). The operator is the reference's +and envelope plasma (`src/response/dispersive.cpp`). The operator is the reference's `G[w (G E)^3]`: the temporal collar on the products, before `Π₊` and the output multipliers. Everywhere the multipliers move off the nodes, to wherever the route already holds a spectrum: @@ -1208,7 +1208,7 @@ column's population and free-current integrands with the problem's $2K+1$ taps b sum, reading $K$ samples ahead, so the fused one-sample traversal cannot carry it. - **The team column** (`GeneralWorkspace::team_band_plasma_column`, described in - `native/include/solprop_native/plasma.hpp`). The plasma runs in whole-row passes per node: the + `native/include/solprop_native/response/plasma.hpp`). The plasma runs in whole-row passes per node: the rate; the filtered rate (`team_band_filter`), then the exposure's scan (`team_trapezoid_window`), then pointwise the fraction, the loss guard and the weight $f\,q/m$; per component the free-current integrand, its filter, then its scan with the current formed from the prefix. The @@ -1252,7 +1252,7 @@ The band rule's cost against the trapezoid's is in [benchmarks.md](benchmarks.md#band-rule-plasma-integration). **native device builds** run a different column, written down in -`native/include/solprop_native/plasma.hpp`: parallel filters and scans form the exposure and the free current, with the +`native/include/solprop_native/response/plasma.hpp`: parallel filters and scans form the exposure and the free current, with the pointwise population and loss-current steps between, on both the team and the tiled device paths. The stencil keeps the reference's summation order, so it is deterministic and independent of the team size; the scan associates differently from the serial sum, so @@ -1266,7 +1266,7 @@ build defaults to its own: direct reads on CUDA, staging on HIP [SEP 23](seps/archive/sep-0023-far-field-aperture-spectra.md)'s six far-field statistics and `on_axis_spectrum` are one native pass over the stored state on the Cartesian and line kinds -(`GeneralState::aperture_statistics`, `native/include/solprop_native/far_field.hpp`, +(`GeneralState::aperture_statistics`, `native/include/solprop_native/response/far_field.hpp`, `FarFieldPass`), in place of the shared host pass `spatial.aperture_statistics` that `numpy_ref` keeps as the oracle. The pass has its own flag mask beside `spatial_statistics` (values from 32: the six names and `on_axis_spectrum`, the `W = 1` slot, which on the Cartesian and line kinds @@ -1405,7 +1405,7 @@ the edge's tables. The timing probe is `Workspace.probe(start)`, and `diagnostic one record of metadata, memory, sizing, planning and crossed bytes, beside the live `counters()` a run reads at its start and close. Nothing configures a workspace after it is built. Each C++ record that crosses declares its fields once, beside its definition -(`include/solprop_native/fields.hpp`), and one converter (`src/records.hpp`) reads a +(`include/solprop_native/runtime/fields.hpp`), and one converter (`src/bindings/records.hpp`) reads a dataclass, mapping or tuple into it by that table, refusing a missing or unknown field, a wrong dtype, rank or extent relation, a strided array and an integer for an enumeration by the field's path before anything is allocated; `_core.schemas()` exports the tables and a @@ -1451,7 +1451,7 @@ use separate FFT plans for the final partial chunk, aliased onto existing storag Kokkos-FFT plans at `FFTW_ESTIMATE` and consumes measured wisdom in the linked FFTW library, which the workspace seeds before it makes its first plan: a plan made before its wisdom stays at estimate for the run. One policy (`fftw_seed_effort`, -`include/solprop_native/fftw_wisdom.hpp`) decides every seed by the transform's shape and +`include/solprop_native/transforms/fftw_wisdom.hpp`) decides every seed by the transform's shape and width. Unset, row transforms (temporal, modal, Raman and plasma) are measured on one thread and left at estimate when threaded, because measured wisdom can select slower threaded plans; free space's transverse planes are measured at any width. An explicit @@ -1587,7 +1587,7 @@ response column's `if constexpr` blocks carry a capture anchor, because CUDA 13' lambdas otherwise refuse the capture. Every top-level kernel dispatches through `launch::parallel_for`, `parallel_reduce` and -`parallel_scan` (`native/include/solprop_native/launch.hpp`). On a device they hint a +`parallel_scan` (`native/include/solprop_native/runtime/launch.hpp`). On a device they hint a functor of at most 3840 B light-weight, which Kokkos passes as the kernel's argument; Kokkos's HIP default copied any functor of 512 B or more into constant memory, a blit kernel behind a wait on the previous such launch, which kept the host from queuing a @@ -1610,7 +1610,7 @@ execution instances are its one stream. A device holds the two held plasma curre fine and coarse rows in one buffer that one plan transforms. On a device an elementwise Adams trial's source evaluation runs in a graph region -(`graph::Region`, `native/include/solprop_native/graph.hpp`). The region's launches, its +(`graph::Region`, `native/include/solprop_native/runtime/graph.hpp`). The region's launches, its device-to-device copies and its transforms go through `launch::parallel_for`, `graph::deep_copy` and `graph::execute`, which key each operation by its kind and the value bytes of its policy and functor, or of its views, plan and scalars. The first trial @@ -1659,7 +1659,7 @@ prefactors, before field-sized allocation (the CPU build runs them, §1). Suppor cross as stage scalars; z dependence alone is not a refusal. **The general modal route** is a second execution path beside the node pipeline -(`src/general_modal.cpp`, `GeneralModalSource`), exactly as it is one in `numpy_ref` +(`src/response/general_modal.cpp`, `GeneralModalSource`), exactly as it is one in `numpy_ref` (§2). Per node chunk: the coarse-grid synthesis of the three Cartesian components from the complex map `S`, placed into a zeroed fine spectrum; one complex inverse transform to the analytic signal; the rank-four contraction on the real field or on the analytic @@ -1693,7 +1693,7 @@ terms summed in order from zero, `GeneralModalSource::combine`) into the two map synthesis and projection read; nothing is allocated in a trial, and the ledger adds the weights (`general_map_weights`). -**The coupled flow** (`src/coupled_flow.cpp`, `CoupledFlowAction`) executes the +**The coupled flow** (`src/response/coupled_flow.cpp`, `CoupledFlowAction`) executes the constant-Schur and frozen-residual sections, every section the translation builds. The adapter uploads each section's eigenbasis (`solprop.translate.coupled.schur_eigenbasis`: `W`, `Y`, the eigenvalues and the route per @@ -1721,7 +1721,7 @@ subtracts its channel's `kappa_sq` last. Ten fixed steps agree with `numpy_ref` 1.2e-14 to 6.6e-14 against bounds of 2.5e-11 and more, and thread counts agree within those bounds. -**The dispersive electronic response** (`src/dispersive.cpp`, `DispersiveResponse`; +**The dispersive electronic response** (`src/response/dispersive.cpp`, `DispersiveResponse`; [SEP 6](seps/archive/sep-0006-dispersive-nonlinearity.md)) runs on every route `numpy_ref` runs it, with the structure of §3.11 route by route: the fused node path (one or two components), `SeparableDispersive` on the Cartesian free-space, line and @@ -1779,7 +1779,7 @@ at most: three and four threads and radix 8 with 32-point leaves at others), which moves ten realistic steps by at most 8 u: within the transform's rounding bound, so it constrains nothing. This threshold and the stage threshold switch only on `__APPLE__` - (`include/solprop_native/cpu_execution.hpp`): Windows and linux-aarch64 builds take the values + (`include/solprop_native/runtime/cpu_execution.hpp`): Windows and linux-aarch64 builds take the values measured on x86-64 Linux (Cascade Lake and Zen 3), which have not been measured on those platforms ([#328](https://github.com/LupoLab/solprop/issues/328)). - **One stage phase.** `sum_t dPhi_t Phi_t` is needed three times per stage (the diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index 0d6ae2bfc..80b31dc74 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -1473,7 +1473,7 @@ grows as an attempt gets cheaper, and it is above 0.1 % on the two smallest grid ### The Adams passes on x86-64 lanes The exponential Adams trial's predict and correct passes on a host block of 32 elements -held as lanes (`include/solprop_native/adams_phi.hpp`, `adams_lanes_*` in +held as lanes (`include/solprop_native/march/adams_phi.hpp`, `adams_lanes_*` in `src/general/trial.cpp`) against the element-by-element form, at 740b24d. Before the change the passes ran each element through latency-bound scalar strict code (the phi-functions' series, Horner pass and squarings, the compensation and the defects), diff --git a/docs/dev/decisions.md b/docs/dev/decisions.md index 18112333c..d21e93593 100644 --- a/docs/dev/decisions.md +++ b/docs/dev/decisions.md @@ -1116,7 +1116,7 @@ Withdrawn (development-process record, not a design decision). qualified to their rounding budget and to SEP 16's work–precision cases. - **Amendment, device builds:** the protected helpers are strict on CPU builds only. CUDA and HIP builds leave the strict blocks empty (`SOLPROP_NATIVE_STRICT_FUNCTION` - and `SOLPROP_NATIVE_STRICT_BLOCK` in `native/include/solprop_native/fp.hpp`, under + and `SOLPROP_NATIVE_STRICT_BLOCK` in `native/include/solprop_native/runtime/fp.hpp`, under `__CUDACC__` or `__HIPCC__`, which includes nvcc's host pass) and follow their compilers' contraction defaults; `build_info().optimization_policy` records "compiler-default contraction". On a device the φ series, the phase sums, the @@ -1575,7 +1575,7 @@ Withdrawn (development-process record, not a design decision). - Apple M5 Max: the system libm 3.25 ns; SLEEF's AdvSIMD functions gain nothing, and Accelerate's vForce 1.3 times at 1.7 ulp. - **Decision:** - - One private layer, `native/include/solprop_native/vmath.hpp`, gives the host kernels + - One private layer, `native/include/solprop_native/runtime/vmath.hpp`, gives the host kernels batch `exp`, `log`, `sincos` and the complex exponential over contiguous arrays. A kernel forms a block's arguments into a buffer, calls the layer once over it, and finishes each element from the results. Converted: the Runge–Kutta stage entry, the @@ -1639,7 +1639,7 @@ Withdrawn (development-process record, not a design decision). contract's section 11 specifies them in full. - `native` executes `Multistep()` in either formulation, the one-order-higher PEC comparisons and replayed schedules on its resident engine for a run of attempts - per call (`native/src/adams_march.cpp`, compiled without contraction + per call (`native/src/march/adams_march.cpp`, compiled without contraction or fast-math), returning only at a host event: a reached target, an accepted step on which the driver has work, the monitor, the call's bound (256 attempts or 0.1 s), the budget, the floor or a native error. solprop books the call's attempts diff --git a/docs/dev/dependencies.md b/docs/dev/dependencies.md index c921ae2af..0630eb1c3 100644 --- a/docs/dev/dependencies.md +++ b/docs/dev/dependencies.md @@ -75,7 +75,7 @@ solprop-native has no runtime dependency on solprop. **SLEEF 3.9.0** ([sleef.org](https://sleef.org), Boost Software License 1.0) is the native CPU build's host vector maths on x86-64 (ADR-059): vector `exp`, `log` and `sincos`, its u10 functions (1.0 ulp over each function's domain), called over blocks -by `native/include/solprop_native/vmath.hpp`. It is fetched at its release tag like +by `native/include/solprop_native/runtime/vmath.hpp`. It is fetched at its release tag like Kokkos (`FETCHCONTENT_SOURCE_DIR_SLEEF` names a local copy for an offline build), built static as its own CMake project with only `libsleef` and the variants the layer calls (SSE2, AVX2, AVX-512F on request; AdvSIMD on aarch64), and compiled into the extension, whose wheel diff --git a/docs/dev/kernel-contract.md b/docs/dev/kernel-contract.md index 124891014..e8dbdad1c 100644 --- a/docs/dev/kernel-contract.md +++ b/docs/dev/kernel-contract.md @@ -1341,7 +1341,7 @@ stripped extension labels only exported functions). The boundary checks above an ## 12. The Runge–Kutta march *Normative from [SEP 45](seps/archive/sep-0045-native-rk-march.md) phase 3. `native` executes it -(`native/src/rk_march.cpp`), adaptive or replayed, with `PIController`'s norm, weighted or +(`native/src/march/rk_march.cpp`), adaptive or replayed, with `PIController`'s norm, weighted or not, on CPU and device builds. `PIController` is final (ADR-077), so its decisions are the only ones either backend takes.* diff --git a/docs/dev/stepper.md b/docs/dev/stepper.md index 3e07cf99b..7ecbbc299 100644 --- a/docs/dev/stepper.md +++ b/docs/dev/stepper.md @@ -281,7 +281,7 @@ closes the workspace afterwards, SEP 52 §5.5). The native execution session ([S `solprop_native.session`) owns the loop itself: the native Runge-Kutta march replays the schedule, and each step is one in-place native step whose stage scalars, the table keys of `table_steps` and the flows' increments, the native state forms from its stage -tables (`native/src/low_storage.cpp`), with no return to Python; a failure partway +tables (`native/src/march/low_storage.cpp`), with no return to Python; a failure partway through a step leaves no consistent state, so the session's terminal event is poisoned and claims no checkpoint. There is no estimate: the history's norm is 0.0, "not estimated". The step is **in flight** from its start to its commit (the engine's `in_flight`, which the march @@ -404,13 +404,13 @@ frozen-residual flow's generator applied once per sample. **The native Runge–Kutta march** (ADR-069, [SEP 45](seps/archive/sep-0045-native-rk-march.md)): `RungeKuttaMarch.march_to`'s decisions are pure functions of a `RungeKuttaPolicy` (`backends/rk_core.py`), held to a decision corpus recorded before the refactor -(`tests/data/rk_traces.json.gz`), and `native` executes them (`native/src/rk_march.cpp`) on +(`tests/data/rk_traces.json.gz`), and `native` executes them (`native/src/march/rk_march.cpp`) on its staged trial and native norm, a call of attempts at a time, returning only at a host event and taking the boundary checks itself; the kernel contract's section 12 is the interface. It covers interpolated planes (the step across a plane returns to solprop, which holds its extension; in SEP 47's native session the march holds it through its engine, `rk::Engine::extend`, finishes the step itself and the session observes the plane where the -state lives, `native/src/dense.cpp`), a replayed schedule, a coupled flow (its guards transcribed), a z table +state lives, `native/src/march/dense.cpp`), a replayed schedule, a coupled flow (its guards transcribed), a z table the transverse edge (read on the host, as the Adams march reads it) and a weighted norm (reduced natively in both loops), on CPU and device builds alike. `PIController` is final (ADR-077): defining a subclass raises `StepperError`, so its decisions and its norm are the @@ -424,7 +424,7 @@ record, `MarchPolicy`, formed by `march_policy` from the method and the settings the decisions are pure functions of the policy, the march's state and a trial's norms (`backends/adams_core.py`). The kernel contract's section 11 specifies them. `native` executes `Multistep()` in either formulation and replayed schedules from that record on its resident engine -(`native/src/adams_march.cpp`), a run of attempts per call, and returns to solprop +(`native/src/march/adams_march.cpp`), a run of attempts per call, and returns to solprop only at an event: the target reached or crossed, an accepted step on which the driver has work (a statistics row, a natural stop), a boundary check whose verdict needs solprop, the call's bound, the budget, the floor or a native error. The march takes the diff --git a/docs/dev/tooling.md b/docs/dev/tooling.md index 04e8a9f43..d754f67b5 100644 --- a/docs/dev/tooling.md +++ b/docs/dev/tooling.md @@ -243,7 +243,7 @@ to it. - `SOLPROP_THREADS`, the thread budget the native backend's pools stay within, and `SOLPROP_OVERSUBSCRIBE`, which lets an explicit count exceed the CPUs (ADR-052). - The native backend's plans and neighbours: `SOLPROP_FFTW_PLAN` (`estimate` in CI and - tests, `measure` by default, read by `native/src/fftw_wisdom.cpp`), + tests, `measure` by default, read by `native/src/transforms/fftw_wisdom.cpp`), `SOLPROP_FFTW_WISDOM` (a file name, beside which the native backend keeps the plans its runs measure instead of the calibration store, ADR-076) and `SOLPROP_BLAS_THREADS` (the width of the other BLAS libraries while a native diff --git a/docs/guide/performance/building-native.md b/docs/guide/performance/building-native.md index f6c72311a..5dd854876 100644 --- a/docs/guide/performance/building-native.md +++ b/docs/guide/performance/building-native.md @@ -427,7 +427,7 @@ access, point it at source trees of the same versions: The CPU kernels form their transcendentals (the stage and step propagators, the exponential Adams method's flows, phi-functions and compensation, the ionisation rates and fractions) in passes over blocks of elements through one layer, -`native/include/solprop_native/vmath.hpp` ([ADR-059](../../dev/decisions.md)). On x86-64 +`native/include/solprop_native/runtime/vmath.hpp` ([ADR-059](../../dev/decisions.md)). On x86-64 it calls [SLEEF](https://sleef.org)'s vector functions, each within 1.0 ulp of the correctly rounded value over its whole domain: AVX2 where the build's target has it (the wheels' x86-64-v3 baseline and any local build on a processor from 2013 or later), SSE2 diff --git a/native/AGENTS.md b/native/AGENTS.md index 2e366e5a8..a58ed900a 100644 --- a/native/AGENTS.md +++ b/native/AGENTS.md @@ -124,12 +124,12 @@ the gate. - Use RAII, Kokkos views, `Kokkos::complex`, device-callable math, named kernels and allocations (`solprop_native.*` labels), explicit `LayoutRight` and explicit execution-space instances. Dispatch top-level kernels through `launch::parallel_for`, - `parallel_reduce` and `parallel_scan` (`include/solprop_native/launch.hpp`), which pass + `parallel_reduce` and `parallel_scan` (`include/solprop_native/runtime/launch.hpp`), which pass a device kernel's functor as its launch argument; team-level loops stay `Kokkos::`. - Allocate field-sized views, scratch and FFT plans once, when a workspace is built. A trial step performs no allocation, planning, logging, Python call or unplanned fence. The one exception is the host source hook (solprop SEP 49, - `include/solprop_native/host_source.hpp`): on a CPU build, a bound host source is + `include/solprop_native/runtime/host_source.hpp`): on a CPU build, a bound host source is called from the thread that called `advance`, outside every parallel region, and the binding layer takes the interpreter lock for each call; device builds refuse it. - Keep data resident in the execution space until it is observed; `field_to_host` is @@ -150,7 +150,7 @@ the gate. `fast` makes Clang ignore the pragmas). A protected block (a stable increment, a small-argument function, a defect or a compensated sum) stays strict with `SOLPROP_NATIVE_STRICT_FUNCTION` and `SOLPROP_NATIVE_STRICT_BLOCK` - (`include/solprop_native/fp.hpp`; GCC ignores `#pragma STDC FP_CONTRACT`) and spells + (`include/solprop_native/runtime/fp.hpp`; GCC ignores `#pragma STDC FP_CONTRACT`) and spells its complex arithmetic with the `strict_*` helpers, since `Kokkos::complex`'s operators keep their header's contraction when inlined. Reassociation is scoped to the loops that need it. No umbrella fast-math, `nnan`/`ninf`, `nsz`, approximate @@ -167,7 +167,7 @@ the gate. CUDA and HIP builds leave the strict blocks empty and follow their compilers' contraction defaults, held to the measured device envelopes (solprop's ADR-049, amended) until device round-to-nearest helpers land. -- A host kernel's hot transcendentals go through `include/solprop_native/vmath.hpp` +- A host kernel's hot transcendentals go through `include/solprop_native/runtime/vmath.hpp` (solprop's ADR-059): form a block's arguments into a buffer, call the batch function once over it, and finish each element from the results, behind `if constexpr (vmath::vectorised)` where the per-element form is faster with the @@ -194,8 +194,8 @@ the gate. compute it, and stay in step). It records solprop's version without comparing it; never pin solprop's version. - A record that crosses declares its fields once, in a field table beside its C++ - definition (`include/solprop_native/fields.hpp`), and is read by the one converter - (`src/records.hpp`), never by a hand-written dictionary reader; `_core.schemas()` + definition (`include/solprop_native/runtime/fields.hpp`), and is read by the one converter + (`src/bindings/records.hpp`), never by a hand-written dictionary reader; `_core.schemas()` exports the tables, which a test holds to the Python records. A binding only converts, releases the GIL and forwards: logic goes in the C++ library, where ctest reaches it. diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 4cea4518e..4016e6d5e 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -95,7 +95,7 @@ elseif(SOLPROP_NATIVE_ENABLE_CUDA AND elseif(NOT SOLPROP_NATIVE_ENABLE_CUDA AND CMAKE_CXX_COMPILER_ID STREQUAL "GNU" AND CMAKE_CXX_COMPILER_VERSION VERSION_LESS 14) - # GCC 13 compiles the out-of-line strict helpers (include/solprop_native/fp.hpp) + # GCC 13 compiles the out-of-line strict helpers (include/solprop_native/runtime/fp.hpp) # through the stack, stalling every call on a failed store-to-load forward: coupled # Multistep steps took 1.25 to 1.45 times as long as with GCC 14 (Cascade Lake, # Zen 5). @@ -119,7 +119,7 @@ if(NOT APPLE AND NOT SOLPROP_NATIVE_ENABLE_CUDA AND endif() # solprop SEP 25: floating-point contraction (fused multiply-add) is on by default for -# CPU builds, and the few protected blocks opt out (include/solprop_native/fp.hpp). +# CPU builds, and the few protected blocks opt out (include/solprop_native/runtime/fp.hpp). # Clang's plain `-ffp-contract=fast` ignores `#pragma clang fp contract(off)` and # `#pragma STDC FP_CONTRACT OFF`; `fast-honor-pragmas` contracts across statements and # keeps the protected blocks strict. GCC ignores both pragmas: its protected functions @@ -274,7 +274,7 @@ set(SOLPROP_NATIVE_DEVICE_TEAM_SIZE "256" CACHE STRING "Maximum accelerator response and scan team width") set(SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE "512" CACHE STRING "Fixed accelerator deterministic reduction block width") -# solprop SEP 18's band rule (include/solprop_native/plasma.hpp). The staging changes +# solprop SEP 18's band rule (include/solprop_native/response/plasma.hpp). The staging changes # only how a device team reads a filter's halo, through the same arithmetic (Q8). Its # default is the faster read measured on each vendor's device: on an MI325X (HIP) # staging the row in team scratch, 1.25 times the trapezoid's RDW window against 1.36 @@ -800,7 +800,7 @@ endfunction() if(SOLPROP_NATIVE_ENABLE_COVERAGE) # Keep label collection in a separate, uninstrumented Tools module. The runner # loads it only for a separate execution-label pass, never a timing sample. - add_library(solprop_native_labels MODULE src/coverage_labels.cpp) + add_library(solprop_native_labels MODULE src/runtime/coverage_labels.cpp) target_include_directories(solprop_native_labels SYSTEM PRIVATE "${kokkos_SOURCE_DIR}/core/src") target_compile_options(solprop_native_labels PRIVATE @@ -810,13 +810,13 @@ if(SOLPROP_NATIVE_ENABLE_COVERAGE) install(TARGETS solprop_native_labels LIBRARY DESTINATION solprop_native) endif() -add_library(solprop_native_core STATIC src/cpu_execution.cpp src/crossing.cpp +add_library(solprop_native_core STATIC src/runtime/cpu_execution.cpp src/runtime/crossing.cpp src/general/checks.cpp src/general/setup.cpp src/general/trial.cpp - src/general/statistics.cpp src/general/state.cpp src/general/memory.cpp src/general_modal.cpp src/coupled_flow.cpp src/dispersive.cpp src/far_field.cpp - src/adams_march.cpp src/watch.cpp src/rk_march.cpp src/engines.cpp src/session.cpp - src/statistics_plan.cpp src/session_widths.cpp src/low_storage.cpp src/transverse.cpp - src/session_edge.cpp src/dense.cpp src/workspace.cpp src/problem.cpp - src/reference_timing.cpp src/fftw_wisdom.cpp) + src/general/statistics.cpp src/general/state.cpp src/general/memory.cpp src/response/general_modal.cpp src/response/coupled_flow.cpp src/response/dispersive.cpp src/response/far_field.cpp + src/march/adams_march.cpp src/session/watch.cpp src/march/rk_march.cpp src/march/engines.cpp src/session/session.cpp + src/session/statistics_plan.cpp src/session/session_widths.cpp src/march/low_storage.cpp src/session/transverse.cpp + src/session/session_edge.cpp src/march/dense.cpp src/problem/workspace.cpp src/problem/problem.cpp + src/transforms/reference_timing.cpp src/transforms/fftw_wisdom.cpp) # solprop SEP 39, SEP 45 and SEP 47: the Adams march's decisions, the boundary watch's # predictions, the Runge-Kutta march's positions and norm, the low-storage replay's # positions and table keys, the native session's scheduling, export, statistics @@ -826,9 +826,9 @@ add_library(solprop_native_core STATIC src/cpu_execution.cpp src/crossing.cpp # and the last wins). MSVC's /fp:precise does not contract by default. # SOLPROP_NATIVE_STRICT_TU marks them, so that the general workspace's contracting # headers (src/general/) refuse to be included there. -set(SOLPROP_NATIVE_STRICT_SOURCES src/adams_march.cpp src/watch.cpp src/rk_march.cpp - src/session.cpp src/statistics_plan.cpp src/session_widths.cpp src/low_storage.cpp - src/transverse.cpp src/session_edge.cpp src/dense.cpp) +set(SOLPROP_NATIVE_STRICT_SOURCES src/march/adams_march.cpp src/session/watch.cpp src/march/rk_march.cpp + src/session/session.cpp src/session/statistics_plan.cpp src/session/session_widths.cpp src/march/low_storage.cpp + src/session/transverse.cpp src/session/session_edge.cpp src/march/dense.cpp) set_source_files_properties(${SOLPROP_NATIVE_STRICT_SOURCES} PROPERTIES COMPILE_DEFINITIONS SOLPROP_NATIVE_STRICT_TU) if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang|AMDClang)$") @@ -875,7 +875,7 @@ elseif(SOLPROP_NATIVE_BLAS_BACKEND STREQUAL "rocBLAS") ) endif() if(SOLPROP_NATIVE_RAMAN_FFT_PROVIDER STREQUAL "ACCELERATE") - target_sources(solprop_native_core PRIVATE src/raman_fft.cpp) + target_sources(solprop_native_core PRIVATE src/transforms/raman_fft.cpp) target_compile_definitions(solprop_native_core PUBLIC SOLPROP_NATIVE_RAMAN_ACCELERATE=1) target_link_libraries(solprop_native_core PUBLIC "-framework Accelerate") endif() @@ -901,8 +901,8 @@ add_custom_command( VERBATIM ) -nanobind_add_module(_core STABLE_ABI NOMINSIZE NB_SUPPRESS_WARNINGS src/bindings.cpp - src/bindings_transverse.cpp src/bindings_helpers.cpp src/records.cpp +nanobind_add_module(_core STABLE_ABI NOMINSIZE NB_SUPPRESS_WARNINGS src/bindings/bindings.cpp + src/bindings/bindings_transverse.cpp src/bindings/bindings_helpers.cpp src/bindings/records.cpp "${SOLPROP_NATIVE_SOURCE_DIGEST_CPP}") target_link_libraries(_core PRIVATE solprop_native_core ${CMAKE_DL_LIBS}) solprop_native_mark_dependency_headers_system(_core) diff --git a/native/README.md b/native/README.md index 62814199a..caf68a27c 100644 --- a/native/README.md +++ b/native/README.md @@ -134,8 +134,8 @@ The private extension `_core` is a thin seam (solprop SEP 52 section 5.3): (`Workspace.probe(start)`); `Workspace.diagnostics()` returns every fact the metadata reads. A session or probe leases the workspace until it closes. - **Records by field table.** Each C++ record declares its fields once, beside its - definition (`include/solprop_native/fields.hpp`), and one converter - (`src/records.hpp`) reads a dataclass, mapping or tuple into it, refusing a missing + definition (`include/solprop_native/runtime/fields.hpp`), and one converter + (`src/bindings/records.hpp`) reads a dataclass, mapping or tuple into it, refusing a missing or unknown field, a wrong dtype, rank or extent relation, a strided array or an enumeration given by anything but its name, by the field's path before copying anything. `_core.schemas()` exports every table, and a test holds each Python record @@ -144,7 +144,7 @@ The private extension `_core` is a thin seam (solprop SEP 52 section 5.3): storage. - **Bindings only marshal.** A binding converts, releases the interpreter lock and forwards. The lease, the coupled flow's guard, the marches' construction and the - probe live in the C++ library (`native::Workspace`, `src/workspace.cpp`), where the + probe live in the C++ library (`native::Workspace`, `src/problem/workspace.cpp`), where the native tests reach them. - **`_testing` promises nothing.** The test-only entry points live in the submodule `_core._testing`, shipped in the wheel for solprop's own tests; they take the diff --git a/native/cmake/VectorMath.cmake b/native/cmake/VectorMath.cmake index b1d83ef37..2cac69f82 100644 --- a/native/cmake/VectorMath.cmake +++ b/native/cmake/VectorMath.cmake @@ -3,7 +3,7 @@ # SPDX-License-Identifier: GPL-3.0-or-later # Host vector maths for the CPU kernels' transcendental passes (solprop ADR-059, -# include/solprop_native/vmath.hpp): SLEEF's vector exp, log and sincos, its 1.0 ulp +# include/solprop_native/runtime/vmath.hpp): SLEEF's vector exp, log and sincos, its 1.0 ulp # (u10) functions, called over contiguous blocks. glibc's scalar exp, cos and sin cost a # complex exponential 10 to 30 ns on an AMD EPYC 7763 and 9.8 ns on an Intel Xeon Gold # 6240, and SLEEF's AVX2 and AVX-512 functions take it 3 to 3.6 times faster at the same diff --git a/native/include/solprop_native/general_state.hpp b/native/include/solprop_native/general/general_state.hpp similarity index 99% rename from native/include/solprop_native/general_state.hpp rename to native/include/solprop_native/general/general_state.hpp index eb7b35a44..977004dc3 100644 --- a/native/include/solprop_native/general_state.hpp +++ b/native/include/solprop_native/general/general_state.hpp @@ -4,10 +4,10 @@ #pragma once -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/problem.hpp" -#include "solprop_native/readings.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/session/readings.hpp" #include #include diff --git a/native/include/solprop_native/adams_march.hpp b/native/include/solprop_native/march/adams_march.hpp similarity index 99% rename from native/include/solprop_native/adams_march.hpp rename to native/include/solprop_native/march/adams_march.hpp index ef485aced..ecbe03757 100644 --- a/native/include/solprop_native/adams_march.hpp +++ b/native/include/solprop_native/march/adams_march.hpp @@ -4,8 +4,8 @@ #pragma once -#include "solprop_native/host_source.hpp" -#include "solprop_native/watch.hpp" +#include "solprop_native/runtime/host_source.hpp" +#include "solprop_native/session/watch.hpp" #include #include diff --git a/native/include/solprop_native/adams_phi.hpp b/native/include/solprop_native/march/adams_phi.hpp similarity index 99% rename from native/include/solprop_native/adams_phi.hpp rename to native/include/solprop_native/march/adams_phi.hpp index ff8af3cd0..6b1304167 100644 --- a/native/include/solprop_native/adams_phi.hpp +++ b/native/include/solprop_native/march/adams_phi.hpp @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: GPL-3.0-or-later #pragma once -#include "solprop_native/execution.hpp" -#include "solprop_native/fp.hpp" -#include "solprop_native/general_state.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/fp.hpp" #include diff --git a/native/include/solprop_native/chebyshev.hpp b/native/include/solprop_native/march/chebyshev.hpp similarity index 99% rename from native/include/solprop_native/chebyshev.hpp rename to native/include/solprop_native/march/chebyshev.hpp index 4f6b3e6de..ce4587932 100644 --- a/native/include/solprop_native/chebyshev.hpp +++ b/native/include/solprop_native/march/chebyshev.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/fp.hpp" +#include "solprop_native/runtime/fp.hpp" #include #include diff --git a/native/include/solprop_native/dense.hpp b/native/include/solprop_native/march/dense.hpp similarity index 96% rename from native/include/solprop_native/dense.hpp rename to native/include/solprop_native/march/dense.hpp index 73a5c8e2a..57ee56f42 100644 --- a/native/include/solprop_native/dense.hpp +++ b/native/include/solprop_native/march/dense.hpp @@ -4,9 +4,9 @@ #pragma once -#include "solprop_native/general_state.hpp" -#include "solprop_native/session_dense.hpp" -#include "solprop_native/stage_tables.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/march/stage_tables.hpp" +#include "solprop_native/session/session_dense.hpp" #include #include diff --git a/native/include/solprop_native/engines.hpp b/native/include/solprop_native/march/engines.hpp similarity index 95% rename from native/include/solprop_native/engines.hpp rename to native/include/solprop_native/march/engines.hpp index 464f17c14..8796cea90 100644 --- a/native/include/solprop_native/engines.hpp +++ b/native/include/solprop_native/march/engines.hpp @@ -4,13 +4,13 @@ #pragma once -#include "solprop_native/adams_march.hpp" -#include "solprop_native/general_state.hpp" -#include "solprop_native/host_source.hpp" -#include "solprop_native/rk_march.hpp" -#include "solprop_native/session_resident.hpp" -#include "solprop_native/stage_tables.hpp" -#include "solprop_native/transverse.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/march/adams_march.hpp" +#include "solprop_native/march/rk_march.hpp" +#include "solprop_native/march/stage_tables.hpp" +#include "solprop_native/runtime/host_source.hpp" +#include "solprop_native/session/session_resident.hpp" +#include "solprop_native/session/transverse.hpp" #include #include diff --git a/native/include/solprop_native/lagrange_basis.hpp b/native/include/solprop_native/march/lagrange_basis.hpp similarity index 99% rename from native/include/solprop_native/lagrange_basis.hpp rename to native/include/solprop_native/march/lagrange_basis.hpp index 67b73bb7e..1e4b38691 100644 --- a/native/include/solprop_native/lagrange_basis.hpp +++ b/native/include/solprop_native/march/lagrange_basis.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/fp.hpp" +#include "solprop_native/runtime/fp.hpp" #include diff --git a/native/include/solprop_native/low_storage.hpp b/native/include/solprop_native/march/low_storage.hpp similarity index 96% rename from native/include/solprop_native/low_storage.hpp rename to native/include/solprop_native/march/low_storage.hpp index c1fd96a08..e352bf081 100644 --- a/native/include/solprop_native/low_storage.hpp +++ b/native/include/solprop_native/march/low_storage.hpp @@ -4,11 +4,11 @@ #pragma once -#include "solprop_native/engines.hpp" -#include "solprop_native/fields.hpp" -#include "solprop_native/general_state.hpp" -#include "solprop_native/rk_march.hpp" -#include "solprop_native/stage_tables.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/march/engines.hpp" +#include "solprop_native/march/rk_march.hpp" +#include "solprop_native/march/stage_tables.hpp" +#include "solprop_native/runtime/fields.hpp" #include #include diff --git a/native/include/solprop_native/rk_march.hpp b/native/include/solprop_native/march/rk_march.hpp similarity index 99% rename from native/include/solprop_native/rk_march.hpp rename to native/include/solprop_native/march/rk_march.hpp index d4aa6be0e..fec42ed81 100644 --- a/native/include/solprop_native/rk_march.hpp +++ b/native/include/solprop_native/march/rk_march.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/adams_march.hpp" +#include "solprop_native/march/adams_march.hpp" #include #include diff --git a/native/include/solprop_native/stage_tables.hpp b/native/include/solprop_native/march/stage_tables.hpp similarity index 99% rename from native/include/solprop_native/stage_tables.hpp rename to native/include/solprop_native/march/stage_tables.hpp index 458e99ad7..822aeb5a6 100644 --- a/native/include/solprop_native/stage_tables.hpp +++ b/native/include/solprop_native/march/stage_tables.hpp @@ -4,11 +4,11 @@ #pragma once -#include "solprop_native/chebyshev.hpp" -#include "solprop_native/fields.hpp" -#include "solprop_native/general_state.hpp" -#include "solprop_native/lagrange_basis.hpp" -#include "solprop_native/refuse.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/march/chebyshev.hpp" +#include "solprop_native/march/lagrange_basis.hpp" +#include "solprop_native/runtime/fields.hpp" +#include "solprop_native/runtime/refuse.hpp" #include #include diff --git a/native/include/solprop_native/z_table.hpp b/native/include/solprop_native/march/z_table.hpp similarity index 98% rename from native/include/solprop_native/z_table.hpp rename to native/include/solprop_native/march/z_table.hpp index 823baedfb..b5467d012 100644 --- a/native/include/solprop_native/z_table.hpp +++ b/native/include/solprop_native/march/z_table.hpp @@ -4,10 +4,10 @@ #pragma once -#include "solprop_native/chebyshev.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/launch.hpp" -#include "solprop_native/refuse.hpp" +#include "solprop_native/march/chebyshev.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/launch.hpp" +#include "solprop_native/runtime/refuse.hpp" #include #include diff --git a/native/include/solprop_native/problem.hpp b/native/include/solprop_native/problem/problem.hpp similarity index 99% rename from native/include/solprop_native/problem.hpp rename to native/include/solprop_native/problem/problem.hpp index c7ca287c5..913604de1 100644 --- a/native/include/solprop_native/problem.hpp +++ b/native/include/solprop_native/problem/problem.hpp @@ -4,9 +4,9 @@ #pragma once -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/fields.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/fields.hpp" #include #include diff --git a/native/include/solprop_native/run_records.hpp b/native/include/solprop_native/problem/run_records.hpp similarity index 97% rename from native/include/solprop_native/run_records.hpp rename to native/include/solprop_native/problem/run_records.hpp index 73420f040..e2f4e0323 100644 --- a/native/include/solprop_native/run_records.hpp +++ b/native/include/solprop_native/problem/run_records.hpp @@ -4,18 +4,18 @@ #pragma once -#include "solprop_native/adams_march.hpp" -#include "solprop_native/engines.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/fields.hpp" -#include "solprop_native/general_state.hpp" -#include "solprop_native/low_storage.hpp" -#include "solprop_native/readings.hpp" -#include "solprop_native/rk_march.hpp" -#include "solprop_native/stage_tables.hpp" -#include "solprop_native/statistics_plan.hpp" -#include "solprop_native/transverse.hpp" -#include "solprop_native/watch.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/march/adams_march.hpp" +#include "solprop_native/march/engines.hpp" +#include "solprop_native/march/low_storage.hpp" +#include "solprop_native/march/rk_march.hpp" +#include "solprop_native/march/stage_tables.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/fields.hpp" +#include "solprop_native/session/readings.hpp" +#include "solprop_native/session/statistics_plan.hpp" +#include "solprop_native/session/transverse.hpp" +#include "solprop_native/session/watch.hpp" #include #include diff --git a/native/include/solprop_native/workspace.hpp b/native/include/solprop_native/problem/workspace.hpp similarity index 96% rename from native/include/solprop_native/workspace.hpp rename to native/include/solprop_native/problem/workspace.hpp index 42821aafb..8e85c3da1 100644 --- a/native/include/solprop_native/workspace.hpp +++ b/native/include/solprop_native/problem/workspace.hpp @@ -4,16 +4,16 @@ #pragma once -#include "solprop_native/adams_march.hpp" -#include "solprop_native/engines.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/general_state.hpp" -#include "solprop_native/low_storage.hpp" -#include "solprop_native/problem.hpp" -#include "solprop_native/rk_march.hpp" -#include "solprop_native/run_records.hpp" -#include "solprop_native/session.hpp" -#include "solprop_native/stage_tables.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/march/adams_march.hpp" +#include "solprop_native/march/engines.hpp" +#include "solprop_native/march/low_storage.hpp" +#include "solprop_native/march/rk_march.hpp" +#include "solprop_native/march/stage_tables.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/problem/run_records.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/session/session.hpp" #include #include diff --git a/native/include/solprop_native/coupled_flow.hpp b/native/include/solprop_native/response/coupled_flow.hpp similarity index 99% rename from native/include/solprop_native/coupled_flow.hpp rename to native/include/solprop_native/response/coupled_flow.hpp index c8d96dff8..950d954c4 100644 --- a/native/include/solprop_native/coupled_flow.hpp +++ b/native/include/solprop_native/response/coupled_flow.hpp @@ -4,8 +4,8 @@ #pragma once -#include "solprop_native/execution.hpp" -#include "solprop_native/problem.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/runtime/execution.hpp" #include #include diff --git a/native/include/solprop_native/dispersive.hpp b/native/include/solprop_native/response/dispersive.hpp similarity index 99% rename from native/include/solprop_native/dispersive.hpp rename to native/include/solprop_native/response/dispersive.hpp index d9417f776..f003bb63e 100644 --- a/native/include/solprop_native/dispersive.hpp +++ b/native/include/solprop_native/response/dispersive.hpp @@ -4,10 +4,10 @@ #pragma once -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/fft_plan.hpp" -#include "solprop_native/problem.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/transforms/fft_plan.hpp" #include diff --git a/native/include/solprop_native/envelope_plasma.hpp b/native/include/solprop_native/response/envelope_plasma.hpp similarity index 100% rename from native/include/solprop_native/envelope_plasma.hpp rename to native/include/solprop_native/response/envelope_plasma.hpp diff --git a/native/include/solprop_native/far_field.hpp b/native/include/solprop_native/response/far_field.hpp similarity index 99% rename from native/include/solprop_native/far_field.hpp rename to native/include/solprop_native/response/far_field.hpp index 7a3e8bf0b..1437ea2a6 100644 --- a/native/include/solprop_native/far_field.hpp +++ b/native/include/solprop_native/response/far_field.hpp @@ -4,9 +4,9 @@ #pragma once -#include "solprop_native/execution.hpp" -#include "solprop_native/fp.hpp" -#include "solprop_native/problem.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/fp.hpp" #include diff --git a/native/include/solprop_native/flow_functions.hpp b/native/include/solprop_native/response/flow_functions.hpp similarity index 98% rename from native/include/solprop_native/flow_functions.hpp rename to native/include/solprop_native/response/flow_functions.hpp index 54144f0d5..5250cd7e9 100644 --- a/native/include/solprop_native/flow_functions.hpp +++ b/native/include/solprop_native/response/flow_functions.hpp @@ -4,8 +4,8 @@ #pragma once -#include "solprop_native/execution.hpp" -#include "solprop_native/general_state.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/runtime/execution.hpp" #include diff --git a/native/include/solprop_native/general_modal.hpp b/native/include/solprop_native/response/general_modal.hpp similarity index 97% rename from native/include/solprop_native/general_modal.hpp rename to native/include/solprop_native/response/general_modal.hpp index bf40db619..37cdd3dac 100644 --- a/native/include/solprop_native/general_modal.hpp +++ b/native/include/solprop_native/response/general_modal.hpp @@ -4,12 +4,12 @@ #pragma once -#include "solprop_native/build.hpp" -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/dispersive.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/problem.hpp" -#include "solprop_native/raman_fft.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/response/dispersive.hpp" +#include "solprop_native/runtime/build.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/transforms/raman_fft.hpp" #include diff --git a/native/include/solprop_native/plasma.hpp b/native/include/solprop_native/response/plasma.hpp similarity index 99% rename from native/include/solprop_native/plasma.hpp rename to native/include/solprop_native/response/plasma.hpp index 88ab80609..606e5ea51 100644 --- a/native/include/solprop_native/plasma.hpp +++ b/native/include/solprop_native/response/plasma.hpp @@ -4,10 +4,10 @@ #pragma once -#include "solprop_native/dispatch.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/launch.hpp" -#include "solprop_native/problem.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/runtime/dispatch.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/launch.hpp" #include diff --git a/native/include/solprop_native/build.hpp b/native/include/solprop_native/runtime/build.hpp similarity index 100% rename from native/include/solprop_native/build.hpp rename to native/include/solprop_native/runtime/build.hpp diff --git a/native/include/solprop_native/cpu_execution.hpp b/native/include/solprop_native/runtime/cpu_execution.hpp similarity index 99% rename from native/include/solprop_native/cpu_execution.hpp rename to native/include/solprop_native/runtime/cpu_execution.hpp index cb1527048..fd977c9f4 100644 --- a/native/include/solprop_native/cpu_execution.hpp +++ b/native/include/solprop_native/runtime/cpu_execution.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/execution.hpp" +#include "solprop_native/runtime/execution.hpp" #include #include diff --git a/native/include/solprop_native/crossing.hpp b/native/include/solprop_native/runtime/crossing.hpp similarity index 100% rename from native/include/solprop_native/crossing.hpp rename to native/include/solprop_native/runtime/crossing.hpp diff --git a/native/include/solprop_native/dispatch.hpp b/native/include/solprop_native/runtime/dispatch.hpp similarity index 100% rename from native/include/solprop_native/dispatch.hpp rename to native/include/solprop_native/runtime/dispatch.hpp diff --git a/native/include/solprop_native/execution.hpp b/native/include/solprop_native/runtime/execution.hpp similarity index 98% rename from native/include/solprop_native/execution.hpp rename to native/include/solprop_native/runtime/execution.hpp index e29a34dc2..6387e0f65 100644 --- a/native/include/solprop_native/execution.hpp +++ b/native/include/solprop_native/runtime/execution.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/fields.hpp" +#include "solprop_native/runtime/fields.hpp" #include #include diff --git a/native/include/solprop_native/fields.hpp b/native/include/solprop_native/runtime/fields.hpp similarity index 97% rename from native/include/solprop_native/fields.hpp rename to native/include/solprop_native/runtime/fields.hpp index 787f6b8e0..41cc7928b 100644 --- a/native/include/solprop_native/fields.hpp +++ b/native/include/solprop_native/runtime/fields.hpp @@ -33,8 +33,9 @@ * count, or as a symbol another array's extent binds (the Raman table's * `(n_terms, 2, n_to)`). A count field takes its value from a symbol the arrays bound, * so that a count an array's extents give is read from the array and never sent beside - * it. The binding layer's converter (`src/records.hpp`) reads a Python dataclass or - * mapping into the record and back, and exports the tables (`_core.schemas()`). + * it. The binding layer's converter (`src/bindings/records.hpp`) reads a Python + * dataclass or mapping into the record and back, and exports the tables + * (`_core.schemas()`). * * Python-free and Kokkos-free: descriptors only. */ diff --git a/native/include/solprop_native/fp.hpp b/native/include/solprop_native/runtime/fp.hpp similarity index 100% rename from native/include/solprop_native/fp.hpp rename to native/include/solprop_native/runtime/fp.hpp diff --git a/native/include/solprop_native/graph.hpp b/native/include/solprop_native/runtime/graph.hpp similarity index 100% rename from native/include/solprop_native/graph.hpp rename to native/include/solprop_native/runtime/graph.hpp diff --git a/native/include/solprop_native/host_source.hpp b/native/include/solprop_native/runtime/host_source.hpp similarity index 100% rename from native/include/solprop_native/host_source.hpp rename to native/include/solprop_native/runtime/host_source.hpp diff --git a/native/include/solprop_native/launch.hpp b/native/include/solprop_native/runtime/launch.hpp similarity index 99% rename from native/include/solprop_native/launch.hpp rename to native/include/solprop_native/runtime/launch.hpp index 7d2ae3ab4..0672ccee1 100644 --- a/native/include/solprop_native/launch.hpp +++ b/native/include/solprop_native/runtime/launch.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/graph.hpp" +#include "solprop_native/runtime/graph.hpp" #include diff --git a/native/include/solprop_native/refuse.hpp b/native/include/solprop_native/runtime/refuse.hpp similarity index 100% rename from native/include/solprop_native/refuse.hpp rename to native/include/solprop_native/runtime/refuse.hpp diff --git a/native/include/solprop_native/setup_data.hpp b/native/include/solprop_native/runtime/setup_data.hpp similarity index 96% rename from native/include/solprop_native/setup_data.hpp rename to native/include/solprop_native/runtime/setup_data.hpp index 0f244363a..2a84f9f9d 100644 --- a/native/include/solprop_native/setup_data.hpp +++ b/native/include/solprop_native/runtime/setup_data.hpp @@ -4,8 +4,8 @@ #pragma once -#include "solprop_native/execution.hpp" -#include "solprop_native/graph.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/graph.hpp" #include diff --git a/native/include/solprop_native/vmath.hpp b/native/include/solprop_native/runtime/vmath.hpp similarity index 100% rename from native/include/solprop_native/vmath.hpp rename to native/include/solprop_native/runtime/vmath.hpp diff --git a/native/include/solprop_native/readings.hpp b/native/include/solprop_native/session/readings.hpp similarity index 99% rename from native/include/solprop_native/readings.hpp rename to native/include/solprop_native/session/readings.hpp index a58689f88..07b5fbd88 100644 --- a/native/include/solprop_native/readings.hpp +++ b/native/include/solprop_native/session/readings.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/fields.hpp" +#include "solprop_native/runtime/fields.hpp" #include diff --git a/native/include/solprop_native/session.hpp b/native/include/solprop_native/session/session.hpp similarity index 98% rename from native/include/solprop_native/session.hpp rename to native/include/solprop_native/session/session.hpp index fb4e782ce..880e0f174 100644 --- a/native/include/solprop_native/session.hpp +++ b/native/include/solprop_native/session/session.hpp @@ -4,13 +4,13 @@ #pragma once -#include "solprop_native/adams_march.hpp" -#include "solprop_native/rk_march.hpp" -#include "solprop_native/session_dense.hpp" -#include "solprop_native/session_resident.hpp" -#include "solprop_native/statistics_plan.hpp" -#include "solprop_native/transverse.hpp" -#include "solprop_native/watch.hpp" +#include "solprop_native/march/adams_march.hpp" +#include "solprop_native/march/rk_march.hpp" +#include "solprop_native/session/session_dense.hpp" +#include "solprop_native/session/session_resident.hpp" +#include "solprop_native/session/statistics_plan.hpp" +#include "solprop_native/session/transverse.hpp" +#include "solprop_native/session/watch.hpp" #include #include diff --git a/native/include/solprop_native/session_dense.hpp b/native/include/solprop_native/session/session_dense.hpp similarity index 98% rename from native/include/solprop_native/session_dense.hpp rename to native/include/solprop_native/session/session_dense.hpp index ea30cf2d0..45ed6746e 100644 --- a/native/include/solprop_native/session_dense.hpp +++ b/native/include/solprop_native/session/session_dense.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/adams_march.hpp" +#include "solprop_native/march/adams_march.hpp" /* * Dense output in the native execution session (solprop SEP 47 section 5.3): a save diff --git a/native/include/solprop_native/session_resident.hpp b/native/include/solprop_native/session/session_resident.hpp similarity index 97% rename from native/include/solprop_native/session_resident.hpp rename to native/include/solprop_native/session/session_resident.hpp index 8d321dfff..e7872b4ec 100644 --- a/native/include/solprop_native/session_resident.hpp +++ b/native/include/solprop_native/session/session_resident.hpp @@ -4,8 +4,8 @@ #pragma once -#include "solprop_native/statistics_plan.hpp" -#include "solprop_native/transverse.hpp" +#include "solprop_native/session/statistics_plan.hpp" +#include "solprop_native/session/transverse.hpp" #include #include diff --git a/native/include/solprop_native/statistics_plan.hpp b/native/include/solprop_native/session/statistics_plan.hpp similarity index 100% rename from native/include/solprop_native/statistics_plan.hpp rename to native/include/solprop_native/session/statistics_plan.hpp diff --git a/native/include/solprop_native/transverse.hpp b/native/include/solprop_native/session/transverse.hpp similarity index 99% rename from native/include/solprop_native/transverse.hpp rename to native/include/solprop_native/session/transverse.hpp index 67fb3e262..3707ca2f3 100644 --- a/native/include/solprop_native/transverse.hpp +++ b/native/include/solprop_native/session/transverse.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/fields.hpp" +#include "solprop_native/runtime/fields.hpp" #include #include diff --git a/native/include/solprop_native/watch.hpp b/native/include/solprop_native/session/watch.hpp similarity index 99% rename from native/include/solprop_native/watch.hpp rename to native/include/solprop_native/session/watch.hpp index 31d68a55c..f5c8f4c1e 100644 --- a/native/include/solprop_native/watch.hpp +++ b/native/include/solprop_native/session/watch.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/fields.hpp" +#include "solprop_native/runtime/fields.hpp" #include #include diff --git a/native/include/solprop_native/fft_plan.hpp b/native/include/solprop_native/transforms/fft_plan.hpp similarity index 98% rename from native/include/solprop_native/fft_plan.hpp rename to native/include/solprop_native/transforms/fft_plan.hpp index 86940008e..0e755ac82 100644 --- a/native/include/solprop_native/fft_plan.hpp +++ b/native/include/solprop_native/transforms/fft_plan.hpp @@ -4,9 +4,9 @@ #pragma once -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/fftw_wisdom.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/transforms/fftw_wisdom.hpp" #include @@ -83,12 +83,12 @@ void seed_row_fft_wisdom(const Execution& execution, const Input& input, } // namespace solprop_native #endif -#include "solprop_native/graph.hpp" +#include "solprop_native/runtime/graph.hpp" #include #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -#include "solprop_native/launch.hpp" +#include "solprop_native/runtime/launch.hpp" #include #include diff --git a/native/include/solprop_native/fftw_wisdom.hpp b/native/include/solprop_native/transforms/fftw_wisdom.hpp similarity index 98% rename from native/include/solprop_native/fftw_wisdom.hpp rename to native/include/solprop_native/transforms/fftw_wisdom.hpp index 46cd93994..99c9c2aed 100644 --- a/native/include/solprop_native/fftw_wisdom.hpp +++ b/native/include/solprop_native/transforms/fftw_wisdom.hpp @@ -4,8 +4,8 @@ #pragma once -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/execution.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/execution.hpp" /* * The host FFTW plan policy and wisdom (solprop SEP 46, ADR-076): the effort a diff --git a/native/include/solprop_native/gemm.hpp b/native/include/solprop_native/transforms/gemm.hpp similarity index 99% rename from native/include/solprop_native/gemm.hpp rename to native/include/solprop_native/transforms/gemm.hpp index 5290f4fb8..6a192d9f8 100644 --- a/native/include/solprop_native/gemm.hpp +++ b/native/include/solprop_native/transforms/gemm.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/launch.hpp" +#include "solprop_native/runtime/launch.hpp" #include #include diff --git a/native/include/solprop_native/raman_fft.hpp b/native/include/solprop_native/transforms/raman_fft.hpp similarity index 98% rename from native/include/solprop_native/raman_fft.hpp rename to native/include/solprop_native/transforms/raman_fft.hpp index 515c28468..fc3a5b05e 100644 --- a/native/include/solprop_native/raman_fft.hpp +++ b/native/include/solprop_native/transforms/raman_fft.hpp @@ -4,8 +4,8 @@ #pragma once -#include "solprop_native/fft_plan.hpp" -#include "solprop_native/launch.hpp" +#include "solprop_native/runtime/launch.hpp" +#include "solprop_native/transforms/fft_plan.hpp" #include diff --git a/native/include/solprop_native/reference_timing.hpp b/native/include/solprop_native/transforms/reference_timing.hpp similarity index 100% rename from native/include/solprop_native/reference_timing.hpp rename to native/include/solprop_native/transforms/reference_timing.hpp diff --git a/native/include/solprop_native/slab.hpp b/native/include/solprop_native/transforms/slab.hpp similarity index 96% rename from native/include/solprop_native/slab.hpp rename to native/include/solprop_native/transforms/slab.hpp index 2a3914210..cc5aa03f8 100644 --- a/native/include/solprop_native/slab.hpp +++ b/native/include/solprop_native/transforms/slab.hpp @@ -4,8 +4,8 @@ #pragma once -#include "solprop_native/execution.hpp" -#include "solprop_native/launch.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/launch.hpp" #include diff --git a/native/src/bindings.cpp b/native/src/bindings/bindings.cpp similarity index 99% rename from native/src/bindings.cpp rename to native/src/bindings/bindings.cpp index 45f189c29..fa979e986 100644 --- a/native/src/bindings.cpp +++ b/native/src/bindings/bindings.cpp @@ -5,29 +5,29 @@ #include "bindings_helpers.hpp" #include "bindings_transverse.hpp" #include "records.hpp" -#include "solprop_native/adams_march.hpp" -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/crossing.hpp" -#include "solprop_native/dense.hpp" -#include "solprop_native/engines.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/fftw_wisdom.hpp" -#include "solprop_native/fields.hpp" -#include "solprop_native/general_state.hpp" -#include "solprop_native/host_source.hpp" -#include "solprop_native/lagrange_basis.hpp" -#include "solprop_native/low_storage.hpp" -#include "solprop_native/problem.hpp" -#include "solprop_native/readings.hpp" -#include "solprop_native/reference_timing.hpp" -#include "solprop_native/refuse.hpp" -#include "solprop_native/rk_march.hpp" -#include "solprop_native/run_records.hpp" -#include "solprop_native/session.hpp" -#include "solprop_native/stage_tables.hpp" -#include "solprop_native/vmath.hpp" -#include "solprop_native/watch.hpp" -#include "solprop_native/workspace.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/march/adams_march.hpp" +#include "solprop_native/march/dense.hpp" +#include "solprop_native/march/engines.hpp" +#include "solprop_native/march/lagrange_basis.hpp" +#include "solprop_native/march/low_storage.hpp" +#include "solprop_native/march/rk_march.hpp" +#include "solprop_native/march/stage_tables.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/problem/run_records.hpp" +#include "solprop_native/problem/workspace.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/crossing.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/fields.hpp" +#include "solprop_native/runtime/host_source.hpp" +#include "solprop_native/runtime/refuse.hpp" +#include "solprop_native/runtime/vmath.hpp" +#include "solprop_native/session/readings.hpp" +#include "solprop_native/session/session.hpp" +#include "solprop_native/session/watch.hpp" +#include "solprop_native/transforms/fftw_wisdom.hpp" +#include "solprop_native/transforms/reference_timing.hpp" #include diff --git a/native/src/bindings_helpers.cpp b/native/src/bindings/bindings_helpers.cpp similarity index 97% rename from native/src/bindings_helpers.cpp rename to native/src/bindings/bindings_helpers.cpp index bace21e4e..ea8b9a384 100644 --- a/native/src/bindings_helpers.cpp +++ b/native/src/bindings/bindings_helpers.cpp @@ -4,10 +4,10 @@ #include "bindings_helpers.hpp" -#include "solprop_native/chebyshev.hpp" -#include "solprop_native/lagrange_basis.hpp" -#include "solprop_native/refuse.hpp" -#include "solprop_native/vmath.hpp" +#include "solprop_native/march/chebyshev.hpp" +#include "solprop_native/march/lagrange_basis.hpp" +#include "solprop_native/runtime/refuse.hpp" +#include "solprop_native/runtime/vmath.hpp" #include #include diff --git a/native/src/bindings_helpers.hpp b/native/src/bindings/bindings_helpers.hpp similarity index 100% rename from native/src/bindings_helpers.hpp rename to native/src/bindings/bindings_helpers.hpp diff --git a/native/src/bindings_transverse.cpp b/native/src/bindings/bindings_transverse.cpp similarity index 100% rename from native/src/bindings_transverse.cpp rename to native/src/bindings/bindings_transverse.cpp diff --git a/native/src/bindings_transverse.hpp b/native/src/bindings/bindings_transverse.hpp similarity index 96% rename from native/src/bindings_transverse.hpp rename to native/src/bindings/bindings_transverse.hpp index 3db36a545..9afe98ad1 100644 --- a/native/src/bindings_transverse.hpp +++ b/native/src/bindings/bindings_transverse.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/transverse.hpp" +#include "solprop_native/session/transverse.hpp" #include diff --git a/native/src/records.cpp b/native/src/bindings/records.cpp similarity index 100% rename from native/src/records.cpp rename to native/src/bindings/records.cpp diff --git a/native/src/records.hpp b/native/src/bindings/records.hpp similarity index 99% rename from native/src/records.hpp rename to native/src/bindings/records.hpp index dbd76b4b3..a9687c862 100644 --- a/native/src/records.hpp +++ b/native/src/bindings/records.hpp @@ -4,7 +4,7 @@ #pragma once -#include "solprop_native/fields.hpp" +#include "solprop_native/runtime/fields.hpp" #include #include @@ -27,8 +27,8 @@ /* * The field-table converter (solprop SEP 52 section 5.3): one template that reads a * Python dataclass, a mapping or a tuple into a C++ record by its table - * (`solprop_native/fields.hpp`) and writes the record back as a dictionary, and the - * schema export of every table. + * (`solprop_native/runtime/fields.hpp`) and writes the record back as a dictionary, and + * the schema export of every table. * * Reading takes one pass over the input, under the interpreter lock, into a new * record: each field is checked -- a missing or unknown field, a wrong type or dtype, diff --git a/native/src/general/general_workspace.hpp b/native/src/general/general_workspace.hpp index fb083f67e..baae40e8b 100644 --- a/native/src/general/general_workspace.hpp +++ b/native/src/general/general_workspace.hpp @@ -12,7 +12,7 @@ #endif #include "kernels.hpp" -#include "solprop_native/host_source.hpp" +#include "solprop_native/runtime/host_source.hpp" namespace solprop_native { @@ -441,9 +441,10 @@ class GeneralWorkspace { void device_trapezoid_plasma(std::size_t stage, std::size_t first_node, std::size_t node_count); #endif - // solprop SEP 18's band rule as a team column (include/solprop_native/plasma.hpp): - // filter, then scan, for the population and free current, in whole-row passes. - // It writes the current separately from the collared non-plasma response. + // solprop SEP 18's band rule as a team column + // (include/solprop_native/response/plasma.hpp): filter, then scan, for the population + // and free current, in whole-row passes. It writes the current separately from the + // collared non-plasma response. template void team_band_plasma_column(std::size_t stage, std::size_t first_node, diff --git a/native/src/general/kernels.hpp b/native/src/general/kernels.hpp index 5dce67da1..e1d122f55 100644 --- a/native/src/general/kernels.hpp +++ b/native/src/general/kernels.hpp @@ -12,28 +12,28 @@ #error "general/kernels.hpp contracts: a strict translation unit must not include it" #endif -#include "solprop_native/adams_phi.hpp" -#include "solprop_native/build.hpp" -#include "solprop_native/coupled_flow.hpp" -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/crossing.hpp" -#include "solprop_native/dispatch.hpp" -#include "solprop_native/dispersive.hpp" -#include "solprop_native/envelope_plasma.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/far_field.hpp" -#include "solprop_native/fft_plan.hpp" -#include "solprop_native/flow_functions.hpp" -#include "solprop_native/fp.hpp" -#include "solprop_native/gemm.hpp" -#include "solprop_native/general_modal.hpp" -#include "solprop_native/plasma.hpp" -#include "solprop_native/raman_fft.hpp" -#include "solprop_native/refuse.hpp" -#include "solprop_native/setup_data.hpp" -#include "solprop_native/slab.hpp" -#include "solprop_native/vmath.hpp" -#include "solprop_native/z_table.hpp" +#include "solprop_native/march/adams_phi.hpp" +#include "solprop_native/march/z_table.hpp" +#include "solprop_native/response/coupled_flow.hpp" +#include "solprop_native/response/dispersive.hpp" +#include "solprop_native/response/envelope_plasma.hpp" +#include "solprop_native/response/far_field.hpp" +#include "solprop_native/response/flow_functions.hpp" +#include "solprop_native/response/general_modal.hpp" +#include "solprop_native/response/plasma.hpp" +#include "solprop_native/runtime/build.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/crossing.hpp" +#include "solprop_native/runtime/dispatch.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/fp.hpp" +#include "solprop_native/runtime/refuse.hpp" +#include "solprop_native/runtime/setup_data.hpp" +#include "solprop_native/runtime/vmath.hpp" +#include "solprop_native/transforms/fft_plan.hpp" +#include "solprop_native/transforms/gemm.hpp" +#include "solprop_native/transforms/raman_fft.hpp" +#include "solprop_native/transforms/slab.hpp" #include #include diff --git a/native/src/general/trial.cpp b/native/src/general/trial.cpp index 35cac1b5f..7cbee78cd 100644 --- a/native/src/general/trial.cpp +++ b/native/src/general/trial.cpp @@ -7064,9 +7064,9 @@ GeneralWorkspace::band_team_policy(const std::size_t node_count, } // solprop SEP 18's band rule as a team column in whole-row passes (solprop SEP 25 -// section 6.7.4; include/solprop_native/plasma.hpp): per channel the rates, their -// filter and scan give the exposure; the fraction (floored where it is formed), the -// loss guard and the free-current weight follow pointwise, a mixture adding each +// section 6.7.4; include/solprop_native/response/plasma.hpp): per channel the rates, +// their filter and scan give the exposure; the fraction (floored where it is formed), +// the loss guard and the free-current weight follow pointwise, a mixture adding each // channel's share in channel order. Per component the free-current integrand is // filtered and scanned, and the scan's store forms the current from it, the guard and // the field. SEP 27 uses this current directly, without a third running integral. @@ -7546,8 +7546,8 @@ GeneralWorkspace::tiled_band_policy(const std::size_t rows, const bool scratch, } // The band column in tiles (solprop SEP 25 section 6.7.6; -// include/solprop_native/plasma.hpp): each running integral is a local kernel of one -// team per (row, tile), which loads, filters and scans its tile from zero +// include/solprop_native/response/plasma.hpp): each running integral is a local kernel +// of one team per (row, tile), which loads, filters and scans its tile from zero // (`team_band_tile`), then a finish kernel of one team per (row, tile), which adds the // row's earlier tile totals (`team_tile_offset`) and forms the pass's product from the // prefix, in the team column's arithmetic: diff --git a/native/src/adams_march.cpp b/native/src/march/adams_march.cpp similarity index 99% rename from native/src/adams_march.cpp rename to native/src/march/adams_march.cpp index c716cd582..60845f872 100644 --- a/native/src/adams_march.cpp +++ b/native/src/march/adams_march.cpp @@ -2,10 +2,10 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/adams_march.hpp" +#include "solprop_native/march/adams_march.hpp" -#include "solprop_native/host_source.hpp" -#include "solprop_native/refuse.hpp" +#include "solprop_native/runtime/host_source.hpp" +#include "solprop_native/runtime/refuse.hpp" #include #include diff --git a/native/src/dense.cpp b/native/src/march/dense.cpp similarity index 97% rename from native/src/dense.cpp rename to native/src/march/dense.cpp index bc7d2b330..ddbc9a425 100644 --- a/native/src/dense.cpp +++ b/native/src/march/dense.cpp @@ -2,10 +2,10 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/dense.hpp" +#include "solprop_native/march/dense.hpp" -#include "solprop_native/engines.hpp" -#include "solprop_native/lagrange_basis.hpp" +#include "solprop_native/march/engines.hpp" +#include "solprop_native/march/lagrange_basis.hpp" #include #include diff --git a/native/src/engines.cpp b/native/src/march/engines.cpp similarity index 99% rename from native/src/engines.cpp rename to native/src/march/engines.cpp index c21b19e0a..c2a627f22 100644 --- a/native/src/engines.cpp +++ b/native/src/march/engines.cpp @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/engines.hpp" +#include "solprop_native/march/engines.hpp" #include #include diff --git a/native/src/low_storage.cpp b/native/src/march/low_storage.cpp similarity index 99% rename from native/src/low_storage.cpp rename to native/src/march/low_storage.cpp index 42d82b880..de7ca9dd7 100644 --- a/native/src/low_storage.cpp +++ b/native/src/march/low_storage.cpp @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/low_storage.hpp" +#include "solprop_native/march/low_storage.hpp" -#include "solprop_native/adams_march.hpp" +#include "solprop_native/march/adams_march.hpp" #include #include diff --git a/native/src/rk_march.cpp b/native/src/march/rk_march.cpp similarity index 99% rename from native/src/rk_march.cpp rename to native/src/march/rk_march.cpp index 9f71015c5..4ea0ef7dc 100644 --- a/native/src/rk_march.cpp +++ b/native/src/march/rk_march.cpp @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/rk_march.hpp" +#include "solprop_native/march/rk_march.hpp" -#include "solprop_native/refuse.hpp" +#include "solprop_native/runtime/refuse.hpp" #include #include diff --git a/native/src/problem.cpp b/native/src/problem/problem.cpp similarity index 97% rename from native/src/problem.cpp rename to native/src/problem/problem.cpp index b241d240b..65a71e3a0 100644 --- a/native/src/problem.cpp +++ b/native/src/problem/problem.cpp @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/problem.hpp" +#include "solprop_native/problem/problem.hpp" -#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" #include #include diff --git a/native/src/workspace.cpp b/native/src/problem/workspace.cpp similarity index 99% rename from native/src/workspace.cpp rename to native/src/problem/workspace.cpp index fda3b31b8..9115788d3 100644 --- a/native/src/workspace.cpp +++ b/native/src/problem/workspace.cpp @@ -2,11 +2,11 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/workspace.hpp" +#include "solprop_native/problem/workspace.hpp" -#include "solprop_native/dense.hpp" -#include "solprop_native/refuse.hpp" -#include "solprop_native/run_records.hpp" +#include "solprop_native/march/dense.hpp" +#include "solprop_native/problem/run_records.hpp" +#include "solprop_native/runtime/refuse.hpp" #include #include diff --git a/native/src/coupled_flow.cpp b/native/src/response/coupled_flow.cpp similarity index 99% rename from native/src/coupled_flow.cpp rename to native/src/response/coupled_flow.cpp index 997c585b1..cff4a1e4b 100644 --- a/native/src/coupled_flow.cpp +++ b/native/src/response/coupled_flow.cpp @@ -2,11 +2,11 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/coupled_flow.hpp" +#include "solprop_native/response/coupled_flow.hpp" -#include "solprop_native/launch.hpp" -#include "solprop_native/refuse.hpp" -#include "solprop_native/setup_data.hpp" +#include "solprop_native/runtime/launch.hpp" +#include "solprop_native/runtime/refuse.hpp" +#include "solprop_native/runtime/setup_data.hpp" #include #include diff --git a/native/src/dispersive.cpp b/native/src/response/dispersive.cpp similarity index 99% rename from native/src/dispersive.cpp rename to native/src/response/dispersive.cpp index 72f2ac5e4..132d20072 100644 --- a/native/src/dispersive.cpp +++ b/native/src/response/dispersive.cpp @@ -2,12 +2,12 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/dispersive.hpp" +#include "solprop_native/response/dispersive.hpp" -#include "solprop_native/gemm.hpp" -#include "solprop_native/launch.hpp" -#include "solprop_native/setup_data.hpp" -#include "solprop_native/slab.hpp" +#include "solprop_native/runtime/launch.hpp" +#include "solprop_native/runtime/setup_data.hpp" +#include "solprop_native/transforms/gemm.hpp" +#include "solprop_native/transforms/slab.hpp" #include #include diff --git a/native/src/far_field.cpp b/native/src/response/far_field.cpp similarity index 99% rename from native/src/far_field.cpp rename to native/src/response/far_field.cpp index ed880ef2c..dd5952af1 100644 --- a/native/src/far_field.cpp +++ b/native/src/response/far_field.cpp @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/far_field.hpp" +#include "solprop_native/response/far_field.hpp" -#include "solprop_native/launch.hpp" +#include "solprop_native/runtime/launch.hpp" #include diff --git a/native/src/general_modal.cpp b/native/src/response/general_modal.cpp similarity index 99% rename from native/src/general_modal.cpp rename to native/src/response/general_modal.cpp index 0e24887c6..16118ddb2 100644 --- a/native/src/general_modal.cpp +++ b/native/src/response/general_modal.cpp @@ -2,13 +2,13 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/general_modal.hpp" +#include "solprop_native/response/general_modal.hpp" -#include "solprop_native/dispatch.hpp" -#include "solprop_native/fft_plan.hpp" -#include "solprop_native/launch.hpp" -#include "solprop_native/refuse.hpp" -#include "solprop_native/setup_data.hpp" +#include "solprop_native/runtime/dispatch.hpp" +#include "solprop_native/runtime/launch.hpp" +#include "solprop_native/runtime/refuse.hpp" +#include "solprop_native/runtime/setup_data.hpp" +#include "solprop_native/transforms/fft_plan.hpp" #include #include diff --git a/native/src/coverage_labels.cpp b/native/src/runtime/coverage_labels.cpp similarity index 100% rename from native/src/coverage_labels.cpp rename to native/src/runtime/coverage_labels.cpp diff --git a/native/src/cpu_execution.cpp b/native/src/runtime/cpu_execution.cpp similarity index 99% rename from native/src/cpu_execution.cpp rename to native/src/runtime/cpu_execution.cpp index 3cb5cfda9..b7e1f2c93 100644 --- a/native/src/cpu_execution.cpp +++ b/native/src/runtime/cpu_execution.cpp @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" -#include "solprop_native/execution.hpp" +#include "solprop_native/runtime/execution.hpp" #include diff --git a/native/src/crossing.cpp b/native/src/runtime/crossing.cpp similarity index 98% rename from native/src/crossing.cpp rename to native/src/runtime/crossing.cpp index c5ab49c6b..a192e8ba3 100644 --- a/native/src/crossing.cpp +++ b/native/src/runtime/crossing.cpp @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/crossing.hpp" +#include "solprop_native/runtime/crossing.hpp" -#include "solprop_native/build.hpp" +#include "solprop_native/runtime/build.hpp" #include #include diff --git a/native/src/session.cpp b/native/src/session/session.cpp similarity index 99% rename from native/src/session.cpp rename to native/src/session/session.cpp index afcd55b01..59f1e9b29 100644 --- a/native/src/session.cpp +++ b/native/src/session/session.cpp @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/session.hpp" +#include "solprop_native/session/session.hpp" -#include "solprop_native/refuse.hpp" +#include "solprop_native/runtime/refuse.hpp" #include #include diff --git a/native/src/session_edge.cpp b/native/src/session/session_edge.cpp similarity index 98% rename from native/src/session_edge.cpp rename to native/src/session/session_edge.cpp index 53de02546..157d91bc5 100644 --- a/native/src/session_edge.cpp +++ b/native/src/session/session_edge.cpp @@ -8,7 +8,7 @@ // warning makes. The edge itself is `transverse::Edge`; the readings are the resident // state's (`Resident::edge_reading`), so nothing here returns to Python. -#include "solprop_native/session.hpp" +#include "solprop_native/session/session.hpp" #include #include diff --git a/native/src/session_widths.cpp b/native/src/session/session_widths.cpp similarity index 99% rename from native/src/session_widths.cpp rename to native/src/session/session_widths.cpp index 801db2180..02f6f8ef6 100644 --- a/native/src/session_widths.cpp +++ b/native/src/session/session_widths.cpp @@ -16,9 +16,9 @@ #pragma GCC optimize("fp-contract=off") #endif -#include "solprop_native/general_state.hpp" -#include "solprop_native/refuse.hpp" -#include "solprop_native/statistics_plan.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/runtime/refuse.hpp" +#include "solprop_native/session/statistics_plan.hpp" #include #include diff --git a/native/src/statistics_plan.cpp b/native/src/session/statistics_plan.cpp similarity index 99% rename from native/src/statistics_plan.cpp rename to native/src/session/statistics_plan.cpp index f68265d69..bbfaa3870 100644 --- a/native/src/statistics_plan.cpp +++ b/native/src/session/statistics_plan.cpp @@ -2,14 +2,14 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/statistics_plan.hpp" - -#include "solprop_native/chebyshev.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/far_field.hpp" -#include "solprop_native/general_state.hpp" -#include "solprop_native/problem.hpp" -#include "solprop_native/refuse.hpp" +#include "solprop_native/session/statistics_plan.hpp" + +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/march/chebyshev.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/response/far_field.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/refuse.hpp" #include #include diff --git a/native/src/transverse.cpp b/native/src/session/transverse.cpp similarity index 99% rename from native/src/transverse.cpp rename to native/src/session/transverse.cpp index 18617139e..3ea70ddfa 100644 --- a/native/src/transverse.cpp +++ b/native/src/session/transverse.cpp @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/transverse.hpp" +#include "solprop_native/session/transverse.hpp" #include #include diff --git a/native/src/watch.cpp b/native/src/session/watch.cpp similarity index 99% rename from native/src/watch.cpp rename to native/src/session/watch.cpp index 785bcd8d1..137aba02c 100644 --- a/native/src/watch.cpp +++ b/native/src/session/watch.cpp @@ -2,9 +2,9 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/watch.hpp" +#include "solprop_native/session/watch.hpp" -#include "solprop_native/refuse.hpp" +#include "solprop_native/runtime/refuse.hpp" #include #include diff --git a/native/src/solprop_native/_helpers.py b/native/src/solprop_native/_helpers.py index c6d262584..fd3cbc354 100644 --- a/native/src/solprop_native/_helpers.py +++ b/native/src/solprop_native/_helpers.py @@ -8,8 +8,8 @@ runs it (``NativeBackend.helpers``, solprop SEP 29 section 6.6), so that a native run never reaches solprop's Python forms and the NumPy reference never reaches this package. :data:`NATIVE` is this backend's record: the C++ transcription in -``include/solprop_native/chebyshev.hpp``, which performs solprop's operations in its -order under strict arithmetic. +``include/solprop_native/march/chebyshev.hpp``, which performs solprop's operations in +its order under strict arithmetic. The record names no Adams basis. The native Adams march is compiled and forms its Lagrange basis and coefficient tables in C++ (``lagrange_basis.hpp``) every trial, so no diff --git a/native/src/solprop_native/_transverse.py b/native/src/solprop_native/_transverse.py index 914e33e36..ee9c0f31f 100644 --- a/native/src/solprop_native/_transverse.py +++ b/native/src/solprop_native/_transverse.py @@ -7,13 +7,14 @@ solprop SEP 47 section 6.4 and the maintainer's decision 4 at its acceptance: the native session owns a free-space run's transverse edge (SEP 40 section 5.3) with every decision and record (``transverse::Edge``, -``native/include/solprop_native/transverse.hpp``). The edge's configuration and a -resume's continuation are solprop's run records (:mod:`solprop.backends.records`, SEP -52), which the extension's record converter (``native/src/records.hpp``) reads. This -module only translates the rest: it forms the input noise's readings a launch carries, -formats the native record as :meth:`~solprop.transverse_monitor.TransverseEdge.summary` -lays it out, and formats a native verdict as the edge's warning or exception with the -edge's own texts. It decides nothing. +``native/include/solprop_native/session/transverse.hpp``). The edge's configuration and +a resume's continuation are solprop's run records (:mod:`solprop.backends.records`, SEP +52), which the extension's record converter (``native/src/bindings/records.hpp``) reads. +This module only translates the rest: it forms the input noise's readings a launch +carries, formats the native record as +:meth:`~solprop.transverse_monitor.TransverseEdge.summary` lays it out, and formats a +native verdict as the edge's warning or exception with the edge's own texts. It decides +nothing. """ from __future__ import annotations diff --git a/native/src/solprop_native/_watch.py b/native/src/solprop_native/_watch.py index c953671dc..f30b6f7f3 100644 --- a/native/src/solprop_native/_watch.py +++ b/native/src/solprop_native/_watch.py @@ -6,12 +6,12 @@ solprop SEP 47 section 6.4 and the maintainer's decision 4 at its acceptance: the native session owns the boundary monitor's every decision and record -(``watch::Monitor``, ``native/include/solprop_native/watch.hpp``). The monitor's +(``watch::Monitor``, ``native/include/solprop_native/session/watch.hpp``). The monitor's configuration and a resume's continuation are solprop's run records (:mod:`solprop.backends.records`, SEP 52), which the extension's record converter -(``native/src/records.hpp``) reads. This module only translates what comes back: it -formats the native record as :meth:`~solprop.monitor.BoundaryMonitor.summary` lays it -out, and a native verdict as solprop's warning or exception. It decides nothing. +(``native/src/bindings/records.hpp``) reads. This module only translates what comes +back: it formats the native record as :meth:`~solprop.monitor.BoundaryMonitor.summary` +lays it out, and a native verdict as solprop's warning or exception. It decides nothing. """ from __future__ import annotations diff --git a/native/src/solprop_native/session.py b/native/src/solprop_native/session.py index 4a84d68fb..6e92d5fe5 100644 --- a/native/src/solprop_native/session.py +++ b/native/src/solprop_native/session.py @@ -6,10 +6,10 @@ :class:`NativeSession` implements solprop's :class:`~solprop.backends.session.RunSession` over ``_core.Session``, the native -session (``native/include/solprop_native/session.hpp``). The native session is the one -mutable owner of the trajectory: the march's policy, state and history, the plane -schedule, the periodic statistics rows (taken natively after each commit and kept in a -resident buffer until the next observation), the stop flag, the snapshot latch, the +session (``native/include/solprop_native/session/session.hpp``). The native session is +the one mutable owner of the trajectory: the march's policy, state and history, the +plane schedule, the periodic statistics rows (taken natively after each commit and kept +in a resident buffer until the next observation), the stop flag, the snapshot latch, the committed-state generation and the pending batch. This adapter only translates: it prepares the session once from a :class:`~solprop_native.backend.NativeBackend`'s lowering (the native problem, the stage tables and the CPU execution record), turns @@ -50,7 +50,7 @@ from the resident state with no return to Python, and a monitored resume continues both records natively from solprop's run records (:mod:`solprop.backends.records`, read by the extension's record converter, - ``native/src/records.hpp``); + ``native/src/bindings/records.hpp``); - dense output (``interpolate_planes``), for Adams and the embedded Runge-Kutta tableau alike: a plane the march steps across is interpolated natively inside the step that crossed it, the committed state held aside, and observed in order before diff --git a/native/src/fftw_wisdom.cpp b/native/src/transforms/fftw_wisdom.cpp similarity index 99% rename from native/src/fftw_wisdom.cpp rename to native/src/transforms/fftw_wisdom.cpp index eb2943536..5b7cf4e05 100644 --- a/native/src/fftw_wisdom.cpp +++ b/native/src/transforms/fftw_wisdom.cpp @@ -2,10 +2,10 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/fftw_wisdom.hpp" +#include "solprop_native/transforms/fftw_wisdom.hpp" -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/execution.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/execution.hpp" #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) #include diff --git a/native/src/raman_fft.cpp b/native/src/transforms/raman_fft.cpp similarity index 99% rename from native/src/raman_fft.cpp rename to native/src/transforms/raman_fft.cpp index f062d8c84..182e724f9 100644 --- a/native/src/raman_fft.cpp +++ b/native/src/transforms/raman_fft.cpp @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/raman_fft.hpp" +#include "solprop_native/transforms/raman_fft.hpp" #include #include diff --git a/native/src/reference_timing.cpp b/native/src/transforms/reference_timing.cpp similarity index 95% rename from native/src/reference_timing.cpp rename to native/src/transforms/reference_timing.cpp index 07e75b33d..c308f6747 100644 --- a/native/src/reference_timing.cpp +++ b/native/src/transforms/reference_timing.cpp @@ -2,11 +2,11 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/reference_timing.hpp" +#include "solprop_native/transforms/reference_timing.hpp" -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/launch.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/launch.hpp" #include #include diff --git a/native/tests/native/execution.hpp b/native/tests/native/execution.hpp index e525d5435..b17b401b5 100644 --- a/native/tests/native/execution.hpp +++ b/native/tests/native/execution.hpp @@ -9,7 +9,7 @@ // `_execution.builtin` returns it without a spin. Native code holds no values of its // own; a workspace refuses a problem without a record. -#include "solprop_native/cpu_execution.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" namespace solprop_native::test { diff --git a/native/tests/native/test_adams_march.cpp b/native/tests/native/test_adams_march.cpp index e588e8e5b..5fb0e6e7a 100644 --- a/native/tests/native/test_adams_march.cpp +++ b/native/tests/native/test_adams_march.cpp @@ -8,7 +8,7 @@ // policy is refused by the field at fault, that a run on a toy engine keeps its // ledger and lands on its target, and where a stop request ends a call (SEP 47). -#include "solprop_native/adams_march.hpp" +#include "solprop_native/march/adams_march.hpp" #include diff --git a/native/tests/native/test_adams_phi.cpp b/native/tests/native/test_adams_phi.cpp index faf33b3ac..2518e9e77 100644 --- a/native/tests/native/test_adams_phi.cpp +++ b/native/tests/native/test_adams_phi.cpp @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/adams_phi.hpp" +#include "solprop_native/march/adams_phi.hpp" #include @@ -16,9 +16,10 @@ #include // The exponential Adams march's phi-functions on a host block's lanes -// (include/solprop_native/adams_phi.hpp) against the element-by-element `adams_phi`: -// every lane's values are the scalar form's bits, whatever its neighbours, on each of -// the three branches, at the branch and halving thresholds, and under a carrier frame. +// (include/solprop_native/march/adams_phi.hpp) against the element-by-element +// `adams_phi`: every lane's values are the scalar form's bits, whatever its neighbours, +// on each of the three branches, at the branch and halving thresholds, and under a +// carrier frame. #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) namespace { namespace native = solprop_native; diff --git a/native/tests/native/test_aperture.cpp b/native/tests/native/test_aperture.cpp index 8d08b4f27..743026991 100644 --- a/native/tests/native/test_aperture.cpp +++ b/native/tests/native/test_aperture.cpp @@ -11,9 +11,9 @@ // bits; hard-edge membership is shared. #include "bounds.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/far_field.hpp" -#include "solprop_native/problem.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/response/far_field.hpp" +#include "solprop_native/runtime/execution.hpp" #include #include diff --git a/native/tests/native/test_core.cpp b/native/tests/native/test_core.cpp index 8d426325b..30d33a7eb 100644 --- a/native/tests/native/test_core.cpp +++ b/native/tests/native/test_core.cpp @@ -4,12 +4,12 @@ #include "bounds.hpp" #include "execution.hpp" -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/gemm.hpp" -#include "solprop_native/general_state.hpp" -#include "solprop_native/plasma.hpp" -#include "solprop_native/problem.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/response/plasma.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/transforms/gemm.hpp" #include #include diff --git a/native/tests/native/test_coupled_flow.cpp b/native/tests/native/test_coupled_flow.cpp index 99480b055..dbfe4238e 100644 --- a/native/tests/native/test_coupled_flow.cpp +++ b/native/tests/native/test_coupled_flow.cpp @@ -3,11 +3,11 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "execution.hpp" -#include "solprop_native/coupled_flow.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/flow_functions.hpp" -#include "solprop_native/general_state.hpp" -#include "solprop_native/problem.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/response/coupled_flow.hpp" +#include "solprop_native/response/flow_functions.hpp" +#include "solprop_native/runtime/execution.hpp" #include diff --git a/native/tests/native/test_crossing.cpp b/native/tests/native/test_crossing.cpp index ee9b3f998..d5019cf77 100644 --- a/native/tests/native/test_crossing.cpp +++ b/native/tests/native/test_crossing.cpp @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/crossing.hpp" +#include "solprop_native/runtime/crossing.hpp" #include diff --git a/native/tests/native/test_dispersive.cpp b/native/tests/native/test_dispersive.cpp index c5aee5dee..d0f329c64 100644 --- a/native/tests/native/test_dispersive.cpp +++ b/native/tests/native/test_dispersive.cpp @@ -3,10 +3,10 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "execution.hpp" -#include "solprop_native/dispersive.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/general_state.hpp" -#include "solprop_native/problem.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/response/dispersive.hpp" +#include "solprop_native/runtime/execution.hpp" #include #include diff --git a/native/tests/native/test_envelope_plasma.cpp b/native/tests/native/test_envelope_plasma.cpp index c53a7eabd..a4e507892 100644 --- a/native/tests/native/test_envelope_plasma.cpp +++ b/native/tests/native/test_envelope_plasma.cpp @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/envelope_plasma.hpp" -#include "solprop_native/execution.hpp" +#include "solprop_native/response/envelope_plasma.hpp" +#include "solprop_native/runtime/execution.hpp" #include #include diff --git a/native/tests/native/test_fft.cpp b/native/tests/native/test_fft.cpp index ec4f7c5d1..1c3522ca7 100644 --- a/native/tests/native/test_fft.cpp +++ b/native/tests/native/test_fft.cpp @@ -2,11 +2,11 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/fft_plan.hpp" -#include "solprop_native/fftw_wisdom.hpp" -#include "solprop_native/raman_fft.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/transforms/fft_plan.hpp" +#include "solprop_native/transforms/fftw_wisdom.hpp" +#include "solprop_native/transforms/raman_fft.hpp" #include #include diff --git a/native/tests/native/test_fp.cpp b/native/tests/native/test_fp.cpp index cfaa3819b..9f57e364f 100644 --- a/native/tests/native/test_fp.cpp +++ b/native/tests/native/test_fp.cpp @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/execution.hpp" -#include "solprop_native/fp.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/runtime/fp.hpp" #include diff --git a/native/tests/native/test_general_modal.cpp b/native/tests/native/test_general_modal.cpp index b226bb520..4f3ac5cd4 100644 --- a/native/tests/native/test_general_modal.cpp +++ b/native/tests/native/test_general_modal.cpp @@ -3,12 +3,12 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "execution.hpp" -#include "solprop_native/cpu_execution.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/general_modal.hpp" -#include "solprop_native/general_state.hpp" -#include "solprop_native/problem.hpp" -#include "solprop_native/readings.hpp" +#include "solprop_native/general/general_state.hpp" +#include "solprop_native/problem/problem.hpp" +#include "solprop_native/response/general_modal.hpp" +#include "solprop_native/runtime/cpu_execution.hpp" +#include "solprop_native/runtime/execution.hpp" +#include "solprop_native/session/readings.hpp" #include #include diff --git a/native/tests/native/test_helpers.cpp b/native/tests/native/test_helpers.cpp index f9a012cee..df90f3ddc 100644 --- a/native/tests/native/test_helpers.cpp +++ b/native/tests/native/test_helpers.cpp @@ -2,8 +2,8 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/chebyshev.hpp" -#include "solprop_native/lagrange_basis.hpp" +#include "solprop_native/march/chebyshev.hpp" +#include "solprop_native/march/lagrange_basis.hpp" #include diff --git a/native/tests/native/test_low_storage.cpp b/native/tests/native/test_low_storage.cpp index 302386208..8c04261b3 100644 --- a/native/tests/native/test_low_storage.cpp +++ b/native/tests/native/test_low_storage.cpp @@ -9,8 +9,8 @@ // stepped in place, the replay's records and the poisoned terminal of a step left in // flight, by a failure or a divergence. -#include "solprop_native/low_storage.hpp" -#include "solprop_native/session.hpp" +#include "solprop_native/march/low_storage.hpp" +#include "solprop_native/session/session.hpp" #include diff --git a/native/tests/native/test_rk_march.cpp b/native/tests/native/test_rk_march.cpp index e5ce98eea..53b089076 100644 --- a/native/tests/native/test_rk_march.cpp +++ b/native/tests/native/test_rk_march.cpp @@ -6,7 +6,7 @@ // solprop's by solprop's tests; these check what those cannot see: that the unit rounds // every product and sum separately, and the order of a stage's position and the norm. -#include "solprop_native/rk_march.hpp" +#include "solprop_native/march/rk_march.hpp" #include diff --git a/native/tests/native/test_session.cpp b/native/tests/native/test_session.cpp index 66de519ba..749727962 100644 --- a/native/tests/native/test_session.cpp +++ b/native/tests/native/test_session.cpp @@ -9,7 +9,7 @@ // the cadence of the resident rows, yields that change nothing, the stop and snapshot // controls, the export's recipe to the bit, and the strict unit's own rounding. -#include "solprop_native/session.hpp" +#include "solprop_native/session/session.hpp" #include diff --git a/native/tests/native/test_session_monitor.cpp b/native/tests/native/test_session_monitor.cpp index 6f222d670..29b396f59 100644 --- a/native/tests/native/test_session_monitor.cpp +++ b/native/tests/native/test_session_monitor.cpp @@ -11,7 +11,7 @@ // work done once after it, a stop a terminal failure there, a stale step checked at // its plane, and the final check before completion. -#include "solprop_native/session.hpp" +#include "solprop_native/session/session.hpp" #include diff --git a/native/tests/native/test_session_transverse.cpp b/native/tests/native/test_session_transverse.cpp index 4d41f40b0..e18cb45fd 100644 --- a/native/tests/native/test_session_transverse.cpp +++ b/native/tests/native/test_session_transverse.cpp @@ -12,7 +12,7 @@ // a plane read before its observation, and a collar's current taken at every step // with no more batches than the planes'. -#include "solprop_native/session.hpp" +#include "solprop_native/session/session.hpp" #include diff --git a/native/tests/native/test_transverse.cpp b/native/tests/native/test_transverse.cpp index 5b5afd167..f25302eed 100644 --- a/native/tests/native/test_transverse.cpp +++ b/native/tests/native/test_transverse.cpp @@ -9,7 +9,7 @@ // configuration is refused by the field at fault, the tie order of the worst face, a // point's several notices served one at a time, and the spacing's ceiling. -#include "solprop_native/transverse.hpp" +#include "solprop_native/session/transverse.hpp" #include diff --git a/native/tests/native/test_vmath.cpp b/native/tests/native/test_vmath.cpp index c428d79c1..3d5f6276b 100644 --- a/native/tests/native/test_vmath.cpp +++ b/native/tests/native/test_vmath.cpp @@ -3,7 +3,7 @@ // SPDX-License-Identifier: GPL-3.0-or-later #include "bounds.hpp" -#include "solprop_native/vmath.hpp" +#include "solprop_native/runtime/vmath.hpp" #include @@ -13,9 +13,9 @@ #include #include -// The host vector-maths layer (include/solprop_native/vmath.hpp): every block length, -// the special values, and agreement with the system libm. Its accuracy against a -// high-precision reference over the kernels' argument ranges is +// The host vector-maths layer (include/solprop_native/runtime/vmath.hpp): every block +// length, the special values, and agreement with the system libm. Its accuracy against +// a high-precision reference over the kernels' argument ranges is // tests/test_vector_math.py. namespace { namespace vmath = solprop_native::vmath; diff --git a/native/tests/native/test_watch.cpp b/native/tests/native/test_watch.cpp index 5bffa8e72..09c6df71d 100644 --- a/native/tests/native/test_watch.cpp +++ b/native/tests/native/test_watch.cpp @@ -7,7 +7,7 @@ // see: that the unit rounds every product and sum separately, that a policy is refused // by the field at fault, and the split limit's sequential order. -#include "solprop_native/watch.hpp" +#include "solprop_native/session/watch.hpp" #include diff --git a/native/tests/native/test_workspace.cpp b/native/tests/native/test_workspace.cpp index d57e14311..967b8ad28 100644 --- a/native/tests/native/test_workspace.cpp +++ b/native/tests/native/test_workspace.cpp @@ -7,8 +7,8 @@ // timing probe, each moved out of the binding unit and tested here. #include "execution.hpp" -#include "solprop_native/run_records.hpp" -#include "solprop_native/workspace.hpp" +#include "solprop_native/problem/run_records.hpp" +#include "solprop_native/problem/workspace.hpp" #include diff --git a/native/tests/native/test_z_table.cpp b/native/tests/native/test_z_table.cpp index 7c6d657ed..5f286e42c 100644 --- a/native/tests/native/test_z_table.cpp +++ b/native/tests/native/test_z_table.cpp @@ -2,10 +2,10 @@ // // SPDX-License-Identifier: GPL-3.0-or-later -#include "solprop_native/chebyshev.hpp" -#include "solprop_native/execution.hpp" -#include "solprop_native/stage_tables.hpp" -#include "solprop_native/z_table.hpp" +#include "solprop_native/march/chebyshev.hpp" +#include "solprop_native/march/stage_tables.hpp" +#include "solprop_native/march/z_table.hpp" +#include "solprop_native/runtime/execution.hpp" #include diff --git a/native/tests/test_band_plasma.py b/native/tests/test_band_plasma.py index 8a5d73f01..30656c3d6 100644 --- a/native/tests/test_band_plasma.py +++ b/native/tests/test_band_plasma.py @@ -8,8 +8,8 @@ symmetric taps before the cumulative trapezoid, closes the window's ends by point reflection and floors the exposure at zero (solprop ``RunningIntegral``). Every build runs it as a team column in whole-row passes, a band filter then a chunked scan per -running integral (``include/solprop_native/plasma.hpp``); a host team splits a row -between its threads when a chunk has fewer nodes than threads, and skips each row's +running integral (``include/solprop_native/response/plasma.hpp``); a host team splits a +row between its threads when a chunk has fewer nodes than threads, and skips each row's quiet samples. Its statistics and run diagnostics come from the band scan. The bounds are the equivalence ladder's (solprop's ``accelerator_bounds``): the transforms' depth plus the plasma column's sums, which the band rule widens by its ``2K + 1`` taps per diff --git a/native/tests/test_band_stress.py b/native/tests/test_band_stress.py index 9304d8ab5..ba38b9315 100644 --- a/native/tests/test_band_stress.py +++ b/native/tests/test_band_stress.py @@ -6,10 +6,10 @@ The host column runs whole-row passes: each running integral is a band filter and a chunked scan, a team may split a row between its threads, and each pass is limited to -its row's live window (``include/solprop_native/plasma.hpp``). These cases aim at what -that can get wrong: a preionised field whose rate is identically zero (the current -lives with no exposure), a field just above threshold (a tiny exposure from a few -cycle peaks), pulses cut by either end of the window (the point reflections and the +its row's live window (``include/solprop_native/response/plasma.hpp``). These cases aim +at what that can get wrong: a preionised field whose rate is identically zero (the +current lives with no exposure), a field just above threshold (a tiny exposure from a +few cycle peaks), pulses cut by either end of the window (the point reflections and the windows reaching an end), a mixture there (several channels), and a two-component field on the radial route. Each is compared at one evaluation, one fixed step and ten, under a fixed-order method, with :mod:`test_band_plasma`'s bounds, decided before these diff --git a/native/tests/test_coupled_flow.py b/native/tests/test_coupled_flow.py index 042b0d8fc..f8c41553a 100644 --- a/native/tests/test_coupled_flow.py +++ b/native/tests/test_coupled_flow.py @@ -7,7 +7,7 @@ The engine carries each stage input forward and its source back by the step's actions, adds a frozen-residual flow's explicit residual, and carries the accepted step by the whole action -(``src/coupled_flow.cpp``); it also forms the actions, from each +(``src/response/coupled_flow.cpp``); it also forms the actions, from each section's eigenbasis or by the reference's scaling and squaring, and measures the whole step's conditioning, which the adapter checks against the certificate. Two capillary cases are propagated ten fixed steps on both backends: a scalar coherent diff --git a/native/tests/test_fftw_plan_refusal.py b/native/tests/test_fftw_plan_refusal.py index a5fde57f7..8c3f5b697 100644 --- a/native/tests/test_fftw_plan_refusal.py +++ b/native/tests/test_fftw_plan_refusal.py @@ -5,7 +5,7 @@ """A mistyped FFTW planning effort is refused (SEP 43 inventory, item 73; #460). Native takes the effort from ``SOLPROP_FFTW_PLAN`` (``fftw_seed_effort`` in -``src/fftw_wisdom.cpp``). Read as ``measure``, a typo would change a +``src/transforms/fftw_wisdom.cpp``). Read as ``measure``, a typo would change a run's plans in silence, so it is refused in a fresh process, as the neighbouring thread variables are. """ diff --git a/native/tests/test_general_modal.py b/native/tests/test_general_modal.py index ca0797f60..de396aae8 100644 --- a/native/tests/test_general_modal.py +++ b/native/tests/test_general_modal.py @@ -5,8 +5,8 @@ """The general modal route of solprop on the native backend, against ``numpy_ref``. The route's source is a second execution path beside the -node pipeline (``src/general_modal.cpp``); everything before and after it -- the -lab-frame stage input, the in-stage absorber, the prefactor and the interaction +node pipeline (``src/response/general_modal.cpp``); everything before and after it -- +the lab-frame stage input, the in-stage absorber, the prefactor and the interaction picture -- is the engine's existing code. Each case propagates ten fixed steps on both backends from the same problem and compares the fields against a bound derived before the run (:func:`_parity_bound`). diff --git a/src/solprop/backends/engines.py b/src/solprop/backends/engines.py index 98d8715b4..f75205de4 100644 --- a/src/solprop/backends/engines.py +++ b/src/solprop/backends/engines.py @@ -8,9 +8,9 @@ state-sized operation a march asks of the workspace that holds the state. The two protocols here mirror the native marches' C++ interfaces one-to-one, every argument included: :class:`RungeKuttaEngine` is ``rk::Engine`` -(``native/include/solprop_native/rk_march.hpp``), which the low-storage step implements -too, and :class:`AdamsEngine` is ``march::Engine`` -(``native/include/solprop_native/adams_march.hpp``). The reference's engines are +(``native/include/solprop_native/march/rk_march.hpp``), which the low-storage step +implements too, and :class:`AdamsEngine` is ``march::Engine`` +(``native/include/solprop_native/march/adams_march.hpp``). The reference's engines are :class:`~solprop.backends.numpy_ref.engines.NumpyRungeKutta`, the low-storage step :class:`~solprop.backends.low_storage.HostLowStorage` and the Adams history :class:`~solprop.backends.numpy_ref.multistep.HostHistory`. diff --git a/src/solprop/backends/numpy_ref/dense.py b/src/solprop/backends/numpy_ref/dense.py index 5b593f3d3..cd888a6f6 100644 --- a/src/solprop/backends/numpy_ref/dense.py +++ b/src/solprop/backends/numpy_ref/dense.py @@ -5,7 +5,7 @@ """The reference's dense output: a plane interpolated inside the step that crossed it. The counterparts of native's ``engines::RungeKuttaDense`` and ``engines::AdamsDense`` -(``native/src/dense.cpp``), which implement the session's ``Dense``: the session +(``native/src/march/dense.cpp``), which implement the session's ``Dense``: the session (:class:`~solprop.backends.host.session.HostSession`) decides when a plane is observed inside the last accepted step, and asks one of these to hold the committed state aside, make the state the interpolated field there, and restore the committed state before diff --git a/tests/backends/test_engines.py b/tests/backends/test_engines.py index 04d8df3c0..09a4b9a24 100644 --- a/tests/backends/test_engines.py +++ b/tests/backends/test_engines.py @@ -33,7 +33,8 @@ "reading": "(self) -> solprop.backends.engines.BoundaryReading", "in_flight": "(self) -> bool", } -"""``rk::Engine`` (``native/include/solprop_native/rk_march.hpp``), member by member.""" +"""``rk::Engine`` (``native/include/solprop_native/march/rk_march.hpp``), member by +member.""" ADAMS_ENGINE = { "start": "(self, z: float) -> None", @@ -52,7 +53,7 @@ "end_attempt": "(self, outcome: solprop.backends.engines.AttemptOutcome) -> None", "reading": "(self) -> solprop.backends.engines.BoundaryReading", } -"""``march::Engine`` (``native/include/solprop_native/adams_march.hpp``).""" +"""``march::Engine`` (``native/include/solprop_native/march/adams_march.hpp``).""" RECORDS = { AcceptRecord: ("stiffness", "removed", "found"), @@ -62,7 +63,8 @@ """The records the protocols pass, field by field.""" OUTCOMES = {"REJECTED": 0, "ACCEPTED": 1, "ABANDONED": 2} -"""``HostOutcome`` (``native/include/solprop_native/host_source.hpp``), by value.""" +"""``HostOutcome`` (``native/include/solprop_native/runtime/host_source.hpp``), by +value.""" @pytest.mark.parametrize( diff --git a/tests/backends/test_native_floating_point_licence.py b/tests/backends/test_native_floating_point_licence.py index ab4e5ba4c..82269a482 100644 --- a/tests/backends/test_native_floating_point_licence.py +++ b/tests/backends/test_native_floating_point_licence.py @@ -46,12 +46,12 @@ ("src/general/kernels.hpp", "decayed_fractions"), ("src/general/kernels.hpp", "envelope_fraction"), # The small-argument phi-functions (multistep._phi, _weights). - ("include/solprop_native/adams_phi.hpp", "adams_phi_fixed"), - ("include/solprop_native/adams_phi.hpp", "adams_lanes_classify"), - ("include/solprop_native/adams_phi.hpp", "adams_lanes_phi"), - ("include/solprop_native/adams_phi.hpp", "adams_lanes_large_exponential"), - ("include/solprop_native/lagrange_basis.hpp", "basis_rows"), - ("include/solprop_native/lagrange_basis.hpp", "trial_tables"), + ("include/solprop_native/march/adams_phi.hpp", "adams_phi_fixed"), + ("include/solprop_native/march/adams_phi.hpp", "adams_lanes_classify"), + ("include/solprop_native/march/adams_phi.hpp", "adams_lanes_phi"), + ("include/solprop_native/march/adams_phi.hpp", "adams_lanes_large_exponential"), + ("include/solprop_native/march/lagrange_basis.hpp", "basis_rows"), + ("include/solprop_native/march/lagrange_basis.hpp", "trial_tables"), # The stable phase increments. ("src/general/kernels.hpp", "phase_sum"), ("src/general/kernels.hpp", "phase_block"), @@ -70,9 +70,9 @@ ("src/general/trial.cpp", "adams_lanes_rate"), ("src/general/trial.cpp", "adams_defect_block"), # The far field's plane radius (mask_plane_radius). - ("include/solprop_native/far_field.hpp", "far_field_radius"), - ("include/solprop_native/far_field.hpp", "far_field_offset"), - ("include/solprop_native/far_field.hpp", "far_field_scaled_radius"), + ("include/solprop_native/response/far_field.hpp", "far_field_radius"), + ("include/solprop_native/response/far_field.hpp", "far_field_offset"), + ("include/solprop_native/response/far_field.hpp", "far_field_scaled_radius"), ] """Each helper the licence protects, and the file that defines it. @@ -117,7 +117,7 @@ def test_every_protected_helper_is_strict(path, name): def test_a_protected_marker_still_does_something(): """The macro names its two mechanisms: the list cannot see an empty one.""" - source = NATIVE / "include" / "solprop_native" / "fp.hpp" + source = NATIVE / "include" / "solprop_native" / "runtime" / "fp.hpp" if not source.exists(): pytest.skip("the native sources are not in this tree") text = source.read_text(encoding="utf-8") @@ -144,16 +144,16 @@ def test_a_protected_marker_still_does_something(): STRICT_SOURCES = ( - "src/adams_march.cpp", - "src/watch.cpp", - "src/rk_march.cpp", - "src/session.cpp", - "src/statistics_plan.cpp", - "src/session_widths.cpp", - "src/low_storage.cpp", - "src/transverse.cpp", - "src/session_edge.cpp", - "src/dense.cpp", + "src/march/adams_march.cpp", + "src/session/watch.cpp", + "src/march/rk_march.cpp", + "src/session/session.cpp", + "src/session/statistics_plan.cpp", + "src/session/session_widths.cpp", + "src/march/low_storage.cpp", + "src/session/transverse.cpp", + "src/session/session_edge.cpp", + "src/march/dense.cpp", ) """The translation units built without contraction (ADR-060, SEP 45, SEP 47). @@ -178,7 +178,7 @@ def test_the_build_system_asks_for_none_of_the_refused_flags(): # The strict translation units round as Python's do, on every compiler, and refuse # the general workspace's contracting headers. strict = re.search( - r"set\(SOLPROP_NATIVE_STRICT_SOURCES\s+((?:src/\w+\.cpp\s*)+)\)", text + r"set\(SOLPROP_NATIVE_STRICT_SOURCES\s+((?:src/[\w/]+\.cpp\s*)+)\)", text ) assert strict is not None assert tuple(strict.group(1).split()) == STRICT_SOURCES diff --git a/tests/backends/test_native_rk_traces.py b/tests/backends/test_native_rk_traces.py index 8e81b4fe3..70ad7a21a 100644 --- a/tests/backends/test_native_rk_traces.py +++ b/tests/backends/test_native_rk_traces.py @@ -5,7 +5,7 @@ """The native Runge-Kutta march against the decision corpus (SEP 45 section 7.2). ``tests/data/rk_traces.json.gz`` holds the decisions solprop's march made on scripted -norms before SEP 45 refactored it. The native march (``native/src/rk_march.cpp``), +norms before SEP 45 refactored it. The native march (``native/src/march/rk_march.cpp``), driven through its adapter on a scripted engine, must make every one of them: each attempt's step and norm, acceptance, landing, the start-up rule, the caps, natural stops, and how it stopped -- at a call's bound of one attempt, seven, and the default, diff --git a/tests/backends/test_native_vector_math.py b/tests/backends/test_native_vector_math.py index f89f0feb9..4039886ac 100644 --- a/tests/backends/test_native_vector_math.py +++ b/tests/backends/test_native_vector_math.py @@ -5,9 +5,9 @@ """The native backend's host vector maths against a 50-digit reference (ADR-059). The CPU kernels form their transcendentals in passes over blocks through one layer -(``native/include/solprop_native/vmath.hpp``): SLEEF's u10 functions on x86-64 builds, -the system libm on Arm. Either way each of ``exp``, ``log``, ``sin`` and ``cos`` is -within 1.0 ulp of the correctly rounded value (SLEEF documents 1.0 ulp over each +(``native/include/solprop_native/runtime/vmath.hpp``): SLEEF's u10 functions on x86-64 +builds, the system libm on Arm. Either way each of ``exp``, ``log``, ``sin`` and ``cos`` +is within 1.0 ulp of the correctly rounded value (SLEEF documents 1.0 ulp over each function's whole domain; glibc's and Apple's are within it), measured here against mpmath at 50 significant digits over the argument ranges the kernels form: diff --git a/tests/flow_parity.py b/tests/flow_parity.py index e7a2750bc..5bdb6d3ee 100644 --- a/tests/flow_parity.py +++ b/tests/flow_parity.py @@ -405,8 +405,8 @@ def augmented_formation_bound(n_modes, count, size, spread, row_sum, real): ``M_k = k! phi_(k+1)(X)``, ``X = x I + h T`` with ``x`` the moments' argument (``Delta Phi``, less ``i sigma`` under a carrier frame), off the first block row of the exponential of the augmented matrix ``A`` of ``count + 1`` blocks - (``native/include/solprop_native/flow_functions.hpp``), whose block rows past the - first are a shift. The native passes are the full augmented scaling and + (``native/include/solprop_native/response/flow_functions.hpp``), whose block rows + past the first are a shift. The native passes are the full augmented scaling and squaring's to the sign of a zero, at ``N = n (count + 1)`` channels and the ``s`` squarings of ``A``'s row-sum norm `row_sum`. From 3650dfa58af017690c6301815a58e607c966fc4d Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 16 Nov 2026 12:52:06 +0000 Subject: [PATCH 21/21] Include the standard headers the split native headers use MSVC's library does not bring in transitively as libc++ does, which broke the Windows arm64 build after the run records left workspace.hpp. Also point a test docstring at cpu_execution.cpp. Co-Authored-By: Claude Opus 5.5 --- .../include/solprop_native/march/stage_tables.hpp | 1 + .../include/solprop_native/problem/run_records.hpp | 2 ++ native/include/solprop_native/problem/workspace.hpp | 1 + .../solprop_native/response/general_modal.hpp | 1 + .../solprop_native/runtime/cpu_execution.hpp | 1 + .../solprop_native/session/statistics_plan.hpp | 1 + .../include/solprop_native/session/transverse.hpp | 1 + native/include/solprop_native/session/watch.hpp | 1 + .../include/solprop_native/transforms/fft_plan.hpp | 1 + tests/backends/forced_threads.py | 13 +++++++------ 10 files changed, 17 insertions(+), 6 deletions(-) diff --git a/native/include/solprop_native/march/stage_tables.hpp b/native/include/solprop_native/march/stage_tables.hpp index 822aeb5a6..594cf9cec 100644 --- a/native/include/solprop_native/march/stage_tables.hpp +++ b/native/include/solprop_native/march/stage_tables.hpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include diff --git a/native/include/solprop_native/problem/run_records.hpp b/native/include/solprop_native/problem/run_records.hpp index e2f4e0323..e1374c5e7 100644 --- a/native/include/solprop_native/problem/run_records.hpp +++ b/native/include/solprop_native/problem/run_records.hpp @@ -20,11 +20,13 @@ #include #include #include +#include #include #include #include #include #include +#include #include /* diff --git a/native/include/solprop_native/problem/workspace.hpp b/native/include/solprop_native/problem/workspace.hpp index 8e85c3da1..592508987 100644 --- a/native/include/solprop_native/problem/workspace.hpp +++ b/native/include/solprop_native/problem/workspace.hpp @@ -16,6 +16,7 @@ #include "solprop_native/session/session.hpp" #include +#include #include #include #include diff --git a/native/include/solprop_native/response/general_modal.hpp b/native/include/solprop_native/response/general_modal.hpp index 37cdd3dac..74037c47b 100644 --- a/native/include/solprop_native/response/general_modal.hpp +++ b/native/include/solprop_native/response/general_modal.hpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace solprop_native { diff --git a/native/include/solprop_native/runtime/cpu_execution.hpp b/native/include/solprop_native/runtime/cpu_execution.hpp index fd977c9f4..49fcbab8b 100644 --- a/native/include/solprop_native/runtime/cpu_execution.hpp +++ b/native/include/solprop_native/runtime/cpu_execution.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include diff --git a/native/include/solprop_native/session/statistics_plan.hpp b/native/include/solprop_native/session/statistics_plan.hpp index 17ce56da9..2c0340089 100644 --- a/native/include/solprop_native/session/statistics_plan.hpp +++ b/native/include/solprop_native/session/statistics_plan.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include /* diff --git a/native/include/solprop_native/session/transverse.hpp b/native/include/solprop_native/session/transverse.hpp index 3707ca2f3..fb558e4d7 100644 --- a/native/include/solprop_native/session/transverse.hpp +++ b/native/include/solprop_native/session/transverse.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include diff --git a/native/include/solprop_native/session/watch.hpp b/native/include/solprop_native/session/watch.hpp index f5c8f4c1e..be5d33973 100644 --- a/native/include/solprop_native/session/watch.hpp +++ b/native/include/solprop_native/session/watch.hpp @@ -11,6 +11,7 @@ #include #include #include +#include #include /* diff --git a/native/include/solprop_native/transforms/fft_plan.hpp b/native/include/solprop_native/transforms/fft_plan.hpp index 0e755ac82..bb07e77b9 100644 --- a/native/include/solprop_native/transforms/fft_plan.hpp +++ b/native/include/solprop_native/transforms/fft_plan.hpp @@ -94,6 +94,7 @@ void seed_row_fft_wisdom(const Execution& execution, const Input& input, #include #include #include +#include #endif namespace solprop_native { diff --git a/tests/backends/forced_threads.py b/tests/backends/forced_threads.py index 8a455b57b..fcc2c7492 100644 --- a/tests/backends/forced_threads.py +++ b/tests/backends/forced_threads.py @@ -5,12 +5,13 @@ """Run a case on ``native`` with every thread pool forced, in a fresh process. Native sizes each operation by the work it holds (``operation_threads`` in -``native/src/core.cpp``), so a small case at ``SOLPROP_NATIVE_THREADS=4`` runs most of -its kernels on one thread and compares one thread with one thread. An explicit pool -bypasses that rule, which is what ``numba_cpu``'s ``regime="large"`` was for. So -:func:`forced_run` names every pool, and a test reads the widths actually taken from the -run's ``execution_sizing`` through :func:`widest` and asserts them. A comparison whose -threads never ran would otherwise pass. +``native/src/runtime/cpu_execution.cpp``), so a small case at +``SOLPROP_NATIVE_THREADS=4`` runs most of its kernels on one thread and compares one +thread with one thread. An explicit pool bypasses that rule, which is what +``numba_cpu``'s ``regime="large"`` was for. So :func:`forced_run` names every pool, and +a test reads the widths actually taken from the run's ``execution_sizing`` through +:func:`widest` and asserts them. A comparison whose threads never ran would otherwise +pass. The Kokkos runtime reads its pools once per process (``matched.py``), so each width is a fresh process running ``native_forced_thread_probe.py``; :func:`forced_run` returns the

("PlanePlan", fields::value("z", &P::z), + fields::array("planes", &P::planes, E::of("n_z")), + fields::value("next_plane", &P::next_plane), + fields::value("launch_pending", &P::launch_pending), + fields::value("step_planes", &P::step_planes), + fields::value("dense", &P::dense), + fields::value("period", &P::period), + fields::value("replay_steps", &P::replay_steps), + fields::value("replay_length", &P::replay_length), + fields::value("attempt_capacity", &P::attempt_capacity), + fields::value("row_capacity", &P::row_capacity), + fields::value("call_attempts", &P::call_attempts), + fields::value("call_seconds", &P::call_seconds)); +} + +/** `solprop.backends.records.Resumption`: the numbers a resume reads by name. */ +struct ResumptionRecord { + bool record = false; + bool collar_quiet = false; + std::map numbers; + std::optional>> unwatched; + std::optional> resumed_at; + std::optional>> planes; +}; + +[[nodiscard]] constexpr auto describe(const ResumptionRecord* /*unused*/) { + using R = ResumptionRecord; + return fields::table( + "Resumption", fields::value("record", &R::record), + fields::value("collar_quiet", &R::collar_quiet), + fields::value("numbers", &R::numbers), fields::value("unwatched", &R::unwatched), + fields::value("resumed_at", &R::resumed_at), fields::value("planes", &R::planes)); +} + +/** `solprop.backends.records.EdgeResumption`. */ +struct EdgeResumptionRecord { + bool record = false; + std::map numbers; + std::optional> edge_baseline; + std::optional> edge_max; + std::optional> edge_max_raw; + std::optional reads; + std::optional transverse_status; +}; + +[[nodiscard]] constexpr auto describe(const EdgeResumptionRecord* /*unused*/) { + using R = EdgeResumptionRecord; + return fields::table("EdgeResumption", fields::value("record", &R::record), + fields::value("numbers", &R::numbers), + fields::value("edge_baseline", &R::edge_baseline), + fields::value("edge_max", &R::edge_max), + fields::value("edge_max_raw", &R::edge_max_raw), + fields::value("reads", &R::reads), + fields::value("transverse_status", &R::transverse_status)); +} + +/** `solprop.backends.records.MonitorContinuation`. */ +struct ContinuationRecord { + ResumptionRecord record; + EdgeResumptionRecord edge; + bool plane = false; + std::vector> unwatched; +}; + +[[nodiscard]] constexpr auto describe(const ContinuationRecord* /*unused*/) { + using C = ContinuationRecord; + return fields::table("MonitorContinuation", fields::value("record", &C::record), + fields::value("edge", &C::edge), + fields::value("plane", &C::plane), + fields::value("unwatched", &C::unwatched)); +} + +/** `solprop.backends.records.MonitorRecords`. */ +struct MonitorRecordsRecord { + watch::Config config; + std::optional edge; + std::optional continuation; +}; + +[[nodiscard]] constexpr auto describe(const MonitorRecordsRecord* /*unused*/) { + using M = MonitorRecordsRecord; + return fields::table("MonitorRecords", fields::value("config", &M::config), + fields::value("edge", &M::edge), + fields::value("continuation", &M::continuation)); +} + +/** `solprop.backends.records.RunRecords`: the method's policy is the Adams march's or + the Runge-Kutta march's, told apart by their fields. */ +struct RunRecordsRecord { + std::variant policy; + std::optional low_storage; + std::optional schedule; + PlanePlanRecord planes; + std::optional monitor; +}; + +[[nodiscard]] constexpr auto describe(const RunRecordsRecord* /*unused*/) { + using R = RunRecordsRecord; + return fields::table("RunRecords", fields::value("policy", &R::policy), + fields::value("low_storage", &R::low_storage), + fields::value("schedule", &R::schedule), + fields::value("planes", &R::planes), + fields::value("monitor", &R::monitor)); +} + +/** One persistent row of a resumed Adams history, `(n_slots, rows, n_omega)`. */ +struct HistoryRow { + std::string name; + std::vector values; + std::size_t n_slots = 0; + std::size_t n_rows = 0; + std::size_t n_omega = 0; +}; + +[[nodiscard]] constexpr auto describe(const HistoryRow* /*unused*/) { + using H = HistoryRow; + using E = fields::Extent; + return fields::table("HistoryRow", fields::value("name", &H::name), + fields::array("values", &H::values, E::of("n_slots"), + E::of("n_rows"), E::of("n_omega")), + fields::count("n_slots", &H::n_slots), + fields::count("n_rows", &H::n_rows), + fields::count("n_omega", &H::n_omega)); +} + +/** What one run starts from (`Workspace::start`): its records, the initial field and + noise as the state takes them (`set_initial`), borrowed from the caller for the + call, the march's state at the start (the Adams march's or the Runge-Kutta march's, + none for a fresh state of its own), a resumed history's rows, the statistics plans, + the monitor's launch readings of the input noise (its floor: energy, then four + regions' shares; and the edge's), and the run's monitor weights and edge tables where + the state lives. */ +struct RunStart { + RunRecordsRecord records; + std::span field; + std::span noise; + std::optional adams; + std::optional runge_kutta; + std::vector rows; + std::shared_ptr plane_statistics; + std::shared_ptr row_statistics; + std::optional> floor; + std::optional edge_noise; + std::optional monitor; + std::optional edge; +}; + +[[nodiscard]] constexpr auto describe(const RunStart* /*unused*/) { + using R = RunStart; + using E = fields::Extent; + return fields::table( + "RunStart", fields::value("records", &R::records), + fields::array("field", &R::field, E::any()), + fields::array("noise", &R::noise, E::any()), fields::value("adams", &R::adams), + fields::value("runge_kutta", &R::runge_kutta), fields::value("rows", &R::rows), + fields::value("plane_statistics", &R::plane_statistics), + fields::value("row_statistics", &R::row_statistics), + fields::value("floor", &R::floor), fields::value("edge_noise", &R::edge_noise), + fields::value("monitor", &R::monitor), fields::value("edge", &R::edge)); +} + +/** The Runge-Kutta march's policy from its record (kernel contract section 12): the + controller's gains, factors, lag and acceptance rule flattened in, and the + equivalent norm's power `estimator_order + 1` and its reciprocal. */ +[[nodiscard]] rk::Policy to_native(const RungeKuttaPolicyRecord& record); +/** A fixed schedule's steps and edges, accumulated in step order. */ +[[nodiscard]] march::Schedule to_native(const ScheduleRecord& record); +/** A monitor record's resumption, each number by name; refuses an unknown name. */ +[[nodiscard]] watch::Resumption to_native(const ResumptionRecord& record); +/** An edge record's resumption, likewise. */ +[[nodiscard]] transverse::Resumption to_native(const EdgeResumptionRecord& record); + +/** A test's injected failure by name: the stage (`startup`, `trial`, `trial_source` + or `accept`) and the category (`convergence`, `value`, `backend`, `sigint` or + `sigterm`). */ +[[nodiscard]] engines::MarchFault +fault_from(const std::string& stage, std::size_t after, const std::string& category); + +/** The stage tables' consistency (the checks `adams_stage_tables` makes). */ +void validate(const stage::Tables& tables); + +} // namespace solprop_native diff --git a/native/include/solprop_native/workspace.hpp b/native/include/solprop_native/workspace.hpp index 54dd941c6..42821aafb 100644 --- a/native/include/solprop_native/workspace.hpp +++ b/native/include/solprop_native/workspace.hpp @@ -7,21 +7,17 @@ #include "solprop_native/adams_march.hpp" #include "solprop_native/engines.hpp" #include "solprop_native/execution.hpp" -#include "solprop_native/fields.hpp" #include "solprop_native/general_state.hpp" #include "solprop_native/low_storage.hpp" #include "solprop_native/problem.hpp" -#include "solprop_native/readings.hpp" #include "solprop_native/rk_march.hpp" +#include "solprop_native/run_records.hpp" #include "solprop_native/session.hpp" #include "solprop_native/stage_tables.hpp" -#include "solprop_native/transverse.hpp" -#include "solprop_native/watch.hpp" #include #include #include -#include #include #include #include @@ -36,491 +32,9 @@ * applied from the problem's `Setup`; the lease between it and the one session or * probe that uses it; the coupled flow's guard; the marches' construction from a run's * records; and the timing probe of section 5.5. - * - * The binding converts Python's records into the structs here (each with its field - * table, `fields.hpp`), releases the interpreter lock and forwards; the logic is here, - * where the native tests reach it. */ namespace solprop_native { -// ------------------------------------------------------------------ the set-once data - -/** One general z table made resident at construction (solprop SEP 29 G1): its - breakpoints, admitted span and coefficients `(n_edges - 1, n_terms, n_values)`. */ -struct ZTable { - TableKind kind = TableKind::phi; - std::vector edges; - double lower = 0.0; - double upper = 0.0; - std::vector coefficients; - std::size_t n_terms = 0; - std::size_t n_values = 0; -}; - -[[nodiscard]] constexpr auto describe(const ZTable* /*unused*/) { - using E = fields::Extent; - return fields::table( - "ZTable", fields::value("kind", &ZTable::kind), - fields::array("edges", &ZTable::edges, E::of("n_edges")), - fields::value("lower", &ZTable::lower), fields::value("upper", &ZTable::upper), - fields::array("coefficients", &ZTable::coefficients, E::of("n_edges", -1), - E::of("n_terms"), E::of("n_values")), - fields::count("n_terms", &ZTable::n_terms), - fields::count("n_values", &ZTable::n_values)); -} - -/** The z tables' position slots (`stage::ZSlots::configure`). */ -struct ZSlotsConfig { - bool phi = false; - bool prefactor = false; - std::size_t slots = 0; - bool rate = false; -}; - -[[nodiscard]] constexpr auto describe(const ZSlotsConfig* /*unused*/) { - using Z = ZSlotsConfig; - return fields::table("ZSlots", fields::value("phi", &Z::phi), - fields::value("prefactor", &Z::prefactor), - fields::value("slots", &Z::slots), - fields::value("rate", &Z::rate)); -} - -/** A coupled flow's certificate and the guard's constants (solprop SEP 45 phase 4): - each section's row-sum norm, each section's conditioning bound per active bin - `(n_sections, n_bins)`, the Taylor scale, the most squarings and the rounding - allowance, as solprop's `_flow_squarings` and `_check_flow_conditioning` read - them. */ -struct FlowGuard { - std::vector row_sum; - std::vector condition_bound; - double taylor_scale = 0.0; - int max_squarings = 0; - double allowance = 0.0; -}; - -[[nodiscard]] constexpr auto describe(const FlowGuard* /*unused*/) { - using F = FlowGuard; - using E = fields::Extent; - return fields::table("FlowGuard", - fields::array("row_sum", &F::row_sum, E::of("n_sections")), - fields::array("condition_bound", &F::condition_bound, - E::of("n_sections"), E::of("n_bins")), - fields::value("taylor_scale", &F::taylor_scale), - fields::value("max_squarings", &F::max_squarings), - fields::value("allowance", &F::allowance)); -} - -/** The Runge-Kutta trial's tableau nodes and error control (solprop SEP 45): the - nodes, the tolerances, a weighted controller's weights (empty for none) and the - tableau's dense-output weights (empty without dense output). */ -struct RungeKuttaConfig { - std::vector nodes; - double rtol = 0.0; - double atol = 0.0; - std::vector weights; - std::vector dense_weights; -}; - -[[nodiscard]] constexpr auto describe(const RungeKuttaConfig* /*unused*/) { - using R = RungeKuttaConfig; - using E = fields::Extent; - return fields::table( - "RungeKuttaConfig", fields::array("nodes", &R::nodes, E::of("n_stages")), - fields::value("rtol", &R::rtol), fields::value("atol", &R::atol), - fields::array("weights", &R::weights, E::any()), - fields::array("dense_weights", &R::dense_weights, E::any())); -} - -/** The resident Adams history (solprop SEP 16, SEP 39): its rows, whether trials carry - the transported compensation, whether the steps are kept for interpolation, the - fused (interaction) formulation, and the controller's scale. */ -struct AdamsConfig { - std::size_t slots = 0; - bool compensated = false; - bool interpolate = false; - bool fused = false; - double rtol = 0.0; - double atol = 0.0; - std::vector weights; -}; - -[[nodiscard]] constexpr auto describe(const AdamsConfig* /*unused*/) { - using A = AdamsConfig; - using E = fields::Extent; - return fields::table( - "AdamsConfig", fields::value("slots", &A::slots), - fields::value("compensated", &A::compensated), - fields::value("interpolate", &A::interpolate), fields::value("fused", &A::fused), - fields::value("rtol", &A::rtol), fields::value("atol", &A::atol), - fields::array("weights", &A::weights, E::any())); -} - -/** The boundary monitor's weights where the state lives (solprop SEP 21), a run's: each - run's monitor is its own (`RunRequest.for_run`). `split` says the monitor reads the - split collar's crossing, whose walk-off buckets are the problem's (`Occupancy`). */ -struct MonitorBinding { - std::vector weights; - std::vector column_weights; - bool split = false; -}; - -[[nodiscard]] constexpr auto describe(const MonitorBinding* /*unused*/) { - using M = MonitorBinding; - using E = fields::Extent; - return fields::table("MonitorBinding", - fields::array("weights", &M::weights, E::any()), - fields::array("column_weights", &M::column_weights, E::any()), - fields::value("split", &M::split)); -} - -/** The split collar's walk-off bucket of each bin (solprop #440), the problem's: set - once, and bound to the state by the first run whose monitor reads it. */ -struct Occupancy { - std::vector buckets; - std::size_t count = 0; -}; - -[[nodiscard]] constexpr auto describe(const Occupancy* /*unused*/) { - using O = Occupancy; - using E = fields::Extent; - return fields::table( - "Occupancy", - fields::array("buckets", &O::buckets, fields::Dtype::uint8, E::any()), - fields::value("count", &O::count)); -} - -/** The transverse edge's tables where the state lives (solprop SEP 44), a run's: the - faces', the clip's and the totals' weights, a reduction's synthesis (empty for - none), the first row, and a requested collar's lines. */ -struct EdgeBinding { - std::vector faces; - std::vector clip; - std::vector totals; - std::vector synthesis; - std::size_t first_row = 0; - std::optional lines; -}; - -[[nodiscard]] constexpr auto describe(const EdgeBinding* /*unused*/) { - using B = EdgeBinding; - using E = fields::Extent; - return fields::table("EdgeBinding", fields::array("faces", &B::faces, E::any()), - fields::array("clip", &B::clip, E::any()), - fields::array("totals", &B::totals, E::any()), - fields::array("synthesis", &B::synthesis, E::any()), - fields::value("first_row", &B::first_row), - fields::value("lines", &B::lines)); -} - -/** Everything fixed for a workspace beside the problem's groups (SEP 52 section 5.3). - Each part is absent where the problem has none; `crossing_workers` is the threads a - crossing's copy may use (zero keeps the state's default). */ -struct Setup { - std::optional stage_tables; - std::optional z_slots; - std::vector z_tables; - std::optional flow_guard; - std::optional runge_kutta; - std::optional adams; - std::optional low_storage; - std::optional occupancy; - std::size_t crossing_workers = 0; -}; - -[[nodiscard]] constexpr auto describe(const Setup* /*unused*/) { - using S = Setup; - return fields::table( - "Setup", fields::value("stage_tables", &S::stage_tables), - fields::value("z_slots", &S::z_slots), fields::value("z_tables", &S::z_tables), - fields::value("flow_guard", &S::flow_guard), - fields::value("runge_kutta", &S::runge_kutta), fields::value("adams", &S::adams), - fields::value("low_storage", &S::low_storage), - fields::value("occupancy", &S::occupancy), - fields::value("crossing_workers", &S::crossing_workers)); -} - -// -------------------------------------------------------------------- the run records -// The structs below mirror `solprop.backends.records`, field for field, so that one -// table holds each to its Python dataclass; `to_native` forms what the marches and the -// session read from them. - -/** `solprop.stepper.controller.PIController`. */ -struct ControllerRecord { - double rtol = 0.0; - double atol = 0.0; - double safety = 0.0; - double max_factor = 0.0; - double min_factor = 0.0; - double beta1 = 0.0; - double beta2 = 0.0; - std::int64_t lag = 0; - bool reject_free = false; - std::optional> weights; - std::int64_t estimator_order = 0; - std::string region; -}; - -[[nodiscard]] constexpr auto describe(const ControllerRecord* /*unused*/) { - using C = ControllerRecord; - using E = fields::Extent; - return fields::table( - "PIController", fields::value("rtol", &C::rtol), fields::value("atol", &C::atol), - fields::value("safety", &C::safety), fields::value("max_factor", &C::max_factor), - fields::value("min_factor", &C::min_factor), fields::value("beta1", &C::beta1), - fields::value("beta2", &C::beta2), fields::value("lag", &C::lag), - fields::value("reject_free", &C::reject_free), - fields::array("weights", &C::weights, E::of("n_omega")), - fields::value("estimator_order", &C::estimator_order), - fields::value("region", &C::region)); -} - -/** `solprop.backends.records.RungeKuttaPolicy`. */ -struct RungeKuttaPolicyRecord { - ControllerRecord controller; - double dz_min = 0.0; - double dz_max = 0.0; - bool auto_init = false; - double stretch = 0.0; - double z_end = 0.0; - std::vector save_planes; - bool interpolate = false; - std::vector hard_kinks; - std::vector breakpoints; - int landing_ulps = 0; - double start_growth = 0.0; - std::optional quiet_interval; -}; - -[[nodiscard]] constexpr auto describe(const RungeKuttaPolicyRecord* /*unused*/) { - using R = RungeKuttaPolicyRecord; - return fields::table( - "RungeKuttaPolicy", fields::value("controller", &R::controller), - fields::value("dz_min", &R::dz_min), fields::value("dz_max", &R::dz_max), - fields::value("auto_init", &R::auto_init), fields::value("stretch", &R::stretch), - fields::value("z_end", &R::z_end), fields::value("save_planes", &R::save_planes), - fields::value("interpolate", &R::interpolate), - fields::value("hard_kinks", &R::hard_kinks), - fields::value("breakpoints", &R::breakpoints), - fields::value("landing_ulps", &R::landing_ulps), - fields::value("start_growth", &R::start_growth), - fields::value("quiet_interval", &R::quiet_interval)); -} - -/** `solprop.stepper.schedule.FixedSchedule`: its steps and start; the edges are - accumulated in step order (`FixedSchedule.edges`). */ -struct ScheduleRecord { - std::vector steps; - double z_start = 0.0; -}; - -[[nodiscard]] constexpr auto describe(const ScheduleRecord* /*unused*/) { - using E = fields::Extent; - return fields::table( - "FixedSchedule", fields::array("steps", &ScheduleRecord::steps, E::of("n_steps")), - fields::value("z_start", &ScheduleRecord::z_start)); -} - -/** `solprop.backends.records.PlanePlan`. */ -struct PlanePlanRecord { - double z = 0.0; - std::vector planes; - std::size_t next_plane = 0; - bool launch_pending = true; - std::uint64_t step_planes = 0; - bool dense = false; - std::uint64_t period = 0; - std::size_t replay_steps = 0; - double replay_length = 0.0; - std::size_t attempt_capacity = 0; - std::size_t row_capacity = 0; - std::size_t call_attempts = 0; - double call_seconds = 0.0; -}; - -[[nodiscard]] constexpr auto describe(const PlanePlanRecord* /*unused*/) { - using P = PlanePlanRecord; - using E = fields::Extent; - return fields::table