diff --git a/docs/dev/backends.md b/docs/dev/backends.md index 7e25c40d9..149a32ea0 100644 --- a/docs/dev/backends.md +++ b/docs/dev/backends.md @@ -653,7 +653,7 @@ two-stage reductions, and moving the whole inner loop into compiled code: one na executes a Runge–Kutta trial (§3.5). On a CPU build's trapezoid column the whole nonlinear term at one node — the rate lookup, the plasma's two running sums, the loss-current guard, the Kerr cubic and the Raman drive — is **one traversal of `t`**, the plasma's sums carried -in scalars across the loop (`response_plasma`, `native/src/general.cpp`); the band column +in scalars across the loop (`response_plasma`, `native/src/general/trial.cpp`); the band column (§3.12) uses complete row passes instead. The solution and the embedded estimate are running accumulators (ADR-009), which is also why there is no dense output by default. An isolated step timing is not comparable with a whole-run average, which carries the @@ -662,7 +662,7 @@ once-per-run FFTW planning and the output's planes. ### 3.1a What exists The adapter `native/src/solprop_native/backend.py`, its session -`native/src/solprop_native/session.py` and the engine `native/src/general.cpp`. The public default is `"auto"`: the +`native/src/solprop_native/session.py` and the engine under `native/src/general/`. The public default is `"auto"`: the native backend, with no fallback (§1, ADR-029). `numpy_ref` stays selectable by name. diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index bf936696f..959fd91d3 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -1474,7 +1474,7 @@ grows as an attempt gets cheaper, and it is above 0.1 % on the two smallest grid The exponential Adams trial's predict and correct passes on a host block of 32 elements held as lanes (`include/solprop_native/adams_phi.hpp`, `adams_lanes_*` in -`src/general.cpp`) against the element-by-element form, at 740b24d. Before the change +`src/general/trial.cpp`) against the element-by-element form, at 740b24d. Before the change the passes ran each element through latency-bound scalar strict code (the phi-functions' series, Horner pass and squarings, the compensation and the defects), GCC calling every strict helper out of line from the contracting kernels; per element diff --git a/docs/dev/open-questions.md b/docs/dev/open-questions.md index b0f3a2efe..fcfd19f3a 100644 --- a/docs/dev/open-questions.md +++ b/docs/dev/open-questions.md @@ -19,7 +19,7 @@ comparison with a reference, a measurement — settles it. **The on-axis spectrum's end bins on the native backend.** The `on_axis_spectrum` statistic is computed natively on the native backend -(`GeneralWorkspace::spatial_statistics` in `native/src/general.cpp`) and in +(`GeneralWorkspace::spatial_statistics` in `native/src/general/statistics.cpp`) and in Python on the others (`solprop/backends/spatial.py`, through {func}`~solprop.fourier.spectral_density`). On a field grid the two agree at every interior bin and differ at the zero-frequency and Nyquist bins: the native kernel does diff --git a/docs/dev/seps/sep-0002-two-photon-absorption.md b/docs/dev/seps/sep-0002-two-photon-absorption.md index 0d6d3e780..d7cf9342b 100644 --- a/docs/dev/seps/sep-0002-two-photon-absorption.md +++ b/docs/dev/seps/sep-0002-two-photon-absorption.md @@ -99,7 +99,7 @@ The relevant current code is: admits `MaterialKerr` and `SilicaTensorResponse`; those types do not already provide arbitrary complex susceptibility or TPA. - In `solprop-kokkos`, `src/solprop_kokkos/backend.py`, - `include/solprop_kokkos/core.hpp`, `src/core.cpp`, `src/general.cpp` and + `include/solprop_kokkos/core.hpp`, `src/core.cpp`, `src/general/` and `include/solprop_kokkos/envelope_plasma.hpp`: admission, native data and resident execution. A source cannot be added solely to the Python oracle. diff --git a/docs/dev/seps/sep-0009-propagation-statistics.md b/docs/dev/seps/sep-0009-propagation-statistics.md index d34051072..eea0ce900 100644 --- a/docs/dev/seps/sep-0009-propagation-statistics.md +++ b/docs/dev/seps/sep-0009-propagation-statistics.md @@ -137,7 +137,7 @@ product and must not reinterpret the existing statistic. `backends.numpy_ref.backend`, `backends.numba_cpu.backend`, `backends.spatial` and `backends.driver.StepDriver` own the reference, compiled and periodic evaluation. In `solprop-kokkos`, `src/solprop_kokkos/backend.py` packs/unpacks native statistics; -`src/general.cpp` owns the modal statistics pass, and `src/general_modal.cpp` the +`src/general/statistics.cpp` owns the modal statistics pass, and `src/general_modal.cpp` the general modal execution. The current Kokkos temporal FWHM path transfers a total power trace for host crossing interpolation; a new family of per-mode/time-resolved measurements must budget that transfer rather than assume all statistics are scalars. diff --git a/native/.clang-format b/native/.clang-format index 15a0947b2..eb8a12c74 100644 --- a/native/.clang-format +++ b/native/.clang-format @@ -12,7 +12,7 @@ InsertNewlineAtEOF: true CommentPragmas: '^ (IWYU pragma:|NOLINT)' # No RemoveSemicolon: clang-format parses `const auto f = KOKKOS_LAMBDA(...) {...};` as a # function definition and removes the semicolon the declaration needs. Declaring the -# macro (`Macros:`) fixes that but makes formatting general.cpp minutes slower. +# macro (`Macros:`) fixes that but makes formatting src/general/ minutes slower. # Include blocks: this package's and the directory's own headers ("..."), then the # third-party libraries, then the standard library, then platform headers (the operating # system, the compiler and the device runtimes). An include under #if stays in its own diff --git a/native/.clang-tidy b/native/.clang-tidy index 6609abb16..3dc841f41 100644 --- a/native/.clang-tidy +++ b/native/.clang-tidy @@ -24,8 +24,9 @@ # - readability-trailing-comma: style only, and clang-format owns layout. # - readability-braces-around-statements: the LLVM convention leaves single # statements unbraced, and adding braces around bodies with #if is unsafe. -# - readability-function-size, readability-function-cognitive-complexity: the long -# setup functions and hot kernels are split first (a later change), then gated. +# - readability-function-cognitive-complexity: numerical kernels branch on their +# problem's features by design. readability-function-size is on, at 300 lines; +# the hot kernels above it say why where they are defined. # - readability-avoid-nested-conditional-operator: nested conditionals initialise # const locals, in kernels too; an immediately invoked lambda reads worse. # - readability-convert-member-functions-to-static: it sees the CPU build only, where @@ -62,7 +63,6 @@ Checks: > -readability-uppercase-literal-suffix, -readability-trailing-comma, -readability-braces-around-statements, - -readability-function-size, -readability-function-cognitive-complexity, -readability-avoid-nested-conditional-operator, -readability-make-member-function-const, @@ -79,6 +79,8 @@ WarningsAsErrors: '*' HeaderFilterRegex: '.*/(include|src)/.*' FormatStyle: file CheckOptions: + # A function longer than this is split; its statements stay at the default 800. + readability-function-size.LineThreshold: 300 # `.value()` is a checked access: it throws std::bad_optional_access. bugprone-unchecked-optional-access.IgnoreValueCalls: true # A cast to void is an explicit discard (unique_ptr::release once a capsule owns it). diff --git a/native/AGENTS.md b/native/AGENTS.md index 7d3117dfa..b15668949 100644 --- a/native/AGENTS.md +++ b/native/AGENTS.md @@ -195,7 +195,8 @@ the gate. - Release the GIL around long native work and hold it while touching Python objects. Never call Python from a Kokkos or OpenMP worker, or during teardown. - Public Python has complete annotations, NumPy-style docstrings, `__all__`, `py.typed` - and a complete stub for the private extension. Install a `NullHandler`; never + and a complete stub for the private extension; its bound problem's part is generated + (`scripts/problem_stub.py`, checked in the lint stage). Install a `NullHandler`; never configure the root logger or print diagnostics from library code. - Thread settings are read once, at import. Never change process thread settings (or a BLAS thread pool) after import. diff --git a/native/CMakeLists.txt b/native/CMakeLists.txt index 8bb38be9b..0d23fc6da 100644 --- a/native/CMakeLists.txt +++ b/native/CMakeLists.txt @@ -42,8 +42,11 @@ if(SOLPROP_NATIVE_ENABLE_COVERAGE) -O1 -g -fprofile-instr-generate -fcoverage-mapping -fprofile-update=atomic) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION OFF) set(CMAKE_INTERPROCEDURAL_OPTIMIZATION_RELEASE OFF) + # The source digest's set (cmake/SourceDigest.cmake), src/general/'s headers included. file(GLOB_RECURSE _coverage_sources CONFIGURE_DEPENDS RELATIVE "${PROJECT_SOURCE_DIR}" - "${PROJECT_SOURCE_DIR}/src/*.cpp" "${PROJECT_SOURCE_DIR}/include/*.hpp") + "${PROJECT_SOURCE_DIR}/src/*.cpp" "${PROJECT_SOURCE_DIR}/src/*.hpp" + "${PROJECT_SOURCE_DIR}/src/*.h" "${PROJECT_SOURCE_DIR}/include/*.cpp" + "${PROJECT_SOURCE_DIR}/include/*.hpp" "${PROJECT_SOURCE_DIR}/include/*.h") list(SORT _coverage_sources) set(_coverage_source_record "") foreach(_source IN LISTS _coverage_sources) @@ -805,8 +808,9 @@ if(SOLPROP_NATIVE_ENABLE_COVERAGE) install(TARGETS solprop_native_labels LIBRARY DESTINATION solprop_native) endif() -add_library(solprop_native_core STATIC src/core.cpp src/crossing.cpp src/general.cpp - src/general_modal.cpp src/coupled_flow.cpp src/dispersive.cpp src/far_field.cpp +add_library(solprop_native_core STATIC src/core.cpp src/crossing.cpp + src/general/checks.cpp src/general/setup.cpp src/general/trial.cpp + src/general/statistics.cpp src/general/state.cpp src/general_modal.cpp src/coupled_flow.cpp src/dispersive.cpp src/far_field.cpp src/adams_march.cpp src/watch.cpp src/rk_march.cpp src/engines.cpp src/session.cpp src/session_statistics.cpp src/session_widths.cpp src/low_storage.cpp src/transverse.cpp src/session_edge.cpp src/dense.cpp) @@ -817,10 +821,15 @@ add_library(solprop_native_core STATIC src/core.cpp src/crossing.cpp src/general # judgement round as solprop's Python does, so their # translation units take no contraction on any build (the flag follows the global one, # and the last wins). MSVC's /fp:precise does not contract by default. +# SOLPROP_NATIVE_STRICT_TU marks them, so that the general workspace's contracting +# headers (src/general/) refuse to be included there. +set(SOLPROP_NATIVE_STRICT_SOURCES src/adams_march.cpp src/watch.cpp src/rk_march.cpp + src/session.cpp src/session_statistics.cpp src/session_widths.cpp src/low_storage.cpp + src/transverse.cpp src/session_edge.cpp src/dense.cpp) +set_source_files_properties(${SOLPROP_NATIVE_STRICT_SOURCES} + PROPERTIES COMPILE_DEFINITIONS SOLPROP_NATIVE_STRICT_TU) if(CMAKE_CXX_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang|AMDClang)$") - set_source_files_properties(src/adams_march.cpp src/watch.cpp src/rk_march.cpp - src/session.cpp src/session_statistics.cpp src/session_widths.cpp src/low_storage.cpp - src/transverse.cpp src/session_edge.cpp src/dense.cpp + set_source_files_properties(${SOLPROP_NATIVE_STRICT_SOURCES} PROPERTIES COMPILE_OPTIONS "-ffp-contract=off") endif() target_include_directories(solprop_native_core PUBLIC diff --git a/native/include/solprop_native/build.hpp b/native/include/solprop_native/build.hpp new file mode 100644 index 000000000..b9cd978c9 --- /dev/null +++ b/native/include/solprop_native/build.hpp @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +/** The build's facts as constants, for `if constexpr` where both branches compile in + every build (CPU, CUDA and HIP); code that names a device or host-only facility + keeps its `#if`. */ +namespace solprop_native::build { + +#ifdef KOKKOS_ENABLE_CUDA +inline constexpr bool cuda = true; +#else +inline constexpr bool cuda = false; +#endif + +#ifdef KOKKOS_ENABLE_HIP +inline constexpr bool hip = true; +#else +inline constexpr bool hip = false; +#endif + +/** A CUDA or HIP build: the kernels' execution space is a device. */ +inline constexpr bool device = cuda || hip; + +/** A host build: the kernels run on the CPU. */ +inline constexpr bool host = !device; + +#ifdef KOKKOS_ENABLE_OPENMP +inline constexpr bool openmp = true; +#else +inline constexpr bool openmp = false; +#endif + +} // namespace solprop_native::build diff --git a/native/include/solprop_native/core.hpp b/native/include/solprop_native/core.hpp index ded441b5a..4e7c7ba30 100644 --- a/native/include/solprop_native/core.hpp +++ b/native/include/solprop_native/core.hpp @@ -471,238 +471,310 @@ inline constexpr std::size_t plasma_max_taps = 9; /** The six parts of SEP 18's run diagnostics, in solprop's order. */ inline constexpr std::size_t plasma_diagnostic_parts = 6; -/** Fixed, Python-free problem data uploaded once by a native state. */ +/** Fixed, Python-free problem data uploaded once by a native state, grouped as + solprop's kernel contract groups it. The adapter fills it field by field (`lower` in + solprop_native's backend); `validate` checks presence and extents before a state + allocates anything. */ struct Problem { - std::size_t n_x{}, n_y{}; - std::vector spatial_k_squared, spatial_k_squared_low, spatial_numerator, - spatial_clamped; - std::vector spatial_kperp_sq, spatial_kperp_sq_low, spatial_frame, - spatial_spectral_rate; - // solprop kernel schema 45 (#231): the user's share of the spectral rate, read only - // by `absorbed_power`; the operator takes the total. - std::vector spatial_user_rate; - std::vector spatial_absorber_rate; - // solprop kernel schema 31 (SEP 22 phase 2): a transverse collar's power rate per - // column, added to the field attenuation on active bins; zeros leave the operator. - std::vector spatial_kperp_rate; - double spatial_evanescent_rate{}; - // solprop SEP 23 (free space only): the stored transverse axes k_x (n_x) and k_y - // (n_y) in rad/m, FFT order, column ix + n_x iy, which the far-field statistics read; - // and the slots of the problem's own far-field request, which size the band partials - // at construction (zero for none). Empty and zero elsewhere. - std::vector spatial_k_x, spatial_k_y; - std::size_t spatial_far_field_slots{}; - // solprop's hybrid planar kind (kernel schema 15): n_p slab - // channels in x on the real (n_x, n_p) synthesis and (n_p, n_x) projection, times a - // raw DFT in y. Columns are (p, k_y) with the channel fastest, n_modes = n_p * n_y, - // and nodes (x, y) with x fastest, n_nodes = n_x * n_y; node_chunk holds whole y - // rows. Row-major (n_p, n_omega) wall terms kappa_p^2 enter the separable operator, - // and spatial_absorber_rate is then per column. Zero n_p and empty kappa elsewhere. - std::size_t n_p{}; - std::vector spatial_kappa_sq; - // solprop kernel schema 35 (SEP 34): a non-separable spatial run's plane measure, - // the entrance's material squares (n_omega) and transverse squares (n_modes) whose - // root is solprop's `longitudinal_root`; `flux_table` when the material square - // varies along z and is uploaded as a `TableKind::flux` table of n_omega values. A - // separable run's measure is its own operator's (these stay empty). `flux_closure` - // allocates the magnetic closure's buffers at construction, for a request with a - // local flux statistic. - std::vector flux_k_squared, flux_k_squared_low; - std::vector flux_kperp_sq, flux_kperp_sq_low; - bool flux_table{}; - bool flux_closure{}; - /** Free space or the hybrid planar kind: the separable operator and a y FFT. */ - [[nodiscard]] bool separable_geometry() const noexcept { return n_x != 0; } - [[nodiscard]] bool slab_geometry() const noexcept { return n_p != 0; } - [[nodiscard]] bool free_geometry() const noexcept { return n_x != 0 && n_p == 0; } - std::size_t n_omega{}; - std::size_t n_t{}; - std::size_t n_to{}; - std::size_t n_modes{1}; - std::size_t n_pol_state{1}; - std::size_t n_pol_transform{1}; - std::size_t n_nodes{1}; - std::size_t node_chunk{1}; + /** The state's extents and field family. */ + struct Dims { + std::size_t n_omega{}; + std::size_t n_t{}; + std::size_t n_to{}; + std::size_t n_modes{1}; + std::size_t n_pol_state{1}; + std::size_t n_pol_transform{1}; + std::size_t n_nodes{1}; + std::size_t node_chunk{1}; + FieldFamily family{FieldFamily::field_real}; + }; + /** The frequency and time grids, their windows and masks. */ + struct Grid { + std::vector crop_index; + std::vector active_mask; + std::vector norm_mask; + std::vector omega_window; + std::vector omega; + std::vector power_weight; + std::vector time_window; + std::vector time_window_derivative; + std::vector carrier; + double dt_fine{}; + double dt{}; + }; + /** The separable geometries' operator (free space and the hybrid planar kind) and a + radial run's absorber and on-axis rows. */ + struct Spatial { + std::size_t n_x{}, n_y{}; + std::vector k_squared, k_squared_low, numerator, clamped; + std::vector kperp_sq, kperp_sq_low, frame, spectral_rate; + // solprop kernel schema 45 (#231): the user's share of the spectral rate, read only + // by `absorbed_power`; the operator takes the total. + std::vector user_rate; + std::vector absorber_rate; + // solprop kernel schema 31 (SEP 22 phase 2): a transverse collar's power rate per + // column, added to the field attenuation on active bins; zeros leave the operator. + std::vector kperp_rate; + double evanescent_rate{}; + // solprop SEP 23 (free space only): the stored transverse axes k_x (n_x) and k_y + // (n_y) in rad/m, FFT order, column ix + n_x iy, which the far-field statistics + // read; and the slots of the problem's own far-field request, which size the band + // partials at construction (zero for none). Empty and zero elsewhere. + std::vector k_x, k_y; + std::size_t far_field_slots{}; + // solprop's hybrid planar kind (kernel schema 15): n_p slab channels in x on the + // real (n_x, n_p) synthesis and (n_p, n_x) projection, times a raw DFT in y. + // Columns are (p, k_y) with the channel fastest, n_modes = n_p * n_y, and nodes + // (x, y) with x fastest, n_nodes = n_x * n_y; node_chunk holds whole y rows. + // Row-major (n_p, n_omega) wall terms kappa_p^2 enter the separable operator, and + // absorber_rate is then per column. Zero n_p and empty kappa elsewhere. + std::size_t n_p{}; + std::vector kappa_sq; + // A radial run's spatial absorber and its on-axis statistics' rows. + std::vector absorber_matrix; + std::vector spectral_weights; + std::vector on_axis; + }; + /** solprop kernel schema 35 (SEP 34): a non-separable spatial run's plane measure, + the entrance's material squares (n_omega) and transverse squares (n_modes) whose + root is solprop's `longitudinal_root`; `table` when the material square varies + along z and is uploaded as a `TableKind::flux` table of n_omega values. A + separable run's measure is its own operator's (these stay empty). `closure` + allocates the magnetic closure's buffers at construction, for a request with a + local flux statistic. */ + struct Flux { + std::vector k_squared, k_squared_low; + std::vector kperp_sq, kperp_sq_low; + bool table{}; + bool closure{}; + }; + /** The Runge-Kutta tableau and the march's own choices. */ + struct Stepper { + std::size_t n_stages{}; + std::vector tableau_matrix; + std::vector tableau_nodes; + std::vector solution_weights; + std::vector error_weights; + bool first_same_as_last{}; + bool weighted_controller{}; + bool modified_noise{}; + // solprop SEP 24 phase 3 (kernel schema 32): the low-storage RK4IP march. With it + // no Runge-Kutta view is allocated (stages, accumulators, accept buffer, residual, + // recycled stage, whole-step propagator, stage inverse); the accumulator register + // is, the stage register unless it is elided (solprop's + // `low_storage_stage_register`), and `low_storage_flows` flow volumes (0 + // recomputed, 1 a constant operator's table, 2 a varying operator's step pair). + bool low_storage{}; + bool low_storage_stage_register{true}; + std::size_t low_storage_flows{}; + // Whether a separable route's first input crosses to the transverse scratch + // through tiles of team scratch, as a device's does. + bool low_storage_tiled_input{}; + }; + /** The linear operator's fixed terms, row-major (terms, columns, n_omega). solprop + SEP 29 G1: Phi or the prefactor is the general z table when `phi_table` + (`prefactor_table`); its terms are then `n_phi_terms` (`n_prefactor_terms`) slots + that `table_evaluate` fills with the table's values at a trial's positions, and + `phi_terms` (`prefactor_terms`) is empty. */ + /** The shapes the bindings received for the arrays whose layout counts beside them + give, by field name: `validate` holds each to its counts (a vector keeps its + elements alone). Empty for a problem built in C++. */ + using Shapes = std::map>; + struct Linear { + std::size_t n_phi_terms{}; + std::size_t n_prefactor_terms{}; + bool phi_table{}; + bool prefactor_table{}; + std::vector phi_terms; + std::vector prefactor_terms; + Shapes shapes; + }; + /** The temporal absorber, applied in each stage or split around the step. */ + struct Absorber { + bool in_stage{}; + bool split_step{}; + std::vector fine; + std::vector coarse; + }; + /** The modal synthesis and projection onto the nodes. */ + struct Transform { + std::vector synthesis_matrix; + std::vector projection_matrix; + std::vector projection_coarse_matrix; + double synthesis{}; + double projection{}; + }; + /** The instantaneous Kerr response. */ + struct Kerr { + bool nonlinear{}; + double coefficient{}; + double self_factor{}; + double conjugate_factor{}; + double harmonic_factor{}; + bool take_real{}; + }; + /** The field and envelope plasma columns and their rate tables. */ + struct Plasma { + bool enabled{}; + bool envelope{}; + // solprop's private, test-only gated plasma benchmark: the field plasma's current + // enters as w(t) times its running integral, added to the local polarisation, + // instead of the held current's two collared spectra. Never set by a user run. + bool gated{}; + bool stash{}; + // Channel-major 4 x 512 split angular rules and coarse-axis current factors. + std::vector envelope_rule_nodes, envelope_rule_weights, + envelope_drude_factor; + std::vector envelope_loss_factor; + double charge_mass_ratio{}; + double ionisation_potential{}; + double log_field_min{}; + double d_log_field{}; + double field_min{}; + double field_max{}; + double rate_max{}; + std::size_t rate_intervals{}; + // Row-major (4, rate_intervals). + std::vector rate_coefficients; + std::size_t n_channels{1}; + // solprop SEP 18 (kernel schema 25): the field plasma column's running-integral + // rule, one per problem and shared by every channel. Each running integral is the + // cumulative trapezoid of the integrand filtered by the symmetric taps + // (c_0, ..., c_K), centre first, with the window's ends closed by point + // reflection, and the exposure floored at zero where the fraction is formed. The + // trapezoid rule is `band == false` with the one identity tap, and runs the + // version-24 column unchanged. + bool band{}; + std::vector taps{1.0}; + [[nodiscard]] std::size_t half_width() const noexcept { + return taps.empty() ? 0 : taps.size() - 1; + } + // Channel-major rows: log_min, d_log, field_min, field_max, rate_max, + // interval count, coefficient offset, charge/mass ratio, ionisation potential. + std::vector channel_parameters, channel_coefficients; + Shapes shapes; + }; + /** The delayed Raman response. */ + struct Raman { + bool enabled{}; + bool thg{}; + bool tensor_capable{}; + std::size_t n_terms{}; + double fraction{}; + double epsilon0{}; + // Row-major (n_terms, 2 n_to) and (n_terms, 2). + std::vector undamped; + std::vector tensor_weights; + std::size_t n_components{1}; + std::vector row_component, component_fraction; + Shapes shapes; + }; + /** solprop's general modal route. Row-major: synthesis (map terms, nodes, + components, modes, n_omega); projection (map terms, modes, nodes, components, + n_omega); tensor (nodes, C, C, C, C); delayed tensor (responses, terms, C, C, C, + C); delayed spectrum (responses, terms, n_to + 1); node response (nodes), the + owning response plus one and zero for none. Empty otherwise. The map terms are + map_intervals x map_n_deg Chebyshev coefficients along z, one for a constant + guide; a moving guide's are combined per stage over + TrialScalars::transform_weights. */ + struct General { + bool enabled{}; + bool real_form{}; + std::size_t map_intervals{1}, map_n_deg{1}; + std::size_t components{}, nodes{}, node_chunk{}; + std::size_t responses{}, terms{}; + double pad_scale{}, epsilon0{}; + std::vector synthesis, projection, frequency_factor, delayed_spectrum; + std::vector tensor, delayed_tensor; + std::vector node_response; + }; + /** solprop's linear.coupled_flow. `slot` (n_omega) numbers the bins a section's + generator is non-zero at 1, 2, ... in increasing frequency, zero elsewhere. With + an explicit residual (terms > 0), row-major frozen blocks (sections, active, + modes, modes) and residual terms (terms, active, modes, modes); both empty + otherwise. The actions are loaded per step. + + The actions' native formation, all present or all empty: each section's + eigenbasis of its Schur factor, row-major left `W` and right `Y` (sections, + active, modes, modes) and eigenvalues (sections, active, modes); the route + (sections, active), zero where a bin takes its eigenbasis and 1, 2, ... + numbering in row-major order the bins formed by scaling and squaring, whose Schur + factors `Z` and `T` are the rows of the fallback arrays (fallback, modes, + modes). */ + struct Flow { + bool enabled{}; + std::size_t active{}, sections{}, terms{}; + std::vector slot; + std::vector frozen, residual_terms; + std::vector left, right, eigenvalues; + std::vector route; + std::vector fallback_vectors, fallback_triangular; + }; + /** solprop SEP 6's frequency-dependent electronic response + (`responses.dispersive_kerr`, kernel schema 24) on the fused node path, laid out + by the adapter as solprop's `dispersive_terms_layout` lays it out + (`dispersive.hpp`). The form is 0 (a field grid's full cubic), 1 (the + number-conserving sector) or 2 (an envelope's full cubic). Row-major on the + leaf's spectral layout of `bins` (n_to / 2 + 1 on a field grid, n_to on an + envelope): the input multipliers (fields, bins), each output term's multiplier + (terms, bins) with its product row and factor, the static coefficients and the + polynomial's pair constants (channels, 2) per channel, the groups as offsets + (groups + 1) into members (channels), each symmetric mixture's channels' `g^4` + (channels, bins) and the input and output scales (bins). On the general modal + route (SEP 6 Phase 3b) each group's key is its node region (groups), the + polynomial forms one cube per distinct key, and every general node's region + (general nodes) says where a channel acts; both are empty elsewhere, where every + channel acts at every node. Empty otherwise. */ + struct Dispersive { + bool enabled{}; + bool symmetric{}; + int form{}; + std::vector inputs, term_multipliers, coefficients, pair_constants, + quartics, input_scale, output_scale; + std::vector term_rows, term_channels, group_offsets, group_members, + group_keys, node_region; + }; + + Dims dims; + Grid grid; + Spatial spatial; + Flux flux; + Stepper stepper; + Linear linear; + Absorber absorber; + Transform transform; + Kerr kerr; + Plasma plasma; + Raman raman; + General general; + Flow flow; + Dispersive dispersive; // How the workspace's CPU operations thread (solprop SEP 46): set by the caller, // refused when unset (`require_cpu_execution`). CpuExecution cpu_execution{}; // How the workspace plans its host transforms (solprop SEP 46 phase 3). FftPlanning fft_planning{}; - std::size_t n_stages{}; - // solprop SEP 24 phase 3 (kernel schema 32): the low-storage RK4IP march. With it no - // Runge-Kutta view is allocated (stages, accumulators, accept buffer, residual, - // recycled stage, whole-step propagator, stage inverse); the accumulator register - // is, the stage register unless it is elided (solprop's - // `low_storage_stage_register`), and `low_storage_flows` flow volumes (0 - // recomputed, 1 a constant operator's table, 2 a varying operator's step pair). - bool low_storage{}; - bool low_storage_stage_register{true}; - std::size_t low_storage_flows{}; - // Whether a separable route's first input crosses to the transverse scratch through - // tiles of team scratch, as a device's does. - bool low_storage_tiled_input{}; - std::size_t n_phi_terms{}; - std::size_t n_prefactor_terms{}; - // solprop SEP 29 G1: Phi or the prefactor is the general z table. Its terms are then - // `n_phi_terms` (or `n_prefactor_terms`) slots that `table_evaluate` fills with the - // table's values at a trial's positions, and `phi_terms` (`prefactor_terms`) is - // empty. - bool phi_table{}; - bool prefactor_table{}; - FieldFamily family{FieldFamily::field_real}; - bool nonlinear{}; - bool modified_noise{}; - bool in_stage_absorber{}; - bool split_step_absorber{}; - bool first_same_as_last{}; - std::vector crop_index; - std::vector active_mask; - std::vector norm_mask; - std::vector omega_window; - std::vector omega; - std::vector power_weight; - std::vector time_window; - std::vector time_window_derivative; - std::vector absorber_fine; - std::vector absorber_coarse; - std::vector spatial_absorber_matrix; - std::vector spatial_spectral_weights; - std::vector spatial_on_axis; - std::vector carrier; - std::vector phi_terms; - std::vector prefactor_terms; - std::vector synthesis_matrix; - std::vector projection_matrix; - std::vector projection_coarse_matrix; - std::vector tableau_matrix; - std::vector tableau_nodes; - std::vector solution_weights; - std::vector error_weights; - double synthesis{}; - double projection{}; - double kerr_coefficient{}; - double self_factor{}; - double conjugate_factor{}; - double harmonic_factor{}; - bool take_real{}; - bool plasma{}; - bool envelope_plasma{}; - // solprop's private, test-only gated plasma benchmark: the field plasma's current - // enters as w(t) times its running integral, added to the local polarisation, - // instead of the held current's two collared spectra. Never set by a user run. - bool plasma_gated{}; - // Channel-major 4 x 512 split angular rules and coarse-axis current factors. - std::vector envelope_rule_nodes, envelope_rule_weights, envelope_drude_factor; - std::vector envelope_loss_factor; - bool raman{}; - bool raman_thg{}; - bool raman_tensor_capable{}; - bool stash_plasma{}; - bool weighted_controller{}; - double dt_fine{}; - double dt{}; - std::size_t n_raman_terms{}; - double raman_fraction{}; - double raman_epsilon0{}; - std::vector raman_undamped; - std::vector raman_tensor_weights; - double charge_mass_ratio{}; - double ionisation_potential{}; - double log_field_min{}; - double d_log_field{}; - double field_min{}; - double field_max{}; - double rate_max{}; - std::size_t rate_intervals{}; - std::vector rate_coefficients; - std::size_t n_channels{1}; - // solprop SEP 18 (kernel schema 25): the field plasma column's running-integral - // rule, one per problem and shared by every channel. Each running integral is the - // cumulative trapezoid of the integrand filtered by the symmetric taps - // (c_0, ..., c_K), centre first, with the window's ends closed by point reflection, - // and the exposure floored at zero where the fraction is formed. The trapezoid rule - // is `plasma_band == false` with the one identity tap, and runs the version-24 - // column unchanged. - bool plasma_band{}; - std::vector plasma_taps{1.0}; - [[nodiscard]] std::size_t plasma_half_width() const noexcept { - return plasma_taps.empty() ? 0 : plasma_taps.size() - 1; + + /** Refuse, naming the field, data whose presence or extents disagree with the + problem's own flags and extents: the checks that need no execution space. */ + void validate() const; + + /** Free space or the hybrid planar kind: the separable operator and a y FFT. */ + [[nodiscard]] bool separable_geometry() const noexcept { return spatial.n_x != 0; } + [[nodiscard]] bool slab_geometry() const noexcept { return spatial.n_p != 0; } + [[nodiscard]] bool free_geometry() const noexcept { + return spatial.n_x != 0 && spatial.n_p == 0; + } + [[nodiscard]] std::size_t n_columns() const noexcept { + return dims.n_modes * dims.n_pol_state; } - // Channel-major rows: log_min, d_log, field_min, field_max, rate_max, - // interval count, coefficient offset, charge/mass ratio, ionisation potential. - std::vector channel_parameters, channel_coefficients; - std::size_t n_raman_components{1}; - std::vector raman_row_component, raman_component_fraction; - // solprop's general modal route. Row-major: synthesis (map terms, - // nodes, components, modes, n_omega); projection (map terms, modes, nodes, - // components, n_omega); tensor (nodes, C, C, C, C); delayed tensor (responses, - // terms, C, C, C, C); delayed spectrum (responses, terms, n_to + 1); node response - // (nodes), the owning response plus one and zero for none. Empty otherwise. The - // map terms are general_map_intervals x general_map_n_deg Chebyshev coefficients - // along z, one for a constant guide; a moving guide's are combined per stage over - // TrialScalars::transform_weights. - bool general_modal{}; - bool general_real_form{}; - std::size_t general_map_intervals{1}, general_map_n_deg{1}; - std::size_t general_components{}, general_nodes{}, general_node_chunk{}; - std::size_t general_responses{}, general_terms{}; - double general_pad_scale{}, general_epsilon0{}; - std::vector general_synthesis, general_projection, general_frequency_factor, - general_delayed_spectrum; - std::vector general_tensor, general_delayed_tensor; - std::vector general_node_response; - // solprop's linear.coupled_flow. flow_slot (n_omega) - // numbers the bins a section's generator is non-zero at 1, 2, ... in increasing - // frequency, zero elsewhere. With an explicit residual (flow_terms > 0), row-major - // frozen blocks (sections, active, modes, modes) and residual terms (terms, active, - // modes, modes); both empty otherwise. The actions are loaded per step. - bool coupled_flow{}; - std::size_t flow_active{}, flow_sections{}, flow_terms{}; - std::vector flow_slot; - std::vector flow_frozen, flow_residual_terms; - // The actions' native formation, all present or all - // empty: each section's eigenbasis of its Schur factor, row-major left `W` and - // right `Y` (sections, active, modes, modes) and eigenvalues (sections, active, - // modes); the route (sections, active), zero where a bin takes its eigenbasis and - // 1, 2, ... numbering in row-major order the bins formed by scaling and squaring, - // whose Schur factors `Z` and `T` are the rows of the fallback arrays (fallback, - // modes, modes). - std::vector flow_left, flow_right, flow_eigenvalues; - std::vector flow_route; - std::vector flow_fallback_vectors, flow_fallback_triangular; - - // solprop SEP 6's frequency-dependent electronic response - // (`responses.dispersive_kerr`, kernel schema 24) on the fused node path, laid out by - // the adapter as solprop's `dispersive_terms_layout` lays it out (`dispersive.hpp`). - // The form is 0 (a field grid's full cubic), 1 (the number-conserving sector) or 2 - // (an envelope's full cubic). Row-major on the leaf's spectral layout of `bins` (n_to - // / 2 + 1 on a field grid, n_to on an envelope): the input multipliers (fields, - // bins), each output term's multiplier (terms, bins) with its product row and factor, - // the static coefficients and the polynomial's pair constants (channels, 2) per - // channel, the groups as offsets (groups + 1) into members (channels), each symmetric - // mixture's channels' `g^4` (channels, bins) and the input and output scales (bins). - // On the general modal route (SEP 6 Phase 3b) each group's key is its node region - // (groups), the polynomial forms one cube per distinct key, and every general node's - // region (general_nodes) says where a channel acts; both are empty elsewhere, where - // every channel acts at every node. Empty otherwise. - bool dispersive{}; - bool dispersive_symmetric{}; - int dispersive_form{}; - std::vector dispersive_inputs, dispersive_term_multipliers, - dispersive_coefficients, dispersive_pair_constants, dispersive_quartics, - dispersive_input_scale, dispersive_output_scale; - std::vector dispersive_term_rows, dispersive_term_channels, - dispersive_group_offsets, dispersive_group_members, dispersive_group_keys, - dispersive_node_region; - - [[nodiscard]] std::size_t n_columns() const noexcept { return n_modes * n_pol_state; } [[nodiscard]] std::size_t effective_node_pol() const noexcept { - return n_pol_transform * n_pol_state; + return dims.n_pol_transform * dims.n_pol_state; } [[nodiscard]] std::size_t raman_drives() const noexcept { if (effective_node_pol() == 2) - for (std::size_t index = 1; index < raman_tensor_weights.size(); index += 2) - if (raman_tensor_weights[index] != 0.0) + for (std::size_t index = 1; index < raman.tensor_weights.size(); index += 2) + if (raman.tensor_weights[index] != 0.0) return 3; return 1; } @@ -1090,6 +1162,12 @@ class GeneralState { void close() noexcept; private: + /** `function(workspace, arguments...)` inside the workspace's CPU team scope + (`CpuTeamScope`); defined and used in src/general/state.cpp alone, so that no unit + outside src/general/ sees the workspace class. */ + template + decltype(auto) with_team(Function function, Arguments&&... arguments) const; + std::unique_ptr workspace_; }; diff --git a/native/include/solprop_native/coupled_flow.hpp b/native/include/solprop_native/coupled_flow.hpp index 8faf107b5..02a38973f 100644 --- a/native/include/solprop_native/coupled_flow.hpp +++ b/native/include/solprop_native/coupled_flow.hpp @@ -135,9 +135,6 @@ class CoupledFlowAction { [[nodiscard]] const ComplexView4& frozen() const noexcept { return frozen_; } [[nodiscard]] std::size_t n_terms() const noexcept { return terms_; } [[nodiscard]] std::size_t n_active() const noexcept { return active_; } - [[nodiscard]] std::size_t n_sections() const noexcept { return sections_; } - /** Whether every section's every bin is on the eigenbasis route. */ - [[nodiscard]] bool full_route() const noexcept { return forms_ && fallback_ == 0; } // The bins off the eigenbasis route (solprop SEP 47 phase 4): each section's bin's // one-based index among them in row-major (section, bin) order, zero on the route, // and their Schur factors `Z` and `T`, which the exponential Adams formulation forms @@ -157,13 +154,11 @@ class CoupledFlowAction { [[nodiscard]] std::map memory_ledger() const; /** The same rows, all zero, for a problem that carries no coupled flow. */ [[nodiscard]] static std::map empty_ledger(); - /** Release every View; repeated calls have no effect. */ - void close() noexcept; private: std::size_t n_modes_{}, n_omega_{}, n_stages_{}, active_{}, sections_{}, terms_{}; std::size_t section_{}, fallback_{}; - bool ready_{}, closed_{}, forms_{}; + bool ready_{}, forms_{}; std::vector identity_; // Per section, whether a bin is off the eigenbasis route. std::vector section_fallback_; diff --git a/native/include/solprop_native/dispatch.hpp b/native/include/solprop_native/dispatch.hpp new file mode 100644 index 000000000..71f2c7af8 --- /dev/null +++ b/native/include/solprop_native/dispatch.hpp @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include + +#include +#include + +namespace solprop_native { + +/** `function.template operator()()` for the first of `Values` equal to `value`, + else `otherwise()`: a run-time extent as a compile-time one, the `switch` ladder + written once. Device-callable; every branch returns the same type. */ +template +KOKKOS_FORCEINLINE_FUNCTION decltype(auto) dispatch_among(const std::size_t value, + const Function& function, + const Otherwise& otherwise) { + if (value == First) + return function.template operator()(); + if constexpr (sizeof...(Rest) == 0) + return otherwise(); + else + return dispatch_among(value, function, otherwise); +} + +namespace detail { + +template +KOKKOS_FORCEINLINE_FUNCTION decltype(auto) +dispatch_width(std::index_sequence /*offsets*/, const std::size_t width, + const Function& function, const Otherwise& otherwise) { + return dispatch_among<(First + Offsets)...>(width, function, otherwise); +} + +} // namespace detail + +/** `dispatch_among` over the widths `First, First + 1, ..., Last`. */ +template +KOKKOS_FORCEINLINE_FUNCTION decltype(auto) dispatch_width(const std::size_t width, + const Function& function, + const Otherwise& otherwise) { + static_assert(First <= Last, "a width range holds at least one width"); + return detail::dispatch_width(std::make_index_sequence{}, + width, function, otherwise); +} + +} // namespace solprop_native diff --git a/native/include/solprop_native/dispersive.hpp b/native/include/solprop_native/dispersive.hpp index a9c7cbedc..d900541d7 100644 --- a/native/include/solprop_native/dispersive.hpp +++ b/native/include/solprop_native/dispersive.hpp @@ -314,7 +314,6 @@ class DispersiveWeights { [[nodiscard]] const RealView2& weights() const noexcept { return weights_; } /** The bytes of every table and staging buffer, the ledger's multipliers row. */ [[nodiscard]] std::size_t bytes() const; - void close() noexcept; // NVCC extended lambdas require their enclosing member functions to be public. /** Weigh the channels at a stage; a no-op when its densities are the last ones. */ @@ -419,8 +418,6 @@ class DispersiveResponse { [[nodiscard]] std::map memory_ledger() const; /** The same rows, all zero, for a problem without the response. */ [[nodiscard]] static std::map empty_ledger(); - /** Release plans before Views; repeated calls have no effect. */ - void close() noexcept; // NVCC extended lambdas require their enclosing member functions to be public. void synthesise(std::size_t field, std::size_t first_node, std::size_t node_count, @@ -439,7 +436,7 @@ class DispersiveResponse { std::size_t spectrum_bins_{}; DispersiveForm form_{DispersiveForm::real}; bool analytic_{}, embedded_{}, alone_{}, lone_{}, in_place_{}; - bool borrowed_{}, shifted_{}, coarse_pending_{}, closed_{}; + bool borrowed_{}, shifted_{}, coarse_pending_{}; ExecutionSpace kernels_, response_, tail_response_, dense_, tail_dense_, matrix_copy_; std::size_t response_chunk_{}; [[nodiscard]] const ExecutionSpace& response_for(std::size_t count) const { @@ -535,7 +532,6 @@ class SeparableDispersive { void combine(); [[nodiscard]] std::map memory_ledger() const; - void close() noexcept; // NVCC extended lambdas require their enclosing member functions to be public. void slab_offsets(std::size_t units); @@ -549,7 +545,6 @@ class SeparableDispersive { std::size_t n_x_{}, n_p_{}; DispersiveForm form_{DispersiveForm::real}; bool slab_{}, analytic_{}, alone_{}, lone_{}, in_place_{}, borrowed_{}, shifted_{}; - bool closed_{}; ExecutionSpace kernels_, tail_kernels_, response_, tail_response_, dense_, tail_dense_; std::size_t tail_rows_{}, active_rows_{}; @@ -630,7 +625,6 @@ class GeneralDispersive { void combine(bool overwrite); [[nodiscard]] std::map memory_ledger() const; - void close() noexcept; // NVCC extended lambdas require their enclosing member functions to be public. template @@ -641,7 +635,7 @@ class GeneralDispersive { using ForwardRealPlan = KokkosFFT::Plan; std::size_t rows_{}, components_{}, n_to_{}, spectrum_bins_{}; - bool real_{}, shifted_{}, closed_{}; + bool real_{}, shifted_{}; ExecutionSpace kernels_, tail_kernels_, response_, tail_response_; std::size_t tail_rows_{}, active_rows_{}, last_chunk_{}; bool active_tail_{}; diff --git a/native/include/solprop_native/far_field.hpp b/native/include/solprop_native/far_field.hpp index 061fec5e7..5367927f6 100644 --- a/native/include/solprop_native/far_field.hpp +++ b/native/include/solprop_native/far_field.hpp @@ -362,7 +362,7 @@ class FarFieldPass { }; FarFieldPass(const Problem& problem, ExecutionSpace execution); - ~FarFieldPass() = default; + ~FarFieldPass(); FarFieldPass(const FarFieldPass&) = delete; FarFieldPass& operator=(const FarFieldPass&) = delete; FarFieldPass(FarFieldPass&&) = delete; @@ -395,7 +395,6 @@ class FarFieldPass { [[nodiscard]] std::size_t partial_bytes() const noexcept; /** The bytes of the column axes, tables and packed output. */ [[nodiscard]] std::size_t table_bytes() const noexcept; - void close() noexcept; /** What the rows have in common: the distance from the origin is evaluated once per element when they share one distance and scale row, and the distance from the diff --git a/native/include/solprop_native/general_modal.hpp b/native/include/solprop_native/general_modal.hpp index afa151fcb..611bb40ae 100644 --- a/native/include/solprop_native/general_modal.hpp +++ b/native/include/solprop_native/general_modal.hpp @@ -4,6 +4,7 @@ #pragma once +#include "solprop_native/build.hpp" #include "solprop_native/core.hpp" #include "solprop_native/dispersive.hpp" #include "solprop_native/raman_fft.hpp" @@ -67,11 +68,7 @@ constexpr std::size_t general_projection_lanes(const std::size_t modes, * (exactly, a power of two): half the inverse transform's work and half the field's * memory traffic. Host builds only; device builds keep the complex transform. */ -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -inline constexpr bool general_real_signal_enabled = false; -#else -inline constexpr bool general_real_signal_enabled = true; -#endif +inline constexpr bool general_real_signal_enabled = build::host; /** * Refuse a general modal problem whose extents or data this source cannot execute. @@ -173,8 +170,6 @@ class GeneralModalSource { void combine(std::size_t stage); /** Create the persistent FFT plans over the chunk buffers. */ void plan(); - /** The provider of the delayed pair transforms, or "none" without them. */ - [[nodiscard]] std::string delayed_fft_provider() const; /** Evaluate the source of the state in `scaled()` into `source()`. */ void evaluate(); /** Persistent allocations, by role, in bytes. */ @@ -188,8 +183,6 @@ class GeneralModalSource { } /** The same rows, all zero, for a problem that does not take the route. */ [[nodiscard]] static std::map empty_ledger(); - /** Release plans before Views; repeated calls have no effect. */ - void close() noexcept; // NVCC extended lambdas require their enclosing member functions to be public. void combine_maps(std::size_t stage); @@ -272,7 +265,6 @@ class GeneralModalSource { std::unique_ptr tail_drive_plan_; std::unique_ptr tail_history_plan_; std::unique_ptr dispersive_; - bool closed_{}; }; } // namespace solprop_native diff --git a/native/include/solprop_native/graph.hpp b/native/include/solprop_native/graph.hpp index 3e9b1e040..b1208bf7d 100644 --- a/native/include/solprop_native/graph.hpp +++ b/native/include/solprop_native/graph.hpp @@ -461,10 +461,6 @@ class Region { /** The next operation cannot be in a graph: issue what the region holds, stop. */ void refuse(); - /** Runs that launched the recorded graph, and runs that captured one. */ - [[nodiscard]] std::size_t replays() const noexcept { return replays_; } - [[nodiscard]] std::size_t captures() const noexcept { return captures_; } - [[nodiscard]] bool disabled() const noexcept { return disabled_; } /** Drop the graph and its records, as when the views they name change. */ void reset() noexcept { if (exec_ != nullptr) @@ -501,8 +497,6 @@ class Region { bool disabled_ = false; std::size_t next_ = 0; std::vector operations_; - std::size_t replays_ = 0; - std::size_t captures_ = 0; }; // The run departs from the recorded sequence after `next_` matched operations: they @@ -572,7 +566,6 @@ inline void Region::finish() { detail::destroy(detail::end_capture(stream_)); mode_ = Mode::off; detail::launch(exec_, stream_); - ++replays_; return; } // The run ended before the recorded sequence: its matched part is the graph. @@ -586,7 +579,6 @@ inline void Region::finish() { exec_ = nullptr; exec_ = detail::instantiate(graph); detail::launch(exec_, stream_); - ++captures_; } // A run that throws: what it issued before the throw still runs, as without a graph, diff --git a/native/include/solprop_native/plasma.hpp b/native/include/solprop_native/plasma.hpp index 6b501f191..55fb46541 100644 --- a/native/include/solprop_native/plasma.hpp +++ b/native/include/solprop_native/plasma.hpp @@ -5,6 +5,7 @@ #pragma once #include "solprop_native/core.hpp" +#include "solprop_native/dispatch.hpp" #include @@ -41,10 +42,10 @@ namespace solprop_native { * column writes the same separate current and non-plasma polarisation streams. * * A CPU build runs it as a team column (`GeneralWorkspace::team_band_plasma_column`, - * src/general.cpp): the field is a row over t, and each running integral is a band - * filter over the row (`team_band_filter`, a parallel stencil with a K-sample halo, - * reflected at the row's ends) followed by a chunked trapezoid scan of the filtered row - * (`team_trapezoid_window`). These are solprop SEP 25 section 6.7.4 + * src/general/trial.cpp): the field is a row over t, and each running integral is a + * band filter over the row (`team_band_filter`, a parallel stencil with a K-sample + * halo, reflected at the row's ends) followed by a chunked trapezoid scan of the + * filtered row (`team_trapezoid_window`). These are solprop SEP 25 section 6.7.4 * whole-row passes, with the final polarisation integral removed by SEP 27: * * 1. per channel: rates (parallel over t; the raw rates are kept for the loss current @@ -215,44 +216,12 @@ KOKKOS_INLINE_FUNCTION void band_filter_interior(const double* row, double* out, KOKKOS_INLINE_FUNCTION void band_filter_interior(const double* row, double* out, const int low, const int high, const double* taps, const int half) { - switch (half) { - case 0: { - band_filter_interior<0>(row, out, low, high, taps); - return; - } - case 1: { - band_filter_interior<1>(row, out, low, high, taps); - return; - } - case 2: { - band_filter_interior<2>(row, out, low, high, taps); - return; - } - case 3: { - band_filter_interior<3>(row, out, low, high, taps); - return; - } - case 4: { - band_filter_interior<4>(row, out, low, high, taps); - return; - } - case 5: { - band_filter_interior<5>(row, out, low, high, taps); - return; - } - case 6: { - band_filter_interior<6>(row, out, low, high, taps); - return; - } - case 7: { - band_filter_interior<7>(row, out, low, high, taps); - return; - } - default: { + const auto filter = [&] { + band_filter_interior(Width)>(row, out, low, high, taps); + }; + dispatch_width<0, band_max_half - 1>(static_cast(half), filter, [&] { band_filter_interior(row, out, low, high, taps); - return; - } - } + }); } /** A staged band filter's tile in a team's scratch. */ diff --git a/native/include/solprop_native/refuse.hpp b/native/include/solprop_native/refuse.hpp new file mode 100644 index 000000000..ab8e85707 --- /dev/null +++ b/native/include/solprop_native/refuse.hpp @@ -0,0 +1,40 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +#include +#include +#include +#include + +namespace solprop_native { + +namespace detail { + +template void append_part(std::string& text, const Part& part) { + if constexpr (std::is_arithmetic_v && !std::is_same_v && + !std::is_same_v) + text += std::to_string(part); + else + text += std::string_view(part); +} + +} // namespace detail + +/** `parts` concatenated: text as given, numbers as `std::to_string` writes them (a + double in `%f`), so that a message built from them keeps its exact text. */ +template [[nodiscard]] std::string concat(const Parts&... parts) { + std::string text; + (detail::append_part(text, parts), ...); + return text; +} + +/** Throw `Error` with the message `concat(parts...)`. */ +template +[[noreturn]] void refuse(const Parts&... parts) { + throw Error(concat(parts...)); +} + +} // namespace solprop_native diff --git a/native/include/solprop_native/stage_tables.hpp b/native/include/solprop_native/stage_tables.hpp index 4977c8c72..209cb6008 100644 --- a/native/include/solprop_native/stage_tables.hpp +++ b/native/include/solprop_native/stage_tables.hpp @@ -7,6 +7,7 @@ #include "solprop_native/chebyshev.hpp" #include "solprop_native/core.hpp" #include "solprop_native/lagrange_basis.hpp" +#include "solprop_native/refuse.hpp" #include #include @@ -57,8 +58,7 @@ struct Table { chebyshev::clenshaw_points(&z, 1, edges.data(), edges.size(), lower, upper, coefficients.data(), n_terms, n_values, out); if (bad >= 0) - throw std::invalid_argument("z = " + std::to_string(z) + - " is outside a stage table's span"); + refuse("z = ", z, " is outside a stage table's span"); } /** The table's single value at `z`. The scalar tables are checked to hold one value @@ -77,8 +77,7 @@ struct Table { */ void closing(const double z, double* out, const bool before = true) const { if (z < lower || z > upper) - throw std::invalid_argument("z = " + std::to_string(z) + - " is outside a stage table's span"); + refuse("z = ", z, " is outside a stage table's span"); std::size_t index = chebyshev::interval_index(z, edges.data(), edges.size()); if (before && index > 0 && z <= edges[index]) --index; @@ -260,10 +259,8 @@ struct ZSlots { if (std::ranges::find(wanted, z) == wanted.end()) wanted.push_back(z); if (wanted.size() > slots) - throw std::invalid_argument("a native call reads the z table at " + - std::to_string(wanted.size()) + - " positions, more than the " + std::to_string(slots) + - " slots this problem holds"); + refuse("a native call reads the z table at ", wanted.size(), + " positions, more than the ", slots, " slots this problem holds"); std::vector slot_of(wanted.size(), slots); std::vector taken(slots, false); for (std::size_t index = 0; index < wanted.size(); ++index) { @@ -401,8 +398,7 @@ inline Row row(const Tables& tables, Anchor& anchor, const double z0, const doub out.prefactor[at_slot] = 1.0; } else if (tables.prefactor == Terms::constant) { if (!(tables.prefactor_low <= z1 && z1 <= tables.prefactor_high)) - throw std::invalid_argument("z = " + std::to_string(z1) + - " lies outside the prefactor's table"); + refuse("z = ", z1, " lies outside the prefactor's table"); out.prefactor = {1.0}; } else { out.prefactor.resize(tables.prefactor_weights.n_values); @@ -566,14 +562,13 @@ inline TrialScalars sample(const Tables& tables, Anchor& anchor, inline void require_scalar_tables(const Tables& tables) { const auto scalar = [](const Table& table, const std::string& name) { if (table.present() && table.n_values != 1) - throw std::invalid_argument("the stage table " + name + " holds " + - std::to_string(table.n_values) + - " values where one is read"); + refuse("the stage table ", name, " holds ", table.n_values, + " values where one is read"); }; const auto scalars = [&scalar](const std::vector& list, const std::string& name) { for (std::size_t index = 0; index < list.size(); ++index) - scalar(list[index], name + "[" + std::to_string(index) + "]"); + scalar(list[index], concat(name, "[", index, "]")); }; scalar(tables.density, "density"); scalar(tables.taper, "taper"); diff --git a/native/include/solprop_native/z_table.hpp b/native/include/solprop_native/z_table.hpp index d0082b564..e429e09c0 100644 --- a/native/include/solprop_native/z_table.hpp +++ b/native/include/solprop_native/z_table.hpp @@ -6,6 +6,7 @@ #include "solprop_native/chebyshev.hpp" #include "solprop_native/core.hpp" +#include "solprop_native/refuse.hpp" #include #include @@ -68,9 +69,8 @@ inline Points locate(const Table& table, const std::vector& slots, if (slots.size() != positions.size()) throw std::invalid_argument("z table: one slot per position"); if (positions.size() > max_points) - throw std::invalid_argument("z table: at most " + std::to_string(max_points) + - " positions per evaluation, got " + - std::to_string(positions.size())); + refuse("z table: at most ", max_points, " positions per evaluation, got ", + positions.size()); if (table.edges.size() < 2) throw std::logic_error("z table: no table has been uploaded"); Points points; @@ -78,10 +78,8 @@ inline Points locate(const Table& table, const std::vector& slots, for (std::size_t p = 0; p < positions.size(); ++p) { const double z = positions[p]; if (!(z >= table.lower && z <= table.upper)) - throw std::out_of_range("z = " + std::to_string(z) + - " m is outside the z table's span [" + - std::to_string(table.edges.front()) + ", " + - std::to_string(table.edges.back()) + "] m"); + refuse("z = ", z, " m is outside the z table's span [", + table.edges.front(), ", ", table.edges.back(), "] m"); std::size_t k = chebyshev::interval_index(z, table.edges.data(), table.edges.size()); if (before && k > 0 && z <= table.edges[k]) diff --git a/native/scripts/check.sh b/native/scripts/check.sh index 1361f8975..c5fa4924d 100755 --- a/native/scripts/check.sh +++ b/native/scripts/check.sh @@ -62,6 +62,8 @@ mkdir -p build if stage lint; then uv run ruff format --check . uv run ruff check . +# The bound problem's part of _core.pyi is stubgen's, from the build installed above. +uv run python scripts/problem_stub.py --check uv run python - "$repo_root" "$workspace_root" <<'PY' import json import sys diff --git a/native/scripts/problem_stub.py b/native/scripts/problem_stub.py new file mode 100644 index 000000000..64db04083 --- /dev/null +++ b/native/scripts/problem_stub.py @@ -0,0 +1,74 @@ +# SPDX-FileCopyrightText: 2026 John C. Travers and contributors +# +# SPDX-License-Identifier: GPL-3.0-or-later + +"""Write the bound problem's part of ``_core.pyi`` from ``nanobind.stubgen``. + +The extension binds ``FieldFamily`` and the nested ``Problem`` aggregates; stubgen +writes each field as a property and its setter, typed by the array caster. This keeps +the setter's type as a plain attribute, so the stub stays short and strict pyright +accepts the adapter's assignments, and replaces the text between the stub's markers. +``--check`` exits 1 instead when the committed stub differs (the native lint stage). +""" + +import re +import subprocess +import sys +from pathlib import Path + +from nanobind.stubgen import StubGen + +import solprop_native._core as core + +STUB = Path(__file__).resolve().parents[1] / "src/solprop_native/_core.pyi" +BEGIN = ( + "# Generated by native/scripts/problem_stub.py from nanobind.stubgen.\n" + "# Array fields are copied in and out: assign a whole array, since an item\n" + "# assigned through a read copy changes nothing.\n" +) +END = "\n# End of the generated problem stub.\n" +PAIR = re.compile( + r"@property\n\s*def (\w+)\(self\) -> .+: \.\.\.\n\n" + r"\s*@\1\.setter\n\s*def \1\(self, arg: (.+), /\) -> None: \.\.\." +) + + +def generated() -> str: + """Return the problem's stub section, formatted as ruff formats a stub.""" + stub = StubGen(core, quiet=True, include_docstrings=False) + for name in ("FieldFamily", "Problem"): + stub.put(getattr(core, name), name) + text = PAIR.sub(r"\1: \2", stub.get()) + text = text.replace("numpy.typing.NDArray[numpy.", "npt.NDArray[np.") + text = text[text.index("class FieldFamily") :] + text = re.sub(r"\n\n( +\w+: )", r"\n\1", text) + text = re.sub(r"(def __init__\(self\) -> None: \.\.\.)\n\n", r"\1\n", text) + formatted = subprocess.run( + [sys.executable, "-m", "ruff", "format", "--stdin-filename", str(STUB), "-"], + input=text, + capture_output=True, + text=True, + check=True, + ).stdout + return BEGIN + formatted + END + + +def main(argv: list[str]) -> int: + """Rewrite the section, or with ``--check`` report whether it is stale.""" + text = STUB.read_text(encoding="utf-8") + # The section runs from the marker's first line to the end marker. + start = text.index(BEGIN.splitlines(keepends=True)[0]) + end = text.index(END, start) + len(END) + head, current, tail = text[:start], text[start:end], text[end:] + fresh = generated() + if "--check" in argv: + if current != fresh: + print(f"{STUB.name} is stale: run native/scripts/problem_stub.py") + return 1 + return 0 + STUB.write_text(head + fresh + tail, encoding="utf-8") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/native/src/adams_march.cpp b/native/src/adams_march.cpp index a06152acb..8a32a61a8 100644 --- a/native/src/adams_march.cpp +++ b/native/src/adams_march.cpp @@ -5,6 +5,7 @@ #include "solprop_native/adams_march.hpp" #include "solprop_native/core.hpp" +#include "solprop_native/refuse.hpp" #include #include @@ -56,9 +57,8 @@ std::size_t bisect_right(const std::vector& values, const double x) { return static_cast(std::ranges::upper_bound(values, x) - values.begin()); } -void refuse(const char* field, const double value) { - throw std::invalid_argument(std::string("the Adams march's policy has ") + field + - " = " + std::to_string(value)); +[[noreturn]] void refuse_policy(const char* field, const double value) { + refuse("the Adams march's policy has ", field, " = ", value); } bool increasing(const std::vector& values) { @@ -85,46 +85,46 @@ double strict_probe(const double a, const double b, const double c) { void validate(const Policy& policy) { if (policy.order < 1 || policy.order > 6) - refuse("order", static_cast(policy.order)); + refuse_policy("order", static_cast(policy.order)); if (policy.real_intervals.size() < policy.order) - refuse("real_intervals", static_cast(policy.real_intervals.size())); + refuse_policy("real_intervals", static_cast(policy.real_intervals.size())); if (!(policy.safety > 0.0 && policy.safety <= 1.0)) - refuse("safety", policy.safety); + refuse_policy("safety", policy.safety); if (!(policy.max_growth > 1.0)) - refuse("max_growth", policy.max_growth); + refuse_policy("max_growth", policy.max_growth); if (!(policy.min_factor > 0.0 && policy.min_factor < 1.0)) - refuse("min_factor", policy.min_factor); + refuse_policy("min_factor", policy.min_factor); if (!(policy.rejection_growth > 0.0 && policy.rejection_growth <= 1.0)) - refuse("rejection_growth", policy.rejection_growth); + refuse_policy("rejection_growth", policy.rejection_growth); if (!(policy.hysteresis >= 1.0)) - refuse("hysteresis", policy.hysteresis); + refuse_policy("hysteresis", policy.hysteresis); if (policy.restart_after < 1) - refuse("restart_after", static_cast(policy.restart_after)); + refuse_policy("restart_after", static_cast(policy.restart_after)); if (!(policy.collar_stability > 0.0)) - refuse("collar_stability", policy.collar_stability); + refuse_policy("collar_stability", policy.collar_stability); if (!(policy.collar_rate >= 0.0)) - refuse("collar_rate", policy.collar_rate); + refuse_policy("collar_rate", policy.collar_rate); if (!(policy.carrier_cap > 0.0)) - refuse("carrier_cap", policy.carrier_cap); + refuse_policy("carrier_cap", policy.carrier_cap); if (policy.has_budget && policy.max_calls < 1) - refuse("max_calls", static_cast(policy.max_calls)); + refuse_policy("max_calls", static_cast(policy.max_calls)); if (!(policy.dz_min >= 0.0 && policy.dz_min <= policy.dz_max) || !std::isfinite(policy.dz_max)) - refuse("dz_min", policy.dz_min); + refuse_policy("dz_min", policy.dz_min); if (!(policy.stretch >= 1.0) || !std::isfinite(policy.stretch)) - refuse("stretch", policy.stretch); + refuse_policy("stretch", policy.stretch); if (!std::isfinite(policy.z_end)) - refuse("z_end", policy.z_end); + refuse_policy("z_end", policy.z_end); if (policy.breakpoints.size() < 2 || !increasing(policy.breakpoints)) - refuse("breakpoints", static_cast(policy.breakpoints.size())); + refuse_policy("breakpoints", static_cast(policy.breakpoints.size())); if (!increasing(policy.save_planes)) - refuse("save_planes", static_cast(policy.save_planes.size())); + refuse_policy("save_planes", static_cast(policy.save_planes.size())); if (!increasing(policy.hard_kinks)) - refuse("hard_kinks", static_cast(policy.hard_kinks.size())); + refuse_policy("hard_kinks", static_cast(policy.hard_kinks.size())); if (policy.landing_ulps < 1) - refuse("landing_ulps", policy.landing_ulps); + refuse_policy("landing_ulps", policy.landing_ulps); if (policy.has_quiet_interval && !(policy.quiet_interval > 0.0)) - refuse("quiet_interval", policy.quiet_interval); + refuse_policy("quiet_interval", policy.quiet_interval); } double factor(const Policy& policy, const double norm, const std::size_t order, @@ -258,17 +258,14 @@ double collar_reentry(const Policy& policy, const double proposal, Landing landing(const Policy& policy, const double z, const double dz, const double z_end) { if (!(dz > 0.0) || !std::isfinite(dz)) - throw std::invalid_argument("dz must be finite and positive, got " + - std::to_string(dz)); + refuse("dz must be finite and positive, got ", dz); const auto& edges = policy.breakpoints; if (!(edges.front() <= z && z <= edges.back())) - throw std::invalid_argument("z = " + std::to_string(z) + - " m lies outside the operator table"); + refuse("z = ", z, " m lies outside the operator table"); const double tolerance = policy.landing_ulps * ulp(py_max(std::fabs(z), std::fabs(z_end))); if (z >= z_end - tolerance) - throw std::invalid_argument("z = " + std::to_string(z) + - " m is already at or past z_end"); + refuse("z = ", z, " m is already at or past z_end"); const std::size_t found = bisect_right(edges, z); std::size_t index = found == 0 ? 0 : found - 1; index = std::min(index, edges.size() - 2); diff --git a/native/src/bindings.cpp b/native/src/bindings.cpp index d737e9651..3bf902ce3 100644 --- a/native/src/bindings.cpp +++ b/native/src/bindings.cpp @@ -12,6 +12,7 @@ #include "solprop_native/fft_plan.hpp" #include "solprop_native/lagrange_basis.hpp" #include "solprop_native/low_storage.hpp" +#include "solprop_native/refuse.hpp" #include "solprop_native/rk_march.hpp" #include "solprop_native/session.hpp" #include "solprop_native/stage_tables.hpp" @@ -158,16 +159,6 @@ std::vector index_vector(const IndexInput values) { return result; } -native::FieldFamily field_family(const std::string& name) { - if (name == "field_real") - return native::FieldFamily::field_real; - if (name == "field_analytic") - return native::FieldFamily::field_analytic; - if (name == "envelope") - return native::FieldFamily::envelope; - throw std::invalid_argument("unknown field family " + name); -} - native::TableKind table_kind(const std::string& name) { if (name == "phi") return native::TableKind::phi; @@ -197,9 +188,8 @@ native::CpuExecution cpu_execution_from(const nb::dict& values) { const std::size_t expected = 2 * native::cpu_operation_count + cpu_execution_scalars.size(); if (values.size() != expected) - throw std::invalid_argument("a CPU execution record has " + - std::to_string(expected) + " fields, got " + - std::to_string(values.size())); + native::refuse("a CPU execution record has ", expected, " fields, got ", + values.size()); const auto field = [&values](const std::string& name) { if (!values.contains(name.c_str())) throw std::invalid_argument("a CPU execution record misses a field: " + name); @@ -316,6 +306,24 @@ nb::dict fft_planning_dict(const native::FftPlanning& planning) { return result; } +// The planning a workspace is asked for, as `fft_planning_from` reads it back. +nb::dict fft_request_dict(const native::FftPlanning& planning) { + static constexpr std::array modes{"legacy", "profile", "capture"}; + nb::dict efforts; + efforts["rows"] = planning.rows; + efforts["threaded_rows"] = planning.threaded_rows; + efforts["planes"] = planning.planes; + nb::dict result; + result["mode"] = modes[static_cast(planning.mode)]; + result["efforts"] = efforts; + result["wisdom"] = planning.wisdom; + result["capture_effort"] = planning.capture_effort; + result["time_limit"] = planning.time_limit; + result["stored"] = planning.stored; + result["keep"] = planning.keep; + return result; +} + nb::dict cpu_execution_dict(const native::CpuExecution& execution) { nb::dict result; for (std::size_t index = 0; index < native::cpu_operation_count; ++index) { @@ -334,390 +342,367 @@ nb::dict cpu_execution_dict(const native::CpuExecution& execution) { return result; } -native::Problem problem_from(const nb::dict& values) { - native::Problem result; - if (!values.contains("cpu_execution")) - throw std::invalid_argument("a native problem needs its CPU execution record " - "(cpu_execution, solprop SEP 46)"); - try { - result.cpu_execution = - cpu_execution_from(nb::cast(values["cpu_execution"])); - } catch (const nb::cast_error&) { - throw std::invalid_argument("a native problem's cpu_execution must be a dict"); - } -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - if (values.contains("fft_planning")) { - try { - result.fft_planning = - fft_planning_from(nb::cast(values["fft_planning"])); - } catch (const nb::cast_error&) { - throw std::invalid_argument("a native problem's fft_planning must be a dict"); - } - } -#endif - try { - if (values.contains("n_x")) { - result.n_x = nb::cast(values["n_x"]); - result.n_y = nb::cast(values["n_y"]); - if (values.contains("n_p")) - result.n_p = nb::cast(values["n_p"]); - if (values.contains("spatial_far_field_slots")) - result.spatial_far_field_slots = - nb::cast(values["spatial_far_field_slots"]); - } - result.n_omega = nb::cast(values["n_omega"]); - result.n_t = nb::cast(values["n_t"]); - result.n_to = nb::cast(values["n_to"]); - result.n_stages = nb::cast(values["n_stages"]); - if (values.contains("n_modes")) { - result.n_modes = nb::cast(values["n_modes"]); - result.n_pol_state = nb::cast(values["n_pol_state"]); - result.n_pol_transform = nb::cast(values["n_pol_transform"]); - result.n_nodes = nb::cast(values["n_nodes"]); - result.node_chunk = nb::cast(values["node_chunk"]); - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("problem dimensions must be integers"); - } - try { - result.family = field_family(nb::cast(values["family"])); - result.nonlinear = nb::cast(values["nonlinear"]); - result.modified_noise = nb::cast(values["modified_noise"]); - result.in_stage_absorber = nb::cast(values["in_stage_absorber"]); - result.split_step_absorber = nb::cast(values["split_step_absorber"]); - result.first_same_as_last = nb::cast(values["first_same_as_last"]); - if (values.contains("low_storage")) { - result.low_storage = nb::cast(values["low_storage"]); - result.low_storage_stage_register = - nb::cast(values["low_storage_stage_register"]); - result.low_storage_flows = nb::cast(values["low_storage_flows"]); - result.low_storage_tiled_input = - nb::cast(values["low_storage_tiled_input"]); - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("problem family and flags have wrong types"); +// ------------------------------------------------------------- the problem's fields +// A vector field of a bound struct crosses as an exact NumPy array: float64, +// complex128, uint64 or bool, C-contiguous on the CPU and of any rank, copied in, and +// out as a one-dimensional copy. Anything else is refused, naming the field. + +template struct ExactArray; +template <> struct ExactArray { + using Input = RealInput; + using Output = nb::ndarray, nb::c_contig>; + using Stored = double; + static constexpr const char* dtype = "float64"; + static constexpr const char* scalar = "numpy.float64"; +}; +template <> struct ExactArray { + using Input = ComplexInput; + using Output = + nb::ndarray, nb::ndim<1>, nb::c_contig>; + using Stored = std::complex; + static constexpr const char* dtype = "complex128"; + static constexpr const char* scalar = "numpy.complex128"; +}; +template <> struct ExactArray { + using Input = IndexInput; + using Output = nb::ndarray, nb::c_contig>; + using Stored = std::uint64_t; + static constexpr const char* dtype = "uint64"; + static constexpr const char* scalar = "numpy.uint64"; +}; +template <> struct ExactArray { + using Input = BoolInput; + using Output = nb::ndarray, nb::c_contig>; + using Stored = bool; + static constexpr const char* dtype = "bool"; + static constexpr const char* scalar = "numpy.bool_"; +}; + +template +std::vector exact_vector(const nb::handle value, const std::string& field, + const std::size_t rank, + std::vector* shape = nullptr) { + typename ExactArray::Input array; + if (!nb::try_cast(value, array, false) || (rank != 0 && array.ndim() != rank)) + native::refuse(field, " must be a C-contiguous ", ExactArray::dtype, + " array", rank == 0 ? "" : native::concat(" of rank ", rank)); + if (shape != nullptr) { + shape->resize(array.ndim()); + for (std::size_t axis = 0; axis < array.ndim(); ++axis) + (*shape)[axis] = array.shape(axis); + } + if constexpr (std::is_same_v) + return real_vector(array); + else if constexpr (std::is_same_v) + return complex_vector(array); + else if constexpr (std::is_same_v) + return index_vector(array); + else + return bool_vector(array); +} + +template +// NOLINTNEXTLINE(readability-redundant-typename): nvcc's front end needs it +typename ExactArray::Output array_of(const std::vector& values) { + // NOLINTNEXTLINE(readability-redundant-typename): nvcc's front end needs it + using Stored = typename ExactArray::Stored; + auto storage = std::make_unique(values.size()); + for (std::size_t index = 0; index < values.size(); ++index) { + if constexpr (std::is_same_v) + storage[index] = {values[index].real(), values[index].imag()}; + else + storage[index] = static_cast(values[index]); } - try { - result.crop_index = index_vector(exact_array(values["crop_index"])); - result.active_mask = bool_vector(exact_array(values["active_mask"])); - result.norm_mask = bool_vector(exact_array(values["norm_mask"])); - result.omega_window = real_vector(exact_array(values["omega_window"])); - if (values.contains("omega")) - result.omega = real_vector(exact_array(values["omega"])); - if (values.contains("power_weight")) - result.power_weight = real_vector(exact_array(values["power_weight"])); - result.time_window = real_vector(exact_array(values["time_window"])); - result.time_window_derivative = - real_vector(exact_array(values["time_window_derivative"])); - result.absorber_fine = real_vector(exact_array(values["absorber_fine"])); - result.absorber_coarse = - real_vector(exact_array(values["absorber_coarse"])); - if (result.separable_geometry()) { - result.spatial_evanescent_rate = - nb::cast(values["spatial_evanescent_rate"]); - result.spatial_k_squared = - complex_vector(exact_array(values["spatial_k_squared"])); - result.spatial_k_squared_low = - complex_vector(exact_array(values["spatial_k_squared_low"])); - result.spatial_numerator = - complex_vector(exact_array(values["spatial_numerator"])); - result.spatial_clamped = - complex_vector(exact_array(values["spatial_clamped"])); - result.spatial_kperp_sq = - real_vector(exact_array(values["spatial_kperp_sq"])); - result.spatial_kperp_sq_low = - real_vector(exact_array(values["spatial_kperp_sq_low"])); - result.spatial_frame = - real_vector(exact_array(values["spatial_frame"])); - result.spatial_spectral_rate = - real_vector(exact_array(values["spatial_spectral_rate"])); - result.spatial_user_rate = - real_vector(exact_array(values["spatial_user_rate"])); - result.spatial_absorber_rate = - real_vector(exact_array(values["spatial_absorber_rate"])); - result.spatial_kperp_rate = - real_vector(exact_array(values["spatial_kperp_rate"])); - if (result.slab_geometry()) - result.spatial_kappa_sq = - complex_vector(exact_array(values["spatial_kappa_sq"])); - if (result.free_geometry()) { - result.spatial_k_x = real_vector(exact_array(values["spatial_k_x"])); - result.spatial_k_y = real_vector(exact_array(values["spatial_k_y"])); - } - } - if (values.contains("spatial_absorber_matrix")) - result.spatial_absorber_matrix = - real_vector(exact_array(values["spatial_absorber_matrix"])); - if (values.contains("spatial_spectral_weights")) { - result.spatial_spectral_weights = - real_vector(exact_array(values["spatial_spectral_weights"])); - result.spatial_on_axis = - complex_vector(exact_array(values["spatial_on_axis"])); - } - if (values.contains("flux_k_squared")) { - result.flux_k_squared = - complex_vector(exact_array(values["flux_k_squared"])); - result.flux_k_squared_low = - complex_vector(exact_array(values["flux_k_squared_low"])); - result.flux_kperp_sq = - real_vector(exact_array(values["flux_kperp_sq"])); - result.flux_kperp_sq_low = - real_vector(exact_array(values["flux_kperp_sq_low"])); - result.flux_table = nb::cast(values["flux_table"]); - } - if (values.contains("flux_closure")) - result.flux_closure = nb::cast(values["flux_closure"]); - result.carrier = complex_vector(exact_array(values["carrier"])); - if (values.contains("synthesis_matrix")) { - result.synthesis_matrix = - real_vector(exact_array(values["synthesis_matrix"])); - result.projection_matrix = - real_vector(exact_array(values["projection_matrix"])); - result.projection_coarse_matrix = - real_vector(exact_array(values["projection_coarse_matrix"])); + nb::capsule owner(storage.get(), [](void* held) noexcept { + const std::unique_ptr owned(static_cast(held)); + }); + const std::array shape{values.size()}; + return {storage.release(), 1, shape.data(), owner}; +} + +template struct IsVector : std::false_type {}; +template struct IsVector> : std::true_type {}; + +/** One group of the problem's fields, bound as a nested class: scalars read and + written as they are, vectors as exact arrays (`exact_vector`). */ +template class Fields { +public: + Fields(nb::handle scope, const char* name, std::string path) + : class_(scope, name), path_(std::move(path)) { + class_.def(nb::init<>()); + } + /** Bind `member` as `name`; a vector field of a nonzero `rank` takes arrays of that + rank alone, as the extents a count beside it gives are laid out. */ + template + Fields& field(const char* name, Value Group::* member, const std::size_t rank = 0) { + if constexpr (IsVector::value) { + // NOLINTNEXTLINE(readability-redundant-typename): nvcc's front end needs it + using Element = typename Value::value_type; + // The setter's stub names the one dtype it takes. + const std::string setter = + native::concat("def ", name, "(self, arg: numpy.typing.NDArray[", + ExactArray::scalar, "], /) -> None"); + class_.def_prop_rw( + name, [member](const Group& self) { return array_of(self.*member); }, + [member, rank, name, path = path_ + "." + name](Group& self, + const nb::handle value) { + // A group whose layouts its counts give keeps each array's shape for + // `Problem::validate`. + if constexpr (requires { self.shapes; }) { + std::vector shape; + self.*member = exact_vector(value, path, rank, &shape); + self.shapes[name] = std::move(shape); + } else { + static_cast(name); + self.*member = exact_vector(value, path, rank); + } + }, + nb::for_getter(nb::rv_policy::automatic), + nb::for_setter(nb::sig(setter.c_str()))); + } else { + static_cast(rank); + class_.def_rw(name, member); } - } catch (const nb::cast_error&) { - throw std::invalid_argument("problem masks and transform arrays have wrong dtypes"); - } - ComplexInput phi; - ComplexInput prefactor; - try { - phi = exact_array(values["phi_terms"]); - prefactor = exact_array(values["prefactor_terms"]); - } catch (const nb::cast_error&) { - throw std::invalid_argument("fixed terms must be complex128 arrays"); - } - if ((phi.ndim() != 2 && phi.ndim() != 3) || - (prefactor.ndim() != 2 && prefactor.ndim() != 3)) { - throw std::invalid_argument("fixed terms must be rank-two or rank-three arrays"); - } - result.n_phi_terms = phi.shape(0); - result.n_prefactor_terms = prefactor.shape(0); - result.phi_terms = complex_vector(phi); - result.prefactor_terms = complex_vector(prefactor); - // solprop SEP 29 G1: a z table's terms are slots its evaluation fills; the leading - // extent of an empty terms array names how many. - try { - if (values.contains("phi_table")) - result.phi_table = nb::cast(values["phi_table"]); - if (values.contains("prefactor_table")) - result.prefactor_table = nb::cast(values["prefactor_table"]); - } catch (const nb::cast_error&) { - throw std::invalid_argument("z table flags must be booleans"); - } - try { - result.tableau_matrix = - real_vector(exact_array(values["tableau_matrix"])); - result.tableau_nodes = real_vector(exact_array(values["tableau_nodes"])); - result.solution_weights = - real_vector(exact_array(values["solution_weights"])); - result.error_weights = real_vector(exact_array(values["error_weights"])); - } catch (const nb::cast_error&) { - throw std::invalid_argument("tableau values must be float64 arrays"); + return *this; } - try { - result.synthesis = nb::cast(values["synthesis"]); - result.projection = nb::cast(values["projection"]); - result.kerr_coefficient = nb::cast(values["kerr_coefficient"]); - result.self_factor = nb::cast(values["self_factor"]); - result.conjugate_factor = nb::cast(values["conjugate_factor"]); - result.harmonic_factor = nb::cast(values["harmonic_factor"]); - result.take_real = nb::cast(values["take_real"]); - if (values.contains("plasma")) { - result.plasma = nb::cast(values["plasma"]); - result.raman = nb::cast(values["raman"]); - result.raman_thg = nb::cast(values["raman_thg"]); - result.stash_plasma = nb::cast(values["stash_plasma"]); - result.weighted_controller = nb::cast(values["weighted_controller"]); - result.dt_fine = nb::cast(values["dt_fine"]); - result.dt = nb::cast(values["dt"]); - result.raman_fraction = nb::cast(values["raman_fraction"]); - result.raman_epsilon0 = nb::cast(values["raman_epsilon0"]); - const auto raman_undamped = exact_array(values["raman_undamped"]); - if (result.raman && (raman_undamped.ndim() != 2 || raman_undamped.shape(0) == 0 || - result.n_to > std::numeric_limits::max() / 2 || - raman_undamped.shape(1) != 2 * result.n_to)) - throw std::invalid_argument( - "raman_undamped must have shape (n_terms, 2 * n_to)"); - result.n_raman_terms = result.raman ? raman_undamped.shape(0) : 0; - if (result.raman) { - result.raman_undamped = real_vector(raman_undamped); - result.raman_tensor_capable = nb::cast(values["raman_tensor_capable"]); - const auto weights = exact_array(values["raman_tensor_weights"]); - if (weights.ndim() != 2 || weights.shape(0) != result.n_raman_terms || - weights.shape(1) != 2) - throw std::invalid_argument( - "raman_tensor_weights must have shape (n_terms, 2)"); - result.raman_tensor_weights = real_vector(weights); - } - result.charge_mass_ratio = nb::cast(values["charge_mass_ratio"]); - result.ionisation_potential = nb::cast(values["ionisation_potential"]); - result.log_field_min = nb::cast(values["log_field_min"]); - result.d_log_field = nb::cast(values["d_log_field"]); - result.field_min = nb::cast(values["field_min"]); - result.field_max = nb::cast(values["field_max"]); - result.rate_max = nb::cast(values["rate_max"]); - const auto coefficients = exact_array(values["rate_coefficients"]); - if (coefficients.ndim() != 2 || coefficients.shape(0) != 4) - throw std::invalid_argument( - "rate_coefficients must have shape (4, n_intervals)"); - result.rate_intervals = coefficients.shape(1); - result.rate_coefficients = real_vector(coefficients); - // solprop SEP 18: the running-integral rule. Translation designed and - // certified the taps; the core checks their extent, finiteness and the - // trapezoid's one identity tap. - if (values.contains("plasma_gated")) - result.plasma_gated = nb::cast(values["plasma_gated"]); - if (values.contains("plasma_band")) { - result.plasma_band = nb::cast(values["plasma_band"]); - const auto taps = exact_array(values["plasma_taps"]); - if (taps.ndim() != 1) - throw std::invalid_argument("plasma_taps must be one-dimensional"); - result.plasma_taps = real_vector(taps); - } - if (values.contains("n_channels")) { - result.n_channels = nb::cast(values["n_channels"]); - result.channel_parameters = - real_vector(exact_array(values["channel_parameters"])); - result.channel_coefficients = - real_vector(exact_array(values["channel_coefficients"])); - result.n_raman_components = nb::cast(values["n_raman_components"]); - result.raman_row_component = - real_vector(exact_array(values["raman_row_component"])); - result.raman_component_fraction = - real_vector(exact_array(values["raman_component_fraction"])); - } - if (values.contains("envelope_plasma")) { - result.envelope_plasma = nb::cast(values["envelope_plasma"]); - if (result.envelope_plasma) { - const auto nodes = exact_array(values["envelope_rule_nodes"]); - const auto weights = exact_array(values["envelope_rule_weights"]); - for (const auto& rule : {nodes, weights}) - if (rule.ndim() != 3 || rule.shape(0) != result.n_channels || - rule.shape(1) != 4 || rule.shape(2) != 512) + +private: + nb::class_ class_; + std::string path_; +}; + +void bind_problem(nb::module_& module_) { + using P = native::Problem; + nb::enum_(module_, "FieldFamily") + .value("field_real", native::FieldFamily::field_real) + .value("field_analytic", native::FieldFamily::field_analytic) + .value("envelope", native::FieldFamily::envelope); + nb::class_

problem(module_, "Problem"); + Fields(problem, "Dims", "dims") + .field("n_omega", &P::Dims::n_omega) + .field("n_t", &P::Dims::n_t) + .field("n_to", &P::Dims::n_to) + .field("n_modes", &P::Dims::n_modes) + .field("n_pol_state", &P::Dims::n_pol_state) + .field("n_pol_transform", &P::Dims::n_pol_transform) + .field("n_nodes", &P::Dims::n_nodes) + .field("node_chunk", &P::Dims::node_chunk) + .field("family", &P::Dims::family); + Fields(problem, "Grid", "grid") + .field("crop_index", &P::Grid::crop_index) + .field("active_mask", &P::Grid::active_mask) + .field("norm_mask", &P::Grid::norm_mask) + .field("omega_window", &P::Grid::omega_window) + .field("omega", &P::Grid::omega) + .field("power_weight", &P::Grid::power_weight) + .field("time_window", &P::Grid::time_window) + .field("time_window_derivative", &P::Grid::time_window_derivative) + .field("carrier", &P::Grid::carrier) + .field("dt_fine", &P::Grid::dt_fine) + .field("dt", &P::Grid::dt); + Fields(problem, "Spatial", "spatial") + .field("n_x", &P::Spatial::n_x) + .field("n_y", &P::Spatial::n_y) + .field("k_squared", &P::Spatial::k_squared) + .field("k_squared_low", &P::Spatial::k_squared_low) + .field("numerator", &P::Spatial::numerator) + .field("clamped", &P::Spatial::clamped) + .field("kperp_sq", &P::Spatial::kperp_sq) + .field("kperp_sq_low", &P::Spatial::kperp_sq_low) + .field("frame", &P::Spatial::frame) + .field("spectral_rate", &P::Spatial::spectral_rate) + .field("user_rate", &P::Spatial::user_rate) + .field("absorber_rate", &P::Spatial::absorber_rate) + .field("kperp_rate", &P::Spatial::kperp_rate) + .field("evanescent_rate", &P::Spatial::evanescent_rate) + .field("k_x", &P::Spatial::k_x) + .field("k_y", &P::Spatial::k_y) + .field("far_field_slots", &P::Spatial::far_field_slots) + .field("n_p", &P::Spatial::n_p) + .field("kappa_sq", &P::Spatial::kappa_sq) + .field("absorber_matrix", &P::Spatial::absorber_matrix) + .field("spectral_weights", &P::Spatial::spectral_weights) + .field("on_axis", &P::Spatial::on_axis); + Fields(problem, "Flux", "flux") + .field("k_squared", &P::Flux::k_squared) + .field("k_squared_low", &P::Flux::k_squared_low) + .field("kperp_sq", &P::Flux::kperp_sq) + .field("kperp_sq_low", &P::Flux::kperp_sq_low) + .field("table", &P::Flux::table) + .field("closure", &P::Flux::closure); + Fields(problem, "Stepper", "stepper") + .field("n_stages", &P::Stepper::n_stages) + .field("tableau_matrix", &P::Stepper::tableau_matrix) + .field("tableau_nodes", &P::Stepper::tableau_nodes) + .field("solution_weights", &P::Stepper::solution_weights) + .field("error_weights", &P::Stepper::error_weights) + .field("first_same_as_last", &P::Stepper::first_same_as_last) + .field("weighted_controller", &P::Stepper::weighted_controller) + .field("modified_noise", &P::Stepper::modified_noise) + .field("low_storage", &P::Stepper::low_storage) + .field("low_storage_stage_register", &P::Stepper::low_storage_stage_register) + .field("low_storage_flows", &P::Stepper::low_storage_flows) + .field("low_storage_tiled_input", &P::Stepper::low_storage_tiled_input); + Fields(problem, "Linear", "linear") + .field("n_phi_terms", &P::Linear::n_phi_terms) + .field("n_prefactor_terms", &P::Linear::n_prefactor_terms) + .field("phi_table", &P::Linear::phi_table) + .field("prefactor_table", &P::Linear::prefactor_table) + .field("phi_terms", &P::Linear::phi_terms) + .field("prefactor_terms", &P::Linear::prefactor_terms); + Fields(problem, "Absorber", "absorber") + .field("in_stage", &P::Absorber::in_stage) + .field("split_step", &P::Absorber::split_step) + .field("fine", &P::Absorber::fine) + .field("coarse", &P::Absorber::coarse); + Fields(problem, "Transform", "transform") + .field("synthesis_matrix", &P::Transform::synthesis_matrix) + .field("projection_matrix", &P::Transform::projection_matrix) + .field("projection_coarse_matrix", &P::Transform::projection_coarse_matrix) + .field("synthesis", &P::Transform::synthesis) + .field("projection", &P::Transform::projection); + Fields(problem, "Kerr", "kerr") + .field("nonlinear", &P::Kerr::nonlinear) + .field("coefficient", &P::Kerr::coefficient) + .field("self_factor", &P::Kerr::self_factor) + .field("conjugate_factor", &P::Kerr::conjugate_factor) + .field("harmonic_factor", &P::Kerr::harmonic_factor) + .field("take_real", &P::Kerr::take_real); + Fields(problem, "Plasma", "plasma") + .field("enabled", &P::Plasma::enabled) + .field("envelope", &P::Plasma::envelope) + .field("gated", &P::Plasma::gated) + .field("stash", &P::Plasma::stash) + .field("envelope_rule_nodes", &P::Plasma::envelope_rule_nodes, 3) + .field("envelope_rule_weights", &P::Plasma::envelope_rule_weights, 3) + .field("envelope_drude_factor", &P::Plasma::envelope_drude_factor, 1) + .field("envelope_loss_factor", &P::Plasma::envelope_loss_factor, 1) + .field("charge_mass_ratio", &P::Plasma::charge_mass_ratio) + .field("ionisation_potential", &P::Plasma::ionisation_potential) + .field("log_field_min", &P::Plasma::log_field_min) + .field("d_log_field", &P::Plasma::d_log_field) + .field("field_min", &P::Plasma::field_min) + .field("field_max", &P::Plasma::field_max) + .field("rate_max", &P::Plasma::rate_max) + .field("rate_intervals", &P::Plasma::rate_intervals) + .field("rate_coefficients", &P::Plasma::rate_coefficients, 2) + .field("n_channels", &P::Plasma::n_channels) + .field("band", &P::Plasma::band) + .field("taps", &P::Plasma::taps, 1) + .field("channel_parameters", &P::Plasma::channel_parameters) + .field("channel_coefficients", &P::Plasma::channel_coefficients); + Fields(problem, "Raman", "raman") + .field("enabled", &P::Raman::enabled) + .field("thg", &P::Raman::thg) + .field("tensor_capable", &P::Raman::tensor_capable) + .field("n_terms", &P::Raman::n_terms) + .field("fraction", &P::Raman::fraction) + .field("epsilon0", &P::Raman::epsilon0) + .field("undamped", &P::Raman::undamped, 2) + .field("tensor_weights", &P::Raman::tensor_weights, 2) + .field("n_components", &P::Raman::n_components) + .field("row_component", &P::Raman::row_component) + .field("component_fraction", &P::Raman::component_fraction); + Fields(problem, "General", "general") + .field("enabled", &P::General::enabled) + .field("real_form", &P::General::real_form) + .field("map_intervals", &P::General::map_intervals) + .field("map_n_deg", &P::General::map_n_deg) + .field("components", &P::General::components) + .field("nodes", &P::General::nodes) + .field("node_chunk", &P::General::node_chunk) + .field("responses", &P::General::responses) + .field("terms", &P::General::terms) + .field("pad_scale", &P::General::pad_scale) + .field("epsilon0", &P::General::epsilon0) + .field("synthesis", &P::General::synthesis) + .field("projection", &P::General::projection) + .field("frequency_factor", &P::General::frequency_factor) + .field("delayed_spectrum", &P::General::delayed_spectrum) + .field("tensor", &P::General::tensor) + .field("delayed_tensor", &P::General::delayed_tensor) + .field("node_response", &P::General::node_response); + Fields(problem, "Flow", "flow") + .field("enabled", &P::Flow::enabled) + .field("active", &P::Flow::active) + .field("sections", &P::Flow::sections) + .field("terms", &P::Flow::terms) + .field("slot", &P::Flow::slot) + .field("frozen", &P::Flow::frozen) + .field("residual_terms", &P::Flow::residual_terms) + .field("left", &P::Flow::left) + .field("right", &P::Flow::right) + .field("eigenvalues", &P::Flow::eigenvalues) + .field("route", &P::Flow::route) + .field("fallback_vectors", &P::Flow::fallback_vectors) + .field("fallback_triangular", &P::Flow::fallback_triangular); + Fields(problem, "Dispersive", "dispersive") + .field("enabled", &P::Dispersive::enabled) + .field("symmetric", &P::Dispersive::symmetric) + .field("form", &P::Dispersive::form) + .field("inputs", &P::Dispersive::inputs) + .field("term_multipliers", &P::Dispersive::term_multipliers) + .field("coefficients", &P::Dispersive::coefficients) + .field("pair_constants", &P::Dispersive::pair_constants) + .field("quartics", &P::Dispersive::quartics) + .field("input_scale", &P::Dispersive::input_scale) + .field("output_scale", &P::Dispersive::output_scale) + .field("term_rows", &P::Dispersive::term_rows) + .field("term_channels", &P::Dispersive::term_channels) + .field("group_offsets", &P::Dispersive::group_offsets) + .field("group_members", &P::Dispersive::group_members) + .field("group_keys", &P::Dispersive::group_keys) + .field("node_region", &P::Dispersive::node_region); + problem.def(nb::init<>()) + .def_rw("dims", &P::dims) + .def_rw("grid", &P::grid) + .def_rw("spatial", &P::spatial) + .def_rw("flux", &P::flux) + .def_rw("stepper", &P::stepper) + .def_rw("linear", &P::linear) + .def_rw("absorber", &P::absorber) + .def_rw("transform", &P::transform) + .def_rw("kerr", &P::kerr) + .def_rw("plasma", &P::plasma) + .def_rw("raman", &P::raman) + .def_rw("general", &P::general) + .def_rw("flow", &P::flow) + .def_rw("dispersive", &P::dispersive) + // solprop SEP 46: the CPU execution record and the FFT planning cross as the + // records solprop_native's `_execution` and `_wisdom` write. + .def_prop_rw( + "cpu_execution", + [](const P& self) { return cpu_execution_dict(self.cpu_execution); }, + [](P& self, const nb::handle values) { + if (!nb::isinstance(values)) throw std::invalid_argument( - "envelope rules require (n_channels, 4, 512)"); - const auto drude = exact_array(values["envelope_drude_factor"]); - const auto loss = exact_array(values["envelope_loss_factor"]); - if (drude.ndim() != 1 || drude.shape(0) != result.n_omega || - loss.ndim() != 1 || loss.shape(0) != result.n_omega) - throw std::invalid_argument("envelope current factors require (n_omega,)"); - result.envelope_rule_nodes = real_vector(nodes); - result.envelope_rule_weights = real_vector(weights); - result.envelope_drude_factor = real_vector(drude); - result.envelope_loss_factor = complex_vector(loss); - } - } - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("response scalars have wrong types"); - } - if (values.contains("dispersive")) { - try { - result.dispersive = nb::cast(values["dispersive"]); - if (result.dispersive) { - result.dispersive_symmetric = nb::cast(values["dispersive_symmetric"]); - result.dispersive_form = nb::cast(values["dispersive_form"]); - result.dispersive_inputs = - real_vector(exact_array(values["dispersive_inputs"])); - result.dispersive_term_multipliers = - real_vector(exact_array(values["dispersive_term_multipliers"])); - result.dispersive_term_rows = - index_vector(exact_array(values["dispersive_term_rows"])); - result.dispersive_term_channels = - index_vector(exact_array(values["dispersive_term_channels"])); - result.dispersive_coefficients = - real_vector(exact_array(values["dispersive_coefficients"])); - result.dispersive_pair_constants = - real_vector(exact_array(values["dispersive_pair_constants"])); - result.dispersive_group_offsets = - index_vector(exact_array(values["dispersive_group_offsets"])); - result.dispersive_group_members = - index_vector(exact_array(values["dispersive_group_members"])); - result.dispersive_quartics = - real_vector(exact_array(values["dispersive_quartics"])); - result.dispersive_input_scale = - real_vector(exact_array(values["dispersive_input_scale"])); - result.dispersive_output_scale = - real_vector(exact_array(values["dispersive_output_scale"])); - // The general modal route's group keys and node regions; absent elsewhere. - if (values.contains("dispersive_group_keys")) - result.dispersive_group_keys = - index_vector(exact_array(values["dispersive_group_keys"])); - if (values.contains("dispersive_node_region")) - result.dispersive_node_region = - index_vector(exact_array(values["dispersive_node_region"])); - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("dispersive response data have wrong types"); - } - } - if (values.contains("general_modal")) { - try { - result.general_modal = nb::cast(values["general_modal"]); - if (result.general_modal) { - result.general_real_form = nb::cast(values["general_real_form"]); - result.general_map_intervals = - nb::cast(values["general_map_intervals"]); - result.general_map_n_deg = nb::cast(values["general_map_n_deg"]); - result.general_components = nb::cast(values["general_components"]); - result.general_nodes = nb::cast(values["general_nodes"]); - result.general_node_chunk = nb::cast(values["general_node_chunk"]); - result.general_responses = nb::cast(values["general_responses"]); - result.general_terms = nb::cast(values["general_terms"]); - result.general_pad_scale = nb::cast(values["general_pad_scale"]); - result.general_epsilon0 = nb::cast(values["general_epsilon0"]); - result.general_synthesis = - complex_vector(exact_array(values["general_synthesis"])); - result.general_projection = - complex_vector(exact_array(values["general_projection"])); - result.general_tensor = - real_vector(exact_array(values["general_tensor"])); - result.general_frequency_factor = complex_vector( - exact_array(values["general_frequency_factor"])); - result.general_delayed_tensor = - real_vector(exact_array(values["general_delayed_tensor"])); - result.general_delayed_spectrum = complex_vector( - exact_array(values["general_delayed_spectrum"])); - result.general_node_response = - index_vector(exact_array(values["general_node_response"])); - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("general modal data have wrong types"); - } - } - if (values.contains("coupled_flow")) { - try { - result.coupled_flow = nb::cast(values["coupled_flow"]); - if (result.coupled_flow) { - result.flow_active = nb::cast(values["flow_active"]); - result.flow_sections = nb::cast(values["flow_sections"]); - result.flow_terms = nb::cast(values["flow_terms"]); - result.flow_slot = index_vector(exact_array(values["flow_slot"])); - result.flow_frozen = - complex_vector(exact_array(values["flow_frozen"])); - result.flow_residual_terms = - complex_vector(exact_array(values["flow_residual_terms"])); - if (values.contains("flow_left")) { - result.flow_left = - complex_vector(exact_array(values["flow_left"])); - result.flow_right = - complex_vector(exact_array(values["flow_right"])); - result.flow_eigenvalues = - complex_vector(exact_array(values["flow_eigenvalues"])); - result.flow_route = - index_vector(exact_array(values["flow_route"])); - result.flow_fallback_vectors = complex_vector( - exact_array(values["flow_fallback_vectors"])); - result.flow_fallback_triangular = complex_vector( - exact_array(values["flow_fallback_triangular"])); - } - } - } catch (const nb::cast_error&) { - throw std::invalid_argument("coupled flow data have wrong types"); - } - } - return result; + "a native problem's cpu_execution must be a dict"); + self.cpu_execution = cpu_execution_from(nb::borrow(values)); + }, + nb::for_getter(nb::sig("def cpu_execution(self) -> dict[str, int | bool]")), + nb::for_setter(nb::sig( + "def cpu_execution(self, arg: dict[str, int | bool], /) -> None")), + nb::for_setter(nb::arg("values").none())) + .def_prop_rw( + "fft_planning", + [](const P& self) { return fft_request_dict(self.fft_planning); }, + [](P& self, const nb::handle values) { + if (!nb::isinstance(values)) + throw std::invalid_argument( + "a native problem's fft_planning must be a dict"); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + self.fft_planning = fft_planning_from(nb::borrow(values)); +#else + // A device build plans its transforms with the vendor's FFT alone. + static_cast(self); + static_cast(values); +#endif + }, + nb::for_getter(nb::sig("def fft_planning(self) -> dict[str, object]")), + nb::for_setter( + nb::sig("def fft_planning(self, arg: dict[str, object], /) -> None")), + nb::for_setter(nb::arg("values").none())) + .def("validate", &P::validate); } native::TrialScalars trial_from(const RealInput phi_delta, const RealInput phi_at, @@ -838,8 +823,8 @@ native::stage::Tables stage_tables_from(const nb::dict values) { const int chord = nb::cast(values["chord"]); if (chord < static_cast(native::stage::Chord::none) || chord > static_cast(native::stage::Chord::table)) - throw std::invalid_argument("the stage tables are inconsistent: chord kind " + - std::to_string(chord) + " is unknown"); + native::refuse("the stage tables are inconsistent: chord kind ", chord, + " is unknown"); tables.chord = static_cast(chord); tables.chord_rate = nb::cast(values["chord_rate"]); tables.chord_terms = nb::cast>(values["chord_terms"]); @@ -1623,11 +1608,12 @@ class ScriptedRkMarch { class BoundState { public: - explicit BoundState(const nb::dict& problem) : BoundState(problem_from(problem)) {} explicit BoundState(native::Problem problem) - : columns_(problem.n_columns()), elements_(problem.n_columns() * problem.n_omega), - stages_(problem.n_stages), raman_terms_(problem.n_raman_terms), - raman_(problem.raman) { + : columns_(problem.n_columns()), + elements_(problem.n_columns() * problem.dims.n_omega), + stages_(problem.stepper.n_stages), raman_terms_(problem.raman.n_terms), + raman_(problem.raman.enabled) { + problem.validate(); general_state_ = std::make_shared(std::move(problem)); const std::scoped_lock lock(runtime_state.mutex); runtime_state.states.push_back(this); @@ -1720,12 +1706,11 @@ class BoundState { squarings = static_cast(std::ceil(std::log2(largest / guard.taylor_scale))); if (squarings > guard.max_squarings) - throw native::BackendRefusal( + native::refuse( "native cannot take a coupled flow action whose exponent has row-sum " - "norm " + - py_g(largest, 4) + ": it would need " + std::to_string(squarings) + - " squarings, above the " + std::to_string(guard.max_squarings) + - " this backend admits. Cap dz_max"); + "norm ", + py_g(largest, 4), ": it would need ", squarings, " squarings, above the ", + guard.max_squarings, " this backend admits. Cap dz_max"); } const auto measured = general_state_->form_flow_actions(section, dz, fractions); const std::size_t width = guard.condition_bound.size() / guard.row_sum.size(); @@ -1744,13 +1729,12 @@ class BoundState { } } if (largest > 1.0) - throw native::BackendRefusal( - "native measured a coupled flow conditioning of " + - py_g(measured[worst], 12) + " over a step of " + py_g(dz, 4) + - " m, above the " + py_g(declared[worst], 12) + - " the certificate declares at active bin " + std::to_string(worst) + - " of section " + std::to_string(section) + " -- exceeding it by a factor " + - py_g(largest, 12) + + native::refuse( + "native measured a coupled flow conditioning of ", + py_g(measured[worst], 12), " over a step of ", py_g(dz, 4), + " m, above the ", py_g(declared[worst], 12), + " the certificate declares at active bin ", worst, " of section ", section, + " -- exceeding it by a factor ", py_g(largest, 12), " of the rounding allowance: the inverse action carries the nonlinear " "source back by the measured factor. Lower dz_max, which is what the " "bound was declared over"); @@ -2429,13 +2413,11 @@ class BoundSession { nb::ndarray, nb::c_contig, nb::device::cpu> out) { const std::size_t n = session_->elements(); if (out.size() != n) - throw std::invalid_argument("an export destination has " + - std::to_string(out.size()) + " elements, expected " + - std::to_string(n)); + native::refuse("an export destination has ", out.size(), " elements, expected ", + n); if (weight && weight->size() != n) - throw std::invalid_argument("an export's power weight has " + - std::to_string(weight->size()) + - " elements, expected " + std::to_string(n)); + native::refuse("an export's power weight has ", weight->size(), + " elements, expected ", n); const double* values = weight ? weight->data() : nullptr; auto* destination = reinterpret_cast(out.data()); const nb::gil_scoped_release release; @@ -3146,6 +3128,7 @@ nb::dict build_record() { } // namespace +// NOLINTNEXTLINE(readability-function-size): the module's one table of bindings, kept whole NB_MODULE(_core, module_) { // Each registration makes the Python exception type; the handle need not outlive it. const nb::exception envelope_convergence_error( @@ -3361,6 +3344,7 @@ NB_MODULE(_core, module_) { }, "text"_a); native::bindings::bind_helpers(module_); + bind_problem(module_); native::bindings::bind_transverse(module_); nb::class_(module_, "ScriptedMarch") .def(nb::init(), "policy"_a, @@ -3375,7 +3359,7 @@ NB_MODULE(_core, module_) { .def("log", &ScriptedRkMarch::log) .def("used", &ScriptedRkMarch::used); nb::class_(module_, "State") - .def(nb::init()) + .def(nb::init(), "problem"_a) .def( "set_initial", [](BoundState& self, const ComplexInput field, const ComplexInput noise) { @@ -3564,10 +3548,8 @@ NB_MODULE(_core, module_) { const auto extents = state.adams_row_extents(name); if (rows.ndim() != 3 || rows.shape(0) != extents[0] || rows.shape(1) != extents[1] || rows.shape(2) != extents[2]) - throw std::invalid_argument( - "the Adams history row '" + name + "' must have shape (" + - std::to_string(extents[0]) + ", " + std::to_string(extents[1]) + - ", " + std::to_string(extents[2]) + ")"); + native::refuse("the Adams history row '", name, "' must have shape (", + extents[0], ", ", extents[1], ", ", extents[2], ")"); const auto values = complex_vector(rows); const nb::gil_scoped_release release; state.adams_rows_from_host(name, values); diff --git a/native/src/core.cpp b/native/src/core.cpp index 29ff8696a..4674e1aa9 100644 --- a/native/src/core.cpp +++ b/native/src/core.cpp @@ -11,7 +11,9 @@ #include #include #include +#include #include +#include #include #ifdef KOKKOS_ENABLE_OPENMP @@ -321,11 +323,69 @@ int scoped_openmp_default() noexcept { #endif } +void Problem::validate() const { + const auto require = [](const bool holds, const char* message) { + if (!holds) + throw std::invalid_argument(message); + }; + require(cpu_execution != CpuExecution{}, + "a native problem needs its CPU execution record (cpu_execution, solprop " + "SEP 46)"); + // A bound array's recorded shape is the layout its counts give; an array set in C++ + // records none. + const auto shaped = [](const Shapes& shapes, const char* field, + const std::vector& expected) { + const auto found = shapes.find(field); + return found == shapes.end() || found->second == expected; + }; + for (const auto& [field, count, rows] : + {std::tuple{"phi_terms", linear.n_phi_terms, + "phi_terms must hold n_phi_terms rows along its first axis"}, + std::tuple{"prefactor_terms", linear.n_prefactor_terms, + "prefactor_terms must hold n_prefactor_terms rows along its first " + "axis"}}) + if (const auto found = linear.shapes.find(field); found != linear.shapes.end()) { + const auto& shape = found->second; + require(shape.size() == 2 || shape.size() == 3, + "fixed terms must be rank-two or rank-three arrays"); + require(shape[0] == count, rows); + } + if (raman.enabled) { + require(raman.n_terms != 0 && + dims.n_to <= + std::numeric_limits::max() / (2 * raman.n_terms) && + raman.undamped.size() == raman.n_terms * 2 * dims.n_to && + shaped(raman.shapes, "undamped", {raman.n_terms, 2 * dims.n_to}), + "raman_undamped must have shape (n_terms, 2 * n_to)"); + require(raman.tensor_weights.size() == 2 * raman.n_terms && + shaped(raman.shapes, "tensor_weights", {raman.n_terms, 2}), + "raman_tensor_weights must have shape (n_terms, 2)"); + } else { + require(raman.n_terms == 0, + "raman.n_terms counts the terms of an enabled response"); + } + require(plasma.rate_coefficients.size() == 4 * plasma.rate_intervals && + shaped(plasma.shapes, "rate_coefficients", {4, plasma.rate_intervals}), + "rate_coefficients must have shape (4, n_intervals)"); + if (plasma.envelope) { + constexpr std::size_t rule = std::size_t{4} * 512; + const std::vector layout{plasma.n_channels, 4, 512}; + require(plasma.envelope_rule_nodes.size() == plasma.n_channels * rule && + plasma.envelope_rule_weights.size() == plasma.n_channels * rule && + shaped(plasma.shapes, "envelope_rule_nodes", layout) && + shaped(plasma.shapes, "envelope_rule_weights", layout), + "envelope rules require (n_channels, 4, 512)"); + require(plasma.envelope_drude_factor.size() == dims.n_omega && + plasma.envelope_loss_factor.size() == dims.n_omega, + "envelope current factors require (n_omega,)"); + } +} + bool compact_problem(const Problem& problem) noexcept { - return problem.n_columns() == 1 && problem.n_nodes == 1 && - problem.n_pol_transform == 1 && !problem.general_modal && - !problem.coupled_flow && !problem.separable_geometry() && - problem.n_to < problem.cpu_execution.compact_max_fine_samples; + return problem.n_columns() == 1 && problem.dims.n_nodes == 1 && + problem.dims.n_pol_transform == 1 && !problem.general.enabled && + !problem.flow.enabled && !problem.separable_geometry() && + problem.dims.n_to < problem.cpu_execution.compact_max_fine_samples; } const ExecutionSpace& CpuExecutionPool::get(const CpuOperation operation, diff --git a/native/src/coupled_flow.cpp b/native/src/coupled_flow.cpp index 1ee2c368c..b43bb591d 100644 --- a/native/src/coupled_flow.cpp +++ b/native/src/coupled_flow.cpp @@ -5,6 +5,7 @@ #include "solprop_native/coupled_flow.hpp" #include "solprop_native/launch.hpp" +#include "solprop_native/refuse.hpp" #include #include @@ -27,9 +28,7 @@ std::size_t product(const std::size_t left, const std::size_t right, const char* void require_size(const std::size_t actual, const std::size_t expected, const char* name) { if (actual != expected) - throw std::invalid_argument(std::string("coupled flow ") + name + " has " + - std::to_string(actual) + " values, expected " + - std::to_string(expected)); + refuse("coupled flow ", name, " has ", actual, " values, expected ", expected); } bool finite(const std::vector& values) { @@ -55,8 +54,8 @@ KOKKOS_INLINE_FUNCTION Complex flow_exponential(const Complex value) { std::size_t block_size(const std::size_t count, const Problem& problem, const char* name) { - return product(product(count, problem.flow_active, name), - product(problem.n_modes, problem.n_modes, name), name); + return product(product(count, problem.flow.active, name), + product(problem.dims.n_modes, problem.dims.n_modes, name), name); } using ComplexView3 = CoupledFlowAction::ComplexView3; @@ -263,26 +262,26 @@ namespace { std::size_t fallback_count(const Problem& problem) { return static_cast( - std::count_if(problem.flow_route.begin(), problem.flow_route.end(), + std::count_if(problem.flow.route.begin(), problem.flow.route.end(), [](const std::size_t value) { return value != 0; })); } void require_flow_eigenbasis(const Problem& problem) { - if (problem.flow_left.empty() && problem.flow_right.empty() && - problem.flow_eigenvalues.empty() && problem.flow_route.empty() && - problem.flow_fallback_vectors.empty() && problem.flow_fallback_triangular.empty()) + if (problem.flow.left.empty() && problem.flow.right.empty() && + problem.flow.eigenvalues.empty() && problem.flow.route.empty() && + problem.flow.fallback_vectors.empty() && problem.flow.fallback_triangular.empty()) return; - const auto sections = problem.flow_sections; - require_size(problem.flow_left.size(), block_size(sections, problem, "eigenbasis"), + const auto sections = problem.flow.sections; + require_size(problem.flow.left.size(), block_size(sections, problem, "eigenbasis"), "left eigenbasis"); - require_size(problem.flow_right.size(), block_size(sections, problem, "eigenbasis"), + require_size(problem.flow.right.size(), block_size(sections, problem, "eigenbasis"), "right eigenbasis"); - const auto bins = product(sections, problem.flow_active, "route"); - require_size(problem.flow_eigenvalues.size(), - product(bins, problem.n_modes, "eigenvalues"), "eigenvalues"); - require_size(problem.flow_route.size(), bins, "route"); + const auto bins = product(sections, problem.flow.active, "route"); + require_size(problem.flow.eigenvalues.size(), + product(bins, problem.dims.n_modes, "eigenvalues"), "eigenvalues"); + require_size(problem.flow.route.size(), bins, "route"); std::size_t expected = 1; - for (const auto route : problem.flow_route) { + for (const auto route : problem.flow.route) { if (route == 0) continue; if (route != expected) @@ -291,43 +290,44 @@ void require_flow_eigenbasis(const Problem& problem) { ++expected; } const auto fallback = expected - 1; - const auto blocks = product( - fallback, product(problem.n_modes, problem.n_modes, "fallback"), "fallback"); - require_size(problem.flow_fallback_vectors.size(), blocks, "fallback vectors"); - require_size(problem.flow_fallback_triangular.size(), blocks, "fallback triangular"); - if (!finite(problem.flow_left) || !finite(problem.flow_right) || - !finite(problem.flow_eigenvalues) || !finite(problem.flow_fallback_vectors) || - !finite(problem.flow_fallback_triangular)) + const auto blocks = + product(fallback, product(problem.dims.n_modes, problem.dims.n_modes, "fallback"), + "fallback"); + require_size(problem.flow.fallback_vectors.size(), blocks, "fallback vectors"); + require_size(problem.flow.fallback_triangular.size(), blocks, "fallback triangular"); + if (!finite(problem.flow.left) || !finite(problem.flow.right) || + !finite(problem.flow.eigenvalues) || !finite(problem.flow.fallback_vectors) || + !finite(problem.flow.fallback_triangular)) throw std::invalid_argument("coupled flow eigenbasis data must be finite"); } } // namespace void require_coupled_flow(const Problem& problem) { - if (!problem.coupled_flow) { - if (problem.flow_active != 0 || problem.flow_sections != 0 || - problem.flow_terms != 0 || !problem.flow_slot.empty() || - !problem.flow_frozen.empty() || !problem.flow_residual_terms.empty() || - !problem.flow_left.empty() || !problem.flow_route.empty()) + if (!problem.flow.enabled) { + if (problem.flow.active != 0 || problem.flow.sections != 0 || + problem.flow.terms != 0 || !problem.flow.slot.empty() || + !problem.flow.frozen.empty() || !problem.flow.residual_terms.empty() || + !problem.flow.left.empty() || !problem.flow.route.empty()) throw std::invalid_argument("coupled flow arrays require coupled_flow"); return; } // The action contracts the channel axis of a scalar state; the synthesis may resolve // two transverse components, which the flow never reads. - if (problem.n_pol_state != 1 || problem.separable_geometry()) + if (problem.dims.n_pol_state != 1 || problem.separable_geometry()) throw std::invalid_argument("a coupled flow mixes one channel axis of a scalar " "modal state, with no polarisation axis"); - if (problem.modified_noise) + if (problem.stepper.modified_noise) throw std::invalid_argument("a coupled flow carries no noise model"); - if (problem.n_stages == 0) + if (problem.stepper.n_stages == 0) throw std::invalid_argument("a coupled flow needs a tableau"); - if (problem.flow_active == 0 || problem.flow_active > problem.n_omega) + if (problem.flow.active == 0 || problem.flow.active > problem.dims.n_omega) throw std::invalid_argument("flow_active must lie in [1, n_omega]"); - if (problem.flow_sections == 0) + if (problem.flow.sections == 0) throw std::invalid_argument("a coupled flow has at least one section"); - require_size(problem.flow_slot.size(), problem.n_omega, "slot"); + require_size(problem.flow.slot.size(), problem.dims.n_omega, "slot"); std::size_t expected = 1; - for (const auto slot : problem.flow_slot) { + for (const auto slot : problem.flow.slot) { if (slot == 0) continue; if (slot != expected) @@ -335,33 +335,34 @@ void require_coupled_flow(const Problem& problem) { "in increasing frequency"); ++expected; } - if (expected - 1 != problem.flow_active) + if (expected - 1 != problem.flow.active) throw std::invalid_argument("flow_slot names a different number of active bins " "than flow_active"); - static_cast(block_size(problem.n_stages + 1, problem, "actions")); + static_cast(block_size(problem.stepper.n_stages + 1, problem, "actions")); require_flow_eigenbasis(problem); - if (problem.flow_terms == 0) { - if (!problem.flow_frozen.empty() || !problem.flow_residual_terms.empty()) + if (problem.flow.terms == 0) { + if (!problem.flow.frozen.empty() || !problem.flow.residual_terms.empty()) throw std::invalid_argument("a coupled flow with no explicit residual carries " "no frozen blocks or residual terms"); return; } - require_size(problem.flow_frozen.size(), - block_size(problem.flow_sections, problem, "frozen"), "frozen"); - require_size(problem.flow_residual_terms.size(), - block_size(problem.flow_terms, problem, "residual terms"), + require_size(problem.flow.frozen.size(), + block_size(problem.flow.sections, problem, "frozen"), "frozen"); + require_size(problem.flow.residual_terms.size(), + block_size(problem.flow.terms, problem, "residual terms"), "residual terms"); - if (!finite(problem.flow_frozen) || !finite(problem.flow_residual_terms)) + if (!finite(problem.flow.frozen) || !finite(problem.flow.residual_terms)) throw std::invalid_argument("coupled flow residual data must be finite"); } CoupledFlowAction::CoupledFlowAction(const Problem& problem, ExecutionSpace kernels) - : n_modes_(problem.n_modes), n_omega_(problem.n_omega), n_stages_(problem.n_stages), - active_(problem.flow_active), sections_(problem.flow_sections), - terms_(problem.flow_terms), identity_(problem.n_stages + 1, 0U), - listed_counts_(problem.n_stages, 0U), kernels_(std::move(kernels)) { + : n_modes_(problem.dims.n_modes), n_omega_(problem.dims.n_omega), + n_stages_(problem.stepper.n_stages), active_(problem.flow.active), + sections_(problem.flow.sections), terms_(problem.flow.terms), + identity_(problem.stepper.n_stages + 1, 0U), + listed_counts_(problem.stepper.n_stages, 0U), kernels_(std::move(kernels)) { require_coupled_flow(problem); - if (!problem.coupled_flow) + if (!problem.flow.enabled) throw std::invalid_argument("the problem does not declare a coupled flow"); const auto slots = n_stages_ + 1; slot_ = IndexView("solprop_native.flow.slot", n_omega_); @@ -387,11 +388,11 @@ CoupledFlowAction::CoupledFlowAction(const Problem& problem, ExecutionSpace kern host_active_terms_ = Kokkos::create_mirror_view(active_terms_); host_active_counts_ = Kokkos::create_mirror_view(active_counts_); lab_ = ComplexView2("solprop_native.flow.lab", n_modes_, n_omega_); - upload(kernels_, frozen_, problem.flow_frozen); - upload(kernels_, terms_view_, problem.flow_residual_terms); + upload(kernels_, frozen_, problem.flow.frozen); + upload(kernels_, terms_view_, problem.flow.residual_terms); } - upload(kernels_, slot_, problem.flow_slot); - forms_ = !problem.flow_left.empty(); + upload(kernels_, slot_, problem.flow.slot); + forms_ = !problem.flow.left.empty(); if (forms_) { fallback_ = fallback_count(problem); left_ = ComplexView4("solprop_native.flow.left", sections_, active_, n_modes_, @@ -414,16 +415,16 @@ CoupledFlowAction::CoupledFlowAction(const Problem& problem, ExecutionSpace kern host_fractions_ = Kokkos::create_mirror_view(fractions_); host_measured_ = Kokkos::create_mirror_view(measured_); measured_host_.assign(active_, 0.0); - upload(kernels_, left_, problem.flow_left); - upload(kernels_, right_, problem.flow_right); - upload(kernels_, eigenvalues_, problem.flow_eigenvalues); - upload(kernels_, route_, problem.flow_route); + upload(kernels_, left_, problem.flow.left); + upload(kernels_, right_, problem.flow.right); + upload(kernels_, eigenvalues_, problem.flow.eigenvalues); + upload(kernels_, route_, problem.flow.route); section_fallback_.assign(sections_, 0U); - for (std::size_t index = 0; index < problem.flow_route.size(); ++index) - if (problem.flow_route[index] != 0) + for (std::size_t index = 0; index < problem.flow.route.size(); ++index) + if (problem.flow.route[index] != 0) section_fallback_[index / active_] = 1U; - upload(kernels_, fallback_vectors_, problem.flow_fallback_vectors); - upload(kernels_, fallback_triangular_, problem.flow_fallback_triangular); + upload(kernels_, fallback_vectors_, problem.flow.fallback_vectors); + upload(kernels_, fallback_triangular_, problem.flow.fallback_triangular); } kernels_.fence("solprop_native coupled flow upload"); } @@ -431,8 +432,6 @@ CoupledFlowAction::CoupledFlowAction(const Problem& problem, ExecutionSpace kern const std::vector& CoupledFlowAction::form_actions(const std::size_t section, const double dz, const std::vector& fractions) { - if (closed_) - throw std::logic_error("the coupled flow is closed"); if (!forms_) throw std::logic_error("this coupled flow carries no eigenbasis to form its " "actions from"); @@ -522,13 +521,11 @@ bool CoupledFlowAction::falls_back(const std::size_t section) const { return section < section_fallback_.size() && section_fallback_[section] != 0U; } -CoupledFlowAction::~CoupledFlowAction() { close(); } +CoupledFlowAction::~CoupledFlowAction() { kernels_.fence(); } void CoupledFlowAction::set_actions(const std::vector& forward, const std::vector& inverse, const std::vector& identity) { - if (closed_) - throw std::logic_error("the coupled flow is closed"); require_size(forward.size(), forward_.size(), "forward actions"); require_size(inverse.size(), inverse_.size(), "inverse actions"); require_size(identity.size(), n_stages_ + 1, "identity flags"); @@ -545,8 +542,6 @@ void CoupledFlowAction::set_actions(const std::vector& forward, void CoupledFlowAction::set_trial(const std::size_t section, const std::vector& weights) { - if (closed_) - throw std::logic_error("the coupled flow is closed"); if (!ready_) throw std::logic_error("no coupled flow actions have been loaded"); if (section >= sections_) @@ -578,8 +573,6 @@ void CoupledFlowAction::set_trial(const std::size_t section, } std::vector CoupledFlowAction::actions_to_host(const bool inverse) const { - if (closed_) - throw std::logic_error("the coupled flow is closed"); if (!ready_) throw std::logic_error("no coupled flow actions have been loaded"); kernels_.fence("solprop_native coupled flow observation"); @@ -793,42 +786,4 @@ std::map CoupledFlowAction::memory_ledger() const { }; } -void CoupledFlowAction::close() noexcept { - if (closed_) - return; - kernels_.fence(); - slot_ = {}; - forward_ = {}; - inverse_ = {}; - frozen_ = {}; - terms_view_ = {}; - weights_ = {}; - active_terms_ = {}; - active_counts_ = {}; - host_active_terms_ = {}; - host_active_counts_ = {}; - input_ = {}; - lab_ = {}; - drive_ = {}; - state_ = {}; - solution_ = {}; - error_ = {}; - left_ = {}; - right_ = {}; - eigenvalues_ = {}; - route_ = {}; - fallback_vectors_ = {}; - fallback_triangular_ = {}; - fallback_work_ = {}; - exponents_ = {}; - gram_ = {}; - fractions_ = {}; - measured_ = {}; - host_fractions_ = {}; - host_measured_ = {}; - ready_ = false; - forms_ = false; - closed_ = true; -} - } // namespace solprop_native diff --git a/native/src/crossing.cpp b/native/src/crossing.cpp index c4fc436f4..c5ab49c6b 100644 --- a/native/src/crossing.cpp +++ b/native/src/crossing.cpp @@ -4,6 +4,8 @@ #include "solprop_native/crossing.hpp" +#include "solprop_native/build.hpp" + #include #include #include @@ -21,12 +23,6 @@ namespace solprop_native::crossing { namespace { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -constexpr bool device_build = true; -#else -constexpr bool device_build = false; -#endif - // Kokkos holds its copies to one thread at a time per instance, so the workers call the // vendor's synchronous copy, which is safe from many threads: it is what was measured. void device_read(void* destination, const void* source, const std::size_t bytes) { @@ -91,7 +87,7 @@ std::size_t worker_count() { } // namespace bool takes_ring(const std::size_t bytes) noexcept { - return device_build && bytes >= ring_threshold; + return build::device && bytes >= ring_threshold; } void BufferDeleter::operator()(double* pointer) const noexcept { std::free(pointer); } diff --git a/native/src/dispersive.cpp b/native/src/dispersive.cpp index dfdc2f8c2..b3281544f 100644 --- a/native/src/dispersive.cpp +++ b/native/src/dispersive.cpp @@ -574,50 +574,50 @@ template struct GeneralPairs { } // namespace void require_dispersive(const Problem& problem) { - if (!problem.dispersive) { - if (!problem.dispersive_inputs.empty() || - !problem.dispersive_coefficients.empty() || - !problem.dispersive_term_rows.empty() || - !problem.dispersive_group_offsets.empty() || - !problem.dispersive_group_keys.empty() || - !problem.dispersive_node_region.empty()) + if (!problem.dispersive.enabled) { + if (!problem.dispersive.inputs.empty() || + !problem.dispersive.coefficients.empty() || + !problem.dispersive.term_rows.empty() || + !problem.dispersive.group_offsets.empty() || + !problem.dispersive.group_keys.empty() || + !problem.dispersive.node_region.empty()) throw std::invalid_argument( "dispersive data were supplied while the dispersive response is disabled"); return; } - if (problem.dispersive_form < 0 || problem.dispersive_form > 2) + if (problem.dispersive.form < 0 || problem.dispersive.form > 2) throw std::invalid_argument("dispersive_form must be 0, 1 or 2"); - const auto form = static_cast(problem.dispersive_form); - const bool general = problem.general_modal; + const auto form = static_cast(problem.dispersive.form); + const bool general = problem.general.enabled; // The general route synthesises its own analytic signal on the real field family; // elsewhere the family follows the sector. - const bool family_fits = general - ? problem.family == FieldFamily::field_real && - form != DispersiveForm::harmonic - : (problem.family == FieldFamily::field_real && - form == DispersiveForm::real) || - (problem.family == FieldFamily::field_analytic && - form == DispersiveForm::conserving) || - (problem.family == FieldFamily::envelope && - form != DispersiveForm::real); + const bool family_fits = + general ? problem.dims.family == FieldFamily::field_real && + form != DispersiveForm::harmonic + : (problem.dims.family == FieldFamily::field_real && + form == DispersiveForm::real) || + (problem.dims.family == FieldFamily::field_analytic && + form == DispersiveForm::conserving) || + (problem.dims.family == FieldFamily::envelope && + form != DispersiveForm::real); if (!family_fits) throw std::invalid_argument( "the dispersive form does not fit the field family: a field grid's full cubic " "takes the real family, its number-conserving sector the analytic one (the " "general modal route's both the real one), and an envelope's both sectors the " "envelope family"); - const std::size_t bins = problem.family == FieldFamily::envelope && !general - ? problem.n_to - : problem.n_to / 2 + 1; - const std::size_t channels = problem.dispersive_coefficients.size(); + const std::size_t bins = problem.dims.family == FieldFamily::envelope && !general + ? problem.dims.n_to + : problem.dims.n_to / 2 + 1; + const std::size_t channels = problem.dispersive.coefficients.size(); if (channels == 0) throw std::invalid_argument("the dispersive response needs a channel"); - require_size(problem.dispersive_input_scale.size(), bins, "input_scale"); - require_size(problem.dispersive_output_scale.size(), bins, "output_scale"); - require_size(problem.dispersive_pair_constants.size(), + require_size(problem.dispersive.input_scale.size(), bins, "input_scale"); + require_size(problem.dispersive.output_scale.size(), bins, "output_scale"); + require_size(problem.dispersive.pair_constants.size(), product_of(2, channels, "pair constants"), "pair_constants"); - const auto& offsets = problem.dispersive_group_offsets; - const auto& members = problem.dispersive_group_members; + const auto& offsets = problem.dispersive.group_offsets; + const auto& members = problem.dispersive.group_members; if (offsets.size() < 2 || offsets.front() != 0 || offsets.back() != members.size() || members.size() != channels) throw std::invalid_argument( @@ -632,18 +632,18 @@ void require_dispersive(const Problem& problem) { for (std::size_t group = 0; group + 1 < offsets.size(); ++group) { if (offsets[group + 1] <= offsets[group]) throw std::invalid_argument("every dispersive group needs a channel"); - if (problem.dispersive_symmetric && offsets[group + 1] - offsets[group] > 1) + if (problem.dispersive.symmetric && offsets[group + 1] - offsets[group] > 1) ++mixtures; } const std::size_t groups = offsets.size() - 1; // Keys and node regions belong to the general route, where a channel acts at its own // region's nodes; elsewhere every channel acts at every node. - const auto& keys = problem.dispersive_group_keys; + const auto& keys = problem.dispersive.group_keys; if (general) { require_size(keys.size(), groups, "group_keys"); - require_size(problem.dispersive_node_region.size(), problem.general_nodes, + require_size(problem.dispersive.node_region.size(), problem.general.nodes, "node_region"); - } else if (!keys.empty() || !problem.dispersive_node_region.empty()) { + } else if (!keys.empty() || !problem.dispersive.node_region.empty()) { throw std::invalid_argument( "dispersive group keys and node regions belong to the general modal route"); } @@ -653,35 +653,36 @@ void require_dispersive(const Problem& problem) { if (std::ranges::find(distinct, key) == distinct.end()) distinct.push_back(key); } - const std::size_t fields = problem.dispersive_symmetric ? groups : 2 * groups; + const std::size_t fields = problem.dispersive.symmetric ? groups : 2 * groups; const std::size_t products = - problem.dispersive_symmetric ? groups : distinct.size() + 2 * groups; - require_size(problem.dispersive_inputs.size(), product_of(fields, bins, "inputs"), + problem.dispersive.symmetric ? groups : distinct.size() + 2 * groups; + require_size(problem.dispersive.inputs.size(), product_of(fields, bins, "inputs"), "inputs"); - const std::size_t terms = problem.dispersive_term_rows.size(); + const std::size_t terms = problem.dispersive.term_rows.size(); if (terms == 0) throw std::invalid_argument("the dispersive response needs an output term"); - require_size(problem.dispersive_term_channels.size(), terms, "term_channels"); - require_size(problem.dispersive_term_multipliers.size(), + require_size(problem.dispersive.term_channels.size(), terms, "term_channels"); + require_size(problem.dispersive.term_multipliers.size(), product_of(terms, bins, "term multipliers"), "term_multipliers"); for (std::size_t term = 0; term < terms; ++term) - if (problem.dispersive_term_rows[term] >= products || - problem.dispersive_term_channels[term] > channels + mixtures) + if (problem.dispersive.term_rows[term] >= products || + problem.dispersive.term_channels[term] > channels + mixtures) throw std::invalid_argument("a dispersive term names no product or factor"); - require_size(problem.dispersive_quartics.size(), + require_size(problem.dispersive.quartics.size(), mixtures == 0 ? 0 : product_of(channels, bins, "quartics"), "quartics"); // The products and their spectra, and the node slots: the general route's chunk, or // at most the modal columns. - const auto rows = general ? product_of(problem.general_node_chunk, - problem.general_components, "dispersive rows") + const auto rows = general ? product_of(problem.general.node_chunk, + problem.general.components, "dispersive rows") : problem.n_columns(); const auto field = product_of(product_of(products + fields, rows, "dispersive products"), - problem.n_to, "dispersive products"); + problem.dims.n_to, "dispersive products"); static_cast(product_of(field, sizeof(Complex), "dispersive product bytes")); static_cast(product_of( - product_of(2 * problem.node_chunk, problem.effective_node_pol(), "slots"), - product_of(problem.n_to, sizeof(Complex), "slot bytes"), "dispersive slots")); + product_of(2 * problem.dims.node_chunk, problem.effective_node_pol(), "slots"), + product_of(problem.dims.n_to, sizeof(Complex), "slot bytes"), + "dispersive slots")); } void dispersive_mixture_rows( @@ -720,19 +721,19 @@ void dispersive_mixture_rows( DispersiveWeights::DispersiveWeights(const Problem& problem, ExecutionSpace kernels) : kernels_(std::move(kernels)) { require_dispersive(problem); - if (!problem.dispersive) + if (!problem.dispersive.enabled) throw std::invalid_argument("the problem does not carry the dispersive response"); - channels_ = problem.dispersive_coefficients.size(); - bins_ = problem.dispersive_input_scale.size(); - n_stages_ = problem.n_stages; - symmetric_ = problem.dispersive_symmetric; - coefficients_ = problem.dispersive_coefficients; - pair_constants_ = problem.dispersive_pair_constants; - group_offsets_ = problem.dispersive_group_offsets; - group_members_ = problem.dispersive_group_members; + channels_ = problem.dispersive.coefficients.size(); + bins_ = problem.dispersive.input_scale.size(); + n_stages_ = problem.stepper.n_stages; + symmetric_ = problem.dispersive.symmetric; + coefficients_ = problem.dispersive.coefficients; + pair_constants_ = problem.dispersive.pair_constants; + group_offsets_ = problem.dispersive.group_offsets; + group_members_ = problem.dispersive.group_members; groups_ = group_offsets_.size() - 1; - keys_ = problem.dispersive_group_keys.empty() ? std::vector(groups_, 0) - : problem.dispersive_group_keys; + keys_ = problem.dispersive.group_keys.empty() ? std::vector(groups_, 0) + : problem.dispersive.group_keys; if (!symmetric_) { // One cube per distinct key, in the order the groups first name it (solprop // `dispersive_terms_layout`). @@ -753,7 +754,7 @@ DispersiveWeights::DispersiveWeights(const Problem& problem, ExecutionSpace kern mixtures_ = mixture_groups_.size(); static_inputs_ = RealView2("solprop_native.dispersive.static_inputs", fields_, bins_); inputs_ = RealView2("solprop_native.dispersive.inputs", fields_, bins_); - const auto terms = problem.dispersive_term_rows.size(); + const auto terms = problem.dispersive.term_rows.size(); term_multipliers_ = RealView2("solprop_native.dispersive.term_multipliers", terms, bins_); outputs_ = RealView2("solprop_native.dispersive.outputs", terms, bins_); @@ -768,17 +769,17 @@ DispersiveWeights::DispersiveWeights(const Problem& problem, ExecutionSpace kern term_rows_ = IndexView("solprop_native.dispersive.term_rows", terms); term_channels_ = IndexView("solprop_native.dispersive.term_channels", terms); members_ = IndexView("solprop_native.dispersive.members", channels_); - upload(kernels_, static_inputs_, problem.dispersive_inputs); + upload(kernels_, static_inputs_, problem.dispersive.inputs); graph::deep_copy(kernels_, inputs_, static_inputs_); - upload(kernels_, term_multipliers_, problem.dispersive_term_multipliers); + upload(kernels_, term_multipliers_, problem.dispersive.term_multipliers); graph::deep_copy(kernels_, outputs_, term_multipliers_); if (mixtures_ != 0) - upload(kernels_, quartics_, problem.dispersive_quartics); - upload(kernels_, input_scale_, problem.dispersive_input_scale); - upload(kernels_, output_scale_, problem.dispersive_output_scale); - upload(kernels_, term_rows_, problem.dispersive_term_rows); - upload(kernels_, term_channels_, problem.dispersive_term_channels); - upload(kernels_, members_, problem.dispersive_group_members); + upload(kernels_, quartics_, problem.dispersive.quartics); + upload(kernels_, input_scale_, problem.dispersive.input_scale); + upload(kernels_, output_scale_, problem.dispersive.output_scale); + upload(kernels_, term_rows_, problem.dispersive.term_rows); + upload(kernels_, term_channels_, problem.dispersive.term_channels); + upload(kernels_, members_, problem.dispersive.group_members); pairs_.assign(2 * groups_, 0.0); densities_.assign(n_stages_ * channels_, 0.0); last_.assign(channels_, 0.0); @@ -888,37 +889,22 @@ std::size_t DispersiveWeights::bytes() const { (term_rows_.size() + term_channels_.size() + members_.size()) * index + host; } -void DispersiveWeights::close() noexcept { - static_inputs_ = {}; - inputs_ = {}; - term_multipliers_ = {}; - outputs_ = {}; - weights_ = {}; - quartics_ = {}; - input_scale_ = {}; - output_scale_ = {}; - factors_ = {}; - for (auto& staging : host_factors_) - staging = {}; - term_rows_ = {}; - term_channels_ = {}; - members_ = {}; -} - // --- the fused node path ------------------------------------------------------------- DispersiveResponse::DispersiveResponse(const Problem& problem, CpuExecutionPool& pool, StageViews views) - : n_modes_(problem.n_modes), n_pol_state_(problem.n_pol_state), - n_pol_transform_(problem.n_pol_transform), + : n_modes_(problem.dims.n_modes), n_pol_state_(problem.dims.n_pol_state), + n_pol_transform_(problem.dims.n_pol_transform), components_(problem.effective_node_pol()), columns_(problem.n_columns()), - n_nodes_(problem.n_nodes), node_chunk_(problem.node_chunk), - n_chunks_((problem.n_nodes + problem.node_chunk - 1) / problem.node_chunk), - n_to_(problem.n_to), form_(static_cast(problem.dispersive_form)), + n_nodes_(problem.dims.n_nodes), node_chunk_(problem.dims.node_chunk), + n_chunks_((problem.dims.n_nodes + problem.dims.node_chunk - 1) / + problem.dims.node_chunk), + n_to_(problem.dims.n_to), + form_(static_cast(problem.dispersive.form)), stage_(std::move(views)), weights_(problem, - pool.get(CpuOperation::stage, problem.dispersive_inputs.size())) { - if (problem.general_modal || problem.separable_geometry()) + pool.get(CpuOperation::stage, problem.dispersive.inputs.size())) { + if (problem.general.enabled || problem.separable_geometry()) throw std::invalid_argument( "the fused dispersive response runs on the legacy transform's node path"); const bool complex = form_ != DispersiveForm::real; @@ -927,8 +913,8 @@ DispersiveResponse::DispersiveResponse(const Problem& problem, CpuExecutionPool& const auto fields = weights_.fields(); const auto products = weights_.products(); kernels_ = pool.get(CpuOperation::stage, products * columns_ * n_to_); - response_chunk_ = problem.node_chunk; - const auto tail = problem.n_nodes % response_chunk_; + response_chunk_ = problem.dims.node_chunk; + const auto tail = problem.dims.n_nodes % response_chunk_; response_ = pool.get(CpuOperation::response, response_chunk_ * components_ * n_to_, n_to_); tail_response_ = pool.get(CpuOperation::response, tail * components_ * n_to_, n_to_); @@ -939,14 +925,14 @@ DispersiveResponse::DispersiveResponse(const Problem& problem, CpuExecutionPool& matrix_copy_ = pool.get(CpuOperation::copy, n_modes_ * n_nodes_ * n_pol_transform_); dense_ = pool.get(CpuOperation::dense, dense_elements); tail_dense_ = pool.get(CpuOperation::dense, tail_dense_elements); - analytic_ = problem.family == FieldFamily::field_analytic; - embedded_ = !problem.projection_coarse_matrix.empty(); + analytic_ = problem.dims.family == FieldFamily::field_analytic; + embedded_ = !problem.transform.projection_coarse_matrix.empty(); // Alone, nothing else is projected or transformed beside the response; a lone one is // also a single product, which reads the stage's own node buffer. - alone_ = !problem.nonlinear && !problem.plasma && !problem.raman && - !problem.envelope_plasma; + alone_ = !problem.kerr.nonlinear && !problem.plasma.enabled && + !problem.raman.enabled && !problem.plasma.envelope; lone_ = alone_ && symmetric && products == 1; - in_place_ = lone_ && !problem.in_stage_absorber; + in_place_ = lone_ && !problem.absorber.in_stage; borrowed_ = lone_ && !analytic_; shifted_ = analytic_ && !symmetric; @@ -1010,7 +996,7 @@ DispersiveResponse::DispersiveResponse(const Problem& problem, CpuExecutionPool& kernels_.fence("solprop_native dispersive construction"); } -DispersiveResponse::~DispersiveResponse() { close(); } +DispersiveResponse::~DispersiveResponse() { kernels_.fence(); } void DispersiveResponse::plan(CpuExecutionPool& transforms) { const auto space_for = [&](const auto& rows) -> const ExecutionSpace& { @@ -1272,8 +1258,6 @@ std::map DispersiveResponse::empty_ledger() { } std::map DispersiveResponse::memory_ledger() const { - if (closed_) - return empty_ledger(); const auto real = sizeof(double); const auto complex = sizeof(Complex); return { @@ -1293,53 +1277,24 @@ std::map DispersiveResponse::memory_ledger() const { }; } -void DispersiveResponse::close() noexcept { - if (closed_) - return; - closed_ = true; - kernels_.fence(); - inverse_real_.reset(); - inverse_complex_.reset(); - forward_real_.reset(); - forward_complex_.reset(); - coarse_real_plan_.reset(); - coarse_complex_plan_.reset(); - stage_ = {}; - weights_.close(); - filtered_spectrum_ = {}; - spectra_ = {}; - filtered_complex_ = {}; - slots_complex_ = {}; - products_complex_ = {}; - coarse_complex_ = {}; - filtered_real_ = {}; - slots_real_ = {}; - products_real_ = {}; - coarse_real_ = {}; - packed_projection_ = {}; - packed_coarse_ = {}; - dc_ = {}; - offset_ = {}; -} - // --- the Cartesian free-space, line and planar routes // --------------------------------- SeparableDispersive::SeparableDispersive(const Problem& problem, CpuExecutionPool& pool, StageViews views) - : n_to_(problem.n_to), n_x_(problem.n_x), n_p_(problem.n_p), - form_(static_cast(problem.dispersive_form)), + : n_to_(problem.dims.n_to), n_x_(problem.spatial.n_x), n_p_(problem.spatial.n_p), + form_(static_cast(problem.dispersive.form)), slab_(problem.slab_geometry()), stage_(std::move(views)), weights_(problem, - pool.get(CpuOperation::stage, problem.dispersive_inputs.size())) { + pool.get(CpuOperation::stage, problem.dispersive.inputs.size())) { if (!problem.separable_geometry()) throw std::invalid_argument("the separable dispersive response runs on the " "Cartesian free-space, line and planar routes"); // A chunk's temporal rows: a free-space chunk's nodes times their components, a // planar block's (p, y) rows; and the rows of its node buffer. components_ = problem.effective_node_pol(); - node_rows_ = problem.node_chunk * components_; - rows_ = slab_ ? (problem.node_chunk / n_x_) * n_p_ : node_rows_; + node_rows_ = problem.dims.node_chunk * components_; + rows_ = slab_ ? (problem.dims.node_chunk / n_x_) * n_p_ : node_rows_; require_size(stage_.fine_spectrum.extent(0), rows_, "chunk rows"); const bool complex = form_ != DispersiveForm::real; spectrum_bins_ = complex ? n_to_ : n_to_ / 2 + 1; @@ -1347,8 +1302,8 @@ SeparableDispersive::SeparableDispersive(const Problem& problem, CpuExecutionPoo const auto fields = weights_.fields(); const auto products = weights_.products(); kernels_ = pool.get(CpuOperation::stage, products * rows_ * n_to_); - response_chunk_ = problem.node_chunk; - const auto tail = problem.n_nodes % response_chunk_; + response_chunk_ = problem.dims.node_chunk; + const auto tail = problem.dims.n_nodes % response_chunk_; active_rows_ = rows_; if constexpr (cpu_size_policy_enabled) tail_rows_ = slab_ ? (tail / n_x_) * n_p_ : tail * components_; @@ -1358,9 +1313,9 @@ SeparableDispersive::SeparableDispersive(const Problem& problem, CpuExecutionPoo tail_response_ = pool.get(CpuOperation::response, tail * components_ * n_to_, n_to_); dense_ = pool.get(CpuOperation::dense, response_chunk_ * components_ * n_to_ * n_p_); tail_dense_ = pool.get(CpuOperation::dense, tail * components_ * n_to_ * n_p_); - analytic_ = problem.family == FieldFamily::field_analytic; - alone_ = !problem.nonlinear && !problem.plasma && !problem.raman && - !problem.envelope_plasma; + analytic_ = problem.dims.family == FieldFamily::field_analytic; + alone_ = !problem.kerr.nonlinear && !problem.plasma.enabled && + !problem.raman.enabled && !problem.plasma.envelope; lone_ = alone_ && symmetric && products == 1; // The in-stage absorber is a pass of its own here, which packs the chunk afresh. in_place_ = lone_; @@ -1417,7 +1372,7 @@ SeparableDispersive::SeparableDispersive(const Problem& problem, CpuExecutionPoo kernels_.fence("solprop_native separable dispersive construction"); } -SeparableDispersive::~SeparableDispersive() { close(); } +SeparableDispersive::~SeparableDispersive() { kernels_.fence(); } void SeparableDispersive::plan(CpuExecutionPool& transforms) { const auto space_for = [&](const auto& rows) -> const ExecutionSpace& { @@ -1657,8 +1612,6 @@ void SeparableDispersive::combine() { } std::map SeparableDispersive::memory_ledger() const { - if (closed_) - return DispersiveResponse::empty_ledger(); const auto real = sizeof(double); const auto complex = sizeof(Complex); // Free space's filtered fields are blocks of its products; the planar route's its @@ -1677,45 +1630,18 @@ std::map SeparableDispersive::memory_ledger() const { return ledger; } -void SeparableDispersive::close() noexcept { - if (closed_) - return; - closed_ = true; - kernels_.fence(); - tail_inverse_real_.clear(); - tail_inverse_complex_.clear(); - tail_forward_real_.clear(); - tail_forward_complex_.clear(); - inverse_real_.reset(); - inverse_complex_.reset(); - forward_real_.reset(); - forward_complex_.reset(); - stage_ = {}; - weights_.close(); - filtered_spectrum_ = {}; - spectra_ = {}; - filtered_complex_ = {}; - slots_complex_ = {}; - products_complex_ = {}; - filtered_real_ = {}; - slots_real_ = {}; - products_real_ = {}; - dc_ = {}; - offset_ = {}; -} - // --- the general modal route -------------------------------------------------------- GeneralDispersive::GeneralDispersive(const Problem& problem, CpuExecutionPool& pool, StageViews views) - : rows_(problem.general_node_chunk * problem.general_components), - components_(problem.general_components), n_to_(problem.n_to), - real_(static_cast(problem.dispersive_form) == + : rows_(problem.general.node_chunk * problem.general.components), + components_(problem.general.components), n_to_(problem.dims.n_to), + real_(static_cast(problem.dispersive.form) == DispersiveForm::real), stage_(std::move(views)), weights_(problem, - pool.get(CpuOperation::stage, problem.dispersive_inputs.size())) { - if (!problem.general_modal) + pool.get(CpuOperation::stage, problem.dispersive.inputs.size())) { + if (!problem.general.enabled) throw std::invalid_argument( "the general dispersive response runs on the general modal route"); require_size(stage_.spectrum.extent(0), rows_, "chunk rows"); @@ -1724,10 +1650,10 @@ GeneralDispersive::GeneralDispersive(const Problem& problem, CpuExecutionPool& p const auto fields = weights_.fields() * rows_; const auto products = weights_.products() * rows_; kernels_ = pool.get(CpuOperation::stage, products * n_to_); - response_chunk_ = problem.general_node_chunk; - const auto tail = problem.general_nodes % response_chunk_; + response_chunk_ = problem.general.node_chunk; + const auto tail = problem.general.nodes % response_chunk_; active_rows_ = rows_; - last_chunk_ = problem.general_nodes / response_chunk_; + last_chunk_ = problem.general.nodes / response_chunk_; if constexpr (cpu_size_policy_enabled) tail_rows_ = tail * components_; tail_kernels_ = @@ -1748,13 +1674,13 @@ GeneralDispersive::GeneralDispersive(const Problem& problem, CpuExecutionPool& p if (shifted_) dc_ = ComplexView("solprop_native.dispersive.zero_frequency", rows_); node_region_ = - IndexView("solprop_native.dispersive.node_region", problem.general_nodes); - upload(kernels_, node_region_, problem.dispersive_node_region); + IndexView("solprop_native.dispersive.node_region", problem.general.nodes); + upload(kernels_, node_region_, problem.dispersive.node_region); // Which chunks have a node of any channel's region, decided once. - const auto& region = problem.dispersive_node_region; - const auto chunk = problem.general_node_chunk; - for (std::size_t first = 0; first < problem.general_nodes; first += chunk) { - const auto last = std::min(first + chunk, problem.general_nodes); + const auto& region = problem.dispersive.node_region; + const auto chunk = problem.general.node_chunk; + for (std::size_t first = 0; first < problem.general.nodes; first += chunk) { + const auto last = std::min(first + chunk, problem.general.nodes); bool any = false; for (std::size_t node = first; node < last && !any; ++node) for (std::size_t group = 0; group < weights_.groups() && !any; ++group) @@ -1764,7 +1690,7 @@ GeneralDispersive::GeneralDispersive(const Problem& problem, CpuExecutionPool& p kernels_.fence("solprop_native general dispersive construction"); } -GeneralDispersive::~GeneralDispersive() { close(); } +GeneralDispersive::~GeneralDispersive() { kernels_.fence(); } void GeneralDispersive::plan(CpuExecutionPool& transforms) { constexpr int axis = -1; @@ -1901,8 +1827,6 @@ void GeneralDispersive::combine(const bool overwrite) { } std::map GeneralDispersive::memory_ledger() const { - if (closed_) - return DispersiveResponse::empty_ledger(); const auto real = sizeof(double); const auto complex = sizeof(Complex); auto ledger = DispersiveResponse::empty_ledger(); @@ -1918,26 +1842,4 @@ std::map GeneralDispersive::memory_ledger() const { return ledger; } -void GeneralDispersive::close() noexcept { - if (closed_) - return; - closed_ = true; - kernels_.fence(); - tail_inverse_.clear(); - tail_forward_real_.clear(); - tail_forward_complex_.clear(); - inverse_.reset(); - forward_real_.reset(); - forward_complex_.reset(); - stage_ = {}; - weights_.close(); - node_region_ = {}; - filtered_spectrum_ = {}; - filtered_ = {}; - products_complex_ = {}; - products_real_ = {}; - spectra_ = {}; - dc_ = {}; -} - } // namespace solprop_native diff --git a/native/src/far_field.cpp b/native/src/far_field.cpp index 693ba2626..ed880ef2c 100644 --- a/native/src/far_field.cpp +++ b/native/src/far_field.cpp @@ -74,17 +74,18 @@ std::size_t far_field_band_count(const std::size_t n_x, const std::size_t n_y, } FarFieldPass::FarFieldPass(const Problem& problem, ExecutionSpace execution) - : execution_(std::move(execution)), n_x_(problem.n_x), n_y_(problem.n_y), - n_omega_(problem.n_omega), n_pol_(problem.n_pol_state), n_t_(problem.n_t), - dt_(problem.dt), envelope_(problem.family == FieldFamily::envelope) { - if (!problem.free_geometry() || problem.spatial_k_x.size() != n_x_ || - problem.spatial_k_y.size() != n_y_) + : execution_(std::move(execution)), n_x_(problem.spatial.n_x), + n_y_(problem.spatial.n_y), n_omega_(problem.dims.n_omega), + n_pol_(problem.dims.n_pol_state), n_t_(problem.dims.n_t), dt_(problem.grid.dt), + envelope_(problem.dims.family == FieldFamily::envelope) { + if (!problem.free_geometry() || problem.spatial.k_x.size() != n_x_ || + problem.spatial.k_y.size() != n_y_) throw std::invalid_argument( "the far-field pass needs a Cartesian or line grid and its column axes"); - for (const double value : problem.spatial_k_x) + for (const double value : problem.spatial.k_x) if (!std::isfinite(value)) throw std::invalid_argument("spatial_k_x must be finite"); - for (const double value : problem.spatial_k_y) + for (const double value : problem.spatial.k_y) if (!std::isfinite(value)) throw std::invalid_argument("spatial_k_y must be finite"); k_x_ = RealView("solprop_native.far_field.k_x", n_x_); @@ -92,13 +93,13 @@ FarFieldPass::FarFieldPass(const Problem& problem, ExecutionSpace execution) const auto host_x = Kokkos::create_mirror_view(k_x_); const auto host_y = Kokkos::create_mirror_view(k_y_); for (std::size_t index = 0; index < n_x_; ++index) - host_x(index) = problem.spatial_k_x[index]; + host_x(index) = problem.spatial.k_x[index]; for (std::size_t index = 0; index < n_y_; ++index) - host_y(index) = problem.spatial_k_y[index]; + host_y(index) = problem.spatial.k_y[index]; Kokkos::deep_copy(execution_, k_x_, host_x); Kokkos::deep_copy(execution_, k_y_, host_y); - if (problem.spatial_far_field_slots != 0) { - const auto slots = problem.spatial_far_field_slots; + if (problem.spatial.far_field_slots != 0) { + const auto slots = problem.spatial.far_field_slots; const auto bands = far_field_band_count(n_x_, n_y_, slots, n_omega_); partials_ = RealView("solprop_native.far_field.partials", bands * slots * n_omega_); } @@ -113,16 +114,7 @@ std::size_t FarFieldPass::table_bytes() const noexcept { return (k_x_.size() + k_y_.size() + tables_.size() + packed_.size()) * sizeof(double); } -void FarFieldPass::close() noexcept { - execution_.fence(); - k_x_ = {}; - k_y_ = {}; - partials_ = {}; - tables_ = {}; - packed_ = {}; - host_tables_ = {}; - host_packed_ = {}; -} +FarFieldPass::~FarFieldPass() { execution_.fence(); } std::vector FarFieldPass::run(const std::uint64_t flags, const std::vector& apertures, diff --git a/native/src/general.cpp b/native/src/general.cpp deleted file mode 100644 index 4abf76725..000000000 --- a/native/src/general.cpp +++ /dev/null @@ -1,18301 +0,0 @@ -// SPDX-FileCopyrightText: 2026 John C. Travers and contributors -// -// SPDX-License-Identifier: GPL-3.0-or-later - -#include "solprop_native/adams_phi.hpp" -#include "solprop_native/core.hpp" -#include "solprop_native/coupled_flow.hpp" -#include "solprop_native/crossing.hpp" -#include "solprop_native/dispersive.hpp" -#include "solprop_native/envelope_plasma.hpp" -#include "solprop_native/far_field.hpp" -#include "solprop_native/fft_plan.hpp" -#include "solprop_native/flow_functions.hpp" -#include "solprop_native/fp.hpp" -#include "solprop_native/gemm.hpp" -#include "solprop_native/general_modal.hpp" -#include "solprop_native/plasma.hpp" -#include "solprop_native/raman_fft.hpp" -#include "solprop_native/slab.hpp" -#include "solprop_native/vmath.hpp" -#include "solprop_native/z_table.hpp" - -#include -#include -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -#include -#ifdef __SSE2__ -#include -#define SOLPROP_NATIVE_STREAMING_STORES 1 -#endif -#endif - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#ifdef __APPLE__ -#include -#elif defined(__GLIBC__) -#include -#endif - -namespace solprop_native { -namespace { -/** The three squared norms an accepted step reports, as one reduction value. */ -struct NormSums { - double old_sum = 0.0; - double new_sum = 0.0; - double error_sum = 0.0; - KOKKOS_INLINE_FUNCTION NormSums& operator+=(const NormSums& other) { - old_sum += other.old_sum; - new_sum += other.new_sum; - error_sum += other.error_sum; - return *this; - } -}; -} // namespace -} // namespace solprop_native - -template <> struct Kokkos::reduction_identity { - KOKKOS_FUNCTION static solprop_native::NormSums sum() { return {}; } -}; - -namespace solprop_native { -namespace { -/** A plane's scaled fluence, one reduction value: its sum and its index moments. */ -struct FluenceMoments { - double total = 0.0; - double x = 0.0; - double y = 0.0; - KOKKOS_INLINE_FUNCTION FluenceMoments& operator+=(const FluenceMoments& other) { - total += other.total; - x += other.x; - y += other.y; - return *this; - } -}; -} // namespace -} // namespace solprop_native - -template <> struct Kokkos::reduction_identity { - KOKKOS_FUNCTION static solprop_native::FluenceMoments sum() { return {}; } -}; - -namespace solprop_native { -namespace { - -// Sets a coupled flow aside for a stage evaluation and puts it back however the scope -// is left: the Adams sample is the material source alone (solprop SEP 16 phase 5). -class FlowAside { -public: - explicit FlowAside(std::unique_ptr& slot) - : slot_(slot), held_(std::exchange(slot, nullptr)) {} - // Restores the slot; clang-tidy 18 misreads it as trivial when GCC 13's intrinsic - // headers fail to parse. - // NOLINTNEXTLINE(modernize-use-equals-default) - ~FlowAside() { slot_ = std::move(held_); } - FlowAside(const FlowAside&) = delete; - FlowAside& operator=(const FlowAside&) = delete; - FlowAside(FlowAside&&) = delete; - FlowAside& operator=(FlowAside&&) = delete; - -private: - std::unique_ptr& slot_; - std::unique_ptr held_; -}; - -// One frozen-residual weight row from the host into a View's row. -template -void upload_weights(const ExecutionSpace& space, const Row& out, - const std::vector& weights) { - if (weights.size() < out.extent(0)) - throw std::invalid_argument("a coupled flow's weights are shorter than its terms"); - const Kokkos::View> - host(weights.data(), out.extent(0)); - graph::deep_copy(space, out, host); -} - -// A frozen-residual flow's generator at one position applied to a field, `sum_t w_t -// G_t u` on the block's bins and zero off them (the terms are zero off the scored -// bins). A zero weight is skipped, which leaves every bit as the full sum has it. -template -void apply_generator(const ExecutionSpace& space, const std::size_t n, - const std::size_t modes, - const Kokkos::View& slots, - const Kokkos::View& terms, - const Kokkos::View& weights, - const Kokkos::View& input, - const Out& out) { - const auto n_terms = weights.extent(0); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - launch::parallel_for( - "solprop_native.adams.generator", Range2(space, {0, 0}, {n, modes}), - KOKKOS_LAMBDA(const std::size_t index, const std::size_t i) { - const auto one = slots(index); - Complex value{}; - if (one != 0) { - const auto bin = one - 1; - for (std::size_t entry = 0; entry < n_terms; ++entry) { - const double weight = weights(entry); - if (weight == 0.0) - continue; - for (std::size_t j = 0; j < modes; ++j) - value += weight * terms(entry, bin, i, j) * input(j, index); - } - } - out(i, index) = value; - }); -#else - launch::parallel_for( - "solprop_native.adams.generator", - Kokkos::RangePolicy(space, 0, n), - KOKKOS_LAMBDA(const std::size_t index) { - const auto one = slots(index); - for (std::size_t i = 0; i < modes; ++i) { - Complex value{}; - if (one != 0) { - const auto bin = one - 1; - for (std::size_t entry = 0; entry < n_terms; ++entry) { - const double weight = weights(entry); - if (weight == 0.0) - continue; - for (std::size_t j = 0; j < modes; ++j) - value += weight * terms(entry, bin, i, j) * input(j, index); - } - } - out(i, index) = value; - } - }); -#endif -} - -template -KOKKOS_INLINE_FUNCTION void capture_anchor(const Ts&... /*unused*/) {} - -// Net process allocator growth includes opaque vendor FFT plan/workspace -// allocations. It is measured separately from Kokkos Views and is not a peak bound. -std::size_t heap_bytes() { -#ifdef __APPLE__ - malloc_statistics_t statistics{}; - malloc_zone_statistics(nullptr, &statistics); - return statistics.size_in_use; -#elif defined(__GLIBC__) - // mallinfo2 is glibc 2.33 or later; a wheel built against an older glibc (the - // manylinux_2_28 baseline) finds it at run time when the system provides it. Its - // layout is ten size_t counters, in glibc's declared order. - struct Mallinfo2 { - std::size_t arena, ordblks, smblks, hblks, hblkhd, usmblks, fsmblks, uordblks, - fordblks, keepcost; - }; - using Query = Mallinfo2 (*)(); - static const auto query = reinterpret_cast(dlsym(RTLD_DEFAULT, "mallinfo2")); - if (query == nullptr) - return 0; - const auto statistics = query(); - return statistics.uordblks + statistics.hblkhd; -#else - // Neither allocator interface exists (musl, Windows): the ledger row then measures - // nothing, and construction must still succeed. - return 0; -#endif -} -std::size_t device_free_bytes() { -#ifdef KOKKOS_ENABLE_CUDA - std::size_t available{}, total{}; - if (cudaMemGetInfo(&available, &total) != cudaSuccess) - throw std::runtime_error("cudaMemGetInfo failed during FFT planning"); - return available; -#elif defined(KOKKOS_ENABLE_HIP) - std::size_t available{}, total{}; - if (hipMemGetInfo(&available, &total) != hipSuccess) - throw std::runtime_error("hipMemGetInfo failed during FFT planning"); - return available; -#else - return 0; -#endif -} - -using ComplexView2 = Kokkos::View; -using ComplexView3 = Kokkos::View; -using RealView2 = Kokkos::View; -using RealView3 = Kokkos::View; - -template -KOKKOS_INLINE_FUNCTION void -write_raman_drive(const RealView2& drive, const std::size_t node, - const std::size_t time, const Complex x, const Complex y, - const std::size_t count) { - const double xx = RealField ? x.real() * x.real() - : 0.5 * (x.real() * x.real() + x.imag() * x.imag()); - if constexpr (Components == 1) { - drive(node, time) = xx; - } else { - const double yy = RealField ? y.real() * y.real() - : 0.5 * (y.real() * y.real() + y.imag() * y.imag()); - drive(count * node, time) = xx + yy; - if (count == 3) { - drive(count * node + 1, time) = xx - yy; - drive(count * node + 2, time) = RealField - ? 2.0 * x.real() * y.real() - : x.real() * y.real() + x.imag() * y.imag(); - } - } -} -using ByteView = Kokkos::View; -using IndexView = Kokkos::View; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -// A device splits a few long real rows over its multiprocessors (`RealRowPlan`). -using InverseRealPlan = RealRowPlan; -using ForwardRealPlan = RealRowPlan; -#else -using InverseRealPlan = KokkosFFT::Plan; -using ForwardRealPlan = KokkosFFT::Plan; -#endif -using RamanForwardPlan = RamanFftPlan; -using RamanInversePlan = RamanFftPlan; -using ComplexPlan = KokkosFFT::Plan; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -// A device's plans of the transverse scratch run one after another, so they share one -// work area (`transform_work_`) in place of holding one each (solprop #529). A -// Kerr-only free-space envelope takes the stage's time axis with the plane's in one -// rank-3 transform (`volume_kerr_`). -using TransversePlan = KokkosFFT::DynPlan; -using VolumePlan = TransversePlan; -#else -using TransversePlan = KokkosFFT::Plan; -using VolumePlan = KokkosFFT::Plan; -#endif -// The hybrid planar kind transforms y alone, the trailing axis of (bins, n_p, n_y). -using SlabPlan = KokkosFFT::Plan; -// A requested collar's padded transform, along the last axis of a batch (solprop SEP -// 44). -using StripPlan = KokkosFFT::Plan; - -// Each current keeps its own chunk accumulator, including stacked embedded -// quadrature. Sharing the latter would mix currents between partial chunks. -struct EnvelopeStream { - ComplexView2 node, projected, coarse, stacked, spectrum, coarse_spectrum; - std::unique_ptr forward, coarse_forward; - [[nodiscard]] std::size_t bytes() const { - return sizeof(Complex) * - (node.size() + projected.size() + coarse.size() + stacked.size() + - spectrum.size() + coarse_spectrum.size()); - } -}; - -// Real current products own their projections, including an embedded quadrature. -struct FieldCurrentStream { - RealView2 node, projected, coarse, stacked; - ComplexView2 spectrum, coarse_spectrum; - std::unique_ptr forward, coarse_forward; - [[nodiscard]] std::size_t bytes() const { - return sizeof(double) * - (node.size() + projected.size() + coarse.size() + stacked.size()) + - sizeof(Complex) * (spectrum.size() + coarse_spectrum.size()); - } -}; - -// CPU experiments favour one task per column; accelerator lanes retain their -// two-dimensional launch. Both invoke the same element functor in the same order. -template -void cartesian_for(const char* label, const ExecutionSpace& execution, - const std::size_t rows, const std::size_t width, - const Function& element) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - launch::parallel_for(label, Range2(execution, {0, 0}, {rows, width}), element); -#else - launch::parallel_for( - label, Kokkos::RangePolicy(execution, 0, rows), - KOKKOS_LAMBDA(const std::size_t row) { - for (std::size_t inner = 0; inner < width; ++inner) - // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide - element(row, inner); - }); -#endif -} - -// A traversal between a separable geometry's column-major registers `(column, omega)` -// and the transverse scratch `(omega, y, x)` (SEP 28). On CPU one task per column -// reads the scratch a whole plane apart at every bin; tiles of adjacent columns and -// bins, columns fastest within a tile, read both layouts in contiguous runs. The -// element is the same functor; each (column, bin) is visited once. -// 32 columns by 16 bins: the best of 32x16, 64x32 and 256x8 on one Zen 5 socket -// (SEP 28; Apple silicon is within a few per cent across them). -[[maybe_unused]] inline constexpr std::size_t crossing_tile_columns = 32; -[[maybe_unused]] inline constexpr std::size_t crossing_tile_bins = 16; - -// A store that bypasses the caches where the CPU has one (x86 SSE2): a traversal that -// writes a large volume it does not read (the scatter into the transverse scratch) -// otherwise reads every line before writing it. Ordered by `stream_fence`. Device -// builds compile out the host traversals that use these, hence `maybe_unused`. -[[maybe_unused]] KOKKOS_INLINE_FUNCTION void stream_store(Complex& target, - const Complex value) { -#ifdef SOLPROP_NATIVE_STREAMING_STORES - _mm_stream_pd(reinterpret_cast(&target), - _mm_set_pd(value.imag(), value.real())); -#else - target = value; -#endif -} - -[[maybe_unused]] KOKKOS_INLINE_FUNCTION void stream_fence() { -#ifdef SOLPROP_NATIVE_STREAMING_STORES - _mm_sfence(); -#endif -} - -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -// One host tile of a crossing: its rows `[first_row, last_row)` and bins -// `[first_bin, last_bin)`, at most `crossing_tile_columns` by `crossing_tile_bins`. -struct CrossingTile { - std::size_t first_row, last_row, first_bin, last_bin; - - [[nodiscard]] std::size_t size() const noexcept { - return (last_row - first_row) * (last_bin - first_bin); - } - // An element's place in a tile buffer, rows outermost. - [[nodiscard]] std::size_t at(const std::size_t row, - const std::size_t bin) const noexcept { - return (row - first_row) * (last_bin - first_bin) + (bin - first_bin); - } - // Every element in the traversal order `crossing_for` names. - template void visit(const Function& element) const { - if constexpr (BinsInner) - for (std::size_t row = first_row; row < last_row; ++row) - for (std::size_t inner = first_bin; inner < last_bin; ++inner) - element(row, inner); - else - for (std::size_t inner = first_bin; inner < last_bin; ++inner) - for (std::size_t row = first_row; row < last_row; ++row) - element(row, inner); - } -}; - -// The host tiles of a crossing, one task each: `crossing_for`'s traversal, for a -// kernel that forms a whole tile's transcendentals in one pass first. -template -void crossing_tiles(const char* label, const ExecutionSpace& execution, - const std::size_t rows, const std::size_t width, - const Function& visit) { - constexpr std::size_t tile_rows = crossing_tile_columns; - constexpr std::size_t tile_bins = crossing_tile_bins; - const std::size_t row_tiles = (rows + tile_rows - 1) / tile_rows; - const std::size_t bin_tiles = (width + tile_bins - 1) / tile_bins; - launch::parallel_for( - label, Kokkos::RangePolicy(execution, 0, row_tiles * bin_tiles), - KOKKOS_LAMBDA(const std::size_t index) { - const std::size_t first_row = (index / bin_tiles) * tile_rows; - const std::size_t first_bin = (index % bin_tiles) * tile_bins; - visit(CrossingTile{ - .first_row = first_row, - .last_row = rows - first_row < tile_rows ? rows : first_row + tile_rows, - .first_bin = first_bin, - .last_bin = width - first_bin < tile_bins ? width : first_bin + tile_bins}); - // Each thread orders its own streaming stores before the region ends. - if constexpr (Streaming) - stream_fence(); - }); -} - -// A tile's propagators `exp(argument)`, as `propagator_exp` forms them, in one pass -// of the vector-maths layer (solprop ADR-059, issue #338): the cosines and sines, -// then the magnitudes, `exp(+-0)` exactly one on a lossless element without the -// call. Fill `real` and `imag` for the tile's elements, then `form`. -struct PropagatorTile { - static constexpr std::size_t capacity = crossing_tile_columns * crossing_tile_bins; - // The static analyser cannot follow that `form` and `operator[]` read only what the - // tile's visit wrote, so its build zeroes the arrays; a real build leaves them unset. -#ifdef __clang_analyzer__ - double real[capacity]{}, imag[capacity]{}, cosine[capacity]{}, sine[capacity]{}; -#else - double real[capacity], imag[capacity], cosine[capacity], sine[capacity]; -#endif - - void form(const std::size_t count) { - vmath::sincos(imag, sine, cosine, count); - bool lossless = true; - for (std::size_t k = 0; k < count; ++k) - lossless = lossless && real[k] == 0.0; - if (lossless) - return; -#ifdef __clang_analyzer__ - double magnitude[capacity]{}; -#else - double magnitude[capacity]; -#endif - vmath::exp(real, magnitude, count); - for (std::size_t k = 0; k < count; ++k) { - const double scale = real[k] == 0.0 ? 1.0 : magnitude[k]; - cosine[k] *= scale; - sine[k] *= scale; - } - } - Complex operator[](const std::size_t k) const { return {cosine[k], sine[k]}; } -}; -#endif - -// On a device, `RowFastest` maps adjacent rows (columns of the plane) to adjacent -// threads, so the transverse scratch side of a gather or scatter is coalesced; the -// chunk-local side is the strided one. -// On the host, `BinsInner` visits a tile's rows outermost and its bins innermost: a -// traversal that writes the rows (a pack from the scratch) then stores each row's run -// at once, about 1.7 times faster for the free-space chunk's pack on an Intel Xeon Gold -// 6240 at one thread. -template -void crossing_for(const char* label, const ExecutionSpace& execution, - const std::size_t rows, const std::size_t width, - const Function& element) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - if constexpr (RowFastest) { - using Left = Kokkos::MDRangePolicy< - ExecutionSpace, Kokkos::Rank<2, Kokkos::Iterate::Left, Kokkos::Iterate::Left>, - Kokkos::IndexType>; - launch::parallel_for( - label, - Left(execution, {0, 0}, - {static_cast(rows), static_cast(width)}), - KOKKOS_LAMBDA(const std::int64_t row, const std::int64_t inner) { - element(static_cast(row), static_cast(inner)); - }); - } else { - cartesian_for(label, execution, rows, width, element); - } -#else - crossing_tiles( - label, execution, rows, width, - [&](const CrossingTile& tile) { tile.visit(element); }); -#endif -} - -struct SpatialElement { - Complex linear, prefactor; -}; - -// A device crossing through a 32-by-32 tile of team scratch (SEP 28): the tile is read -// with adjacent threads along the side that is contiguous for `read` (rows when -// `read_rows`, else bins) and written with adjacent threads along the other, so both -// the transverse scratch and the chunk's rows are accessed coalesced. -template -void tiled_crossing(const char* label, const ExecutionSpace& execution, - const std::size_t rows, const std::size_t width, - const bool read_rows, const Read& read, const Write& write) { - constexpr int tile = 32; - const int depth = std::min(8, execution.concurrency()); - using Policy = Kokkos::TeamPolicy; - using Member = Policy::member_type; - using Shared = Kokkos::View>; - const std::size_t row_tiles = (rows + tile - 1) / tile; - const std::size_t bin_tiles = (width + tile - 1) / tile; - const auto bytes = Shared::shmem_size(tile, tile + 1); - launch::parallel_for( - label, - Policy(execution, static_cast(row_tiles * bin_tiles), depth, tile) - .set_scratch_size(0, Kokkos::PerTeam(bytes)), - KOKKOS_LAMBDA(const Member& team) { - // The buffer holds (bin, row) within the tile. - Shared buffer(team.team_scratch(0), tile, tile + 1); - const auto which = static_cast(team.league_rank()); - const std::size_t first_row = (which / bin_tiles) * tile; - const std::size_t first_bin = (which % bin_tiles) * tile; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, tile), [&](const int outer) { - Kokkos::parallel_for( - Kokkos::ThreadVectorRange(team, tile), [&](const int lane) { - const int row_local = read_rows ? lane : outer; - const int bin_local = read_rows ? outer : lane; - const std::size_t row = first_row + static_cast(row_local); - const std::size_t bin = first_bin + static_cast(bin_local); - if (row < rows && bin < width) - buffer(bin_local, row_local) = read(row, bin); - }); - }); - team.team_barrier(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, tile), [&](const int outer) { - Kokkos::parallel_for( - Kokkos::ThreadVectorRange(team, tile), [&](const int lane) { - const int row_local = read_rows ? outer : lane; - const int bin_local = read_rows ? lane : outer; - const std::size_t row = first_row + static_cast(row_local); - const std::size_t bin = first_bin + static_cast(bin_local); - if (row < rows && bin < width) - write(row, bin, buffer(bin_local, row_local)); - }); - }); - }); -} - -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -// A device update of registers and the transverse scratch together (SEP 28): a tile of -// the scratch is staged in team scratch with adjacent threads along rows, `body` -// visits each (row, bin) with adjacent threads along bins (the registers' contiguous -// side) and may change its cell, and the tile goes back to the scratch along rows -// when `writes`. -template -void tiled_update(const char* label, const ExecutionSpace& execution, - const std::size_t rows, const std::size_t width, const bool writes, - const Read& read, const Body& body, const Write& write) { - constexpr int tile = 32; - constexpr int depth = 8; - using Policy = Kokkos::TeamPolicy; - using Member = Policy::member_type; - using Shared = Kokkos::View>; - const std::size_t row_tiles = (rows + tile - 1) / tile; - const std::size_t bin_tiles = (width + tile - 1) / tile; - const auto bytes = Shared::shmem_size(tile, tile + 1); - launch::parallel_for( - label, - Policy(execution, static_cast(row_tiles * bin_tiles), depth, tile) - .set_scratch_size(0, Kokkos::PerTeam(bytes)), - KOKKOS_LAMBDA(const Member& team) { - Shared buffer(team.team_scratch(0), tile, tile + 1); - const auto which = static_cast(team.league_rank()); - const std::size_t first_row = (which / bin_tiles) * tile; - const std::size_t first_bin = (which % bin_tiles) * tile; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, tile), [&](const int outer) { - Kokkos::parallel_for( - Kokkos::ThreadVectorRange(team, tile), [&](const int lane) { - const std::size_t row = first_row + static_cast(lane); - const std::size_t bin = first_bin + static_cast(outer); - if (row < rows && bin < width) - buffer(outer, lane) = read(row, bin); - }); - }); - team.team_barrier(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, tile), [&](const int outer) { - Kokkos::parallel_for( - Kokkos::ThreadVectorRange(team, tile), [&](const int lane) { - const std::size_t row = first_row + static_cast(outer); - const std::size_t bin = first_bin + static_cast(lane); - if (row < rows && bin < width) - body(row, bin, buffer(lane, outer)); - }); - }); - if (!writes) - return; - team.team_barrier(); - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, tile), [&](const int outer) { - Kokkos::parallel_for( - Kokkos::ThreadVectorRange(team, tile), [&](const int lane) { - const std::size_t row = first_row + static_cast(lane); - const std::size_t bin = first_bin + static_cast(outer); - if (row < rows && bin < width) - write(row, bin, buffer(outer, lane)); - }); - }); - }); -} -#endif - -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -// Plan, then discard, a batched complex transform of transverse planes at the plan -// policy's effort, leaving its wisdom. It overwrites both arrays; the caller clears -// them. Row transforms are seeded by `seed_row_fft_wisdom` (fft_plan.hpp). -inline void seed_fftw_wisdom(const ExecutionSpace& space, Complex* input, - Complex* output, const std::vector& extents, - const int howmany, const int input_distance, - const int output_distance, const int sign) { - std::string sizes; - for (const int extent : extents) - sizes += (sizes.empty() ? "" : "x") + std::to_string(extent); - const std::string key = fft_plan_key( - sign == FFTW_FORWARD ? "planes-forward" : "planes-backward", sizes, howmany, - input_distance, output_distance, input, output, space.concurrency()); - fftw_init_threads(); - const auto chosen = begin_seed(key, FftShape::planes, space.concurrency()); - if (!chosen) - return; - fftw_plan_with_nthreads(space.concurrency()); - auto* in = reinterpret_cast(input); - auto* out = reinterpret_cast(output); - auto* const seeded = fftw_plan_many_dft( - static_cast(extents.size()), extents.data(), howmany, in, nullptr, 1, - input_distance, out, nullptr, 1, output_distance, sign, *chosen); - // The policy has always tolerated a plane it cannot seed; a profile records it. - if (seeded == nullptr && (*chosen & FFTW_WISDOM_ONLY) == 0U && - (detail::planning == nullptr || - detail::planning->mode == FftPlanning::Mode::legacy)) { - release_seed(key, false); - return; - } - end_seed(key, *chosen, seeded != nullptr); - if (seeded != nullptr) - fftw_destroy_plan(seeded); -} -#endif - -// Where a separable geometry's column (or a temporal row, which is the same index) -// lives in the transverse scratch. Free space: (.., n_y, n_x), x fastest. The hybrid -// planar kind: (.., n_p, n_y) with the y transform trailing, the channel fastest in -// the column index p + n_p * iy. -struct TransverseLayout { - std::size_t inner{}; - bool channel_fastest{}; - // Elements per transverse plane of the row-major scratch `(planes, ., .)`. - std::size_t plane{}; - // A column's offset within a plane: the column itself on free space (y, x with x - // fastest), `p * n_y + iy` on the planar kind. - KOKKOS_INLINE_FUNCTION std::size_t offset(const std::size_t column) const { - return channel_fastest ? (column % inner) * (plane / inner) + column / inner - : column; - } - template - KOKKOS_INLINE_FUNCTION auto& cell(const View& scratch, const std::size_t plane_index, - const std::size_t column) const { - return scratch.data()[plane_index * plane + offset(column)]; - } - KOKKOS_INLINE_FUNCTION std::size_t first(const std::size_t column) const { - return channel_fastest ? column % inner : column / inner; - } - KOKKOS_INLINE_FUNCTION std::size_t second(const std::size_t column) const { - return channel_fastest ? column / inner : column % inner; - } -}; - -// A node's physical spectrum in a transverse volume after the inverse transverse plan: -// on free space the node's own cell; on the hybrid planar kind the slab rule over its y -// row's channels, without the temporal pad scale the planar synthesis carries, since -// the statistics read the coarse field (solprop's `collect_spatial`). -struct PhysicalSpectrum { - TransverseLayout layout{}; - RealView2 synthesis; - std::size_t n_x{}, n_p{}; - double crop_scale = 1.0; - bool slab = false; - template - KOKKOS_INLINE_FUNCTION Complex read(const View& source, const std::size_t plane, - const std::size_t node) const { - if (!slab) - return layout.cell(source, plane, node); - const auto ix = node % n_x; - const auto rows = (node / n_x) * n_p; - Complex sum{}; - for (std::size_t channel = 0; channel < n_p; ++channel) - sum += synthesis(ix, channel) * - source(plane, layout.first(rows + channel), layout.second(rows + channel)); - return crop_scale * sum; - } -}; - -// Where a separable route's low-storage registers (the accumulator, the stage register -// and the flow table) hold the element (column, omega). On the host, at the transverse -// scratch's own position: a fold then reads them in the scratch's order beside it, and -// only the state crosses layouts. On a device, at the state's (column-major). -struct RegisterIndex { - TransverseLayout layout{}; - std::size_t n_pol{}, n{}; - bool plane{}; - // One polarisation: a column is its mode, without a division (a flag of its own, - // which the compiler cannot fold back into `column / 1`). - bool scalar{}; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // A stage of the volume route on a device (`volume_order_`): the scratch is `(y, x, - // omega)`, the state's and the registers' own order, so nothing crosses. - bool volume{}; -#endif - KOKKOS_INLINE_FUNCTION std::size_t mode(const std::size_t column) const { - return scalar ? column : column / n_pol; - } - KOKKOS_INLINE_FUNCTION std::size_t pol(const std::size_t column) const { - return scalar ? 0 : column % n_pol; - } - // The transverse scratch's position of the element, on any build. - KOKKOS_INLINE_FUNCTION std::size_t cell(const std::size_t column, - const std::size_t omega) const { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - if (volume) - return column * n + omega; -#endif - return (omega * n_pol + pol(column)) * layout.plane + layout.offset(mode(column)); - } - KOKKOS_INLINE_FUNCTION std::size_t operator()(const std::size_t column, - const std::size_t omega) const { - return plane ? cell(column, omega) : column * n + omega; - } -}; - -// One chunk of a separable geometry's response pass: whole transverse units (a node -// on free space, a y row of n_p columns and n_x nodes on the hybrid planar kind), its -// temporal rows (state columns) and its physical nodes. -struct TransverseChunk { - std::size_t first_row{}, rows{}, first_node{}, nodes{}; -}; - -struct SpatialCoefficients { - ComplexView square, square_low, numerator, clamped; - RealView transverse, transverse_low, frame, spectral_rate, user_rate, absorber; - // A transverse collar's power rate per column (solprop kernel schema 31). - RealView transverse_rate; - // The hybrid planar kind's wall terms, (channels, n_omega); empty on free space. - ComplexView2 kappa; - std::size_t channels{}; - double evanescent{}; - // The longitudinal square q of a column and bin and its principal root, solprop's - // `longitudinal_root`: the one evaluation the operator and the plane measure share. - KOKKOS_INLINE_FUNCTION Complex longitudinal_square(const std::size_t mode, - const std::size_t omega) const { - Complex longitudinal = - (square(omega) - transverse(mode)) + (square_low(omega) - transverse_low(mode)); - // q = k^2 eps_co - k_y^2 - kappa_p^2, the wall term subtracted last as in solprop's - // separable_values (kernel schema 15), so a free-space column's arithmetic is - // untouched. - if (channels != 0) - longitudinal = longitudinal - kappa(mode % channels, omega); - return longitudinal; - } - // A real q (a lossless medium) takes the real root, the same bits as the complex - // root of q + 0i at a fraction of its cost (SEP 28): sqrt(q) or i sqrt(-q). - static KOKKOS_INLINE_FUNCTION Complex root_of(const Complex longitudinal) { - return longitudinal.imag() == 0.0 - ? (longitudinal.real() >= 0.0 - ? Complex(Kokkos::sqrt(longitudinal.real()), 0.0) - : Complex(0.0, Kokkos::sqrt(-longitudinal.real()))) - : Kokkos::sqrt(longitudinal); - } - // The conjugate of the plane's dimensionless forward admittance `c k_z / omega` - // (solprop SEP 34, `axial_admittance`) on an active propagating bin at positive - // frequency, zero elsewhere: the factor an internal (conjugate) spectrum takes to - // its magnetic closure. Its real part weighs energy and spectra. - KOKKOS_INLINE_FUNCTION Complex closing(const std::size_t mode, - const std::size_t omega, const bool active, - const double frequency) const { - const Complex longitudinal = longitudinal_square(mode, omega); - if (!active || !(longitudinal.real() > 0.0) || !(frequency > 0.0)) - return Complex{}; - const Complex root = root_of(longitudinal); - constexpr double speed_of_light = 299792458.0; - return Complex(speed_of_light * root.real(), -(speed_of_light * root.imag())) / - frequency; - } - // `value`'s prefactor alone, the same arithmetic: what a fold whose flows are - // tabulated reads of the element. - KOKKOS_INLINE_FUNCTION Complex prefactor(const std::size_t mode, - const std::size_t omega, - const bool active) const { - if (!active) - return Complex{}; - const Complex longitudinal = longitudinal_square(mode, omega); - if (!(longitudinal.real() > 0.0)) - return clamped(omega); - const Complex root = root_of(longitudinal); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return numerator(omega) * (1.0 / root.real()); -#else - return numerator(omega) / root.real(); -#endif - } - KOKKOS_INLINE_FUNCTION SpatialElement value(const std::size_t mode, - const std::size_t omega, - const bool active) const { - const Complex longitudinal = longitudinal_square(mode, omega); - const Complex root = root_of(longitudinal); - const bool propagating = longitudinal.real() > 0.0; - const double attenuation = - propagating ? root.imag() : Kokkos::min(Kokkos::abs(root.imag()), evanescent); - // The transverse collar's rate joins the attenuation inside the active band, as - // in solprop's separable_values (SEP 22, Q6); a zero rate changes no bit. - const Complex linear = active ? Complex(-attenuation - 0.5 * transverse_rate(mode), - -(root.real() - frame(omega))) - : Complex{}; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // One binary64 reciprocal and two products in place of two divisions, at every - // element of the stage passes: a device's division is several fused steps (at - // most an ulp from the quotients). - const Complex prefactor = - propagating ? numerator(omega) * (1.0 / root.real()) : clamped(omega); -#else - const Complex prefactor = - propagating ? numerator(omega) / root.real() : clamped(omega); -#endif - return {.linear = linear - 0.5 * spectral_rate(omega), - .prefactor = active ? prefactor : Complex{}}; - } -}; - -KOKKOS_INLINE_FUNCTION Complex exponential(const Complex value) { - const double magnitude = Kokkos::exp(value.real()); - return {magnitude * Kokkos::cos(value.imag()), magnitude * Kokkos::sin(value.imag())}; -} - -// `cos theta` and `sin theta`. On a device they are one `sincos` call, which shares -// the argument's reduction between the two (on a GPU of reduced binary64 rate the -// stage propagators are about half of the stage entry); the host keeps the two -// library calls, so its values are unchanged. -KOKKOS_INLINE_FUNCTION void cosine_sine(const double theta, double& cosine, - double& sine) { -#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__) - ::sincos(theta, &sine, &cosine); -#else - cosine = Kokkos::cos(theta); - sine = Kokkos::sin(theta); -#endif -} - -// A flow's magnitude: on a lossless bin `exp(+-0)`, exactly one, taken without a -// binary64 `exp` per element. Out of line on a host, whose call to its library was out -// of line already: inlined there, this branch took the tabulated fold's whole kernel -// past the compiler's inlining limit, 40 % slower (solprop's docs/dev/benchmarks.md). -#if defined(__GNUC__) && !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -[[gnu::noinline]] -#endif -KOKKOS_INLINE_FUNCTION double flow_magnitude(const double real) { - return real == 0.0 ? 1.0 : Kokkos::exp(real); -} - -// A stage or step propagator `exp(argument)`: `exponential` with the cosine and sine -// from `cosine_sine`, so it is `exponential` to the bit on the host. -KOKKOS_INLINE_FUNCTION Complex propagator_exp(const Complex argument) { - const double magnitude = flow_magnitude(argument.real()); - double cosine = 0.0; - double sine = 0.0; - cosine_sine(argument.imag(), cosine, sine); - return {magnitude * cosine, magnitude * sine}; -} - -// `exponential`, or on a lossless problem (a signed-zero real part) the same value -// with its magnitude `exp(+-0)`, exactly one, taken without the call. The cosine and -// sine come from `cosine_sine`, so on the host it is `exponential` to the bit. -template -KOKKOS_INLINE_FUNCTION Complex entry_exponential(const Complex value) { - if constexpr (Lossless) { - double cosine = 0.0; - double sine = 0.0; - cosine_sine(value.imag(), cosine, sine); - return {cosine, sine}; - } else { - return propagator_exp(value); - } -} - -KOKKOS_INLINE_FUNCTION double squared_abs(const Complex value) { - return value.real() * value.real() + value.imag() * value.imag(); -} - -KOKKOS_INLINE_FUNCTION double trapezoid(const double current, const double previous, - const double dt) { - return dt * (current + previous) / 2.0; -} - -// The fraction ionised from an exposure's decay `exp(-X)`, `1 - (1 - p) exp(-X)`, as -// `bodies.fraction_at` rounds it. A small response: strict arithmetic (solprop SEP 25 -// section 4.2, `fp.hpp`), so the product and the difference round separately. -SOLPROP_NATIVE_STRICT_FUNCTION -KOKKOS_INLINE_FUNCTION double decayed_fraction(const double decay, - const double preionisation) { - SOLPROP_NATIVE_STRICT_BLOCK - const double survival = (1.0 - preionisation) * decay; - return 1.0 - survival; -} - -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -// `decayed_fraction` of each of a host block's `count` decays, in place: one strict -// loop that vectorises, each product and difference rounded as there. -SOLPROP_NATIVE_STRICT_FUNCTION -inline void decayed_fractions(double* values, const std::size_t count, - const double preionisation) { - SOLPROP_NATIVE_STRICT_BLOCK - for (std::size_t k = 0; k < count; ++k) { - const double survival = (1.0 - preionisation) * values[k]; - values[k] = 1.0 - survival; - } -} -#endif - -// The fraction ionised from an exposure, `decayed_fraction` of its `exp(-X)`. -KOKKOS_INLINE_FUNCTION double ionised_fraction(const double exposure, - const double preionisation) { - return decayed_fraction(Kokkos::exp(-exposure), preionisation); -} - -// The envelope column's fraction, `p + (1 - p) (-expm1(-X))`, strict as above. -SOLPROP_NATIVE_STRICT_FUNCTION -KOKKOS_INLINE_FUNCTION double envelope_fraction(const double exposure, - const double preionisation) { - SOLPROP_NATIVE_STRICT_BLOCK - const double ionised = (1.0 - preionisation) * (-Kokkos::expm1(-exposure)); - return preionisation + ionised; -} - -// The band rule's fraction: the exposure floored at zero where the fraction is formed -// (SEP 18 section 4.5), `max(X, 0)` as NumPy's `maximum` takes it, so a NaN exposure -// stays NaN rather than becoming the preionisation. -KOKKOS_INLINE_FUNCTION double band_fraction(const double exposure, - const double preionisation) { - return ionised_fraction(exposure < 0.0 ? 0.0 : exposure, preionisation); -} - -// A multistep compensation's residual term `(D - x / dz) u`: the difference of two -// nearly equal rates, strict (solprop SEP 25 section 4.2), each complex operation -// spelt with `fp.hpp`'s helpers. -SOLPROP_NATIVE_STRICT_FUNCTION -KOKKOS_INLINE_FUNCTION Complex compensation_residual(const Complex rate, - const Complex exponent, - const double inverse_dz, - const Complex state) { - SOLPROP_NATIVE_STRICT_BLOCK - return strict_mul(strict_sub(rate, strict_scale(inverse_dz, exponent)), state); -} - -// `bodies.loss_scale`: the loss current's coefficient of the field, or zero. -KOKKOS_INLINE_FUNCTION double band_loss_scale(const double rate, const double fraction, - const double squared, - const double ionisation_potential) { - if (rate > 0.0 && squared > 0.0) - return ionisation_potential * rate * (1.0 - fraction) / squared; - return 0.0; -} - -// The rate table's cubic in `log(rate)` at a field's `log |E|` within the table's -// range: the interval's local coordinate and its Horner form (`ionisation_rate`). -template -KOKKOS_INLINE_FUNCTION double -ionisation_log_rate(const double log_field, const View& coefficients, - const double log_field_min, const double d_log_field, - const std::size_t intervals, const std::size_t offset = 0) { - long raw_index = static_cast((log_field - log_field_min) / d_log_field); - if (raw_index < 0) - raw_index = 0; - // NOLINTNEXTLINE(modernize-use-integer-sign-comparison): device code; std::cmp_* is host-only - if (raw_index >= static_cast(intervals)) - raw_index = static_cast(intervals - 1); - const auto index = static_cast(raw_index); - const double local = - log_field - (log_field_min + static_cast(index) * d_log_field); - return ((coefficients(offset + index) * local + - coefficients(offset + intervals + index)) * - local + - coefficients(offset + 2 * intervals + index)) * - local + - coefficients(offset + 3 * intervals + index); -} - -// The ionisation rate at a field magnitude: the table's limits outside its range, and -// within it the exponential of its cubic in `log |E|`. -template -KOKKOS_INLINE_FUNCTION double -ionisation_rate(const double magnitude, const View& coefficients, - const double log_field_min, const double d_log_field, - const double field_min, const double field_max, const double rate_max, - const std::size_t intervals, const std::size_t offset = 0) { - if (magnitude > field_max) - return rate_max; - if (magnitude < field_min) - return 0.0; - return Kokkos::exp(ionisation_log_rate(Kokkos::log(magnitude), coefficients, - log_field_min, d_log_field, intervals, - offset)); -} - -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -// The samples a host pass of the plasma column takes at a time. -constexpr std::size_t rate_block = 256; - -// `ionisation_rate`'s values at `live <= rate_block` magnitudes within the table's -// range, in place: one `log` and one `exp` pass of the vector-maths layer (solprop -// ADR-059) around the cubic. A host block gathers the magnitudes it passes, so the -// samples outside the range (most of a pulse's wings) cost what they cost before. -template -void ionisation_rates_within(double* values, const std::size_t live, - const View& coefficients, const double log_field_min, - const double d_log_field, const std::size_t intervals, - const std::size_t offset = 0) { - // The cubic writes a local block, which no table gather can alias, so the loop - // vectorises with the table's gathers wherever this is compiled out of line. - double log_rates[rate_block]; - vmath::log(values, values, live); - for (std::size_t k = 0; k < live; ++k) - log_rates[k] = ionisation_log_rate(values[k], coefficients, log_field_min, - d_log_field, intervals, offset); - vmath::exp(log_rates, values, live); -} -#endif - -// A host span as an unmanaged view of a state-shaped memory-space view, for one copy -// between them: no staging mirror, no element loop. A span whose data is not aligned -// as `Complex` (a NumPy buffer need not be) is staged in an aligned vector first. -void upload_span(const ExecutionSpace& execution_space, crossing::Ring& ring, - const ComplexView2& destination, - const std::span source) { - if (destination.size() != source.size()) - throw std::invalid_argument("fixed problem array has the wrong extent"); - if (reinterpret_cast(source.data()) % alignof(Complex) != 0) { - const std::vector aligned(source.begin(), source.end()); - upload_span(execution_space, ring, destination, aligned); - execution_space.fence("solprop_native unaligned upload"); - return; - } - if (crossing::takes_ring(source.size_bytes())) { - execution_space.fence("solprop_native upload"); - ring.from_host(destination.data(), source.data(), source.size_bytes()); - return; - } - const Kokkos::View> - host(source.data(), destination.extent(0), destination.extent(1)); - graph::deep_copy(execution_space, destination, host); -} - -template -void copy_vector(const ExecutionSpace& execution_space, const View& destination, - const std::vector& source) { - if (destination.size() != source.size()) - throw std::invalid_argument("fixed problem array has the wrong extent"); - auto mirror = Kokkos::create_mirror_view(destination); - auto* data = mirror.data(); - for (std::size_t index = 0; index < source.size(); ++index) - data[index] = source[index]; - graph::deep_copy(execution_space, destination, mirror); -} - -void require_extent(const std::size_t actual, const std::size_t expected, - const char* name) { - if (actual != expected) - throw std::invalid_argument(std::string(name) + " has the wrong extent"); -} - -std::size_t checked_product(const std::size_t left, const std::size_t right, - const char* name) { - if (left != 0 && right > std::numeric_limits::max() / left) - throw std::length_error(std::string(name) + " extent overflows size_t"); - return left * right; -} - -std::size_t checked_sum(const std::size_t left, const std::size_t right, - const char* name) { - if (right > std::numeric_limits::max() - left) - throw std::length_error(std::string(name) + " extent overflows size_t"); - return left + right; -} - -void require_allocation(const std::size_t elements, const std::size_t element_bytes, - const char* name) { - static_cast(checked_product(elements, element_bytes, name)); -} - -void require_general_problem(const Problem& problem) { - require_general_modal(problem); - require_coupled_flow(problem); - require_dispersive(problem); - if (problem.n_omega == 0 || problem.n_t == 0 || problem.n_to == 0 || - problem.n_modes == 0 || problem.n_pol_state == 0 || problem.n_nodes == 0 || - problem.n_pol_transform == 0) - throw std::invalid_argument("all transform and FFT extents must be positive"); - const auto effective_node_pol = checked_product( - problem.n_pol_transform, problem.n_pol_state, "effective node polarisation"); - if (effective_node_pol > 2) - throw std::invalid_argument("at most two effective node components are supported"); - if (problem.node_chunk == 0 || problem.node_chunk > problem.n_nodes) - throw std::invalid_argument("node_chunk must lie in [1, n_nodes]"); - if (problem.n_stages == 0) - throw std::invalid_argument("the tableau must contain a stage"); - const auto columns = - checked_product(problem.n_modes, problem.n_pol_state, "field columns"); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - constexpr auto policy_limit = - static_cast(std::numeric_limits::max()); - if (problem.n_to > policy_limit || problem.n_t > policy_limit || - problem.n_omega > policy_limit || columns > policy_limit || - problem.node_chunk > policy_limit) - throw std::length_error("accelerator policy extent exceeds signed 32-bit range"); -#endif - const auto rows = - checked_product(problem.n_nodes, problem.n_pol_transform, "transform rows"); - const auto field = checked_product(columns, problem.n_omega, "field"); - const auto modal_time = checked_product(columns, problem.n_to, "modal time"); - const auto chunk_rows = - checked_product(problem.node_chunk, effective_node_pol, "node chunk rows"); - const auto node_time = checked_product(chunk_rows, problem.n_to, "node time"); - // The hybrid planar kind carries its x half alone, (n_x, n_p) and its transpose. - const auto transform = - problem.free_geometry() ? 0 - : problem.slab_geometry() - ? checked_product(problem.n_x, problem.n_p, "slab transform matrix") - : checked_product(rows, problem.n_modes, "transform matrix"); - const auto stage_field = checked_product(problem.n_stages, field, "stage field"); - require_allocation(field, sizeof(Complex), "field bytes"); - require_allocation(modal_time, sizeof(Complex), "modal-time bytes"); - require_allocation(node_time, sizeof(Complex), "node-time bytes"); - require_allocation(transform, sizeof(double), "transform-matrix bytes"); - if (!problem.projection_coarse_matrix.empty() && - problem.family != FieldFamily::envelope) { - const auto stacked_rows = - checked_product(2, problem.n_modes, "stacked projection rows"); - require_allocation( - checked_product(stacked_rows, problem.n_to, "stacked projection"), - sizeof(double), "stacked-projection bytes"); - } - require_allocation(stage_field, sizeof(Complex), "stage-field bytes"); - if (problem.plasma && !problem.envelope_plasma) { - require_allocation( - checked_product(chunk_rows, problem.n_omega, "held nodal spectrum"), - sizeof(Complex), "held nodal spectrum bytes"); - require_allocation(checked_product(2, modal_time, "held stacked projection"), - sizeof(double), "held stacked projection bytes"); - require_allocation( - checked_product(problem.n_modes, - checked_product(problem.node_chunk, problem.n_pol_transform, - "held projection rows"), - "held projection"), - sizeof(double), "held projection bytes"); - } - require_extent(problem.crop_index.size(), problem.n_omega, "crop_index"); - require_extent(problem.active_mask.size(), problem.n_omega, "active_mask"); - require_extent(problem.norm_mask.size(), problem.n_omega, "norm_mask"); - require_extent(problem.omega_window.size(), problem.n_omega, "omega_window"); - if (!problem.omega.empty()) - require_extent(problem.omega.size(), problem.n_omega, "omega"); - if (!problem.spatial_spectral_weights.empty()) { - require_extent(problem.spatial_spectral_weights.size(), problem.n_modes, - "spatial_spectral_weights"); - require_extent(problem.spatial_on_axis.size(), problem.n_modes, "spatial_on_axis"); - if ((!problem.slab_geometry() && problem.n_nodes != problem.n_modes) || - problem.n_pol_transform != 1) - throw std::invalid_argument( - "radial statistics require a scalar square transform"); - for (const double weight : problem.spatial_spectral_weights) - if (!std::isfinite(weight) || weight <= 0.0) - throw std::invalid_argument( - "spatial spectral weights must be finite and positive"); - } else if (!problem.spatial_on_axis.empty()) { - throw std::invalid_argument("spatial on-axis weights require the spectral measure"); - } - if (problem.slab_geometry()) { - // solprop's hybrid planar kind: columns (p, k_y), nodes (x, y), a scalar state, - // whole y rows per node chunk and no embedded coarse rule (solprop's reference - // scores no quadrature alarm on this kind). - if (problem.n_x < 1 || problem.n_y < 1 || - checked_product(problem.n_p, problem.n_y, "planar columns") != - problem.n_modes || - checked_product(problem.n_x, problem.n_y, "planar nodes") != problem.n_nodes || - problem.n_pol_transform != 1 || problem.n_pol_state != 1 || - problem.node_chunk % problem.n_x != 0 || problem.n_phi_terms != 1 || - problem.n_prefactor_terms != 1 || !problem.projection_coarse_matrix.empty()) - throw std::invalid_argument( - "the hybrid planar kind requires matching axes, a scalar state, whole y " - "rows per node chunk and scalar separable data"); - require_extent(problem.spatial_kappa_sq.size(), - checked_product(problem.n_p, problem.n_omega, "planar wall terms"), - "spatial_kappa_sq"); - for (const auto value : problem.spatial_kappa_sq) - if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) - throw std::invalid_argument("spatial_kappa_sq must be finite"); - } else if (!problem.spatial_kappa_sq.empty()) { - throw std::invalid_argument("spatial_kappa_sq belongs to the hybrid planar kind"); - } - // n_y == 1 is solprop's line uniform in y (grid.LineGrid): one y row carrying - // k_y = 0, transformed by the same two-axis plan with a unit extent. - if (problem.free_geometry() && - (problem.n_x < 2 || problem.n_y < 1 || - checked_product(problem.n_x, problem.n_y, "Cartesian shape") != - problem.n_modes || - problem.n_nodes != problem.n_modes || problem.n_pol_transform != 1 || - problem.n_phi_terms != 1 || problem.n_prefactor_terms != 1)) - throw std::invalid_argument( - "Cartesian geometry requires matching axes and scalar separable data"); - if (problem.separable_geometry()) { - if (!std::isfinite(problem.spatial_evanescent_rate) || - problem.spatial_evanescent_rate <= 0) - throw std::invalid_argument( - "Cartesian evanescent rate must be finite and positive"); - require_extent(problem.spatial_k_squared.size(), problem.n_omega, - "spatial_k_squared"); - require_extent(problem.spatial_k_squared_low.size(), problem.n_omega, - "spatial_k_squared_low"); - require_extent(problem.spatial_numerator.size(), problem.n_omega, - "spatial_numerator"); - require_extent(problem.spatial_clamped.size(), problem.n_omega, "spatial_clamped"); - require_extent(problem.spatial_frame.size(), problem.n_omega, "spatial_frame"); - require_extent(problem.spatial_spectral_rate.size(), problem.n_omega, - "spatial_spectral_rate"); - require_extent(problem.spatial_user_rate.size(), problem.n_omega, - "spatial_user_rate"); - require_extent(problem.spatial_kperp_sq.size(), problem.n_modes, - "spatial_kperp_sq"); - require_extent(problem.spatial_kperp_sq_low.size(), problem.n_modes, - "spatial_kperp_sq_low"); - require_extent(problem.spatial_absorber_rate.size(), problem.n_modes, - "spatial_absorber_rate"); - require_extent(problem.spatial_kperp_rate.size(), problem.n_modes, - "spatial_kperp_rate"); - for (const auto value : problem.spatial_k_squared) - if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) - throw std::invalid_argument("spatial_k_squared must be finite"); - for (const auto value : problem.spatial_k_squared_low) - if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) - throw std::invalid_argument("spatial_k_squared_low must be finite"); - for (const auto value : problem.spatial_numerator) - if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) - throw std::invalid_argument("spatial_numerator must be finite"); - for (const auto value : problem.spatial_clamped) - if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) - throw std::invalid_argument("spatial_clamped must be finite"); - for (const auto value : problem.spatial_frame) - if (!std::isfinite(value)) - throw std::invalid_argument("spatial_frame is invalid"); - for (const auto value : problem.spatial_spectral_rate) - if (!std::isfinite(value) || value < 0) - throw std::invalid_argument("spatial_spectral_rate is invalid"); - for (std::size_t bin = 0; bin < problem.n_omega; ++bin) - if (const double value = problem.spatial_user_rate[bin]; - !std::isfinite(value) || value < 0 || - value > problem.spatial_spectral_rate[bin]) - throw std::invalid_argument("spatial_user_rate is invalid"); - for (const auto value : problem.spatial_kperp_sq) - if (!std::isfinite(value) || value < 0) - throw std::invalid_argument("spatial_kperp_sq is invalid"); - for (const auto value : problem.spatial_kperp_sq_low) - if (!std::isfinite(value)) - throw std::invalid_argument("spatial_kperp_sq_low must be finite"); - for (const auto value : problem.spatial_absorber_rate) - if (!std::isfinite(value) || value < 0) - throw std::invalid_argument("spatial_absorber_rate is invalid"); - for (const auto value : problem.spatial_kperp_rate) - if (!std::isfinite(value) || value < 0) - throw std::invalid_argument("spatial_kperp_rate is invalid"); - if (problem.free_geometry()) { - // solprop SEP 23: the stored transverse axes the far-field statistics read. - require_extent(problem.spatial_k_x.size(), problem.n_x, "spatial_k_x"); - require_extent(problem.spatial_k_y.size(), problem.n_y, "spatial_k_y"); - for (const auto value : problem.spatial_k_x) - if (!std::isfinite(value)) - throw std::invalid_argument("spatial_k_x must be finite"); - for (const auto value : problem.spatial_k_y) - if (!std::isfinite(value)) - throw std::invalid_argument("spatial_k_y must be finite"); - } - } else if (problem.n_y != 0 || problem.n_p != 0) { - throw std::invalid_argument("Cartesian axes must be supplied together"); - } - if (!problem.free_geometry() && - (!problem.spatial_k_x.empty() || !problem.spatial_k_y.empty() || - problem.spatial_far_field_slots != 0)) - throw std::invalid_argument("the far-field statistics belong to free space"); - require_extent(problem.time_window.size(), problem.n_to, "time_window"); - if (problem.plasma && !problem.envelope_plasma) { - require_extent(problem.time_window_derivative.size(), problem.n_to, - "time_window_derivative"); - require_extent(problem.omega.size(), problem.n_omega, "held current omega"); - for (const auto value : problem.time_window_derivative) - if (!std::isfinite(value)) - throw std::invalid_argument("time_window_derivative is not finite"); - for (std::size_t index = 0; index < problem.n_omega; ++index) - if (problem.active_mask[index] != 0 && - (!(problem.omega[index] > 0.0) || !std::isfinite(problem.omega[index]))) - throw std::invalid_argument( - "held plasma current requires positive active omega"); - } - require_extent(problem.carrier.size(), problem.n_to, "carrier"); - require_extent(problem.synthesis_matrix.size(), transform, "synthesis_matrix"); - require_extent(problem.projection_matrix.size(), transform, "projection_matrix"); - if (!problem.projection_coarse_matrix.empty()) - require_extent(problem.projection_coarse_matrix.size(), transform, - "projection_coarse_matrix"); - // A z table's terms are slots its evaluation fills, so nothing crosses for them. - if ((problem.phi_table || problem.prefactor_table) && problem.separable_geometry()) - throw std::invalid_argument("a separable operator takes no z table"); - require_extent(problem.phi_terms.size(), - problem.separable_geometry() || problem.phi_table - ? 0 - : checked_product(problem.n_phi_terms, field, "phi_terms"), - "phi_terms"); - require_extent( - problem.prefactor_terms.size(), - problem.separable_geometry() || problem.prefactor_table - ? 0 - : checked_product(problem.n_prefactor_terms, field, "prefactor_terms"), - "prefactor_terms"); - require_extent(problem.tableau_matrix.size(), - checked_product(problem.n_stages, problem.n_stages, "tableau"), - "tableau_matrix"); - require_extent(problem.tableau_nodes.size(), problem.n_stages, "tableau_nodes"); - require_extent(problem.solution_weights.size(), problem.n_stages, "solution_weights"); - require_extent(problem.error_weights.size(), problem.n_stages, "error_weights"); - if (problem.in_stage_absorber) - require_extent(problem.absorber_fine.size(), problem.n_to, "absorber_fine"); - if (!problem.spatial_absorber_matrix.empty()) { - require_extent( - problem.spatial_absorber_matrix.size(), - checked_product(problem.n_modes, problem.n_modes, "spatial absorber"), - "spatial_absorber_matrix"); - if (!problem.in_stage_absorber || problem.n_pol_transform != 1 || - problem.n_nodes != problem.n_modes) - throw std::invalid_argument("spatial absorber requires scalar radial geometry"); - } - if (problem.split_step_absorber) - require_extent(problem.absorber_coarse.size(), problem.n_t, "absorber_coarse"); - const std::size_t fine_size = - problem.family == FieldFamily::field_real ? problem.n_to / 2 + 1 : problem.n_to; - if (problem.envelope_plasma && problem.family != FieldFamily::envelope) - throw std::invalid_argument("envelope plasma requires an envelope field family"); - std::vector seen(fine_size, false); - for (const auto index : problem.crop_index) { - if (index >= fine_size) - throw std::invalid_argument("crop_index lies outside the fine spectrum"); - if (seen[index]) - throw std::invalid_argument("crop_index contains a duplicate fine-spectrum bin"); - seen[index] = true; - } - if (problem.plasma && problem.n_channels == 0) - throw std::invalid_argument("plasma requires at least one channel"); - if (problem.n_channels > 1 || problem.envelope_plasma) { - if (!problem.plasma) - throw std::invalid_argument("channels require plasma"); - require_extent(problem.channel_parameters.size(), - checked_product(problem.n_channels, 9, "channel parameters"), - "channel_parameters"); - if (!std::ranges::all_of(problem.channel_parameters, - [](double value) { return std::isfinite(value); }) || - !std::ranges::all_of(problem.channel_coefficients, - [](double value) { return std::isfinite(value); })) - throw std::invalid_argument("channel parameters and coefficients must be finite"); - std::size_t expected_offset = 0; - // Each interval holds four coefficients. - const std::size_t total_intervals = problem.channel_coefficients.size() / 4; - for (std::size_t channel = 0; channel < problem.n_channels; ++channel) { - const double* row = problem.channel_parameters.data() + 9 * channel; - if (!(row[1] > 0.0 && row[2] > 0.0 && row[3] > row[2] && row[4] >= 0.0 && - row[5] >= 1.0 && row[5] == std::floor(row[5]) && - row[6] == static_cast(expected_offset) && row[7] > 0.0 && - row[8] > 0.0) || - row[5] > static_cast(total_intervals)) - throw std::invalid_argument( - "channel constants and coefficient ranges must be positive and consistent"); - expected_offset = checked_sum( - expected_offset, - checked_product(4, static_cast(row[5]), "channel table"), - "channel table offsets"); - } - require_extent(problem.channel_coefficients.size(), expected_offset, - "channel_coefficients"); - } - if (problem.raman && problem.n_raman_components == 0) - throw std::invalid_argument("Raman requires at least one component"); - if (problem.n_raman_components > 1) { - if (!problem.raman) - throw std::invalid_argument("Raman components require Raman"); - require_extent(problem.raman_row_component.size(), problem.n_raman_terms, - "raman_row_component"); - require_extent(problem.raman_component_fraction.size(), problem.n_raman_components, - "raman_component_fraction"); - for (double value : problem.raman_row_component) - if (!std::isfinite(value) || value < 0.0 || - value >= static_cast(problem.n_raman_components) || - value != std::floor(value)) - throw std::invalid_argument("Raman row component index is invalid"); - for (double value : problem.raman_component_fraction) - if (!std::isfinite(value) || value <= 0.0) - throw std::invalid_argument( - "Raman component fractions must be finite and positive"); - } - if (problem.plasma) { - if (!(problem.dt_fine > 0.0) || !(problem.d_log_field > 0.0) || - !(problem.field_min > 0.0) || !(problem.field_max > problem.field_min) || - !(problem.charge_mass_ratio > 0.0) || !(problem.ionisation_potential > 0.0) || - problem.rate_intervals == 0) - throw std::invalid_argument("plasma constants must be positive and consistent"); - require_extent(problem.rate_coefficients.size(), - checked_product(4, problem.rate_intervals, "rate coefficients"), - "rate_coefficients"); - } - // solprop SEP 18's running-integral rule. Translation certified the taps; only their - // extent, finiteness, the trapezoid's identity tap and the route are checked here. - if (problem.plasma_taps.empty() || problem.plasma_taps.size() > plasma_max_taps || - !std::ranges::all_of(problem.plasma_taps, - [](double value) { return std::isfinite(value); })) - throw std::invalid_argument("plasma_taps must hold one to nine finite taps"); - if (!problem.plasma_band && - (problem.plasma_taps.size() != 1 || problem.plasma_taps[0] != 1.0)) - throw std::invalid_argument("the trapezoid rule is the one identity tap"); - if (problem.plasma_gated && - (!problem.plasma || problem.envelope_plasma || problem.general_modal)) - throw std::invalid_argument( - "the gated plasma benchmark needs a field plasma column off the general route"); - if (problem.plasma_band) { - if (!problem.plasma || problem.envelope_plasma || - problem.family == FieldFamily::envelope) - throw std::invalid_argument( - "the band integration rule needs a field-grid plasma column"); - if (problem.n_to <= problem.plasma_half_width()) - throw std::invalid_argument( - "the band integration rule's filter needs more than K samples"); - } - if (problem.envelope_plasma) { - if (!problem.plasma || problem.family != FieldFamily::envelope || - problem.harmonic_factor != 0.0 || problem.raman_thg || problem.stash_plasma) - throw std::invalid_argument("envelope plasma requires separate envelope currents " - "without THG or field stashing"); - const auto rule_size = - checked_product(problem.n_channels, std::size_t{4} * 512, "envelope rules"); - require_extent(problem.envelope_rule_nodes.size(), rule_size, - "envelope_rule_nodes"); - require_extent(problem.envelope_rule_weights.size(), rule_size, - "envelope_rule_weights"); - for (std::size_t channel = 0; channel < problem.n_channels; ++channel) - for (std::size_t level = 0; level < 4; ++level) { - const auto order = std::size_t{64} << level; - double sum = 0.0; - double previous = -1.0; - for (std::size_t index = 0; index < 512; ++index) { - const auto offset = (channel * 4 + level) * 512 + index; - const auto node = problem.envelope_rule_nodes[offset]; - const auto weight = problem.envelope_rule_weights[offset]; - if (!std::isfinite(node) || !std::isfinite(weight) || - (index < order ? !(node > previous && node < 1.0 && weight > 0.0) - : node != 0.0 || weight != 0.0)) - throw std::invalid_argument( - "envelope quadrature nodes or weights are invalid"); - if (index < order) { - previous = node; - sum += weight; - } - } - if (std::abs(sum - 2.0) > - 8 * static_cast(order) * std::numeric_limits::epsilon()) - throw std::invalid_argument( - "envelope quadrature weights must integrate a constant"); - } - require_extent(problem.envelope_drude_factor.size(), problem.n_omega, - "envelope_drude_factor"); - require_extent(problem.envelope_loss_factor.size(), problem.n_omega, - "envelope_loss_factor"); - for (std::size_t index = 0; index < problem.n_omega; ++index) { - const auto drude = problem.envelope_drude_factor[index]; - const auto loss = problem.envelope_loss_factor[index]; - const bool active = problem.active_mask[index] != 0 && problem.omega[index] > 0.0; - if (!std::isfinite(drude) || !std::isfinite(loss.real()) || - !std::isfinite(loss.imag()) || loss.real() != 0.0 || - (active ? !(drude < 0.0 && loss.imag() < 0.0) - : drude != 0.0 || loss.imag() != 0.0)) - throw std::invalid_argument("envelope current factors require negative active " - "factors and zero inactive bins"); - } - } - if (problem.raman) { - if (effective_node_pol != 1 && !problem.raman_tensor_capable) - throw std::invalid_argument( - "vector Raman requires tensor-capable terms; found scalar_only"); - if (problem.n_raman_terms == 0) - throw std::invalid_argument("a Raman problem must contain at least one term"); - require_extent(problem.raman_tensor_weights.size(), - checked_product(problem.n_raman_terms, 2, "Raman tensor weights"), - "raman_tensor_weights"); - for (std::size_t term = 0; term < problem.n_raman_terms; ++term) { - const double trace = problem.raman_tensor_weights[2 * term]; - const double traceless = problem.raman_tensor_weights[2 * term + 1]; - if (!((trace == 1.0 && traceless == 0.0) || (trace == 0.25 && traceless == 0.75))) - throw std::invalid_argument( - "Raman tensor weights must be isotropic or rank two"); - } - const auto doubled = checked_product(2, problem.n_to, "Raman doubled grid"); - const auto raman_rows = - checked_product(problem.node_chunk, problem.raman_drives(), "Raman drive rows"); - require_allocation(checked_product(raman_rows, doubled, "Raman drive"), - sizeof(double), "Raman drive bytes"); - require_extent(problem.raman_undamped.size(), - checked_product(problem.n_raman_terms, doubled, "raman_undamped"), - "raman_undamped"); - if (!(problem.raman_fraction > 0.0) || !std::isfinite(problem.raman_fraction) || - !(problem.dt_fine > 0.0) || !std::isfinite(problem.dt_fine) || - !(problem.raman_epsilon0 > 0.0) || !std::isfinite(problem.raman_epsilon0)) - throw std::invalid_argument( - "Raman fraction, dt_fine and epsilon0 must be positive and finite"); - if (!std::ranges::all_of(problem.raman_undamped, - [](const double value) { return std::isfinite(value); })) - throw std::invalid_argument("raman_undamped must contain only finite values"); - } else if (problem.raman_thg || problem.n_raman_terms != 0 || - !problem.raman_undamped.empty() || !problem.raman_tensor_weights.empty() || - problem.raman_tensor_capable) { - throw std::invalid_argument("Raman data were supplied while Raman is disabled"); - } -} - -// The samples of one tile of the device trapezoid plasma's two-level scans (unused on -// a host build). -[[maybe_unused]] inline constexpr std::size_t plasma_tile_samples = 1024; - -// Descriptors borrow allocations retained by the member GeneralState owners. They -// contain no ownership or reference counts and are refreshed after state swaps. -struct UpdateColumn { - const Complex *current{}, *solution{}, *error{}, *whole{}; - Complex *advanced{}, *residual{}; - const unsigned char* in_norm{}; - double *old_partial{}, *new_partial{}, *error_partial{}; - // The state the solution is added to: `current` itself, or a coupled flow's - // whole-step action of it (solprop `accept_step(U u, ..., U k)`); the old norm is - // always taken on `current`. - const Complex* carried{}; -}; - -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -/** - * The fewest bins a host thread of a reduction team takes: below it the team's - * synchronisation costs more than the thread's share of the pass. - */ -constexpr std::size_t host_reduction_bins_per_thread = 2048; - -/** A host team per column, sharing the threads when the columns are fewer. */ -int host_reduction_team_size(const ExecutionSpace& execution, const std::size_t columns, - const std::size_t bins) { - const auto threads = static_cast(std::max(1, execution.concurrency())); - const auto per_column = - std::max(1, threads / std::max(1, columns)); - const auto by_bins = std::max(1, bins / host_reduction_bins_per_thread); - return static_cast(std::min(per_column, by_bins)); -} -#endif - -// The teams a column's norm reductions are split over on a device, each writing its -// own partial sums: one team per column fills a GPU only when the columns are many, so -// a one-node fibre's column takes up to `reduction_target_teams` teams of at least -// `reduction_min_bins` bins each. The host keeps one slot per column. The slots depend -// on the member's columns and bins alone, so a batch member and its solo run sum in one -// order. -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -constexpr std::size_t reduction_target_teams = 256; -constexpr std::size_t reduction_min_bins = 1024; -#endif -inline std::size_t reduction_slots(const std::size_t columns, const std::size_t n) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const std::size_t wanted = - (reduction_target_teams + std::max(columns, 1) - 1) / - std::max(columns, 1); - const std::size_t most = - std::max(1, (n + reduction_min_bins - 1) / reduction_min_bins); - return std::max(1, std::min(wanted, most)); -#else - static_cast(columns); - static_cast(n); - return 1; -#endif -} - -// The bins `[first, end)` of slot `slot` of `slots` over `n`. -KOKKOS_INLINE_FUNCTION void slot_range(const std::size_t n, const std::size_t slots, - const std::size_t slot, std::size_t& first, - std::size_t& end) { - const std::size_t width = (n + slots - 1) / slots; - first = Kokkos::min(n, slot * width); - end = Kokkos::min(n, first + width); -} - -// The update and its three binary64 norms in one pass. On the host one team per column, -// its threads splitting the bins when the columns are fewer than the threads; on a -// device `slots` teams per column, each over its own bins, writing its partial sums at -// offset `slot` from the column's partial pointers. The sums associate as the teams -// split them (solprop SEP 25), and the host reduces them. -template -void accept_update(const ExecutionSpace& execution, const std::size_t count, - const std::size_t n, const double dz, const Columns& columns, - const char* label, const std::size_t slots) { - using ReductionPolicy = Kokkos::TeamPolicy; - using ReductionMember = ReductionPolicy::member_type; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const int team_size = SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE; -#else - const int team_size = host_reduction_team_size(execution, count, n); -#endif - const ReductionPolicy reduction_policy(execution, static_cast(count * slots), - team_size); - launch::parallel_for( - label, reduction_policy, KOKKOS_LAMBDA(const ReductionMember& team) { - const auto league = static_cast(team.league_rank()); - const auto column = league / slots; - const auto slot = league % slots; - std::size_t first = 0; - std::size_t end = 0; - slot_range(n, slots, slot, first, end); - const auto values = columns(column); - NormSums sums; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(first), - static_cast(end)), - [&](const int index_value, NormSums& update) { - const auto index = static_cast(index_value); - const Complex candidate = - values.whole[index] * - (values.carried[index] + dz * values.solution[index]); - values.advanced[index] = candidate; - const Complex one_residual = - values.whole[index] * (dz * values.error[index]); - values.residual[index] = one_residual; - if (values.in_norm[index] == 0U) - return; - update.old_sum += squared_abs(values.current[index]); - update.new_sum += squared_abs(candidate); - update.error_sum += squared_abs(one_residual); - }, - Kokkos::Sum(sums)); - Kokkos::single(Kokkos::PerTeam(team), [&] { - values.old_partial[slot] = sums.old_sum; - values.new_partial[slot] = sums.new_sum; - values.error_partial[slot] = sums.error_sum; - }); - }); -} - -// The bins a host work item of the stage's entry kernel takes at a time: every bin's -// phase sum is the same sequence of rounded products and additions in term order as -// a per-bin loop, laid out so that the bins vectorise. -constexpr std::size_t stage_block = 64; -// The pinned staging buffers a state's per-trial scalars cross from (`upload_trial`). -constexpr std::size_t stage_staging_ring = 8; - -// The pointwise bodies of a stage over the coarse spectrum: its entry (the phase -// increment, the combination with the earlier stages, the lab frame, the noise and the -// pad) and its finish (prefactor, window, absorber, return to the interaction picture -// and accumulation). The entry kernel runs the finish of the stage before it in the -// same traversal when nothing lies between them (solprop SEP 25 section 6.7): both -// are pointwise over the bins, and the combination reads only its own bin's stages. -// How a stage's entry forms its propagator pair (`StageTerms::combine`). -enum class EntryPhase { general, lossless, repeated }; - -// The transcendental factors of a stage's combination at one bin: the phase's -// `cos b` and `sin b`, its magnitudes `e^a` and `e^-a` (one on a lossless problem), -// and the noise phase's cosine and sine. -struct EntryFactors { - double cosine = 0.0, sine = 0.0, magnitude = 1.0, inverse = 1.0; - double noise_cosine = 0.0, noise_sine = 0.0; -}; - -// A host work item's factors for its block of bins, formed by the vector-maths layer -// in one pass of each function (`StageTerms::entry_factors_block`); the factors a -// stage does not use stay zero. -struct EntryBlock { - double cosine[stage_block], sine[stage_block]; - double magnitude[stage_block], inverse[stage_block]; - double noise_cosine[stage_block], noise_sine[stage_block]; - - [[nodiscard]] EntryFactors at(const std::size_t bin) const { - return {.cosine = cosine[bin], - .sine = sine[bin], - .magnitude = magnitude[bin], - .inverse = inverse[bin], - .noise_cosine = noise_cosine[bin], - .noise_sine = noise_sine[bin]}; - } -}; - -struct StageTerms { - ComplexView2 current, fine, whole, solution, error, stage_phase, stage_inverse, noise; - ComplexView2 flow_input, flow_lab, flow_drive, general_scaled, general_source; - ComplexView2 projected, absorber; - ComplexView3 stages, phi_terms, prefactor_terms; - RealView tableau, phi_delta, phi_at, prefactor_weights, window; - IndexView crop, gaps; - ByteView active; - std::size_t gap_ranges = 0; - std::size_t n = 0, n_phi = 0, n_stages = 0, n_prefactor = 0, n_to = 0; - double inverse_n_to = 0.0, general_pad = 0.0; - bool add_noise = false, analytic = false, general = false, flow = false; - bool carried = false, keep_lab = false, zero_response = false; - bool in_stage_absorber = false; - // Every coarse bin is the fine bin of its own index (`crop` is the identity). - bool identity_crop = false; - - // The stage's phase increment `sum_t dPhi_t Phi_t` at one bin: strict (solprop SEP - // 25 section 4.2), each product and sum rounded in term order. A zero weight's term - // is skipped: its product with a finite term is a signed zero, which leaves the - // running sum (started at +0) unchanged, so the skip is exact. A z table's terms - // are its values at a trial's positions, weighed by +-1 and mostly zero (SEP 29 G1). - SOLPROP_NATIVE_STRICT_FUNCTION - KOKKOS_INLINE_FUNCTION Complex phase_sum(const std::size_t offset, - const std::size_t column, - const std::size_t index) const { - SOLPROP_NATIVE_STRICT_BLOCK - double real = 0.0; - double imag = 0.0; - for (std::size_t term = 0; term < n_phi; ++term) { - const double weight = phi_delta(offset + term); - if (weight == 0.0) - continue; - const Complex fixed = phi_terms(term, column, index); - const double scaled_real = weight * fixed.real(); - const double scaled_imag = weight * fixed.imag(); - real = real + scaled_real; - imag = imag + scaled_imag; - } - return {real, imag}; - } - - // The same sums over a block of bins, laid out so that the bins vectorise. - SOLPROP_NATIVE_STRICT_FUNCTION - KOKKOS_INLINE_FUNCTION void - phase_block(const std::size_t offset, const std::size_t column, - const std::size_t first, const std::size_t count, - double (&real)[stage_block], double (&imag)[stage_block]) const { - SOLPROP_NATIVE_STRICT_BLOCK - for (std::size_t bin = 0; bin < stage_block; ++bin) { - real[bin] = 0.0; - imag[bin] = 0.0; - } - for (std::size_t term = 0; term < n_phi; ++term) { - const double weight = phi_delta(offset + term); - if (weight == 0.0) - continue; - for (std::size_t bin = 0; bin < count; ++bin) { - const Complex fixed = phi_terms(term, column, first + bin); - const double scaled_real = weight * fixed.real(); - const double scaled_imag = weight * fixed.imag(); - real[bin] = real[bin] + scaled_real; - imag[bin] = imag[bin] + scaled_imag; - } - } - } - - // The noise's phase: strict (solprop SEP 25 section 4.2). - SOLPROP_NATIVE_STRICT_FUNCTION - KOKKOS_INLINE_FUNCTION Complex noise_phase(const std::size_t stage, - const std::size_t column, - const std::size_t index) const { - SOLPROP_NATIVE_STRICT_BLOCK - Complex sum{}; - for (std::size_t term = 0; term < n_phi; ++term) { - const double weight = phi_at(stage * n_phi + term); - // Exact, as in `phase_sum`. - if (weight != 0.0) - sum = strict_accumulate(sum, weight, phi_terms(term, column, index)); - } - return sum; - } - - // The factors of stage `stage`'s combination at one bin, one bin at a time (a - // device's entry; the host forms a block's together, `entry_factors_block`). The - // propagator `exp(phase)` and its inverse `exp(-phase)` share their cosine and sine, - // each magnitude is its own `exp`, and a lossless problem's are exactly one, taken - // without the calls (`combine`). - template - KOKKOS_INLINE_FUNCTION EntryFactors entry_factors(const std::size_t stage, - const std::size_t column, - const std::size_t index, - const Complex phase) const { - EntryFactors factors; - if constexpr (Mode != EntryPhase::repeated) - cosine_sine(phase.imag(), factors.cosine, factors.sine); - if constexpr (Mode == EntryPhase::general) { - factors.magnitude = Kokkos::exp(phase.real()); - factors.inverse = Kokkos::exp(-phase.real()); - } - if (add_noise) { - const Complex phase_at = noise_phase(stage, column, index); - factors.noise_cosine = Kokkos::cos(phase_at.imag()); - factors.noise_sine = Kokkos::sin(phase_at.imag()); - } - return factors; - } - -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // Whether the first `count` of a block's phases have real parts that are all zeros. - static bool lossless_block(const double (&real)[stage_block], - const std::size_t count) { - bool zero = true; - for (std::size_t bin = 0; bin < count; ++bin) - zero = zero && real[bin] == 0.0; - return zero; - } - - // A host work item's factors for its `count` bins from `first`, each function one - // pass of the vector-maths layer over the block (solprop ADR-059): the phases' sines - // and cosines, their magnitudes and inverse magnitudes, and the noise phases'. - template - [[maybe_unused]] void - entry_factors_block(const std::size_t stage, const std::size_t column, - const std::size_t first, const std::size_t count, - const double (&real)[stage_block], - const double (&imag)[stage_block], EntryBlock& block) const { - if constexpr (Mode != EntryPhase::repeated) - vmath::sincos(imag, block.sine, block.cosine, count); - if constexpr (Mode == EntryPhase::general) { - // A block whose phases' real parts are all (signed) zeros -- a lossy problem's - // loss often lies in a few bands -- has both magnitudes `exp(+-0)`, exactly one, - // without the calls. - if (lossless_block(real, count)) { - for (std::size_t bin = 0; bin < count; ++bin) { - block.magnitude[bin] = 1.0; - block.inverse[bin] = 1.0; - } - } else { - double negated[stage_block]; - for (std::size_t bin = 0; bin < count; ++bin) - negated[bin] = -real[bin]; - vmath::exp(real, block.magnitude, count); - vmath::exp(negated, block.inverse, count); - } - } - if (add_noise) { - double angle[stage_block]{}; - for (std::size_t bin = 0; bin < count; ++bin) - angle[bin] = noise_phase(stage, column, first + bin).imag(); - vmath::sincos(angle, block.noise_sine, block.noise_cosine, count); - } - } -#endif - - // The combination of stage `stage` at one bin, from the stage's phase increment and - // its factors (`entry_factors`). The propagator `exp(phase)` and its inverse - // `exp(-phase)`, which the finish applies, share their cosine and sine (solprop SEP - // 25 section 6.7): the angle's parts are even and odd, so `exp(-phase)` is - // `e^{-a} (cos b, -sin b)` with its own magnitude, never a reciprocal of the forward - // value. A coupled flow's residual reads the phase. A lossless problem - // (`EntryPhase::lossless`: every Phi term's real part exactly zero, so the phase's - // is a signed zero) takes both magnitudes as `exp(+-0)`, exactly one, without the - // calls. A lossless stage whose phase weights equal the stage before's - // (`EntryPhase::repeated`) has that stage's inverse still in place, and its - // propagator is that inverse's conjugate, `(cos b, -(-sin b))`, to the bit. - // Returns the propagator. - template - KOKKOS_INLINE_FUNCTION Complex combine(const std::size_t stage, const double dz, - const double solution_weight, - const double error_weight, - const std::size_t column, - const std::size_t index, const Complex phase, - const EntryFactors& factors) const { - Complex propagator; - if constexpr (Mode == EntryPhase::repeated) { - const Complex inverse = stage_inverse(column, index); - propagator = Complex(inverse.real(), -inverse.imag()); - } else if constexpr (Mode == EntryPhase::lossless) { - propagator = Complex(factors.cosine, factors.sine); - stage_inverse(column, index) = Complex(factors.cosine, -factors.sine); - } else { - propagator = - Complex(factors.magnitude * factors.cosine, factors.magnitude * factors.sine); - stage_inverse(column, index) = - Complex(factors.inverse * factors.cosine, -(factors.inverse * factors.sine)); - } - if (flow) - stage_phase(column, index) = phase; - Complex value{}; - if (carried) { - value = propagator * flow_input(column, index); - } else { - Complex accumulated{}; - for (std::size_t previous = 0; previous < stage; ++previous) - accumulated += - tableau(stage * n_stages + previous) * stages(previous, column, index); - value = propagator * (current(column, index) + dz * accumulated); - } - if (add_noise) - value += noise(column, index) * Complex(factors.noise_cosine, factors.noise_sine); - const double analytic_scale = - analytic && index > 0 && 2 * crop(index) < n_to ? 2.0 : 1.0; - fine(column, crop(index)) = analytic_scale * inverse_n_to * value; - if (keep_lab) - flow_lab(column, index) = value; - // The general route synthesises on the coarse grid: the pad scale and the - // analytic doubling go on the channel spectra, the 1/n_to on its transform. - if (general) - general_scaled(column, index) = - value * (general_pad * general_doubling(crop(index), n_to)); - if (zero_response && flow) { - flow_drive(column, index) = Complex{}; - } else if (zero_response) { - stages(stage, column, index) = Complex{}; - if (stage == 0) { - solution(column, index) = Complex{}; - error(column, index) = Complex{}; - } else { - solution(column, index) += solution_weight * Complex{}; - error(column, index) += error_weight * Complex{}; - } - } - return propagator; - } - - // The fine bins no coarse bin fills, positions `[begin, end)` of this column's - // list, zeroed for the inverse transform (which may destroy its input). The list is - // held as runs of adjacent bins, `gaps(3 r + {0, 1, 2})` the run's first position, - // first bin and length, so each run zeroes contiguously. - KOKKOS_INLINE_FUNCTION void zero_gaps(const std::size_t column, - const std::size_t begin, - const std::size_t end) const { - for (std::size_t run = 0; run < gap_ranges; ++run) { - const std::size_t position = gaps(3 * run); - const std::size_t bin = gaps(3 * run + 1); - const std::size_t length = gaps(3 * run + 2); - const std::size_t lower = begin > position ? begin : position; - const std::size_t upper = end < position + length ? end : position + length; - for (std::size_t gap = lower; gap < upper; ++gap) - fine(column, bin + (gap - position)) = Complex{}; - } - } - - // The finish of stage `stage` at one bin, with the inverse propagator its - // combination formed. - KOKKOS_INLINE_FUNCTION void finish(const std::size_t stage, - const double solution_weight, - const double error_weight, - const std::size_t column, - const std::size_t index) const { - // A zero weight is skipped: a z table's terms are its values at a trial's - // positions, one-hot weighted (SEP 29 G1). The sum's value is unchanged; at most - // the sign of an exactly zero sum could differ. - Complex prefactor{}; - for (std::size_t term = 0; term < n_prefactor; ++term) { - const double weight = prefactor_weights(stage * n_prefactor + term); - if (weight != 0.0) - prefactor += weight * prefactor_terms(term, column, index); - } - Complex value = - prefactor * window(index) * - (general ? general_source(column, index) : projected(column, crop(index))); - if (in_stage_absorber && active(index) != 0U) - value += absorber(column, crop(index)); - value *= stage_inverse(column, index); - if (flow) { - flow_drive(column, index) = value; - return; - } - stages(stage, column, index) = value; - if (stage == 0) { - solution(column, index) = solution_weight * value; - error(column, index) = error_weight * value; - } else { - solution(column, index) += solution_weight * value; - error(column, index) += error_weight * value; - } - } - -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // Whether a host work item takes its finish and combination as the block forms - // below: a stage that projects a source into the stage rows, off a coupled flow and - // the general route, whose stores the per-bin forms keep. - [[nodiscard]] bool blocks_vectorise() const noexcept { - return !flow && !general && !zero_response; - } - - // `finish` over the `count` bins of a block from `first`, each bin the same - // arithmetic in the same order, the block's parts held as separate real and - // imaginary lanes so that the bins vectorise; only the crop's gathers stay per bin. - [[maybe_unused]] void finish_block(const std::size_t stage, - const double solution_weight, - const double error_weight, - const std::size_t column, const std::size_t first, - const std::size_t count) const { - double real[stage_block]; - double imag[stage_block]; - double source_real[stage_block]; - double source_imag[stage_block]; - for (std::size_t bin = 0; bin < count; ++bin) { - real[bin] = 0.0; - imag[bin] = 0.0; - } - for (std::size_t term = 0; term < n_prefactor; ++term) { - const double weight = prefactor_weights(stage * n_prefactor + term); - if (weight == 0.0) - continue; - const Complex* values = &prefactor_terms(term, column, first); - for (std::size_t bin = 0; bin < count; ++bin) { - real[bin] += weight * values[bin].real(); - imag[bin] += weight * values[bin].imag(); - } - } - const std::size_t* bins = &crop(first); - const auto gather = [&](const Complex* row) { - if (identity_crop) - for (std::size_t bin = 0; bin < count; ++bin) { - source_real[bin] = row[first + bin].real(); - source_imag[bin] = row[first + bin].imag(); - } - else - for (std::size_t bin = 0; bin < count; ++bin) { - source_real[bin] = row[bins[bin]].real(); - source_imag[bin] = row[bins[bin]].imag(); - } - }; - gather(&projected(column, 0)); - const double* weights = &window(first); - for (std::size_t bin = 0; bin < count; ++bin) { - const double scaled_real = real[bin] * weights[bin]; - const double scaled_imag = imag[bin] * weights[bin]; - real[bin] = scaled_real * source_real[bin] - scaled_imag * source_imag[bin]; - imag[bin] = scaled_real * source_imag[bin] + scaled_imag * source_real[bin]; - } - if (in_stage_absorber) { - gather(&absorber(column, 0)); - const unsigned char* band = &active(first); - for (std::size_t bin = 0; bin < count; ++bin) { - const bool inside = band[bin] != 0U; - real[bin] = inside ? real[bin] + source_real[bin] : real[bin]; - imag[bin] = inside ? imag[bin] + source_imag[bin] : imag[bin]; - } - } - const Complex* inverse = &stage_inverse(column, first); - Complex* row = &stages(stage, column, first); - Complex* total = &solution(column, first); - Complex* estimate = &error(column, first); - for (std::size_t bin = 0; bin < count; ++bin) { - const double value_real = - real[bin] * inverse[bin].real() - imag[bin] * inverse[bin].imag(); - const double value_imag = - real[bin] * inverse[bin].imag() + imag[bin] * inverse[bin].real(); - row[bin] = Complex(value_real, value_imag); - if (stage == 0) { - total[bin] = - Complex(solution_weight * value_real, solution_weight * value_imag); - estimate[bin] = Complex(error_weight * value_real, error_weight * value_imag); - } else { - total[bin] = Complex(total[bin].real() + solution_weight * value_real, - total[bin].imag() + solution_weight * value_imag); - estimate[bin] = Complex(estimate[bin].real() + error_weight * value_real, - estimate[bin].imag() + error_weight * value_imag); - } - } - } - - // `combine` over the `count` bins of a block from `first`, from the block's factors, - // in the lanes of `finish_block`; with `store_whole`, each propagator is also the - // whole step's. - template - [[maybe_unused]] void - combine_block(const std::size_t stage, const double dz, const std::size_t column, - const std::size_t first, const std::size_t count, - const EntryBlock& factors, const bool store_whole) const { - double real[stage_block]; - double imag[stage_block]; - double sum_real[stage_block]; - double sum_imag[stage_block]; - Complex* inverse = &stage_inverse(column, first); - for (std::size_t bin = 0; bin < count; ++bin) { - if constexpr (Mode == EntryPhase::repeated) { - real[bin] = inverse[bin].real(); - imag[bin] = -inverse[bin].imag(); - } else if constexpr (Mode == EntryPhase::lossless) { - real[bin] = factors.cosine[bin]; - imag[bin] = factors.sine[bin]; - inverse[bin] = Complex(factors.cosine[bin], -factors.sine[bin]); - } else { - real[bin] = factors.magnitude[bin] * factors.cosine[bin]; - imag[bin] = factors.magnitude[bin] * factors.sine[bin]; - inverse[bin] = Complex(factors.inverse[bin] * factors.cosine[bin], - -(factors.inverse[bin] * factors.sine[bin])); - } - } - if (store_whole) { - Complex* propagator = &whole(column, first); - for (std::size_t bin = 0; bin < count; ++bin) - propagator[bin] = Complex(real[bin], imag[bin]); - } - for (std::size_t bin = 0; bin < count; ++bin) { - sum_real[bin] = 0.0; - sum_imag[bin] = 0.0; - } - for (std::size_t previous = 0; previous < stage; ++previous) { - const double weight = tableau(stage * n_stages + previous); - const Complex* earlier = &stages(previous, column, first); - for (std::size_t bin = 0; bin < count; ++bin) { - sum_real[bin] += weight * earlier[bin].real(); - sum_imag[bin] += weight * earlier[bin].imag(); - } - } - const Complex* state = ¤t(column, first); - for (std::size_t bin = 0; bin < count; ++bin) { - const double input_real = state[bin].real() + dz * sum_real[bin]; - const double input_imag = state[bin].imag() + dz * sum_imag[bin]; - const double value_real = real[bin] * input_real - imag[bin] * input_imag; - const double value_imag = real[bin] * input_imag + imag[bin] * input_real; - real[bin] = value_real; - imag[bin] = value_imag; - } - if (add_noise) { - const Complex* draw = &noise(column, first); - for (std::size_t bin = 0; bin < count; ++bin) { - const double noise_real = draw[bin].real() * factors.noise_cosine[bin] - - draw[bin].imag() * factors.noise_sine[bin]; - const double noise_imag = draw[bin].real() * factors.noise_sine[bin] + - draw[bin].imag() * factors.noise_cosine[bin]; - real[bin] += noise_real; - imag[bin] += noise_imag; - } - } - Complex* padded = &fine(column, 0); - const std::size_t* bins = &crop(first); - if (identity_crop && !analytic) { - for (std::size_t bin = 0; bin < count; ++bin) - padded[first + bin] = - Complex(inverse_n_to * real[bin], inverse_n_to * imag[bin]); - return; - } - for (std::size_t bin = 0; bin < count; ++bin) { - const double analytic_scale = - analytic && first + bin > 0 && 2 * bins[bin] < n_to ? 2.0 : 1.0; - const double scale = analytic_scale * inverse_n_to; - padded[bins[bin]] = Complex(scale * real[bin], scale * imag[bin]); - } - } -#endif -}; - -} // namespace - -// What a Cartesian plane's widths read of its fluence (`plane_fluence_moments`). -inline constexpr std::size_t plane_moment_slots = 8; - -class GeneralWorkspace { -public: - explicit GeneralWorkspace(Problem problem) : problem_(std::move(problem)) { - require_general_problem(problem_); - require_cpu_execution(problem_.cpu_execution); - identity_crop_ = problem_.crop_index.size() == problem_.n_omega; - for (std::size_t index = 0; identity_crop_ && index < problem_.n_omega; ++index) - identity_crop_ = problem_.crop_index[index] == index; - one_mode_products_ = one_mode_products_enabled && - problem_.family == FieldFamily::field_real && - problem_.n_modes == 1 && problem_.n_pol_state == 1 && - problem_.n_pol_transform == 1 && problem_.n_nodes > 1 && - problem_.node_chunk == problem_.n_nodes; - compact_ = compact_problem(problem_); - if (compact_) - for (auto* pool : {&kernel_pool_, &fft_pool_, &raman_pool_, &raman_tail_pool_}) - pool->compact(); - execution_space_ = - kernel_pool_.get(CpuOperation::stage, problem_.n_columns() * problem_.n_omega); - const auto nodes = problem_.node_chunk; - const auto response_elements = - nodes * problem_.effective_node_pol() * problem_.n_to; - response_execution_space_ = - kernel_pool_.get(CpuOperation::response, response_elements, problem_.n_to); - // On one thread the vendor GEMM's serial kernels are as fast. - one_mode_products_ = - one_mode_products_ && response_execution_space_.concurrency() > 1; - const auto tail = problem_.n_nodes % nodes; - response_tail_execution_space_ = kernel_pool_.get( - CpuOperation::response, tail * problem_.effective_node_pol() * problem_.n_to, - problem_.n_to); - const auto dense_elements = response_elements * problem_.n_modes; - const auto tail_dense_elements = - tail * problem_.effective_node_pol() * problem_.n_to * problem_.n_modes; - transform_execution_space_ = kernel_pool_.get(CpuOperation::dense, dense_elements); - transform_tail_execution_space_ = - kernel_pool_.get(CpuOperation::dense, tail_dense_elements); - temporal_execution_space_ = - kernel_pool_.get(CpuOperation::copy, problem_.n_columns() * problem_.n_to); - // Packing copies only matrix entries, and splitting copies only output samples; - // neither repeats the dense transform's inner product. - const auto matrix_elements = nodes * problem_.n_pol_transform * problem_.n_modes; - const auto tail_matrix_elements = - tail * problem_.n_pol_transform * problem_.n_modes; - matrix_copy_space_ = kernel_pool_.get(CpuOperation::copy, matrix_elements); - tail_matrix_copy_space_ = - kernel_pool_.get(CpuOperation::copy, tail_matrix_elements); - stacked_matrix_copy_space_ = - kernel_pool_.get(CpuOperation::copy, 2 * matrix_elements); - tail_stacked_matrix_copy_space_ = - kernel_pool_.get(CpuOperation::copy, 2 * tail_matrix_elements); - node_spectral_space_ = kernel_pool_.get( - CpuOperation::copy, nodes * problem_.effective_node_pol() * problem_.n_omega); - flow_execution_space_ = - kernel_pool_.get(CpuOperation::flow, problem_.flow_active * problem_.n_modes * - problem_.n_modes * problem_.n_modes); - if (!problem_.spatial_absorber_matrix.empty()) - absorber_execution_space_ = kernel_pool_.get( - CpuOperation::dense, problem_.n_columns() * problem_.n_to * problem_.n_modes); - // solprop SEP 46: the workspace's own FFT planning, wisdom isolated from the rest - // of the process in a calibration's modes. - const FftPlanningScope planning(problem_.fft_planning); - allocate(); - upload(); - const auto heap_before = heap_bytes(); - const auto device_before = device_free_bytes(); - const auto started = std::chrono::steady_clock::now(); - plan(); - execution_space_.fence("solprop_native planned workspace accounting"); - const auto heap_after = heap_bytes(); - const auto device_after = device_free_bytes(); - plan_heap_growth_ = heap_after > heap_before ? heap_after - heap_before : 0; - plan_device_growth_ = - device_before > device_after ? device_before - device_after : 0; - planning_seconds_ = - std::chrono::duration(std::chrono::steady_clock::now() - started) - .count(); - execution_space_.fence("solprop_native construction"); - } - - void set_initial(std::span field, std::span noise); - std::vector trial_step(double dz, const TrialScalars& scalars); - void prepare_trial(double dz, const TrialScalars& scalars); - std::vector adams_source(const std::vector& input, - const TrialScalars& scalars); - void adopt(const std::vector& field, double dz); - double low_storage_step(double dz, const TrialScalars& scalars, bool rebuild_table); - void low_storage_flows_formed(bool rebuild_table); - void low_storage_separable_pass(std::size_t pass, double dz); - void low_storage_separable_input(std::size_t pass); - void low_storage_separable_fold(std::size_t pass, int mode, int term, double dz); - void low_storage_modal_pass(std::size_t pass, double dz); - void low_storage_modal_input(std::size_t pass); - void adams_allocate(std::size_t slots, bool compensated, bool interpolate); - void adams_compensation_start(std::size_t slot, const std::vector& rate); - void adams_start(std::size_t slot, const TrialScalars& scalars); - double adams_magnitude(); - // The controller's norm scale on the device: rtol * w_i * |u| + atol over the - // in-band magnitude, formed each trial without the magnitude crossing to the host. - void adams_scale(double rtol, double atol, const std::vector& weights); - void adams_upload_divisor(const std::vector& divisor); - std::vector adams_fused_trial(double dz, const AdamsWeights& weights, - const std::vector& divisor, - const TrialScalars& scalars); - void adams_fused_accept(double dz, const AdamsWeights& keep, std::size_t free, - const AdamsWeights& dense); - void adams_hold(); - void adams_restore(); - void adams_observe(const AdamsWeights& weights, const TrialScalars& scalars); - std::vector - adams_exponential_trial(double dz, const ExponentialWeights& weights, - const std::vector& divisor, const TrialScalars& step, - const TrialScalars& sample, const std::vector& rate); - void adams_exponential_accept(double dz, std::size_t free, const AdamsWeights& dense); - // The end of an accepted Adams step (a host fence; nothing on a device). - void adams_accepted_boundary() const; - void adams_exponential_observe(double xi, double dz, - const ExponentialWeights& weights); - // solprop SEP 47: a Runge-Kutta march's dense output in a native session. - void rk_dense_allocate(const std::vector& weights); - void rk_dense_hold(double dz); - void rk_dense_observe(double theta, const TrialScalars& carry); - void rk_dense_restore(); - void rk_dense_release() noexcept; - std::vector adams_row_names() const; - std::array adams_row_extents(const std::string& name) const; - std::vector adams_rows_to_host(const std::string& name) const; - void adams_rows_to_host(const std::string& name, std::complex* destination, - std::size_t size) const; - void adams_rows_from_host(const std::string& name, - const std::vector& values); - std::vector collect_trial(); - void table_upload(TableKind kind, std::vector edges, double lower, - double upper, const std::complex* coefficients, - std::size_t n_intervals, std::size_t n_terms, std::size_t n_values); - void table_evaluate(const std::vector& slots, - const std::vector& positions); - void table_rate(std::size_t slot, double z, bool before); - std::vector table_terms_to_host(TableKind kind, std::size_t slot) const; - std::vector phase_sum_to_host(const std::vector& weights); - std::vector weighted_error(const std::vector& scale); - void set_flow_actions(const std::vector& forward, - const std::vector& inverse, - const std::vector& identity); - std::vector form_flow_actions(std::size_t section, double dz, - const std::vector& fractions); - std::vector flow_actions_to_host(bool inverse) const; - std::vector adams_block_to_host() const; - std::vector adams_exponent_to_host() const; - void prepare_raman(const std::vector& rates, - const std::vector& densities); - std::vector spatial_statistics(std::uint64_t flags, double intensity_factor, - double z); - std::vector plane_fluence_trace(double center_x, double center_y, double dx, - double dy, double extent, double spacing, - std::size_t n_radii, - const std::vector& cosines, - const std::vector& sines); - std::vector axis_spectrum(const std::vector& matrix); - std::vector - axis_populations(const std::vector& matrix, - const std::vector& channel_preionisation); - std::vector aperture_statistics(std::uint64_t flags, - const std::vector& apertures, - int exponent, - const std::vector& regions, - double intensity_factor); - std::vector statistics(std::uint64_t flags, - const std::vector& windows, - const std::vector& on_axis, double density, - double taper, double preionisation, - double intensity_factor, - const std::vector& channel_preionisation, - const std::vector& power_weight); - std::vector whitened_statistics(const std::vector& root, - std::uint64_t flags, - const std::vector& windows, - double intensity_factor, bool spectral); - void finish(double dz); - void promote_fsal(); - void clear_fsal(); - // The boundary monitor found the temporal collar quiet (solprop SEP 21). - void set_absorber_quiet(const bool quiet) noexcept { absorber_quiet_ = quiet; } - void bind_monitor(const std::vector& weights, - const std::vector& column_weights); - void unbind_monitor() noexcept { monitor_bound_ = false; } - void bind_occupancy(const std::vector& buckets, std::size_t count); - std::vector monitor_reading(); - void bind_edge(const std::vector& faces, const std::vector& clip, - const std::vector& totals, - const std::vector& synthesis, std::size_t first_row); - void bind_edge_lines(const EdgeLines& lines); - EdgeNumbers edge_reading(bool faces, bool whole, bool totals, bool strip, - bool want_current); - // What reading the edge holds in the state's memory space (solprop SEP 44). - [[nodiscard]] std::size_t edge_bytes() const noexcept { - std::size_t total = - (edge_faces_.size() + edge_clip_.size() + edge_totals_.size() + - edge_synthesis_.size() + edge_columns_.size() + edge_scale_.size()) * - sizeof(double); - for (const auto& line : edge_lines_) - total += line.positions.size() * sizeof(std::size_t) + - line.window.size() * sizeof(double); - return total + edge_strip_.size() * sizeof(Complex); - } - // The bytes through this state's crossings since it was built, each way (solprop SEP - // 44). - [[nodiscard]] CrossedBytes crossed_bytes() const noexcept { return crossed_; } - std::vector field_to_host() const; - void field_to_host(std::span destination) const; - const Complex* host_field() const { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - if constexpr (!static_cast( - Kokkos::SpaceAccessibility::accessible)) - return nullptr; - execution_space_.fence("solprop_native host_field"); - return current_.data(); - } - void count_to_host(const std::size_t bytes) const noexcept { - crossed_.to_host += bytes; - } - void cross_to_host(const ComplexView2& source, std::span destination) const; - void set_crossing_workers(const std::size_t workers) noexcept { - crossing_ring_.set_workers(workers); - } - std::vector stage_to_host(std::size_t stage) const; - void stage_to_host(std::size_t stage, std::span destination) const; - std::vector projected_to_host(bool coarse); - std::map dispersive_path() const; - std::map memory_ledger() const; - /** The team a compact problem's FFTW loops take (`CpuTeamScope`): its pools' - widest instance, which every threaded kernel and transform of it uses. Zero for - any other problem: the runtime's team. */ - [[nodiscard]] int cpu_team() const noexcept { - return compact_ ? std::max({kernel_pool_.widest(), fft_pool_.widest(), - raman_pool_.widest(), raman_tail_pool_.widest()}) - : 0; - } - const CpuExecution& cpu_execution() const noexcept { - return kernel_pool_.execution(); - } - CpuSizing execution_sizing() const { - auto result = kernel_pool_.sizing(); - result.merge(fft_pool_.sizing()); - result.merge(raman_pool_.sizing()); - result.merge(raman_tail_pool_.sizing()); - if (general_) - result.merge(general_->execution_sizing()); - return result; - } - std::map base_memory_ledger() const; - double planning_seconds() const noexcept { return planning_seconds_; } - double raman_planning_seconds() const noexcept { return raman_planning_seconds_; } - std::string raman_fft_provider() const { - return raman_drive_forward_plan_ ? raman_drive_forward_plan_->provider() : "none"; - } - std::map raman_fft_tasks() const { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const int full = 0; - const int tail = 0; - const int kernel = 0; - const int products = 0; -#else - const int full = raman_execution_space_.concurrency(); - const int tail = raman_tail_execution_space_.concurrency(); - const int kernel = raman_kernel_execution_space_.concurrency(); - const int products = raman_product_execution_space_.concurrency(); -#endif - // `products` is nonzero when the modal products replace the node drives. - return { - {"full", - raman_drive_forward_plan_ ? raman_drive_forward_plan_->tasks(full) : 0}, - {"tail", raman_tail_forward_plan_ ? raman_tail_forward_plan_->tasks(tail) : 0}, - {"kernel", - raman_kernel_forward_plan_ ? raman_kernel_forward_plan_->tasks(kernel) : 0}, - {"products", raman_product_forward_plan_ - ? raman_product_forward_plan_->tasks(products) - : 0}}; - } - std::size_t raman_kernel_transforms() const noexcept { - return raman_kernel_transforms_; - } - // The last accepted step's score, read from the device when asked for. - double quadrature_error() const; - // The largest score since the last call, then reset (NaN when none was scored). - double quadrature_peak(); - std::string synthesis_provider() const; - std::string projection_provider() const; - static std::string_view gemm_shape_policy(); - std::size_t response_team_size() const noexcept; - bool fsal_ready() const noexcept { return has_recycled_; } - std::size_t rk_source_evaluations() const noexcept { return rk_source_evaluations_; } - std::size_t axis_scans() const noexcept { return axis_scans_; } - bool closed() const noexcept { return closed_; } - void close() noexcept; - - // NVCC extended device lambdas require their enclosing member to be public. - // GeneralWorkspace is translation-unit-local, so this does not widen the API. - std::size_t fine_spectrum_size() const { - return problem_.family == FieldFamily::field_real ? problem_.n_to / 2 + 1 - : problem_.n_to; - } - std::size_t projected_spectrum_size() const { - return problem_.family == FieldFamily::envelope ? problem_.n_to - : problem_.n_to / 2 + 1; - } - void require_open() const { - if (closed_) - throw std::logic_error("native state is closed"); - } - void require_trial_scalars(const TrialScalars& scalars) const; - void require_adams() const; - void adams_phase(); - void adams_evaluate_source(const TrialScalars& scalars); - void adams_evaluate(const TrialScalars& scalars); - std::vector - adams_elementwise_trial(double dz, const ExponentialWeights& weights, - const std::vector& divisor, const TrialScalars& step, - const TrialScalars& sample, const std::vector& rate); - void adams_rate(const std::vector& rate, const ComplexView2& out); - void adams_fallback_squarings(std::size_t section, double dz, - const ExponentialWeights& weights); - // A trial's `rows` norms: its reduction over `policy` into the pinned buffer, then - // the trial's one fence, on the reduction's own instance. A reduction into memory the - // device writes returns before it completes. - template - std::vector adams_norms(const char* label, const Policy& policy, - const Reducer& reducer, const std::size_t rows) { - const Kokkos::View> - result(adams_norms_.data(), rows); - launch::parallel_reduce(label, policy, reducer, result); - policy.space().fence("solprop_native Adams trial boundary"); - std::vector norms(rows); - for (std::size_t row = 0; row < rows; ++row) - norms[row] = std::sqrt(result(row)); - adams_trial_ready_ = true; - return norms; - } - void allocate(); - void upload(); - void upload_trial(const TrialScalars& scalars, const TrialScalars* sample = nullptr, - const std::vector* rate = nullptr); - void use_uploaded_sample(const TrialScalars& sample); - void bind_stage_scalars(std::size_t slot); - void note_trial_scalars(const TrialScalars& scalars); - void plan(); - void reuse_fsal(); - void evaluate_stage(std::size_t stage, double dz); - // `accumulate` false: an Adams trial reads the stage row alone. - void evaluate_stage_source(std::size_t stage, double dz, bool accumulate = true); - bool modal_core(std::size_t stage); - void general_core(std::size_t stage); - StageTerms stage_terms(std::size_t stage) const; - std::vector> fine_gap_runs() const; - void stage_entry(std::size_t stage, double dz); - template void stage_entry_with(std::size_t stage, double dz); - void finish_or_defer(std::size_t stage); - // Free space and the hybrid planar kind (`Problem::separable_geometry`). - TransverseLayout transverse_layout() const noexcept { - const std::size_t plane = transverse_.extent(1) * transverse_.extent(2); - return problem_.slab_geometry() ? TransverseLayout{.inner = problem_.n_p, - .channel_fastest = true, - .plane = plane} - : TransverseLayout{.inner = problem_.n_x, - .channel_fastest = false, - .plane = plane}; - } - RegisterIndex register_index() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - constexpr bool plane = false; -#else - constexpr bool plane = true; -#endif - return {.layout = transverse_layout(), - .n_pol = problem_.n_pol_state, - .n = problem_.n_omega, - .plane = plane, - .scalar = problem_.n_pol_state == 1}; - } - // A stage's scratch positions: the volume route's own order on a device, or the - // scratch's (`register_index`). Every other pass over the scratch forms its own input - // in `(omega, y, x)`, since the scratch is dead between them. - RegisterIndex stage_index() const noexcept { - auto index = register_index(); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - index.volume = volume_order_; -#endif - return index; - } - // The unscaled transverse DFT's sample count: the plane's, or the slab's y axis. - double inverse_transverse_count() const noexcept { - return 1.0 / static_cast(problem_.slab_geometry() ? problem_.n_y - : problem_.n_modes); - } - std::size_t transverse_units() const noexcept { - return problem_.slab_geometry() ? problem_.n_y : problem_.n_nodes; - } - std::size_t transverse_units_per_chunk() const noexcept { - return problem_.slab_geometry() ? problem_.node_chunk / problem_.n_x - : problem_.node_chunk; - } - TransverseChunk transverse_chunk(std::size_t first_unit) const noexcept; - void transverse_forward(); - void transverse_inverse(); - void transverse_inverse(const ComplexView3& scratch); - void flux_at(double z); - void require_flux_closure(); - void fill_statistics_closure(); - void evaluate_separable_stage(std::size_t stage, double dz, bool accumulate); - void seed_free_wisdom(); - void seed_modal_wisdom(); - void seed_stream_wisdom(); - void separable_core(std::size_t stage, bool material); - void separable_spatial_core(); - void separable_volume_core(std::size_t stage); - void separable_stage_input(std::size_t stage, double dz); - void separable_load_current(); - void separable_pack_chunk(const TransverseChunk& chunk, bool material); - void separable_nodes(const TransverseChunk& chunk); - void slab_project(const TransverseChunk& chunk, const EnvelopeStream* stream); - void separable_finish_transform(std::size_t stage, bool material); - void separable_time_statistics(std::size_t axis_peak, std::size_t peak, bool fluence, - double intensity_factor); - void separable_fluence(double intensity_factor); - FarFieldPass::Inputs far_field_inputs() const; - void plane_fluence_moments(double (&moments)[plane_moment_slots]); - void scale_transform(std::size_t stage, double taper); - void synthesise(std::size_t first_node, std::size_t node_count); - void response(std::size_t stage, std::size_t first_node, std::size_t node_count); - void prepare_raman_stage(std::size_t stage, const double* rates, - const double* densities = nullptr); - void convolve_raman(std::size_t stage, std::size_t node_count); - void finish_raman(std::size_t stage, std::size_t node_count); - // A scalar state's Raman drive is a quadratic form of its modal fields, so the - // causal convolution is applied once per unordered modal product and each node's - // convolved drive is reconstructed from them (`raman_modal_products`). - [[nodiscard]] std::size_t raman_modal_products() const noexcept; - void convolve_raman_products(); - void finish_raman_products(std::size_t stage, std::size_t first_node, - std::size_t node_count); - template - void response_plasma(std::size_t stage, std::size_t first_node, - std::size_t node_count); - template - void response_nonlinear(std::size_t stage, std::size_t first_node, - std::size_t node_count); - template - void response_stash(std::size_t stage, std::size_t first_node, - std::size_t node_count); - template - void response_harmonic(std::size_t stage, std::size_t first_node, - std::size_t node_count); - template - void response_raman(std::size_t stage, std::size_t first_node, - std::size_t node_count); - template - void response_impl(std::size_t stage, std::size_t first_node, std::size_t node_count); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // The device's response (`device_response`) and its trapezoid plasma's two-level - // scans. - template - void device_response(std::size_t stage, std::size_t first_node, - std::size_t node_count); - template - void device_trapezoid_plasma(std::size_t stage, std::size_t first_node, - std::size_t node_count); -#endif - // solprop SEP 18's band rule as a team column (include/solprop_native/plasma.hpp): - // filter, then scan, for the population and free current, in whole-row passes. - // It writes the current separately from the collared non-plasma response. - template - void team_band_plasma_column(std::size_t stage, std::size_t first_node, - std::size_t node_count); - // The same column in tiles (solprop SEP 25 section 6.7.6, the device form): each - // running integral is a local kernel, one team per (row, tile) filtering and scanning - // its tile from zero, and a finish kernel adding the earlier tiles' totals and - // forming the pass's product. - template - void tiled_band_plasma_column(std::size_t stage, std::size_t first_node, - std::size_t node_count); - // The tiled column's teams: `rows` x tiles of them, as wide as the build's response - // team on a device (or `band_tile_threads_` on the host) and the kernel allows, with - // the tile's scratch when `scratch`. - template - [[nodiscard]] Kokkos::TeamPolicy - tiled_band_policy(std::size_t rows, bool scratch, const Functor& functor) const; - void tiled_band_local(const RealView2& source, std::size_t rows, const char* label); - // The band rule's first stage alone at every node of a chunk: the floored final - // exposure and largest rate per channel, and with `Diagnose` the six per-node parts - // of the run diagnostics; team scans with team reductions. - template - void team_band_statistics(std::size_t first_node, std::size_t node_count, - const RealView2& output, double preionisation); - // The band column's teams: on a device one per node, as wide as the build's team and - // the kernel's resources allow; on the host the threads shared between the nodes. - // Each team walks the nodes `slot, slot + league, ...` with its slot's rows. - template - [[nodiscard]] Kokkos::TeamPolicy - band_team_policy(std::size_t node_count, const Functor& functor) const; - [[nodiscard]] std::size_t band_league(std::size_t node_count) const; - [[nodiscard]] int band_host_team_size(std::size_t node_count) const; - void project(std::size_t first_node, std::size_t node_count, bool first_chunk, - const EnvelopeStream* stream = nullptr, - const FieldCurrentStream* field_stream = nullptr); - bool field_plasma() const noexcept { - return problem_.plasma && !problem_.envelope_plasma; - } - bool held_plasma() const noexcept { return field_plasma() && !problem_.plasma_gated; } - // solprop's private gated benchmark: the current's running integral, gated by the - // time window, joins the local polarisation before it is projected. - bool gated_plasma() const noexcept { return field_plasma() && problem_.plasma_gated; } - bool held_radial() const noexcept { - return held_plasma() && !problem_.separable_geometry() && - !problem_.spatial_spectral_weights.empty(); - } - // Whether a device takes a modal stage's forward transforms -- the projection's - // (fine and coarse), the in-stage absorber's and the held currents', each a real row - // of n_to samples -- as one batched transform at the end of the stage - // (`transform_forward_pack`). On an AMD Instinct MI325X each transform of a - // single-mode run was four rocFFT kernels of a few microseconds, launched one by one; - // on an NVIDIA A40 a transform of a four-mode capillary's rows took about 96 - // microseconds on 16 of its 84 multiprocessors, whatever the rows. Batched, the - // stage's rows share them. A device never defers the coarse quadrature - // (`deferred_quadrature_eligible`), so the coarse rows are projected at every stage. - // A host packs them too, unless the problem is compact or SOLPROP_NATIVE_FORWARD_PACK - // is 0 (solprop issue #584): one transform of every row, threaded by its shape, in - // place of one a transform over only the modes' rows, which left most of the - // threads idle: on 18 cores of an Intel Xeon Gold 6240 the pack ran rdw 0.90, - // argon_blueshift 0.88 and ctc 0.92 of their windows. A host keeps the coarse rows - // after the fine ones so that a deferred quadrature's stages transform the fine block - // alone. - bool packs_forward() const noexcept { -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // A compact problem keeps its separate transforms: its one row a transform runs - // serially below the FFT start, and the pack's few rows on the compact team ran - // averaged helium 1.35 times slower on 18 cores of an Intel Xeon Gold 6240 and - // 1.11 on 16 of an AMD EPYC 7763 (solprop issue #584). - if (!host_forward_pack() || compact_) - return false; -#endif - return problem_.family == FieldFamily::field_real && - !problem_.separable_geometry() && !problem_.general_modal && - !problem_.dispersive && !problem_.envelope_plasma && !held_radial() && - (problem_.in_stage_absorber || held_plasma() || - !problem_.projection_coarse_matrix.empty()); - } - static bool host_forward_pack() noexcept { - static const bool enabled = [] { - const char* text = std::getenv("SOLPROP_NATIVE_FORWARD_PACK"); - return text == nullptr || std::string_view(text) != "0"; - }(); - return enabled; - } - void transform_forward_pack(); - void held_response(std::size_t stage, std::size_t first_node, std::size_t node_count, - const TransverseChunk* chunk = nullptr); - void gated_response(std::size_t node_count); - void held_transform(bool coarse_only = false, bool transformed = false); - void finish_quadrature(); - bool deferred_quadrature_eligible() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return false; -#else - // The validated seven-stage FSAL tableau identifies the ordinary DOPRI5 - // route without another seam field. Other consumers retain eager spectra. - return problem_.first_same_as_last && problem_.n_stages == 7 && - !problem_.low_storage && !problem_.separable_geometry() && !general_ && - !flow_ && !dispersive_ && !problem_.envelope_plasma && !held_radial() && - !problem_.projection_coarse_matrix.empty(); -#endif - } - void held_radial_projection(std::size_t first_node, std::size_t node_count); - void envelope_cycle(std::size_t channel, std::size_t node_count, - const ComplexView2& field_rows, std::size_t n_pol); - void envelope_statistics(std::size_t channel, std::size_t first_node, - std::size_t node_count, const RealView2& statistics, - const ComplexView2& field, std::size_t n_pol); - void envelope_population(std::size_t channel, std::size_t stage, - std::size_t node_count); - void envelope_response(std::size_t stage, std::size_t first_node, - std::size_t node_count, - const TransverseChunk* chunk = nullptr); - void envelope_transform(); - void finish_stage(std::size_t stage); - void score_quadrature(); - void fill_statistics_time(bool windowed, double lower, double upper, - bool weighted = false); - void fill_statistics_pair(bool windowed, double lower, double upper); - void form_statistics_power(); - void form_statistics_total_power(); - void statistics_modal_time(); - void scan_plasma_statistics(std::size_t first_node, std::size_t node_count, - const RealView2& output, bool diagnose = false, - double preionisation = 0.0); - void form_absorber(); - void split_step(double dz, const ComplexView2& advanced); - void form_split_factor(double dz); - void split_columns(double dz, std::size_t active_rows = 0, double parseval = 0.0, - std::size_t first_column = 0); - void split_state_rows(double dz, const ComplexView2& state); - void add_split_removal(double dz, std::size_t active_rows, double parseval, - std::size_t first_column, const ComplexView2& complex_time, - double power_scale); - void separable_to_time(const TransverseChunk& chunk); - void separable_to_frequency(const TransverseChunk& chunk); - void separable_split_step(double dz, const ComplexView2& advanced); - void mask(const ComplexView2& values); - bool use_packed_radial_blas() const noexcept; - // A square radial analytic map synthesises the retained spectrum, before padding, - // and transforms each node (`retained_synthesis`). - bool retained_synthesis() const noexcept; - void synthesise_retained(std::size_t first_node, std::size_t node_count); - void packed_radial_gemm(const RealView2& matrix, const ComplexView2& source, - const ComplexView2& destination); - bool use_blas_for_synthesis() const noexcept; - bool use_interleaved_synthesis() const noexcept; - bool use_interleaved_projection() const noexcept; - bool hybrid_shape_eligible() const noexcept; - bool use_blas_for_projection() const noexcept; - bool use_chunked_blas_for_synthesis() const noexcept; - bool folds_single_node() const noexcept; - bool use_chunked_blas_for_projection() const noexcept; - [[nodiscard]] const ExecutionSpace& fft_space_for(std::size_t samples, - std::size_t length); - /** A batch's FFT instance, sized by its rows: extent 0 counts the rows, the - remaining extents make one row (a transverse plane counts as one row). */ - template - [[nodiscard]] const ExecutionSpace& fft_space_for_rows(const View& rows) { - return fft_space_for(rows.size(), row_length(rows)); - } - - Problem problem_; - // A compact problem's pools (`compact_problem`) and the team its FFTW loops take. - bool compact_{}; - // One real mode at several nodes in one chunk: synthesis and projection through - // the one-mode kernels on the response team (gemm.hpp). - bool one_mode_products_{}; - CpuExecutionPool kernel_pool_{CpuPool::kernels, problem_.cpu_execution}; - CpuExecutionPool fft_pool_{CpuPool::fft, problem_.cpu_execution}; - CpuExecutionPool raman_pool_{CpuPool::raman_fft, problem_.cpu_execution}; - CpuExecutionPool raman_tail_pool_{CpuPool::raman_tail_fft, problem_.cpu_execution}; - mutable ExecutionSpace execution_space_; - ExecutionSpace response_execution_space_, response_tail_execution_space_; - ExecutionSpace transform_execution_space_, transform_tail_execution_space_; - ExecutionSpace matrix_copy_space_, tail_matrix_copy_space_; - ExecutionSpace stacked_matrix_copy_space_, tail_stacked_matrix_copy_space_; - ExecutionSpace node_spectral_space_; - const ExecutionSpace& matrix_copy_space_for(std::size_t nodes, - bool stacked = false) const { - if (stacked) - return nodes == problem_.node_chunk ? stacked_matrix_copy_space_ - : tail_stacked_matrix_copy_space_; - return nodes == problem_.node_chunk ? matrix_copy_space_ : tail_matrix_copy_space_; - } - ExecutionSpace fft_execution_space_; - ExecutionSpace temporal_execution_space_, flow_execution_space_, - absorber_execution_space_; - // The exponential Adams trial's passes and its accepted step's rows, sized by - // adams_allocate when the history's rows are known. - ExecutionSpace adams_execution_space_, adams_copy_space_; - ExecutionSpace raman_multiply_space_, raman_tail_multiply_space_, - raman_rebuild_space_, raman_scale_space_; - ExecutionSpace raman_execution_space_, raman_tail_execution_space_, - raman_kernel_execution_space_; - ExecutionSpace raman_product_execution_space_, raman_product_space_; - const ExecutionSpace& response_space_for(std::size_t nodes) const { - return nodes == problem_.node_chunk ? response_execution_space_ - : response_tail_execution_space_; - } - const ExecutionSpace& transform_space_for(std::size_t nodes) const { - return nodes == problem_.node_chunk ? transform_execution_space_ - : transform_tail_execution_space_; - } - bool closed_{}; - bool initialized_{}; - // A constant operator's low-storage table is built (set on a rebuild, cleared by - // `set_initial` and `close`): a step that reuses a table never built refuses. - bool table_ready_{}; - bool stages_ready_{}; - bool has_noise_{}; - bool has_recycled_{}; - std::size_t rk_source_evaluations_{}; - // On-axis row scans (`axis_populations`), for the session's record (SEP 19). - std::size_t axis_scans_{}; - bool pending_fsal_{}; - // Only the final trial source supplies the accepted quadrature score. Its - // coarse time buffers survive until scoring, or until a new source replaces - // them. The stored accepted score and peak are independent of these flags. - bool defer_quadrature_{}; - bool project_coarse_{true}; - bool coarse_pending_{}; - // Set by a trial for a stage whose finish may run in the next stage's entry kernel; - // `finish_pending_` names the stage's finish still to run there. - bool defer_finish_{}; - bool finish_pending_{}; - // `problem_.crop_index` is the identity (`StageTerms::identity_crop`). - bool identity_crop_{}; - // The bound trial's tapers on the host, and the one the scaled transforms hold - // (`scale_transform`). - std::vector trial_taper_; - double scaled_taper_{}; - bool scaled_taper_ready_{}; - std::size_t pending_stage_{}; - bool stash_is_current_{}; - std::size_t plan_heap_growth_{}, plan_device_growth_{}; - double planning_seconds_{}; - double raman_planning_seconds_{}; - // Every fixed Phi term is purely imaginary: the stage phases have no real part. - bool lossless_phase_ = false; - // Per stage of the current trial, whether its phase weights equal the stage - // before's, and whether the whole step's equal the last stage's (`upload_trial`); - // the last stage whose entry formed the pair in this trial. - std::vector phase_repeats_; - bool whole_repeats_ = false; - std::size_t entered_stage_ = std::numeric_limits::max(); - std::size_t raman_kernel_transforms_{}; - bool raman_kernel_ready_{}; - std::vector cached_raman_rates_, cached_raman_densities_; - mutable double quadrature_error_{Kokkos::Experimental::quiet_NaN_v}; - bool adams_device_scale_ = false; - double adams_rtol_ = 0.0, adams_atol_ = 0.0; - RealView adams_scale_weights_; - Kokkos::View adams_magnitude_sum_; - // The device's (last, peak) scores: an accepted step scores the quadrature on the - // device and nothing crosses until the score is read. - RealView quadrature_scores_; - mutable bool quadrature_stale_ = false; - SpatialCoefficients spatial_; - // solprop SEP 34: the plane measure's coefficients. A separable run's are its own - // operator's views (aliased, nothing more held); a radial run's are the entrance - // squares, the material square replaced by the material z table's value at the - // statistic's plane when the medium varies (`flux_table_`). - SpatialCoefficients flux_; - z_table::Table flux_table_; - // The magnetic closure's transverse volume on a separable run, beside - // `transverse_`, allocated when a local flux statistic is asked for. - ComplexView3 flux_transverse_; - ComplexView3 transverse_; - std::unique_ptr transverse_forward_, transverse_inverse_; - // On a device, a free-space envelope whose material response is the Kerr term alone, - // on a time grid the spectrum fills (n_t = n_to = n_omega, the crop the identity), - // with one polarisation: the stage core transforms the whole scratch volume in - // (omega, y, x) at once and forms the response there, in place of the chunks' - // transposes and temporal transforms (`separable_volume_core`). - bool volume_kerr_ = false; - // The volume route's stages hold the scratch as `(y, x, omega)`, `volume_scratch_`, - // on a device (solprop #365): its rank-3 transform then keeps frequency contiguous - // (cuFFT's `{N,N,T}`, rocFFT's `{T,N,N}`), and its input and fold no longer cross - // layouts. - bool volume_order_ = false; - ComplexView3 volume_scratch_; - std::unique_ptr volume_forward_, volume_inverse_; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - ComplexView transform_work_; -#endif - std::unique_ptr slab_forward_, slab_inverse_; - ComplexView2 current_, advanced_, solution_, error_, residual_, recycled_, noise_; - // solprop SEP 24 phase 3: the low-storage march's registers B and C (the state is - // A) and its flows. - ComplexView2 register_accumulator_, register_stage_; - ComplexView3 low_storage_flows_; - ComplexView2 whole_propagator_, fine_spectrum_, projected_spectrum_; - // The stage's `sum_t dPhi_t Phi_t`, formed by the combination for a coupled flow's - // residual, and its inverse propagator `exp(-sum)`, which the finish applies. - ComplexView2 stage_phase_, stage_inverse_; - ComplexView2 projected_coarse_spectrum_, absorber_spectrum_, modal_complex_; - ComplexView2 statistics_spectrum_, statistics_time_, statistics_weighted_time_; - RealView2 statistics_power_weight_; - bool statistics_unit_weight_ = true; - ComplexView2 node_complex_, polarisation_complex_, projected_complex_; - ComplexView2 projected_coarse_complex_, absorber_complex_, coarse_spectrum_; - ComplexView2 coarse_time_complex_, scaled_synthesis_complex_; - ComplexView2 scaled_projection_complex_, scaled_projection_coarse_complex_; - ComplexView2 scaled_projection_stacked_complex_, projected_stacked_complex_; - ComplexView2 chunk_synthesis_complex_, tail_synthesis_complex_; - ComplexView2 chunk_projection_complex_, tail_projection_complex_; - ComplexView2 chunk_projection_stacked_complex_, tail_projection_stacked_complex_; - ComplexView2 node_raman_spectrum_, raman_kernel_spectrum_; - ComplexView2 raman_tail_spectrum_; - ComplexView2 spatial_absorber_complex_; - ComplexView3 stages_, phi_terms_, prefactor_terms_; - // solprop SEP 29 G1: the general z tables' coefficients, uploaded once; their values - // at a trial's positions are written into the term slots above. - z_table::Table phi_table_, prefactor_table_, rate_table_; - RealView2 radial_blas_input_, radial_blas_output_; - // The retained synthesis's coarse modal and node spectra. - ComplexView2 retained_modal_, retained_node_; - bool retained_synthesis_ = false; - RealView2 modal_real_, node_real_, polarisation_real_, projected_real_; - RealView2 projected_coarse_real_, projected_stacked_real_, absorber_real_; - RealView2 coarse_time_real_; - RealView2 node_raman_drive_, node_raman_convolved_, raman_undamped_; - RealView2 raman_tail_drive_, raman_tail_convolved_; - // The modal products' rows, the leading rows of the node drive and its spectrum: - // the drive rows hold each product, then its convolution. - RealView2 raman_product_drive_; - ComplexView2 raman_product_spectrum_; - std::size_t raman_products_ = 0; - RealView2 raman_kernel_time_, stage_raman_rates_, stage_raman_densities_; - RealView2 raman_tensor_weights_; - RealView raman_row_component_, raman_component_fraction_; - RealView2 synthesis_, projection_, projection_coarse_; - RealView2 spatial_absorber_real_; - RealView2 scaled_synthesis_, scaled_projection_, scaled_projection_coarse_; - RealView2 scaled_projection_stacked_; - RealView2 chunk_synthesis_, tail_synthesis_; - RealView2 chunk_projection_, tail_projection_; - RealView2 chunk_projection_stacked_, tail_projection_stacked_; - RealView2 partials_, quadrature_partials_, stash_candidate_, stash_current_; - RealView2 statistics_power_, plasma_statistics_scratch_; - // solprop SEP 34: each node's `|E| |G|` beside its signed flux, which bounds a radial - // fluence's rounding; allocated with the closure's buffers. - RealView2 statistics_magnitude_; - // `spatial_statistics_partials_` holds a plane's scaled fluence for its trace. - bool plane_fluence_held_ = false; - RealView2 plasma_rate_, plasma_exposure_, plasma_fraction_; - RealView2 plasma_current_integrand_, plasma_free_current_; - // The device trapezoid plasma's tile totals, by component row, and its tile count - // (`device_trapezoid_plasma`). - RealView2 plasma_tile_totals_; - std::size_t plasma_tiles_{}; - RealView2 plasma_outer_current_; - RealView weighted_scale_, weighted_partials_; - RealView spatial_spectral_weights_; - ComplexView spatial_on_axis_; - RealView2 spatial_statistics_partials_; - RealView statistics_total_power_; - RealView statistics_on_axis_, statistics_packed_; - // An envelope's on-axis row on a one-component node scan, its x and y components - // (`axis_populations`), allocated by the first statistics call that needs it. - ComplexView2 statistics_axis_row_; - // solprop SEP 47: a general modal basis's whitened columns, the metric's square - // root at one position and their sums, allocated by the first statistics call that - // needs them (outside any trial step). - ComplexView2 statistics_whitened_; - ComplexView statistics_root_; - RealView statistics_whitened_sums_; - RealView tableau_, omega_, omega_window_, time_window_, absorber_fine_, - absorber_coarse_; - RealView stage_phi_delta_, stage_phi_at_, stage_prefactor_; - RealView stage_density_, stage_taper_, stage_preionisation_, rate_coefficients_; - RealView stage_kerr_density_, stage_plasma_density_, stage_raman_density_; - // Every per-stage scalar view above, a Raman run's stage rates and amplitude - // densities and a compensated Adams trial's rate weights live in one buffer of two - // slots, uploaded in one copy a trial from the next of a ring of pinned host buffers - // (`upload_trial`, solprop SEP 39 section 6.7): an Adams trial's step scalars in - // slot 0 and its sample's in slot 1. The views point into the slot in use. - RealView stage_scalars_; - using StagingView = decltype(Kokkos::create_mirror_view( - Kokkos::SharedHostPinnedSpace{}, std::declval())); - std::vector stage_staging_; - std::size_t stage_staging_next_ = 0; - std::size_t stage_slot_size_ = 0; - std::array stage_offsets_{}; - RealView stage_rate_weights_; - RealView2 channel_parameters_, stage_channel_density_, stage_channel_preionisation_; - RealView channel_coefficients_; - Kokkos::View channel_scratch_; - // solprop SEP 18 on a CPU build: the running-integral rule's taps, one row of ring - // scratch per node of a chunk (the band column's, or the statistics scan's alone - // under the trapezoid) and the run diagnostics' six parts per node. - RealView plasma_taps_; - RealView2 plasma_diagnostics_; - // The band column's scratch rows per kind: one per concurrently running team. - std::size_t band_slots_{}; - // Whether the band rule runs the tiled column (`tiled_band_plasma_column`), its - // tile's samples, the tiles of a row, a host tile team's threads, and the tiles' - // totals by row. - bool band_tiled_{band_tiled(thread_policy())}; - // Each column's norm reductions on a device are this many teams (`reduction_slots`): - // the norm partials hold `columns * reduction_slots_` slots, reduced on the host. - std::size_t reduction_slots_{reduction_slots(problem_.n_columns(), problem_.n_omega)}; - std::size_t band_tile_{}; - std::size_t band_tile_count_{}; - int band_tile_threads_{band_tile_threads(thread_policy())}; - RealView2 band_tile_totals_; - ComplexView carrier_; - std::array envelope_streams_; - std::array held_streams_; - // A device's two held currents, fine and coarse rows of each, in one buffer and its - // spectrum, which one plan transforms (`held_transform`); empty on a host and on the - // radial route. - RealView2 held_pack_; - ComplexView2 held_pack_spectrum_; - std::unique_ptr held_pack_forward_; - // A device's forward rows of a stage, one buffer and its spectrum (`packs_forward`): - // the in-stage absorber's rows, then the projection's, then the held pack's, and the - // plans of all of them and of the rows after the absorber's (a quiet collar's stage). - RealView2 forward_pack_, forward_pack_tail_; - ComplexView2 forward_pack_spectrum_, forward_pack_tail_spectrum_; - std::unique_ptr forward_pack_plan_, forward_pack_tail_plan_; - std::size_t forward_pack_absorber_rows_ = 0; - // A host's deferred quadrature (`deferred_quadrature_eligible`): the pack's fine - // block (the absorber's, the projection's and the held currents' fine rows) with and - // without the absorber's, for the stages, and its coarse block for the finish. - RealView2 forward_pack_fine_, forward_pack_fine_tail_, forward_pack_coarse_; - ComplexView2 forward_pack_fine_spectrum_, forward_pack_fine_tail_spectrum_, - forward_pack_coarse_spectrum_; - std::unique_ptr forward_pack_fine_plan_, - forward_pack_fine_tail_plan_, forward_pack_coarse_plan_; - RealView2 held_current_, held_projection_; - // The gated benchmark's running integral of each current row, and the band rule's - // filtered rows (`gated_response`). - RealView2 gated_prefix_, gated_filtered_; - ComplexView2 held_nodal_spectrum_, held_projected_spectrum_; - RealView held_derivative_, held_inverse_; - RealView3 envelope_rule_nodes_, envelope_rule_weights_, envelope_moments_; - RealView2 envelope_rates_, envelope_exposure_; - IndexView envelope_failures_, envelope_status_; - Kokkos::View envelope_host_status_; - RealView envelope_drude_factor_; - ComplexView envelope_loss_factor_; - ByteView active_mask_, norm_mask_; - // solprop SEP 16 phase 3b: the resident Adams history and its trial's scratch. - ComplexView3 adams_store_, adams_slopes_, adams_moments_; - ComplexView2 adams_source_, adams_propagator_, adams_exponent_, adams_saved_; - ComplexView2 adams_held_, adams_dense_start_, adams_dense_end_, adams_dense_exponent_; - Kokkos::View adams_divisor_; - // A Runge-Kutta trial's norm partials cross through this mirror, pinned on a device - // (the view itself on a host build), so that no trial takes a fresh host buffer. The - // copy is asynchronous: it is read after the trial's fence. - decltype(Kokkos::create_mirror_view(Kokkos::SharedHostPinnedSpace{}, - std::declval())) pinned_partials_; - // A trial's squared norms, which its reduction writes and the host reads: in pinned - // host memory on a device, which the reduction's last block writes directly, so that - // no copy from the device follows it before the trial's one fence. - Kokkos::View adams_norms_; - // An elementwise trial's source evaluation, replayed as a device graph. - graph::Region adams_source_graph_; - // A varying operator's compensation: each sample's state and rate, the trial's - // carried rows, its predicted field, endpoint term and rate, and dense observation's - // terms. - ComplexView3 adams_states_, adams_rates_, adams_carried_, adams_dense_terms_; - ComplexView2 adams_lab_, adams_endpoint_, adams_endpoint_rate_; - Kokkos::View adams_rate_weights_; - // A coupled flow's exponential trial (solprop SEP 16 phase 6): the step's flow and - // moments on the block's bins, `(count + 1, n_active, n_modes, n_modes)` with the - // flow first; a frozen-residual flow's weights at each sample and at the trial's end. - Kokkos::View adams_block_; - // A host's channel vectors per bin, `(n_omega, (adams_max_samples + 1) n_modes)`, - // beside a block wider than `adams_max_channels` (solprop SEP 47 phase 4); empty - // otherwise, when a host task holds them on its stack. - ComplexView2 adams_channels_; - // Each build's functions of the bins off the flow's eigenbasis route, `(n_active, - // augmented_blocks(slots), n_modes, n_modes)` by active bin: the augmented - // exponential's first block row and its scratch for the trial's section (solprop - // SEP 47 phase 4); empty without a bin off the route. - Kokkos::View adams_fallback_; - // A frozen-residual flow's generator at each sample applied to its state, `G(z_j) - // u_j`, formed once per sample; the trial's at its end, and the weights it is formed - // from. A trial forms `R_s(z_j) u_j = G(z_j) u_j - G_s u_j` against its own section. - ComplexView3 adams_applied_; - ComplexView2 adams_end_applied_; - Kokkos::View adams_end_weights_; - std::size_t adams_dense_count_{}; - // solprop SEP 47: a Runge-Kutta step's continuous extension for dense output, its - // five coefficients `(5, columns, n_omega)`; the bins its flow changes by more than - // an e-fold; the committed state held across an observation; the tableau's - // dense-output weights; and whether a step is held. - ComplexView3 rk_dense_coefficients_; - Kokkos::View rk_dense_stiff_; - ComplexView2 rk_dense_held_; - std::vector rk_dense_weights_; - bool rk_dense_ready_{}; - bool adams_trial_ready_{}; - bool adams_trial_compensated_{}; - bool adams_trial_coupled_{}; - bool adams_dense_compensated_{}; - // The persistent row view a resume exports or installs, by name (SEP 29 G2). - ComplexView3 adams_row_view(const std::string& name) const; - IndexView crop_index_; - // The fine bins outside the crop, zeroed by the stage's entry kernel. - IndexView fine_gaps_; - // The separable core's fine bins outside the crop, zeroed per chunk instead of the - // whole fine spectrum (SEP 28); empty when the crop fills every fine bin. - IndexView separable_gaps_; - // Free space's node field and polarisation alias the chunk's time rows and their - // projected rows (SEP 28): the copies between them are identities. - bool separable_aliased_{}; - // The in-stage collar with no temporal rate beside it: its rate depends on the - // column alone, so its pass needs no temporal pair (solprop SEP 40 section 4.7). - bool spatial_only_absorber_{}; - std::size_t temporal_tail_rows_{}; - TailRowPlans temporal_tail_inverse_real_, - coarse_tail_inverse_real_; - TailRowPlans temporal_tail_forward_real_, - coarse_tail_forward_real_; - TailRowPlans - temporal_tail_inverse_complex_, temporal_tail_forward_complex_, - coarse_tail_inverse_complex_, coarse_tail_forward_complex_; - std::unique_ptr inverse_real_plan_; - std::unique_ptr inverse_complex_plan_; - std::unique_ptr forward_real_plan_; - std::unique_ptr forward_complex_plan_; - std::unique_ptr forward_coarse_real_plan_; - std::unique_ptr forward_coarse_complex_plan_; - std::unique_ptr statistics_inverse_plan_; - std::unique_ptr absorber_real_plan_; - std::unique_ptr absorber_complex_plan_; - std::unique_ptr coarse_inverse_real_plan_; - // A free-space envelope's split collar transforms the state's own rows in place, this - // many at a time, and holds no coarse buffers (`split_state_rows`). - std::size_t split_rows_{}; - std::unique_ptr split_inverse_plan_, split_forward_plan_; - std::unique_ptr split_tail_inverse_plan_, split_tail_forward_plan_; - std::unique_ptr coarse_inverse_complex_plan_; - std::unique_ptr coarse_forward_real_plan_; - std::unique_ptr coarse_forward_complex_plan_; - std::unique_ptr raman_drive_forward_plan_; - std::unique_ptr raman_convolution_inverse_plan_; - std::unique_ptr raman_tail_forward_plan_; - std::unique_ptr raman_tail_inverse_plan_; - std::unique_ptr raman_product_forward_plan_; - std::unique_ptr raman_product_inverse_plan_; - std::unique_ptr raman_kernel_forward_plan_; - // The retained synthesis's node transforms, full and tail chunks: from the leading - // rows of the modal time buffer, which then holds node spectra, to the node field. - std::unique_ptr retained_inverse_plan_, retained_tail_inverse_plan_; - // solprop's general modal route; null on every other route. - std::unique_ptr general_; - bool absorber_quiet_ = false; - // solprop's boundary monitor (SEP 21, ADR-057), bound for a run that watches its - // window: its four time weights `(4, n_t)` (low guard, high guard, the collar's - // 1 - W, the collar's rate), each state column's quadrature weight, and what the - // split-step collar took since the last reading beside one split's sums, the - // energies of what the last step's split found and the share it takes at each - // time. Allocated at the first bind, a few rows, and kept. - bool monitor_bound_ = false; - RealView2 monitor_weights_; - RealView monitor_column_weights_; - RealView monitor_share_, monitor_half_; - // A device's split factor per coarse sample, formed once an accepted step. - RealView split_factor_; - Kokkos::View monitor_removed_; - Kokkos::View monitor_split_sum_, monitor_found_; - // The split collar's walk-off bucket of each (mode, frequency) bin and the state's - // energy per bucket, which a reading returns for the crossing limit (solprop - // #440); empty unbound. - ByteView occupancy_buckets_; - std::size_t occupancy_count_ = 0; - mutable CrossedBytes crossed_; - // What the transverse edge's reading sums with, once a monitor is bound (SEP 44). - RealView2 edge_faces_, edge_clip_, edge_totals_, edge_synthesis_; - // A radial edge's node fields at the rows a read uses, real and imaginary parts as - // columns, when the state has one polarisation (`edge_reading`); empty otherwise. - RealView2 edge_nodes_; - RealView edge_columns_; - std::size_t edge_first_row_ = 0; - // A requested collar's lines, and one padded batch per distinct shape, kept across a - // runner's points. Native chooses none of what's in them. - struct EdgeLine { - std::size_t axis{}, nodes{}, rows{}, batch{}, first{}, last{}; - double factor{}, measure{}, step{}; - std::size_t onset{EdgeLines::no_onset}, before{}, after{}; - IndexView positions; - RealView window; - }; - struct EdgeBatch { - ComplexView2 buffer; - std::unique_ptr plan; - std::size_t heap_growth{}, device_growth{}; - }; - std::vector edge_lines_; - std::map, EdgeBatch> edge_batches_; - ComplexView edge_strip_; - RealView edge_scale_; - double edge_k_min_{}; - // A large crossing's pinned chunks, once one has been made (`crossing.hpp`). - mutable crossing::Ring crossing_ring_; - [[nodiscard]] bool monitoring_split() const noexcept { - return monitor_bound_ && problem_.split_step_absorber; - } - // Whether the stage adds the in-stage absorber's term. A quiet temporal collar is - // left out on the modal routes, as solprop's reference leaves it out, unless a - // real-space collar shares the term or the route is the general modal one or a - // coupled flow, which keep it (solprop SEP 21). - [[nodiscard]] bool absorbing() const noexcept { - return problem_.in_stage_absorber && - !(absorber_quiet_ && !general_ && !problem_.coupled_flow && - problem_.spatial_absorber_matrix.empty()); - } - // solprop's linear.coupled_flow; null otherwise. - std::unique_ptr flow_; - // solprop SEP 6's dispersive electronic response on the fused node path, and on the - // Cartesian free-space, line and planar routes; null otherwise. The general modal - // route's is its source's. - std::unique_ptr dispersive_; - // solprop SEP 23's far-field pass, on free space with spatial statistics. - std::unique_ptr far_field_; - std::unique_ptr separable_dispersive_; - // The response's layout and stage weights on whichever route runs it, or null. - DispersiveWeights* dispersive_weights() const noexcept { - if (dispersive_) - return &dispersive_->weights(); - if (separable_dispersive_) - return &separable_dispersive_->weights(); - if (general_ && general_->dispersive() != nullptr) - return &general_->dispersive()->weights(); - return nullptr; - } -}; - -// The hybrid policy's shape guard. A host build sends every eligible transform to the -// vendor GEMM: measured on Apple's Accelerate with one thread, it beat the custom -// kernel at every shape tried -- a radial field's transforms 1.1 to 1.5 times faster -// at 2 to 6 nodes and 1.6 to 35 times at 16 to 256 nodes and 256 to 2048 samples, a -// one-mode field capillary's 2.4 to 3.1 times -- and lost at none. A device build -// takes the vendor GEMM from eight modes (with eight nodes and 1024 samples): below -// that the product is a few rows deep, which the custom kernels take in one pass over -// the samples. On an NVIDIA A40 a four-mode capillary's stacked projection (8 rows, -// 36 nodes, 10240 samples) took 71 us in cuBLAS and its synthesis 27 us; at 256 -// radial modes the vendor GEMM runs near the FP64 peak. -bool GeneralWorkspace::hybrid_shape_eligible() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return problem_.n_modes >= 8 && problem_.n_nodes >= 8 && problem_.n_to >= 1024; -#else - return true; -#endif -} - -bool GeneralWorkspace::use_packed_radial_blas() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return false; -#else - // Two packed work arrays add two complex-equivalent time columns to the - // shared conservative twenty-column inventory. Decide before allocation. - constexpr std::size_t max_samples = - std::size_t{64} * 1024 * 1024 / (22 * sizeof(Complex)); - return !problem_.separable_geometry() && !problem_.spatial_spectral_weights.empty() && - problem_.n_modes == problem_.n_nodes && - problem_.node_chunk == problem_.n_nodes && problem_.n_pol_state == 1 && - problem_.n_pol_transform == 1 && problem_.family == FieldFamily::envelope && - problem_.projection_coarse_matrix.empty() && problem_.n_nodes >= 256 && - problem_.n_to >= 256 && problem_.n_nodes <= max_samples / problem_.n_to; -#endif -} - -// The retained-spectrum radial synthesis, on a host and a device. A real static -// synthesis commutes with the temporal transform: synthesising the coarse bins the -// stage entry fills and padding each node's spectrum gives the nodes' fields, so the -// product covers n_omega complex columns rather than the n_to complex samples of the -// padded, oversampled analytic signal. On a square map the node transforms replace the -// modal ones one for one; the held radial plasma's projection already contracts after -// cropping (`held_radial_projection`). An in-stage absorber or a Raman drive reads the -// modal time field, and a dispersive response its own synthesis, so they keep the -// time-domain product; skinny capillary maps would multiply the transforms. -bool GeneralWorkspace::retained_synthesis() const noexcept { - return problem_.family == FieldFamily::field_analytic && - !problem_.spatial_spectral_weights.empty() && !problem_.separable_geometry() && - !problem_.general_modal && !problem_.dispersive && - !problem_.in_stage_absorber && !problem_.raman && - problem_.n_modes == problem_.n_nodes && problem_.n_nodes > 1 && - problem_.n_pol_state == 1 && problem_.n_pol_transform == 1 && - problem_.n_omega < problem_.n_to; -} - -void GeneralWorkspace::synthesise_retained(const std::size_t first_node, - const std::size_t node_count) { - Kokkos::Profiling::pushRegion("solprop_native.transform.synthesis.retained"); - const auto crop = crop_index_; - const auto n = problem_.n_omega; - // The coarse modal spectra once per evaluation, at its first chunk (the stage core - // and the statistics' synthesis both start at node zero). Rows run in parallel at - // the modal copies' width (`temporal_execution_space_`), each a contiguous pass. - if (first_node == 0) { - const auto fine = fine_spectrum_; - const auto modal = retained_modal_; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // A device takes a thread per element. - launch::parallel_for( - "solprop_native.transform.retained_gather", - Range2(execution_space_, {0, 0}, {problem_.n_modes, n}), - KOKKOS_LAMBDA(const std::size_t mode, const std::size_t index) { - modal(mode, index) = fine(mode, crop(index)); - }); -#else - launch::parallel_for( - "solprop_native.transform.retained_gather", - Kokkos::RangePolicy(temporal_execution_space_, 0, - problem_.n_modes), - KOKKOS_LAMBDA(const std::size_t mode) { - for (std::size_t index = 0; index < n; ++index) - modal(mode, index) = fine(mode, crop(index)); - }); -#endif - } - using Interleaved = Kokkos::View>; - static_assert(sizeof(Complex) == 2 * sizeof(double)); - const Interleaved synthesis(scaled_synthesis_.data() + first_node * problem_.n_modes, - node_count, problem_.n_modes); - const Interleaved input(reinterpret_cast(retained_modal_.data()), - problem_.n_modes, 2 * n); - const Interleaved output(reinterpret_cast(retained_node_.data()), node_count, - 2 * n); - engine_gemm(transform_space_for(node_count), synthesis, input, output, 0.0); - // Each node's padded spectrum in the modal time buffer's leading rows, which are - // free as no modal transform runs on this path. Only the retained bins are - // written: the others were zeroed when the plans were made, and an out-of-place - // complex transform leaves its input as it was. - const auto node = retained_node_; - const auto padded = modal_complex_; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // A device zeroes the chunk's padded rows first, a fill at its bandwidth, rather - // than relying on the modal buffer's other bins staying zero, and pads a thread per - // element. - graph::deep_copy(execution_space_, - Kokkos::subview(padded, std::make_pair(std::size_t{0}, node_count), - Kokkos::ALL()), - Complex{}); - launch::parallel_for( - "solprop_native.transform.retained_pad", - Range2(execution_space_, {0, 0}, {node_count, n}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { - padded(row, crop(index)) = node(row, index); - }); -#else - launch::parallel_for( - "solprop_native.transform.retained_pad", - Kokkos::RangePolicy(temporal_execution_space_, 0, node_count), - KOKKOS_LAMBDA(const std::size_t row) { - for (std::size_t index = 0; index < n; ++index) - padded(row, crop(index)) = node(row, index); - }); -#endif - const bool tail = node_count != problem_.node_chunk; - const auto rows = std::make_pair(std::size_t{0}, node_count); - graph::execute(tail ? *retained_tail_inverse_plan_ : *retained_inverse_plan_, - Kokkos::subview(modal_complex_, rows, Kokkos::ALL()), - Kokkos::subview(node_complex_, rows, Kokkos::ALL()), - KokkosFFT::Normalization::none); - Kokkos::Profiling::popRegion(); -} - -void GeneralWorkspace::packed_radial_gemm(const RealView2& matrix, - const ComplexView2& source, - const ComplexView2& destination) { - const auto input = radial_blas_input_; - const auto output = radial_blas_output_; - const auto rows = problem_.n_nodes; - const auto times = problem_.n_to; - Kokkos::Profiling::pushRegion("solprop_native.radial.packed_real_gemm"); - launch::parallel_for( - "solprop_native.radial.pack", - Range2(temporal_execution_space_, {0, 0}, {rows, times}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { - input(row, 2 * time) = source(row, time).real(); - input(row, 2 * time + 1) = source(row, time).imag(); - }); - engine_gemm(transform_execution_space_, matrix, input, output, 0.0); - launch::parallel_for( - "solprop_native.radial.unpack", - Range2(temporal_execution_space_, {0, 0}, {rows, times}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { - destination(row, time) = - Complex(output(row, 2 * time), output(row, 2 * time + 1)); - }); - Kokkos::Profiling::popRegion(); -} - -bool GeneralWorkspace::use_blas_for_synthesis() const noexcept { - if (use_packed_radial_blas()) - return false; - // The hybrid planar kind contracts its small x rule per y row natively. - if (problem_.separable_geometry()) - return false; - if (problem_.n_pol_state != 1) - return false; -#if !defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) && \ - !defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) && \ - !defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) - // Without a vendor BLAS an envelope keeps the custom kernels. - if (problem_.family == FieldFamily::envelope) - return false; -#endif - if (problem_.family != FieldFamily::envelope && - problem_.family != FieldFamily::field_real) - return false; -#if defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) - // Spatial envelope matrices can be large even with a short temporal grid. - if (!problem_.spatial_spectral_weights.empty() && - problem_.family == FieldFamily::envelope && - problem_.projection_coarse_matrix.empty() && - problem_.n_modes == problem_.n_nodes && problem_.n_nodes >= 256 && - problem_.n_to >= 256) - return true; -#endif - return hybrid_shape_eligible(); -} - -// An analytic field's or an envelope's synthesis is a real matrix times complex -// samples: viewed as real numbers, its samples are a real matrix twice as wide, so the -// transform is one real GEMM through the selected BLAS, with no copy and no complex -// BLAS. An envelope's projection is the same (`use_interleaved_projection`). -// -// A device takes it at the shapes where it already takes BLAS for the projection -// (`hybrid_shape_eligible`): there the custom synthesis kernel cost 4.6 times the -// projection's GEMM at 512 radial nodes (an AMD CDNA 3 device, 2026-10-06). -bool GeneralWorkspace::use_interleaved_synthesis() const noexcept { - if (problem_.family == FieldFamily::field_real || problem_.separable_geometry() || - problem_.n_pol_state != 1 || problem_.n_pol_transform != 1) - return false; - return hybrid_shape_eligible(); -} - -bool GeneralWorkspace::use_interleaved_projection() const noexcept { - if (problem_.family != FieldFamily::envelope || problem_.separable_geometry() || - problem_.n_pol_state != 1 || problem_.n_pol_transform != 1) - return false; - return hybrid_shape_eligible(); -} - -bool GeneralWorkspace::use_blas_for_projection() const noexcept { - if (use_packed_radial_blas()) - return false; - // The hybrid planar kind contracts its small x rule per y row natively. - if (problem_.separable_geometry()) - return false; - if (problem_.n_pol_state != 1) - return false; -#if !defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) && \ - !defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) && \ - !defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) - // Without a vendor BLAS an envelope keeps the custom kernels. - if (problem_.family == FieldFamily::envelope) - return false; -#endif -#if defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) - // Spatial envelope matrices can be large even with a short temporal grid. - if (!problem_.spatial_spectral_weights.empty() && - problem_.family == FieldFamily::envelope && - problem_.projection_coarse_matrix.empty() && - problem_.n_modes == problem_.n_nodes && problem_.n_nodes >= 256 && - problem_.n_to >= 256) - return true; -#endif - return hybrid_shape_eligible(); -} - -bool GeneralWorkspace::use_chunked_blas_for_synthesis() const noexcept { - return problem_.node_chunk < problem_.n_nodes && use_blas_for_synthesis(); -} - -// One node, one mode and one scalar component: the synthesis and the projection are -// each one product, which the response column forms itself (solprop SEP 25 section -// 6.7), in place of the stage's taper scaling, two one-element GEMMs and the zero -// fill of the projection. Each folded product is the GEMM's own one rounding, -// `(s * taper) * x` and `(p / taper) * y`. -// The host's columns only: a device keeps its own column kernels. -bool GeneralWorkspace::folds_single_node() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return false; -#else - return !general_ && !dispersive_ && !held_radial() && !gated_plasma() && - !problem_.separable_geometry() && problem_.n_modes == 1 && - problem_.n_nodes == 1 && problem_.node_chunk == 1 && - problem_.n_pol_state == 1 && problem_.n_pol_transform == 1 && - problem_.projection_coarse_matrix.empty() && !problem_.raman && - !problem_.envelope_plasma && radial_blas_input_.empty(); -#endif -} - -bool GeneralWorkspace::use_chunked_blas_for_projection() const noexcept { - return problem_.node_chunk < problem_.n_nodes && use_blas_for_projection(); -} - -std::string GeneralWorkspace::synthesis_provider() const { - const auto provider = selected_blas_provider_name(); - if (folds_single_node()) - return "folded"; - if (use_packed_radial_blas()) - return std::string(provider) + "-packed-real"; - if (problem_.free_geometry()) - return "kokkos-fft"; - if (retained_synthesis_) - return std::string(provider) + "-retained-interleaved-real"; - if (use_interleaved_synthesis()) - return std::string(provider) + "-interleaved-real"; - return std::string(use_blas_for_synthesis() ? provider : std::string_view("custom")); -} - -std::string GeneralWorkspace::projection_provider() const { - const auto provider = selected_blas_provider_name(); - if (folds_single_node()) - return "folded"; - if (use_packed_radial_blas()) - return std::string(provider) + "-packed-real"; - if (problem_.free_geometry()) - return "kokkos-fft"; - if (use_interleaved_projection()) - return std::string(provider) + "-interleaved-real"; - return std::string(use_blas_for_projection() ? provider : std::string_view("custom")); -} - -std::string_view GeneralWorkspace::gemm_shape_policy() { - return "host radial scalar envelope uses packed real BLAS for whole transforms " - "with n_nodes/n_to >= 256 and a 22-column inventory within 64 MiB; " - "device radial scalar envelope uses vendor complex BLAS with " - "n_nodes/n_to >= 256; otherwise hybrid uses KokkosBlas for every whole " - "transform on the host, and on a device for those with n_modes >= 8, " - "n_nodes >= 8 and n_to >= 1024: real ones directly, an analytic field's " - "synthesis as one real GEMM over its samples' interleaved real view, and its " - "projection; host radial analytic synthesis also uses unallocated row views " - "for partial chunks; an envelope's synthesis and projection the same way; " - "OpenBLAS, " - "cuBLAS and rocBLAS builds also use " - "preconverted complex GEMM for eligible envelope transforms; eligible chunked " - "transforms are packed into persistent buffers for vendor BLAS; vector state " - "uses custom; a host build without a vendor BLAS runs KokkosBlas's native " - "arithmetic through a tiled host kernel, bit for bit; a compute capability " - "8.6 build forms the real GEMMs with a device kernel in place of cuBLAS"; -} - -std::size_t GeneralWorkspace::response_team_size() const noexcept { - constexpr auto configured = static_cast(SOLPROP_NATIVE_DEVICE_TEAM_SIZE); -#ifdef KOKKOS_ENABLE_CUDA - // Measured on a Hopper GPU, the crossover is sharp: 4,096-sample fibre problems - // favour teams of 256, while 8,192 samples and more favour 512. Keep an explicitly - // configured smaller team intact. - if (problem_.n_to < 8192) - return std::min(configured, std::size_t{256}); -#endif - return configured; -} - -void GeneralWorkspace::allocate() { - if (problem_.low_storage - ? problem_.low_storage_flows > 2 || problem_.coupled_flow || - (!problem_.low_storage_stage_register && !problem_.free_geometry()) - : problem_.low_storage_flows != 0 || !problem_.low_storage_stage_register) - throw std::invalid_argument( - "the low-storage march holds 0 to 2 flows beside no coupled flow, and elides " - "its stage register on free space alone"); - // The exchange fold of an in-stage absorber re-forms its input from the stage - // register, and a separable fold applies one flow volume, a constant operator's - // table (solprop SEP 24 section 4.3): neither has a meaning to fall back on. - if (problem_.low_storage && problem_.in_stage_absorber && - !problem_.low_storage_stage_register) - throw std::invalid_argument("the low-storage march cannot elide its stage register " - "beside an in-stage absorber"); - if (problem_.low_storage && problem_.separable_geometry() && - problem_.low_storage_flows > 1) - throw std::invalid_argument( - "the low-storage march holds at most one flow volume on " - "a separable geometry"); - const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - const auto chunk_rows = problem_.node_chunk * problem_.effective_node_pol(); - const auto transform_rows = - problem_.separable_geometry() ? 0 : problem_.n_nodes * problem_.n_pol_transform; - // A planar chunk's temporal rows are its y rows' columns, not its nodes. - const auto temporal_columns = problem_.slab_geometry() - ? transverse_units_per_chunk() * problem_.n_p - : (problem_.free_geometry() ? chunk_rows : columns); - separable_aliased_ = problem_.free_geometry() && !problem_.slab_geometry() && - !problem_.dispersive && temporal_columns == chunk_rows; - const auto statistic_nodes = - problem_.separable_geometry() ? problem_.node_chunk : problem_.n_modes; - // Spatial statistics reconstruct one chunk of physical nodes at a time. - const auto statistics_rows = problem_.slab_geometry() ? chunk_rows : temporal_columns; - if (problem_.separable_geometry()) { - // Keeping transform axes trailing avoids Kokkos-FFT's per-execution - // transpose allocations. The canonical state retains omega-contiguous columns. - transverse_ = - problem_.slab_geometry() - ? ComplexView3("solprop_native.slab.transverse", n, problem_.n_p, - problem_.n_y) - : ComplexView3("solprop_native.free.transverse", problem_.n_pol_state * n, - problem_.n_y, problem_.n_x); - spatial_.square = ComplexView("solprop_native.free.k_squared", n); - spatial_.square_low = ComplexView("solprop_native.free.k_squared_low", n); - spatial_.numerator = ComplexView("solprop_native.free.numerator", n); - spatial_.clamped = ComplexView("solprop_native.free.clamped", n); - spatial_.transverse = RealView("solprop_native.free.kperp_sq", problem_.n_modes); - spatial_.transverse_low = - RealView("solprop_native.free.kperp_sq_low", problem_.n_modes); - spatial_.frame = RealView("solprop_native.free.frame", n); - spatial_.spectral_rate = RealView("solprop_native.free.spectral_rate", n); - spatial_.user_rate = RealView("solprop_native.free.user_rate", n); - spatial_.absorber = RealView("solprop_native.free.absorber", problem_.n_modes); - spatial_.transverse_rate = - RealView("solprop_native.free.kperp_rate", problem_.n_modes); - spatial_.evanescent = problem_.spatial_evanescent_rate; - if (problem_.slab_geometry()) { - spatial_.kappa = ComplexView2("solprop_native.slab.kappa_sq", problem_.n_p, n); - spatial_.channels = problem_.n_p; - } - flux_ = spatial_; - if (problem_.flux_closure) - flux_transverse_ = - ComplexView3("solprop_native.flux.closure_transverse", transverse_.extent(0), - transverse_.extent(1), transverse_.extent(2)); - } else if (!problem_.flux_k_squared.empty()) { - flux_.square = ComplexView("solprop_native.flux.k_squared", n); - flux_.square_low = ComplexView("solprop_native.flux.k_squared_low", n); - flux_.transverse = RealView("solprop_native.flux.kperp_sq", problem_.n_modes); - flux_.transverse_low = - RealView("solprop_native.flux.kperp_sq_low", problem_.n_modes); - } - current_ = ComplexView2("solprop_native.state.current", columns, n); - if (problem_.low_storage) { - // solprop SEP 24 section 6.7: the march holds its registers beside the state and - // none of the Runge-Kutta views; the stage's input is formed in the route's own - // scratch (`transverse_`, or the fine spectrum of the modal routes). - register_accumulator_ = - ComplexView2("solprop_native.low_storage.accumulator", columns, n); - if (problem_.low_storage_stage_register) - register_stage_ = ComplexView2("solprop_native.low_storage.stage", columns, n); - low_storage_flows_ = ComplexView3("solprop_native.low_storage.flows", - problem_.low_storage_flows, columns, n); - } else { - advanced_ = ComplexView2("solprop_native.state.advanced", columns, n); - stages_ = ComplexView3("solprop_native.step.stages", problem_.n_stages, columns, n); - solution_ = ComplexView2("solprop_native.step.solution", columns, n); - error_ = ComplexView2("solprop_native.step.error", columns, n); - residual_ = ComplexView2("solprop_native.step.residual", columns, n); - recycled_ = ComplexView2("solprop_native.step.recycled", columns, n); - whole_propagator_ = - ComplexView2("solprop_native.step.whole_propagator", columns, n); - if (!problem_.separable_geometry()) - stage_inverse_ = ComplexView2("solprop_native.step.stage_inverse", columns, n); - } - if (problem_.modified_noise) - noise_ = ComplexView2("solprop_native.noise", columns, n); - if (!problem_.separable_geometry() && problem_.coupled_flow) - stage_phase_ = ComplexView2("solprop_native.step.stage_phase", columns, n); - partials_ = - RealView2("solprop_native.step.error_partials", 3, - checked_product(columns, reduction_slots_, "norm partial slots")); - if (!problem_.projection_coarse_matrix.empty()) - quadrature_partials_ = - RealView2("solprop_native.transform.quadrature_partials", 2, columns); - quadrature_scores_ = RealView("solprop_native.transform.quadrature_scores", 2); - Kokkos::deep_copy(quadrature_scores_, Kokkos::Experimental::quiet_NaN_v); - if (!problem_.low_storage) - pinned_partials_ = - Kokkos::create_mirror_view(Kokkos::SharedHostPinnedSpace{}, partials_); - if (problem_.weighted_controller) { - weighted_scale_ = RealView("solprop_native.step.weighted_scale", n); - weighted_partials_ = - RealView("solprop_native.step.weighted_partials", columns * reduction_slots_); - } - if (!problem_.spatial_spectral_weights.empty()) { - spatial_spectral_weights_ = - RealView("solprop_native.spatial.spectral_weights", problem_.n_modes); - spatial_on_axis_ = ComplexView("solprop_native.spatial.on_axis", problem_.n_modes); - // One row per column for the energy and per node for the peaks and fluence, - // which differ on the hybrid planar kind alone. - spatial_statistics_partials_ = - RealView2("solprop_native.spatial.statistics_partials", - std::max(problem_.n_modes, problem_.n_nodes), 3); - } - statistics_spectrum_ = - ComplexView2("solprop_native.statistics.spectrum", statistics_rows, problem_.n_t); - statistics_time_ = - ComplexView2("solprop_native.statistics.time", statistics_rows, problem_.n_t); - statistics_power_weight_ = - RealView2("solprop_native.statistics.power_weight", - problem_.spatial_spectral_weights.empty() ? columns : 0, n); - graph::deep_copy(execution_space_, statistics_power_weight_, 1.0); - // A spatial run's rows hold the magnetic closure in time (solprop SEP 34) when a - // local flux statistic is asked for. - statistics_weighted_time_ = - ComplexView2("solprop_native.statistics.weighted_time", - problem_.spatial_spectral_weights.empty() ? temporal_columns - : problem_.flux_closure ? statistics_rows - : 0, - problem_.n_t); - statistics_power_ = - RealView2("solprop_native.statistics.power", statistic_nodes, problem_.n_t); - if (problem_.flux_closure && !problem_.separable_geometry()) - statistics_magnitude_ = RealView2("solprop_native.statistics.flux_magnitude", - statistic_nodes, problem_.n_t); - statistics_total_power_ = - RealView("solprop_native.statistics.total_power", problem_.n_t); - statistics_on_axis_ = - RealView("solprop_native.statistics.on_axis", 2 * problem_.n_modes); - const auto base_statistics = checked_sum( - checked_sum(problem_.n_t, - checked_product(2, problem_.n_columns(), "statistics columns"), - "statistics packed output"), - checked_product(3, problem_.n_modes, "statistics modes"), - "statistics packed output"); - statistics_packed_ = - RealView("solprop_native.statistics.packed", - checked_sum(base_statistics, 5, "statistics packed output")); - crop_index_ = IndexView("solprop_native.transform.crop_index", n); - active_mask_ = ByteView("solprop_native.mask.active", n); - norm_mask_ = ByteView("solprop_native.mask.norm", n); - omega_window_ = RealView("solprop_native.window.omega", n); - omega_ = RealView("solprop_native.statistics.omega", problem_.omega.size()); - time_window_ = RealView("solprop_native.window.time", problem_.n_to); - carrier_ = ComplexView("solprop_native.kerr.carrier", problem_.n_to); - phi_terms_ = ComplexView3("solprop_native.linear.phi_terms", - problem_.separable_geometry() ? 0 : problem_.n_phi_terms, - columns, n); - prefactor_terms_ = ComplexView3( - "solprop_native.response.prefactor_terms", - problem_.separable_geometry() ? 0 : problem_.n_prefactor_terms, columns, n); - tableau_ = - RealView("solprop_native.step.tableau", problem_.n_stages * problem_.n_stages); - { - // One device buffer for every per-stage scalar view, so that a trial's scalars - // cross in one copy instead of one per view: on a device each small copy from - // pageable memory is a transfer of its own. - const auto stages = problem_.n_stages; - const auto channels = problem_.plasma ? problem_.n_channels : 0; - const auto raman_terms = problem_.raman ? problem_.n_raman_terms : 0; - const auto raman_components = problem_.raman && problem_.n_raman_components > 1 - ? problem_.n_raman_components - : 0; - const std::array sizes{(stages + 1) * problem_.n_phi_terms, - stages * problem_.n_phi_terms, - stages * problem_.n_prefactor_terms, - stages, - stages, - stages, - stages, - stages * channels, - stages * channels, - stages, - stages, - stages * raman_terms, - stages * raman_components, - problem_.n_phi_terms}; - std::size_t total = 0; - for (std::size_t region = 0; region < sizes.size(); ++region) { - stage_offsets_[region] = total; - total += sizes[region]; - } - stage_offsets_[sizes.size()] = total; - stage_slot_size_ = total; - stage_scalars_ = RealView("solprop_native.step.scalars", 2 * total); - bind_stage_scalars(0); - // No trial's tapers yet: every scaling runs until one is bound. - trial_taper_.assign(problem_.n_stages, std::numeric_limits::quiet_NaN()); - // A pinned buffer's copy is asynchronous, so its source must not be rewritten - // until the copy has run: `upload_trial` fences before the ring comes round again. - // A host build's mirror is the buffer itself. - stage_staging_.clear(); - for (std::size_t copy = 0; copy < stage_staging_ring; ++copy) - stage_staging_.push_back( - Kokkos::create_mirror_view(Kokkos::SharedHostPinnedSpace{}, stage_scalars_)); - } - if (problem_.raman) { - const auto doubled = 2 * problem_.n_to; - node_raman_drive_ = - RealView2("solprop_native.raman.node_drive", - problem_.node_chunk * problem_.raman_drives(), doubled); - node_raman_spectrum_ = - ComplexView2("solprop_native.raman.node_spectrum", - problem_.node_chunk * problem_.raman_drives(), problem_.n_to + 1); - node_raman_convolved_ = - RealView2("solprop_native.raman.node_convolved", - problem_.node_chunk * problem_.raman_drives(), doubled); - raman_undamped_ = - RealView2("solprop_native.raman.undamped", problem_.n_raman_terms, doubled); - raman_kernel_time_ = RealView2("solprop_native.raman.kernel_time", - problem_.raman_kernels(), doubled); - raman_kernel_spectrum_ = ComplexView2("solprop_native.raman.kernel_spectrum", - problem_.raman_kernels(), problem_.n_to + 1); - raman_tensor_weights_ = - RealView2("solprop_native.raman.tensor_weights", problem_.n_raman_terms, 2); - cached_raman_rates_.resize(problem_.n_raman_terms); - if (problem_.n_raman_components > 1) { - raman_row_component_ = - RealView("solprop_native.raman.row_component", problem_.n_raman_terms); - raman_component_fraction_ = RealView("solprop_native.raman.component_fraction", - problem_.n_raman_components); - cached_raman_densities_.resize(problem_.n_raman_components); - } - } - if (use_packed_radial_blas()) { - radial_blas_input_ = RealView2("solprop_native.radial.blas_input", problem_.n_nodes, - 2 * problem_.n_to); - radial_blas_output_ = RealView2("solprop_native.radial.blas_output", - problem_.n_nodes, 2 * problem_.n_to); - } - // The hybrid planar kind's slab rule in x, (n_x, n_p) and (n_p, n_x). - const auto synthesis_rows = problem_.slab_geometry() ? problem_.n_x : transform_rows; - const auto synthesis_columns = - problem_.slab_geometry() ? problem_.n_p : problem_.n_modes; - synthesis_ = RealView2("solprop_native.transform.synthesis", synthesis_rows, - synthesis_columns); - projection_ = RealView2("solprop_native.transform.projection", synthesis_columns, - synthesis_rows); - scaled_synthesis_ = RealView2("solprop_native.transform.scaled_synthesis", - transform_rows, problem_.n_modes); - scaled_projection_ = RealView2("solprop_native.transform.scaled_projection", - problem_.n_modes, transform_rows); - if (problem_.family == FieldFamily::envelope && use_blas_for_synthesis()) { - scaled_synthesis_complex_ = - ComplexView2("solprop_native.transform.scaled_synthesis_complex", - transform_rows, problem_.n_modes); - scaled_projection_complex_ = - ComplexView2("solprop_native.transform.scaled_projection_complex", - problem_.n_modes, transform_rows); - } - if (!problem_.projection_coarse_matrix.empty()) { - projection_coarse_ = RealView2("solprop_native.transform.projection_coarse", - problem_.n_modes, transform_rows); - scaled_projection_coarse_ = - RealView2("solprop_native.transform.scaled_projection_coarse", problem_.n_modes, - transform_rows); - if (problem_.family == FieldFamily::envelope && use_blas_for_projection()) { - scaled_projection_coarse_complex_ = - ComplexView2("solprop_native.transform.scaled_projection_coarse_complex", - problem_.n_modes, transform_rows); - scaled_projection_stacked_complex_ = - ComplexView2("solprop_native.transform.scaled_projection_stacked_complex", - 2 * problem_.n_modes, transform_rows); - projected_stacked_complex_ = - ComplexView2("solprop_native.response.projected_stacked_complex", 2 * columns, - problem_.n_to); - } else if (problem_.family != FieldFamily::envelope) { - scaled_projection_stacked_ = - RealView2("solprop_native.transform.scaled_projection_stacked", - 2 * problem_.n_modes, transform_rows); - projected_stacked_real_ = - RealView2("solprop_native.response.projected_stacked_real", - 2 * temporal_columns, problem_.n_to); - } - } - const auto chunk_transform_rows = problem_.node_chunk * problem_.n_pol_transform; - const auto tail_nodes = problem_.n_nodes % problem_.node_chunk; - const auto tail_transform_rows = tail_nodes * problem_.n_pol_transform; - if (use_chunked_blas_for_synthesis()) { - if (problem_.family == FieldFamily::envelope) { - chunk_synthesis_complex_ = - ComplexView2("solprop_native.transform.chunk_synthesis_complex", - chunk_transform_rows, problem_.n_modes); - if (tail_transform_rows != 0) - tail_synthesis_complex_ = - ComplexView2("solprop_native.transform.tail_synthesis_complex", - tail_transform_rows, problem_.n_modes); - } else { - chunk_synthesis_ = RealView2("solprop_native.transform.chunk_synthesis", - chunk_transform_rows, problem_.n_modes); - if (tail_transform_rows != 0) - tail_synthesis_ = RealView2("solprop_native.transform.tail_synthesis", - tail_transform_rows, problem_.n_modes); - } - } - if (use_chunked_blas_for_projection()) { - const bool stacked = !problem_.projection_coarse_matrix.empty(); - const auto projection_rows = stacked ? 2 * problem_.n_modes : problem_.n_modes; - if (problem_.family == FieldFamily::envelope) { - if (stacked) { - chunk_projection_stacked_complex_ = - ComplexView2("solprop_native.transform.chunk_projection_stacked_complex", - projection_rows, chunk_transform_rows); - if (tail_transform_rows != 0) - tail_projection_stacked_complex_ = - ComplexView2("solprop_native.transform.tail_projection_stacked_complex", - projection_rows, tail_transform_rows); - } else { - chunk_projection_complex_ = - ComplexView2("solprop_native.transform.chunk_projection_complex", - projection_rows, chunk_transform_rows); - if (tail_transform_rows != 0) - tail_projection_complex_ = - ComplexView2("solprop_native.transform.tail_projection_complex", - projection_rows, tail_transform_rows); - } - } else if (stacked) { - chunk_projection_stacked_ = - RealView2("solprop_native.transform.chunk_projection_stacked", - projection_rows, chunk_transform_rows); - if (tail_transform_rows != 0) - tail_projection_stacked_ = - RealView2("solprop_native.transform.tail_projection_stacked", - projection_rows, tail_transform_rows); - } else { - chunk_projection_ = RealView2("solprop_native.transform.chunk_projection", - projection_rows, chunk_transform_rows); - if (tail_transform_rows != 0) - tail_projection_ = RealView2("solprop_native.transform.tail_projection", - projection_rows, tail_transform_rows); - } - } - fine_spectrum_ = ComplexView2("solprop_native.fft.fine_spectrum", temporal_columns, - fine_spectrum_size()); - if (!problem_.separable_geometry() && problem_.n_omega != fine_spectrum_size()) - fine_gaps_ = IndexView("solprop_native.fft.fine_gaps", 3 * fine_gap_runs().size()); - if (problem_.separable_geometry() && problem_.n_omega != fine_spectrum_size()) { - std::vector bins; - for (const auto& [bin, length] : fine_gap_runs()) - for (std::size_t offset = 0; offset < length; ++offset) - bins.push_back(bin + offset); - separable_gaps_ = IndexView("solprop_native.free.fine_gaps", bins.size()); - copy_vector(execution_space_, separable_gaps_, bins); - } - // A device's packed forward rows (`packs_forward`): the absorber's, the projection's, - // the coarse projection's and the held pack's, in that order, carved from one buffer - // and one spectrum. - const bool pack = packs_forward(); - const std::size_t absorber_rows = - pack && problem_.in_stage_absorber ? temporal_columns : 0; - const std::size_t coarse_rows = - pack && !problem_.projection_coarse_matrix.empty() ? temporal_columns : 0; - const std::size_t pack_rows = - absorber_rows + temporal_columns + coarse_rows + - (held_plasma() ? 2 * (temporal_columns + coarse_rows) : 0); - const auto pack_slice = [&](const std::size_t first, const std::size_t count) { - return std::pair(first, first + count); - }; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - constexpr bool host_pack_layout = false; -#else - constexpr bool host_pack_layout = true; -#endif - // A host's pack: [absorber | fine | held fine x2 | coarse | held coarse x2]; a - // device's: [absorber | fine | coarse | (held fine, held coarse) x2]. - const std::size_t held_fine_rows = held_plasma() ? 2 * temporal_columns : 0; - const std::size_t fine_end = absorber_rows + temporal_columns + held_fine_rows; - const std::size_t coarse_first = - host_pack_layout ? fine_end : absorber_rows + temporal_columns; - forward_pack_absorber_rows_ = absorber_rows; - if (pack) { - forward_pack_ = - RealView2("solprop_native.fft.forward_pack", pack_rows, problem_.n_to); - forward_pack_spectrum_ = ComplexView2("solprop_native.fft.forward_pack_spectrum", - pack_rows, projected_spectrum_size()); - if (absorber_rows != 0 && pack_rows - absorber_rows > temporal_columns) { - forward_pack_tail_ = Kokkos::subview( - forward_pack_, pack_slice(absorber_rows, pack_rows - absorber_rows), - Kokkos::ALL); - forward_pack_tail_spectrum_ = Kokkos::subview( - forward_pack_spectrum_, pack_slice(absorber_rows, pack_rows - absorber_rows), - Kokkos::ALL); - } - if (host_pack_layout && coarse_rows != 0 && deferred_quadrature_eligible()) { - forward_pack_fine_ = - Kokkos::subview(forward_pack_, pack_slice(0, fine_end), Kokkos::ALL); - forward_pack_fine_spectrum_ = - Kokkos::subview(forward_pack_spectrum_, pack_slice(0, fine_end), Kokkos::ALL); - if (absorber_rows != 0 && fine_end - absorber_rows > temporal_columns) { - forward_pack_fine_tail_ = Kokkos::subview( - forward_pack_, pack_slice(absorber_rows, fine_end - absorber_rows), - Kokkos::ALL); - forward_pack_fine_tail_spectrum_ = Kokkos::subview( - forward_pack_spectrum_, pack_slice(absorber_rows, fine_end - absorber_rows), - Kokkos::ALL); - } - if (held_plasma()) { - forward_pack_coarse_ = Kokkos::subview( - forward_pack_, pack_slice(fine_end, pack_rows - fine_end), Kokkos::ALL); - forward_pack_coarse_spectrum_ = - Kokkos::subview(forward_pack_spectrum_, - pack_slice(fine_end, pack_rows - fine_end), Kokkos::ALL); - } - } - } - projected_spectrum_ = - pack ? ComplexView2(Kokkos::subview(forward_pack_spectrum_, - pack_slice(absorber_rows, temporal_columns), - Kokkos::ALL)) - : ComplexView2("solprop_native.fft.projected_spectrum", temporal_columns, - projected_spectrum_size()); - if (!problem_.projection_coarse_matrix.empty()) - projected_coarse_spectrum_ = - coarse_rows != 0 - ? ComplexView2(Kokkos::subview(forward_pack_spectrum_, - pack_slice(coarse_first, coarse_rows), - Kokkos::ALL)) - : ComplexView2("solprop_native.fft.projected_coarse_spectrum", columns, - projected_spectrum_size()); - absorber_spectrum_ = - absorber_rows != 0 - ? ComplexView2(Kokkos::subview(forward_pack_spectrum_, - pack_slice(0, absorber_rows), Kokkos::ALL)) - : ComplexView2("solprop_native.fft.absorber_spectrum", temporal_columns, - projected_spectrum_size()); - if (problem_.family == FieldFamily::field_real) { - modal_real_ = RealView2("solprop_native.fft.modal_time_real", temporal_columns, - problem_.n_to); - node_real_ = separable_aliased_ ? modal_real_ - : RealView2("solprop_native.response.node_real", - chunk_rows, problem_.n_to); - } else { - modal_complex_ = ComplexView2("solprop_native.fft.modal_time_complex", - temporal_columns, problem_.n_to); - node_complex_ = separable_aliased_ - ? modal_complex_ - : ComplexView2("solprop_native.response.node_complex", - chunk_rows, problem_.n_to); - retained_synthesis_ = retained_synthesis(); - if (retained_synthesis_) { - retained_modal_ = ComplexView2("solprop_native.transform.retained_modal", - temporal_columns, problem_.n_omega); - retained_node_ = ComplexView2("solprop_native.transform.retained_node", - chunk_rows, problem_.n_omega); - } - } - if (problem_.family == FieldFamily::envelope) { - projected_complex_ = ComplexView2("solprop_native.response.projected_complex", - temporal_columns, problem_.n_to); - polarisation_complex_ = - separable_aliased_ - ? projected_complex_ - : ComplexView2("solprop_native.response.polarisation_complex", chunk_rows, - problem_.n_to); - if (!problem_.projection_coarse_matrix.empty()) - projected_coarse_complex_ = - ComplexView2("solprop_native.response.projected_coarse_complex", - temporal_columns, problem_.n_to); - absorber_complex_ = ComplexView2("solprop_native.absorber.fine_complex", - temporal_columns, problem_.n_to); - } else { - projected_real_ = - pack ? RealView2(Kokkos::subview(forward_pack_, - pack_slice(absorber_rows, temporal_columns), - Kokkos::ALL)) - : RealView2("solprop_native.response.projected_real", temporal_columns, - problem_.n_to); - polarisation_real_ = separable_aliased_ - ? projected_real_ - : RealView2("solprop_native.response.polarisation_real", - chunk_rows, problem_.n_to); - if (!problem_.projection_coarse_matrix.empty()) - projected_coarse_real_ = - coarse_rows != 0 - ? RealView2(Kokkos::subview( - forward_pack_, pack_slice(coarse_first, coarse_rows), Kokkos::ALL)) - : RealView2("solprop_native.response.projected_coarse_real", - temporal_columns, problem_.n_to); - absorber_real_ = - absorber_rows != 0 - ? RealView2(Kokkos::subview(forward_pack_, pack_slice(0, absorber_rows), - Kokkos::ALL)) - : RealView2("solprop_native.absorber.fine_real", temporal_columns, - problem_.n_to); - } - if (field_plasma()) - held_current_ = RealView2("solprop_native.held.current", chunk_rows, problem_.n_to); - if (gated_plasma()) { - gated_prefix_ = RealView2("solprop_native.gated.prefix", chunk_rows, problem_.n_to); - if (problem_.plasma_band) - gated_filtered_ = - RealView2("solprop_native.gated.filtered", chunk_rows, problem_.n_to); - } - if (held_plasma()) { - held_derivative_ = RealView("solprop_native.held.derivative", problem_.n_to); - held_inverse_ = RealView("solprop_native.held.inverse", problem_.n_omega); - const bool radial = held_radial(); - const auto rows = radial ? chunk_rows : temporal_columns; - const bool coarse = !problem_.projection_coarse_matrix.empty(); - // On a device the two currents' fine and coarse rows are one buffer, a stream's - // fine rows then its coarse ones, so that one transform takes all four: on an AMD - // Instinct MI325X each transform of a single-mode run was four library kernels of a - // few microseconds, launched one by one. On the radial route the three streams' - // fine rows are one buffer, which a node chunk's one transform takes. -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const bool packed = !radial; - const bool radial_packed = radial; -#else - const bool packed = false; - const bool radial_packed = false; -#endif - // A host's pack holds the streams' rows itself, each fine block then each coarse. - const bool host_held = host_pack_layout && pack && !radial; - if (packed && pack) { - // The stage's forward pack ends with the held pack's rows. - const auto held_rows = - pack_slice(absorber_rows + rows + coarse_rows, 2 * (rows + coarse_rows)); - held_pack_ = Kokkos::subview(forward_pack_, held_rows, Kokkos::ALL); - held_pack_spectrum_ = - Kokkos::subview(forward_pack_spectrum_, held_rows, Kokkos::ALL); - } else if (packed) { - const std::size_t held_rows = 2 * (coarse ? 2 * rows : rows); - held_pack_ = RealView2("solprop_native.held.pack", held_rows, problem_.n_to); - held_pack_spectrum_ = ComplexView2("solprop_native.held.pack_spectrum", held_rows, - problem_.n_to / 2 + 1); - } else if (radial_packed) { - held_pack_ = RealView2("solprop_native.held.pack", 3 * rows, problem_.n_to); - held_pack_spectrum_ = ComplexView2("solprop_native.held.pack_spectrum", 3 * rows, - problem_.n_to / 2 + 1); - } - for (std::size_t index = 0; index < (radial ? 3U : 2U); ++index) { - auto& stream = held_streams_[index]; - const auto label = - std::string("solprop_native.held.stream") + std::to_string(index) + "."; - if (index < 2) - stream.node = RealView2(label + "node", chunk_rows, problem_.n_to); - if (host_held && index < 2) { - const auto fine = pack_slice(absorber_rows + rows + index * rows, rows); - stream.projected = Kokkos::subview(forward_pack_, fine, Kokkos::ALL); - stream.spectrum = Kokkos::subview(forward_pack_spectrum_, fine, Kokkos::ALL); - if (coarse) { - const auto held_coarse = pack_slice(fine_end + rows + index * rows, rows); - stream.coarse = Kokkos::subview(forward_pack_, held_coarse, Kokkos::ALL); - stream.coarse_spectrum = - Kokkos::subview(forward_pack_spectrum_, held_coarse, Kokkos::ALL); - stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.n_to); - } - continue; - } - if (packed) { - // Each of the two streams holds half of the pack. - const std::size_t first = index * (held_pack_.extent(0) / 2); - const auto rows_from = [&](const std::size_t start, const std::size_t count) { - return std::pair(start, start + count); - }; - stream.projected = - Kokkos::subview(held_pack_, rows_from(first, rows), Kokkos::ALL); - stream.spectrum = - Kokkos::subview(held_pack_spectrum_, rows_from(first, rows), Kokkos::ALL); - if (coarse) { - stream.coarse = - Kokkos::subview(held_pack_, rows_from(first + rows, rows), Kokkos::ALL); - stream.coarse_spectrum = Kokkos::subview( - held_pack_spectrum_, rows_from(first + rows, rows), Kokkos::ALL); - stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.n_to); - } - continue; - } - if (radial_packed) { - // Each of the three streams holds a third of the pack. - const auto third = pack_slice(index * rows, rows); - stream.projected = Kokkos::subview(held_pack_, third, Kokkos::ALL); - stream.spectrum = Kokkos::subview(held_pack_spectrum_, third, Kokkos::ALL); - } else { - stream.projected = RealView2(label + "projected", rows, problem_.n_to); - stream.spectrum = ComplexView2(label + "spectrum", rows, problem_.n_to / 2 + 1); - } - if (coarse) { - stream.coarse = RealView2(label + "coarse", rows, problem_.n_to); - stream.coarse_spectrum = - ComplexView2(label + "coarse_spectrum", rows, problem_.n_to / 2 + 1); - stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.n_to); - } - } - if (radial) { - held_nodal_spectrum_ = ComplexView2("solprop_native.held.nodal_spectrum", - chunk_rows, problem_.n_omega); - held_projected_spectrum_ = ComplexView2("solprop_native.held.projected_spectrum", - problem_.n_columns(), problem_.n_omega); - held_projection_ = RealView2("solprop_native.held.projection", problem_.n_modes, - problem_.node_chunk * problem_.n_pol_transform); - } - } - absorber_fine_ = - RealView("solprop_native.absorber.rate_fine", problem_.absorber_fine.size()); - absorber_coarse_ = - RealView("solprop_native.absorber.rate_coarse", problem_.absorber_coarse.size()); - if (!problem_.spatial_absorber_matrix.empty()) { - if (problem_.family == FieldFamily::envelope) { - spatial_absorber_complex_ = - ComplexView2("solprop_native.absorber.spatial_complex", problem_.n_modes, - problem_.n_modes); - std::vector matrix; - matrix.reserve(problem_.spatial_absorber_matrix.size()); - for (const auto value : problem_.spatial_absorber_matrix) - matrix.emplace_back(value, 0.0); - copy_vector(execution_space_, spatial_absorber_complex_, matrix); - } else { - spatial_absorber_real_ = RealView2("solprop_native.absorber.spatial_real", - problem_.n_modes, problem_.n_modes); - copy_vector(execution_space_, spatial_absorber_real_, - problem_.spatial_absorber_matrix); - } - } - if (problem_.envelope_plasma) { - envelope_rule_nodes_ = - RealView3("solprop_native.envelope.rule_nodes", problem_.n_channels, 4, 512); - envelope_rule_weights_ = - RealView3("solprop_native.envelope.rule_weights", problem_.n_channels, 4, 512); - envelope_moments_ = RealView3("solprop_native.envelope.moments", 2, - problem_.node_chunk, problem_.n_to); - envelope_rates_ = - RealView2("solprop_native.envelope.rates", problem_.node_chunk, problem_.n_to); - envelope_exposure_ = RealView2("solprop_native.envelope.exposure", - problem_.node_chunk, problem_.n_to); - envelope_failures_ = IndexView("solprop_native.envelope.failures", - problem_.node_chunk * problem_.n_to); - envelope_status_ = IndexView("solprop_native.envelope.status", 1); - envelope_host_status_ = Kokkos::View( - "solprop_native.envelope.host_status", 1); - envelope_drude_factor_ = - RealView("solprop_native.envelope.drude_factor", problem_.n_omega); - envelope_loss_factor_ = - ComplexView("solprop_native.envelope.loss_factor", problem_.n_omega); - for (std::size_t index = 0; index < envelope_streams_.size(); ++index) { - auto& stream = envelope_streams_[index]; - const auto label = - std::string("solprop_native.envelope.") + (index == 0 ? "drude." : "loss."); - stream.node = ComplexView2(label + "node", chunk_rows, problem_.n_to); - stream.projected = - ComplexView2(label + "projected", temporal_columns, problem_.n_to); - stream.spectrum = - ComplexView2(label + "spectrum", temporal_columns, problem_.n_to); - if (!problem_.projection_coarse_matrix.empty()) { - stream.coarse = ComplexView2(label + "coarse", temporal_columns, problem_.n_to); - stream.coarse_spectrum = - ComplexView2(label + "coarse_spectrum", temporal_columns, problem_.n_to); - } - if (!projected_stacked_complex_.empty()) - stream.stacked = - ComplexView2(label + "stacked", 2 * temporal_columns, problem_.n_to); - } - } - if (problem_.plasma) { - if (problem_.n_channels > 1 || problem_.envelope_plasma) { - channel_parameters_ = - RealView2("solprop_native.plasma.channel_parameters", problem_.n_channels, 9); - channel_coefficients_ = RealView("solprop_native.plasma.channel_coefficients", - problem_.channel_coefficients.size()); - if (!problem_.envelope_plasma) - channel_scratch_ = - decltype(channel_scratch_)("solprop_native.plasma.channel_scratch", - problem_.n_channels, problem_.node_chunk, 4); - } - rate_coefficients_ = RealView("solprop_native.plasma.rate_coefficients", - problem_.rate_coefficients.size()); - plasma_statistics_scratch_ = - RealView2("solprop_native.plasma.statistics_scratch", problem_.n_nodes, - 1 + 2 * problem_.n_channels); - if (!problem_.envelope_plasma) { - // solprop SEP 18: the rule's taps and the run diagnostics' six parts per node. - plasma_taps_ = - RealView("solprop_native.plasma.taps", problem_.plasma_taps.size()); - plasma_diagnostics_ = RealView2("solprop_native.plasma.diagnostics", - problem_.n_nodes, plasma_diagnostic_parts); - } - if (!problem_.envelope_plasma) { - // The band rule's team column and its statistics (`team_band_plasma_column`). - // A device keeps the trapezoid device column's rows, one per node of a chunk, - // and the band column reuses them. A host team owns one row of each per slot, - // and at most one team per thread runs at once (solprop SEP 25 section - // 6.7.5): the rate, exposure and filtered rows for every field plasma (the - // trapezoid's run diagnostics come from the band statistics), and under the - // band rule the guard, a mixture's weight and a row per component. The - // tiled column (solprop SEP 25 section 6.7.6) keeps the device's rows on every - // build, the weight rows holding the free-current local prefixes, and the tiles' - // totals by row. - const bool tiled = band_tiled_ && problem_.plasma_band; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - band_slots_ = problem_.node_chunk; - const auto guard_rows = chunk_rows; - const auto weight_rows = chunk_rows; - const auto component_rows = chunk_rows; -#else - if (tiled) { - band_slots_ = problem_.node_chunk; - } else { - band_slots_ = std::min(problem_.node_chunk, - static_cast(std::max( - 1, response_execution_space_.concurrency()))); - } - const bool band = problem_.plasma_band; - const auto guard_rows = tiled ? chunk_rows : band ? band_slots_ : 0; - const auto weight_rows = tiled ? chunk_rows - : band && problem_.n_channels > 1 ? band_slots_ - : 0; - const auto component_rows = - band ? checked_product(band_slots_, problem_.effective_node_pol(), - "band component rows") - : 0; -#endif - if (tiled) { - // The widest tile whose two scratch rows the device's team scratch holds. - using Policy = Kokkos::TeamPolicy; - const auto scratch_limit = - static_cast(Policy::scratch_size_max(0)); - std::size_t largest = band_tile_max; - while (largest > band_tile_min && - band_tile_bytes(largest) > scratch_limit) - largest /= 2; - if (band_tile_bytes(largest) > scratch_limit) - throw std::length_error("the band tile exceeds the team scratch"); - band_tile_ = band_tile_samples(problem_.n_to, largest); - band_tile_count_ = band_tile_count(problem_.n_to, band_tile_); - require_allocation(checked_product(chunk_rows, band_tile_count_, "band tiles"), - sizeof(double), "band-tile bytes"); - band_tile_totals_ = RealView2("solprop_native.plasma.band_tile_totals", - chunk_rows, band_tile_count_); - } - const auto band_rows = checked_sum( - checked_sum(checked_product(3, band_slots_, "band rows"), guard_rows, - "band rows"), - checked_sum(weight_rows, component_rows, "band rows"), "band rows"); - require_allocation(checked_product(band_rows, problem_.n_to, "band rows"), - sizeof(double), "band-row bytes"); - plasma_rate_ = - RealView2("solprop_native.plasma.rate", band_slots_, problem_.n_to); - plasma_exposure_ = - RealView2("solprop_native.plasma.exposure", band_slots_, problem_.n_to); - plasma_fraction_ = - RealView2("solprop_native.plasma.fraction", band_slots_, problem_.n_to); - if (component_rows != 0) { - plasma_current_integrand_ = RealView2("solprop_native.plasma.current_integrand", - component_rows, problem_.n_to); - } - if (guard_rows != 0) - plasma_free_current_ = - RealView2("solprop_native.plasma.free_current", guard_rows, problem_.n_to); - if (weight_rows != 0) - plasma_outer_current_ = RealView2("solprop_native.plasma.outer_current", - weight_rows, problem_.n_to); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - if (!problem_.plasma_band) { - plasma_tiles_ = (problem_.n_to + plasma_tile_samples - 1) / plasma_tile_samples; - plasma_tile_totals_ = - RealView2("solprop_native.plasma.tile_totals", chunk_rows, plasma_tiles_); - } -#endif - } - if (problem_.stash_plasma) { - stash_candidate_ = RealView2("solprop_native.plasma.stash_candidate", - problem_.n_nodes, 1 + 2 * problem_.n_channels); - stash_current_ = RealView2("solprop_native.plasma.stash_current", - problem_.n_nodes, 1 + 2 * problem_.n_channels); - } - } - if (problem_.split_step_absorber) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - split_factor_ = RealView("solprop_native.absorber.split_factor", problem_.n_t); -#endif - if (problem_.free_geometry() && problem_.family == FieldFamily::envelope) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // In place on the state's rows, a block holds no buffer of its own, only its - // plans' work area (9 MiB on cuFFT for every row at 768 x 768 x 600), so a device - // takes every row at once whatever the node budget: 14 kernels and 10 transforms - // a step there, against 119 and 220 at the node chunk (solprop #361). A CPU keeps - // the chunk, where FFTW's threaded plan of a whole state was much the slower. - split_rows_ = problem_.n_columns(); -#else - split_rows_ = temporal_columns; -#endif - } else { - coarse_spectrum_ = - ComplexView2("solprop_native.absorber.coarse_spectrum", temporal_columns, n); - if (problem_.family == FieldFamily::envelope) - coarse_time_complex_ = - ComplexView2("solprop_native.absorber.coarse_time_complex", - temporal_columns, problem_.n_t); - else - coarse_time_real_ = RealView2("solprop_native.absorber.coarse_time_real", - temporal_columns, problem_.n_t); - } - } - if (problem_.general_modal) { - // solprop SEP 47: a general modal basis's statistics whiten the columns by the - // metric's root (`whitened_statistics`), into scratch held from the start. - statistics_whitened_ = ComplexView2("solprop_native.statistics.whitened", - problem_.n_columns(), problem_.n_omega); - statistics_root_ = - ComplexView("solprop_native.statistics.metric_root", - checked_product(problem_.n_omega, - checked_product(problem_.n_modes, problem_.n_modes, - "metric root"), - "metric root")); - statistics_whitened_sums_ = RealView("solprop_native.statistics.whitened_sums", - problem_.n_columns() + problem_.n_omega); - general_ = std::make_unique(problem_); - // The source owns execution-space copies; the host maps are the largest arrays a - // general guide carries and nothing reads them again. - std::vector().swap(problem_.general_synthesis); - std::vector().swap(problem_.general_projection); - std::vector().swap(problem_.general_tensor); - std::vector().swap(problem_.general_frequency_factor); - std::vector().swap(problem_.general_delayed_tensor); - std::vector().swap(problem_.general_delayed_spectrum); - std::vector().swap(problem_.general_node_response); - } - if (problem_.free_geometry() && !problem_.spatial_spectral_weights.empty()) - far_field_ = std::make_unique(problem_, execution_space_); - if (problem_.dispersive && problem_.separable_geometry()) { - // The planar kind's slab rule in x; free space has none. - const bool slab = problem_.slab_geometry(); - separable_dispersive_ = std::make_unique( - problem_, kernel_pool_, - SeparableDispersive::StageViews{.fine_spectrum = fine_spectrum_, - .projected_spectrum = projected_spectrum_, - .node_real = node_real_, - .polarisation_real = polarisation_real_, - .node_complex = node_complex_, - .polarisation_complex = polarisation_complex_, - .synthesis = slab ? synthesis_ : RealView2{}, - .projection = slab ? projection_ : RealView2{}, - .time_window = time_window_, - .carrier = carrier_}); - } else if (problem_.dispersive && !problem_.general_modal) { - dispersive_ = std::make_unique( - problem_, kernel_pool_, - DispersiveResponse::StageViews{ - .fine_spectrum = fine_spectrum_, - .projected_spectrum = projected_spectrum_, - .projected_coarse_spectrum = projected_coarse_spectrum_, - .node_real = node_real_, - .polarisation_real = polarisation_real_, - .node_complex = node_complex_, - .polarisation_complex = polarisation_complex_, - .scaled_synthesis = scaled_synthesis_, - .scaled_projection = scaled_projection_, - .scaled_projection_coarse = scaled_projection_coarse_, - .time_window = time_window_, - .carrier = carrier_}); - } - if (problem_.dispersive) { - // The response holds execution-space copies of its layout (the general route's - // source built its own above). - std::vector().swap(problem_.dispersive_inputs); - std::vector().swap(problem_.dispersive_term_multipliers); - std::vector().swap(problem_.dispersive_quartics); - } - if (problem_.coupled_flow) { - flow_ = std::make_unique(problem_, flow_execution_space_); - std::vector().swap(problem_.flow_frozen); - std::vector().swap(problem_.flow_residual_terms); - std::vector().swap(problem_.flow_left); - std::vector().swap(problem_.flow_right); - std::vector().swap(problem_.flow_eigenvalues); - std::vector().swap(problem_.flow_route); - std::vector().swap(problem_.flow_fallback_vectors); - std::vector().swap(problem_.flow_fallback_triangular); - } -} - -void GeneralWorkspace::upload() { - if (problem_.n_raman_components > 1) { - copy_vector(execution_space_, raman_row_component_, problem_.raman_row_component); - copy_vector(execution_space_, raman_component_fraction_, - problem_.raman_component_fraction); - } - if (problem_.separable_geometry()) { - copy_vector(execution_space_, spatial_.square, problem_.spatial_k_squared); - copy_vector(execution_space_, spatial_.square_low, problem_.spatial_k_squared_low); - copy_vector(execution_space_, spatial_.numerator, problem_.spatial_numerator); - copy_vector(execution_space_, spatial_.clamped, problem_.spatial_clamped); - copy_vector(execution_space_, spatial_.transverse, problem_.spatial_kperp_sq); - copy_vector(execution_space_, spatial_.transverse_low, - problem_.spatial_kperp_sq_low); - copy_vector(execution_space_, spatial_.frame, problem_.spatial_frame); - copy_vector(execution_space_, spatial_.spectral_rate, - problem_.spatial_spectral_rate); - copy_vector(execution_space_, spatial_.user_rate, problem_.spatial_user_rate); - copy_vector(execution_space_, spatial_.absorber, problem_.spatial_absorber_rate); - copy_vector(execution_space_, spatial_.transverse_rate, - problem_.spatial_kperp_rate); - if (problem_.slab_geometry()) - copy_vector(execution_space_, spatial_.kappa, problem_.spatial_kappa_sq); - } else if (!problem_.flux_k_squared.empty()) { - copy_vector(execution_space_, flux_.square, problem_.flux_k_squared); - copy_vector(execution_space_, flux_.square_low, problem_.flux_k_squared_low); - copy_vector(execution_space_, flux_.transverse, problem_.flux_kperp_sq); - copy_vector(execution_space_, flux_.transverse_low, problem_.flux_kperp_sq_low); - } - if (problem_.n_channels > 1 || problem_.envelope_plasma) { - copy_vector(execution_space_, channel_parameters_, problem_.channel_parameters); - copy_vector(execution_space_, channel_coefficients_, problem_.channel_coefficients); - } - copy_vector(execution_space_, crop_index_, problem_.crop_index); - if (!fine_gaps_.empty()) { - std::vector runs; - std::size_t position = 0; - for (const auto& [bin, length] : fine_gap_runs()) { - runs.insert(runs.end(), {position, bin, length}); - position += length; - } - copy_vector(execution_space_, fine_gaps_, runs); - } - if (problem_.envelope_plasma) { - copy_vector(execution_space_, envelope_rule_nodes_, problem_.envelope_rule_nodes); - copy_vector(execution_space_, envelope_rule_weights_, - problem_.envelope_rule_weights); - copy_vector(execution_space_, envelope_drude_factor_, - problem_.envelope_drude_factor); - copy_vector(execution_space_, envelope_loss_factor_, problem_.envelope_loss_factor); - } - copy_vector(execution_space_, active_mask_, problem_.active_mask); - copy_vector(execution_space_, norm_mask_, problem_.norm_mask); - copy_vector(execution_space_, omega_window_, problem_.omega_window); - copy_vector(execution_space_, omega_, problem_.omega); - copy_vector(execution_space_, time_window_, problem_.time_window); - if (held_plasma()) { - copy_vector(execution_space_, held_derivative_, problem_.time_window_derivative); - std::vector inverse(problem_.n_omega, 0.0); - for (std::size_t index = 0; index < problem_.n_omega; ++index) - if (problem_.active_mask[index] != 0) - inverse[index] = 1.0 / problem_.omega[index]; - copy_vector(execution_space_, held_inverse_, inverse); - } - copy_vector(execution_space_, carrier_, problem_.carrier); - // A z table's slots start at zero (Kokkos initialises a view) until evaluated. - lossless_phase_ = false; - if (!problem_.phi_table) { - copy_vector(execution_space_, phi_terms_, problem_.phi_terms); - lossless_phase_ = std::ranges::all_of( - problem_.phi_terms, [](const Complex& term) { return term.real() == 0.0; }); - } - if (!problem_.prefactor_table) - copy_vector(execution_space_, prefactor_terms_, problem_.prefactor_terms); - copy_vector(execution_space_, tableau_, problem_.tableau_matrix); - copy_vector(execution_space_, spatial_spectral_weights_, - problem_.spatial_spectral_weights); - copy_vector(execution_space_, spatial_on_axis_, problem_.spatial_on_axis); - copy_vector(execution_space_, synthesis_, problem_.synthesis_matrix); - copy_vector(execution_space_, projection_, problem_.projection_matrix); - if (!problem_.projection_coarse_matrix.empty()) - copy_vector(execution_space_, projection_coarse_, - problem_.projection_coarse_matrix); - copy_vector(execution_space_, absorber_fine_, problem_.absorber_fine); - spatial_only_absorber_ = std::ranges::all_of( - problem_.absorber_fine, [](const double rate) { return rate == 0.0; }); - copy_vector(execution_space_, absorber_coarse_, problem_.absorber_coarse); - if (problem_.plasma) - copy_vector(execution_space_, rate_coefficients_, problem_.rate_coefficients); - if (!plasma_taps_.empty()) - copy_vector(execution_space_, plasma_taps_, problem_.plasma_taps); - if (problem_.raman) { - copy_vector(execution_space_, raman_undamped_, problem_.raman_undamped); - copy_vector(execution_space_, raman_tensor_weights_, problem_.raman_tensor_weights); - graph::deep_copy(execution_space_, node_raman_drive_, 0.0); - } -} - -// Size from the plan's real-side batch samples, using the shared complete-window -// thresholds in core.hpp. Explicit FFT-pool requests bypass automatic sizing. -// FFTW's factorisation can change with the width; comparisons use solprop's SEP 25 -// transform/propagation rounding budgets rather than bitwise agreement. -const ExecutionSpace& GeneralWorkspace::fft_space_for(const std::size_t samples, - const std::size_t length) { - // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide - return fft_pool_.get(CpuOperation::fft, samples, length); -} - -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -// Measured plans for free space's batched transforms (SEP 28): the transverse pair in -// place on the scratch and, on the complex fields, the chunk's temporal pair, each at -// the plan policy's effort for its shape and width (`fftw_seed_effort`). The scratch -// buffers are cleared after planning overwrote them. -void GeneralWorkspace::seed_free_wisdom() { - const auto planes = static_cast(transverse_.extent(0)); - const auto n_y = static_cast(transverse_.extent(1)); - const auto n_x = static_cast(transverse_.extent(2)); - for (const int sign : {FFTW_FORWARD, FFTW_BACKWARD}) - seed_fftw_wisdom(fft_space_for_rows(transverse_), transverse_.data(), - transverse_.data(), {n_y, n_x}, planes, n_y * n_x, n_y * n_x, - sign); - graph::deep_copy(execution_space_, transverse_, Complex{}); - // The chunk's temporal pair is the modal routes' (`separable_to_time`, - // `separable_to_frequency`), on every field family: the real family's pair and the - // analytic family's forward transform were left at FFTW_ESTIMATE (solprop #560). - // A fine spectrum wider than n_to is not a plain row pair; the helper leaves it. - seed_modal_wisdom(); -} - -// Measured plans for the modal routes' temporal pair, the two transforms of every -// source evaluation: Kokkos-FFT plans them at FFTW_ESTIMATE, which picks markedly -// slower codelets for long rows (a 16384-sample envelope GNLSE ran its pair at about -// twice the measured cost). The plan policy (`fftw_seed_effort`) decides the effort. -void GeneralWorkspace::seed_modal_wisdom() { - if (problem_.family == FieldFamily::field_real) { - seed_row_fft_wisdom(fft_space_for_rows(modal_real_), fine_spectrum_, modal_real_, - KokkosFFT::Direction::backward); - graph::deep_copy(execution_space_, modal_real_, 0.0); - } else { - seed_row_fft_wisdom(fft_space_for_rows(modal_complex_), fine_spectrum_, - modal_complex_, KokkosFFT::Direction::backward); - graph::deep_copy(execution_space_, modal_complex_, Complex{}); - } - if (problem_.family == FieldFamily::envelope) { - seed_row_fft_wisdom(fft_space_for_rows(projected_complex_), projected_complex_, - projected_spectrum_, KokkosFFT::Direction::forward); - graph::deep_copy(execution_space_, projected_complex_, Complex{}); - } else { - seed_row_fft_wisdom(fft_space_for_rows(projected_real_), projected_real_, - projected_spectrum_, KokkosFFT::Direction::forward); - graph::deep_copy(execution_space_, projected_real_, 0.0); - } - graph::deep_copy(execution_space_, fine_spectrum_, Complex{}); - graph::deep_copy(execution_space_, projected_spectrum_, Complex{}); -} - -// The plasma streams' forward rows, fine and coarse. Most share a shape with the -// projected pair seeded above, whose wisdom FFTW then finds at once; the radial held -// streams' chunk rows do not. -void GeneralWorkspace::seed_stream_wisdom() { - // The forward pack's plans (`packs_forward`); seeding overwrites their rows. - for (const auto& [rows, spectrum] : - {std::pair{forward_pack_, forward_pack_spectrum_}, - std::pair{forward_pack_tail_, forward_pack_tail_spectrum_}, - std::pair{forward_pack_fine_, forward_pack_fine_spectrum_}, - std::pair{forward_pack_fine_tail_, forward_pack_fine_tail_spectrum_}, - std::pair{forward_pack_coarse_, forward_pack_coarse_spectrum_}}) { - if (rows.empty()) - continue; - seed_row_fft_wisdom(fft_space_for_rows(rows), rows, spectrum, - KokkosFFT::Direction::forward); - } - if (!forward_pack_.empty()) { - graph::deep_copy(execution_space_, forward_pack_, 0.0); - graph::deep_copy(execution_space_, forward_pack_spectrum_, Complex{}); - } - if (held_plasma()) - for (auto& stream : held_streams_) { - if (stream.projected.empty()) - continue; - seed_row_fft_wisdom(fft_space_for_rows(stream.projected), stream.projected, - stream.spectrum, KokkosFFT::Direction::forward); - graph::deep_copy(execution_space_, stream.projected, 0.0); - graph::deep_copy(execution_space_, stream.spectrum, Complex{}); - if (stream.coarse.empty()) - continue; - seed_row_fft_wisdom(fft_space_for_rows(stream.coarse), stream.coarse, - stream.coarse_spectrum, KokkosFFT::Direction::forward); - graph::deep_copy(execution_space_, stream.coarse, 0.0); - graph::deep_copy(execution_space_, stream.coarse_spectrum, Complex{}); - } - if (problem_.envelope_plasma) - for (auto& stream : envelope_streams_) { - seed_row_fft_wisdom(fft_space_for_rows(stream.projected), stream.projected, - stream.spectrum, KokkosFFT::Direction::forward); - graph::deep_copy(execution_space_, stream.projected, Complex{}); - graph::deep_copy(execution_space_, stream.spectrum, Complex{}); - if (stream.coarse.empty()) - continue; - seed_row_fft_wisdom(fft_space_for_rows(stream.coarse), stream.coarse, - stream.coarse_spectrum, KokkosFFT::Direction::forward); - graph::deep_copy(execution_space_, stream.coarse, Complex{}); - graph::deep_copy(execution_space_, stream.coarse_spectrum, Complex{}); - } -} -#endif - -void GeneralWorkspace::plan() { -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // Seed every measured shape before the first plan exists: a Kokkos-FFT plan made - // before its wisdom stays at FFTW_ESTIMATE for the run. - if (problem_.free_geometry()) - seed_free_wisdom(); - else if (!problem_.separable_geometry()) - seed_modal_wisdom(); - seed_stream_wisdom(); -#endif - fft_execution_space_ = modal_complex_.size() > modal_real_.size() - ? fft_space_for_rows(modal_complex_) - : fft_space_for_rows(modal_real_); - if (problem_.raman) { - raman_execution_space_ = raman_pool_.get( - CpuOperation::raman, node_raman_drive_.size(), row_length(node_raman_drive_)); - const auto raman_tail = problem_.n_nodes % problem_.node_chunk; - raman_tail_execution_space_ = raman_tail_pool_.get( - CpuOperation::raman, raman_tail * problem_.raman_drives() * 2 * problem_.n_to, - 2 * problem_.n_to); - raman_kernel_execution_space_ = raman_pool_.get( - CpuOperation::raman, raman_kernel_time_.size(), row_length(raman_kernel_time_)); - raman_multiply_space_ = - kernel_pool_.get(CpuOperation::stage, node_raman_spectrum_.size()); - raman_tail_multiply_space_ = - kernel_pool_.get(CpuOperation::stage, - raman_tail * problem_.raman_drives() * (problem_.n_to + 1)); - raman_rebuild_space_ = kernel_pool_.get( - CpuOperation::response, raman_kernel_time_.size() * problem_.n_raman_terms); - raman_scale_space_ = - kernel_pool_.get(CpuOperation::stage, raman_kernel_spectrum_.size()); - raman_products_ = raman_modal_products(); - if (raman_products_ != 0) { - raman_product_execution_space_ = raman_tail_pool_.get( - CpuOperation::raman, raman_products_ * 2 * problem_.n_to, 2 * problem_.n_to); - raman_product_space_ = - kernel_pool_.get(CpuOperation::stage, raman_products_ * 2 * problem_.n_to); - } - } - if (general_) - general_->plan(); - if (dispersive_) - dispersive_->plan(fft_pool_); - if (separable_dispersive_) - separable_dispersive_->plan(fft_pool_); - if (problem_.slab_geometry()) { - // The y transforms are many short ones; below the threaded-plan threshold they - // run on one FFTW thread, as every other small plan (`fft_space_for`). - constexpr int y_axis = -1; - const auto& space = fft_space_for_rows(transverse_); - slab_forward_ = std::make_unique(space, transverse_, transverse_, - KokkosFFT::Direction::forward, y_axis); - slab_inverse_ = std::make_unique(space, transverse_, transverse_, - KokkosFFT::Direction::backward, y_axis); - } else if (problem_.free_geometry()) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - constexpr std::size_t axes = 2; -#else - const KokkosFFT::axis_type<2> axes{-2, -1}; -#endif - transverse_forward_ = std::make_unique( - fft_space_for_rows(transverse_), transverse_, transverse_, - KokkosFFT::Direction::forward, axes); - transverse_inverse_ = std::make_unique( - fft_space_for_rows(transverse_), transverse_, transverse_, - KokkosFFT::Direction::backward, axes); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - bool identity_crop = problem_.crop_index.size() == problem_.n_omega; - for (std::size_t index = 0; identity_crop && index < problem_.n_omega; ++index) - identity_crop = problem_.crop_index[index] == index; - volume_kerr_ = identity_crop && problem_.family == FieldFamily::envelope && - problem_.n_pol_state == 1 && problem_.effective_node_pol() == 1 && - problem_.n_to == problem_.n_omega && problem_.n_t == problem_.n_to && - separable_aliased_ && problem_.nonlinear && !problem_.plasma && - !problem_.raman && !problem_.envelope_plasma && - !problem_.dispersive && separable_dispersive_ == nullptr && - problem_.harmonic_factor == 0.0 && separable_gaps_.empty(); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // At 768 x 768 x 600 on an NVIDIA H200, cuFFT's `{N,N,T}` takes 14.7 ms a transform - // against 17.6 ms for `{T,N,N}` (solprop #365). An in-stage absorber evaluates its - // term from the same input in the scratch's order, so it keeps that order. - volume_order_ = volume_kerr_ && !problem_.in_stage_absorber; -#endif - if (volume_order_) - volume_scratch_ = ComplexView3(transverse_.data(), problem_.n_y, problem_.n_x, - problem_.n_omega); - if (volume_kerr_) { - constexpr std::size_t volume_axes = 3; - const auto& volume = volume_order_ ? volume_scratch_ : transverse_; - volume_forward_ = std::make_unique( - execution_space_, volume, volume, KokkosFFT::Direction::forward, volume_axes); - volume_inverse_ = - std::make_unique(execution_space_, volume, volume, - KokkosFFT::Direction::backward, volume_axes); - } - std::size_t work = KokkosFFT::compute_required_workspace_size( - *transverse_forward_, *transverse_inverse_); - if (volume_kerr_) - work = std::max(work, KokkosFFT::compute_required_workspace_size( - *volume_forward_, *volume_inverse_)); - if (work != 0) { - transform_work_ = - ComplexView(Kokkos::view_alloc("solprop_native.free.transform_work", - Kokkos::WithoutInitializing), - work); - transverse_forward_->set_work_area(transform_work_); - transverse_inverse_->set_work_area(transform_work_); - if (volume_kerr_) { - volume_forward_->set_work_area(transform_work_); - volume_inverse_->set_work_area(transform_work_); - } - } -#endif - } - constexpr int axis = -1; - if (held_plasma()) { - for (auto& stream : held_streams_) { - if (stream.projected.empty()) - continue; - stream.forward = std::make_unique( - fft_space_for_rows(stream.projected), stream.projected, stream.spectrum, - KokkosFFT::Direction::forward, axis); - if (!stream.coarse.empty()) - stream.coarse_forward = std::make_unique( - fft_space_for_rows(stream.coarse), stream.coarse, stream.coarse_spectrum, - KokkosFFT::Direction::forward, axis); - } - if (!held_pack_.empty() && forward_pack_.empty()) - held_pack_forward_ = std::make_unique( - fft_space_for_rows(held_pack_), held_pack_, held_pack_spectrum_, - KokkosFFT::Direction::forward, axis); - } - if (problem_.envelope_plasma) - for (auto& stream : envelope_streams_) { - stream.forward = std::make_unique( - fft_space_for_rows(stream.projected), stream.projected, stream.spectrum, - KokkosFFT::Direction::forward, axis); - if (!stream.coarse.empty()) - stream.coarse_forward = std::make_unique( - fft_space_for_rows(stream.coarse), stream.coarse, stream.coarse_spectrum, - KokkosFFT::Direction::forward, axis); - } - statistics_inverse_plan_ = std::make_unique( - fft_space_for_rows(statistics_time_), statistics_spectrum_, statistics_time_, - KokkosFFT::Direction::backward, axis); - if (problem_.family == FieldFamily::field_real) { - inverse_real_plan_ = std::make_unique( - fft_space_for_rows(modal_real_), fine_spectrum_, modal_real_, - KokkosFFT::Direction::backward, axis); - } else { - inverse_complex_plan_ = std::make_unique( - fft_space_for_rows(modal_complex_), fine_spectrum_, modal_complex_, - KokkosFFT::Direction::backward, axis); - } - if (retained_synthesis_) { - const auto plan_rows = [&](const std::size_t count) { - const auto rows = std::make_pair(std::size_t{0}, count); - const ComplexView2 padded = Kokkos::subview(modal_complex_, rows, Kokkos::ALL()); - const ComplexView2 node = Kokkos::subview(node_complex_, rows, Kokkos::ALL()); - const auto& space = fft_space_for_rows(node); -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - seed_row_fft_wisdom(space, padded, node, KokkosFFT::Direction::backward); -#endif - auto plan = std::make_unique(space, padded, node, - KokkosFFT::Direction::backward, axis); - // The padded rows' unretained bins stay zero from here on - // (`synthesise_retained`), whatever planning wrote. - graph::deep_copy(execution_space_, padded, Complex{}); - graph::deep_copy(execution_space_, node, Complex{}); - return plan; - }; - retained_inverse_plan_ = plan_rows(problem_.node_chunk); - if (const auto tail = problem_.n_nodes % problem_.node_chunk; tail != 0) - retained_tail_inverse_plan_ = plan_rows(tail); - } - if (problem_.family == FieldFamily::envelope) { - forward_complex_plan_ = std::make_unique( - fft_space_for_rows(projected_complex_), projected_complex_, projected_spectrum_, - KokkosFFT::Direction::forward, axis); - absorber_complex_plan_ = std::make_unique( - fft_space_for_rows(absorber_complex_), absorber_complex_, absorber_spectrum_, - KokkosFFT::Direction::forward, axis); - if (!problem_.projection_coarse_matrix.empty()) - forward_coarse_complex_plan_ = std::make_unique( - fft_space_for_rows(projected_coarse_complex_), projected_coarse_complex_, - projected_coarse_spectrum_, KokkosFFT::Direction::forward, axis); - } else { - forward_real_plan_ = std::make_unique( - fft_space_for_rows(projected_real_), projected_real_, projected_spectrum_, - KokkosFFT::Direction::forward, axis); - absorber_real_plan_ = std::make_unique( - fft_space_for_rows(absorber_real_), absorber_real_, absorber_spectrum_, - KokkosFFT::Direction::forward, axis); - if (!forward_pack_.empty()) - forward_pack_plan_ = std::make_unique( - fft_space_for_rows(forward_pack_), forward_pack_, forward_pack_spectrum_, - KokkosFFT::Direction::forward, axis); - if (!forward_pack_tail_.empty()) - forward_pack_tail_plan_ = std::make_unique( - fft_space_for_rows(forward_pack_tail_), forward_pack_tail_, - forward_pack_tail_spectrum_, KokkosFFT::Direction::forward, axis); - if (!forward_pack_fine_.empty()) - forward_pack_fine_plan_ = std::make_unique( - fft_space_for_rows(forward_pack_fine_), forward_pack_fine_, - forward_pack_fine_spectrum_, KokkosFFT::Direction::forward, axis); - if (!forward_pack_fine_tail_.empty()) - forward_pack_fine_tail_plan_ = std::make_unique( - fft_space_for_rows(forward_pack_fine_tail_), forward_pack_fine_tail_, - forward_pack_fine_tail_spectrum_, KokkosFFT::Direction::forward, axis); - if (!forward_pack_coarse_.empty()) - forward_pack_coarse_plan_ = std::make_unique( - fft_space_for_rows(forward_pack_coarse_), forward_pack_coarse_, - forward_pack_coarse_spectrum_, KokkosFFT::Direction::forward, axis); - if (!problem_.projection_coarse_matrix.empty()) - forward_coarse_real_plan_ = std::make_unique( - fft_space_for_rows(projected_coarse_real_), projected_coarse_real_, - projected_coarse_spectrum_, KokkosFFT::Direction::forward, axis); - } - if constexpr (cpu_size_policy_enabled) { - if (problem_.separable_geometry()) { - const auto full_units = transverse_units_per_chunk(); - const auto tail_units = transverse_units() % full_units; - const auto rows_per_unit = - problem_.slab_geometry() ? problem_.n_p : problem_.n_pol_state; - temporal_tail_rows_ = tail_units * rows_per_unit; - if (temporal_tail_rows_ != 0) { - const auto& space = - fft_space_for(temporal_tail_rows_ * problem_.n_to, problem_.n_to); - const auto rows = fine_spectrum_.extent(0); - if (problem_.family == FieldFamily::field_real) - temporal_tail_inverse_real_.create(space, fine_spectrum_, modal_real_, rows, - temporal_tail_rows_, - KokkosFFT::Direction::backward); - else - temporal_tail_inverse_complex_.create(space, fine_spectrum_, modal_complex_, - rows, temporal_tail_rows_, - KokkosFFT::Direction::backward); - if (problem_.family == FieldFamily::envelope) - temporal_tail_forward_complex_.create( - space, projected_complex_, projected_spectrum_, rows, temporal_tail_rows_, - KokkosFFT::Direction::forward); - else - temporal_tail_forward_real_.create( - space, projected_real_, projected_spectrum_, rows, temporal_tail_rows_, - KokkosFFT::Direction::forward); - // The tail plans' measured planning overwrote their rows. - graph::deep_copy(execution_space_, fine_spectrum_, Complex{}); - graph::deep_copy(execution_space_, projected_spectrum_, Complex{}); - if (problem_.family == FieldFamily::envelope) - graph::deep_copy(execution_space_, projected_complex_, Complex{}); - else - graph::deep_copy(execution_space_, projected_real_, 0.0); - if (problem_.family == FieldFamily::field_real) - graph::deep_copy(execution_space_, modal_real_, 0.0); - else - graph::deep_copy(execution_space_, modal_complex_, Complex{}); - } - } - } - if (problem_.raman) { - const auto started = std::chrono::steady_clock::now(); - raman_drive_forward_plan_ = std::make_unique( - raman_execution_space_, node_raman_drive_, node_raman_spectrum_, - KokkosFFT::Direction::forward, axis); - raman_convolution_inverse_plan_ = std::make_unique( - raman_execution_space_, node_raman_spectrum_, node_raman_convolved_, - KokkosFFT::Direction::backward, axis); - const auto tail = problem_.n_nodes % problem_.node_chunk; - if (tail != 0) { - // Reuse the leading rows of the full chunk buffers. A separate persistent - // plan avoids transforming stale rows in the final partial node chunk. - const auto rows = std::make_pair(std::size_t{0}, tail * problem_.raman_drives()); - raman_tail_drive_ = Kokkos::subview(node_raman_drive_, rows, Kokkos::ALL()); - raman_tail_spectrum_ = Kokkos::subview(node_raman_spectrum_, rows, Kokkos::ALL()); - raman_tail_convolved_ = - Kokkos::subview(node_raman_convolved_, rows, Kokkos::ALL()); - raman_tail_forward_plan_ = std::make_unique( - raman_tail_execution_space_, raman_tail_drive_, raman_tail_spectrum_, - KokkosFFT::Direction::forward, axis); - raman_tail_inverse_plan_ = std::make_unique( - raman_tail_execution_space_, raman_tail_spectrum_, raman_tail_convolved_, - KokkosFFT::Direction::backward, axis); - } - if (raman_products_ != 0) { - const auto rows = std::make_pair(std::size_t{0}, raman_products_); - raman_product_drive_ = Kokkos::subview(node_raman_drive_, rows, Kokkos::ALL()); - raman_product_spectrum_ = - Kokkos::subview(node_raman_spectrum_, rows, Kokkos::ALL()); - raman_product_forward_plan_ = std::make_unique( - raman_product_execution_space_, raman_product_drive_, raman_product_spectrum_, - KokkosFFT::Direction::forward, axis); - raman_product_inverse_plan_ = std::make_unique( - raman_product_execution_space_, raman_product_spectrum_, raman_product_drive_, - KokkosFFT::Direction::backward, axis); - } - raman_kernel_forward_plan_ = std::make_unique( - raman_kernel_execution_space_, raman_kernel_time_, raman_kernel_spectrum_, - KokkosFFT::Direction::forward, axis); - raman_planning_seconds_ = - std::chrono::duration(std::chrono::steady_clock::now() - started) - .count(); - } - if (!problem_.split_step_absorber) - return; - if (split_rows_ != 0) { - // In place on the state's rows; a later execution may name another row block or the - // other of the state's two views, of the same extents. - const auto& state = problem_.low_storage ? current_ : advanced_; - const auto plan_rows = [&](const std::size_t count, - const KokkosFFT::Direction direction) { - const ComplexView2 rows = - Kokkos::subview(state, std::make_pair(std::size_t{0}, count), Kokkos::ALL()); - const auto& space = fft_space_for_rows(rows); -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // Seeding may overwrite the rows; nothing is loaded yet. - seed_row_fft_wisdom(space, rows, rows, direction); - graph::deep_copy(execution_space_, rows, Complex{}); -#endif - return std::make_unique(space, rows, rows, direction, axis); - }; - split_inverse_plan_ = plan_rows(split_rows_, KokkosFFT::Direction::backward); - split_forward_plan_ = plan_rows(split_rows_, KokkosFFT::Direction::forward); - if (const auto tail = problem_.n_columns() % split_rows_; tail != 0) { - split_tail_inverse_plan_ = plan_rows(tail, KokkosFFT::Direction::backward); - split_tail_forward_plan_ = plan_rows(tail, KokkosFFT::Direction::forward); - } - return; - } - if (problem_.family == FieldFamily::envelope) { - coarse_inverse_complex_plan_ = std::make_unique( - fft_space_for_rows(coarse_time_complex_), coarse_spectrum_, - coarse_time_complex_, KokkosFFT::Direction::backward, axis); - coarse_forward_complex_plan_ = std::make_unique( - fft_space_for_rows(coarse_time_complex_), coarse_time_complex_, - coarse_spectrum_, KokkosFFT::Direction::forward, axis); - } else { - coarse_inverse_real_plan_ = std::make_unique( - fft_space_for_rows(coarse_time_real_), coarse_spectrum_, coarse_time_real_, - KokkosFFT::Direction::backward, axis); - coarse_forward_real_plan_ = std::make_unique( - fft_space_for_rows(coarse_time_real_), coarse_time_real_, coarse_spectrum_, - KokkosFFT::Direction::forward, axis); - } - if (temporal_tail_rows_ != 0) { - const auto& space = fft_space_for(temporal_tail_rows_ * problem_.n_t, problem_.n_t); - const auto rows = coarse_spectrum_.extent(0); - if (problem_.family == FieldFamily::envelope) { - coarse_tail_inverse_complex_.create(space, coarse_spectrum_, coarse_time_complex_, - rows, temporal_tail_rows_, - KokkosFFT::Direction::backward); - coarse_tail_forward_complex_.create(space, coarse_time_complex_, coarse_spectrum_, - rows, temporal_tail_rows_, - KokkosFFT::Direction::forward); - } else { - coarse_tail_inverse_real_.create(space, coarse_spectrum_, coarse_time_real_, rows, - temporal_tail_rows_, - KokkosFFT::Direction::backward); - coarse_tail_forward_real_.create(space, coarse_time_real_, coarse_spectrum_, rows, - temporal_tail_rows_, - KokkosFFT::Direction::forward); - } - // The tail plans' measured planning overwrote their rows. - graph::deep_copy(execution_space_, coarse_spectrum_, Complex{}); - if (problem_.family == FieldFamily::envelope) - graph::deep_copy(execution_space_, coarse_time_complex_, Complex{}); - else - graph::deep_copy(execution_space_, coarse_time_real_, 0.0); - } -} - -void GeneralWorkspace::set_initial(const std::span field, - const std::span noise) { - require_open(); - require_extent(field.size(), problem_.n_columns() * problem_.n_omega, - "initial field"); - if (!noise.empty()) - require_extent(noise.size(), field.size(), "noise"); - upload_span(execution_space_, crossing_ring_, current_, field); - crossed_.from_host += field.size_bytes(); - if (noise.empty()) { - if (problem_.modified_noise) - graph::deep_copy(execution_space_, noise_, Complex{}); - has_noise_ = false; - } else { - if (!problem_.modified_noise) - throw std::invalid_argument( - "a native noise field is valid only for modified noise"); - upload_span(execution_space_, crossing_ring_, noise_, noise); - crossed_.from_host += noise.size_bytes(); - has_noise_ = true; - } - mask(current_); - if (!stash_current_.empty()) - graph::deep_copy(execution_space_, stash_current_, 0.0); - execution_space_.fence("solprop_native set_initial"); - initialized_ = true; - rk_source_evaluations_ = 0; - table_ready_ = false; - stages_ready_ = false; - stash_is_current_ = false; - quadrature_error_ = Kokkos::Experimental::quiet_NaN_v; - quadrature_stale_ = false; - graph::deep_copy(execution_space_, quadrature_scores_, - Kokkos::Experimental::quiet_NaN_v); - clear_fsal(); -} - -void GeneralWorkspace::require_trial_scalars(const TrialScalars& scalars) const { - const auto checked_values = [](const std::vector& values, const char* name, - bool nonnegative, bool fraction) { - for (double value : values) - if (!std::isfinite(value) || (nonnegative && value < 0.0) || - (fraction && value > 1.0)) - throw std::invalid_argument(std::string(name) + " contains an invalid value"); - }; - if (!scalars.kerr_density.empty()) - require_extent(scalars.kerr_density.size(), problem_.n_stages, "kerr_density"); - checked_values(scalars.kerr_density, "kerr_density", false, false); - if (problem_.plasma && - (problem_.n_channels > 1 || !scalars.plasma_densities.empty() || - !scalars.plasma_preionisation.empty())) { - const auto count = - checked_product(problem_.n_stages, problem_.n_channels, "stage channel values"); - require_extent(scalars.plasma_densities.size(), count, "plasma_densities"); - require_extent(scalars.plasma_preionisation.size(), count, "plasma_preionisation"); - checked_values(scalars.plasma_densities, "plasma_densities", true, false); - checked_values(scalars.plasma_preionisation, "plasma_preionisation", true, true); - } - if (problem_.raman && - (problem_.n_raman_components > 1 || !scalars.raman_densities.empty())) { - require_extent(scalars.raman_densities.size(), - checked_product(problem_.n_stages, problem_.n_raman_components, - "stage Raman values"), - "raman_densities"); - checked_values(scalars.raman_densities, "raman_densities", true, false); - } - if (const auto* weights = dispersive_weights()) - weights->require_densities(scalars.dispersive_densities); - else if (!scalars.dispersive_densities.empty()) - throw std::invalid_argument("dispersive_densities weigh SEP 6's dispersive " - "response, and this problem has none"); - if (!general_ && !scalars.transform_weights.empty()) - throw std::invalid_argument("transform_weights weight a general modal guide's " - "moving maps, and this problem has none"); - require_extent(scalars.phi_delta.size(), - (problem_.n_stages + 1) * problem_.n_phi_terms, "phi_delta"); - require_extent(scalars.phi_at.size(), problem_.n_stages * problem_.n_phi_terms, - "phi_at"); - require_extent(scalars.prefactor.size(), - problem_.n_stages * problem_.n_prefactor_terms, "prefactor"); - require_extent(scalars.density.size(), problem_.n_stages, "density"); - require_extent(scalars.taper.size(), problem_.n_stages, "taper"); - if (problem_.plasma) - require_extent(scalars.preionisation.size(), problem_.n_stages, "preionisation"); - if (problem_.raman) { - require_extent(scalars.raman_rates.size(), - problem_.n_stages * problem_.n_raman_terms, "raman_rates"); - if (!std::ranges::all_of(scalars.raman_rates, [](const double value) { - return std::isfinite(value) && value >= 0.0; - })) - throw std::invalid_argument("Raman decay rates must be finite and non-negative"); - } -} - -// Point the per-stage scalar views at a slot of the packed buffer. -void GeneralWorkspace::bind_stage_scalars(const std::size_t slot) { - const auto stages = problem_.n_stages; - const auto channels = problem_.plasma ? problem_.n_channels : 0; - double* const base = stage_scalars_.data() + slot * stage_slot_size_; - const auto region = [&](const std::size_t index) { - return base + stage_offsets_[index]; - }; - const auto extent = [&](const std::size_t index) { - return stage_offsets_[index + 1] - stage_offsets_[index]; - }; - stage_phi_delta_ = RealView(region(0), extent(0)); - stage_phi_at_ = RealView(region(1), extent(1)); - stage_prefactor_ = RealView(region(2), extent(2)); - stage_density_ = RealView(region(3), stages); - stage_kerr_density_ = RealView(region(4), stages); - stage_plasma_density_ = RealView(region(5), stages); - stage_raman_density_ = RealView(region(6), stages); - stage_channel_density_ = RealView2(region(7), stages, channels); - stage_channel_preionisation_ = RealView2(region(8), stages, channels); - stage_taper_ = RealView(region(9), stages); - stage_preionisation_ = RealView(region(10), stages); - if (problem_.raman) { - stage_raman_rates_ = RealView2(region(11), stages, problem_.n_raman_terms); - if (problem_.n_raman_components > 1) - stage_raman_densities_ = - RealView2(region(12), stages, problem_.n_raman_components); - } - // A compensated trial's rate weights are its step's, in slot 0 whichever slot the - // stage views read. - stage_rate_weights_ = - RealView(stage_scalars_.data() + stage_offsets_[13], extent(13)); -} - -// The host's record of a trial's scalars beside the views: which stages' phase -// weights repeat the stage before's, and the dispersive response's densities. -void GeneralWorkspace::note_trial_scalars(const TrialScalars& scalars) { - // Rows of equal weights give bitwise equal phases (a zero weight's term is skipped). - const auto n_phi = problem_.n_phi_terms; - const auto same_row = [&](const std::size_t row) { - return std::equal( - scalars.phi_delta.begin() + static_cast(row * n_phi), - scalars.phi_delta.begin() + static_cast((row + 1) * n_phi), - scalars.phi_delta.begin() + static_cast((row - 1) * n_phi)); - }; - trial_taper_.assign(scalars.taper.begin(), scalars.taper.end()); - phase_repeats_.assign(problem_.n_stages, false); - for (std::size_t stage = 1; stage < problem_.n_stages; ++stage) - phase_repeats_[stage] = same_row(stage); - whole_repeats_ = problem_.n_stages > 0 && same_row(problem_.n_stages); - entered_stage_ = std::numeric_limits::max(); - if (auto* weights = dispersive_weights()) - weights->set_densities(scalars.dispersive_densities); -} - -// Fill the next staging buffer with the trial's scalars, at each view's place in -// slot 0 -- and an Adams trial's sample's in slot 1, and a compensated trial's rate -// weights -- then copy it across once (a host build's mirror is the buffer itself) -// and point the views at slot 0. `use_uploaded_sample` turns them to slot 1. -void GeneralWorkspace::upload_trial(const TrialScalars& scalars, - const TrialScalars* sample, - const std::vector* rate) { - auto staging = stage_staging_[stage_staging_next_]; - const bool staged = staging.data() != stage_scalars_.data(); - // The ring comes round: every earlier copy from it has run once the space drains. - if (staged && stage_staging_next_ == 0) - execution_space_.fence("solprop_native stage staging reuse"); - const auto& previous = - stage_staging_[(stage_staging_next_ + stage_staging_.size() - 1) % - stage_staging_.size()]; - stage_staging_next_ = (stage_staging_next_ + 1) % stage_staging_.size(); - if (staged) - // A device's staging buffer starts from the last values uploaded, so that a view - // this trial does not set keeps them, as the separate copies did. - std::copy(previous.data(), previous.data() + staging.size(), staging.data()); - const auto fill_slot = [&](const std::size_t slot, const TrialScalars& values) { - double* const base = staging.data() + slot * stage_slot_size_; - const auto place = [&](const std::size_t region, - const std::vector& source) { - if (source.size() != stage_offsets_[region + 1] - stage_offsets_[region]) - throw std::invalid_argument("fixed problem array has the wrong extent"); - std::ranges::copy(source, base + stage_offsets_[region]); - }; - const auto fill = [&](const std::size_t region, const double value) { - std::fill(base + stage_offsets_[region], base + stage_offsets_[region + 1], - value); - }; - place(0, values.phi_delta); - place(1, values.phi_at); - place(2, values.prefactor); - place(3, values.density); - place(4, values.kerr_density.empty() ? values.density : values.kerr_density); - if (problem_.plasma) { - if (values.plasma_densities.empty()) { - place(5, values.density); - place(7, values.density); - place(8, values.preionisation); - } else { - place(7, values.plasma_densities); - place(8, values.plasma_preionisation); - // The sole channel already has stage-major contiguous scalar vectors. - // Multiple channels consume their own matrices and need no extracted copy. - if (problem_.n_channels == 1) { - place(5, values.plasma_densities); - place(10, values.plasma_preionisation); - } - } - } - if (problem_.raman) { - if (values.raman_densities.empty()) - place(6, values.density); - else if (problem_.n_raman_components == 1) - place(6, values.raman_densities); - else - fill(6, 1.0); - place(11, values.raman_rates); - if (problem_.n_raman_components > 1) - place(12, values.raman_densities); - } - place(9, values.taper); - if (problem_.plasma && values.plasma_preionisation.empty()) - place(10, values.preionisation); - }; - fill_slot(0, scalars); - if (sample != nullptr) - fill_slot(1, *sample); - if (rate != nullptr) { - require_extent(rate->size(), problem_.n_phi_terms, "Adams rate weights"); - std::ranges::copy(*rate, staging.data() + stage_offsets_[13]); - } - graph::deep_copy(execution_space_, stage_scalars_, staging); - bind_stage_scalars(0); - note_trial_scalars(scalars); -} - -// An Adams trial's sample scalars, uploaded with its step's: the views turn to slot 1. -void GeneralWorkspace::use_uploaded_sample(const TrialScalars& sample) { - bind_stage_scalars(1); - note_trial_scalars(sample); -} - -void GeneralWorkspace::prepare_raman(const std::vector& rates, - const std::vector& densities) { - require_open(); - if (!problem_.raman) - throw std::logic_error("this state has no Raman workspace"); - require_extent(rates.size(), problem_.n_raman_terms, "Raman decay rates"); - if (!std::ranges::all_of(rates, [](const double value) { - return std::isfinite(value) && value >= 0.0; - })) - throw std::invalid_argument("Raman decay rates must be finite and non-negative"); - auto host = Kokkos::create_mirror_view(stage_raman_rates_); - for (std::size_t term = 0; term < problem_.n_raman_terms; ++term) - host(0, term) = rates[term]; - graph::deep_copy(execution_space_, stage_raman_rates_, host); - if (problem_.n_raman_components > 1) { - require_extent(densities.size(), problem_.n_raman_components, - "Raman amplitude densities"); - if (!std::ranges::all_of(densities, [](double value) { - return std::isfinite(value) && value >= 0.0; - })) - throw std::invalid_argument( - "Raman amplitude densities must be finite and non-negative"); - auto density_host = Kokkos::create_mirror_view(stage_raman_densities_); - for (std::size_t component = 0; component < densities.size(); ++component) - density_host(0, component) = densities[component]; - graph::deep_copy(execution_space_, stage_raman_densities_, density_host); - } - prepare_raman_stage(0, rates.data(), densities.data()); - execution_space_.fence("solprop_native Raman diagnostic boundary"); -} - -void GeneralWorkspace::prepare_raman_stage(const std::size_t stage, const double* rates, - const double* densities) { - bool unchanged = raman_kernel_ready_; - for (std::size_t term = 0; term < problem_.n_raman_terms; ++term) - unchanged = unchanged && cached_raman_rates_[term] == rates[term]; - if (problem_.n_raman_components > 1) - for (std::size_t component = 0; component < problem_.n_raman_components; - ++component) - unchanged = - unchanged && cached_raman_densities_[component] == densities[component]; - if (unchanged) - return; - for (std::size_t term = 0; term < problem_.n_raman_terms; ++term) - cached_raman_rates_[term] = rates[term]; - if (problem_.n_raman_components > 1) - for (std::size_t component = 0; component < problem_.n_raman_components; - ++component) - cached_raman_densities_[component] = densities[component]; - - Kokkos::Profiling::pushRegion("solprop_native.raman.kernel_rebuild"); - const auto undamped = raman_undamped_; - const auto weights = raman_tensor_weights_; - const auto kernel = raman_kernel_time_; - const auto stage_rates = stage_raman_rates_; - const auto amplitudes = stage_raman_densities_; - const auto row_component = raman_row_component_; - const auto fractions = raman_component_fraction_; - const bool multiple = problem_.n_raman_components > 1; - const bool tensor = problem_.effective_node_pol() == 2; - const auto terms = problem_.n_raman_terms; - const auto doubled = 2 * problem_.n_to; - const double fraction = problem_.raman_fraction; - const double dt = problem_.dt_fine; - launch::parallel_for( - "solprop_native.raman.kernel_rebuild", - Range2(raman_rebuild_space_, {0, 0}, {problem_.raman_kernels(), doubled}), - KOKKOS_LAMBDA(const std::size_t channel, const std::size_t time) { - double total = 0.0; - const double delay = static_cast(time) * dt; - if (multiple) { - for (std::size_t component = 0; component < amplitudes.extent(1); - ++component) { - double response = 0.0; - for (std::size_t term = 0; term < terms; ++term) { - if (static_cast(row_component(term)) == component) { - const double damped = undamped(term, time) * - Kokkos::exp(-stage_rates(stage, term) * delay); - response += tensor ? weights(term, channel) * damped : damped; - } - } - total += amplitudes(stage, component) * fractions(component) * response; - } - kernel(channel, time) = total; - } else { - for (std::size_t term = 0; term < terms; ++term) { - const double damped = - undamped(term, time) * Kokkos::exp(-stage_rates(stage, term) * delay); - total += tensor ? weights(term, channel) * damped : damped; - } - kernel(channel, time) = fraction * total; - } - }); - Kokkos::Profiling::popRegion(); - - Kokkos::Profiling::pushRegion("solprop_native.raman.kernel_fft"); - execute_raman(*raman_kernel_forward_plan_, raman_kernel_time_, raman_kernel_spectrum_, - KokkosFFT::Normalization::none); - const auto spectrum = raman_kernel_spectrum_; - launch::parallel_for( - "solprop_native.raman.kernel_dt_scale", - Range2(raman_scale_space_, {0, 0}, {problem_.raman_kernels(), problem_.n_to + 1}), - KOKKOS_LAMBDA(const std::size_t channel, const std::size_t index) { - spectrum(channel, index) *= dt; - }); - Kokkos::Profiling::popRegion(); - - raman_kernel_ready_ = true; - ++raman_kernel_transforms_; -} - -void GeneralWorkspace::prepare_trial(const double dz, const TrialScalars& scalars) { - require_open(); - if (problem_.low_storage) - throw std::logic_error( - "trial_step: a low-storage state of " + std::to_string(current_.extent(0)) + - " x " + std::to_string(current_.extent(1)) + - " holds no Runge-Kutta views; its steps are low_storage_step's"); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - require_trial_scalars(scalars); - coarse_pending_ = false; - upload_trial(scalars); - if (flow_) - flow_->set_trial(scalars.flow_section, scalars.flow_weights); - if (general_) - general_->set_weights(scalars.transform_weights); - std::size_t first = 0; - if (has_recycled_) { - reuse_fsal(); - first = 1; - } - // A stage's finish runs in the next stage's entry kernel (solprop SEP 25 section - // 6.7) unless a coupled flow carries the stage back first, or the stage is the last. - finish_pending_ = false; - const bool deferrable = !flow_ && !problem_.separable_geometry(); - defer_quadrature_ = deferred_quadrature_eligible(); - for (std::size_t stage = first; stage < problem_.n_stages; ++stage) { - try { - if (problem_.raman) - prepare_raman_stage( - stage, scalars.raman_rates.data() + stage * problem_.n_raman_terms, - problem_.n_raman_components > 1 - ? scalars.raman_densities.data() + stage * problem_.n_raman_components - : nullptr); - defer_finish_ = deferrable && stage + 1 < problem_.n_stages; - project_coarse_ = !defer_quadrature_ || stage + 1 == problem_.n_stages; - evaluate_stage(stage, dz); - } catch (...) { - defer_finish_ = false; - finish_pending_ = false; - defer_quadrature_ = false; - project_coarse_ = true; - coarse_pending_ = false; - throw; - } - } - defer_finish_ = false; - defer_quadrature_ = false; - project_coarse_ = true; - if (finish_pending_) - throw std::logic_error("a deferred stage finish was never run"); - stages_ready_ = true; -} - -std::vector GeneralWorkspace::trial_step(const double dz, - const TrialScalars& scalars) { - prepare_trial(dz, scalars); - if (flow_) - flow_->carry_step(current_, solution_, error_); - const auto current = current_; - const auto carried = flow_ ? flow_->carried_state() : current_; - const auto solution = flow_ ? flow_->carried_solution() : solution_; - const auto error = flow_ ? flow_->carried_error() : error_; - const auto whole = whole_propagator_; - const auto advanced = advanced_; - const auto residual = residual_; - const auto in_norm = norm_mask_; - const auto partials = partials_; - const auto count = problem_.n_columns(); - const auto n = problem_.n_omega; - const auto slots = reduction_slots_; - accept_update( - execution_space_, count, n, dz, - KOKKOS_LAMBDA(const std::size_t column) { - const auto slot = column * slots; - return UpdateColumn{.current = current.data() + column * current.stride(0), - .solution = solution.data() + column * solution.stride(0), - .error = error.data() + column * error.stride(0), - .whole = whole.data() + column * whole.stride(0), - .advanced = advanced.data() + column * advanced.stride(0), - .residual = residual.data() + column * residual.stride(0), - .in_norm = in_norm.data(), - .old_partial = partials.data() + slot, - .new_partial = partials.data() + partials.stride(0) + slot, - .error_partial = - partials.data() + 2 * partials.stride(0) + slot, - .carried = carried.data() + column * carried.stride(0)}; - }, - "solprop_native.step.accept_error_partials", slots); - return collect_trial(); -} - -// solprop's low-call Adams march (SEP 13, SEP 14 phase 2): one source evaluation at a -// field given in the step's anchor frame, `U(anchor, position) N(U(position, anchor) -// v)`. It is stage zero of a trial whose stage-zero scalars are the evaluation -// position's: the combination with no earlier stage carries the input to the position -// and the finish brings the source back, so the shared stage pipeline, its responses -// and its in-stage absorber are the march's too. The accepted state is held in -// `advanced_`, which only the Runge-Kutta accept writes, while the input takes its -// place. -std::vector GeneralWorkspace::adams_source(const std::vector& input, - const TrialScalars& scalars) { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - // Every route's stage source serves the Adams march (solprop SEP 16 phase 3): the - // separable stage takes its flow from the stage's own Phi increment, as the modal one - // does. Beside a coupled flow the sample is the material source alone (solprop SEP - // 16 phase 5): the exponential formulation carries the residual in its compensation - // and the flow in its moments, so the stage runs without the flow's pipeline. - require_extent(input.size(), problem_.n_columns() * problem_.n_omega, "Adams input"); - // The flow is set aside for the evaluation and put back whatever happens. - auto held_flow = std::exchange(flow_, nullptr); - // An aggregate, so it declares no copy or move; it is never copied or moved. - // NOLINTNEXTLINE(cppcoreguidelines-special-member-functions): a local scope guard - struct RestoreFlow { - std::unique_ptr& slot; - std::unique_ptr& held; - // NOLINTNEXTLINE(modernize-use-equals-default): restores the slot (see FlowAside). - ~RestoreFlow() { slot = std::move(held); } - } restore{.slot = flow_, .held = held_flow}; - require_trial_scalars(scalars); - upload_trial(scalars); - if (general_) - general_->set_weights(scalars.transform_weights); - graph::deep_copy(execution_space_, advanced_, current_); - copy_vector(execution_space_, current_, input); - if (problem_.raman) - prepare_raman_stage(0, scalars.raman_rates.data(), - problem_.n_raman_components > 1 ? scalars.raman_densities.data() - : nullptr); - evaluate_stage_source(0, 0.0); - graph::deep_copy(execution_space_, current_, advanced_); - stages_ready_ = true; - // No Runge-Kutta stage of this state exists to recycle. - pending_fsal_ = false; - has_recycled_ = false; - return stage_to_host(0); -} - -// Adopt an Adams step's end state, given on the host before its boundary treatments: -// the shared finish masks it, or applies the split-step absorber, and makes it current. -void GeneralWorkspace::adopt(const std::vector& field, const double dz) { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - require_extent(field.size(), problem_.n_columns() * problem_.n_omega, "Adams state"); - copy_vector(execution_space_, advanced_, field); - finish(dz); - // The stash a response pass left is the predicted field's, not this one's. - stash_is_current_ = false; -} - -// --- solprop SEP 24 phase 3: the low-storage RK4IP march ----------------------------- -// -// Classical RK4 in the interaction picture anchored at the step's midpoint (solprop's -// backends/low_storage.py, SEP 24 section 4.3). The state is the register A; B -// accumulates the bracket of the accepted step and C holds the next pass's lab-frame -// input, unless it is elided (solprop's `low_storage_stage_register`): then each of the -// first three finishes writes that input straight into the transverse scratch. A step -// is four passes, each forming its input in the route's scratch, running the stage core -// the Runge-Kutta march shares (`separable_core`, `modal_core`; SEP 24 Q5) and folding -// the source into the registers per element: -// -// pass 1, F1 at z_n: A <- P A; B <- A + h/6 P S; C <- A + h/2 P S -// pass 2, F2 at z_m: B <- B + h/3 S; C <- A + h/2 S -// pass 3, F3 at z_m: B <- B + h/3 S; C <- Q (A + h S) -// pass 4, F4 at end: A <- Q B + h/6 S -// -// Every pass reads the source at its own position, so nothing carries it back to an -// anchor, and only forward flows are applied. The step is accepted in place; the -// caller's finish masks it or applies the split-step collar. - -namespace { - -constexpr int fold_complete = 0; -constexpr int fold_exchange = 1; -constexpr int fold_second = 2; -constexpr int term_none = 0; -constexpr int term_material = 1; -constexpr int term_absorber = 2; - -// The scalar row each pass reads: the step's start, its midpoint twice, its end. -constexpr std::size_t low_storage_rows[4] = {0, 1, 1, 2}; - -// One evaluation's fold into the registers at one element: solprop's -// `bodies.low_storage_fold`, the finish column of SEP 24 section 4.3. A pass that -// evaluates twice -- the material response, then the in-stage absorber's term, from the -// same input -- folds the first at the exchange (pass 3's C left unflowed) and the -// second after it, whose sum is the pass's. -KOKKOS_INLINE_FUNCTION void -low_storage_fold(const int stage, const int mode, const double step, - const Complex source, Complex& state, Complex& accumulator, - Complex& stage_register, const Complex first, const Complex second) { - if (stage == 0) { - const Complex slope = first * source; - if (mode == fold_second) { - accumulator = accumulator + (step / 6.0) * slope; - stage_register = stage_register + (0.5 * step) * slope; - return; - } - const Complex moved = first * state; - state = moved; - accumulator = moved + (step / 6.0) * slope; - stage_register = moved + (0.5 * step) * slope; - return; - } - if (stage == 1) { - accumulator = accumulator + (step / 3.0) * source; - stage_register = - (mode == fold_second ? stage_register : state) + (0.5 * step) * source; - return; - } - if (stage == 2) { - accumulator = accumulator + (step / 3.0) * source; - if (mode == fold_complete) - stage_register = second * (state + step * source); - else if (mode == fold_exchange) - stage_register = state + step * source; - else - stage_register = second * (stage_register + step * source); - return; - } - if (mode == fold_second) { - state = state + (step / 6.0) * source; - return; - } - state = second * accumulator + (step / 6.0) * source; -} - -// A register traversal over the state's elements. The last fold of a step also reduces -// the accepted state's squared norm per column into `partials(0, column)`, one team per -// column as the Runge-Kutta accept reduces its own (its sums associate as the team -// splits them, solprop SEP 25). -template -void register_traversal(const char* label, const ExecutionSpace& execution, - const std::size_t columns, const std::size_t n, - const RealView2& partials, const bool reduce, - const Element& element) { - const auto traversal_for = [](const char* name, const ExecutionSpace& space, - const std::size_t rows, const std::size_t width, - const auto& body) { - if constexpr (Crossing) - crossing_for(name, space, rows, width, body); - else - cartesian_for(name, space, rows, width, body); - }; - if (!reduce) { - traversal_for( - label, execution, columns, n, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - static_cast(element(column, index)); - }); - return; - } -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - if constexpr (Crossing) { - // Blocks of adjacent columns, each column's sum in ascending bin order. - constexpr std::size_t block = crossing_tile_columns; - constexpr std::size_t bins = crossing_tile_bins; - const std::size_t blocks = (columns + block - 1) / block; - launch::parallel_for( - label, Kokkos::RangePolicy(execution, 0, blocks), - KOKKOS_LAMBDA(const std::size_t which) { - const std::size_t first = which * block; - const std::size_t count = columns - first < block ? columns - first : block; - double totals[block] = {}; - for (std::size_t start = 0; start < n; start += bins) { - const std::size_t stop = n - start < bins ? n : start + bins; - for (std::size_t index = start; index < stop; ++index) - for (std::size_t offset = 0; offset < count; ++offset) - totals[offset] += element(first + offset, index); - } - for (std::size_t offset = 0; offset < count; ++offset) - partials(0, first + offset) = totals[offset]; - }); - return; - } -#endif - using Policy = Kokkos::TeamPolicy; - using Member = Policy::member_type; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const Policy policy(execution, static_cast(columns), - SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); - launch::parallel_for( - label, policy, KOKKOS_LAMBDA(const Member& team) { - const auto column = static_cast(team.league_rank()); - double total = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n)), - [&](const int index, double& update) { - update += element(column, static_cast(index)); - }, - total); - Kokkos::single(Kokkos::PerTeam(team), [&]() { partials(0, column) = total; }); - }); -#else - const Policy policy(execution, static_cast(columns), - host_reduction_team_size(execution, columns, n)); - launch::parallel_for( - label, policy, KOKKOS_LAMBDA(const Member& team) { - const auto column = static_cast(team.league_rank()); - double total = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, n), - [&](const std::size_t index, double& update) { - update += element(column, index); - }, - total); - Kokkos::single(Kokkos::PerTeam(team), [&] { partials(0, column) = total; }); - }); -#endif -} - -} // namespace - -double GeneralWorkspace::low_storage_step(const double dz, const TrialScalars& scalars, - const bool rebuild_table) { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - if (!problem_.low_storage) - throw std::logic_error("this native state was not built for the low-storage march"); - if (!(dz > 0.0) || !std::isfinite(dz)) - throw std::invalid_argument("the low-storage step must be positive and finite"); - if (problem_.n_stages < 3) - throw std::invalid_argument("the low-storage march reads three scalar rows"); - if (rebuild_table && problem_.low_storage_flows != 1) - throw std::logic_error("only a constant operator's table is rebuilt"); - if (problem_.low_storage_flows == 1 && !rebuild_table && !table_ready_) - throw std::logic_error("the low-storage table has not been built since the initial " - "field was loaded"); - require_trial_scalars(scalars); - upload_trial(scalars); - if (general_) - general_->set_weights(scalars.transform_weights); - low_storage_flows_formed(rebuild_table); - if (rebuild_table) - table_ready_ = true; - for (std::size_t pass = 0; pass < 4; ++pass) { - const auto row = low_storage_rows[pass]; - if (problem_.raman) - prepare_raman_stage( - row, scalars.raman_rates.data() + row * problem_.n_raman_terms, - problem_.n_raman_components > 1 - ? scalars.raman_densities.data() + row * problem_.n_raman_components - : nullptr); - if (problem_.separable_geometry()) - low_storage_separable_pass(pass, dz); - else - low_storage_modal_pass(pass, dz); - } - // No Runge-Kutta stage exists to observe or recycle. - stages_ready_ = false; - stash_is_current_ = false; - execution_space_.fence("solprop_native low-storage step"); - double total = 0.0; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // One number crosses, not the whole of the partials' table. - const auto partials = partials_; - launch::parallel_reduce( - "solprop_native.low_storage.norm_total", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_columns()), - KOKKOS_LAMBDA(const std::size_t column, double& sum) { - sum += partials(0, column); - }, - Kokkos::Sum(total)); -#else - for (std::size_t column = 0; column < problem_.n_columns(); ++column) - total += partials_(0, column); -#endif - return total; -} - -// The step's held flows: a constant operator's table `exp(h_t D / 2)`, built from the -// table's interval (scalar row 2 of `phi_delta`) when the caller's key changed, or a -// varying operator's `P` and `Q` from rows 0 and 1, every step. Recomputed flows are -// formed in the finishes that apply them and hold nothing. -void GeneralWorkspace::low_storage_flows_formed(const bool rebuild_table) { - const auto flows = low_storage_flows_; - const auto count = problem_.low_storage_flows; - if (count == 0 || (count == 1 && !rebuild_table)) - return; - const auto n = problem_.n_omega; -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // With vector maths a host task forms a column's exponents a block of bins at a - // time and their exponentials in one pass (solprop ADR-059); a separable route's - // task, a block of adjacent columns at one bin, its table's run in the scratch's - // layout (`RegisterIndex`). - if constexpr (vmath::vectorised) { - if (problem_.separable_geometry()) { - const auto spatial = spatial_; - const auto delta = stage_phi_delta_; - const auto active = active_mask_; - const auto index = register_index(); - const auto columns = problem_.n_columns(); - const std::size_t blocks = (columns + stage_block - 1) / stage_block; - launch::parallel_for( - "solprop_native.low_storage.separable_table", - Kokkos::RangePolicy(execution_space_, 0, n * blocks), - KOKKOS_LAMBDA(const std::size_t item) { - const std::size_t omega = item / blocks; - const std::size_t first = (item % blocks) * stage_block; - const std::size_t width = - columns - first < stage_block ? columns - first : stage_block; - const bool in_band = active(omega) != 0U; - double real[stage_block]; - double imag[stage_block]; - for (std::size_t offset = 0; offset < width; ++offset) { - const Complex x = - delta(2) * - spatial.value(index.mode(first + offset), omega, in_band).linear; - real[offset] = x.real(); - imag[offset] = x.imag(); - } - vmath::exp_complex(real, imag, real, imag, width); - for (std::size_t offset = 0; offset < width; ++offset) - flows.data()[index(first + offset, omega)] = - Complex(real[offset], imag[offset]); - }); - return; - } - const auto terms = stage_terms(0); - const auto n_phi = problem_.n_phi_terms; - const std::size_t blocks = (n + stage_block - 1) / stage_block; - launch::parallel_for( - "solprop_native.low_storage.modal_flows", - Kokkos::RangePolicy(execution_space_, 0, - problem_.n_columns() * blocks), - KOKKOS_LAMBDA(const std::size_t item) { - const std::size_t column = item / blocks; - const std::size_t first = (item % blocks) * stage_block; - const std::size_t bins = n - first < stage_block ? n - first : stage_block; - double real[stage_block]; - double imag[stage_block]; - const auto form = [&](const std::size_t flow, const std::size_t offset) { - terms.phase_block(offset, column, first, bins, real, imag); - vmath::exp_complex(real, imag, real, imag, bins); - for (std::size_t bin = 0; bin < bins; ++bin) - flows(flow, column, first + bin) = Complex(real[bin], imag[bin]); - }; - if (count == 1) { - form(0, 2 * n_phi); - } else { - form(0, 0); - form(1, n_phi); - } - }); - return; - } -#endif - if (problem_.separable_geometry()) { - const auto spatial = spatial_; - const auto delta = stage_phi_delta_; - const auto active = active_mask_; - const auto n_pol = problem_.n_pol_state; - const auto index = register_index(); - cartesian_for( - "solprop_native.low_storage.separable_table", execution_space_, - problem_.n_columns(), n, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - const auto values = spatial.value(column / n_pol, omega, active(omega) != 0U); - flows.data()[index(column, omega)] = propagator_exp(delta(2) * values.linear); - }); - return; - } - const auto terms = stage_terms(0); - const auto n_phi = problem_.n_phi_terms; - cartesian_for( - "solprop_native.low_storage.modal_flows", execution_space_, problem_.n_columns(), - n, KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - if (count == 1) { - flows(0, column, index) = - propagator_exp(terms.phase_sum(2 * n_phi, column, index)); - } else { - flows(0, column, index) = propagator_exp(terms.phase_sum(0, column, index)); - flows(1, column, index) = - propagator_exp(terms.phase_sum(n_phi, column, index)); - } - }); -} - -// One pass on a separable geometry (free space and the hybrid planar kind). Its input -// is the register A (pass 1) or C, with the noise draw at the pass's position, in the -// transverse scratch -- already written by the previous finish when C is elided. Under -// an in-stage absorber the pass evaluates twice from the same input: the material -// source, then the absorber's term, the exchange between them taking the material -// output out of the scratch and re-forming the input (SEP 24 section 4.3). -void GeneralWorkspace::low_storage_separable_pass(const std::size_t pass, - const double dz) { - const bool material = problem_.nonlinear || problem_.plasma || problem_.raman || - separable_dispersive_ != nullptr; - const bool absorber = problem_.in_stage_absorber; - const bool elided = !problem_.low_storage_stage_register; - const auto row = low_storage_rows[pass]; - if ((material || absorber) && (pass == 0 || !elided)) - low_storage_separable_input(pass); - if (material && absorber) { - separable_core(row, true); - low_storage_separable_fold(pass, fold_exchange, term_material, dz); - separable_core(row, false); - low_storage_separable_fold(pass, fold_second, term_absorber, dz); - } else if (material || absorber) { - separable_core(row, material); - low_storage_separable_fold(pass, fold_complete, - material ? term_material : term_absorber, dz); - } else { - low_storage_separable_fold(pass, fold_complete, term_none, dz); - } -} - -namespace { -// Each column's masked squared norm of `state` into `partials(0, column)`, in a team -// per column: a device's final fold traverses like the others and reduces here, over -// one volume, instead of reducing inside the fold's crossing (SEP 28). -void column_norms(const char* label, const ExecutionSpace& execution, - const ComplexView2& state, const ByteView& in_norm, - const RealView2& partials) { - using Policy = Kokkos::TeamPolicy; - using Member = Policy::member_type; - const auto n = state.extent(1); - launch::parallel_for( - label, Policy(execution, static_cast(state.extent(0)), 128), - KOKKOS_LAMBDA(const Member& team) { - const auto column = static_cast(team.league_rank()); - double total = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n)), - [&](const int index, double& update) { - const auto omega = static_cast(index); - if (in_norm(omega) != 0U) - update += squared_abs(state(column, omega)); - }, - total); - Kokkos::single(Kokkos::PerTeam(team), [&] { partials(0, column) = total; }); - }); -} -} // namespace - -void GeneralWorkspace::low_storage_separable_input(const std::size_t pass) { - const auto state = current_; - const auto stage_register = register_stage_; - const auto index = stage_index(); - const auto spatial = spatial_; - const auto at = stage_phi_at_; - const auto active = active_mask_; - const auto noise = noise_; - const auto scratch = transverse_; - const bool add_noise = problem_.modified_noise && has_noise_; - const double scale = inverse_transverse_count(); - const auto row = low_storage_rows[pass]; - const auto input = KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - const auto mode = index.mode(column); - Complex value = - pass == 0 ? state(column, omega) : stage_register.data()[index(column, omega)]; - if (add_noise) { - const auto values = spatial.value(mode, omega, active(omega) != 0U); - const double angle = at(row) * values.linear.imag(); - value += noise(column, omega) * Complex(Kokkos::cos(angle), Kokkos::sin(angle)); - } - return value * scale; - }; - // In the volume route's order the scratch is the state's, so the input streams. - if (problem_.low_storage_tiled_input && !volume_order_) { - // Read along the state's bins and written along the scratch's rows, a tile at a - // time. - tiled_crossing( - "solprop_native.low_storage.separable_input", execution_space_, - problem_.n_columns(), problem_.n_omega, false, input, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, - const Complex value) { - scratch.data()[index.cell(column, omega)] = value; - }); - return; - } - crossing_for( - "solprop_native.low_storage.separable_input", execution_space_, - problem_.n_columns(), problem_.n_omega, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - scratch.data()[index.cell(column, omega)] = input(column, omega); - }); -} - -void GeneralWorkspace::low_storage_separable_fold(const std::size_t pass, - const int mode, const int term, - const double dz) { - const auto state = current_; - const auto accumulator = register_accumulator_; - const auto stage_register = register_stage_; - const auto flows = low_storage_flows_; - const bool tabulated = problem_.low_storage_flows != 0; - const auto scratch = transverse_; - const auto spatial = spatial_; - const auto window = omega_window_; - const auto active = active_mask_; - const auto in_norm = norm_mask_; - const auto noise = noise_; - const auto delta = stage_phi_delta_; - const auto at = stage_phi_at_; - // A device gathers and scatters the scratch through its own tiles. - [[maybe_unused]] const auto n_pol = problem_.n_pol_state; - [[maybe_unused]] const auto layout = transverse_layout(); - const auto index = stage_index(); - const bool add_noise = problem_.modified_noise && has_noise_; - const double scale = inverse_transverse_count(); - const int stage = static_cast(pass); - const bool exchange = mode == fold_exchange; - const bool second = mode == fold_second; - // Elided, each of the first three passes writes the next pass's input in place of C. - const bool forward = !problem_.low_storage_stage_register && pass < 3; - const bool final = pass == 3 && !exchange; - // What this fold reads and writes, invariant over the traversal: each moves only the - // registers its row of the flow table names. - const bool flows_here = - stage == 0 || (stage == 2 && !exchange) || (stage == 3 && !second); - const bool reads_register = (second && stage < 3) || (exchange && stage > 0); - const bool reads_accumulator = stage > 0 || second; - const bool writes_state = (stage == 0 && !second) || stage == 3; - const bool writes_register = stage < 3 && !forward; - const auto own = low_storage_rows[pass]; - const auto next = low_storage_rows[pass < 3 ? pass + 1 : 3]; - const bool linear_here = add_noise || (flows_here && !tabulated); - const std::size_t flow_row = stage == 0 ? 0 : 1; - const double step = dz; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // A device reduces the accepted state's norms in a pass of their own. - const bool fused_norm = false; -#else - const bool fused_norm = final; -#endif - // The element, given its cell of the transverse scratch and its state (`held`). - const auto body = KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, - Complex& cell, Complex& held) { - const auto column_mode = index.mode(column); - const bool in_band = active(omega) != 0U; - // Tabulated flows without noise, or a pass with no flow, read only the prefactor of - // the element. - SpatialElement values{}; - if (linear_here) - values = spatial.value(column_mode, omega, in_band); - else if (term == term_material) - values.prefactor = spatial.prefactor(column_mode, omega, in_band); - Complex source{}; - if (term == term_material) - source = values.prefactor * window(omega) * cell; - else if (term == term_absorber && in_band) - source = cell; - const std::size_t slot = index(column, omega); - Complex flow(1.0, 0.0); - if (flows_here) - flow = tabulated ? flows.data()[slot] - : propagator_exp(delta(flow_row) * values.linear); - Complex a = held; - Complex b = reads_accumulator ? accumulator.data()[slot] : Complex{}; - Complex c = reads_register ? stage_register.data()[slot] : Complex{}; - if (exchange) { - Complex value = stage == 0 ? a : c; - if (add_noise) { - const double angle = at(own) * values.linear.imag(); - value += noise(column, omega) * Complex(Kokkos::cos(angle), Kokkos::sin(angle)); - } - cell = value * scale; - } - low_storage_fold(stage, mode, step, source, a, b, c, flow, flow); - if (forward) { - Complex value = c; - if (add_noise) { - const double angle = at(next) * values.linear.imag(); - value += noise(column, omega) * Complex(Kokkos::cos(angle), Kokkos::sin(angle)); - } - cell = value * scale; - } - if (writes_state) - held = a; - if (stage < 3) - accumulator.data()[slot] = b; - if (writes_register) - stage_register.data()[slot] = c; - return final && in_norm(omega) != 0U ? squared_abs(a) : 0.0; - }; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // In the volume route's order, the scratch's cell sits beside the registers' and the - // state's, so the fold streams all three. - if (volume_order_) - cartesian_for( - "solprop_native.low_storage.separable_fold", execution_space_, - problem_.n_columns(), problem_.n_omega, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - static_cast(body(column, omega, - scratch.data()[index.cell(column, omega)], - state(column, omega))); - }); - else - tiled_update( - "solprop_native.low_storage.separable_fold", execution_space_, - problem_.n_columns(), problem_.n_omega, exchange || forward, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - return layout.cell(scratch, omega * n_pol + column % n_pol, column / n_pol); - }, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, - Complex& cell) { - static_cast(body(column, omega, cell, state(column, omega))); - }, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, - const Complex value) { - layout.cell(scratch, omega * n_pol + column % n_pol, column / n_pol) = value; - }); -#else - // The host's registers sit at the scratch cell's own position (`RegisterIndex`), so a - // block of adjacent columns reads them, and the scratch, in contiguous runs at every - // bin; only the state is column-major. A work item takes a block of columns over - // every bin, a tile of bins at a time: it loads the tile's state a column (a - // contiguous run) at a time, folds the tile in the scratch's order, and stores the - // state back. The final fold sums each column's norm in ascending bin order (as - // `register_traversal`). - constexpr std::size_t block = crossing_tile_columns; - constexpr std::size_t bins = crossing_tile_bins; - const std::size_t columns = problem_.n_columns(); - const std::size_t n = problem_.n_omega; - const std::size_t blocks = (columns + block - 1) / block; - const auto partials = partials_; - launch::parallel_for( - "solprop_native.low_storage.separable_fold", - Kokkos::RangePolicy(execution_space_, 0, blocks), - KOKKOS_LAMBDA(const std::size_t which) { - const std::size_t first = which * block; - const std::size_t count = columns - first < block ? columns - first : block; - double totals[block] = {}; - Complex tile[bins][block]; - for (std::size_t start = 0; start < n; start += bins) { - const std::size_t width = n - start < bins ? n - start : bins; - for (std::size_t offset = 0; offset < count; ++offset) { - const Complex* row = &state(first + offset, start); - for (std::size_t bin = 0; bin < width; ++bin) - tile[bin][offset] = row[bin]; - } - for (std::size_t bin = 0; bin < width; ++bin) - for (std::size_t offset = 0; offset < count; ++offset) { - const std::size_t column = first + offset; - const std::size_t omega = start + bin; - totals[offset] += - body(column, omega, scratch.data()[index(column, omega)], - tile[bin][offset]); - } - if (writes_state) - for (std::size_t offset = 0; offset < count; ++offset) { - Complex* row = &state(first + offset, start); - for (std::size_t bin = 0; bin < width; ++bin) - row[bin] = tile[bin][offset]; - } - } - if (fused_norm) - for (std::size_t offset = 0; offset < count; ++offset) - partials(0, first + offset) = totals[offset]; - }); -#endif - if (final && !fused_norm) - column_norms("solprop_native.low_storage.separable_norms", execution_space_, state, - in_norm, partials_); -} - -// One pass on a modal route (mode sets, radial, general modal): its input, the -// register A (pass 1) or C with the noise draw at the pass's position, padded into the -// fine spectrum (and the general route's scaled channels); the shared core; and one -// fold of the source -- prefactor, window and the in-stage absorber's term at the -// pass's position, with no flow back to an anchor. -void GeneralWorkspace::low_storage_modal_pass(const std::size_t pass, const double dz) { - const auto row = low_storage_rows[pass]; - const bool responds = general_ - ? general_->responds() || problem_.in_stage_absorber - : problem_.nonlinear || problem_.plasma || problem_.raman || - problem_.in_stage_absorber || dispersive_ != nullptr; - if (responds) - low_storage_modal_input(pass); - const bool evaluated = modal_core(row); - const auto terms = stage_terms(row); - const auto state = current_; - const auto accumulator = register_accumulator_; - const auto stage_register = register_stage_; - const auto flows = low_storage_flows_; - const auto count = problem_.low_storage_flows; - const auto in_norm = norm_mask_; - const auto n_phi = problem_.n_phi_terms; - const auto n_prefactor = problem_.n_prefactor_terms; - const int stage = static_cast(pass); - const bool final = pass == 3; - const bool flows_here = stage != 1; - const std::size_t flow_row = stage == 0 ? 0 : 1; - const double step = dz; - register_traversal( - "solprop_native.low_storage.modal_fold", execution_space_, problem_.n_columns(), - problem_.n_omega, partials_, final, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - Complex source{}; - if (evaluated) { - // Zero weights skipped, as in `StageTerms::finish`. - Complex prefactor{}; - for (std::size_t term = 0; term < n_prefactor; ++term) { - const double weight = terms.prefactor_weights(row * n_prefactor + term); - if (weight != 0.0) - prefactor += weight * terms.prefactor_terms(term, column, index); - } - source = prefactor * terms.window(index) * - (terms.general ? terms.general_source(column, index) - : terms.projected(column, terms.crop(index))); - if (terms.in_stage_absorber && terms.active(index) != 0U) - source += terms.absorber(column, terms.crop(index)); - } - Complex flow(1.0, 0.0); - if (flows_here) - flow = count == 1 ? flows(0, column, index) - : count == 2 - ? flows(flow_row, column, index) - : propagator_exp(terms.phase_sum(flow_row * n_phi, column, index)); - Complex a = state(column, index); - Complex b = stage > 0 ? accumulator(column, index) : Complex{}; - Complex c{}; - low_storage_fold(stage, fold_complete, step, source, a, b, c, flow, flow); - if (stage == 0 || stage == 3) - state(column, index) = a; - if (stage < 3) { - accumulator(column, index) = b; - stage_register(column, index) = c; - } - return final && in_norm(index) != 0U ? squared_abs(a) : 0.0; - }); -} - -void GeneralWorkspace::low_storage_modal_input(const std::size_t pass) { - const auto source = pass == 0 ? current_ : register_stage_; - const auto row = low_storage_rows[pass]; - const auto terms = stage_terms(row); - const auto n = problem_.n_omega; - const auto gaps = fine_spectrum_size() - n; - cartesian_for( - "solprop_native.low_storage.modal_input", execution_space_, problem_.n_columns(), - n, KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - Complex value = source(column, index); - if (terms.add_noise) { - const Complex phase_at = terms.noise_phase(row, column, index); - value += terms.noise(column, index) * - Complex(Kokkos::cos(phase_at.imag()), Kokkos::sin(phase_at.imag())); - } - const double analytic_scale = - terms.analytic && index > 0 && 2 * terms.crop(index) < terms.n_to ? 2.0 - : 1.0; - terms.fine(column, terms.crop(index)) = - analytic_scale * terms.inverse_n_to * value; - if (terms.general) - terms.general_scaled(column, index) = - value * - (terms.general_pad * general_doubling(terms.crop(index), terms.n_to)); - for (std::size_t gap = index; gap < gaps; gap += n) - terms.zero_gaps(column, gap, gap + 1); - }); -} - -// --- solprop SEP 16 phase 3b: the Adams history resident in the engine --------------- -// -// solprop's HostAdamsMarch keeps the controller and hands every state-sized operation -// of its fused path to an engine (solprop ADR-043). These are that engine's passes: the -// history's rows, the trial's predictor, source, corrected field and defect norms, the -// adoption and transport of an accepted step and dense observation all stay in device -// memory; only the trial's scalars, its coefficient tables (by value, in AdamsWeights) -// and the norms cross. Each pass is the host engine's arithmetic (solprop's -// backends/multistep.py, HostHistory and AdamsKernels), checked against it in parity. - -void GeneralWorkspace::adams_allocate(const std::size_t slots, const bool compensated, - const bool interpolate) { - require_open(); - if (slots == 0 || slots > adams_max_samples) - throw std::invalid_argument("the Adams history needs 1 to 8 rows"); - if (compensated && problem_.separable_geometry()) - throw std::logic_error("a separable operator needs no compensation"); - const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - if (compensated && adams_states_.extent(0) != slots) { - adams_states_ = ComplexView3("solprop_native.adams.states", slots, columns, n); - adams_rates_ = ComplexView3("solprop_native.adams.rates", slots, columns, n); - adams_carried_ = ComplexView3("solprop_native.adams.carried", slots, columns, n); - adams_lab_ = ComplexView2("solprop_native.adams.predicted", columns, n); - adams_endpoint_ = ComplexView2("solprop_native.adams.endpoint", columns, n); - adams_endpoint_rate_ = - ComplexView2("solprop_native.adams.endpoint_rate", columns, n); - adams_rate_weights_ = Kokkos::View( - "solprop_native.adams.rate_weights", problem_.n_phi_terms); - } - // Dense observations are opt-in. Keeping their slopes and saved fields on a run - // that lands on its planes costs (slots + 4) whole fields without reading them. - if (interpolate) { - if (adams_slopes_.extent(0) != slots) { - adams_slopes_ = - ComplexView3("solprop_native.adams.dense_slopes", slots, columns, n); - adams_held_ = ComplexView2("solprop_native.adams.held_state", columns, n); - adams_dense_start_ = ComplexView2("solprop_native.adams.dense_start", columns, n); - adams_dense_end_ = ComplexView2("solprop_native.adams.dense_end", columns, n); - adams_dense_exponent_ = - ComplexView2("solprop_native.adams.dense_exponent", columns, n); - } - if (compensated && adams_dense_terms_.extent(0) != slots) - adams_dense_terms_ = - ComplexView3("solprop_native.adams.dense_terms", slots, columns, n); - } else { - adams_slopes_ = {}; - adams_held_ = {}; - adams_dense_start_ = {}; - adams_dense_end_ = {}; - adams_dense_exponent_ = {}; - adams_dense_terms_ = {}; - adams_dense_count_ = 0; - } - // Beside a coupled flow the trial's block functions, and a frozen-residual flow's - // weights per sample, live beside the history (solprop SEP 16 phase 6). - if (flow_) { - if (!flow_->forms_actions()) - throw std::logic_error("the resident Adams engine needs the flow's eigenbasis " - "and route"); - // A bin off the route takes the augmented exponential: a trial reads one - // section, so the scratch is one row per active bin, whatever the sections. A - // device's carry also keeps a bin's channel vectors there, in blocks 3 and 4 - // (solprop SEP 47 phase 4), which `augmented_blocks` holds for any history. - const auto rows = flow_->n_fallback() == 0 ? 0 : flow_->n_active(); - const auto blocks = flow_functions::augmented_blocks(slots); - if (rows == 0) - adams_fallback_ = {}; - else if (adams_fallback_.extent(0) != rows || adams_fallback_.extent(1) != blocks) - adams_fallback_ = Kokkos::View( - "solprop_native.adams.fallback_functions", rows, blocks, problem_.n_modes, - problem_.n_modes); - if (columns != problem_.n_modes) - throw std::logic_error("the resident Adams engine steps a coupled block of one " - "batch row"); -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // A wider block's host passes hold its channel vectors in a row per bin; a - // device's passes take a thread per (bin, channel) and hold none, at any width - // (solprop SEP 47 phase 4). - const std::size_t width = problem_.n_modes > adams_max_channels - ? (adams_max_samples + 1) * problem_.n_modes - : 0; - if (adams_channels_.extent(1) != width) - adams_channels_ = width == 0 - ? ComplexView2{} - : ComplexView2("solprop_native.adams.channels", n, width); -#endif - if (adams_block_.extent(0) != slots + 1) - adams_block_ = Kokkos::View( - "solprop_native.adams.block", slots + 1, flow_->n_active(), problem_.n_modes, - problem_.n_modes); - if (compensated && flow_->n_terms() != 0 && adams_applied_.extent(0) != slots) { - adams_applied_ = ComplexView3("solprop_native.adams.applied", slots, columns, n); - adams_end_applied_ = ComplexView2("solprop_native.adams.end_applied", columns, n); - adams_end_weights_ = Kokkos::View( - "solprop_native.adams.end_weights", flow_->n_terms()); - } - } - // The norm's scale per bin, written each trial; allocated here so that the history's - // memory is held, and counted, from construction (solprop's adams_memory). - if (adams_divisor_.extent(0) != n) - adams_divisor_ = - Kokkos::View("solprop_native.adams.divisor", n); - if (adams_norms_.extent(0) == 0) - adams_norms_ = Kokkos::View( - "solprop_native.adams.norms", adams_max_norms); - // The history's views may move: a graph that names them is dropped. - adams_source_graph_.reset(); - // A trial's elementwise passes cost about ten stage entries an element, rising with - // the moments formed, so they take a width of their own; the accepted step's rows - // are copies. - adams_execution_space_ = kernel_pool_.get(CpuOperation::adams, columns * n * slots); - adams_copy_space_ = kernel_pool_.get(CpuOperation::copy, columns * n); - if (adams_store_.extent(0) == slots) - return; - adams_store_ = ComplexView3("solprop_native.adams.store", slots, columns, n); - adams_moments_ = ComplexView3("solprop_native.adams.moments", slots, columns, n); - adams_source_ = ComplexView2("solprop_native.adams.source", columns, n); - adams_propagator_ = ComplexView2("solprop_native.adams.propagator", columns, n); - adams_exponent_ = ComplexView2("solprop_native.adams.exponent", columns, n); - adams_saved_ = ComplexView2("solprop_native.adams.saved_state", columns, n); -} - -void GeneralWorkspace::require_adams() const { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - if (adams_store_.extent(0) == 0) - throw std::logic_error("the resident Adams history is not allocated"); -} - -namespace { - -// The whole step's phase increment `sum_t dPhi_t Phi_t` at one element, in term order: -// strict (solprop SEP 25 section 4.2). -template -SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex adams_phase_sum( - const Terms& phi_terms, const Weights& phi_delta, const std::size_t offset, - const std::size_t n_phi, const std::size_t column, const std::size_t index) { - SOLPROP_NATIVE_STRICT_BLOCK - double real = 0.0; - double imag = 0.0; - for (std::size_t term = 0; term < n_phi; ++term) { - const double weight = phi_delta(offset + term); - // Exact, as in `StageTerms::phase_sum`. - if (weight == 0.0) - continue; - const Complex fixed = phi_terms(term, column, index); - real = real + weight * fixed.real(); - imag = imag + weight * fixed.imag(); - } - return {real, imag}; -} - -} // namespace - -// The whole step's exponent from the uploaded weights, and its exponential: on a -// separable route the step's length times each element's operator, as the stage -// kernel forms the whole propagator; otherwise `sum_t dPhi_t Phi_t` in term order. -void GeneralWorkspace::adams_phase() { - if (problem_.separable_geometry()) { - const auto spatial = spatial_; - const auto active = active_mask_; - const auto delta = stage_phi_delta_; - const auto exponent = adams_exponent_; - const auto propagator = adams_propagator_; - const auto n_pol = problem_.n_pol_state; - const auto n_stages = problem_.n_stages; - launch::parallel_for( - "solprop_native.adams.separable_phase", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - const auto values = spatial.value(column / n_pol, index, active(index) != 0U); - const Complex x = delta(n_stages) * values.linear; - exponent(column, index) = x; - propagator(column, index) = exponential(x); - }); - return; - } - const auto phi_terms = phi_terms_; - const auto phi_delta = stage_phi_delta_; - const auto exponent = adams_exponent_; - const auto propagator = adams_propagator_; - const auto n_phi = problem_.n_phi_terms; - const auto offset = problem_.n_stages * n_phi; - launch::parallel_for( - "solprop_native.adams.phase", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - // The step's phase increment: strict (solprop SEP 25 section 4.2). - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) - SOLPROP_NATIVE_STRICT_FUNCTION { - SOLPROP_NATIVE_STRICT_BLOCK - const Complex x = - adams_phase_sum(phi_terms, phi_delta, offset, n_phi, column, index); - exponent(column, index) = x; - propagator(column, index) = exponential(x); - }); -} - -// The stage-zero source at an anchor-frame field held in current_, as adams_source -// evaluates it, into stages_(0). -void GeneralWorkspace::adams_evaluate_source(const TrialScalars& scalars) { - if (general_) - general_->set_weights(scalars.transform_weights); - if (problem_.raman) - prepare_raman_stage(0, scalars.raman_rates.data(), - problem_.n_raman_components > 1 ? scalars.raman_densities.data() - : nullptr); - evaluate_stage_source(0, 0.0, false); - stages_ready_ = true; - pending_fsal_ = false; - has_recycled_ = false; -} - -// The same, with current_ restored from adams_saved_ afterwards. -void GeneralWorkspace::adams_evaluate(const TrialScalars& scalars) { - adams_evaluate_source(scalars); - graph::deep_copy(execution_space_, current_, adams_saved_); -} - -void GeneralWorkspace::adams_start(const std::size_t slot, - const TrialScalars& scalars) { - require_adams(); - if (slot >= adams_store_.extent(0)) - throw std::out_of_range("Adams history row"); - require_trial_scalars(scalars); - upload_trial(scalars); - graph::deep_copy(execution_space_, adams_saved_, current_); - { - const FlowAside aside(flow_); - adams_evaluate(scalars); - } - // A frozen-residual flow's generator at the sample applied to its state, on the far - // side of a kink that restarted the history (solprop SEP 16 phase 6). - if (adams_applied_.extent(0) != 0) { - upload_weights(execution_space_, adams_end_weights_, scalars.flow_weights); - apply_generator(execution_space_, problem_.n_omega, problem_.n_modes, - flow_->block_slots(), flow_->residual_terms(), adams_end_weights_, - current_, - Kokkos::subview(adams_applied_, slot, Kokkos::ALL, Kokkos::ALL)); - } - const auto stages = stages_; - const auto store = adams_store_; - launch::parallel_for( - "solprop_native.adams.start", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - store(slot, column, index) = stages(0, column, index); - }); -} - -void GeneralWorkspace::adams_scale(const double rtol, const double atol, - const std::vector& weights) { - require_open(); - if (!weights.empty()) - require_extent(weights.size(), problem_.n_omega, "Adams scale weights"); - adams_rtol_ = rtol; - adams_atol_ = atol; - adams_scale_weights_ = RealView("solprop_native.adams.scale_weights", weights.size()); - if (!weights.empty()) - copy_vector(execution_space_, adams_scale_weights_, weights); - adams_magnitude_sum_ = - Kokkos::View("solprop_native.adams.magnitude_sum"); - adams_device_scale_ = true; -} - -// A trial's divisor: the host's when one is given, else formed on the device from the -// accepted state, as solprop's PIController.scale forms it from the same magnitude. -void GeneralWorkspace::adams_upload_divisor(const std::vector& divisor) { - if (adams_divisor_.extent(0) != problem_.n_omega) - adams_divisor_ = Kokkos::View( - "solprop_native.adams.divisor", problem_.n_omega); - if (!divisor.empty() || !adams_device_scale_) { - require_extent(divisor.size(), problem_.n_omega, "Adams norm divisor"); - copy_vector(execution_space_, adams_divisor_, divisor); - return; - } - const auto current = current_; - const auto in_norm = norm_mask_; - const auto n = problem_.n_omega; - // A threaded reduction in whatever order measures fastest (solprop SEP 25). - launch::parallel_reduce( - "solprop_native.adams.magnitude", - Kokkos::RangePolicy(execution_space_, 0, - problem_.n_columns() * n), - KOKKOS_LAMBDA(const std::size_t item, double& update) { - const auto index = item % n; - if (in_norm(index) != 0) - update += squared_abs(current(item / n, index)); - }, - adams_magnitude_sum_); - const auto sum = adams_magnitude_sum_; - const auto weights = adams_scale_weights_; - const bool weighted = weights.extent(0) != 0; - const double rtol = adams_rtol_; - const double atol = adams_atol_; - const auto scale = adams_divisor_; - launch::parallel_for( - "solprop_native.adams.divisor", - Kokkos::RangePolicy(execution_space_, 0, n), - KOKKOS_LAMBDA(const std::size_t index) { - const double magnitude = Kokkos::sqrt(sum()); - scale(index) = weighted ? rtol * weights(index) * magnitude + atol - : rtol * magnitude + atol; - }); -} - -double GeneralWorkspace::adams_magnitude() { - require_adams(); - const auto current = current_; - const auto in_norm = norm_mask_; - const auto n = problem_.n_omega; - double total = 0.0; - launch::parallel_reduce( - "solprop_native.adams.magnitude", - Kokkos::RangePolicy(execution_space_, 0, - problem_.n_columns() * n), - KOKKOS_LAMBDA(const std::size_t item, double& update) { - const auto index = item % n; - if (in_norm(index) != 0) - update += squared_abs(current(item / n, index)); - }, - total); - return Kokkos::sqrt(total); -} - -namespace { - -/** The squared scaled norms of up to three defects, reduced in one traversal. */ -struct AdamsDefectNorms { - using value_type = double[]; - using size_type = std::size_t; - size_type value_count; - Kokkos::View store; - Kokkos::View source; - Kokkos::View propagator; - Kokkos::View in_norm; - Kokkos::View divisor; - AdamsWeights weights; - double dz; - std::size_t n; - - // A defect is a divided difference of the samples: strict (solprop SEP 25 4.2). - SOLPROP_NATIVE_STRICT_FUNCTION - KOKKOS_INLINE_FUNCTION void operator()(const std::size_t item, - value_type update) const { - SOLPROP_NATIVE_STRICT_BLOCK - const auto column = item / n; - const auto index = item % n; - if (in_norm(index) == 0) - return; - const Complex u = propagator(column, index); - const double scale = divisor(index); - for (size_type row = 0; row < value_count; ++row) { - Complex defect = strict_scale(weights.rows[row][0], source(column, index)); - for (std::size_t t = 1; t < weights.width; ++t) - defect = strict_accumulate(defect, weights.rows[row][t], - store(weights.slot[t - 1], column, index)); - const Complex scaled = - strict_divide(strict_mul(u, strict_scale(dz, defect)), scale); - update[row] += squared_abs(scaled); - } - } - KOKKOS_INLINE_FUNCTION void join(value_type destination, - const value_type source_values) const { - for (size_type row = 0; row < value_count; ++row) - destination[row] += source_values[row]; - } - KOKKOS_INLINE_FUNCTION void init(value_type values) const { - for (size_type row = 0; row < value_count; ++row) - values[row] = 0.0; - } -}; - -} // namespace - -std::vector -GeneralWorkspace::adams_fused_trial(const double dz, const AdamsWeights& weights, - const std::vector& divisor, - const TrialScalars& scalars) { - require_adams(); - if (flow_) - throw std::logic_error("the Adams source is evaluated without a coupled flow"); - if (weights.n_rows == 0 || weights.n_rows > 3 || weights.width == 0 || - weights.width > adams_max_samples || weights.order >= weights.width) - throw std::invalid_argument("Adams coefficient tables are out of range"); - for (std::size_t t = 0; t + 1 < weights.width; ++t) - if (weights.slot[t] >= adams_store_.extent(0)) - throw std::out_of_range("Adams history row"); - require_trial_scalars(scalars); - upload_trial(scalars); - adams_upload_divisor(divisor); - adams_phase(); - const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); - const auto store = adams_store_; - const auto current = current_; - const auto predicted = adams_saved_; - // The predictor in the anchor's frame, a_n + dz sum_t w_t g_t, is the field the - // stage-zero source evaluates; the accepted state is swapped aside meanwhile. - launch::parallel_for( - "solprop_native.adams.predict", policy, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - Complex total{}; - for (std::size_t t = 0; t < weights.order; ++t) - total += weights.predictor[t] * store(weights.slot[t], column, index); - predicted(column, index) = current(column, index) + dz * total; - }); - std::swap(current_, adams_saved_); - adams_evaluate_source(scalars); - std::swap(current_, adams_saved_); - const auto stages = stages_; - const auto source = adams_source_; - const auto propagator = adams_propagator_; - const auto advanced = advanced_; - launch::parallel_for( - "solprop_native.adams.correct", policy, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - const Complex g = stages(0, column, index); - source(column, index) = g; - Complex total = weights.corrector[0] * g; - for (std::size_t t = 1; t < weights.width; ++t) - total += weights.corrector[t] * store(weights.slot[t - 1], column, index); - advanced(column, index) = - propagator(column, index) * (current(column, index) + dz * total); - }); - const AdamsDefectNorms reducer{.value_count = weights.n_rows, - .store = store, - .source = source, - .propagator = propagator, - .in_norm = norm_mask_, - .divisor = adams_divisor_, - .weights = weights, - .dz = dz, - .n = n}; - return adams_norms( - "solprop_native.adams.defect_norms", - Kokkos::RangePolicy(execution_space_, 0, columns * n), reducer, - weights.n_rows); -} - -void GeneralWorkspace::adams_fused_accept(const double dz, const AdamsWeights& keep, - const std::size_t free, - const AdamsWeights& dense) { - require_adams(); - if (!adams_trial_ready_) - throw std::logic_error("no Adams trial is held"); - if (free >= adams_store_.extent(0)) - throw std::out_of_range("Adams history row"); - if (dense.width > 0 && adams_slopes_.extent(0) == 0) - throw std::logic_error("Adams dense observation storage is not allocated"); - adams_trial_ready_ = false; - const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); - const auto store = adams_store_; - const auto source = adams_source_; - const auto propagator = adams_propagator_; - const auto slopes = adams_slopes_; - // The step's own interpolant, in its anchor's frame, before the transport moves the - // rows: the slopes (source, newest, ...) and the state it starts from. - adams_dense_count_ = dense.width; - if (dense.width > 0) { - graph::deep_copy(execution_space_, adams_dense_start_, current_); - graph::deep_copy(execution_space_, adams_dense_exponent_, adams_exponent_); - launch::parallel_for( - "solprop_native.adams.dense_slopes", policy, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - slopes(0, column, index) = source(column, index); - for (std::size_t t = 1; t < dense.width; ++t) - slopes(t, column, index) = store(dense.slot[t - 1], column, index); - }); - } - finish(dz); - stash_is_current_ = false; - if (dense.width > 0) { - const auto current = current_; - const auto end = adams_dense_end_; - launch::parallel_for( - "solprop_native.adams.dense_end", policy, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - end(column, index) = current(column, index) / propagator(column, index); - }); - } - // The history re-anchored at the step's end, and the new derivative in row `free`. - launch::parallel_for( - "solprop_native.adams.transport", policy, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - const Complex u = propagator(column, index); - for (std::size_t t = 0; t < keep.width; ++t) - store(keep.slot[t], column, index) = store(keep.slot[t], column, index) * u; - store(free, column, index) = u * source(column, index); - }); - adams_accepted_boundary(); -} - -void GeneralWorkspace::adams_hold() { - require_adams(); - if (adams_held_.empty()) - throw std::logic_error("Adams dense observation storage is not allocated"); - graph::deep_copy(execution_space_, adams_held_, current_); -} - -void GeneralWorkspace::adams_restore() { - require_adams(); - if (adams_held_.empty()) - throw std::logic_error("Adams dense observation storage is not allocated"); - graph::deep_copy(execution_space_, current_, adams_held_); - stash_is_current_ = false; - execution_space_.fence("solprop_native Adams restore boundary"); -} - -// --- solprop SEP 29 G2: the resident history's rows across a resume ---------------- -// -// The persistent rows are the samples (store), a compensated history's sampled states -// and rates, and a frozen-residual flow's applied generators G(z_j) u_j: everything a -// later trial reads that an earlier step wrote. The rest of the Adams views are a -// trial's scratch, rebuilt by the next trial, and the step's bookkeeping lives in -// solprop's march. The copies are lossless: the view's own LayoutRight order, element -// by element, with no arithmetic. - -ComplexView3 GeneralWorkspace::adams_row_view(const std::string& name) const { - require_adams(); - ComplexView3 view; - if (name == "store") - view = adams_store_; - else if (name == "states") - view = adams_states_; - else if (name == "rates") - view = adams_rates_; - else if (name == "applied") - view = adams_applied_; - else - throw std::invalid_argument("unknown Adams history row '" + name + "'"); - if (view.extent(0) == 0) - throw std::logic_error("the Adams history row '" + name + "' is not allocated"); - return view; -} - -std::vector GeneralWorkspace::adams_row_names() const { - require_adams(); - std::vector names{"store"}; - if (adams_states_.extent(0) != 0) - names.insert(names.end(), {"states", "rates"}); - if (adams_applied_.extent(0) != 0) - names.emplace_back("applied"); - return names; -} - -std::array -GeneralWorkspace::adams_row_extents(const std::string& name) const { - const auto view = adams_row_view(name); - return {view.extent(0), view.extent(1), view.extent(2)}; -} - -std::vector -GeneralWorkspace::adams_rows_to_host(const std::string& name) const { - std::vector> values(adams_row_view(name).size()); - adams_rows_to_host(name, values.data(), values.size()); - return {values.begin(), values.end()}; -} - -// One copy, straight into the caller's buffer (a NumPy array's storage at the seam): -// no intermediate host vector, so a kept save costs what the host engine's copy does. -void GeneralWorkspace::adams_rows_to_host(const std::string& name, - std::complex* destination, - const std::size_t size) const { - const auto view = adams_row_view(name); - if (size != view.size()) - throw std::invalid_argument("the Adams history row '" + name + - "' has the wrong extent"); - execution_space_.fence("solprop_native Adams rows_to_host"); - crossed_.to_host += size * sizeof(Complex); - if (crossing::takes_ring(size * sizeof(Complex))) { - crossing_ring_.to_host(destination, view.data(), size * sizeof(Complex)); - return; - } - const auto host = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, view); - const auto* source = host.data(); - for (std::size_t index = 0; index < size; ++index) - destination[index] = {source[index].real(), source[index].imag()}; -} - -void GeneralWorkspace::adams_rows_from_host(const std::string& name, - const std::vector& values) { - const auto view = adams_row_view(name); - if (values.size() != view.size()) - throw std::invalid_argument("the Adams history row '" + name + - "' has the wrong extent"); - copy_vector(execution_space_, view, values); - // A trial in flight and the last step's interpolant belong to the history replaced. - adams_trial_ready_ = false; - adams_dense_count_ = 0; - execution_space_.fence("solprop_native Adams rows_from_host"); -} - -// solprop SEP 29 G1: the general z tables. A table is uploaded once, after the state -// is constructed and before its first trial, which is the construction's allocation. -void GeneralWorkspace::table_upload(const TableKind kind, std::vector edges, - const double lower, const double upper, - const std::complex* coefficients, - const std::size_t n_intervals, - const std::size_t n_terms, - const std::size_t n_values) { - require_open(); - const bool prefactor = kind == TableKind::prefactor; - const bool flux = kind == TableKind::flux; - if (flux ? !problem_.flux_table - : (prefactor ? !problem_.prefactor_table : !problem_.phi_table)) - throw std::logic_error(std::string("this problem's ") + - (flux ? "plane measure" - : prefactor ? "prefactor" - : "Phi") + - " is not a z table"); - if (n_intervals == 0 || n_terms == 0 || edges.size() != n_intervals + 1) - throw std::invalid_argument("a z table needs one more breakpoint than intervals " - "and at least one coefficient"); - if (flux && n_values != problem_.n_omega) - throw std::invalid_argument("the plane measure's material table holds n_omega = " + - std::to_string(problem_.n_omega) + " values"); - if (!flux && n_values != problem_.n_columns() * problem_.n_omega) - throw std::invalid_argument("a z table's values must be the state's (" + - std::to_string(problem_.n_columns()) + " x " + - std::to_string(problem_.n_omega) + ")"); - for (std::size_t index = 0; index < edges.size(); ++index) - if (!std::isfinite(edges[index]) || - (index > 0 && !(edges[index] > edges[index - 1]))) - throw std::invalid_argument("a z table's breakpoints must be finite and " - "strictly increasing"); - if (!(lower <= edges.front() && upper >= edges.back())) - throw std::invalid_argument("a z table's admitted span must contain its edges"); - auto& table = kind == TableKind::phi ? phi_table_ - : kind == TableKind::prefactor ? prefactor_table_ - : flux ? flux_table_ - : rate_table_; - table.coefficients = Kokkos::View( - kind == TableKind::phi ? "solprop_native.linear.phi_table" - : kind == TableKind::prefactor ? "solprop_native.response.prefactor_table" - : flux ? "solprop_native.flux.material_table" - : "solprop_native.linear.rate_table", - n_intervals, n_terms, n_values); - auto mirror = Kokkos::create_mirror_view(table.coefficients); - auto* data = mirror.data(); - const std::size_t count = n_intervals * n_terms * n_values; - for (std::size_t index = 0; index < count; ++index) - data[index] = - chebyshev::Coefficient>::load(coefficients[index]); - graph::deep_copy(execution_space_, table.coefficients, mirror); - table.edges = std::move(edges); - table.lower = lower; - table.upper = upper; - execution_space_.fence("solprop_native z table upload"); -} - -// The tables' values at a trial's positions into their term slots: Phi's and the -// prefactor's, each a pass over its coefficients for every position at once. -void GeneralWorkspace::table_evaluate(const std::vector& slots, - const std::vector& positions) { - require_open(); - if (!phi_table_.present() && !prefactor_table_.present()) - throw std::logic_error("no z table has been uploaded"); - for (const auto slot : slots) { - if ((phi_table_.present() && slot >= problem_.n_phi_terms) || - (prefactor_table_.present() && slot >= problem_.n_prefactor_terms)) - throw std::out_of_range("z table slot " + std::to_string(slot) + - " is not a term of this problem"); - } - if (phi_table_.present()) - z_table::evaluate("solprop_native.linear.phi_table_values", execution_space_, - phi_table_, z_table::locate(phi_table_, slots, positions, false), - phi_terms_.data()); - if (prefactor_table_.present()) - z_table::evaluate("solprop_native.response.prefactor_table_values", - execution_space_, prefactor_table_, - z_table::locate(prefactor_table_, slots, positions, false), - prefactor_terms_.data()); -} - -// solprop SEP 34: the plane measure's material square at `z`, a radial varying -// medium's, into the flux coefficients the statistics read; nothing for any other. -void GeneralWorkspace::flux_at(const double z) { - if (!flux_table_.present()) - return; - z_table::evaluate("solprop_native.flux.material_values", execution_space_, - flux_table_, z_table::locate(flux_table_, {0}, {z}, false), - flux_.square.data()); -} - -// D(z) = dPhi/dz from the rate table into one Phi term slot, read on the side of an -// interior breakpoint `before` asks for (solprop's `phi_rate`). -void GeneralWorkspace::table_rate(const std::size_t slot, const double z, - const bool before) { - require_open(); - if (!rate_table_.present()) - throw std::logic_error("no rate table has been uploaded"); - if (slot >= problem_.n_phi_terms) - throw std::out_of_range("z table slot " + std::to_string(slot) + - " is not a term of this problem"); - z_table::evaluate("solprop_native.linear.rate_table_values", execution_space_, - rate_table_, z_table::locate(rate_table_, {slot}, {z}, before), - phi_terms_.data()); -} - -std::vector -GeneralWorkspace::table_terms_to_host(const TableKind kind, - const std::size_t slot) const { - require_open(); - const auto& terms = kind == TableKind::prefactor ? prefactor_terms_ : phi_terms_; - if (slot >= terms.extent(0)) - throw std::out_of_range("term slot " + std::to_string(slot) + - " is not a term of this problem"); - execution_space_.fence("solprop_native z table terms_to_host"); - const auto host = Kokkos::create_mirror_view_and_copy( - Kokkos::HostSpace{}, Kokkos::subview(terms, slot, Kokkos::ALL, Kokkos::ALL)); - std::vector values(host.size()); - for (std::size_t column = 0; column < host.extent(0); ++column) - for (std::size_t index = 0; index < host.extent(1); ++index) - values[column * host.extent(1) + index] = host(column, index); - return values; -} - -// The stage kernels' own phase sum (`StageTerms::phase_sum`) over the Phi terms for one -// row of weights, into a fresh host array: a diagnostic, outside any trial. -std::vector -GeneralWorkspace::phase_sum_to_host(const std::vector& weights) { - require_open(); - if (problem_.separable_geometry()) - throw std::logic_error("a separable operator has no Phi terms"); - require_extent(weights.size(), problem_.n_phi_terms, "phase weights"); - RealView row("solprop_native.diagnostic.phase_weights", weights.size()); - copy_vector(execution_space_, row, weights); - ComplexView2 phase("solprop_native.diagnostic.phase", problem_.n_columns(), - problem_.n_omega); - auto terms = stage_terms(0); - terms.phi_delta = row; - launch::parallel_for( - "solprop_native.diagnostic.phase_sum", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - phase(column, index) = terms.phase_sum(0, column, index); - }); - execution_space_.fence("solprop_native phase_sum_to_host"); - const auto host = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, phase); - return {host.data(), host.data() + host.size()}; -} - -// The field at a fraction of the last accepted step: the step's interpolant in its -// anchor's frame -- the Adams corrector's (`weights.predictor` holds dz-scaled -// integrated weights over the slopes) -- with a bin the step's flow changes by more -// than an e-fold taking the anchor's value, carried to the observation by the uploaded -// trial's flow and masked into current_. -void GeneralWorkspace::adams_observe(const AdamsWeights& weights, - const TrialScalars& scalars) { - require_adams(); - if (adams_dense_count_ == 0) - throw std::logic_error("no Adams step is held for dense observation"); - require_trial_scalars(scalars); - upload_trial(scalars); - adams_phase(); - const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - const auto count = adams_dense_count_; - const auto start = adams_dense_start_; - const auto exponent = adams_dense_exponent_; - const auto slopes = adams_slopes_; - const auto carry = adams_propagator_; - const auto current = current_; - launch::parallel_for( - "solprop_native.adams.observe", Range2(execution_space_, {0, 0}, {columns, n}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - Complex total{}; - for (std::size_t t = 0; t < count; ++t) - total += weights.predictor[t] * slopes(t, column, index); - Complex value = start(column, index) + total; - if (Kokkos::abs(exponent(column, index).real()) > 1.0) - value = start(column, index); - current(column, index) = carry(column, index) * value; - }); - mask(current_); - stash_is_current_ = false; - execution_space_.fence("solprop_native Adams observation boundary"); -} - -// --- solprop SEP 16 phase 3b: the exponential formulation, resident -// ------------------- -// -// The exponential formulation's passes (solprop ADR-041, ADR-043): untransported -// samples integrated exactly against the step's flow exp(x), x = Phi(z + h) - Phi(z). -// The moments M_k = k! phi_{k+1}(x) are formed once a trial per element by the -// reference's algorithm (solprop backends/exponential.py, phi_functions), then every -// formula of the trial is -// applied in the monomial basis of its interpolant; the defects use the factored form -// (a divided difference of the samples times one moment combination). - -namespace { - -/** k! for k < 2 adams_max_samples, the moments' scale. */ -KOKKOS_INLINE_FUNCTION double adams_factorial(const std::size_t k) { - double value = 1.0; - for (std::size_t i = 2; i <= k; ++i) - value *= static_cast(i); - return value; -} - -/** - * The phi-functions a trial's moments take: at x, or under a carrier frame at - * x - sigma with that argument's own exponential (solprop SEP 35 section 4.1). The - * flow stays exp(x), which the caller holds. - */ -KOKKOS_INLINE_FUNCTION Complex adams_moment_argument(const ExponentialWeights& weights, - const Complex x) { - return weights.shifted ? x - Complex(0.0, weights.shift) : x; -} - -/** `adams_phi` at `adams_moment_argument`, with the flow `exp_x`; `Fixed`, when not - zero, is `order` known at compile time (`adams_phi_fixed`). */ -template -KOKKOS_INLINE_FUNCTION void adams_moment_phi(const ExponentialWeights& weights, - const Complex x, const Complex exp_x, - const std::size_t order, Complex* out) { - const Complex argument = adams_moment_argument(weights, x); - const double size = Kokkos::abs(argument); - if (!weights.shifted) { - adams_phi_fixed(argument, size, exp_x, order, out); - return; - } - // exp(x - sigma) = exp(x) exp(-sigma): one product with the trial's exp(-sigma) - // rather than a second exponential per element (a few ulp, inside the moments' - // share of the rounding budget). - adams_phi_fixed( - argument, size, - strict_mul(exp_x, Complex(weights.unshift_re, weights.unshift_im)), order, out); -} - -/** - * History row `t` of a trial (`weights.slot[t]`) at one element, rotated by a carrier - * frame's factor exp(sigma (1 - theta_t)) as it is read (solprop SEP 35 section 6.2): - * the store keeps the untransported samples. - */ -template -KOKKOS_INLINE_FUNCTION Complex adams_history(const Store& store, - const ExponentialWeights& weights, - const std::size_t t, - const std::size_t column, - const std::size_t index) { - const Complex value = store(weights.slot[t], column, index); - if (!weights.shifted) - return value; - return strict_mul(Complex(weights.factor_re[t], weights.factor_im[t]), value); -} - -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -/** - * A device element's moments, written to `moments`, and its prediction - * `exp(x) u + dz sum_k M_k sum_j c_jk F_j`, as the elementwise trial's predict pass - * forms them, with `formed` phi-functions. `Fixed`, when not zero, is `formed` known at - * compile time: the loops over the phi-functions unroll and the thread holds them in - * registers, where an indexed array lives in private (scratch) memory, which on an - * AMD Instinct MI325X made this the costliest kernel of a small trial. - */ -template -KOKKOS_INLINE_FUNCTION Complex adams_device_prediction( - const ExponentialWeights& weights, const std::size_t formed, const Complex x, - const Complex propagate, const Complex accepted, const double dz, - const Moments& moments, const Store& store, const std::size_t column, - const std::size_t index) { - constexpr std::size_t bound = Fixed != 0 ? Fixed : 2 * adams_max_samples; - Complex phi[bound + 1]; - adams_moment_phi(weights, x, propagate, formed, phi); - for (std::size_t k = 0; k < bound; ++k) { - if (k >= weights.count) - break; - moments(k, column, index) = adams_factorial(k) * phi[k + 1]; - } - Complex total{}; - for (std::size_t k = 0; k < bound; ++k) { - if (k >= weights.order) - break; - Complex combined{}; - for (std::size_t t = 0; t < weights.order; ++t) - combined += - weights.predictor[t][k] * adams_history(store, weights, t, column, index); - const Complex moment = adams_factorial(k) * phi[k + 1]; - total += moment * combined; - } - return propagate * accepted + dz * total; -} - -/** `adams_device_prediction` at the compile-time bound equal to `formed`, or at a - run-time one past the table's eight samples. */ -template -KOKKOS_INLINE_FUNCTION Complex -adams_device_predict(const ExponentialWeights& weights, const std::size_t formed, - const Complex x, const Complex propagate, const Complex accepted, - const double dz, const Moments& moments, const Store& store, - const std::size_t column, const std::size_t index) { - static_assert(adams_max_samples == 8, "one case per sample count"); - switch (formed) { - case 1: - return adams_device_prediction<1>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 2: - return adams_device_prediction<2>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 3: - return adams_device_prediction<3>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 4: - return adams_device_prediction<4>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 5: - return adams_device_prediction<5>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 6: - return adams_device_prediction<6>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 7: - return adams_device_prediction<7>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - case 8: - return adams_device_prediction<8>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - default: - return adams_device_prediction<0>(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); - } -} -#endif - -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -/** The channel vectors a host task of a coupled pass holds: up to `adams_max_samples + - 1` rows of a bin's channels. */ -inline constexpr std::size_t adams_channel_stack = - (adams_max_samples + 1) * adams_max_channels; - -/** - * Where a host task holds bin `index`'s channel vectors, rows of `modes` entries: - * `stack` up to `adams_max_channels` channels, beyond it the bin's row of `rows` - * (`adams_channels_`, solprop SEP 47 phase 4). Every pass reads only what it wrote. - */ -// A bin's row of `adams_channels_` is contiguous: the view's last extent is its -// fastest. -static_assert(std::is_same_v, - "adams_channel_rows reads a bin's row as contiguous memory"); - -inline Complex* -adams_channel_rows(Complex* stack, - const Kokkos::View& rows, - const std::size_t index) { - return rows.extent(1) == 0 ? stack : &rows(index, 0); -} -#endif - -// The defects' norms beside a coupled flow (solprop SEP 16 phase 6): per bin, the -// samples' channel vectors; off the block each channel's defect is the diagonal one, -// on it each moment combination is a matrix applied to the divided difference. -struct CoupledDefectNorms { - using value_type = double[]; - using size_type = std::size_t; - size_type value_count; - Kokkos::View store; - Kokkos::View moments; - Kokkos::View source; - Kokkos::View carried; - Kokkos::View endpoint; - Kokkos::View block; - Kokkos::View slots; - Kokkos::View in_norm; - Kokkos::View divisor; - ExponentialWeights weights; - double dz; - std::size_t modes; - // A host's channel vectors beyond `adams_max_channels` (`adams_channels_`). - Kokkos::View channels; - -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - SOLPROP_NATIVE_STRICT_FUNCTION - KOKKOS_INLINE_FUNCTION void operator()(const std::size_t item, - value_type update) const { - SOLPROP_NATIVE_STRICT_BLOCK - // An item is a (bin, channel) pair: a thread holds no channel vectors. - const auto index = item / modes; - const auto i = item % modes; - if (in_norm(index) == 0) - return; - const double scale = divisor(index); - const auto one = slots(index); - // Channel c's divided difference for one defect. - const auto difference = [&](const size_type row, const std::size_t c) { - Complex total = strict_scale(weights.rows[row][0], source(c, index)); - for (std::size_t t = 1; t < weights.width; ++t) - total = strict_accumulate(total, weights.rows[row][t], - adams_history(store, weights, t - 1, c, index)); - return total; - }; - for (size_type row = 0; row < value_count; ++row) { - Complex defect{}; - if (one == 0) { - Complex weight{}; - for (std::size_t k = 0; k < weights.count; ++k) - weight = strict_accumulate(weight, weights.rows[row][weights.width + k], - moments(k, i, index)); - defect = strict_scale(dz, strict_mul(difference(row, i), weight)); - } else { - const auto bin = one - 1; - Complex total{}; - for (std::size_t j = 0; j < modes; ++j) { - Complex weight{}; - for (std::size_t k = 0; k < weights.count; ++k) - weight = strict_accumulate(weight, weights.rows[row][weights.width + k], - block(k + 1, bin, i, j)); - total = strict_add(total, strict_mul(weight, difference(row, j))); - } - defect = strict_scale(dz, total); - } - if (weights.compensated) { - Complex compensation = - strict_scale(weights.comp_rows[row][0], endpoint(i, index)); - for (std::size_t t = 0; t + 1 < weights.width; ++t) - compensation = strict_accumulate(compensation, weights.comp_rows[row][t + 1], - carried(t, i, index)); - defect = strict_accumulate(defect, dz, compensation); - } - update[row] += squared_abs(strict_divide(defect, scale)); - } - } -#else - // The defects and their compensation terms: strict (solprop SEP 25 section 4.2). - SOLPROP_NATIVE_STRICT_FUNCTION - KOKKOS_INLINE_FUNCTION void operator()(const std::size_t index, - value_type update) const { - SOLPROP_NATIVE_STRICT_BLOCK - if (in_norm(index) == 0) - return; - const double scale = divisor(index); - const auto one = slots(index); - // The samples' channel vectors, row t at `values + t modes`, and the divided - // difference after them. - Complex stack[adams_channel_stack]; - Complex* const values = adams_channel_rows(stack, channels, index); - Complex* const difference = values + adams_max_samples * modes; - for (std::size_t i = 0; i < modes; ++i) { - values[i] = source(i, index); - for (std::size_t t = 1; t < weights.width; ++t) - values[t * modes + i] = adams_history(store, weights, t - 1, i, index); - } - for (size_type row = 0; row < value_count; ++row) { - for (std::size_t i = 0; i < modes; ++i) { - Complex total = strict_scale(weights.rows[row][0], values[i]); - for (std::size_t t = 1; t < weights.width; ++t) - total = strict_accumulate(total, weights.rows[row][t], values[t * modes + i]); - difference[i] = total; - } - for (std::size_t i = 0; i < modes; ++i) { - Complex defect{}; - if (one == 0) { - Complex weight{}; - for (std::size_t k = 0; k < weights.count; ++k) - weight = strict_accumulate(weight, weights.rows[row][weights.width + k], - moments(k, i, index)); - defect = strict_scale(dz, strict_mul(difference[i], weight)); - } else { - const auto bin = one - 1; - Complex total{}; - for (std::size_t j = 0; j < modes; ++j) { - Complex weight{}; - for (std::size_t k = 0; k < weights.count; ++k) - weight = strict_accumulate(weight, weights.rows[row][weights.width + k], - block(k + 1, bin, i, j)); - total = strict_add(total, strict_mul(weight, difference[j])); - } - defect = strict_scale(dz, total); - } - if (weights.compensated) { - Complex compensation = - strict_scale(weights.comp_rows[row][0], endpoint(i, index)); - for (std::size_t t = 0; t + 1 < weights.width; ++t) - compensation = strict_accumulate( - compensation, weights.comp_rows[row][t + 1], carried(t, i, index)); - defect = strict_accumulate(defect, dz, compensation); - } - update[row] += squared_abs(strict_divide(defect, scale)); - } - } - } -#endif - KOKKOS_INLINE_FUNCTION void join(value_type destination, - const value_type source_values) const { - for (size_type row = 0; row < value_count; ++row) - destination[row] += source_values[row]; - } - KOKKOS_INLINE_FUNCTION void init(value_type values) const { - for (size_type row = 0; row < value_count; ++row) - values[row] = 0.0; - } -}; - -} // namespace - -// D(z) = sum_t w_t Phi_t from the operator's fixed terms (solprop's -// translate.linop.rate_weights), into `out`. -void GeneralWorkspace::adams_rate(const std::vector& rate, - const ComplexView2& out) { - require_extent(rate.size(), problem_.n_phi_terms, "Adams rate weights"); - if (adams_rate_weights_.extent(0) != rate.size()) - adams_rate_weights_ = Kokkos::View( - "solprop_native.adams.rate_weights", rate.size()); - copy_vector(execution_space_, adams_rate_weights_, rate); - const auto weights = adams_rate_weights_; - const auto phi_terms = phi_terms_; - const auto n_phi = problem_.n_phi_terms; - launch::parallel_for( - "solprop_native.adams.rate", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - Complex value{}; - for (std::size_t term = 0; term < n_phi; ++term) - value += weights(term) * phi_terms(term, column, index); - out(column, index) = value; - }); -} - -// The startup sample's state and rate, beside the source adams_start stored. -void GeneralWorkspace::adams_compensation_start(const std::size_t slot, - const std::vector& rate) { - require_adams(); - if (adams_states_.extent(0) == 0) - throw std::logic_error("the Adams history carries no compensation"); - if (slot >= adams_states_.extent(0)) - throw std::out_of_range("Adams history row"); - const auto rates = Kokkos::subview(adams_rates_, slot, Kokkos::ALL, Kokkos::ALL); - adams_rate(rate, adams_endpoint_rate_); - graph::deep_copy(execution_space_, rates, adams_endpoint_rate_); - graph::deep_copy(execution_space_, - Kokkos::subview(adams_states_, slot, Kokkos::ALL, Kokkos::ALL), - current_); -} - -// The exponential Adams march's compensation at one bin, each a single strict function -// (solprop SEP 25 section 4.2) called once per bin: GCC compiles a strict function -// with its own options and does not inline it into a contracting kernel, but inlines -// the strict helpers it calls, which share those options. One call per bin, rather -// than one per helper, keeps the protected arithmetic strict on every compiler at the -// cost of a call (and Clang, which inlines every level, is unchanged). - -// The predictor's carried samples `e^{s_t x} (R_t - x / dz) u_t`, written to -// `carried`, and the prediction `value + dz sum_t c_t carried_t`. A span of one (the -// newest sample, at the step's start) takes the step's flow `flow = e^x` as it is: -// `strict_scale(1, x)` is x, so its exponential is the same bits. A host block with -// vector maths forms the same values on its lanes -// (`adams_lanes_predicted_compensation`). -// The strict compensation helpers keep their linkage: changing it could change how GCC -// inlines strict code (fp.hpp), which needs its own A/B. -// NOLINTBEGIN(misc-use-internal-linkage): strict helpers, linkage unchanged -template -SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex -adams_predicted_compensation(const ExponentialWeights& weights, const Complex x, - const Complex flow, const double inverse_dz, - const double dz, const Complex value, const Rates& rates, - const States& states, const Carried& carried, - const std::size_t column, const std::size_t index) { - SOLPROP_NATIVE_STRICT_BLOCK - for (std::size_t t = 0; t + 1 < weights.width; ++t) { - const auto slot = weights.slot[t]; - Complex term = compensation_residual(rates(slot, column, index), x, inverse_dz, - states(slot, column, index)); - const double span = weights.span[t]; - if (span == 1.0) - term = strict_mul(flow, term); - else if (span != 0.0) - term = strict_mul(exponential(strict_scale(span, x)), term); - carried(t, column, index) = term; - } - Complex compensation{}; - for (std::size_t t = 0; t < weights.order; ++t) - compensation = strict_accumulate(compensation, weights.comp_predictor[t], - carried(t, column, index)); - return strict_accumulate(value, dz, compensation); -} - -// The corrector's endpoint rate `sum_t w_t Phi_t` and compensation `(R - x / dz) u`, -// written to `endpoint_rate` and `endpoint`, and the correction -// `value + dz (c_0 endpoint + sum_t c_{t+1} carried_t)`. -template -SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex -adams_corrected_compensation(const ExponentialWeights& weights, const std::size_t n_phi, - const Weights& rate_weights, const Terms& phi_terms, - const Complex x, const double inverse_dz, - const Complex lab, const double dz, const Complex value, - const Points& endpoint_rate, const Points& endpoint, - const Carried& carried, const std::size_t column, - const std::size_t index) { - SOLPROP_NATIVE_STRICT_BLOCK - Complex rate_here{}; - for (std::size_t term = 0; term < n_phi; ++term) - rate_here = strict_accumulate(rate_here, rate_weights(term), - phi_terms(term, column, index)); - endpoint_rate(column, index) = rate_here; - const Complex compensated = compensation_residual(rate_here, x, inverse_dz, lab); - endpoint(column, index) = compensated; - Complex compensation = strict_scale(weights.comp_corrector[0], compensated); - for (std::size_t t = 0; t < weights.order; ++t) - compensation = strict_accumulate(compensation, weights.comp_corrector[t + 1], - carried(t, column, index)); - return strict_accumulate(value, dz, compensation); -} - -// Dense observation's compensation `value + sum_t dz w_t e^{(xi - theta_t) x} term_t`. -template -SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex -adams_observed_compensation(const ExponentialWeights& weights, const std::size_t count, - const double xi, const double dz, const Complex chord, - const Complex value, const Terms& terms, - const std::size_t column, const std::size_t index) { - SOLPROP_NATIVE_STRICT_BLOCK - Complex total = value; - for (std::size_t t = 0; t < count; ++t) - total = strict_add( - total, - strict_mul( - strict_scale(dz * weights.dense_weight[t], - exponential(strict_scale(xi - weights.dense_theta[t], chord))), - terms(t, column, index))); - return total; -} - -// NOLINTEND(misc-use-internal-linkage) -namespace { - -// The elements a host task of the elementwise trial's passes takes in turn. Each pass -// runs its loops over the block, so that what one loop writes the next reads from -// cache, and a strict loop is one call a block on a compiler that keeps strict -// functions out of line; a device takes a thread an element. -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -constexpr std::size_t adams_block = 1; -#else -constexpr std::size_t adams_block = adams_lane_width; -#endif - -// The host blocks run as lanes where the vector maths forms their exponentials in -// passes (`vmath::vectorised`); elsewhere (devices, the system libm on Arm) each -// element forms its own, and the lanes are not compiled. -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && \ - defined(SOLPROP_NATIVE_VECTOR_MATH_SLEEF) -#define SOLPROP_NATIVE_ADAMS_LANES 1 -#endif - -// The step's exponent and flow on the elements [first, last) of a column. The phase -// increment: strict (solprop SEP 25 section 4.2). A host block with vector maths -// forms both on its lanes instead (`adams_lanes_predict`). -template -SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION void -adams_phase_block(const Terms& phi_terms, const Weights& phi_delta, - const std::size_t offset, const std::size_t n_phi, - const std::size_t column, const std::size_t first, - const std::size_t last, const Points& exponent, const Points& flow) { - SOLPROP_NATIVE_STRICT_BLOCK - for (std::size_t index = first; index < last; ++index) { - const Complex x = - adams_phase_sum(phi_terms, phi_delta, offset, n_phi, column, index); - exponent(column, index) = x; - flow(column, index) = exponential(x); - } -} - -#ifdef SOLPROP_NATIVE_ADAMS_LANES - -// The history rows a pass reads, rotated by a carrier frame's factors -// (`adams_history`). Strict. -template -SOLPROP_NATIVE_STRICT_FUNCTION inline void -adams_lanes_history(AdamsLanes& lanes, const ExponentialWeights& weights, - const Store& store, const std::size_t rows, - const std::size_t column, const std::size_t first) { - SOLPROP_NATIVE_STRICT_BLOCK - const std::size_t count = lanes.count; - for (std::size_t t = 0; t < rows; ++t) { - const Complex* row = &store(weights.slot[t], column, first); - if (!weights.shifted) { - for (std::size_t b = 0; b < count; ++b) { - lanes.hr[t][b] = row[b].real(); - lanes.hi[t][b] = row[b].imag(); - } - continue; - } - const double factor_r = weights.factor_re[t]; - const double factor_i = weights.factor_im[t]; - for (std::size_t b = 0; b < count; ++b) { - const double rr = factor_r * row[b].real(); - const double ii = factor_i * row[b].imag(); - const double ri = factor_r * row[b].imag(); - const double ir = factor_i * row[b].real(); - lanes.hr[t][b] = rr - ii; - lanes.hi[t][b] = ri + ir; - } - } -} - -// The spans' arguments s_t x of a varying operator's carried samples, into `sr`/`si` -// (strict_scale), for the exponentials the caller forms over them. -SOLPROP_NATIVE_STRICT_FUNCTION -inline void adams_lanes_span(AdamsLanes& lanes, const double span, - const std::size_t t) { - SOLPROP_NATIVE_STRICT_BLOCK - for (std::size_t b = 0; b < lanes.count; ++b) { - lanes.sr[t][b] = span * lanes.xr[b]; - lanes.si[t][b] = span * lanes.xi[b]; - } -} - -// The predictor's carried samples `e^{s_t x} (R_t - x / dz) u_t` into `cr`/`ci`, and -// the prediction in `vr`/`vi` plus `dz sum_t c_t carried_t`, as -// `adams_predicted_compensation`. Strict. -template -SOLPROP_NATIVE_STRICT_FUNCTION inline void -adams_lanes_predicted_compensation(AdamsLanes& lanes, const ExponentialWeights& weights, - const double inverse_dz, const double dz, - const Rates& rates, const States& states, - const std::size_t column, const std::size_t first) { - SOLPROP_NATIVE_STRICT_BLOCK - const std::size_t count = lanes.count; - for (std::size_t t = 0; t + 1 < weights.width; ++t) { - const auto slot = weights.slot[t]; - const Complex* rate = &rates(slot, column, first); - const Complex* state = &states(slot, column, first); - const double span = weights.span[t]; - for (std::size_t b = 0; b < count; ++b) { - const double scaled_r = inverse_dz * lanes.xr[b]; - const double scaled_i = inverse_dz * lanes.xi[b]; - const double difference_r = rate[b].real() - scaled_r; - const double difference_i = rate[b].imag() - scaled_i; - const double rr = difference_r * state[b].real(); - const double ii = difference_i * state[b].imag(); - const double ri = difference_r * state[b].imag(); - const double ir = difference_i * state[b].real(); - lanes.cr[t][b] = rr - ii; - lanes.ci[t][b] = ri + ir; - } - if (span == 0.0) - continue; - const double* factor_r = span == 1.0 ? lanes.fr : lanes.sr[t]; - const double* factor_i = span == 1.0 ? lanes.fi : lanes.si[t]; - for (std::size_t b = 0; b < count; ++b) { - const double rr = factor_r[b] * lanes.cr[t][b]; - const double ii = factor_i[b] * lanes.ci[t][b]; - const double ri = factor_r[b] * lanes.ci[t][b]; - const double ir = factor_i[b] * lanes.cr[t][b]; - lanes.cr[t][b] = rr - ii; - lanes.ci[t][b] = ri + ir; - } - } - double total_r[AdamsLanes::B]; - double total_i[AdamsLanes::B]; - for (std::size_t b = 0; b < count; ++b) { - total_r[b] = 0.0; - total_i[b] = 0.0; - } - for (std::size_t t = 0; t < weights.order; ++t) { - const double weight = weights.comp_predictor[t]; - for (std::size_t b = 0; b < count; ++b) { - const double real = weight * lanes.cr[t][b]; - const double imag = weight * lanes.ci[t][b]; - total_r[b] = total_r[b] + real; - total_i[b] = total_i[b] + imag; - } - } - for (std::size_t b = 0; b < count; ++b) { - const double real = dz * total_r[b]; - const double imag = dz * total_i[b]; - lanes.vr[b] = lanes.vr[b] + real; - lanes.vi[b] = lanes.vi[b] + imag; - } -} - -// A varying operator's endpoint rate `sum_t w_t Phi_t` into `dr`/`di` and -// `endpoint_rate`, as `adams_corrected_compensation` forms it. Strict. -template -SOLPROP_NATIVE_STRICT_FUNCTION inline void -adams_lanes_rate(AdamsLanes& lanes, const std::size_t n_phi, - const Weights& rate_weights, const Terms& phi_terms, - const Points& endpoint_rate, const std::size_t column, - const std::size_t first) { - SOLPROP_NATIVE_STRICT_BLOCK - const std::size_t count = lanes.count; - for (std::size_t b = 0; b < count; ++b) { - lanes.dr[b] = 0.0; - lanes.di[b] = 0.0; - } - for (std::size_t term = 0; term < n_phi; ++term) { - const double weight = rate_weights(term); - const Complex* fixed = &phi_terms(term, column, first); - for (std::size_t b = 0; b < count; ++b) { - const double real = weight * fixed[b].real(); - const double imag = weight * fixed[b].imag(); - lanes.dr[b] = lanes.dr[b] + real; - lanes.di[b] = lanes.di[b] + imag; - } - } - Complex* rate_out = &endpoint_rate(column, first); - for (std::size_t b = 0; b < count; ++b) - rate_out[b] = Complex(lanes.dr[b], lanes.di[b]); -} - -// The corrector's compensation `(R - x / dz) u` at the endpoint rate `dr`/`di` into -// `nr`/`ni`, and the correction in `vr`/`vi` plus `dz (c_0 endpoint + sum_t c_{t+1} -// carried_t)`, as `adams_corrected_compensation`; `ur`/`ui` hold the lab-frame field. -// Strict. -SOLPROP_NATIVE_STRICT_FUNCTION -inline void adams_lanes_corrected_compensation(AdamsLanes& lanes, - const ExponentialWeights& weights, - const double inverse_dz, - const double dz) { - SOLPROP_NATIVE_STRICT_BLOCK - constexpr std::size_t B = AdamsLanes::B; - const std::size_t count = lanes.count; - double total_r[B]; - double total_i[B]; - const double weight0 = weights.comp_corrector[0]; - for (std::size_t b = 0; b < count; ++b) { - const double scaled_r = inverse_dz * lanes.xr[b]; - const double scaled_i = inverse_dz * lanes.xi[b]; - const double difference_r = lanes.dr[b] - scaled_r; - const double difference_i = lanes.di[b] - scaled_i; - const double rr = difference_r * lanes.ur[b]; - const double ii = difference_i * lanes.ui[b]; - const double ri = difference_r * lanes.ui[b]; - const double ir = difference_i * lanes.ur[b]; - lanes.nr[b] = rr - ii; - lanes.ni[b] = ri + ir; - total_r[b] = weight0 * lanes.nr[b]; - total_i[b] = weight0 * lanes.ni[b]; - } - for (std::size_t t = 0; t < weights.order; ++t) { - const double weight = weights.comp_corrector[t + 1]; - for (std::size_t b = 0; b < count; ++b) { - const double real = weight * lanes.cr[t][b]; - const double imag = weight * lanes.ci[t][b]; - total_r[b] = total_r[b] + real; - total_i[b] = total_i[b] + imag; - } - } - for (std::size_t b = 0; b < count; ++b) { - const double real = dz * total_r[b]; - const double imag = dz * total_i[b]; - lanes.vr[b] = lanes.vr[b] + real; - lanes.vi[b] = lanes.vi[b] + imag; - } -} - -// The squared scaled norms of up to three factored defects of every lane into -// `defect`, as `adams_defect_block` forms them: `wr`/`wi` hold the new sample, -// `hr`/`hi` the history, `mr`/`mi` the moments, `nr`/`ni` the endpoint's and `cr`/`ci` -// the carried compensation, `scale` the divisors. Strict. -SOLPROP_NATIVE_STRICT_FUNCTION -inline void adams_lanes_defects(AdamsLanes& lanes, const ExponentialWeights& weights, - const std::size_t scored, const double dz) { - SOLPROP_NATIVE_STRICT_BLOCK - constexpr std::size_t B = AdamsLanes::B; - const std::size_t count = lanes.count; - for (std::size_t row = 0; row < scored; ++row) { - double difference_r[B]; - double difference_i[B]; - double weight_r[B]; - double weight_i[B]; - const double weight0 = weights.rows[row][0]; - for (std::size_t b = 0; b < count; ++b) { - difference_r[b] = weight0 * lanes.wr[b]; - difference_i[b] = weight0 * lanes.wi[b]; - weight_r[b] = 0.0; - weight_i[b] = 0.0; - } - for (std::size_t t = 1; t < weights.width; ++t) { - const double weight = weights.rows[row][t]; - for (std::size_t b = 0; b < count; ++b) { - const double real = weight * lanes.hr[t - 1][b]; - const double imag = weight * lanes.hi[t - 1][b]; - difference_r[b] = difference_r[b] + real; - difference_i[b] = difference_i[b] + imag; - } - } - for (std::size_t k = 0; k < weights.count; ++k) { - const double weight = weights.rows[row][weights.width + k]; - for (std::size_t b = 0; b < count; ++b) { - const double real = weight * lanes.mr[k][b]; - const double imag = weight * lanes.mi[k][b]; - weight_r[b] = weight_r[b] + real; - weight_i[b] = weight_i[b] + imag; - } - } - double defect_r[B]; - double defect_i[B]; - for (std::size_t b = 0; b < count; ++b) { - const double rr = difference_r[b] * weight_r[b]; - const double ii = difference_i[b] * weight_i[b]; - const double ri = difference_r[b] * weight_i[b]; - const double ir = difference_i[b] * weight_r[b]; - const double product_r = rr - ii; - const double product_i = ri + ir; - defect_r[b] = dz * product_r; - defect_i[b] = dz * product_i; - } - if (weights.compensated) { - double total_r[B]; - double total_i[B]; - const double first = weights.comp_rows[row][0]; - for (std::size_t b = 0; b < count; ++b) { - total_r[b] = first * lanes.nr[b]; - total_i[b] = first * lanes.ni[b]; - } - for (std::size_t t = 0; t + 1 < weights.width; ++t) { - const double weight = weights.comp_rows[row][t + 1]; - for (std::size_t b = 0; b < count; ++b) { - const double real = weight * lanes.cr[t][b]; - const double imag = weight * lanes.ci[t][b]; - total_r[b] = total_r[b] + real; - total_i[b] = total_i[b] + imag; - } - } - for (std::size_t b = 0; b < count; ++b) { - const double real = dz * total_r[b]; - const double imag = dz * total_i[b]; - defect_r[b] = defect_r[b] + real; - defect_i[b] = defect_i[b] + imag; - } - } - for (std::size_t b = 0; b < count; ++b) { - const double real = defect_r[b] / lanes.scale[b]; - const double imag = defect_i[b] / lanes.scale[b]; - const double rr = real * real; - const double ii = imag * imag; - lanes.defect[row][b] = rr + ii; - } - } -} -#endif - -#ifdef SOLPROP_NATIVE_ADAMS_LANES -// The whole step's phase increment `sum_t dPhi_t Phi_t` of every lane into `xr`/`xi`, -// as `adams_phase_sum` forms it, and with `Rate` a varying operator's endpoint rate -// `sum_t w_t Phi_t` into `dr`/`di`, as `adams_corrected_compensation` forms it, in the -// same pass over the terms. Strict. -template -SOLPROP_NATIVE_STRICT_FUNCTION inline void -adams_lanes_phase(AdamsLanes& lanes, const Terms& phi_terms, const Weights& phi_delta, - const RateWeights& rate_weights, const std::size_t offset, - const std::size_t n_phi, const std::size_t column, - const std::size_t first) { - SOLPROP_NATIVE_STRICT_BLOCK - const std::size_t count = lanes.count; - for (std::size_t b = 0; b < count; ++b) { - lanes.xr[b] = 0.0; - lanes.xi[b] = 0.0; - lanes.dr[b] = 0.0; - lanes.di[b] = 0.0; - } - for (std::size_t term = 0; term < n_phi; ++term) { - const double weight = phi_delta(offset + term); - const Complex* fixed = &phi_terms(term, column, first); - // A zero weight is skipped, exactly, as in `StageTerms::phase_sum`. - if constexpr (Rate) { - const double rate = rate_weights(term); - if (weight == 0.0) - for (std::size_t b = 0; b < count; ++b) { - lanes.dr[b] = lanes.dr[b] + rate * fixed[b].real(); - lanes.di[b] = lanes.di[b] + rate * fixed[b].imag(); - } - else - for (std::size_t b = 0; b < count; ++b) { - lanes.xr[b] = lanes.xr[b] + weight * fixed[b].real(); - lanes.xi[b] = lanes.xi[b] + weight * fixed[b].imag(); - lanes.dr[b] = lanes.dr[b] + rate * fixed[b].real(); - lanes.di[b] = lanes.di[b] + rate * fixed[b].imag(); - } - } else { - if (weight == 0.0) - continue; - for (std::size_t b = 0; b < count; ++b) { - lanes.xr[b] = lanes.xr[b] + weight * fixed[b].real(); - lanes.xi[b] = lanes.xi[b] + weight * fixed[b].imag(); - } - } - } -} -#endif - -#ifdef SOLPROP_NATIVE_ADAMS_LANES -/** - * The predict pass on a host block's lanes, the elements [first, last) of a column - * whose exponents are formed: the flow, the moments, the predictor's carried samples - * and the prediction, as the element-by-element form writes them. Returns the block's - * share of the accepted state's squared magnitude, summed in element order. - */ -template -inline double adams_lanes_predict( - const Terms& phi_terms, const Weights& phi_delta, const RateWeights& rate_weights, - const Points& endpoint_rate, const std::size_t offset, const std::size_t n_phi, - const ExponentialWeights& weights, const std::size_t formed, const double dz, - const double inverse_dz, const Points& exponent, const Points& flow, - const Points& current, const Points& predicted, const Points& lab, - const Cube& moments, const Cube& store, const Cube& carried, const Cube& states, - const Cube& rates, const Mask& in_norm, const std::size_t column, - const std::size_t first, const std::size_t last) { - constexpr std::size_t B = AdamsLanes::B; - AdamsLanes lanes; - const std::size_t n = last - first; - lanes.count = n; - if constexpr (Phase) { - // The corrector's endpoint rate reads the same terms: formed here, in the same - // pass, and read back by the correct pass. - if (weights.compensated) { - adams_lanes_phase(lanes, phi_terms, phi_delta, rate_weights, offset, n_phi, - column, first); - Complex* rate = &endpoint_rate(column, first); - for (std::size_t b = 0; b < n; ++b) - rate[b] = Complex(lanes.dr[b], lanes.di[b]); - } else { - adams_lanes_phase(lanes, phi_terms, phi_delta, rate_weights, offset, n_phi, - column, first); - } - Complex* x = &exponent(column, first); - for (std::size_t b = 0; b < n; ++b) - x[b] = Complex(lanes.xr[b], lanes.xi[b]); - } else { - const Complex* x = &exponent(column, first); - for (std::size_t b = 0; b < n; ++b) { - lanes.xr[b] = x[b].real(); - lanes.xi[b] = x[b].imag(); - } - } - // Formed before the vector maths writes them: clang-tidy's analyzer does not follow - // its stores (a few stores a block). - for (std::size_t b = 0; b < n; ++b) { - lanes.fr[b] = 0.0; - lanes.fi[b] = 0.0; - } - vmath::exp_complex(lanes.xr, lanes.xi, lanes.fr, lanes.fi, n); - Complex* flow_out = &flow(column, first); - for (std::size_t b = 0; b < n; ++b) - flow_out[b] = Complex(lanes.fr[b], lanes.fi[b]); - adams_lanes_classify(lanes, weights); - vmath::exp_complex(lanes.yr, lanes.yi, lanes.er, lanes.ei, n); - if (lanes.any_large) - adams_lanes_large_exponential(lanes, weights); - adams_lanes_phi(lanes, formed); - if (weights.compensated) - for (std::size_t t = 0; t + 1 < weights.width; ++t) { - const double span = weights.span[t]; - if (span == 1.0 || span == 0.0) - continue; - adams_lanes_span(lanes, span, t); - vmath::exp_complex(lanes.sr[t], lanes.si[t], lanes.sr[t], lanes.si[t], n); - } - // The moments M_k = k! phi_{k+1}. - const std::size_t moment_rows = formed; - for (std::size_t k = 0; k < moment_rows; ++k) { - const double factorial = adams_factorial(k); - for (std::size_t b = 0; b < n; ++b) { - lanes.mr[k][b] = factorial * lanes.pr[k + 1][b]; - lanes.mi[k][b] = factorial * lanes.pi[k + 1][b]; - } - } - for (std::size_t k = 0; k < weights.count; ++k) { - Complex* out = &moments(k, column, first); - for (std::size_t b = 0; b < n; ++b) - out[b] = Complex(lanes.mr[k][b], lanes.mi[k][b]); - } - adams_lanes_history(lanes, weights, store, weights.order, column, first); - // The prediction exp(x) u + dz sum_k M_k sum_j c_jk F_j. - const Complex* accepted = ¤t(column, first); - double total_r[B]; - double total_i[B]; - for (std::size_t b = 0; b < n; ++b) { - lanes.ur[b] = accepted[b].real(); - lanes.ui[b] = accepted[b].imag(); - total_r[b] = 0.0; - total_i[b] = 0.0; - } - for (std::size_t k = 0; k < weights.order; ++k) { - double combined_r[B]; - double combined_i[B]; - for (std::size_t b = 0; b < n; ++b) { - combined_r[b] = 0.0; - combined_i[b] = 0.0; - } - for (std::size_t t = 0; t < weights.order; ++t) { - const double weight = weights.predictor[t][k]; - for (std::size_t b = 0; b < n; ++b) { - combined_r[b] += weight * lanes.hr[t][b]; - combined_i[b] += weight * lanes.hi[t][b]; - } - } - for (std::size_t b = 0; b < n; ++b) { - total_r[b] += lanes.mr[k][b] * combined_r[b] - lanes.mi[k][b] * combined_i[b]; - total_i[b] += lanes.mr[k][b] * combined_i[b] + lanes.mi[k][b] * combined_r[b]; - } - } - for (std::size_t b = 0; b < n; ++b) { - lanes.vr[b] = - lanes.fr[b] * lanes.ur[b] - lanes.fi[b] * lanes.ui[b] + dz * total_r[b]; - lanes.vi[b] = - lanes.fr[b] * lanes.ui[b] + lanes.fi[b] * lanes.ur[b] + dz * total_i[b]; - } - double squares = 0.0; - for (std::size_t b = 0; b < n; ++b) - if (in_norm(first + b) != 0) - squares += lanes.ur[b] * lanes.ur[b] + lanes.ui[b] * lanes.ui[b]; - if (weights.compensated) { - adams_lanes_predicted_compensation(lanes, weights, inverse_dz, dz, rates, states, - column, first); - for (std::size_t t = 0; t + 1 < weights.width; ++t) { - Complex* out = &carried(t, column, first); - for (std::size_t b = 0; b < n; ++b) - out[b] = Complex(lanes.cr[t][b], lanes.ci[t][b]); - } - Complex* lab_out = &lab(column, first); - for (std::size_t b = 0; b < n; ++b) - lab_out[b] = Complex(lanes.vr[b], lanes.vi[b]); - } - Complex* out = &predicted(column, first); - for (std::size_t b = 0; b < n; ++b) - out[b] = Complex(lanes.vr[b], lanes.vi[b]); - return squares; -} -#endif - -// The squared scaled norms of up to three factored defects on the in-band elements of -// [first, last) of a column, added to `update`: a divided difference of the samples -// times one moment combination, and a varying operator's compensation term, over the -// divisor `scale[index - first]`. The defects and their compensation: strict (solprop -// SEP 25 section 4.2). -template -SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION void -adams_defect_block(const ExponentialWeights& weights, const std::size_t rows, - const Store& store, const Moments& moments, const Points& source, - const Carried& carried, const Points& endpoint, const Mask& in_norm, - const double dz, const double* scale, const std::size_t column, - const std::size_t first, const std::size_t last, double* update) { - SOLPROP_NATIVE_STRICT_BLOCK - // The block's sums, added to the reduction's once. A trial scores at most three - // defects (adams_exponential_trial refuses more); the bound says so here too. - constexpr std::size_t max_rows = 3; - const std::size_t scored = rows < max_rows ? rows : max_rows; - double sums[max_rows] = {0.0, 0.0, 0.0}; - for (std::size_t index = first; index < last; ++index) { - if (in_norm(index) == 0) - continue; - for (std::size_t row = 0; row < scored; ++row) { - Complex difference = strict_scale(weights.rows[row][0], source(column, index)); - for (std::size_t t = 1; t < weights.width; ++t) - difference = - strict_accumulate(difference, weights.rows[row][t], - adams_history(store, weights, t - 1, column, index)); - Complex weight{}; - for (std::size_t k = 0; k < weights.count; ++k) - weight = strict_accumulate(weight, weights.rows[row][weights.width + k], - moments(k, column, index)); - Complex defect = strict_scale(dz, strict_mul(difference, weight)); - if (weights.compensated) { - Complex compensation = - strict_scale(weights.comp_rows[row][0], endpoint(column, index)); - for (std::size_t t = 0; t + 1 < weights.width; ++t) - compensation = strict_accumulate(compensation, weights.comp_rows[row][t + 1], - carried(t, column, index)); - defect = strict_accumulate(defect, dz, compensation); - } - sums[row] += squared_abs(strict_divide(defect, scale[index - first])); - } - } - for (std::size_t row = 0; row < scored; ++row) - update[row] += sums[row]; -} - -/** - * The elementwise trial's last pass (solprop SEP 16 phase 3b), a block of a column a - * task: the new sample and the corrected field `exp(x) u + dz sum_k M_k sum_j c_jk - * F_j` with a varying operator's compensation, then the squared scaled norms of up to - * three factored defects, reduced in the same traversal. solprop's own controller's - * divisor is formed here from the accepted state's magnitude, as its - * PIController.scale forms it; any other controller's is read from `divisor`. - */ -struct ExponentialCorrection { - using value_type = double[]; - using size_type = std::size_t; - size_type value_count; - Kokkos::View store, moments, stages, carried; - Kokkos::View phi_terms; - Kokkos::View source, flow, current, advanced, - exponent; - Kokkos::View lab, endpoint, endpoint_rate; - Kokkos::View rate_weights; - Kokkos::View in_norm; - Kokkos::View divisor, scale_weights; - Kokkos::View magnitude_sum; - ExponentialWeights weights; - double dz, inverse_dz, rtol, atol; - std::size_t n, blocks, n_phi; - bool device_scale; - // The host lanes' predict pass formed the endpoint rate (off a separable route). - bool rate_formed; - -#ifdef SOLPROP_NATIVE_ADAMS_LANES - // The pass on a host block's lanes (`AdamsLanes`), as the element-by-element form - // below writes it; the norms' sums are taken in element order. - void lanes_block(const std::size_t column, const std::size_t first, - const std::size_t last, const double magnitude, - value_type update) const { - constexpr std::size_t B = AdamsLanes::B; - AdamsLanes lanes; - const std::size_t count = last - first; - lanes.count = count; - const Complex* sample = &stages(0, column, first); - Complex* sample_out = &source(column, first); - for (std::size_t b = 0; b < count; ++b) { - lanes.wr[b] = sample[b].real(); - lanes.wi[b] = sample[b].imag(); - sample_out[b] = sample[b]; - } - adams_lanes_history(lanes, weights, store, weights.width - 1, column, first); - for (std::size_t k = 0; k < weights.count; ++k) { - const Complex* moment = &moments(k, column, first); - for (std::size_t b = 0; b < count; ++b) { - lanes.mr[k][b] = moment[b].real(); - lanes.mi[k][b] = moment[b].imag(); - } - } - double total_r[B]; - double total_i[B]; - for (std::size_t b = 0; b < count; ++b) { - total_r[b] = 0.0; - total_i[b] = 0.0; - } - for (std::size_t k = 0; k < weights.count; ++k) { - double combined_r[B]; - double combined_i[B]; - const double weight0 = weights.corrector[0][k]; - for (std::size_t b = 0; b < count; ++b) { - combined_r[b] = weight0 * lanes.wr[b]; - combined_i[b] = weight0 * lanes.wi[b]; - } - for (std::size_t t = 1; t < weights.width; ++t) { - const double weight = weights.corrector[t][k]; - for (std::size_t b = 0; b < count; ++b) { - combined_r[b] += weight * lanes.hr[t - 1][b]; - combined_i[b] += weight * lanes.hi[t - 1][b]; - } - } - for (std::size_t b = 0; b < count; ++b) { - total_r[b] += lanes.mr[k][b] * combined_r[b] - lanes.mi[k][b] * combined_i[b]; - total_i[b] += lanes.mr[k][b] * combined_i[b] + lanes.mi[k][b] * combined_r[b]; - } - } - const Complex* propagate = &flow(column, first); - const Complex* accepted = ¤t(column, first); - for (std::size_t b = 0; b < count; ++b) { - const Complex value = - propagate[b] * accepted[b] + dz * Complex(total_r[b], total_i[b]); - lanes.vr[b] = value.real(); - lanes.vi[b] = value.imag(); - } - if (weights.compensated) { - const Complex* x = &exponent(column, first); - const Complex* field = &lab(column, first); - for (std::size_t b = 0; b < count; ++b) { - lanes.xr[b] = x[b].real(); - lanes.xi[b] = x[b].imag(); - lanes.ur[b] = field[b].real(); - lanes.ui[b] = field[b].imag(); - } - for (std::size_t t = 0; t + 1 < weights.width; ++t) { - const Complex* row = &carried(t, column, first); - for (std::size_t b = 0; b < count; ++b) { - lanes.cr[t][b] = row[b].real(); - lanes.ci[t][b] = row[b].imag(); - } - } - if (rate_formed) { - const Complex* rate = &endpoint_rate(column, first); - for (std::size_t b = 0; b < count; ++b) { - lanes.dr[b] = rate[b].real(); - lanes.di[b] = rate[b].imag(); - } - } else { - adams_lanes_rate(lanes, n_phi, rate_weights, phi_terms, endpoint_rate, column, - first); - } - adams_lanes_corrected_compensation(lanes, weights, inverse_dz, dz); - Complex* out = &endpoint(column, first); - for (std::size_t b = 0; b < count; ++b) - out[b] = Complex(lanes.nr[b], lanes.ni[b]); - } - Complex* out = &advanced(column, first); - for (std::size_t b = 0; b < count; ++b) - out[b] = Complex(lanes.vr[b], lanes.vi[b]); - if (device_scale) { - if (scale_weights.extent(0) != 0) - for (std::size_t b = 0; b < count; ++b) - lanes.scale[b] = rtol * scale_weights(first + b) * magnitude + atol; - else - for (std::size_t b = 0; b < count; ++b) - lanes.scale[b] = rtol * magnitude + atol; - } else { - for (std::size_t b = 0; b < count; ++b) - lanes.scale[b] = divisor(first + b); - } - constexpr std::size_t max_rows = 3; - const std::size_t scored = value_count < max_rows ? value_count : max_rows; - adams_lanes_defects(lanes, weights, scored, dz); - double sums[max_rows] = {0.0, 0.0, 0.0}; - for (std::size_t b = 0; b < count; ++b) { - if (in_norm(first + b) == 0) - continue; - for (std::size_t row = 0; row < scored; ++row) - sums[row] += lanes.defect[row][b]; - } - for (std::size_t row = 0; row < scored; ++row) - update[row] += sums[row]; - } -#endif - -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // One device element's pass, `Width` the trial's samples known at compile time: as - // the loops below and `adams_defect_block` form it, in the same operations and - // order, but with the history rows and moments read once into registers. Through - // the views, which the pass also writes, every use reloads them; on an AMD Instinct - // MI325X this pass is latency-bound on a single-mode trial's few thousand elements. - template - KOKKOS_INLINE_FUNCTION void - device_element(const std::size_t column, const std::size_t index, - const double magnitude, value_type update) const { - const Complex g = stages(0, column, index); - source(column, index) = g; - Complex history[Width]; - for (std::size_t t = 0; t + 1 < Width; ++t) - history[t] = adams_history(store, weights, t, column, index); - Complex moment[Width]; - for (std::size_t k = 0; k < Width; ++k) - if (k < weights.count) - moment[k] = moments(k, column, index); - Complex total{}; - for (std::size_t k = 0; k < Width; ++k) { - if (k >= weights.count) - break; - Complex combined = weights.corrector[0][k] * g; - for (std::size_t t = 1; t < Width; ++t) - combined += weights.corrector[t][k] * history[t - 1]; - total += moment[k] * combined; - } - Complex value = flow(column, index) * current(column, index) + dz * total; - if (weights.compensated) - value = adams_corrected_compensation(weights, n_phi, rate_weights, phi_terms, - exponent(column, index), inverse_dz, - lab(column, index), dz, value, endpoint_rate, - endpoint, carried, column, index); - advanced(column, index) = value; - if (in_norm(index) == 0) - return; - const double scale = device_scale - ? (scale_weights.extent(0) != 0 - ? rtol * scale_weights(index) * magnitude + atol - : rtol * magnitude + atol) - : divisor(index); - constexpr std::size_t max_rows = 3; - const std::size_t scored = value_count < max_rows ? value_count : max_rows; - for (std::size_t row = 0; row < max_rows; ++row) { - if (row >= scored) - break; - Complex difference = strict_scale(weights.rows[row][0], g); - for (std::size_t t = 1; t < Width; ++t) - difference = - strict_accumulate(difference, weights.rows[row][t], history[t - 1]); - Complex weight{}; - for (std::size_t k = 0; k < Width; ++k) { - if (k >= weights.count) - break; - weight = strict_accumulate(weight, weights.rows[row][Width + k], moment[k]); - } - Complex defect = strict_scale(dz, strict_mul(difference, weight)); - if (weights.compensated) { - Complex compensation = - strict_scale(weights.comp_rows[row][0], endpoint(column, index)); - for (std::size_t t = 0; t + 1 < Width; ++t) - compensation = strict_accumulate(compensation, weights.comp_rows[row][t + 1], - carried(t, column, index)); - defect = strict_accumulate(defect, dz, compensation); - } - update[row] += squared_abs(strict_divide(defect, scale)); - } - } -#endif - - KOKKOS_INLINE_FUNCTION void operator()(const std::size_t task, - value_type update) const { - const auto column = task / blocks; - const auto first = (task % blocks) * adams_block; - const auto last = first + adams_block < n ? first + adams_block : n; - const double magnitude = device_scale ? Kokkos::sqrt(magnitude_sum()) : 0.0; -#ifdef SOLPROP_NATIVE_ADAMS_LANES - if constexpr (vmath::vectorised) { - lanes_block(column, first, last, magnitude, update); - return; - } -#endif -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - static_assert(adams_block == 1, "a device thread takes one element"); - if (first < last) { - switch (weights.width) { - case 1: - return device_element<1>(column, first, magnitude, update); - case 2: - return device_element<2>(column, first, magnitude, update); - case 3: - return device_element<3>(column, first, magnitude, update); - case 4: - return device_element<4>(column, first, magnitude, update); - case 5: - return device_element<5>(column, first, magnitude, update); - case 6: - return device_element<6>(column, first, magnitude, update); - case 7: - return device_element<7>(column, first, magnitude, update); - case 8: - return device_element<8>(column, first, magnitude, update); - default: - break; - } - } -#endif - double scale[adams_block]{}; - for (std::size_t index = first; index < last; ++index) { - const Complex g = stages(0, column, index); - source(column, index) = g; - Complex total{}; - for (std::size_t k = 0; k < weights.count; ++k) { - Complex combined = weights.corrector[0][k] * g; - for (std::size_t t = 1; t < weights.width; ++t) - combined += weights.corrector[t][k] * - adams_history(store, weights, t - 1, column, index); - total += moments(k, column, index) * combined; - } - Complex value = flow(column, index) * current(column, index) + dz * total; - // The new sample's compensation at the predicted field, untransported, with the - // rate read on the side of the step's end the step came from. - if (weights.compensated) - value = adams_corrected_compensation( - weights, n_phi, rate_weights, phi_terms, exponent(column, index), - inverse_dz, lab(column, index), dz, value, endpoint_rate, endpoint, carried, - column, index); - advanced(column, index) = value; - if (in_norm(index) == 0) - continue; - if (device_scale) - scale[index - first] = scale_weights.extent(0) != 0 - ? rtol * scale_weights(index) * magnitude + atol - : rtol * magnitude + atol; - else - scale[index - first] = divisor(index); - } - adams_defect_block(weights, value_count, store, moments, source, carried, endpoint, - in_norm, dz, scale, column, first, last, update); - } - KOKKOS_INLINE_FUNCTION void join(value_type destination, - const value_type source_values) const { - for (size_type row = 0; row < value_count; ++row) - destination[row] += source_values[row]; - } - KOKKOS_INLINE_FUNCTION void init(value_type values) const { - for (size_type row = 0; row < value_count; ++row) - values[row] = 0.0; - } -}; - -} // namespace - -// The exponential trial without a coupled flow, every formula elementwise, in three -// passes on the trial's own width (solprop SEP 16 phase 3b). The first forms the -// step's exponent, flow and moments, a varying operator's carried samples and the -// prediction, and reduces the accepted state's magnitude for the norm's divisor; the -// second is the source at the prediction; the third is ExponentialCorrection. Each -// threaded pass costs a fork and a join, which on a host is most of what one of these -// passes costs on a few thousand elements. The prediction is written beside the -// accepted state and the two exchange places around the source, so the accepted -// state is never copied. -std::vector GeneralWorkspace::adams_elementwise_trial( - const double dz, const ExponentialWeights& weights, - const std::vector& divisor, const TrialScalars& step, - const TrialScalars& sample, const std::vector& rate) { - const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - const bool compensated = weights.compensated; - // solprop's own controller's scale is formed from the magnitude below; any other - // controller's divisor crosses. - const bool device_scale = divisor.empty() && adams_device_scale_; - if (!device_scale) - adams_upload_divisor(divisor); - // The corrector forms a compensated trial's endpoint rate and compensation; only its - // weights cross, with the step's and the sample's scalars, in the trial's one copy. - upload_trial(step, &sample, compensated ? &rate : nullptr); - const std::size_t blocks = (n + adams_block - 1) / adams_block; -#ifdef KOKKOS_ENABLE_CUDA - // A device thread takes one element of a long FP64 chain, and Kokkos's CUDA range - // reduction gives each block 256 of them: a four-mode capillary's 10496 elements - // were 41 blocks on an NVIDIA A40's 84 multiprocessors. Blocks of 128 take them - // all. - using Tasks = Kokkos::RangePolicy>; -#else - using Tasks = Kokkos::RangePolicy; -#endif - const auto tasks = Tasks(adams_execution_space_, 0, columns * blocks); - const bool separable = problem_.separable_geometry(); - const auto spatial = spatial_; - const auto active = active_mask_; - const auto n_pol = problem_.n_pol_state; - const auto n_stages = problem_.n_stages; - const auto phi_terms = phi_terms_; - const auto phi_delta = stage_phi_delta_; - const auto n_phi = problem_.n_phi_terms; - const auto offset = n_stages * n_phi; - const auto exponent = adams_exponent_; - const auto flow = adams_propagator_; - const auto moments = adams_moments_; - const auto store = adams_store_; - const auto current = current_; - const auto predicted = adams_saved_; - const auto lab = adams_lab_; - const auto carried = adams_carried_; - const auto states = adams_states_; - const auto rates = adams_rates_; - const auto in_norm = norm_mask_; - const std::size_t count = weights.count; - const std::size_t formed = count > weights.order ? count : weights.order; - const double inverse_dz = 1.0 / dz; -#ifdef SOLPROP_NATIVE_ADAMS_LANES - // A host block's predict pass forms the corrector's endpoint rate beside the phase, - // from the weights the trial uploaded with its scalars. - const auto endpoint_rates = stage_rate_weights_; - const auto endpoint_rate = adams_endpoint_rate_; -#endif - // The step's exponent, on a separable route the step's length times each element's - // operator as the stage kernel forms the whole propagator, and its flow; its - // moments (kept for the corrector and the norms); a varying operator's carried - // samples; and the prediction, exp(x) u + dz sum_k M_k sum_j c_jk F_j and the - // compensation's predictor, the field the lab-frame source is evaluated at. - const auto predict = KOKKOS_LAMBDA(const std::size_t task, double& magnitude) { - const auto column = task / blocks; - const auto first = (task % blocks) * adams_block; - const auto last = first + adams_block < n ? first + adams_block : n; - // With vector maths a host block forms its exponentials in passes and the rest of - // the pass on its lanes (`adams_lanes_predict`); otherwise each element forms its - // own. -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - constexpr bool passes = vmath::vectorised; -#else - // Read only by the host's lanes (Clang's -Wunused-variable on HIP builds). - [[maybe_unused]] constexpr bool passes = false; -#endif - if (separable) - for (std::size_t index = first; index < last; ++index) { - const Complex x = - phi_delta(n_stages) * - spatial.value(column / n_pol, index, active(index) != 0U).linear; - exponent(column, index) = x; - // A device lambda may not first capture a view inside `if constexpr` (nvcc): - // devices never form passes, so the guard is the host's alone. -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - if constexpr (!passes) -#endif - flow(column, index) = exponential(x); - } -#ifdef SOLPROP_NATIVE_ADAMS_LANES - if constexpr (passes) { - magnitude += separable - ? adams_lanes_predict( - phi_terms, phi_delta, endpoint_rates, endpoint_rate, - offset, n_phi, weights, formed, dz, inverse_dz, exponent, - flow, current, predicted, lab, moments, store, carried, - states, rates, in_norm, column, first, last) - : adams_lanes_predict( - phi_terms, phi_delta, endpoint_rates, endpoint_rate, - offset, n_phi, weights, formed, dz, inverse_dz, exponent, - flow, current, predicted, lab, moments, store, carried, - states, rates, in_norm, column, first, last); - return; - } -#endif - if (!separable) - adams_phase_block(phi_terms, phi_delta, offset, n_phi, column, first, last, - exponent, flow); - double squares = 0.0; - for (std::size_t index = first; index < last; ++index) { - const Complex x = exponent(column, index); - const Complex propagate = flow(column, index); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const Complex accepted = current(column, index); - if (in_norm(index) != 0) - squares += squared_abs(accepted); - Complex value = adams_device_predict(weights, formed, x, propagate, accepted, dz, - moments, store, column, index); -#else - Complex phi[2 * adams_max_samples + 1]; - adams_moment_phi(weights, x, propagate, formed, phi); - for (std::size_t k = 0; k < count; ++k) - moments(k, column, index) = adams_factorial(k) * phi[k + 1]; - const Complex accepted = current(column, index); - if (in_norm(index) != 0) - squares += squared_abs(accepted); - Complex total{}; - for (std::size_t k = 0; k < weights.order; ++k) { - Complex combined{}; - for (std::size_t t = 0; t < weights.order; ++t) - combined += - weights.predictor[t][k] * adams_history(store, weights, t, column, index); - const Complex moment = adams_factorial(k) * phi[k + 1]; - total += moment * combined; - } - Complex value = propagate * accepted + dz * total; -#endif - if (compensated) { - value = - adams_predicted_compensation(weights, x, propagate, inverse_dz, dz, value, - rates, states, carried, column, index); - lab(column, index) = value; - } - predicted(column, index) = value; - } - magnitude += squares; - }; - if (device_scale) { - // A threaded reduction in whatever order measures fastest (solprop SEP 25). - launch::parallel_reduce("solprop_native.adams.exponential_predict", tasks, predict, - adams_magnitude_sum_); - } else { - double unused = 0.0; - launch::parallel_reduce("solprop_native.adams.exponential_predict", tasks, predict, - unused); - } - std::swap(current_, adams_saved_); - use_uploaded_sample(sample); - // On a device the source's launches replay as one graph while they repeat. - adams_source_graph_.run(execution_space_, [&] { adams_evaluate_source(sample); }); - std::swap(current_, adams_saved_); - const ExponentialCorrection reducer{.value_count = weights.n_rows, - .store = store, - .moments = moments, - .stages = stages_, - .carried = carried, - .phi_terms = phi_terms, - .source = adams_source_, - .flow = flow, - .current = current_, - .advanced = advanced_, - .exponent = exponent, - .lab = lab, - .endpoint = adams_endpoint_, - .endpoint_rate = adams_endpoint_rate_, - .rate_weights = stage_rate_weights_, - .in_norm = in_norm, - .divisor = adams_divisor_, - .scale_weights = adams_scale_weights_, - .magnitude_sum = adams_magnitude_sum_, - .weights = weights, - .dz = dz, - .inverse_dz = inverse_dz, - .rtol = adams_rtol_, - .atol = adams_atol_, - .n = n, - .blocks = blocks, - .n_phi = n_phi, - .device_scale = device_scale, - .rate_formed = !separable}; - auto norms = adams_norms("solprop_native.adams.exponential_correct", tasks, reducer, - weights.n_rows); - adams_trial_compensated_ = compensated; - adams_trial_coupled_ = false; - return norms; -} - -// Refuse a trial whose bins off the flow's eigenbasis route would need more squarings -// than solprop's reference admits for a flow action (solprop SEP 47 phase 4): the -// augmented exponential of each bin's shifted block for the moments, and the block's -// own `span dz T` for each carried sample. Before any pass of the trial moves the -// state, so a refusal leaves the accepted step as it was. -void GeneralWorkspace::adams_fallback_squarings(const std::size_t section, - const double dz, - const ExponentialWeights& weights) { - const auto slots = flow_->block_slots(); - const auto route = flow_->route(); - const auto triangular = flow_->fallback_triangular(); - const auto exponent = adams_exponent_; - const auto modes = problem_.n_modes; - const std::size_t count = weights.count; - const auto policy = - Kokkos::RangePolicy(flow_execution_space_, 0, problem_.n_omega); - // The moments' augmented exponent, and the carries' `span dz T`, apart, so that a - // refusal names which. - int moments = 0; - int carries = 0; - launch::parallel_reduce( - "solprop_native.adams.fallback_squarings", policy, - KOKKOS_LAMBDA(const std::size_t index, int& most, int& carried) { - const auto one = slots(index); - if (one == 0) - return; - const auto path = route(section, one - 1); - if (path == 0) - return; - const Complex argument = adams_moment_argument(weights, exponent(0, index)); - const int squarings = flow_functions::augmented_squarings( - argument, dz, triangular, path - 1, modes, count); - most = squarings > most ? squarings : most; - if (weights.compensated) - for (std::size_t t = 0; t + 1 < weights.width; ++t) { - const int carry = flow_functions::augmented_squarings( - Complex{}, weights.span[t] * dz, triangular, path - 1, modes, 0); - carried = carry > carried ? carry : carried; - } - }, - Kokkos::Max(moments), Kokkos::Max(carries)); - const std::string bin = - "native cannot take the flow and moments of a coupled flow's bin off its " - "eigenbasis route"; - if (moments == flow_functions::non_finite || carries == flow_functions::non_finite) - throw BackendRefusal(bin + ": the step's exponent there is not finite"); - if (moments <= flow_functions::max_squarings && - carries <= flow_functions::max_squarings) - return; - std::array length{}; - std::snprintf(length.data(), length.size(), "%.4g", dz); - const std::string limit = " would need more than " + - std::to_string(flow_functions::max_squarings) + - " squarings, the most this backend admits. Cap dz_max"; - if (moments > flow_functions::max_squarings) - throw BackendRefusal(bin + " over a step of " + length.data() + " m: its exponent" + - limit); - throw BackendRefusal(bin + " over a step of " + length.data() + - " m: a history sample's carry to the step's end" + limit); -} - -std::vector GeneralWorkspace::adams_exponential_trial( - const double dz, const ExponentialWeights& weights, - const std::vector& divisor, const TrialScalars& step, - const TrialScalars& sample, const std::vector& rate) { - require_adams(); - if (weights.compensated && adams_states_.extent(0) == 0) - throw std::logic_error("the Adams history carries no compensation"); - // Beside a coupled flow the step's flow and moments are matrices on the block's bins - // (solprop SEP 16 phases 5 and 6); the sample is the material source alone. - const bool coupled = static_cast(flow_); - if (coupled && adams_block_.extent(0) == 0) - throw std::logic_error("the Adams history holds no coupled block"); - if (weights.n_rows == 0 || weights.n_rows > 3 || weights.width == 0 || - weights.width > adams_store_.extent(0) || weights.count > weights.width || - weights.order >= weights.width) - throw std::invalid_argument("Adams coefficient tables are out of range"); - for (std::size_t t = 0; t + 1 < weights.width; ++t) - if (weights.slot[t] >= adams_store_.extent(0)) - throw std::out_of_range("Adams history row"); - require_trial_scalars(step); - require_trial_scalars(sample); - if (!coupled) - return adams_elementwise_trial(dz, weights, divisor, step, sample, rate); - adams_upload_divisor(divisor); - // The step's exponent and flow, and its moments, once for every formula; the - // sample's scalars cross in the same copy. - upload_trial(step, &sample); - adams_phase(); - const std::size_t section = step.flow_section; - const bool residual = adams_applied_.extent(0) != 0; - if (residual) - upload_weights(execution_space_, adams_end_weights_, step.flow_weights); - const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); - const auto bins = Kokkos::RangePolicy(flow_execution_space_, 0, n); - const auto exponent = adams_exponent_; - const auto flow = adams_propagator_; - const auto moments = adams_moments_; - const std::size_t count = weights.count; - const auto slots = flow_->block_slots(); - // The elementwise moments, off the block's bins: on them the block's matrices below - // take their place. - launch::parallel_for( - "solprop_native.adams.moments", policy, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - if (slots(index) != 0) - return; - Complex phi[2 * adams_max_samples + 1]; - adams_moment_phi(weights, exponent(column, index), flow(column, index), count, - phi); - for (std::size_t k = 0; k < count; ++k) - moments(k, column, index) = adams_factorial(k) * phi[k + 1]; - }); - // The block's flow and moments, W f(x + h lambda) Y from the section's eigenbasis on - // its route; a bin off it is formed from its Schur factors by the augmented - // exponential (`flow_functions`), on a host and a device alike. - const auto block = adams_block_; - // A host's channel vectors of a block wider than `adams_max_channels`. - [[maybe_unused]] const auto channels = adams_channels_; - const auto modes = problem_.n_modes; - // The bins off the eigenbasis route (solprop SEP 47 phase 4). - const auto route = flow_->route(); - const auto fallback_vectors = flow_->fallback_vectors(); - const auto fallback_triangular = flow_->fallback_triangular(); - const auto fallback = adams_fallback_; - const bool falls_back = flow_->falls_back(section); - if (flow_->falls_back(section)) - adams_fallback_squarings(section, dz, weights); - const auto left = flow_->left(); - const auto right = flow_->right(); - const auto eigenvalues = flow_->eigenvalues(); - const auto terms = - residual ? flow_->residual_terms() : CoupledFlowAction::ComplexView4{}; - const auto frozen = residual ? flow_->frozen() : CoupledFlowAction::ComplexView4{}; - // A device takes a thread per (bin, channel) in the coupled passes below: its - // lanes outnumber the bins and a channel vector does not fit a thread's - // registers. A host keeps a task per bin, which forms a bin's shared terms once. -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - launch::parallel_for( - "solprop_native.adams.block_functions", - Range2(flow_execution_space_, {0, 0}, {n, modes * modes}), - KOKKOS_LAMBDA(const std::size_t index, const std::size_t pair) { - const auto one = slots(index); - if (one == 0) - return; - const auto bin = one - 1; - // A bin off the route is the next pass's. - if (falls_back && route(section, bin) != 0) - return; - const auto i = pair / modes; - const auto j = pair % modes; - const Complex shift = exponent(0, index); - // One entry (i, j) of every function's matrix, a thread each: the sum over - // the eigenvalues in their order, each function formed where it is needed. - Complex total[adams_max_samples + 1]; - for (std::size_t k = 0; k <= count; ++k) - total[k] = Complex{}; - Complex phi[2 * adams_max_samples + 1]; - for (std::size_t m = 0; m < modes; ++m) { - const Complex x = shift + dz * eigenvalues(section, bin, m); - const Complex exp_x = exponential(x); - // The flow is exp(h B); the moments are the shifted block's (solprop SEP - // 35 section 4.5): every eigenvalue moves by -sigma. - adams_moment_phi(weights, x, exp_x, count, phi); - const Complex l = left(section, bin, i, m); - const Complex r = right(section, bin, m, j); - total[0] += l * exp_x * r; - for (std::size_t k = 0; k < count; ++k) - total[k + 1] += l * (adams_factorial(k) * phi[k + 1]) * r; - } - for (std::size_t k = 0; k <= count; ++k) - block(k, bin, i, j) = total[k]; - }); - if (falls_back) - // Off the eigenbasis route, a thread per bin: the host task's passes, in its order - // (the first block row of the augmented exponential of the shifted block, carried - // back by Z), its matrices in the bin's scratch row rather than on a stack. - launch::parallel_for( - "solprop_native.adams.fallback_block_functions", bins, - KOKKOS_LAMBDA(const std::size_t index) { - const auto one = slots(index); - if (one == 0) - return; - const auto bin = one - 1; - const auto path = route(section, bin); - if (path == 0) - return; - const auto f = path - 1; - const Complex argument = adams_moment_argument(weights, exponent(0, index)); - flow_functions::augmented_row( - argument, dz, fallback_triangular, f, modes, count, - flow_functions::augmented_squarings(argument, dz, fallback_triangular, f, - modes, count), - fallback, bin); - const Complex factor = weights.shifted - ? exponential(Complex(0.0, weights.shift)) - : Complex(1.0, 0.0); - const auto product = count + 1; - for (std::size_t k = 0; k <= count; ++k) { - for (std::size_t i = 0; i < modes; ++i) - for (std::size_t j = 0; j < modes; ++j) { - Complex total{}; - for (std::size_t m = 0; m <= j; ++m) - total += fallback_vectors(f, i, m) * fallback(bin, k, m, j); - fallback(bin, product, i, j) = total; - } - const Complex scale = - k == 0 ? factor : Complex(adams_factorial(k - 1), 0.0); - for (std::size_t i = 0; i < modes; ++i) - for (std::size_t j = 0; j < modes; ++j) { - Complex total{}; - for (std::size_t m = 0; m < modes; ++m) - total += fallback(bin, product, i, m) * - Kokkos::conj(fallback_vectors(f, j, m)); - block(k, bin, i, j) = scale * total; - } - } - }); -#else - static_cast(falls_back); - launch::parallel_for( - "solprop_native.adams.block_functions", bins, - KOKKOS_LAMBDA(const std::size_t index) { - const auto one = slots(index); - if (one == 0) - return; - const auto bin = one - 1; - const auto path = route(section, bin); - if (path != 0) { - // Off the eigenbasis route: the first block row of the augmented - // exponential of the shifted block, carried back by Z, the moments - // scaled by k! and the flow by the carrier's exp(sigma) (solprop's - // `coupled_moments`). - const auto f = path - 1; - const Complex argument = adams_moment_argument(weights, exponent(0, index)); - flow_functions::augmented_row( - argument, dz, fallback_triangular, f, modes, count, - flow_functions::augmented_squarings(argument, dz, fallback_triangular, f, - modes, count), - fallback, bin); - const Complex factor = weights.shifted - ? exponential(Complex(0.0, weights.shift)) - : Complex(1.0, 0.0); - const auto product = count + 1; - for (std::size_t k = 0; k <= count; ++k) { - for (std::size_t i = 0; i < modes; ++i) - for (std::size_t j = 0; j < modes; ++j) { - Complex total{}; - for (std::size_t m = 0; m <= j; ++m) - total += fallback_vectors(f, i, m) * fallback(bin, k, m, j); - fallback(bin, product, i, j) = total; - } - const Complex scale = - k == 0 ? factor : Complex(adams_factorial(k - 1), 0.0); - for (std::size_t i = 0; i < modes; ++i) - for (std::size_t j = 0; j < modes; ++j) { - Complex total{}; - for (std::size_t m = 0; m < modes; ++m) - total += fallback(bin, product, i, m) * - Kokkos::conj(fallback_vectors(f, j, m)); - block(k, bin, i, j) = scale * total; - } - } - return; - } - const Complex shift = exponent(0, index); - // Eigenvalue m's functions at `values + m (adams_max_samples + 1)`. - constexpr std::size_t stride = adams_max_samples + 1; - Complex stack[adams_channel_stack]; - Complex* const values = adams_channel_rows(stack, channels, index); - Complex phi[2 * adams_max_samples + 1]; - for (std::size_t m = 0; m < modes; ++m) { - const Complex x = shift + dz * eigenvalues(section, bin, m); - const Complex exp_x = exponential(x); - values[m * stride] = exp_x; - adams_moment_phi(weights, x, exp_x, count, phi); - for (std::size_t k = 0; k < count; ++k) - values[m * stride + k + 1] = adams_factorial(k) * phi[k + 1]; - } - for (std::size_t k = 0; k <= count; ++k) - for (std::size_t i = 0; i < modes; ++i) - for (std::size_t j = 0; j < modes; ++j) { - Complex total{}; - for (std::size_t m = 0; m < modes; ++m) - total += left(section, bin, i, m) * values[m * stride + k] * - right(section, bin, m, j); - block(k, bin, i, j) = total; - } - }); -#endif - // A varying operator's samples carried to the step's end in the chord's flow: - // exp((1 - theta_j) x) (D(z_j) - x / dz) u_j, the difference of rates formed first. - // Beside a coupled flow a frozen-residual flow's R_s(z_j) u_j joins each, against - // the step's own section, and the block's bins are carried by its matrix flow. - const auto carried = adams_carried_; - const auto states = adams_states_; - const auto rates = adams_rates_; - const auto applied = adams_applied_; - const bool compensated = weights.compensated; - const double inverse_dz = 1.0 / dz; - if (compensated) -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - launch::parallel_for( - "solprop_native.adams.coupled_carry", - Range2(execution_space_, {0, 0}, {n, modes}), - KOKKOS_LAMBDA(const std::size_t index, const std::size_t i) { - const auto one = slots(index); - // A bin off the route is the next pass's. - if (falls_back && one != 0 && route(section, one - 1) != 0) - return; - // A sample's term on channel c, formed where it is needed: a thread holds no - // channel vectors. - const auto term = [&](const std::size_t slot, const std::size_t c) { - Complex value = - compensation_residual(rates(slot, c, index), exponent(c, index), - inverse_dz, states(slot, c, index)); - if (residual && one != 0) { - const auto bin = one - 1; - // R_s(z_j) u_j: the sample's own generator applied once, less the - // step's section's frozen block. - Complex extra = applied(slot, c, index); - for (std::size_t j = 0; j < modes; ++j) - extra = strict_sub(extra, strict_mul(frozen(section, bin, c, j), - states(slot, j, index))); - value = strict_add(value, extra); - } - return value; - }; - for (std::size_t t = 0; t + 1 < weights.width; ++t) { - const auto slot = weights.slot[t]; - const double span = weights.span[t]; - Complex result; - if (span == 0.0) { - result = term(slot, i); - } else if (one == 0) { - result = exponential(span * exponent(i, index)) * term(slot, i); - } else { - const auto bin = one - 1; - const Complex shift = exponent(0, index); - Complex total{}; - for (std::size_t m = 0; m < modes; ++m) { - Complex value{}; - for (std::size_t j = 0; j < modes; ++j) - value += right(section, bin, m, j) * term(slot, j); - total += - left(section, bin, i, m) * - (exponential(span * (shift + dz * eigenvalues(section, bin, m))) * - value); - } - result = total; - } - carried(t, i, index) = result; - } - }); - if (compensated && falls_back) - // Off the eigenbasis route, a thread per bin on the block functions' instance, - // whose scratch row it reuses: the host task's carry in its order, the bin's - // channel vectors in blocks 3 (the term) and 4 (its rotation) of the row. - launch::parallel_for( - "solprop_native.adams.fallback_carry", bins, - KOKKOS_LAMBDA(const std::size_t index) { - const auto one = slots(index); - if (one == 0) - return; - const auto bin = one - 1; - const auto path = route(section, bin); - if (path == 0) - return; - const auto f = path - 1; - constexpr std::size_t term = 3; - constexpr std::size_t rotated = 4; - for (std::size_t t = 0; t + 1 < weights.width; ++t) { - const auto slot = weights.slot[t]; - for (std::size_t i = 0; i < modes; ++i) - fallback(bin, term, 0, i) = - compensation_residual(rates(slot, i, index), exponent(i, index), - inverse_dz, states(slot, i, index)); - if (residual) - for (std::size_t i = 0; i < modes; ++i) { - // R_s(z_j) u_j: the sample's own generator applied once, less the - // step's section's frozen block. - Complex value = applied(slot, i, index); - for (std::size_t j = 0; j < modes; ++j) - value = strict_sub(value, strict_mul(frozen(section, bin, i, j), - states(slot, j, index))); - fallback(bin, term, 0, i) = - strict_add(fallback(bin, term, 0, i), value); - } - const double span = weights.span[t]; - if (span != 0.0) { - // The reference's flow matrix Z exp(span x) Z^H with x = shift I + dz T, - // the shift's exponential apart: exp(span shift) Z exp(span dz T) Z^H. - const double length = span * dz; - flow_functions::augmented_row( - Complex{}, length, fallback_triangular, f, modes, 0, - flow_functions::augmented_squarings(Complex{}, length, - fallback_triangular, f, modes, 0), - fallback, bin); - const Complex factor = exponential(span * exponent(0, index)); - for (std::size_t i = 0; i < modes; ++i) - for (std::size_t j = 0; j < modes; ++j) { - Complex total{}; - for (std::size_t m = 0; m <= j; ++m) - total += fallback_vectors(f, i, m) * fallback(bin, 0, m, j); - fallback(bin, 1, i, j) = total; - } - for (std::size_t i = 0; i < modes; ++i) - for (std::size_t j = 0; j < modes; ++j) { - Complex total{}; - for (std::size_t m = 0; m < modes; ++m) - total += fallback(bin, 1, i, m) * - Kokkos::conj(fallback_vectors(f, j, m)); - fallback(bin, 2, i, j) = factor * total; - } - for (std::size_t i = 0; i < modes; ++i) { - Complex value{}; - for (std::size_t j = 0; j < modes; ++j) - value += fallback(bin, 2, i, j) * fallback(bin, term, 0, j); - fallback(bin, rotated, 0, i) = value; - } - for (std::size_t i = 0; i < modes; ++i) - fallback(bin, term, 0, i) = fallback(bin, rotated, 0, i); - } - for (std::size_t i = 0; i < modes; ++i) - carried(t, i, index) = fallback(bin, term, 0, i); - } - }); -#else - launch::parallel_for( - "solprop_native.adams.coupled_carry", bins, - KOKKOS_LAMBDA(const std::size_t index) { - const auto one = slots(index); - Complex stack[adams_channel_stack]; - Complex* const term = adams_channel_rows(stack, channels, index); - Complex* const rotated = term + modes; - for (std::size_t t = 0; t + 1 < weights.width; ++t) { - const auto slot = weights.slot[t]; - for (std::size_t i = 0; i < modes; ++i) - term[i] = compensation_residual(rates(slot, i, index), exponent(i, index), - inverse_dz, states(slot, i, index)); - if (residual && one != 0) { - const auto bin = one - 1; - for (std::size_t i = 0; i < modes; ++i) { - // R_s(z_j) u_j: the sample's own generator applied once, less the - // step's section's frozen block. - Complex value = applied(slot, i, index); - for (std::size_t j = 0; j < modes; ++j) - value = strict_sub(value, strict_mul(frozen(section, bin, i, j), - states(slot, j, index))); - term[i] = strict_add(term[i], value); - } - } - const double span = weights.span[t]; - if (span != 0.0) { - if (one == 0) { - for (std::size_t i = 0; i < modes; ++i) - term[i] = exponential(span * exponent(i, index)) * term[i]; - } else if (const auto path = route(section, one - 1); path != 0) { - // Off the eigenbasis route, the reference's flow matrix - // Z exp(span x) Z^H with x = shift I + dz T, the shift's exponential - // apart: exp(span shift) Z exp(span dz T) Z^H, then applied. - const auto f = path - 1; - const auto bin = one - 1; - const double length = span * dz; - flow_functions::augmented_row( - Complex{}, length, fallback_triangular, f, modes, 0, - flow_functions::augmented_squarings( - Complex{}, length, fallback_triangular, f, modes, 0), - fallback, bin); - const Complex factor = exponential(span * exponent(0, index)); - for (std::size_t i = 0; i < modes; ++i) - for (std::size_t j = 0; j < modes; ++j) { - Complex total{}; - for (std::size_t m = 0; m <= j; ++m) - total += fallback_vectors(f, i, m) * fallback(bin, 0, m, j); - fallback(bin, 1, i, j) = total; - } - for (std::size_t i = 0; i < modes; ++i) - for (std::size_t j = 0; j < modes; ++j) { - Complex total{}; - for (std::size_t m = 0; m < modes; ++m) - total += fallback(bin, 1, i, m) * - Kokkos::conj(fallback_vectors(f, j, m)); - fallback(bin, 2, i, j) = factor * total; - } - for (std::size_t i = 0; i < modes; ++i) { - Complex value{}; - for (std::size_t j = 0; j < modes; ++j) - value += fallback(bin, 2, i, j) * term[j]; - rotated[i] = value; - } - for (std::size_t i = 0; i < modes; ++i) - term[i] = rotated[i]; - } else { - const auto bin = one - 1; - const Complex shift = exponent(0, index); - for (std::size_t m = 0; m < modes; ++m) { - Complex value{}; - for (std::size_t j = 0; j < modes; ++j) - value += right(section, bin, m, j) * term[j]; - rotated[m] = - exponential(span * (shift + dz * eigenvalues(section, bin, m))) * - value; - } - for (std::size_t i = 0; i < modes; ++i) { - Complex value{}; - for (std::size_t m = 0; m < modes; ++m) - value += left(section, bin, i, m) * rotated[m]; - term[i] = value; - } - } - } - for (std::size_t i = 0; i < modes; ++i) - carried(t, i, index) = term[i]; - } - }); -#endif - // The predicted lab-frame field, exp(x) u + dz sum_k M_k sum_j c_jk F_j and the - // compensation's predictor, is the field the lab-frame source is evaluated at; the - // accepted state is kept aside. - const auto store = adams_store_; - const auto current = current_; - const auto saved = adams_saved_; - const auto lab = adams_lab_; - graph::deep_copy(execution_space_, saved, current); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - launch::parallel_for( - "solprop_native.adams.coupled_predict", - Range2(execution_space_, {0, 0}, {n, modes}), - KOKKOS_LAMBDA(const std::size_t index, const std::size_t i) { - const auto one = slots(index); - // Channel c's combinations sum_t w_tk F_t, its samples read once. - const auto combine = [&](const std::size_t c, Complex* out) { - Complex samples[adams_max_samples]; - for (std::size_t t = 0; t < weights.order; ++t) - samples[t] = adams_history(store, weights, t, c, index); - for (std::size_t k = 0; k < weights.order; ++k) { - Complex total{}; - for (std::size_t t = 0; t < weights.order; ++t) - total += weights.predictor[t][k] * samples[t]; - out[k] = total; - } - }; - Complex combined[adams_max_samples]; - Complex value{}; - if (one == 0) { - combine(i, combined); - Complex total{}; - for (std::size_t k = 0; k < weights.order; ++k) - total += moments(k, i, index) * combined[k]; - value = flow(i, index) * saved(i, index) + dz * total; - } else { - const auto bin = one - 1; - Complex linear{}; - Complex total{}; - for (std::size_t j = 0; j < modes; ++j) { - linear += block(0, bin, i, j) * saved(j, index); - combine(j, combined); - for (std::size_t k = 0; k < weights.order; ++k) - total += block(k + 1, bin, i, j) * combined[k]; - } - value = linear + dz * total; - } - if (compensated) { - Complex compensation{}; - for (std::size_t t = 0; t < weights.order; ++t) - compensation = strict_accumulate(compensation, weights.comp_predictor[t], - carried(t, i, index)); - value = strict_accumulate(value, dz, compensation); - lab(i, index) = value; - } - current(i, index) = value; - }); -#else - launch::parallel_for( - "solprop_native.adams.coupled_predict", bins, - KOKKOS_LAMBDA(const std::size_t index) { - const auto one = slots(index); - // Combination k of channel i at `combined[k modes + i]`. - Complex stack[adams_channel_stack]; - Complex* const combined = adams_channel_rows(stack, channels, index); - for (std::size_t k = 0; k < weights.order; ++k) - for (std::size_t i = 0; i < modes; ++i) { - Complex total{}; - for (std::size_t t = 0; t < weights.order; ++t) - total += - weights.predictor[t][k] * adams_history(store, weights, t, i, index); - combined[k * modes + i] = total; - } - for (std::size_t i = 0; i < modes; ++i) { - Complex value{}; - if (one == 0) { - Complex total{}; - for (std::size_t k = 0; k < weights.order; ++k) - total += moments(k, i, index) * combined[k * modes + i]; - value = flow(i, index) * saved(i, index) + dz * total; - } else { - const auto bin = one - 1; - Complex linear{}; - Complex total{}; - for (std::size_t j = 0; j < modes; ++j) { - linear += block(0, bin, i, j) * saved(j, index); - for (std::size_t k = 0; k < weights.order; ++k) - total += block(k + 1, bin, i, j) * combined[k * modes + j]; - } - value = linear + dz * total; - } - if (compensated) { - Complex compensation{}; - for (std::size_t t = 0; t < weights.order; ++t) - compensation = strict_accumulate(compensation, weights.comp_predictor[t], - carried(t, i, index)); - value = strict_accumulate(value, dz, compensation); - lab(i, index) = value; - } - current(i, index) = value; - } - }); -#endif - use_uploaded_sample(sample); - { - const FlowAside aside(flow_); - adams_evaluate(sample); - } - // The new sample's compensation at the predicted field, untransported, with the - // rate read on the side of the step's end the step came from; beside a coupled flow - // a frozen-residual flow's residual at the step's end joins it. - const auto endpoint = adams_endpoint_; - const auto end_applied = adams_end_applied_; - if (compensated) { - adams_rate(rate, adams_endpoint_rate_); - const auto endpoint_rate = adams_endpoint_rate_; - launch::parallel_for( - "solprop_native.adams.endpoint", policy, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - endpoint(column, index) = compensation_residual( - endpoint_rate(column, index), exponent(column, index), inverse_dz, - lab(column, index)); - }); - if (residual) { - // The generator at the step's end applied to the predicted field: the new - // sample's G(z_n + h) u, kept for it on acceptance. - apply_generator(execution_space_, n, modes, slots, terms, adams_end_weights_, lab, - end_applied); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - launch::parallel_for( - "solprop_native.adams.endpoint_residual", - Range2(execution_space_, {0, 0}, {n, modes}), - KOKKOS_LAMBDA(const std::size_t index, const std::size_t i) { - const auto one = slots(index); - if (one == 0) - return; - const auto bin = one - 1; - Complex value = end_applied(i, index); - for (std::size_t j = 0; j < modes; ++j) - value = strict_sub(value, - strict_mul(frozen(section, bin, i, j), lab(j, index))); - endpoint(i, index) = strict_add(endpoint(i, index), value); - }); -#else - launch::parallel_for( - "solprop_native.adams.endpoint_residual", bins, - KOKKOS_LAMBDA(const std::size_t index) { - const auto one = slots(index); - if (one == 0) - return; - const auto bin = one - 1; - for (std::size_t i = 0; i < modes; ++i) { - Complex value = end_applied(i, index); - for (std::size_t j = 0; j < modes; ++j) - value = strict_sub( - value, strict_mul(frozen(section, bin, i, j), lab(j, index))); - endpoint(i, index) = strict_add(endpoint(i, index), value); - } - }); -#endif - } - } - const auto stages = stages_; - const auto source = adams_source_; - const auto advanced = advanced_; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - launch::parallel_for( - "solprop_native.adams.coupled_correct", - Range2(execution_space_, {0, 0}, {n, modes}), - KOKKOS_LAMBDA(const std::size_t index, const std::size_t i) { - const auto one = slots(index); - source(i, index) = stages(0, i, index); - // Channel c's combinations sum_t w_tk F_t with the new sample first. - const auto combine = [&](const std::size_t c, Complex* out) { - const Complex g = stages(0, c, index); - Complex samples[adams_max_samples]; - for (std::size_t t = 1; t < weights.width; ++t) - samples[t] = adams_history(store, weights, t - 1, c, index); - for (std::size_t k = 0; k < count; ++k) { - Complex total = weights.corrector[0][k] * g; - for (std::size_t t = 1; t < weights.width; ++t) - total += weights.corrector[t][k] * samples[t]; - out[k] = total; - } - }; - Complex combined[adams_max_samples]; - Complex value{}; - if (one == 0) { - combine(i, combined); - Complex total{}; - for (std::size_t k = 0; k < count; ++k) - total += moments(k, i, index) * combined[k]; - value = flow(i, index) * current(i, index) + dz * total; - } else { - const auto bin = one - 1; - Complex linear{}; - Complex total{}; - for (std::size_t j = 0; j < modes; ++j) { - linear += block(0, bin, i, j) * current(j, index); - combine(j, combined); - for (std::size_t k = 0; k < count; ++k) - total += block(k + 1, bin, i, j) * combined[k]; - } - value = linear + dz * total; - } - if (compensated) { - Complex compensation = - strict_scale(weights.comp_corrector[0], endpoint(i, index)); - for (std::size_t t = 0; t < weights.order; ++t) - compensation = strict_accumulate( - compensation, weights.comp_corrector[t + 1], carried(t, i, index)); - value = strict_accumulate(value, dz, compensation); - } - advanced(i, index) = value; - }); -#else - launch::parallel_for( - "solprop_native.adams.coupled_correct", bins, - KOKKOS_LAMBDA(const std::size_t index) { - const auto one = slots(index); - Complex stack[adams_channel_stack]; - Complex* const combined = adams_channel_rows(stack, channels, index); - for (std::size_t i = 0; i < modes; ++i) { - const Complex g = stages(0, i, index); - source(i, index) = g; - for (std::size_t k = 0; k < count; ++k) { - Complex total = weights.corrector[0][k] * g; - for (std::size_t t = 1; t < weights.width; ++t) - total += weights.corrector[t][k] * - adams_history(store, weights, t - 1, i, index); - combined[k * modes + i] = total; - } - } - for (std::size_t i = 0; i < modes; ++i) { - Complex value{}; - if (one == 0) { - Complex total{}; - for (std::size_t k = 0; k < count; ++k) - total += moments(k, i, index) * combined[k * modes + i]; - value = flow(i, index) * current(i, index) + dz * total; - } else { - const auto bin = one - 1; - Complex linear{}; - Complex total{}; - for (std::size_t j = 0; j < modes; ++j) { - linear += block(0, bin, i, j) * current(j, index); - for (std::size_t k = 0; k < count; ++k) - total += block(k + 1, bin, i, j) * combined[k * modes + j]; - } - value = linear + dz * total; - } - if (compensated) { - Complex compensation = - strict_scale(weights.comp_corrector[0], endpoint(i, index)); - for (std::size_t t = 0; t < weights.order; ++t) - compensation = strict_accumulate( - compensation, weights.comp_corrector[t + 1], carried(t, i, index)); - value = strict_accumulate(value, dz, compensation); - } - advanced(i, index) = value; - } - }); -#endif - const CoupledDefectNorms reducer{.value_count = weights.n_rows, - .store = store, - .moments = moments, - .source = source, - .carried = carried, - .endpoint = endpoint, - .block = block, - .slots = slots, - .in_norm = norm_mask_, - .divisor = adams_divisor_, - .weights = weights, - .dz = dz, - .modes = modes, - .channels = adams_channels_}; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // A device item is a (bin, channel) pair, a host item a bin. - const auto items = n * modes; -#else - const auto items = n; -#endif - auto norms = - adams_norms("solprop_native.adams.coupled_norms", - Kokkos::RangePolicy(flow_execution_space_, 0, items), - reducer, weights.n_rows); - adams_trial_compensated_ = compensated; - adams_trial_coupled_ = residual && compensated; - return norms; -} - -void GeneralWorkspace::adams_exponential_accept(const double dz, const std::size_t free, - const AdamsWeights& dense) { - require_adams(); - if (!adams_trial_ready_) - throw std::logic_error("no Adams trial is held"); - if (free >= adams_store_.extent(0)) - throw std::out_of_range("Adams history row"); - if (dense.width > 0 && adams_slopes_.extent(0) == 0) - throw std::logic_error("Adams dense observation storage is not allocated"); - adams_trial_ready_ = false; - const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); - const auto store = adams_store_; - const auto source = adams_source_; - const auto slopes = adams_slopes_; - adams_dense_count_ = dense.width; - if (dense.width > 0) { - // The step's samples and the state it starts from, before the new sample takes a - // row: the accepted corrector's interpolant for dense observation. - graph::deep_copy(execution_space_, adams_dense_start_, current_); - graph::deep_copy(execution_space_, adams_dense_exponent_, adams_exponent_); - launch::parallel_for( - "solprop_native.adams.exponential_dense", policy, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - slopes(0, column, index) = source(column, index); - for (std::size_t t = 1; t < dense.width; ++t) - slopes(t, column, index) = store(dense.slot[t - 1], column, index); - }); - if (adams_trial_compensated_) { - // The interpolant's compensation terms, untransported: the endpoint's and - // (D(z_j) - x / dz) u_j of each history sample it reads. - const auto terms = adams_dense_terms_; - const auto endpoint = adams_endpoint_; - const auto exponent = adams_exponent_; - const auto states = adams_states_; - const auto rates = adams_rates_; - const double inverse_dz = 1.0 / dz; - launch::parallel_for( - "solprop_native.adams.dense_terms", policy, - // Compensation terms only: strict as one function (solprop SEP 25 4.2). - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) - SOLPROP_NATIVE_STRICT_FUNCTION { - SOLPROP_NATIVE_STRICT_BLOCK - terms(0, column, index) = endpoint(column, index); - for (std::size_t t = 1; t < dense.width; ++t) { - const auto slot = dense.slot[t - 1]; - terms(t, column, index) = compensation_residual( - rates(slot, column, index), exponent(column, index), inverse_dz, - states(slot, column, index)); - } - }); - } - } - adams_dense_compensated_ = dense.width > 0 && adams_trial_compensated_; - // The accepted field takes `finish`'s boundary treatment: a split-step absorber's - // half steps, or the band's mask in the pass below. Untransported samples: nothing - // moves; the new sample takes row `free`, and for a varying operator its state and - // rate the rows beside it. - const bool masked = !problem_.split_step_absorber; - if (!masked) - split_step(dz, advanced_); - const bool compensated = adams_trial_compensated_; - const auto active = active_mask_; - const auto advanced = advanced_; - const auto states = adams_states_; - const auto rates = adams_rates_; - const auto lab = adams_lab_; - const auto endpoint_rate = adams_endpoint_rate_; - const std::size_t blocks = (n + adams_block - 1) / adams_block; - launch::parallel_for( - "solprop_native.adams.exponential_accept", - Kokkos::RangePolicy(adams_copy_space_, 0, columns * blocks), - KOKKOS_LAMBDA(const std::size_t task) { - const auto column = task / blocks; - const auto first = (task % blocks) * adams_block; - const auto last = first + adams_block < n ? first + adams_block : n; - for (std::size_t index = first; index < last; ++index) { - if (masked && active(index) == 0U) - advanced(column, index) = Complex{}; - store(free, column, index) = source(column, index); - if (compensated) { - states(free, column, index) = lab(column, index); - rates(free, column, index) = endpoint_rate(column, index); - } - } - }); - std::swap(current_, advanced_); - // The last evaluation was at the prediction, so no response stash is the state's. - stash_is_current_ = false; - // A frozen-residual flow's weights at the new sample, the trial's end. - if (adams_trial_coupled_) - graph::deep_copy(execution_space_, - Kokkos::subview(adams_applied_, free, Kokkos::ALL, Kokkos::ALL), - adams_end_applied_); - adams_accepted_boundary(); -} - -// A host's execution instances are partitions of its thread pool, and the next -// trial's passes may run on another one than this step's, so the accepted step ends -// with a fence there. A device's instances are its one stream, which already orders -// the next trial's work after this step's: there the host goes on to the next trial's -// tables while the device finishes the step (on an AMD Instinct MI325X a trial's -// kernels take a few microseconds each, so the wait was a visible part of a trial). -void GeneralWorkspace::adams_accepted_boundary() const { -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - execution_space_.fence("solprop_native Adams accepted-step boundary"); -#endif -} - -// The field at the fraction `xi` of the last accepted step: the accepted corrector's -// interpolant integrated exactly against the flow, exp(xi x) u_n + dz sum_k -// xi^(k+1) M_k(xi x) sum_j c_jk F_j (`weights.corrector`), masked into current_. -void GeneralWorkspace::adams_exponential_observe(const double xi, const double dz, - const ExponentialWeights& weights) { - require_adams(); - if (adams_dense_count_ == 0) - throw std::logic_error("no Adams step is held for dense observation"); - const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - const std::size_t count = adams_dense_count_; - const auto start = adams_dense_start_; - const auto exponent = adams_dense_exponent_; - const auto slopes = adams_slopes_; - const auto current = current_; - const auto terms = adams_dense_terms_; - const bool compensated = adams_dense_compensated_; - launch::parallel_for( - "solprop_native.adams.exponential_observe", - Range2(execution_space_, {0, 0}, {columns, n}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - const Complex x = xi * exponent(column, index); - const Complex e = exponential(x); - Complex phi[2 * adams_max_samples + 1]; - if (weights.shifted) { - // The accepted step's own sigma (solprop SEP 35 section 4.1): the moments at - // xi (x - sigma), its rotated samples, and exp(-sigma (1 - xi)) on the sum. - const Complex residual = - xi * (exponent(column, index) - Complex(0.0, weights.shift)); - adams_phi(residual, Kokkos::abs(residual), exponential(residual), count, phi); - } else { - adams_phi(x, Kokkos::abs(x), e, count, phi); - } - Complex total{}; - double power = xi; - for (std::size_t k = 0; k < count; ++k) { - Complex combined{}; - for (std::size_t t = 0; t < count; ++t) { - Complex slope = slopes(t, column, index); - if (weights.shifted && t > 0) - slope = strict_mul( - Complex(weights.factor_re[t - 1], weights.factor_im[t - 1]), slope); - combined += weights.corrector[t][k] * slope; - } - total += (power * (adams_factorial(k) * phi[k + 1])) * combined; - power *= xi; - } - if (weights.shifted) - total = exponential(Complex(0.0, -weights.shift * (1.0 - xi))) * total; - Complex value = e * start(column, index) + dz * total; - if (compensated) { - // e^{(xi - theta_j) x} carries each term; for the endpoint it is an inverse - // flow, so a bin the step damps by more than an e-fold takes the anchor's. - const Complex chord = exponent(column, index); - value = adams_observed_compensation(weights, count, xi, dz, chord, value, - terms, column, index); - if (chord.real() < -1.0) - value = e * start(column, index); - } - current(column, index) = value; - }); - mask(current_); - stash_is_current_ = false; - execution_space_.fence("solprop_native Adams observation boundary"); -} - -// --- solprop SEP 47: a Runge-Kutta march's dense output in a native session --------- -// -// DOPRI5's continuous extension (Hairer, Norsett and Wanner, II.6) in the step's -// interaction picture, as solprop's `extension_coefficients`, `extension_value` and -// `HostDriver._observe_inside` form it: with `v0` the state at the step's anchor, `k_i` -// its stages in the anchor's frame and `v1 - v0 = h sum_i b_i k_i`, the coefficients -// `v0`, `v1 - v0`, `h k1 - (v1 - v0)`, `(v1 - v0) - h k7 - (h k1 - (v1 - v0))` and -// `h sum_i d_i k_i`, evaluated at the fraction theta as `r1 + theta (r2 + (1 - theta) -// (r3 + theta (r4 + (1 - theta) r5)))`. A bin the step's flow changes by more than an -// e-fold takes the anchor's value: no polynomial in theta follows its exponentially -// growing interaction-picture derivative. The value is carried from the anchor to the -// plane by `U(position, anchor)` and masked. Every product and sum is rounded on its -// own, in solprop's order. - -namespace { - -/** The most stages a dense extension reads. */ -constexpr std::size_t rk_dense_max_stages = 16; - -/** A tableau's solution and dense-output weights, captured by a kernel. */ -struct RungeKuttaDenseWeights { - double solution[rk_dense_max_stages]{}; - double dense[rk_dense_max_stages]{}; - std::size_t stages{}; -}; - -/** The step's phase increment at one element from the uploaded weights' last row: on a - separable route the step's length times the element's operator, otherwise `sum_t - dPhi_t Phi_t` in term order (`adams_phase`'s exponent). */ -struct StepExponent { - bool separable{}; - SpatialCoefficients spatial; - ByteView active; - RealView delta; - ComplexView3 phi_terms; - std::size_t n_pol{}; - std::size_t n_stages{}; - std::size_t n_phi{}; - KOKKOS_INLINE_FUNCTION Complex operator()(const std::size_t column, - const std::size_t index) const { - if (separable) { - const auto values = spatial.value(column / n_pol, index, active(index) != 0U); - return delta(n_stages) * values.linear; - } - return adams_phase_sum(phi_terms, delta, n_stages * n_phi, n_phi, column, index); - } -}; - -} // namespace - -void GeneralWorkspace::rk_dense_allocate(const std::vector& weights) { - require_open(); - if (problem_.low_storage || stages_.extent(0) == 0) - throw std::logic_error("Runge-Kutta dense output needs the Runge-Kutta stages"); - if (weights.size() != problem_.n_stages || weights.size() > rk_dense_max_stages) - throw std::invalid_argument("the dense-output weights are one per stage"); - const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - rk_dense_weights_ = weights; - if (rk_dense_coefficients_.extent(0) == 0) { - rk_dense_coefficients_ = - ComplexView3("solprop_native.rk.dense_coefficients", 5, columns, n); - rk_dense_stiff_ = Kokkos::View( - "solprop_native.rk.dense_stiff", columns, n); - rk_dense_held_ = ComplexView2("solprop_native.rk.dense_held", columns, n); - } - rk_dense_ready_ = false; -} - -void GeneralWorkspace::rk_dense_hold(const double dz) { - require_open(); - if (rk_dense_coefficients_.extent(0) == 0) - throw std::logic_error("Runge-Kutta dense output storage is not allocated"); - if (!stages_ready_) - throw std::logic_error("no Runge-Kutta trial is held for a dense extension"); - RungeKuttaDenseWeights weights; - weights.stages = problem_.n_stages; - for (std::size_t stage = 0; stage < weights.stages; ++stage) { - weights.solution[stage] = problem_.solution_weights[stage]; - weights.dense[stage] = rk_dense_weights_[stage]; - } - // The trial's own scalars are bound: their last row is the whole step's. - const StepExponent step{.separable = problem_.separable_geometry(), - .spatial = spatial_, - .active = active_mask_, - .delta = stage_phi_delta_, - .phi_terms = phi_terms_, - .n_pol = problem_.n_pol_state, - .n_stages = problem_.n_stages, - .n_phi = problem_.n_phi_terms}; - const auto current = current_; - const auto stages = stages_; - const auto coefficients = rk_dense_coefficients_; - const auto stiff = rk_dense_stiff_; - launch::parallel_for( - "solprop_native.rk.dense_hold", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) - SOLPROP_NATIVE_STRICT_FUNCTION { - SOLPROP_NATIVE_STRICT_BLOCK - Complex span{}; - Complex dense{}; - for (std::size_t stage = 0; stage < weights.stages; ++stage) { - const Complex slope = stages(stage, column, index); - span = strict_accumulate(span, weights.solution[stage], slope); - dense = strict_accumulate(dense, weights.dense[stage], slope); - } - span = strict_scale(dz, span); - const Complex first = - strict_sub(strict_scale(dz, stages(0, column, index)), span); - const Complex last = strict_sub( - strict_sub(span, - strict_scale(dz, stages(weights.stages - 1, column, index))), - first); - coefficients(0, column, index) = current(column, index); - coefficients(1, column, index) = span; - coefficients(2, column, index) = first; - coefficients(3, column, index) = last; - coefficients(4, column, index) = strict_scale(dz, dense); - const Complex x = step(column, index); - stiff(column, index) = Kokkos::abs(x.real()) > 1.0 ? 1U : 0U; - }); - rk_dense_ready_ = true; -} - -void GeneralWorkspace::rk_dense_observe(const double theta, const TrialScalars& carry) { - require_open(); - if (!rk_dense_ready_) - throw std::logic_error("no Runge-Kutta step is held for a dense observation"); - require_trial_scalars(carry); - graph::deep_copy(execution_space_, rk_dense_held_, current_); - upload_trial(carry); - const StepExponent flow{.separable = problem_.separable_geometry(), - .spatial = spatial_, - .active = active_mask_, - .delta = stage_phi_delta_, - .phi_terms = phi_terms_, - .n_pol = problem_.n_pol_state, - .n_stages = problem_.n_stages, - .n_phi = problem_.n_phi_terms}; - const double rest = 1.0 - theta; - const auto coefficients = rk_dense_coefficients_; - const auto stiff = rk_dense_stiff_; - const auto current = current_; - launch::parallel_for( - "solprop_native.rk.dense_observe", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, - const std::size_t index) SOLPROP_NATIVE_STRICT_FUNCTION { - SOLPROP_NATIVE_STRICT_BLOCK - const Complex start = coefficients(0, column, index); - Complex value = start; - if (stiff(column, index) == 0U) { - Complex inner = strict_scale(rest, coefficients(4, column, index)); - inner = - strict_scale(theta, strict_add(coefficients(3, column, index), inner)); - inner = strict_scale(rest, strict_add(coefficients(2, column, index), inner)); - inner = - strict_scale(theta, strict_add(coefficients(1, column, index), inner)); - value = strict_add(start, inner); - } - current(column, index) = strict_mul(exponential(flow(column, index)), value); - }); - mask(current_); - stash_is_current_ = false; - // A plane inside a step already taken has no quadrature score of its own, as on - // the driver's path, which reloads the observed field. - quadrature_error_ = Kokkos::Experimental::quiet_NaN_v; - quadrature_stale_ = false; - execution_space_.fence("solprop_native Runge-Kutta observation boundary"); -} - -void GeneralWorkspace::rk_dense_restore() { - require_open(); - if (rk_dense_held_.extent(0) == 0) - throw std::logic_error("Runge-Kutta dense output storage is not allocated"); - graph::deep_copy(execution_space_, current_, rk_dense_held_); - // As the driver's march reloads the committed state (`_adams_restore`): the next - // step evaluates its first stage afresh rather than reuse the crossing step's last, - // so that the session's run is the driver's whatever FFT plans round each. - clear_fsal(); - execution_space_.fence("solprop_native Runge-Kutta restore boundary"); -} - -void GeneralWorkspace::rk_dense_release() noexcept { - rk_dense_coefficients_ = {}; - rk_dense_stiff_ = {}; - rk_dense_held_ = {}; - // A moved-in empty vector releases the storage and cannot throw (`= {}` may). - rk_dense_weights_ = std::vector(); - rk_dense_ready_ = false; -} - -std::vector GeneralWorkspace::collect_trial() { - // A coupled flow's last stage is carried back by the step's own inverse action, so - // it is not the next step's first stage; solprop's reference takes no FSAL there. - pending_fsal_ = - problem_.first_same_as_last && !problem_.split_step_absorber && !flow_; - graph::deep_copy(execution_space_, pinned_partials_, partials_); - execution_space_.fence("solprop_native trial boundary"); - const auto& host = pinned_partials_; - // Every column's slots, in order: the host's reduction is the caller's sum of a row. - const auto slots = host.extent(1); - std::vector result(3 * slots); - for (std::size_t row = 0; row < 3; ++row) - for (std::size_t slot = 0; slot < slots; ++slot) - result[row * slots + slot] = host(row, slot); - return result; -} - -void GeneralWorkspace::reuse_fsal() { - const auto stages = stages_; - const auto recycled = recycled_; - const auto solution = solution_; - const auto error = error_; - const auto whole = whole_propagator_; - const double solution_weight = problem_.solution_weights[0]; - const double error_weight = problem_.error_weights[0]; - const auto n = problem_.n_omega; - launch::parallel_for( - "solprop_native.stage.reuse_fsal", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - const Complex value = recycled(column, index); - stages(0, column, index) = value; - solution(column, index) = solution_weight * value; - error(column, index) = error_weight * value; - }); -} - -TransverseChunk -GeneralWorkspace::transverse_chunk(const std::size_t first_unit) const noexcept { - const auto units = - std::min(transverse_units_per_chunk(), transverse_units() - first_unit); - if (problem_.slab_geometry()) - return {.first_row = first_unit * problem_.n_p, - .rows = units * problem_.n_p, - .first_node = first_unit * problem_.n_x, - .nodes = units * problem_.n_x}; - return { - .first_row = first_unit, .rows = units, .first_node = first_unit, .nodes = units}; -} - -void GeneralWorkspace::transverse_forward() { - if (slab_forward_) - graph::execute(*slab_forward_, transverse_, transverse_, - KokkosFFT::Normalization::none); - else - graph::execute(*transverse_forward_, transverse_, transverse_, - KokkosFFT::Normalization::none); -} - -void GeneralWorkspace::transverse_inverse() { transverse_inverse(transverse_); } - -// The in-place inverse transverse plan on another volume of the same extents: the -// plan's own scratch or the magnetic closure's (solprop SEP 34). -void GeneralWorkspace::transverse_inverse(const ComplexView3& scratch) { - if (slab_inverse_) - graph::execute(*slab_inverse_, scratch, scratch, KokkosFFT::Normalization::none); - else - graph::execute(*transverse_inverse_, scratch, scratch, - KokkosFFT::Normalization::none); -} - -void GeneralWorkspace::separable_stage_input(const std::size_t stage, const double dz) { - const auto current = current_; - const auto stages = stages_; - const auto tableau = tableau_; - const auto spatial = spatial_; - const auto delta = stage_phi_delta_; - const auto at = stage_phi_at_; - const auto active = active_mask_; - const auto noise = noise_; - const auto scratch = transverse_; - const auto whole = whole_propagator_; - const auto index = stage_index(); - const auto n_stages = problem_.n_stages; - const bool add_noise = problem_.modified_noise && has_noise_; - const double inverse_area_count = inverse_transverse_count(); - // The stage input at one element, given its spatial value and its propagator. - const auto input = KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, - const Complex linear, const Complex propagator) { - Complex accumulated{}; - for (std::size_t previous = 0; previous < stage; ++previous) - accumulated += - tableau(stage * n_stages + previous) * stages(previous, column, omega); - Complex value = propagator * (current(column, omega) + dz * accumulated); - if (add_noise) { - const double angle = at(stage) * linear.imag(); - value += noise(column, omega) * Complex(Kokkos::cos(angle), Kokkos::sin(angle)); - } - scratch.data()[index.cell(column, omega)] = value * inverse_area_count; - }; - // One element's propagators: devices, and a host whose vector maths is the system - // libm, where the per-element form is the faster (solprop ADR-059). - const auto element = - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - const auto values = spatial.value(index.mode(column), omega, active(omega) != 0U); - input(column, omega, values.linear, propagator_exp(delta(stage) * values.linear)); - if (stage + 1 == n_stages) - whole(column, omega) = propagator_exp(delta(n_stages) * values.linear); - }; -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - if constexpr (vmath::vectorised) { - // Each tile's propagators in one pass of the vector maths, then the elements in - // `crossing_for`'s order: the propagators' cosines and sines took a fifth of a - // one-thread `line` run one element at a time (solprop issue #338). - const bool last = stage + 1 == n_stages; - crossing_tiles( - "solprop_native.free.stage_input", execution_space_, problem_.n_columns(), - problem_.n_omega, [=](const CrossingTile& tile) { - PropagatorTile step; - PropagatorTile finish; - Complex linear[PropagatorTile::capacity]; - tile.visit([&](const std::size_t column, const std::size_t omega) { - const auto k = tile.at(column, omega); - linear[k] = - spatial.value(index.mode(column), omega, active(omega) != 0U).linear; - const Complex argument = delta(stage) * linear[k]; - step.real[k] = argument.real(); - step.imag[k] = argument.imag(); - if (last) { - const Complex whole_argument = delta(n_stages) * linear[k]; - finish.real[k] = whole_argument.real(); - finish.imag[k] = whole_argument.imag(); - } - }); - step.form(tile.size()); - if (last) - finish.form(tile.size()); - tile.visit([&](const std::size_t column, const std::size_t omega) { - const auto k = tile.at(column, omega); - input(column, omega, linear[k], step[k]); - if (last) - whole(column, omega) = finish[k]; - }); - }); - return; - } -#endif - crossing_for("solprop_native.free.stage_input", execution_space_, - problem_.n_columns(), problem_.n_omega, element); -} - -void GeneralWorkspace::separable_load_current() { - const auto current = current_; - const auto scratch = transverse_; - const auto index = register_index(); - const double scale = inverse_transverse_count(); - crossing_for( - "solprop_native.free.current_transverse_input", execution_space_, - problem_.n_columns(), problem_.n_omega, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - scratch.data()[index.cell(column, omega)] = scale * current(column, omega); - }); - transverse_inverse(); -} - -void GeneralWorkspace::separable_pack_chunk(const TransverseChunk& chunk, - const bool material) { - const auto fine = fine_spectrum_; - // Every crop bin of the chunk's rows is written below; only the bins outside the - // crop need zeros, each chunk, since the inverse transform may destroy its input. - // Rows past a short last chunk keep earlier finite values nothing reads back. - if (!separable_gaps_.empty()) { - const auto gaps = separable_gaps_; - cartesian_for( - "solprop_native.free.zero_fine_gaps", execution_space_, - chunk.rows * problem_.n_pol_state, gaps.extent(0), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t gap) { - fine(row, gaps(gap)) = Complex{}; - }); - } - const auto scratch = transverse_; - const auto crop = crop_index_; - const auto n_pol = problem_.n_pol_state; - [[maybe_unused]] const auto layout = transverse_layout(); - const auto first = chunk.first_row; - const auto n = problem_.n_omega; - const auto n_to = problem_.n_to; - const bool analytic = problem_.family == FieldFamily::field_analytic; - // Free space's material field carries the pad scale here; the hybrid planar kind's - // slab synthesis carries it (solprop's `Transform.synthesis`). - const double scale = - 1.0 / static_cast(material && !problem_.slab_geometry() ? problem_.n_t - : problem_.n_to); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - tiled_crossing( - "solprop_native.free.pack_temporal_chunk", execution_space_, chunk.rows * n_pol, - n, true, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - return layout.cell(scratch, omega * n_pol + row % n_pol, first + row / n_pol); - }, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega, - const Complex value) { - const double factor = - analytic && omega > 0 && 2 * crop(omega) < n_to ? 2.0 : 1.0; - fine(row, crop(omega)) = factor * scale * value; - }); -#else - // The chunk's row `row` is the state's column `first_column + row`. - const auto index = register_index(); - const auto first_column = first * n_pol; - crossing_for( - "solprop_native.free.pack_temporal_chunk", execution_space_, chunk.rows * n_pol, - n, KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - // The analytic signal doubles the bins strictly between zero and the fine - // grid's Nyquist bin, the same rule as the modal pad and the reference. - const double factor = - analytic && omega > 0 && 2 * crop(omega) < n_to ? 2.0 : 1.0; - fine(row, crop(omega)) = - factor * scale * scratch.data()[index.cell(first_column + row, omega)]; - }); -#endif -} - -// The chunk's physical nodes from its temporal rows: free space's are the rows -// themselves; the hybrid planar kind's are the slab synthesis of each y row. -void GeneralWorkspace::separable_nodes(const TransverseChunk& chunk) { - const bool real_field = problem_.family == FieldFamily::field_real; - const auto n_to = problem_.n_to; - const auto n_pol = problem_.n_pol_state; - if (problem_.slab_geometry()) { - const auto units = chunk.nodes / problem_.n_x; - if (real_field) - slab_contract("solprop_native.slab.synthesis", execution_space_, synthesis_, - modal_real_, node_real_, units, n_to); - else - slab_contract("solprop_native.slab.synthesis", execution_space_, synthesis_, - modal_complex_, node_complex_, units, n_to); - return; - } - if (separable_aliased_) - return; - const auto time = modal_real_; - const auto nodes = node_real_; - const auto complex_time = modal_complex_; - const auto complex_nodes = node_complex_; - cartesian_for( - "solprop_native.free.physical_columns", execution_space_, chunk.rows * n_pol, - n_to, KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { - if (real_field) - nodes(row, index) = time(row, index); - else - complex_nodes(row, index) = complex_time(row, index); - }); -} - -// The hybrid planar kind's slab projection of a chunk's node source onto its y rows' -// columns: the polarisation, or one envelope-plasma current stream. -void GeneralWorkspace::slab_project(const TransverseChunk& chunk, - const EnvelopeStream* stream) { - const auto units = chunk.nodes / problem_.n_x; - const auto n_to = problem_.n_to; - if (stream != nullptr) - slab_contract("solprop_native.slab.projection", execution_space_, projection_, - stream->node, stream->projected, units, n_to); - else if (problem_.family == FieldFamily::envelope) - slab_contract("solprop_native.slab.projection", execution_space_, projection_, - polarisation_complex_, projected_complex_, units, n_to); - else - slab_contract("solprop_native.slab.projection", execution_space_, projection_, - polarisation_real_, projected_real_, units, n_to); -} - -void GeneralWorkspace::separable_finish_transform(const std::size_t stage, - const bool material) { - const auto scratch = transverse_; - const auto spatial = spatial_; - const auto stages = stages_; - const auto window = omega_window_; - const auto active = active_mask_; - const auto delta = stage_phi_delta_; - const auto index = stage_index(); - // The finish at one element, given its prefactor and its inverse propagator. - const auto finish = KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, - const Complex prefactor, const Complex inverse) { - Complex value = scratch.data()[index.cell(column, omega)]; - value *= - material ? prefactor * window(omega) : Complex(active(omega) != 0U ? 1.0 : 0.0); - value *= inverse; - if (material) - stages(stage, column, omega) = value; - else - stages(stage, column, omega) += value; - }; - // One element, on devices and without host vector maths (as the stage input). - const auto element = - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - const auto values = spatial.value(index.mode(column), omega, active(omega) != 0U); - finish(column, omega, values.prefactor, - propagator_exp(-delta(stage) * values.linear)); - }; -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - if constexpr (vmath::vectorised) { - // As the stage input: each tile's inverse propagators in one pass (issue #338). - crossing_tiles( - "solprop_native.free.finish_transform", execution_space_, problem_.n_columns(), - problem_.n_omega, [=](const CrossingTile& tile) { - PropagatorTile inverse; - Complex prefactor[PropagatorTile::capacity]; - tile.visit([&](const std::size_t column, const std::size_t omega) { - const auto k = tile.at(column, omega); - const auto values = - spatial.value(index.mode(column), omega, active(omega) != 0U); - prefactor[k] = values.prefactor; - const Complex argument = -delta(stage) * values.linear; - inverse.real[k] = argument.real(); - inverse.imag[k] = argument.imag(); - }); - inverse.form(tile.size()); - tile.visit([&](const std::size_t column, const std::size_t omega) { - const auto k = tile.at(column, omega); - finish(column, omega, prefactor[k], inverse[k]); - }); - }); - return; - } -#endif - crossing_for("solprop_native.free.finish_transform", - execution_space_, problem_.n_columns(), - problem_.n_omega, element); -} - -// Free space (solprop `NumpyRefBackend._nonlinear_free`) and the hybrid planar kind -// (`_nonlinear_slab`): the stage input to the transverse spectrum, the inverse -// transverse transform, then chunk by chunk the temporal rows to time, the material -// response at the physical nodes (for the planar kind, synthesised from and projected -// back onto the rows in time) or the in-stage absorber on the rows, and back; the -// forward transverse transform and each column's prefactor and propagator. -void GeneralWorkspace::evaluate_separable_stage(const std::size_t stage, - const double dz, - const bool accumulate) { - auto* const dispersive = separable_dispersive_.get(); - const bool nonlinear = - problem_.nonlinear || problem_.plasma || problem_.raman || dispersive != nullptr; - const auto stages = stages_; - const auto solution = solution_; - const auto error = error_; - const auto n = problem_.n_omega; - // A material finish assigns the stage row; only an absorber-only or a linear stage - // accumulates into (or leaves) a cleared row. - if (!nonlinear) - cartesian_for( - "solprop_native.free.clear_stage", execution_space_, problem_.n_columns(), n, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - stages(stage, column, omega) = Complex{}; - }); - separable_stage_input(stage, dz); - for (const bool material : {true, false}) { - if ((material && !nonlinear) || (!material && !problem_.in_stage_absorber)) - continue; - if (!material) - separable_stage_input(stage, dz); - separable_core(stage, material); - separable_finish_transform(stage, material); - } - if (!accumulate) - return; - const double weight = problem_.solution_weights[stage]; - const double error_weight = problem_.error_weights[stage]; - cartesian_for( - "solprop_native.free.accumulate_stage", execution_space_, problem_.n_columns(), n, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - const Complex value = stages(stage, column, omega); - if (stage == 0) { - solution(column, omega) = weight * value; - error(column, omega) = error_weight * value; - } else { - solution(column, omega) += weight * value; - error(column, omega) += error_weight * value; - } - }); -} - -void GeneralWorkspace::separable_to_time(const TransverseChunk& chunk) { - const bool tail = temporal_tail_rows_ != 0 && - chunk.rows * problem_.n_pol_state == temporal_tail_rows_; - if (problem_.family == FieldFamily::field_real) { - if (tail) - temporal_tail_inverse_real_.execute(); - else - graph::execute(*inverse_real_plan_, fine_spectrum_, modal_real_, - KokkosFFT::Normalization::none); - } else { - if (tail) - temporal_tail_inverse_complex_.execute(); - else - graph::execute(*inverse_complex_plan_, fine_spectrum_, modal_complex_, - KokkosFFT::Normalization::none); - } -} - -void GeneralWorkspace::separable_to_frequency(const TransverseChunk& chunk) { - const bool tail = temporal_tail_rows_ != 0 && - chunk.rows * problem_.n_pol_state == temporal_tail_rows_; - if (problem_.family == FieldFamily::envelope) { - if (tail) - temporal_tail_forward_complex_.execute(); - else - graph::execute(*forward_complex_plan_, projected_complex_, projected_spectrum_, - KokkosFFT::Normalization::none); - } else { - if (tail) - temporal_tail_forward_real_.execute(); - else - graph::execute(*forward_real_plan_, projected_real_, projected_spectrum_, - KokkosFFT::Normalization::none); - } -} - -// `volume_kerr_`'s stage core: the scratch's (omega, y, x) volume to time and the -// plane at once, the Kerr polarisation in place sample by sample -- the device -// column's arithmetic (`device_response`), the pad scale 1/n_t that the chunk's pack -// would apply taken with the field -- and back. The rank-3 transform is the chunks' -// temporal transforms and the plane's together, the same DFT to rounding, without the -// pack's and the crop's transposes. -void GeneralWorkspace::separable_volume_core(const std::size_t stage) { - Kokkos::Profiling::pushRegion("solprop_native.free.volume_core"); - const auto scratch = volume_order_ ? volume_scratch_ : transverse_; - graph::execute(*volume_inverse_, scratch, scratch, KokkosFFT::Normalization::none); - const auto window = time_window_; - const auto kerr_density = stage_kerr_density_; - const double kerr_coefficient = problem_.kerr_coefficient; - const double self_factor = problem_.self_factor; - const double conjugate_factor = problem_.conjugate_factor; - const double scale = 1.0 / static_cast(problem_.n_t); - const auto plane = transverse_.extent(1) * transverse_.extent(2); - const bool time_fastest = volume_order_; - const auto n_t = problem_.n_t; - launch::parallel_for( - "solprop_native.free.volume_kerr", - Kokkos::RangePolicy(execution_space_, 0, scratch.size()), - KOKKOS_LAMBDA(const std::size_t item) { - const std::size_t time = time_fastest ? item % n_t : item / plane; - Complex& cell = scratch.data()[item]; - const Complex value = scale * cell; - const double modulus = squared_abs(value); - const Complex square = value * value; - const double factor = kerr_coefficient * kerr_density(stage); - const double weighted_real = conjugate_factor * square.real(); - const double weighted_imag = conjugate_factor * square.imag(); - const double weighted_self = self_factor * modulus; - const Complex result( - factor * (weighted_self * value.real() + - (weighted_real * value.real() + weighted_imag * value.imag())), - factor * (weighted_self * value.imag() + - (weighted_imag * value.real() - weighted_real * value.imag()))); - cell = result * window(time); - }); - graph::execute(*volume_forward_, scratch, scratch, KokkosFFT::Normalization::none); - Kokkos::Profiling::popRegion(); -} - -// The in-stage collar's term alone (solprop SEP 40 section 4.7): with no temporal -// rate beside it, the rate depends on the column alone and commutes with the temporal -// transforms, so the term is the rate times the scratch's spectrum, in the transverse -// physical space, and the pass takes only the transverse pair. The temporal pair -// would return the crop's bins to rounding; the bins it would drop lie outside the -// active band, which the finish masks. -void GeneralWorkspace::separable_spatial_core() { - Kokkos::Profiling::pushRegion("solprop_native.free.spatial_core"); - transverse_inverse(); - const auto index = register_index(); - const auto spatial = spatial_; - const auto scratch = transverse_; - const auto n_pol = problem_.n_pol_state; - cartesian_for( - "solprop_native.free.absorber_spectrum", execution_space_, problem_.n_columns(), - problem_.n_omega, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - scratch.data()[index.cell(column, omega)] *= - -0.5 * spatial.absorber(column / n_pol); - }); - transverse_forward(); - Kokkos::Profiling::popRegion(); -} - -// The separable stage's core (solprop SEP 24 Q5): from its input in the transverse -// scratch, the material response (or, `material` false, the in-stage absorber's term) -// back into the scratch, in frequency, chunk by chunk through the temporal transforms. -// Shared by the Runge-Kutta stage and the low-storage march's passes, which differ in -// how they form the input and fold the output. -void GeneralWorkspace::separable_core(const std::size_t stage, const bool material) { - if (material && volume_kerr_) { - separable_volume_core(stage); - return; - } - if (!material && spatial_only_absorber_) { - separable_spatial_core(); - return; - } - // SEP 6's dispersive response runs chunk by chunk beside the material's response - // column, or alone in its place (`SeparableDispersive`). - auto* const dispersive = separable_dispersive_.get(); - const bool alone = dispersive != nullptr && dispersive->alone(); - const auto n_pol = problem_.n_pol_state; - // The host's scratch positions without a division (`RegisterIndex`). - [[maybe_unused]] const auto index = register_index(); - [[maybe_unused]] const auto layout = transverse_layout(); - const auto n = problem_.n_omega; - const auto n_to = problem_.n_to; - const auto real_time = modal_real_; - const auto complex_time = modal_complex_; - const auto polarisation_real = polarisation_real_; - const auto polarisation_complex = polarisation_complex_; - const auto projected_real = projected_real_; - const auto projected_complex = projected_complex_; - const auto temporal_rate = absorber_fine_; - const auto spatial = spatial_; - const auto spectrum = projected_spectrum_; - const auto crop = crop_index_; - const auto scratch = transverse_; - const bool real_field = problem_.family == FieldFamily::field_real; - const bool envelope = problem_.family == FieldFamily::envelope; - const bool slab = problem_.slab_geometry(); - // Profiling regions name the core's stages for a Kokkos Tools timer (SEP 28 phase - // 0); with no tool loaded each is one test of a flag. The kernels inside carry - // their own labels. - Kokkos::Profiling::pushRegion("solprop_native.free.core"); - if (material && dispersive != nullptr) - dispersive->begin(stage); - Kokkos::Profiling::pushRegion("solprop_native.free.transverse_inverse"); - transverse_inverse(); - Kokkos::Profiling::popRegion(); - for (std::size_t unit = 0; unit < transverse_units(); - unit += transverse_units_per_chunk()) { - const auto chunk = transverse_chunk(unit); - const auto first = chunk.first_row; - Kokkos::Profiling::pushRegion("solprop_native.free.pack"); - separable_pack_chunk(chunk, material); - Kokkos::Profiling::popRegion(); - // The response filters the packed spectrum first: the inverse transform destroys - // it. In place, that transform gives the filtered field. - if (material && dispersive != nullptr) - dispersive->prepare(chunk.nodes); - Kokkos::Profiling::pushRegion("solprop_native.free.to_time"); - separable_to_time(chunk); - Kokkos::Profiling::popRegion(); - // The projected rows are overwritten below by the polarisation (or are the - // polarisation, aliased) or by the absorber's term; only a planar projection or a - // response that projects nothing of the stage's needs them zeroed first. - const bool overwrites = - !slab && - (!material || !alone || (dispersive != nullptr && dispersive->borrowed())); - Kokkos::Profiling::pushRegion("solprop_native.free.clear_projected"); - if (!overwrites) { - if (envelope) - graph::deep_copy(execution_space_, projected_complex_, Complex{}); - else - graph::deep_copy(execution_space_, projected_real_, 0.0); - } - Kokkos::Profiling::popRegion(); - if (material) { - separable_nodes(chunk); - Kokkos::Profiling::pushRegion("solprop_native.free.response"); - if (!alone) - response(stage, chunk.first_node, chunk.nodes); - if (problem_.envelope_plasma) - envelope_response(stage, chunk.first_node, chunk.nodes, &chunk); - if (problem_.raman) - convolve_raman(stage, chunk.nodes); - if (dispersive != nullptr) - dispersive->respond(chunk.nodes); - if (held_plasma()) - held_response(stage, chunk.first_node, chunk.nodes, &chunk); - if (gated_plasma()) - gated_response(chunk.nodes); - Kokkos::Profiling::popRegion(); - // Alone with products of its own, nothing of the stage's is projected. - const bool stage_projects = !alone || dispersive->borrowed(); - if (stage_projects && slab) - slab_project(chunk, nullptr); - else if (stage_projects && !separable_aliased_) - cartesian_for( - "solprop_native.free.polarisation_columns", execution_space_, - chunk.rows * n_pol, n_to, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { - if (envelope) - projected_complex(row, time) = polarisation_complex(row, time); - else - projected_real(row, time) = polarisation_real(row, time); - }); - } else { - // The collar's rate depends on y alone on the planar kind, so it acts on the - // rows, whose time signals are the slab amplitudes' (solprop's reference). - cartesian_for( - "solprop_native.free.absorber_columns", execution_space_, chunk.rows * n_pol, - n_to, KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { - const double rate = - -0.5 * (spatial.absorber(first + row / n_pol) + temporal_rate(time)); - if (envelope) - projected_complex(row, time) = rate * complex_time(row, time); - else - projected_real(row, time) = - rate * - (real_field ? real_time(row, time) : complex_time(row, time).real()); - }); - } - const bool responds = material && dispersive != nullptr; - if (responds) - dispersive->finish(); - Kokkos::Profiling::pushRegion("solprop_native.free.to_frequency"); - if (!responds || !dispersive->replaces_transform()) { - separable_to_frequency(chunk); - } - Kokkos::Profiling::popRegion(); - // The response's spectrum joins the chunk's, before the envelope currents'. - if (responds) - dispersive->combine(); - if (material && problem_.envelope_plasma) - envelope_transform(); - if (material && held_plasma()) - held_transform(); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - tiled_crossing( - "solprop_native.free.crop_temporal_chunk", execution_space_, chunk.rows * n_pol, - n, false, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - return spectrum(row, crop(omega)); - }, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega, - const Complex value) { - layout.cell(scratch, omega * n_pol + row % n_pol, first + row / n_pol) = - value; - }); -#else - const auto first_column = first * n_pol; - crossing_for( - "solprop_native.free.crop_temporal_chunk", execution_space_, chunk.rows * n_pol, - n, KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - stream_store(scratch.data()[index.cell(first_column + row, omega)], - spectrum(row, crop(omega))); - }); -#endif - } - Kokkos::Profiling::pushRegion("solprop_native.free.transverse_forward"); - transverse_forward(); - Kokkos::Profiling::popRegion(); - Kokkos::Profiling::popRegion(); -} - -// The fine bins outside the crop as runs `(first bin, length)` in bin order. -std::vector> -GeneralWorkspace::fine_gap_runs() const { - std::vector filled(fine_spectrum_size(), false); - for (const auto index : problem_.crop_index) - filled[index] = true; - std::vector> runs; - for (std::size_t index = 0; index < filled.size(); ++index) { - if (filled[index]) - continue; - if (!runs.empty() && runs.back().first + runs.back().second == index) - ++runs.back().second; - else - runs.emplace_back(index, 1); - } - return runs; -} - -StageTerms GeneralWorkspace::stage_terms(const std::size_t stage) const { - StageTerms terms; - terms.current = current_; - terms.fine = fine_spectrum_; - terms.whole = whole_propagator_; - terms.solution = solution_; - terms.error = error_; - terms.stage_phase = stage_phase_; - terms.stage_inverse = stage_inverse_; - terms.noise = noise_; - terms.flow = static_cast(flow_); - terms.carried = terms.flow && !flow_->identity(stage); - terms.keep_lab = terms.flow && flow_->residual(); - if (terms.flow) { - terms.flow_input = flow_->input(); - terms.flow_drive = flow_->drive(); - } - if (terms.keep_lab) - terms.flow_lab = flow_->lab(); - terms.general = static_cast(general_); - if (terms.general) { - terms.general_scaled = general_->scaled(); - terms.general_source = general_->source(); - } - terms.projected = projected_spectrum_; - terms.absorber = absorber_spectrum_; - terms.stages = stages_; - terms.phi_terms = phi_terms_; - terms.prefactor_terms = prefactor_terms_; - terms.tableau = tableau_; - terms.phi_delta = stage_phi_delta_; - terms.phi_at = stage_phi_at_; - terms.prefactor_weights = stage_prefactor_; - terms.window = omega_window_; - terms.crop = crop_index_; - terms.gaps = fine_gaps_; - terms.gap_ranges = fine_gaps_.extent(0) / 3; - terms.active = active_mask_; - terms.n = problem_.n_omega; - terms.n_phi = problem_.n_phi_terms; - terms.n_stages = problem_.n_stages; - terms.n_prefactor = problem_.n_prefactor_terms; - terms.n_to = problem_.n_to; - terms.inverse_n_to = 1.0 / static_cast(problem_.n_to); - terms.general_pad = problem_.general_pad_scale; - terms.add_noise = problem_.modified_noise && has_noise_; - terms.analytic = problem_.family == FieldFamily::field_analytic; - const bool general_active = - terms.general && (general_->responds() || problem_.in_stage_absorber); - terms.zero_response = terms.general - ? !general_active - : !problem_.nonlinear && !problem_.plasma && - !problem_.raman && !absorbing() && !dispersive_; - terms.in_stage_absorber = absorbing(); - terms.identity_crop = identity_crop_; - return terms; -} - -// The stage's entry kernel: the deferred finish of the stage before it (when a trial -// left one pending), the stage's phase increment and the whole step's at the last -// stage, the combination, and the zero pad of the fine spectrum, in one traversal of -// the bins. The finish and the combination are each the arithmetic of their own -// kernels, so fusing them moves no rounding. A lossless problem's kernel makes no -// `exp` calls, and a lossless stage repeating the stage before's phase (the last two -// stages of a FSAL tableau, and the whole step's at the last) reuses its pair -// (`StageTerms::combine`). -void GeneralWorkspace::stage_entry(const std::size_t stage, const double dz) { - const bool repeated = lossless_phase_ && stage > 0 && entered_stage_ == stage - 1 && - phase_repeats_[stage]; - entered_stage_ = stage; - if (repeated) - stage_entry_with(stage, dz); - else if (lossless_phase_) - stage_entry_with(stage, dz); - else - stage_entry_with(stage, dz); -} - -template -void GeneralWorkspace::stage_entry_with(const std::size_t stage, const double dz) { - constexpr bool lossless = Mode != EntryPhase::general; - const bool finish_previous = finish_pending_; - finish_pending_ = false; - if (finish_previous && (stage == 0 || pending_stage_ + 1 != stage)) - throw std::logic_error("a deferred stage finish lost its order"); - const std::size_t previous = finish_previous ? stage - 1 : 0; - const double previous_solution = problem_.solution_weights[previous]; - const double previous_error = problem_.error_weights[previous]; - const double solution_weight = problem_.solution_weights[stage]; - const double error_weight = problem_.error_weights[stage]; - const auto terms = stage_terms(stage); - const auto n = problem_.n_omega; - const auto n_phi = problem_.n_phi_terms; - const auto n_stages = problem_.n_stages; - const auto gaps = problem_.separable_geometry() ? 0 : fine_spectrum_size() - n; - const bool last = stage + 1 == n_stages; - // The whole step's propagator is the last stage's when their phases are equal. - const bool reuse_whole = last && lossless && whole_repeats_; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // On a device, one thread per bin: the host layout's blocks of 64 bins, each a work - // item holding two 64-element arrays, gave a GPU a few dozen threads walking every - // term, which made a varying capillary no faster on a GPU than on a CPU. - launch::parallel_for( - "solprop_native.stage.entry", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - if (finish_previous) - terms.finish(previous, previous_solution, previous_error, column, index); - const Complex phase = terms.phase_sum(stage * n_phi, column, index); - const Complex propagator = terms.template combine( - stage, dz, solution_weight, error_weight, column, index, phase, - terms.template entry_factors(stage, column, index, phase)); - if (reuse_whole) { - terms.whole(column, index) = propagator; - } else if (last) { - // At a last stage whose node is the step's end (DOPRI5's) the phase sums are - // equal on a lossy problem too, and the stage's propagator is the step's. - const Complex whole = terms.phase_sum(n_stages * n_phi, column, index); - terms.whole(column, index) = - whole == phase ? propagator : entry_exponential(whole); - } - for (std::size_t gap = index; gap < gaps; gap += n) - terms.zero_gaps(column, gap, gap + 1); - }); -#else - const std::size_t blocks = (n + stage_block - 1) / stage_block; - const bool block_forms = terms.blocks_vectorise(); - launch::parallel_for( - "solprop_native.stage.entry", - Kokkos::RangePolicy(execution_space_, 0, - problem_.n_columns() * blocks), - KOKKOS_LAMBDA(const std::size_t item) { - const std::size_t column = item / blocks; - const std::size_t block = item % blocks; - const std::size_t first = block * stage_block; - const std::size_t count = n - first < stage_block ? n - first : stage_block; - if (finish_previous && block_forms) - terms.finish_block(previous, previous_solution, previous_error, column, first, - count); - else if (finish_previous) - for (std::size_t bin = 0; bin < count; ++bin) - terms.finish(previous, previous_solution, previous_error, column, - first + bin); - double real[stage_block]; - double imag[stage_block]; - terms.phase_block(stage * n_phi, column, first, count, real, imag); - // With vector maths the block's transcendentals are passes of the layer; with - // the system libm each bin keeps its own calls, which measured faster there - // (the passes' buffers cost 1 to 3 % on an AMD EPYC 7763 with glibc's). - if constexpr (vmath::vectorised) { - EntryBlock factors{}; - terms.template entry_factors_block(stage, column, first, count, real, - imag, factors); - if (block_forms) - terms.template combine_block(stage, dz, column, first, count, factors, - reuse_whole); - else - for (std::size_t bin = 0; bin < count; ++bin) { - const Complex propagator = terms.template combine( - stage, dz, solution_weight, error_weight, column, first + bin, - Complex(real[bin], imag[bin]), factors.at(bin)); - if (reuse_whole) - terms.whole(column, first + bin) = propagator; - } - if (last && !reuse_whole) { - // The whole step's propagator, `entry_exponential` over the block. - terms.phase_block(n_stages * n_phi, column, first, count, real, imag); - // `e^{+-0} (cos b, sin b)` is `(cos b, sin b)` to the bit. - if (lossless || StageTerms::lossless_block(real, count)) - vmath::sincos(imag, factors.sine, factors.cosine, count); - else - vmath::exp_complex(real, imag, factors.cosine, factors.sine, count); - for (std::size_t bin = 0; bin < count; ++bin) - terms.whole(column, first + bin) = - Complex(factors.cosine[bin], factors.sine[bin]); - } - } else { - for (std::size_t bin = 0; bin < count; ++bin) { - const Complex phase(real[bin], imag[bin]); - const Complex propagator = terms.template combine( - stage, dz, solution_weight, error_weight, column, first + bin, phase, - terms.template entry_factors(stage, column, first + bin, phase)); - if (reuse_whole) - terms.whole(column, first + bin) = propagator; - } - if (last && !reuse_whole) { - terms.phase_block(n_stages * n_phi, column, first, count, real, imag); - for (std::size_t bin = 0; bin < count; ++bin) - terms.whole(column, first + bin) = - entry_exponential(Complex(real[bin], imag[bin])); - } - } - terms.zero_gaps(column, block * gaps / blocks, (block + 1) * gaps / blocks); - }); -#endif -} - -// A stage's finish, or (inside a trial, when the next stage's entry follows with -// nothing between) its deferral to that entry kernel. -void GeneralWorkspace::finish_or_defer(const std::size_t stage) { - if (defer_finish_) { - finish_pending_ = true; - pending_stage_ = stage; - return; - } - finish_stage(stage); -} - -void GeneralWorkspace::evaluate_stage(const std::size_t stage, const double dz) { - ++rk_source_evaluations_; - if (!flow_) { - evaluate_stage_source(stage, dz); - return; - } - // solprop `NumpyRefBackend._trial_step` on a coupled flow: the stage input is - // carried forward by the stage's action, the source (with a frozen-residual flow's - // explicit residual) is evaluated into the flow's drive, and carried back. - if (!flow_->identity(stage)) - flow_->carry_input(stage, dz, current_, stages_, tableau_); - evaluate_stage_source(stage, dz); - flow_->add_residual(stage, stage_phase_); - flow_->carry_back(stage, problem_.solution_weights[stage], - problem_.error_weights[stage], stages_, solution_, error_); -} - -void GeneralWorkspace::evaluate_stage_source(const std::size_t stage, const double dz, - const bool accumulate) { - if (!defer_quadrature_) - coarse_pending_ = false; - if (problem_.separable_geometry()) { - evaluate_separable_stage(stage, dz, accumulate); - return; - } - stage_entry(stage, dz); - if (modal_core(stage)) - finish_or_defer(stage); -} - -// The modal routes' stage core (solprop SEP 24 Q5): from the input in the fine spectrum -// (and the general route's scaled channels) to the source in the projected spectra, -// the in-stage absorber's beside it. Shared by the Runge-Kutta stage and the -// low-storage march's passes. Returns whether a source was evaluated: a stage without -// a response has none to finish. -bool GeneralWorkspace::modal_core(const std::size_t stage) { - if (general_) { - const bool general_active = general_->responds() || problem_.in_stage_absorber; - if (general_active) - general_core(stage); - return general_active; - } - if (!problem_.nonlinear && !problem_.plasma && !problem_.raman && !absorbing() && - !dispersive_) - return false; - // A device's forward rows go in one transform after the absorber's are formed. - const bool packed = !forward_pack_.empty(); - // SEP 6's dispersive response filters the padded spectrum first: the inverse - // transform destroys it. In place, that transform gives the filtered field. - if (dispersive_) - dispersive_->prepare(stage); - { - const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.inverse"); - if (problem_.family == FieldFamily::field_real) - graph::execute(*inverse_real_plan_, fine_spectrum_, modal_real_, - KokkosFFT::Normalization::none); - else if (!retained_synthesis_) - graph::execute(*inverse_complex_plan_, fine_spectrum_, modal_complex_, - KokkosFFT::Normalization::none); - } - // A single node's synthesis and projection are folded into its response column. - const bool folded = folds_single_node(); - if (!folded) - scale_transform(stage, trial_taper_[stage]); - // Alone, the response is the stage's only source: no response column runs, and - // unless its product is the stage's polarisation nothing is projected or - // transformed but its own. - const bool alone = dispersive_ && dispersive_->alone(); - const bool stage_projects = !alone || dispersive_->borrowed(); - const bool stage_synthesises = - !folded && (!dispersive_ || !dispersive_->synthesises_nodes()); - // A projection writes every sample of its targets at the first chunk (a GEMM with - // a zero beta, the custom kernel by assignment, the folded column by its own - // product), so only a stage that does not project needs them zeroed. - if (!stage_projects) { - if (problem_.family == FieldFamily::envelope) { - graph::deep_copy(execution_space_, projected_complex_, Complex{}); - if (!projected_coarse_complex_.empty()) - graph::deep_copy(execution_space_, projected_coarse_complex_, Complex{}); - } else { - graph::deep_copy(execution_space_, projected_real_, 0.0); - if (!projected_coarse_real_.empty()) - graph::deep_copy(execution_space_, projected_coarse_real_, 0.0); - } - } - if (dispersive_) - dispersive_->begin(); - if (raman_products_ != 0) - convolve_raman_products(); - bool first_chunk = true; - std::size_t chunk = 0; - for (std::size_t first_node = 0; first_node < problem_.n_nodes; - first_node += problem_.node_chunk) { - const auto node_count = - std::min(problem_.node_chunk, problem_.n_nodes - first_node); - if (stage_synthesises) - synthesise(first_node, node_count); - if (!alone) - response(stage, first_node, node_count); - if (raman_products_ != 0) - finish_raman_products(stage, first_node, node_count); - else if (problem_.raman) - convolve_raman(stage, node_count); - if (dispersive_) - dispersive_->respond(chunk, first_node, node_count); - if (held_plasma()) { - const Kokkos::Profiling::ScopedRegion region("solprop_native.held.response"); - held_response(stage, first_node, node_count); - } - if (gated_plasma()) - gated_response(node_count); - if (stage_projects && !folded && !held_radial()) { - const Kokkos::Profiling::ScopedRegion region("solprop_native.projection"); - project(first_node, node_count, first_chunk); - } - if (problem_.envelope_plasma) - envelope_response(stage, first_node, node_count); - first_chunk = false; - ++chunk; - } - if (dispersive_) - dispersive_->finish(); - if (!held_radial() && (!dispersive_ || !dispersive_->replaces_transform())) { - const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.forward"); - if (problem_.family == FieldFamily::envelope) { - graph::execute(*forward_complex_plan_, projected_complex_, projected_spectrum_, - KokkosFFT::Normalization::none); - if (forward_coarse_complex_plan_ && !defer_quadrature_) - graph::execute(*forward_coarse_complex_plan_, projected_coarse_complex_, - projected_coarse_spectrum_, KokkosFFT::Normalization::none); - } else if (!packed) { - graph::execute(*forward_real_plan_, projected_real_, projected_spectrum_, - KokkosFFT::Normalization::none); - if (forward_coarse_real_plan_ && !defer_quadrature_) - graph::execute(*forward_coarse_real_plan_, projected_coarse_real_, - projected_coarse_spectrum_, KokkosFFT::Normalization::none); - } - } - // The response's spectrum joins the stage's, before the envelope currents'. - if (dispersive_) - dispersive_->combine(); - if (absorbing()) { - form_absorber(); - const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.absorber"); - if (problem_.family == FieldFamily::envelope) - graph::execute(*absorber_complex_plan_, absorber_complex_, absorber_spectrum_, - KokkosFFT::Normalization::none); - else if (!packed) - graph::execute(*absorber_real_plan_, absorber_real_, absorber_spectrum_, - KokkosFFT::Normalization::none); - } - if (packed) - transform_forward_pack(); - if (problem_.envelope_plasma) - envelope_transform(); - if (held_plasma() && !held_radial()) - held_transform(false, packed); - if (defer_quadrature_ && project_coarse_) - coarse_pending_ = true; - return true; -} - -// The general modal route's stage core: the shared lab-frame input, the in-stage -// absorber on the modal time field as on every route, and the general source in place -// of the legacy node pipeline; the finish applies the shared normalisation and the -// return to the interaction picture (solprop `NumpyRefBackend._nonlinear_general`). -void GeneralWorkspace::general_core(const std::size_t stage) { - if (problem_.in_stage_absorber) { - graph::execute(*inverse_real_plan_, fine_spectrum_, modal_real_, - KokkosFFT::Normalization::none); - form_absorber(); - graph::execute(*absorber_real_plan_, absorber_real_, absorber_spectrum_, - KokkosFFT::Normalization::none); - } - // A guide whose every region is linear has an identically zero source, which - // solprop's reference short-circuits in the same way. - if (general_->responds()) { - general_->combine(stage); - if (general_->dispersive() != nullptr) - general_->dispersive()->begin(stage); - general_->evaluate(); - } else { - graph::deep_copy(execution_space_, general_->source(), Complex{}); - } -} - -// The transforms scaled by stage `stage`'s taper, whose value `taper` the host holds. -// They depend on that value alone: a stage whose taper equals the last one scaled (a -// guide without a taper has one throughout) keeps them, the same bits. -void GeneralWorkspace::scale_transform(const std::size_t stage, - const double taper_value) { - if (scaled_taper_ready_ && taper_value == scaled_taper_) - return; - scaled_taper_ready_ = true; - scaled_taper_ = taper_value; - const auto synthesis = synthesis_; - const auto projection = projection_; - const auto projection_coarse = projection_coarse_; - const auto scaled_synthesis = scaled_synthesis_; - const auto scaled_projection = scaled_projection_; - const auto scaled_projection_coarse = scaled_projection_coarse_; - const auto scaled_projection_stacked = scaled_projection_stacked_; - const auto scaled_synthesis_complex = scaled_synthesis_complex_; - const auto scaled_projection_complex = scaled_projection_complex_; - const auto scaled_projection_coarse_complex = scaled_projection_coarse_complex_; - const auto scaled_projection_stacked_complex = scaled_projection_stacked_complex_; - const auto taper = stage_taper_; - const auto rows = problem_.n_nodes * problem_.n_pol_transform; - const auto modes = problem_.n_modes; - launch::parallel_for( - "solprop_native.transform.scale", - Range2(execution_space_, {0, 0}, {rows, problem_.n_modes}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t mode) { - const double scale = taper(stage); - scaled_synthesis(row, mode) = synthesis(row, mode) * scale; - scaled_projection(mode, row) = projection(mode, row) / scale; - if (!scaled_synthesis_complex.empty()) { - scaled_synthesis_complex(row, mode) = - Complex(synthesis(row, mode) * scale, 0.0); - scaled_projection_complex(mode, row) = - Complex(projection(mode, row) / scale, 0.0); - } - if (!scaled_projection_coarse.empty()) - scaled_projection_coarse(mode, row) = projection_coarse(mode, row) / scale; - if (!scaled_projection_coarse_complex.empty()) - scaled_projection_coarse_complex(mode, row) = - Complex(projection_coarse(mode, row) / scale, 0.0); - if (!scaled_projection_stacked.empty()) { - scaled_projection_stacked(mode, row) = projection(mode, row) / scale; - scaled_projection_stacked(modes + mode, row) = - projection_coarse(mode, row) / scale; - } - if (!scaled_projection_stacked_complex.empty()) { - scaled_projection_stacked_complex(mode, row) = - Complex(projection(mode, row) / scale, 0.0); - scaled_projection_stacked_complex(modes + mode, row) = - Complex(projection_coarse(mode, row) / scale, 0.0); - } - }); -} - -void GeneralWorkspace::synthesise(const std::size_t first_node, - const std::size_t node_count) { - if (retained_synthesis_) { - synthesise_retained(first_node, node_count); - return; - } - const bool whole_transform = - first_node == 0 && node_count == problem_.n_nodes && problem_.n_pol_state == 1; - if (whole_transform && !radial_blas_input_.empty()) { - packed_radial_gemm(scaled_synthesis_, modal_complex_, node_complex_); - return; - } - // A radial analytic field can exceed the node budget even at 256 nodes. Its - // row chunks are contiguous in LayoutRight, including the short final chunk: - // alias those rows directly instead of falling through to scalar synthesis. A - // device takes the same GEMM wherever its shape guard admits the interleaved - // synthesis, which `synthesis_provider` reports. - const bool radial_analytic_chunk = problem_.family == FieldFamily::field_analytic && - !problem_.spatial_spectral_weights.empty(); - if ((whole_transform || radial_analytic_chunk) && use_interleaved_synthesis()) { - Kokkos::Profiling::pushRegion( - "solprop_native.transform.synthesis.kokkosblas_interleaved"); - using Interleaved = Kokkos::View>; - // Complex samples are pairs of doubles, real part first: the same rows read as - // twice as many real columns. - static_assert(sizeof(Complex) == 2 * sizeof(double)); - const Interleaved modal(reinterpret_cast(modal_complex_.data()), - problem_.n_modes, 2 * problem_.n_to); - const Interleaved synthesis(scaled_synthesis_.data() + - first_node * problem_.n_modes, - node_count, problem_.n_modes); - const Interleaved node(reinterpret_cast(node_complex_.data()), node_count, - 2 * problem_.n_to); - engine_gemm(transform_space_for(node_count), synthesis, modal, node, 0.0); - Kokkos::Profiling::popRegion(); - return; - } - if (whole_transform && use_blas_for_synthesis()) { - Kokkos::Profiling::pushRegion("solprop_native.transform.synthesis.kokkosblas"); - if (problem_.family == FieldFamily::envelope) - selected_gemm(transform_space_for(node_count), scaled_synthesis_complex_, - modal_complex_, node_complex_, Complex{}, true); - else if (one_mode_products_) - one_mode_synthesis(response_space_for(node_count), scaled_synthesis_, modal_real_, - node_real_); - else - selected_gemm(transform_space_for(node_count), scaled_synthesis_, modal_real_, - node_real_, 0.0, true); - Kokkos::Profiling::popRegion(); - return; - } - if (use_chunked_blas_for_synthesis()) { - Kokkos::Profiling::pushRegion( - "solprop_native.transform.synthesis.kokkosblas_chunked"); - const auto rows = node_count * problem_.n_pol_transform; - const auto first_row = first_node * problem_.n_pol_transform; - const bool tail = node_count != problem_.node_chunk; - if (problem_.family == FieldFamily::envelope) { - const auto packed = tail ? tail_synthesis_complex_ : chunk_synthesis_complex_; - const auto source = scaled_synthesis_complex_; - launch::parallel_for( - "solprop_native.transform.pack_synthesis_complex", - Range2(matrix_copy_space_for(node_count), {0, 0}, {rows, problem_.n_modes}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t mode) { - packed(row, mode) = source(first_row + row, mode); - }); - const auto output = Kokkos::subview( - node_complex_, std::make_pair(std::size_t{0}, rows), Kokkos::ALL()); - selected_gemm(transform_space_for(node_count), packed, modal_complex_, output, - Complex{}, true); - } else { - const auto packed = tail ? tail_synthesis_ : chunk_synthesis_; - const auto source = scaled_synthesis_; - launch::parallel_for( - "solprop_native.transform.pack_synthesis", - Range2(matrix_copy_space_for(node_count), {0, 0}, {rows, problem_.n_modes}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t mode) { - packed(row, mode) = source(first_row + row, mode); - }); - const auto output = Kokkos::subview( - node_real_, std::make_pair(std::size_t{0}, rows), Kokkos::ALL()); - selected_gemm(transform_space_for(node_count), packed, modal_real_, output, 0.0, - true); - } - Kokkos::Profiling::popRegion(); - return; - } - Kokkos::Profiling::pushRegion("solprop_native.transform.synthesis.custom"); - const auto synthesis = scaled_synthesis_; - const auto modal_real = modal_real_; - const auto modal_complex = modal_complex_; - const auto node_real = node_real_; - const auto node_complex = node_complex_; - const auto n_modes = problem_.n_modes; - const auto n_pol_state = problem_.n_pol_state; - const auto n_pol_transform = problem_.n_pol_transform; - const auto n_to = problem_.n_to; - const bool real_family = problem_.family == FieldFamily::field_real; - const auto effective_rows = node_count * problem_.effective_node_pol(); - launch::parallel_for( - "solprop_native.transform.synthesis.custom", - Range2(transform_space_for(node_count), {0, 0}, {effective_rows, n_to}), - KOKKOS_LAMBDA(const std::size_t local_row, const std::size_t time) { - const std::size_t state_pol = local_row % n_pol_state; - const std::size_t transform_local = local_row / n_pol_state; - const std::size_t transform_row = - first_node * n_pol_transform + transform_local; - if (real_family) { - double value = 0.0; - for (std::size_t mode = 0; mode < n_modes; ++mode) - value += synthesis(transform_row, mode) * - modal_real(mode * n_pol_state + state_pol, time); - node_real(local_row, time) = value; - } else { - Complex value{}; - for (std::size_t mode = 0; mode < n_modes; ++mode) - value += synthesis(transform_row, mode) * - modal_complex(mode * n_pol_state + state_pol, time); - node_complex(local_row, time) = value; - } - }); - Kokkos::Profiling::popRegion(); -} - -template -void GeneralWorkspace::response_harmonic(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count) { - if constexpr (Nonlinear && Family != FieldFamily::field_real) { - if (problem_.harmonic_factor != 0.0) { - response_raman( - stage, first_node, node_count); - return; - } - } - response_raman(stage, first_node, - node_count); -} - -template -void GeneralWorkspace::response_raman(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count) { - if (problem_.raman) { - if (problem_.raman_thg) { - response_impl(stage, first_node, node_count); - } else { - response_impl(stage, first_node, node_count); - } - return; - } - response_impl(stage, first_node, node_count); -} - -template -void GeneralWorkspace::response_stash(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count) { - if constexpr (Plasma) { - // Clang's host loop is measurably faster when the maximum-rate accumulator is - // present. Keep that loop shape for every plasma problem, while the view and its - // final-stage stores remain conditional on requested periodic statistics. - response_harmonic(stage, first_node, - node_count); - return; - } - response_harmonic(stage, first_node, - node_count); -} - -template -void GeneralWorkspace::response_nonlinear(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count) { - if (problem_.nonlinear) { - response_stash(stage, first_node, node_count); - } else { - response_stash(stage, first_node, node_count); - } -} - -template -void GeneralWorkspace::response_plasma(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count) { - if (problem_.plasma && !problem_.envelope_plasma) { - if constexpr (Family != FieldFamily::envelope) { - // SEP 27: the band column writes the physical current separately. The - // ordinary traversal forms and collars only the Kerr/Raman polarisation. - if (problem_.plasma_band) { - if (band_tiled_) { - if (problem_.n_channels > 1) - tiled_band_plasma_column(stage, first_node, - node_count); - else - tiled_band_plasma_column(stage, first_node, - node_count); - } else if (problem_.n_channels > 1) { - team_band_plasma_column(stage, first_node, - node_count); - } else { - team_band_plasma_column(stage, first_node, - node_count); - } - response_nonlinear(stage, first_node, node_count); - return; - } - } - response_nonlinear(stage, first_node, node_count); - } else { - response_nonlinear(stage, first_node, node_count); - } -} - -void GeneralWorkspace::response(const std::size_t stage, const std::size_t first_node, - const std::size_t node_count) { - const auto components = problem_.effective_node_pol(); - switch (problem_.family) { - case FieldFamily::field_real: - if (components == 1) - response_plasma(stage, first_node, node_count); - else - response_plasma(stage, first_node, node_count); - return; - case FieldFamily::field_analytic: - if (components == 1) - response_plasma(stage, first_node, node_count); - else - response_plasma(stage, first_node, node_count); - return; - case FieldFamily::envelope: - if (components == 1) - response_plasma(stage, first_node, node_count); - else - response_plasma(stage, first_node, node_count); - } -} - -template -void GeneralWorkspace::response_impl(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count) { - // A single node's transforms fold into the column (`folds_single_node`), compiled - // apart so that the other columns' arithmetic and layout are untouched. - if constexpr (Components == 1 && !Raman && !Folded) { - if (folds_single_node()) { - response_impl(stage, first_node, node_count); - return; - } - } - if constexpr (!Multiple) { - if (problem_.n_channels > 1 && Plasma) { - response_impl(stage, first_node, node_count); - return; - } - } - const auto node_real = node_real_; - const auto node_complex = node_complex_; - const auto out_real = polarisation_real_; - const auto held_current = held_current_; - const auto out_complex = polarisation_complex_; - const auto raman_drive = node_raman_drive_; - const auto raman_drives = problem_.raman_drives(); - // The modal products replace the nodes' drives (`raman_modal_products`). - const bool node_drives = raman_products_ == 0; - const auto window = time_window_; - const auto carrier = carrier_; - const auto rate_coefficients = rate_coefficients_; - const auto stash = stash_candidate_; - const auto density = stage_plasma_density_; - const auto kerr_density = stage_kerr_density_; - const auto channels = problem_.n_channels; - const auto channel_parameters = channel_parameters_; - const auto channel_coefficients = channel_coefficients_; - const auto channel_density = stage_channel_density_; - const auto channel_preionisation = stage_channel_preionisation_; - const auto scratch = channel_scratch_; - const auto preionisation = stage_preionisation_; - const auto n_to = problem_.n_to; - const double kerr_coefficient = problem_.kerr_coefficient; - const double self_factor = problem_.self_factor; - const double conjugate_factor = problem_.conjugate_factor; - const double harmonic_factor = problem_.harmonic_factor; - const double charge_mass_ratio = problem_.charge_mass_ratio; - const double ionisation_potential = problem_.ionisation_potential; - const double dt = problem_.dt_fine; - const double log_field_min = problem_.log_field_min; - const double d_log_field = problem_.d_log_field; - const double field_min = problem_.field_min; - const double field_max = problem_.field_max; - const double rate_max = problem_.rate_max; - const auto rate_intervals = problem_.rate_intervals; - const auto n_stages = problem_.n_stages; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // The device's form (`device_response`); a mixture of channels takes the generic - // column below. - if constexpr (!Multiple) { - device_response(stage, first_node, node_count); - return; - } -#endif - // A folded single node reads the modal field and writes the projected one, each - // through its one-element transform (`folds_single_node`); otherwise both scales - // are one, which changes no bit. - constexpr bool folded = Folded; - const auto input_real = folded ? modal_real_ : node_real_; - const auto input_complex = folded ? modal_complex_ : node_complex_; - const auto target_real = folded ? projected_real_ : polarisation_real_; - const auto target_complex = folded ? projected_complex_ : polarisation_complex_; - const auto synthesis = synthesis_; - const auto projection = projection_; - const auto taper = stage_taper_; - launch::parallel_for( - "solprop_native.response.column", - Kokkos::RangePolicy(response_space_for(node_count), 0, - node_count), - KOKKOS_LAMBDA(const std::size_t node) { - // CUDA extended-lambda restriction 12 requires the first capture outside - // if-constexpr blocks. Reference binding keeps every compile-time branch. - capture_anchor(channels, scratch, channel_preionisation, node_real, - node_complex, channel_coefficients, channel_parameters, dt, - channel_density, rate_coefficients, log_field_min, d_log_field, - field_min, field_max, rate_max, rate_intervals, - ionisation_potential, charge_mass_ratio, density, - conjugate_factor, self_factor, harmonic_factor, carrier, - raman_drive, raman_drives, node_drives, out_real, held_current, - window, out_complex, stash, n_stages, first_node); - const std::size_t base = node * Components; - const double input_scale = folded ? synthesis(0, 0) * taper(stage) : 1.0; - const double output_scale = folded ? projection(0, 0) / taper(stage) : 1.0; - double exposure = 0.0; - double last_rate = 0.0; - double largest_rate = 0.0; - double fraction = Plasma && !Multiple ? preionisation(stage) : 0.0; - double free_first = 0.0; - double free_second = 0.0; - double last_first = 0.0; - double last_second = 0.0; - if constexpr (Multiple) { - for (std::size_t channel = 0; channel < channels; ++channel) { - scratch(channel, node, 0) = 0.0; - scratch(channel, node, 1) = 0.0; - scratch(channel, node, 2) = channel_preionisation(stage, channel); - scratch(channel, node, 3) = 0.0; - } - } - // Cache contiguous pointwise addresses to avoid repeated View stride loads. - // Plasma retains its measured recurrence-loop layout; applying - // the same change there regresses HE plasma by 8-9% (kokkos-reopt report). - auto* const real_input = input_real.data(); - auto* const complex_input = input_complex.data(); - auto* const real_target = target_real.data(); - auto* const complex_target = target_complex.data(); - auto* const window_data = window.data(); - const double pointwise_factor = - Plasma ? 0.0 : kerr_coefficient * kerr_density(stage); - for (std::size_t time = 0; time < n_to; ++time) { - Complex first_value{}; - Complex second_value{}; - if constexpr (Family == FieldFamily::field_real) { - first_value = - Complex(input_scale * (Plasma ? input_real(base, time) - : real_input[base * n_to + time]), - 0.0); - if constexpr (Components == 2) - second_value = - Complex(input_scale * (Plasma ? input_real(base + 1, time) - : real_input[(base + 1) * n_to + time]), - 0.0); - } else { - first_value = input_scale * (Plasma ? input_complex(base, time) - : complex_input[base * n_to + time]); - if constexpr (Components == 2) - second_value = - input_scale * (Plasma ? input_complex(base + 1, time) - : complex_input[(base + 1) * n_to + time]); - } - const double modulus = squared_abs(first_value) + - (Components == 2 ? squared_abs(second_value) : 0.0); - const Complex square = - first_value * first_value + - (Components == 2 ? second_value * second_value : Complex{}); - if constexpr (Plasma) { - // IntelLLVM's precise model retains an unused hypot outside this branch. - const double magnitude = - Components == 2 ? Kokkos::hypot(first_value.real(), second_value.real()) - : Kokkos::abs(first_value.real()); - double guard = 0.0; - double weight = 0.0; - if constexpr (Multiple) { - double work = 0.0; - for (std::size_t channel = 0; channel < channels; ++channel) { - const double rate = ionisation_rate( - magnitude, channel_coefficients, channel_parameters(channel, 0), - channel_parameters(channel, 1), channel_parameters(channel, 2), - channel_parameters(channel, 3), channel_parameters(channel, 4), - static_cast(channel_parameters(channel, 5)), - static_cast(channel_parameters(channel, 6))); - double channel_exposure = scratch(channel, node, 0); - const double previous_exposure = channel_exposure; - if (time > 0) - channel_exposure += trapezoid(rate, scratch(channel, node, 1), dt); - double channel_fraction = scratch(channel, node, 2); - if (time == 0 || channel_exposure != previous_exposure) - channel_fraction = ionised_fraction( - channel_exposure, channel_preionisation(stage, channel)); - scratch(channel, node, 0) = channel_exposure; - scratch(channel, node, 1) = rate; - scratch(channel, node, 2) = channel_fraction; - if (rate > scratch(channel, node, 3)) - scratch(channel, node, 3) = rate; - const double partial = channel_density(stage, channel); - weight += partial * channel_fraction * channel_parameters(channel, 7); - work += partial * channel_parameters(channel, 8) * rate * - (1.0 - channel_fraction); - } - guard = work > 0.0 && magnitude * magnitude > 0.0 - ? work / (magnitude * magnitude) - : 0.0; - } else { - const double rate = ionisation_rate(magnitude, rate_coefficients, - log_field_min, d_log_field, field_min, - field_max, rate_max, rate_intervals); - const double previous_exposure = exposure; - if (time > 0) - exposure += trapezoid(rate, last_rate, dt); - last_rate = rate; - if constexpr (Stash) { - if (rate > largest_rate) - largest_rate = rate; - } - // Preserve the first subtraction's rounding even for pre-ionized - // input; afterwards an unchanged rounded exposure has the same exp. - if (time == 0 || exposure != previous_exposure) - fraction = ionised_fraction(exposure, preionisation(stage)); - guard = rate > 0.0 && magnitude * magnitude > 0.0 - ? ionisation_potential * rate * (1.0 - fraction) / - (magnitude * magnitude) - : 0.0; - weight = fraction * charge_mass_ratio; - } - double integrand = weight * first_value.real(); - if (time > 0) - free_first += trapezoid(integrand, last_first, dt); - double current = free_first + guard * first_value.real(); - last_first = integrand; - held_current(base, time) = Multiple ? current : density(stage) * current; - if constexpr (Components == 2) { - integrand = weight * second_value.real(); - if (time > 0) - free_second += trapezoid(integrand, last_second, dt); - current = free_second + guard * second_value.real(); - last_second = integrand; - held_current(base + 1, time) = - Multiple ? current : density(stage) * current; - } - } - const double factor = - Plasma ? kerr_coefficient * kerr_density(stage) : pointwise_factor; - Complex first{}; - Complex second{}; - if constexpr (Nonlinear) { - if constexpr (Family == FieldFamily::field_real) { - first += Complex(factor * modulus * first_value.real(), 0.0); - if constexpr (Components == 2) - second += Complex(factor * modulus * second_value.real(), 0.0); - } else { - const auto analytic = [&](const Complex value) { - const double weighted_real = conjugate_factor * square.real(); - const double weighted_imag = conjugate_factor * square.imag(); - const double weighted_self = self_factor * modulus; - Complex result(factor * (weighted_self * value.real() + - (weighted_real * value.real() + - weighted_imag * value.imag())), - factor * (weighted_self * value.imag() + - (weighted_imag * value.real() - - weighted_real * value.imag()))); - if constexpr (Harmonic) - result += - ((factor * harmonic_factor) * carrier(time)) * square * value; - return result; - }; - first += analytic(first_value); - if constexpr (Components == 2) - second += analytic(second_value); - } - } - if constexpr (Raman) { - if (node_drives) - write_raman_drive( - raman_drive, node, time, first_value, second_value, raman_drives); - } - if constexpr (Family != FieldFamily::envelope) { - (Plasma ? target_real(base, time) : real_target[base * n_to + time]) = - Raman ? first.real() - : output_scale * (first.real() * - (Plasma ? window(time) : window_data[time])); - if constexpr (Components == 2) - (Plasma ? target_real(base + 1, time) - : real_target[(base + 1) * n_to + time]) = - Raman ? second.real() - : output_scale * (second.real() * - (Plasma ? window(time) : window_data[time])); - } else { - (Plasma ? target_complex(base, time) : complex_target[base * n_to + time]) = - Raman ? first - : output_scale * - (first * (Plasma ? window(time) : window_data[time])); - if constexpr (Components == 2) - (Plasma ? target_complex(base + 1, time) - : complex_target[(base + 1) * n_to + time]) = - Raman ? second - : output_scale * - (second * (Plasma ? window(time) : window_data[time])); - } - } - if constexpr (Stash) { - if (stash.empty() || stage + 1 != n_stages) - return; - if constexpr (Multiple) { - for (std::size_t channel = 0; channel < channels; ++channel) { - stash(first_node + node, 2 * channel) = scratch(channel, node, 2); - stash(first_node + node, 2 * channel + 1) = scratch(channel, node, 3); - } - } else { - stash(first_node + node, 0) = fraction; - stash(first_node + node, 1) = largest_rate; - } - } - }); -} - -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -// The device's response with no team tied to a node, so that a one-node fibre fills the -// device: -// -// - the pointwise terms (Kerr, the harmonic, the Raman drive and the windowed -// polarisation) as one two-dimensional range over (node, time), in the column's -// arithmetic sample by sample; -// - the trapezoid plasma as two-level scans over tiles of `plasma_tile_samples`: -// each running integral is a kernel of one team per (row, tile) that sums the -// tile's increments into its total (`plasma_tile_totals_`), then one per (row, -// tile) that adds the row's earlier totals and scans the tile again from that -// offset, forming the exposure's fraction or the current from each prefix. -template -void GeneralWorkspace::device_response(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count) { - const auto node_real = node_real_; - const auto node_complex = node_complex_; - const auto out_real = polarisation_real_; - const auto out_complex = polarisation_complex_; - const auto raman_drive = node_raman_drive_; - const auto raman_drives = problem_.raman_drives(); - const auto window = time_window_; - const auto carrier = carrier_; - const auto kerr_density = stage_kerr_density_; - const auto n_to = problem_.n_to; - const double kerr_coefficient = problem_.kerr_coefficient; - const double self_factor = problem_.self_factor; - const double conjugate_factor = problem_.conjugate_factor; - const double harmonic_factor = problem_.harmonic_factor; - if constexpr (Plasma) - device_trapezoid_plasma(stage, first_node, node_count); - launch::parallel_for( - Plasma ? "solprop_native.response.device_pointwise_after_plasma" - : "solprop_native.response.device_pointwise", - Range2(response_space_for(node_count), {0, 0}, {node_count, n_to}), - KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { - capture_anchor(self_factor, conjugate_factor, node_real, node_complex, out_real, - out_complex, raman_drive, raman_drives, window, carrier, - kerr_density, kerr_coefficient, harmonic_factor, stage); - const std::size_t base = node * Components; - const auto value_at = [&](const std::size_t row) { - if constexpr (Family == FieldFamily::field_real) - return Complex(node_real(row, time), 0.0); - else - return Complex(node_complex(row, time)); - }; - const Complex first_value = value_at(base); - Complex second_value{}; - if constexpr (Components == 2) - second_value = value_at(base + 1); - const double modulus = squared_abs(first_value) + - (Components == 2 ? squared_abs(second_value) : 0.0); - const Complex square = - first_value * first_value + - (Components == 2 ? second_value * second_value : Complex{}); - const double factor = kerr_coefficient * kerr_density(stage); - Complex first{}; - Complex second{}; - if constexpr (Nonlinear) { - if constexpr (Family == FieldFamily::field_real) { - first += Complex(factor * modulus * first_value.real(), 0.0); - if constexpr (Components == 2) - second += Complex(factor * modulus * second_value.real(), 0.0); - } else { - const auto analytic = [&](const Complex value) { - const double weighted_real = conjugate_factor * square.real(); - const double weighted_imag = conjugate_factor * square.imag(); - const double weighted_self = self_factor * modulus; - Complex result( - factor * - (weighted_self * value.real() + - (weighted_real * value.real() + weighted_imag * value.imag())), - factor * - (weighted_self * value.imag() + - (weighted_imag * value.real() - weighted_real * value.imag()))); - if constexpr (Harmonic) - result += ((factor * harmonic_factor) * carrier(time)) * square * value; - return result; - }; - first += analytic(first_value); - if constexpr (Components == 2) - second += analytic(second_value); - } - } - if constexpr (Raman) - write_raman_drive(raman_drive, node, time, first_value, - second_value, raman_drives); - if constexpr (Family != FieldFamily::envelope) { - out_real(base, time) = Raman ? first.real() : first.real() * window(time); - if constexpr (Components == 2) - out_real(base + 1, time) = - Raman ? second.real() : second.real() * window(time); - } else { - out_complex(base, time) = Raman ? first : first * window(time); - if constexpr (Components == 2) - out_complex(base + 1, time) = Raman ? second : second * window(time); - } - }); -} - -// The trapezoid plasma's two-level scans on a device (`device_response`). Rows of the -// rate, exposure and fraction views are the chunk's nodes; rows of the integrand and -// of the tile totals are its component rows. -template -void GeneralWorkspace::device_trapezoid_plasma(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count) { - using TeamPolicy = Kokkos::TeamPolicy; - using TeamMember = TeamPolicy::member_type; - const auto node_real = node_real_; - const auto node_complex = node_complex_; - const auto held_current = held_current_; - const auto rates = plasma_rate_; - const auto exposures = plasma_exposure_; - const auto fractions = plasma_fraction_; - const auto integrands = plasma_current_integrand_; - const auto totals = plasma_tile_totals_; - const auto stash = stash_candidate_; - const auto rate_coefficients = rate_coefficients_; - const auto density = stage_plasma_density_; - const auto preionisation = stage_preionisation_; - const double charge_mass_ratio = problem_.charge_mass_ratio; - const double ionisation_potential = problem_.ionisation_potential; - const double dt = problem_.dt_fine; - const double log_field_min = problem_.log_field_min; - const double d_log_field = problem_.d_log_field; - const double field_min = problem_.field_min; - const double field_max = problem_.field_max; - const double rate_max = problem_.rate_max; - const auto rate_intervals = problem_.rate_intervals; - const auto n_to = problem_.n_to; - const auto samples = static_cast(n_to); - const auto width = static_cast(plasma_tile_samples); - const auto tiles = static_cast(plasma_tiles_); - const auto last = n_to - 1; - const bool stashing = Stash && !stash.empty() && stage + 1 == problem_.n_stages; - const std::size_t rows = node_count * Components; - const auto team_size = static_cast(response_team_size()); - // The field a row reads: a real row's value, or an analytic or envelope row's real - // part. - const auto raw = [=] KOKKOS_FUNCTION(const std::size_t row, const std::size_t time) { - capture_anchor(node_real, node_complex); - if constexpr (Family == FieldFamily::field_real) - return node_real(row, time); - else - return node_complex(row, time).real(); - }; - - // 1. The rates. - launch::parallel_for( - "solprop_native.response.device_plasma_rate", - Range2(response_space_for(node_count), {0, 0}, {node_count, n_to}), - KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { - const std::size_t base = node * Components; - const double first = raw(base, time); - double magnitude = 0.0; - if constexpr (Components == 2) - magnitude = Kokkos::hypot(first, raw(base + 1, time)); - else - magnitude = Kokkos::abs(first); - rates(node, time) = - ionisation_rate(magnitude, rate_coefficients, log_field_min, d_log_field, - field_min, field_max, rate_max, rate_intervals); - }); - // 2. The exposure: tile totals, then each tile's prefixes from its offset, and the - // fraction from each. - const auto exposure_local = KOKKOS_LAMBDA(const TeamMember& team) { - capture_anchor(rates, totals, samples, width, tiles, dt); - const auto node = static_cast(team.league_rank() / tiles); - const int tile = team.league_rank() % tiles; - const int first = tile * width; - const int end = Kokkos::min(first + width, samples); - const double total = team_trapezoid_total( - team, - [&](const int sample) { return rates(node, static_cast(sample)); }, - first, end, dt); - Kokkos::single(Kokkos::PerTeam(team), - [&] { totals(node, static_cast(tile)) = total; }); - }; - launch::parallel_for("solprop_native.response.device_plasma_exposure_tiles", - TeamPolicy(response_space_for(node_count), - static_cast(node_count) * tiles, team_size), - exposure_local); - const auto exposure_finish = KOKKOS_LAMBDA(const TeamMember& team) { - capture_anchor(rates, totals, exposures, fractions, preionisation, samples, width, - tiles, dt, stage); - const auto node = static_cast(team.league_rank() / tiles); - const int tile = team.league_rank() % tiles; - const int first = tile * width; - const int end = Kokkos::min(first + width, samples); - const double offset = team_tile_offset(team, totals, node, tile); - team_trapezoid_finish( - team, - [&](const int sample) { return rates(node, static_cast(sample)); }, - first, end, dt, offset, - [&](const int sample, const double prefix) { - const auto time = static_cast(sample); - exposures(node, time) = prefix; - fractions(node, time) = ionised_fraction(prefix, preionisation(stage)); - }); - }; - launch::parallel_for("solprop_native.response.device_plasma_exposure", - TeamPolicy(response_space_for(node_count), - static_cast(node_count) * tiles, team_size), - exposure_finish); - // 3. The free-current integrand, per component row. - launch::parallel_for( - "solprop_native.response.device_plasma_integrand", - Range2(response_space_for(node_count), {0, 0}, {rows, n_to}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { - integrands(row, time) = - fractions(row / Components, time) * charge_mass_ratio * raw(row, time); - }); - // 4. The free current: tile totals, then each tile's prefixes from its offset and - // the current from each. - const auto current_local = KOKKOS_LAMBDA(const TeamMember& team) { - capture_anchor(integrands, totals, samples, width, tiles, dt); - const auto row = static_cast(team.league_rank() / tiles); - const int tile = team.league_rank() % tiles; - const int first = tile * width; - const int end = Kokkos::min(first + width, samples); - const double total = team_trapezoid_total( - team, - [&](const int sample) { - return integrands(row, static_cast(sample)); - }, - first, end, dt); - Kokkos::single(Kokkos::PerTeam(team), - [&] { totals(row, static_cast(tile)) = total; }); - }; - launch::parallel_for("solprop_native.response.device_plasma_current_tiles", - TeamPolicy(response_space_for(node_count), - static_cast(rows) * tiles, team_size), - current_local); - const auto current_finish = KOKKOS_LAMBDA(const TeamMember& team) { - capture_anchor(integrands, totals, held_current, rates, fractions, density, samples, - width, tiles, dt, ionisation_potential, stage); - const auto row = static_cast(team.league_rank() / tiles); - const int tile = team.league_rank() % tiles; - const int first = tile * width; - const int end = Kokkos::min(first + width, samples); - const double offset = team_tile_offset(team, totals, row, tile); - team_trapezoid_finish( - team, - [&](const int sample) { - return integrands(row, static_cast(sample)); - }, - first, end, dt, offset, - [&](const int sample, const double prefix) { - const auto time = static_cast(sample); - const std::size_t node = row / Components; - const std::size_t base = node * Components; - const double first_field = raw(base, time); - const double second_field = Components == 2 ? raw(base + 1, time) : 0.0; - const double magnitude_squared = - first_field * first_field + second_field * second_field; - const double rate = rates(node, time); - const double guard = rate > 0.0 && magnitude_squared > 0.0 - ? ionisation_potential * rate * - (1.0 - fractions(node, time)) / - magnitude_squared - : 0.0; - held_current(row, time) = density(stage) * (prefix + guard * raw(row, time)); - }); - }; - launch::parallel_for("solprop_native.response.device_plasma_current", - TeamPolicy(response_space_for(node_count), - static_cast(rows) * tiles, team_size), - current_finish); - // 5. The stash: the last sample's fraction and the largest rate, per node. - if (stashing) { - const auto peak = KOKKOS_LAMBDA(const TeamMember& team) { - capture_anchor(rates, fractions, stash, samples, last, first_node); - const auto node = static_cast(team.league_rank()); - double largest = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, samples), - [&](const int sample, double& update) { - const double rate = rates(node, static_cast(sample)); - if (rate > update) - update = rate; - }, - Kokkos::Max(largest)); - Kokkos::single(Kokkos::PerTeam(team), [&] { - stash(first_node + node, 0) = fractions(node, last); - stash(first_node + node, 1) = largest; - }); - }; - launch::parallel_for("solprop_native.response.device_plasma_stash", - TeamPolicy(response_space_for(node_count), - static_cast(node_count), team_size), - peak); - } -} -#endif - -static_assert(band_max_half + 1 == static_cast(plasma_max_taps), - "the team band filter's tap bound is the problem's"); - -namespace { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) -/** Whether the band column skips each row's quiet samples (`band_support`): host only. - */ -constexpr bool band_windows = false; -#else -constexpr bool band_windows = true; -/** - * The fewest samples a host thread of a band team takes from a row: below it a thread's - * share of a pass no longer repays the team's barriers between the passes. Measured on - * the mode-averaged helium RDW case (n_to = 8192, one node) on a loaded Apple M-series - * host: teams of two and four threads ran the band column faster than one, six slower; - * 2048 caps that row at four. To be re-measured on each host class (solprop SEP 25). - */ -constexpr std::size_t band_host_samples_per_thread = 2048; -#endif - -/** The first and last indices whose value is non-zero, as a Kokkos MinMax reduction. */ -using BandLive = Kokkos::MinMaxScalar; - -/** The band window of a row whose non-zero samples `live` found. */ -KOKKOS_INLINE_FUNCTION BandWindow band_window_of(const BandLive& live, - const int samples, const int half) { - if constexpr (band_windows) - return band_support(live.min_val, live.max_val, samples, half); - else - return {.first = 0, .last = samples - 1}; -} - -/** Count `index` into `live` when `value` is not exactly zero. */ -KOKKOS_INLINE_FUNCTION void band_note(BandLive& live, const int index, - const double value) { - if (value != 0.0) { - live.min_val = Kokkos::min(live.min_val, index); - live.max_val = Kokkos::max(live.max_val, index); - } -} - -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) -/** - * Count a host block's non-zero values into `live`, as `band_note` of each: - * `values[k]` is the sample `offset + k`. Only the first and the last non-zero value - * matter, so the block is searched from either end. - */ -inline void band_note_block(BandLive& live, const double* values, - const std::size_t count, const int offset) { - std::size_t low = 0; - while (low < count && values[low] == 0.0) - ++low; - if (low == count) - return; - std::size_t high = count - 1; - while (values[high] == 0.0) - --high; - live.min_val = std::min(live.min_val, offset + static_cast(low)); - live.max_val = std::max(live.max_val, offset + static_cast(high)); -} - -/** - * `ionisation_rate` of a host block's `count <= rate_block` magnitudes into `row`, - * with the table's limits `lower`, `upper` and `highest` as it takes them (a NaN - * magnitude is live); `within(values, live)` runs `ionisation_rates_within` on the - * live ones. A mostly live block runs every sample through the layer, a quiet one's - * magnitude replaced by the lower limit, and then keeps the limits' values: each value - * is the same function of its own magnitude, so the passes' lanes do not change it, - * and no sample is gathered. A mostly quiet block gathers its live magnitudes. - */ -template -void band_rates_block(const double* magnitudes, double* row, const std::size_t count, - const double lower, const double upper, const double highest, - const Within& within) { - double values[rate_block]; - std::size_t live = 0; - for (std::size_t k = 0; k < count; ++k) - live += ((magnitudes[k] > upper) | (magnitudes[k] < lower)) ? 0 : 1; - if (2 * live >= count && lower > 0.0) { - for (std::size_t k = 0; k < count; ++k) { - const double magnitude = magnitudes[k]; - values[k] = ((magnitude > upper) | (magnitude < lower)) ? lower : magnitude; - } - within(values, count); - for (std::size_t k = 0; k < count; ++k) { - const double magnitude = magnitudes[k]; - row[k] = magnitude > upper ? highest : magnitude < lower ? 0.0 : values[k]; - } - return; - } - if (live == 0) { - // A quiet block takes only the limits. - for (std::size_t k = 0; k < count; ++k) - row[k] = magnitudes[k] > upper ? highest : 0.0; - return; - } - std::size_t active[rate_block]; - std::size_t gathered = 0; - for (std::size_t k = 0; k < count; ++k) { - const double magnitude = magnitudes[k]; - row[k] = magnitude > upper ? highest : 0.0; - if (!(magnitude > upper) && !(magnitude < lower)) { - values[gathered] = magnitude; - active[gathered] = k; - ++gathered; - } - } - within(values, gathered); - for (std::size_t k = 0; k < gathered; ++k) - row[active[k]] = values[k]; -} -#endif -} // namespace - -int GeneralWorkspace::band_host_team_size(const std::size_t node_count) const { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - static_cast(node_count); - return 1; -#else - // solprop SEP 25 section 6.7.5: share the threads between the chunk's nodes, so a - // chunk of fewer nodes than threads runs each node's passes over chunks of t. - const auto threads = static_cast( - std::max(1, response_space_for(node_count).concurrency())); - const auto per_node = - std::max(1, threads / std::max(1, node_count)); - const auto by_samples = - std::max(1, problem_.n_to / band_host_samples_per_thread); - return static_cast(std::min(per_node, by_samples)); -#endif -} - -std::size_t GeneralWorkspace::band_league(const std::size_t node_count) const { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return node_count; -#else - return std::min(node_count, band_slots_); -#endif -} - -template -Kokkos::TeamPolicy -GeneralWorkspace::band_team_policy(const std::size_t node_count, - const Functor& functor) const { - using Policy = Kokkos::TeamPolicy; - constexpr auto policy_limit = - static_cast(std::numeric_limits::max()); - if (node_count > policy_limit || problem_.n_to > policy_limit) - throw std::length_error("band team column extent exceeds signed 32-bit range"); - const auto league = static_cast(band_league(node_count)); - const auto scratch = - Kokkos::PerTeam(band_staging_bytes()); - // The widest team this kernel can launch with its scratch: a device's registers per - // thread can hold the band column below the configured width. - Policy probe(response_space_for(node_count), league, 1); - probe.set_scratch_size(0, scratch); - const int widest = probe.team_size_max(functor, Kokkos::ParallelForTag()); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const int requested = static_cast(response_team_size()); -#else - const int requested = band_host_team_size(node_count); -#endif - Policy policy(response_space_for(node_count), league, - std::max(1, std::min(requested, widest))); - policy.set_scratch_size(0, scratch); - return policy; -} - -// solprop SEP 18's band rule as a team column in whole-row passes (solprop SEP 25 -// section 6.7.4; include/solprop_native/plasma.hpp): per channel the rates, their -// filter and scan give the exposure; the fraction (floored where it is formed), the -// loss guard and the free-current weight follow pointwise, a mixture adding each -// channel's share in channel order. Per component the free-current integrand is -// filtered and scanned, and the scan's store forms the current from it, the guard and -// the field. SEP 27 uses this current directly, without a third running integral. -// -// The rows, by slot (a team's own; the node on a device): `plasma_rate_` the raw rates, -// `plasma_fraction_` the filtered row, `plasma_exposure_` the exposure, -// `plasma_free_current_` the loss guard (a mixture's work until the last channel), -// `plasma_outer_current_` a mixture's weight, `plasma_current_integrand_` the -// free-current integrand. The separate `held_current_` stores the total current. Each -// filter and scan runs on its row's live window (`band_support`) and writes the rest: -// zero before it, the final prefix after it, and an unchanged exposure's fraction. -template -void GeneralWorkspace::team_band_plasma_column(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count) { - if constexpr (Components == 1 && !Folded) { - if (folds_single_node()) { - team_band_plasma_column(stage, first_node, - node_count); - return; - } - } - static_assert(Family != FieldFamily::envelope, - "the band rule integrates a field-grid plasma column"); - using TeamMember = Kokkos::TeamPolicy::member_type; - // A folded single node reads the modal field through its one-element synthesis - // (`folds_single_node`). - constexpr bool folded = Folded; - const auto node_real = folded ? modal_real_ : node_real_; - const auto node_complex = folded ? modal_complex_ : node_complex_; - const auto synthesis = synthesis_; - const auto taper = stage_taper_; - const auto out = held_current_; - const auto rates = plasma_rate_; - const auto filtered = plasma_fraction_; - const auto exposures = plasma_exposure_; - const auto guards = plasma_free_current_; - const auto weights = plasma_outer_current_; - const auto integrands = plasma_current_integrand_; - const auto taps_view = plasma_taps_; - const auto stash = stash_candidate_; - const bool stashing = !stash.empty() && stage + 1 == problem_.n_stages; - const auto rate_coefficients = rate_coefficients_; - const auto channel_parameters = channel_parameters_; - const auto channel_coefficients = channel_coefficients_; - const auto channel_density = stage_channel_density_; - const auto channel_preionisation = stage_channel_preionisation_; - const auto density = stage_plasma_density_; - const auto preionisation = stage_preionisation_; - const double charge_mass_ratio = problem_.charge_mass_ratio; - const double ionisation_potential = problem_.ionisation_potential; - const double dt = problem_.dt_fine; - const double log_field_min = problem_.log_field_min; - const double d_log_field = problem_.d_log_field; - const double field_min = problem_.field_min; - const double field_max = problem_.field_max; - const double rate_max = problem_.rate_max; - const auto rate_intervals = problem_.rate_intervals; - const std::size_t channels = Multiple ? problem_.n_channels : 1; - const auto samples = static_cast(problem_.n_to); -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - const std::size_t sample_count = problem_.n_to; -#endif - const auto half = static_cast(problem_.plasma_half_width()); - const std::size_t last = problem_.n_to - 1; - const std::size_t league = band_league(node_count); - const auto column = KOKKOS_LAMBDA(const TeamMember& team) { - // CUDA extended-lambda restriction 12 requires the first capture outside - // if-constexpr blocks. - capture_anchor(node_real, node_complex, out, rates, filtered, exposures, guards, - weights, integrands, taps_view, stash, stashing, rate_coefficients, - channel_parameters, channel_coefficients, channel_density, - channel_preionisation, density, preionisation, charge_mass_ratio, - ionisation_potential, dt, log_field_min, d_log_field, field_min, - field_max, rate_max, rate_intervals, channels, samples, half, last, - first_node, stage, league, synthesis, taper); - const auto slot = static_cast(team.league_rank()); - const std::size_t own = slot * Components; - double taps[plasma_max_taps] = {}; - for (int k = 0; k <= half; ++k) - taps[k] = taps_view(static_cast(k)); - const auto tile = band_tile(team); - const double input_scale = folded ? synthesis(0, 0) * taper(stage) : 1.0; - for (std::size_t node = slot; node < node_count; node += league) { - const std::size_t base = node * Components; - const auto field_at = [&](const std::size_t row, const std::size_t time) { - if constexpr (Family == FieldFamily::field_real) - return input_scale * node_real(row, time); - else - return input_scale * node_complex(row, time).real(); - }; - const auto magnitude_at = [&](const std::size_t time) { - const double first = field_at(base, time); - if constexpr (Components == 2) - return Kokkos::hypot(first, field_at(base + 1, time)); - else - return Kokkos::abs(first); - }; - [[maybe_unused]] const auto rate_of = [&](const double magnitude, - const std::size_t channel) { - if constexpr (Multiple) - return ionisation_rate( - magnitude, channel_coefficients, channel_parameters(channel, 0), - channel_parameters(channel, 1), channel_parameters(channel, 2), - channel_parameters(channel, 3), channel_parameters(channel, 4), - static_cast(channel_parameters(channel, 5)), - static_cast(channel_parameters(channel, 6))); - else - return ionisation_rate(magnitude, rate_coefficients, log_field_min, - d_log_field, field_min, field_max, rate_max, - rate_intervals); - }; - BandLive integrand_live; - integrand_live.min_val = samples; - integrand_live.max_val = -1; - -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // With vector maths a host team thread takes blocks of samples and gathers the - // live ones' transcendentals into passes of the layer (`ionisation_rates_within`, - // the decays below), writing the rest as it goes; with the system libm each - // sample keeps its own calls, as on a device. - [[maybe_unused]] const auto blocks = - static_cast((sample_count + rate_block - 1) / rate_block); - [[maybe_unused]] const auto rates_into = - [&](const std::size_t first, const std::size_t count, - const std::size_t channel, BandLive& update) { - const double lower = Multiple ? channel_parameters(channel, 2) : field_min; - const double upper = Multiple ? channel_parameters(channel, 3) : field_max; - const double highest = Multiple ? channel_parameters(channel, 4) : rate_max; - const auto within = [&](double* values, const std::size_t live) { - if constexpr (Multiple) - ionisation_rates_within( - values, live, channel_coefficients, channel_parameters(channel, 0), - channel_parameters(channel, 1), - static_cast(channel_parameters(channel, 5)), - static_cast(channel_parameters(channel, 6))); - else - ionisation_rates_within(values, live, rate_coefficients, log_field_min, - d_log_field, rate_intervals); - }; - double magnitudes[rate_block]; - for (std::size_t k = 0; k < count; ++k) - magnitudes[k] = magnitude_at(first + k); - double* const row = &rates(slot, first); - band_rates_block(magnitudes, row, count, lower, upper, highest, within); - band_note_block(update, row, count, static_cast(first)); - }; -#endif - - // 1. Per channel: rates -> filter -> scan -> exposure, then the fraction, the - // guard and the weight pointwise. - for (std::size_t channel = 0; channel < channels; ++channel) { - BandLive rate_live; -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - if constexpr (vmath::vectorised) { - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, blocks), - [&](const int block, BandLive& update) { - const auto first = static_cast(block) * rate_block; - rates_into(first, std::min(rate_block, sample_count - first), channel, - update); - }, - Kokkos::MinMax(rate_live)); - } else -#endif - { - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, samples), - [&](const int sample, BandLive& update) { - const auto time = static_cast(sample); - const double rate = rate_of(magnitude_at(time), channel); - rates(slot, time) = rate; - band_note(update, sample, rate); - }, - Kokkos::MinMax(rate_live)); - } - team.team_barrier(); - const BandWindow window = band_window_of(rate_live, samples, half); - if (window.first <= window.last) - team_band_filter(team, tile, rates, slot, filtered, slot, - samples, taps, half, window.first, - window.last + 1); - const double total = - team_trapezoid_window(team, filtered, slot, window, samples, dt, - [&](const int sample, const double prefix) { - exposures(slot, static_cast(sample)) = - prefix; - }); - const double initial = - Multiple ? channel_preionisation(stage, channel) : preionisation(stage); - // Outside [window.first, window.last + 1] the exposure is 0 before and `total` - // after, so the fraction there is one of two values (band_support). - const int changing_end = Kokkos::min(window.last + 2, samples); - const double before = band_fraction(0.0, initial); - const double after = band_fraction(total, initial); - [[maybe_unused]] const auto fraction_at = [&](const int sample) { - if (sample < window.first) - return before; - if (sample >= changing_end) - return after; - return band_fraction(exposures(slot, static_cast(sample)), - initial); - }; -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // A host block's fractions with vector maths, each handed to `apply` with its - // sample: the changing samples' decays `exp(-max(X, 0))` in one pass, each - // fraction then `band_fraction`'s strict arithmetic. - [[maybe_unused]] const auto fractions_into = - [&](const std::size_t first, const std::size_t count, const auto& apply) { - double decay[rate_block]; - std::size_t active[rate_block]; - std::size_t live = 0; - for (std::size_t time = first; time < first + count; ++time) { - const auto sample = static_cast(time); - if (sample < window.first) { - apply(time, before); - } else if (sample >= changing_end) { - apply(time, after); - } else { - const double exposure = exposures(slot, time); - decay[live] = -(exposure < 0.0 ? 0.0 : exposure); - active[live] = time; - ++live; - } - } - vmath::exp(decay, decay, live); - for (std::size_t k = 0; k < live; ++k) - apply(active[k], decayed_fraction(decay[k], initial)); - }; - // A single gas's host block of free-current integrands and loss guards as - // loops over the block's samples: the fractions as `fractions_into` forms - // them (the changing samples' decays in one pass, `decayed_fractions` in - // another), then `band_loss_scale` and the weights with their branch taken - // as a selection, and the integrand rows' live samples searched afterwards. - [[maybe_unused]] const auto integrands_into = [&](const std::size_t first, - const std::size_t count, - BandLive& update) { - double fraction[rate_block]; - const auto start = static_cast(first); - const auto stop = static_cast(first + count); - for (std::size_t k = 0; k < count; ++k) - fraction[k] = start + static_cast(k) < window.first ? before : after; - const int begin = Kokkos::max(window.first, start); - const int end = Kokkos::min(changing_end, stop); - if (begin < end) { - double* const changing = fraction + (begin - start); - const auto changing_count = static_cast(end - begin); - const double* const exposure = - &exposures(slot, static_cast(begin)); - for (std::size_t k = 0; k < changing_count; ++k) - changing[k] = -(exposure[k] < 0.0 ? 0.0 : exposure[k]); - vmath::exp(changing, changing, changing_count); - decayed_fractions(changing, changing_count, initial); - } - const double* const rate_row = &rates(slot, first); - double* const guard_row = &guards(slot, first); - for (std::size_t k = 0; k < count; ++k) { - const std::size_t time = first + k; - const double magnitude = magnitude_at(time); - const double squared = magnitude * magnitude; - const double rate = rate_row[k]; - const bool loses = rate > 0.0 && squared > 0.0; - const double scale = ionisation_potential * rate * (1.0 - fraction[k]) / - (loses ? squared : 1.0); - guard_row[k] = loses ? scale : 0.0; - const double weight = fraction[k] * charge_mass_ratio; - for (std::size_t component = 0; component < Components; ++component) - integrands(own + component, time) = - weight * field_at(base + component, time); - } - for (std::size_t component = 0; component < Components; ++component) - band_note_block(update, &integrands(own + component, first), count, start); - }; -#endif - if constexpr (Multiple) { - const double partial = channel_density(stage, channel); - const double charge = channel_parameters(channel, 7); - const double potential = channel_parameters(channel, 8); - const auto share_at = [&](const std::size_t time, const double fraction) { - const double rate = rates(slot, time); - const double share = partial * fraction * charge; - const double paid = partial * potential * rate * (1.0 - fraction); - const double weight = channel == 0 ? 0.0 : weights(slot, time); - const double work = channel == 0 ? 0.0 : guards(slot, time); - weights(slot, time) = weight + share; - guards(slot, time) = work + paid; - }; -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - if constexpr (vmath::vectorised) { - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, blocks), [&](const int block) { - const auto first = static_cast(block) * rate_block; - fractions_into(first, std::min(rate_block, sample_count - first), - share_at); - }); - } else -#endif - { - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, samples), [&](const int sample) { - share_at(static_cast(sample), fraction_at(sample)); - }); - } - } else { - const auto integrand_at = [&](const std::size_t time, const double fraction, - BandLive& update) { - const double magnitude = magnitude_at(time); - const double squared = magnitude * magnitude; - guards(slot, time) = band_loss_scale(rates(slot, time), fraction, squared, - ionisation_potential); - const double weight = fraction * charge_mass_ratio; - for (std::size_t component = 0; component < Components; ++component) { - const double value = weight * field_at(base + component, time); - integrands(own + component, time) = value; - band_note(update, static_cast(time), value); - } - }; -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - if constexpr (vmath::vectorised) { - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, blocks), - [&](const int block, BandLive& update) { - const auto first = static_cast(block) * rate_block; - integrands_into(first, std::min(rate_block, sample_count - first), - update); - }, - Kokkos::MinMax(integrand_live)); - } else -#endif - { - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, samples), - [&](const int sample, BandLive& update) { - integrand_at(static_cast(sample), fraction_at(sample), - update); - }, - Kokkos::MinMax(integrand_live)); - } - } - if (stashing) { - double largest = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, samples), - [&](const int sample, double& update) { - const double rate = rates(slot, static_cast(sample)); - if (rate > update) - update = rate; - }, - Kokkos::Max(largest)); - Kokkos::single(Kokkos::PerTeam(team), [&] { - stash(first_node + node, 2 * channel) = - band_fraction(exposures(slot, last), initial); - stash(first_node + node, 2 * channel + 1) = largest > 0.0 ? largest : 0.0; - }); - } - team.team_barrier(); - } - if constexpr (Multiple) { - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, samples), - [&](const int sample, BandLive& update) { - const auto time = static_cast(sample); - const double magnitude = magnitude_at(time); - const double squared = magnitude * magnitude; - guards(slot, time) = - band_loss_scale(1.0, 0.0, squared, guards(slot, time)); - const double weight = weights(slot, time); - for (std::size_t component = 0; component < Components; ++component) { - const double value = weight * field_at(base + component, time); - integrands(own + component, time) = value; - band_note(update, sample, value); - } - }, - Kokkos::MinMax(integrand_live)); - } - team.team_barrier(); - - // 2. Per component: the free current, and from it the current. - const BandWindow free_window = band_window_of(integrand_live, samples, half); - // A single gas's density, read once: inside the store below the compiler must - // reload it after each store to the current, which might alias it. - const double stage_density = Multiple ? 0.0 : density(stage); - for (std::size_t component = 0; component < Components; ++component) { - const std::size_t row = base + component; - if (free_window.first <= free_window.last) - team_band_filter(team, tile, integrands, own + component, - filtered, slot, samples, taps, half, - free_window.first, free_window.last + 1); - team_trapezoid_window(team, filtered, slot, free_window, samples, dt, - [&](const int sample, const double prefix) { - const auto time = static_cast(sample); - const double drive = - guards(slot, time) * field_at(row, time); - out(row, time) = Multiple - ? prefix + drive - : stage_density * (prefix + drive); - }); - } - } - }; - launch::parallel_for("solprop_native.response.team_band_plasma_column", - band_team_policy(node_count, column), column); -} - -namespace { -/** - * A band column's node field: `scale` times the real row of a field-family node (or of - * the folded single node's modal row), and its magnitude over a node's components. - */ -template struct BandField { - RealView2 real; - ComplexView2 complex; - KOKKOS_INLINE_FUNCTION double at(const std::size_t row, const std::size_t time, - const double scale) const { - if constexpr (Family == FieldFamily::field_real) - return scale * real(row, time); - else - return scale * complex(row, time).real(); - } - KOKKOS_INLINE_FUNCTION double - magnitude(const std::size_t base, const std::size_t time, const double scale) const { - const double first = at(base, time, scale); - if constexpr (Components == 2) - return Kokkos::hypot(first, at(base + 1, time, scale)); - else - return Kokkos::abs(first); - } -}; - -/** A band column's ionisation rate: the single gas's table, or a mixture channel's. */ -template struct BandRate { - RealView coefficients; - RealView2 parameters; - RealView channel_coefficients; - double log_field_min{}, d_log_field{}, field_min{}, field_max{}, rate_max{}; - std::size_t intervals{}; - KOKKOS_INLINE_FUNCTION double operator()(const double magnitude, - const std::size_t channel) const { - if constexpr (Multiple) - return ionisation_rate(magnitude, channel_coefficients, parameters(channel, 0), - parameters(channel, 1), parameters(channel, 2), - parameters(channel, 3), parameters(channel, 4), - static_cast(parameters(channel, 5)), - static_cast(parameters(channel, 6))); - else - return ionisation_rate(magnitude, coefficients, log_field_min, d_log_field, - field_min, field_max, rate_max, intervals); - } -}; -} // namespace - -template -Kokkos::TeamPolicy -GeneralWorkspace::tiled_band_policy(const std::size_t rows, const bool scratch, - const Functor& functor) const { - using Policy = Kokkos::TeamPolicy; - constexpr auto policy_limit = - static_cast(std::numeric_limits::max()); - if (band_tile_count_ == 0 || rows > policy_limit / band_tile_count_ || - problem_.n_to > policy_limit) - throw std::length_error("tiled band column extent exceeds signed 32-bit range"); - const auto league = static_cast(rows * band_tile_count_); - const auto bytes = scratch ? band_tile_bytes(band_tile_) : 0; - Policy probe(response_execution_space_, league, 1); - probe.set_scratch_size(0, Kokkos::PerTeam(bytes)); - const int widest = probe.team_size_max(functor, Kokkos::ParallelForTag()); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const auto requested = static_cast(std::min(response_team_size(), band_tile_)); -#else - const int requested = std::max(1, band_tile_threads_); -#endif - Policy policy(response_execution_space_, league, - std::max(1, std::min(requested, widest))); - policy.set_scratch_size(0, Kokkos::PerTeam(bytes)); - return policy; -} - -// The band column in tiles (solprop SEP 25 section 6.7.6; -// include/solprop_native/plasma.hpp): each running integral is a local kernel of one -// team per (row, tile), which loads, filters and scans its tile from zero -// (`team_band_tile`), then a finish kernel of one team per (row, tile), which adds the -// row's earlier tile totals (`team_tile_offset`) and forms the pass's product from the -// prefix, in the team column's arithmetic: -// -// 1. per channel: the local kernel forms the rates in its window (keeping its own -// tile's in `plasma_rate_`) and the exposure's local prefixes -// (`plasma_exposure_`); the finish forms the fraction, the loss guard -// (`plasma_free_current_`) and the free-current integrand -// (`plasma_current_integrand_`), a mixture accumulating its weight -// (`plasma_outer_current_`) and work channel by channel; -// 2. per component row: the integrand's local prefixes (`plasma_outer_current_`, -// free after the first pass), then the current (`held_current_`). -// -// Rows are by node within the chunk, the totals by row (`band_tile_totals_`). The -// stash's fraction comes from the last sample's finish and its largest rate from a -// reduction over the rate row. No window is skipped: every sample is formed. -template -void GeneralWorkspace::tiled_band_plasma_column(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count) { - if constexpr (Components == 1 && !Folded) { - if (folds_single_node()) { - tiled_band_plasma_column(stage, first_node, - node_count); - return; - } - } - static_assert(Family != FieldFamily::envelope, - "the band rule integrates a field-grid plasma column"); - using TeamMember = Kokkos::TeamPolicy::member_type; - constexpr bool folded = Folded; - const BandField field{folded ? modal_real_ : node_real_, - folded ? modal_complex_ : node_complex_}; - const BandRate rate{ - rate_coefficients_, channel_parameters_, channel_coefficients_, - problem_.log_field_min, problem_.d_log_field, problem_.field_min, - problem_.field_max, problem_.rate_max, problem_.rate_intervals}; - const auto synthesis = synthesis_; - const auto taper = stage_taper_; - const auto out = held_current_; - const auto rates = plasma_rate_; - const auto exposures = plasma_exposure_; - const auto guards = plasma_free_current_; - const auto weights = plasma_outer_current_; - const auto locals = plasma_outer_current_; - const auto integrands = plasma_current_integrand_; - const auto totals = band_tile_totals_; - const auto taps_view = plasma_taps_; - const auto stash = stash_candidate_; - const bool stashing = !stash.empty() && stage + 1 == problem_.n_stages; - const auto channel_parameters = channel_parameters_; - const auto channel_density = stage_channel_density_; - const auto channel_preionisation = stage_channel_preionisation_; - const auto density = stage_plasma_density_; - const auto preionisation = stage_preionisation_; - const double charge_mass_ratio = problem_.charge_mass_ratio; - const double ionisation_potential = problem_.ionisation_potential; - const double dt = problem_.dt_fine; - const std::size_t channels = Multiple ? problem_.n_channels : 1; - const auto samples = static_cast(problem_.n_to); - const auto half = static_cast(problem_.plasma_half_width()); - const auto width = static_cast(band_tile_); - const auto tiles = static_cast(band_tile_count_); - const int last = samples - 1; - const std::size_t rows = node_count * Components; - - // 1. Per channel: the exposure, then the fraction, the guard and the integrand. - for (std::size_t channel = 0; channel < channels; ++channel) { - const auto local = KOKKOS_LAMBDA(const TeamMember& team) { - capture_anchor(field, rate, rates, exposures, totals, taps_view, synthesis, taper, - stage, channel, samples, half, width, tiles, dt); - const auto node = static_cast(team.league_rank() / tiles); - const int index = team.league_rank() % tiles; - const int first = index * width; - const int end = Kokkos::min(first + width, samples); - double taps[plasma_max_taps] = {}; - for (int k = 0; k <= half; ++k) - taps[k] = taps_view(static_cast(k)); - const double scale = folded ? synthesis(0, 0) * taper(stage) : 1.0; - const std::size_t base = node * Components; - const double total = team_band_tile( - team, - [&](const int sample) { - const auto time = static_cast(sample); - const double value = rate(field.magnitude(base, time, scale), channel); - if (sample >= first && sample < end) - rates(node, time) = value; - return value; - }, - first, end, samples, taps, half, dt, - [&](const int sample, const double prefix) { - exposures(node, static_cast(sample)) = prefix; - }); - Kokkos::single(Kokkos::PerTeam(team), - [&] { totals(node, static_cast(index)) = total; }); - }; - launch::parallel_for("solprop_native.response.tiled_band_exposure", - tiled_band_policy(node_count, true, local), local); - const auto finish = KOKKOS_LAMBDA(const TeamMember& team) { - capture_anchor(field, rates, exposures, guards, weights, integrands, totals, - stash, stashing, synthesis, taper, channel_parameters, - channel_density, channel_preionisation, preionisation, - charge_mass_ratio, ionisation_potential, channels, channel, - samples, width, tiles, last, first_node, stage); - const auto node = static_cast(team.league_rank() / tiles); - const int index = team.league_rank() % tiles; - const int first = index * width; - const int end = Kokkos::min(first + width, samples); - const double offset = team_tile_offset(team, totals, node, index); - const double scale = folded ? synthesis(0, 0) * taper(stage) : 1.0; - const std::size_t base = node * Components; - const double initial = - Multiple ? channel_preionisation(stage, channel) : preionisation(stage); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, first, end), [&](const int sample) { - const auto time = static_cast(sample); - const double fraction = - band_fraction(offset + exposures(node, time), initial); - const double value = rates(node, time); - if constexpr (Multiple) { - const double partial = channel_density(stage, channel); - const double charge = channel_parameters(channel, 7); - const double potential = channel_parameters(channel, 8); - const double share = partial * fraction * charge; - const double paid = partial * potential * value * (1.0 - fraction); - const double weight = (channel == 0 ? 0.0 : weights(node, time)) + share; - const double work = (channel == 0 ? 0.0 : guards(node, time)) + paid; - if (channel + 1 == channels) { - const double magnitude = field.magnitude(base, time, scale); - guards(node, time) = - band_loss_scale(1.0, 0.0, magnitude * magnitude, work); - for (std::size_t component = 0; component < Components; ++component) - integrands(base + component, time) = - weight * field.at(base + component, time, scale); - } else { - weights(node, time) = weight; - guards(node, time) = work; - } - } else { - const double magnitude = field.magnitude(base, time, scale); - guards(node, time) = band_loss_scale( - value, fraction, magnitude * magnitude, ionisation_potential); - const double weight = fraction * charge_mass_ratio; - for (std::size_t component = 0; component < Components; ++component) - integrands(base + component, time) = - weight * field.at(base + component, time, scale); - } - if (stashing && sample == last) - stash(first_node + node, 2 * channel) = fraction; - }); - }; - launch::parallel_for("solprop_native.response.tiled_band_fraction", - tiled_band_policy(node_count, false, finish), finish); - if (stashing) { - const auto peak = KOKKOS_LAMBDA(const TeamMember& team) { - capture_anchor(rates, stash, channel, samples, first_node); - const auto node = static_cast(team.league_rank()); - double largest = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, samples), - [&](const int sample, double& update) { - const double value = rates(node, static_cast(sample)); - if (value > update) - update = value; - }, - Kokkos::Max(largest)); - Kokkos::single(Kokkos::PerTeam(team), [&] { - stash(first_node + node, 2 * channel + 1) = largest > 0.0 ? largest : 0.0; - }); - }; - launch::parallel_for("solprop_native.response.tiled_band_peak_rate", - Kokkos::TeamPolicy( - response_space_for(node_count), - static_cast(node_count), Kokkos::AUTO()), - peak); - } - } - - // 2. Per component row: the free current, followed by the pointwise loss current. - tiled_band_local(integrands, rows, "solprop_native.response.tiled_band_free_current"); - const auto current = KOKKOS_LAMBDA(const TeamMember& team) { - capture_anchor(field, guards, locals, out, density, totals, synthesis, taper, stage, - samples, width, tiles); - const auto row = static_cast(team.league_rank() / tiles); - const int index = team.league_rank() % tiles; - const int first = index * width; - const int end = Kokkos::min(first + width, samples); - const double offset = team_tile_offset(team, totals, row, index); - const double scale = folded ? synthesis(0, 0) * taper(stage) : 1.0; - const std::size_t node = row / Components; - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, first, end), [&](const int sample) { - const auto time = static_cast(sample); - const double prefix = offset + locals(row, time); - const double drive = guards(node, time) * field.at(row, time, scale); - out(row, time) = - Multiple ? prefix + drive : density(stage) * (prefix + drive); - }); - }; - launch::parallel_for("solprop_native.response.tiled_band_current", - tiled_band_policy(rows, false, current), current); -} - -// The tiled column's local kernel over component rows: `source`'s band-filtered tile -// prefixes from zero into the free-current rows (`plasma_outer_current_`), -// and each tile's total into `band_tile_totals_`. -void GeneralWorkspace::tiled_band_local(const RealView2& source, const std::size_t rows, - const char* label) { - using TeamMember = Kokkos::TeamPolicy::member_type; - const auto locals = plasma_outer_current_; - const auto totals = band_tile_totals_; - const auto taps_view = plasma_taps_; - const double dt = problem_.dt_fine; - const auto samples = static_cast(problem_.n_to); - const auto half = static_cast(problem_.plasma_half_width()); - const auto width = static_cast(band_tile_); - const auto tiles = static_cast(band_tile_count_); - const auto local = KOKKOS_LAMBDA(const TeamMember& team) { - const auto row = static_cast(team.league_rank() / tiles); - const int index = team.league_rank() % tiles; - const int first = index * width; - const int end = Kokkos::min(first + width, samples); - double taps[plasma_max_taps] = {}; - for (int k = 0; k <= half; ++k) - taps[k] = taps_view(static_cast(k)); - const double total = team_band_tile( - team, - [&](const int sample) { return source(row, static_cast(sample)); }, - first, end, samples, taps, half, dt, - [&](const int sample, const double prefix) { - locals(row, static_cast(sample)) = prefix; - }); - Kokkos::single(Kokkos::PerTeam(team), - [&] { totals(row, static_cast(index)) = total; }); - }; - launch::parallel_for(label, tiled_band_policy(rows, true, local), local); -} - -// The band rule's statistics scan: per channel the rates, their band filter and -// trapezoid scan, then team reductions over the rows. The floored final exposure and -// the largest rate always; with `Diagnose` the six parts of SEP 18's run diagnostics, -// the largest over the channels: the exposure's largest dip and rise (maxima of the -// scan's increments), the largest |paid - freed| and freed (the paid work a sum of the -// trapezoid's increments of the raw rate's owed work), and the largest |f E| in the -// first 2K samples and anywhere. Maxima combine exactly; the sums associate as the -// team's scan does. The rows are the band column's (`plasma_rate_`, -// `plasma_fraction_` filtered and then owed work, `plasma_exposure_` exposure and then -// fraction), by slot as the column walks the nodes. Not a step's hot path: it runs -// whole rows, without the column's live windows. -template -void GeneralWorkspace::team_band_statistics(const std::size_t first_node, - const std::size_t node_count, - const RealView2& output, - const double preionisation) { - using TeamMember = Kokkos::TeamPolicy::member_type; - const auto node_real = node_real_; - const auto node_complex = node_complex_; - const bool real_family = problem_.family == FieldFamily::field_real; - const auto rates = plasma_rate_; - const auto filtered = plasma_fraction_; - const auto exposures = plasma_exposure_; - const auto taps_view = plasma_taps_; - const auto diagnostics = plasma_diagnostics_; - const auto coefficients = rate_coefficients_; - const auto parameters = channel_parameters_; - const auto channel_coefficients = channel_coefficients_; - const auto channel_preionisation = stage_channel_preionisation_; - const auto channels = problem_.n_channels; - const bool multiple = channels > 1; - const auto n_pol = problem_.effective_node_pol(); - const auto samples = static_cast(problem_.n_to); - const auto intervals = problem_.rate_intervals; - const auto dt = problem_.dt_fine; - const auto log_field_min = problem_.log_field_min; - const auto d_log_field = problem_.d_log_field; - const auto field_min = problem_.field_min; - const auto field_max = problem_.field_max; - const auto rate_max = problem_.rate_max; - const auto half = static_cast(problem_.plasma_half_width()); - const int reach = Kokkos::min(2 * half, samples); - const std::size_t last = problem_.n_to - 1; - const std::size_t league = band_league(node_count); - const auto scan = KOKKOS_LAMBDA(const TeamMember& team) { - capture_anchor(node_real, node_complex, real_family, rates, filtered, exposures, - taps_view, diagnostics, coefficients, parameters, - channel_coefficients, channel_preionisation, channels, multiple, - n_pol, samples, intervals, dt, log_field_min, d_log_field, field_min, - field_max, rate_max, half, reach, last, first_node, output, - preionisation, league); - const auto slot = static_cast(team.league_rank()); - double taps[plasma_max_taps] = {}; - for (int k = 0; k <= half; ++k) - taps[k] = taps_view(static_cast(k)); - const auto tile = band_tile(team); - for (std::size_t node = slot; node < node_count; node += league) { - const auto base = node * n_pol; - const auto field_at = [&](const std::size_t row, const std::size_t time) { - return real_family ? node_real(row, time) : node_complex(row, time).real(); - }; - double parts[plasma_diagnostic_parts] = {}; - for (std::size_t channel = 0; channel < channels; ++channel) { - const double initial = - multiple ? channel_preionisation(0, channel) : preionisation; -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // With vector maths a host thread forms blocks of rates as the column does - // (`band_rates_block`). - if constexpr (vmath::vectorised) { - const double lower = multiple ? parameters(channel, 2) : field_min; - const double upper = multiple ? parameters(channel, 3) : field_max; - const double highest = multiple ? parameters(channel, 4) : rate_max; - const auto within = [&](double* values, const std::size_t live) { - if (multiple) - ionisation_rates_within(values, live, channel_coefficients, - parameters(channel, 0), parameters(channel, 1), - static_cast(parameters(channel, 5)), - static_cast(parameters(channel, 6))); - else - ionisation_rates_within(values, live, coefficients, log_field_min, - d_log_field, intervals); - }; - const auto sample_count = static_cast(samples); - const auto blocks = - static_cast((sample_count + rate_block - 1) / rate_block); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, blocks), [&](const int block) { - const auto start = static_cast(block) * rate_block; - const auto count = std::min(rate_block, sample_count - start); - double magnitudes[rate_block]; - for (std::size_t k = 0; k < count; ++k) { - const double first = field_at(base, start + k); - magnitudes[k] = - n_pol == 2 ? Kokkos::hypot(first, field_at(base + 1, start + k)) - : Kokkos::abs(first); - } - band_rates_block(magnitudes, &rates(slot, start), count, lower, upper, - highest, within); - }); - } else { -#endif - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, samples), [&](const int sample) { - const auto time = static_cast(sample); - const double first = field_at(base, time); - const double magnitude = - n_pol == 2 ? Kokkos::hypot(first, field_at(base + 1, time)) - : Kokkos::abs(first); - rates(slot, time) = - multiple - ? ionisation_rate( - magnitude, channel_coefficients, parameters(channel, 0), - parameters(channel, 1), parameters(channel, 2), - parameters(channel, 3), parameters(channel, 4), - static_cast(parameters(channel, 5)), - static_cast(parameters(channel, 6))) - : ionisation_rate(magnitude, coefficients, log_field_min, - d_log_field, field_min, field_max, rate_max, - intervals); - }); -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - } -#endif - team.team_barrier(); - team_band_filter(team, tile, rates, slot, filtered, slot, samples, - taps, half); - team_trapezoid_prefix(team, filtered, slot, samples, dt, - [&](const int sample, const double prefix) { - exposures(slot, static_cast(sample)) = - prefix; - }); - double largest = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, samples), - [&](const int sample, double& update) { - const double rate = rates(slot, static_cast(sample)); - if (rate > update) - update = rate; - }, - Kokkos::Max(largest)); - const double exposure = exposures(slot, last); - Kokkos::single(Kokkos::PerTeam(team), [&] { - output(first_node + node, 2 * channel) = exposure > 0.0 ? exposure : 0.0; - output(first_node + node, 2 * channel + 1) = largest > 0.0 ? largest : 0.0; - }); - if constexpr (Diagnose) { - // The exposure's increments, as the scan formed them. - const auto increment = [&](const int sample) { - const auto time = static_cast(sample); - const double current = filtered(slot, time); - const double previous = filtered(slot, time - 1); - return dt * (current + previous) / 2.0; - }; - double dip = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, 1, samples), - [&](const int sample, double& update) { - const double fall = -increment(sample); - if (fall > update) - update = fall; - }, - Kokkos::Max(dip)); - double rise = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, 1, samples), - [&](const int sample, double& update) { - const double step = increment(sample); - if (step > update) - update = step; - }, - Kokkos::Max(rise)); - team.team_barrier(); - // The filtered row becomes the owed work, the exposure row the fraction. -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // With vector maths a host thread takes blocks: the decays in one pass, - // then `decayed_fractions`, as the column forms them. - if constexpr (vmath::vectorised) { - const auto sample_count = static_cast(samples); - const auto blocks = - static_cast((sample_count + rate_block - 1) / rate_block); - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, blocks), [&](const int block) { - const auto start = static_cast(block) * rate_block; - const auto count = std::min(rate_block, sample_count - start); - double* const exposed = &exposures(slot, start); - double* const owed = &filtered(slot, start); - const double* const rate = &rates(slot, start); - double fraction[rate_block]; - for (std::size_t k = 0; k < count; ++k) - fraction[k] = -(exposed[k] < 0.0 ? 0.0 : exposed[k]); - vmath::exp(fraction, fraction, count); - decayed_fractions(fraction, count, initial); - for (std::size_t k = 0; k < count; ++k) { - owed[k] = rate[k] * (1.0 - fraction[k]); - exposed[k] = fraction[k]; - } - }); - } else { -#endif - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, samples), [&](const int sample) { - const auto time = static_cast(sample); - const double fraction = band_fraction(exposures(slot, time), initial); - filtered(slot, time) = rates(slot, time) * (1.0 - fraction); - exposures(slot, time) = fraction; - }); -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - } -#endif - team.team_barrier(); - double paid = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, 1, samples), - [&](const int sample, double& sum) { sum += increment(sample); }, paid); - const auto level = [&](const int sample, double& update) { - const auto time = static_cast(sample); - const double fraction = exposures(slot, time); - for (std::size_t component = 0; component < n_pol; ++component) { - const double value = - Kokkos::abs(fraction * field_at(base + component, time)); - if (value > update) - update = value; - } - }; - double early = 0.0; - Kokkos::parallel_reduce(Kokkos::TeamThreadRange(team, reach), level, - Kokkos::Max(early)); - double peak = 0.0; - Kokkos::parallel_reduce(Kokkos::TeamThreadRange(team, samples), level, - Kokkos::Max(peak)); - const double freed = exposures(slot, last) - exposures(slot, 0); - const double values[plasma_diagnostic_parts] = { - dip > 0.0 ? dip : 0.0, rise > 0.0 ? rise : 0.0, - Kokkos::abs(paid - freed), freed, - early > 0.0 ? early : 0.0, peak > 0.0 ? peak : 0.0}; - for (std::size_t part = 0; part < plasma_diagnostic_parts; ++part) - parts[part] = Kokkos::max(parts[part], values[part]); - } - team.team_barrier(); - } - if constexpr (Diagnose) - Kokkos::single(Kokkos::PerTeam(team), [&] { - for (std::size_t part = 0; part < plasma_diagnostic_parts; ++part) - diagnostics(first_node + node, part) = parts[part]; - }); - team.team_barrier(); - } - }; - launch::parallel_for("solprop_native.plasma.team_band_statistics", - band_team_policy(node_count, scan), scan); -} - -void GeneralWorkspace::convolve_raman(const std::size_t stage, - const std::size_t node_count) { - const bool tail = node_count != problem_.node_chunk; - Kokkos::Profiling::pushRegion("solprop_native.raman.drive_fft"); - execute_raman(tail ? *raman_tail_forward_plan_ : *raman_drive_forward_plan_, - tail ? raman_tail_drive_ : node_raman_drive_, - tail ? raman_tail_spectrum_ : node_raman_spectrum_, - KokkosFFT::Normalization::none); - Kokkos::Profiling::popRegion(); - - Kokkos::Profiling::pushRegion("solprop_native.raman.spectral_multiply"); - const auto node_spectrum = node_raman_spectrum_; - const auto kernel_spectrum = raman_kernel_spectrum_; - const auto drives = problem_.raman_drives(); - const double inverse_doubled = 1.0 / static_cast(2 * problem_.n_to); - launch::parallel_for( - "solprop_native.raman.spectral_multiply", - Range2(tail ? raman_tail_multiply_space_ : raman_multiply_space_, {0, 0}, - {node_count * drives, problem_.n_to + 1}), - KOKKOS_LAMBDA(const std::size_t node, const std::size_t index) { - node_spectrum(node, index) = - (node_spectrum(node, index) * - kernel_spectrum(node % drives == 0 ? 0 : 1, index)) * - inverse_doubled; - }); - Kokkos::Profiling::popRegion(); - - Kokkos::Profiling::pushRegion("solprop_native.raman.inverse_fft"); - execute_raman(tail ? *raman_tail_inverse_plan_ : *raman_convolution_inverse_plan_, - tail ? raman_tail_spectrum_ : node_raman_spectrum_, - tail ? raman_tail_convolved_ : node_raman_convolved_, - KokkosFFT::Normalization::none); - Kokkos::Profiling::popRegion(); - finish_raman(stage, node_count); -} - -void GeneralWorkspace::finish_raman(const std::size_t stage, - const std::size_t node_count) { - Kokkos::Profiling::pushRegion("solprop_native.raman.finish"); - const auto node_real = node_real_; - const auto node_complex = node_complex_; - const auto out_real = polarisation_real_; - const auto out_complex = polarisation_complex_; - const auto convolved = node_raman_convolved_; - const auto window = time_window_; - const auto density = stage_raman_density_; - const auto family = problem_.family; - const auto components = problem_.effective_node_pol(); - const auto drives = problem_.raman_drives(); - const double epsilon0 = problem_.raman_epsilon0; - launch::parallel_for( - "solprop_native.raman.finish", - Range2(response_space_for(node_count), {0, 0}, {node_count, problem_.n_to}), - KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { - if (components == 2) { - const auto base = 2 * node; - const auto row = drives * node; - const double u = convolved(row, time); - const double v = drives == 3 ? convolved(row + 1, time) : 0.0; - const double w = drives == 3 ? convolved(row + 2, time) : 0.0; - const double factor = epsilon0 * density(stage); - if (family == FieldFamily::envelope) { - const Complex x = node_complex(base, time); - const Complex y = node_complex(base + 1, time); - out_complex(base, time) = - (out_complex(base, time) + factor * ((u + v) * x + w * y)) * - window(time); - out_complex(base + 1, time) = - (out_complex(base + 1, time) + factor * (w * x + (u - v) * y)) * - window(time); - } else { - const double x = family == FieldFamily::field_real - ? node_real(base, time) - : node_complex(base, time).real(); - const double y = family == FieldFamily::field_real - ? node_real(base + 1, time) - : node_complex(base + 1, time).real(); - out_real(base, time) = - (out_real(base, time) + factor * ((u + v) * x + w * y)) * window(time); - out_real(base + 1, time) = - (out_real(base + 1, time) + factor * (w * x + (u - v) * y)) * - window(time); - } - return; - } - const double factor = epsilon0 * density(stage) * convolved(node, time); - if (family == FieldFamily::envelope) { - out_complex(node, time) = - (out_complex(node, time) + factor * node_complex(node, time)) * - window(time); - } else { - const double field = family == FieldFamily::field_real - ? node_real(node, time) - : node_complex(node, time).real(); - out_real(node, time) = (out_real(node, time) + factor * field) * window(time); - } - }); - Kokkos::Profiling::popRegion(); -} - -// The modal-product convolution (after the CPU experiment's Raman port). With a real -// static synthesis S, a scalar node field is x_j = sum_m S_jm a_m, so the Raman drive -// x_j^2 (or the cycle average |x_j|^2 / 2) is sum_{m <= n} c_mn S_jm S_jn P_mn with -// P_mn = Re(a_m) Re(a_n) (or (Re(a_m conj a_n)) / 2) and c_mn = 2 off the diagonal. -// The causal convolution is linear: convolving each of the M (M + 1) / 2 products once -// and reconstructing each node's convolved drive from them replaces one drive per node. -// It is chosen where it transforms fewer rows than the nodes and its rows fit the node -// drive's own scratch; the stage-dependent kernel, its support and the taper (inside -// S) are those of the node path. The reconstruction sums the products in a different -// order from squaring the synthesised field: the two agree to rounding of the drive's -// scale, sum_m |S_jm a_m| squared. Host builds only: a device keeps the node drives. -std::size_t GeneralWorkspace::raman_modal_products() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - return 0; -#else - if (!problem_.raman || problem_.general_modal || problem_.dispersive || - problem_.separable_geometry() || problem_.effective_node_pol() != 1 || - problem_.n_pol_state != 1 || problem_.n_pol_transform != 1 || - problem_.raman_drives() != 1 || problem_.n_nodes < 2) - return 0; - const auto products = problem_.n_modes * (problem_.n_modes + 1) / 2; - return products < problem_.n_nodes && products <= problem_.node_chunk ? products : 0; -#endif -} - -void GeneralWorkspace::convolve_raman_products() { - Kokkos::Profiling::pushRegion("solprop_native.raman.products"); - const auto drive = raman_product_drive_; - const auto modal_real = modal_real_; - const auto modal_complex = modal_complex_; - const auto modes = problem_.n_modes; - const auto n_to = problem_.n_to; - const bool real_family = problem_.family == FieldFamily::field_real; - const bool squared = problem_.raman_thg; - launch::parallel_for( - "solprop_native.raman.products", - Range2(raman_product_space_, {0, 0}, {raman_products_, 2 * n_to}), - KOKKOS_LAMBDA(const std::size_t product, const std::size_t time) { - if (time >= n_to) { - drive(product, time) = 0.0; - return; - } - // The unordered pair (first, second), first <= second, in row-major order. - std::size_t first = 0; - std::size_t second = product; - while (second >= modes - first) { - second -= modes - first; - ++first; - } - second += first; - const double multiplicity = first == second ? 1.0 : 2.0; - double value; - if (real_family) { - value = modal_real(first, time) * modal_real(second, time); - if (!squared) - value *= 0.5; - } else { - const Complex a = modal_complex(first, time); - const Complex b = modal_complex(second, time); - value = squared ? a.real() * b.real() - : 0.5 * (a.real() * b.real() + a.imag() * b.imag()); - } - drive(product, time) = multiplicity * value; - }); - execute_raman(*raman_product_forward_plan_, raman_product_drive_, - raman_product_spectrum_, KokkosFFT::Normalization::none); - const auto spectrum = raman_product_spectrum_; - const auto kernel_spectrum = raman_kernel_spectrum_; - const double inverse_doubled = 1.0 / static_cast(2 * n_to); - launch::parallel_for( - "solprop_native.raman.products_multiply", - Range2(raman_product_space_, {0, 0}, {raman_products_, n_to + 1}), - KOKKOS_LAMBDA(const std::size_t product, const std::size_t index) { - spectrum(product, index) = - (spectrum(product, index) * kernel_spectrum(0, index)) * inverse_doubled; - }); - execute_raman(*raman_product_inverse_plan_, raman_product_spectrum_, - raman_product_drive_, KokkosFFT::Normalization::none); - Kokkos::Profiling::popRegion(); -} - -// `finish_raman`'s scalar form with each node's convolved drive reconstructed from -// the convolved modal products. -void GeneralWorkspace::finish_raman_products(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count) { - Kokkos::Profiling::pushRegion("solprop_native.raman.finish_products"); - const auto node_real = node_real_; - const auto node_complex = node_complex_; - const auto out_real = polarisation_real_; - const auto out_complex = polarisation_complex_; - const auto convolved = raman_product_drive_; - const auto synthesis = scaled_synthesis_; - const auto window = time_window_; - const auto density = stage_raman_density_; - const auto family = problem_.family; - const auto modes = problem_.n_modes; - const double epsilon0 = problem_.raman_epsilon0; - const auto n_to = problem_.n_to; - // Blocks of samples per node, the products accumulated in their order over each - // block as contiguous rows: one pass over the block per product, which vectorises, - // where a per-sample loop over the products strides between rows. - constexpr std::size_t block = 512; - const std::size_t blocks = (n_to + block - 1) / block; - const double* const products = convolved.data(); - const std::size_t stride = convolved.stride(0); - launch::parallel_for( - "solprop_native.raman.finish_products", - Kokkos::RangePolicy(response_space_for(node_count), 0, - node_count * blocks), - KOKKOS_LAMBDA(const std::size_t item) { - const std::size_t node = item / blocks; - const std::size_t begin = (item % blocks) * block; - const std::size_t count = n_to - begin < block ? n_to - begin : block; - const std::size_t row = first_node + node; - double drive[block]; - for (std::size_t offset = 0; offset < count; ++offset) - drive[offset] = 0.0; - std::size_t product = 0; - for (std::size_t first = 0; first < modes; ++first) { - const double weight = synthesis(row, first); - for (std::size_t second = first; second < modes; ++second, ++product) { - const double pair = weight * synthesis(row, second); - const double* const values = products + product * stride + begin; - for (std::size_t offset = 0; offset < count; ++offset) - drive[offset] += pair * values[offset]; - } - } - const double scale = epsilon0 * density(stage); - for (std::size_t offset = 0; offset < count; ++offset) { - const std::size_t time = begin + offset; - const double factor = scale * drive[offset]; - if (family == FieldFamily::envelope) { - out_complex(node, time) = - (out_complex(node, time) + factor * node_complex(node, time)) * - window(time); - } else { - const double field = family == FieldFamily::field_real - ? node_real(node, time) - : node_complex(node, time).real(); - out_real(node, time) = - (out_real(node, time) + factor * field) * window(time); - } - } - }); - Kokkos::Profiling::popRegion(); -} - -// One channel's cycle-averaged rates at each node and their running integral, from -// the polarisation ellipse of `field_rows`' `n_pol` rows per node: the node buffer -// with its components, or an envelope's on-axis row (`axis_populations`). -void GeneralWorkspace::envelope_cycle(const std::size_t channel, - const std::size_t node_count, - const ComplexView2& field_rows, - const std::size_t n_pol) { - // The kernels capture the view by value, never the caller's reference, which a - // device cannot read. - // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) - const auto field = field_rows; - const auto parameters = channel_parameters_; - const auto coefficients = channel_coefficients_; - const auto nodes = - Kokkos::subview(envelope_rule_nodes_, channel, Kokkos::ALL(), Kokkos::ALL()); - const auto weights = - Kokkos::subview(envelope_rule_weights_, channel, Kokkos::ALL(), Kokkos::ALL()); - const auto moments = envelope_moments_; - const auto rates = envelope_rates_; - const auto failures = envelope_failures_; - const auto n_to = problem_.n_to; - launch::parallel_for( - "solprop_native.envelope.cycle_moments", - Range2(response_space_for(node_count), {0, 0}, {node_count, n_to}), - KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { - const auto axes = - envelope_axes(field(node * n_pol, time), - n_pol == 2 ? field(node * n_pol + 1, time) : Complex{}); - const auto rate_at = [&](const double value) { - return ionisation_rate(value, coefficients, parameters(channel, 0), - parameters(channel, 1), parameters(channel, 2), - parameters(channel, 3), parameters(channel, 4), - static_cast(parameters(channel, 5)), - static_cast(parameters(channel, 6))); - }; - const auto pair = - envelope_moments(axes.major, axes.ratio, parameters(channel, 2), - parameters(channel, 3), rate_at, nodes, weights); - moments(0, node, time) = pair.cosine; - moments(1, node, time) = pair.sine; - rates(node, time) = pair.cosine + axes.ratio * axes.ratio * pair.sine; - failures(node * n_to + time) = pair.failed ? 1 : 0; - }); - const auto status = envelope_status_; - std::size_t no_failure = 0; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // A serial device scan dominated the short combined Raman/plasma profiles. - // Integer minimum preserves the first failing sample exactly; - // its identity denotes success without an additional normalization launch. - no_failure = std::numeric_limits::max(); - using StatusPolicy = Kokkos::TeamPolicy; - using StatusMember = StatusPolicy::member_type; - const auto samples = node_count * n_to; - // One team uses only launch-time shared memory; a global reduction could grow - // the execution-space scratch allocator inside the first trial. - launch::parallel_for( - "solprop_native.envelope.angular_status", - StatusPolicy(response_space_for(node_count), 1, - SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE), - KOKKOS_LAMBDA(const StatusMember& team) { - std::size_t first_failure = Kokkos::reduction_identity::min(); - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, samples), - [&](const std::size_t sample, std::size_t& minimum) { - if (failures(sample) != 0) - minimum = Kokkos::min(minimum, sample + 1); - }, - Kokkos::Min(first_failure)); - Kokkos::single(Kokkos::PerTeam(team), [&]() { status(0) = first_failure; }); - }); -#else - launch::parallel_for( - "solprop_native.envelope.angular_status", - Kokkos::RangePolicy(response_space_for(node_count), 0, 1), - KOKKOS_LAMBDA(const std::size_t) { - status(0) = 0; - for (std::size_t sample = 0; sample < node_count * n_to; ++sample) - if (failures(sample) != 0) { - status(0) = sample + 1; - break; - } - }); -#endif - // This planned scalar boundary rejects failed refinement before a current FFT. - graph::deep_copy(response_space_for(node_count), envelope_host_status_, - envelope_status_); - response_space_for(node_count).fence("solprop_native envelope refinement boundary"); - if (envelope_host_status_(0) != no_failure) - throw EnvelopeConvergenceError("envelope plasma channel " + - std::to_string(channel) + - " exceeded the angular refinement budget"); - const auto rows = std::make_pair(std::size_t{0}, node_count); - portable_team_trapezoid_scan( - response_space_for(node_count), - Kokkos::subview(envelope_rates_, rows, Kokkos::ALL()), - Kokkos::subview(envelope_exposure_, rows, Kokkos::ALL()), problem_.dt_fine); -} - -// One channel's exposure and peak cycle-averaged rate at each node of `field` -// (`envelope_cycle`), into the rows of `statistics` from `first_node`. -void GeneralWorkspace::envelope_statistics(const std::size_t channel, - const std::size_t first_node, - const std::size_t node_count, - const RealView2& statistics, - const ComplexView2& field, - const std::size_t n_pol) { - envelope_cycle(channel, node_count, field, n_pol); - const auto rates = envelope_rates_; - const auto exposure = envelope_exposure_; - const auto n_to = problem_.n_to; - launch::parallel_for( - "solprop_native.envelope.population_statistics", - Kokkos::RangePolicy(response_space_for(node_count), 0, - node_count), - KOKKOS_LAMBDA(const std::size_t node) { - double peak = 0.0; - for (std::size_t time = 0; time < n_to; ++time) - peak = Kokkos::max(peak, rates(node, time)); - statistics(first_node + node, 2 * channel) = exposure(node, n_to - 1); - statistics(first_node + node, 2 * channel + 1) = peak; - }); -} - -void GeneralWorkspace::envelope_population(const std::size_t channel, - const std::size_t stage, - const std::size_t node_count) { - const auto n_pol = problem_.effective_node_pol(); - envelope_cycle(channel, node_count, node_complex_, n_pol); - const auto exposure = envelope_exposure_; - const auto n_to = problem_.n_to; - const auto parameters = channel_parameters_; - const auto densities = stage_channel_density_; - const auto initials = stage_channel_preionisation_; - const auto moments = envelope_moments_; - const auto field = node_complex_; - const auto window = time_window_; - const auto drive = envelope_streams_[0].node; - const auto loss = envelope_streams_[1].node; - launch::parallel_for( - "solprop_native.envelope.currents", - Range2(response_space_for(node_count), {0, 0}, {node_count, n_to}), - KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { - const double initial = initials(stage, channel); - const double fraction = envelope_fraction(exposure(node, time), initial); - const double survival = (1.0 - initial) * Kokkos::exp(-exposure(node, time)); - const double density = densities(stage, channel); - const auto axes = - envelope_axes(field(node * n_pol, time), - n_pol == 2 ? field(node * n_pol + 1, time) : Complex{}); - const auto current = - envelope_loss(axes, moments(0, node, time), moments(1, node, time), - parameters(channel, 8) * density * survival); - const double drude = parameters(channel, 7) * density * fraction; - for (std::size_t pol = 0; pol < n_pol; ++pol) { - const auto row = node * n_pol + pol; - const auto drive_value = window(time) * drude * field(row, time); - const auto loss_value = window(time) * current[pol]; - if (channel == 0) { - drive(row, time) = drive_value; - loss(row, time) = loss_value; - } else { - drive(row, time) += drive_value; - loss(row, time) += loss_value; - } - } - }); -} - -void GeneralWorkspace::envelope_response(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count, - const TransverseChunk* chunk) { - for (std::size_t channel = 0; channel < problem_.n_channels; ++channel) - envelope_population(channel, stage, node_count); - for (auto& stream : envelope_streams_) { - if (problem_.slab_geometry()) { - // As the polarisation: the chunk's y rows, zero beyond a final partial chunk. - graph::deep_copy(response_space_for(node_count), stream.projected, Complex{}); - slab_project(*chunk, &stream); - } else if (problem_.free_geometry()) { - // The persistent maximum-chunk plan transforms zero padding in a final - // partial chunk; only its actual rows are ever copied to the spatial FFT. - graph::deep_copy(response_space_for(node_count), stream.projected, Complex{}); - const auto rows = - std::make_pair(std::size_t{0}, node_count * problem_.effective_node_pol()); - graph::deep_copy(response_space_for(node_count), - Kokkos::subview(stream.projected, rows, Kokkos::ALL()), - Kokkos::subview(stream.node, rows, Kokkos::ALL())); - } else { - project(first_node, node_count, first_node == 0, &stream); - } - } -} - -// SEP 27 forms two independently collared current products. Every maximum-chunk -// row is written, including a partial chunk's padding, before a persistent FFT plan -// can read it. Modal projection precedes the FFT; radial projection follows it. -void GeneralWorkspace::held_response(const std::size_t stage, - const std::size_t first_node, - const std::size_t node_count, - const TransverseChunk* chunk) { - const auto current = held_current_; - const auto window = time_window_; - const auto derivative = held_derivative_; - const auto product = held_streams_[0].node; - const auto edge = held_streams_[1].node; - const auto live = node_count * problem_.effective_node_pol(); - const auto n_to = problem_.n_to; - cartesian_for( - "solprop_native.held.products", response_space_for(node_count), current.extent(0), - n_to, KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { - const double value = row < live ? current(row, time) : 0.0; - product(row, time) = window(time) * value; - edge(row, time) = derivative(time) * value; - }); - const bool folded = folds_single_node(); - const bool radial = held_radial(); - for (std::size_t index = 0; index < 2; ++index) { - auto& stream = held_streams_[index]; - if (folded) { - const auto source = stream.node; - const auto target = stream.projected; - const auto projection = projection_; - const auto taper = stage_taper_; - cartesian_for( - "solprop_native.held.folded_projection", response_space_for(node_count), 1, - n_to, KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { - target(row, time) = projection(0, 0) / taper(stage) * source(row, time); - }); - } else if (problem_.slab_geometry()) { - graph::deep_copy(response_space_for(node_count), stream.projected, 0.0); - slab_contract("solprop_native.held.slab_projection", - response_space_for(node_count), projection_, stream.node, - stream.projected, chunk->nodes / problem_.n_x, n_to); - } else if (problem_.free_geometry() || radial) { - graph::deep_copy(response_space_for(node_count), stream.projected, stream.node); - } else { - project(first_node, node_count, first_node == 0, nullptr, &stream); - } - } - if (radial) { - const auto source = polarisation_real_; - const auto target = held_streams_[2].projected; - cartesian_for( - "solprop_native.held.nonplasma", response_space_for(node_count), - target.extent(0), n_to, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { - target(row, time) = row < live ? source(row, time) : 0.0; - }); - held_radial_projection(first_node, node_count); - } -} - -// solprop's private gated benchmark (`NumpyRefBackend._column_sources`): the chunk's -// live current rows are integrated by the problem's running-integral rule, the band -// rule's filter then the trapezoid (solprop SEP 18), and w(t) times each prefix joins -// the local polarisation, which is then projected as the held current's streams would -// have been. The scans are the device column's portable primitives; no hot column -// changes, and a run without the benchmark never reaches this. -void GeneralWorkspace::gated_response(const std::size_t node_count) { - const Kokkos::Profiling::ScopedRegion region("solprop_native.gated.response"); - const auto live = std::pair( - 0, node_count * problem_.effective_node_pol()); - const auto current = Kokkos::subview(held_current_, live, Kokkos::ALL); - const auto prefix = Kokkos::subview(gated_prefix_, live, Kokkos::ALL); - const auto& space = response_space_for(node_count); - if (problem_.plasma_band) - portable_team_band_scan(space, current, - Kokkos::subview(gated_filtered_, live, Kokkos::ALL), prefix, - plasma_taps_, problem_.dt_fine); - else - portable_team_trapezoid_scan(space, current, prefix, problem_.dt_fine); - const auto window = time_window_; - const auto polarisation = polarisation_real_; - cartesian_for( - "solprop_native.gated.polarisation", space, live.second, problem_.n_to, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { - polarisation(row, time) += window(time) * prefix(row, time); - }); -} - -// A stage's forward rows in one transform: all of them when the absorber's are due, -// else those after the absorber's (or the projection's alone). A host deferring its -// quadrature transforms the fine block only; `finish_quadrature` takes the coarse. -void GeneralWorkspace::transform_forward_pack() { - const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.forward_pack"); - const bool fine = defer_quadrature_ && forward_pack_fine_plan_; - if (absorbing() || forward_pack_absorber_rows_ == 0) { - if (fine) - graph::execute(*forward_pack_fine_plan_, forward_pack_fine_, - forward_pack_fine_spectrum_, KokkosFFT::Normalization::none); - else - graph::execute(*forward_pack_plan_, forward_pack_, forward_pack_spectrum_, - KokkosFFT::Normalization::none); - } else if (fine ? static_cast(forward_pack_fine_tail_plan_) - : static_cast(forward_pack_tail_plan_)) { - if (fine) - graph::execute(*forward_pack_fine_tail_plan_, forward_pack_fine_tail_, - forward_pack_fine_tail_spectrum_, KokkosFFT::Normalization::none); - else - graph::execute(*forward_pack_tail_plan_, forward_pack_tail_, - forward_pack_tail_spectrum_, KokkosFFT::Normalization::none); - } else { - graph::execute(*forward_real_plan_, projected_real_, projected_spectrum_, - KokkosFFT::Normalization::none); - } -} - -void GeneralWorkspace::held_transform(const bool coarse_only, const bool transformed) { - const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.held_streams"); - // The packed rows are every transform this call takes when it takes the fine ones - // and, where there are coarse rows, those too. The stage's forward pack, when it - // took them, has transformed the streams' rows already. - const bool every_row = - !coarse_only && (!held_streams_[0].coarse_forward || !defer_quadrature_); - if (held_pack_forward_ && every_row) { - graph::execute(*held_pack_forward_, held_pack_, held_pack_spectrum_, - KokkosFFT::Normalization::none); - } else if (!transformed) { - for (std::size_t index = 0; index < 2; ++index) { - auto& stream = held_streams_[index]; - if (!coarse_only) - graph::execute(*stream.forward, stream.projected, stream.spectrum, - KokkosFFT::Normalization::none); - if (stream.coarse_forward && (coarse_only || !defer_quadrature_)) - graph::execute(*stream.coarse_forward, stream.coarse, stream.coarse_spectrum, - KokkosFFT::Normalization::none); - } - } - const auto product = held_streams_[0].spectrum; - const auto edge = held_streams_[1].spectrum; - const auto coarse_product = held_streams_[0].coarse_spectrum; - const auto coarse_edge = held_streams_[1].coarse_spectrum; - const auto projected = projected_spectrum_; - const auto coarse = projected_coarse_spectrum_; - const auto inverse = held_inverse_; - const auto active = active_mask_; - const auto crop = crop_index_; - const bool include_coarse = !coarse.empty() && (coarse_only || !defer_quadrature_); - cartesian_for( - "solprop_native.held.spectral_assembly", execution_space_, projected.extent(0), - problem_.n_omega, KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - const auto bin = crop(omega); - if (active(omega)) { - const double value = inverse(omega); - const Complex factor(0.0, -value); - if (!coarse_only) - projected(row, bin) += - factor * product(row, bin) + value * value * edge(row, bin); - if (include_coarse) - coarse(row, bin) += factor * coarse_product(row, bin) + - value * value * coarse_edge(row, bin); - } else { - if (!coarse_only) - projected(row, bin) = Complex{}; - if (include_coarse) - coarse(row, bin) = Complex{}; - } - }); -} - -void GeneralWorkspace::held_radial_projection(const std::size_t first_node, - const std::size_t node_count) { - if (held_pack_forward_) - graph::execute(*held_pack_forward_, held_pack_, held_pack_spectrum_, - KokkosFFT::Normalization::none); - else - for (auto& stream : held_streams_) - graph::execute(*stream.forward, stream.projected, stream.spectrum, - KokkosFFT::Normalization::none); - const auto product = held_streams_[0].spectrum; - const auto edge = held_streams_[1].spectrum; - const auto polarisation = held_streams_[2].spectrum; - const auto nodal = held_nodal_spectrum_; - const auto inverse = held_inverse_; - const auto active = active_mask_; - const auto crop = crop_index_; - const auto n = problem_.n_omega; - const auto live = node_count * problem_.effective_node_pol(); - cartesian_for( - "solprop_native.held.radial_assembly", node_spectral_space_, nodal.extent(0), n, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - Complex value{}; - if (row < live && active(omega)) { - const auto bin = crop(omega); - const double inv = inverse(omega); - value = polarisation(row, bin) + Complex(0.0, -inv) * product(row, bin) + - inv * inv * edge(row, bin); - } - nodal(row, omega) = value; - }); - const auto packed = held_projection_; - const auto projection = scaled_projection_; - const auto first_row = first_node * problem_.n_pol_transform; - const auto live_rows = node_count * problem_.n_pol_transform; - cartesian_for( - "solprop_native.held.radial_projection_matrix", matrix_copy_space_, - packed.extent(0), packed.extent(1), - KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { - packed(mode, row) = row < live_rows ? projection(mode, first_row + row) : 0.0; - }); - // Interleaved real and imaginary parts share the scalar radial projection. This - // is one GEMM of the combined cropped source, independent of state components. - using Interleaved = Kokkos::View>; - static_assert(sizeof(Complex) == 2 * sizeof(double)); - const auto columns = 2 * problem_.n_pol_state * n; - const Interleaved input(reinterpret_cast(nodal.data()), packed.extent(1), - columns); - const Interleaved output(reinterpret_cast(held_projected_spectrum_.data()), - problem_.n_modes, columns); - selected_gemm(transform_space_for(node_count), packed, input, output, 0.0, true); - if (first_node == 0) - graph::deep_copy(temporal_execution_space_, projected_spectrum_, Complex{}); - const auto source = held_projected_spectrum_; - const auto target = projected_spectrum_; - cartesian_for( - "solprop_native.held.radial_accumulate", execution_space_, target.extent(0), n, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - if (active(omega)) - target(row, crop(omega)) += source(row, omega); - }); -} - -void GeneralWorkspace::envelope_transform() { - for (auto& stream : envelope_streams_) { - graph::execute(*stream.forward, stream.projected, stream.spectrum, - KokkosFFT::Normalization::none); - if (stream.coarse_forward) - graph::execute(*stream.coarse_forward, stream.coarse, stream.coarse_spectrum, - KokkosFFT::Normalization::none); - } - const auto drive = envelope_streams_[0].spectrum; - const auto loss = envelope_streams_[1].spectrum; - const auto coarse_drive = envelope_streams_[0].coarse_spectrum; - const auto coarse_loss = envelope_streams_[1].coarse_spectrum; - const auto projected = projected_spectrum_; - const auto coarse = projected_coarse_spectrum_; - const auto drude_factor = envelope_drude_factor_; - const auto loss_factor = envelope_loss_factor_; - const auto crop = crop_index_; - launch::parallel_for( - "solprop_native.envelope.spectral_assembly", - Range2(execution_space_, {0, 0}, {projected.extent(0), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - const auto index = crop(omega); - projected(column, index) += drude_factor(omega) * drive(column, index) + - loss_factor(omega) * loss(column, index); - if (!coarse.empty()) - coarse(column, index) += drude_factor(omega) * coarse_drive(column, index) + - loss_factor(omega) * coarse_loss(column, index); - }); -} - -void GeneralWorkspace::project(const std::size_t first_node, - const std::size_t node_count, const bool first_chunk, - const EnvelopeStream* stream, - const FieldCurrentStream* field_stream) { - const auto real_source = - field_stream == nullptr ? polarisation_real_ : field_stream->node; - const auto real_target = - field_stream == nullptr ? projected_real_ : field_stream->projected; - const auto real_coarse_target = - field_stream == nullptr ? projected_coarse_real_ : field_stream->coarse; - const auto real_stacked_target = - field_stream == nullptr ? projected_stacked_real_ : field_stream->stacked; - const auto node_source = stream == nullptr ? polarisation_complex_ : stream->node; - const auto projected_target = - stream == nullptr ? projected_complex_ : stream->projected; - const auto coarse_target = - stream == nullptr ? projected_coarse_complex_ : stream->coarse; - const auto stacked_target = - stream == nullptr ? projected_stacked_complex_ : stream->stacked; - const bool whole_transform = - first_node == 0 && node_count == problem_.n_nodes && problem_.n_pol_state == 1; - if (whole_transform && !radial_blas_input_.empty()) { - packed_radial_gemm(scaled_projection_, node_source, projected_target); - return; - } - if (whole_transform && use_interleaved_projection()) { - Kokkos::Profiling::pushRegion( - "solprop_native.transform.projection.kokkosblas_interleaved"); - using Interleaved = Kokkos::View>; - static_assert(sizeof(Complex) == 2 * sizeof(double)); - const Interleaved nodes(reinterpret_cast(node_source.data()), - problem_.n_nodes, 2 * problem_.n_to); - const Interleaved fine(reinterpret_cast(projected_target.data()), - problem_.n_modes, 2 * problem_.n_to); - engine_gemm(transform_space_for(node_count), scaled_projection_, nodes, fine, 0.0); - if (project_coarse_ && !problem_.projection_coarse_matrix.empty()) { - const Interleaved coarse(reinterpret_cast(coarse_target.data()), - problem_.n_modes, 2 * problem_.n_to); - engine_gemm(transform_space_for(node_count), scaled_projection_coarse_, nodes, - coarse, 0.0); - } - Kokkos::Profiling::popRegion(); - return; - } - // The stacked projections' splits copy on the stage's instance: on the temporal - // copy's serial instance one took about 90 us at 16 threads on the four-mode - // capillary, beside full-width kernels (docs/dev/benchmarks.md, "Stacked projection - // splits"). - if (whole_transform && use_blas_for_projection()) { - Kokkos::Profiling::pushRegion("solprop_native.transform.projection.kokkosblas"); - if (problem_.family == FieldFamily::envelope && project_coarse_ && - !scaled_projection_stacked_complex_.empty()) { - selected_gemm(transform_space_for(node_count), scaled_projection_stacked_complex_, - node_source, stacked_target, Complex{}, true); - const auto& stacked = stacked_target; - const auto& fine = projected_target; - const auto& coarse = coarse_target; - const auto columns = problem_.n_columns(); - launch::parallel_for( - "solprop_native.transform.projection.split_stacked_complex", - Range2(execution_space_, {0, 0}, {columns, problem_.n_to}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { - fine(column, time) = stacked(column, time); - coarse(column, time) = stacked(columns + column, time); - }); - } else if (problem_.family == FieldFamily::envelope) { - selected_gemm(transform_space_for(node_count), scaled_projection_complex_, - node_source, projected_target, Complex{}, true); - } else if (one_mode_products_) { - const bool stacked = project_coarse_ && !scaled_projection_stacked_.empty(); - one_mode_projection(response_space_for(node_count), - stacked ? scaled_projection_stacked_ : scaled_projection_, - real_source, real_target, real_coarse_target); - } else if (project_coarse_ && !scaled_projection_stacked_.empty()) { - selected_gemm(transform_space_for(node_count), scaled_projection_stacked_, - real_source, real_stacked_target, 0.0, true); - const auto& stacked = real_stacked_target; - const auto& fine = real_target; - const auto& coarse = real_coarse_target; - const auto columns = problem_.n_columns(); - launch::parallel_for( - "solprop_native.transform.projection.split_stacked", - Range2(execution_space_, {0, 0}, {columns, problem_.n_to}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { - fine(column, time) = stacked(column, time); - coarse(column, time) = stacked(columns + column, time); - }); - } else { - selected_gemm(transform_space_for(node_count), scaled_projection_, real_source, - real_target, 0.0, true); - } - Kokkos::Profiling::popRegion(); - return; - } - if (use_chunked_blas_for_projection()) { - Kokkos::Profiling::pushRegion( - "solprop_native.transform.projection.kokkosblas_chunked"); - const auto rows = node_count * problem_.n_pol_transform; - const auto first_row = first_node * problem_.n_pol_transform; - const bool tail = node_count != problem_.node_chunk; - const bool last_chunk = first_node + node_count == problem_.n_nodes; - const bool stacked = project_coarse_ && !problem_.projection_coarse_matrix.empty(); - if (problem_.family == FieldFamily::envelope) { - const auto polarisation = Kokkos::subview( - node_source, std::make_pair(std::size_t{0}, rows), Kokkos::ALL()); - if (stacked) { - const auto packed = - tail ? tail_projection_stacked_complex_ : chunk_projection_stacked_complex_; - const auto source = scaled_projection_stacked_complex_; - launch::parallel_for( - "solprop_native.transform.pack_projection_stacked_complex", - Range2(matrix_copy_space_for(node_count, true), {0, 0}, - {2 * problem_.n_modes, rows}), - KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { - packed(mode, row) = source(mode, first_row + row); - }); - selected_gemm(transform_space_for(node_count), packed, polarisation, - stacked_target, first_chunk ? Complex{} : Complex{1.0, 0.0}, - true); - if (last_chunk) { - const auto& projected = stacked_target; - const auto& fine = projected_target; - const auto& coarse = coarse_target; - const auto columns = problem_.n_columns(); - launch::parallel_for( - "solprop_native.transform.projection.split_stacked_complex", - Range2(execution_space_, {0, 0}, {columns, problem_.n_to}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { - fine(column, time) = projected(column, time); - coarse(column, time) = projected(columns + column, time); - }); - } - } else { - // The first half of the existing stacked scratch is a contiguous fine - // matrix. Alias it when this trial stage does not need the coarse rows. - const auto scratch = - tail ? tail_projection_stacked_complex_ : chunk_projection_stacked_complex_; - const ComplexView2 packed = - scratch.empty() - ? (tail ? tail_projection_complex_ : chunk_projection_complex_) - : ComplexView2(Kokkos::subview( - scratch, std::pair{std::size_t{0}, problem_.n_modes}, - Kokkos::ALL())); - const auto source = scaled_projection_complex_; - launch::parallel_for( - "solprop_native.transform.pack_projection_complex", - Range2(matrix_copy_space_for(node_count), {0, 0}, {problem_.n_modes, rows}), - KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { - packed(mode, row) = source(mode, first_row + row); - }); - selected_gemm(transform_space_for(node_count), packed, polarisation, - projected_target, first_chunk ? Complex{} : Complex{1.0, 0.0}, - true); - } - } else { - const auto polarisation = Kokkos::subview( - real_source, std::make_pair(std::size_t{0}, rows), Kokkos::ALL()); - if (stacked) { - const auto packed = tail ? tail_projection_stacked_ : chunk_projection_stacked_; - const auto source = scaled_projection_stacked_; - launch::parallel_for( - "solprop_native.transform.pack_projection_stacked", - Range2(matrix_copy_space_for(node_count, true), {0, 0}, - {2 * problem_.n_modes, rows}), - KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { - packed(mode, row) = source(mode, first_row + row); - }); - selected_gemm(transform_space_for(node_count), packed, polarisation, - real_stacked_target, first_chunk ? 0.0 : 1.0, true); - if (last_chunk) { - const auto& projected = real_stacked_target; - const auto& fine = real_target; - const auto& coarse = real_coarse_target; - const auto columns = problem_.n_columns(); - launch::parallel_for( - "solprop_native.transform.projection.split_stacked", - Range2(execution_space_, {0, 0}, {columns, problem_.n_to}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { - fine(column, time) = projected(column, time); - coarse(column, time) = projected(columns + column, time); - }); - } - } else { - const auto scratch = - tail ? tail_projection_stacked_ : chunk_projection_stacked_; - const RealView2 packed = - scratch.empty() - ? (tail ? tail_projection_ : chunk_projection_) - : RealView2(Kokkos::subview(scratch, - std::pair{std::size_t{0}, problem_.n_modes}, - Kokkos::ALL())); - const auto source = scaled_projection_; - launch::parallel_for( - "solprop_native.transform.pack_projection", - Range2(matrix_copy_space_for(node_count), {0, 0}, {problem_.n_modes, rows}), - KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { - packed(mode, row) = source(mode, first_row + row); - }); - selected_gemm(transform_space_for(node_count), packed, polarisation, - real_target, first_chunk ? 0.0 : 1.0, true); - } - } - Kokkos::Profiling::popRegion(); - return; - } - Kokkos::Profiling::pushRegion("solprop_native.transform.projection.custom"); - const auto projection = scaled_projection_; - const auto projection_coarse = scaled_projection_coarse_; - const auto& polar_real = real_source; - const auto& polar_complex = node_source; - const auto& projected_real = real_target; - const auto& projected_complex = projected_target; - const auto& projected_coarse_real = real_coarse_target; - const auto& projected_coarse_complex = coarse_target; - const auto n_pol_state = problem_.n_pol_state; - const auto n_pol_transform = problem_.n_pol_transform; - const auto n_to = problem_.n_to; - const bool envelope = problem_.family == FieldFamily::envelope; - const bool coarse = project_coarse_ && !projection_coarse.empty(); - launch::parallel_for( - "solprop_native.transform.projection.custom", - Range2(transform_space_for(node_count), {0, 0}, {problem_.n_columns(), n_to}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { - const std::size_t mode = column / n_pol_state; - const std::size_t state_pol = column % n_pol_state; - if (envelope) { - Complex fine{}; - Complex rough{}; - for (std::size_t node = 0; node < node_count; ++node) { - for (std::size_t transform_pol = 0; transform_pol < n_pol_transform; - ++transform_pol) { - const std::size_t global_row = - (first_node + node) * n_pol_transform + transform_pol; - const std::size_t local_row = - (node * n_pol_transform + transform_pol) * n_pol_state + state_pol; - const Complex value = polar_complex(local_row, time); - fine += projection(mode, global_row) * value; - if (coarse) - rough += projection_coarse(mode, global_row) * value; - } - } - if (first_chunk) { - projected_complex(column, time) = fine; - if (coarse) - projected_coarse_complex(column, time) = rough; - } else { - projected_complex(column, time) += fine; - if (coarse) - projected_coarse_complex(column, time) += rough; - } - } else { - double fine = 0.0; - double rough = 0.0; - for (std::size_t node = 0; node < node_count; ++node) { - for (std::size_t transform_pol = 0; transform_pol < n_pol_transform; - ++transform_pol) { - const std::size_t global_row = - (first_node + node) * n_pol_transform + transform_pol; - const std::size_t local_row = - (node * n_pol_transform + transform_pol) * n_pol_state + state_pol; - const double value = polar_real(local_row, time); - fine += projection(mode, global_row) * value; - if (coarse) - rough += projection_coarse(mode, global_row) * value; - } - } - if (first_chunk) { - projected_real(column, time) = fine; - if (coarse) - projected_coarse_real(column, time) = rough; - } else { - projected_real(column, time) += fine; - if (coarse) - projected_coarse_real(column, time) += rough; - } - } - }); - Kokkos::Profiling::popRegion(); -} - -void GeneralWorkspace::form_absorber() { - const bool spatial = !problem_.spatial_absorber_matrix.empty(); - const auto modes = problem_.n_modes; - const auto polarisations = problem_.n_pol_state; - // Flatten (polarisation,time) together: the same scalar radial matrix acts - // independently on each component. The views alias existing FFT work buffers. - if (spatial && problem_.family == FieldFamily::envelope) { - using Matrix = Kokkos::View>; - const Matrix input(modal_complex_.data(), modes, polarisations * problem_.n_to); - const Matrix output(absorber_complex_.data(), modes, polarisations * problem_.n_to); - selected_gemm(absorber_execution_space_, spatial_absorber_complex_, input, output, - Complex{}, true); - } else if (spatial && problem_.family == FieldFamily::field_real) { - using Matrix = Kokkos::View>; - const Matrix input(modal_real_.data(), modes, polarisations * problem_.n_to); - const Matrix output(absorber_real_.data(), modes, polarisations * problem_.n_to); - selected_gemm(absorber_execution_space_, spatial_absorber_real_, input, output, 0.0, - true); - } - const auto rate = absorber_fine_; - const auto spatial_matrix = spatial_absorber_real_; - const auto modal_real = modal_real_; - const auto modal_complex = modal_complex_; - const auto absorber_real = absorber_real_; - const auto absorber_complex = absorber_complex_; - const auto family = problem_.family; - launch::parallel_for( - "solprop_native.absorber.nonlinear_term", - Range2(temporal_execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_to}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { - if (family == FieldFamily::envelope) { - absorber_complex(column, time) = - (spatial ? absorber_complex(column, time) : Complex{}) - - 0.5 * rate(time) * modal_complex(column, time); - } else if (family == FieldFamily::field_real) { - absorber_real(column, time) = (spatial ? absorber_real(column, time) : 0.0) - - 0.5 * rate(time) * modal_real(column, time); - } else { - double transverse = 0.0; - if (spatial) - for (std::size_t source = 0; source < modes; ++source) - transverse += - spatial_matrix(column / polarisations, source) * - modal_complex(source * polarisations + column % polarisations, time) - .real(); - absorber_real(column, time) = - transverse - 0.5 * rate(time) * modal_complex(column, time).real(); - } - }); -} - -void GeneralWorkspace::finish_stage(const std::size_t stage) { - const auto terms = stage_terms(stage); - const double solution_weight = problem_.solution_weights[stage]; - const double error_weight = problem_.error_weights[stage]; -#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - // The host's blocks of bins, as the stage's entry kernel takes them. - if (terms.blocks_vectorise()) { - const auto n = problem_.n_omega; - const std::size_t blocks = (n + stage_block - 1) / stage_block; - launch::parallel_for( - "solprop_native.stage.finish_accumulate", - Kokkos::RangePolicy(execution_space_, 0, - problem_.n_columns() * blocks), - KOKKOS_LAMBDA(const std::size_t item) { - const std::size_t first = (item % blocks) * stage_block; - terms.finish_block(stage, solution_weight, error_weight, item / blocks, first, - n - first < stage_block ? n - first : stage_block); - }); - return; - } -#endif - launch::parallel_for( - "solprop_native.stage.finish_accumulate", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - terms.finish(stage, solution_weight, error_weight, column, index); - }); -} - -void GeneralWorkspace::set_flow_actions(const std::vector& forward, - const std::vector& inverse, - const std::vector& identity) { - require_open(); - if (!flow_) - throw std::logic_error("this state carries no coupled flow"); - flow_->set_actions(forward, inverse, identity); -} - -std::vector GeneralWorkspace::adams_block_to_host() const { - require_adams(); - if (adams_block_.extent(0) == 0) - throw std::logic_error("the Adams history holds no coupled block"); - std::vector values(adams_block_.size()); - const auto mirror = - Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, adams_block_); - std::copy(mirror.data(), mirror.data() + mirror.size(), values.begin()); - return values; -} - -std::vector GeneralWorkspace::adams_exponent_to_host() const { - require_adams(); - std::vector values(adams_exponent_.size()); - const auto mirror = - Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, adams_exponent_); - std::copy(mirror.data(), mirror.data() + mirror.size(), values.begin()); - return values; -} - -std::vector GeneralWorkspace::flow_actions_to_host(const bool inverse) const { - require_open(); - if (!flow_) - throw std::logic_error("this state carries no coupled flow"); - return flow_->actions_to_host(inverse); -} - -std::vector -GeneralWorkspace::form_flow_actions(const std::size_t section, const double dz, - const std::vector& fractions) { - require_open(); - if (!flow_) - throw std::logic_error("this state carries no coupled flow"); - return flow_->form_actions(section, dz, fractions); -} - -std::vector GeneralWorkspace::weighted_error(const std::vector& scale) { - require_open(); - if (!problem_.weighted_controller) - throw std::logic_error("this state has no weighted controller workspace"); - require_extent(scale.size(), problem_.n_omega, "weighted controller scale"); - copy_vector(execution_space_, weighted_scale_, scale); - const auto residual = residual_; - const auto in_norm = norm_mask_; - const auto native_scale = weighted_scale_; - const auto partials = weighted_partials_; - const auto n = problem_.n_omega; - using TeamPolicy = Kokkos::TeamPolicy; - using TeamMember = TeamPolicy::member_type; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const int team_size = SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE; -#else - // A host team per column, sharing the threads when the columns are fewer. - const int team_size = - host_reduction_team_size(execution_space_, problem_.n_columns(), n); -#endif - // On a device each column is `reduction_slots_` teams over its own bins; the - // caller sums the slots. - const auto slots = reduction_slots_; - const TeamPolicy policy(execution_space_, - static_cast(problem_.n_columns() * slots), team_size); - launch::parallel_for( - "solprop_native.step.weighted_error_partials", policy, - KOKKOS_LAMBDA(const TeamMember& team) { - const auto league = static_cast(team.league_rank()); - const auto column = league / slots; - std::size_t first = 0; - std::size_t end = 0; - slot_range(n, slots, league % slots, first, end); - double sum = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(first), - static_cast(end)), - [&](const int index_value, double& update) { - const auto index = static_cast(index_value); - if (in_norm(index) == 0U) - return; - const Complex value = residual(column, index) / native_scale(index); - update += squared_abs(value); - }, - sum); - Kokkos::single(Kokkos::PerTeam(team), [&] { partials(league) = sum; }); - }); - execution_space_.fence("solprop_native weighted-error boundary"); - const auto host = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, partials); - std::vector result(problem_.n_columns() * slots); - for (std::size_t slot = 0; slot < result.size(); ++slot) - result[slot] = host(slot); - return result; -} - -void GeneralWorkspace::fill_statistics_time(const bool windowed, const double lower, - const double upper, const bool weighted) { - if (omega_.empty()) - throw std::logic_error("native statistics need the coarse frequency axis"); - graph::deep_copy(execution_space_, statistics_spectrum_, Complex{}); - const auto current = current_; - const auto spectrum = statistics_spectrum_; - const auto omega = omega_; - const auto power_weight = statistics_power_weight_; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const bool envelope = problem_.family == FieldFamily::envelope; - launch::parallel_for( - "solprop_native.statistics.prepare_analytic", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - if (windowed && !(omega(index) >= lower && omega(index) < upper)) - return; - const double analytic_scale = - envelope || index == 0 || index + 1 == n ? 1.0 : 2.0; - spectrum(column, index) = - analytic_scale / static_cast(n_t) * current(column, index); - if (weighted) - spectrum(column, index) *= power_weight(column, index); - }); - graph::execute(*statistics_inverse_plan_, statistics_spectrum_, statistics_time_, - KokkosFFT::Normalization::none); -} - -void GeneralWorkspace::fill_statistics_pair(const bool windowed, const double lower, - const double upper) { - if (!statistics_unit_weight_) { - fill_statistics_time(windowed, lower, upper, true); - graph::deep_copy(execution_space_, statistics_weighted_time_, statistics_time_); - } - fill_statistics_time(windowed, lower, upper); -} - -void GeneralWorkspace::form_statistics_power() { - const auto time_field = statistics_time_; - const auto weighted_field = statistics_weighted_time_; - const bool unit_weight = statistics_unit_weight_; - const auto power = statistics_power_; - const auto n_pol = problem_.n_pol_state; - launch::parallel_for( - "solprop_native.statistics.power", - Range2(execution_space_, {0, 0}, {problem_.n_modes, problem_.n_t}), - KOKKOS_LAMBDA(const std::size_t mode, const std::size_t time) { - double sum = 0.0; - for (std::size_t pol = 0; pol < n_pol; ++pol) - sum += unit_weight ? squared_abs(time_field(mode * n_pol + pol, time)) - : (time_field(mode * n_pol + pol, time) * - Kokkos::conj(weighted_field(mode * n_pol + pol, time))) - .real(); - power(mode, time) = sum; - }); -} - -void GeneralWorkspace::form_statistics_total_power() { - const auto power = statistics_power_; - const auto total = statistics_total_power_; - const auto modes = problem_.n_modes; - launch::parallel_for( - "solprop_native.statistics.total_power", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_t), - KOKKOS_LAMBDA(const std::size_t time) { - double sum = 0.0; - for (std::size_t mode = 0; mode < modes; ++mode) - sum += power(mode, time); - total(time) = sum; - }); -} - -void GeneralWorkspace::scan_plasma_statistics(const std::size_t first_node, - const std::size_t node_count, - const RealView2& output, - const bool diagnose, - const double preionisation) { - if (problem_.envelope_plasma) { - if (diagnose) - throw std::invalid_argument("the plasma integration diagnostics read the " - "field-grid plasma column, and this problem " - "carries none"); - for (std::size_t channel = 0; channel < problem_.n_channels; ++channel) - envelope_statistics(channel, first_node, node_count, output, node_complex_, - problem_.effective_node_pol()); - return; - } - // solprop SEP 18: the band rule's statistics, and every problem's run diagnostics, - // come from the band scan; the trapezoid's own scan below is unchanged. - if (problem_.plasma_band || diagnose) { - if (diagnose) - team_band_statistics(first_node, node_count, output, preionisation); - else - team_band_statistics(first_node, node_count, output, preionisation); - return; - } - const auto node_real = node_real_; - const auto node_complex = node_complex_; - const bool real_family = problem_.family == FieldFamily::field_real; - const auto coefficients = rate_coefficients_; - const auto n_pol = problem_.effective_node_pol(); - const auto n_to = problem_.n_to; - const auto intervals = problem_.rate_intervals; - const auto dt = problem_.dt_fine; - const auto log_field_min = problem_.log_field_min; - const auto d_log_field = problem_.d_log_field; - const auto field_min = problem_.field_min; - const auto field_max = problem_.field_max; - const auto rate_max = problem_.rate_max; - if (problem_.n_channels > 1) { - const auto parameters = channel_parameters_; - const auto channel_coefficients = channel_coefficients_; - const auto channels = problem_.n_channels; - launch::parallel_for( - "solprop_native.plasma.channel_statistics_scan", - Kokkos::RangePolicy(response_space_for(node_count), 0, - node_count), - KOKKOS_LAMBDA(const std::size_t node) { - for (std::size_t channel = 0; channel < channels; ++channel) { - double exposure = 0.0; - double previous_rate = 0.0; - double largest_rate = 0.0; - for (std::size_t time = 0; time < n_to; ++time) { - const double first = - (real_family ? node_real(node * n_pol, time) - : node_complex(node * n_pol, time).real()); - const double second = - n_pol == 2 - ? (real_family ? node_real(node * n_pol + 1, time) - : node_complex(node * n_pol + 1, time).real()) - : 0.0; - const double magnitude = - n_pol == 2 ? Kokkos::hypot(first, second) : Kokkos::abs(first); - const double rate = ionisation_rate( - magnitude, channel_coefficients, parameters(channel, 0), - parameters(channel, 1), parameters(channel, 2), - parameters(channel, 3), parameters(channel, 4), - static_cast(parameters(channel, 5)), - static_cast(parameters(channel, 6))); - if (time > 0) - exposure += trapezoid(rate, previous_rate, dt); - previous_rate = rate; - if (rate > largest_rate) - largest_rate = rate; - } - output(first_node + node, 2 * channel) = exposure; - output(first_node + node, 2 * channel + 1) = largest_rate; - } - }); - return; - } -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const auto rates = plasma_rate_; - const auto exposures = plasma_exposure_; - using TeamPolicy = Kokkos::TeamPolicy; - using TeamMember = TeamPolicy::member_type; - const TeamPolicy policy(response_space_for(node_count), static_cast(node_count), - static_cast(response_team_size())); - launch::parallel_for( - "solprop_native.plasma.device_statistics_scan", policy, - KOKKOS_LAMBDA(const TeamMember& team) { - const auto node = static_cast(team.league_rank()); - const auto base = node * n_pol; - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, static_cast(n_to)), - [&](const int time_value) { - const auto time = static_cast(time_value); - const double first = (real_family ? node_real(base, time) - : node_complex(base, time).real()); - const double second = - n_pol == 2 ? (real_family ? node_real(base + 1, time) - : node_complex(base + 1, time).real()) - : 0.0; - const double magnitude = - n_pol == 2 ? Kokkos::hypot(first, second) : Kokkos::abs(first); - rates(node, time) = - ionisation_rate(magnitude, coefficients, log_field_min, d_log_field, - field_min, field_max, rate_max, intervals); - }); - team.team_barrier(); - Kokkos::parallel_scan( - Kokkos::TeamThreadRange(team, static_cast(n_to)), - [&](const int time_value, double& update, const bool final) { - const auto time = static_cast(time_value); - if (time != 0) - update += trapezoid(rates(node, time), rates(node, time - 1), dt); - if (final) - exposures(node, time) = update; - }); - double largest_rate = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n_to)), - [&](const int time_value, double& largest) { - const auto time = static_cast(time_value); - if (rates(node, time) > largest) - largest = rates(node, time); - }, - Kokkos::Max(largest_rate)); - Kokkos::single(Kokkos::PerTeam(team), [&]() { - output(first_node + node, 0) = exposures(node, n_to - 1); - output(first_node + node, 1) = largest_rate; - }); - }); -#else - launch::parallel_for( - "solprop_native.plasma.statistics_scan", - Kokkos::RangePolicy(response_space_for(node_count), 0, - node_count), - KOKKOS_LAMBDA(const std::size_t node) { - const auto base = node * n_pol; - double exposure = 0.0; - double previous_rate = 0.0; - double largest_rate = 0.0; - for (std::size_t time = 0; time < n_to; ++time) { - const double first = - (real_family ? node_real(base, time) : node_complex(base, time).real()); - const double second = - n_pol == 2 ? (real_family ? node_real(base + 1, time) - : node_complex(base + 1, time).real()) - : 0.0; - const double magnitude = - n_pol == 2 ? Kokkos::hypot(first, second) : Kokkos::abs(first); - const double rate = - ionisation_rate(magnitude, coefficients, log_field_min, d_log_field, - field_min, field_max, rate_max, intervals); - if (time > 0) - exposure += trapezoid(rate, previous_rate, dt); - previous_rate = rate; - if (rate > largest_rate) - largest_rate = rate; - } - output(first_node + node, 0) = exposure; - output(first_node + node, 1) = largest_rate; - }); -#endif -} - -std::vector GeneralWorkspace::statistics( - const std::uint64_t flags, const std::vector& windows, - const std::vector& on_axis, const double density, const double taper, - const double preionisation, const double intensity_factor, - const std::vector& channel_preionisation, - const std::vector& power_weight) { - require_open(); - const auto& selected_weight = - power_weight.empty() ? problem_.power_weight : power_weight; - statistics_unit_weight_ = - std::ranges::all_of(selected_weight, [](double value) { return value == 1.0; }); - if (!selected_weight.empty()) { - require_extent(selected_weight.size(), statistics_power_weight_.size(), - "statistics power weight"); - auto host_weight = Kokkos::create_mirror_view(statistics_power_weight_); - for (std::size_t column = 0; column < problem_.n_columns(); ++column) - for (std::size_t index = 0; index < problem_.n_omega; ++index) { - const double value = selected_weight[column * problem_.n_omega + index]; - if (!(value > 0.0) || !std::isfinite(value)) - throw std::invalid_argument( - "statistics power weight must be positive and finite"); - host_weight(column, index) = value; - } - graph::deep_copy(execution_space_, statistics_power_weight_, host_weight); - } else { - graph::deep_copy(execution_space_, statistics_power_weight_, 1.0); - } - const auto power_weights = statistics_power_weight_; - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - if (windows.size() % 2 != 0) - throw std::invalid_argument("statistics windows must contain lower/upper pairs"); - const auto requested = [flags](const StatisticsFlag flag) { - return (flags & static_cast(flag)) != 0U; - }; - // The temporal statistics reconstruct whole columns; on free geometry their - // buffers hold one node chunk, and the reference refuses these names on any - // spatial problem, so the adapter never asks. Refuse here as well rather than - // write past the chunk. - // The modal power weight has no rows on a spatial problem, radial included, and - // the energy kernels below read it for every column. - if ((problem_.separable_geometry() || statistics_power_weight_.extent(0) == 0) && - (requested(statistics_omega0) || requested(statistics_energy) || - requested(statistics_energy_window) || requested(statistics_peak_power) || - requested(statistics_peak_power_window) || - requested(statistics_peak_power_total) || requested(statistics_fwhm))) - throw std::invalid_argument( - "temporal statistics are not spatial saved-field statistics"); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - using StatisticsTeamPolicy = Kokkos::TeamPolicy; - using StatisticsTeamMember = StatisticsTeamPolicy::member_type; -#endif - const auto n_windows = windows.size() / 2; - constexpr auto absent = std::numeric_limits::max(); - std::size_t packed_size = 0; - const auto reserve = [&packed_size, requested](const StatisticsFlag flag, - const std::size_t count) { - if (!requested(flag)) - return absent; - const auto offset = packed_size; - packed_size = checked_sum(packed_size, count, "statistics packed output"); - return offset; - }; - const auto omega0_offset = reserve(statistics_omega0, problem_.n_columns()); - const auto energy_offset = reserve(statistics_energy, problem_.n_columns()); - const auto energy_window_offset = - reserve(statistics_energy_window, checked_product(n_windows, problem_.n_columns(), - "windowed-energy output")); - const auto peak_offset = reserve(statistics_peak_power, problem_.n_modes); - const auto peak_window_offset = - reserve(statistics_peak_power_window, - checked_product(n_windows, problem_.n_modes, "windowed-peak output")); - const auto total_peak_offset = reserve(statistics_peak_power_total, 1); - const auto density_offset = reserve(statistics_electron_density, 1); - const auto rate_offset = reserve(statistics_peak_ionisation_rate, 1); - const auto intensity_offset = reserve(statistics_peak_intensity, 1); - const auto quadrature_offset = reserve(statistics_quadrature_error, 1); - const auto fwhm_offset = reserve(statistics_fwhm, problem_.n_t); - const auto populations_offset = - reserve(statistics_populations, - checked_product(2, - checked_product(problem_.n_channels, problem_.n_nodes, - "population output"), - "population output")); - // solprop SEP 18's run diagnostics, packed last: six maxima over every node. - const auto diagnostics_offset = - reserve(statistics_plasma_integration, plasma_diagnostic_parts); - const bool diagnose = diagnostics_offset != absent; - if (populations_offset != absent || (diagnose && problem_.n_channels > 1)) { - require_extent(channel_preionisation.size(), problem_.n_channels, - "statistics channel preionisation"); - for (double value : channel_preionisation) - if (!std::isfinite(value) || value < 0.0 || value > 1.0) - throw std::invalid_argument( - "statistics channel preionisation must be in [0, 1]"); - auto initial = Kokkos::create_mirror_view(stage_channel_preionisation_); - for (std::size_t channel = 0; channel < problem_.n_channels; ++channel) - initial(0, channel) = channel_preionisation[channel]; - graph::deep_copy(execution_space_, stage_channel_preionisation_, initial); - } - - if (statistics_packed_.size() < packed_size) { - execution_space_.fence("solprop_native statistics output resize"); - statistics_packed_ = RealView("solprop_native.statistics.packed", packed_size); - } - const auto packed = statistics_packed_; - if (packed_size != 0) - graph::deep_copy(execution_space_, packed, - Kokkos::Experimental::quiet_NaN_v); - - if (omega0_offset != absent || energy_offset != absent) { - if (omega_.empty() || !(problem_.dt > 0.0)) - throw std::logic_error("native statistics need the coarse frequency axis and dt"); - const auto current = current_; - const auto omega = omega_; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const auto dt = problem_.dt; - const bool envelope = problem_.family == FieldFamily::envelope; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const StatisticsTeamPolicy policy(execution_space_, - static_cast(problem_.n_columns()), - SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); - launch::parallel_for( - "solprop_native.statistics.device_pack_spectral", policy, - KOKKOS_LAMBDA(const StatisticsTeamMember& team) { - const auto column = static_cast(team.league_rank()); - double density_sum = 0.0; - double weighted_sum = 0.0; - double energy_sum = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n)), - [&](const int index_value, double& update) { - const auto index = static_cast(index_value); - update += (squared_abs(current(column, index)) * - power_weights(column, index)); - }, - density_sum); - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n)), - [&](const int index_value, double& update) { - const auto index = static_cast(index_value); - update += omega(index) * (squared_abs(current(column, index)) * - power_weights(column, index)); - }, - weighted_sum); - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n)), - [&](const int index_value, double& update) { - const auto index = static_cast(index_value); - const double weight = - envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; - update += weight * (squared_abs(current(column, index)) * - power_weights(column, index)); - }, - energy_sum); - Kokkos::single(Kokkos::PerTeam(team), [&]() { - if (omega0_offset != absent) - packed(omega0_offset + column) = - density_sum > 0.0 ? weighted_sum / density_sum - : Kokkos::Experimental::quiet_NaN_v; - if (energy_offset != absent) - packed(energy_offset + column) = - dt / static_cast(n_t) * energy_sum; - }); - }); -#else - launch::parallel_for( - "solprop_native.statistics.pack_spectral", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_columns()), - KOKKOS_LAMBDA(const std::size_t column) { - double density_sum = 0.0; - double weighted_sum = 0.0; - double energy_sum = 0.0; - for (std::size_t index = 0; index < n; ++index) { - const double power = - (squared_abs(current(column, index)) * power_weights(column, index)); - density_sum += power; - weighted_sum += omega(index) * power; - const double weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; - energy_sum += weight * power; - } - if (omega0_offset != absent) - packed(omega0_offset + column) = - density_sum > 0.0 ? weighted_sum / density_sum - : Kokkos::Experimental::quiet_NaN_v; - if (energy_offset != absent) - packed(energy_offset + column) = dt / static_cast(n_t) * energy_sum; - }); -#endif - } - - if (energy_window_offset != absent) { - if (omega_.empty() || !(problem_.dt > 0.0)) - throw std::logic_error("native statistics need the coarse frequency axis and dt"); - const auto current = current_; - const auto omega = omega_; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const auto dt = problem_.dt; - const auto columns = problem_.n_columns(); - const bool envelope = problem_.family == FieldFamily::envelope; - for (std::size_t window = 0; window < n_windows; ++window) { - const double lower = windows[2 * window]; - const double upper = windows[2 * window + 1]; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const StatisticsTeamPolicy policy(execution_space_, static_cast(columns), - SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); - launch::parallel_for( - "solprop_native.statistics.device_pack_energy_window", policy, - KOKKOS_LAMBDA(const StatisticsTeamMember& team) { - const auto column = static_cast(team.league_rank()); - double sum = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n)), - [&](const int index_value, double& update) { - const auto index = static_cast(index_value); - if (!(omega(index) >= lower && omega(index) < upper)) - return; - const double weight = - envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; - update += weight * (squared_abs(current(column, index)) * - power_weights(column, index)); - }, - sum); - Kokkos::single(Kokkos::PerTeam(team), [&]() { - packed(energy_window_offset + window * columns + column) = - dt / static_cast(n_t) * sum; - }); - }); -#else - launch::parallel_for( - "solprop_native.statistics.pack_energy_window", - Kokkos::RangePolicy(execution_space_, 0, columns), - KOKKOS_LAMBDA(const std::size_t column) { - double sum = 0.0; - for (std::size_t index = 0; index < n; ++index) { - if (!(omega(index) >= lower && omega(index) < upper)) - continue; - const double weight = - envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; - sum += weight * (squared_abs(current(column, index)) * - power_weights(column, index)); - } - packed(energy_window_offset + window * columns + column) = - dt / static_cast(n_t) * sum; - }); -#endif - } - } - - const bool needs_base_power = peak_offset != absent || total_peak_offset != absent || - intensity_offset != absent || fwhm_offset != absent; - if (needs_base_power) { - fill_statistics_pair(false, 0.0, 0.0); - form_statistics_power(); - } - if (peak_offset != absent) { - const auto power = statistics_power_; - const auto n_t = problem_.n_t; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const StatisticsTeamPolicy policy(execution_space_, - static_cast(problem_.n_modes), - SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); - launch::parallel_for( - "solprop_native.statistics.device_pack_peak_power", policy, - KOKKOS_LAMBDA(const StatisticsTeamMember& team) { - const auto mode = static_cast(team.league_rank()); - double peak = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n_t)), - [&](const int time_value, double& update) { - const auto time = static_cast(time_value); - if (power(mode, time) > update) - update = power(mode, time); - }, - Kokkos::Max(peak)); - Kokkos::single(Kokkos::PerTeam(team), - [&]() { packed(peak_offset + mode) = peak; }); - }); -#else - launch::parallel_for( - "solprop_native.statistics.pack_peak_power", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_modes), - KOKKOS_LAMBDA(const std::size_t mode) { - double peak = 0.0; - for (std::size_t time = 0; time < n_t; ++time) - if (power(mode, time) > peak) - peak = power(mode, time); - packed(peak_offset + mode) = peak; - }); -#endif - } - if (total_peak_offset != absent || fwhm_offset != absent) { - form_statistics_total_power(); - if (total_peak_offset != absent) { - const auto total = statistics_total_power_; - const auto n_t = problem_.n_t; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const StatisticsTeamPolicy policy(execution_space_, 1, - SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); - launch::parallel_for( - "solprop_native.statistics.device_pack_peak_power_total", policy, - KOKKOS_LAMBDA(const StatisticsTeamMember& team) { - double peak = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n_t)), - [&](const int time_value, double& update) { - const auto time = static_cast(time_value); - if (total(time) > update) - update = total(time); - }, - Kokkos::Max(peak)); - Kokkos::single(Kokkos::PerTeam(team), - [&]() { packed(total_peak_offset) = peak; }); - }); -#else - launch::parallel_for( - "solprop_native.statistics.pack_peak_power_total", - Kokkos::RangePolicy(execution_space_, 0, 1), - KOKKOS_LAMBDA(const std::size_t) { - double peak = 0.0; - for (std::size_t time = 0; time < n_t; ++time) - if (total(time) > peak) - peak = total(time); - packed(total_peak_offset) = peak; - }); -#endif - } - if (fwhm_offset != absent) { - const auto destination = Kokkos::subview( - packed, std::make_pair(fwhm_offset, fwhm_offset + problem_.n_t)); - graph::deep_copy(execution_space_, destination, statistics_total_power_); - } - } - if (intensity_offset != absent) { - if (on_axis.empty() || on_axis.size() % problem_.n_modes != 0 || - on_axis.size() / problem_.n_modes > 2) - throw std::invalid_argument( - "on-axis synthesis must have one or two rows and n_modes columns"); - graph::deep_copy(execution_space_, statistics_on_axis_, 0.0); - const auto destination = Kokkos::subview( - statistics_on_axis_, std::make_pair(0, on_axis.size())); - copy_vector(execution_space_, destination, on_axis); - const auto matrix = statistics_on_axis_; - const auto field = statistics_time_; - const auto weighted_field = statistics_weighted_time_; - const bool unit_weight = statistics_unit_weight_; - const auto rows = on_axis.size() / problem_.n_modes; - const auto modes = problem_.n_modes; - const auto n_pol = problem_.n_pol_state; - const auto n_t = problem_.n_t; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const StatisticsTeamPolicy policy(execution_space_, 1, - SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); - launch::parallel_for( - "solprop_native.statistics.device_pack_peak_intensity", policy, - KOKKOS_LAMBDA(const StatisticsTeamMember& team) { - double peak = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n_t)), - [&](const int time_value, double& update) { - const auto time = static_cast(time_value); - double power = 0.0; - for (std::size_t row = 0; row < rows; ++row) { - for (std::size_t pol = 0; pol < n_pol; ++pol) { - Complex value{}, weighted_value{}; - for (std::size_t mode = 0; mode < modes; ++mode) { - value += - matrix(row * modes + mode) * field(mode * n_pol + pol, time); - if (!unit_weight) - weighted_value += matrix(row * modes + mode) * - weighted_field(mode * n_pol + pol, time); - } - power += unit_weight - ? squared_abs(value) - : (value * Kokkos::conj(weighted_value)).real(); - } - } - if (power > update) - update = power; - }, - Kokkos::Max(peak)); - Kokkos::single(Kokkos::PerTeam(team), - [&]() { packed(intensity_offset) = intensity_factor * peak; }); - }); -#else - launch::parallel_for( - "solprop_native.statistics.pack_peak_intensity", - Kokkos::RangePolicy(execution_space_, 0, 1), - KOKKOS_LAMBDA(const std::size_t) { - double peak = 0.0; - for (std::size_t time = 0; time < n_t; ++time) { - double power = 0.0; - for (std::size_t row = 0; row < rows; ++row) { - for (std::size_t pol = 0; pol < n_pol; ++pol) { - Complex value{}; - Complex weighted_value{}; - for (std::size_t mode = 0; mode < modes; ++mode) { - value += matrix(row * modes + mode) * field(mode * n_pol + pol, time); - if (!unit_weight) - weighted_value += matrix(row * modes + mode) * - weighted_field(mode * n_pol + pol, time); - } - power += unit_weight ? squared_abs(value) - : (value * Kokkos::conj(weighted_value)).real(); - } - } - if (power > peak) - peak = power; - } - packed(intensity_offset) = intensity_factor * peak; - }); -#endif - } - - if (peak_window_offset != absent) { - for (std::size_t window = 0; window < n_windows; ++window) { - fill_statistics_pair(true, windows[2 * window], windows[2 * window + 1]); - form_statistics_power(); - const auto power = statistics_power_; - const auto n_t = problem_.n_t; - const auto modes = problem_.n_modes; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const StatisticsTeamPolicy policy(execution_space_, static_cast(modes), - SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); - launch::parallel_for( - "solprop_native.statistics.device_pack_peak_power_window", policy, - KOKKOS_LAMBDA(const StatisticsTeamMember& team) { - const auto mode = static_cast(team.league_rank()); - double peak = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n_t)), - [&](const int time_value, double& update) { - const auto time = static_cast(time_value); - if (power(mode, time) > update) - update = power(mode, time); - }, - Kokkos::Max(peak)); - Kokkos::single(Kokkos::PerTeam(team), [&]() { - packed(peak_window_offset + window * modes + mode) = peak; - }); - }); -#else - launch::parallel_for( - "solprop_native.statistics.pack_peak_power_window", - Kokkos::RangePolicy(execution_space_, 0, modes), - KOKKOS_LAMBDA(const std::size_t mode) { - double peak = 0.0; - for (std::size_t time = 0; time < n_t; ++time) - if (power(mode, time) > peak) - peak = power(mode, time); - packed(peak_window_offset + window * modes + mode) = peak; - }); -#endif - } - } - - if (density_offset != absent || rate_offset != absent || - populations_offset != absent || diagnose) { - if (!problem_.plasma) - throw std::logic_error("this problem declares no compiled plasma response"); - if (diagnose && problem_.envelope_plasma) - throw std::invalid_argument("the plasma integration diagnostics read the " - "field-grid plasma column, and this problem " - "carries none"); - RealView2 source = stash_current_; - // The diagnostics rescan the plane, and the plasma statistics then read the scan. - const bool source_is_fraction = stash_is_current_ && !diagnose; - if (!source_is_fraction) { - source = plasma_statistics_scratch_; - if (problem_.separable_geometry()) { - separable_load_current(); - for (std::size_t unit = 0; unit < transverse_units(); - unit += transverse_units_per_chunk()) { - const auto chunk = transverse_chunk(unit); - separable_pack_chunk(chunk, true); - separable_to_time(chunk); - separable_nodes(chunk); - scan_plasma_statistics(chunk.first_node, chunk.nodes, source, diagnose, - preionisation); - } - } else { - statistics_modal_time(); - graph::deep_copy(execution_space_, stage_taper_, taper); - scale_transform(problem_.n_stages - 1, taper); - for (std::size_t first = 0; first < problem_.n_nodes; - first += problem_.node_chunk) { - const auto count = std::min(problem_.node_chunk, problem_.n_nodes - first); - synthesise(first, count); - scan_plasma_statistics(first, count, source, diagnose, preionisation); - } - } - } - if (populations_offset != absent) { - const bool envelope = problem_.envelope_plasma; - const auto channels = problem_.n_channels; - const auto initial = stage_channel_preionisation_; - launch::parallel_for( - "solprop_native.statistics.pack_populations", - Range2(execution_space_, {0, 0}, {problem_.n_nodes, channels}), - KOKKOS_LAMBDA(const std::size_t node, const std::size_t channel) { - const auto offset = populations_offset + 2 * (node * channels + channel); - packed(offset) = - source_is_fraction ? source(node, 2 * channel) - : envelope - ? envelope_fraction(source(node, 2 * channel), initial(0, channel)) - : ionised_fraction(source(node, 2 * channel), initial(0, channel)); - packed(offset + 1) = source(node, 2 * channel + 1); - }); - } - const auto nodes = problem_.n_nodes; - launch::parallel_for( - "solprop_native.statistics.pack_plasma", - Kokkos::RangePolicy(execution_space_, 0, 1), - KOKKOS_LAMBDA(const std::size_t) { - std::size_t densest = 0; - double largest_fraction = -1.0; - for (std::size_t node = 0; node < nodes; ++node) { - const double fraction = - source_is_fraction ? source(node, 0) - : ionised_fraction(source(node, 0), preionisation); - if (fraction > largest_fraction) { - densest = node; - largest_fraction = fraction; - } - } - if (density_offset != absent) - packed(density_offset) = density * largest_fraction; - if (rate_offset != absent) - packed(rate_offset) = source(densest, 1); - }); - if (diagnose) { - // Each part is a maximum, so the nodes combine in any order. - const auto parts = plasma_diagnostics_; - launch::parallel_for( - "solprop_native.statistics.pack_plasma_integration", - Kokkos::RangePolicy(execution_space_, 0, - plasma_diagnostic_parts), - KOKKOS_LAMBDA(const std::size_t part) { - double largest = 0.0; - for (std::size_t node = 0; node < nodes; ++node) - largest = Kokkos::max(largest, parts(node, part)); - packed(diagnostics_offset + part) = largest; - }); - } - } - if (quadrature_offset != absent) { - const double value = quadrature_error(); - launch::parallel_for( - "solprop_native.statistics.pack_quadrature", - Kokkos::RangePolicy(execution_space_, 0, 1), - KOKKOS_LAMBDA(const std::size_t) { packed(quadrature_offset) = value; }); - } - - if (packed_size == 0) - return {}; - const auto active = - Kokkos::subview(packed, std::make_pair(0, packed_size)); - auto host = Kokkos::create_mirror_view(active); - graph::deep_copy(execution_space_, host, active); - execution_space_.fence("solprop_native statistics packed boundary"); - std::vector result(packed_size); - for (std::size_t index = 0; index < packed_size; ++index) - result[index] = host(index); - return result; -} - -std::vector GeneralWorkspace::whitened_statistics( - const std::vector& root, const std::uint64_t flags, - const std::vector& windows, const double intensity_factor, - const bool spectral) { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - constexpr std::uint64_t admitted = statistics_omega0 | statistics_energy | - statistics_peak_power_total | statistics_fwhm; - if ((flags & ~admitted) != 0U) - throw std::invalid_argument("a general modal basis's statistics are omega0, the " - "energy, the peak total power and the widths"); - const auto columns = problem_.n_columns(); - const auto n = problem_.n_omega; - const auto modes = problem_.n_modes; - const auto n_pol = problem_.n_pol_state; - const bool whiten = !root.empty(); - if (whiten) { - require_extent( - root.size(), - checked_product(n, checked_product(modes, modes, "metric root"), "metric root"), - "metric root"); - if (statistics_whitened_.extent(0) != columns || - statistics_whitened_.extent(1) != n) - throw std::logic_error("whitened statistics are a general modal basis's"); - copy_vector(execution_space_, statistics_root_, root); - const auto field = current_; - const auto whitened = statistics_whitened_; - const auto matrix = statistics_root_; - // `c_mp(w) = sum_n R_mn(w) a_np(w)`, `numpy.einsum("kmn,npk->mpk", ...)`'s terms. - launch::parallel_for( - "solprop_native.statistics.whiten", - Range2(execution_space_, {0, 0}, {columns, n}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - const auto mode = column / n_pol; - const auto pol = column % n_pol; - Complex sum{}; - for (std::size_t other = 0; other < modes; ++other) - sum += matrix((index * modes + mode) * modes + other) * - field(other * n_pol + pol, index); - whitened(column, index) = sum; - }); - } - const auto sums_size = columns + (spectral ? n : 0); - if (statistics_whitened_sums_.size() < sums_size) - throw std::logic_error("whitened statistics are a general modal basis's"); - std::vector result; - { - // The statistics passes read `current_`: they read the whitened columns while - // this scope holds them there, and the state's own afterwards, even on a throw. - struct Swap { - ComplexView2& first; - ComplexView2& second; - bool active; - Swap(ComplexView2& a, ComplexView2& b, const bool on) - : first(a), second(b), active(on) { - if (active) - std::swap(first, second); - } - ~Swap() { - if (active) - std::swap(first, second); - } - Swap(const Swap&) = delete; - Swap& operator=(const Swap&) = delete; - Swap(Swap&&) = delete; - Swap& operator=(Swap&&) = delete; - } swap(current_, statistics_whitened_, whiten); - if (flags != 0U) - result = statistics(flags, windows, {}, 0.0, 1.0, 0.0, intensity_factor, {}, {}); - const auto field = current_; - const auto sums = statistics_whitened_sums_; - launch::parallel_for( - "solprop_native.statistics.whitened_column_sums", - Kokkos::RangePolicy(execution_space_, 0, columns), - KOKKOS_LAMBDA(const std::size_t column) { - double sum = 0.0; - for (std::size_t index = 0; index < n; ++index) - sum += squared_abs(field(column, index)); - sums(column) = sum; - }); - if (spectral) - launch::parallel_for( - "solprop_native.statistics.whitened_bin_sums", - Kokkos::RangePolicy(execution_space_, 0, n), - KOKKOS_LAMBDA(const std::size_t index) { - double sum = 0.0; - for (std::size_t column = 0; column < columns; ++column) - sum += squared_abs(field(column, index)); - sums(columns + index) = sum; - }); - } - const auto active = Kokkos::subview(statistics_whitened_sums_, - std::make_pair(0, sums_size)); - auto host = Kokkos::create_mirror_view(active); - graph::deep_copy(execution_space_, host, active); - execution_space_.fence("solprop_native whitened statistics boundary"); - result.reserve(result.size() + sums_size); - for (std::size_t index = 0; index < sums_size; ++index) - result.push_back(host(index)); - return result; -} - -// solprop SEP 34: the buffers of the magnetic closure's synthesis, allocated at -// construction when the problem's own request has a local flux statistic and here, -// outside any trial step, when a later request first needs them. -void GeneralWorkspace::require_flux_closure() { - const auto rows = statistics_time_.extent(0); - const bool time = statistics_weighted_time_.extent(0) != rows; - const bool magnitude = !problem_.separable_geometry() && - statistics_magnitude_.extent(0) != statistics_power_.extent(0); - const bool volume = problem_.separable_geometry() && flux_transverse_.empty(); - if (!time && !magnitude && !volume) - return; - execution_space_.fence("solprop_native flux closure allocation"); - if (time) - statistics_weighted_time_ = - ComplexView2("solprop_native.statistics.weighted_time", rows, problem_.n_t); - if (magnitude) - statistics_magnitude_ = - RealView2("solprop_native.statistics.flux_magnitude", - statistics_power_.extent(0), statistics_power_.extent(1)); - if (volume) - flux_transverse_ = - ComplexView3("solprop_native.flux.closure_transverse", transverse_.extent(0), - transverse_.extent(1), transverse_.extent(2)); -} - -// The magnetic closure of every column in time (a radial run's, solprop SEP 34): the -// analytic spectrum times the conjugate admittance, inverted into the statistics' -// time rows and kept in `statistics_weighted_time_`. -void GeneralWorkspace::fill_statistics_closure() { - graph::deep_copy(execution_space_, statistics_spectrum_, Complex{}); - const auto current = current_; - const auto spectrum = statistics_spectrum_; - const auto omega = omega_; - const auto active = active_mask_; - const auto flux = flux_; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const auto n_pol = problem_.n_pol_state; - const bool envelope = problem_.family == FieldFamily::envelope; - launch::parallel_for( - "solprop_native.statistics.prepare_closure", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - const double analytic_scale = - envelope || index == 0 || index + 1 == n ? 1.0 : 2.0; - const Complex closing = - flux.closing(column / n_pol, index, active(index) != 0U, omega(index)); - spectrum(column, index) = analytic_scale / static_cast(n_t) * - (closing * current(column, index)); - }); - graph::execute(*statistics_inverse_plan_, statistics_spectrum_, statistics_time_, - KokkosFFT::Normalization::none); - graph::deep_copy(execution_space_, statistics_weighted_time_, statistics_time_); -} - -void GeneralWorkspace::separable_time_statistics(const std::size_t axis_peak, - const std::size_t peak, - const bool fluence, - const double intensity_factor) { - constexpr auto absent = std::numeric_limits::max(); - constexpr double lowest = std::numeric_limits::lowest(); - const auto current = current_; - const auto spectrum = statistics_spectrum_; - const auto field = statistics_time_; - const auto closure = statistics_weighted_time_; - const auto packed = statistics_packed_; - const auto axis_weights = spatial_on_axis_; - const auto scratch = transverse_; - const auto closure_scratch = flux_transverse_; - const auto power = statistics_power_; - const auto partials = spatial_statistics_partials_; - const auto synthesis = synthesis_; - const auto flux = flux_; - const auto active = active_mask_; - const auto omega = omega_; - const auto n_modes = problem_.n_modes; - const auto n_nodes = problem_.n_nodes; - const auto n_pol = problem_.n_pol_state; - const auto layout = transverse_layout(); - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const bool envelope = problem_.family == FieldFamily::envelope; - if (axis_peak != absent) { - // The axis's closure first, kept in the closure rows, then its field: each the - // coherent on-axis sum of the analytic spectrum (solprop SEP 34). On free space the - // far-field pass forms those sums in its team-local bands. A sum per bin over every - // column, as the slab's few columns take here, is a thread's serial walk across a - // whole state on a device. A plane's on-axis spectrum reads the same slots. - if (far_field_) - static_cast(far_field_->run(far_field_axis, {}, 2, {}, intensity_factor, - current, far_field_inputs())); - for (const bool closed : {true, false}) { - graph::deep_copy(execution_space_, spectrum, Complex{}); - if (far_field_) - far_field_->axis_spectrum(closed, spectrum); - else - launch::parallel_for( - "solprop_native.free.statistics_axis_input", - Range2(execution_space_, {0, 0}, {n_pol, n}), - KOKKOS_LAMBDA(const std::size_t pol, const std::size_t index) { - const bool live = active(index) != 0U; - Complex sum{}; - for (std::size_t mode = 0; mode < n_modes; ++mode) { - const Complex value = current(mode * n_pol + pol, index); - sum += axis_weights(mode) * - (closed ? flux.closing(mode, index, live, omega(index)) * value - : value); - } - const double scale = - (envelope || index == 0 || index + 1 == n ? 1.0 : 2.0) / - static_cast(n_t); - spectrum(pol, index) = scale * sum; - }); - graph::execute(*statistics_inverse_plan_, statistics_spectrum_, statistics_time_, - KokkosFFT::Normalization::none); - if (closed) - graph::deep_copy( - execution_space_, - Kokkos::subview(closure, std::make_pair(0, n_pol), - Kokkos::ALL), - Kokkos::subview(field, std::make_pair(0, n_pol), Kokkos::ALL)); - } - launch::parallel_for( - "solprop_native.free.statistics_axis_reduce", - Kokkos::RangePolicy(execution_space_, 0, 1), - KOKKOS_LAMBDA(const std::size_t) { - double maximum = lowest; - for (std::size_t time = 0; time < n_t; ++time) { - double value = 0.0; - for (std::size_t pol = 0; pol < n_pol; ++pol) - value += (field(pol, time) * Kokkos::conj(closure(pol, time))).real(); - maximum = Kokkos::max(maximum, value); - } - packed(axis_peak) = intensity_factor * maximum; - }); - } - if (peak == absent && !fluence) - return; - const auto transverse_count = - static_cast(problem_.slab_geometry() ? problem_.n_y : n_modes); - launch::parallel_for( - "solprop_native.free.statistics_transverse_input", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - const auto mode = column / n_pol; - const Complex value = current(column, index) / transverse_count; - const auto plane = index * n_pol + column % n_pol; - layout.cell(scratch, plane, mode) = value; - layout.cell(closure_scratch, plane, mode) = - flux.closing(mode, index, active(index) != 0U, omega(index)) * value; - }); - transverse_inverse(); - transverse_inverse(flux_transverse_); - if (fluence) - separable_fluence(intensity_factor); - if (peak == absent) - return; - const bool slab = problem_.slab_geometry(); - const auto n_x = problem_.n_x; - const auto n_p = problem_.n_p; - // The planar synthesis carries the temporal pad scale; the statistics read the - // coarse field, so it is divided out (solprop's `collect_spatial`). - const double crop_scale = - static_cast(problem_.n_t) / static_cast(problem_.n_to); - for (std::size_t unit = 0; unit < transverse_units(); - unit += transverse_units_per_chunk()) { - const auto chunk = transverse_chunk(unit); - const auto first = chunk.first_node; - const auto first_row = chunk.first_row; - const auto count = chunk.nodes; - // The chunk's closure first, kept in the closure rows, then its field. - for (const bool closed : {true, false}) { - const auto source = closed ? closure_scratch : scratch; - graph::deep_copy(execution_space_, spectrum, Complex{}); - launch::parallel_for( - "solprop_native.free.statistics_temporal_input", - Range2(execution_space_, {0, 0}, {count * n_pol, n}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { - const double scale = - (envelope || index == 0 || index + 1 == n ? 1.0 : 2.0) / - static_cast(n_t); - if (slab) { - // Node ix of the chunk's y row iy: the slab synthesis of that row. - const auto ix = row % n_x; - const auto rows = first_row + (row / n_x) * n_p; - Complex sum{}; - for (std::size_t channel = 0; channel < n_p; ++channel) - sum += - synthesis(ix, channel) * source(index, layout.first(rows + channel), - layout.second(rows + channel)); - spectrum(row, index) = scale * (crop_scale * sum); - } else { - const auto node = first + row / n_pol; - spectrum(row, index) = - scale * layout.cell(source, index * n_pol + row % n_pol, node); - } - }); - graph::execute(*statistics_inverse_plan_, statistics_spectrum_, statistics_time_, - KokkosFFT::Normalization::none); - if (closed) - graph::deep_copy( - execution_space_, - Kokkos::subview(closure, std::make_pair(0, count * n_pol), - Kokkos::ALL), - Kokkos::subview(field, std::make_pair(0, count * n_pol), - Kokkos::ALL)); - } - launch::parallel_for( - "solprop_native.free.statistics_intensity", - Range2(execution_space_, {0, 0}, {count, n_t}), - KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { - double value = 0.0; - for (std::size_t pol = 0; pol < n_pol; ++pol) - value += (field(node * n_pol + pol, time) * - Kokkos::conj(closure(node * n_pol + pol, time))) - .real(); - power(node, time) = intensity_factor * value; - }); - launch::parallel_for( - "solprop_native.free.statistics_node_peak", - Kokkos::RangePolicy(execution_space_, 0, count), - KOKKOS_LAMBDA(const std::size_t node) { - double maximum = lowest; - for (std::size_t time = 0; time < n_t; ++time) - maximum = Kokkos::max(maximum, power(node, time)); - partials(first + node, 1) = maximum; - }); - } - launch::parallel_reduce( - "solprop_native.free.statistics_peak", - Kokkos::RangePolicy(execution_space_, 0, n_nodes), - KOKKOS_LAMBDA(const std::size_t node, double& maximum) { - maximum = Kokkos::max(maximum, partials(node, 1)); - }, - Kokkos::Max(Kokkos::subview(packed, peak))); -} - -// Each node's signed fluence into `partials(node, 0)` and a bound on its magnitude into -// `partials(node, 2)`, from the physical spectra of the transverse volumes, with no -// time transform. The analytic signal's spectrum is `s_k a_k`, with `s_k` one or two -// over `n_t`, so Parseval's theorem over the `n_t` samples gives `sum_t Re(E(t) -// conj(G(t))) = (1 / n_t) sum_k w_k Re(a_k conj(b_k))`, with `w_k` one or four: exactly -// the time sum the reference takes. The same pairs bound that sum's magnitude: by -// Cauchy-Schwarz over time and polarisation, `sum_t |E(t)| |G(t)| <= (1 / n_t) -// sqrt(sum_k w_k |a_k|^2 sum_k w_k |b_k|^2)`, with equality where `G` is proportional -// to `E`, as in a beam whose admittance varies little across its band. So a rounding -// budget set from it is never below the reference's. -void GeneralWorkspace::separable_fluence(const double intensity_factor) { - const auto scratch = transverse_; - const auto closure_scratch = flux_transverse_; - const auto partials = spatial_statistics_partials_; - const PhysicalSpectrum spectrum{.layout = transverse_layout(), - .synthesis = synthesis_, - .n_x = problem_.n_x, - .n_p = problem_.n_p, - .crop_scale = static_cast(problem_.n_t) / - static_cast(problem_.n_to), - .slab = problem_.slab_geometry()}; - const auto n_nodes = problem_.n_nodes; - const auto n_pol = problem_.n_pol_state; - const auto n = problem_.n_omega; - const bool envelope = problem_.family == FieldFamily::envelope; - const double measure = - intensity_factor * problem_.dt / static_cast(problem_.n_t); -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - // One node per thread, whole planes per block: adjacent threads read adjacent cells. - launch::parallel_for( - "solprop_native.free.statistics_fluence", - Kokkos::RangePolicy(execution_space_, 0, n_nodes), - KOKKOS_LAMBDA(const std::size_t node) { - double sum = 0.0, electric_size = 0.0, magnetic_size = 0.0; - for (std::size_t index = 0; index < n; ++index) { - const double weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; - for (std::size_t pol = 0; pol < n_pol; ++pol) { - const Complex electric = spectrum.read(scratch, index * n_pol + pol, node); - const Complex magnetic = - spectrum.read(closure_scratch, index * n_pol + pol, node); - sum += weight * (electric.real() * magnetic.real() + - electric.imag() * magnetic.imag()); - electric_size += weight * squared_abs(electric); - magnetic_size += weight * squared_abs(magnetic); - } - } - partials(node, 0) = measure * sum; - partials(node, 2) = - measure * Kokkos::sqrt(electric_size) * Kokkos::sqrt(magnetic_size); - }); -#else - // A task takes a run of adjacent nodes and reads each plane's cells of the run in - // order: one node at a time would touch a new plane at every sample. - constexpr std::size_t run = 256; - const std::size_t runs = (n_nodes + run - 1) / run; - launch::parallel_for( - "solprop_native.free.statistics_fluence", - Kokkos::RangePolicy(execution_space_, 0, runs), - KOKKOS_LAMBDA(const std::size_t item) { - const std::size_t first = item * run; - const std::size_t count = n_nodes - first < run ? n_nodes - first : run; - double sum[run] = {}; - double electric_size[run] = {}; - double magnetic_size[run] = {}; - for (std::size_t index = 0; index < n; ++index) { - const double weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; - for (std::size_t pol = 0; pol < n_pol; ++pol) { - const auto plane = index * n_pol + pol; - for (std::size_t offset = 0; offset < count; ++offset) { - const Complex electric = spectrum.read(scratch, plane, first + offset); - const Complex magnetic = - spectrum.read(closure_scratch, plane, first + offset); - sum[offset] += weight * (electric.real() * magnetic.real() + - electric.imag() * magnetic.imag()); - electric_size[offset] += weight * squared_abs(electric); - magnetic_size[offset] += weight * squared_abs(magnetic); - } - } - } - for (std::size_t offset = 0; offset < count; ++offset) { - partials(first + offset, 0) = measure * sum[offset]; - partials(first + offset, 2) = measure * Kokkos::sqrt(electric_size[offset]) * - Kokkos::sqrt(magnetic_size[offset]); - } - }); -#endif -} - -// What a plane's widths read of its fluence (`separable_fluence`), each a reduction -// over the nodes, so that the profile itself stays on the device: `moments` takes the -// lowest value and its node, the largest magnitude, the largest value and the count of -// values that aren't finite and, when the plane has a positive value and none that -// isn't finite, the scaled profile's sum and index moments. That scaled profile is left -// in `partials(node, 0)` for `plane_fluence_trace`. Each reduction blocks: one plane's -// work. -void GeneralWorkspace::plane_fluence_moments(double (&moments)[plane_moment_slots]) { - const auto partials = spatial_statistics_partials_; - const auto n_nodes = problem_.n_nodes; - const auto n_x = problem_.n_x; - const Kokkos::RangePolicy nodes(execution_space_, 0, n_nodes); - std::size_t nonfinite = 0; - launch::parallel_reduce( - "solprop_native.free.plane_fluence_finite", nodes, - KOKKOS_LAMBDA(const std::size_t node, std::size_t& update) { - if (!Kokkos::isfinite(partials(node, 0))) - ++update; - }, - Kokkos::Sum(nonfinite)); - Kokkos::MinLoc::value_type worst{}; - launch::parallel_reduce( - "solprop_native.free.plane_fluence_worst", nodes, - KOKKOS_LAMBDA(const std::size_t node, - Kokkos::MinLoc::value_type& update) { - if (partials(node, 0) < update.val) { - update.val = partials(node, 0); - update.loc = node; - } - }, - Kokkos::MinLoc(worst)); - double size = 0.0; - launch::parallel_reduce( - "solprop_native.free.plane_fluence_size", nodes, - KOKKOS_LAMBDA(const std::size_t node, double& update) { - update = Kokkos::max(update, partials(node, 2)); - }, - Kokkos::Max(size)); - double highest = 0.0; - launch::parallel_reduce( - "solprop_native.free.plane_fluence_highest", nodes, - KOKKOS_LAMBDA(const std::size_t node, double& update) { - update = Kokkos::max(update, partials(node, 0)); - }, - Kokkos::Max(highest)); - // A negative value within its budget is read as zero, as the reference reads it. - const double maximum = Kokkos::max(highest, 0.0); - FluenceMoments sums; - plane_fluence_held_ = nonfinite == 0 && maximum > 0.0; - if (plane_fluence_held_) - launch::parallel_reduce( - "solprop_native.free.plane_fluence_moments", nodes, - KOKKOS_LAMBDA(const std::size_t node, FluenceMoments& update) { - // Scaling cancels in the centroid and the widths and keeps them from - // overflowing on a bright beam. - const double sample = Kokkos::max(partials(node, 0), 0.0) / maximum; - partials(node, 0) = sample; - update.total += sample; - const std::size_t row = node / n_x; - update.x += sample * static_cast(node % n_x); - update.y += sample * static_cast(row); - }, - Kokkos::Sum(sums)); - moments[0] = worst.val; - moments[1] = static_cast(worst.loc); - moments[2] = size; - moments[3] = maximum; - moments[4] = static_cast(nonfinite); - moments[5] = sums.total; - moments[6] = sums.x; - moments[7] = sums.y; -} - -// The second moment and the azimuthal average of a plane's scaled fluence, left by a -// call that asked for its moments. The rule is solprop's -// (`solprop.spatial.cartesian_fluence_widths`), which owns the rings and hands them -// over: each ring samples the bilinear interpolant at `cosines.size()` angles about the -// centroid, with the centre in metres from the axes' first samples, so that only the -// trace crosses, not the profile. Returned: the second-moment sum, then the trace. -std::vector GeneralWorkspace::plane_fluence_trace( - const double center_x, const double center_y, const double dx, const double dy, - const double extent, const double spacing, const std::size_t n_radii, - const std::vector& cosines, const std::vector& sines) { - require_open(); - if (!plane_fluence_held_) - throw std::logic_error("a plane's fluence trace follows the call that holds it"); - plane_fluence_held_ = false; - if (cosines.size() != sines.size() || (n_radii != 0 && cosines.empty()) || - !(dx > 0.0) || !(dy > 0.0) || !(extent > 0.0) || - (n_radii != 0 && !(spacing > 0.0))) - throw std::invalid_argument("invalid fluence rings"); - const auto n_angles = cosines.size(); - const auto packed_size = - checked_sum(checked_sum(1, n_radii, "fluence trace"), - checked_product(2, n_angles, "fluence rings"), "fluence trace"); - if (statistics_packed_.size() < packed_size) { - execution_space_.fence("solprop_native spatial statistics output resize"); - statistics_packed_ = RealView("solprop_native.statistics.packed", packed_size); - } - const auto packed = statistics_packed_; - const auto samples = spatial_statistics_partials_; - const auto n_x = problem_.n_x; - const auto n_y = problem_.n_y; - const auto first_ring = n_radii + 1; - if (n_angles != 0) { - std::vector rings(cosines); - rings.insert(rings.end(), sines.begin(), sines.end()); - copy_vector(execution_space_, - Kokkos::subview(packed, std::make_pair(first_ring, packed_size)), - rings); - } - double second = 0.0; - launch::parallel_reduce( - "solprop_native.free.plane_fluence_second", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_nodes), - KOKKOS_LAMBDA(const std::size_t node, double& update) { - const double x = (static_cast(node % n_x) * dx - center_x) / extent; - const std::size_t row = node / n_x; - const double y = (static_cast(row) * dy - center_y) / extent; - update += samples(node, 0) * (x * x + y * y); - }, - Kokkos::Sum(second)); - const auto last_x = static_cast(n_x - 1); - const auto last_y = static_cast(n_y - 1); - // One thread per ring: a ring's samples are few, and a thread's sum needs no - // reduction. - launch::parallel_for( - "solprop_native.free.plane_fluence_rings", - Kokkos::RangePolicy(execution_space_, 0, n_radii), - KOKKOS_LAMBDA(const std::size_t ring) { - const double distance = spacing * static_cast(ring); - double sum = 0.0; - for (std::size_t angle = 0; angle < n_angles; ++angle) { - const double horizontal = Kokkos::min( - Kokkos::max((center_x + distance * packed(first_ring + angle)) / dx, 0.0), - last_x); - const double vertical = Kokkos::min( - Kokkos::max( - (center_y + distance * packed(first_ring + n_angles + angle)) / dy, - 0.0), - last_y); - const auto ix = - Kokkos::min(static_cast(horizontal), n_x - std::size_t{2}); - const auto iy = - Kokkos::min(static_cast(vertical), n_y - std::size_t{2}); - const double fx = horizontal - static_cast(ix); - const double fy = vertical - static_cast(iy); - const auto low = iy * n_x + ix; - const auto high = low + n_x; - sum += - (1.0 - fy) * ((1.0 - fx) * samples(low, 0) + fx * samples(low + 1, 0)) + - fy * ((1.0 - fx) * samples(high, 0) + fx * samples(high + 1, 0)); - } - packed(1 + ring) = sum / static_cast(n_angles); - }); - const auto trace = - Kokkos::subview(packed, std::make_pair(std::size_t{1}, first_ring)); - auto host = Kokkos::create_mirror_view(trace); - graph::deep_copy(execution_space_, host, trace); - execution_space_.fence("solprop_native fluence trace boundary"); - std::vector result(first_ring); - result[0] = second; - for (std::size_t ring = 0; ring < n_radii; ++ring) - result[1 + ring] = host(ring); - return result; -} - -// The field's spectrum at one physical point, the matrix `(n_rows, n_modes)` contracted -// with the modal state: `(n_rows * n_pol, n_omega)` samples, each a real and an -// imaginary part in turn. The transitional session's on-axis statistics scan this row -// on the host (solprop SEP 19), so that it is all a plane's scan brings across; the -// native session scans it in place (`axis_populations`). -std::vector -GeneralWorkspace::axis_spectrum(const std::vector& matrix) { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - if (problem_.separable_geometry() || matrix.empty() || - matrix.size() % problem_.n_modes != 0 || - matrix.size() / problem_.n_modes > statistics_on_axis_.size() / problem_.n_modes) - throw std::invalid_argument( - "an on-axis row needs a modal state and at most two rows of n_modes columns"); - const auto n_rows = matrix.size() / problem_.n_modes; - const auto n_pol = problem_.n_pol_state; - const auto n_modes = problem_.n_modes; - const auto n = problem_.n_omega; - const auto columns = checked_product(n_rows, n_pol, "on-axis columns"); - const auto samples = checked_product(columns, n, "on-axis samples"); - const auto packed_size = checked_product(2, samples, "on-axis samples"); - if (statistics_packed_.size() < packed_size) { - execution_space_.fence("solprop_native statistics output resize"); - statistics_packed_ = RealView("solprop_native.statistics.packed", packed_size); - } - copy_vector(execution_space_, - Kokkos::subview(statistics_on_axis_, - std::make_pair(0, matrix.size())), - matrix); - const auto weights = statistics_on_axis_; - const auto packed = statistics_packed_; - const auto current = current_; - launch::parallel_for( - "solprop_native.statistics.axis_spectrum", - Range2(execution_space_, {0, 0}, {columns, n}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - const auto row = column / n_pol; - const auto pol = column % n_pol; - Complex sum{}; - for (std::size_t mode = 0; mode < n_modes; ++mode) - sum += weights(row * n_modes + mode) * current(mode * n_pol + pol, index); - const auto at = 2 * (column * n + index); - packed(at) = sum.real(); - packed(at + 1) = sum.imag(); - }); - const auto filled = - Kokkos::subview(packed, std::make_pair(0, packed_size)); - auto host = Kokkos::create_mirror_view(filled); - graph::deep_copy(execution_space_, host, filled); - execution_space_.fence("solprop_native on-axis row boundary"); - std::vector result(samples); - for (std::size_t index = 0; index < samples; ++index) - result[index] = Complex(host(2 * index), host(2 * index + 1)); - return result; -} - -// The modal state on the fine time grid, in `modal_real_` or `modal_complex_`: the -// coarse bins padded at `crop_index_` and transformed back, the node scans' own first -// pass (`statistics`); a retained synthesis transforms at its nodes instead. -void GeneralWorkspace::statistics_modal_time() { - graph::deep_copy(execution_space_, fine_spectrum_, Complex{}); - const auto current = current_; - const auto fine = fine_spectrum_; - const auto crop = crop_index_; - const auto inverse_n_to = 1.0 / static_cast(problem_.n_to); - const auto n_to = problem_.n_to; - const bool analytic = problem_.family == FieldFamily::field_analytic; - launch::parallel_for( - "solprop_native.plasma.statistics_pad", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - const double analytic_scale = - analytic && index > 0 && 2 * crop(index) < n_to ? 2.0 : 1.0; - fine(column, crop(index)) = - analytic_scale * inverse_n_to * current(column, index); - }); - if (problem_.family == FieldFamily::field_real) - graph::execute(*inverse_real_plan_, fine_spectrum_, modal_real_, - KokkosFFT::Normalization::none); - else if (!retained_synthesis_) - graph::execute(*inverse_complex_plan_, fine_spectrum_, modal_complex_, - KokkosFFT::Normalization::none); -} - -// solprop SEP 19's on-axis plasma populations, scanned natively (solprop SEP 47 phase -// 4): the modal state on the fine grid (`statistics_modal_time`), contracted with -// `matrix` -- one or two rows of `n_modes` columns, the pad scale and the taper in -// them, as `solprop.backends.populations.axis_row_matrix` forms them -- into the first -// node's buffer, and scanned there by the node scan itself, every channel with its -// tables and running-integral rule. Returns each channel's final fraction and peak -// rate, `(n_channels, 2)`. Two components on a one-component node buffer are the -// field's magnitude, `hypot(E_x, E_y)`, which is all a field grid's scan reads; an -// envelope's scan reads their polarisation ellipse, so they go to a row of their own -// and the envelope scan reads it as a node of two components -// (`solprop.backends.populations.envelope_populations` on the host's row). -std::vector -GeneralWorkspace::axis_populations(const std::vector& matrix, - const std::vector& channel_preionisation) { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - if (!problem_.plasma || problem_.n_channels == 0) - throw std::invalid_argument( - "the on-axis plasma statistics need a compiled plasma response"); - if (problem_.separable_geometry() || problem_.general_modal || retained_synthesis_) - throw std::invalid_argument( - "the on-axis plasma statistics read a resolved modal state"); - ++axis_scans_; - const auto n_modes = problem_.n_modes; - const auto n_pol_state = problem_.n_pol_state; - const auto node_pol = problem_.effective_node_pol(); - if (matrix.empty() || matrix.size() % n_modes != 0 || matrix.size() / n_modes > 2) - throw std::invalid_argument( - "an on-axis row needs one or two rows of n_modes columns"); - auto rows = matrix.size() / n_modes; - // A y row of zeros on a one-component node scan is dropped: `hypot(E_x, 0)` is - // `|E_x|`, and an envelope's ellipse with no y axis is its x component's. - if (rows == 2 && n_pol_state == 1 && node_pol == 1 && - std::all_of(matrix.begin() + static_cast(n_modes), matrix.end(), - [](const double value) { return value == 0.0; })) - rows = 1; - const auto components = rows * n_pol_state; - const bool pair = components == 2 && node_pol == 1; - if (components != node_pol && !pair) - throw std::invalid_argument( - "an on-axis row has one component on each of the node scan's"); - // A pair on a field grid is combined into its magnitude in the node buffer; an - // envelope's is kept, x and y, in its own row. - const bool ellipse = pair && problem_.envelope_plasma; - const bool combine = pair && !ellipse; - if (ellipse && statistics_axis_row_.extent(0) == 0) { - execution_space_.fence("solprop_native on-axis row allocation"); - statistics_axis_row_ = - ComplexView2("solprop_native.statistics.axis_row", 2, problem_.n_to); - } - require_extent(channel_preionisation.size(), problem_.n_channels, - "on-axis channel preionisation"); - for (const double value : channel_preionisation) - if (!std::isfinite(value) || value < 0.0 || value > 1.0) - throw std::invalid_argument("on-axis channel preionisation must be in [0, 1]"); - auto initial_host = Kokkos::create_mirror_view(stage_channel_preionisation_); - for (std::size_t channel = 0; channel < problem_.n_channels; ++channel) - initial_host(0, channel) = channel_preionisation[channel]; - graph::deep_copy(execution_space_, stage_channel_preionisation_, initial_host); - const std::vector held( - matrix.begin(), matrix.begin() + static_cast(rows * n_modes)); - copy_vector( - execution_space_, - Kokkos::subview(statistics_on_axis_, std::make_pair(0, held.size())), - held); - statistics_modal_time(); - const auto weights = statistics_on_axis_; - const auto modal_real = modal_real_; - const auto modal_complex = modal_complex_; - const auto node_real = node_real_; - const auto node_complex = node_complex_; - const auto axis_row = statistics_axis_row_; - const bool real_family = problem_.family == FieldFamily::field_real; - const auto written = combine ? std::size_t{1} : components; - launch::parallel_for( - "solprop_native.statistics.axis_row", - Range2(execution_space_, {0, 0}, {written, problem_.n_to}), - KOKKOS_LAMBDA(const std::size_t component, const std::size_t time) { - // Component `row * n_pol_state + pol`, `passes.synthesise`'s layout. - const auto sum_real = [&](const std::size_t row, const std::size_t pol) { - double value = 0.0; - for (std::size_t mode = 0; mode < n_modes; ++mode) - value += weights(row * n_modes + mode) * - modal_real(mode * n_pol_state + pol, time); - return value; - }; - const auto sum_complex = [&](const std::size_t row, const std::size_t pol) { - Complex value{}; - for (std::size_t mode = 0; mode < n_modes; ++mode) - value += weights(row * n_modes + mode) * - modal_complex(mode * n_pol_state + pol, time); - return value; - }; - const auto row = component / n_pol_state; - const auto pol = component % n_pol_state; - if (ellipse) { - axis_row(component, time) = sum_complex(row, pol); - } else if (combine) { - // Both rows on a scalar state: the magnitude the scan would form. - if (real_family) - node_real(0, time) = Kokkos::hypot(sum_real(0, 0), sum_real(1, 0)); - else - node_complex(0, time) = Complex( - Kokkos::hypot(sum_complex(0, 0).real(), sum_complex(1, 0).real()), 0.0); - } else if (real_family) { - node_real(component, time) = sum_real(row, pol); - } else { - node_complex(component, time) = sum_complex(row, pol); - } - }); - const auto source = plasma_statistics_scratch_; - if (ellipse) - for (std::size_t channel = 0; channel < problem_.n_channels; ++channel) - envelope_statistics(channel, 0, 1, source, statistics_axis_row_, 2); - else - scan_plasma_statistics(0, 1, source, false, channel_preionisation[0]); - const auto channels = problem_.n_channels; - const auto packed_size = 2 * channels; - if (statistics_packed_.size() < packed_size) { - execution_space_.fence("solprop_native statistics output resize"); - statistics_packed_ = RealView("solprop_native.statistics.packed", packed_size); - } - const auto packed = statistics_packed_; - const bool envelope = problem_.envelope_plasma; - const auto initial = stage_channel_preionisation_; - // `statistics`' own packing of a node's populations. - launch::parallel_for( - "solprop_native.statistics.pack_axis_populations", - Kokkos::RangePolicy(execution_space_, 0, channels), - KOKKOS_LAMBDA(const std::size_t channel) { - packed(2 * channel) = - envelope ? envelope_fraction(source(0, 2 * channel), initial(0, channel)) - : ionised_fraction(source(0, 2 * channel), initial(0, channel)); - packed(2 * channel + 1) = source(0, 2 * channel + 1); - }); - const auto filled = - Kokkos::subview(packed, std::make_pair(0, packed_size)); - auto host = Kokkos::create_mirror_view(filled); - graph::deep_copy(execution_space_, host, filled); - execution_space_.fence("solprop_native on-axis populations boundary"); - std::vector result(packed_size); - for (std::size_t index = 0; index < packed_size; ++index) - result[index] = host(index); - return result; -} - -std::vector GeneralWorkspace::aperture_statistics( - const std::uint64_t flags, const std::vector& apertures, const int exponent, - const std::vector& regions, const double intensity_factor) { - require_open(); - if (!initialized_ || !far_field_) - throw std::logic_error( - "far-field statistics need an initial field on a Cartesian or line grid"); - return far_field_->run(flags, apertures, exponent, regions, intensity_factor, - current_, far_field_inputs()); -} - -FarFieldPass::Inputs GeneralWorkspace::far_field_inputs() const { - return FarFieldPass::Inputs{.weights = spatial_spectral_weights_, - .on_axis = spatial_on_axis_, - .square = spatial_.square, - .square_low = spatial_.square_low, - .transverse = spatial_.transverse, - .transverse_low = spatial_.transverse_low, - .spectral_rate = spatial_.spectral_rate, - .user_rate = spatial_.user_rate, - .transverse_rate = spatial_.transverse_rate, - .omega = omega_, - .active = active_mask_, - .evanescent = spatial_.evanescent}; -} - -std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t flags, - const double intensity_factor, - const double z) { - require_open(); - if (!initialized_ || spatial_spectral_weights_.empty()) - throw std::logic_error( - "spatial statistics need an initial field and spatial geometry"); - if (flux_.square.empty()) - throw std::logic_error("spatial statistics need the run's plane measure"); - if ((flags & ~std::uint64_t{63}) != 0 || !std::isfinite(intensity_factor) || - intensity_factor <= 0.0 || !std::isfinite(z)) - throw std::invalid_argument( - "invalid spatial statistic flags, intensity factor or position"); - if ((flags & 32U) != 0 && - (!problem_.free_geometry() || problem_.slab_geometry() || problem_.n_y < 2)) - throw std::invalid_argument("a plane's fluence moments need a Cartesian plane"); - constexpr auto absent = std::numeric_limits::max(); - constexpr double lowest = std::numeric_limits::lowest(); - std::size_t packed_size = 0; - const auto reserve = [flags, &packed_size](const std::uint64_t bit, - const std::size_t count) { - if ((flags & bit) == 0) - return absent; - const auto offset = packed_size; - packed_size = checked_sum(packed_size, count, "spatial statistics output"); - return offset; - }; - const auto energy_offset = reserve(1, 1); - const auto axis_peak_offset = reserve(2, 1); - const auto peak_offset = reserve(4, 1); - const auto spectrum_offset = reserve(8, problem_.n_omega); - // Widths read the signed fluence at the nodes, the separately reconstructed axis, - // which only a radial width reads, and the largest magnitude, which fixes the - // fluence's rounding budget (solprop SEP 34). - const auto fluence_offset = - reserve(16, checked_sum(problem_.n_nodes, 2, "spatial statistics output")); - // On a Cartesian plane the widths reduce here instead, and the moments cross. - const auto moments_offset = reserve(32, plane_moment_slots); - if (packed_size == 0) - return {}; - if (statistics_packed_.size() < packed_size) { - execution_space_.fence("solprop_native spatial statistics output resize"); - statistics_packed_ = RealView("solprop_native.statistics.packed", packed_size); - } - // Every field statistic is the axial flux of the medium at the plane z. - flux_at(z); - const bool local = (flags & (2U | 4U | 16U | 32U)) != 0; - if (local) - require_flux_closure(); - const auto packed = statistics_packed_; - const auto current = current_; - const auto spectral_weights = spatial_spectral_weights_; - const auto axis_weights = spatial_on_axis_; - const auto partials = spatial_statistics_partials_; - const auto flux = flux_; - const auto active = active_mask_; - const auto omega = omega_; - const auto n_modes = problem_.n_modes; - const auto n_nodes = problem_.n_nodes; - const auto n_pol = problem_.n_pol_state; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const auto dt = problem_.dt; - const bool envelope = problem_.family == FieldFamily::envelope; - if (energy_offset != absent) { - launch::parallel_for( - "solprop_native.spatial.energy_partials", - Kokkos::RangePolicy(execution_space_, 0, n_modes), - KOKKOS_LAMBDA(const std::size_t mode) { - double sum = 0.0; - for (std::size_t index = 0; index < n; ++index) { - const double admittance = - flux.closing(mode, index, active(index) != 0U, omega(index)).real(); - if (admittance == 0.0) - continue; - double squared = 0.0; - for (std::size_t pol = 0; pol < n_pol; ++pol) - squared += squared_abs(current(mode * n_pol + pol, index)); - const auto weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; - sum += weight * admittance * squared; - } - partials(mode, 0) = intensity_factor * spectral_weights(mode) * dt / - static_cast(n_t) * sum; - }); - launch::parallel_reduce( - "solprop_native.spatial.pack_energy", - Kokkos::RangePolicy(execution_space_, 0, n_modes), - KOKKOS_LAMBDA(const std::size_t mode, double& sum) { - sum += partials(mode, 0); - }, - Kokkos::Sum(Kokkos::subview(packed, energy_offset))); - } - if (spectrum_offset != absent) { - // The coherent on-axis amplitude paired at each frequency with its closure. - launch::parallel_for( - "solprop_native.spatial.axis_spectrum", - Kokkos::RangePolicy(execution_space_, 0, n), - KOKKOS_LAMBDA(const std::size_t index) { - const bool live = active(index) != 0U; - double power = 0.0; - for (std::size_t pol = 0; pol < n_pol; ++pol) { - Complex electric{}; - Complex magnetic{}; - for (std::size_t mode = 0; mode < n_modes; ++mode) { - const Complex value = - axis_weights(mode) * current(mode * n_pol + pol, index); - electric += value; - magnetic += flux.closing(mode, index, live, omega(index)) * value; - } - power += (electric * Kokkos::conj(magnetic)).real(); - } - const auto weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; - packed(spectrum_offset + index) = - intensity_factor * dt * dt / - (2.0 * Kokkos::numbers::pi_v)*weight * power; - }); - } - double moments[plane_moment_slots] = {}; - if (problem_.separable_geometry()) { - separable_time_statistics(axis_peak_offset, peak_offset, - fluence_offset != absent || moments_offset != absent, - intensity_factor); - if (fluence_offset != absent) { - launch::parallel_for( - "solprop_native.free.fluence_rows", - Kokkos::RangePolicy(execution_space_, 0, n_nodes), - KOKKOS_LAMBDA(const std::size_t node) { - packed(fluence_offset + node) = partials(node, 0); - }); - launch::parallel_reduce( - "solprop_native.free.fluence_largest", - Kokkos::RangePolicy(execution_space_, 0, n_nodes), - KOKKOS_LAMBDA(const std::size_t node, double& maximum) { - maximum = Kokkos::max(maximum, partials(node, 2)); - }, - Kokkos::Max( - Kokkos::subview(packed, fluence_offset + n_nodes + 1))); - } - if (moments_offset != absent) - plane_fluence_moments(moments); - } else if (local) { - if (n_nodes != n_modes) - throw std::logic_error("radial statistics need one node per Hankel column"); - fill_statistics_closure(); - fill_statistics_time(false, 0.0, 0.0); - const auto field = statistics_time_; - const auto closure = statistics_weighted_time_; - const auto axis_power = statistics_total_power_; - if (axis_peak_offset != absent || fluence_offset != absent) { - launch::parallel_for( - "solprop_native.spatial.axis_intensity", - Kokkos::RangePolicy(execution_space_, 0, n_t), - KOKKOS_LAMBDA(const std::size_t time) { - double power = 0.0; - for (std::size_t pol = 0; pol < n_pol; ++pol) { - Complex electric{}; - Complex magnetic{}; - for (std::size_t mode = 0; mode < n_modes; ++mode) { - electric += axis_weights(mode) * field(mode * n_pol + pol, time); - magnetic += axis_weights(mode) * closure(mode * n_pol + pol, time); - } - power += (electric * Kokkos::conj(magnetic)).real(); - } - axis_power(time) = intensity_factor * power; - }); - launch::parallel_for( - "solprop_native.spatial.pack_axis", - Kokkos::RangePolicy(execution_space_, 0, 1), - KOKKOS_LAMBDA(const std::size_t) { - double peak = lowest; - double sum = 0.0; - for (std::size_t time = 0; time < n_t; ++time) { - peak = Kokkos::max(peak, axis_power(time)); - sum += axis_power(time); - } - if (axis_peak_offset != absent) - packed(axis_peak_offset) = peak; - if (fluence_offset != absent) - packed(fluence_offset + n_nodes) = dt * sum; - }); - } - if (peak_offset != absent || fluence_offset != absent) { - const auto power = statistics_power_; - const auto magnitude = statistics_magnitude_; - const auto synthesis = synthesis_; - const double crop_scale = - static_cast(problem_.n_t) / static_cast(problem_.n_to); - // Each node's field and closure, `S F` and `S G` over the modes, as GEMMs over - // interleaved real columns (each sample's real and imaginary parts; a mode's - // polarisations are adjacent rows): the field's into the spectrum buffer, free - // once both time fields are formed, then the closure's over the field, which - // nothing reads after its synthesis. Local statistics hold one node per mode, so - // each buffer has the rows. - using Interleaved = Kokkos::View>; - static_assert(sizeof(Complex) == 2 * sizeof(double)); - const std::size_t width = 2 * n_pol * n_t; - if (statistics_spectrum_.size() < n_nodes * n_pol * n_t || - field.size() < n_nodes * n_pol * n_t) - throw std::logic_error("radial statistics need a row of each buffer per node"); - const Interleaved weights(synthesis.data(), n_nodes, n_modes); - const Interleaved field_rows(reinterpret_cast(field.data()), n_modes, - width); - const Interleaved closure_rows(reinterpret_cast(closure.data()), n_modes, - width); - const Interleaved electric_rows( - reinterpret_cast(statistics_spectrum_.data()), n_nodes, width); - const Interleaved magnetic_rows(reinterpret_cast(field.data()), n_nodes, - width); - engine_gemm(transform_execution_space_, weights, field_rows, electric_rows, 0.0); - engine_gemm(transform_execution_space_, weights, closure_rows, magnetic_rows, - 0.0); - launch::parallel_for( - "solprop_native.spatial.physical_intensity", - Range2(execution_space_, {0, 0}, {n_nodes, n_t}), - KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { - double total = 0.0; - double size = 0.0; - for (std::size_t pol = 0; pol < n_pol; ++pol) { - const std::size_t at = 2 * (pol * n_t + time); - const Complex electric = - crop_scale * - Complex(electric_rows(node, at), electric_rows(node, at + 1)); - const Complex magnetic = - crop_scale * - Complex(magnetic_rows(node, at), magnetic_rows(node, at + 1)); - total += (electric * Kokkos::conj(magnetic)).real(); - size += Kokkos::sqrt(squared_abs(electric) * squared_abs(magnetic)); - } - power(node, time) = intensity_factor * total; - magnitude(node, time) = intensity_factor * size; - }); - launch::parallel_for( - "solprop_native.spatial.fluence_partials", - Kokkos::RangePolicy(execution_space_, 0, n_nodes), - KOKKOS_LAMBDA(const std::size_t node) { - double peak = lowest; - double sum = 0.0; - double size = 0.0; - for (std::size_t time = 0; time < n_t; ++time) { - peak = Kokkos::max(peak, power(node, time)); - sum += power(node, time); - size += magnitude(node, time); - } - partials(node, 1) = peak; - partials(node, 2) = dt * size; - if (fluence_offset != absent) - packed(fluence_offset + node) = dt * sum; - }); - if (fluence_offset != absent) - launch::parallel_reduce( - "solprop_native.spatial.fluence_largest", - Kokkos::RangePolicy(execution_space_, 0, n_nodes), - KOKKOS_LAMBDA(const std::size_t node, double& maximum) { - maximum = Kokkos::max(maximum, partials(node, 2)); - }, - Kokkos::Max( - Kokkos::subview(packed, fluence_offset + n_nodes + 1))); - if (peak_offset != absent) - launch::parallel_for( - "solprop_native.spatial.pack_peak", - Kokkos::RangePolicy(execution_space_, 0, 1), - KOKKOS_LAMBDA(const std::size_t) { - double peak = lowest; - for (std::size_t node = 0; node < n_nodes; ++node) - peak = Kokkos::max(peak, partials(node, 1)); - packed(peak_offset) = peak; - }); - } - } - const auto filled = - Kokkos::subview(packed, std::make_pair(0, packed_size)); - auto host = Kokkos::create_mirror_view(filled); - graph::deep_copy(execution_space_, host, filled); - execution_space_.fence("solprop_native spatial statistics packed boundary"); - std::vector result(packed_size); - for (std::size_t index = 0; index < packed_size; ++index) - result[index] = host(index); - // The separable routes don't form the axis's fluence: only a radial width reads it. - if (fluence_offset != absent && problem_.separable_geometry()) - result[fluence_offset + n_nodes] = Kokkos::Experimental::quiet_NaN_v; - if (moments_offset != absent) - std::copy_n(moments, plane_moment_slots, result.data() + moments_offset); - return result; -} - -void GeneralWorkspace::finish(const double dz) { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - if (problem_.low_storage) { - // The march accepted the step in place (solprop SEP 24 section 6.2): the - // boundary treatments act on the state itself, and the stash a response pass - // left is the last evaluation's, at a predicted field, not this one's. - if (problem_.split_step_absorber) - split_step(dz, current_); - else - mask(current_); - stash_is_current_ = false; - execution_space_.fence("solprop_native accepted-step boundary"); - return; - } - if (problem_.split_step_absorber) - split_step(dz, advanced_); - else - mask(advanced_); - std::swap(current_, advanced_); - if (!stash_current_.empty()) - graph::deep_copy(execution_space_, stash_current_, stash_candidate_); - stash_is_current_ = !stash_current_.empty(); - execution_space_.fence("solprop_native accepted-step boundary"); -} - -void GeneralWorkspace::finish_quadrature() { - const Kokkos::Profiling::ScopedRegion region("solprop_native.quadrature.finish"); - if (!coarse_pending_) - return; - if (problem_.family == FieldFamily::envelope) - graph::execute(*forward_coarse_complex_plan_, projected_coarse_complex_, - projected_coarse_spectrum_, KokkosFFT::Normalization::none); - else if (forward_pack_coarse_plan_) - graph::execute(*forward_pack_coarse_plan_, forward_pack_coarse_, - forward_pack_coarse_spectrum_, KokkosFFT::Normalization::none); - else - graph::execute(*forward_coarse_real_plan_, projected_coarse_real_, - projected_coarse_spectrum_, KokkosFFT::Normalization::none); - if (held_plasma()) - held_transform(true, static_cast(forward_pack_coarse_plan_)); - coarse_pending_ = false; -} - -void GeneralWorkspace::score_quadrature() { - quadrature_error_ = Kokkos::Experimental::quiet_NaN_v; - quadrature_stale_ = false; - if (quadrature_partials_.empty() || - (!problem_.nonlinear && !problem_.plasma && !dispersive_)) - return; - finish_quadrature(); - // SEP 6's dispersive products join the embedded rule's spectrum once, when scored. - if (dispersive_) - dispersive_->finish_coarse(); - - const auto fine = projected_spectrum_; - const auto coarse = projected_coarse_spectrum_; - const auto crop = crop_index_; - const auto window = omega_window_; - const auto prefactor_terms = prefactor_terms_; - const auto prefactor_weights = stage_prefactor_; - const auto partials = quadrature_partials_; - const auto last = problem_.n_stages - 1; - const auto n = problem_.n_omega; - const auto n_prefactor = problem_.n_prefactor_terms; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - using TeamPolicy = Kokkos::TeamPolicy; - using TeamMember = TeamPolicy::member_type; - const TeamPolicy policy(execution_space_, static_cast(problem_.n_columns()), - SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); - launch::parallel_for( - "solprop_native.transform.device_quadrature_partials", policy, - KOKKOS_LAMBDA(const TeamMember& team) { - const auto column = static_cast(team.league_rank()); - double difference_sum = 0.0; - double fine_sum = 0.0; - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n)), - [&](const int index_value, double& update) { - const auto index = static_cast(index_value); - Complex prefactor{}; - for (std::size_t term = 0; term < n_prefactor; ++term) - prefactor += prefactor_weights(last * n_prefactor + term) * - prefactor_terms(term, column, index); - const Complex weight = prefactor * window(index); - const Complex fine_value = weight * fine(column, crop(index)); - const Complex coarse_value = weight * coarse(column, crop(index)); - update += squared_abs(coarse_value - fine_value); - }, - difference_sum); - Kokkos::parallel_reduce( - Kokkos::TeamThreadRange(team, static_cast(n)), - [&](const int index_value, double& update) { - const auto index = static_cast(index_value); - Complex prefactor{}; - for (std::size_t term = 0; term < n_prefactor; ++term) - prefactor += prefactor_weights(last * n_prefactor + term) * - prefactor_terms(term, column, index); - const Complex weight = prefactor * window(index); - update += squared_abs(weight * fine(column, crop(index))); - }, - fine_sum); - Kokkos::single(Kokkos::PerTeam(team), [&]() { - partials(0, column) = difference_sum; - partials(1, column) = fine_sum; - }); - }); -#else - launch::parallel_for( - "solprop_native.transform.quadrature_partials", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_columns()), - KOKKOS_LAMBDA(const std::size_t column) { - double difference_sum = 0.0; - double fine_sum = 0.0; - for (std::size_t index = 0; index < n; ++index) { - Complex prefactor{}; - for (std::size_t term = 0; term < n_prefactor; ++term) - prefactor += prefactor_weights(last * n_prefactor + term) * - prefactor_terms(term, column, index); - const Complex weight = prefactor * window(index); - const Complex fine_value = weight * fine(column, crop(index)); - const Complex coarse_value = weight * coarse(column, crop(index)); - fine_sum += squared_abs(fine_value); - difference_sum += squared_abs(coarse_value - fine_value); - } - partials(0, column) = difference_sum; - partials(1, column) = fine_sum; - }); -#endif - // The columns' sums in their order, on the device: the last score and the peak - // since the backend last read it. - const auto scores = quadrature_scores_; - const auto columns = problem_.n_columns(); - launch::parallel_for( - "solprop_native.transform.quadrature_score", - Kokkos::RangePolicy(execution_space_, 0, 1), - KOKKOS_LAMBDA(const std::size_t) { - double difference_sum = 0.0; - double fine_sum = 0.0; - for (std::size_t column = 0; column < columns; ++column) { - difference_sum += partials(0, column); - fine_sum += partials(1, column); - } - double value = Kokkos::Experimental::quiet_NaN_v; - if (fine_sum != 0.0) - value = Kokkos::sqrt(difference_sum / fine_sum); - scores(0) = value; - if (fine_sum != 0.0 && !(scores(1) >= value)) - scores(1) = value; - }); - quadrature_stale_ = true; -} - -double GeneralWorkspace::quadrature_error() const { - if (quadrature_stale_) { - const auto last = Kokkos::subview(quadrature_scores_, 0); - Kokkos::deep_copy(execution_space_, quadrature_error_, last); - execution_space_.fence("solprop_native quadrature score"); - quadrature_stale_ = false; - } - return quadrature_error_; -} - -double GeneralWorkspace::quadrature_peak() { - if (quadrature_scores_.extent(0) == 0) - return Kokkos::Experimental::quiet_NaN_v; - double peak = 0.0; - const auto stored = Kokkos::subview(quadrature_scores_, 1); - graph::deep_copy(execution_space_, peak, stored); - execution_space_.fence("solprop_native quadrature peak"); - graph::deep_copy(execution_space_, stored, Kokkos::Experimental::quiet_NaN_v); - return peak; -} - -void GeneralWorkspace::promote_fsal() { - require_open(); - score_quadrature(); - if (!pending_fsal_) - return; - const auto stages = stages_; - const auto whole = whole_propagator_; - const auto recycled = recycled_; - const auto last = problem_.n_stages - 1; - launch::parallel_for( - "solprop_native.stage.promote_fsal", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - recycled(column, index) = whole(column, index) * stages(last, column, index); - }); - has_recycled_ = true; - pending_fsal_ = false; -} - -void GeneralWorkspace::clear_fsal() { - has_recycled_ = false; - pending_fsal_ = false; - coarse_pending_ = false; - if (!stash_current_.empty()) - graph::deep_copy(execution_space_, stash_current_, 0.0); - stash_is_current_ = false; -} - -void GeneralWorkspace::mask(const ComplexView2& values) { - const auto active = active_mask_; - launch::parallel_for( - "solprop_native.mask.accepted", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - if (active(index) == 0U) - values(column, index) = Complex{}; - }); -} - -namespace { -// The absorber split's factor `exp(-rate h / 2)` on `rows` rows of `time`. The factor -// depends on the sample alone: a device forms it once a step for every sample, and a -// host task forms it once for a tile of samples, through the vector-maths layer -// (solprop ADR-059), and applies it to a block of rows, the same expression to the bit -// on the libm route. A device reads each sample's factor from `factor`, formed once a -// step by `form_split_factor` with the same expression: one binary64 exponential per -// sample rather than per element, which on a device with a reduced binary64 rate is -// most of the pass. -template -void split_apply(const char* label, const ExecutionSpace& execution, - const TimeView& time, const RealView& rate, const RealView& factors, - const double dz, const std::size_t rows, const std::size_t samples, - const double scale = 1.0) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - static_cast(rate); - static_cast(dz); - launch::parallel_for( - label, Range2(execution, {0, 0}, {rows, samples}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { - time(row, index) *= scale * factors(index); - }); -#else - static_cast(factors); - constexpr std::size_t tile = 256; - constexpr std::size_t row_block = 64; - const std::size_t tiles = (samples + tile - 1) / tile; - const std::size_t blocks = (rows + row_block - 1) / row_block; - launch::parallel_for( - label, Kokkos::RangePolicy(execution, 0, tiles * blocks), - KOKKOS_LAMBDA(const std::size_t task) { - const std::size_t first = (task % tiles) * tile; - const std::size_t count = samples - first < tile ? samples - first : tile; - const std::size_t row_first = (task / tiles) * row_block; - const std::size_t row_last = - rows - row_first < row_block ? rows : row_first + row_block; - double factor[tile]; - for (std::size_t sample = 0; sample < count; ++sample) - factor[sample] = -0.5 * rate(first + sample) * dz; - vmath::exp(factor, factor, count); - for (std::size_t sample = 0; sample < count; ++sample) - factor[sample] *= scale; - for (std::size_t row = row_first; row < row_last; ++row) - for (std::size_t sample = 0; sample < count; ++sample) - time(row, first + sample) *= factor[sample]; - }); -#endif -} -} // namespace - -void GeneralWorkspace::split_columns(const double dz, std::size_t active_rows, - const double parseval, - const std::size_t first_column) { - const bool tail = temporal_tail_rows_ != 0 && active_rows == temporal_tail_rows_; - if (active_rows == 0) - active_rows = coarse_spectrum_.extent(0); - if (problem_.family == FieldFamily::envelope) { - if (tail) - coarse_tail_inverse_complex_.execute(); - else - graph::execute(*coarse_inverse_complex_plan_, coarse_spectrum_, - coarse_time_complex_, KokkosFFT::Normalization::none); - if (monitoring_split()) - add_split_removal(dz, active_rows, parseval, first_column, coarse_time_complex_, - 1.0); - split_apply("solprop_native.absorber.split_apply_complex", execution_space_, - coarse_time_complex_, absorber_coarse_, split_factor_, dz, active_rows, - problem_.n_t); - if (tail) - coarse_tail_forward_complex_.execute(); - else - graph::execute(*coarse_forward_complex_plan_, coarse_time_complex_, - coarse_spectrum_, KokkosFFT::Normalization::none); - } else { - if (tail) - coarse_tail_inverse_real_.execute(); - else - graph::execute(*coarse_inverse_real_plan_, coarse_spectrum_, coarse_time_real_, - KokkosFFT::Normalization::none); - if (monitoring_split()) - add_split_removal(dz, active_rows, parseval, first_column, {}, 1.0); - split_apply("solprop_native.absorber.split_apply_real", execution_space_, - coarse_time_real_, absorber_coarse_, split_factor_, dz, active_rows, - problem_.n_t); - if (tail) - coarse_tail_forward_real_.execute(); - else - graph::execute(*coarse_forward_real_plan_, coarse_time_real_, coarse_spectrum_, - KokkosFFT::Normalization::none); - } -} - -// Free space's complex rows: the split's factor depends on time alone, so it acts alike -// on every column of any transverse basis, and the temporal pair runs in place on the -// state's own rows, a block of them at a time. Nothing is copied to a coarse buffer and -// back, the transform's normalisation goes with the factor, and the band's mask is the -// one pass after the blocks. Equal to the reference's split in real space up to the -// transforms' rounding, at any block size. -void GeneralWorkspace::split_state_rows(const double dz, const ComplexView2& state) { - const double inverse_time = 1.0 / static_cast(problem_.n_t); - const auto columns = problem_.n_columns(); - for (std::size_t first = 0; first < columns; first += split_rows_) { - const auto rows = std::min(split_rows_, columns - first); - const ComplexView2 block = - Kokkos::subview(state, std::make_pair(first, first + rows), Kokkos::ALL()); - const bool tail = rows != split_rows_; - graph::execute(tail ? *split_tail_inverse_plan_ : *split_inverse_plan_, block, - block, KokkosFFT::Normalization::none); - if (monitoring_split()) - add_split_removal(dz, rows, 0.0, first, block, inverse_time * inverse_time); - split_apply("solprop_native.absorber.split_apply_complex", execution_space_, block, - absorber_coarse_, split_factor_, dz, rows, problem_.n_t, inverse_time); - graph::execute(tail ? *split_tail_forward_plan_ : *split_forward_plan_, block, - block, KokkosFFT::Normalization::none); - } - mask(state); -} - -// What one split takes of the power it finds in the coarse time rows, `sum P (1 - -// exp(-alpha h))` with the boundary monitor's column weights, added to the run's -// accumulator in the memory space (solprop SEP 21, ADR-057): the monitor's reading -// drains it at a check, so every accepted step is counted, checked or not. The same -// pass adds to the step's `sum P'` and the guards' and the collar's sums of `P'`, the -// power found halfway through the split's factor, `P exp(-alpha h / 2)`, which the -// monitor judges the occupancies on: the symmetric splitting's state at the step's -// end. A separable route's rows are real-space rows, measured by `parseval`; zero -// means the rows are the stored columns from `first_column` on, weighted by their -// quadrature. `complex_time` holds a complex route's rows, whose power `power_scale` -// brings to the state's measure: the state's own rows carry no transform normalisation. -void GeneralWorkspace::add_split_removal(const double dz, const std::size_t active_rows, - const double parseval, - const std::size_t first_column, - const ComplexView2& complex_time, - const double power_scale) { - const auto rate = absorber_coarse_; - const auto share = monitor_share_; - const auto half = monitor_half_; - const auto weights = monitor_column_weights_; - const bool stored = parseval == 0.0; - // The share the split takes, and half its power factor, depend on time alone: - // formed once, not per row. - launch::parallel_for( - "solprop_native.monitor.split_share", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_t), - KOKKOS_LAMBDA(const std::size_t index) { - share(index) = -Kokkos::expm1(-rate(index) * dz); - half(index) = Kokkos::exp(-0.5 * rate(index) * dz); - }); - const auto guards = monitor_weights_; - const auto samples = Range2(execution_space_, {0, 0}, {active_rows, problem_.n_t}); - if (problem_.family == FieldFamily::envelope) { - launch::parallel_reduce( - "solprop_native.monitor.split_removal_complex", samples, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t index, - SplitSums& update) { - const double power = (stored ? weights(first_column + row) : parseval) * - (power_scale * squared_abs(complex_time(row, index))); - update.values[0] += power * share(index); - const double found = half(index) * power; - update.values[1] += found; - update.values[2] += guards(0, index) * found; - update.values[3] += guards(1, index) * found; - update.values[4] += guards(2, index) * found; - }, - Kokkos::Sum(monitor_split_sum_)); - } else { - const auto time = coarse_time_real_; - launch::parallel_reduce( - "solprop_native.monitor.split_removal_real", samples, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t index, - SplitSums& update) { - const double value = time(row, index); - const double power = - (stored ? weights(first_column + row) : parseval) * value * value; - update.values[0] += power * share(index); - const double found = half(index) * power; - update.values[1] += found; - update.values[2] += guards(0, index) * found; - update.values[3] += guards(1, index) * found; - update.values[4] += guards(2, index) * found; - }, - Kokkos::Sum(monitor_split_sum_)); - } - const auto removed = monitor_removed_; - const auto split = monitor_split_sum_; - const auto found = monitor_found_; - launch::parallel_for( - "solprop_native.monitor.split_removal_add", - Kokkos::RangePolicy(execution_space_, 0, 1), - KOKKOS_LAMBDA(const std::size_t) { - removed() += split().values[0]; - found() += split(); - }); -} - -// Bind solprop's boundary monitor (SEP 21): its four time weights, row-major `(4, -// n_t)`, and its columns' quadrature by mode (empty for unit weights). The views are -// allocated at the first bind and kept; the split's accumulator starts at zero. -void GeneralWorkspace::bind_monitor(const std::vector& weights, - const std::vector& column_weights) { - require_open(); - const auto n_t = problem_.n_t; - const auto columns = problem_.n_columns(); - require_extent(weights.size(), 4 * n_t, "monitor weights"); - if (!column_weights.empty()) - require_extent(column_weights.size(), problem_.n_modes, "monitor column weights"); - if (monitor_weights_.extent(0) == 0) { - monitor_weights_ = RealView2("solprop_native.monitor.weights", 4, n_t); - monitor_column_weights_ = - RealView("solprop_native.monitor.column_weights", columns); - monitor_share_ = RealView("solprop_native.monitor.split_share", n_t); - monitor_half_ = RealView("solprop_native.monitor.split_half", n_t); - monitor_removed_ = - Kokkos::View("solprop_native.monitor.removed"); - monitor_split_sum_ = - Kokkos::View("solprop_native.monitor.split_sum"); - monitor_found_ = - Kokkos::View("solprop_native.monitor.split_found"); - } - auto host_weights = Kokkos::create_mirror_view(monitor_weights_); - for (std::size_t row = 0; row < 4; ++row) - for (std::size_t index = 0; index < n_t; ++index) - host_weights(row, index) = weights[row * n_t + index]; - graph::deep_copy(execution_space_, monitor_weights_, host_weights); - // A state column is `mode * n_pol + pol`. - std::vector expanded(columns, 1.0); - if (!column_weights.empty()) - for (std::size_t column = 0; column < columns; ++column) - expanded[column] = column_weights[column / problem_.n_pol_state]; - copy_vector(execution_space_, monitor_column_weights_, expanded); - graph::deep_copy(execution_space_, monitor_removed_, 0.0); - graph::deep_copy(execution_space_, monitor_found_, SplitSums{}); - execution_space_.fence("solprop_native monitor bind"); - monitor_bound_ = true; -} - -// The boundary monitor's reading of the accepted state (solprop SEP 21, ADR-057): -// `sum_t P`, the four weighted sums `sum_t w_k P` and the split's accumulator, which -// this drains; `P(t)` is the columns' weighted `|E(t)|^2` on the coarse time axis. -// The state is transformed chunk by chunk through buffers idle between steps: on a -// field grid under the split scheme the collar's coarse pair, whose inverse real -// transform is the one solprop's reference takes; otherwise the statistics buffers, -// with the analytic signal's spectrum, whose real part is that transform's result. -namespace { -// The boundary monitor's sums over `count` rows of `n_t` samples: each sample's power, -// `power(row, index)`, in total and weighed by each of the four regions' weights. A -// host work item takes a run of one row's samples, its sums in sample order, so that -// no sample divides its flat index by the row length; a device takes one sample. -template -MonitorSums monitor_sums(const char* label, const ExecutionSpace& execution, - const std::size_t count, const std::size_t n_t, - const Weights& weights, const Power& power) { - MonitorSums sums; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - launch::parallel_reduce( - label, Kokkos::RangePolicy(execution, 0, count * n_t), - KOKKOS_LAMBDA(const std::size_t item, MonitorSums& update) { - const auto index = item % n_t; - const double value = power(item / n_t, index); - update.values[0] += value; - for (std::size_t region = 0; region < 4; ++region) - update.values[region + 1] += weights(region, index) * value; - }, - Kokkos::Sum(sums)); -#else - constexpr std::size_t run = 512; - const std::size_t runs = (n_t + run - 1) / run; - launch::parallel_reduce( - label, Kokkos::RangePolicy(execution, 0, count * runs), - KOKKOS_LAMBDA(const std::size_t item, MonitorSums& update) { - const std::size_t row = item / runs; - const std::size_t first = (item % runs) * run; - const std::size_t last = n_t - first < run ? n_t : first + run; - double total = 0.0; - double regions[4] = {0.0, 0.0, 0.0, 0.0}; - for (std::size_t index = first; index < last; ++index) { - // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide - const double value = power(row, index); - total += value; - for (std::size_t region = 0; region < 4; ++region) - regions[region] += weights(region, index) * value; - } - update.values[0] += total; - for (std::size_t region = 0; region < 4; ++region) - update.values[region + 1] += regions[region]; - }, - Kokkos::Sum(sums)); -#endif - return sums; -} -} // namespace - -namespace { -// The field on a line's node, in a row of the transverse scratch (solprop SEP 44): a -// row is a plane and a position across the line, and no route has a branch here. -struct EdgeScratchValue { - ComplexView3 scratch; - std::size_t units{}, across{}, row_stride{}, node_stride{}; - KOKKOS_INLINE_FUNCTION Complex operator()(const std::size_t row, - const std::size_t node) const { - return scratch.data()[(row / across) * units + (row % across) * row_stride + - node * node_stride]; - } -}; - -// The same radially: `sqrt(r) E`, a dense product per value, in a row of (pol, omega). -struct EdgeRadialValue { - RealView2 synthesis; - ComplexView2 current; - RealView scale; - std::size_t n_pol{}, n{}, modes{}; - KOKKOS_INLINE_FUNCTION Complex operator()(const std::size_t row, - const std::size_t node) const { - const auto pol = row / n; - const auto omega = row % n; - Complex value{}; - for (std::size_t mode = 0; mode < modes; ++mode) - value += synthesis(node, mode) * current(mode * n_pol + pol, omega); - return scale(node) * value; - } -}; - -// One line's `(inward, windowed)`, unscaled: a Kaiser-windowed transform along its -// normal, padded twice, batch by batch so that the buffer stays bounded. -template -MonitorSums edge_strip_line(const ExecutionSpace& execution, const Value& value, - const std::size_t rows, const std::size_t batch, - const std::size_t nodes, const std::size_t first, - const std::size_t last, const IndexView& positions, - const RealView& window, const ComplexView2& buffer, - StripPlan& plan) { - const std::size_t count = 2 * nodes; - MonitorSums sums; - for (std::size_t start = 0; start < rows; start += batch) { - launch::parallel_for( - "solprop_native.edge.strip_fill", - Kokkos::RangePolicy(execution, 0, batch), - KOKKOS_LAMBDA(const std::size_t row) { - // A short last batch leaves its other rows zero, whose energy is none. - const bool live = start + row < rows; - for (std::size_t index = 0; index < nodes; ++index) - buffer(row, index) = - live ? window(index) * value(start + row, positions(index)) : Complex{}; - for (std::size_t index = nodes; index < count; ++index) - buffer(row, index) = Complex{}; - }); - graph::execute(plan, buffer, buffer, KokkosFFT::Normalization::none); - MonitorSums part; - launch::parallel_reduce( - "solprop_native.edge.strip_sums", - Kokkos::RangePolicy(execution, 0, batch), - KOKKOS_LAMBDA(const std::size_t row, MonitorSums& update) { - for (std::size_t index = 0; index < count; ++index) { - const double power = squared_abs(buffer(row, index)); - update.values[1] += power; - if (index >= first && index < last) - update.values[0] += power; - } - }, - Kokkos::Sum(part)); - sums += part; - } - return sums; -} - -// One line's rectified inward current: `Im(E* dE/ds) / k`, rectified per row before -// it's summed, as a collar's current is defined (solprop SEP 40 section 5.3). -template -double edge_current_line(const ExecutionSpace& execution, const Value& value, - const std::size_t rows, const std::size_t node, - const std::size_t before, const std::size_t after, - const double step, const double k_min) { - double total = 0.0; - launch::parallel_reduce( - "solprop_native.edge.current", - Kokkos::RangePolicy(execution, 0, rows), - KOKKOS_LAMBDA(const std::size_t row, double& update) { - const Complex here = value(row, node); - const Complex slope = (value(row, after) - value(row, before)) / step; - const double flux = - (here.real() * slope.imag() - here.imag() * slope.real()) / k_min; - update += flux > 0.0 ? flux : 0.0; - }, - Kokkos::Sum(total)); - return total; -} - -// The transverse edge's region weights (solprop SEP 44), from the rows given on: a -// radial read synthesises only the outer rows its regions can touch. -struct EdgeWeights { - RealView2 weights; - std::size_t first{}; - KOKKOS_INLINE_FUNCTION double operator()(const std::size_t region, - const std::size_t index) const { - return weights(region, first + index); - } -}; -} // namespace - -// Bind what the transverse edge's reading sums with (solprop SEP 44): three tables of -// four regions' weights on the nodes, and radially the synthesis and the first row a -// read uses. Native holds no face, mask or judgement. The views are allocated at the -// first bind and kept. -void GeneralWorkspace::bind_edge(const std::vector& faces, - const std::vector& clip, - const std::vector& totals, - const std::vector& synthesis, - const std::size_t first_row) { - require_open(); - const auto nodes = faces.size() / 4; - require_extent(faces.size(), 4 * nodes, "edge faces"); - require_extent(clip.size(), 4 * nodes, "edge clip"); - require_extent(totals.size(), 4 * nodes, "edge totals"); - if (problem_.separable_geometry()) { - require_extent(nodes, transverse_units(), "edge nodes"); - require_extent(synthesis.size(), 0, "a separable edge's synthesis"); - } else { - require_extent(nodes, problem_.n_modes, "edge nodes"); - require_extent(synthesis.size(), nodes * nodes, "edge synthesis"); - } - if (first_row >= nodes) - throw std::invalid_argument("the edge's first row lies past its nodes"); - if (edge_faces_.extent(0) == 0) { - edge_faces_ = RealView2("solprop_native.edge.faces", 4, nodes); - edge_clip_ = RealView2("solprop_native.edge.clip", 4, nodes); - edge_totals_ = RealView2("solprop_native.edge.totals", 4, nodes); - edge_columns_ = RealView("solprop_native.edge.columns", problem_.n_modes); - if (!synthesis.empty()) - edge_synthesis_ = RealView2("solprop_native.edge.synthesis", nodes, nodes); - if (!synthesis.empty() && problem_.n_pol_state == 1 && - current_.extent(1) == problem_.n_omega) - edge_nodes_ = RealView2( - Kokkos::view_alloc(Kokkos::WithoutInitializing, "solprop_native.edge.nodes"), - nodes, 2 * problem_.n_omega); - } - copy_vector(execution_space_, edge_faces_, faces); - copy_vector(execution_space_, edge_clip_, clip); - copy_vector(execution_space_, edge_totals_, totals); - if (!synthesis.empty()) - copy_vector(execution_space_, edge_synthesis_, synthesis); - edge_first_row_ = first_row; - execution_space_.fence("solprop_native edge bind"); -} - -// Bind a requested collar's lines (solprop SEP 44), which Python describes: their -// nodes, windows, inward bins and onsets. A batch of one shape serves every line that -// has it. -void GeneralWorkspace::bind_edge_lines(const EdgeLines& lines) { - require_open(); - const auto n_lines = lines.axes.size(); - require_extent(lines.counts.size(), n_lines, "edge line counts"); - require_extent(lines.inward.size(), 2 * n_lines, "edge line bins"); - require_extent(lines.factors.size(), n_lines, "edge line factors"); - require_extent(lines.measures.size(), n_lines, "edge line measures"); - require_extent(lines.onsets.size(), 3 * n_lines, "edge line onsets"); - require_extent(lines.steps.size(), n_lines, "edge line steps"); - require_extent(lines.windows.size(), lines.positions.size(), "edge line windows"); - if (std::accumulate(lines.counts.begin(), lines.counts.end(), std::size_t{0}) != - lines.positions.size()) - throw std::invalid_argument("the edge's lines don't account for their nodes"); - const auto planes = problem_.n_pol_state * problem_.n_omega; - // A runner's next point reuses the plans of its shapes, and only those stay held. - decltype(edge_batches_) batches; - std::vector bound; - bound.reserve(n_lines); - std::size_t offset = 0; - for (std::size_t index = 0; index < n_lines; ++index) { - EdgeLine line; - line.axis = lines.axes[index]; - line.nodes = lines.counts[index]; - line.rows = planes * (problem_.slab_geometry() ? problem_.n_p - : !problem_.separable_geometry() ? std::size_t{1} - : line.axis == 1 ? problem_.n_y - : problem_.n_x); - line.first = lines.inward[2 * index]; - line.last = lines.inward[2 * index + 1]; - line.factor = lines.factors[index]; - line.measure = lines.measures[index]; - line.step = lines.steps[index]; - line.onset = lines.onsets[3 * index]; - line.before = lines.onsets[3 * index + 1]; - line.after = lines.onsets[3 * index + 2]; - if (line.nodes != 0) { - const auto first = static_cast(offset); - const auto last = static_cast(offset + line.nodes); - line.positions = IndexView("solprop_native.edge.line_nodes", line.nodes); - line.window = RealView("solprop_native.edge.line_window", line.nodes); - copy_vector(execution_space_, line.positions, - std::vector(lines.positions.begin() + first, - lines.positions.begin() + last)); - copy_vector(execution_space_, line.window, - std::vector(lines.windows.begin() + first, - lines.windows.begin() + last)); - offset += line.nodes; - const std::size_t count = 2 * line.nodes; - line.batch = - std::min(line.rows, std::max(1, lines.batch_bytes / - (sizeof(Complex) * count))); - } - bound.push_back(std::move(line)); - } - // The strips run in turn, so one allocation serves every shape: a positive and a - // negative face's lines commonly differ by a node, and would otherwise hold two. - std::size_t largest = 0; - for (const auto& line : bound) - largest = std::max(largest, line.batch * 2 * line.nodes); - if (edge_strip_.size() != largest) - edge_strip_ = - largest == 0 - ? ComplexView{} - : ComplexView(Kokkos::view_alloc(Kokkos::WithoutInitializing, - std::string("solprop_native.edge.strip")), - largest); - for (const auto& line : bound) { - if (line.nodes == 0) - continue; - const std::size_t count = 2 * line.nodes; - const std::pair shape{line.batch, count}; - if (!batches.contains(shape)) { - EdgeBatch batch; - if (auto kept = edge_batches_.extract(shape); kept) - batch = std::move(kept.mapped()); - // Not tracked: `edge_strip_` owns the memory, and outlives every shape's view. - batch.buffer = ComplexView2(edge_strip_.data(), line.batch, count); - if (!batch.plan) { - // A plan's own memory is measured as the workspace's are, and listed with them. - const auto heap_before = heap_bytes(); - const auto device_before = device_free_bytes(); - constexpr int along = -1; - batch.plan = std::make_unique(fft_space_for_rows(batch.buffer), - batch.buffer, batch.buffer, - KokkosFFT::Direction::forward, along); - execution_space_.fence("solprop_native edge strip plan"); - const auto heap_after = heap_bytes(); - const auto device_after = device_free_bytes(); - batch.heap_growth = heap_after > heap_before ? heap_after - heap_before : 0; - batch.device_growth = - device_before > device_after ? device_before - device_after : 0; - } - batches.emplace(shape, std::move(batch)); - } - } - edge_lines_ = std::move(bound); - edge_batches_ = std::move(batches); - edge_k_min_ = lines.k_min; - if (lines.scale.empty()) { - edge_scale_ = {}; - } else { - edge_scale_ = RealView("solprop_native.edge.line_scale", lines.scale.size()); - copy_vector(execution_space_, edge_scale_, lines.scale); - } - execution_space_.fence("solprop_native edge lines"); -} - -// The state's reading for the transverse edge: its region sums and each mode's energy, -// with only those numbers crossing. With `whole` (a launch) every table on every node; -// otherwise the faces', and with `totals` a collar's, on the rows a read uses; and with -// a requested collar its strip and current, on the same transform. The transverse -// scratch is free at a committed position: a step's first pass writes it again. -EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, - const bool totals, const bool strip, - const bool want_current) { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - if (edge_faces_.extent(0) == 0) - throw std::logic_error("no transverse edge is bound"); - const auto current = current_; - const auto n_pol = problem_.n_pol_state; - const auto n = problem_.n_omega; - EdgeNumbers reading; - const bool separable = problem_.separable_geometry(); - if (separable) - separable_load_current(); - const auto take = [&reading](const MonitorSums& sums) { - reading.sums.insert(reading.sums.end(), sums.values.data() + 1, - sums.values.data() + 5); - }; - if (!faces) { - // Only what follows was asked for. - } else if (separable) { - const auto scratch = transverse_; - const auto units = transverse_units(); - const auto planes = scratch.size() / units; - const auto power = KOKKOS_LAMBDA(const std::size_t plane, const std::size_t node) { - return squared_abs(scratch.data()[plane * units + node]); - }; - take(monitor_sums("solprop_native.edge.faces", execution_space_, planes, units, - edge_faces_, power)); - if (whole) - take(monitor_sums("solprop_native.edge.clip", execution_space_, planes, units, - edge_clip_, power)); - if (whole || totals) - take(monitor_sums("solprop_native.edge.totals", execution_space_, planes, units, - edge_totals_, power)); - } else if (edge_nodes_.extent(0) != 0) { - // One state polarisation: the rows wanted synthesised in one product, the state's - // complex samples read as twice as many real columns, and every table's sums then - // from those fields. A launch reads every node: its total is the grid's. The - // per-sample sums this replaces synthesised each row once a table; on an Apple M5 - // Max the 256- and 512-node radial filaments ran 0.83 and 0.75 of their one-thread - // windows, 0.92 and 0.87 at six threads. - using Interleaved = Kokkos::View>; - static_assert(sizeof(Complex) == 2 * sizeof(double)); - const auto nodes = problem_.n_modes; - const auto first = whole ? std::size_t{0} : edge_first_row_; - const auto rows = nodes - first; - const Interleaved state(reinterpret_cast(current_.data()), nodes, 2 * n); - const Interleaved synthesis(edge_synthesis_.data() + first * nodes, rows, nodes); - const Interleaved fields(edge_nodes_.data(), rows, 2 * n); - Kokkos::Profiling::pushRegion("solprop_native.edge.synthesis"); - engine_gemm(transform_space_for(rows), synthesis, state, fields, 0.0); - Kokkos::Profiling::popRegion(); - const auto power = KOKKOS_LAMBDA(const std::size_t plane, const std::size_t row) { - const double real = fields(row, 2 * plane); - const double imaginary = fields(row, 2 * plane + 1); - return real * real + imaginary * imaginary; - }; - const auto sums = [&](const char* label, const RealView2& weights) { - return monitor_sums(label, execution_space_, n, rows, - EdgeWeights{.weights = weights, .first = first}, power); - }; - take(sums("solprop_native.edge.faces", edge_faces_)); - if (whole) - take(sums("solprop_native.edge.clip", edge_clip_)); - if (whole || totals) - take(sums("solprop_native.edge.totals", edge_totals_)); - } else { - // Each plane's synthesis on the rows wanted. A launch reads every node: its total - // is the grid's. A dense product per sample, on a read's few outer rows. - const auto synthesis = edge_synthesis_; - const auto nodes = problem_.n_modes; - const auto first = whole ? std::size_t{0} : edge_first_row_; - const auto power = KOKKOS_LAMBDA(const std::size_t plane, const std::size_t row) { - const auto pol = plane / n; - const auto omega = plane % n; - Complex value{}; - for (std::size_t mode = 0; mode < nodes; ++mode) - value += synthesis(first + row, mode) * current(mode * n_pol + pol, omega); - return squared_abs(value); - }; - const auto sums = [&](const char* label, const RealView2& weights) { - return monitor_sums(label, execution_space_, n_pol * n, nodes - first, - EdgeWeights{.weights = weights, .first = first}, power); - }; - take(sums("solprop_native.edge.faces", edge_faces_)); - if (whole) - take(sums("solprop_native.edge.clip", edge_clip_)); - if (whole || totals) - take(sums("solprop_native.edge.totals", edge_totals_)); - } - if (faces) { - const auto energies = edge_columns_; - launch::parallel_for( - "solprop_native.edge.columns", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_modes), - KOKKOS_LAMBDA(const std::size_t mode) { - double sum = 0.0; - for (std::size_t pol = 0; pol < n_pol; ++pol) - for (std::size_t index = 0; index < n; ++index) - sum += squared_abs(current(mode * n_pol + pol, index)); - energies(mode) = sum; - }); - execution_space_.fence("solprop_native edge columns"); - const auto host = - Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, energies); - reading.columns.assign(host.data(), host.data() + host.size()); - } - // The same two kernels on any route: only where a row's value comes from differs. - const auto take_line = [&](const EdgeLine& line, const auto& value) { - if (strip) { - MonitorSums sums; - if (line.nodes != 0) { - auto& batch = edge_batches_.at({line.batch, 2 * line.nodes}); - sums = edge_strip_line(execution_space_, value, line.rows, line.batch, - line.nodes, line.first, line.last, line.positions, - line.window, batch.buffer, *batch.plan); - } - reading.strip.push_back(sums.values[0] * line.factor); - reading.strip.push_back(sums.values[1] * line.factor); - } - if (want_current && line.onset != EdgeLines::no_onset) - reading.current += - line.measure * edge_current_line(execution_space_, value, line.rows, - line.onset, line.before, line.after, - line.step, edge_k_min_); - }; - if (strip || want_current) { - for (const auto& line : edge_lines_) { - if (!separable) { - take_line(line, EdgeRadialValue{.synthesis = edge_synthesis_, - .current = current, - .scale = edge_scale_, - .n_pol = n_pol, - .n = n, - .modes = problem_.n_modes}); - } else if (problem_.slab_geometry()) { - take_line(line, EdgeScratchValue{.scratch = transverse_, - .units = transverse_units(), - .across = 1, - .row_stride = 0, - .node_stride = 1}); - } else if (line.axis == 1) { - take_line(line, EdgeScratchValue{.scratch = transverse_, - .units = transverse_units(), - .across = problem_.n_y, - .row_stride = problem_.n_x, - .node_stride = 1}); - } else { - take_line(line, EdgeScratchValue{.scratch = transverse_, - .units = transverse_units(), - .across = problem_.n_x, - .row_stride = 1, - .node_stride = problem_.n_x}); - } - } - } - return reading; -} - -std::vector GeneralWorkspace::monitor_reading() { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - if (!monitor_bound_) - throw std::logic_error("no boundary monitor is bound"); - const auto current = current_; - const auto spectrum = statistics_spectrum_; - const auto time = statistics_time_; - const auto weights = monitor_weights_; - const auto column_weights = monitor_column_weights_; - const auto n = problem_.n_omega; - const auto n_t = problem_.n_t; - const bool envelope = problem_.family == FieldFamily::envelope; - // A one-sided spectrum's last bin is the Nyquist bin only on an even axis. - const bool nyquist = n_t % 2 == 0; - const auto columns = problem_.n_columns(); - MonitorSums sums; - // Half the analytic signal's transform work, on buffers that already exist. - if (problem_.split_step_absorber && !envelope) { - const auto coarse = coarse_spectrum_; - const auto samples = coarse_time_real_; - const double inverse_n = 1.0 / static_cast(n_t); - const auto chunk_rows = coarse_spectrum_.extent(0); - for (std::size_t first = 0; first < columns; first += chunk_rows) { - const auto count = std::min(chunk_rows, columns - first); - launch::parallel_for( - "solprop_native.monitor.coarse_spectrum", - Range2(execution_space_, {0, 0}, {count, n}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { - coarse(row, index) = inverse_n * current(first + row, index); - }); - graph::execute(*coarse_inverse_real_plan_, coarse, samples, - KokkosFFT::Normalization::none); - sums += monitor_sums( - "solprop_native.monitor.sums_real", execution_space_, count, n_t, weights, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { - const double value = samples(row, index); - return column_weights(first + row) * value * value; - }); - } - } - const auto rows = statistics_spectrum_.extent(0); - const bool analytic = !(problem_.split_step_absorber && !envelope); - for (std::size_t first = 0; analytic && first < columns; first += rows) { - const auto count = std::min(rows, columns - first); - // Every bin of the rows read is written, the empty half of a one-sided spectrum - // with zeros; a partial chunk's other rows are transformed but never read. - launch::parallel_for( - "solprop_native.monitor.spectrum", - Range2(execution_space_, {0, 0}, {count, n_t}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { - if (index >= n) { - spectrum(row, index) = Complex{}; - return; - } - const bool single = envelope || index == 0 || (nyquist && index + 1 == n); - spectrum(row, index) = (single ? 1.0 : 2.0) / static_cast(n_t) * - current(first + row, index); - }); - graph::execute(*statistics_inverse_plan_, spectrum, time, - KokkosFFT::Normalization::none); - sums += monitor_sums( - "solprop_native.monitor.sums", execution_space_, count, n_t, weights, - KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { - const Complex value = time(row, index); - return column_weights(first + row) * - (envelope ? squared_abs(value) : value.real() * value.real()); - }); - } - double removed = 0.0; - SplitSums found; - if (problem_.split_step_absorber) { - execution_space_.fence("solprop_native monitor reading"); - removed = - Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, monitor_removed_)(); - found = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, monitor_found_)(); - graph::deep_copy(execution_space_, monitor_removed_, 0.0); - } - std::vector reading{sums.values[0], sums.values[1], sums.values[2], - sums.values[3], sums.values[4], removed, - found.values[1], found.values[2], found.values[3], - found.values[4]}; - if (occupancy_count_ != 0) { - // The state's energy per walk-off bucket, beside the five: a state column is - // `mode * n_pol + pol`, and the buckets are by mode. A reduction per column, whose - // per-thread histograms need no atomics: on a device an atomic per element took - // most of a reading, several steps' time on an AMD Instinct MI325X. - const auto buckets = occupancy_buckets_; - const auto n_pol = problem_.n_pol_state; - for (std::size_t first = 0; first < occupancy_count_; - first += occupancy_reduction_slots) { - OccupancySums occupancy; - launch::parallel_reduce( - "solprop_native.monitor.occupancy", - Kokkos::RangePolicy(execution_space_, 0, columns), - KOKKOS_LAMBDA(const std::size_t row, OccupancySums& update) { - const std::size_t offset = (row / n_pol) * n; - const double weight = column_weights(row); - for (std::size_t index = 0; index < n; ++index) { - const std::size_t bucket = buckets(offset + index); - if (bucket >= first && bucket - first < occupancy_reduction_slots) - update.values[bucket - first] += - weight * squared_abs(current(row, index)); - } - }, - Kokkos::Sum(occupancy)); - reading.insert(reading.end(), occupancy.values.data(), - occupancy.values.data() + - std::min(occupancy_reduction_slots, occupancy_count_ - first)); - } - } - execution_space_.fence("solprop_native monitor reading"); - return reading; -} - -// Bind the split collar's walk-off bucket of each (mode, frequency) bin (solprop -// #440): from here on a monitor reading also returns the state's energy per bucket. -// The table is allocated at the first bind and kept. -void GeneralWorkspace::bind_occupancy(const std::vector& buckets, - const std::size_t count) { - require_open(); - require_extent(buckets.size(), problem_.n_modes * problem_.n_omega, - "occupancy buckets"); - if (count == 0 || count > occupancy_bucket_limit) - throw std::invalid_argument("the occupancy needs between 1 and 128 buckets"); - for (const auto bucket : buckets) - if (bucket >= count) - throw std::invalid_argument("an occupancy bucket lies past the bucket count"); - if (occupancy_count_ == 0) - occupancy_buckets_ = - ByteView("solprop_native.monitor.occupancy_buckets", buckets.size()); - occupancy_count_ = count; - auto host = Kokkos::create_mirror_view(occupancy_buckets_); - for (std::size_t index = 0; index < buckets.size(); ++index) - host(index) = buckets[index]; - graph::deep_copy(execution_space_, occupancy_buckets_, host); - execution_space_.fence("solprop_native occupancy bind"); -} - -// The split-step temporal absorber on a separable geometry: the rate is the same at -// every node, so it acts on the temporal rows (free space's nodes, the planar kind's -// y rows of slab amplitudes) between the two transverse transforms. -void GeneralWorkspace::separable_split_step(const double dz, - const ComplexView2& advanced) { - if (split_rows_ != 0) { - split_state_rows(dz, advanced); - return; - } - if (problem_.free_geometry()) { - // The split's factor depends on time alone, so it acts alike on every column of - // any transverse basis (solprop's reference measures its removal on the stored - // columns for this reason): free space splits its stored transverse-spectrum - // columns directly, chunk by chunk of the temporal rows, as a modal route splits - // its modes, with no transverse transform pair. Equal to the reference's split in - // real space up to the transforms' rounding. - const auto coarse = coarse_spectrum_; - const auto active = active_mask_; - const auto n_pol = problem_.n_pol_state; - const auto n = problem_.n_omega; - const double inverse_time = 1.0 / static_cast(problem_.n_t); - for (std::size_t unit = 0; unit < transverse_units(); - unit += transverse_units_per_chunk()) { - const auto chunk = transverse_chunk(unit); - const auto first = chunk.first_row * n_pol; - const auto rows = chunk.rows * n_pol; - launch::parallel_for( - "solprop_native.free.split_columns_input", - Range2(execution_space_, {0, 0}, {rows, n}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - coarse(row, omega) = inverse_time * advanced(first + row, omega); - }); - split_columns(dz, rows, 0.0, first); - launch::parallel_for( - "solprop_native.free.split_columns_output", - Range2(execution_space_, {0, 0}, {rows, n}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - advanced(first + row, omega) = - active(omega) != 0U ? coarse(row, omega) : Complex{}; - }); - } - return; - } - const auto scratch = transverse_; - const auto coarse = coarse_spectrum_; - const auto active = active_mask_; - const auto n_pol = problem_.n_pol_state; - const auto layout = transverse_layout(); - const auto n = problem_.n_omega; - const double inverse_nodes = inverse_transverse_count(); - const double inverse_time = 1.0 / static_cast(problem_.n_t); - launch::parallel_for( - "solprop_native.free.split_input", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - const auto mode = row / n_pol; - layout.cell(scratch, omega * n_pol + row % n_pol, mode) = - inverse_nodes * advanced(row, omega); - }); - transverse_inverse(); - for (std::size_t unit = 0; unit < transverse_units(); - unit += transverse_units_per_chunk()) { - const auto chunk = transverse_chunk(unit); - const auto first = chunk.first_row; - const auto count = chunk.rows; - graph::deep_copy(execution_space_, coarse, Complex{}); - launch::parallel_for( - "solprop_native.free.split_temporal_input", - Range2(execution_space_, {0, 0}, {count * n_pol, n}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - const auto mode = first + row / n_pol; - coarse(row, omega) = - inverse_time * layout.cell(scratch, omega * n_pol + row % n_pol, mode); - }); - // The rows are real-space rows here: by Parseval their power is the stored - // columns' over the transverse count, which the monitor's measure is in. - split_columns(dz, count * n_pol, 1.0 / inverse_nodes); - launch::parallel_for( - "solprop_native.free.split_temporal_output", - Range2(execution_space_, {0, 0}, {count * n_pol, n}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - const auto mode = first + row / n_pol; - layout.cell(scratch, omega * n_pol + row % n_pol, mode) = coarse(row, omega); - }); - } - transverse_forward(); - launch::parallel_for( - "solprop_native.free.split_output", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), - KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { - const auto mode = row / n_pol; - advanced(row, omega) = - active(omega) != 0U - ? layout.cell(scratch, omega * n_pol + row % n_pol, mode) - : Complex{}; - }); -} - -void GeneralWorkspace::form_split_factor(const double dz) { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const auto rate = absorber_coarse_; - const auto factor = split_factor_; - launch::parallel_for( - "solprop_native.absorber.split_factor", - Kokkos::RangePolicy(execution_space_, 0, problem_.n_t), - KOKKOS_LAMBDA(const std::size_t index) { - factor(index) = Kokkos::exp(-0.5 * rate(index) * dz); - }); -#else - static_cast(dz); -#endif -} - -void GeneralWorkspace::split_step(const double dz, const ComplexView2& advanced) { - form_split_factor(dz); - if (monitoring_split()) { - // The chunks below add what this step's split finds; the last step's is dropped. - const auto found = monitor_found_; - launch::parallel_for( - "solprop_native.monitor.split_found_clear", - Kokkos::RangePolicy(execution_space_, 0, 1), - KOKKOS_LAMBDA(const std::size_t) { found() = SplitSums{}; }); - } - if (problem_.separable_geometry()) { - separable_split_step(dz, advanced); - return; - } - const auto coarse = coarse_spectrum_; - const double inverse_n = 1.0 / static_cast(problem_.n_t); - launch::parallel_for( - "solprop_native.absorber.split_pad", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - coarse(column, index) = inverse_n * advanced(column, index); - }); - split_columns(dz); - const auto active = active_mask_; - launch::parallel_for( - "solprop_native.absorber.split_commit", - Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.n_omega}), - KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { - advanced(column, index) = - active(index) != 0U ? coarse(column, index) : Complex{}; - }); -} - -std::vector GeneralWorkspace::field_to_host() const { - std::vector result(problem_.n_columns() * problem_.n_omega); - field_to_host(result); - return result; -} - -// One copy from the memory space into the caller's buffer: no staging mirror and no -// element loop, so a large state crosses once (a device's one transfer). -void GeneralWorkspace::cross_to_host(const ComplexView2& source, - const std::span destination) const { - crossed_.to_host += destination.size_bytes(); - if (crossing::takes_ring(destination.size_bytes())) { - crossing_ring_.to_host(destination.data(), source.data(), destination.size_bytes()); - return; - } - const Kokkos::View> - host(destination.data(), source.extent(0), source.extent(1)); - // On the stage's instance: a compact problem's team, which the default instance's - // whole-runtime copy would end and restart. - graph::deep_copy(execution_space_, host, source); - execution_space_.fence("solprop_native crossing"); -} - -void GeneralWorkspace::field_to_host(const std::span destination) const { - require_open(); - if (!initialized_) - throw std::logic_error("no initial field has been loaded"); - require_extent(destination.size(), problem_.n_columns() * problem_.n_omega, - "field destination"); - execution_space_.fence("solprop_native field_to_host"); - cross_to_host(current_, destination); -} - -std::vector GeneralWorkspace::stage_to_host(const std::size_t stage) const { - std::vector result(problem_.n_columns() * problem_.n_omega); - stage_to_host(stage, result); - return result; -} - -void GeneralWorkspace::stage_to_host(const std::size_t stage, - const std::span destination) const { - require_open(); - if (!stages_ready_) - throw std::logic_error("no trial stages have been evaluated"); - if (stage >= problem_.n_stages) - throw std::out_of_range("stage index is outside the tableau"); - require_extent(destination.size(), problem_.n_columns() * problem_.n_omega, - "stage destination"); - execution_space_.fence("solprop_native stage_to_host"); - const auto selected = Kokkos::subview(stages_, stage, Kokkos::ALL, Kokkos::ALL); - crossed_.to_host += destination.size_bytes(); - if (crossing::takes_ring(destination.size_bytes())) { - crossing_ring_.to_host(destination.data(), selected.data(), - destination.size_bytes()); - return; - } - const auto host = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, selected); - for (std::size_t column = 0; column < problem_.n_columns(); ++column) - for (std::size_t index = 0; index < problem_.n_omega; ++index) - destination[column * problem_.n_omega + index] = host(column, index); -} - -std::vector GeneralWorkspace::projected_to_host(const bool coarse) { - require_open(); - if (!stages_ready_) - throw std::logic_error("no trial stages have been evaluated"); - if (coarse && projected_coarse_spectrum_.extent(0) == 0) - throw std::logic_error("this quadrature has no embedded rule"); - if (coarse && dispersive_) - dispersive_->finish_coarse(); - if (coarse) - finish_quadrature(); - execution_space_.fence("solprop_native projected_to_host"); - const auto host = Kokkos::create_mirror_view_and_copy( - Kokkos::HostSpace{}, coarse ? projected_coarse_spectrum_ : projected_spectrum_); - return {host.data(), host.data() + host.size()}; -} - -std::map GeneralWorkspace::dispersive_path() const { - if (separable_dispersive_) - return {{"alone", separable_dispersive_->alone()}, - {"in_place", separable_dispersive_->in_place()}, - {"borrowed", separable_dispersive_->borrowed()}}; - // The general route: alone when no regional tensor or delayed response runs. - if (general_ && general_->dispersive() != nullptr) - return {{"alone", !general_->regional()}, {"in_place", false}, {"borrowed", false}}; - if (!dispersive_) - return {}; - return {{"alone", dispersive_->alone()}, - {"in_place", dispersive_->in_place()}, - {"borrowed", dispersive_->borrowed()}}; -} - -std::map GeneralWorkspace::memory_ledger() const { - auto ledger = - general_ ? general_->memory_ledger() : GeneralModalSource::empty_ledger(); - ledger.merge(flow_ ? flow_->memory_ledger() : CoupledFlowAction::empty_ledger()); - ledger.merge(dispersive_ ? dispersive_->memory_ledger() - : separable_dispersive_ ? separable_dispersive_->memory_ledger() - : general_ && general_->dispersive() != nullptr - ? general_->dispersive()->memory_ledger() - : DispersiveResponse::empty_ledger()); - ledger.merge(base_memory_ledger()); - return ledger; -} - -std::map GeneralWorkspace::base_memory_ledger() const { - std::size_t edge_plan_heap = 0; - std::size_t edge_plan_device = 0; - for (const auto& [shape, batch] : edge_batches_) { - edge_plan_heap += batch.heap_growth; - edge_plan_device += batch.device_growth; - } - const auto complex_bytes = sizeof(Complex); - const auto real_bytes = sizeof(double); - const auto byte_bytes = sizeof(unsigned char); - const auto index_bytes = sizeof(std::size_t); - const auto field = problem_.n_columns() * problem_.n_omega * complex_bytes; - const auto modal = - modal_real_.size() * real_bytes + modal_complex_.size() * complex_bytes; - // Aliased views (free space, SEP 28) own no storage of their own. - const auto node = separable_aliased_ ? 0 - : node_real_.size() * real_bytes + - node_complex_.size() * complex_bytes; - const auto polarisation = separable_aliased_ - ? 0 - : polarisation_real_.size() * real_bytes + - polarisation_complex_.size() * complex_bytes; - const auto projected = - projected_real_.size() * real_bytes + projected_complex_.size() * complex_bytes; - return { - {"held_current_host_derivative", - problem_.time_window_derivative.capacity() * real_bytes}, - {"held_current", held_current_.size() * real_bytes}, - {"gated_current_prefix", - (gated_prefix_.size() + gated_filtered_.size()) * real_bytes}, - {"held_current_gj", held_streams_[0].bytes()}, - {"held_current_edge", held_streams_[1].bytes()}, - {"held_current_nonplasma", held_streams_[2].bytes()}, - {"held_current_projection", held_projection_.size() * real_bytes}, - {"held_current_radial_spectra", - (held_nodal_spectrum_.size() + held_projected_spectrum_.size()) * complex_bytes}, - {"held_current_tables", - (held_derivative_.size() + held_inverse_.size()) * real_bytes}, - {"envelope_drude_stream", envelope_streams_[0].bytes()}, - {"envelope_loss_stream", envelope_streams_[1].bytes()}, - {"envelope_cycles", - (envelope_rule_nodes_.size() + envelope_rule_weights_.size() + - envelope_moments_.size() + envelope_rates_.size() + envelope_exposure_.size()) * - real_bytes}, - {"envelope_factors", envelope_drude_factor_.size() * real_bytes + - envelope_loss_factor_.size() * complex_bytes}, - {"envelope_status", (envelope_failures_.size() + envelope_status_.size() + - envelope_host_status_.size()) * - index_bytes}, - {"radial_blas_buffers", - (radial_blas_input_.size() + radial_blas_output_.size()) * real_bytes}, - {"state", field}, - {"advanced", advanced_.size() * complex_bytes}, - {"stages", stages_.size() * complex_bytes}, - {"accumulator_solution", solution_.size() * complex_bytes}, - {"accumulator_error", error_.size() * complex_bytes}, - {"weighted_residual", residual_.size() * complex_bytes}, - {"recycled_stage", recycled_.size() * complex_bytes}, - // A native session's Runge-Kutta dense output (solprop SEP 47 section 5.3). - {"rk_dense", - (rk_dense_coefficients_.size() + rk_dense_held_.size()) * complex_bytes + - rk_dense_stiff_.size() * byte_bytes}, - {"noise", problem_.modified_noise ? field : 0}, - // The Runge-Kutta march's whole-step propagator, or the low-storage march's - // table or step pair (solprop's `propagators` row). - {"propagators", - (whole_propagator_.size() + low_storage_flows_.size()) * complex_bytes}, - {"register_accumulator", register_accumulator_.size() * complex_bytes}, - {"register_stage", register_stage_.size() * complex_bytes}, - {"stage_phase", stage_phase_.size() * complex_bytes}, - {"stage_inverse", stage_inverse_.size() * complex_bytes}, - {"fine_spectrum", fine_spectrum_.size() * complex_bytes}, - {"retained_synthesis", - (retained_modal_.size() + retained_node_.size()) * complex_bytes}, - {"fine_gaps", (fine_gaps_.size() + separable_gaps_.size()) * index_bytes}, - {"modal_time", modal}, - {"node_field", node}, - {"node_polarisation", polarisation}, - {"projected", projected}, - {"projected_coarse", problem_.projection_coarse_matrix.empty() ? 0 : projected}, - {"projected_spectrum", projected_spectrum_.size() * complex_bytes}, - {"projected_coarse_spectrum", projected_coarse_spectrum_.size() * complex_bytes}, - {"absorber_spectrum", absorber_spectrum_.size() * complex_bytes}, - {"absorber_time", - absorber_complex_.size() * complex_bytes + absorber_real_.size() * real_bytes}, - {"split_absorber_spectrum", coarse_spectrum_.size() * complex_bytes}, - {"split_absorber_time", coarse_time_complex_.size() * complex_bytes + - coarse_time_real_.size() * real_bytes}, - {"transform_matrices", - (synthesis_.size() + projection_.size() + projection_coarse_.size()) * - real_bytes}, - // The native problem owns host copies as well as the execution-space views. - // Count retained vector capacity, including construction-only coefficients. - {"host_problem_arrays", - (problem_.envelope_loss_factor.capacity() + problem_.carrier.capacity() + - problem_.general_synthesis.capacity() + problem_.general_projection.capacity() + - problem_.general_frequency_factor.capacity() + - problem_.general_delayed_spectrum.capacity() + problem_.phi_terms.capacity() + - problem_.prefactor_terms.capacity() + problem_.spatial_on_axis.capacity() + - problem_.spatial_k_squared.capacity() + - problem_.spatial_k_squared_low.capacity() + - problem_.spatial_numerator.capacity() + problem_.spatial_clamped.capacity() + - problem_.spatial_kappa_sq.capacity()) * - complex_bytes + - (problem_.envelope_rule_nodes.capacity() + - problem_.envelope_rule_weights.capacity() + - problem_.envelope_drude_factor.capacity() + - problem_.omega_window.capacity() + problem_.omega.capacity() + - problem_.power_weight.capacity() + problem_.time_window.capacity() + - problem_.absorber_fine.capacity() + problem_.absorber_coarse.capacity() + - problem_.spatial_absorber_matrix.capacity() + - problem_.spatial_spectral_weights.capacity() + - problem_.spatial_kperp_sq.capacity() + - problem_.spatial_kperp_sq_low.capacity() + - problem_.spatial_frame.capacity() + - problem_.spatial_spectral_rate.capacity() + - problem_.spatial_user_rate.capacity() + - problem_.spatial_absorber_rate.capacity() + - problem_.spatial_kperp_rate.capacity() + problem_.spatial_k_x.capacity() + - problem_.spatial_k_y.capacity() + problem_.synthesis_matrix.capacity() + - problem_.projection_matrix.capacity() + - problem_.projection_coarse_matrix.capacity() + - problem_.tableau_matrix.capacity() + problem_.tableau_nodes.capacity() + - problem_.solution_weights.capacity() + problem_.error_weights.capacity() + - problem_.raman_undamped.capacity() + - problem_.raman_tensor_weights.capacity() + - problem_.rate_coefficients.capacity() + - problem_.channel_parameters.capacity() + - problem_.channel_coefficients.capacity() + - problem_.raman_row_component.capacity() + - problem_.raman_component_fraction.capacity() + - problem_.general_tensor.capacity() + - problem_.general_delayed_tensor.capacity()) * - real_bytes + - problem_.general_node_response.capacity() * index_bytes + - problem_.crop_index.capacity() * index_bytes + - (problem_.active_mask.capacity() + problem_.norm_mask.capacity()) * - byte_bytes}, - {"scaled_transform_matrices", - (scaled_synthesis_.size() + scaled_projection_.size() + - scaled_projection_coarse_.size() + scaled_projection_stacked_.size()) * - real_bytes + - (scaled_synthesis_complex_.size() + scaled_projection_complex_.size() + - scaled_projection_coarse_complex_.size() + - scaled_projection_stacked_complex_.size()) * - complex_bytes}, - {"stacked_projection", projected_stacked_real_.size() * real_bytes + - projected_stacked_complex_.size() * complex_bytes}, - {"chunk_transform_buffers", - (chunk_synthesis_.size() + tail_synthesis_.size() + chunk_projection_.size() + - tail_projection_.size() + chunk_projection_stacked_.size() + - tail_projection_stacked_.size()) * - real_bytes + - (chunk_synthesis_complex_.size() + tail_synthesis_complex_.size() + - chunk_projection_complex_.size() + tail_projection_complex_.size() + - chunk_projection_stacked_complex_.size() + - tail_projection_stacked_complex_.size()) * - complex_bytes}, - {"transverse_scratch", transverse_.size() * complex_bytes}, - {"spatial_coefficients", - (spatial_.square.size() + spatial_.square_low.size() + - spatial_.numerator.size() + spatial_.clamped.size() + spatial_.kappa.size()) * - complex_bytes + - (spatial_.transverse.size() + spatial_.transverse_low.size() + - spatial_.frame.size() + spatial_.spectral_rate.size() + - spatial_.user_rate.size() + spatial_.absorber.size() + - spatial_.transverse_rate.size()) * - real_bytes}, - {"adams_history", - (adams_store_.size() + adams_slopes_.size() + adams_moments_.size() + - adams_source_.size() + adams_propagator_.size() + adams_exponent_.size() + - adams_saved_.size() + adams_held_.size() + adams_dense_start_.size() + - adams_dense_end_.size() + adams_dense_exponent_.size() + adams_states_.size() + - adams_rates_.size() + adams_carried_.size() + adams_dense_terms_.size() + - adams_lab_.size() + adams_endpoint_.size() + adams_endpoint_rate_.size() + - adams_block_.size() + adams_channels_.size() + adams_fallback_.size() + - adams_applied_.size() + adams_end_applied_.size()) * - complex_bytes + - (adams_divisor_.size() + adams_rate_weights_.size() + - adams_end_weights_.size() + adams_scale_weights_.size() + - (adams_magnitude_sum_.data() != nullptr ? 1 : 0)) * - real_bytes}, - {"fixed_phi_terms", phi_terms_.size() * complex_bytes}, - {"fixed_prefactor_terms", prefactor_terms_.size() * complex_bytes}, - {"z_tables", phi_table_.bytes() + prefactor_table_.bytes() + rate_table_.bytes()}, - {"statistics_metric", statistics_whitened_.size() * complex_bytes + - statistics_root_.size() * complex_bytes + - statistics_whitened_sums_.size() * real_bytes}, - // solprop SEP 34: a radial run's plane-measure squares and its material table - // (a separable run's alias its operator's), and the magnetic closure's - // transverse volume of a separable run's local flux statistics. - {"flux_coefficients", - problem_.separable_geometry() - ? 0 - : (flux_.square.size() + flux_.square_low.size()) * complex_bytes + - (flux_.transverse.size() + flux_.transverse_low.size()) * real_bytes}, - {"flux_table", flux_table_.bytes()}, - {"flux_closure_transverse", flux_transverse_.size() * complex_bytes}, - {"tableau", tableau_.size() * real_bytes}, - // The per-stage scalars are held twice, in the packed buffer's two slots, and - // a device's pinned host ring holds copies of the whole buffer. - {"stage_scalars", - 2 * - (stage_phi_delta_.size() + stage_phi_at_.size() + stage_prefactor_.size() + - stage_density_.size() + stage_taper_.size() + stage_preionisation_.size() + - stage_rate_weights_.size()) * - real_bytes}, - {"stage_scalar_staging", - stage_staging_.empty() || stage_staging_.front().data() == stage_scalars_.data() - ? 0 - : stage_staging_.size() * stage_scalars_.size() * real_bytes}, - // An Adams trial's norms, read back in pinned host memory on a device, and a - // Runge-Kutta trial's mirror of its partials (the view itself on a host build). - {"trial_norms", - (adams_norms_.size() + - (pinned_partials_.data() == partials_.data() ? 0 : pinned_partials_.size())) * - real_bytes}, - {"crop_index", crop_index_.size() * index_bytes}, - {"masks", (active_mask_.size() + norm_mask_.size()) * byte_bytes}, - {"windows", (omega_window_.size() + time_window_.size()) * real_bytes}, - {"spatial_statistics_geometry", spatial_spectral_weights_.size() * real_bytes + - spatial_on_axis_.size() * complex_bytes}, - {"spatial_statistics_partials", spatial_statistics_partials_.size() * real_bytes}, - {"far_field_partials", far_field_ ? far_field_->partial_bytes() : 0}, - {"far_field_tables", far_field_ ? far_field_->table_bytes() : 0}, - {"statistics_frequency_axis", omega_.size() * real_bytes}, - {"carrier", carrier_.size() * complex_bytes}, - {"node_raman_drive", node_raman_drive_.size() * real_bytes}, - {"node_raman_spectrum", node_raman_spectrum_.size() * complex_bytes}, - {"node_raman_convolved", node_raman_convolved_.size() * real_bytes}, - {"raman_undamped", raman_undamped_.size() * real_bytes}, - {"raman_tensor_weights", raman_tensor_weights_.size() * real_bytes}, - {"raman_kernel_time", raman_kernel_time_.size() * real_bytes}, - {"raman_kernel_spectrum", raman_kernel_spectrum_.size() * complex_bytes}, - {"stage_raman_rates", 2 * stage_raman_rates_.size() * real_bytes}, - {"absorber_rates", - (absorber_fine_.size() + absorber_coarse_.size() + split_factor_.size()) * - real_bytes}, - {"spatial_absorber_matrix", spatial_absorber_real_.size() * real_bytes + - spatial_absorber_complex_.size() * complex_bytes}, - {"plasma_rate_table", rate_coefficients_.size() * real_bytes}, - {"channel_parameters", channel_parameters_.size() * real_bytes}, - {"channel_rate_tables", channel_coefficients_.size() * real_bytes}, - {"channel_scratch", channel_scratch_.size() * real_bytes}, - {"material_stage_values", - 2 * - (stage_kerr_density_.size() + stage_plasma_density_.size() + - stage_raman_density_.size() + stage_channel_density_.size() + - stage_channel_preionisation_.size()) * - real_bytes}, - {"raman_component_values", - (2 * stage_raman_densities_.size() + raman_row_component_.size() + - raman_component_fraction_.size() + cached_raman_densities_.capacity()) * - real_bytes}, - {"plasma_stash", (stash_candidate_.size() + stash_current_.size()) * real_bytes}, - {"plasma_statistics_scratch", plasma_statistics_scratch_.size() * real_bytes}, - {"plasma_integration_diagnostics", - (plasma_taps_.size() + plasma_diagnostics_.size()) * real_bytes}, - {"plasma_device_scan", - (plasma_rate_.size() + plasma_exposure_.size() + plasma_fraction_.size() + - plasma_current_integrand_.size() + plasma_free_current_.size() + - plasma_outer_current_.size() + band_tile_totals_.size() + - plasma_tile_totals_.size()) * - real_bytes}, - {"error_partials", 3 * problem_.n_columns() * reduction_slots_ * real_bytes}, - {"quadrature_partials", problem_.projection_coarse_matrix.empty() - ? 0 - : 2 * problem_.n_columns() * real_bytes}, - {"weighted_control", - (weighted_scale_.size() + weighted_partials_.size()) * real_bytes}, - {"statistics_spectrum", statistics_spectrum_.size() * complex_bytes}, - {"statistics_time", statistics_time_.size() * complex_bytes}, - {"statistics_weighted_time", statistics_weighted_time_.size() * complex_bytes}, - {"statistics_power_weight", statistics_power_weight_.size() * sizeof(double)}, - {"statistics_power", statistics_power_.size() * real_bytes}, - // The boundary monitor's weights and accumulators, once a monitor is bound. - {"monitor_weights", - (monitor_weights_.size() + monitor_column_weights_.size() + - monitor_share_.size() + monitor_half_.size() + monitor_removed_.size()) * - real_bytes + - occupancy_buckets_.size() + - (monitor_split_sum_.size() + monitor_found_.size()) * sizeof(SplitSums)}, - {"statistics_flux_magnitude", statistics_magnitude_.size() * real_bytes}, - {"host_crossing_ring", crossing_ring_.bytes()}, - {"transverse_edge", edge_bytes()}, - {"statistics_outputs", (statistics_total_power_.size() + - statistics_on_axis_.size() + statistics_packed_.size()) * - real_bytes + - statistics_axis_row_.size() * complex_bytes}, - {"fft_plan_heap_growth", plan_heap_growth_ + edge_plan_heap}, - {"fft_plan_device_growth", plan_device_growth_ + edge_plan_device}, - }; -} - -void GeneralWorkspace::close() noexcept { - if (closed_) - return; - execution_space_.fence(); - if (general_) - general_->close(); - general_.reset(); - if (flow_) - flow_->close(); - flow_.reset(); - if (dispersive_) - dispersive_->close(); - dispersive_.reset(); - if (far_field_) - far_field_->close(); - far_field_.reset(); - if (separable_dispersive_) - separable_dispersive_->close(); - separable_dispersive_.reset(); - for (auto& stream : held_streams_) { - stream.forward.reset(); - stream.coarse_forward.reset(); - stream = {}; - } - held_pack_forward_.reset(); - held_pack_ = {}; - held_pack_spectrum_ = {}; - forward_pack_plan_.reset(); - forward_pack_tail_plan_.reset(); - forward_pack_fine_plan_.reset(); - forward_pack_fine_tail_plan_.reset(); - forward_pack_coarse_plan_.reset(); - forward_pack_fine_ = {}; - forward_pack_fine_tail_ = {}; - forward_pack_coarse_ = {}; - forward_pack_fine_spectrum_ = {}; - forward_pack_fine_tail_spectrum_ = {}; - forward_pack_coarse_spectrum_ = {}; - forward_pack_ = {}; - forward_pack_tail_ = {}; - forward_pack_spectrum_ = {}; - forward_pack_tail_spectrum_ = {}; - held_current_ = {}; - gated_prefix_ = {}; - gated_filtered_ = {}; - held_projection_ = {}; - held_nodal_spectrum_ = {}; - held_projected_spectrum_ = {}; - held_derivative_ = {}; - held_inverse_ = {}; - for (auto& stream : envelope_streams_) - stream = {}; - envelope_rule_nodes_ = {}; - envelope_rule_weights_ = {}; - envelope_moments_ = {}; - envelope_rates_ = {}; - envelope_exposure_ = {}; - envelope_drude_factor_ = {}; - envelope_loss_factor_ = {}; - envelope_failures_ = {}; - envelope_status_ = {}; - envelope_host_status_ = {}; - radial_blas_input_ = {}; - radial_blas_output_ = {}; - transverse_forward_.reset(); - transverse_inverse_.reset(); - volume_forward_.reset(); - volume_inverse_.reset(); - volume_scratch_ = {}; -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - transform_work_ = {}; -#endif - slab_forward_.reset(); - slab_inverse_.reset(); - transverse_ = {}; - spatial_ = {}; - flux_ = {}; - flux_table_ = {}; - flux_transverse_ = {}; - statistics_magnitude_ = {}; - temporal_tail_inverse_real_.clear(); - temporal_tail_inverse_complex_.clear(); - temporal_tail_forward_real_.clear(); - temporal_tail_forward_complex_.clear(); - coarse_tail_inverse_real_.clear(); - coarse_tail_inverse_complex_.clear(); - coarse_tail_forward_real_.clear(); - coarse_tail_forward_complex_.clear(); - inverse_real_plan_.reset(); - inverse_complex_plan_.reset(); - forward_real_plan_.reset(); - forward_complex_plan_.reset(); - forward_coarse_real_plan_.reset(); - forward_coarse_complex_plan_.reset(); - statistics_inverse_plan_.reset(); - absorber_real_plan_.reset(); - absorber_complex_plan_.reset(); - coarse_inverse_real_plan_.reset(); - split_inverse_plan_.reset(); - split_forward_plan_.reset(); - split_tail_inverse_plan_.reset(); - split_tail_forward_plan_.reset(); - coarse_inverse_complex_plan_.reset(); - coarse_forward_real_plan_.reset(); - coarse_forward_complex_plan_.reset(); - raman_drive_forward_plan_.reset(); - raman_convolution_inverse_plan_.reset(); - raman_tail_forward_plan_.reset(); - raman_tail_inverse_plan_.reset(); - raman_tail_drive_ = {}; - raman_tail_spectrum_ = {}; - raman_tail_convolved_ = {}; - retained_inverse_plan_.reset(); - retained_tail_inverse_plan_.reset(); - retained_modal_ = {}; - retained_node_ = {}; - raman_product_forward_plan_.reset(); - raman_product_inverse_plan_.reset(); - raman_product_drive_ = {}; - raman_product_spectrum_ = {}; - raman_kernel_forward_plan_.reset(); - current_ = {}; - register_accumulator_ = {}; - register_stage_ = {}; - low_storage_flows_ = {}; - advanced_ = {}; - stages_ = {}; - solution_ = {}; - error_ = {}; - residual_ = {}; - recycled_ = {}; - noise_ = {}; - whole_propagator_ = {}; - stage_phase_ = {}; - stage_inverse_ = {}; - adams_store_ = {}; - adams_slopes_ = {}; - adams_moments_ = {}; - adams_source_ = {}; - adams_propagator_ = {}; - adams_exponent_ = {}; - adams_saved_ = {}; - adams_held_ = {}; - adams_dense_start_ = {}; - adams_dense_end_ = {}; - adams_dense_exponent_ = {}; - adams_divisor_ = {}; - adams_norms_ = {}; - adams_source_graph_.reset(); - adams_states_ = {}; - adams_rates_ = {}; - adams_carried_ = {}; - adams_dense_terms_ = {}; - adams_lab_ = {}; - adams_endpoint_ = {}; - adams_endpoint_rate_ = {}; - adams_rate_weights_ = {}; - adams_block_ = {}; - adams_channels_ = {}; - adams_fallback_ = {}; - adams_applied_ = {}; - adams_end_applied_ = {}; - adams_end_weights_ = {}; - fine_spectrum_ = {}; - projected_spectrum_ = {}; - projected_coarse_spectrum_ = {}; - absorber_spectrum_ = {}; - statistics_spectrum_ = {}; - statistics_time_ = {}; - statistics_weighted_time_ = {}; - statistics_power_weight_ = {}; - modal_complex_ = {}; - node_complex_ = {}; - polarisation_complex_ = {}; - projected_complex_ = {}; - projected_coarse_complex_ = {}; - absorber_complex_ = {}; - coarse_spectrum_ = {}; - split_factor_ = {}; - coarse_time_complex_ = {}; - scaled_synthesis_complex_ = {}; - scaled_projection_complex_ = {}; - scaled_projection_coarse_complex_ = {}; - scaled_projection_stacked_complex_ = {}; - projected_stacked_complex_ = {}; - chunk_synthesis_complex_ = {}; - tail_synthesis_complex_ = {}; - chunk_projection_complex_ = {}; - tail_projection_complex_ = {}; - chunk_projection_stacked_complex_ = {}; - tail_projection_stacked_complex_ = {}; - node_raman_spectrum_ = {}; - raman_kernel_spectrum_ = {}; - modal_real_ = {}; - node_real_ = {}; - polarisation_real_ = {}; - projected_real_ = {}; - projected_coarse_real_ = {}; - projected_stacked_real_ = {}; - absorber_real_ = {}; - coarse_time_real_ = {}; - node_raman_drive_ = {}; - node_raman_convolved_ = {}; - raman_undamped_ = {}; - raman_tensor_weights_ = {}; - raman_kernel_time_ = {}; - stage_raman_rates_ = {}; - synthesis_ = {}; - projection_ = {}; - projection_coarse_ = {}; - scaled_synthesis_ = {}; - scaled_projection_ = {}; - scaled_projection_coarse_ = {}; - scaled_projection_stacked_ = {}; - chunk_synthesis_ = {}; - tail_synthesis_ = {}; - chunk_projection_ = {}; - tail_projection_ = {}; - chunk_projection_stacked_ = {}; - tail_projection_stacked_ = {}; - partials_ = {}; - pinned_partials_ = {}; - quadrature_partials_ = {}; - weighted_scale_ = {}; - weighted_partials_ = {}; - stash_candidate_ = {}; - stash_current_ = {}; - statistics_power_ = {}; - plasma_statistics_scratch_ = {}; - plasma_rate_ = {}; - plasma_exposure_ = {}; - plasma_fraction_ = {}; - plasma_current_integrand_ = {}; - plasma_free_current_ = {}; - plasma_outer_current_ = {}; - band_tile_totals_ = {}; - plasma_tile_totals_ = {}; - spatial_spectral_weights_ = {}; - spatial_on_axis_ = {}; - spatial_statistics_partials_ = {}; - statistics_total_power_ = {}; - statistics_on_axis_ = {}; - statistics_packed_ = {}; - statistics_axis_row_ = {}; - monitor_bound_ = false; - monitor_weights_ = {}; - monitor_column_weights_ = {}; - monitor_share_ = {}; - monitor_half_ = {}; - monitor_removed_ = {}; - monitor_split_sum_ = {}; - monitor_found_ = {}; - occupancy_buckets_ = {}; - occupancy_count_ = 0; - crossing_ring_.release(); - edge_batches_.clear(); - edge_strip_ = {}; - edge_lines_.clear(); - edge_scale_ = {}; - edge_faces_ = {}; - edge_clip_ = {}; - edge_totals_ = {}; - edge_synthesis_ = {}; - edge_columns_ = {}; - tableau_ = {}; - omega_window_ = {}; - omega_ = {}; - time_window_ = {}; - absorber_fine_ = {}; - absorber_coarse_ = {}; - spatial_absorber_real_ = {}; - spatial_absorber_complex_ = {}; - stage_phi_delta_ = {}; - stage_scalars_ = {}; - stage_rate_weights_ = {}; - stage_staging_.clear(); - stage_phi_at_ = {}; - stage_prefactor_ = {}; - stage_density_ = {}; - stage_taper_ = {}; - stage_preionisation_ = {}; - rate_coefficients_ = {}; - channel_parameters_ = {}; - channel_coefficients_ = {}; - channel_scratch_ = {}; - plasma_taps_ = {}; - plasma_diagnostics_ = {}; - stage_channel_density_ = {}; - stage_channel_preionisation_ = {}; - stage_kerr_density_ = {}; - stage_plasma_density_ = {}; - stage_raman_density_ = {}; - stage_raman_densities_ = {}; - raman_row_component_ = {}; - raman_component_fraction_ = {}; - cached_raman_densities_.clear(); - carrier_ = {}; - active_mask_ = {}; - norm_mask_ = {}; - crop_index_ = {}; - fine_gaps_ = {}; - separable_gaps_ = {}; - // FFT plans must release their execution instances before runtime shutdown. - raman_tail_execution_space_ = ExecutionSpace(); - raman_execution_space_ = ExecutionSpace(); - raman_kernel_execution_space_ = ExecutionSpace(); - raman_product_execution_space_ = ExecutionSpace(); - raman_product_space_ = ExecutionSpace(); - temporal_execution_space_ = ExecutionSpace(); - flow_execution_space_ = ExecutionSpace(); - absorber_execution_space_ = ExecutionSpace(); - adams_execution_space_ = ExecutionSpace(); - adams_copy_space_ = ExecutionSpace(); - raman_multiply_space_ = ExecutionSpace(); - raman_tail_multiply_space_ = ExecutionSpace(); - raman_rebuild_space_ = ExecutionSpace(); - raman_scale_space_ = ExecutionSpace(); - response_execution_space_ = ExecutionSpace(); - response_tail_execution_space_ = ExecutionSpace(); - transform_execution_space_ = ExecutionSpace(); - transform_tail_execution_space_ = ExecutionSpace(); - matrix_copy_space_ = ExecutionSpace(); - tail_matrix_copy_space_ = ExecutionSpace(); - stacked_matrix_copy_space_ = ExecutionSpace(); - tail_stacked_matrix_copy_space_ = ExecutionSpace(); - node_spectral_space_ = ExecutionSpace(); - kernel_pool_.clear(); - fft_pool_.clear(); - raman_pool_.clear(); - raman_tail_pool_.clear(); - fft_execution_space_ = ExecutionSpace(); - execution_space_ = ExecutionSpace(); - closed_ = true; - initialized_ = false; - table_ready_ = false; - has_recycled_ = false; - pending_fsal_ = false; - coarse_pending_ = false; - stash_is_current_ = false; - raman_kernel_ready_ = false; -} - -GeneralState::GeneralState(Problem problem) - : workspace_(std::make_unique(std::move(problem))) {} -GeneralState::~GeneralState() { close(); } -void GeneralState::set_initial(const std::span field, - const std::span noise) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->set_initial(field, noise); -} -std::vector GeneralState::trial_step(const double dz, - const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->trial_step(dz, scalars); -} -void GeneralState::prepare_trial(const double dz, const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->prepare_trial(dz, scalars); -} -std::vector GeneralState::adams_source(const std::vector& input, - const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_source(input, scalars); -} -void GeneralState::adopt(const std::vector& field, const double dz) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adopt(field, dz); -} -double GeneralState::low_storage_step(const double dz, const TrialScalars& scalars, - const bool rebuild_table) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->low_storage_step(dz, scalars, rebuild_table); -} -void GeneralState::adams_allocate(const std::size_t slots, const bool compensated, - const bool interpolate) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_allocate(slots, compensated, interpolate); -} -void GeneralState::adams_compensation_start(const std::size_t slot, - const std::vector& rate) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_compensation_start(slot, rate); -} -void GeneralState::adams_start(const std::size_t slot, const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_start(slot, scalars); -} -double GeneralState::adams_magnitude() { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_magnitude(); -} -void GeneralState::adams_scale(const double rtol, const double atol, - const std::vector& weights) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_scale(rtol, atol, weights); -} -std::vector GeneralState::adams_fused_trial(const double dz, - const AdamsWeights& weights, - const std::vector& divisor, - const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_fused_trial(dz, weights, divisor, scalars); -} -void GeneralState::adams_fused_accept(const double dz, const AdamsWeights& keep, - const std::size_t free, - const AdamsWeights& dense) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_fused_accept(dz, keep, free, dense); -} -void GeneralState::adams_hold() { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_hold(); -} -void GeneralState::adams_restore() { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_restore(); -} -void GeneralState::table_upload(const TableKind kind, std::vector edges, - const double lower, const double upper, - const std::complex* coefficients, - const std::size_t n_intervals, - const std::size_t n_terms, const std::size_t n_values) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->table_upload(kind, std::move(edges), lower, upper, coefficients, - n_intervals, n_terms, n_values); -} -void GeneralState::table_evaluate(const std::vector& slots, - const std::vector& positions) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->table_evaluate(slots, positions); -} -void GeneralState::table_rate(const std::size_t slot, const double z, - const bool before) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->table_rate(slot, z, before); -} -std::vector GeneralState::table_terms_to_host(const TableKind kind, - const std::size_t slot) const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->table_terms_to_host(kind, slot); -} -std::vector -GeneralState::phase_sum_to_host(const std::vector& weights) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->phase_sum_to_host(weights); -} -std::vector GeneralState::adams_row_names() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_row_names(); -} -std::array -GeneralState::adams_row_extents(const std::string& name) const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_row_extents(name); -} -std::vector GeneralState::adams_rows_to_host(const std::string& name) const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_rows_to_host(name); -} -void GeneralState::adams_rows_to_host(const std::string& name, - std::complex* destination, - const std::size_t size) const { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_rows_to_host(name, destination, size); -} -void GeneralState::adams_rows_from_host(const std::string& name, - const std::vector& values) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_rows_from_host(name, values); -} -void GeneralState::adams_observe(const AdamsWeights& weights, - const TrialScalars& scalars) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_observe(weights, scalars); -} -std::vector GeneralState::adams_exponential_trial( - const double dz, const ExponentialWeights& weights, - const std::vector& divisor, const TrialScalars& step, - const TrialScalars& sample, const std::vector& rate) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_exponential_trial(dz, weights, divisor, step, sample, rate); -} -void GeneralState::adams_exponential_accept(const double dz, const std::size_t free, - const AdamsWeights& dense) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_exponential_accept(dz, free, dense); -} -void GeneralState::rk_dense_allocate(const std::vector& weights) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->rk_dense_allocate(weights); -} -void GeneralState::rk_dense_hold(const double dz) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->rk_dense_hold(dz); -} -void GeneralState::rk_dense_observe(const double theta, const TrialScalars& carry) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->rk_dense_observe(theta, carry); -} -void GeneralState::rk_dense_restore() { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->rk_dense_restore(); -} -void GeneralState::rk_dense_release() noexcept { - if (workspace_) - workspace_->rk_dense_release(); -} -void GeneralState::adams_exponential_observe(const double xi, const double dz, - const ExponentialWeights& weights) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->adams_exponential_observe(xi, dz, weights); -} -std::vector GeneralState::weighted_error(const std::vector& scale) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->weighted_error(scale); -} -void GeneralState::set_flow_actions(const std::vector& forward, - const std::vector& inverse, - const std::vector& identity) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->set_flow_actions(forward, inverse, identity); -} -std::vector GeneralState::flow_actions_to_host(const bool inverse) const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->flow_actions_to_host(inverse); -} -std::vector GeneralState::adams_block_to_host() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_block_to_host(); -} -std::vector GeneralState::adams_exponent_to_host() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->adams_exponent_to_host(); -} -std::vector -GeneralState::form_flow_actions(const std::size_t section, const double dz, - const std::vector& fractions) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->form_flow_actions(section, dz, fractions); -} -void GeneralState::prepare_raman(const std::vector& rates, - const std::vector& densities) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->prepare_raman(rates, densities); -} -std::vector -GeneralState::statistics(const std::uint64_t flags, const std::vector& windows, - const std::vector& on_axis, const double density, - const double taper, const double preionisation, - const double intensity_factor, - const std::vector& channel_preionisation, - const std::vector& power_weight) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->statistics(flags, windows, on_axis, density, taper, preionisation, - intensity_factor, channel_preionisation, power_weight); -} -std::vector -GeneralState::whitened_statistics(const std::vector& root, - const std::uint64_t flags, - const std::vector& windows, - const double intensity_factor, const bool spectral) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->whitened_statistics(root, flags, windows, intensity_factor, - spectral); -} -std::vector GeneralState::spatial_statistics(const std::uint64_t flags, - const double intensity_factor, - const double z) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->spatial_statistics(flags, intensity_factor, z); -} -std::vector GeneralState::plane_fluence_trace( - const double center_x, const double center_y, const double dx, const double dy, - const double extent, const double spacing, const std::size_t n_radii, - const std::vector& cosines, const std::vector& sines) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->plane_fluence_trace(center_x, center_y, dx, dy, extent, spacing, - n_radii, cosines, sines); -} -std::vector GeneralState::axis_spectrum(const std::vector& matrix) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->axis_spectrum(matrix); -} -std::vector -GeneralState::axis_populations(const std::vector& matrix, - const std::vector& channel_preionisation) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->axis_populations(matrix, channel_preionisation); -} -std::vector GeneralState::aperture_statistics( - const std::uint64_t flags, const std::vector& apertures, const int exponent, - const std::vector& regions, const double intensity_factor) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->aperture_statistics(flags, apertures, exponent, regions, - intensity_factor); -} -void GeneralState::finish(const double dz) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->finish(dz); -} -void GeneralState::bind_monitor(const std::vector& weights, - const std::vector& column_weights) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->bind_monitor(weights, column_weights); -} -void GeneralState::unbind_monitor() noexcept { - if (workspace_) - workspace_->unbind_monitor(); -} -std::vector GeneralState::monitor_reading() { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->monitor_reading(); -} -void GeneralState::bind_edge(const std::vector& faces, - const std::vector& clip, - const std::vector& totals, - const std::vector& synthesis, - const std::size_t first_row) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->bind_edge(faces, clip, totals, synthesis, first_row); -} -CrossedBytes GeneralState::crossed_bytes() const noexcept { - return workspace_->crossed_bytes(); -} -void GeneralState::bind_edge_lines(const EdgeLines& lines) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->bind_edge_lines(lines); -} -EdgeNumbers GeneralState::edge_reading(const bool faces, const bool whole, - const bool totals, const bool strip, - const bool current) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->edge_reading(faces, whole, totals, strip, current); -} -void GeneralState::bind_occupancy(const std::vector& buckets, - const std::size_t count) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->bind_occupancy(buckets, count); -} -void GeneralState::promote_fsal() { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->promote_fsal(); -} -void GeneralState::clear_fsal() { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->clear_fsal(); -} -void GeneralState::set_absorber_quiet(const bool quiet) { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->set_absorber_quiet(quiet); -} -double GeneralState::quadrature_error() const { - return workspace_ ? workspace_->quadrature_error() - : Kokkos::Experimental::quiet_NaN_v; -} -double GeneralState::quadrature_peak() { - return workspace_ ? workspace_->quadrature_peak() - : Kokkos::Experimental::quiet_NaN_v; -} -std::string GeneralState::synthesis_provider() const { - return workspace_ ? workspace_->synthesis_provider() : "closed"; -} -std::string GeneralState::projection_provider() const { - return workspace_ ? workspace_->projection_provider() : "closed"; -} -std::string GeneralState::gemm_shape_policy() const { - const CpuTeamScope team(workspace_->cpu_team()); - return std::string(GeneralWorkspace::gemm_shape_policy()); -} -std::size_t GeneralState::response_team_size() const noexcept { - return workspace_ ? workspace_->response_team_size() : 0; -} -std::vector GeneralState::field_to_host() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->field_to_host(); -} -void GeneralState::set_crossing_workers(const std::size_t workers) noexcept { - if (workspace_) - workspace_->set_crossing_workers(workers); -} -void GeneralState::field_to_host(const std::span destination) const { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->field_to_host(destination); -} -const Complex* GeneralState::host_field() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->host_field(); -} -void GeneralState::count_to_host(const std::size_t bytes) const noexcept { - workspace_->count_to_host(bytes); -} -void GeneralState::stage_to_host(const std::size_t stage, - const std::span destination) const { - const CpuTeamScope team(workspace_->cpu_team()); - workspace_->stage_to_host(stage, destination); -} -std::vector GeneralState::stage_to_host(const std::size_t stage) const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->stage_to_host(stage); -} -std::vector GeneralState::projected_to_host(const bool coarse) { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->projected_to_host(coarse); -} -std::map GeneralState::dispersive_path() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->dispersive_path(); -} -std::map GeneralState::memory_ledger() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->memory_ledger(); -} -CpuSizing GeneralState::execution_sizing() const { - const CpuTeamScope team(workspace_->cpu_team()); - return workspace_->execution_sizing(); -} -const CpuExecution& GeneralState::cpu_execution() const { - return workspace_->cpu_execution(); -} -const FftPlanning& GeneralState::fft_planning() const { - return workspace_->problem_.fft_planning; -} -double GeneralState::planning_seconds() const noexcept { - return workspace_ ? workspace_->planning_seconds() : 0.0; -} -double GeneralState::raman_planning_seconds() const noexcept { - return workspace_ ? workspace_->raman_planning_seconds() : 0.0; -} -std::string GeneralState::raman_fft_provider() const { - return workspace_ ? workspace_->raman_fft_provider() : "none"; -} -std::map GeneralState::raman_fft_tasks() const { - return workspace_ ? workspace_->raman_fft_tasks() : std::map{}; -} -std::size_t GeneralState::raman_kernel_transforms() const noexcept { - return workspace_ ? workspace_->raman_kernel_transforms() : 0; -} -bool GeneralState::fsal_ready() const noexcept { - return workspace_ && workspace_->fsal_ready(); -} -std::size_t GeneralState::rk_source_evaluations() const noexcept { - return workspace_ ? workspace_->rk_source_evaluations() : 0; -} -std::size_t GeneralState::axis_scans() const noexcept { - return workspace_ ? workspace_->axis_scans() : 0; -} -bool GeneralState::closed() const noexcept { - return !workspace_ || workspace_->closed(); -} -void GeneralState::close() noexcept { - if (workspace_) - workspace_->close(); -} - -} // namespace solprop_native diff --git a/native/src/general/checks.cpp b/native/src/general/checks.cpp new file mode 100644 index 000000000..0d18d3f2b --- /dev/null +++ b/native/src/general/checks.cpp @@ -0,0 +1,562 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +// What the general workspace refuses before it allocates, and a trial's scalars. + +#include "workspace.hpp" + +#include + +namespace solprop_native { + +namespace { + +// The extents that size the workspace's allocations. +struct Extents { + std::size_t effective_node_pol{}, field{}, transform{}; +}; + +// The extents are positive and every allocation's size fits a size_t. +Extents require_extents(const Problem& problem) { + if (problem.dims.n_omega == 0 || problem.dims.n_t == 0 || problem.dims.n_to == 0 || + problem.dims.n_modes == 0 || problem.dims.n_pol_state == 0 || + problem.dims.n_nodes == 0 || problem.dims.n_pol_transform == 0) + throw std::invalid_argument("all transform and FFT extents must be positive"); + const auto effective_node_pol = + checked_product(problem.dims.n_pol_transform, problem.dims.n_pol_state, + "effective node polarisation"); + if (effective_node_pol > 2) + throw std::invalid_argument("at most two effective node components are supported"); + if (problem.dims.node_chunk == 0 || problem.dims.node_chunk > problem.dims.n_nodes) + throw std::invalid_argument("node_chunk must lie in [1, n_nodes]"); + if (problem.stepper.n_stages == 0) + throw std::invalid_argument("the tableau must contain a stage"); + const auto columns = + checked_product(problem.dims.n_modes, problem.dims.n_pol_state, "field columns"); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + constexpr auto policy_limit = + static_cast(std::numeric_limits::max()); + if (problem.dims.n_to > policy_limit || problem.dims.n_t > policy_limit || + problem.dims.n_omega > policy_limit || columns > policy_limit || + problem.dims.node_chunk > policy_limit) + throw std::length_error("accelerator policy extent exceeds signed 32-bit range"); +#endif + const auto rows = checked_product(problem.dims.n_nodes, problem.dims.n_pol_transform, + "transform rows"); + const auto field = checked_product(columns, problem.dims.n_omega, "field"); + const auto modal_time = checked_product(columns, problem.dims.n_to, "modal time"); + const auto chunk_rows = + checked_product(problem.dims.node_chunk, effective_node_pol, "node chunk rows"); + const auto node_time = checked_product(chunk_rows, problem.dims.n_to, "node time"); + // The hybrid planar kind carries its x half alone, (n_x, n_p) and its transpose. + const auto transform = + problem.free_geometry() ? 0 + : problem.slab_geometry() + ? checked_product(problem.spatial.n_x, problem.spatial.n_p, + "slab transform matrix") + : checked_product(rows, problem.dims.n_modes, "transform matrix"); + const auto stage_field = + checked_product(problem.stepper.n_stages, field, "stage field"); + require_allocation(field, sizeof(Complex), "field bytes"); + require_allocation(modal_time, sizeof(Complex), "modal-time bytes"); + require_allocation(node_time, sizeof(Complex), "node-time bytes"); + require_allocation(transform, sizeof(double), "transform-matrix bytes"); + if (!problem.transform.projection_coarse_matrix.empty() && + problem.dims.family != FieldFamily::envelope) { + const auto stacked_rows = + checked_product(2, problem.dims.n_modes, "stacked projection rows"); + require_allocation( + checked_product(stacked_rows, problem.dims.n_to, "stacked projection"), + sizeof(double), "stacked-projection bytes"); + } + require_allocation(stage_field, sizeof(Complex), "stage-field bytes"); + if (problem.plasma.enabled && !problem.plasma.envelope) { + require_allocation( + checked_product(chunk_rows, problem.dims.n_omega, "held nodal spectrum"), + sizeof(Complex), "held nodal spectrum bytes"); + require_allocation(checked_product(2, modal_time, "held stacked projection"), + sizeof(double), "held stacked projection bytes"); + require_allocation(checked_product(problem.dims.n_modes, + checked_product(problem.dims.node_chunk, + problem.dims.n_pol_transform, + "held projection rows"), + "held projection"), + sizeof(double), "held projection bytes"); + } + require_extent(problem.grid.crop_index.size(), problem.dims.n_omega, "crop_index"); + require_extent(problem.grid.active_mask.size(), problem.dims.n_omega, "active_mask"); + require_extent(problem.grid.norm_mask.size(), problem.dims.n_omega, "norm_mask"); + require_extent(problem.grid.omega_window.size(), problem.dims.n_omega, + "omega_window"); + if (!problem.grid.omega.empty()) + require_extent(problem.grid.omega.size(), problem.dims.n_omega, "omega"); + return { + .effective_node_pol = effective_node_pol, .field = field, .transform = transform}; +} + +// The spatial measure's and the separable and planar geometries' data. +void require_spatial(const Problem& problem) { + if (!problem.spatial.spectral_weights.empty()) { + require_extent(problem.spatial.spectral_weights.size(), problem.dims.n_modes, + "spatial_spectral_weights"); + require_extent(problem.spatial.on_axis.size(), problem.dims.n_modes, + "spatial_on_axis"); + if ((!problem.slab_geometry() && problem.dims.n_nodes != problem.dims.n_modes) || + problem.dims.n_pol_transform != 1) + throw std::invalid_argument( + "radial statistics require a scalar square transform"); + for (const double weight : problem.spatial.spectral_weights) + if (!std::isfinite(weight) || weight <= 0.0) + throw std::invalid_argument( + "spatial spectral weights must be finite and positive"); + } else if (!problem.spatial.on_axis.empty()) { + throw std::invalid_argument("spatial on-axis weights require the spectral measure"); + } + if (problem.slab_geometry()) { + // solprop's hybrid planar kind: columns (p, k_y), nodes (x, y), a scalar state, + // whole y rows per node chunk and no embedded coarse rule (solprop's reference + // scores no quadrature alarm on this kind). + if (problem.spatial.n_x < 1 || problem.spatial.n_y < 1 || + checked_product(problem.spatial.n_p, problem.spatial.n_y, "planar columns") != + problem.dims.n_modes || + checked_product(problem.spatial.n_x, problem.spatial.n_y, "planar nodes") != + problem.dims.n_nodes || + problem.dims.n_pol_transform != 1 || problem.dims.n_pol_state != 1 || + problem.dims.node_chunk % problem.spatial.n_x != 0 || + problem.linear.n_phi_terms != 1 || problem.linear.n_prefactor_terms != 1 || + !problem.transform.projection_coarse_matrix.empty()) + throw std::invalid_argument( + "the hybrid planar kind requires matching axes, a scalar state, whole y " + "rows per node chunk and scalar separable data"); + require_extent( + problem.spatial.kappa_sq.size(), + checked_product(problem.spatial.n_p, problem.dims.n_omega, "planar wall terms"), + "spatial_kappa_sq"); + for (const auto value : problem.spatial.kappa_sq) + if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) + throw std::invalid_argument("spatial_kappa_sq must be finite"); + } else if (!problem.spatial.kappa_sq.empty()) { + throw std::invalid_argument("spatial_kappa_sq belongs to the hybrid planar kind"); + } + // n_y == 1 is solprop's line uniform in y (grid.LineGrid): one y row carrying + // k_y = 0, transformed by the same two-axis plan with a unit extent. + if (problem.free_geometry() && + (problem.spatial.n_x < 2 || problem.spatial.n_y < 1 || + checked_product(problem.spatial.n_x, problem.spatial.n_y, "Cartesian shape") != + problem.dims.n_modes || + problem.dims.n_nodes != problem.dims.n_modes || + problem.dims.n_pol_transform != 1 || problem.linear.n_phi_terms != 1 || + problem.linear.n_prefactor_terms != 1)) + throw std::invalid_argument( + "Cartesian geometry requires matching axes and scalar separable data"); + if (problem.separable_geometry()) { + if (!std::isfinite(problem.spatial.evanescent_rate) || + problem.spatial.evanescent_rate <= 0) + throw std::invalid_argument( + "Cartesian evanescent rate must be finite and positive"); + const auto& spatial = problem.spatial; + const auto bins = problem.dims.n_omega; + const auto modes = problem.dims.n_modes; + for (const auto& [size, extent, name] : + {std::tuple{spatial.k_squared.size(), bins, "spatial_k_squared"}, + std::tuple{spatial.k_squared_low.size(), bins, "spatial_k_squared_low"}, + std::tuple{spatial.numerator.size(), bins, "spatial_numerator"}, + std::tuple{spatial.clamped.size(), bins, "spatial_clamped"}, + std::tuple{spatial.frame.size(), bins, "spatial_frame"}, + std::tuple{spatial.spectral_rate.size(), bins, "spatial_spectral_rate"}, + std::tuple{spatial.user_rate.size(), bins, "spatial_user_rate"}, + std::tuple{spatial.kperp_sq.size(), modes, "spatial_kperp_sq"}, + std::tuple{spatial.kperp_sq_low.size(), modes, "spatial_kperp_sq_low"}, + std::tuple{spatial.absorber_rate.size(), modes, "spatial_absorber_rate"}, + std::tuple{spatial.kperp_rate.size(), modes, "spatial_kperp_rate"}}) + require_extent(size, extent, name); + for (const auto& [values, message] : + {std::pair{&spatial.k_squared, "spatial_k_squared must be finite"}, + std::pair{&spatial.k_squared_low, "spatial_k_squared_low must be finite"}, + std::pair{&spatial.numerator, "spatial_numerator must be finite"}, + std::pair{&spatial.clamped, "spatial_clamped must be finite"}}) + for (const auto value : *values) + if (!std::isfinite(value.real()) || !std::isfinite(value.imag())) + throw std::invalid_argument(message); + // Each real array: finite, and when `nonnegative` not below zero. + const auto require_real = [](const std::vector& values, + const bool nonnegative, const char* message) { + for (const auto value : values) + if (!std::isfinite(value) || (nonnegative && value < 0)) + throw std::invalid_argument(message); + }; + require_real(spatial.frame, false, "spatial_frame is invalid"); + require_real(spatial.spectral_rate, true, "spatial_spectral_rate is invalid"); + for (std::size_t bin = 0; bin < problem.dims.n_omega; ++bin) + if (const double value = problem.spatial.user_rate[bin]; + !std::isfinite(value) || value < 0 || + value > problem.spatial.spectral_rate[bin]) + throw std::invalid_argument("spatial_user_rate is invalid"); + require_real(spatial.kperp_sq, true, "spatial_kperp_sq is invalid"); + require_real(spatial.kperp_sq_low, false, "spatial_kperp_sq_low must be finite"); + require_real(spatial.absorber_rate, true, "spatial_absorber_rate is invalid"); + require_real(spatial.kperp_rate, true, "spatial_kperp_rate is invalid"); + if (problem.free_geometry()) { + // solprop SEP 23: the stored transverse axes the far-field statistics read. + require_extent(problem.spatial.k_x.size(), problem.spatial.n_x, "spatial_k_x"); + require_extent(problem.spatial.k_y.size(), problem.spatial.n_y, "spatial_k_y"); + require_real(spatial.k_x, false, "spatial_k_x must be finite"); + require_real(spatial.k_y, false, "spatial_k_y must be finite"); + } + } else if (problem.spatial.n_y != 0 || problem.spatial.n_p != 0) { + throw std::invalid_argument("Cartesian axes must be supplied together"); + } + if (!problem.free_geometry() && + (!problem.spatial.k_x.empty() || !problem.spatial.k_y.empty() || + problem.spatial.far_field_slots != 0)) + throw std::invalid_argument("the far-field statistics belong to free space"); +} + +// The grid's windows, the transforms, the linear operator, the tableau and the +// absorbers. +void require_operator(const Problem& problem, const Extents& extents) { + const auto field = extents.field; + const auto transform = extents.transform; + require_extent(problem.grid.time_window.size(), problem.dims.n_to, "time_window"); + if (problem.plasma.enabled && !problem.plasma.envelope) { + require_extent(problem.grid.time_window_derivative.size(), problem.dims.n_to, + "time_window_derivative"); + require_extent(problem.grid.omega.size(), problem.dims.n_omega, + "held current omega"); + for (const auto value : problem.grid.time_window_derivative) + if (!std::isfinite(value)) + throw std::invalid_argument("time_window_derivative is not finite"); + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) + if (problem.grid.active_mask[index] != 0 && + (!(problem.grid.omega[index] > 0.0) || + !std::isfinite(problem.grid.omega[index]))) + throw std::invalid_argument( + "held plasma current requires positive active omega"); + } + require_extent(problem.grid.carrier.size(), problem.dims.n_to, "carrier"); + require_extent(problem.transform.synthesis_matrix.size(), transform, + "synthesis_matrix"); + require_extent(problem.transform.projection_matrix.size(), transform, + "projection_matrix"); + if (!problem.transform.projection_coarse_matrix.empty()) + require_extent(problem.transform.projection_coarse_matrix.size(), transform, + "projection_coarse_matrix"); + // A z table's terms are slots its evaluation fills, so nothing crosses for them. + if ((problem.linear.phi_table || problem.linear.prefactor_table) && + problem.separable_geometry()) + throw std::invalid_argument("a separable operator takes no z table"); + require_extent(problem.linear.phi_terms.size(), + problem.separable_geometry() || problem.linear.phi_table + ? 0 + : checked_product(problem.linear.n_phi_terms, field, "phi_terms"), + "phi_terms"); + require_extent( + problem.linear.prefactor_terms.size(), + problem.separable_geometry() || problem.linear.prefactor_table + ? 0 + : checked_product(problem.linear.n_prefactor_terms, field, "prefactor_terms"), + "prefactor_terms"); + require_extent( + problem.stepper.tableau_matrix.size(), + checked_product(problem.stepper.n_stages, problem.stepper.n_stages, "tableau"), + "tableau_matrix"); + require_extent(problem.stepper.tableau_nodes.size(), problem.stepper.n_stages, + "tableau_nodes"); + require_extent(problem.stepper.solution_weights.size(), problem.stepper.n_stages, + "solution_weights"); + require_extent(problem.stepper.error_weights.size(), problem.stepper.n_stages, + "error_weights"); + if (problem.absorber.in_stage) + require_extent(problem.absorber.fine.size(), problem.dims.n_to, "absorber_fine"); + if (!problem.spatial.absorber_matrix.empty()) { + require_extent( + problem.spatial.absorber_matrix.size(), + checked_product(problem.dims.n_modes, problem.dims.n_modes, "spatial absorber"), + "spatial_absorber_matrix"); + if (!problem.absorber.in_stage || problem.dims.n_pol_transform != 1 || + problem.dims.n_nodes != problem.dims.n_modes) + throw std::invalid_argument("spatial absorber requires scalar radial geometry"); + } + if (problem.absorber.split_step) + require_extent(problem.absorber.coarse.size(), problem.dims.n_t, "absorber_coarse"); + const std::size_t fine_size = problem.dims.family == FieldFamily::field_real + ? problem.dims.n_to / 2 + 1 + : problem.dims.n_to; + if (problem.plasma.envelope && problem.dims.family != FieldFamily::envelope) + throw std::invalid_argument("envelope plasma requires an envelope field family"); + std::vector seen(fine_size, false); + for (const auto index : problem.grid.crop_index) { + if (index >= fine_size) + throw std::invalid_argument("crop_index lies outside the fine spectrum"); + if (seen[index]) + throw std::invalid_argument("crop_index contains a duplicate fine-spectrum bin"); + seen[index] = true; + } +} + +// The plasma channels' and the Raman components' tables. +void require_components(const Problem& problem) { + if (problem.plasma.enabled && problem.plasma.n_channels == 0) + throw std::invalid_argument("plasma requires at least one channel"); + if (problem.plasma.n_channels > 1 || problem.plasma.envelope) { + if (!problem.plasma.enabled) + throw std::invalid_argument("channels require plasma"); + require_extent(problem.plasma.channel_parameters.size(), + checked_product(problem.plasma.n_channels, 9, "channel parameters"), + "channel_parameters"); + if (!std::ranges::all_of(problem.plasma.channel_parameters, + [](double value) { return std::isfinite(value); }) || + !std::ranges::all_of(problem.plasma.channel_coefficients, + [](double value) { return std::isfinite(value); })) + throw std::invalid_argument("channel parameters and coefficients must be finite"); + std::size_t expected_offset = 0; + // Each interval holds four coefficients. + const std::size_t total_intervals = problem.plasma.channel_coefficients.size() / 4; + for (std::size_t channel = 0; channel < problem.plasma.n_channels; ++channel) { + const double* row = problem.plasma.channel_parameters.data() + 9 * channel; + if (!(row[1] > 0.0 && row[2] > 0.0 && row[3] > row[2] && row[4] >= 0.0 && + row[5] >= 1.0 && row[5] == std::floor(row[5]) && + row[6] == static_cast(expected_offset) && row[7] > 0.0 && + row[8] > 0.0) || + row[5] > static_cast(total_intervals)) + throw std::invalid_argument( + "channel constants and coefficient ranges must be positive and consistent"); + expected_offset = checked_sum( + expected_offset, + checked_product(4, static_cast(row[5]), "channel table"), + "channel table offsets"); + } + require_extent(problem.plasma.channel_coefficients.size(), expected_offset, + "channel_coefficients"); + } + if (problem.raman.enabled && problem.raman.n_components == 0) + throw std::invalid_argument("Raman requires at least one component"); + if (problem.raman.n_components > 1) { + if (!problem.raman.enabled) + throw std::invalid_argument("Raman components require Raman"); + require_extent(problem.raman.row_component.size(), problem.raman.n_terms, + "raman_row_component"); + require_extent(problem.raman.component_fraction.size(), problem.raman.n_components, + "raman_component_fraction"); + for (double value : problem.raman.row_component) + if (!std::isfinite(value) || value < 0.0 || + value >= static_cast(problem.raman.n_components) || + value != std::floor(value)) + throw std::invalid_argument("Raman row component index is invalid"); + for (double value : problem.raman.component_fraction) + if (!std::isfinite(value) || value <= 0.0) + throw std::invalid_argument( + "Raman component fractions must be finite and positive"); + } +} + +// The plasma column: its constants, integration rule and envelope currents. +void require_plasma(const Problem& problem) { + if (problem.plasma.enabled) { + if (!(problem.grid.dt_fine > 0.0) || !(problem.plasma.d_log_field > 0.0) || + !(problem.plasma.field_min > 0.0) || + !(problem.plasma.field_max > problem.plasma.field_min) || + !(problem.plasma.charge_mass_ratio > 0.0) || + !(problem.plasma.ionisation_potential > 0.0) || + problem.plasma.rate_intervals == 0) + throw std::invalid_argument("plasma constants must be positive and consistent"); + require_extent( + problem.plasma.rate_coefficients.size(), + checked_product(4, problem.plasma.rate_intervals, "rate coefficients"), + "rate_coefficients"); + } + // solprop SEP 18's running-integral rule. Translation certified the taps; only their + // extent, finiteness, the trapezoid's identity tap and the route are checked here. + if (problem.plasma.taps.empty() || problem.plasma.taps.size() > plasma_max_taps || + !std::ranges::all_of(problem.plasma.taps, + [](double value) { return std::isfinite(value); })) + throw std::invalid_argument("plasma_taps must hold one to nine finite taps"); + if (!problem.plasma.band && + (problem.plasma.taps.size() != 1 || problem.plasma.taps[0] != 1.0)) + throw std::invalid_argument("the trapezoid rule is the one identity tap"); + if (problem.plasma.gated && + (!problem.plasma.enabled || problem.plasma.envelope || problem.general.enabled)) + throw std::invalid_argument( + "the gated plasma benchmark needs a field plasma column off the general route"); + if (problem.plasma.band) { + if (!problem.plasma.enabled || problem.plasma.envelope || + problem.dims.family == FieldFamily::envelope) + throw std::invalid_argument( + "the band integration rule needs a field-grid plasma column"); + if (problem.dims.n_to <= problem.plasma.half_width()) + throw std::invalid_argument( + "the band integration rule's filter needs more than K samples"); + } + if (problem.plasma.envelope) { + if (!problem.plasma.enabled || problem.dims.family != FieldFamily::envelope || + problem.kerr.harmonic_factor != 0.0 || problem.raman.thg || + problem.plasma.stash) + throw std::invalid_argument("envelope plasma requires separate envelope currents " + "without THG or field stashing"); + const auto rule_size = checked_product(problem.plasma.n_channels, + std::size_t{4} * 512, "envelope rules"); + require_extent(problem.plasma.envelope_rule_nodes.size(), rule_size, + "envelope_rule_nodes"); + require_extent(problem.plasma.envelope_rule_weights.size(), rule_size, + "envelope_rule_weights"); + for (std::size_t channel = 0; channel < problem.plasma.n_channels; ++channel) + for (std::size_t level = 0; level < 4; ++level) { + const auto order = std::size_t{64} << level; + double sum = 0.0; + double previous = -1.0; + for (std::size_t index = 0; index < 512; ++index) { + const auto offset = (channel * 4 + level) * 512 + index; + const auto node = problem.plasma.envelope_rule_nodes[offset]; + const auto weight = problem.plasma.envelope_rule_weights[offset]; + if (!std::isfinite(node) || !std::isfinite(weight) || + (index < order ? !(node > previous && node < 1.0 && weight > 0.0) + : node != 0.0 || weight != 0.0)) + throw std::invalid_argument( + "envelope quadrature nodes or weights are invalid"); + if (index < order) { + previous = node; + sum += weight; + } + } + if (std::abs(sum - 2.0) > + 8 * static_cast(order) * std::numeric_limits::epsilon()) + throw std::invalid_argument( + "envelope quadrature weights must integrate a constant"); + } + require_extent(problem.plasma.envelope_drude_factor.size(), problem.dims.n_omega, + "envelope_drude_factor"); + require_extent(problem.plasma.envelope_loss_factor.size(), problem.dims.n_omega, + "envelope_loss_factor"); + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) { + const auto drude = problem.plasma.envelope_drude_factor[index]; + const auto loss = problem.plasma.envelope_loss_factor[index]; + const bool active = + problem.grid.active_mask[index] != 0 && problem.grid.omega[index] > 0.0; + if (!std::isfinite(drude) || !std::isfinite(loss.real()) || + !std::isfinite(loss.imag()) || loss.real() != 0.0 || + (active ? !(drude < 0.0 && loss.imag() < 0.0) + : drude != 0.0 || loss.imag() != 0.0)) + throw std::invalid_argument("envelope current factors require negative active " + "factors and zero inactive bins"); + } + } +} + +// The Raman response's terms and kernel. +void require_raman(const Problem& problem, const std::size_t effective_node_pol) { + if (problem.raman.enabled) { + if (effective_node_pol != 1 && !problem.raman.tensor_capable) + throw std::invalid_argument( + "vector Raman requires tensor-capable terms; found scalar_only"); + if (problem.raman.n_terms == 0) + throw std::invalid_argument("a Raman problem must contain at least one term"); + require_extent(problem.raman.tensor_weights.size(), + checked_product(problem.raman.n_terms, 2, "Raman tensor weights"), + "raman_tensor_weights"); + for (std::size_t term = 0; term < problem.raman.n_terms; ++term) { + const double trace = problem.raman.tensor_weights[2 * term]; + const double traceless = problem.raman.tensor_weights[2 * term + 1]; + if (!((trace == 1.0 && traceless == 0.0) || (trace == 0.25 && traceless == 0.75))) + throw std::invalid_argument( + "Raman tensor weights must be isotropic or rank two"); + } + const auto doubled = checked_product(2, problem.dims.n_to, "Raman doubled grid"); + const auto raman_rows = checked_product(problem.dims.node_chunk, + problem.raman_drives(), "Raman drive rows"); + require_allocation(checked_product(raman_rows, doubled, "Raman drive"), + sizeof(double), "Raman drive bytes"); + require_extent(problem.raman.undamped.size(), + checked_product(problem.raman.n_terms, doubled, "raman_undamped"), + "raman_undamped"); + if (!(problem.raman.fraction > 0.0) || !std::isfinite(problem.raman.fraction) || + !(problem.grid.dt_fine > 0.0) || !std::isfinite(problem.grid.dt_fine) || + !(problem.raman.epsilon0 > 0.0) || !std::isfinite(problem.raman.epsilon0)) + throw std::invalid_argument( + "Raman fraction, dt_fine and epsilon0 must be positive and finite"); + if (!std::ranges::all_of(problem.raman.undamped, + [](const double value) { return std::isfinite(value); })) + throw std::invalid_argument("raman_undamped must contain only finite values"); + } else if (problem.raman.thg || problem.raman.n_terms != 0 || + !problem.raman.undamped.empty() || !problem.raman.tensor_weights.empty() || + problem.raman.tensor_capable) { + throw std::invalid_argument("Raman data were supplied while Raman is disabled"); + } +} + +} // namespace + +void require_general_problem(const Problem& problem) { + require_general_modal(problem); + require_coupled_flow(problem); + require_dispersive(problem); + const auto extents = require_extents(problem); + require_spatial(problem); + require_operator(problem, extents); + require_components(problem); + require_plasma(problem); + require_raman(problem, extents.effective_node_pol); +} + +void GeneralWorkspace::require_trial_scalars(const TrialScalars& scalars) const { + const auto checked_values = [](const std::vector& values, const char* name, + bool nonnegative, bool fraction) { + for (double value : values) + if (!std::isfinite(value) || (nonnegative && value < 0.0) || + (fraction && value > 1.0)) + throw std::invalid_argument(std::string(name) + " contains an invalid value"); + }; + if (!scalars.kerr_density.empty()) + require_extent(scalars.kerr_density.size(), problem_.stepper.n_stages, + "kerr_density"); + checked_values(scalars.kerr_density, "kerr_density", false, false); + if (problem_.plasma.enabled && + (problem_.plasma.n_channels > 1 || !scalars.plasma_densities.empty() || + !scalars.plasma_preionisation.empty())) { + const auto count = checked_product( + problem_.stepper.n_stages, problem_.plasma.n_channels, "stage channel values"); + require_extent(scalars.plasma_densities.size(), count, "plasma_densities"); + require_extent(scalars.plasma_preionisation.size(), count, "plasma_preionisation"); + checked_values(scalars.plasma_densities, "plasma_densities", true, false); + checked_values(scalars.plasma_preionisation, "plasma_preionisation", true, true); + } + if (problem_.raman.enabled && + (problem_.raman.n_components > 1 || !scalars.raman_densities.empty())) { + require_extent(scalars.raman_densities.size(), + checked_product(problem_.stepper.n_stages, + problem_.raman.n_components, "stage Raman values"), + "raman_densities"); + checked_values(scalars.raman_densities, "raman_densities", true, false); + } + if (const auto* weights = dispersive_weights()) + weights->require_densities(scalars.dispersive_densities); + else if (!scalars.dispersive_densities.empty()) + throw std::invalid_argument("dispersive_densities weigh SEP 6's dispersive " + "response, and this problem has none"); + if (!general_ && !scalars.transform_weights.empty()) + throw std::invalid_argument("transform_weights weight a general modal guide's " + "moving maps, and this problem has none"); + require_extent(scalars.phi_delta.size(), + (problem_.stepper.n_stages + 1) * problem_.linear.n_phi_terms, + "phi_delta"); + require_extent(scalars.phi_at.size(), + problem_.stepper.n_stages * problem_.linear.n_phi_terms, "phi_at"); + require_extent(scalars.prefactor.size(), + problem_.stepper.n_stages * problem_.linear.n_prefactor_terms, + "prefactor"); + require_extent(scalars.density.size(), problem_.stepper.n_stages, "density"); + require_extent(scalars.taper.size(), problem_.stepper.n_stages, "taper"); + if (problem_.plasma.enabled) + require_extent(scalars.preionisation.size(), problem_.stepper.n_stages, + "preionisation"); + if (problem_.raman.enabled) { + require_extent(scalars.raman_rates.size(), + problem_.stepper.n_stages * problem_.raman.n_terms, "raman_rates"); + if (!std::ranges::all_of(scalars.raman_rates, [](const double value) { + return std::isfinite(value) && value >= 0.0; + })) + throw std::invalid_argument("Raman decay rates must be finite and non-negative"); + } +} + +} // namespace solprop_native diff --git a/native/src/general/kernels.hpp b/native/src/general/kernels.hpp new file mode 100644 index 000000000..10e2d8562 --- /dev/null +++ b/native/src/general/kernels.hpp @@ -0,0 +1,1723 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +// The general workspace's helpers: device-callable functions, kernel functors and the +// workspace's view and plan types, shared by its translation units (there is no +// relocatable device code, so a helper a kernel calls is defined here, inline). +// Contracting code: a strict translation unit never includes it. +#ifdef SOLPROP_NATIVE_STRICT_TU +#error "general/kernels.hpp contracts: a strict translation unit must not include it" +#endif + +#include "solprop_native/adams_phi.hpp" +#include "solprop_native/build.hpp" +#include "solprop_native/core.hpp" +#include "solprop_native/coupled_flow.hpp" +#include "solprop_native/crossing.hpp" +#include "solprop_native/dispatch.hpp" +#include "solprop_native/dispersive.hpp" +#include "solprop_native/envelope_plasma.hpp" +#include "solprop_native/far_field.hpp" +#include "solprop_native/fft_plan.hpp" +#include "solprop_native/flow_functions.hpp" +#include "solprop_native/fp.hpp" +#include "solprop_native/gemm.hpp" +#include "solprop_native/general_modal.hpp" +#include "solprop_native/plasma.hpp" +#include "solprop_native/raman_fft.hpp" +#include "solprop_native/refuse.hpp" +#include "solprop_native/slab.hpp" +#include "solprop_native/vmath.hpp" +#include "solprop_native/z_table.hpp" + +#include +#include +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +#include +#ifdef __SSE2__ +#include +#define SOLPROP_NATIVE_STREAMING_STORES 1 +#endif +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __APPLE__ +#include +#elif defined(__GLIBC__) +#include +#endif +namespace solprop_native { +namespace general {} +// The helpers are found as an unnamed namespace's would be, under their own name so +// that every unit that includes them sees one definition. +using namespace general; +} // namespace solprop_native + +namespace solprop_native::general { +/** The three squared norms an accepted step reports, as one reduction value. */ +struct NormSums { + double old_sum = 0.0; + double new_sum = 0.0; + double error_sum = 0.0; + KOKKOS_INLINE_FUNCTION NormSums& operator+=(const NormSums& other) { + old_sum += other.old_sum; + new_sum += other.new_sum; + error_sum += other.error_sum; + return *this; + } +}; +} // namespace solprop_native::general + +template <> struct Kokkos::reduction_identity { + KOKKOS_FUNCTION static solprop_native::NormSums sum() { return {}; } +}; + +namespace solprop_native::general { +/** A plane's scaled fluence, one reduction value: its sum and its index moments. */ +struct FluenceMoments { + double total = 0.0; + double x = 0.0; + double y = 0.0; + KOKKOS_INLINE_FUNCTION FluenceMoments& operator+=(const FluenceMoments& other) { + total += other.total; + x += other.x; + y += other.y; + return *this; + } +}; +} // namespace solprop_native::general + +template <> struct Kokkos::reduction_identity { + KOKKOS_FUNCTION static solprop_native::FluenceMoments sum() { return {}; } +}; + +namespace solprop_native::general { + +// Sets a coupled flow aside for a stage evaluation and puts it back however the scope +// is left: the Adams sample is the material source alone (solprop SEP 16 phase 5). +class FlowAside { +public: + explicit FlowAside(std::unique_ptr& slot) + : slot_(slot), held_(std::exchange(slot, nullptr)) {} + // Restores the slot; clang-tidy 18 misreads it as trivial when GCC 13's intrinsic + // headers fail to parse. + // NOLINTNEXTLINE(modernize-use-equals-default) + ~FlowAside() { slot_ = std::move(held_); } + FlowAside(const FlowAside&) = delete; + FlowAside& operator=(const FlowAside&) = delete; + FlowAside(FlowAside&&) = delete; + FlowAside& operator=(FlowAside&&) = delete; + +private: + std::unique_ptr& slot_; + std::unique_ptr held_; +}; + +// One frozen-residual weight row from the host into a View's row. +template +void upload_weights(const ExecutionSpace& space, const Row& out, + const std::vector& weights) { + if (weights.size() < out.extent(0)) + throw std::invalid_argument("a coupled flow's weights are shorter than its terms"); + const Kokkos::View> + host(weights.data(), out.extent(0)); + graph::deep_copy(space, out, host); +} + +// A frozen-residual flow's generator at one position applied to a field, `sum_t w_t +// G_t u` on the block's bins and zero off them (the terms are zero off the scored +// bins). A zero weight is skipped, which leaves every bit as the full sum has it. +template +void apply_generator(const ExecutionSpace& space, const std::size_t n, + const std::size_t modes, + const Kokkos::View& slots, + const Kokkos::View& terms, + const Kokkos::View& weights, + const Kokkos::View& input, + const Out& out) { + const auto n_terms = weights.extent(0); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + launch::parallel_for( + "solprop_native.adams.generator", Range2(space, {0, 0}, {n, modes}), + KOKKOS_LAMBDA(const std::size_t index, const std::size_t i) { + const auto one = slots(index); + Complex value{}; + if (one != 0) { + const auto bin = one - 1; + for (std::size_t entry = 0; entry < n_terms; ++entry) { + const double weight = weights(entry); + if (weight == 0.0) + continue; + for (std::size_t j = 0; j < modes; ++j) + value += weight * terms(entry, bin, i, j) * input(j, index); + } + } + out(i, index) = value; + }); +#else + launch::parallel_for( + "solprop_native.adams.generator", + Kokkos::RangePolicy(space, 0, n), + KOKKOS_LAMBDA(const std::size_t index) { + const auto one = slots(index); + for (std::size_t i = 0; i < modes; ++i) { + Complex value{}; + if (one != 0) { + const auto bin = one - 1; + for (std::size_t entry = 0; entry < n_terms; ++entry) { + const double weight = weights(entry); + if (weight == 0.0) + continue; + for (std::size_t j = 0; j < modes; ++j) + value += weight * terms(entry, bin, i, j) * input(j, index); + } + } + out(i, index) = value; + } + }); +#endif +} + +template +KOKKOS_INLINE_FUNCTION void capture_anchor(const Ts&... /*unused*/) {} + +// Net process allocator growth includes opaque vendor FFT plan/workspace +// allocations. It is measured separately from Kokkos Views and is not a peak bound. +inline std::size_t heap_bytes() { +#ifdef __APPLE__ + malloc_statistics_t statistics{}; + malloc_zone_statistics(nullptr, &statistics); + return statistics.size_in_use; +#elif defined(__GLIBC__) + // mallinfo2 is glibc 2.33 or later; a wheel built against an older glibc (the + // manylinux_2_28 baseline) finds it at run time when the system provides it. Its + // layout is ten size_t counters, in glibc's declared order. + struct Mallinfo2 { + std::size_t arena, ordblks, smblks, hblks, hblkhd, usmblks, fsmblks, uordblks, + fordblks, keepcost; + }; + using Query = Mallinfo2 (*)(); + static const auto query = reinterpret_cast(dlsym(RTLD_DEFAULT, "mallinfo2")); + if (query == nullptr) + return 0; + const auto statistics = query(); + return statistics.uordblks + statistics.hblkhd; +#else + // Neither allocator interface exists (musl, Windows): the ledger row then measures + // nothing, and construction must still succeed. + return 0; +#endif +} +inline std::size_t device_free_bytes() { +#ifdef KOKKOS_ENABLE_CUDA + std::size_t available{}, total{}; + if (cudaMemGetInfo(&available, &total) != cudaSuccess) + throw std::runtime_error("cudaMemGetInfo failed during FFT planning"); + return available; +#elif defined(KOKKOS_ENABLE_HIP) + std::size_t available{}, total{}; + if (hipMemGetInfo(&available, &total) != hipSuccess) + throw std::runtime_error("hipMemGetInfo failed during FFT planning"); + return available; +#else + return 0; +#endif +} + +using ComplexView2 = Kokkos::View; +using ComplexView3 = Kokkos::View; +using RealView2 = Kokkos::View; +using RealView3 = Kokkos::View; + +template +KOKKOS_INLINE_FUNCTION void +write_raman_drive(const RealView2& drive, const std::size_t node, + const std::size_t time, const Complex x, const Complex y, + const std::size_t count) { + const double xx = RealField ? x.real() * x.real() + : 0.5 * (x.real() * x.real() + x.imag() * x.imag()); + if constexpr (Components == 1) { + drive(node, time) = xx; + } else { + const double yy = RealField ? y.real() * y.real() + : 0.5 * (y.real() * y.real() + y.imag() * y.imag()); + drive(count * node, time) = xx + yy; + if (count == 3) { + drive(count * node + 1, time) = xx - yy; + drive(count * node + 2, time) = RealField + ? 2.0 * x.real() * y.real() + : x.real() * y.real() + x.imag() * y.imag(); + } + } +} +using ByteView = Kokkos::View; +using IndexView = Kokkos::View; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +// A device splits a few long real rows over its multiprocessors (`RealRowPlan`). +using InverseRealPlan = RealRowPlan; +using ForwardRealPlan = RealRowPlan; +#else +using InverseRealPlan = KokkosFFT::Plan; +using ForwardRealPlan = KokkosFFT::Plan; +#endif +using RamanForwardPlan = RamanFftPlan; +using RamanInversePlan = RamanFftPlan; +using ComplexPlan = KokkosFFT::Plan; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +// A device's plans of the transverse scratch run one after another, so they share one +// work area (`transform_work_`) in place of holding one each (solprop #529). A +// Kerr-only free-space envelope takes the stage's time axis with the plane's in one +// rank-3 transform (`volume_kerr_`). +using TransversePlan = KokkosFFT::DynPlan; +using VolumePlan = TransversePlan; +#else +using TransversePlan = KokkosFFT::Plan; +using VolumePlan = KokkosFFT::Plan; +#endif +// The hybrid planar kind transforms y alone, the trailing axis of (bins, n_p, n_y). +using SlabPlan = KokkosFFT::Plan; +// A requested collar's padded transform, along the last axis of a batch (solprop SEP +// 44). +using StripPlan = KokkosFFT::Plan; + +// Each current keeps its own chunk accumulator, including stacked embedded +// quadrature. Sharing the latter would mix currents between partial chunks. +struct EnvelopeStream { + ComplexView2 node, projected, coarse, stacked, spectrum, coarse_spectrum; + std::unique_ptr forward, coarse_forward; + [[nodiscard]] std::size_t bytes() const { + return sizeof(Complex) * + (node.size() + projected.size() + coarse.size() + stacked.size() + + spectrum.size() + coarse_spectrum.size()); + } +}; + +// Real current products own their projections, including an embedded quadrature. +struct FieldCurrentStream { + RealView2 node, projected, coarse, stacked; + ComplexView2 spectrum, coarse_spectrum; + std::unique_ptr forward, coarse_forward; + [[nodiscard]] std::size_t bytes() const { + return sizeof(double) * + (node.size() + projected.size() + coarse.size() + stacked.size()) + + sizeof(Complex) * (spectrum.size() + coarse_spectrum.size()); + } +}; + +// CPU experiments favour one task per column; accelerator lanes retain their +// two-dimensional launch. Both invoke the same element functor in the same order. +template +void cartesian_for(const char* label, const ExecutionSpace& execution, + const std::size_t rows, const std::size_t width, + const Function& element) { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + launch::parallel_for(label, Range2(execution, {0, 0}, {rows, width}), element); +#else + launch::parallel_for( + label, Kokkos::RangePolicy(execution, 0, rows), + KOKKOS_LAMBDA(const std::size_t row) { + for (std::size_t inner = 0; inner < width; ++inner) + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide + element(row, inner); + }); +#endif +} + +// A traversal between a separable geometry's column-major registers `(column, omega)` +// and the transverse scratch `(omega, y, x)` (SEP 28). On CPU one task per column +// reads the scratch a whole plane apart at every bin; tiles of adjacent columns and +// bins, columns fastest within a tile, read both layouts in contiguous runs. The +// element is the same functor; each (column, bin) is visited once. +// 32 columns by 16 bins: the best of 32x16, 64x32 and 256x8 on one Zen 5 socket +// (SEP 28; Apple silicon is within a few per cent across them). +[[maybe_unused]] inline constexpr std::size_t crossing_tile_columns = 32; +[[maybe_unused]] inline constexpr std::size_t crossing_tile_bins = 16; + +// A store that bypasses the caches where the CPU has one (x86 SSE2): a traversal that +// writes a large volume it does not read (the scatter into the transverse scratch) +// otherwise reads every line before writing it. Ordered by `stream_fence`. Device +// builds compile out the host traversals that use these, hence `maybe_unused`. +[[maybe_unused]] KOKKOS_INLINE_FUNCTION void stream_store(Complex& target, + const Complex value) { +#ifdef SOLPROP_NATIVE_STREAMING_STORES + _mm_stream_pd(reinterpret_cast(&target), + _mm_set_pd(value.imag(), value.real())); +#else + target = value; +#endif +} + +[[maybe_unused]] KOKKOS_INLINE_FUNCTION void stream_fence() { +#ifdef SOLPROP_NATIVE_STREAMING_STORES + _mm_sfence(); +#endif +} + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +// One host tile of a crossing: its rows `[first_row, last_row)` and bins +// `[first_bin, last_bin)`, at most `crossing_tile_columns` by `crossing_tile_bins`. +struct CrossingTile { + std::size_t first_row, last_row, first_bin, last_bin; + + [[nodiscard]] std::size_t size() const noexcept { + return (last_row - first_row) * (last_bin - first_bin); + } + // An element's place in a tile buffer, rows outermost. + [[nodiscard]] std::size_t at(const std::size_t row, + const std::size_t bin) const noexcept { + return (row - first_row) * (last_bin - first_bin) + (bin - first_bin); + } + // Every element in the traversal order `crossing_for` names. + template void visit(const Function& element) const { + if constexpr (BinsInner) + for (std::size_t row = first_row; row < last_row; ++row) + for (std::size_t inner = first_bin; inner < last_bin; ++inner) + element(row, inner); + else + for (std::size_t inner = first_bin; inner < last_bin; ++inner) + for (std::size_t row = first_row; row < last_row; ++row) + element(row, inner); + } +}; + +// The host tiles of a crossing, one task each: `crossing_for`'s traversal, for a +// kernel that forms a whole tile's transcendentals in one pass first. +template +void crossing_tiles(const char* label, const ExecutionSpace& execution, + const std::size_t rows, const std::size_t width, + const Function& visit) { + constexpr std::size_t tile_rows = crossing_tile_columns; + constexpr std::size_t tile_bins = crossing_tile_bins; + const std::size_t row_tiles = (rows + tile_rows - 1) / tile_rows; + const std::size_t bin_tiles = (width + tile_bins - 1) / tile_bins; + launch::parallel_for( + label, Kokkos::RangePolicy(execution, 0, row_tiles * bin_tiles), + KOKKOS_LAMBDA(const std::size_t index) { + const std::size_t first_row = (index / bin_tiles) * tile_rows; + const std::size_t first_bin = (index % bin_tiles) * tile_bins; + visit(CrossingTile{ + .first_row = first_row, + .last_row = rows - first_row < tile_rows ? rows : first_row + tile_rows, + .first_bin = first_bin, + .last_bin = width - first_bin < tile_bins ? width : first_bin + tile_bins}); + // Each thread orders its own streaming stores before the region ends. + if constexpr (Streaming) + stream_fence(); + }); +} + +// A tile's propagators `exp(argument)`, as `propagator_exp` forms them, in one pass +// of the vector-maths layer (solprop ADR-059, issue #338): the cosines and sines, +// then the magnitudes, `exp(+-0)` exactly one on a lossless element without the +// call. Fill `real` and `imag` for the tile's elements, then `form`. +struct PropagatorTile { + static constexpr std::size_t capacity = crossing_tile_columns * crossing_tile_bins; + // The static analyser cannot follow that `form` and `operator[]` read only what the + // tile's visit wrote, so its build zeroes the arrays; a real build leaves them unset. +#ifdef __clang_analyzer__ + double real[capacity]{}, imag[capacity]{}, cosine[capacity]{}, sine[capacity]{}; +#else + double real[capacity], imag[capacity], cosine[capacity], sine[capacity]; +#endif + + void form(const std::size_t count) { + vmath::sincos(imag, sine, cosine, count); + bool lossless = true; + for (std::size_t k = 0; k < count; ++k) + lossless = lossless && real[k] == 0.0; + if (lossless) + return; +#ifdef __clang_analyzer__ + double magnitude[capacity]{}; +#else + double magnitude[capacity]; +#endif + vmath::exp(real, magnitude, count); + for (std::size_t k = 0; k < count; ++k) { + const double scale = real[k] == 0.0 ? 1.0 : magnitude[k]; + cosine[k] *= scale; + sine[k] *= scale; + } + } + Complex operator[](const std::size_t k) const { return {cosine[k], sine[k]}; } +}; +#endif + +// On a device, `RowFastest` maps adjacent rows (columns of the plane) to adjacent +// threads, so the transverse scratch side of a gather or scatter is coalesced; the +// chunk-local side is the strided one. +// On the host, `BinsInner` visits a tile's rows outermost and its bins innermost: a +// traversal that writes the rows (a pack from the scratch) then stores each row's run +// at once, about 1.7 times faster for the free-space chunk's pack on an Intel Xeon Gold +// 6240 at one thread. +template +void crossing_for(const char* label, const ExecutionSpace& execution, + const std::size_t rows, const std::size_t width, + const Function& element) { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + if constexpr (RowFastest) { + using Left = Kokkos::MDRangePolicy< + ExecutionSpace, Kokkos::Rank<2, Kokkos::Iterate::Left, Kokkos::Iterate::Left>, + Kokkos::IndexType>; + launch::parallel_for( + label, + Left(execution, {0, 0}, + {static_cast(rows), static_cast(width)}), + KOKKOS_LAMBDA(const std::int64_t row, const std::int64_t inner) { + element(static_cast(row), static_cast(inner)); + }); + } else { + cartesian_for(label, execution, rows, width, element); + } +#else + crossing_tiles( + label, execution, rows, width, + [&](const CrossingTile& tile) { tile.visit(element); }); +#endif +} + +struct SpatialElement { + Complex linear, prefactor; +}; + +// A device crossing through a 32-by-32 tile of team scratch (SEP 28): the tile is read +// with adjacent threads along the side that is contiguous for `read` (rows when +// `read_rows`, else bins) and written with adjacent threads along the other, so both +// the transverse scratch and the chunk's rows are accessed coalesced. +template +void tiled_crossing(const char* label, const ExecutionSpace& execution, + const std::size_t rows, const std::size_t width, + const bool read_rows, const Read& read, const Write& write) { + constexpr int tile = 32; + const int depth = std::min(8, execution.concurrency()); + using Policy = Kokkos::TeamPolicy; + using Member = Policy::member_type; + using Shared = + Kokkos::View>; + const std::size_t row_tiles = (rows + tile - 1) / tile; + const std::size_t bin_tiles = (width + tile - 1) / tile; + const auto bytes = Shared::shmem_size(tile, tile + 1); + launch::parallel_for( + label, + Policy(execution, static_cast(row_tiles * bin_tiles), depth, tile) + .set_scratch_size(0, Kokkos::PerTeam(bytes)), + KOKKOS_LAMBDA(const Member& team) { + // The buffer holds (bin, row) within the tile. + Shared buffer(team.team_scratch(0), tile, tile + 1); + const auto which = static_cast(team.league_rank()); + const std::size_t first_row = (which / bin_tiles) * tile; + const std::size_t first_bin = (which % bin_tiles) * tile; + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, tile), [&](const int outer) { + Kokkos::parallel_for( + Kokkos::ThreadVectorRange(team, tile), [&](const int lane) { + const int row_local = read_rows ? lane : outer; + const int bin_local = read_rows ? outer : lane; + const std::size_t row = first_row + static_cast(row_local); + const std::size_t bin = first_bin + static_cast(bin_local); + if (row < rows && bin < width) + buffer(bin_local, row_local) = read(row, bin); + }); + }); + team.team_barrier(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, tile), [&](const int outer) { + Kokkos::parallel_for( + Kokkos::ThreadVectorRange(team, tile), [&](const int lane) { + const int row_local = read_rows ? outer : lane; + const int bin_local = read_rows ? lane : outer; + const std::size_t row = first_row + static_cast(row_local); + const std::size_t bin = first_bin + static_cast(bin_local); + if (row < rows && bin < width) + write(row, bin, buffer(bin_local, row_local)); + }); + }); + }); +} + +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +// A device update of registers and the transverse scratch together (SEP 28): a tile of +// the scratch is staged in team scratch with adjacent threads along rows, `body` +// visits each (row, bin) with adjacent threads along bins (the registers' contiguous +// side) and may change its cell, and the tile goes back to the scratch along rows +// when `writes`. +template +void tiled_update(const char* label, const ExecutionSpace& execution, + const std::size_t rows, const std::size_t width, const bool writes, + const Read& read, const Body& body, const Write& write) { + constexpr int tile = 32; + constexpr int depth = 8; + using Policy = Kokkos::TeamPolicy; + using Member = Policy::member_type; + using Shared = Kokkos::View>; + const std::size_t row_tiles = (rows + tile - 1) / tile; + const std::size_t bin_tiles = (width + tile - 1) / tile; + const auto bytes = Shared::shmem_size(tile, tile + 1); + launch::parallel_for( + label, + Policy(execution, static_cast(row_tiles * bin_tiles), depth, tile) + .set_scratch_size(0, Kokkos::PerTeam(bytes)), + KOKKOS_LAMBDA(const Member& team) { + Shared buffer(team.team_scratch(0), tile, tile + 1); + const auto which = static_cast(team.league_rank()); + const std::size_t first_row = (which / bin_tiles) * tile; + const std::size_t first_bin = (which % bin_tiles) * tile; + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, tile), [&](const int outer) { + Kokkos::parallel_for( + Kokkos::ThreadVectorRange(team, tile), [&](const int lane) { + const std::size_t row = first_row + static_cast(lane); + const std::size_t bin = first_bin + static_cast(outer); + if (row < rows && bin < width) + buffer(outer, lane) = read(row, bin); + }); + }); + team.team_barrier(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, tile), [&](const int outer) { + Kokkos::parallel_for( + Kokkos::ThreadVectorRange(team, tile), [&](const int lane) { + const std::size_t row = first_row + static_cast(outer); + const std::size_t bin = first_bin + static_cast(lane); + if (row < rows && bin < width) + body(row, bin, buffer(lane, outer)); + }); + }); + if (!writes) + return; + team.team_barrier(); + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, tile), [&](const int outer) { + Kokkos::parallel_for( + Kokkos::ThreadVectorRange(team, tile), [&](const int lane) { + const std::size_t row = first_row + static_cast(lane); + const std::size_t bin = first_bin + static_cast(outer); + if (row < rows && bin < width) + write(row, bin, buffer(outer, lane)); + }); + }); + }); +} +#endif + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +// Plan, then discard, a batched complex transform of transverse planes at the plan +// policy's effort, leaving its wisdom. It overwrites both arrays; the caller clears +// them. Row transforms are seeded by `seed_row_fft_wisdom` (fft_plan.hpp). +inline void seed_fftw_wisdom(const ExecutionSpace& space, Complex* input, + Complex* output, const std::vector& extents, + const int howmany, const int input_distance, + const int output_distance, const int sign) { + std::string sizes; + for (const int extent : extents) + sizes += (sizes.empty() ? "" : "x") + std::to_string(extent); + const std::string key = fft_plan_key( + sign == FFTW_FORWARD ? "planes-forward" : "planes-backward", sizes, howmany, + input_distance, output_distance, input, output, space.concurrency()); + fftw_init_threads(); + const auto chosen = begin_seed(key, FftShape::planes, space.concurrency()); + if (!chosen) + return; + fftw_plan_with_nthreads(space.concurrency()); + auto* in = reinterpret_cast(input); + auto* out = reinterpret_cast(output); + auto* const seeded = fftw_plan_many_dft( + static_cast(extents.size()), extents.data(), howmany, in, nullptr, 1, + input_distance, out, nullptr, 1, output_distance, sign, *chosen); + // The policy has always tolerated a plane it cannot seed; a profile records it. + if (seeded == nullptr && (*chosen & FFTW_WISDOM_ONLY) == 0U && + (detail::planning == nullptr || + detail::planning->mode == FftPlanning::Mode::legacy)) { + release_seed(key, false); + return; + } + end_seed(key, *chosen, seeded != nullptr); + if (seeded != nullptr) + fftw_destroy_plan(seeded); +} +#endif + +// Where a separable geometry's column (or a temporal row, which is the same index) +// lives in the transverse scratch. Free space: (.., n_y, n_x), x fastest. The hybrid +// planar kind: (.., n_p, n_y) with the y transform trailing, the channel fastest in +// the column index p + n_p * iy. +struct TransverseLayout { + std::size_t inner{}; + bool channel_fastest{}; + // Elements per transverse plane of the row-major scratch `(planes, ., .)`. + std::size_t plane{}; + // A column's offset within a plane: the column itself on free space (y, x with x + // fastest), `p * n_y + iy` on the planar kind. + KOKKOS_INLINE_FUNCTION std::size_t offset(const std::size_t column) const { + return channel_fastest ? (column % inner) * (plane / inner) + column / inner + : column; + } + template + KOKKOS_INLINE_FUNCTION auto& cell(const View& scratch, const std::size_t plane_index, + const std::size_t column) const { + return scratch.data()[plane_index * plane + offset(column)]; + } + KOKKOS_INLINE_FUNCTION std::size_t first(const std::size_t column) const { + return channel_fastest ? column % inner : column / inner; + } + KOKKOS_INLINE_FUNCTION std::size_t second(const std::size_t column) const { + return channel_fastest ? column / inner : column % inner; + } +}; + +// A node's physical spectrum in a transverse volume after the inverse transverse plan: +// on free space the node's own cell; on the hybrid planar kind the slab rule over its y +// row's channels, without the temporal pad scale the planar synthesis carries, since +// the statistics read the coarse field (solprop's `collect_spatial`). +struct PhysicalSpectrum { + TransverseLayout layout{}; + RealView2 synthesis; + std::size_t n_x{}, n_p{}; + double crop_scale = 1.0; + bool slab = false; + template + KOKKOS_INLINE_FUNCTION Complex read(const View& source, const std::size_t plane, + const std::size_t node) const { + if (!slab) + return layout.cell(source, plane, node); + const auto ix = node % n_x; + const auto rows = (node / n_x) * n_p; + Complex sum{}; + for (std::size_t channel = 0; channel < n_p; ++channel) + sum += synthesis(ix, channel) * + source(plane, layout.first(rows + channel), layout.second(rows + channel)); + return crop_scale * sum; + } +}; + +// Where a separable route's low-storage registers (the accumulator, the stage register +// and the flow table) hold the element (column, omega). On the host, at the transverse +// scratch's own position: a fold then reads them in the scratch's order beside it, and +// only the state crosses layouts. On a device, at the state's (column-major). +struct RegisterIndex { + TransverseLayout layout{}; + std::size_t n_pol{}, n{}; + bool plane{}; + // One polarisation: a column is its mode, without a division (a flag of its own, + // which the compiler cannot fold back into `column / 1`). + bool scalar{}; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // A stage of the volume route on a device (`volume_order_`): the scratch is `(y, x, + // omega)`, the state's and the registers' own order, so nothing crosses. + bool volume{}; +#endif + KOKKOS_INLINE_FUNCTION std::size_t mode(const std::size_t column) const { + return scalar ? column : column / n_pol; + } + KOKKOS_INLINE_FUNCTION std::size_t pol(const std::size_t column) const { + return scalar ? 0 : column % n_pol; + } + // The transverse scratch's position of the element, on any build. + KOKKOS_INLINE_FUNCTION std::size_t cell(const std::size_t column, + const std::size_t omega) const { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + if (volume) + return column * n + omega; +#endif + return (omega * n_pol + pol(column)) * layout.plane + layout.offset(mode(column)); + } + KOKKOS_INLINE_FUNCTION std::size_t operator()(const std::size_t column, + const std::size_t omega) const { + return plane ? cell(column, omega) : column * n + omega; + } +}; + +// One chunk of a separable geometry's response pass: whole transverse units (a node +// on free space, a y row of n_p columns and n_x nodes on the hybrid planar kind), its +// temporal rows (state columns) and its physical nodes. +struct TransverseChunk { + std::size_t first_row{}, rows{}, first_node{}, nodes{}; +}; + +struct SpatialCoefficients { + ComplexView square, square_low, numerator, clamped; + RealView transverse, transverse_low, frame, spectral_rate, user_rate, absorber; + // A transverse collar's power rate per column (solprop kernel schema 31). + RealView transverse_rate; + // The hybrid planar kind's wall terms, (channels, n_omega); empty on free space. + ComplexView2 kappa; + std::size_t channels{}; + double evanescent{}; + // The longitudinal square q of a column and bin and its principal root, solprop's + // `longitudinal_root`: the one evaluation the operator and the plane measure share. + KOKKOS_INLINE_FUNCTION Complex longitudinal_square(const std::size_t mode, + const std::size_t omega) const { + Complex longitudinal = + (square(omega) - transverse(mode)) + (square_low(omega) - transverse_low(mode)); + // q = k^2 eps_co - k_y^2 - kappa_p^2, the wall term subtracted last as in solprop's + // separable_values (kernel schema 15), so a free-space column's arithmetic is + // untouched. + if (channels != 0) + longitudinal = longitudinal - kappa(mode % channels, omega); + return longitudinal; + } + // A real q (a lossless medium) takes the real root, the same bits as the complex + // root of q + 0i at a fraction of its cost (SEP 28): sqrt(q) or i sqrt(-q). + static KOKKOS_INLINE_FUNCTION Complex root_of(const Complex longitudinal) { + return longitudinal.imag() == 0.0 + ? (longitudinal.real() >= 0.0 + ? Complex(Kokkos::sqrt(longitudinal.real()), 0.0) + : Complex(0.0, Kokkos::sqrt(-longitudinal.real()))) + : Kokkos::sqrt(longitudinal); + } + // The conjugate of the plane's dimensionless forward admittance `c k_z / omega` + // (solprop SEP 34, `axial_admittance`) on an active propagating bin at positive + // frequency, zero elsewhere: the factor an internal (conjugate) spectrum takes to + // its magnetic closure. Its real part weighs energy and spectra. + KOKKOS_INLINE_FUNCTION Complex closing(const std::size_t mode, + const std::size_t omega, const bool active, + const double frequency) const { + const Complex longitudinal = longitudinal_square(mode, omega); + if (!active || !(longitudinal.real() > 0.0) || !(frequency > 0.0)) + return Complex{}; + const Complex root = root_of(longitudinal); + constexpr double speed_of_light = 299792458.0; + return Complex(speed_of_light * root.real(), -(speed_of_light * root.imag())) / + frequency; + } + // `value`'s prefactor alone, the same arithmetic: what a fold whose flows are + // tabulated reads of the element. + KOKKOS_INLINE_FUNCTION Complex prefactor(const std::size_t mode, + const std::size_t omega, + const bool active) const { + if (!active) + return Complex{}; + const Complex longitudinal = longitudinal_square(mode, omega); + if (!(longitudinal.real() > 0.0)) + return clamped(omega); + const Complex root = root_of(longitudinal); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + return numerator(omega) * (1.0 / root.real()); +#else + return numerator(omega) / root.real(); +#endif + } + KOKKOS_INLINE_FUNCTION SpatialElement value(const std::size_t mode, + const std::size_t omega, + const bool active) const { + const Complex longitudinal = longitudinal_square(mode, omega); + const Complex root = root_of(longitudinal); + const bool propagating = longitudinal.real() > 0.0; + const double attenuation = + propagating ? root.imag() : Kokkos::min(Kokkos::abs(root.imag()), evanescent); + // The transverse collar's rate joins the attenuation inside the active band, as + // in solprop's separable_values (SEP 22, Q6); a zero rate changes no bit. + const Complex linear = active ? Complex(-attenuation - 0.5 * transverse_rate(mode), + -(root.real() - frame(omega))) + : Complex{}; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // One binary64 reciprocal and two products in place of two divisions, at every + // element of the stage passes: a device's division is several fused steps (at + // most an ulp from the quotients). + const Complex prefactor = + propagating ? numerator(omega) * (1.0 / root.real()) : clamped(omega); +#else + const Complex prefactor = + propagating ? numerator(omega) / root.real() : clamped(omega); +#endif + return {.linear = linear - 0.5 * spectral_rate(omega), + .prefactor = active ? prefactor : Complex{}}; + } +}; + +KOKKOS_INLINE_FUNCTION Complex exponential(const Complex value) { + const double magnitude = Kokkos::exp(value.real()); + return {magnitude * Kokkos::cos(value.imag()), magnitude * Kokkos::sin(value.imag())}; +} + +// `cos theta` and `sin theta`. On a device they are one `sincos` call, which shares +// the argument's reduction between the two (on a GPU of reduced binary64 rate the +// stage propagators are about half of the stage entry); the host keeps the two +// library calls, so its values are unchanged. +KOKKOS_INLINE_FUNCTION void cosine_sine(const double theta, double& cosine, + double& sine) { +#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__) + ::sincos(theta, &sine, &cosine); +#else + cosine = Kokkos::cos(theta); + sine = Kokkos::sin(theta); +#endif +} + +// A flow's magnitude: on a lossless bin `exp(+-0)`, exactly one, taken without a +// binary64 `exp` per element. Out of line on a host, whose call to its library was out +// of line already: inlined there, this branch took the tabulated fold's whole kernel +// past the compiler's inlining limit, 40 % slower (solprop's docs/dev/benchmarks.md). +#if defined(__GNUC__) && !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +[[gnu::noinline]] +#endif +KOKKOS_INLINE_FUNCTION double flow_magnitude(const double real) { + return real == 0.0 ? 1.0 : Kokkos::exp(real); +} + +// A stage or step propagator `exp(argument)`: `exponential` with the cosine and sine +// from `cosine_sine`, so it is `exponential` to the bit on the host. +KOKKOS_INLINE_FUNCTION Complex propagator_exp(const Complex argument) { + const double magnitude = flow_magnitude(argument.real()); + double cosine = 0.0; + double sine = 0.0; + cosine_sine(argument.imag(), cosine, sine); + return {magnitude * cosine, magnitude * sine}; +} + +// `exponential`, or on a lossless problem (a signed-zero real part) the same value +// with its magnitude `exp(+-0)`, exactly one, taken without the call. The cosine and +// sine come from `cosine_sine`, so on the host it is `exponential` to the bit. +template +KOKKOS_INLINE_FUNCTION Complex entry_exponential(const Complex value) { + if constexpr (Lossless) { + double cosine = 0.0; + double sine = 0.0; + cosine_sine(value.imag(), cosine, sine); + return {cosine, sine}; + } else { + return propagator_exp(value); + } +} + +KOKKOS_INLINE_FUNCTION double squared_abs(const Complex value) { + return value.real() * value.real() + value.imag() * value.imag(); +} + +KOKKOS_INLINE_FUNCTION double trapezoid(const double current, const double previous, + const double dt) { + return dt * (current + previous) / 2.0; +} + +// The fraction ionised from an exposure's decay `exp(-X)`, `1 - (1 - p) exp(-X)`, as +// `bodies.fraction_at` rounds it. A small response: strict arithmetic (solprop SEP 25 +// section 4.2, `fp.hpp`), so the product and the difference round separately. +SOLPROP_NATIVE_STRICT_FUNCTION +KOKKOS_INLINE_FUNCTION double decayed_fraction(const double decay, + const double preionisation) { + SOLPROP_NATIVE_STRICT_BLOCK + const double survival = (1.0 - preionisation) * decay; + return 1.0 - survival; +} + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +// `decayed_fraction` of each of a host block's `count` decays, in place: one strict +// loop that vectorises, each product and difference rounded as there. +SOLPROP_NATIVE_STRICT_FUNCTION +inline void decayed_fractions(double* values, const std::size_t count, + const double preionisation) { + SOLPROP_NATIVE_STRICT_BLOCK + for (std::size_t k = 0; k < count; ++k) { + const double survival = (1.0 - preionisation) * values[k]; + values[k] = 1.0 - survival; + } +} +#endif + +// The fraction ionised from an exposure, `decayed_fraction` of its `exp(-X)`. +KOKKOS_INLINE_FUNCTION double ionised_fraction(const double exposure, + const double preionisation) { + return decayed_fraction(Kokkos::exp(-exposure), preionisation); +} + +// The envelope column's fraction, `p + (1 - p) (-expm1(-X))`, strict as above. +SOLPROP_NATIVE_STRICT_FUNCTION +KOKKOS_INLINE_FUNCTION double envelope_fraction(const double exposure, + const double preionisation) { + SOLPROP_NATIVE_STRICT_BLOCK + const double ionised = (1.0 - preionisation) * (-Kokkos::expm1(-exposure)); + return preionisation + ionised; +} + +// The band rule's fraction: the exposure floored at zero where the fraction is formed +// (SEP 18 section 4.5), `max(X, 0)` as NumPy's `maximum` takes it, so a NaN exposure +// stays NaN rather than becoming the preionisation. +KOKKOS_INLINE_FUNCTION double band_fraction(const double exposure, + const double preionisation) { + return ionised_fraction(exposure < 0.0 ? 0.0 : exposure, preionisation); +} + +// A multistep compensation's residual term `(D - x / dz) u`: the difference of two +// nearly equal rates, strict (solprop SEP 25 section 4.2), each complex operation +// spelt with `fp.hpp`'s helpers. +SOLPROP_NATIVE_STRICT_FUNCTION +KOKKOS_INLINE_FUNCTION Complex compensation_residual(const Complex rate, + const Complex exponent, + const double inverse_dz, + const Complex state) { + SOLPROP_NATIVE_STRICT_BLOCK + return strict_mul(strict_sub(rate, strict_scale(inverse_dz, exponent)), state); +} + +// `bodies.loss_scale`: the loss current's coefficient of the field, or zero. +KOKKOS_INLINE_FUNCTION double band_loss_scale(const double rate, const double fraction, + const double squared, + const double ionisation_potential) { + if (rate > 0.0 && squared > 0.0) + return ionisation_potential * rate * (1.0 - fraction) / squared; + return 0.0; +} + +// The rate table's cubic in `log(rate)` at a field's `log |E|` within the table's +// range: the interval's local coordinate and its Horner form (`ionisation_rate`). +template +KOKKOS_INLINE_FUNCTION double +ionisation_log_rate(const double log_field, const View& coefficients, + const double log_field_min, const double d_log_field, + const std::size_t intervals, const std::size_t offset = 0) { + long raw_index = static_cast((log_field - log_field_min) / d_log_field); + if (raw_index < 0) + raw_index = 0; + // NOLINTNEXTLINE(modernize-use-integer-sign-comparison): device code; std::cmp_* is host-only + if (raw_index >= static_cast(intervals)) + raw_index = static_cast(intervals - 1); + const auto index = static_cast(raw_index); + const double local = + log_field - (log_field_min + static_cast(index) * d_log_field); + return ((coefficients(offset + index) * local + + coefficients(offset + intervals + index)) * + local + + coefficients(offset + 2 * intervals + index)) * + local + + coefficients(offset + 3 * intervals + index); +} + +// The ionisation rate at a field magnitude: the table's limits outside its range, and +// within it the exponential of its cubic in `log |E|`. +template +KOKKOS_INLINE_FUNCTION double +ionisation_rate(const double magnitude, const View& coefficients, + const double log_field_min, const double d_log_field, + const double field_min, const double field_max, const double rate_max, + const std::size_t intervals, const std::size_t offset = 0) { + if (magnitude > field_max) + return rate_max; + if (magnitude < field_min) + return 0.0; + return Kokkos::exp(ionisation_log_rate(Kokkos::log(magnitude), coefficients, + log_field_min, d_log_field, intervals, + offset)); +} + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +// The samples a host pass of the plasma column takes at a time. +constexpr std::size_t rate_block = 256; + +// `ionisation_rate`'s values at `live <= rate_block` magnitudes within the table's +// range, in place: one `log` and one `exp` pass of the vector-maths layer (solprop +// ADR-059) around the cubic. A host block gathers the magnitudes it passes, so the +// samples outside the range (most of a pulse's wings) cost what they cost before. +template +void ionisation_rates_within(double* values, const std::size_t live, + const View& coefficients, const double log_field_min, + const double d_log_field, const std::size_t intervals, + const std::size_t offset = 0) { + // The cubic writes a local block, which no table gather can alias, so the loop + // vectorises with the table's gathers wherever this is compiled out of line. + double log_rates[rate_block]; + vmath::log(values, values, live); + for (std::size_t k = 0; k < live; ++k) + log_rates[k] = ionisation_log_rate(values[k], coefficients, log_field_min, + d_log_field, intervals, offset); + vmath::exp(log_rates, values, live); +} +#endif + +// A host span as an unmanaged view of a state-shaped memory-space view, for one copy +// between them: no staging mirror, no element loop. A span whose data is not aligned +// as `Complex` (a NumPy buffer need not be) is staged in an aligned vector first. +inline void upload_span(const ExecutionSpace& execution_space, crossing::Ring& ring, + const ComplexView2& destination, + const std::span source) { + if (destination.size() != source.size()) + throw std::invalid_argument("fixed problem array has the wrong extent"); + if (reinterpret_cast(source.data()) % alignof(Complex) != 0) { + const std::vector aligned(source.begin(), source.end()); + upload_span(execution_space, ring, destination, aligned); + execution_space.fence("solprop_native unaligned upload"); + return; + } + if (crossing::takes_ring(source.size_bytes())) { + execution_space.fence("solprop_native upload"); + ring.from_host(destination.data(), source.data(), source.size_bytes()); + return; + } + const Kokkos::View> + host(source.data(), destination.extent(0), destination.extent(1)); + graph::deep_copy(execution_space, destination, host); +} + +template +void copy_vector(const ExecutionSpace& execution_space, const View& destination, + const std::vector& source) { + if (destination.size() != source.size()) + throw std::invalid_argument("fixed problem array has the wrong extent"); + auto mirror = Kokkos::create_mirror_view(destination); + auto* data = mirror.data(); + for (std::size_t index = 0; index < source.size(); ++index) + data[index] = source[index]; + graph::deep_copy(execution_space, destination, mirror); +} + +inline void require_extent(const std::size_t actual, const std::size_t expected, + const char* name) { + if (actual != expected) + throw std::invalid_argument(std::string(name) + " has the wrong extent"); +} + +inline std::size_t checked_product(const std::size_t left, const std::size_t right, + const char* name) { + if (left != 0 && right > std::numeric_limits::max() / left) + throw std::length_error(std::string(name) + " extent overflows size_t"); + return left * right; +} + +inline std::size_t checked_sum(const std::size_t left, const std::size_t right, + const char* name) { + if (right > std::numeric_limits::max() - left) + throw std::length_error(std::string(name) + " extent overflows size_t"); + return left + right; +} + +inline void require_allocation(const std::size_t elements, + const std::size_t element_bytes, const char* name) { + static_cast(checked_product(elements, element_bytes, name)); +} + +// The samples of one tile of the device trapezoid plasma's two-level scans (unused on +// a host build). +[[maybe_unused]] inline constexpr std::size_t plasma_tile_samples = 1024; + +// Descriptors borrow allocations retained by the member GeneralState owners. They +// contain no ownership or reference counts and are refreshed after state swaps. +struct UpdateColumn { + const Complex *current{}, *solution{}, *error{}, *whole{}; + Complex *advanced{}, *residual{}; + const unsigned char* in_norm{}; + double *old_partial{}, *new_partial{}, *error_partial{}; + // The state the solution is added to: `current` itself, or a coupled flow's + // whole-step action of it (solprop `accept_step(U u, ..., U k)`); the old norm is + // always taken on `current`. + const Complex* carried{}; +}; + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +/** + * The fewest bins a host thread of a reduction team takes: below it the team's + * synchronisation costs more than the thread's share of the pass. + */ +constexpr std::size_t host_reduction_bins_per_thread = 2048; + +/** A host team per column, sharing the threads when the columns are fewer. */ +inline int host_reduction_team_size(const ExecutionSpace& execution, + const std::size_t columns, const std::size_t bins) { + const auto threads = static_cast(std::max(1, execution.concurrency())); + const auto per_column = + std::max(1, threads / std::max(1, columns)); + const auto by_bins = std::max(1, bins / host_reduction_bins_per_thread); + return static_cast(std::min(per_column, by_bins)); +} +#endif + +// The teams a column's norm reductions are split over on a device, each writing its +// own partial sums: one team per column fills a GPU only when the columns are many, so +// a one-node fibre's column takes up to `reduction_target_teams` teams of at least +// `reduction_min_bins` bins each. The host keeps one slot per column. The slots depend +// on the member's columns and bins alone, so a batch member and its solo run sum in one +// order. +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +constexpr std::size_t reduction_target_teams = 256; +constexpr std::size_t reduction_min_bins = 1024; +#endif +inline std::size_t reduction_slots(const std::size_t columns, const std::size_t n) { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const std::size_t wanted = + (reduction_target_teams + std::max(columns, 1) - 1) / + std::max(columns, 1); + const std::size_t most = + std::max(1, (n + reduction_min_bins - 1) / reduction_min_bins); + return std::max(1, std::min(wanted, most)); +#else + static_cast(columns); + static_cast(n); + return 1; +#endif +} + +// The bins `[first, end)` of slot `slot` of `slots` over `n`. +KOKKOS_INLINE_FUNCTION void slot_range(const std::size_t n, const std::size_t slots, + const std::size_t slot, std::size_t& first, + std::size_t& end) { + const std::size_t width = (n + slots - 1) / slots; + first = Kokkos::min(n, slot * width); + end = Kokkos::min(n, first + width); +} + +// The update and its three binary64 norms in one pass. On the host one team per column, +// its threads splitting the bins when the columns are fewer than the threads; on a +// device `slots` teams per column, each over its own bins, writing its partial sums at +// offset `slot` from the column's partial pointers. The sums associate as the teams +// split them (solprop SEP 25), and the host reduces them. +template +void accept_update(const ExecutionSpace& execution, const std::size_t count, + const std::size_t n, const double dz, const Columns& columns, + const char* label, const std::size_t slots) { + using ReductionPolicy = Kokkos::TeamPolicy; + using ReductionMember = ReductionPolicy::member_type; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const int team_size = SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE; +#else + const int team_size = host_reduction_team_size(execution, count, n); +#endif + const ReductionPolicy reduction_policy(execution, static_cast(count * slots), + team_size); + launch::parallel_for( + label, reduction_policy, KOKKOS_LAMBDA(const ReductionMember& team) { + const auto league = static_cast(team.league_rank()); + const auto column = league / slots; + const auto slot = league % slots; + std::size_t first = 0; + std::size_t end = 0; + slot_range(n, slots, slot, first, end); + const auto values = columns(column); + NormSums sums; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(first), + static_cast(end)), + [&](const int index_value, NormSums& update) { + const auto index = static_cast(index_value); + const Complex candidate = + values.whole[index] * + (values.carried[index] + dz * values.solution[index]); + values.advanced[index] = candidate; + const Complex one_residual = + values.whole[index] * (dz * values.error[index]); + values.residual[index] = one_residual; + if (values.in_norm[index] == 0U) + return; + update.old_sum += squared_abs(values.current[index]); + update.new_sum += squared_abs(candidate); + update.error_sum += squared_abs(one_residual); + }, + Kokkos::Sum(sums)); + Kokkos::single(Kokkos::PerTeam(team), [&] { + values.old_partial[slot] = sums.old_sum; + values.new_partial[slot] = sums.new_sum; + values.error_partial[slot] = sums.error_sum; + }); + }); +} + +// The bins a host work item of the stage's entry kernel takes at a time: every bin's +// phase sum is the same sequence of rounded products and additions in term order as +// a per-bin loop, laid out so that the bins vectorise. +constexpr std::size_t stage_block = 64; +// The pinned staging buffers a state's per-trial scalars cross from (`upload_trial`). +constexpr std::size_t stage_staging_ring = 8; + +// The pointwise bodies of a stage over the coarse spectrum: its entry (the phase +// increment, the combination with the earlier stages, the lab frame, the noise and the +// pad) and its finish (prefactor, window, absorber, return to the interaction picture +// and accumulation). The entry kernel runs the finish of the stage before it in the +// same traversal when nothing lies between them (solprop SEP 25 section 6.7): both +// are pointwise over the bins, and the combination reads only its own bin's stages. +// How a stage's entry forms its propagator pair (`StageTerms::combine`). +enum class EntryPhase { general, lossless, repeated }; + +// The transcendental factors of a stage's combination at one bin: the phase's +// `cos b` and `sin b`, its magnitudes `e^a` and `e^-a` (one on a lossless problem), +// and the noise phase's cosine and sine. +struct EntryFactors { + double cosine = 0.0, sine = 0.0, magnitude = 1.0, inverse = 1.0; + double noise_cosine = 0.0, noise_sine = 0.0; +}; + +// A host work item's factors for its block of bins, formed by the vector-maths layer +// in one pass of each function (`StageTerms::entry_factors_block`); the factors a +// stage does not use stay zero. +struct EntryBlock { + double cosine[stage_block], sine[stage_block]; + double magnitude[stage_block], inverse[stage_block]; + double noise_cosine[stage_block], noise_sine[stage_block]; + + [[nodiscard]] EntryFactors at(const std::size_t bin) const { + return {.cosine = cosine[bin], + .sine = sine[bin], + .magnitude = magnitude[bin], + .inverse = inverse[bin], + .noise_cosine = noise_cosine[bin], + .noise_sine = noise_sine[bin]}; + } +}; + +struct StageTerms { + ComplexView2 current, fine, whole, solution, error, stage_phase, stage_inverse, noise; + ComplexView2 flow_input, flow_lab, flow_drive, general_scaled, general_source; + ComplexView2 projected, absorber; + ComplexView3 stages, phi_terms, prefactor_terms; + RealView tableau, phi_delta, phi_at, prefactor_weights, window; + IndexView crop, gaps; + ByteView active; + std::size_t gap_ranges = 0; + std::size_t n = 0, n_phi = 0, n_stages = 0, n_prefactor = 0, n_to = 0; + double inverse_n_to = 0.0, general_pad = 0.0; + bool add_noise = false, analytic = false, general = false, flow = false; + bool carried = false, keep_lab = false, zero_response = false; + bool in_stage_absorber = false; + // Every coarse bin is the fine bin of its own index (`crop` is the identity). + bool identity_crop = false; + + // The stage's phase increment `sum_t dPhi_t Phi_t` at one bin: strict (solprop SEP + // 25 section 4.2), each product and sum rounded in term order. A zero weight's term + // is skipped: its product with a finite term is a signed zero, which leaves the + // running sum (started at +0) unchanged, so the skip is exact. A z table's terms + // are its values at a trial's positions, weighed by +-1 and mostly zero (SEP 29 G1). + SOLPROP_NATIVE_STRICT_FUNCTION + KOKKOS_INLINE_FUNCTION Complex phase_sum(const std::size_t offset, + const std::size_t column, + const std::size_t index) const { + SOLPROP_NATIVE_STRICT_BLOCK + double real = 0.0; + double imag = 0.0; + for (std::size_t term = 0; term < n_phi; ++term) { + const double weight = phi_delta(offset + term); + if (weight == 0.0) + continue; + const Complex fixed = phi_terms(term, column, index); + const double scaled_real = weight * fixed.real(); + const double scaled_imag = weight * fixed.imag(); + real = real + scaled_real; + imag = imag + scaled_imag; + } + return {real, imag}; + } + + // The same sums over a block of bins, laid out so that the bins vectorise. + SOLPROP_NATIVE_STRICT_FUNCTION + KOKKOS_INLINE_FUNCTION void + phase_block(const std::size_t offset, const std::size_t column, + const std::size_t first, const std::size_t count, + double (&real)[stage_block], double (&imag)[stage_block]) const { + SOLPROP_NATIVE_STRICT_BLOCK + for (std::size_t bin = 0; bin < stage_block; ++bin) { + real[bin] = 0.0; + imag[bin] = 0.0; + } + for (std::size_t term = 0; term < n_phi; ++term) { + const double weight = phi_delta(offset + term); + if (weight == 0.0) + continue; + for (std::size_t bin = 0; bin < count; ++bin) { + const Complex fixed = phi_terms(term, column, first + bin); + const double scaled_real = weight * fixed.real(); + const double scaled_imag = weight * fixed.imag(); + real[bin] = real[bin] + scaled_real; + imag[bin] = imag[bin] + scaled_imag; + } + } + } + + // The noise's phase: strict (solprop SEP 25 section 4.2). + SOLPROP_NATIVE_STRICT_FUNCTION + KOKKOS_INLINE_FUNCTION Complex noise_phase(const std::size_t stage, + const std::size_t column, + const std::size_t index) const { + SOLPROP_NATIVE_STRICT_BLOCK + Complex sum{}; + for (std::size_t term = 0; term < n_phi; ++term) { + const double weight = phi_at(stage * n_phi + term); + // Exact, as in `phase_sum`. + if (weight != 0.0) + sum = strict_accumulate(sum, weight, phi_terms(term, column, index)); + } + return sum; + } + + // The factors of stage `stage`'s combination at one bin, one bin at a time (a + // device's entry; the host forms a block's together, `entry_factors_block`). The + // propagator `exp(phase)` and its inverse `exp(-phase)` share their cosine and sine, + // each magnitude is its own `exp`, and a lossless problem's are exactly one, taken + // without the calls (`combine`). + template + KOKKOS_INLINE_FUNCTION EntryFactors entry_factors(const std::size_t stage, + const std::size_t column, + const std::size_t index, + const Complex phase) const { + EntryFactors factors; + if constexpr (Mode != EntryPhase::repeated) + cosine_sine(phase.imag(), factors.cosine, factors.sine); + if constexpr (Mode == EntryPhase::general) { + factors.magnitude = Kokkos::exp(phase.real()); + factors.inverse = Kokkos::exp(-phase.real()); + } + if (add_noise) { + const Complex phase_at = noise_phase(stage, column, index); + factors.noise_cosine = Kokkos::cos(phase_at.imag()); + factors.noise_sine = Kokkos::sin(phase_at.imag()); + } + return factors; + } + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // Whether the first `count` of a block's phases have real parts that are all zeros. + static bool lossless_block(const double (&real)[stage_block], + const std::size_t count) { + bool zero = true; + for (std::size_t bin = 0; bin < count; ++bin) + zero = zero && real[bin] == 0.0; + return zero; + } + + // A host work item's factors for its `count` bins from `first`, each function one + // pass of the vector-maths layer over the block (solprop ADR-059): the phases' sines + // and cosines, their magnitudes and inverse magnitudes, and the noise phases'. + template + [[maybe_unused]] void + entry_factors_block(const std::size_t stage, const std::size_t column, + const std::size_t first, const std::size_t count, + const double (&real)[stage_block], + const double (&imag)[stage_block], EntryBlock& block) const { + if constexpr (Mode != EntryPhase::repeated) + vmath::sincos(imag, block.sine, block.cosine, count); + if constexpr (Mode == EntryPhase::general) { + // A block whose phases' real parts are all (signed) zeros -- a lossy problem's + // loss often lies in a few bands -- has both magnitudes `exp(+-0)`, exactly one, + // without the calls. + if (lossless_block(real, count)) { + for (std::size_t bin = 0; bin < count; ++bin) { + block.magnitude[bin] = 1.0; + block.inverse[bin] = 1.0; + } + } else { + double negated[stage_block]; + for (std::size_t bin = 0; bin < count; ++bin) + negated[bin] = -real[bin]; + vmath::exp(real, block.magnitude, count); + vmath::exp(negated, block.inverse, count); + } + } + if (add_noise) { + double angle[stage_block]{}; + for (std::size_t bin = 0; bin < count; ++bin) + angle[bin] = noise_phase(stage, column, first + bin).imag(); + vmath::sincos(angle, block.noise_sine, block.noise_cosine, count); + } + } +#endif + + // The combination of stage `stage` at one bin, from the stage's phase increment and + // its factors (`entry_factors`). The propagator `exp(phase)` and its inverse + // `exp(-phase)`, which the finish applies, share their cosine and sine (solprop SEP + // 25 section 6.7): the angle's parts are even and odd, so `exp(-phase)` is + // `e^{-a} (cos b, -sin b)` with its own magnitude, never a reciprocal of the forward + // value. A coupled flow's residual reads the phase. A lossless problem + // (`EntryPhase::lossless`: every Phi term's real part exactly zero, so the phase's + // is a signed zero) takes both magnitudes as `exp(+-0)`, exactly one, without the + // calls. A lossless stage whose phase weights equal the stage before's + // (`EntryPhase::repeated`) has that stage's inverse still in place, and its + // propagator is that inverse's conjugate, `(cos b, -(-sin b))`, to the bit. + // Returns the propagator. + template + KOKKOS_INLINE_FUNCTION Complex combine(const std::size_t stage, const double dz, + const double solution_weight, + const double error_weight, + const std::size_t column, + const std::size_t index, const Complex phase, + const EntryFactors& factors) const { + Complex propagator; + if constexpr (Mode == EntryPhase::repeated) { + const Complex inverse = stage_inverse(column, index); + propagator = Complex(inverse.real(), -inverse.imag()); + } else if constexpr (Mode == EntryPhase::lossless) { + propagator = Complex(factors.cosine, factors.sine); + stage_inverse(column, index) = Complex(factors.cosine, -factors.sine); + } else { + propagator = + Complex(factors.magnitude * factors.cosine, factors.magnitude * factors.sine); + stage_inverse(column, index) = + Complex(factors.inverse * factors.cosine, -(factors.inverse * factors.sine)); + } + if (flow) + stage_phase(column, index) = phase; + Complex value{}; + if (carried) { + value = propagator * flow_input(column, index); + } else { + Complex accumulated{}; + for (std::size_t previous = 0; previous < stage; ++previous) + accumulated += + tableau(stage * n_stages + previous) * stages(previous, column, index); + value = propagator * (current(column, index) + dz * accumulated); + } + if (add_noise) + value += noise(column, index) * Complex(factors.noise_cosine, factors.noise_sine); + const double analytic_scale = + analytic && index > 0 && 2 * crop(index) < n_to ? 2.0 : 1.0; + fine(column, crop(index)) = analytic_scale * inverse_n_to * value; + if (keep_lab) + flow_lab(column, index) = value; + // The general route synthesises on the coarse grid: the pad scale and the + // analytic doubling go on the channel spectra, the 1/n_to on its transform. + if (general) + general_scaled(column, index) = + value * (general_pad * general_doubling(crop(index), n_to)); + if (zero_response && flow) { + flow_drive(column, index) = Complex{}; + } else if (zero_response) { + stages(stage, column, index) = Complex{}; + if (stage == 0) { + solution(column, index) = Complex{}; + error(column, index) = Complex{}; + } else { + solution(column, index) += solution_weight * Complex{}; + error(column, index) += error_weight * Complex{}; + } + } + return propagator; + } + + // The fine bins no coarse bin fills, positions `[begin, end)` of this column's + // list, zeroed for the inverse transform (which may destroy its input). The list is + // held as runs of adjacent bins, `gaps(3 r + {0, 1, 2})` the run's first position, + // first bin and length, so each run zeroes contiguously. + KOKKOS_INLINE_FUNCTION void zero_gaps(const std::size_t column, + const std::size_t begin, + const std::size_t end) const { + for (std::size_t run = 0; run < gap_ranges; ++run) { + const std::size_t position = gaps(3 * run); + const std::size_t bin = gaps(3 * run + 1); + const std::size_t length = gaps(3 * run + 2); + const std::size_t lower = begin > position ? begin : position; + const std::size_t upper = end < position + length ? end : position + length; + for (std::size_t gap = lower; gap < upper; ++gap) + fine(column, bin + (gap - position)) = Complex{}; + } + } + + // The finish of stage `stage` at one bin, with the inverse propagator its + // combination formed. + KOKKOS_INLINE_FUNCTION void finish(const std::size_t stage, + const double solution_weight, + const double error_weight, + const std::size_t column, + const std::size_t index) const { + // A zero weight is skipped: a z table's terms are its values at a trial's + // positions, one-hot weighted (SEP 29 G1). The sum's value is unchanged; at most + // the sign of an exactly zero sum could differ. + Complex prefactor{}; + for (std::size_t term = 0; term < n_prefactor; ++term) { + const double weight = prefactor_weights(stage * n_prefactor + term); + if (weight != 0.0) + prefactor += weight * prefactor_terms(term, column, index); + } + Complex value = + prefactor * window(index) * + (general ? general_source(column, index) : projected(column, crop(index))); + if (in_stage_absorber && active(index) != 0U) + value += absorber(column, crop(index)); + value *= stage_inverse(column, index); + if (flow) { + flow_drive(column, index) = value; + return; + } + stages(stage, column, index) = value; + if (stage == 0) { + solution(column, index) = solution_weight * value; + error(column, index) = error_weight * value; + } else { + solution(column, index) += solution_weight * value; + error(column, index) += error_weight * value; + } + } + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // Whether a host work item takes its finish and combination as the block forms + // below: a stage that projects a source into the stage rows, off a coupled flow and + // the general route, whose stores the per-bin forms keep. + [[nodiscard]] bool blocks_vectorise() const noexcept { + return !flow && !general && !zero_response; + } + + // `finish` over the `count` bins of a block from `first`, each bin the same + // arithmetic in the same order, the block's parts held as separate real and + // imaginary lanes so that the bins vectorise; only the crop's gathers stay per bin. + [[maybe_unused]] void finish_block(const std::size_t stage, + const double solution_weight, + const double error_weight, + const std::size_t column, const std::size_t first, + const std::size_t count) const { + double real[stage_block]; + double imag[stage_block]; + double source_real[stage_block]; + double source_imag[stage_block]; + for (std::size_t bin = 0; bin < count; ++bin) { + real[bin] = 0.0; + imag[bin] = 0.0; + } + for (std::size_t term = 0; term < n_prefactor; ++term) { + const double weight = prefactor_weights(stage * n_prefactor + term); + if (weight == 0.0) + continue; + const Complex* values = &prefactor_terms(term, column, first); + for (std::size_t bin = 0; bin < count; ++bin) { + real[bin] += weight * values[bin].real(); + imag[bin] += weight * values[bin].imag(); + } + } + const std::size_t* bins = &crop(first); + const auto gather = [&](const Complex* row) { + if (identity_crop) + for (std::size_t bin = 0; bin < count; ++bin) { + source_real[bin] = row[first + bin].real(); + source_imag[bin] = row[first + bin].imag(); + } + else + for (std::size_t bin = 0; bin < count; ++bin) { + source_real[bin] = row[bins[bin]].real(); + source_imag[bin] = row[bins[bin]].imag(); + } + }; + gather(&projected(column, 0)); + const double* weights = &window(first); + for (std::size_t bin = 0; bin < count; ++bin) { + const double scaled_real = real[bin] * weights[bin]; + const double scaled_imag = imag[bin] * weights[bin]; + real[bin] = scaled_real * source_real[bin] - scaled_imag * source_imag[bin]; + imag[bin] = scaled_real * source_imag[bin] + scaled_imag * source_real[bin]; + } + if (in_stage_absorber) { + gather(&absorber(column, 0)); + const unsigned char* band = &active(first); + for (std::size_t bin = 0; bin < count; ++bin) { + const bool inside = band[bin] != 0U; + real[bin] = inside ? real[bin] + source_real[bin] : real[bin]; + imag[bin] = inside ? imag[bin] + source_imag[bin] : imag[bin]; + } + } + const Complex* inverse = &stage_inverse(column, first); + Complex* row = &stages(stage, column, first); + Complex* total = &solution(column, first); + Complex* estimate = &error(column, first); + for (std::size_t bin = 0; bin < count; ++bin) { + const double value_real = + real[bin] * inverse[bin].real() - imag[bin] * inverse[bin].imag(); + const double value_imag = + real[bin] * inverse[bin].imag() + imag[bin] * inverse[bin].real(); + row[bin] = Complex(value_real, value_imag); + if (stage == 0) { + total[bin] = + Complex(solution_weight * value_real, solution_weight * value_imag); + estimate[bin] = Complex(error_weight * value_real, error_weight * value_imag); + } else { + total[bin] = Complex(total[bin].real() + solution_weight * value_real, + total[bin].imag() + solution_weight * value_imag); + estimate[bin] = Complex(estimate[bin].real() + error_weight * value_real, + estimate[bin].imag() + error_weight * value_imag); + } + } + } + + // `combine` over the `count` bins of a block from `first`, from the block's factors, + // in the lanes of `finish_block`; with `store_whole`, each propagator is also the + // whole step's. + template + [[maybe_unused]] void + combine_block(const std::size_t stage, const double dz, const std::size_t column, + const std::size_t first, const std::size_t count, + const EntryBlock& factors, const bool store_whole) const { + double real[stage_block]; + double imag[stage_block]; + double sum_real[stage_block]; + double sum_imag[stage_block]; + Complex* inverse = &stage_inverse(column, first); + for (std::size_t bin = 0; bin < count; ++bin) { + if constexpr (Mode == EntryPhase::repeated) { + real[bin] = inverse[bin].real(); + imag[bin] = -inverse[bin].imag(); + } else if constexpr (Mode == EntryPhase::lossless) { + real[bin] = factors.cosine[bin]; + imag[bin] = factors.sine[bin]; + inverse[bin] = Complex(factors.cosine[bin], -factors.sine[bin]); + } else { + real[bin] = factors.magnitude[bin] * factors.cosine[bin]; + imag[bin] = factors.magnitude[bin] * factors.sine[bin]; + inverse[bin] = Complex(factors.inverse[bin] * factors.cosine[bin], + -(factors.inverse[bin] * factors.sine[bin])); + } + } + if (store_whole) { + Complex* propagator = &whole(column, first); + for (std::size_t bin = 0; bin < count; ++bin) + propagator[bin] = Complex(real[bin], imag[bin]); + } + for (std::size_t bin = 0; bin < count; ++bin) { + sum_real[bin] = 0.0; + sum_imag[bin] = 0.0; + } + for (std::size_t previous = 0; previous < stage; ++previous) { + const double weight = tableau(stage * n_stages + previous); + const Complex* earlier = &stages(previous, column, first); + for (std::size_t bin = 0; bin < count; ++bin) { + sum_real[bin] += weight * earlier[bin].real(); + sum_imag[bin] += weight * earlier[bin].imag(); + } + } + const Complex* state = ¤t(column, first); + for (std::size_t bin = 0; bin < count; ++bin) { + const double input_real = state[bin].real() + dz * sum_real[bin]; + const double input_imag = state[bin].imag() + dz * sum_imag[bin]; + const double value_real = real[bin] * input_real - imag[bin] * input_imag; + const double value_imag = real[bin] * input_imag + imag[bin] * input_real; + real[bin] = value_real; + imag[bin] = value_imag; + } + if (add_noise) { + const Complex* draw = &noise(column, first); + for (std::size_t bin = 0; bin < count; ++bin) { + const double noise_real = draw[bin].real() * factors.noise_cosine[bin] - + draw[bin].imag() * factors.noise_sine[bin]; + const double noise_imag = draw[bin].real() * factors.noise_sine[bin] + + draw[bin].imag() * factors.noise_cosine[bin]; + real[bin] += noise_real; + imag[bin] += noise_imag; + } + } + Complex* padded = &fine(column, 0); + const std::size_t* bins = &crop(first); + if (identity_crop && !analytic) { + for (std::size_t bin = 0; bin < count; ++bin) + padded[first + bin] = + Complex(inverse_n_to * real[bin], inverse_n_to * imag[bin]); + return; + } + for (std::size_t bin = 0; bin < count; ++bin) { + const double analytic_scale = + analytic && first + bin > 0 && 2 * bins[bin] < n_to ? 2.0 : 1.0; + const double scale = analytic_scale * inverse_n_to; + padded[bins[bin]] = Complex(scale * real[bin], scale * imag[bin]); + } + } +#endif +}; + +} // namespace solprop_native::general diff --git a/native/src/general/setup.cpp b/native/src/general/setup.cpp new file mode 100644 index 000000000..f53b7a601 --- /dev/null +++ b/native/src/general/setup.cpp @@ -0,0 +1,1919 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +// The general workspace's construction: its allocations, uploads, FFT plans and +// memory ledger. + +#include "workspace.hpp" + +namespace solprop_native { + +GeneralWorkspace::GeneralWorkspace(Problem problem) : problem_(std::move(problem)) { + require_general_problem(problem_); + require_cpu_execution(problem_.cpu_execution); + identity_crop_ = problem_.grid.crop_index.size() == problem_.dims.n_omega; + for (std::size_t index = 0; identity_crop_ && index < problem_.dims.n_omega; ++index) + identity_crop_ = problem_.grid.crop_index[index] == index; + one_mode_products_ = + one_mode_products_enabled && problem_.dims.family == FieldFamily::field_real && + problem_.dims.n_modes == 1 && problem_.dims.n_pol_state == 1 && + problem_.dims.n_pol_transform == 1 && problem_.dims.n_nodes > 1 && + problem_.dims.node_chunk == problem_.dims.n_nodes; + compact_ = compact_problem(problem_); + if (compact_) + for (auto* pool : {&kernel_pool_, &fft_pool_, &raman_pool_, &raman_tail_pool_}) + pool->compact(); + execution_space_ = kernel_pool_.get(CpuOperation::stage, + problem_.n_columns() * problem_.dims.n_omega); + const auto nodes = problem_.dims.node_chunk; + const auto response_elements = + nodes * problem_.effective_node_pol() * problem_.dims.n_to; + response_execution_space_ = + kernel_pool_.get(CpuOperation::response, response_elements, problem_.dims.n_to); + // On one thread the vendor GEMM's serial kernels are as fast. + one_mode_products_ = + one_mode_products_ && response_execution_space_.concurrency() > 1; + const auto tail = problem_.dims.n_nodes % nodes; + response_tail_execution_space_ = kernel_pool_.get( + CpuOperation::response, tail * problem_.effective_node_pol() * problem_.dims.n_to, + problem_.dims.n_to); + const auto dense_elements = response_elements * problem_.dims.n_modes; + const auto tail_dense_elements = + tail * problem_.effective_node_pol() * problem_.dims.n_to * problem_.dims.n_modes; + transform_execution_space_ = kernel_pool_.get(CpuOperation::dense, dense_elements); + transform_tail_execution_space_ = + kernel_pool_.get(CpuOperation::dense, tail_dense_elements); + temporal_execution_space_ = + kernel_pool_.get(CpuOperation::copy, problem_.n_columns() * problem_.dims.n_to); + // Packing copies only matrix entries, and splitting copies only output samples; + // neither repeats the dense transform's inner product. + const auto matrix_elements = + nodes * problem_.dims.n_pol_transform * problem_.dims.n_modes; + const auto tail_matrix_elements = + tail * problem_.dims.n_pol_transform * problem_.dims.n_modes; + matrix_copy_space_ = kernel_pool_.get(CpuOperation::copy, matrix_elements); + tail_matrix_copy_space_ = kernel_pool_.get(CpuOperation::copy, tail_matrix_elements); + stacked_matrix_copy_space_ = + kernel_pool_.get(CpuOperation::copy, 2 * matrix_elements); + tail_stacked_matrix_copy_space_ = + kernel_pool_.get(CpuOperation::copy, 2 * tail_matrix_elements); + node_spectral_space_ = + kernel_pool_.get(CpuOperation::copy, + nodes * problem_.effective_node_pol() * problem_.dims.n_omega); + flow_execution_space_ = kernel_pool_.get( + CpuOperation::flow, problem_.flow.active * problem_.dims.n_modes * + problem_.dims.n_modes * problem_.dims.n_modes); + if (!problem_.spatial.absorber_matrix.empty()) + absorber_execution_space_ = kernel_pool_.get( + CpuOperation::dense, + problem_.n_columns() * problem_.dims.n_to * problem_.dims.n_modes); + // solprop SEP 46: the workspace's own FFT planning, wisdom isolated from the rest + // of the process in a calibration's modes. + const FftPlanningScope planning(problem_.fft_planning); + allocate(); + upload(); + const auto heap_before = heap_bytes(); + const auto device_before = device_free_bytes(); + const auto started = std::chrono::steady_clock::now(); + plan(); + execution_space_.fence("solprop_native planned workspace accounting"); + const auto heap_after = heap_bytes(); + const auto device_after = device_free_bytes(); + plan_heap_growth_ = heap_after > heap_before ? heap_after - heap_before : 0; + plan_device_growth_ = device_before > device_after ? device_before - device_after : 0; + planning_seconds_ = + std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + execution_space_.fence("solprop_native construction"); +} + +void GeneralWorkspace::allocate() { + if (problem_.stepper.low_storage + ? problem_.stepper.low_storage_flows > 2 || problem_.flow.enabled || + (!problem_.stepper.low_storage_stage_register && + !problem_.free_geometry()) + : problem_.stepper.low_storage_flows != 0 || + !problem_.stepper.low_storage_stage_register) + throw std::invalid_argument( + "the low-storage march holds 0 to 2 flows beside no coupled flow, and elides " + "its stage register on free space alone"); + // The exchange fold of an in-stage absorber re-forms its input from the stage + // register, and a separable fold applies one flow volume, a constant operator's + // table (solprop SEP 24 section 4.3): neither has a meaning to fall back on. + if (problem_.stepper.low_storage && problem_.absorber.in_stage && + !problem_.stepper.low_storage_stage_register) + throw std::invalid_argument("the low-storage march cannot elide its stage register " + "beside an in-stage absorber"); + if (problem_.stepper.low_storage && problem_.separable_geometry() && + problem_.stepper.low_storage_flows > 1) + throw std::invalid_argument( + "the low-storage march holds at most one flow volume on " + "a separable geometry"); + const auto columns = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + const auto chunk_rows = problem_.dims.node_chunk * problem_.effective_node_pol(); + const auto transform_rows = + problem_.separable_geometry() + ? 0 + : problem_.dims.n_nodes * problem_.dims.n_pol_transform; + // A planar chunk's temporal rows are its y rows' columns, not its nodes. + const auto temporal_columns = + problem_.slab_geometry() ? transverse_units_per_chunk() * problem_.spatial.n_p + : (problem_.free_geometry() ? chunk_rows : columns); + separable_aliased_ = problem_.free_geometry() && !problem_.slab_geometry() && + !problem_.dispersive.enabled && temporal_columns == chunk_rows; + const auto statistic_nodes = + problem_.separable_geometry() ? problem_.dims.node_chunk : problem_.dims.n_modes; + // Spatial statistics reconstruct one chunk of physical nodes at a time. + const auto statistics_rows = problem_.slab_geometry() ? chunk_rows : temporal_columns; + allocate_separable(n); + current_ = ComplexView2("solprop_native.state.current", columns, n); + allocate_low_storage(columns, n); + if (problem_.stepper.modified_noise) + noise_ = ComplexView2("solprop_native.noise", columns, n); + if (!problem_.separable_geometry() && problem_.flow.enabled) + stage_phase_ = ComplexView2("solprop_native.step.stage_phase", columns, n); + partials_ = + RealView2("solprop_native.step.error_partials", 3, + checked_product(columns, reduction_slots_, "norm partial slots")); + if (!problem_.transform.projection_coarse_matrix.empty()) + quadrature_partials_ = + RealView2("solprop_native.transform.quadrature_partials", 2, columns); + quadrature_scores_ = RealView("solprop_native.transform.quadrature_scores", 2); + Kokkos::deep_copy(quadrature_scores_, Kokkos::Experimental::quiet_NaN_v); + if (!problem_.stepper.low_storage) + pinned_partials_ = + Kokkos::create_mirror_view(Kokkos::SharedHostPinnedSpace{}, partials_); + if (problem_.stepper.weighted_controller) { + weighted_scale_ = RealView("solprop_native.step.weighted_scale", n); + weighted_partials_ = + RealView("solprop_native.step.weighted_partials", columns * reduction_slots_); + } + allocate_statistics(columns, n, temporal_columns, statistic_nodes, statistics_rows); + crop_index_ = IndexView("solprop_native.transform.crop_index", n); + active_mask_ = ByteView("solprop_native.mask.active", n); + norm_mask_ = ByteView("solprop_native.mask.norm", n); + omega_window_ = RealView("solprop_native.window.omega", n); + omega_ = RealView("solprop_native.statistics.omega", problem_.grid.omega.size()); + time_window_ = RealView("solprop_native.window.time", problem_.dims.n_to); + carrier_ = ComplexView("solprop_native.kerr.carrier", problem_.dims.n_to); + phi_terms_ = ComplexView3( + "solprop_native.linear.phi_terms", + problem_.separable_geometry() ? 0 : problem_.linear.n_phi_terms, columns, n); + prefactor_terms_ = ComplexView3( + "solprop_native.response.prefactor_terms", + problem_.separable_geometry() ? 0 : problem_.linear.n_prefactor_terms, columns, + n); + tableau_ = RealView("solprop_native.step.tableau", + problem_.stepper.n_stages * problem_.stepper.n_stages); + { + // One device buffer for every per-stage scalar view, so that a trial's scalars + // cross in one copy instead of one per view: on a device each small copy from + // pageable memory is a transfer of its own. + const auto stages = problem_.stepper.n_stages; + const auto channels = problem_.plasma.enabled ? problem_.plasma.n_channels : 0; + const auto raman_terms = problem_.raman.enabled ? problem_.raman.n_terms : 0; + const auto raman_components = + problem_.raman.enabled && problem_.raman.n_components > 1 + ? problem_.raman.n_components + : 0; + const std::array sizes{(stages + 1) * problem_.linear.n_phi_terms, + stages * problem_.linear.n_phi_terms, + stages * problem_.linear.n_prefactor_terms, + stages, + stages, + stages, + stages, + stages * channels, + stages * channels, + stages, + stages, + stages * raman_terms, + stages * raman_components, + problem_.linear.n_phi_terms}; + std::size_t total = 0; + for (std::size_t region = 0; region < sizes.size(); ++region) { + stage_offsets_[region] = total; + total += sizes[region]; + } + stage_offsets_[sizes.size()] = total; + stage_slot_size_ = total; + stage_scalars_ = RealView("solprop_native.step.scalars", 2 * total); + bind_stage_scalars(0); + // No trial's tapers yet: every scaling runs until one is bound. + trial_taper_.assign(problem_.stepper.n_stages, + std::numeric_limits::quiet_NaN()); + // A pinned buffer's copy is asynchronous, so its source must not be rewritten + // until the copy has run: `upload_trial` fences before the ring comes round again. + // A host build's mirror is the buffer itself. + stage_staging_.clear(); + for (std::size_t copy = 0; copy < stage_staging_ring; ++copy) + stage_staging_.push_back( + Kokkos::create_mirror_view(Kokkos::SharedHostPinnedSpace{}, stage_scalars_)); + } + allocate_raman(); + allocate_transforms(columns, transform_rows, temporal_columns); + fine_spectrum_ = ComplexView2("solprop_native.fft.fine_spectrum", temporal_columns, + fine_spectrum_size()); + if (!problem_.separable_geometry() && problem_.dims.n_omega != fine_spectrum_size()) + fine_gaps_ = IndexView("solprop_native.fft.fine_gaps", 3 * fine_gap_runs().size()); + if (problem_.separable_geometry() && problem_.dims.n_omega != fine_spectrum_size()) { + std::vector bins; + for (const auto& [bin, length] : fine_gap_runs()) + for (std::size_t offset = 0; offset < length; ++offset) + bins.push_back(bin + offset); + separable_gaps_ = IndexView("solprop_native.free.fine_gaps", bins.size()); + copy_vector(execution_space_, separable_gaps_, bins); + } + allocate_forward_rows(columns, chunk_rows, temporal_columns); + allocate_absorbers(); + allocate_envelope_plasma(chunk_rows, temporal_columns); + allocate_plasma(chunk_rows); + allocate_split(n, temporal_columns); + allocate_parts(); +} + +// A separable geometry's transverse scratch, gaps and statistics geometry. +void GeneralWorkspace::allocate_separable(const std::size_t n) { + if (problem_.separable_geometry()) { + // Keeping transform axes trailing avoids Kokkos-FFT's per-execution + // transpose allocations. The canonical state retains omega-contiguous columns. + transverse_ = problem_.slab_geometry() + ? ComplexView3("solprop_native.slab.transverse", n, + problem_.spatial.n_p, problem_.spatial.n_y) + : ComplexView3("solprop_native.free.transverse", + problem_.dims.n_pol_state * n, + problem_.spatial.n_y, problem_.spatial.n_x); + spatial_.square = ComplexView("solprop_native.free.k_squared", n); + spatial_.square_low = ComplexView("solprop_native.free.k_squared_low", n); + spatial_.numerator = ComplexView("solprop_native.free.numerator", n); + spatial_.clamped = ComplexView("solprop_native.free.clamped", n); + spatial_.transverse = + RealView("solprop_native.free.kperp_sq", problem_.dims.n_modes); + spatial_.transverse_low = + RealView("solprop_native.free.kperp_sq_low", problem_.dims.n_modes); + spatial_.frame = RealView("solprop_native.free.frame", n); + spatial_.spectral_rate = RealView("solprop_native.free.spectral_rate", n); + spatial_.user_rate = RealView("solprop_native.free.user_rate", n); + spatial_.absorber = RealView("solprop_native.free.absorber", problem_.dims.n_modes); + spatial_.transverse_rate = + RealView("solprop_native.free.kperp_rate", problem_.dims.n_modes); + spatial_.evanescent = problem_.spatial.evanescent_rate; + if (problem_.slab_geometry()) { + spatial_.kappa = + ComplexView2("solprop_native.slab.kappa_sq", problem_.spatial.n_p, n); + spatial_.channels = problem_.spatial.n_p; + } + flux_ = spatial_; + if (problem_.flux.closure) + flux_transverse_ = + ComplexView3("solprop_native.flux.closure_transverse", transverse_.extent(0), + transverse_.extent(1), transverse_.extent(2)); + } else if (!problem_.flux.k_squared.empty()) { + flux_.square = ComplexView("solprop_native.flux.k_squared", n); + flux_.square_low = ComplexView("solprop_native.flux.k_squared_low", n); + flux_.transverse = RealView("solprop_native.flux.kperp_sq", problem_.dims.n_modes); + flux_.transverse_low = + RealView("solprop_native.flux.kperp_sq_low", problem_.dims.n_modes); + } +} + +// The low-storage march's registers and flows (solprop SEP 24). +void GeneralWorkspace::allocate_low_storage(const std::size_t columns, + const std::size_t n) { + if (problem_.stepper.low_storage) { + // solprop SEP 24 section 6.7: the march holds its registers beside the state and + // none of the Runge-Kutta views; the stage's input is formed in the route's own + // scratch (`transverse_`, or the fine spectrum of the modal routes). + register_accumulator_ = + ComplexView2("solprop_native.low_storage.accumulator", columns, n); + if (problem_.stepper.low_storage_stage_register) + register_stage_ = ComplexView2("solprop_native.low_storage.stage", columns, n); + low_storage_flows_ = ComplexView3("solprop_native.low_storage.flows", + problem_.stepper.low_storage_flows, columns, n); + } else { + advanced_ = ComplexView2("solprop_native.state.advanced", columns, n); + stages_ = ComplexView3("solprop_native.step.stages", problem_.stepper.n_stages, + columns, n); + solution_ = ComplexView2("solprop_native.step.solution", columns, n); + error_ = ComplexView2("solprop_native.step.error", columns, n); + residual_ = ComplexView2("solprop_native.step.residual", columns, n); + recycled_ = ComplexView2("solprop_native.step.recycled", columns, n); + whole_propagator_ = + ComplexView2("solprop_native.step.whole_propagator", columns, n); + if (!problem_.separable_geometry()) + stage_inverse_ = ComplexView2("solprop_native.step.stage_inverse", columns, n); + } +} + +// The statistics' spectra, time rows, weights and packed outputs. +void GeneralWorkspace::allocate_statistics(const std::size_t columns, + const std::size_t n, + const std::size_t temporal_columns, + const std::size_t statistic_nodes, + const std::size_t statistics_rows) { + if (!problem_.spatial.spectral_weights.empty()) { + spatial_spectral_weights_ = + RealView("solprop_native.spatial.spectral_weights", problem_.dims.n_modes); + spatial_on_axis_ = + ComplexView("solprop_native.spatial.on_axis", problem_.dims.n_modes); + // One row per column for the energy and per node for the peaks and fluence, + // which differ on the hybrid planar kind alone. + spatial_statistics_partials_ = + RealView2("solprop_native.spatial.statistics_partials", + std::max(problem_.dims.n_modes, problem_.dims.n_nodes), 3); + } + statistics_spectrum_ = ComplexView2("solprop_native.statistics.spectrum", + statistics_rows, problem_.dims.n_t); + statistics_time_ = ComplexView2("solprop_native.statistics.time", statistics_rows, + problem_.dims.n_t); + statistics_power_weight_ = + RealView2("solprop_native.statistics.power_weight", + problem_.spatial.spectral_weights.empty() ? columns : 0, n); + graph::deep_copy(execution_space_, statistics_power_weight_, 1.0); + // A spatial run's rows hold the magnetic closure in time (solprop SEP 34) when a + // local flux statistic is asked for. + statistics_weighted_time_ = + ComplexView2("solprop_native.statistics.weighted_time", + problem_.spatial.spectral_weights.empty() ? temporal_columns + : problem_.flux.closure ? statistics_rows + : 0, + problem_.dims.n_t); + statistics_power_ = + RealView2("solprop_native.statistics.power", statistic_nodes, problem_.dims.n_t); + if (problem_.flux.closure && !problem_.separable_geometry()) + statistics_magnitude_ = RealView2("solprop_native.statistics.flux_magnitude", + statistic_nodes, problem_.dims.n_t); + statistics_total_power_ = + RealView("solprop_native.statistics.total_power", problem_.dims.n_t); + statistics_on_axis_ = + RealView("solprop_native.statistics.on_axis", 2 * problem_.dims.n_modes); + const auto base_statistics = checked_sum( + checked_sum(problem_.dims.n_t, + checked_product(2, problem_.n_columns(), "statistics columns"), + "statistics packed output"), + checked_product(3, problem_.dims.n_modes, "statistics modes"), + "statistics packed output"); + statistics_packed_ = + RealView("solprop_native.statistics.packed", + checked_sum(base_statistics, 5, "statistics packed output")); +} + +// The Raman convolution's drives, kernel and stage rates. +void GeneralWorkspace::allocate_raman() { + if (problem_.raman.enabled) { + const auto doubled = 2 * problem_.dims.n_to; + node_raman_drive_ = + RealView2("solprop_native.raman.node_drive", + problem_.dims.node_chunk * problem_.raman_drives(), doubled); + node_raman_spectrum_ = ComplexView2( + "solprop_native.raman.node_spectrum", + problem_.dims.node_chunk * problem_.raman_drives(), problem_.dims.n_to + 1); + node_raman_convolved_ = + RealView2("solprop_native.raman.node_convolved", + problem_.dims.node_chunk * problem_.raman_drives(), doubled); + raman_undamped_ = + RealView2("solprop_native.raman.undamped", problem_.raman.n_terms, doubled); + raman_kernel_time_ = RealView2("solprop_native.raman.kernel_time", + problem_.raman_kernels(), doubled); + raman_kernel_spectrum_ = + ComplexView2("solprop_native.raman.kernel_spectrum", problem_.raman_kernels(), + problem_.dims.n_to + 1); + raman_tensor_weights_ = + RealView2("solprop_native.raman.tensor_weights", problem_.raman.n_terms, 2); + cached_raman_rates_.resize(problem_.raman.n_terms); + if (problem_.raman.n_components > 1) { + raman_row_component_ = + RealView("solprop_native.raman.row_component", problem_.raman.n_terms); + raman_component_fraction_ = RealView("solprop_native.raman.component_fraction", + problem_.raman.n_components); + cached_raman_densities_.resize(problem_.raman.n_components); + } + } +} + +// The synthesis and projection matrices, scaled and chunked, and their time rows. +void GeneralWorkspace::allocate_transforms(const std::size_t columns, + const std::size_t transform_rows, + const std::size_t temporal_columns) { + if (use_packed_radial_blas()) { + radial_blas_input_ = RealView2("solprop_native.radial.blas_input", + problem_.dims.n_nodes, 2 * problem_.dims.n_to); + radial_blas_output_ = RealView2("solprop_native.radial.blas_output", + problem_.dims.n_nodes, 2 * problem_.dims.n_to); + } + // The hybrid planar kind's slab rule in x, (n_x, n_p) and (n_p, n_x). + const auto synthesis_rows = + problem_.slab_geometry() ? problem_.spatial.n_x : transform_rows; + const auto synthesis_columns = + problem_.slab_geometry() ? problem_.spatial.n_p : problem_.dims.n_modes; + synthesis_ = RealView2("solprop_native.transform.synthesis", synthesis_rows, + synthesis_columns); + projection_ = RealView2("solprop_native.transform.projection", synthesis_columns, + synthesis_rows); + scaled_synthesis_ = RealView2("solprop_native.transform.scaled_synthesis", + transform_rows, problem_.dims.n_modes); + scaled_projection_ = RealView2("solprop_native.transform.scaled_projection", + problem_.dims.n_modes, transform_rows); + if (problem_.dims.family == FieldFamily::envelope && use_blas_for_synthesis()) { + scaled_synthesis_complex_ = + ComplexView2("solprop_native.transform.scaled_synthesis_complex", + transform_rows, problem_.dims.n_modes); + scaled_projection_complex_ = + ComplexView2("solprop_native.transform.scaled_projection_complex", + problem_.dims.n_modes, transform_rows); + } + if (!problem_.transform.projection_coarse_matrix.empty()) { + projection_coarse_ = RealView2("solprop_native.transform.projection_coarse", + problem_.dims.n_modes, transform_rows); + scaled_projection_coarse_ = + RealView2("solprop_native.transform.scaled_projection_coarse", + problem_.dims.n_modes, transform_rows); + if (problem_.dims.family == FieldFamily::envelope && use_blas_for_projection()) { + scaled_projection_coarse_complex_ = + ComplexView2("solprop_native.transform.scaled_projection_coarse_complex", + problem_.dims.n_modes, transform_rows); + scaled_projection_stacked_complex_ = + ComplexView2("solprop_native.transform.scaled_projection_stacked_complex", + 2 * problem_.dims.n_modes, transform_rows); + projected_stacked_complex_ = + ComplexView2("solprop_native.response.projected_stacked_complex", 2 * columns, + problem_.dims.n_to); + } else if (problem_.dims.family != FieldFamily::envelope) { + scaled_projection_stacked_ = + RealView2("solprop_native.transform.scaled_projection_stacked", + 2 * problem_.dims.n_modes, transform_rows); + projected_stacked_real_ = + RealView2("solprop_native.response.projected_stacked_real", + 2 * temporal_columns, problem_.dims.n_to); + } + } + const auto chunk_transform_rows = + problem_.dims.node_chunk * problem_.dims.n_pol_transform; + const auto tail_nodes = problem_.dims.n_nodes % problem_.dims.node_chunk; + const auto tail_transform_rows = tail_nodes * problem_.dims.n_pol_transform; + if (use_chunked_blas_for_synthesis()) { + if (problem_.dims.family == FieldFamily::envelope) { + chunk_synthesis_complex_ = + ComplexView2("solprop_native.transform.chunk_synthesis_complex", + chunk_transform_rows, problem_.dims.n_modes); + if (tail_transform_rows != 0) + tail_synthesis_complex_ = + ComplexView2("solprop_native.transform.tail_synthesis_complex", + tail_transform_rows, problem_.dims.n_modes); + } else { + chunk_synthesis_ = RealView2("solprop_native.transform.chunk_synthesis", + chunk_transform_rows, problem_.dims.n_modes); + if (tail_transform_rows != 0) + tail_synthesis_ = RealView2("solprop_native.transform.tail_synthesis", + tail_transform_rows, problem_.dims.n_modes); + } + } + if (use_chunked_blas_for_projection()) { + const bool stacked = !problem_.transform.projection_coarse_matrix.empty(); + const auto projection_rows = + stacked ? 2 * problem_.dims.n_modes : problem_.dims.n_modes; + if (problem_.dims.family == FieldFamily::envelope) { + if (stacked) { + chunk_projection_stacked_complex_ = + ComplexView2("solprop_native.transform.chunk_projection_stacked_complex", + projection_rows, chunk_transform_rows); + if (tail_transform_rows != 0) + tail_projection_stacked_complex_ = + ComplexView2("solprop_native.transform.tail_projection_stacked_complex", + projection_rows, tail_transform_rows); + } else { + chunk_projection_complex_ = + ComplexView2("solprop_native.transform.chunk_projection_complex", + projection_rows, chunk_transform_rows); + if (tail_transform_rows != 0) + tail_projection_complex_ = + ComplexView2("solprop_native.transform.tail_projection_complex", + projection_rows, tail_transform_rows); + } + } else if (stacked) { + chunk_projection_stacked_ = + RealView2("solprop_native.transform.chunk_projection_stacked", + projection_rows, chunk_transform_rows); + if (tail_transform_rows != 0) + tail_projection_stacked_ = + RealView2("solprop_native.transform.tail_projection_stacked", + projection_rows, tail_transform_rows); + } else { + chunk_projection_ = RealView2("solprop_native.transform.chunk_projection", + projection_rows, chunk_transform_rows); + if (tail_transform_rows != 0) + tail_projection_ = RealView2("solprop_native.transform.tail_projection", + projection_rows, tail_transform_rows); + } + } +} + +// The forward rows and spectra: the projection's, the absorber's, a device's pack and +// the held and gated plasma currents'. +void GeneralWorkspace::allocate_forward_rows(const std::size_t columns, + const std::size_t chunk_rows, + const std::size_t temporal_columns) { + // A device's packed forward rows (`packs_forward`): the absorber's, the projection's, + // the coarse projection's and the held pack's, in that order, carved from one buffer + // and one spectrum. + const bool pack = packs_forward(); + const std::size_t absorber_rows = + pack && problem_.absorber.in_stage ? temporal_columns : 0; + const std::size_t coarse_rows = + pack && !problem_.transform.projection_coarse_matrix.empty() ? temporal_columns + : 0; + const std::size_t pack_rows = + absorber_rows + temporal_columns + coarse_rows + + (held_plasma() ? 2 * (temporal_columns + coarse_rows) : 0); + const auto pack_slice = [&](const std::size_t first, const std::size_t count) { + return std::pair(first, first + count); + }; + constexpr bool host_pack_layout = build::host; + // A host's pack: [absorber | fine | held fine x2 | coarse | held coarse x2]; a + // device's: [absorber | fine | coarse | (held fine, held coarse) x2]. + const std::size_t held_fine_rows = held_plasma() ? 2 * temporal_columns : 0; + const std::size_t fine_end = absorber_rows + temporal_columns + held_fine_rows; + const std::size_t coarse_first = + host_pack_layout ? fine_end : absorber_rows + temporal_columns; + forward_pack_absorber_rows_ = absorber_rows; + if (pack) { + forward_pack_ = + RealView2("solprop_native.fft.forward_pack", pack_rows, problem_.dims.n_to); + forward_pack_spectrum_ = ComplexView2("solprop_native.fft.forward_pack_spectrum", + pack_rows, projected_spectrum_size()); + if (absorber_rows != 0 && pack_rows - absorber_rows > temporal_columns) { + forward_pack_tail_ = Kokkos::subview( + forward_pack_, pack_slice(absorber_rows, pack_rows - absorber_rows), + Kokkos::ALL); + forward_pack_tail_spectrum_ = Kokkos::subview( + forward_pack_spectrum_, pack_slice(absorber_rows, pack_rows - absorber_rows), + Kokkos::ALL); + } + if (host_pack_layout && coarse_rows != 0 && deferred_quadrature_eligible()) { + forward_pack_fine_ = + Kokkos::subview(forward_pack_, pack_slice(0, fine_end), Kokkos::ALL); + forward_pack_fine_spectrum_ = + Kokkos::subview(forward_pack_spectrum_, pack_slice(0, fine_end), Kokkos::ALL); + if (absorber_rows != 0 && fine_end - absorber_rows > temporal_columns) { + forward_pack_fine_tail_ = Kokkos::subview( + forward_pack_, pack_slice(absorber_rows, fine_end - absorber_rows), + Kokkos::ALL); + forward_pack_fine_tail_spectrum_ = Kokkos::subview( + forward_pack_spectrum_, pack_slice(absorber_rows, fine_end - absorber_rows), + Kokkos::ALL); + } + if (held_plasma()) { + forward_pack_coarse_ = Kokkos::subview( + forward_pack_, pack_slice(fine_end, pack_rows - fine_end), Kokkos::ALL); + forward_pack_coarse_spectrum_ = + Kokkos::subview(forward_pack_spectrum_, + pack_slice(fine_end, pack_rows - fine_end), Kokkos::ALL); + } + } + } + projected_spectrum_ = + pack ? ComplexView2(Kokkos::subview(forward_pack_spectrum_, + pack_slice(absorber_rows, temporal_columns), + Kokkos::ALL)) + : ComplexView2("solprop_native.fft.projected_spectrum", temporal_columns, + projected_spectrum_size()); + if (!problem_.transform.projection_coarse_matrix.empty()) + projected_coarse_spectrum_ = + coarse_rows != 0 + ? ComplexView2(Kokkos::subview(forward_pack_spectrum_, + pack_slice(coarse_first, coarse_rows), + Kokkos::ALL)) + : ComplexView2("solprop_native.fft.projected_coarse_spectrum", columns, + projected_spectrum_size()); + absorber_spectrum_ = + absorber_rows != 0 + ? ComplexView2(Kokkos::subview(forward_pack_spectrum_, + pack_slice(0, absorber_rows), Kokkos::ALL)) + : ComplexView2("solprop_native.fft.absorber_spectrum", temporal_columns, + projected_spectrum_size()); + if (problem_.dims.family == FieldFamily::field_real) { + modal_real_ = RealView2("solprop_native.fft.modal_time_real", temporal_columns, + problem_.dims.n_to); + node_real_ = separable_aliased_ ? modal_real_ + : RealView2("solprop_native.response.node_real", + chunk_rows, problem_.dims.n_to); + } else { + modal_complex_ = ComplexView2("solprop_native.fft.modal_time_complex", + temporal_columns, problem_.dims.n_to); + node_complex_ = separable_aliased_ + ? modal_complex_ + : ComplexView2("solprop_native.response.node_complex", + chunk_rows, problem_.dims.n_to); + retained_synthesis_ = retained_synthesis(); + if (retained_synthesis_) { + retained_modal_ = ComplexView2("solprop_native.transform.retained_modal", + temporal_columns, problem_.dims.n_omega); + retained_node_ = ComplexView2("solprop_native.transform.retained_node", + chunk_rows, problem_.dims.n_omega); + } + } + if (problem_.dims.family == FieldFamily::envelope) { + projected_complex_ = ComplexView2("solprop_native.response.projected_complex", + temporal_columns, problem_.dims.n_to); + polarisation_complex_ = + separable_aliased_ + ? projected_complex_ + : ComplexView2("solprop_native.response.polarisation_complex", chunk_rows, + problem_.dims.n_to); + if (!problem_.transform.projection_coarse_matrix.empty()) + projected_coarse_complex_ = + ComplexView2("solprop_native.response.projected_coarse_complex", + temporal_columns, problem_.dims.n_to); + absorber_complex_ = ComplexView2("solprop_native.absorber.fine_complex", + temporal_columns, problem_.dims.n_to); + } else { + projected_real_ = + pack ? RealView2(Kokkos::subview(forward_pack_, + pack_slice(absorber_rows, temporal_columns), + Kokkos::ALL)) + : RealView2("solprop_native.response.projected_real", temporal_columns, + problem_.dims.n_to); + polarisation_real_ = separable_aliased_ + ? projected_real_ + : RealView2("solprop_native.response.polarisation_real", + chunk_rows, problem_.dims.n_to); + if (!problem_.transform.projection_coarse_matrix.empty()) + projected_coarse_real_ = + coarse_rows != 0 + ? RealView2(Kokkos::subview( + forward_pack_, pack_slice(coarse_first, coarse_rows), Kokkos::ALL)) + : RealView2("solprop_native.response.projected_coarse_real", + temporal_columns, problem_.dims.n_to); + absorber_real_ = + absorber_rows != 0 + ? RealView2(Kokkos::subview(forward_pack_, pack_slice(0, absorber_rows), + Kokkos::ALL)) + : RealView2("solprop_native.absorber.fine_real", temporal_columns, + problem_.dims.n_to); + } + if (field_plasma()) + held_current_ = + RealView2("solprop_native.held.current", chunk_rows, problem_.dims.n_to); + if (gated_plasma()) { + gated_prefix_ = + RealView2("solprop_native.gated.prefix", chunk_rows, problem_.dims.n_to); + if (problem_.plasma.band) + gated_filtered_ = + RealView2("solprop_native.gated.filtered", chunk_rows, problem_.dims.n_to); + } + if (held_plasma()) { + held_derivative_ = RealView("solprop_native.held.derivative", problem_.dims.n_to); + held_inverse_ = RealView("solprop_native.held.inverse", problem_.dims.n_omega); + const bool radial = held_radial(); + const auto rows = radial ? chunk_rows : temporal_columns; + const bool coarse = !problem_.transform.projection_coarse_matrix.empty(); + // On a device the two currents' fine and coarse rows are one buffer, a stream's + // fine rows then its coarse ones, so that one transform takes all four: on an AMD + // Instinct MI325X each transform of a single-mode run was four library kernels of a + // few microseconds, launched one by one. On the radial route the three streams' + // fine rows are one buffer, which a node chunk's one transform takes. +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const bool packed = !radial; + const bool radial_packed = radial; +#else + const bool packed = false; + const bool radial_packed = false; +#endif + // A host's pack holds the streams' rows itself, each fine block then each coarse. + const bool host_held = host_pack_layout && pack && !radial; + if (packed && pack) { + // The stage's forward pack ends with the held pack's rows. + const auto held_rows = + pack_slice(absorber_rows + rows + coarse_rows, 2 * (rows + coarse_rows)); + held_pack_ = Kokkos::subview(forward_pack_, held_rows, Kokkos::ALL); + held_pack_spectrum_ = + Kokkos::subview(forward_pack_spectrum_, held_rows, Kokkos::ALL); + } else if (packed) { + const std::size_t held_rows = 2 * (coarse ? 2 * rows : rows); + held_pack_ = RealView2("solprop_native.held.pack", held_rows, problem_.dims.n_to); + held_pack_spectrum_ = ComplexView2("solprop_native.held.pack_spectrum", held_rows, + problem_.dims.n_to / 2 + 1); + } else if (radial_packed) { + held_pack_ = RealView2("solprop_native.held.pack", 3 * rows, problem_.dims.n_to); + held_pack_spectrum_ = ComplexView2("solprop_native.held.pack_spectrum", 3 * rows, + problem_.dims.n_to / 2 + 1); + } + for (std::size_t index = 0; index < (radial ? 3U : 2U); ++index) { + auto& stream = held_streams_[index]; + const auto label = + std::string("solprop_native.held.stream") + std::to_string(index) + "."; + if (index < 2) + stream.node = RealView2(label + "node", chunk_rows, problem_.dims.n_to); + if (host_held && index < 2) { + const auto fine = pack_slice(absorber_rows + rows + index * rows, rows); + stream.projected = Kokkos::subview(forward_pack_, fine, Kokkos::ALL); + stream.spectrum = Kokkos::subview(forward_pack_spectrum_, fine, Kokkos::ALL); + if (coarse) { + const auto held_coarse = pack_slice(fine_end + rows + index * rows, rows); + stream.coarse = Kokkos::subview(forward_pack_, held_coarse, Kokkos::ALL); + stream.coarse_spectrum = + Kokkos::subview(forward_pack_spectrum_, held_coarse, Kokkos::ALL); + stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.dims.n_to); + } + continue; + } + if (packed) { + // Each of the two streams holds half of the pack. + const std::size_t first = index * (held_pack_.extent(0) / 2); + const auto rows_from = [&](const std::size_t start, const std::size_t count) { + return std::pair(start, start + count); + }; + stream.projected = + Kokkos::subview(held_pack_, rows_from(first, rows), Kokkos::ALL); + stream.spectrum = + Kokkos::subview(held_pack_spectrum_, rows_from(first, rows), Kokkos::ALL); + if (coarse) { + stream.coarse = + Kokkos::subview(held_pack_, rows_from(first + rows, rows), Kokkos::ALL); + stream.coarse_spectrum = Kokkos::subview( + held_pack_spectrum_, rows_from(first + rows, rows), Kokkos::ALL); + stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.dims.n_to); + } + continue; + } + if (radial_packed) { + // Each of the three streams holds a third of the pack. + const auto third = pack_slice(index * rows, rows); + stream.projected = Kokkos::subview(held_pack_, third, Kokkos::ALL); + stream.spectrum = Kokkos::subview(held_pack_spectrum_, third, Kokkos::ALL); + } else { + stream.projected = RealView2(label + "projected", rows, problem_.dims.n_to); + stream.spectrum = + ComplexView2(label + "spectrum", rows, problem_.dims.n_to / 2 + 1); + } + if (coarse) { + stream.coarse = RealView2(label + "coarse", rows, problem_.dims.n_to); + stream.coarse_spectrum = + ComplexView2(label + "coarse_spectrum", rows, problem_.dims.n_to / 2 + 1); + stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.dims.n_to); + } + } + if (radial) { + held_nodal_spectrum_ = ComplexView2("solprop_native.held.nodal_spectrum", + chunk_rows, problem_.dims.n_omega); + held_projected_spectrum_ = + ComplexView2("solprop_native.held.projected_spectrum", problem_.n_columns(), + problem_.dims.n_omega); + held_projection_ = + RealView2("solprop_native.held.projection", problem_.dims.n_modes, + problem_.dims.node_chunk * problem_.dims.n_pol_transform); + } + } +} + +// The temporal absorbers' rates and a radial run's spatial absorber. +void GeneralWorkspace::allocate_absorbers() { + absorber_fine_ = + RealView("solprop_native.absorber.rate_fine", problem_.absorber.fine.size()); + absorber_coarse_ = + RealView("solprop_native.absorber.rate_coarse", problem_.absorber.coarse.size()); + if (!problem_.spatial.absorber_matrix.empty()) { + if (problem_.dims.family == FieldFamily::envelope) { + spatial_absorber_complex_ = + ComplexView2("solprop_native.absorber.spatial_complex", problem_.dims.n_modes, + problem_.dims.n_modes); + std::vector matrix; + matrix.reserve(problem_.spatial.absorber_matrix.size()); + for (const auto value : problem_.spatial.absorber_matrix) + matrix.emplace_back(value, 0.0); + copy_vector(execution_space_, spatial_absorber_complex_, matrix); + } else { + spatial_absorber_real_ = RealView2("solprop_native.absorber.spatial_real", + problem_.dims.n_modes, problem_.dims.n_modes); + copy_vector(execution_space_, spatial_absorber_real_, + problem_.spatial.absorber_matrix); + } + } +} + +// The envelope plasma's quadrature rules, cycles and current streams. +void GeneralWorkspace::allocate_envelope_plasma(const std::size_t chunk_rows, + const std::size_t temporal_columns) { + if (problem_.plasma.envelope) { + envelope_rule_nodes_ = RealView3("solprop_native.envelope.rule_nodes", + problem_.plasma.n_channels, 4, 512); + envelope_rule_weights_ = RealView3("solprop_native.envelope.rule_weights", + problem_.plasma.n_channels, 4, 512); + envelope_moments_ = RealView3("solprop_native.envelope.moments", 2, + problem_.dims.node_chunk, problem_.dims.n_to); + envelope_rates_ = RealView2("solprop_native.envelope.rates", + problem_.dims.node_chunk, problem_.dims.n_to); + envelope_exposure_ = RealView2("solprop_native.envelope.exposure", + problem_.dims.node_chunk, problem_.dims.n_to); + envelope_failures_ = IndexView("solprop_native.envelope.failures", + problem_.dims.node_chunk * problem_.dims.n_to); + envelope_status_ = IndexView("solprop_native.envelope.status", 1); + envelope_host_status_ = Kokkos::View( + "solprop_native.envelope.host_status", 1); + envelope_drude_factor_ = + RealView("solprop_native.envelope.drude_factor", problem_.dims.n_omega); + envelope_loss_factor_ = + ComplexView("solprop_native.envelope.loss_factor", problem_.dims.n_omega); + for (std::size_t index = 0; index < envelope_streams_.size(); ++index) { + auto& stream = envelope_streams_[index]; + const auto label = + std::string("solprop_native.envelope.") + (index == 0 ? "drude." : "loss."); + stream.node = ComplexView2(label + "node", chunk_rows, problem_.dims.n_to); + stream.projected = + ComplexView2(label + "projected", temporal_columns, problem_.dims.n_to); + stream.spectrum = + ComplexView2(label + "spectrum", temporal_columns, problem_.dims.n_to); + if (!problem_.transform.projection_coarse_matrix.empty()) { + stream.coarse = + ComplexView2(label + "coarse", temporal_columns, problem_.dims.n_to); + stream.coarse_spectrum = ComplexView2(label + "coarse_spectrum", + temporal_columns, problem_.dims.n_to); + } + if (!projected_stacked_complex_.empty()) + stream.stacked = + ComplexView2(label + "stacked", 2 * temporal_columns, problem_.dims.n_to); + } + } +} + +// The plasma column's tables, channels, scans and diagnostics. +void GeneralWorkspace::allocate_plasma(const std::size_t chunk_rows) { + if (problem_.plasma.enabled) { + if (problem_.plasma.n_channels > 1 || problem_.plasma.envelope) { + channel_parameters_ = RealView2("solprop_native.plasma.channel_parameters", + problem_.plasma.n_channels, 9); + channel_coefficients_ = RealView("solprop_native.plasma.channel_coefficients", + problem_.plasma.channel_coefficients.size()); + if (!problem_.plasma.envelope) + channel_scratch_ = decltype(channel_scratch_)( + "solprop_native.plasma.channel_scratch", problem_.plasma.n_channels, + problem_.dims.node_chunk, 4); + } + rate_coefficients_ = RealView("solprop_native.plasma.rate_coefficients", + problem_.plasma.rate_coefficients.size()); + plasma_statistics_scratch_ = + RealView2("solprop_native.plasma.statistics_scratch", problem_.dims.n_nodes, + 1 + 2 * problem_.plasma.n_channels); + if (!problem_.plasma.envelope) { + // solprop SEP 18: the rule's taps and the run diagnostics' six parts per node. + plasma_taps_ = + RealView("solprop_native.plasma.taps", problem_.plasma.taps.size()); + plasma_diagnostics_ = RealView2("solprop_native.plasma.diagnostics", + problem_.dims.n_nodes, plasma_diagnostic_parts); + } + if (!problem_.plasma.envelope) { + // The band rule's team column and its statistics (`team_band_plasma_column`). + // A device keeps the trapezoid device column's rows, one per node of a chunk, + // and the band column reuses them. A host team owns one row of each per slot, + // and at most one team per thread runs at once (solprop SEP 25 section + // 6.7.5): the rate, exposure and filtered rows for every field plasma (the + // trapezoid's run diagnostics come from the band statistics), and under the + // band rule the guard, a mixture's weight and a row per component. The + // tiled column (solprop SEP 25 section 6.7.6) keeps the device's rows on every + // build, the weight rows holding the free-current local prefixes, and the tiles' + // totals by row. + const bool tiled = band_tiled_ && problem_.plasma.band; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + band_slots_ = problem_.dims.node_chunk; + const auto guard_rows = chunk_rows; + const auto weight_rows = chunk_rows; + const auto component_rows = chunk_rows; +#else + if (tiled) { + band_slots_ = problem_.dims.node_chunk; + } else { + band_slots_ = std::min(problem_.dims.node_chunk, + static_cast(std::max( + 1, response_execution_space_.concurrency()))); + } + const bool band = problem_.plasma.band; + const auto guard_rows = tiled ? chunk_rows : band ? band_slots_ : 0; + const auto weight_rows = tiled ? chunk_rows + : band && problem_.plasma.n_channels > 1 ? band_slots_ + : 0; + const auto component_rows = + band ? checked_product(band_slots_, problem_.effective_node_pol(), + "band component rows") + : 0; +#endif + if (tiled) { + // The widest tile whose two scratch rows the device's team scratch holds. + using Policy = Kokkos::TeamPolicy; + const auto scratch_limit = + static_cast(Policy::scratch_size_max(0)); + std::size_t largest = band_tile_max; + while (largest > band_tile_min && + band_tile_bytes(largest) > scratch_limit) + largest /= 2; + if (band_tile_bytes(largest) > scratch_limit) + throw std::length_error("the band tile exceeds the team scratch"); + band_tile_ = band_tile_samples(problem_.dims.n_to, largest); + band_tile_count_ = band_tile_count(problem_.dims.n_to, band_tile_); + require_allocation(checked_product(chunk_rows, band_tile_count_, "band tiles"), + sizeof(double), "band-tile bytes"); + band_tile_totals_ = RealView2("solprop_native.plasma.band_tile_totals", + chunk_rows, band_tile_count_); + } + const auto band_rows = checked_sum( + checked_sum(checked_product(3, band_slots_, "band rows"), guard_rows, + "band rows"), + checked_sum(weight_rows, component_rows, "band rows"), "band rows"); + require_allocation(checked_product(band_rows, problem_.dims.n_to, "band rows"), + sizeof(double), "band-row bytes"); + plasma_rate_ = + RealView2("solprop_native.plasma.rate", band_slots_, problem_.dims.n_to); + plasma_exposure_ = + RealView2("solprop_native.plasma.exposure", band_slots_, problem_.dims.n_to); + plasma_fraction_ = + RealView2("solprop_native.plasma.fraction", band_slots_, problem_.dims.n_to); + if (component_rows != 0) { + plasma_current_integrand_ = RealView2("solprop_native.plasma.current_integrand", + component_rows, problem_.dims.n_to); + } + if (guard_rows != 0) + plasma_free_current_ = RealView2("solprop_native.plasma.free_current", + guard_rows, problem_.dims.n_to); + if (weight_rows != 0) + plasma_outer_current_ = RealView2("solprop_native.plasma.outer_current", + weight_rows, problem_.dims.n_to); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + if (!problem_.plasma.band) { + plasma_tiles_ = + (problem_.dims.n_to + plasma_tile_samples - 1) / plasma_tile_samples; + plasma_tile_totals_ = + RealView2("solprop_native.plasma.tile_totals", chunk_rows, plasma_tiles_); + } +#endif + } + if (problem_.plasma.stash) { + stash_candidate_ = + RealView2("solprop_native.plasma.stash_candidate", problem_.dims.n_nodes, + 1 + 2 * problem_.plasma.n_channels); + stash_current_ = + RealView2("solprop_native.plasma.stash_current", problem_.dims.n_nodes, + 1 + 2 * problem_.plasma.n_channels); + } + } +} + +// The split-step absorber's factor and coarse rows. +void GeneralWorkspace::allocate_split(const std::size_t n, + const std::size_t temporal_columns) { + if (problem_.absorber.split_step) { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + split_factor_ = RealView("solprop_native.absorber.split_factor", problem_.dims.n_t); +#endif + if (problem_.free_geometry() && problem_.dims.family == FieldFamily::envelope) { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // In place on the state's rows, a block holds no buffer of its own, only its + // plans' work area (9 MiB on cuFFT for every row at 768 x 768 x 600), so a device + // takes every row at once whatever the node budget: 14 kernels and 10 transforms + // a step there, against 119 and 220 at the node chunk (solprop #361). A CPU keeps + // the chunk, where FFTW's threaded plan of a whole state was much the slower. + split_rows_ = problem_.n_columns(); +#else + split_rows_ = temporal_columns; +#endif + } else { + coarse_spectrum_ = + ComplexView2("solprop_native.absorber.coarse_spectrum", temporal_columns, n); + if (problem_.dims.family == FieldFamily::envelope) + coarse_time_complex_ = + ComplexView2("solprop_native.absorber.coarse_time_complex", + temporal_columns, problem_.dims.n_t); + else + coarse_time_real_ = RealView2("solprop_native.absorber.coarse_time_real", + temporal_columns, problem_.dims.n_t); + } + } +} + +// The route's parts: the general modal source, the dispersive responses, the far-field +// pass and the coupled flow. +void GeneralWorkspace::allocate_parts() { + if (problem_.general.enabled) { + // solprop SEP 47: a general modal basis's statistics whiten the columns by the + // metric's root (`whitened_statistics`), into scratch held from the start. + statistics_whitened_ = ComplexView2("solprop_native.statistics.whitened", + problem_.n_columns(), problem_.dims.n_omega); + statistics_root_ = ComplexView( + "solprop_native.statistics.metric_root", + checked_product(problem_.dims.n_omega, + checked_product(problem_.dims.n_modes, problem_.dims.n_modes, + "metric root"), + "metric root")); + statistics_whitened_sums_ = RealView("solprop_native.statistics.whitened_sums", + problem_.n_columns() + problem_.dims.n_omega); + general_ = std::make_unique(problem_); + // The source owns execution-space copies; the host maps are the largest arrays a + // general guide carries and nothing reads them again. + std::vector().swap(problem_.general.synthesis); + std::vector().swap(problem_.general.projection); + std::vector().swap(problem_.general.tensor); + std::vector().swap(problem_.general.frequency_factor); + std::vector().swap(problem_.general.delayed_tensor); + std::vector().swap(problem_.general.delayed_spectrum); + std::vector().swap(problem_.general.node_response); + } + if (problem_.free_geometry() && !problem_.spatial.spectral_weights.empty()) + far_field_ = std::make_unique(problem_, execution_space_); + if (problem_.dispersive.enabled && problem_.separable_geometry()) { + // The planar kind's slab rule in x; free space has none. + const bool slab = problem_.slab_geometry(); + separable_dispersive_ = std::make_unique( + problem_, kernel_pool_, + SeparableDispersive::StageViews{.fine_spectrum = fine_spectrum_, + .projected_spectrum = projected_spectrum_, + .node_real = node_real_, + .polarisation_real = polarisation_real_, + .node_complex = node_complex_, + .polarisation_complex = polarisation_complex_, + .synthesis = slab ? synthesis_ : RealView2{}, + .projection = slab ? projection_ : RealView2{}, + .time_window = time_window_, + .carrier = carrier_}); + } else if (problem_.dispersive.enabled && !problem_.general.enabled) { + dispersive_ = std::make_unique( + problem_, kernel_pool_, + DispersiveResponse::StageViews{ + .fine_spectrum = fine_spectrum_, + .projected_spectrum = projected_spectrum_, + .projected_coarse_spectrum = projected_coarse_spectrum_, + .node_real = node_real_, + .polarisation_real = polarisation_real_, + .node_complex = node_complex_, + .polarisation_complex = polarisation_complex_, + .scaled_synthesis = scaled_synthesis_, + .scaled_projection = scaled_projection_, + .scaled_projection_coarse = scaled_projection_coarse_, + .time_window = time_window_, + .carrier = carrier_}); + } + if (problem_.dispersive.enabled) { + // The response holds execution-space copies of its layout (the general route's + // source built its own above). + std::vector().swap(problem_.dispersive.inputs); + std::vector().swap(problem_.dispersive.term_multipliers); + std::vector().swap(problem_.dispersive.quartics); + } + if (problem_.flow.enabled) { + flow_ = std::make_unique(problem_, flow_execution_space_); + std::vector().swap(problem_.flow.frozen); + std::vector().swap(problem_.flow.residual_terms); + std::vector().swap(problem_.flow.left); + std::vector().swap(problem_.flow.right); + std::vector().swap(problem_.flow.eigenvalues); + std::vector().swap(problem_.flow.route); + std::vector().swap(problem_.flow.fallback_vectors); + std::vector().swap(problem_.flow.fallback_triangular); + } +} + +void GeneralWorkspace::upload() { + if (problem_.raman.n_components > 1) { + copy_vector(execution_space_, raman_row_component_, problem_.raman.row_component); + copy_vector(execution_space_, raman_component_fraction_, + problem_.raman.component_fraction); + } + if (problem_.separable_geometry()) { + copy_vector(execution_space_, spatial_.square, problem_.spatial.k_squared); + copy_vector(execution_space_, spatial_.square_low, problem_.spatial.k_squared_low); + copy_vector(execution_space_, spatial_.numerator, problem_.spatial.numerator); + copy_vector(execution_space_, spatial_.clamped, problem_.spatial.clamped); + copy_vector(execution_space_, spatial_.transverse, problem_.spatial.kperp_sq); + copy_vector(execution_space_, spatial_.transverse_low, + problem_.spatial.kperp_sq_low); + copy_vector(execution_space_, spatial_.frame, problem_.spatial.frame); + copy_vector(execution_space_, spatial_.spectral_rate, + problem_.spatial.spectral_rate); + copy_vector(execution_space_, spatial_.user_rate, problem_.spatial.user_rate); + copy_vector(execution_space_, spatial_.absorber, problem_.spatial.absorber_rate); + copy_vector(execution_space_, spatial_.transverse_rate, + problem_.spatial.kperp_rate); + if (problem_.slab_geometry()) + copy_vector(execution_space_, spatial_.kappa, problem_.spatial.kappa_sq); + } else if (!problem_.flux.k_squared.empty()) { + copy_vector(execution_space_, flux_.square, problem_.flux.k_squared); + copy_vector(execution_space_, flux_.square_low, problem_.flux.k_squared_low); + copy_vector(execution_space_, flux_.transverse, problem_.flux.kperp_sq); + copy_vector(execution_space_, flux_.transverse_low, problem_.flux.kperp_sq_low); + } + if (problem_.plasma.n_channels > 1 || problem_.plasma.envelope) { + copy_vector(execution_space_, channel_parameters_, + problem_.plasma.channel_parameters); + copy_vector(execution_space_, channel_coefficients_, + problem_.plasma.channel_coefficients); + } + copy_vector(execution_space_, crop_index_, problem_.grid.crop_index); + if (!fine_gaps_.empty()) { + std::vector runs; + std::size_t position = 0; + for (const auto& [bin, length] : fine_gap_runs()) { + runs.insert(runs.end(), {position, bin, length}); + position += length; + } + copy_vector(execution_space_, fine_gaps_, runs); + } + if (problem_.plasma.envelope) { + copy_vector(execution_space_, envelope_rule_nodes_, + problem_.plasma.envelope_rule_nodes); + copy_vector(execution_space_, envelope_rule_weights_, + problem_.plasma.envelope_rule_weights); + copy_vector(execution_space_, envelope_drude_factor_, + problem_.plasma.envelope_drude_factor); + copy_vector(execution_space_, envelope_loss_factor_, + problem_.plasma.envelope_loss_factor); + } + copy_vector(execution_space_, active_mask_, problem_.grid.active_mask); + copy_vector(execution_space_, norm_mask_, problem_.grid.norm_mask); + copy_vector(execution_space_, omega_window_, problem_.grid.omega_window); + copy_vector(execution_space_, omega_, problem_.grid.omega); + copy_vector(execution_space_, time_window_, problem_.grid.time_window); + if (held_plasma()) { + copy_vector(execution_space_, held_derivative_, + problem_.grid.time_window_derivative); + std::vector inverse(problem_.dims.n_omega, 0.0); + for (std::size_t index = 0; index < problem_.dims.n_omega; ++index) + if (problem_.grid.active_mask[index] != 0) + inverse[index] = 1.0 / problem_.grid.omega[index]; + copy_vector(execution_space_, held_inverse_, inverse); + } + copy_vector(execution_space_, carrier_, problem_.grid.carrier); + // A z table's slots start at zero (Kokkos initialises a view) until evaluated. + lossless_phase_ = false; + if (!problem_.linear.phi_table) { + copy_vector(execution_space_, phi_terms_, problem_.linear.phi_terms); + lossless_phase_ = + std::ranges::all_of(problem_.linear.phi_terms, + [](const Complex& term) { return term.real() == 0.0; }); + } + if (!problem_.linear.prefactor_table) + copy_vector(execution_space_, prefactor_terms_, problem_.linear.prefactor_terms); + copy_vector(execution_space_, tableau_, problem_.stepper.tableau_matrix); + copy_vector(execution_space_, spatial_spectral_weights_, + problem_.spatial.spectral_weights); + copy_vector(execution_space_, spatial_on_axis_, problem_.spatial.on_axis); + copy_vector(execution_space_, synthesis_, problem_.transform.synthesis_matrix); + copy_vector(execution_space_, projection_, problem_.transform.projection_matrix); + if (!problem_.transform.projection_coarse_matrix.empty()) + copy_vector(execution_space_, projection_coarse_, + problem_.transform.projection_coarse_matrix); + copy_vector(execution_space_, absorber_fine_, problem_.absorber.fine); + spatial_only_absorber_ = std::ranges::all_of( + problem_.absorber.fine, [](const double rate) { return rate == 0.0; }); + copy_vector(execution_space_, absorber_coarse_, problem_.absorber.coarse); + if (problem_.plasma.enabled) + copy_vector(execution_space_, rate_coefficients_, + problem_.plasma.rate_coefficients); + if (!plasma_taps_.empty()) + copy_vector(execution_space_, plasma_taps_, problem_.plasma.taps); + if (problem_.raman.enabled) { + copy_vector(execution_space_, raman_undamped_, problem_.raman.undamped); + copy_vector(execution_space_, raman_tensor_weights_, problem_.raman.tensor_weights); + graph::deep_copy(execution_space_, node_raman_drive_, 0.0); + } +} + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +// Measured plans for free space's batched transforms (SEP 28): the transverse pair in +// place on the scratch and, on the complex fields, the chunk's temporal pair, each at +// the plan policy's effort for its shape and width (`fftw_seed_effort`). The scratch +// buffers are cleared after planning overwrote them. +void GeneralWorkspace::seed_free_wisdom() { + const auto planes = static_cast(transverse_.extent(0)); + const auto n_y = static_cast(transverse_.extent(1)); + const auto n_x = static_cast(transverse_.extent(2)); + for (const int sign : {FFTW_FORWARD, FFTW_BACKWARD}) + seed_fftw_wisdom(fft_space_for_rows(transverse_), transverse_.data(), + transverse_.data(), {n_y, n_x}, planes, n_y * n_x, n_y * n_x, + sign); + graph::deep_copy(execution_space_, transverse_, Complex{}); + // The chunk's temporal pair is the modal routes' (`separable_to_time`, + // `separable_to_frequency`), on every field family: the real family's pair and the + // analytic family's forward transform were left at FFTW_ESTIMATE (solprop #560). + // A fine spectrum wider than n_to is not a plain row pair; the helper leaves it. + seed_modal_wisdom(); +} + +// Measured plans for the modal routes' temporal pair, the two transforms of every +// source evaluation: Kokkos-FFT plans them at FFTW_ESTIMATE, which picks markedly +// slower codelets for long rows (a 16384-sample envelope GNLSE ran its pair at about +// twice the measured cost). The plan policy (`fftw_seed_effort`) decides the effort. +void GeneralWorkspace::seed_modal_wisdom() { + if (problem_.dims.family == FieldFamily::field_real) { + seed_row_fft_wisdom(fft_space_for_rows(modal_real_), fine_spectrum_, modal_real_, + KokkosFFT::Direction::backward); + graph::deep_copy(execution_space_, modal_real_, 0.0); + } else { + seed_row_fft_wisdom(fft_space_for_rows(modal_complex_), fine_spectrum_, + modal_complex_, KokkosFFT::Direction::backward); + graph::deep_copy(execution_space_, modal_complex_, Complex{}); + } + if (problem_.dims.family == FieldFamily::envelope) { + seed_row_fft_wisdom(fft_space_for_rows(projected_complex_), projected_complex_, + projected_spectrum_, KokkosFFT::Direction::forward); + graph::deep_copy(execution_space_, projected_complex_, Complex{}); + } else { + seed_row_fft_wisdom(fft_space_for_rows(projected_real_), projected_real_, + projected_spectrum_, KokkosFFT::Direction::forward); + graph::deep_copy(execution_space_, projected_real_, 0.0); + } + graph::deep_copy(execution_space_, fine_spectrum_, Complex{}); + graph::deep_copy(execution_space_, projected_spectrum_, Complex{}); +} + +// The plasma streams' forward rows, fine and coarse. Most share a shape with the +// projected pair seeded above, whose wisdom FFTW then finds at once; the radial held +// streams' chunk rows do not. +void GeneralWorkspace::seed_stream_wisdom() { + // The forward pack's plans (`packs_forward`); seeding overwrites their rows. + for (const auto& [rows, spectrum] : + {std::pair{forward_pack_, forward_pack_spectrum_}, + std::pair{forward_pack_tail_, forward_pack_tail_spectrum_}, + std::pair{forward_pack_fine_, forward_pack_fine_spectrum_}, + std::pair{forward_pack_fine_tail_, forward_pack_fine_tail_spectrum_}, + std::pair{forward_pack_coarse_, forward_pack_coarse_spectrum_}}) { + if (rows.empty()) + continue; + seed_row_fft_wisdom(fft_space_for_rows(rows), rows, spectrum, + KokkosFFT::Direction::forward); + } + if (!forward_pack_.empty()) { + graph::deep_copy(execution_space_, forward_pack_, 0.0); + graph::deep_copy(execution_space_, forward_pack_spectrum_, Complex{}); + } + if (held_plasma()) + for (auto& stream : held_streams_) { + if (stream.projected.empty()) + continue; + seed_row_fft_wisdom(fft_space_for_rows(stream.projected), stream.projected, + stream.spectrum, KokkosFFT::Direction::forward); + graph::deep_copy(execution_space_, stream.projected, 0.0); + graph::deep_copy(execution_space_, stream.spectrum, Complex{}); + if (stream.coarse.empty()) + continue; + seed_row_fft_wisdom(fft_space_for_rows(stream.coarse), stream.coarse, + stream.coarse_spectrum, KokkosFFT::Direction::forward); + graph::deep_copy(execution_space_, stream.coarse, 0.0); + graph::deep_copy(execution_space_, stream.coarse_spectrum, Complex{}); + } + if (problem_.plasma.envelope) + for (auto& stream : envelope_streams_) { + seed_row_fft_wisdom(fft_space_for_rows(stream.projected), stream.projected, + stream.spectrum, KokkosFFT::Direction::forward); + graph::deep_copy(execution_space_, stream.projected, Complex{}); + graph::deep_copy(execution_space_, stream.spectrum, Complex{}); + if (stream.coarse.empty()) + continue; + seed_row_fft_wisdom(fft_space_for_rows(stream.coarse), stream.coarse, + stream.coarse_spectrum, KokkosFFT::Direction::forward); + graph::deep_copy(execution_space_, stream.coarse, Complex{}); + graph::deep_copy(execution_space_, stream.coarse_spectrum, Complex{}); + } +} +#endif + +void GeneralWorkspace::plan() { +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // Seed every measured shape before the first plan exists: a Kokkos-FFT plan made + // before its wisdom stays at FFTW_ESTIMATE for the run. + if (problem_.free_geometry()) + seed_free_wisdom(); + else if (!problem_.separable_geometry()) + seed_modal_wisdom(); + seed_stream_wisdom(); +#endif + fft_execution_space_ = modal_complex_.size() > modal_real_.size() + ? fft_space_for_rows(modal_complex_) + : fft_space_for_rows(modal_real_); + if (problem_.raman.enabled) { + raman_execution_space_ = raman_pool_.get( + CpuOperation::raman, node_raman_drive_.size(), row_length(node_raman_drive_)); + const auto raman_tail = problem_.dims.n_nodes % problem_.dims.node_chunk; + raman_tail_execution_space_ = raman_tail_pool_.get( + CpuOperation::raman, + raman_tail * problem_.raman_drives() * 2 * problem_.dims.n_to, + 2 * problem_.dims.n_to); + raman_kernel_execution_space_ = raman_pool_.get( + CpuOperation::raman, raman_kernel_time_.size(), row_length(raman_kernel_time_)); + raman_multiply_space_ = + kernel_pool_.get(CpuOperation::stage, node_raman_spectrum_.size()); + raman_tail_multiply_space_ = + kernel_pool_.get(CpuOperation::stage, raman_tail * problem_.raman_drives() * + (problem_.dims.n_to + 1)); + raman_rebuild_space_ = kernel_pool_.get( + CpuOperation::response, raman_kernel_time_.size() * problem_.raman.n_terms); + raman_scale_space_ = + kernel_pool_.get(CpuOperation::stage, raman_kernel_spectrum_.size()); + raman_products_ = raman_modal_products(); + if (raman_products_ != 0) { + raman_product_execution_space_ = raman_tail_pool_.get( + CpuOperation::raman, raman_products_ * 2 * problem_.dims.n_to, + 2 * problem_.dims.n_to); + raman_product_space_ = kernel_pool_.get(CpuOperation::stage, + raman_products_ * 2 * problem_.dims.n_to); + } + } + if (general_) + general_->plan(); + if (dispersive_) + dispersive_->plan(fft_pool_); + if (separable_dispersive_) + separable_dispersive_->plan(fft_pool_); + plan_transverse(); + constexpr int axis = -1; + if (held_plasma()) { + for (auto& stream : held_streams_) { + if (stream.projected.empty()) + continue; + stream.forward = std::make_unique( + fft_space_for_rows(stream.projected), stream.projected, stream.spectrum, + KokkosFFT::Direction::forward, axis); + if (!stream.coarse.empty()) + stream.coarse_forward = std::make_unique( + fft_space_for_rows(stream.coarse), stream.coarse, stream.coarse_spectrum, + KokkosFFT::Direction::forward, axis); + } + if (!held_pack_.empty() && forward_pack_.empty()) + held_pack_forward_ = std::make_unique( + fft_space_for_rows(held_pack_), held_pack_, held_pack_spectrum_, + KokkosFFT::Direction::forward, axis); + } + if (problem_.plasma.envelope) + for (auto& stream : envelope_streams_) { + stream.forward = std::make_unique( + fft_space_for_rows(stream.projected), stream.projected, stream.spectrum, + KokkosFFT::Direction::forward, axis); + if (!stream.coarse.empty()) + stream.coarse_forward = std::make_unique( + fft_space_for_rows(stream.coarse), stream.coarse, stream.coarse_spectrum, + KokkosFFT::Direction::forward, axis); + } + statistics_inverse_plan_ = std::make_unique( + fft_space_for_rows(statistics_time_), statistics_spectrum_, statistics_time_, + KokkosFFT::Direction::backward, axis); + if (problem_.dims.family == FieldFamily::field_real) { + inverse_real_plan_ = std::make_unique( + fft_space_for_rows(modal_real_), fine_spectrum_, modal_real_, + KokkosFFT::Direction::backward, axis); + } else { + inverse_complex_plan_ = std::make_unique( + fft_space_for_rows(modal_complex_), fine_spectrum_, modal_complex_, + KokkosFFT::Direction::backward, axis); + } + if (retained_synthesis_) { + const auto plan_rows = [&](const std::size_t count) { + const auto rows = std::make_pair(std::size_t{0}, count); + const ComplexView2 padded = Kokkos::subview(modal_complex_, rows, Kokkos::ALL()); + const ComplexView2 node = Kokkos::subview(node_complex_, rows, Kokkos::ALL()); + const auto& space = fft_space_for_rows(node); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + seed_row_fft_wisdom(space, padded, node, KokkosFFT::Direction::backward); +#endif + auto plan = std::make_unique(space, padded, node, + KokkosFFT::Direction::backward, axis); + // The padded rows' unretained bins stay zero from here on + // (`synthesise_retained`), whatever planning wrote. + graph::deep_copy(execution_space_, padded, Complex{}); + graph::deep_copy(execution_space_, node, Complex{}); + return plan; + }; + retained_inverse_plan_ = plan_rows(problem_.dims.node_chunk); + if (const auto tail = problem_.dims.n_nodes % problem_.dims.node_chunk; tail != 0) + retained_tail_inverse_plan_ = plan_rows(tail); + } + if (problem_.dims.family == FieldFamily::envelope) { + forward_complex_plan_ = std::make_unique( + fft_space_for_rows(projected_complex_), projected_complex_, projected_spectrum_, + KokkosFFT::Direction::forward, axis); + absorber_complex_plan_ = std::make_unique( + fft_space_for_rows(absorber_complex_), absorber_complex_, absorber_spectrum_, + KokkosFFT::Direction::forward, axis); + if (!problem_.transform.projection_coarse_matrix.empty()) + forward_coarse_complex_plan_ = std::make_unique( + fft_space_for_rows(projected_coarse_complex_), projected_coarse_complex_, + projected_coarse_spectrum_, KokkosFFT::Direction::forward, axis); + } else { + forward_real_plan_ = std::make_unique( + fft_space_for_rows(projected_real_), projected_real_, projected_spectrum_, + KokkosFFT::Direction::forward, axis); + absorber_real_plan_ = std::make_unique( + fft_space_for_rows(absorber_real_), absorber_real_, absorber_spectrum_, + KokkosFFT::Direction::forward, axis); + if (!forward_pack_.empty()) + forward_pack_plan_ = std::make_unique( + fft_space_for_rows(forward_pack_), forward_pack_, forward_pack_spectrum_, + KokkosFFT::Direction::forward, axis); + if (!forward_pack_tail_.empty()) + forward_pack_tail_plan_ = std::make_unique( + fft_space_for_rows(forward_pack_tail_), forward_pack_tail_, + forward_pack_tail_spectrum_, KokkosFFT::Direction::forward, axis); + if (!forward_pack_fine_.empty()) + forward_pack_fine_plan_ = std::make_unique( + fft_space_for_rows(forward_pack_fine_), forward_pack_fine_, + forward_pack_fine_spectrum_, KokkosFFT::Direction::forward, axis); + if (!forward_pack_fine_tail_.empty()) + forward_pack_fine_tail_plan_ = std::make_unique( + fft_space_for_rows(forward_pack_fine_tail_), forward_pack_fine_tail_, + forward_pack_fine_tail_spectrum_, KokkosFFT::Direction::forward, axis); + if (!forward_pack_coarse_.empty()) + forward_pack_coarse_plan_ = std::make_unique( + fft_space_for_rows(forward_pack_coarse_), forward_pack_coarse_, + forward_pack_coarse_spectrum_, KokkosFFT::Direction::forward, axis); + if (!problem_.transform.projection_coarse_matrix.empty()) + forward_coarse_real_plan_ = std::make_unique( + fft_space_for_rows(projected_coarse_real_), projected_coarse_real_, + projected_coarse_spectrum_, KokkosFFT::Direction::forward, axis); + } + if constexpr (cpu_size_policy_enabled) { + if (problem_.separable_geometry()) { + const auto full_units = transverse_units_per_chunk(); + const auto tail_units = transverse_units() % full_units; + const auto rows_per_unit = + problem_.slab_geometry() ? problem_.spatial.n_p : problem_.dims.n_pol_state; + temporal_tail_rows_ = tail_units * rows_per_unit; + if (temporal_tail_rows_ != 0) { + const auto& space = + fft_space_for(temporal_tail_rows_ * problem_.dims.n_to, problem_.dims.n_to); + const auto rows = fine_spectrum_.extent(0); + if (problem_.dims.family == FieldFamily::field_real) + temporal_tail_inverse_real_.create(space, fine_spectrum_, modal_real_, rows, + temporal_tail_rows_, + KokkosFFT::Direction::backward); + else + temporal_tail_inverse_complex_.create(space, fine_spectrum_, modal_complex_, + rows, temporal_tail_rows_, + KokkosFFT::Direction::backward); + if (problem_.dims.family == FieldFamily::envelope) + temporal_tail_forward_complex_.create( + space, projected_complex_, projected_spectrum_, rows, temporal_tail_rows_, + KokkosFFT::Direction::forward); + else + temporal_tail_forward_real_.create( + space, projected_real_, projected_spectrum_, rows, temporal_tail_rows_, + KokkosFFT::Direction::forward); + // The tail plans' measured planning overwrote their rows. + graph::deep_copy(execution_space_, fine_spectrum_, Complex{}); + graph::deep_copy(execution_space_, projected_spectrum_, Complex{}); + if (problem_.dims.family == FieldFamily::envelope) + graph::deep_copy(execution_space_, projected_complex_, Complex{}); + else + graph::deep_copy(execution_space_, projected_real_, 0.0); + if (problem_.dims.family == FieldFamily::field_real) + graph::deep_copy(execution_space_, modal_real_, 0.0); + else + graph::deep_copy(execution_space_, modal_complex_, Complex{}); + } + } + } + if (problem_.raman.enabled) + plan_raman(); + if (problem_.absorber.split_step) + plan_split(); +} + +// The transverse plans of a separable geometry: the slab's y transforms, or free +// space's planes (and on a device its volume route). +void GeneralWorkspace::plan_transverse() { + if (problem_.slab_geometry()) { + // The y transforms are many short ones; below the threaded-plan threshold they + // run on one FFTW thread, as every other small plan (`fft_space_for`). + constexpr int y_axis = -1; + const auto& space = fft_space_for_rows(transverse_); + slab_forward_ = std::make_unique(space, transverse_, transverse_, + KokkosFFT::Direction::forward, y_axis); + slab_inverse_ = std::make_unique(space, transverse_, transverse_, + KokkosFFT::Direction::backward, y_axis); + } else if (problem_.free_geometry()) { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + constexpr std::size_t axes = 2; +#else + const KokkosFFT::axis_type<2> axes{-2, -1}; +#endif + transverse_forward_ = std::make_unique( + fft_space_for_rows(transverse_), transverse_, transverse_, + KokkosFFT::Direction::forward, axes); + transverse_inverse_ = std::make_unique( + fft_space_for_rows(transverse_), transverse_, transverse_, + KokkosFFT::Direction::backward, axes); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + bool identity_crop = problem_.grid.crop_index.size() == problem_.dims.n_omega; + for (std::size_t index = 0; identity_crop && index < problem_.dims.n_omega; ++index) + identity_crop = problem_.grid.crop_index[index] == index; + volume_kerr_ = identity_crop && problem_.dims.family == FieldFamily::envelope && + problem_.dims.n_pol_state == 1 && + problem_.effective_node_pol() == 1 && + problem_.dims.n_to == problem_.dims.n_omega && + problem_.dims.n_t == problem_.dims.n_to && separable_aliased_ && + problem_.kerr.nonlinear && !problem_.plasma.enabled && + !problem_.raman.enabled && !problem_.plasma.envelope && + !problem_.dispersive.enabled && separable_dispersive_ == nullptr && + problem_.kerr.harmonic_factor == 0.0 && separable_gaps_.empty(); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // At 768 x 768 x 600 on an NVIDIA H200, cuFFT's `{N,N,T}` takes 14.7 ms a transform + // against 17.6 ms for `{T,N,N}` (solprop #365). An in-stage absorber evaluates its + // term from the same input in the scratch's order, so it keeps that order. + volume_order_ = volume_kerr_ && !problem_.absorber.in_stage; +#endif + if (volume_order_) + volume_scratch_ = ComplexView3(transverse_.data(), problem_.spatial.n_y, + problem_.spatial.n_x, problem_.dims.n_omega); + if (volume_kerr_) { + constexpr std::size_t volume_axes = 3; + const auto& volume = volume_order_ ? volume_scratch_ : transverse_; + volume_forward_ = std::make_unique( + execution_space_, volume, volume, KokkosFFT::Direction::forward, volume_axes); + volume_inverse_ = + std::make_unique(execution_space_, volume, volume, + KokkosFFT::Direction::backward, volume_axes); + } + std::size_t work = KokkosFFT::compute_required_workspace_size( + *transverse_forward_, *transverse_inverse_); + if (volume_kerr_) + work = std::max(work, KokkosFFT::compute_required_workspace_size( + *volume_forward_, *volume_inverse_)); + if (work != 0) { + transform_work_ = + ComplexView(Kokkos::view_alloc("solprop_native.free.transform_work", + Kokkos::WithoutInitializing), + work); + transverse_forward_->set_work_area(transform_work_); + transverse_inverse_->set_work_area(transform_work_); + if (volume_kerr_) { + volume_forward_->set_work_area(transform_work_); + volume_inverse_->set_work_area(transform_work_); + } + } +#endif + } +} + +// The Raman convolution's plans: the chunk's, its tail's, the modal products' and the +// kernel's, timed for the record. +void GeneralWorkspace::plan_raman() { + constexpr int axis = -1; + const auto started = std::chrono::steady_clock::now(); + raman_drive_forward_plan_ = std::make_unique( + raman_execution_space_, node_raman_drive_, node_raman_spectrum_, + KokkosFFT::Direction::forward, axis); + raman_convolution_inverse_plan_ = std::make_unique( + raman_execution_space_, node_raman_spectrum_, node_raman_convolved_, + KokkosFFT::Direction::backward, axis); + const auto tail = problem_.dims.n_nodes % problem_.dims.node_chunk; + if (tail != 0) { + // Reuse the leading rows of the full chunk buffers. A separate persistent + // plan avoids transforming stale rows in the final partial node chunk. + const auto rows = std::make_pair(std::size_t{0}, tail * problem_.raman_drives()); + raman_tail_drive_ = Kokkos::subview(node_raman_drive_, rows, Kokkos::ALL()); + raman_tail_spectrum_ = Kokkos::subview(node_raman_spectrum_, rows, Kokkos::ALL()); + raman_tail_convolved_ = Kokkos::subview(node_raman_convolved_, rows, Kokkos::ALL()); + raman_tail_forward_plan_ = std::make_unique( + raman_tail_execution_space_, raman_tail_drive_, raman_tail_spectrum_, + KokkosFFT::Direction::forward, axis); + raman_tail_inverse_plan_ = std::make_unique( + raman_tail_execution_space_, raman_tail_spectrum_, raman_tail_convolved_, + KokkosFFT::Direction::backward, axis); + } + if (raman_products_ != 0) { + const auto rows = std::make_pair(std::size_t{0}, raman_products_); + raman_product_drive_ = Kokkos::subview(node_raman_drive_, rows, Kokkos::ALL()); + raman_product_spectrum_ = + Kokkos::subview(node_raman_spectrum_, rows, Kokkos::ALL()); + raman_product_forward_plan_ = std::make_unique( + raman_product_execution_space_, raman_product_drive_, raman_product_spectrum_, + KokkosFFT::Direction::forward, axis); + raman_product_inverse_plan_ = std::make_unique( + raman_product_execution_space_, raman_product_spectrum_, raman_product_drive_, + KokkosFFT::Direction::backward, axis); + } + raman_kernel_forward_plan_ = std::make_unique( + raman_kernel_execution_space_, raman_kernel_time_, raman_kernel_spectrum_, + KokkosFFT::Direction::forward, axis); + raman_planning_seconds_ = + std::chrono::duration(std::chrono::steady_clock::now() - started).count(); +} + +// The split-step absorber's plans: in place on the state's rows, or on the coarse grid. +void GeneralWorkspace::plan_split() { + constexpr int axis = -1; + if (split_rows_ != 0) { + // In place on the state's rows; a later execution may name another row block or the + // other of the state's two views, of the same extents. + const auto& state = problem_.stepper.low_storage ? current_ : advanced_; + const auto plan_rows = [&](const std::size_t count, + const KokkosFFT::Direction direction) { + const ComplexView2 rows = + Kokkos::subview(state, std::make_pair(std::size_t{0}, count), Kokkos::ALL()); + const auto& space = fft_space_for_rows(rows); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // Seeding may overwrite the rows; nothing is loaded yet. + seed_row_fft_wisdom(space, rows, rows, direction); + graph::deep_copy(execution_space_, rows, Complex{}); +#endif + return std::make_unique(space, rows, rows, direction, axis); + }; + split_inverse_plan_ = plan_rows(split_rows_, KokkosFFT::Direction::backward); + split_forward_plan_ = plan_rows(split_rows_, KokkosFFT::Direction::forward); + if (const auto tail = problem_.n_columns() % split_rows_; tail != 0) { + split_tail_inverse_plan_ = plan_rows(tail, KokkosFFT::Direction::backward); + split_tail_forward_plan_ = plan_rows(tail, KokkosFFT::Direction::forward); + } + return; + } + if (problem_.dims.family == FieldFamily::envelope) { + coarse_inverse_complex_plan_ = std::make_unique( + fft_space_for_rows(coarse_time_complex_), coarse_spectrum_, + coarse_time_complex_, KokkosFFT::Direction::backward, axis); + coarse_forward_complex_plan_ = std::make_unique( + fft_space_for_rows(coarse_time_complex_), coarse_time_complex_, + coarse_spectrum_, KokkosFFT::Direction::forward, axis); + } else { + coarse_inverse_real_plan_ = std::make_unique( + fft_space_for_rows(coarse_time_real_), coarse_spectrum_, coarse_time_real_, + KokkosFFT::Direction::backward, axis); + coarse_forward_real_plan_ = std::make_unique( + fft_space_for_rows(coarse_time_real_), coarse_time_real_, coarse_spectrum_, + KokkosFFT::Direction::forward, axis); + } + if (temporal_tail_rows_ != 0) { + const auto& space = + fft_space_for(temporal_tail_rows_ * problem_.dims.n_t, problem_.dims.n_t); + const auto rows = coarse_spectrum_.extent(0); + if (problem_.dims.family == FieldFamily::envelope) { + coarse_tail_inverse_complex_.create(space, coarse_spectrum_, coarse_time_complex_, + rows, temporal_tail_rows_, + KokkosFFT::Direction::backward); + coarse_tail_forward_complex_.create(space, coarse_time_complex_, coarse_spectrum_, + rows, temporal_tail_rows_, + KokkosFFT::Direction::forward); + } else { + coarse_tail_inverse_real_.create(space, coarse_spectrum_, coarse_time_real_, rows, + temporal_tail_rows_, + KokkosFFT::Direction::backward); + coarse_tail_forward_real_.create(space, coarse_time_real_, coarse_spectrum_, rows, + temporal_tail_rows_, + KokkosFFT::Direction::forward); + } + // The tail plans' measured planning overwrote their rows. + graph::deep_copy(execution_space_, coarse_spectrum_, Complex{}); + if (problem_.dims.family == FieldFamily::envelope) + graph::deep_copy(execution_space_, coarse_time_complex_, Complex{}); + else + graph::deep_copy(execution_space_, coarse_time_real_, 0.0); + } +} + +namespace { + +// The bytes the views hold: each one's elements times its element's size. +template std::size_t bytes(const Views&... views) { + return (std::size_t{0} + ... + (views.size() * sizeof(typename Views::value_type))); +} + +// The bytes the problem's host vectors reserve. +template std::size_t reserved(const Vectors&... vectors) { + return (std::size_t{0} + ... + + (vectors.capacity() * sizeof(typename Vectors::value_type))); +} + +} // namespace + +std::map GeneralWorkspace::memory_ledger() const { + auto ledger = + general_ ? general_->memory_ledger() : GeneralModalSource::empty_ledger(); + ledger.merge(flow_ ? flow_->memory_ledger() : CoupledFlowAction::empty_ledger()); + ledger.merge(dispersive_ ? dispersive_->memory_ledger() + : separable_dispersive_ ? separable_dispersive_->memory_ledger() + : general_ && general_->dispersive() != nullptr + ? general_->dispersive()->memory_ledger() + : DispersiveResponse::empty_ledger()); + ledger.merge(base_memory_ledger()); + return ledger; +} + +std::map GeneralWorkspace::base_memory_ledger() const { + std::size_t edge_plan_heap = 0; + std::size_t edge_plan_device = 0; + for (const auto& [shape, batch] : edge_batches_) { + edge_plan_heap += batch.heap_growth; + edge_plan_device += batch.device_growth; + } + const auto real_bytes = sizeof(double); + const auto field = problem_.n_columns() * problem_.dims.n_omega * sizeof(Complex); + const auto modal = bytes(modal_real_, modal_complex_); + // Aliased views (free space, SEP 28) own no storage of their own. + const auto node = separable_aliased_ ? 0 : bytes(node_real_, node_complex_); + const auto polarisation = + separable_aliased_ ? 0 : bytes(polarisation_real_, polarisation_complex_); + const auto projected = bytes(projected_real_, projected_complex_); + return { + {"held_current_host_derivative", reserved(problem_.grid.time_window_derivative)}, + {"held_current", bytes(held_current_)}, + {"gated_current_prefix", bytes(gated_prefix_, gated_filtered_)}, + {"held_current_gj", held_streams_[0].bytes()}, + {"held_current_edge", held_streams_[1].bytes()}, + {"held_current_nonplasma", held_streams_[2].bytes()}, + {"held_current_projection", bytes(held_projection_)}, + {"held_current_radial_spectra", + bytes(held_nodal_spectrum_, held_projected_spectrum_)}, + {"held_current_tables", bytes(held_derivative_, held_inverse_)}, + {"envelope_drude_stream", envelope_streams_[0].bytes()}, + {"envelope_loss_stream", envelope_streams_[1].bytes()}, + {"envelope_cycles", + bytes(envelope_rule_nodes_, envelope_rule_weights_, envelope_moments_, + envelope_rates_, envelope_exposure_)}, + {"envelope_factors", bytes(envelope_drude_factor_, envelope_loss_factor_)}, + {"envelope_status", + bytes(envelope_failures_, envelope_status_, envelope_host_status_)}, + {"radial_blas_buffers", bytes(radial_blas_input_, radial_blas_output_)}, + {"state", field}, + {"advanced", bytes(advanced_)}, + {"stages", bytes(stages_)}, + {"accumulator_solution", bytes(solution_)}, + {"accumulator_error", bytes(error_)}, + {"weighted_residual", bytes(residual_)}, + {"recycled_stage", bytes(recycled_)}, + // A native session's Runge-Kutta dense output (solprop SEP 47 section 5.3). + {"rk_dense", bytes(rk_dense_coefficients_, rk_dense_held_, rk_dense_stiff_)}, + {"noise", problem_.stepper.modified_noise ? field : 0}, + // The Runge-Kutta march's whole-step propagator, or the low-storage march's + // table or step pair (solprop's `propagators` row). + {"propagators", bytes(whole_propagator_, low_storage_flows_)}, + {"register_accumulator", bytes(register_accumulator_)}, + {"register_stage", bytes(register_stage_)}, + {"stage_phase", bytes(stage_phase_)}, + {"stage_inverse", bytes(stage_inverse_)}, + {"fine_spectrum", bytes(fine_spectrum_)}, + {"retained_synthesis", bytes(retained_modal_, retained_node_)}, + {"fine_gaps", bytes(fine_gaps_, separable_gaps_)}, + {"modal_time", modal}, + {"node_field", node}, + {"node_polarisation", polarisation}, + {"projected", projected}, + {"projected_coarse", + problem_.transform.projection_coarse_matrix.empty() ? 0 : projected}, + {"projected_spectrum", bytes(projected_spectrum_)}, + {"projected_coarse_spectrum", bytes(projected_coarse_spectrum_)}, + {"absorber_spectrum", bytes(absorber_spectrum_)}, + {"absorber_time", bytes(absorber_complex_, absorber_real_)}, + {"split_absorber_spectrum", bytes(coarse_spectrum_)}, + {"split_absorber_time", bytes(coarse_time_complex_, coarse_time_real_)}, + {"transform_matrices", bytes(synthesis_, projection_, projection_coarse_)}, + // The native problem owns host copies as well as the execution-space views. + // Count retained vector capacity, including construction-only coefficients. + {"host_problem_arrays", + reserved(problem_.grid.crop_index, problem_.grid.active_mask, + problem_.grid.norm_mask, problem_.grid.omega_window, + problem_.grid.omega, problem_.grid.power_weight, + problem_.grid.time_window, problem_.grid.carrier) + + reserved(problem_.spatial.k_squared, problem_.spatial.k_squared_low, + problem_.spatial.numerator, problem_.spatial.clamped, + problem_.spatial.kperp_sq, problem_.spatial.kperp_sq_low, + problem_.spatial.frame, problem_.spatial.spectral_rate, + problem_.spatial.user_rate, problem_.spatial.absorber_rate, + problem_.spatial.kperp_rate, problem_.spatial.k_x, + problem_.spatial.k_y, problem_.spatial.kappa_sq, + problem_.spatial.absorber_matrix, problem_.spatial.spectral_weights, + problem_.spatial.on_axis) + + reserved(problem_.stepper.tableau_matrix, problem_.stepper.tableau_nodes, + problem_.stepper.solution_weights, problem_.stepper.error_weights) + + reserved(problem_.linear.phi_terms, problem_.linear.prefactor_terms) + + reserved(problem_.absorber.fine, problem_.absorber.coarse) + + reserved(problem_.transform.synthesis_matrix, + problem_.transform.projection_matrix, + problem_.transform.projection_coarse_matrix) + + reserved(problem_.plasma.envelope_rule_nodes, + problem_.plasma.envelope_rule_weights, + problem_.plasma.envelope_drude_factor, + problem_.plasma.envelope_loss_factor, + problem_.plasma.rate_coefficients, + problem_.plasma.channel_parameters, + problem_.plasma.channel_coefficients) + + reserved(problem_.raman.undamped, problem_.raman.tensor_weights, + problem_.raman.row_component, problem_.raman.component_fraction) + + reserved(problem_.general.synthesis, problem_.general.projection, + problem_.general.frequency_factor, + problem_.general.delayed_spectrum, problem_.general.tensor, + problem_.general.delayed_tensor, problem_.general.node_response)}, + {"scaled_transform_matrices", + bytes(scaled_synthesis_, scaled_projection_, scaled_projection_coarse_, + scaled_projection_stacked_, scaled_synthesis_complex_, + scaled_projection_complex_, scaled_projection_coarse_complex_, + scaled_projection_stacked_complex_)}, + {"stacked_projection", + bytes(projected_stacked_real_, projected_stacked_complex_)}, + {"chunk_transform_buffers", + bytes(chunk_synthesis_, tail_synthesis_, chunk_projection_, tail_projection_, + chunk_projection_stacked_, tail_projection_stacked_, + chunk_synthesis_complex_, tail_synthesis_complex_, + chunk_projection_complex_, tail_projection_complex_, + chunk_projection_stacked_complex_, tail_projection_stacked_complex_)}, + {"transverse_scratch", bytes(transverse_)}, + {"spatial_coefficients", + bytes(spatial_.square, spatial_.square_low, spatial_.numerator, spatial_.clamped, + spatial_.kappa, spatial_.transverse, spatial_.transverse_low, + spatial_.frame, spatial_.spectral_rate, spatial_.user_rate, + spatial_.absorber, spatial_.transverse_rate)}, + {"adams_history", + bytes(adams_store_, adams_slopes_, adams_moments_, adams_source_, + adams_propagator_, adams_exponent_, adams_saved_, adams_held_, + adams_dense_start_, adams_dense_end_, adams_dense_exponent_, adams_states_, + adams_rates_, adams_carried_, adams_dense_terms_, adams_lab_, + adams_endpoint_, adams_endpoint_rate_, adams_block_, adams_channels_, + adams_fallback_, adams_applied_, adams_end_applied_, adams_divisor_, + adams_rate_weights_, adams_end_weights_, adams_scale_weights_) + + (adams_magnitude_sum_.data() != nullptr ? sizeof(double) : 0)}, + {"fixed_phi_terms", bytes(phi_terms_)}, + {"fixed_prefactor_terms", bytes(prefactor_terms_)}, + {"z_tables", phi_table_.bytes() + prefactor_table_.bytes() + rate_table_.bytes()}, + {"statistics_metric", + bytes(statistics_whitened_, statistics_root_, statistics_whitened_sums_)}, + // solprop SEP 34: a radial run's plane-measure squares and its material table + // (a separable run's alias its operator's), and the magnetic closure's + // transverse volume of a separable run's local flux statistics. + {"flux_coefficients", problem_.separable_geometry() + ? 0 + : bytes(flux_.square, flux_.square_low, + flux_.transverse, flux_.transverse_low)}, + {"flux_table", flux_table_.bytes()}, + {"flux_closure_transverse", bytes(flux_transverse_)}, + {"tableau", bytes(tableau_)}, + // The per-stage scalars are held twice, in the packed buffer's two slots, and + // a device's pinned host ring holds copies of the whole buffer. + {"stage_scalars", + 2 * bytes(stage_phi_delta_, stage_phi_at_, stage_prefactor_, stage_density_, + stage_taper_, stage_preionisation_, stage_rate_weights_)}, + {"stage_scalar_staging", + stage_staging_.empty() || stage_staging_.front().data() == stage_scalars_.data() + ? 0 + : stage_staging_.size() * bytes(stage_scalars_)}, + // An Adams trial's norms, read back in pinned host memory on a device, and a + // Runge-Kutta trial's mirror of its partials (the view itself on a host build). + {"trial_norms", + bytes(adams_norms_) + + (pinned_partials_.data() == partials_.data() ? 0 : bytes(pinned_partials_))}, + {"crop_index", bytes(crop_index_)}, + {"masks", bytes(active_mask_, norm_mask_)}, + {"windows", bytes(omega_window_, time_window_)}, + {"spatial_statistics_geometry", + bytes(spatial_spectral_weights_, spatial_on_axis_)}, + {"spatial_statistics_partials", bytes(spatial_statistics_partials_)}, + {"far_field_partials", far_field_ ? far_field_->partial_bytes() : 0}, + {"far_field_tables", far_field_ ? far_field_->table_bytes() : 0}, + {"statistics_frequency_axis", bytes(omega_)}, + {"carrier", bytes(carrier_)}, + {"node_raman_drive", bytes(node_raman_drive_)}, + {"node_raman_spectrum", bytes(node_raman_spectrum_)}, + {"node_raman_convolved", bytes(node_raman_convolved_)}, + {"raman_undamped", bytes(raman_undamped_)}, + {"raman_tensor_weights", bytes(raman_tensor_weights_)}, + {"raman_kernel_time", bytes(raman_kernel_time_)}, + {"raman_kernel_spectrum", bytes(raman_kernel_spectrum_)}, + {"stage_raman_rates", 2 * bytes(stage_raman_rates_)}, + {"absorber_rates", bytes(absorber_fine_, absorber_coarse_, split_factor_)}, + {"spatial_absorber_matrix", + bytes(spatial_absorber_real_, spatial_absorber_complex_)}, + {"plasma_rate_table", bytes(rate_coefficients_)}, + {"channel_parameters", bytes(channel_parameters_)}, + {"channel_rate_tables", bytes(channel_coefficients_)}, + {"channel_scratch", bytes(channel_scratch_)}, + {"material_stage_values", + 2 * bytes(stage_kerr_density_, stage_plasma_density_, stage_raman_density_, + stage_channel_density_, stage_channel_preionisation_)}, + {"raman_component_values", + 2 * bytes(stage_raman_densities_) + + bytes(raman_row_component_, raman_component_fraction_) + + reserved(cached_raman_densities_)}, + {"plasma_stash", bytes(stash_candidate_, stash_current_)}, + {"plasma_statistics_scratch", bytes(plasma_statistics_scratch_)}, + {"plasma_integration_diagnostics", bytes(plasma_taps_, plasma_diagnostics_)}, + {"plasma_device_scan", + bytes(plasma_rate_, plasma_exposure_, plasma_fraction_, + plasma_current_integrand_, plasma_free_current_, plasma_outer_current_, + band_tile_totals_, plasma_tile_totals_)}, + {"error_partials", 3 * problem_.n_columns() * reduction_slots_ * real_bytes}, + {"quadrature_partials", problem_.transform.projection_coarse_matrix.empty() + ? 0 + : 2 * problem_.n_columns() * real_bytes}, + {"weighted_control", bytes(weighted_scale_, weighted_partials_)}, + {"statistics_spectrum", bytes(statistics_spectrum_)}, + {"statistics_time", bytes(statistics_time_)}, + {"statistics_weighted_time", bytes(statistics_weighted_time_)}, + {"statistics_power_weight", bytes(statistics_power_weight_)}, + {"statistics_power", bytes(statistics_power_)}, + // The boundary monitor's weights and accumulators, once a monitor is bound. + {"monitor_weights", + bytes(monitor_weights_, monitor_column_weights_, monitor_share_, monitor_half_, + monitor_removed_, occupancy_buckets_, monitor_split_sum_, monitor_found_)}, + {"statistics_flux_magnitude", bytes(statistics_magnitude_)}, + {"host_crossing_ring", crossing_ring_.bytes()}, + {"transverse_edge", edge_bytes()}, + {"statistics_outputs", bytes(statistics_total_power_, statistics_on_axis_, + statistics_packed_, statistics_axis_row_)}, + {"fft_plan_heap_growth", plan_heap_growth_ + edge_plan_heap}, + {"fft_plan_device_growth", plan_device_growth_ + edge_plan_device}, + }; +} + +} // namespace solprop_native diff --git a/native/src/general/state.cpp b/native/src/general/state.cpp new file mode 100644 index 000000000..e16896fa4 --- /dev/null +++ b/native/src/general/state.cpp @@ -0,0 +1,462 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +// GeneralState over its workspace: host transfers, closing and the forwarders. + +#include "workspace.hpp" + +namespace solprop_native { + +std::vector GeneralWorkspace::field_to_host() const { + std::vector result(problem_.n_columns() * problem_.dims.n_omega); + field_to_host(result); + return result; +} + +// One copy from the memory space into the caller's buffer: no staging mirror and no +// element loop, so a large state crosses once (a device's one transfer). +void GeneralWorkspace::cross_to_host(const ComplexView2& source, + const std::span destination) const { + crossed_.to_host += destination.size_bytes(); + if (crossing::takes_ring(destination.size_bytes())) { + crossing_ring_.to_host(destination.data(), source.data(), destination.size_bytes()); + return; + } + const Kokkos::View> + host(destination.data(), source.extent(0), source.extent(1)); + // On the stage's instance: a compact problem's team, which the default instance's + // whole-runtime copy would end and restart. + graph::deep_copy(execution_space_, host, source); + execution_space_.fence("solprop_native crossing"); +} + +void GeneralWorkspace::field_to_host(const std::span destination) const { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + require_extent(destination.size(), problem_.n_columns() * problem_.dims.n_omega, + "field destination"); + execution_space_.fence("solprop_native field_to_host"); + cross_to_host(current_, destination); +} + +std::vector GeneralWorkspace::stage_to_host(const std::size_t stage) const { + std::vector result(problem_.n_columns() * problem_.dims.n_omega); + stage_to_host(stage, result); + return result; +} + +void GeneralWorkspace::stage_to_host(const std::size_t stage, + const std::span destination) const { + if (!stages_ready_) + throw std::logic_error("no trial stages have been evaluated"); + if (stage >= problem_.stepper.n_stages) + throw std::out_of_range("stage index is outside the tableau"); + require_extent(destination.size(), problem_.n_columns() * problem_.dims.n_omega, + "stage destination"); + execution_space_.fence("solprop_native stage_to_host"); + const auto selected = Kokkos::subview(stages_, stage, Kokkos::ALL, Kokkos::ALL); + crossed_.to_host += destination.size_bytes(); + if (crossing::takes_ring(destination.size_bytes())) { + crossing_ring_.to_host(destination.data(), selected.data(), + destination.size_bytes()); + return; + } + const auto host = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, selected); + for (std::size_t column = 0; column < problem_.n_columns(); ++column) + for (std::size_t index = 0; index < problem_.dims.n_omega; ++index) + destination[column * problem_.dims.n_omega + index] = host(column, index); +} + +std::vector GeneralWorkspace::projected_to_host(const bool coarse) { + if (!stages_ready_) + throw std::logic_error("no trial stages have been evaluated"); + if (coarse && projected_coarse_spectrum_.extent(0) == 0) + throw std::logic_error("this quadrature has no embedded rule"); + if (coarse && dispersive_) + dispersive_->finish_coarse(); + if (coarse) + finish_quadrature(); + execution_space_.fence("solprop_native projected_to_host"); + const auto host = Kokkos::create_mirror_view_and_copy( + Kokkos::HostSpace{}, coarse ? projected_coarse_spectrum_ : projected_spectrum_); + return {host.data(), host.data() + host.size()}; +} + +std::map GeneralWorkspace::dispersive_path() const { + if (separable_dispersive_) + return {{"alone", separable_dispersive_->alone()}, + {"in_place", separable_dispersive_->in_place()}, + {"borrowed", separable_dispersive_->borrowed()}}; + // The general route: alone when no regional tensor or delayed response runs. + if (general_ && general_->dispersive() != nullptr) + return {{"alone", !general_->regional()}, {"in_place", false}, {"borrowed", false}}; + if (!dispersive_) + return {}; + return {{"alone", dispersive_->alone()}, + {"in_place", dispersive_->in_place()}, + {"borrowed", dispersive_->borrowed()}}; +} + +// Members are destroyed in reverse order of declaration: each plan before the views +// declared ahead of it, the execution instances before the pools that partition them, +// and every one before an owned Kokkos finalisation (native/AGENTS.md). +GeneralWorkspace::~GeneralWorkspace() { execution_space_.fence(); } + +template +decltype(auto) GeneralState::with_team(Function function, + Arguments&&... arguments) const { + if (!workspace_) + throw std::logic_error("native state is closed"); + const CpuTeamScope team(workspace_->cpu_team()); + // NOLINTNEXTLINE(readability-avoid-return-with-void-value): returns what the member returns, void included + return std::invoke(function, *workspace_, std::forward(arguments)...); +} + +GeneralState::GeneralState(Problem problem) + : workspace_(std::make_unique(std::move(problem))) {} +GeneralState::~GeneralState() = default; +void GeneralState::set_initial(const std::span field, + const std::span noise) { + with_team(&GeneralWorkspace::set_initial, field, noise); +} +std::vector GeneralState::trial_step(const double dz, + const TrialScalars& scalars) { + return with_team(&GeneralWorkspace::trial_step, dz, scalars); +} +void GeneralState::prepare_trial(const double dz, const TrialScalars& scalars) { + with_team(&GeneralWorkspace::prepare_trial, dz, scalars); +} +std::vector GeneralState::adams_source(const std::vector& input, + const TrialScalars& scalars) { + return with_team(&GeneralWorkspace::adams_source, input, scalars); +} +void GeneralState::adopt(const std::vector& field, const double dz) { + with_team(&GeneralWorkspace::adopt, field, dz); +} +double GeneralState::low_storage_step(const double dz, const TrialScalars& scalars, + const bool rebuild_table) { + return with_team(&GeneralWorkspace::low_storage_step, dz, scalars, rebuild_table); +} +void GeneralState::adams_allocate(const std::size_t slots, const bool compensated, + const bool interpolate) { + with_team(&GeneralWorkspace::adams_allocate, slots, compensated, interpolate); +} +void GeneralState::adams_compensation_start(const std::size_t slot, + const std::vector& rate) { + with_team(&GeneralWorkspace::adams_compensation_start, slot, rate); +} +void GeneralState::adams_start(const std::size_t slot, const TrialScalars& scalars) { + with_team(&GeneralWorkspace::adams_start, slot, scalars); +} +double GeneralState::adams_magnitude() { + return with_team(&GeneralWorkspace::adams_magnitude); +} +void GeneralState::adams_scale(const double rtol, const double atol, + const std::vector& weights) { + with_team(&GeneralWorkspace::adams_scale, rtol, atol, weights); +} +std::vector GeneralState::adams_fused_trial(const double dz, + const AdamsWeights& weights, + const std::vector& divisor, + const TrialScalars& scalars) { + return with_team(&GeneralWorkspace::adams_fused_trial, dz, weights, divisor, scalars); +} +void GeneralState::adams_fused_accept(const double dz, const AdamsWeights& keep, + const std::size_t free, + const AdamsWeights& dense) { + with_team(&GeneralWorkspace::adams_fused_accept, dz, keep, free, dense); +} +void GeneralState::adams_hold() { with_team(&GeneralWorkspace::adams_hold); } +void GeneralState::adams_restore() { with_team(&GeneralWorkspace::adams_restore); } +void GeneralState::table_upload(const TableKind kind, std::vector edges, + const double lower, const double upper, + const std::complex* coefficients, + const std::size_t n_intervals, + const std::size_t n_terms, const std::size_t n_values) { + with_team(&GeneralWorkspace::table_upload, kind, std::move(edges), lower, upper, + coefficients, n_intervals, n_terms, n_values); +} +void GeneralState::table_evaluate(const std::vector& slots, + const std::vector& positions) { + with_team(&GeneralWorkspace::table_evaluate, slots, positions); +} +void GeneralState::table_rate(const std::size_t slot, const double z, + const bool before) { + with_team(&GeneralWorkspace::table_rate, slot, z, before); +} +std::vector GeneralState::table_terms_to_host(const TableKind kind, + const std::size_t slot) const { + return with_team(&GeneralWorkspace::table_terms_to_host, kind, slot); +} +std::vector +GeneralState::phase_sum_to_host(const std::vector& weights) { + return with_team(&GeneralWorkspace::phase_sum_to_host, weights); +} +std::vector GeneralState::adams_row_names() const { + return with_team(&GeneralWorkspace::adams_row_names); +} +std::array +GeneralState::adams_row_extents(const std::string& name) const { + return with_team(&GeneralWorkspace::adams_row_extents, name); +} +std::vector GeneralState::adams_rows_to_host(const std::string& name) const { + return with_team( + [&](GeneralWorkspace& workspace) { return workspace.adams_rows_to_host(name); }); +} +void GeneralState::adams_rows_to_host(const std::string& name, + std::complex* destination, + const std::size_t size) const { + with_team([&](GeneralWorkspace& workspace) { + workspace.adams_rows_to_host(name, destination, size); + }); +} +void GeneralState::adams_rows_from_host(const std::string& name, + const std::vector& values) { + with_team(&GeneralWorkspace::adams_rows_from_host, name, values); +} +void GeneralState::adams_observe(const AdamsWeights& weights, + const TrialScalars& scalars) { + with_team(&GeneralWorkspace::adams_observe, weights, scalars); +} +std::vector GeneralState::adams_exponential_trial( + const double dz, const ExponentialWeights& weights, + const std::vector& divisor, const TrialScalars& step, + const TrialScalars& sample, const std::vector& rate) { + return with_team(&GeneralWorkspace::adams_exponential_trial, dz, weights, divisor, + step, sample, rate); +} +void GeneralState::adams_exponential_accept(const double dz, const std::size_t free, + const AdamsWeights& dense) { + with_team(&GeneralWorkspace::adams_exponential_accept, dz, free, dense); +} +void GeneralState::rk_dense_allocate(const std::vector& weights) { + with_team(&GeneralWorkspace::rk_dense_allocate, weights); +} +void GeneralState::rk_dense_hold(const double dz) { + with_team(&GeneralWorkspace::rk_dense_hold, dz); +} +void GeneralState::rk_dense_observe(const double theta, const TrialScalars& carry) { + with_team(&GeneralWorkspace::rk_dense_observe, theta, carry); +} +void GeneralState::rk_dense_restore() { + with_team(&GeneralWorkspace::rk_dense_restore); +} +void GeneralState::rk_dense_release() noexcept { + if (workspace_) + workspace_->rk_dense_release(); +} +void GeneralState::adams_exponential_observe(const double xi, const double dz, + const ExponentialWeights& weights) { + with_team(&GeneralWorkspace::adams_exponential_observe, xi, dz, weights); +} +std::vector GeneralState::weighted_error(const std::vector& scale) { + return with_team(&GeneralWorkspace::weighted_error, scale); +} +void GeneralState::set_flow_actions(const std::vector& forward, + const std::vector& inverse, + const std::vector& identity) { + with_team(&GeneralWorkspace::set_flow_actions, forward, inverse, identity); +} +std::vector GeneralState::flow_actions_to_host(const bool inverse) const { + return with_team(&GeneralWorkspace::flow_actions_to_host, inverse); +} +std::vector GeneralState::adams_block_to_host() const { + return with_team(&GeneralWorkspace::adams_block_to_host); +} +std::vector GeneralState::adams_exponent_to_host() const { + return with_team(&GeneralWorkspace::adams_exponent_to_host); +} +std::vector +GeneralState::form_flow_actions(const std::size_t section, const double dz, + const std::vector& fractions) { + return with_team(&GeneralWorkspace::form_flow_actions, section, dz, fractions); +} +void GeneralState::prepare_raman(const std::vector& rates, + const std::vector& densities) { + with_team(&GeneralWorkspace::prepare_raman, rates, densities); +} +std::vector +GeneralState::statistics(const std::uint64_t flags, const std::vector& windows, + const std::vector& on_axis, const double density, + const double taper, const double preionisation, + const double intensity_factor, + const std::vector& channel_preionisation, + const std::vector& power_weight) { + return with_team(&GeneralWorkspace::statistics, flags, windows, on_axis, density, + taper, preionisation, intensity_factor, channel_preionisation, + power_weight); +} +std::vector +GeneralState::whitened_statistics(const std::vector& root, + const std::uint64_t flags, + const std::vector& windows, + const double intensity_factor, const bool spectral) { + return with_team(&GeneralWorkspace::whitened_statistics, root, flags, windows, + intensity_factor, spectral); +} +std::vector GeneralState::spatial_statistics(const std::uint64_t flags, + const double intensity_factor, + const double z) { + return with_team(&GeneralWorkspace::spatial_statistics, flags, intensity_factor, z); +} +std::vector GeneralState::plane_fluence_trace( + const double center_x, const double center_y, const double dx, const double dy, + const double extent, const double spacing, const std::size_t n_radii, + const std::vector& cosines, const std::vector& sines) { + return with_team(&GeneralWorkspace::plane_fluence_trace, center_x, center_y, dx, dy, + extent, spacing, n_radii, cosines, sines); +} +std::vector GeneralState::axis_spectrum(const std::vector& matrix) { + return with_team(&GeneralWorkspace::axis_spectrum, matrix); +} +std::vector +GeneralState::axis_populations(const std::vector& matrix, + const std::vector& channel_preionisation) { + return with_team(&GeneralWorkspace::axis_populations, matrix, channel_preionisation); +} +std::vector GeneralState::aperture_statistics( + const std::uint64_t flags, const std::vector& apertures, const int exponent, + const std::vector& regions, const double intensity_factor) { + return with_team(&GeneralWorkspace::aperture_statistics, flags, apertures, exponent, + regions, intensity_factor); +} +void GeneralState::finish(const double dz) { with_team(&GeneralWorkspace::finish, dz); } +void GeneralState::bind_monitor(const std::vector& weights, + const std::vector& column_weights) { + with_team(&GeneralWorkspace::bind_monitor, weights, column_weights); +} +void GeneralState::unbind_monitor() noexcept { + if (workspace_) + workspace_->unbind_monitor(); +} +std::vector GeneralState::monitor_reading() { + return with_team(&GeneralWorkspace::monitor_reading); +} +void GeneralState::bind_edge(const std::vector& faces, + const std::vector& clip, + const std::vector& totals, + const std::vector& synthesis, + const std::size_t first_row) { + with_team(&GeneralWorkspace::bind_edge, faces, clip, totals, synthesis, first_row); +} +CrossedBytes GeneralState::crossed_bytes() const noexcept { + return workspace_ ? workspace_->crossed_bytes() : CrossedBytes{}; +} +void GeneralState::bind_edge_lines(const EdgeLines& lines) { + with_team(&GeneralWorkspace::bind_edge_lines, lines); +} +EdgeNumbers GeneralState::edge_reading(const bool faces, const bool whole, + const bool totals, const bool strip, + const bool current) { + return with_team(&GeneralWorkspace::edge_reading, faces, whole, totals, strip, + current); +} +void GeneralState::bind_occupancy(const std::vector& buckets, + const std::size_t count) { + with_team(&GeneralWorkspace::bind_occupancy, buckets, count); +} +void GeneralState::promote_fsal() { with_team(&GeneralWorkspace::promote_fsal); } +void GeneralState::clear_fsal() { with_team(&GeneralWorkspace::clear_fsal); } +void GeneralState::set_absorber_quiet(const bool quiet) { + with_team(&GeneralWorkspace::set_absorber_quiet, quiet); +} +double GeneralState::quadrature_error() const { + return workspace_ ? workspace_->quadrature_error() + : Kokkos::Experimental::quiet_NaN_v; +} +double GeneralState::quadrature_peak() { + return workspace_ ? workspace_->quadrature_peak() + : Kokkos::Experimental::quiet_NaN_v; +} +std::string GeneralState::synthesis_provider() const { + return workspace_ ? workspace_->synthesis_provider() : "closed"; +} +std::string GeneralState::projection_provider() const { + return workspace_ ? workspace_->projection_provider() : "closed"; +} +std::string GeneralState::gemm_shape_policy() const { + return with_team([](const GeneralWorkspace& /*workspace*/) { + return std::string(GeneralWorkspace::gemm_shape_policy()); + }); +} +std::size_t GeneralState::response_team_size() const noexcept { + return workspace_ ? workspace_->response_team_size() : 0; +} +std::vector GeneralState::field_to_host() const { + return with_team( + [&](GeneralWorkspace& workspace) { return workspace.field_to_host(); }); +} +void GeneralState::set_crossing_workers(const std::size_t workers) noexcept { + if (workspace_) + workspace_->set_crossing_workers(workers); +} +void GeneralState::field_to_host(const std::span destination) const { + with_team([&](GeneralWorkspace& workspace) { workspace.field_to_host(destination); }); +} +const Complex* GeneralState::host_field() const { + return with_team(&GeneralWorkspace::host_field); +} +void GeneralState::count_to_host(const std::size_t bytes) const noexcept { + if (workspace_) + workspace_->count_to_host(bytes); +} +void GeneralState::stage_to_host(const std::size_t stage, + const std::span destination) const { + with_team([&](GeneralWorkspace& workspace) { + workspace.stage_to_host(stage, destination); + }); +} +std::vector GeneralState::stage_to_host(const std::size_t stage) const { + return with_team( + [&](GeneralWorkspace& workspace) { return workspace.stage_to_host(stage); }); +} +std::vector GeneralState::projected_to_host(const bool coarse) { + return with_team(&GeneralWorkspace::projected_to_host, coarse); +} +std::map GeneralState::dispersive_path() const { + return with_team(&GeneralWorkspace::dispersive_path); +} +std::map GeneralState::memory_ledger() const { + return with_team(&GeneralWorkspace::memory_ledger); +} +CpuSizing GeneralState::execution_sizing() const { + return with_team(&GeneralWorkspace::execution_sizing); +} +const CpuExecution& GeneralState::cpu_execution() const { + return with_team(&GeneralWorkspace::cpu_execution); +} +const FftPlanning& GeneralState::fft_planning() const { + return with_team([](const GeneralWorkspace& workspace) -> const FftPlanning& { + return workspace.problem_.fft_planning; + }); +} +double GeneralState::planning_seconds() const noexcept { + return workspace_ ? workspace_->planning_seconds() : 0.0; +} +double GeneralState::raman_planning_seconds() const noexcept { + return workspace_ ? workspace_->raman_planning_seconds() : 0.0; +} +std::string GeneralState::raman_fft_provider() const { + return workspace_ ? workspace_->raman_fft_provider() : "none"; +} +std::map GeneralState::raman_fft_tasks() const { + return workspace_ ? workspace_->raman_fft_tasks() : std::map{}; +} +std::size_t GeneralState::raman_kernel_transforms() const noexcept { + return workspace_ ? workspace_->raman_kernel_transforms() : 0; +} +bool GeneralState::fsal_ready() const noexcept { + return workspace_ && workspace_->fsal_ready(); +} +std::size_t GeneralState::rk_source_evaluations() const noexcept { + return workspace_ ? workspace_->rk_source_evaluations() : 0; +} +std::size_t GeneralState::axis_scans() const noexcept { + return workspace_ ? workspace_->axis_scans() : 0; +} +bool GeneralState::closed() const noexcept { return !workspace_; } +void GeneralState::close() noexcept { workspace_.reset(); } + +} // namespace solprop_native diff --git a/native/src/general/statistics.cpp b/native/src/general/statistics.cpp new file mode 100644 index 000000000..9b33cfd7e --- /dev/null +++ b/native/src/general/statistics.cpp @@ -0,0 +1,2484 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +// The general workspace's statistics and its boundary monitor's readings. + +#include "workspace.hpp" + +namespace solprop_native { + +void GeneralWorkspace::fill_statistics_time(const bool windowed, const double lower, + const double upper, const bool weighted) { + if (omega_.empty()) + throw std::logic_error("native statistics need the coarse frequency axis"); + graph::deep_copy(execution_space_, statistics_spectrum_, Complex{}); + const auto current = current_; + const auto spectrum = statistics_spectrum_; + const auto omega = omega_; + const auto power_weight = statistics_power_weight_; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const bool envelope = problem_.dims.family == FieldFamily::envelope; + launch::parallel_for( + "solprop_native.statistics.prepare_analytic", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + if (windowed && !(omega(index) >= lower && omega(index) < upper)) + return; + const double analytic_scale = + envelope || index == 0 || index + 1 == n ? 1.0 : 2.0; + spectrum(column, index) = + analytic_scale / static_cast(n_t) * current(column, index); + if (weighted) + spectrum(column, index) *= power_weight(column, index); + }); + graph::execute(*statistics_inverse_plan_, statistics_spectrum_, statistics_time_, + KokkosFFT::Normalization::none); +} + +void GeneralWorkspace::fill_statistics_pair(const bool windowed, const double lower, + const double upper) { + if (!statistics_unit_weight_) { + fill_statistics_time(windowed, lower, upper, true); + graph::deep_copy(execution_space_, statistics_weighted_time_, statistics_time_); + } + fill_statistics_time(windowed, lower, upper); +} + +void GeneralWorkspace::form_statistics_power() { + const auto time_field = statistics_time_; + const auto weighted_field = statistics_weighted_time_; + const bool unit_weight = statistics_unit_weight_; + const auto power = statistics_power_; + const auto n_pol = problem_.dims.n_pol_state; + launch::parallel_for( + "solprop_native.statistics.power", + Range2(execution_space_, {0, 0}, {problem_.dims.n_modes, problem_.dims.n_t}), + KOKKOS_LAMBDA(const std::size_t mode, const std::size_t time) { + double sum = 0.0; + for (std::size_t pol = 0; pol < n_pol; ++pol) + sum += unit_weight ? squared_abs(time_field(mode * n_pol + pol, time)) + : (time_field(mode * n_pol + pol, time) * + Kokkos::conj(weighted_field(mode * n_pol + pol, time))) + .real(); + power(mode, time) = sum; + }); +} + +void GeneralWorkspace::form_statistics_total_power() { + const auto power = statistics_power_; + const auto total = statistics_total_power_; + const auto modes = problem_.dims.n_modes; + launch::parallel_for( + "solprop_native.statistics.total_power", + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_t), + KOKKOS_LAMBDA(const std::size_t time) { + double sum = 0.0; + for (std::size_t mode = 0; mode < modes; ++mode) + sum += power(mode, time); + total(time) = sum; + }); +} + +namespace { + +// The offset of a statistic not asked for. +constexpr auto absent = std::numeric_limits::max(); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +using StatisticsTeamPolicy = Kokkos::TeamPolicy; +using StatisticsTeamMember = StatisticsTeamPolicy::member_type; +#endif + +} // namespace + +std::vector GeneralWorkspace::statistics( + const std::uint64_t flags, const std::vector& windows, + const std::vector& on_axis, const double density, const double taper, + const double preionisation, const double intensity_factor, + const std::vector& channel_preionisation, + const std::vector& power_weight) { + const auto& selected_weight = + power_weight.empty() ? problem_.grid.power_weight : power_weight; + statistics_unit_weight_ = + std::ranges::all_of(selected_weight, [](double value) { return value == 1.0; }); + if (!selected_weight.empty()) { + require_extent(selected_weight.size(), statistics_power_weight_.size(), + "statistics power weight"); + auto host_weight = Kokkos::create_mirror_view(statistics_power_weight_); + for (std::size_t column = 0; column < problem_.n_columns(); ++column) + for (std::size_t index = 0; index < problem_.dims.n_omega; ++index) { + const double value = selected_weight[column * problem_.dims.n_omega + index]; + if (!(value > 0.0) || !std::isfinite(value)) + throw std::invalid_argument( + "statistics power weight must be positive and finite"); + host_weight(column, index) = value; + } + graph::deep_copy(execution_space_, statistics_power_weight_, host_weight); + } else { + graph::deep_copy(execution_space_, statistics_power_weight_, 1.0); + } + const auto power_weights = statistics_power_weight_; + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + if (windows.size() % 2 != 0) + throw std::invalid_argument("statistics windows must contain lower/upper pairs"); + const auto requested = [flags](const StatisticsFlag flag) { + return (flags & static_cast(flag)) != 0U; + }; + // The temporal statistics reconstruct whole columns; on free geometry their + // buffers hold one node chunk, and the reference refuses these names on any + // spatial problem, so the adapter never asks. Refuse here as well rather than + // write past the chunk. + // The modal power weight has no rows on a spatial problem, radial included, and + // the energy kernels below read it for every column. + if ((problem_.separable_geometry() || statistics_power_weight_.extent(0) == 0) && + (requested(statistics_omega0) || requested(statistics_energy) || + requested(statistics_energy_window) || requested(statistics_peak_power) || + requested(statistics_peak_power_window) || + requested(statistics_peak_power_total) || requested(statistics_fwhm))) + throw std::invalid_argument( + "temporal statistics are not spatial saved-field statistics"); + const auto n_windows = windows.size() / 2; + std::size_t packed_size = 0; + const auto reserve = [&packed_size, requested](const StatisticsFlag flag, + const std::size_t count) { + if (!requested(flag)) + return absent; + const auto offset = packed_size; + packed_size = checked_sum(packed_size, count, "statistics packed output"); + return offset; + }; + const auto omega0_offset = reserve(statistics_omega0, problem_.n_columns()); + const auto energy_offset = reserve(statistics_energy, problem_.n_columns()); + const auto energy_window_offset = + reserve(statistics_energy_window, checked_product(n_windows, problem_.n_columns(), + "windowed-energy output")); + const auto peak_offset = reserve(statistics_peak_power, problem_.dims.n_modes); + const auto peak_window_offset = reserve( + statistics_peak_power_window, + checked_product(n_windows, problem_.dims.n_modes, "windowed-peak output")); + const auto total_peak_offset = reserve(statistics_peak_power_total, 1); + const auto density_offset = reserve(statistics_electron_density, 1); + const auto rate_offset = reserve(statistics_peak_ionisation_rate, 1); + const auto intensity_offset = reserve(statistics_peak_intensity, 1); + const auto quadrature_offset = reserve(statistics_quadrature_error, 1); + const auto fwhm_offset = reserve(statistics_fwhm, problem_.dims.n_t); + const auto populations_offset = reserve( + statistics_populations, + checked_product(2, + checked_product(problem_.plasma.n_channels, problem_.dims.n_nodes, + "population output"), + "population output")); + // solprop SEP 18's run diagnostics, packed last: six maxima over every node. + const auto diagnostics_offset = + reserve(statistics_plasma_integration, plasma_diagnostic_parts); + const bool diagnose = diagnostics_offset != absent; + if (populations_offset != absent || (diagnose && problem_.plasma.n_channels > 1)) { + require_extent(channel_preionisation.size(), problem_.plasma.n_channels, + "statistics channel preionisation"); + for (double value : channel_preionisation) + if (!std::isfinite(value) || value < 0.0 || value > 1.0) + throw std::invalid_argument( + "statistics channel preionisation must be in [0, 1]"); + auto initial = Kokkos::create_mirror_view(stage_channel_preionisation_); + for (std::size_t channel = 0; channel < problem_.plasma.n_channels; ++channel) + initial(0, channel) = channel_preionisation[channel]; + graph::deep_copy(execution_space_, stage_channel_preionisation_, initial); + } + + if (statistics_packed_.size() < packed_size) { + execution_space_.fence("solprop_native statistics output resize"); + statistics_packed_ = RealView("solprop_native.statistics.packed", packed_size); + } + const auto packed = statistics_packed_; + if (packed_size != 0) + graph::deep_copy(execution_space_, packed, + Kokkos::Experimental::quiet_NaN_v); + + pack_energy(packed, power_weights, omega0_offset, energy_offset); + statistics_window_energy(packed, power_weights, n_windows, energy_window_offset, + windows); + const bool needs_base_power = peak_offset != absent || total_peak_offset != absent || + intensity_offset != absent || fwhm_offset != absent; + if (needs_base_power) { + fill_statistics_pair(false, 0.0, 0.0); + form_statistics_power(); + } + if (peak_offset != absent) { + const auto power = statistics_power_; + const auto n_t = problem_.dims.n_t; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const StatisticsTeamPolicy policy(execution_space_, + static_cast(problem_.dims.n_modes), + SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); + launch::parallel_for( + "solprop_native.statistics.device_pack_peak_power", policy, + KOKKOS_LAMBDA(const StatisticsTeamMember& team) { + const auto mode = static_cast(team.league_rank()); + double peak = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n_t)), + [&](const int time_value, double& update) { + const auto time = static_cast(time_value); + if (power(mode, time) > update) + update = power(mode, time); + }, + Kokkos::Max(peak)); + Kokkos::single(Kokkos::PerTeam(team), + [&]() { packed(peak_offset + mode) = peak; }); + }); +#else + launch::parallel_for( + "solprop_native.statistics.pack_peak_power", + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_modes), + KOKKOS_LAMBDA(const std::size_t mode) { + double peak = 0.0; + for (std::size_t time = 0; time < n_t; ++time) + if (power(mode, time) > peak) + peak = power(mode, time); + packed(peak_offset + mode) = peak; + }); +#endif + } + if (total_peak_offset != absent || fwhm_offset != absent) { + form_statistics_total_power(); + if (total_peak_offset != absent) { + const auto total = statistics_total_power_; + const auto n_t = problem_.dims.n_t; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const StatisticsTeamPolicy policy(execution_space_, 1, + SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); + launch::parallel_for( + "solprop_native.statistics.device_pack_peak_power_total", policy, + KOKKOS_LAMBDA(const StatisticsTeamMember& team) { + double peak = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n_t)), + [&](const int time_value, double& update) { + const auto time = static_cast(time_value); + if (total(time) > update) + update = total(time); + }, + Kokkos::Max(peak)); + Kokkos::single(Kokkos::PerTeam(team), + [&]() { packed(total_peak_offset) = peak; }); + }); +#else + launch::parallel_for( + "solprop_native.statistics.pack_peak_power_total", + Kokkos::RangePolicy(execution_space_, 0, 1), + KOKKOS_LAMBDA(const std::size_t) { + double peak = 0.0; + for (std::size_t time = 0; time < n_t; ++time) + if (total(time) > peak) + peak = total(time); + packed(total_peak_offset) = peak; + }); +#endif + } + if (fwhm_offset != absent) { + const auto destination = Kokkos::subview( + packed, std::make_pair(fwhm_offset, fwhm_offset + problem_.dims.n_t)); + graph::deep_copy(execution_space_, destination, statistics_total_power_); + } + } + statistics_intensity(packed, intensity_offset, on_axis, intensity_factor); + if (peak_window_offset != absent) { + for (std::size_t window = 0; window < n_windows; ++window) { + fill_statistics_pair(true, windows[2 * window], windows[2 * window + 1]); + form_statistics_power(); + const auto power = statistics_power_; + const auto n_t = problem_.dims.n_t; + const auto modes = problem_.dims.n_modes; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const StatisticsTeamPolicy policy(execution_space_, static_cast(modes), + SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); + launch::parallel_for( + "solprop_native.statistics.device_pack_peak_power_window", policy, + KOKKOS_LAMBDA(const StatisticsTeamMember& team) { + const auto mode = static_cast(team.league_rank()); + double peak = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n_t)), + [&](const int time_value, double& update) { + const auto time = static_cast(time_value); + if (power(mode, time) > update) + update = power(mode, time); + }, + Kokkos::Max(peak)); + Kokkos::single(Kokkos::PerTeam(team), [&]() { + packed(peak_window_offset + window * modes + mode) = peak; + }); + }); +#else + launch::parallel_for( + "solprop_native.statistics.pack_peak_power_window", + Kokkos::RangePolicy(execution_space_, 0, modes), + KOKKOS_LAMBDA(const std::size_t mode) { + double peak = 0.0; + for (std::size_t time = 0; time < n_t; ++time) + if (power(mode, time) > peak) + peak = power(mode, time); + packed(peak_window_offset + window * modes + mode) = peak; + }); +#endif + } + } + + statistics_plasma(packed, density_offset, rate_offset, populations_offset, + diagnostics_offset, diagnose, density, taper, preionisation); + if (quadrature_offset != absent) { + const double value = quadrature_error(); + launch::parallel_for( + "solprop_native.statistics.pack_quadrature", + Kokkos::RangePolicy(execution_space_, 0, 1), + KOKKOS_LAMBDA(const std::size_t) { packed(quadrature_offset) = value; }); + } + + if (packed_size == 0) + return {}; + const auto active = + Kokkos::subview(packed, std::make_pair(0, packed_size)); + auto host = Kokkos::create_mirror_view(active); + graph::deep_copy(execution_space_, host, active); + execution_space_.fence("solprop_native statistics packed boundary"); + std::vector result(packed_size); + for (std::size_t index = 0; index < packed_size; ++index) + result[index] = host(index); + return result; +} + +// The columns' centre frequency and energy (`omega0`, `energy`). +void GeneralWorkspace::pack_energy(const RealView packed, const RealView2 power_weights, + const std::size_t omega0_offset, + const std::size_t energy_offset) { + if (omega0_offset != absent || energy_offset != absent) { + if (omega_.empty() || !(problem_.grid.dt > 0.0)) + throw std::logic_error("native statistics need the coarse frequency axis and dt"); + const auto current = current_; + const auto omega = omega_; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const auto dt = problem_.grid.dt; + const bool envelope = problem_.dims.family == FieldFamily::envelope; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const StatisticsTeamPolicy policy(execution_space_, + static_cast(problem_.n_columns()), + SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); + launch::parallel_for( + "solprop_native.statistics.device_pack_spectral", policy, + KOKKOS_LAMBDA(const StatisticsTeamMember& team) { + const auto column = static_cast(team.league_rank()); + double density_sum = 0.0; + double weighted_sum = 0.0; + double energy_sum = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n)), + [&](const int index_value, double& update) { + const auto index = static_cast(index_value); + update += (squared_abs(current(column, index)) * + power_weights(column, index)); + }, + density_sum); + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n)), + [&](const int index_value, double& update) { + const auto index = static_cast(index_value); + update += omega(index) * (squared_abs(current(column, index)) * + power_weights(column, index)); + }, + weighted_sum); + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n)), + [&](const int index_value, double& update) { + const auto index = static_cast(index_value); + const double weight = + envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; + update += weight * (squared_abs(current(column, index)) * + power_weights(column, index)); + }, + energy_sum); + Kokkos::single(Kokkos::PerTeam(team), [&]() { + if (omega0_offset != absent) + packed(omega0_offset + column) = + density_sum > 0.0 ? weighted_sum / density_sum + : Kokkos::Experimental::quiet_NaN_v; + if (energy_offset != absent) + packed(energy_offset + column) = + dt / static_cast(n_t) * energy_sum; + }); + }); +#else + launch::parallel_for( + "solprop_native.statistics.pack_spectral", + Kokkos::RangePolicy(execution_space_, 0, problem_.n_columns()), + KOKKOS_LAMBDA(const std::size_t column) { + double density_sum = 0.0; + double weighted_sum = 0.0; + double energy_sum = 0.0; + for (std::size_t index = 0; index < n; ++index) { + const double power = + (squared_abs(current(column, index)) * power_weights(column, index)); + density_sum += power; + weighted_sum += omega(index) * power; + const double weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; + energy_sum += weight * power; + } + if (omega0_offset != absent) + packed(omega0_offset + column) = + density_sum > 0.0 ? weighted_sum / density_sum + : Kokkos::Experimental::quiet_NaN_v; + if (energy_offset != absent) + packed(energy_offset + column) = dt / static_cast(n_t) * energy_sum; + }); +#endif + } +} + +// Each column's energy within each time window (`energy_window`). +void GeneralWorkspace::statistics_window_energy(const RealView packed, + const RealView2 power_weights, + const std::size_t n_windows, + const std::size_t energy_window_offset, + const std::vector& windows) { + if (energy_window_offset != absent) { + if (omega_.empty() || !(problem_.grid.dt > 0.0)) + throw std::logic_error("native statistics need the coarse frequency axis and dt"); + const auto current = current_; + const auto omega = omega_; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const auto dt = problem_.grid.dt; + const auto columns = problem_.n_columns(); + const bool envelope = problem_.dims.family == FieldFamily::envelope; + for (std::size_t window = 0; window < n_windows; ++window) { + const double lower = windows[2 * window]; + const double upper = windows[2 * window + 1]; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const StatisticsTeamPolicy policy(execution_space_, static_cast(columns), + SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); + launch::parallel_for( + "solprop_native.statistics.device_pack_energy_window", policy, + KOKKOS_LAMBDA(const StatisticsTeamMember& team) { + const auto column = static_cast(team.league_rank()); + double sum = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n)), + [&](const int index_value, double& update) { + const auto index = static_cast(index_value); + if (!(omega(index) >= lower && omega(index) < upper)) + return; + const double weight = + envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; + update += weight * (squared_abs(current(column, index)) * + power_weights(column, index)); + }, + sum); + Kokkos::single(Kokkos::PerTeam(team), [&]() { + packed(energy_window_offset + window * columns + column) = + dt / static_cast(n_t) * sum; + }); + }); +#else + launch::parallel_for( + "solprop_native.statistics.pack_energy_window", + Kokkos::RangePolicy(execution_space_, 0, columns), + KOKKOS_LAMBDA(const std::size_t column) { + double sum = 0.0; + for (std::size_t index = 0; index < n; ++index) { + if (!(omega(index) >= lower && omega(index) < upper)) + continue; + const double weight = + envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; + sum += weight * (squared_abs(current(column, index)) * + power_weights(column, index)); + } + packed(energy_window_offset + window * columns + column) = + dt / static_cast(n_t) * sum; + }); +#endif + } + } +} + +// The peak on-axis intensity (`peak_intensity`). +void GeneralWorkspace::statistics_intensity(const RealView packed, + const std::size_t intensity_offset, + const std::vector& on_axis, + const double intensity_factor) { + if (intensity_offset != absent) { + if (on_axis.empty() || on_axis.size() % problem_.dims.n_modes != 0 || + on_axis.size() / problem_.dims.n_modes > 2) + throw std::invalid_argument( + "on-axis synthesis must have one or two rows and n_modes columns"); + graph::deep_copy(execution_space_, statistics_on_axis_, 0.0); + const auto destination = Kokkos::subview( + statistics_on_axis_, std::make_pair(0, on_axis.size())); + copy_vector(execution_space_, destination, on_axis); + const auto matrix = statistics_on_axis_; + const auto field = statistics_time_; + const auto weighted_field = statistics_weighted_time_; + const bool unit_weight = statistics_unit_weight_; + const auto rows = on_axis.size() / problem_.dims.n_modes; + const auto modes = problem_.dims.n_modes; + const auto n_pol = problem_.dims.n_pol_state; + const auto n_t = problem_.dims.n_t; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const StatisticsTeamPolicy policy(execution_space_, 1, + SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); + launch::parallel_for( + "solprop_native.statistics.device_pack_peak_intensity", policy, + KOKKOS_LAMBDA(const StatisticsTeamMember& team) { + double peak = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n_t)), + [&](const int time_value, double& update) { + const auto time = static_cast(time_value); + double power = 0.0; + for (std::size_t row = 0; row < rows; ++row) { + for (std::size_t pol = 0; pol < n_pol; ++pol) { + Complex value{}, weighted_value{}; + for (std::size_t mode = 0; mode < modes; ++mode) { + value += + matrix(row * modes + mode) * field(mode * n_pol + pol, time); + if (!unit_weight) + weighted_value += matrix(row * modes + mode) * + weighted_field(mode * n_pol + pol, time); + } + power += unit_weight + ? squared_abs(value) + : (value * Kokkos::conj(weighted_value)).real(); + } + } + if (power > update) + update = power; + }, + Kokkos::Max(peak)); + Kokkos::single(Kokkos::PerTeam(team), + [&]() { packed(intensity_offset) = intensity_factor * peak; }); + }); +#else + launch::parallel_for( + "solprop_native.statistics.pack_peak_intensity", + Kokkos::RangePolicy(execution_space_, 0, 1), + KOKKOS_LAMBDA(const std::size_t) { + double peak = 0.0; + for (std::size_t time = 0; time < n_t; ++time) { + double power = 0.0; + for (std::size_t row = 0; row < rows; ++row) { + for (std::size_t pol = 0; pol < n_pol; ++pol) { + Complex value{}; + Complex weighted_value{}; + for (std::size_t mode = 0; mode < modes; ++mode) { + value += matrix(row * modes + mode) * field(mode * n_pol + pol, time); + if (!unit_weight) + weighted_value += matrix(row * modes + mode) * + weighted_field(mode * n_pol + pol, time); + } + power += unit_weight ? squared_abs(value) + : (value * Kokkos::conj(weighted_value)).real(); + } + } + if (power > peak) + peak = power; + } + packed(intensity_offset) = intensity_factor * peak; + }); +#endif + } +} + +// The plasma statistics: the electron density, the peak ionisation rate, the +// populations and the run diagnostics. +void GeneralWorkspace::statistics_plasma( + const RealView packed, const std::size_t density_offset, + const std::size_t rate_offset, const std::size_t populations_offset, + const std::size_t diagnostics_offset, const bool diagnose, const double density, + const double taper, const double preionisation) { + if (density_offset != absent || rate_offset != absent || + populations_offset != absent || diagnose) { + if (!problem_.plasma.enabled) + throw std::logic_error("this problem declares no compiled plasma response"); + if (diagnose && problem_.plasma.envelope) + throw std::invalid_argument("the plasma integration diagnostics read the " + "field-grid plasma column, and this problem " + "carries none"); + RealView2 source = stash_current_; + // The diagnostics rescan the plane, and the plasma statistics then read the scan. + const bool source_is_fraction = stash_is_current_ && !diagnose; + if (!source_is_fraction) { + source = plasma_statistics_scratch_; + if (problem_.separable_geometry()) { + separable_load_current(); + for (std::size_t unit = 0; unit < transverse_units(); + unit += transverse_units_per_chunk()) { + const auto chunk = transverse_chunk(unit); + separable_pack_chunk(chunk, true); + separable_to_time(chunk); + separable_nodes(chunk); + scan_plasma_statistics(chunk.first_node, chunk.nodes, source, diagnose, + preionisation); + } + } else { + statistics_modal_time(); + graph::deep_copy(execution_space_, stage_taper_, taper); + scale_transform(problem_.stepper.n_stages - 1, taper); + for (std::size_t first = 0; first < problem_.dims.n_nodes; + first += problem_.dims.node_chunk) { + const auto count = + std::min(problem_.dims.node_chunk, problem_.dims.n_nodes - first); + synthesise(first, count); + scan_plasma_statistics(first, count, source, diagnose, preionisation); + } + } + } + if (populations_offset != absent) { + const bool envelope = problem_.plasma.envelope; + const auto channels = problem_.plasma.n_channels; + const auto initial = stage_channel_preionisation_; + launch::parallel_for( + "solprop_native.statistics.pack_populations", + Range2(execution_space_, {0, 0}, {problem_.dims.n_nodes, channels}), + KOKKOS_LAMBDA(const std::size_t node, const std::size_t channel) { + const auto offset = populations_offset + 2 * (node * channels + channel); + packed(offset) = + source_is_fraction ? source(node, 2 * channel) + : envelope + ? envelope_fraction(source(node, 2 * channel), initial(0, channel)) + : ionised_fraction(source(node, 2 * channel), initial(0, channel)); + packed(offset + 1) = source(node, 2 * channel + 1); + }); + } + const auto nodes = problem_.dims.n_nodes; + launch::parallel_for( + "solprop_native.statistics.pack_plasma", + Kokkos::RangePolicy(execution_space_, 0, 1), + KOKKOS_LAMBDA(const std::size_t) { + std::size_t densest = 0; + double largest_fraction = -1.0; + for (std::size_t node = 0; node < nodes; ++node) { + const double fraction = + source_is_fraction ? source(node, 0) + : ionised_fraction(source(node, 0), preionisation); + if (fraction > largest_fraction) { + densest = node; + largest_fraction = fraction; + } + } + if (density_offset != absent) + packed(density_offset) = density * largest_fraction; + if (rate_offset != absent) + packed(rate_offset) = source(densest, 1); + }); + if (diagnose) { + // Each part is a maximum, so the nodes combine in any order. + const auto parts = plasma_diagnostics_; + launch::parallel_for( + "solprop_native.statistics.pack_plasma_integration", + Kokkos::RangePolicy(execution_space_, 0, + plasma_diagnostic_parts), + KOKKOS_LAMBDA(const std::size_t part) { + double largest = 0.0; + for (std::size_t node = 0; node < nodes; ++node) + largest = Kokkos::max(largest, parts(node, part)); + packed(diagnostics_offset + part) = largest; + }); + } + } +} + +std::vector GeneralWorkspace::whitened_statistics( + const std::vector& root, const std::uint64_t flags, + const std::vector& windows, const double intensity_factor, + const bool spectral) { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + constexpr std::uint64_t admitted = statistics_omega0 | statistics_energy | + statistics_peak_power_total | statistics_fwhm; + if ((flags & ~admitted) != 0U) + throw std::invalid_argument("a general modal basis's statistics are omega0, the " + "energy, the peak total power and the widths"); + const auto columns = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + const auto modes = problem_.dims.n_modes; + const auto n_pol = problem_.dims.n_pol_state; + const bool whiten = !root.empty(); + if (whiten) { + require_extent( + root.size(), + checked_product(n, checked_product(modes, modes, "metric root"), "metric root"), + "metric root"); + if (statistics_whitened_.extent(0) != columns || + statistics_whitened_.extent(1) != n) + throw std::logic_error("whitened statistics are a general modal basis's"); + copy_vector(execution_space_, statistics_root_, root); + const auto field = current_; + const auto whitened = statistics_whitened_; + const auto matrix = statistics_root_; + // `c_mp(w) = sum_n R_mn(w) a_np(w)`, `numpy.einsum("kmn,npk->mpk", ...)`'s terms. + launch::parallel_for( + "solprop_native.statistics.whiten", + Range2(execution_space_, {0, 0}, {columns, n}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + const auto mode = column / n_pol; + const auto pol = column % n_pol; + Complex sum{}; + for (std::size_t other = 0; other < modes; ++other) + sum += matrix((index * modes + mode) * modes + other) * + field(other * n_pol + pol, index); + whitened(column, index) = sum; + }); + } + const auto sums_size = columns + (spectral ? n : 0); + if (statistics_whitened_sums_.size() < sums_size) + throw std::logic_error("whitened statistics are a general modal basis's"); + std::vector result; + { + // The statistics passes read `current_`: they read the whitened columns while + // this scope holds them there, and the state's own afterwards, even on a throw. + struct Swap { + ComplexView2& first; + ComplexView2& second; + bool active; + Swap(ComplexView2& a, ComplexView2& b, const bool on) + : first(a), second(b), active(on) { + if (active) + std::swap(first, second); + } + ~Swap() { + if (active) + std::swap(first, second); + } + Swap(const Swap&) = delete; + Swap& operator=(const Swap&) = delete; + Swap(Swap&&) = delete; + Swap& operator=(Swap&&) = delete; + } swap(current_, statistics_whitened_, whiten); + if (flags != 0U) + result = statistics(flags, windows, {}, 0.0, 1.0, 0.0, intensity_factor, {}, {}); + const auto field = current_; + const auto sums = statistics_whitened_sums_; + launch::parallel_for( + "solprop_native.statistics.whitened_column_sums", + Kokkos::RangePolicy(execution_space_, 0, columns), + KOKKOS_LAMBDA(const std::size_t column) { + double sum = 0.0; + for (std::size_t index = 0; index < n; ++index) + sum += squared_abs(field(column, index)); + sums(column) = sum; + }); + if (spectral) + launch::parallel_for( + "solprop_native.statistics.whitened_bin_sums", + Kokkos::RangePolicy(execution_space_, 0, n), + KOKKOS_LAMBDA(const std::size_t index) { + double sum = 0.0; + for (std::size_t column = 0; column < columns; ++column) + sum += squared_abs(field(column, index)); + sums(columns + index) = sum; + }); + } + const auto active = Kokkos::subview(statistics_whitened_sums_, + std::make_pair(0, sums_size)); + auto host = Kokkos::create_mirror_view(active); + graph::deep_copy(execution_space_, host, active); + execution_space_.fence("solprop_native whitened statistics boundary"); + result.reserve(result.size() + sums_size); + for (std::size_t index = 0; index < sums_size; ++index) + result.push_back(host(index)); + return result; +} + +// solprop SEP 34: the buffers of the magnetic closure's synthesis, allocated at +// construction when the problem's own request has a local flux statistic and here, +// outside any trial step, when a later request first needs them. +void GeneralWorkspace::require_flux_closure() { + const auto rows = statistics_time_.extent(0); + const bool time = statistics_weighted_time_.extent(0) != rows; + const bool magnitude = !problem_.separable_geometry() && + statistics_magnitude_.extent(0) != statistics_power_.extent(0); + const bool volume = problem_.separable_geometry() && flux_transverse_.empty(); + if (!time && !magnitude && !volume) + return; + execution_space_.fence("solprop_native flux closure allocation"); + if (time) + statistics_weighted_time_ = ComplexView2("solprop_native.statistics.weighted_time", + rows, problem_.dims.n_t); + if (magnitude) + statistics_magnitude_ = + RealView2("solprop_native.statistics.flux_magnitude", + statistics_power_.extent(0), statistics_power_.extent(1)); + if (volume) + flux_transverse_ = + ComplexView3("solprop_native.flux.closure_transverse", transverse_.extent(0), + transverse_.extent(1), transverse_.extent(2)); +} + +// The magnetic closure of every column in time (a radial run's, solprop SEP 34): the +// analytic spectrum times the conjugate admittance, inverted into the statistics' +// time rows and kept in `statistics_weighted_time_`. +void GeneralWorkspace::fill_statistics_closure() { + graph::deep_copy(execution_space_, statistics_spectrum_, Complex{}); + const auto current = current_; + const auto spectrum = statistics_spectrum_; + const auto omega = omega_; + const auto active = active_mask_; + const auto flux = flux_; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const auto n_pol = problem_.dims.n_pol_state; + const bool envelope = problem_.dims.family == FieldFamily::envelope; + launch::parallel_for( + "solprop_native.statistics.prepare_closure", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + const double analytic_scale = + envelope || index == 0 || index + 1 == n ? 1.0 : 2.0; + const Complex closing = + flux.closing(column / n_pol, index, active(index) != 0U, omega(index)); + spectrum(column, index) = analytic_scale / static_cast(n_t) * + (closing * current(column, index)); + }); + graph::execute(*statistics_inverse_plan_, statistics_spectrum_, statistics_time_, + KokkosFFT::Normalization::none); + graph::deep_copy(execution_space_, statistics_weighted_time_, statistics_time_); +} + +void GeneralWorkspace::separable_time_statistics(const std::size_t axis_peak, + const std::size_t peak, + const bool fluence, + const double intensity_factor) { + constexpr double lowest = std::numeric_limits::lowest(); + const auto current = current_; + const auto spectrum = statistics_spectrum_; + const auto field = statistics_time_; + const auto closure = statistics_weighted_time_; + const auto packed = statistics_packed_; + const auto axis_weights = spatial_on_axis_; + const auto scratch = transverse_; + const auto closure_scratch = flux_transverse_; + const auto power = statistics_power_; + const auto partials = spatial_statistics_partials_; + const auto synthesis = synthesis_; + const auto flux = flux_; + const auto active = active_mask_; + const auto omega = omega_; + const auto n_modes = problem_.dims.n_modes; + const auto n_nodes = problem_.dims.n_nodes; + const auto n_pol = problem_.dims.n_pol_state; + const auto layout = transverse_layout(); + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const bool envelope = problem_.dims.family == FieldFamily::envelope; + if (axis_peak != absent) { + // The axis's closure first, kept in the closure rows, then its field: each the + // coherent on-axis sum of the analytic spectrum (solprop SEP 34). On free space the + // far-field pass forms those sums in its team-local bands. A sum per bin over every + // column, as the slab's few columns take here, is a thread's serial walk across a + // whole state on a device. A plane's on-axis spectrum reads the same slots. + if (far_field_) + static_cast(far_field_->run(far_field_axis, {}, 2, {}, intensity_factor, + current, far_field_inputs())); + for (const bool closed : {true, false}) { + graph::deep_copy(execution_space_, spectrum, Complex{}); + if (far_field_) + far_field_->axis_spectrum(closed, spectrum); + else + launch::parallel_for( + "solprop_native.free.statistics_axis_input", + Range2(execution_space_, {0, 0}, {n_pol, n}), + KOKKOS_LAMBDA(const std::size_t pol, const std::size_t index) { + const bool live = active(index) != 0U; + Complex sum{}; + for (std::size_t mode = 0; mode < n_modes; ++mode) { + const Complex value = current(mode * n_pol + pol, index); + sum += axis_weights(mode) * + (closed ? flux.closing(mode, index, live, omega(index)) * value + : value); + } + const double scale = + (envelope || index == 0 || index + 1 == n ? 1.0 : 2.0) / + static_cast(n_t); + spectrum(pol, index) = scale * sum; + }); + graph::execute(*statistics_inverse_plan_, statistics_spectrum_, statistics_time_, + KokkosFFT::Normalization::none); + if (closed) + graph::deep_copy( + execution_space_, + Kokkos::subview(closure, std::make_pair(0, n_pol), + Kokkos::ALL), + Kokkos::subview(field, std::make_pair(0, n_pol), Kokkos::ALL)); + } + launch::parallel_for( + "solprop_native.free.statistics_axis_reduce", + Kokkos::RangePolicy(execution_space_, 0, 1), + KOKKOS_LAMBDA(const std::size_t) { + double maximum = lowest; + for (std::size_t time = 0; time < n_t; ++time) { + double value = 0.0; + for (std::size_t pol = 0; pol < n_pol; ++pol) + value += (field(pol, time) * Kokkos::conj(closure(pol, time))).real(); + maximum = Kokkos::max(maximum, value); + } + packed(axis_peak) = intensity_factor * maximum; + }); + } + if (peak == absent && !fluence) + return; + const auto transverse_count = + static_cast(problem_.slab_geometry() ? problem_.spatial.n_y : n_modes); + launch::parallel_for( + "solprop_native.free.statistics_transverse_input", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + const auto mode = column / n_pol; + const Complex value = current(column, index) / transverse_count; + const auto plane = index * n_pol + column % n_pol; + layout.cell(scratch, plane, mode) = value; + layout.cell(closure_scratch, plane, mode) = + flux.closing(mode, index, active(index) != 0U, omega(index)) * value; + }); + transverse_inverse(); + transverse_inverse(flux_transverse_); + if (fluence) + separable_fluence(intensity_factor); + if (peak == absent) + return; + const bool slab = problem_.slab_geometry(); + const auto n_x = problem_.spatial.n_x; + const auto n_p = problem_.spatial.n_p; + // The planar synthesis carries the temporal pad scale; the statistics read the + // coarse field, so it is divided out (solprop's `collect_spatial`). + const double crop_scale = + static_cast(problem_.dims.n_t) / static_cast(problem_.dims.n_to); + for (std::size_t unit = 0; unit < transverse_units(); + unit += transverse_units_per_chunk()) { + const auto chunk = transverse_chunk(unit); + const auto first = chunk.first_node; + const auto first_row = chunk.first_row; + const auto count = chunk.nodes; + // The chunk's closure first, kept in the closure rows, then its field. + for (const bool closed : {true, false}) { + const auto source = closed ? closure_scratch : scratch; + graph::deep_copy(execution_space_, spectrum, Complex{}); + launch::parallel_for( + "solprop_native.free.statistics_temporal_input", + Range2(execution_space_, {0, 0}, {count * n_pol, n}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { + const double scale = + (envelope || index == 0 || index + 1 == n ? 1.0 : 2.0) / + static_cast(n_t); + if (slab) { + // Node ix of the chunk's y row iy: the slab synthesis of that row. + const auto ix = row % n_x; + const auto rows = first_row + (row / n_x) * n_p; + Complex sum{}; + for (std::size_t channel = 0; channel < n_p; ++channel) + sum += + synthesis(ix, channel) * source(index, layout.first(rows + channel), + layout.second(rows + channel)); + spectrum(row, index) = scale * (crop_scale * sum); + } else { + const auto node = first + row / n_pol; + spectrum(row, index) = + scale * layout.cell(source, index * n_pol + row % n_pol, node); + } + }); + graph::execute(*statistics_inverse_plan_, statistics_spectrum_, statistics_time_, + KokkosFFT::Normalization::none); + if (closed) + graph::deep_copy( + execution_space_, + Kokkos::subview(closure, std::make_pair(0, count * n_pol), + Kokkos::ALL), + Kokkos::subview(field, std::make_pair(0, count * n_pol), + Kokkos::ALL)); + } + launch::parallel_for( + "solprop_native.free.statistics_intensity", + Range2(execution_space_, {0, 0}, {count, n_t}), + KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { + double value = 0.0; + for (std::size_t pol = 0; pol < n_pol; ++pol) + value += (field(node * n_pol + pol, time) * + Kokkos::conj(closure(node * n_pol + pol, time))) + .real(); + power(node, time) = intensity_factor * value; + }); + launch::parallel_for( + "solprop_native.free.statistics_node_peak", + Kokkos::RangePolicy(execution_space_, 0, count), + KOKKOS_LAMBDA(const std::size_t node) { + double maximum = lowest; + for (std::size_t time = 0; time < n_t; ++time) + maximum = Kokkos::max(maximum, power(node, time)); + partials(first + node, 1) = maximum; + }); + } + launch::parallel_reduce( + "solprop_native.free.statistics_peak", + Kokkos::RangePolicy(execution_space_, 0, n_nodes), + KOKKOS_LAMBDA(const std::size_t node, double& maximum) { + maximum = Kokkos::max(maximum, partials(node, 1)); + }, + Kokkos::Max(Kokkos::subview(packed, peak))); +} + +// Each node's signed fluence into `partials(node, 0)` and a bound on its magnitude into +// `partials(node, 2)`, from the physical spectra of the transverse volumes, with no +// time transform. The analytic signal's spectrum is `s_k a_k`, with `s_k` one or two +// over `n_t`, so Parseval's theorem over the `n_t` samples gives `sum_t Re(E(t) +// conj(G(t))) = (1 / n_t) sum_k w_k Re(a_k conj(b_k))`, with `w_k` one or four: exactly +// the time sum the reference takes. The same pairs bound that sum's magnitude: by +// Cauchy-Schwarz over time and polarisation, `sum_t |E(t)| |G(t)| <= (1 / n_t) +// sqrt(sum_k w_k |a_k|^2 sum_k w_k |b_k|^2)`, with equality where `G` is proportional +// to `E`, as in a beam whose admittance varies little across its band. So a rounding +// budget set from it is never below the reference's. +void GeneralWorkspace::separable_fluence(const double intensity_factor) { + const auto scratch = transverse_; + const auto closure_scratch = flux_transverse_; + const auto partials = spatial_statistics_partials_; + const PhysicalSpectrum spectrum{.layout = transverse_layout(), + .synthesis = synthesis_, + .n_x = problem_.spatial.n_x, + .n_p = problem_.spatial.n_p, + .crop_scale = static_cast(problem_.dims.n_t) / + static_cast(problem_.dims.n_to), + .slab = problem_.slab_geometry()}; + const auto n_nodes = problem_.dims.n_nodes; + const auto n_pol = problem_.dims.n_pol_state; + const auto n = problem_.dims.n_omega; + const bool envelope = problem_.dims.family == FieldFamily::envelope; + const double measure = + intensity_factor * problem_.grid.dt / static_cast(problem_.dims.n_t); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // One node per thread, whole planes per block: adjacent threads read adjacent cells. + launch::parallel_for( + "solprop_native.free.statistics_fluence", + Kokkos::RangePolicy(execution_space_, 0, n_nodes), + KOKKOS_LAMBDA(const std::size_t node) { + double sum = 0.0, electric_size = 0.0, magnetic_size = 0.0; + for (std::size_t index = 0; index < n; ++index) { + const double weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; + for (std::size_t pol = 0; pol < n_pol; ++pol) { + const Complex electric = spectrum.read(scratch, index * n_pol + pol, node); + const Complex magnetic = + spectrum.read(closure_scratch, index * n_pol + pol, node); + sum += weight * (electric.real() * magnetic.real() + + electric.imag() * magnetic.imag()); + electric_size += weight * squared_abs(electric); + magnetic_size += weight * squared_abs(magnetic); + } + } + partials(node, 0) = measure * sum; + partials(node, 2) = + measure * Kokkos::sqrt(electric_size) * Kokkos::sqrt(magnetic_size); + }); +#else + // A task takes a run of adjacent nodes and reads each plane's cells of the run in + // order: one node at a time would touch a new plane at every sample. + constexpr std::size_t run = 256; + const std::size_t runs = (n_nodes + run - 1) / run; + launch::parallel_for( + "solprop_native.free.statistics_fluence", + Kokkos::RangePolicy(execution_space_, 0, runs), + KOKKOS_LAMBDA(const std::size_t item) { + const std::size_t first = item * run; + const std::size_t count = n_nodes - first < run ? n_nodes - first : run; + double sum[run] = {}; + double electric_size[run] = {}; + double magnetic_size[run] = {}; + for (std::size_t index = 0; index < n; ++index) { + const double weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; + for (std::size_t pol = 0; pol < n_pol; ++pol) { + const auto plane = index * n_pol + pol; + for (std::size_t offset = 0; offset < count; ++offset) { + const Complex electric = spectrum.read(scratch, plane, first + offset); + const Complex magnetic = + spectrum.read(closure_scratch, plane, first + offset); + sum[offset] += weight * (electric.real() * magnetic.real() + + electric.imag() * magnetic.imag()); + electric_size[offset] += weight * squared_abs(electric); + magnetic_size[offset] += weight * squared_abs(magnetic); + } + } + } + for (std::size_t offset = 0; offset < count; ++offset) { + partials(first + offset, 0) = measure * sum[offset]; + partials(first + offset, 2) = measure * Kokkos::sqrt(electric_size[offset]) * + Kokkos::sqrt(magnetic_size[offset]); + } + }); +#endif +} + +// What a plane's widths read of its fluence (`separable_fluence`), each a reduction +// over the nodes, so that the profile itself stays on the device: `moments` takes the +// lowest value and its node, the largest magnitude, the largest value and the count of +// values that aren't finite and, when the plane has a positive value and none that +// isn't finite, the scaled profile's sum and index moments. That scaled profile is left +// in `partials(node, 0)` for `plane_fluence_trace`. Each reduction blocks: one plane's +// work. +void GeneralWorkspace::plane_fluence_moments(double (&moments)[plane_moment_slots]) { + const auto partials = spatial_statistics_partials_; + const auto n_nodes = problem_.dims.n_nodes; + const auto n_x = problem_.spatial.n_x; + const Kokkos::RangePolicy nodes(execution_space_, 0, n_nodes); + std::size_t nonfinite = 0; + launch::parallel_reduce( + "solprop_native.free.plane_fluence_finite", nodes, + KOKKOS_LAMBDA(const std::size_t node, std::size_t& update) { + if (!Kokkos::isfinite(partials(node, 0))) + ++update; + }, + Kokkos::Sum(nonfinite)); + Kokkos::MinLoc::value_type worst{}; + launch::parallel_reduce( + "solprop_native.free.plane_fluence_worst", nodes, + KOKKOS_LAMBDA(const std::size_t node, + Kokkos::MinLoc::value_type& update) { + if (partials(node, 0) < update.val) { + update.val = partials(node, 0); + update.loc = node; + } + }, + Kokkos::MinLoc(worst)); + double size = 0.0; + launch::parallel_reduce( + "solprop_native.free.plane_fluence_size", nodes, + KOKKOS_LAMBDA(const std::size_t node, double& update) { + update = Kokkos::max(update, partials(node, 2)); + }, + Kokkos::Max(size)); + double highest = 0.0; + launch::parallel_reduce( + "solprop_native.free.plane_fluence_highest", nodes, + KOKKOS_LAMBDA(const std::size_t node, double& update) { + update = Kokkos::max(update, partials(node, 0)); + }, + Kokkos::Max(highest)); + // A negative value within its budget is read as zero, as the reference reads it. + const double maximum = Kokkos::max(highest, 0.0); + FluenceMoments sums; + plane_fluence_held_ = nonfinite == 0 && maximum > 0.0; + if (plane_fluence_held_) + launch::parallel_reduce( + "solprop_native.free.plane_fluence_moments", nodes, + KOKKOS_LAMBDA(const std::size_t node, FluenceMoments& update) { + // Scaling cancels in the centroid and the widths and keeps them from + // overflowing on a bright beam. + const double sample = Kokkos::max(partials(node, 0), 0.0) / maximum; + partials(node, 0) = sample; + update.total += sample; + const std::size_t row = node / n_x; + update.x += sample * static_cast(node % n_x); + update.y += sample * static_cast(row); + }, + Kokkos::Sum(sums)); + moments[0] = worst.val; + moments[1] = static_cast(worst.loc); + moments[2] = size; + moments[3] = maximum; + moments[4] = static_cast(nonfinite); + moments[5] = sums.total; + moments[6] = sums.x; + moments[7] = sums.y; +} + +// The second moment and the azimuthal average of a plane's scaled fluence, left by a +// call that asked for its moments. The rule is solprop's +// (`solprop.spatial.cartesian_fluence_widths`), which owns the rings and hands them +// over: each ring samples the bilinear interpolant at `cosines.size()` angles about the +// centroid, with the centre in metres from the axes' first samples, so that only the +// trace crosses, not the profile. Returned: the second-moment sum, then the trace. +std::vector GeneralWorkspace::plane_fluence_trace( + const double center_x, const double center_y, const double dx, const double dy, + const double extent, const double spacing, const std::size_t n_radii, + const std::vector& cosines, const std::vector& sines) { + if (!plane_fluence_held_) + throw std::logic_error("a plane's fluence trace follows the call that holds it"); + plane_fluence_held_ = false; + if (cosines.size() != sines.size() || (n_radii != 0 && cosines.empty()) || + !(dx > 0.0) || !(dy > 0.0) || !(extent > 0.0) || + (n_radii != 0 && !(spacing > 0.0))) + throw std::invalid_argument("invalid fluence rings"); + const auto n_angles = cosines.size(); + const auto packed_size = + checked_sum(checked_sum(1, n_radii, "fluence trace"), + checked_product(2, n_angles, "fluence rings"), "fluence trace"); + if (statistics_packed_.size() < packed_size) { + execution_space_.fence("solprop_native spatial statistics output resize"); + statistics_packed_ = RealView("solprop_native.statistics.packed", packed_size); + } + const auto packed = statistics_packed_; + const auto samples = spatial_statistics_partials_; + const auto n_x = problem_.spatial.n_x; + const auto n_y = problem_.spatial.n_y; + const auto first_ring = n_radii + 1; + if (n_angles != 0) { + std::vector rings(cosines); + rings.insert(rings.end(), sines.begin(), sines.end()); + copy_vector(execution_space_, + Kokkos::subview(packed, std::make_pair(first_ring, packed_size)), + rings); + } + double second = 0.0; + launch::parallel_reduce( + "solprop_native.free.plane_fluence_second", + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_nodes), + KOKKOS_LAMBDA(const std::size_t node, double& update) { + const double x = (static_cast(node % n_x) * dx - center_x) / extent; + const std::size_t row = node / n_x; + const double y = (static_cast(row) * dy - center_y) / extent; + update += samples(node, 0) * (x * x + y * y); + }, + Kokkos::Sum(second)); + const auto last_x = static_cast(n_x - 1); + const auto last_y = static_cast(n_y - 1); + // One thread per ring: a ring's samples are few, and a thread's sum needs no + // reduction. + launch::parallel_for( + "solprop_native.free.plane_fluence_rings", + Kokkos::RangePolicy(execution_space_, 0, n_radii), + KOKKOS_LAMBDA(const std::size_t ring) { + const double distance = spacing * static_cast(ring); + double sum = 0.0; + for (std::size_t angle = 0; angle < n_angles; ++angle) { + const double horizontal = Kokkos::min( + Kokkos::max((center_x + distance * packed(first_ring + angle)) / dx, 0.0), + last_x); + const double vertical = Kokkos::min( + Kokkos::max( + (center_y + distance * packed(first_ring + n_angles + angle)) / dy, + 0.0), + last_y); + const auto ix = + Kokkos::min(static_cast(horizontal), n_x - std::size_t{2}); + const auto iy = + Kokkos::min(static_cast(vertical), n_y - std::size_t{2}); + const double fx = horizontal - static_cast(ix); + const double fy = vertical - static_cast(iy); + const auto low = iy * n_x + ix; + const auto high = low + n_x; + sum += + (1.0 - fy) * ((1.0 - fx) * samples(low, 0) + fx * samples(low + 1, 0)) + + fy * ((1.0 - fx) * samples(high, 0) + fx * samples(high + 1, 0)); + } + packed(1 + ring) = sum / static_cast(n_angles); + }); + const auto trace = + Kokkos::subview(packed, std::make_pair(std::size_t{1}, first_ring)); + auto host = Kokkos::create_mirror_view(trace); + graph::deep_copy(execution_space_, host, trace); + execution_space_.fence("solprop_native fluence trace boundary"); + std::vector result(first_ring); + result[0] = second; + for (std::size_t ring = 0; ring < n_radii; ++ring) + result[1 + ring] = host(ring); + return result; +} + +// The field's spectrum at one physical point, the matrix `(n_rows, n_modes)` contracted +// with the modal state: `(n_rows * n_pol, n_omega)` samples, each a real and an +// imaginary part in turn. The transitional session's on-axis statistics scan this row +// on the host (solprop SEP 19), so that it is all a plane's scan brings across; the +// native session scans it in place (`axis_populations`). +std::vector +GeneralWorkspace::axis_spectrum(const std::vector& matrix) { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + if (problem_.separable_geometry() || matrix.empty() || + matrix.size() % problem_.dims.n_modes != 0 || + matrix.size() / problem_.dims.n_modes > + statistics_on_axis_.size() / problem_.dims.n_modes) + throw std::invalid_argument( + "an on-axis row needs a modal state and at most two rows of n_modes columns"); + const auto n_rows = matrix.size() / problem_.dims.n_modes; + const auto n_pol = problem_.dims.n_pol_state; + const auto n_modes = problem_.dims.n_modes; + const auto n = problem_.dims.n_omega; + const auto columns = checked_product(n_rows, n_pol, "on-axis columns"); + const auto samples = checked_product(columns, n, "on-axis samples"); + const auto packed_size = checked_product(2, samples, "on-axis samples"); + if (statistics_packed_.size() < packed_size) { + execution_space_.fence("solprop_native statistics output resize"); + statistics_packed_ = RealView("solprop_native.statistics.packed", packed_size); + } + copy_vector(execution_space_, + Kokkos::subview(statistics_on_axis_, + std::make_pair(0, matrix.size())), + matrix); + const auto weights = statistics_on_axis_; + const auto packed = statistics_packed_; + const auto current = current_; + launch::parallel_for( + "solprop_native.statistics.axis_spectrum", + Range2(execution_space_, {0, 0}, {columns, n}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + const auto row = column / n_pol; + const auto pol = column % n_pol; + Complex sum{}; + for (std::size_t mode = 0; mode < n_modes; ++mode) + sum += weights(row * n_modes + mode) * current(mode * n_pol + pol, index); + const auto at = 2 * (column * n + index); + packed(at) = sum.real(); + packed(at + 1) = sum.imag(); + }); + const auto filled = + Kokkos::subview(packed, std::make_pair(0, packed_size)); + auto host = Kokkos::create_mirror_view(filled); + graph::deep_copy(execution_space_, host, filled); + execution_space_.fence("solprop_native on-axis row boundary"); + std::vector result(samples); + for (std::size_t index = 0; index < samples; ++index) + result[index] = Complex(host(2 * index), host(2 * index + 1)); + return result; +} + +// The modal state on the fine time grid, in `modal_real_` or `modal_complex_`: the +// coarse bins padded at `crop_index_` and transformed back, the node scans' own first +// pass (`statistics`); a retained synthesis transforms at its nodes instead. +void GeneralWorkspace::statistics_modal_time() { + graph::deep_copy(execution_space_, fine_spectrum_, Complex{}); + const auto current = current_; + const auto fine = fine_spectrum_; + const auto crop = crop_index_; + const auto inverse_n_to = 1.0 / static_cast(problem_.dims.n_to); + const auto n_to = problem_.dims.n_to; + const bool analytic = problem_.dims.family == FieldFamily::field_analytic; + launch::parallel_for( + "solprop_native.plasma.statistics_pad", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + const double analytic_scale = + analytic && index > 0 && 2 * crop(index) < n_to ? 2.0 : 1.0; + fine(column, crop(index)) = + analytic_scale * inverse_n_to * current(column, index); + }); + if (problem_.dims.family == FieldFamily::field_real) + graph::execute(*inverse_real_plan_, fine_spectrum_, modal_real_, + KokkosFFT::Normalization::none); + else if (!retained_synthesis_) + graph::execute(*inverse_complex_plan_, fine_spectrum_, modal_complex_, + KokkosFFT::Normalization::none); +} + +// solprop SEP 19's on-axis plasma populations, scanned natively (solprop SEP 47 phase +// 4): the modal state on the fine grid (`statistics_modal_time`), contracted with +// `matrix` -- one or two rows of `n_modes` columns, the pad scale and the taper in +// them, as `solprop.backends.populations.axis_row_matrix` forms them -- into the first +// node's buffer, and scanned there by the node scan itself, every channel with its +// tables and running-integral rule. Returns each channel's final fraction and peak +// rate, `(n_channels, 2)`. Two components on a one-component node buffer are the +// field's magnitude, `hypot(E_x, E_y)`, which is all a field grid's scan reads; an +// envelope's scan reads their polarisation ellipse, so they go to a row of their own +// and the envelope scan reads it as a node of two components +// (`solprop.backends.populations.envelope_populations` on the host's row). +std::vector +GeneralWorkspace::axis_populations(const std::vector& matrix, + const std::vector& channel_preionisation) { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + if (!problem_.plasma.enabled || problem_.plasma.n_channels == 0) + throw std::invalid_argument( + "the on-axis plasma statistics need a compiled plasma response"); + if (problem_.separable_geometry() || problem_.general.enabled || retained_synthesis_) + throw std::invalid_argument( + "the on-axis plasma statistics read a resolved modal state"); + ++axis_scans_; + const auto n_modes = problem_.dims.n_modes; + const auto n_pol_state = problem_.dims.n_pol_state; + const auto node_pol = problem_.effective_node_pol(); + if (matrix.empty() || matrix.size() % n_modes != 0 || matrix.size() / n_modes > 2) + throw std::invalid_argument( + "an on-axis row needs one or two rows of n_modes columns"); + auto rows = matrix.size() / n_modes; + // A y row of zeros on a one-component node scan is dropped: `hypot(E_x, 0)` is + // `|E_x|`, and an envelope's ellipse with no y axis is its x component's. + if (rows == 2 && n_pol_state == 1 && node_pol == 1 && + std::all_of(matrix.begin() + static_cast(n_modes), matrix.end(), + [](const double value) { return value == 0.0; })) + rows = 1; + const auto components = rows * n_pol_state; + const bool pair = components == 2 && node_pol == 1; + if (components != node_pol && !pair) + throw std::invalid_argument( + "an on-axis row has one component on each of the node scan's"); + // A pair on a field grid is combined into its magnitude in the node buffer; an + // envelope's is kept, x and y, in its own row. + const bool ellipse = pair && problem_.plasma.envelope; + const bool combine = pair && !ellipse; + if (ellipse && statistics_axis_row_.extent(0) == 0) { + execution_space_.fence("solprop_native on-axis row allocation"); + statistics_axis_row_ = + ComplexView2("solprop_native.statistics.axis_row", 2, problem_.dims.n_to); + } + require_extent(channel_preionisation.size(), problem_.plasma.n_channels, + "on-axis channel preionisation"); + for (const double value : channel_preionisation) + if (!std::isfinite(value) || value < 0.0 || value > 1.0) + throw std::invalid_argument("on-axis channel preionisation must be in [0, 1]"); + auto initial_host = Kokkos::create_mirror_view(stage_channel_preionisation_); + for (std::size_t channel = 0; channel < problem_.plasma.n_channels; ++channel) + initial_host(0, channel) = channel_preionisation[channel]; + graph::deep_copy(execution_space_, stage_channel_preionisation_, initial_host); + const std::vector held( + matrix.begin(), matrix.begin() + static_cast(rows * n_modes)); + copy_vector( + execution_space_, + Kokkos::subview(statistics_on_axis_, std::make_pair(0, held.size())), + held); + statistics_modal_time(); + const auto weights = statistics_on_axis_; + const auto modal_real = modal_real_; + const auto modal_complex = modal_complex_; + const auto node_real = node_real_; + const auto node_complex = node_complex_; + const auto axis_row = statistics_axis_row_; + const bool real_family = problem_.dims.family == FieldFamily::field_real; + const auto written = combine ? std::size_t{1} : components; + launch::parallel_for( + "solprop_native.statistics.axis_row", + Range2(execution_space_, {0, 0}, {written, problem_.dims.n_to}), + KOKKOS_LAMBDA(const std::size_t component, const std::size_t time) { + // Component `row * n_pol_state + pol`, `passes.synthesise`'s layout. + const auto sum_real = [&](const std::size_t row, const std::size_t pol) { + double value = 0.0; + for (std::size_t mode = 0; mode < n_modes; ++mode) + value += weights(row * n_modes + mode) * + modal_real(mode * n_pol_state + pol, time); + return value; + }; + const auto sum_complex = [&](const std::size_t row, const std::size_t pol) { + Complex value{}; + for (std::size_t mode = 0; mode < n_modes; ++mode) + value += weights(row * n_modes + mode) * + modal_complex(mode * n_pol_state + pol, time); + return value; + }; + const auto row = component / n_pol_state; + const auto pol = component % n_pol_state; + if (ellipse) { + axis_row(component, time) = sum_complex(row, pol); + } else if (combine) { + // Both rows on a scalar state: the magnitude the scan would form. + if (real_family) + node_real(0, time) = Kokkos::hypot(sum_real(0, 0), sum_real(1, 0)); + else + node_complex(0, time) = Complex( + Kokkos::hypot(sum_complex(0, 0).real(), sum_complex(1, 0).real()), 0.0); + } else if (real_family) { + node_real(component, time) = sum_real(row, pol); + } else { + node_complex(component, time) = sum_complex(row, pol); + } + }); + const auto source = plasma_statistics_scratch_; + if (ellipse) + for (std::size_t channel = 0; channel < problem_.plasma.n_channels; ++channel) + envelope_statistics(channel, 0, 1, source, statistics_axis_row_, 2); + else + scan_plasma_statistics(0, 1, source, false, channel_preionisation[0]); + const auto channels = problem_.plasma.n_channels; + const auto packed_size = 2 * channels; + if (statistics_packed_.size() < packed_size) { + execution_space_.fence("solprop_native statistics output resize"); + statistics_packed_ = RealView("solprop_native.statistics.packed", packed_size); + } + const auto packed = statistics_packed_; + const bool envelope = problem_.plasma.envelope; + const auto initial = stage_channel_preionisation_; + // `statistics`' own packing of a node's populations. + launch::parallel_for( + "solprop_native.statistics.pack_axis_populations", + Kokkos::RangePolicy(execution_space_, 0, channels), + KOKKOS_LAMBDA(const std::size_t channel) { + packed(2 * channel) = + envelope ? envelope_fraction(source(0, 2 * channel), initial(0, channel)) + : ionised_fraction(source(0, 2 * channel), initial(0, channel)); + packed(2 * channel + 1) = source(0, 2 * channel + 1); + }); + const auto filled = + Kokkos::subview(packed, std::make_pair(0, packed_size)); + auto host = Kokkos::create_mirror_view(filled); + graph::deep_copy(execution_space_, host, filled); + execution_space_.fence("solprop_native on-axis populations boundary"); + std::vector result(packed_size); + for (std::size_t index = 0; index < packed_size; ++index) + result[index] = host(index); + return result; +} + +std::vector GeneralWorkspace::aperture_statistics( + const std::uint64_t flags, const std::vector& apertures, const int exponent, + const std::vector& regions, const double intensity_factor) { + if (!initialized_ || !far_field_) + throw std::logic_error( + "far-field statistics need an initial field on a Cartesian or line grid"); + return far_field_->run(flags, apertures, exponent, regions, intensity_factor, + current_, far_field_inputs()); +} + +FarFieldPass::Inputs GeneralWorkspace::far_field_inputs() const { + return FarFieldPass::Inputs{.weights = spatial_spectral_weights_, + .on_axis = spatial_on_axis_, + .square = spatial_.square, + .square_low = spatial_.square_low, + .transverse = spatial_.transverse, + .transverse_low = spatial_.transverse_low, + .spectral_rate = spatial_.spectral_rate, + .user_rate = spatial_.user_rate, + .transverse_rate = spatial_.transverse_rate, + .omega = omega_, + .active = active_mask_, + .evanescent = spatial_.evanescent}; +} + +std::vector GeneralWorkspace::spatial_statistics(const std::uint64_t flags, + const double intensity_factor, + const double z) { + if (!initialized_ || spatial_spectral_weights_.empty()) + throw std::logic_error( + "spatial statistics need an initial field and spatial geometry"); + if (flux_.square.empty()) + throw std::logic_error("spatial statistics need the run's plane measure"); + if ((flags & ~std::uint64_t{63}) != 0 || !std::isfinite(intensity_factor) || + intensity_factor <= 0.0 || !std::isfinite(z)) + throw std::invalid_argument( + "invalid spatial statistic flags, intensity factor or position"); + if ((flags & 32U) != 0 && (!problem_.free_geometry() || problem_.slab_geometry() || + problem_.spatial.n_y < 2)) + throw std::invalid_argument("a plane's fluence moments need a Cartesian plane"); + constexpr double lowest = std::numeric_limits::lowest(); + std::size_t packed_size = 0; + const auto reserve = [flags, &packed_size](const std::uint64_t bit, + const std::size_t count) { + if ((flags & bit) == 0) + return absent; + const auto offset = packed_size; + packed_size = checked_sum(packed_size, count, "spatial statistics output"); + return offset; + }; + const auto energy_offset = reserve(1, 1); + const auto axis_peak_offset = reserve(2, 1); + const auto peak_offset = reserve(4, 1); + const auto spectrum_offset = reserve(8, problem_.dims.n_omega); + // Widths read the signed fluence at the nodes, the separately reconstructed axis, + // which only a radial width reads, and the largest magnitude, which fixes the + // fluence's rounding budget (solprop SEP 34). + const auto fluence_offset = + reserve(16, checked_sum(problem_.dims.n_nodes, 2, "spatial statistics output")); + // On a Cartesian plane the widths reduce here instead, and the moments cross. + const auto moments_offset = reserve(32, plane_moment_slots); + if (packed_size == 0) + return {}; + if (statistics_packed_.size() < packed_size) { + execution_space_.fence("solprop_native spatial statistics output resize"); + statistics_packed_ = RealView("solprop_native.statistics.packed", packed_size); + } + // Every field statistic is the axial flux of the medium at the plane z. + flux_at(z); + const bool local = (flags & (2U | 4U | 16U | 32U)) != 0; + if (local) + require_flux_closure(); + const auto packed = statistics_packed_; + const auto current = current_; + const auto spectral_weights = spatial_spectral_weights_; + const auto axis_weights = spatial_on_axis_; + const auto partials = spatial_statistics_partials_; + const auto flux = flux_; + const auto active = active_mask_; + const auto omega = omega_; + const auto n_modes = problem_.dims.n_modes; + const auto n_nodes = problem_.dims.n_nodes; + const auto n_pol = problem_.dims.n_pol_state; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const auto dt = problem_.grid.dt; + const bool envelope = problem_.dims.family == FieldFamily::envelope; + if (energy_offset != absent) { + launch::parallel_for( + "solprop_native.spatial.energy_partials", + Kokkos::RangePolicy(execution_space_, 0, n_modes), + KOKKOS_LAMBDA(const std::size_t mode) { + double sum = 0.0; + for (std::size_t index = 0; index < n; ++index) { + const double admittance = + flux.closing(mode, index, active(index) != 0U, omega(index)).real(); + if (admittance == 0.0) + continue; + double squared = 0.0; + for (std::size_t pol = 0; pol < n_pol; ++pol) + squared += squared_abs(current(mode * n_pol + pol, index)); + const auto weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; + sum += weight * admittance * squared; + } + partials(mode, 0) = intensity_factor * spectral_weights(mode) * dt / + static_cast(n_t) * sum; + }); + launch::parallel_reduce( + "solprop_native.spatial.pack_energy", + Kokkos::RangePolicy(execution_space_, 0, n_modes), + KOKKOS_LAMBDA(const std::size_t mode, double& sum) { + sum += partials(mode, 0); + }, + Kokkos::Sum(Kokkos::subview(packed, energy_offset))); + } + if (spectrum_offset != absent) { + // The coherent on-axis amplitude paired at each frequency with its closure. + launch::parallel_for( + "solprop_native.spatial.axis_spectrum", + Kokkos::RangePolicy(execution_space_, 0, n), + KOKKOS_LAMBDA(const std::size_t index) { + const bool live = active(index) != 0U; + double power = 0.0; + for (std::size_t pol = 0; pol < n_pol; ++pol) { + Complex electric{}; + Complex magnetic{}; + for (std::size_t mode = 0; mode < n_modes; ++mode) { + const Complex value = + axis_weights(mode) * current(mode * n_pol + pol, index); + electric += value; + magnetic += flux.closing(mode, index, live, omega(index)) * value; + } + power += (electric * Kokkos::conj(magnetic)).real(); + } + const auto weight = envelope || index == 0 || index + 1 == n ? 1.0 : 4.0; + packed(spectrum_offset + index) = + intensity_factor * dt * dt / + (2.0 * Kokkos::numbers::pi_v)*weight * power; + }); + } + double moments[plane_moment_slots] = {}; + if (problem_.separable_geometry()) { + separable_time_statistics(axis_peak_offset, peak_offset, + fluence_offset != absent || moments_offset != absent, + intensity_factor); + if (fluence_offset != absent) { + launch::parallel_for( + "solprop_native.free.fluence_rows", + Kokkos::RangePolicy(execution_space_, 0, n_nodes), + KOKKOS_LAMBDA(const std::size_t node) { + packed(fluence_offset + node) = partials(node, 0); + }); + launch::parallel_reduce( + "solprop_native.free.fluence_largest", + Kokkos::RangePolicy(execution_space_, 0, n_nodes), + KOKKOS_LAMBDA(const std::size_t node, double& maximum) { + maximum = Kokkos::max(maximum, partials(node, 2)); + }, + Kokkos::Max( + Kokkos::subview(packed, fluence_offset + n_nodes + 1))); + } + if (moments_offset != absent) + plane_fluence_moments(moments); + } else if (local) { + if (n_nodes != n_modes) + throw std::logic_error("radial statistics need one node per Hankel column"); + fill_statistics_closure(); + fill_statistics_time(false, 0.0, 0.0); + const auto field = statistics_time_; + const auto closure = statistics_weighted_time_; + const auto axis_power = statistics_total_power_; + if (axis_peak_offset != absent || fluence_offset != absent) { + launch::parallel_for( + "solprop_native.spatial.axis_intensity", + Kokkos::RangePolicy(execution_space_, 0, n_t), + KOKKOS_LAMBDA(const std::size_t time) { + double power = 0.0; + for (std::size_t pol = 0; pol < n_pol; ++pol) { + Complex electric{}; + Complex magnetic{}; + for (std::size_t mode = 0; mode < n_modes; ++mode) { + electric += axis_weights(mode) * field(mode * n_pol + pol, time); + magnetic += axis_weights(mode) * closure(mode * n_pol + pol, time); + } + power += (electric * Kokkos::conj(magnetic)).real(); + } + axis_power(time) = intensity_factor * power; + }); + launch::parallel_for( + "solprop_native.spatial.pack_axis", + Kokkos::RangePolicy(execution_space_, 0, 1), + KOKKOS_LAMBDA(const std::size_t) { + double peak = lowest; + double sum = 0.0; + for (std::size_t time = 0; time < n_t; ++time) { + peak = Kokkos::max(peak, axis_power(time)); + sum += axis_power(time); + } + if (axis_peak_offset != absent) + packed(axis_peak_offset) = peak; + if (fluence_offset != absent) + packed(fluence_offset + n_nodes) = dt * sum; + }); + } + if (peak_offset != absent || fluence_offset != absent) { + const auto power = statistics_power_; + const auto magnitude = statistics_magnitude_; + const auto synthesis = synthesis_; + const double crop_scale = static_cast(problem_.dims.n_t) / + static_cast(problem_.dims.n_to); + // Each node's field and closure, `S F` and `S G` over the modes, as GEMMs over + // interleaved real columns (each sample's real and imaginary parts; a mode's + // polarisations are adjacent rows): the field's into the spectrum buffer, free + // once both time fields are formed, then the closure's over the field, which + // nothing reads after its synthesis. Local statistics hold one node per mode, so + // each buffer has the rows. + using Interleaved = Kokkos::View>; + static_assert(sizeof(Complex) == 2 * sizeof(double)); + const std::size_t width = 2 * n_pol * n_t; + if (statistics_spectrum_.size() < n_nodes * n_pol * n_t || + field.size() < n_nodes * n_pol * n_t) + throw std::logic_error("radial statistics need a row of each buffer per node"); + const Interleaved weights(synthesis.data(), n_nodes, n_modes); + const Interleaved field_rows(reinterpret_cast(field.data()), n_modes, + width); + const Interleaved closure_rows(reinterpret_cast(closure.data()), n_modes, + width); + const Interleaved electric_rows( + reinterpret_cast(statistics_spectrum_.data()), n_nodes, width); + const Interleaved magnetic_rows(reinterpret_cast(field.data()), n_nodes, + width); + engine_gemm(transform_execution_space_, weights, field_rows, electric_rows, 0.0); + engine_gemm(transform_execution_space_, weights, closure_rows, magnetic_rows, + 0.0); + launch::parallel_for( + "solprop_native.spatial.physical_intensity", + Range2(execution_space_, {0, 0}, {n_nodes, n_t}), + KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { + double total = 0.0; + double size = 0.0; + for (std::size_t pol = 0; pol < n_pol; ++pol) { + const std::size_t at = 2 * (pol * n_t + time); + const Complex electric = + crop_scale * + Complex(electric_rows(node, at), electric_rows(node, at + 1)); + const Complex magnetic = + crop_scale * + Complex(magnetic_rows(node, at), magnetic_rows(node, at + 1)); + total += (electric * Kokkos::conj(magnetic)).real(); + size += Kokkos::sqrt(squared_abs(electric) * squared_abs(magnetic)); + } + power(node, time) = intensity_factor * total; + magnitude(node, time) = intensity_factor * size; + }); + launch::parallel_for( + "solprop_native.spatial.fluence_partials", + Kokkos::RangePolicy(execution_space_, 0, n_nodes), + KOKKOS_LAMBDA(const std::size_t node) { + double peak = lowest; + double sum = 0.0; + double size = 0.0; + for (std::size_t time = 0; time < n_t; ++time) { + peak = Kokkos::max(peak, power(node, time)); + sum += power(node, time); + size += magnitude(node, time); + } + partials(node, 1) = peak; + partials(node, 2) = dt * size; + if (fluence_offset != absent) + packed(fluence_offset + node) = dt * sum; + }); + if (fluence_offset != absent) + launch::parallel_reduce( + "solprop_native.spatial.fluence_largest", + Kokkos::RangePolicy(execution_space_, 0, n_nodes), + KOKKOS_LAMBDA(const std::size_t node, double& maximum) { + maximum = Kokkos::max(maximum, partials(node, 2)); + }, + Kokkos::Max( + Kokkos::subview(packed, fluence_offset + n_nodes + 1))); + if (peak_offset != absent) + launch::parallel_for( + "solprop_native.spatial.pack_peak", + Kokkos::RangePolicy(execution_space_, 0, 1), + KOKKOS_LAMBDA(const std::size_t) { + double peak = lowest; + for (std::size_t node = 0; node < n_nodes; ++node) + peak = Kokkos::max(peak, partials(node, 1)); + packed(peak_offset) = peak; + }); + } + } + const auto filled = + Kokkos::subview(packed, std::make_pair(0, packed_size)); + auto host = Kokkos::create_mirror_view(filled); + graph::deep_copy(execution_space_, host, filled); + execution_space_.fence("solprop_native spatial statistics packed boundary"); + std::vector result(packed_size); + for (std::size_t index = 0; index < packed_size; ++index) + result[index] = host(index); + // The separable routes don't form the axis's fluence: only a radial width reads it. + if (fluence_offset != absent && problem_.separable_geometry()) + result[fluence_offset + n_nodes] = Kokkos::Experimental::quiet_NaN_v; + if (moments_offset != absent) + std::copy_n(moments, plane_moment_slots, result.data() + moments_offset); + return result; +} + +// Bind solprop's boundary monitor (SEP 21): its four time weights, row-major `(4, +// n_t)`, and its columns' quadrature by mode (empty for unit weights). The views are +// allocated at the first bind and kept; the split's accumulator starts at zero. +void GeneralWorkspace::bind_monitor(const std::vector& weights, + const std::vector& column_weights) { + const auto n_t = problem_.dims.n_t; + const auto columns = problem_.n_columns(); + require_extent(weights.size(), 4 * n_t, "monitor weights"); + if (!column_weights.empty()) + require_extent(column_weights.size(), problem_.dims.n_modes, + "monitor column weights"); + if (monitor_weights_.extent(0) == 0) { + monitor_weights_ = RealView2("solprop_native.monitor.weights", 4, n_t); + monitor_column_weights_ = + RealView("solprop_native.monitor.column_weights", columns); + monitor_share_ = RealView("solprop_native.monitor.split_share", n_t); + monitor_half_ = RealView("solprop_native.monitor.split_half", n_t); + monitor_removed_ = + Kokkos::View("solprop_native.monitor.removed"); + monitor_split_sum_ = + Kokkos::View("solprop_native.monitor.split_sum"); + monitor_found_ = + Kokkos::View("solprop_native.monitor.split_found"); + } + auto host_weights = Kokkos::create_mirror_view(monitor_weights_); + for (std::size_t row = 0; row < 4; ++row) + for (std::size_t index = 0; index < n_t; ++index) + host_weights(row, index) = weights[row * n_t + index]; + graph::deep_copy(execution_space_, monitor_weights_, host_weights); + // A state column is `mode * n_pol + pol`. + std::vector expanded(columns, 1.0); + if (!column_weights.empty()) + for (std::size_t column = 0; column < columns; ++column) + expanded[column] = column_weights[column / problem_.dims.n_pol_state]; + copy_vector(execution_space_, monitor_column_weights_, expanded); + graph::deep_copy(execution_space_, monitor_removed_, 0.0); + graph::deep_copy(execution_space_, monitor_found_, SplitSums{}); + execution_space_.fence("solprop_native monitor bind"); + monitor_bound_ = true; +} + +// The boundary monitor's reading of the accepted state (solprop SEP 21, ADR-057): +// `sum_t P`, the four weighted sums `sum_t w_k P` and the split's accumulator, which +// this drains; `P(t)` is the columns' weighted `|E(t)|^2` on the coarse time axis. +// The state is transformed chunk by chunk through buffers idle between steps: on a +// field grid under the split scheme the collar's coarse pair, whose inverse real +// transform is the one solprop's reference takes; otherwise the statistics buffers, +// with the analytic signal's spectrum, whose real part is that transform's result. +namespace { +// The boundary monitor's sums over `count` rows of `n_t` samples: each sample's power, +// `power(row, index)`, in total and weighed by each of the four regions' weights. A +// host work item takes a run of one row's samples, its sums in sample order, so that +// no sample divides its flat index by the row length; a device takes one sample. +template +MonitorSums monitor_sums(const char* label, const ExecutionSpace& execution, + const std::size_t count, const std::size_t n_t, + const Weights& weights, const Power& power) { + MonitorSums sums; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + launch::parallel_reduce( + label, Kokkos::RangePolicy(execution, 0, count * n_t), + KOKKOS_LAMBDA(const std::size_t item, MonitorSums& update) { + const auto index = item % n_t; + const double value = power(item / n_t, index); + update.values[0] += value; + for (std::size_t region = 0; region < 4; ++region) + update.values[region + 1] += weights(region, index) * value; + }, + Kokkos::Sum(sums)); +#else + constexpr std::size_t run = 512; + const std::size_t runs = (n_t + run - 1) / run; + launch::parallel_reduce( + label, Kokkos::RangePolicy(execution, 0, count * runs), + KOKKOS_LAMBDA(const std::size_t item, MonitorSums& update) { + const std::size_t row = item / runs; + const std::size_t first = (item % runs) * run; + const std::size_t last = n_t - first < run ? n_t : first + run; + double total = 0.0; + double regions[4] = {0.0, 0.0, 0.0, 0.0}; + for (std::size_t index = first; index < last; ++index) { + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide + const double value = power(row, index); + total += value; + for (std::size_t region = 0; region < 4; ++region) + regions[region] += weights(region, index) * value; + } + update.values[0] += total; + for (std::size_t region = 0; region < 4; ++region) + update.values[region + 1] += regions[region]; + }, + Kokkos::Sum(sums)); +#endif + return sums; +} +} // namespace + +namespace { +// The field on a line's node, in a row of the transverse scratch (solprop SEP 44): a +// row is a plane and a position across the line, and no route has a branch here. +struct EdgeScratchValue { + ComplexView3 scratch; + std::size_t units{}, across{}, row_stride{}, node_stride{}; + KOKKOS_INLINE_FUNCTION Complex operator()(const std::size_t row, + const std::size_t node) const { + return scratch.data()[(row / across) * units + (row % across) * row_stride + + node * node_stride]; + } +}; + +// The same radially: `sqrt(r) E`, a dense product per value, in a row of (pol, omega). +struct EdgeRadialValue { + RealView2 synthesis; + ComplexView2 current; + RealView scale; + std::size_t n_pol{}, n{}, modes{}; + KOKKOS_INLINE_FUNCTION Complex operator()(const std::size_t row, + const std::size_t node) const { + const auto pol = row / n; + const auto omega = row % n; + Complex value{}; + for (std::size_t mode = 0; mode < modes; ++mode) + value += synthesis(node, mode) * current(mode * n_pol + pol, omega); + return scale(node) * value; + } +}; + +// One line's `(inward, windowed)`, unscaled: a Kaiser-windowed transform along its +// normal, padded twice, batch by batch so that the buffer stays bounded. +template +MonitorSums edge_strip_line(const ExecutionSpace& execution, const Value& value, + const std::size_t rows, const std::size_t batch, + const std::size_t nodes, const std::size_t first, + const std::size_t last, const IndexView& positions, + const RealView& window, const ComplexView2& buffer, + StripPlan& plan) { + const std::size_t count = 2 * nodes; + MonitorSums sums; + for (std::size_t start = 0; start < rows; start += batch) { + launch::parallel_for( + "solprop_native.edge.strip_fill", + Kokkos::RangePolicy(execution, 0, batch), + KOKKOS_LAMBDA(const std::size_t row) { + // A short last batch leaves its other rows zero, whose energy is none. + const bool live = start + row < rows; + for (std::size_t index = 0; index < nodes; ++index) + buffer(row, index) = + live ? window(index) * value(start + row, positions(index)) : Complex{}; + for (std::size_t index = nodes; index < count; ++index) + buffer(row, index) = Complex{}; + }); + graph::execute(plan, buffer, buffer, KokkosFFT::Normalization::none); + MonitorSums part; + launch::parallel_reduce( + "solprop_native.edge.strip_sums", + Kokkos::RangePolicy(execution, 0, batch), + KOKKOS_LAMBDA(const std::size_t row, MonitorSums& update) { + for (std::size_t index = 0; index < count; ++index) { + const double power = squared_abs(buffer(row, index)); + update.values[1] += power; + if (index >= first && index < last) + update.values[0] += power; + } + }, + Kokkos::Sum(part)); + sums += part; + } + return sums; +} + +// One line's rectified inward current: `Im(E* dE/ds) / k`, rectified per row before +// it's summed, as a collar's current is defined (solprop SEP 40 section 5.3). +template +double edge_current_line(const ExecutionSpace& execution, const Value& value, + const std::size_t rows, const std::size_t node, + const std::size_t before, const std::size_t after, + const double step, const double k_min) { + double total = 0.0; + launch::parallel_reduce( + "solprop_native.edge.current", + Kokkos::RangePolicy(execution, 0, rows), + KOKKOS_LAMBDA(const std::size_t row, double& update) { + const Complex here = value(row, node); + const Complex slope = (value(row, after) - value(row, before)) / step; + const double flux = + (here.real() * slope.imag() - here.imag() * slope.real()) / k_min; + update += flux > 0.0 ? flux : 0.0; + }, + Kokkos::Sum(total)); + return total; +} + +// The transverse edge's region weights (solprop SEP 44), from the rows given on: a +// radial read synthesises only the outer rows its regions can touch. +struct EdgeWeights { + RealView2 weights; + std::size_t first{}; + KOKKOS_INLINE_FUNCTION double operator()(const std::size_t region, + const std::size_t index) const { + return weights(region, first + index); + } +}; +} // namespace + +// Bind what the transverse edge's reading sums with (solprop SEP 44): three tables of +// four regions' weights on the nodes, and radially the synthesis and the first row a +// read uses. Native holds no face, mask or judgement. The views are allocated at the +// first bind and kept. +void GeneralWorkspace::bind_edge(const std::vector& faces, + const std::vector& clip, + const std::vector& totals, + const std::vector& synthesis, + const std::size_t first_row) { + const auto nodes = faces.size() / 4; + require_extent(faces.size(), 4 * nodes, "edge faces"); + require_extent(clip.size(), 4 * nodes, "edge clip"); + require_extent(totals.size(), 4 * nodes, "edge totals"); + if (problem_.separable_geometry()) { + require_extent(nodes, transverse_units(), "edge nodes"); + require_extent(synthesis.size(), 0, "a separable edge's synthesis"); + } else { + require_extent(nodes, problem_.dims.n_modes, "edge nodes"); + require_extent(synthesis.size(), nodes * nodes, "edge synthesis"); + } + if (first_row >= nodes) + throw std::invalid_argument("the edge's first row lies past its nodes"); + if (edge_faces_.extent(0) == 0) { + edge_faces_ = RealView2("solprop_native.edge.faces", 4, nodes); + edge_clip_ = RealView2("solprop_native.edge.clip", 4, nodes); + edge_totals_ = RealView2("solprop_native.edge.totals", 4, nodes); + edge_columns_ = RealView("solprop_native.edge.columns", problem_.dims.n_modes); + if (!synthesis.empty()) + edge_synthesis_ = RealView2("solprop_native.edge.synthesis", nodes, nodes); + if (!synthesis.empty() && problem_.dims.n_pol_state == 1 && + current_.extent(1) == problem_.dims.n_omega) + edge_nodes_ = RealView2( + Kokkos::view_alloc(Kokkos::WithoutInitializing, "solprop_native.edge.nodes"), + nodes, 2 * problem_.dims.n_omega); + } + copy_vector(execution_space_, edge_faces_, faces); + copy_vector(execution_space_, edge_clip_, clip); + copy_vector(execution_space_, edge_totals_, totals); + if (!synthesis.empty()) + copy_vector(execution_space_, edge_synthesis_, synthesis); + edge_first_row_ = first_row; + execution_space_.fence("solprop_native edge bind"); +} + +// Bind a requested collar's lines (solprop SEP 44), which Python describes: their +// nodes, windows, inward bins and onsets. A batch of one shape serves every line that +// has it. +void GeneralWorkspace::bind_edge_lines(const EdgeLines& lines) { + const auto n_lines = lines.axes.size(); + require_extent(lines.counts.size(), n_lines, "edge line counts"); + require_extent(lines.inward.size(), 2 * n_lines, "edge line bins"); + require_extent(lines.factors.size(), n_lines, "edge line factors"); + require_extent(lines.measures.size(), n_lines, "edge line measures"); + require_extent(lines.onsets.size(), 3 * n_lines, "edge line onsets"); + require_extent(lines.steps.size(), n_lines, "edge line steps"); + require_extent(lines.windows.size(), lines.positions.size(), "edge line windows"); + if (std::accumulate(lines.counts.begin(), lines.counts.end(), std::size_t{0}) != + lines.positions.size()) + throw std::invalid_argument("the edge's lines don't account for their nodes"); + const auto planes = problem_.dims.n_pol_state * problem_.dims.n_omega; + // A runner's next point reuses the plans of its shapes, and only those stay held. + decltype(edge_batches_) batches; + std::vector bound; + bound.reserve(n_lines); + std::size_t offset = 0; + for (std::size_t index = 0; index < n_lines; ++index) { + EdgeLine line; + line.axis = lines.axes[index]; + line.nodes = lines.counts[index]; + line.rows = planes * (problem_.slab_geometry() ? problem_.spatial.n_p + : !problem_.separable_geometry() ? std::size_t{1} + : line.axis == 1 ? problem_.spatial.n_y + : problem_.spatial.n_x); + line.first = lines.inward[2 * index]; + line.last = lines.inward[2 * index + 1]; + line.factor = lines.factors[index]; + line.measure = lines.measures[index]; + line.step = lines.steps[index]; + line.onset = lines.onsets[3 * index]; + line.before = lines.onsets[3 * index + 1]; + line.after = lines.onsets[3 * index + 2]; + if (line.nodes != 0) { + const auto first = static_cast(offset); + const auto last = static_cast(offset + line.nodes); + line.positions = IndexView("solprop_native.edge.line_nodes", line.nodes); + line.window = RealView("solprop_native.edge.line_window", line.nodes); + copy_vector(execution_space_, line.positions, + std::vector(lines.positions.begin() + first, + lines.positions.begin() + last)); + copy_vector(execution_space_, line.window, + std::vector(lines.windows.begin() + first, + lines.windows.begin() + last)); + offset += line.nodes; + const std::size_t count = 2 * line.nodes; + line.batch = + std::min(line.rows, std::max(1, lines.batch_bytes / + (sizeof(Complex) * count))); + } + bound.push_back(std::move(line)); + } + // The strips run in turn, so one allocation serves every shape: a positive and a + // negative face's lines commonly differ by a node, and would otherwise hold two. + std::size_t largest = 0; + for (const auto& line : bound) + largest = std::max(largest, line.batch * 2 * line.nodes); + if (edge_strip_.size() != largest) + edge_strip_ = + largest == 0 + ? ComplexView{} + : ComplexView(Kokkos::view_alloc(Kokkos::WithoutInitializing, + std::string("solprop_native.edge.strip")), + largest); + for (const auto& line : bound) { + if (line.nodes == 0) + continue; + const std::size_t count = 2 * line.nodes; + const std::pair shape{line.batch, count}; + if (!batches.contains(shape)) { + EdgeBatch batch; + if (auto kept = edge_batches_.extract(shape); kept) + batch = std::move(kept.mapped()); + // Not tracked: `edge_strip_` owns the memory, and outlives every shape's view. + batch.buffer = ComplexView2(edge_strip_.data(), line.batch, count); + if (!batch.plan) { + // A plan's own memory is measured as the workspace's are, and listed with them. + const auto heap_before = heap_bytes(); + const auto device_before = device_free_bytes(); + constexpr int along = -1; + batch.plan = std::make_unique(fft_space_for_rows(batch.buffer), + batch.buffer, batch.buffer, + KokkosFFT::Direction::forward, along); + execution_space_.fence("solprop_native edge strip plan"); + const auto heap_after = heap_bytes(); + const auto device_after = device_free_bytes(); + batch.heap_growth = heap_after > heap_before ? heap_after - heap_before : 0; + batch.device_growth = + device_before > device_after ? device_before - device_after : 0; + } + batches.emplace(shape, std::move(batch)); + } + } + edge_lines_ = std::move(bound); + edge_batches_ = std::move(batches); + edge_k_min_ = lines.k_min; + if (lines.scale.empty()) { + edge_scale_ = {}; + } else { + edge_scale_ = RealView("solprop_native.edge.line_scale", lines.scale.size()); + copy_vector(execution_space_, edge_scale_, lines.scale); + } + execution_space_.fence("solprop_native edge lines"); +} + +// The state's reading for the transverse edge: its region sums and each mode's energy, +// with only those numbers crossing. With `whole` (a launch) every table on every node; +// otherwise the faces', and with `totals` a collar's, on the rows a read uses; and with +// a requested collar its strip and current, on the same transform. The transverse +// scratch is free at a committed position: a step's first pass writes it again. +EdgeNumbers GeneralWorkspace::edge_reading(const bool faces, const bool whole, + const bool totals, const bool strip, + const bool want_current) { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + if (edge_faces_.extent(0) == 0) + throw std::logic_error("no transverse edge is bound"); + const auto current = current_; + const auto n_pol = problem_.dims.n_pol_state; + const auto n = problem_.dims.n_omega; + EdgeNumbers reading; + const bool separable = problem_.separable_geometry(); + if (separable) + separable_load_current(); + const auto take = [&reading](const MonitorSums& sums) { + reading.sums.insert(reading.sums.end(), sums.values.data() + 1, + sums.values.data() + 5); + }; + if (!faces) { + // Only what follows was asked for. + } else if (separable) { + const auto scratch = transverse_; + const auto units = transverse_units(); + const auto planes = scratch.size() / units; + const auto power = KOKKOS_LAMBDA(const std::size_t plane, const std::size_t node) { + return squared_abs(scratch.data()[plane * units + node]); + }; + take(monitor_sums("solprop_native.edge.faces", execution_space_, planes, units, + edge_faces_, power)); + if (whole) + take(monitor_sums("solprop_native.edge.clip", execution_space_, planes, units, + edge_clip_, power)); + if (whole || totals) + take(monitor_sums("solprop_native.edge.totals", execution_space_, planes, units, + edge_totals_, power)); + } else if (edge_nodes_.extent(0) != 0) { + // One state polarisation: the rows wanted synthesised in one product, the state's + // complex samples read as twice as many real columns, and every table's sums then + // from those fields. A launch reads every node: its total is the grid's. The + // per-sample sums this replaces synthesised each row once a table; on an Apple M5 + // Max the 256- and 512-node radial filaments ran 0.83 and 0.75 of their one-thread + // windows, 0.92 and 0.87 at six threads. + using Interleaved = Kokkos::View>; + static_assert(sizeof(Complex) == 2 * sizeof(double)); + const auto nodes = problem_.dims.n_modes; + const auto first = whole ? std::size_t{0} : edge_first_row_; + const auto rows = nodes - first; + const Interleaved state(reinterpret_cast(current_.data()), nodes, 2 * n); + const Interleaved synthesis(edge_synthesis_.data() + first * nodes, rows, nodes); + const Interleaved fields(edge_nodes_.data(), rows, 2 * n); + Kokkos::Profiling::pushRegion("solprop_native.edge.synthesis"); + engine_gemm(transform_space_for(rows), synthesis, state, fields, 0.0); + Kokkos::Profiling::popRegion(); + const auto power = KOKKOS_LAMBDA(const std::size_t plane, const std::size_t row) { + const double real = fields(row, 2 * plane); + const double imaginary = fields(row, 2 * plane + 1); + return real * real + imaginary * imaginary; + }; + const auto sums = [&](const char* label, const RealView2& weights) { + return monitor_sums(label, execution_space_, n, rows, + EdgeWeights{.weights = weights, .first = first}, power); + }; + take(sums("solprop_native.edge.faces", edge_faces_)); + if (whole) + take(sums("solprop_native.edge.clip", edge_clip_)); + if (whole || totals) + take(sums("solprop_native.edge.totals", edge_totals_)); + } else { + // Each plane's synthesis on the rows wanted. A launch reads every node: its total + // is the grid's. A dense product per sample, on a read's few outer rows. + const auto synthesis = edge_synthesis_; + const auto nodes = problem_.dims.n_modes; + const auto first = whole ? std::size_t{0} : edge_first_row_; + const auto power = KOKKOS_LAMBDA(const std::size_t plane, const std::size_t row) { + const auto pol = plane / n; + const auto omega = plane % n; + Complex value{}; + for (std::size_t mode = 0; mode < nodes; ++mode) + value += synthesis(first + row, mode) * current(mode * n_pol + pol, omega); + return squared_abs(value); + }; + const auto sums = [&](const char* label, const RealView2& weights) { + return monitor_sums(label, execution_space_, n_pol * n, nodes - first, + EdgeWeights{.weights = weights, .first = first}, power); + }; + take(sums("solprop_native.edge.faces", edge_faces_)); + if (whole) + take(sums("solprop_native.edge.clip", edge_clip_)); + if (whole || totals) + take(sums("solprop_native.edge.totals", edge_totals_)); + } + if (faces) { + const auto energies = edge_columns_; + launch::parallel_for( + "solprop_native.edge.columns", + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_modes), + KOKKOS_LAMBDA(const std::size_t mode) { + double sum = 0.0; + for (std::size_t pol = 0; pol < n_pol; ++pol) + for (std::size_t index = 0; index < n; ++index) + sum += squared_abs(current(mode * n_pol + pol, index)); + energies(mode) = sum; + }); + execution_space_.fence("solprop_native edge columns"); + const auto host = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, energies); + reading.columns.assign(host.data(), host.data() + host.size()); + } + // The same two kernels on any route: only where a row's value comes from differs. + const auto take_line = [&](const EdgeLine& line, const auto& value) { + if (strip) { + MonitorSums sums; + if (line.nodes != 0) { + auto& batch = edge_batches_.at({line.batch, 2 * line.nodes}); + sums = edge_strip_line(execution_space_, value, line.rows, line.batch, + line.nodes, line.first, line.last, line.positions, + line.window, batch.buffer, *batch.plan); + } + reading.strip.push_back(sums.values[0] * line.factor); + reading.strip.push_back(sums.values[1] * line.factor); + } + if (want_current && line.onset != EdgeLines::no_onset) + reading.current += + line.measure * edge_current_line(execution_space_, value, line.rows, + line.onset, line.before, line.after, + line.step, edge_k_min_); + }; + if (strip || want_current) { + for (const auto& line : edge_lines_) { + if (!separable) { + take_line(line, EdgeRadialValue{.synthesis = edge_synthesis_, + .current = current, + .scale = edge_scale_, + .n_pol = n_pol, + .n = n, + .modes = problem_.dims.n_modes}); + } else if (problem_.slab_geometry()) { + take_line(line, EdgeScratchValue{.scratch = transverse_, + .units = transverse_units(), + .across = 1, + .row_stride = 0, + .node_stride = 1}); + } else if (line.axis == 1) { + take_line(line, EdgeScratchValue{.scratch = transverse_, + .units = transverse_units(), + .across = problem_.spatial.n_y, + .row_stride = problem_.spatial.n_x, + .node_stride = 1}); + } else { + take_line(line, EdgeScratchValue{.scratch = transverse_, + .units = transverse_units(), + .across = problem_.spatial.n_x, + .row_stride = 1, + .node_stride = problem_.spatial.n_x}); + } + } + } + return reading; +} + +std::vector GeneralWorkspace::monitor_reading() { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + if (!monitor_bound_) + throw std::logic_error("no boundary monitor is bound"); + const auto current = current_; + const auto spectrum = statistics_spectrum_; + const auto time = statistics_time_; + const auto weights = monitor_weights_; + const auto column_weights = monitor_column_weights_; + const auto n = problem_.dims.n_omega; + const auto n_t = problem_.dims.n_t; + const bool envelope = problem_.dims.family == FieldFamily::envelope; + // A one-sided spectrum's last bin is the Nyquist bin only on an even axis. + const bool nyquist = n_t % 2 == 0; + const auto columns = problem_.n_columns(); + MonitorSums sums; + // Half the analytic signal's transform work, on buffers that already exist. + if (problem_.absorber.split_step && !envelope) { + const auto coarse = coarse_spectrum_; + const auto samples = coarse_time_real_; + const double inverse_n = 1.0 / static_cast(n_t); + const auto chunk_rows = coarse_spectrum_.extent(0); + for (std::size_t first = 0; first < columns; first += chunk_rows) { + const auto count = std::min(chunk_rows, columns - first); + launch::parallel_for( + "solprop_native.monitor.coarse_spectrum", + Range2(execution_space_, {0, 0}, {count, n}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { + coarse(row, index) = inverse_n * current(first + row, index); + }); + graph::execute(*coarse_inverse_real_plan_, coarse, samples, + KokkosFFT::Normalization::none); + sums += monitor_sums( + "solprop_native.monitor.sums_real", execution_space_, count, n_t, weights, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { + const double value = samples(row, index); + return column_weights(first + row) * value * value; + }); + } + } + const auto rows = statistics_spectrum_.extent(0); + const bool analytic = !(problem_.absorber.split_step && !envelope); + for (std::size_t first = 0; analytic && first < columns; first += rows) { + const auto count = std::min(rows, columns - first); + // Every bin of the rows read is written, the empty half of a one-sided spectrum + // with zeros; a partial chunk's other rows are transformed but never read. + launch::parallel_for( + "solprop_native.monitor.spectrum", + Range2(execution_space_, {0, 0}, {count, n_t}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { + if (index >= n) { + spectrum(row, index) = Complex{}; + return; + } + const bool single = envelope || index == 0 || (nyquist && index + 1 == n); + spectrum(row, index) = (single ? 1.0 : 2.0) / static_cast(n_t) * + current(first + row, index); + }); + graph::execute(*statistics_inverse_plan_, spectrum, time, + KokkosFFT::Normalization::none); + sums += monitor_sums( + "solprop_native.monitor.sums", execution_space_, count, n_t, weights, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { + const Complex value = time(row, index); + return column_weights(first + row) * + (envelope ? squared_abs(value) : value.real() * value.real()); + }); + } + double removed = 0.0; + SplitSums found; + if (problem_.absorber.split_step) { + execution_space_.fence("solprop_native monitor reading"); + removed = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, monitor_removed_)(); + found = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, monitor_found_)(); + graph::deep_copy(execution_space_, monitor_removed_, 0.0); + } + std::vector reading{sums.values[0], sums.values[1], sums.values[2], + sums.values[3], sums.values[4], removed, + found.values[1], found.values[2], found.values[3], + found.values[4]}; + if (occupancy_count_ != 0) { + // The state's energy per walk-off bucket, beside the five: a state column is + // `mode * n_pol + pol`, and the buckets are by mode. A reduction per column, whose + // per-thread histograms need no atomics: on a device an atomic per element took + // most of a reading, several steps' time on an AMD Instinct MI325X. + const auto buckets = occupancy_buckets_; + const auto n_pol = problem_.dims.n_pol_state; + for (std::size_t first = 0; first < occupancy_count_; + first += occupancy_reduction_slots) { + OccupancySums occupancy; + launch::parallel_reduce( + "solprop_native.monitor.occupancy", + Kokkos::RangePolicy(execution_space_, 0, columns), + KOKKOS_LAMBDA(const std::size_t row, OccupancySums& update) { + const std::size_t offset = (row / n_pol) * n; + const double weight = column_weights(row); + for (std::size_t index = 0; index < n; ++index) { + const std::size_t bucket = buckets(offset + index); + if (bucket >= first && bucket - first < occupancy_reduction_slots) + update.values[bucket - first] += + weight * squared_abs(current(row, index)); + } + }, + Kokkos::Sum(occupancy)); + reading.insert(reading.end(), occupancy.values.data(), + occupancy.values.data() + + std::min(occupancy_reduction_slots, occupancy_count_ - first)); + } + } + execution_space_.fence("solprop_native monitor reading"); + return reading; +} + +// Bind the split collar's walk-off bucket of each (mode, frequency) bin (solprop +// #440): from here on a monitor reading also returns the state's energy per bucket. +// The table is allocated at the first bind and kept. +void GeneralWorkspace::bind_occupancy(const std::vector& buckets, + const std::size_t count) { + require_extent(buckets.size(), problem_.dims.n_modes * problem_.dims.n_omega, + "occupancy buckets"); + if (count == 0 || count > occupancy_bucket_limit) + throw std::invalid_argument("the occupancy needs between 1 and 128 buckets"); + for (const auto bucket : buckets) + if (bucket >= count) + throw std::invalid_argument("an occupancy bucket lies past the bucket count"); + if (occupancy_count_ == 0) + occupancy_buckets_ = + ByteView("solprop_native.monitor.occupancy_buckets", buckets.size()); + occupancy_count_ = count; + auto host = Kokkos::create_mirror_view(occupancy_buckets_); + for (std::size_t index = 0; index < buckets.size(); ++index) + host(index) = buckets[index]; + graph::deep_copy(execution_space_, occupancy_buckets_, host); + execution_space_.fence("solprop_native occupancy bind"); +} + +} // namespace solprop_native diff --git a/native/src/general/trial.cpp b/native/src/general/trial.cpp new file mode 100644 index 000000000..9c69d1278 --- /dev/null +++ b/native/src/general/trial.cpp @@ -0,0 +1,9930 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +// The general workspace's hot set: the trial step, its stages and responses, the +// Adams and low-storage marches and the split-step absorber, in one unit so that the +// compiler sees every call between them. + +#include "workspace.hpp" + +namespace solprop_native { + +// The hybrid policy's shape guard. A host build sends every eligible transform to the +// vendor GEMM: measured on Apple's Accelerate with one thread, it beat the custom +// kernel at every shape tried -- a radial field's transforms 1.1 to 1.5 times faster +// at 2 to 6 nodes and 1.6 to 35 times at 16 to 256 nodes and 256 to 2048 samples, a +// one-mode field capillary's 2.4 to 3.1 times -- and lost at none. A device build +// takes the vendor GEMM from eight modes (with eight nodes and 1024 samples): below +// that the product is a few rows deep, which the custom kernels take in one pass over +// the samples. On an NVIDIA A40 a four-mode capillary's stacked projection (8 rows, +// 36 nodes, 10240 samples) took 71 us in cuBLAS and its synthesis 27 us; at 256 +// radial modes the vendor GEMM runs near the FP64 peak. +bool GeneralWorkspace::hybrid_shape_eligible() const noexcept { + return build::host || (problem_.dims.n_modes >= 8 && problem_.dims.n_nodes >= 8 && + problem_.dims.n_to >= 1024); +} + +bool GeneralWorkspace::use_packed_radial_blas() const noexcept { + if constexpr (build::device) + return false; + // Two packed work arrays add two complex-equivalent time columns to the + // shared conservative twenty-column inventory. Decide before allocation. + constexpr std::size_t max_samples = + std::size_t{64} * 1024 * 1024 / (22 * sizeof(Complex)); + return !problem_.separable_geometry() && !problem_.spatial.spectral_weights.empty() && + problem_.dims.n_modes == problem_.dims.n_nodes && + problem_.dims.node_chunk == problem_.dims.n_nodes && + problem_.dims.n_pol_state == 1 && problem_.dims.n_pol_transform == 1 && + problem_.dims.family == FieldFamily::envelope && + problem_.transform.projection_coarse_matrix.empty() && + problem_.dims.n_nodes >= 256 && problem_.dims.n_to >= 256 && + problem_.dims.n_nodes <= max_samples / problem_.dims.n_to; +} + +// The retained-spectrum radial synthesis, on a host and a device. A real static +// synthesis commutes with the temporal transform: synthesising the coarse bins the +// stage entry fills and padding each node's spectrum gives the nodes' fields, so the +// product covers n_omega complex columns rather than the n_to complex samples of the +// padded, oversampled analytic signal. On a square map the node transforms replace the +// modal ones one for one; the held radial plasma's projection already contracts after +// cropping (`held_radial_projection`). An in-stage absorber or a Raman drive reads the +// modal time field, and a dispersive response its own synthesis, so they keep the +// time-domain product; skinny capillary maps would multiply the transforms. +bool GeneralWorkspace::retained_synthesis() const noexcept { + return problem_.dims.family == FieldFamily::field_analytic && + !problem_.spatial.spectral_weights.empty() && !problem_.separable_geometry() && + !problem_.general.enabled && !problem_.dispersive.enabled && + !problem_.absorber.in_stage && !problem_.raman.enabled && + problem_.dims.n_modes == problem_.dims.n_nodes && problem_.dims.n_nodes > 1 && + problem_.dims.n_pol_state == 1 && problem_.dims.n_pol_transform == 1 && + problem_.dims.n_omega < problem_.dims.n_to; +} + +void GeneralWorkspace::synthesise_retained(const std::size_t first_node, + const std::size_t node_count) { + Kokkos::Profiling::pushRegion("solprop_native.transform.synthesis.retained"); + const auto crop = crop_index_; + const auto n = problem_.dims.n_omega; + // The coarse modal spectra once per evaluation, at its first chunk (the stage core + // and the statistics' synthesis both start at node zero). Rows run in parallel at + // the modal copies' width (`temporal_execution_space_`), each a contiguous pass. + if (first_node == 0) { + const auto fine = fine_spectrum_; + const auto modal = retained_modal_; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // A device takes a thread per element. + launch::parallel_for( + "solprop_native.transform.retained_gather", + Range2(execution_space_, {0, 0}, {problem_.dims.n_modes, n}), + KOKKOS_LAMBDA(const std::size_t mode, const std::size_t index) { + modal(mode, index) = fine(mode, crop(index)); + }); +#else + launch::parallel_for( + "solprop_native.transform.retained_gather", + Kokkos::RangePolicy(temporal_execution_space_, 0, + problem_.dims.n_modes), + KOKKOS_LAMBDA(const std::size_t mode) { + for (std::size_t index = 0; index < n; ++index) + modal(mode, index) = fine(mode, crop(index)); + }); +#endif + } + using Interleaved = Kokkos::View>; + static_assert(sizeof(Complex) == 2 * sizeof(double)); + const Interleaved synthesis(scaled_synthesis_.data() + + first_node * problem_.dims.n_modes, + node_count, problem_.dims.n_modes); + const Interleaved input(reinterpret_cast(retained_modal_.data()), + problem_.dims.n_modes, 2 * n); + const Interleaved output(reinterpret_cast(retained_node_.data()), node_count, + 2 * n); + engine_gemm(transform_space_for(node_count), synthesis, input, output, 0.0); + // Each node's padded spectrum in the modal time buffer's leading rows, which are + // free as no modal transform runs on this path. Only the retained bins are + // written: the others were zeroed when the plans were made, and an out-of-place + // complex transform leaves its input as it was. + const auto node = retained_node_; + const auto padded = modal_complex_; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // A device zeroes the chunk's padded rows first, a fill at its bandwidth, rather + // than relying on the modal buffer's other bins staying zero, and pads a thread per + // element. + graph::deep_copy(execution_space_, + Kokkos::subview(padded, std::make_pair(std::size_t{0}, node_count), + Kokkos::ALL()), + Complex{}); + launch::parallel_for( + "solprop_native.transform.retained_pad", + Range2(execution_space_, {0, 0}, {node_count, n}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { + padded(row, crop(index)) = node(row, index); + }); +#else + launch::parallel_for( + "solprop_native.transform.retained_pad", + Kokkos::RangePolicy(temporal_execution_space_, 0, node_count), + KOKKOS_LAMBDA(const std::size_t row) { + for (std::size_t index = 0; index < n; ++index) + padded(row, crop(index)) = node(row, index); + }); +#endif + const bool tail = node_count != problem_.dims.node_chunk; + const auto rows = std::make_pair(std::size_t{0}, node_count); + graph::execute(tail ? *retained_tail_inverse_plan_ : *retained_inverse_plan_, + Kokkos::subview(modal_complex_, rows, Kokkos::ALL()), + Kokkos::subview(node_complex_, rows, Kokkos::ALL()), + KokkosFFT::Normalization::none); + Kokkos::Profiling::popRegion(); +} + +void GeneralWorkspace::packed_radial_gemm(const RealView2& matrix, + const ComplexView2& source, + const ComplexView2& destination) { + const auto input = radial_blas_input_; + const auto output = radial_blas_output_; + const auto rows = problem_.dims.n_nodes; + const auto times = problem_.dims.n_to; + Kokkos::Profiling::pushRegion("solprop_native.radial.packed_real_gemm"); + launch::parallel_for( + "solprop_native.radial.pack", + Range2(temporal_execution_space_, {0, 0}, {rows, times}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { + input(row, 2 * time) = source(row, time).real(); + input(row, 2 * time + 1) = source(row, time).imag(); + }); + engine_gemm(transform_execution_space_, matrix, input, output, 0.0); + launch::parallel_for( + "solprop_native.radial.unpack", + Range2(temporal_execution_space_, {0, 0}, {rows, times}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { + destination(row, time) = + Complex(output(row, 2 * time), output(row, 2 * time + 1)); + }); + Kokkos::Profiling::popRegion(); +} + +bool GeneralWorkspace::use_blas_for_synthesis() const noexcept { + if (use_packed_radial_blas()) + return false; + // The hybrid planar kind contracts its small x rule per y row natively. + if (problem_.separable_geometry()) + return false; + if (problem_.dims.n_pol_state != 1) + return false; +#if !defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) + // Without a vendor BLAS an envelope keeps the custom kernels. + if (problem_.dims.family == FieldFamily::envelope) + return false; +#endif + if (problem_.dims.family != FieldFamily::envelope && + problem_.dims.family != FieldFamily::field_real) + return false; +#if defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) + // Spatial envelope matrices can be large even with a short temporal grid. + if (!problem_.spatial.spectral_weights.empty() && + problem_.dims.family == FieldFamily::envelope && + problem_.transform.projection_coarse_matrix.empty() && + problem_.dims.n_modes == problem_.dims.n_nodes && problem_.dims.n_nodes >= 256 && + problem_.dims.n_to >= 256) + return true; +#endif + return hybrid_shape_eligible(); +} + +// An analytic field's or an envelope's synthesis is a real matrix times complex +// samples: viewed as real numbers, its samples are a real matrix twice as wide, so the +// transform is one real GEMM through the selected BLAS, with no copy and no complex +// BLAS. An envelope's projection is the same (`use_interleaved_projection`). +// +// A device takes it at the shapes where it already takes BLAS for the projection +// (`hybrid_shape_eligible`): there the custom synthesis kernel cost 4.6 times the +// projection's GEMM at 512 radial nodes (an AMD CDNA 3 device, 2026-10-06). +bool GeneralWorkspace::use_interleaved_synthesis() const noexcept { + if (problem_.dims.family == FieldFamily::field_real || + problem_.separable_geometry() || problem_.dims.n_pol_state != 1 || + problem_.dims.n_pol_transform != 1) + return false; + return hybrid_shape_eligible(); +} + +bool GeneralWorkspace::use_interleaved_projection() const noexcept { + if (problem_.dims.family != FieldFamily::envelope || problem_.separable_geometry() || + problem_.dims.n_pol_state != 1 || problem_.dims.n_pol_transform != 1) + return false; + return hybrid_shape_eligible(); +} + +bool GeneralWorkspace::use_blas_for_projection() const noexcept { + if (use_packed_radial_blas()) + return false; + // The hybrid planar kind contracts its small x rule per y row natively. + if (problem_.separable_geometry()) + return false; + if (problem_.dims.n_pol_state != 1) + return false; +#if !defined(SOLPROP_NATIVE_TPL_BLAS_OPENBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) && \ + !defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) + // Without a vendor BLAS an envelope keeps the custom kernels. + if (problem_.dims.family == FieldFamily::envelope) + return false; +#endif +#if defined(SOLPROP_NATIVE_TPL_BLAS_CUBLAS) || defined(SOLPROP_NATIVE_TPL_BLAS_ROCBLAS) + // Spatial envelope matrices can be large even with a short temporal grid. + if (!problem_.spatial.spectral_weights.empty() && + problem_.dims.family == FieldFamily::envelope && + problem_.transform.projection_coarse_matrix.empty() && + problem_.dims.n_modes == problem_.dims.n_nodes && problem_.dims.n_nodes >= 256 && + problem_.dims.n_to >= 256) + return true; +#endif + return hybrid_shape_eligible(); +} + +bool GeneralWorkspace::use_chunked_blas_for_synthesis() const noexcept { + return problem_.dims.node_chunk < problem_.dims.n_nodes && use_blas_for_synthesis(); +} + +// One node, one mode and one scalar component: the synthesis and the projection are +// each one product, which the response column forms itself (solprop SEP 25 section +// 6.7), in place of the stage's taper scaling, two one-element GEMMs and the zero +// fill of the projection. Each folded product is the GEMM's own one rounding, +// `(s * taper) * x` and `(p / taper) * y`. +// The host's columns only: a device keeps its own column kernels. +bool GeneralWorkspace::folds_single_node() const noexcept { + if constexpr (build::device) + return false; + return !general_ && !dispersive_ && !held_radial() && !gated_plasma() && + !problem_.separable_geometry() && problem_.dims.n_modes == 1 && + problem_.dims.n_nodes == 1 && problem_.dims.node_chunk == 1 && + problem_.dims.n_pol_state == 1 && problem_.dims.n_pol_transform == 1 && + problem_.transform.projection_coarse_matrix.empty() && + !problem_.raman.enabled && !problem_.plasma.envelope && + radial_blas_input_.empty(); +} + +bool GeneralWorkspace::use_chunked_blas_for_projection() const noexcept { + return problem_.dims.node_chunk < problem_.dims.n_nodes && use_blas_for_projection(); +} + +std::string GeneralWorkspace::synthesis_provider() const { + const auto provider = selected_blas_provider_name(); + if (folds_single_node()) + return "folded"; + if (use_packed_radial_blas()) + return std::string(provider) + "-packed-real"; + if (problem_.free_geometry()) + return "kokkos-fft"; + if (retained_synthesis_) + return std::string(provider) + "-retained-interleaved-real"; + if (use_interleaved_synthesis()) + return std::string(provider) + "-interleaved-real"; + return std::string(use_blas_for_synthesis() ? provider : std::string_view("custom")); +} + +std::string GeneralWorkspace::projection_provider() const { + const auto provider = selected_blas_provider_name(); + if (folds_single_node()) + return "folded"; + if (use_packed_radial_blas()) + return std::string(provider) + "-packed-real"; + if (problem_.free_geometry()) + return "kokkos-fft"; + if (use_interleaved_projection()) + return std::string(provider) + "-interleaved-real"; + return std::string(use_blas_for_projection() ? provider : std::string_view("custom")); +} + +std::string_view GeneralWorkspace::gemm_shape_policy() { + return "host radial scalar envelope uses packed real BLAS for whole transforms " + "with n_nodes/n_to >= 256 and a 22-column inventory within 64 MiB; " + "device radial scalar envelope uses vendor complex BLAS with " + "n_nodes/n_to >= 256; otherwise hybrid uses KokkosBlas for every whole " + "transform on the host, and on a device for those with n_modes >= 8, " + "n_nodes >= 8 and n_to >= 1024: real ones directly, an analytic field's " + "synthesis as one real GEMM over its samples' interleaved real view, and its " + "projection; host radial analytic synthesis also uses unallocated row views " + "for partial chunks; an envelope's synthesis and projection the same way; " + "OpenBLAS, " + "cuBLAS and rocBLAS builds also use " + "preconverted complex GEMM for eligible envelope transforms; eligible chunked " + "transforms are packed into persistent buffers for vendor BLAS; vector state " + "uses custom; a host build without a vendor BLAS runs KokkosBlas's native " + "arithmetic through a tiled host kernel, bit for bit; a compute capability " + "8.6 build forms the real GEMMs with a device kernel in place of cuBLAS"; +} + +std::size_t GeneralWorkspace::response_team_size() const noexcept { + constexpr auto configured = static_cast(SOLPROP_NATIVE_DEVICE_TEAM_SIZE); +#ifdef KOKKOS_ENABLE_CUDA + // Measured on a Hopper GPU, the crossover is sharp: 4,096-sample fibre problems + // favour teams of 256, while 8,192 samples and more favour 512. Keep an explicitly + // configured smaller team intact. + if (problem_.dims.n_to < 8192) + return std::min(configured, std::size_t{256}); +#endif + return configured; +} + +// Size from the plan's real-side batch samples, using the shared complete-window +// thresholds in core.hpp. Explicit FFT-pool requests bypass automatic sizing. +// FFTW's factorisation can change with the width; comparisons use solprop's SEP 25 +// transform/propagation rounding budgets rather than bitwise agreement. +const ExecutionSpace& GeneralWorkspace::fft_space_for(const std::size_t samples, + const std::size_t length) { + // NOLINTNEXTLINE(readability-suspicious-call-argument): the names only coincide + return fft_pool_.get(CpuOperation::fft, samples, length); +} + +void GeneralWorkspace::set_initial(const std::span field, + const std::span noise) { + require_extent(field.size(), problem_.n_columns() * problem_.dims.n_omega, + "initial field"); + if (!noise.empty()) + require_extent(noise.size(), field.size(), "noise"); + upload_span(execution_space_, crossing_ring_, current_, field); + crossed_.from_host += field.size_bytes(); + if (noise.empty()) { + if (problem_.stepper.modified_noise) + graph::deep_copy(execution_space_, noise_, Complex{}); + has_noise_ = false; + } else { + if (!problem_.stepper.modified_noise) + throw std::invalid_argument( + "a native noise field is valid only for modified noise"); + upload_span(execution_space_, crossing_ring_, noise_, noise); + crossed_.from_host += noise.size_bytes(); + has_noise_ = true; + } + mask(current_); + if (!stash_current_.empty()) + graph::deep_copy(execution_space_, stash_current_, 0.0); + execution_space_.fence("solprop_native set_initial"); + initialized_ = true; + rk_source_evaluations_ = 0; + table_ready_ = false; + stages_ready_ = false; + stash_is_current_ = false; + quadrature_error_ = Kokkos::Experimental::quiet_NaN_v; + quadrature_stale_ = false; + graph::deep_copy(execution_space_, quadrature_scores_, + Kokkos::Experimental::quiet_NaN_v); + clear_fsal(); +} + +// Point the per-stage scalar views at a slot of the packed buffer. +void GeneralWorkspace::bind_stage_scalars(const std::size_t slot) { + const auto stages = problem_.stepper.n_stages; + const auto channels = problem_.plasma.enabled ? problem_.plasma.n_channels : 0; + double* const base = stage_scalars_.data() + slot * stage_slot_size_; + const auto region = [&](const std::size_t index) { + return base + stage_offsets_[index]; + }; + const auto extent = [&](const std::size_t index) { + return stage_offsets_[index + 1] - stage_offsets_[index]; + }; + stage_phi_delta_ = RealView(region(0), extent(0)); + stage_phi_at_ = RealView(region(1), extent(1)); + stage_prefactor_ = RealView(region(2), extent(2)); + stage_density_ = RealView(region(3), stages); + stage_kerr_density_ = RealView(region(4), stages); + stage_plasma_density_ = RealView(region(5), stages); + stage_raman_density_ = RealView(region(6), stages); + stage_channel_density_ = RealView2(region(7), stages, channels); + stage_channel_preionisation_ = RealView2(region(8), stages, channels); + stage_taper_ = RealView(region(9), stages); + stage_preionisation_ = RealView(region(10), stages); + if (problem_.raman.enabled) { + stage_raman_rates_ = RealView2(region(11), stages, problem_.raman.n_terms); + if (problem_.raman.n_components > 1) + stage_raman_densities_ = + RealView2(region(12), stages, problem_.raman.n_components); + } + // A compensated trial's rate weights are its step's, in slot 0 whichever slot the + // stage views read. + stage_rate_weights_ = + RealView(stage_scalars_.data() + stage_offsets_[13], extent(13)); +} + +// The host's record of a trial's scalars beside the views: which stages' phase +// weights repeat the stage before's, and the dispersive response's densities. +void GeneralWorkspace::note_trial_scalars(const TrialScalars& scalars) { + // Rows of equal weights give bitwise equal phases (a zero weight's term is skipped). + const auto n_phi = problem_.linear.n_phi_terms; + const auto same_row = [&](const std::size_t row) { + return std::equal( + scalars.phi_delta.begin() + static_cast(row * n_phi), + scalars.phi_delta.begin() + static_cast((row + 1) * n_phi), + scalars.phi_delta.begin() + static_cast((row - 1) * n_phi)); + }; + trial_taper_.assign(scalars.taper.begin(), scalars.taper.end()); + phase_repeats_.assign(problem_.stepper.n_stages, false); + for (std::size_t stage = 1; stage < problem_.stepper.n_stages; ++stage) + phase_repeats_[stage] = same_row(stage); + whole_repeats_ = problem_.stepper.n_stages > 0 && same_row(problem_.stepper.n_stages); + entered_stage_ = std::numeric_limits::max(); + if (auto* weights = dispersive_weights()) + weights->set_densities(scalars.dispersive_densities); +} + +// Fill the next staging buffer with the trial's scalars, at each view's place in +// slot 0 -- and an Adams trial's sample's in slot 1, and a compensated trial's rate +// weights -- then copy it across once (a host build's mirror is the buffer itself) +// and point the views at slot 0. `use_uploaded_sample` turns them to slot 1. +void GeneralWorkspace::upload_trial(const TrialScalars& scalars, + const TrialScalars* sample, + const std::vector* rate) { + auto staging = stage_staging_[stage_staging_next_]; + const bool staged = staging.data() != stage_scalars_.data(); + // The ring comes round: every earlier copy from it has run once the space drains. + if (staged && stage_staging_next_ == 0) + execution_space_.fence("solprop_native stage staging reuse"); + const auto& previous = + stage_staging_[(stage_staging_next_ + stage_staging_.size() - 1) % + stage_staging_.size()]; + stage_staging_next_ = (stage_staging_next_ + 1) % stage_staging_.size(); + if (staged) + // A device's staging buffer starts from the last values uploaded, so that a view + // this trial does not set keeps them, as the separate copies did. + std::copy(previous.data(), previous.data() + staging.size(), staging.data()); + const auto fill_slot = [&](const std::size_t slot, const TrialScalars& values) { + double* const base = staging.data() + slot * stage_slot_size_; + const auto place = [&](const std::size_t region, + const std::vector& source) { + if (source.size() != stage_offsets_[region + 1] - stage_offsets_[region]) + throw std::invalid_argument("fixed problem array has the wrong extent"); + std::ranges::copy(source, base + stage_offsets_[region]); + }; + const auto fill = [&](const std::size_t region, const double value) { + std::fill(base + stage_offsets_[region], base + stage_offsets_[region + 1], + value); + }; + place(0, values.phi_delta); + place(1, values.phi_at); + place(2, values.prefactor); + place(3, values.density); + place(4, values.kerr_density.empty() ? values.density : values.kerr_density); + if (problem_.plasma.enabled) { + if (values.plasma_densities.empty()) { + place(5, values.density); + place(7, values.density); + place(8, values.preionisation); + } else { + place(7, values.plasma_densities); + place(8, values.plasma_preionisation); + // The sole channel already has stage-major contiguous scalar vectors. + // Multiple channels consume their own matrices and need no extracted copy. + if (problem_.plasma.n_channels == 1) { + place(5, values.plasma_densities); + place(10, values.plasma_preionisation); + } + } + } + if (problem_.raman.enabled) { + if (values.raman_densities.empty()) + place(6, values.density); + else if (problem_.raman.n_components == 1) + place(6, values.raman_densities); + else + fill(6, 1.0); + place(11, values.raman_rates); + if (problem_.raman.n_components > 1) + place(12, values.raman_densities); + } + place(9, values.taper); + if (problem_.plasma.enabled && values.plasma_preionisation.empty()) + place(10, values.preionisation); + }; + fill_slot(0, scalars); + if (sample != nullptr) + fill_slot(1, *sample); + if (rate != nullptr) { + require_extent(rate->size(), problem_.linear.n_phi_terms, "Adams rate weights"); + std::ranges::copy(*rate, staging.data() + stage_offsets_[13]); + } + graph::deep_copy(execution_space_, stage_scalars_, staging); + bind_stage_scalars(0); + note_trial_scalars(scalars); +} + +// An Adams trial's sample scalars, uploaded with its step's: the views turn to slot 1. +void GeneralWorkspace::use_uploaded_sample(const TrialScalars& sample) { + bind_stage_scalars(1); + note_trial_scalars(sample); +} + +void GeneralWorkspace::prepare_raman(const std::vector& rates, + const std::vector& densities) { + if (!problem_.raman.enabled) + throw std::logic_error("this state has no Raman workspace"); + require_extent(rates.size(), problem_.raman.n_terms, "Raman decay rates"); + if (!std::ranges::all_of(rates, [](const double value) { + return std::isfinite(value) && value >= 0.0; + })) + throw std::invalid_argument("Raman decay rates must be finite and non-negative"); + auto host = Kokkos::create_mirror_view(stage_raman_rates_); + for (std::size_t term = 0; term < problem_.raman.n_terms; ++term) + host(0, term) = rates[term]; + graph::deep_copy(execution_space_, stage_raman_rates_, host); + if (problem_.raman.n_components > 1) { + require_extent(densities.size(), problem_.raman.n_components, + "Raman amplitude densities"); + if (!std::ranges::all_of(densities, [](double value) { + return std::isfinite(value) && value >= 0.0; + })) + throw std::invalid_argument( + "Raman amplitude densities must be finite and non-negative"); + auto density_host = Kokkos::create_mirror_view(stage_raman_densities_); + for (std::size_t component = 0; component < densities.size(); ++component) + density_host(0, component) = densities[component]; + graph::deep_copy(execution_space_, stage_raman_densities_, density_host); + } + prepare_raman_stage(0, rates.data(), densities.data()); + execution_space_.fence("solprop_native Raman diagnostic boundary"); +} + +void GeneralWorkspace::prepare_raman_stage(const std::size_t stage, const double* rates, + const double* densities) { + bool unchanged = raman_kernel_ready_; + for (std::size_t term = 0; term < problem_.raman.n_terms; ++term) + unchanged = unchanged && cached_raman_rates_[term] == rates[term]; + if (problem_.raman.n_components > 1) + for (std::size_t component = 0; component < problem_.raman.n_components; + ++component) + unchanged = + unchanged && cached_raman_densities_[component] == densities[component]; + if (unchanged) + return; + for (std::size_t term = 0; term < problem_.raman.n_terms; ++term) + cached_raman_rates_[term] = rates[term]; + if (problem_.raman.n_components > 1) + for (std::size_t component = 0; component < problem_.raman.n_components; + ++component) + cached_raman_densities_[component] = densities[component]; + + Kokkos::Profiling::pushRegion("solprop_native.raman.kernel_rebuild"); + const auto undamped = raman_undamped_; + const auto weights = raman_tensor_weights_; + const auto kernel = raman_kernel_time_; + const auto stage_rates = stage_raman_rates_; + const auto amplitudes = stage_raman_densities_; + const auto row_component = raman_row_component_; + const auto fractions = raman_component_fraction_; + const bool multiple = problem_.raman.n_components > 1; + const bool tensor = problem_.effective_node_pol() == 2; + const auto terms = problem_.raman.n_terms; + const auto doubled = 2 * problem_.dims.n_to; + const double fraction = problem_.raman.fraction; + const double dt = problem_.grid.dt_fine; + launch::parallel_for( + "solprop_native.raman.kernel_rebuild", + Range2(raman_rebuild_space_, {0, 0}, {problem_.raman_kernels(), doubled}), + KOKKOS_LAMBDA(const std::size_t channel, const std::size_t time) { + double total = 0.0; + const double delay = static_cast(time) * dt; + if (multiple) { + for (std::size_t component = 0; component < amplitudes.extent(1); + ++component) { + double response = 0.0; + for (std::size_t term = 0; term < terms; ++term) { + if (static_cast(row_component(term)) == component) { + const double damped = undamped(term, time) * + Kokkos::exp(-stage_rates(stage, term) * delay); + response += tensor ? weights(term, channel) * damped : damped; + } + } + total += amplitudes(stage, component) * fractions(component) * response; + } + kernel(channel, time) = total; + } else { + for (std::size_t term = 0; term < terms; ++term) { + const double damped = + undamped(term, time) * Kokkos::exp(-stage_rates(stage, term) * delay); + total += tensor ? weights(term, channel) * damped : damped; + } + kernel(channel, time) = fraction * total; + } + }); + Kokkos::Profiling::popRegion(); + + Kokkos::Profiling::pushRegion("solprop_native.raman.kernel_fft"); + execute_raman(*raman_kernel_forward_plan_, raman_kernel_time_, raman_kernel_spectrum_, + KokkosFFT::Normalization::none); + const auto spectrum = raman_kernel_spectrum_; + launch::parallel_for( + "solprop_native.raman.kernel_dt_scale", + Range2(raman_scale_space_, {0, 0}, + {problem_.raman_kernels(), problem_.dims.n_to + 1}), + KOKKOS_LAMBDA(const std::size_t channel, const std::size_t index) { + spectrum(channel, index) *= dt; + }); + Kokkos::Profiling::popRegion(); + + raman_kernel_ready_ = true; + ++raman_kernel_transforms_; +} + +void GeneralWorkspace::prepare_trial(const double dz, const TrialScalars& scalars) { + if (problem_.stepper.low_storage) + refuse("trial_step: a low-storage state of ", current_.extent(0), + " x ", current_.extent(1), + " holds no Runge-Kutta views; its steps are " + "low_storage_step's"); + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + require_trial_scalars(scalars); + coarse_pending_ = false; + upload_trial(scalars); + if (flow_) + flow_->set_trial(scalars.flow_section, scalars.flow_weights); + if (general_) + general_->set_weights(scalars.transform_weights); + std::size_t first = 0; + if (has_recycled_) { + reuse_fsal(); + first = 1; + } + // A stage's finish runs in the next stage's entry kernel (solprop SEP 25 section + // 6.7) unless a coupled flow carries the stage back first, or the stage is the last. + finish_pending_ = false; + const bool deferrable = !flow_ && !problem_.separable_geometry(); + defer_quadrature_ = deferred_quadrature_eligible(); + for (std::size_t stage = first; stage < problem_.stepper.n_stages; ++stage) { + try { + if (problem_.raman.enabled) + prepare_raman_stage( + stage, scalars.raman_rates.data() + stage * problem_.raman.n_terms, + problem_.raman.n_components > 1 + ? scalars.raman_densities.data() + stage * problem_.raman.n_components + : nullptr); + defer_finish_ = deferrable && stage + 1 < problem_.stepper.n_stages; + project_coarse_ = !defer_quadrature_ || stage + 1 == problem_.stepper.n_stages; + evaluate_stage(stage, dz); + } catch (...) { + defer_finish_ = false; + finish_pending_ = false; + defer_quadrature_ = false; + project_coarse_ = true; + coarse_pending_ = false; + throw; + } + } + defer_finish_ = false; + defer_quadrature_ = false; + project_coarse_ = true; + if (finish_pending_) + throw std::logic_error("a deferred stage finish was never run"); + stages_ready_ = true; +} + +std::vector GeneralWorkspace::trial_step(const double dz, + const TrialScalars& scalars) { + prepare_trial(dz, scalars); + if (flow_) + flow_->carry_step(current_, solution_, error_); + const auto current = current_; + const auto carried = flow_ ? flow_->carried_state() : current_; + const auto solution = flow_ ? flow_->carried_solution() : solution_; + const auto error = flow_ ? flow_->carried_error() : error_; + const auto whole = whole_propagator_; + const auto advanced = advanced_; + const auto residual = residual_; + const auto in_norm = norm_mask_; + const auto partials = partials_; + const auto count = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + const auto slots = reduction_slots_; + accept_update( + execution_space_, count, n, dz, + KOKKOS_LAMBDA(const std::size_t column) { + const auto slot = column * slots; + return UpdateColumn{.current = current.data() + column * current.stride(0), + .solution = solution.data() + column * solution.stride(0), + .error = error.data() + column * error.stride(0), + .whole = whole.data() + column * whole.stride(0), + .advanced = advanced.data() + column * advanced.stride(0), + .residual = residual.data() + column * residual.stride(0), + .in_norm = in_norm.data(), + .old_partial = partials.data() + slot, + .new_partial = partials.data() + partials.stride(0) + slot, + .error_partial = + partials.data() + 2 * partials.stride(0) + slot, + .carried = carried.data() + column * carried.stride(0)}; + }, + "solprop_native.step.accept_error_partials", slots); + return collect_trial(); +} + +// solprop's low-call Adams march (SEP 13, SEP 14 phase 2): one source evaluation at a +// field given in the step's anchor frame, `U(anchor, position) N(U(position, anchor) +// v)`. It is stage zero of a trial whose stage-zero scalars are the evaluation +// position's: the combination with no earlier stage carries the input to the position +// and the finish brings the source back, so the shared stage pipeline, its responses +// and its in-stage absorber are the march's too. The accepted state is held in +// `advanced_`, which only the Runge-Kutta accept writes, while the input takes its +// place. +std::vector GeneralWorkspace::adams_source(const std::vector& input, + const TrialScalars& scalars) { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + // Every route's stage source serves the Adams march (solprop SEP 16 phase 3): the + // separable stage takes its flow from the stage's own Phi increment, as the modal one + // does. Beside a coupled flow the sample is the material source alone (solprop SEP + // 16 phase 5): the exponential formulation carries the residual in its compensation + // and the flow in its moments, so the stage runs without the flow's pipeline. + require_extent(input.size(), problem_.n_columns() * problem_.dims.n_omega, + "Adams input"); + // The flow is set aside for the evaluation and put back whatever happens. + auto held_flow = std::exchange(flow_, nullptr); + // An aggregate, so it declares no copy or move; it is never copied or moved. + // NOLINTNEXTLINE(cppcoreguidelines-special-member-functions): a local scope guard + struct RestoreFlow { + std::unique_ptr& slot; + std::unique_ptr& held; + // NOLINTNEXTLINE(modernize-use-equals-default): restores the slot (see FlowAside). + ~RestoreFlow() { slot = std::move(held); } + } restore{.slot = flow_, .held = held_flow}; + require_trial_scalars(scalars); + upload_trial(scalars); + if (general_) + general_->set_weights(scalars.transform_weights); + graph::deep_copy(execution_space_, advanced_, current_); + copy_vector(execution_space_, current_, input); + if (problem_.raman.enabled) + prepare_raman_stage(0, scalars.raman_rates.data(), + problem_.raman.n_components > 1 ? scalars.raman_densities.data() + : nullptr); + evaluate_stage_source(0, 0.0); + graph::deep_copy(execution_space_, current_, advanced_); + stages_ready_ = true; + // No Runge-Kutta stage of this state exists to recycle. + pending_fsal_ = false; + has_recycled_ = false; + return stage_to_host(0); +} + +// Adopt an Adams step's end state, given on the host before its boundary treatments: +// the shared finish masks it, or applies the split-step absorber, and makes it current. +void GeneralWorkspace::adopt(const std::vector& field, const double dz) { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + require_extent(field.size(), problem_.n_columns() * problem_.dims.n_omega, + "Adams state"); + copy_vector(execution_space_, advanced_, field); + finish(dz); + // The stash a response pass left is the predicted field's, not this one's. + stash_is_current_ = false; +} + +// --- solprop SEP 24 phase 3: the low-storage RK4IP march ----------------------------- +// +// Classical RK4 in the interaction picture anchored at the step's midpoint (solprop's +// backends/low_storage.py, SEP 24 section 4.3). The state is the register A; B +// accumulates the bracket of the accepted step and C holds the next pass's lab-frame +// input, unless it is elided (solprop's `low_storage_stage_register`): then each of the +// first three finishes writes that input straight into the transverse scratch. A step +// is four passes, each forming its input in the route's scratch, running the stage core +// the Runge-Kutta march shares (`separable_core`, `modal_core`; SEP 24 Q5) and folding +// the source into the registers per element: +// +// pass 1, F1 at z_n: A <- P A; B <- A + h/6 P S; C <- A + h/2 P S +// pass 2, F2 at z_m: B <- B + h/3 S; C <- A + h/2 S +// pass 3, F3 at z_m: B <- B + h/3 S; C <- Q (A + h S) +// pass 4, F4 at end: A <- Q B + h/6 S +// +// Every pass reads the source at its own position, so nothing carries it back to an +// anchor, and only forward flows are applied. The step is accepted in place; the +// caller's finish masks it or applies the split-step collar. + +namespace { + +constexpr int fold_complete = 0; +constexpr int fold_exchange = 1; +constexpr int fold_second = 2; +constexpr int term_none = 0; +constexpr int term_material = 1; +constexpr int term_absorber = 2; + +// The scalar row each pass reads: the step's start, its midpoint twice, its end. +constexpr std::size_t low_storage_rows[4] = {0, 1, 1, 2}; + +// One evaluation's fold into the registers at one element: solprop's +// `bodies.low_storage_fold`, the finish column of SEP 24 section 4.3. A pass that +// evaluates twice -- the material response, then the in-stage absorber's term, from the +// same input -- folds the first at the exchange (pass 3's C left unflowed) and the +// second after it, whose sum is the pass's. +KOKKOS_INLINE_FUNCTION void +low_storage_fold(const int stage, const int mode, const double step, + const Complex source, Complex& state, Complex& accumulator, + Complex& stage_register, const Complex first, const Complex second) { + if (stage == 0) { + const Complex slope = first * source; + if (mode == fold_second) { + accumulator = accumulator + (step / 6.0) * slope; + stage_register = stage_register + (0.5 * step) * slope; + return; + } + const Complex moved = first * state; + state = moved; + accumulator = moved + (step / 6.0) * slope; + stage_register = moved + (0.5 * step) * slope; + return; + } + if (stage == 1) { + accumulator = accumulator + (step / 3.0) * source; + stage_register = + (mode == fold_second ? stage_register : state) + (0.5 * step) * source; + return; + } + if (stage == 2) { + accumulator = accumulator + (step / 3.0) * source; + if (mode == fold_complete) + stage_register = second * (state + step * source); + else if (mode == fold_exchange) + stage_register = state + step * source; + else + stage_register = second * (stage_register + step * source); + return; + } + if (mode == fold_second) { + state = state + (step / 6.0) * source; + return; + } + state = second * accumulator + (step / 6.0) * source; +} + +// A register traversal over the state's elements. The last fold of a step also reduces +// the accepted state's squared norm per column into `partials(0, column)`, one team per +// column as the Runge-Kutta accept reduces its own (its sums associate as the team +// splits them, solprop SEP 25). +template +void register_traversal(const char* label, const ExecutionSpace& execution, + const std::size_t columns, const std::size_t n, + const RealView2& partials, const bool reduce, + const Element& element) { + const auto traversal_for = [](const char* name, const ExecutionSpace& space, + const std::size_t rows, const std::size_t width, + const auto& body) { + if constexpr (Crossing) + crossing_for(name, space, rows, width, body); + else + cartesian_for(name, space, rows, width, body); + }; + if (!reduce) { + traversal_for( + label, execution, columns, n, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + static_cast(element(column, index)); + }); + return; + } +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + if constexpr (Crossing) { + // Blocks of adjacent columns, each column's sum in ascending bin order. + constexpr std::size_t block = crossing_tile_columns; + constexpr std::size_t bins = crossing_tile_bins; + const std::size_t blocks = (columns + block - 1) / block; + launch::parallel_for( + label, Kokkos::RangePolicy(execution, 0, blocks), + KOKKOS_LAMBDA(const std::size_t which) { + const std::size_t first = which * block; + const std::size_t count = columns - first < block ? columns - first : block; + double totals[block] = {}; + for (std::size_t start = 0; start < n; start += bins) { + const std::size_t stop = n - start < bins ? n : start + bins; + for (std::size_t index = start; index < stop; ++index) + for (std::size_t offset = 0; offset < count; ++offset) + totals[offset] += element(first + offset, index); + } + for (std::size_t offset = 0; offset < count; ++offset) + partials(0, first + offset) = totals[offset]; + }); + return; + } +#endif + using Policy = Kokkos::TeamPolicy; + using Member = Policy::member_type; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const Policy policy(execution, static_cast(columns), + SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); + launch::parallel_for( + label, policy, KOKKOS_LAMBDA(const Member& team) { + const auto column = static_cast(team.league_rank()); + double total = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n)), + [&](const int index, double& update) { + update += element(column, static_cast(index)); + }, + total); + Kokkos::single(Kokkos::PerTeam(team), [&]() { partials(0, column) = total; }); + }); +#else + const Policy policy(execution, static_cast(columns), + host_reduction_team_size(execution, columns, n)); + launch::parallel_for( + label, policy, KOKKOS_LAMBDA(const Member& team) { + const auto column = static_cast(team.league_rank()); + double total = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, n), + [&](const std::size_t index, double& update) { + update += element(column, index); + }, + total); + Kokkos::single(Kokkos::PerTeam(team), [&] { partials(0, column) = total; }); + }); +#endif +} + +} // namespace + +double GeneralWorkspace::low_storage_step(const double dz, const TrialScalars& scalars, + const bool rebuild_table) { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + if (!problem_.stepper.low_storage) + throw std::logic_error("this native state was not built for the low-storage march"); + if (!(dz > 0.0) || !std::isfinite(dz)) + throw std::invalid_argument("the low-storage step must be positive and finite"); + if (problem_.stepper.n_stages < 3) + throw std::invalid_argument("the low-storage march reads three scalar rows"); + if (rebuild_table && problem_.stepper.low_storage_flows != 1) + throw std::logic_error("only a constant operator's table is rebuilt"); + if (problem_.stepper.low_storage_flows == 1 && !rebuild_table && !table_ready_) + throw std::logic_error("the low-storage table has not been built since the initial " + "field was loaded"); + require_trial_scalars(scalars); + upload_trial(scalars); + if (general_) + general_->set_weights(scalars.transform_weights); + low_storage_flows_formed(rebuild_table); + if (rebuild_table) + table_ready_ = true; + for (std::size_t pass = 0; pass < 4; ++pass) { + const auto row = low_storage_rows[pass]; + if (problem_.raman.enabled) + prepare_raman_stage( + row, scalars.raman_rates.data() + row * problem_.raman.n_terms, + problem_.raman.n_components > 1 + ? scalars.raman_densities.data() + row * problem_.raman.n_components + : nullptr); + if (problem_.separable_geometry()) + low_storage_separable_pass(pass, dz); + else + low_storage_modal_pass(pass, dz); + } + // No Runge-Kutta stage exists to observe or recycle. + stages_ready_ = false; + stash_is_current_ = false; + execution_space_.fence("solprop_native low-storage step"); + double total = 0.0; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // One number crosses, not the whole of the partials' table. + const auto partials = partials_; + launch::parallel_reduce( + "solprop_native.low_storage.norm_total", + Kokkos::RangePolicy(execution_space_, 0, problem_.n_columns()), + KOKKOS_LAMBDA(const std::size_t column, double& sum) { + sum += partials(0, column); + }, + Kokkos::Sum(total)); +#else + for (std::size_t column = 0; column < problem_.n_columns(); ++column) + total += partials_(0, column); +#endif + return total; +} + +// The step's held flows: a constant operator's table `exp(h_t D / 2)`, built from the +// table's interval (scalar row 2 of `phi_delta`) when the caller's key changed, or a +// varying operator's `P` and `Q` from rows 0 and 1, every step. Recomputed flows are +// formed in the finishes that apply them and hold nothing. +void GeneralWorkspace::low_storage_flows_formed(const bool rebuild_table) { + const auto flows = low_storage_flows_; + const auto count = problem_.stepper.low_storage_flows; + if (count == 0 || (count == 1 && !rebuild_table)) + return; + const auto n = problem_.dims.n_omega; +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // With vector maths a host task forms a column's exponents a block of bins at a + // time and their exponentials in one pass (solprop ADR-059); a separable route's + // task, a block of adjacent columns at one bin, its table's run in the scratch's + // layout (`RegisterIndex`). + if constexpr (vmath::vectorised) { + if (problem_.separable_geometry()) { + const auto spatial = spatial_; + const auto delta = stage_phi_delta_; + const auto active = active_mask_; + const auto index = register_index(); + const auto columns = problem_.n_columns(); + const std::size_t blocks = (columns + stage_block - 1) / stage_block; + launch::parallel_for( + "solprop_native.low_storage.separable_table", + Kokkos::RangePolicy(execution_space_, 0, n * blocks), + KOKKOS_LAMBDA(const std::size_t item) { + const std::size_t omega = item / blocks; + const std::size_t first = (item % blocks) * stage_block; + const std::size_t width = + columns - first < stage_block ? columns - first : stage_block; + const bool in_band = active(omega) != 0U; + double real[stage_block]; + double imag[stage_block]; + for (std::size_t offset = 0; offset < width; ++offset) { + const Complex x = + delta(2) * + spatial.value(index.mode(first + offset), omega, in_band).linear; + real[offset] = x.real(); + imag[offset] = x.imag(); + } + vmath::exp_complex(real, imag, real, imag, width); + for (std::size_t offset = 0; offset < width; ++offset) + flows.data()[index(first + offset, omega)] = + Complex(real[offset], imag[offset]); + }); + return; + } + const auto terms = stage_terms(0); + const auto n_phi = problem_.linear.n_phi_terms; + const std::size_t blocks = (n + stage_block - 1) / stage_block; + launch::parallel_for( + "solprop_native.low_storage.modal_flows", + Kokkos::RangePolicy(execution_space_, 0, + problem_.n_columns() * blocks), + KOKKOS_LAMBDA(const std::size_t item) { + const std::size_t column = item / blocks; + const std::size_t first = (item % blocks) * stage_block; + const std::size_t bins = n - first < stage_block ? n - first : stage_block; + double real[stage_block]; + double imag[stage_block]; + const auto form = [&](const std::size_t flow, const std::size_t offset) { + terms.phase_block(offset, column, first, bins, real, imag); + vmath::exp_complex(real, imag, real, imag, bins); + for (std::size_t bin = 0; bin < bins; ++bin) + flows(flow, column, first + bin) = Complex(real[bin], imag[bin]); + }; + if (count == 1) { + form(0, 2 * n_phi); + } else { + form(0, 0); + form(1, n_phi); + } + }); + return; + } +#endif + if (problem_.separable_geometry()) { + const auto spatial = spatial_; + const auto delta = stage_phi_delta_; + const auto active = active_mask_; + const auto n_pol = problem_.dims.n_pol_state; + const auto index = register_index(); + cartesian_for( + "solprop_native.low_storage.separable_table", execution_space_, + problem_.n_columns(), n, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + const auto values = spatial.value(column / n_pol, omega, active(omega) != 0U); + flows.data()[index(column, omega)] = propagator_exp(delta(2) * values.linear); + }); + return; + } + const auto terms = stage_terms(0); + const auto n_phi = problem_.linear.n_phi_terms; + cartesian_for( + "solprop_native.low_storage.modal_flows", execution_space_, problem_.n_columns(), + n, KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + if (count == 1) { + flows(0, column, index) = + propagator_exp(terms.phase_sum(2 * n_phi, column, index)); + } else { + flows(0, column, index) = propagator_exp(terms.phase_sum(0, column, index)); + flows(1, column, index) = + propagator_exp(terms.phase_sum(n_phi, column, index)); + } + }); +} + +// One pass on a separable geometry (free space and the hybrid planar kind). Its input +// is the register A (pass 1) or C, with the noise draw at the pass's position, in the +// transverse scratch -- already written by the previous finish when C is elided. Under +// an in-stage absorber the pass evaluates twice from the same input: the material +// source, then the absorber's term, the exchange between them taking the material +// output out of the scratch and re-forming the input (SEP 24 section 4.3). +void GeneralWorkspace::low_storage_separable_pass(const std::size_t pass, + const double dz) { + const bool material = problem_.kerr.nonlinear || problem_.plasma.enabled || + problem_.raman.enabled || separable_dispersive_ != nullptr; + const bool absorber = problem_.absorber.in_stage; + const bool elided = !problem_.stepper.low_storage_stage_register; + const auto row = low_storage_rows[pass]; + if ((material || absorber) && (pass == 0 || !elided)) + low_storage_separable_input(pass); + if (material && absorber) { + separable_core(row, true); + low_storage_separable_fold(pass, fold_exchange, term_material, dz); + separable_core(row, false); + low_storage_separable_fold(pass, fold_second, term_absorber, dz); + } else if (material || absorber) { + separable_core(row, material); + low_storage_separable_fold(pass, fold_complete, + material ? term_material : term_absorber, dz); + } else { + low_storage_separable_fold(pass, fold_complete, term_none, dz); + } +} + +namespace { +// Each column's masked squared norm of `state` into `partials(0, column)`, in a team +// per column: a device's final fold traverses like the others and reduces here, over +// one volume, instead of reducing inside the fold's crossing (SEP 28). +void column_norms(const char* label, const ExecutionSpace& execution, + const ComplexView2& state, const ByteView& in_norm, + const RealView2& partials) { + using Policy = Kokkos::TeamPolicy; + using Member = Policy::member_type; + const auto n = state.extent(1); + launch::parallel_for( + label, Policy(execution, static_cast(state.extent(0)), 128), + KOKKOS_LAMBDA(const Member& team) { + const auto column = static_cast(team.league_rank()); + double total = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n)), + [&](const int index, double& update) { + const auto omega = static_cast(index); + if (in_norm(omega) != 0U) + update += squared_abs(state(column, omega)); + }, + total); + Kokkos::single(Kokkos::PerTeam(team), [&] { partials(0, column) = total; }); + }); +} +} // namespace + +void GeneralWorkspace::low_storage_separable_input(const std::size_t pass) { + const auto state = current_; + const auto stage_register = register_stage_; + const auto index = stage_index(); + const auto spatial = spatial_; + const auto at = stage_phi_at_; + const auto active = active_mask_; + const auto noise = noise_; + const auto scratch = transverse_; + const bool add_noise = problem_.stepper.modified_noise && has_noise_; + const double scale = inverse_transverse_count(); + const auto row = low_storage_rows[pass]; + const auto input = KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + const auto mode = index.mode(column); + Complex value = + pass == 0 ? state(column, omega) : stage_register.data()[index(column, omega)]; + if (add_noise) { + const auto values = spatial.value(mode, omega, active(omega) != 0U); + const double angle = at(row) * values.linear.imag(); + value += noise(column, omega) * Complex(Kokkos::cos(angle), Kokkos::sin(angle)); + } + return value * scale; + }; + // In the volume route's order the scratch is the state's, so the input streams. + if (problem_.stepper.low_storage_tiled_input && !volume_order_) { + // Read along the state's bins and written along the scratch's rows, a tile at a + // time. + tiled_crossing( + "solprop_native.low_storage.separable_input", execution_space_, + problem_.n_columns(), problem_.dims.n_omega, false, input, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, + const Complex value) { + scratch.data()[index.cell(column, omega)] = value; + }); + return; + } + crossing_for( + "solprop_native.low_storage.separable_input", execution_space_, + problem_.n_columns(), problem_.dims.n_omega, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + scratch.data()[index.cell(column, omega)] = input(column, omega); + }); +} + +void GeneralWorkspace::low_storage_separable_fold(const std::size_t pass, + const int mode, const int term, + const double dz) { + const auto state = current_; + const auto accumulator = register_accumulator_; + const auto stage_register = register_stage_; + const auto flows = low_storage_flows_; + const bool tabulated = problem_.stepper.low_storage_flows != 0; + const auto scratch = transverse_; + const auto spatial = spatial_; + const auto window = omega_window_; + const auto active = active_mask_; + const auto in_norm = norm_mask_; + const auto noise = noise_; + const auto delta = stage_phi_delta_; + const auto at = stage_phi_at_; + // A device gathers and scatters the scratch through its own tiles. + [[maybe_unused]] const auto n_pol = problem_.dims.n_pol_state; + [[maybe_unused]] const auto layout = transverse_layout(); + const auto index = stage_index(); + const bool add_noise = problem_.stepper.modified_noise && has_noise_; + const double scale = inverse_transverse_count(); + const int stage = static_cast(pass); + const bool exchange = mode == fold_exchange; + const bool second = mode == fold_second; + // Elided, each of the first three passes writes the next pass's input in place of C. + const bool forward = !problem_.stepper.low_storage_stage_register && pass < 3; + const bool final = pass == 3 && !exchange; + // What this fold reads and writes, invariant over the traversal: each moves only the + // registers its row of the flow table names. + const bool flows_here = + stage == 0 || (stage == 2 && !exchange) || (stage == 3 && !second); + const bool reads_register = (second && stage < 3) || (exchange && stage > 0); + const bool reads_accumulator = stage > 0 || second; + const bool writes_state = (stage == 0 && !second) || stage == 3; + const bool writes_register = stage < 3 && !forward; + const auto own = low_storage_rows[pass]; + const auto next = low_storage_rows[pass < 3 ? pass + 1 : 3]; + const bool linear_here = add_noise || (flows_here && !tabulated); + const std::size_t flow_row = stage == 0 ? 0 : 1; + const double step = dz; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // A device reduces the accepted state's norms in a pass of their own. + const bool fused_norm = false; +#else + const bool fused_norm = final; +#endif + // The element, given its cell of the transverse scratch and its state (`held`). + const auto body = KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, + Complex& cell, Complex& held) { + const auto column_mode = index.mode(column); + const bool in_band = active(omega) != 0U; + // Tabulated flows without noise, or a pass with no flow, read only the prefactor of + // the element. + SpatialElement values{}; + if (linear_here) + values = spatial.value(column_mode, omega, in_band); + else if (term == term_material) + values.prefactor = spatial.prefactor(column_mode, omega, in_band); + Complex source{}; + if (term == term_material) + source = values.prefactor * window(omega) * cell; + else if (term == term_absorber && in_band) + source = cell; + const std::size_t slot = index(column, omega); + Complex flow(1.0, 0.0); + if (flows_here) + flow = tabulated ? flows.data()[slot] + : propagator_exp(delta(flow_row) * values.linear); + Complex a = held; + Complex b = reads_accumulator ? accumulator.data()[slot] : Complex{}; + Complex c = reads_register ? stage_register.data()[slot] : Complex{}; + if (exchange) { + Complex value = stage == 0 ? a : c; + if (add_noise) { + const double angle = at(own) * values.linear.imag(); + value += noise(column, omega) * Complex(Kokkos::cos(angle), Kokkos::sin(angle)); + } + cell = value * scale; + } + low_storage_fold(stage, mode, step, source, a, b, c, flow, flow); + if (forward) { + Complex value = c; + if (add_noise) { + const double angle = at(next) * values.linear.imag(); + value += noise(column, omega) * Complex(Kokkos::cos(angle), Kokkos::sin(angle)); + } + cell = value * scale; + } + if (writes_state) + held = a; + if (stage < 3) + accumulator.data()[slot] = b; + if (writes_register) + stage_register.data()[slot] = c; + return final && in_norm(omega) != 0U ? squared_abs(a) : 0.0; + }; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // In the volume route's order, the scratch's cell sits beside the registers' and the + // state's, so the fold streams all three. + if (volume_order_) + cartesian_for( + "solprop_native.low_storage.separable_fold", execution_space_, + problem_.n_columns(), problem_.dims.n_omega, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + static_cast(body(column, omega, + scratch.data()[index.cell(column, omega)], + state(column, omega))); + }); + else + tiled_update( + "solprop_native.low_storage.separable_fold", execution_space_, + problem_.n_columns(), problem_.dims.n_omega, exchange || forward, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + return layout.cell(scratch, omega * n_pol + column % n_pol, column / n_pol); + }, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, + Complex& cell) { + static_cast(body(column, omega, cell, state(column, omega))); + }, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, + const Complex value) { + layout.cell(scratch, omega * n_pol + column % n_pol, column / n_pol) = value; + }); +#else + // The host's registers sit at the scratch cell's own position (`RegisterIndex`), so a + // block of adjacent columns reads them, and the scratch, in contiguous runs at every + // bin; only the state is column-major. A work item takes a block of columns over + // every bin, a tile of bins at a time: it loads the tile's state a column (a + // contiguous run) at a time, folds the tile in the scratch's order, and stores the + // state back. The final fold sums each column's norm in ascending bin order (as + // `register_traversal`). + constexpr std::size_t block = crossing_tile_columns; + constexpr std::size_t bins = crossing_tile_bins; + const std::size_t columns = problem_.n_columns(); + const std::size_t n = problem_.dims.n_omega; + const std::size_t blocks = (columns + block - 1) / block; + const auto partials = partials_; + launch::parallel_for( + "solprop_native.low_storage.separable_fold", + Kokkos::RangePolicy(execution_space_, 0, blocks), + KOKKOS_LAMBDA(const std::size_t which) { + const std::size_t first = which * block; + const std::size_t count = columns - first < block ? columns - first : block; + double totals[block] = {}; + Complex tile[bins][block]; + for (std::size_t start = 0; start < n; start += bins) { + const std::size_t width = n - start < bins ? n - start : bins; + for (std::size_t offset = 0; offset < count; ++offset) { + const Complex* row = &state(first + offset, start); + for (std::size_t bin = 0; bin < width; ++bin) + tile[bin][offset] = row[bin]; + } + for (std::size_t bin = 0; bin < width; ++bin) + for (std::size_t offset = 0; offset < count; ++offset) { + const std::size_t column = first + offset; + const std::size_t omega = start + bin; + totals[offset] += + body(column, omega, scratch.data()[index(column, omega)], + tile[bin][offset]); + } + if (writes_state) + for (std::size_t offset = 0; offset < count; ++offset) { + Complex* row = &state(first + offset, start); + for (std::size_t bin = 0; bin < width; ++bin) + row[bin] = tile[bin][offset]; + } + } + if (fused_norm) + for (std::size_t offset = 0; offset < count; ++offset) + partials(0, first + offset) = totals[offset]; + }); +#endif + if (final && !fused_norm) + column_norms("solprop_native.low_storage.separable_norms", execution_space_, state, + in_norm, partials_); +} + +// One pass on a modal route (mode sets, radial, general modal): its input, the +// register A (pass 1) or C with the noise draw at the pass's position, padded into the +// fine spectrum (and the general route's scaled channels); the shared core; and one +// fold of the source -- prefactor, window and the in-stage absorber's term at the +// pass's position, with no flow back to an anchor. +void GeneralWorkspace::low_storage_modal_pass(const std::size_t pass, const double dz) { + const auto row = low_storage_rows[pass]; + const bool responds = general_ + ? general_->responds() || problem_.absorber.in_stage + : problem_.kerr.nonlinear || problem_.plasma.enabled || + problem_.raman.enabled || + problem_.absorber.in_stage || dispersive_ != nullptr; + if (responds) + low_storage_modal_input(pass); + const bool evaluated = modal_core(row); + const auto terms = stage_terms(row); + const auto state = current_; + const auto accumulator = register_accumulator_; + const auto stage_register = register_stage_; + const auto flows = low_storage_flows_; + const auto count = problem_.stepper.low_storage_flows; + const auto in_norm = norm_mask_; + const auto n_phi = problem_.linear.n_phi_terms; + const auto n_prefactor = problem_.linear.n_prefactor_terms; + const int stage = static_cast(pass); + const bool final = pass == 3; + const bool flows_here = stage != 1; + const std::size_t flow_row = stage == 0 ? 0 : 1; + const double step = dz; + register_traversal( + "solprop_native.low_storage.modal_fold", execution_space_, problem_.n_columns(), + problem_.dims.n_omega, partials_, final, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + Complex source{}; + if (evaluated) { + // Zero weights skipped, as in `StageTerms::finish`. + Complex prefactor{}; + for (std::size_t term = 0; term < n_prefactor; ++term) { + const double weight = terms.prefactor_weights(row * n_prefactor + term); + if (weight != 0.0) + prefactor += weight * terms.prefactor_terms(term, column, index); + } + source = prefactor * terms.window(index) * + (terms.general ? terms.general_source(column, index) + : terms.projected(column, terms.crop(index))); + if (terms.in_stage_absorber && terms.active(index) != 0U) + source += terms.absorber(column, terms.crop(index)); + } + Complex flow(1.0, 0.0); + if (flows_here) + flow = count == 1 ? flows(0, column, index) + : count == 2 + ? flows(flow_row, column, index) + : propagator_exp(terms.phase_sum(flow_row * n_phi, column, index)); + Complex a = state(column, index); + Complex b = stage > 0 ? accumulator(column, index) : Complex{}; + Complex c{}; + low_storage_fold(stage, fold_complete, step, source, a, b, c, flow, flow); + if (stage == 0 || stage == 3) + state(column, index) = a; + if (stage < 3) { + accumulator(column, index) = b; + stage_register(column, index) = c; + } + return final && in_norm(index) != 0U ? squared_abs(a) : 0.0; + }); +} + +void GeneralWorkspace::low_storage_modal_input(const std::size_t pass) { + const auto source = pass == 0 ? current_ : register_stage_; + const auto row = low_storage_rows[pass]; + const auto terms = stage_terms(row); + const auto n = problem_.dims.n_omega; + const auto gaps = fine_spectrum_size() - n; + cartesian_for( + "solprop_native.low_storage.modal_input", execution_space_, problem_.n_columns(), + n, KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + Complex value = source(column, index); + if (terms.add_noise) { + const Complex phase_at = terms.noise_phase(row, column, index); + value += terms.noise(column, index) * + Complex(Kokkos::cos(phase_at.imag()), Kokkos::sin(phase_at.imag())); + } + const double analytic_scale = + terms.analytic && index > 0 && 2 * terms.crop(index) < terms.n_to ? 2.0 + : 1.0; + terms.fine(column, terms.crop(index)) = + analytic_scale * terms.inverse_n_to * value; + if (terms.general) + terms.general_scaled(column, index) = + value * + (terms.general_pad * general_doubling(terms.crop(index), terms.n_to)); + for (std::size_t gap = index; gap < gaps; gap += n) + terms.zero_gaps(column, gap, gap + 1); + }); +} + +// --- solprop SEP 16 phase 3b: the Adams history resident in the engine --------------- +// +// solprop's HostAdamsMarch keeps the controller and hands every state-sized operation +// of its fused path to an engine (solprop ADR-043). These are that engine's passes: the +// history's rows, the trial's predictor, source, corrected field and defect norms, the +// adoption and transport of an accepted step and dense observation all stay in device +// memory; only the trial's scalars, its coefficient tables (by value, in AdamsWeights) +// and the norms cross. Each pass is the host engine's arithmetic (solprop's +// backends/multistep.py, HostHistory and AdamsKernels), checked against it in parity. + +void GeneralWorkspace::adams_allocate(const std::size_t slots, const bool compensated, + const bool interpolate) { + if (slots == 0 || slots > adams_max_samples) + throw std::invalid_argument("the Adams history needs 1 to 8 rows"); + if (compensated && problem_.separable_geometry()) + throw std::logic_error("a separable operator needs no compensation"); + const auto columns = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + if (compensated && adams_states_.extent(0) != slots) { + adams_states_ = ComplexView3("solprop_native.adams.states", slots, columns, n); + adams_rates_ = ComplexView3("solprop_native.adams.rates", slots, columns, n); + adams_carried_ = ComplexView3("solprop_native.adams.carried", slots, columns, n); + adams_lab_ = ComplexView2("solprop_native.adams.predicted", columns, n); + adams_endpoint_ = ComplexView2("solprop_native.adams.endpoint", columns, n); + adams_endpoint_rate_ = + ComplexView2("solprop_native.adams.endpoint_rate", columns, n); + adams_rate_weights_ = Kokkos::View( + "solprop_native.adams.rate_weights", problem_.linear.n_phi_terms); + } + // Dense observations are opt-in. Keeping their slopes and saved fields on a run + // that lands on its planes costs (slots + 4) whole fields without reading them. + if (interpolate) { + if (adams_slopes_.extent(0) != slots) { + adams_slopes_ = + ComplexView3("solprop_native.adams.dense_slopes", slots, columns, n); + adams_held_ = ComplexView2("solprop_native.adams.held_state", columns, n); + adams_dense_start_ = ComplexView2("solprop_native.adams.dense_start", columns, n); + adams_dense_end_ = ComplexView2("solprop_native.adams.dense_end", columns, n); + adams_dense_exponent_ = + ComplexView2("solprop_native.adams.dense_exponent", columns, n); + } + if (compensated && adams_dense_terms_.extent(0) != slots) + adams_dense_terms_ = + ComplexView3("solprop_native.adams.dense_terms", slots, columns, n); + } else { + adams_slopes_ = {}; + adams_held_ = {}; + adams_dense_start_ = {}; + adams_dense_end_ = {}; + adams_dense_exponent_ = {}; + adams_dense_terms_ = {}; + adams_dense_count_ = 0; + } + // Beside a coupled flow the trial's block functions, and a frozen-residual flow's + // weights per sample, live beside the history (solprop SEP 16 phase 6). + if (flow_) { + if (!flow_->forms_actions()) + throw std::logic_error("the resident Adams engine needs the flow's eigenbasis " + "and route"); + // A bin off the route takes the augmented exponential: a trial reads one + // section, so the scratch is one row per active bin, whatever the sections. A + // device's carry also keeps a bin's channel vectors there, in blocks 3 and 4 + // (solprop SEP 47 phase 4), which `augmented_blocks` holds for any history. + const auto rows = flow_->n_fallback() == 0 ? 0 : flow_->n_active(); + const auto blocks = flow_functions::augmented_blocks(slots); + if (rows == 0) + adams_fallback_ = {}; + else if (adams_fallback_.extent(0) != rows || adams_fallback_.extent(1) != blocks) + adams_fallback_ = Kokkos::View( + "solprop_native.adams.fallback_functions", rows, blocks, + problem_.dims.n_modes, problem_.dims.n_modes); + if (columns != problem_.dims.n_modes) + throw std::logic_error("the resident Adams engine steps a coupled block of one " + "batch row"); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // A wider block's host passes hold its channel vectors in a row per bin; a + // device's passes take a thread per (bin, channel) and hold none, at any width + // (solprop SEP 47 phase 4). + const std::size_t width = problem_.dims.n_modes > adams_max_channels + ? (adams_max_samples + 1) * problem_.dims.n_modes + : 0; + if (adams_channels_.extent(1) != width) + adams_channels_ = width == 0 + ? ComplexView2{} + : ComplexView2("solprop_native.adams.channels", n, width); +#endif + if (adams_block_.extent(0) != slots + 1) + adams_block_ = Kokkos::View( + "solprop_native.adams.block", slots + 1, flow_->n_active(), + problem_.dims.n_modes, problem_.dims.n_modes); + if (compensated && flow_->n_terms() != 0 && adams_applied_.extent(0) != slots) { + adams_applied_ = ComplexView3("solprop_native.adams.applied", slots, columns, n); + adams_end_applied_ = ComplexView2("solprop_native.adams.end_applied", columns, n); + adams_end_weights_ = Kokkos::View( + "solprop_native.adams.end_weights", flow_->n_terms()); + } + } + // The norm's scale per bin, written each trial; allocated here so that the history's + // memory is held, and counted, from construction (solprop's adams_memory). + if (adams_divisor_.extent(0) != n) + adams_divisor_ = + Kokkos::View("solprop_native.adams.divisor", n); + if (adams_norms_.extent(0) == 0) + adams_norms_ = Kokkos::View( + "solprop_native.adams.norms", adams_max_norms); + // The history's views may move: a graph that names them is dropped. + adams_source_graph_.reset(); + // A trial's elementwise passes cost about ten stage entries an element, rising with + // the moments formed, so they take a width of their own; the accepted step's rows + // are copies. + adams_execution_space_ = kernel_pool_.get(CpuOperation::adams, columns * n * slots); + adams_copy_space_ = kernel_pool_.get(CpuOperation::copy, columns * n); + if (adams_store_.extent(0) == slots) + return; + adams_store_ = ComplexView3("solprop_native.adams.store", slots, columns, n); + adams_moments_ = ComplexView3("solprop_native.adams.moments", slots, columns, n); + adams_source_ = ComplexView2("solprop_native.adams.source", columns, n); + adams_propagator_ = ComplexView2("solprop_native.adams.propagator", columns, n); + adams_exponent_ = ComplexView2("solprop_native.adams.exponent", columns, n); + adams_saved_ = ComplexView2("solprop_native.adams.saved_state", columns, n); +} + +void GeneralWorkspace::require_adams() const { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + if (adams_store_.extent(0) == 0) + throw std::logic_error("the resident Adams history is not allocated"); +} + +namespace { + +// The whole step's phase increment `sum_t dPhi_t Phi_t` at one element, in term order: +// strict (solprop SEP 25 section 4.2). +template +SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex adams_phase_sum( + const Terms& phi_terms, const Weights& phi_delta, const std::size_t offset, + const std::size_t n_phi, const std::size_t column, const std::size_t index) { + SOLPROP_NATIVE_STRICT_BLOCK + double real = 0.0; + double imag = 0.0; + for (std::size_t term = 0; term < n_phi; ++term) { + const double weight = phi_delta(offset + term); + // Exact, as in `StageTerms::phase_sum`. + if (weight == 0.0) + continue; + const Complex fixed = phi_terms(term, column, index); + real = real + weight * fixed.real(); + imag = imag + weight * fixed.imag(); + } + return {real, imag}; +} + +} // namespace + +// The whole step's exponent from the uploaded weights, and its exponential: on a +// separable route the step's length times each element's operator, as the stage +// kernel forms the whole propagator; otherwise `sum_t dPhi_t Phi_t` in term order. +void GeneralWorkspace::adams_phase() { + if (problem_.separable_geometry()) { + const auto spatial = spatial_; + const auto active = active_mask_; + const auto delta = stage_phi_delta_; + const auto exponent = adams_exponent_; + const auto propagator = adams_propagator_; + const auto n_pol = problem_.dims.n_pol_state; + const auto n_stages = problem_.stepper.n_stages; + launch::parallel_for( + "solprop_native.adams.separable_phase", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + const auto values = spatial.value(column / n_pol, index, active(index) != 0U); + const Complex x = delta(n_stages) * values.linear; + exponent(column, index) = x; + propagator(column, index) = exponential(x); + }); + return; + } + const auto phi_terms = phi_terms_; + const auto phi_delta = stage_phi_delta_; + const auto exponent = adams_exponent_; + const auto propagator = adams_propagator_; + const auto n_phi = problem_.linear.n_phi_terms; + const auto offset = problem_.stepper.n_stages * n_phi; + launch::parallel_for( + "solprop_native.adams.phase", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + // The step's phase increment: strict (solprop SEP 25 section 4.2). + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) + SOLPROP_NATIVE_STRICT_FUNCTION { + SOLPROP_NATIVE_STRICT_BLOCK + const Complex x = + adams_phase_sum(phi_terms, phi_delta, offset, n_phi, column, index); + exponent(column, index) = x; + propagator(column, index) = exponential(x); + }); +} + +// The stage-zero source at an anchor-frame field held in current_, as adams_source +// evaluates it, into stages_(0). +void GeneralWorkspace::adams_evaluate_source(const TrialScalars& scalars) { + if (general_) + general_->set_weights(scalars.transform_weights); + if (problem_.raman.enabled) + prepare_raman_stage(0, scalars.raman_rates.data(), + problem_.raman.n_components > 1 ? scalars.raman_densities.data() + : nullptr); + evaluate_stage_source(0, 0.0, false); + stages_ready_ = true; + pending_fsal_ = false; + has_recycled_ = false; +} + +// The same, with current_ restored from adams_saved_ afterwards. +void GeneralWorkspace::adams_evaluate(const TrialScalars& scalars) { + adams_evaluate_source(scalars); + graph::deep_copy(execution_space_, current_, adams_saved_); +} + +void GeneralWorkspace::adams_start(const std::size_t slot, + const TrialScalars& scalars) { + require_adams(); + if (slot >= adams_store_.extent(0)) + throw std::out_of_range("Adams history row"); + require_trial_scalars(scalars); + upload_trial(scalars); + graph::deep_copy(execution_space_, adams_saved_, current_); + { + const FlowAside aside(flow_); + adams_evaluate(scalars); + } + // A frozen-residual flow's generator at the sample applied to its state, on the far + // side of a kink that restarted the history (solprop SEP 16 phase 6). + if (adams_applied_.extent(0) != 0) { + upload_weights(execution_space_, adams_end_weights_, scalars.flow_weights); + apply_generator(execution_space_, problem_.dims.n_omega, problem_.dims.n_modes, + flow_->block_slots(), flow_->residual_terms(), adams_end_weights_, + current_, + Kokkos::subview(adams_applied_, slot, Kokkos::ALL, Kokkos::ALL)); + } + const auto stages = stages_; + const auto store = adams_store_; + launch::parallel_for( + "solprop_native.adams.start", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + store(slot, column, index) = stages(0, column, index); + }); +} + +void GeneralWorkspace::adams_scale(const double rtol, const double atol, + const std::vector& weights) { + if (!weights.empty()) + require_extent(weights.size(), problem_.dims.n_omega, "Adams scale weights"); + adams_rtol_ = rtol; + adams_atol_ = atol; + adams_scale_weights_ = RealView("solprop_native.adams.scale_weights", weights.size()); + if (!weights.empty()) + copy_vector(execution_space_, adams_scale_weights_, weights); + adams_magnitude_sum_ = + Kokkos::View("solprop_native.adams.magnitude_sum"); + adams_device_scale_ = true; +} + +// A trial's divisor: the host's when one is given, else formed on the device from the +// accepted state, as solprop's PIController.scale forms it from the same magnitude. +void GeneralWorkspace::adams_upload_divisor(const std::vector& divisor) { + if (adams_divisor_.extent(0) != problem_.dims.n_omega) + adams_divisor_ = Kokkos::View( + "solprop_native.adams.divisor", problem_.dims.n_omega); + if (!divisor.empty() || !adams_device_scale_) { + require_extent(divisor.size(), problem_.dims.n_omega, "Adams norm divisor"); + copy_vector(execution_space_, adams_divisor_, divisor); + return; + } + const auto current = current_; + const auto in_norm = norm_mask_; + const auto n = problem_.dims.n_omega; + // A threaded reduction in whatever order measures fastest (solprop SEP 25). + launch::parallel_reduce( + "solprop_native.adams.magnitude", + Kokkos::RangePolicy(execution_space_, 0, + problem_.n_columns() * n), + KOKKOS_LAMBDA(const std::size_t item, double& update) { + const auto index = item % n; + if (in_norm(index) != 0) + update += squared_abs(current(item / n, index)); + }, + adams_magnitude_sum_); + const auto sum = adams_magnitude_sum_; + const auto weights = adams_scale_weights_; + const bool weighted = weights.extent(0) != 0; + const double rtol = adams_rtol_; + const double atol = adams_atol_; + const auto scale = adams_divisor_; + launch::parallel_for( + "solprop_native.adams.divisor", + Kokkos::RangePolicy(execution_space_, 0, n), + KOKKOS_LAMBDA(const std::size_t index) { + const double magnitude = Kokkos::sqrt(sum()); + scale(index) = weighted ? rtol * weights(index) * magnitude + atol + : rtol * magnitude + atol; + }); +} + +double GeneralWorkspace::adams_magnitude() { + require_adams(); + const auto current = current_; + const auto in_norm = norm_mask_; + const auto n = problem_.dims.n_omega; + double total = 0.0; + launch::parallel_reduce( + "solprop_native.adams.magnitude", + Kokkos::RangePolicy(execution_space_, 0, + problem_.n_columns() * n), + KOKKOS_LAMBDA(const std::size_t item, double& update) { + const auto index = item % n; + if (in_norm(index) != 0) + update += squared_abs(current(item / n, index)); + }, + total); + return Kokkos::sqrt(total); +} + +namespace { + +/** The squared scaled norms of up to three defects, reduced in one traversal. */ +struct AdamsDefectNorms { + using value_type = double[]; + using size_type = std::size_t; + size_type value_count; + Kokkos::View store; + Kokkos::View source; + Kokkos::View propagator; + Kokkos::View in_norm; + Kokkos::View divisor; + AdamsWeights weights; + double dz; + std::size_t n; + + // A defect is a divided difference of the samples: strict (solprop SEP 25 4.2). + SOLPROP_NATIVE_STRICT_FUNCTION + KOKKOS_INLINE_FUNCTION void operator()(const std::size_t item, + value_type update) const { + SOLPROP_NATIVE_STRICT_BLOCK + const auto column = item / n; + const auto index = item % n; + if (in_norm(index) == 0) + return; + const Complex u = propagator(column, index); + const double scale = divisor(index); + for (size_type row = 0; row < value_count; ++row) { + Complex defect = strict_scale(weights.rows[row][0], source(column, index)); + for (std::size_t t = 1; t < weights.width; ++t) + defect = strict_accumulate(defect, weights.rows[row][t], + store(weights.slot[t - 1], column, index)); + const Complex scaled = + strict_divide(strict_mul(u, strict_scale(dz, defect)), scale); + update[row] += squared_abs(scaled); + } + } + KOKKOS_INLINE_FUNCTION void join(value_type destination, + const value_type source_values) const { + for (size_type row = 0; row < value_count; ++row) + destination[row] += source_values[row]; + } + KOKKOS_INLINE_FUNCTION void init(value_type values) const { + for (size_type row = 0; row < value_count; ++row) + values[row] = 0.0; + } +}; + +} // namespace + +std::vector +GeneralWorkspace::adams_fused_trial(const double dz, const AdamsWeights& weights, + const std::vector& divisor, + const TrialScalars& scalars) { + require_adams(); + if (flow_) + throw std::logic_error("the Adams source is evaluated without a coupled flow"); + if (weights.n_rows == 0 || weights.n_rows > 3 || weights.width == 0 || + weights.width > adams_max_samples || weights.order >= weights.width) + throw std::invalid_argument("Adams coefficient tables are out of range"); + for (std::size_t t = 0; t + 1 < weights.width; ++t) + if (weights.slot[t] >= adams_store_.extent(0)) + throw std::out_of_range("Adams history row"); + require_trial_scalars(scalars); + upload_trial(scalars); + adams_upload_divisor(divisor); + adams_phase(); + const auto columns = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); + const auto store = adams_store_; + const auto current = current_; + const auto predicted = adams_saved_; + // The predictor in the anchor's frame, a_n + dz sum_t w_t g_t, is the field the + // stage-zero source evaluates; the accepted state is swapped aside meanwhile. + launch::parallel_for( + "solprop_native.adams.predict", policy, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + Complex total{}; + for (std::size_t t = 0; t < weights.order; ++t) + total += weights.predictor[t] * store(weights.slot[t], column, index); + predicted(column, index) = current(column, index) + dz * total; + }); + std::swap(current_, adams_saved_); + adams_evaluate_source(scalars); + std::swap(current_, adams_saved_); + const auto stages = stages_; + const auto source = adams_source_; + const auto propagator = adams_propagator_; + const auto advanced = advanced_; + launch::parallel_for( + "solprop_native.adams.correct", policy, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + const Complex g = stages(0, column, index); + source(column, index) = g; + Complex total = weights.corrector[0] * g; + for (std::size_t t = 1; t < weights.width; ++t) + total += weights.corrector[t] * store(weights.slot[t - 1], column, index); + advanced(column, index) = + propagator(column, index) * (current(column, index) + dz * total); + }); + const AdamsDefectNorms reducer{.value_count = weights.n_rows, + .store = store, + .source = source, + .propagator = propagator, + .in_norm = norm_mask_, + .divisor = adams_divisor_, + .weights = weights, + .dz = dz, + .n = n}; + return adams_norms( + "solprop_native.adams.defect_norms", + Kokkos::RangePolicy(execution_space_, 0, columns * n), reducer, + weights.n_rows); +} + +void GeneralWorkspace::adams_fused_accept(const double dz, const AdamsWeights& keep, + const std::size_t free, + const AdamsWeights& dense) { + require_adams(); + if (!adams_trial_ready_) + throw std::logic_error("no Adams trial is held"); + if (free >= adams_store_.extent(0)) + throw std::out_of_range("Adams history row"); + if (dense.width > 0 && adams_slopes_.extent(0) == 0) + throw std::logic_error("Adams dense observation storage is not allocated"); + adams_trial_ready_ = false; + const auto columns = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); + const auto store = adams_store_; + const auto source = adams_source_; + const auto propagator = adams_propagator_; + const auto slopes = adams_slopes_; + // The step's own interpolant, in its anchor's frame, before the transport moves the + // rows: the slopes (source, newest, ...) and the state it starts from. + adams_dense_count_ = dense.width; + if (dense.width > 0) { + graph::deep_copy(execution_space_, adams_dense_start_, current_); + graph::deep_copy(execution_space_, adams_dense_exponent_, adams_exponent_); + launch::parallel_for( + "solprop_native.adams.dense_slopes", policy, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + slopes(0, column, index) = source(column, index); + for (std::size_t t = 1; t < dense.width; ++t) + slopes(t, column, index) = store(dense.slot[t - 1], column, index); + }); + } + finish(dz); + stash_is_current_ = false; + if (dense.width > 0) { + const auto current = current_; + const auto end = adams_dense_end_; + launch::parallel_for( + "solprop_native.adams.dense_end", policy, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + end(column, index) = current(column, index) / propagator(column, index); + }); + } + // The history re-anchored at the step's end, and the new derivative in row `free`. + launch::parallel_for( + "solprop_native.adams.transport", policy, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + const Complex u = propagator(column, index); + for (std::size_t t = 0; t < keep.width; ++t) + store(keep.slot[t], column, index) = store(keep.slot[t], column, index) * u; + store(free, column, index) = u * source(column, index); + }); + adams_accepted_boundary(); +} + +void GeneralWorkspace::adams_hold() { + require_adams(); + if (adams_held_.empty()) + throw std::logic_error("Adams dense observation storage is not allocated"); + graph::deep_copy(execution_space_, adams_held_, current_); +} + +void GeneralWorkspace::adams_restore() { + require_adams(); + if (adams_held_.empty()) + throw std::logic_error("Adams dense observation storage is not allocated"); + graph::deep_copy(execution_space_, current_, adams_held_); + stash_is_current_ = false; + execution_space_.fence("solprop_native Adams restore boundary"); +} + +// --- solprop SEP 29 G2: the resident history's rows across a resume ---------------- +// +// The persistent rows are the samples (store), a compensated history's sampled states +// and rates, and a frozen-residual flow's applied generators G(z_j) u_j: everything a +// later trial reads that an earlier step wrote. The rest of the Adams views are a +// trial's scratch, rebuilt by the next trial, and the step's bookkeeping lives in +// solprop's march. The copies are lossless: the view's own LayoutRight order, element +// by element, with no arithmetic. + +ComplexView3 GeneralWorkspace::adams_row_view(const std::string& name) const { + require_adams(); + ComplexView3 view; + if (name == "store") + view = adams_store_; + else if (name == "states") + view = adams_states_; + else if (name == "rates") + view = adams_rates_; + else if (name == "applied") + view = adams_applied_; + else + throw std::invalid_argument("unknown Adams history row '" + name + "'"); + if (view.extent(0) == 0) + throw std::logic_error("the Adams history row '" + name + "' is not allocated"); + return view; +} + +std::vector GeneralWorkspace::adams_row_names() const { + require_adams(); + std::vector names{"store"}; + if (adams_states_.extent(0) != 0) + names.insert(names.end(), {"states", "rates"}); + if (adams_applied_.extent(0) != 0) + names.emplace_back("applied"); + return names; +} + +std::array +GeneralWorkspace::adams_row_extents(const std::string& name) const { + const auto view = adams_row_view(name); + return {view.extent(0), view.extent(1), view.extent(2)}; +} + +std::vector +GeneralWorkspace::adams_rows_to_host(const std::string& name) const { + std::vector> values(adams_row_view(name).size()); + adams_rows_to_host(name, values.data(), values.size()); + return {values.begin(), values.end()}; +} + +// One copy, straight into the caller's buffer (a NumPy array's storage at the seam): +// no intermediate host vector, so a kept save costs what the host engine's copy does. +void GeneralWorkspace::adams_rows_to_host(const std::string& name, + std::complex* destination, + const std::size_t size) const { + const auto view = adams_row_view(name); + if (size != view.size()) + throw std::invalid_argument("the Adams history row '" + name + + "' has the wrong extent"); + execution_space_.fence("solprop_native Adams rows_to_host"); + crossed_.to_host += size * sizeof(Complex); + if (crossing::takes_ring(size * sizeof(Complex))) { + crossing_ring_.to_host(destination, view.data(), size * sizeof(Complex)); + return; + } + const auto host = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, view); + const auto* source = host.data(); + for (std::size_t index = 0; index < size; ++index) + destination[index] = {source[index].real(), source[index].imag()}; +} + +void GeneralWorkspace::adams_rows_from_host(const std::string& name, + const std::vector& values) { + const auto view = adams_row_view(name); + if (values.size() != view.size()) + throw std::invalid_argument("the Adams history row '" + name + + "' has the wrong extent"); + copy_vector(execution_space_, view, values); + // A trial in flight and the last step's interpolant belong to the history replaced. + adams_trial_ready_ = false; + adams_dense_count_ = 0; + execution_space_.fence("solprop_native Adams rows_from_host"); +} + +// solprop SEP 29 G1: the general z tables. A table is uploaded once, after the state +// is constructed and before its first trial, which is the construction's allocation. +void GeneralWorkspace::table_upload(const TableKind kind, std::vector edges, + const double lower, const double upper, + const std::complex* coefficients, + const std::size_t n_intervals, + const std::size_t n_terms, + const std::size_t n_values) { + const bool prefactor = kind == TableKind::prefactor; + const bool flux = kind == TableKind::flux; + if (flux + ? !problem_.flux.table + : (prefactor ? !problem_.linear.prefactor_table : !problem_.linear.phi_table)) + throw std::logic_error(std::string("this problem's ") + + (flux ? "plane measure" + : prefactor ? "prefactor" + : "Phi") + + " is not a z table"); + if (n_intervals == 0 || n_terms == 0 || edges.size() != n_intervals + 1) + throw std::invalid_argument("a z table needs one more breakpoint than intervals " + "and at least one coefficient"); + if (flux && n_values != problem_.dims.n_omega) + refuse("the plane measure's material table holds n_omega = ", problem_.dims.n_omega, + " values"); + if (!flux && n_values != problem_.n_columns() * problem_.dims.n_omega) + refuse("a z table's values must be the state's (", problem_.n_columns(), " x ", + problem_.dims.n_omega, ")"); + for (std::size_t index = 0; index < edges.size(); ++index) + if (!std::isfinite(edges[index]) || + (index > 0 && !(edges[index] > edges[index - 1]))) + throw std::invalid_argument("a z table's breakpoints must be finite and " + "strictly increasing"); + if (!(lower <= edges.front() && upper >= edges.back())) + throw std::invalid_argument("a z table's admitted span must contain its edges"); + auto& table = kind == TableKind::phi ? phi_table_ + : kind == TableKind::prefactor ? prefactor_table_ + : flux ? flux_table_ + : rate_table_; + table.coefficients = Kokkos::View( + kind == TableKind::phi ? "solprop_native.linear.phi_table" + : kind == TableKind::prefactor ? "solprop_native.response.prefactor_table" + : flux ? "solprop_native.flux.material_table" + : "solprop_native.linear.rate_table", + n_intervals, n_terms, n_values); + auto mirror = Kokkos::create_mirror_view(table.coefficients); + auto* data = mirror.data(); + const std::size_t count = n_intervals * n_terms * n_values; + for (std::size_t index = 0; index < count; ++index) + data[index] = + chebyshev::Coefficient>::load(coefficients[index]); + graph::deep_copy(execution_space_, table.coefficients, mirror); + table.edges = std::move(edges); + table.lower = lower; + table.upper = upper; + execution_space_.fence("solprop_native z table upload"); +} + +// The tables' values at a trial's positions into their term slots: Phi's and the +// prefactor's, each a pass over its coefficients for every position at once. +void GeneralWorkspace::table_evaluate(const std::vector& slots, + const std::vector& positions) { + if (!phi_table_.present() && !prefactor_table_.present()) + throw std::logic_error("no z table has been uploaded"); + for (const auto slot : slots) { + if ((phi_table_.present() && slot >= problem_.linear.n_phi_terms) || + (prefactor_table_.present() && slot >= problem_.linear.n_prefactor_terms)) + refuse("z table slot ", slot, + " is not a term of this problem"); + } + if (phi_table_.present()) + z_table::evaluate("solprop_native.linear.phi_table_values", execution_space_, + phi_table_, z_table::locate(phi_table_, slots, positions, false), + phi_terms_.data()); + if (prefactor_table_.present()) + z_table::evaluate("solprop_native.response.prefactor_table_values", + execution_space_, prefactor_table_, + z_table::locate(prefactor_table_, slots, positions, false), + prefactor_terms_.data()); +} + +// solprop SEP 34: the plane measure's material square at `z`, a radial varying +// medium's, into the flux coefficients the statistics read; nothing for any other. +void GeneralWorkspace::flux_at(const double z) { + if (!flux_table_.present()) + return; + z_table::evaluate("solprop_native.flux.material_values", execution_space_, + flux_table_, z_table::locate(flux_table_, {0}, {z}, false), + flux_.square.data()); +} + +// D(z) = dPhi/dz from the rate table into one Phi term slot, read on the side of an +// interior breakpoint `before` asks for (solprop's `phi_rate`). +void GeneralWorkspace::table_rate(const std::size_t slot, const double z, + const bool before) { + if (!rate_table_.present()) + throw std::logic_error("no rate table has been uploaded"); + if (slot >= problem_.linear.n_phi_terms) + refuse("z table slot ", slot, " is not a term of this problem"); + z_table::evaluate("solprop_native.linear.rate_table_values", execution_space_, + rate_table_, z_table::locate(rate_table_, {slot}, {z}, before), + phi_terms_.data()); +} + +std::vector +GeneralWorkspace::table_terms_to_host(const TableKind kind, + const std::size_t slot) const { + const auto& terms = kind == TableKind::prefactor ? prefactor_terms_ : phi_terms_; + if (slot >= terms.extent(0)) + refuse("term slot ", slot, " is not a term of this problem"); + execution_space_.fence("solprop_native z table terms_to_host"); + const auto host = Kokkos::create_mirror_view_and_copy( + Kokkos::HostSpace{}, Kokkos::subview(terms, slot, Kokkos::ALL, Kokkos::ALL)); + std::vector values(host.size()); + for (std::size_t column = 0; column < host.extent(0); ++column) + for (std::size_t index = 0; index < host.extent(1); ++index) + values[column * host.extent(1) + index] = host(column, index); + return values; +} + +// The stage kernels' own phase sum (`StageTerms::phase_sum`) over the Phi terms for one +// row of weights, into a fresh host array: a diagnostic, outside any trial. +std::vector +GeneralWorkspace::phase_sum_to_host(const std::vector& weights) { + if (problem_.separable_geometry()) + throw std::logic_error("a separable operator has no Phi terms"); + require_extent(weights.size(), problem_.linear.n_phi_terms, "phase weights"); + RealView row("solprop_native.diagnostic.phase_weights", weights.size()); + copy_vector(execution_space_, row, weights); + ComplexView2 phase("solprop_native.diagnostic.phase", problem_.n_columns(), + problem_.dims.n_omega); + auto terms = stage_terms(0); + terms.phi_delta = row; + launch::parallel_for( + "solprop_native.diagnostic.phase_sum", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + phase(column, index) = terms.phase_sum(0, column, index); + }); + execution_space_.fence("solprop_native phase_sum_to_host"); + const auto host = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, phase); + return {host.data(), host.data() + host.size()}; +} + +// The field at a fraction of the last accepted step: the step's interpolant in its +// anchor's frame -- the Adams corrector's (`weights.predictor` holds dz-scaled +// integrated weights over the slopes) -- with a bin the step's flow changes by more +// than an e-fold taking the anchor's value, carried to the observation by the uploaded +// trial's flow and masked into current_. +void GeneralWorkspace::adams_observe(const AdamsWeights& weights, + const TrialScalars& scalars) { + require_adams(); + if (adams_dense_count_ == 0) + throw std::logic_error("no Adams step is held for dense observation"); + require_trial_scalars(scalars); + upload_trial(scalars); + adams_phase(); + const auto columns = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + const auto count = adams_dense_count_; + const auto start = adams_dense_start_; + const auto exponent = adams_dense_exponent_; + const auto slopes = adams_slopes_; + const auto carry = adams_propagator_; + const auto current = current_; + launch::parallel_for( + "solprop_native.adams.observe", Range2(execution_space_, {0, 0}, {columns, n}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + Complex total{}; + for (std::size_t t = 0; t < count; ++t) + total += weights.predictor[t] * slopes(t, column, index); + Complex value = start(column, index) + total; + if (Kokkos::abs(exponent(column, index).real()) > 1.0) + value = start(column, index); + current(column, index) = carry(column, index) * value; + }); + mask(current_); + stash_is_current_ = false; + execution_space_.fence("solprop_native Adams observation boundary"); +} + +// --- solprop SEP 16 phase 3b: the exponential formulation, resident +// ------------------- +// +// The exponential formulation's passes (solprop ADR-041, ADR-043): untransported +// samples integrated exactly against the step's flow exp(x), x = Phi(z + h) - Phi(z). +// The moments M_k = k! phi_{k+1}(x) are formed once a trial per element by the +// reference's algorithm (solprop backends/exponential.py, phi_functions), then every +// formula of the trial is +// applied in the monomial basis of its interpolant; the defects use the factored form +// (a divided difference of the samples times one moment combination). + +namespace { + +/** k! for k < 2 adams_max_samples, the moments' scale. */ +KOKKOS_INLINE_FUNCTION double adams_factorial(const std::size_t k) { + double value = 1.0; + for (std::size_t i = 2; i <= k; ++i) + value *= static_cast(i); + return value; +} + +/** + * The phi-functions a trial's moments take: at x, or under a carrier frame at + * x - sigma with that argument's own exponential (solprop SEP 35 section 4.1). The + * flow stays exp(x), which the caller holds. + */ +KOKKOS_INLINE_FUNCTION Complex adams_moment_argument(const ExponentialWeights& weights, + const Complex x) { + return weights.shifted ? x - Complex(0.0, weights.shift) : x; +} + +/** `adams_phi` at `adams_moment_argument`, with the flow `exp_x`; `Fixed`, when not + zero, is `order` known at compile time (`adams_phi_fixed`). */ +template +KOKKOS_INLINE_FUNCTION void adams_moment_phi(const ExponentialWeights& weights, + const Complex x, const Complex exp_x, + const std::size_t order, Complex* out) { + const Complex argument = adams_moment_argument(weights, x); + const double size = Kokkos::abs(argument); + if (!weights.shifted) { + adams_phi_fixed(argument, size, exp_x, order, out); + return; + } + // exp(x - sigma) = exp(x) exp(-sigma): one product with the trial's exp(-sigma) + // rather than a second exponential per element (a few ulp, inside the moments' + // share of the rounding budget). + adams_phi_fixed( + argument, size, + strict_mul(exp_x, Complex(weights.unshift_re, weights.unshift_im)), order, out); +} + +/** + * History row `t` of a trial (`weights.slot[t]`) at one element, rotated by a carrier + * frame's factor exp(sigma (1 - theta_t)) as it is read (solprop SEP 35 section 6.2): + * the store keeps the untransported samples. + */ +template +KOKKOS_INLINE_FUNCTION Complex adams_history(const Store& store, + const ExponentialWeights& weights, + const std::size_t t, + const std::size_t column, + const std::size_t index) { + const Complex value = store(weights.slot[t], column, index); + if (!weights.shifted) + return value; + return strict_mul(Complex(weights.factor_re[t], weights.factor_im[t]), value); +} + +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +/** + * A device element's moments, written to `moments`, and its prediction + * `exp(x) u + dz sum_k M_k sum_j c_jk F_j`, as the elementwise trial's predict pass + * forms them, with `formed` phi-functions. `Fixed`, when not zero, is `formed` known at + * compile time: the loops over the phi-functions unroll and the thread holds them in + * registers, where an indexed array lives in private (scratch) memory, which on an + * AMD Instinct MI325X made this the costliest kernel of a small trial. + */ +template +KOKKOS_INLINE_FUNCTION Complex adams_device_prediction( + const ExponentialWeights& weights, const std::size_t formed, const Complex x, + const Complex propagate, const Complex accepted, const double dz, + const Moments& moments, const Store& store, const std::size_t column, + const std::size_t index) { + constexpr std::size_t bound = Fixed != 0 ? Fixed : 2 * adams_max_samples; + Complex phi[bound + 1]; + adams_moment_phi(weights, x, propagate, formed, phi); + for (std::size_t k = 0; k < bound; ++k) { + if (k >= weights.count) + break; + moments(k, column, index) = adams_factorial(k) * phi[k + 1]; + } + Complex total{}; + for (std::size_t k = 0; k < bound; ++k) { + if (k >= weights.order) + break; + Complex combined{}; + for (std::size_t t = 0; t < weights.order; ++t) + combined += + weights.predictor[t][k] * adams_history(store, weights, t, column, index); + const Complex moment = adams_factorial(k) * phi[k + 1]; + total += moment * combined; + } + return propagate * accepted + dz * total; +} + +/** `adams_device_prediction` at the compile-time bound equal to `formed`, or at a + run-time one past the table's eight samples. */ +template +KOKKOS_INLINE_FUNCTION Complex +adams_device_predict(const ExponentialWeights& weights, const std::size_t formed, + const Complex x, const Complex propagate, const Complex accepted, + const double dz, const Moments& moments, const Store& store, + const std::size_t column, const std::size_t index) { + static_assert(adams_max_samples == 8, "one case per sample count"); + // A switch, not dispatch_width's template lambda: device-only code nvcc has not + // been shown to compile with one (its build of this file once failed internally). + switch (formed) { + case 1: + return adams_device_prediction<1>(weights, formed, x, propagate, accepted, dz, + moments, store, column, index); + case 2: + return adams_device_prediction<2>(weights, formed, x, propagate, accepted, dz, + moments, store, column, index); + case 3: + return adams_device_prediction<3>(weights, formed, x, propagate, accepted, dz, + moments, store, column, index); + case 4: + return adams_device_prediction<4>(weights, formed, x, propagate, accepted, dz, + moments, store, column, index); + case 5: + return adams_device_prediction<5>(weights, formed, x, propagate, accepted, dz, + moments, store, column, index); + case 6: + return adams_device_prediction<6>(weights, formed, x, propagate, accepted, dz, + moments, store, column, index); + case 7: + return adams_device_prediction<7>(weights, formed, x, propagate, accepted, dz, + moments, store, column, index); + case 8: + return adams_device_prediction<8>(weights, formed, x, propagate, accepted, dz, + moments, store, column, index); + default: + return adams_device_prediction<0>(weights, formed, x, propagate, accepted, dz, + moments, store, column, index); + } +} +#endif + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +/** The channel vectors a host task of a coupled pass holds: up to `adams_max_samples + + 1` rows of a bin's channels. */ +inline constexpr std::size_t adams_channel_stack = + (adams_max_samples + 1) * adams_max_channels; + +/** + * Where a host task holds bin `index`'s channel vectors, rows of `modes` entries: + * `stack` up to `adams_max_channels` channels, beyond it the bin's row of `rows` + * (`adams_channels_`, solprop SEP 47 phase 4). Every pass reads only what it wrote. + */ +// A bin's row of `adams_channels_` is contiguous: the view's last extent is its +// fastest. +static_assert(std::is_same_v, + "adams_channel_rows reads a bin's row as contiguous memory"); + +inline Complex* +adams_channel_rows(Complex* stack, + const Kokkos::View& rows, + const std::size_t index) { + return rows.extent(1) == 0 ? stack : &rows(index, 0); +} +#endif + +// The defects' norms beside a coupled flow (solprop SEP 16 phase 6): per bin, the +// samples' channel vectors; off the block each channel's defect is the diagonal one, +// on it each moment combination is a matrix applied to the divided difference. +struct CoupledDefectNorms { + using value_type = double[]; + using size_type = std::size_t; + size_type value_count; + Kokkos::View store; + Kokkos::View moments; + Kokkos::View source; + Kokkos::View carried; + Kokkos::View endpoint; + Kokkos::View block; + Kokkos::View slots; + Kokkos::View in_norm; + Kokkos::View divisor; + ExponentialWeights weights; + double dz; + std::size_t modes; + // A host's channel vectors beyond `adams_max_channels` (`adams_channels_`). + Kokkos::View channels; + +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + SOLPROP_NATIVE_STRICT_FUNCTION + KOKKOS_INLINE_FUNCTION void operator()(const std::size_t item, + value_type update) const { + SOLPROP_NATIVE_STRICT_BLOCK + // An item is a (bin, channel) pair: a thread holds no channel vectors. + const auto index = item / modes; + const auto i = item % modes; + if (in_norm(index) == 0) + return; + const double scale = divisor(index); + const auto one = slots(index); + // Channel c's divided difference for one defect. + const auto difference = [&](const size_type row, const std::size_t c) { + Complex total = strict_scale(weights.rows[row][0], source(c, index)); + for (std::size_t t = 1; t < weights.width; ++t) + total = strict_accumulate(total, weights.rows[row][t], + adams_history(store, weights, t - 1, c, index)); + return total; + }; + for (size_type row = 0; row < value_count; ++row) { + Complex defect{}; + if (one == 0) { + Complex weight{}; + for (std::size_t k = 0; k < weights.count; ++k) + weight = strict_accumulate(weight, weights.rows[row][weights.width + k], + moments(k, i, index)); + defect = strict_scale(dz, strict_mul(difference(row, i), weight)); + } else { + const auto bin = one - 1; + Complex total{}; + for (std::size_t j = 0; j < modes; ++j) { + Complex weight{}; + for (std::size_t k = 0; k < weights.count; ++k) + weight = strict_accumulate(weight, weights.rows[row][weights.width + k], + block(k + 1, bin, i, j)); + total = strict_add(total, strict_mul(weight, difference(row, j))); + } + defect = strict_scale(dz, total); + } + if (weights.compensated) { + Complex compensation = + strict_scale(weights.comp_rows[row][0], endpoint(i, index)); + for (std::size_t t = 0; t + 1 < weights.width; ++t) + compensation = strict_accumulate(compensation, weights.comp_rows[row][t + 1], + carried(t, i, index)); + defect = strict_accumulate(defect, dz, compensation); + } + update[row] += squared_abs(strict_divide(defect, scale)); + } + } +#else + // The defects and their compensation terms: strict (solprop SEP 25 section 4.2). + SOLPROP_NATIVE_STRICT_FUNCTION + KOKKOS_INLINE_FUNCTION void operator()(const std::size_t index, + value_type update) const { + SOLPROP_NATIVE_STRICT_BLOCK + if (in_norm(index) == 0) + return; + const double scale = divisor(index); + const auto one = slots(index); + // The samples' channel vectors, row t at `values + t modes`, and the divided + // difference after them. + Complex stack[adams_channel_stack]; + Complex* const values = adams_channel_rows(stack, channels, index); + Complex* const difference = values + adams_max_samples * modes; + for (std::size_t i = 0; i < modes; ++i) { + values[i] = source(i, index); + for (std::size_t t = 1; t < weights.width; ++t) + values[t * modes + i] = adams_history(store, weights, t - 1, i, index); + } + for (size_type row = 0; row < value_count; ++row) { + for (std::size_t i = 0; i < modes; ++i) { + Complex total = strict_scale(weights.rows[row][0], values[i]); + for (std::size_t t = 1; t < weights.width; ++t) + total = strict_accumulate(total, weights.rows[row][t], values[t * modes + i]); + difference[i] = total; + } + for (std::size_t i = 0; i < modes; ++i) { + Complex defect{}; + if (one == 0) { + Complex weight{}; + for (std::size_t k = 0; k < weights.count; ++k) + weight = strict_accumulate(weight, weights.rows[row][weights.width + k], + moments(k, i, index)); + defect = strict_scale(dz, strict_mul(difference[i], weight)); + } else { + const auto bin = one - 1; + Complex total{}; + for (std::size_t j = 0; j < modes; ++j) { + Complex weight{}; + for (std::size_t k = 0; k < weights.count; ++k) + weight = strict_accumulate(weight, weights.rows[row][weights.width + k], + block(k + 1, bin, i, j)); + total = strict_add(total, strict_mul(weight, difference[j])); + } + defect = strict_scale(dz, total); + } + if (weights.compensated) { + Complex compensation = + strict_scale(weights.comp_rows[row][0], endpoint(i, index)); + for (std::size_t t = 0; t + 1 < weights.width; ++t) + compensation = strict_accumulate( + compensation, weights.comp_rows[row][t + 1], carried(t, i, index)); + defect = strict_accumulate(defect, dz, compensation); + } + update[row] += squared_abs(strict_divide(defect, scale)); + } + } + } +#endif + KOKKOS_INLINE_FUNCTION void join(value_type destination, + const value_type source_values) const { + for (size_type row = 0; row < value_count; ++row) + destination[row] += source_values[row]; + } + KOKKOS_INLINE_FUNCTION void init(value_type values) const { + for (size_type row = 0; row < value_count; ++row) + values[row] = 0.0; + } +}; + +} // namespace + +// D(z) = sum_t w_t Phi_t from the operator's fixed terms (solprop's +// translate.linop.rate_weights), into `out`. +void GeneralWorkspace::adams_rate(const std::vector& rate, + const ComplexView2& out) { + require_extent(rate.size(), problem_.linear.n_phi_terms, "Adams rate weights"); + if (adams_rate_weights_.extent(0) != rate.size()) + adams_rate_weights_ = Kokkos::View( + "solprop_native.adams.rate_weights", rate.size()); + copy_vector(execution_space_, adams_rate_weights_, rate); + const auto weights = adams_rate_weights_; + const auto phi_terms = phi_terms_; + const auto n_phi = problem_.linear.n_phi_terms; + launch::parallel_for( + "solprop_native.adams.rate", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + Complex value{}; + for (std::size_t term = 0; term < n_phi; ++term) + value += weights(term) * phi_terms(term, column, index); + out(column, index) = value; + }); +} + +// The startup sample's state and rate, beside the source adams_start stored. +void GeneralWorkspace::adams_compensation_start(const std::size_t slot, + const std::vector& rate) { + require_adams(); + if (adams_states_.extent(0) == 0) + throw std::logic_error("the Adams history carries no compensation"); + if (slot >= adams_states_.extent(0)) + throw std::out_of_range("Adams history row"); + const auto rates = Kokkos::subview(adams_rates_, slot, Kokkos::ALL, Kokkos::ALL); + adams_rate(rate, adams_endpoint_rate_); + graph::deep_copy(execution_space_, rates, adams_endpoint_rate_); + graph::deep_copy(execution_space_, + Kokkos::subview(adams_states_, slot, Kokkos::ALL, Kokkos::ALL), + current_); +} + +// The exponential Adams march's compensation at one bin, each a single strict function +// (solprop SEP 25 section 4.2) called once per bin: GCC compiles a strict function +// with its own options and does not inline it into a contracting kernel, but inlines +// the strict helpers it calls, which share those options. One call per bin, rather +// than one per helper, keeps the protected arithmetic strict on every compiler at the +// cost of a call (and Clang, which inlines every level, is unchanged). + +// The predictor's carried samples `e^{s_t x} (R_t - x / dz) u_t`, written to +// `carried`, and the prediction `value + dz sum_t c_t carried_t`. A span of one (the +// newest sample, at the step's start) takes the step's flow `flow = e^x` as it is: +// `strict_scale(1, x)` is x, so its exponential is the same bits. A host block with +// vector maths forms the same values on its lanes +// (`adams_lanes_predicted_compensation`). +// The strict compensation helpers keep their linkage: changing it could change how GCC +// inlines strict code (fp.hpp), which needs its own A/B. +// NOLINTBEGIN(misc-use-internal-linkage): strict helpers, linkage unchanged +template +SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex +adams_predicted_compensation(const ExponentialWeights& weights, const Complex x, + const Complex flow, const double inverse_dz, + const double dz, const Complex value, const Rates& rates, + const States& states, const Carried& carried, + const std::size_t column, const std::size_t index) { + SOLPROP_NATIVE_STRICT_BLOCK + for (std::size_t t = 0; t + 1 < weights.width; ++t) { + const auto slot = weights.slot[t]; + Complex term = compensation_residual(rates(slot, column, index), x, inverse_dz, + states(slot, column, index)); + const double span = weights.span[t]; + if (span == 1.0) + term = strict_mul(flow, term); + else if (span != 0.0) + term = strict_mul(exponential(strict_scale(span, x)), term); + carried(t, column, index) = term; + } + Complex compensation{}; + for (std::size_t t = 0; t < weights.order; ++t) + compensation = strict_accumulate(compensation, weights.comp_predictor[t], + carried(t, column, index)); + return strict_accumulate(value, dz, compensation); +} + +// The corrector's endpoint rate `sum_t w_t Phi_t` and compensation `(R - x / dz) u`, +// written to `endpoint_rate` and `endpoint`, and the correction +// `value + dz (c_0 endpoint + sum_t c_{t+1} carried_t)`. +template +SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex +adams_corrected_compensation(const ExponentialWeights& weights, const std::size_t n_phi, + const Weights& rate_weights, const Terms& phi_terms, + const Complex x, const double inverse_dz, + const Complex lab, const double dz, const Complex value, + const Points& endpoint_rate, const Points& endpoint, + const Carried& carried, const std::size_t column, + const std::size_t index) { + SOLPROP_NATIVE_STRICT_BLOCK + Complex rate_here{}; + for (std::size_t term = 0; term < n_phi; ++term) + rate_here = strict_accumulate(rate_here, rate_weights(term), + phi_terms(term, column, index)); + endpoint_rate(column, index) = rate_here; + const Complex compensated = compensation_residual(rate_here, x, inverse_dz, lab); + endpoint(column, index) = compensated; + Complex compensation = strict_scale(weights.comp_corrector[0], compensated); + for (std::size_t t = 0; t < weights.order; ++t) + compensation = strict_accumulate(compensation, weights.comp_corrector[t + 1], + carried(t, column, index)); + return strict_accumulate(value, dz, compensation); +} + +// Dense observation's compensation `value + sum_t dz w_t e^{(xi - theta_t) x} term_t`. +template +SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION Complex +adams_observed_compensation(const ExponentialWeights& weights, const std::size_t count, + const double xi, const double dz, const Complex chord, + const Complex value, const Terms& terms, + const std::size_t column, const std::size_t index) { + SOLPROP_NATIVE_STRICT_BLOCK + Complex total = value; + for (std::size_t t = 0; t < count; ++t) + total = strict_add( + total, + strict_mul( + strict_scale(dz * weights.dense_weight[t], + exponential(strict_scale(xi - weights.dense_theta[t], chord))), + terms(t, column, index))); + return total; +} + +// NOLINTEND(misc-use-internal-linkage) +namespace { + +// The elements a host task of the elementwise trial's passes takes in turn. Each pass +// runs its loops over the block, so that what one loop writes the next reads from +// cache, and a strict loop is one call a block on a compiler that keeps strict +// functions out of line; a device takes a thread an element. +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +constexpr std::size_t adams_block = 1; +#else +constexpr std::size_t adams_block = adams_lane_width; +#endif + +// The host blocks run as lanes where the vector maths forms their exponentials in +// passes (`vmath::vectorised`); elsewhere (devices, the system libm on Arm) each +// element forms its own, and the lanes are not compiled. +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) && \ + defined(SOLPROP_NATIVE_VECTOR_MATH_SLEEF) +#define SOLPROP_NATIVE_ADAMS_LANES 1 +#endif + +// The step's exponent and flow on the elements [first, last) of a column. The phase +// increment: strict (solprop SEP 25 section 4.2). A host block with vector maths +// forms both on its lanes instead (`adams_lanes_predict`). +template +SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION void +adams_phase_block(const Terms& phi_terms, const Weights& phi_delta, + const std::size_t offset, const std::size_t n_phi, + const std::size_t column, const std::size_t first, + const std::size_t last, const Points& exponent, const Points& flow) { + SOLPROP_NATIVE_STRICT_BLOCK + for (std::size_t index = first; index < last; ++index) { + const Complex x = + adams_phase_sum(phi_terms, phi_delta, offset, n_phi, column, index); + exponent(column, index) = x; + flow(column, index) = exponential(x); + } +} + +#ifdef SOLPROP_NATIVE_ADAMS_LANES + +// The history rows a pass reads, rotated by a carrier frame's factors +// (`adams_history`). Strict. +template +SOLPROP_NATIVE_STRICT_FUNCTION inline void +adams_lanes_history(AdamsLanes& lanes, const ExponentialWeights& weights, + const Store& store, const std::size_t rows, + const std::size_t column, const std::size_t first) { + SOLPROP_NATIVE_STRICT_BLOCK + const std::size_t count = lanes.count; + for (std::size_t t = 0; t < rows; ++t) { + const Complex* row = &store(weights.slot[t], column, first); + if (!weights.shifted) { + for (std::size_t b = 0; b < count; ++b) { + lanes.hr[t][b] = row[b].real(); + lanes.hi[t][b] = row[b].imag(); + } + continue; + } + const double factor_r = weights.factor_re[t]; + const double factor_i = weights.factor_im[t]; + for (std::size_t b = 0; b < count; ++b) { + const double rr = factor_r * row[b].real(); + const double ii = factor_i * row[b].imag(); + const double ri = factor_r * row[b].imag(); + const double ir = factor_i * row[b].real(); + lanes.hr[t][b] = rr - ii; + lanes.hi[t][b] = ri + ir; + } + } +} + +// The spans' arguments s_t x of a varying operator's carried samples, into `sr`/`si` +// (strict_scale), for the exponentials the caller forms over them. +SOLPROP_NATIVE_STRICT_FUNCTION +inline void adams_lanes_span(AdamsLanes& lanes, const double span, + const std::size_t t) { + SOLPROP_NATIVE_STRICT_BLOCK + for (std::size_t b = 0; b < lanes.count; ++b) { + lanes.sr[t][b] = span * lanes.xr[b]; + lanes.si[t][b] = span * lanes.xi[b]; + } +} + +// The predictor's carried samples `e^{s_t x} (R_t - x / dz) u_t` into `cr`/`ci`, and +// the prediction in `vr`/`vi` plus `dz sum_t c_t carried_t`, as +// `adams_predicted_compensation`. Strict. +template +SOLPROP_NATIVE_STRICT_FUNCTION inline void +adams_lanes_predicted_compensation(AdamsLanes& lanes, const ExponentialWeights& weights, + const double inverse_dz, const double dz, + const Rates& rates, const States& states, + const std::size_t column, const std::size_t first) { + SOLPROP_NATIVE_STRICT_BLOCK + const std::size_t count = lanes.count; + for (std::size_t t = 0; t + 1 < weights.width; ++t) { + const auto slot = weights.slot[t]; + const Complex* rate = &rates(slot, column, first); + const Complex* state = &states(slot, column, first); + const double span = weights.span[t]; + for (std::size_t b = 0; b < count; ++b) { + const double scaled_r = inverse_dz * lanes.xr[b]; + const double scaled_i = inverse_dz * lanes.xi[b]; + const double difference_r = rate[b].real() - scaled_r; + const double difference_i = rate[b].imag() - scaled_i; + const double rr = difference_r * state[b].real(); + const double ii = difference_i * state[b].imag(); + const double ri = difference_r * state[b].imag(); + const double ir = difference_i * state[b].real(); + lanes.cr[t][b] = rr - ii; + lanes.ci[t][b] = ri + ir; + } + if (span == 0.0) + continue; + const double* factor_r = span == 1.0 ? lanes.fr : lanes.sr[t]; + const double* factor_i = span == 1.0 ? lanes.fi : lanes.si[t]; + for (std::size_t b = 0; b < count; ++b) { + const double rr = factor_r[b] * lanes.cr[t][b]; + const double ii = factor_i[b] * lanes.ci[t][b]; + const double ri = factor_r[b] * lanes.ci[t][b]; + const double ir = factor_i[b] * lanes.cr[t][b]; + lanes.cr[t][b] = rr - ii; + lanes.ci[t][b] = ri + ir; + } + } + double total_r[AdamsLanes::B]; + double total_i[AdamsLanes::B]; + for (std::size_t b = 0; b < count; ++b) { + total_r[b] = 0.0; + total_i[b] = 0.0; + } + for (std::size_t t = 0; t < weights.order; ++t) { + const double weight = weights.comp_predictor[t]; + for (std::size_t b = 0; b < count; ++b) { + const double real = weight * lanes.cr[t][b]; + const double imag = weight * lanes.ci[t][b]; + total_r[b] = total_r[b] + real; + total_i[b] = total_i[b] + imag; + } + } + for (std::size_t b = 0; b < count; ++b) { + const double real = dz * total_r[b]; + const double imag = dz * total_i[b]; + lanes.vr[b] = lanes.vr[b] + real; + lanes.vi[b] = lanes.vi[b] + imag; + } +} + +// A varying operator's endpoint rate `sum_t w_t Phi_t` into `dr`/`di` and +// `endpoint_rate`, as `adams_corrected_compensation` forms it. Strict. +template +SOLPROP_NATIVE_STRICT_FUNCTION inline void +adams_lanes_rate(AdamsLanes& lanes, const std::size_t n_phi, + const Weights& rate_weights, const Terms& phi_terms, + const Points& endpoint_rate, const std::size_t column, + const std::size_t first) { + SOLPROP_NATIVE_STRICT_BLOCK + const std::size_t count = lanes.count; + for (std::size_t b = 0; b < count; ++b) { + lanes.dr[b] = 0.0; + lanes.di[b] = 0.0; + } + for (std::size_t term = 0; term < n_phi; ++term) { + const double weight = rate_weights(term); + const Complex* fixed = &phi_terms(term, column, first); + for (std::size_t b = 0; b < count; ++b) { + const double real = weight * fixed[b].real(); + const double imag = weight * fixed[b].imag(); + lanes.dr[b] = lanes.dr[b] + real; + lanes.di[b] = lanes.di[b] + imag; + } + } + Complex* rate_out = &endpoint_rate(column, first); + for (std::size_t b = 0; b < count; ++b) + rate_out[b] = Complex(lanes.dr[b], lanes.di[b]); +} + +// The corrector's compensation `(R - x / dz) u` at the endpoint rate `dr`/`di` into +// `nr`/`ni`, and the correction in `vr`/`vi` plus `dz (c_0 endpoint + sum_t c_{t+1} +// carried_t)`, as `adams_corrected_compensation`; `ur`/`ui` hold the lab-frame field. +// Strict. +SOLPROP_NATIVE_STRICT_FUNCTION +inline void adams_lanes_corrected_compensation(AdamsLanes& lanes, + const ExponentialWeights& weights, + const double inverse_dz, + const double dz) { + SOLPROP_NATIVE_STRICT_BLOCK + constexpr std::size_t B = AdamsLanes::B; + const std::size_t count = lanes.count; + double total_r[B]; + double total_i[B]; + const double weight0 = weights.comp_corrector[0]; + for (std::size_t b = 0; b < count; ++b) { + const double scaled_r = inverse_dz * lanes.xr[b]; + const double scaled_i = inverse_dz * lanes.xi[b]; + const double difference_r = lanes.dr[b] - scaled_r; + const double difference_i = lanes.di[b] - scaled_i; + const double rr = difference_r * lanes.ur[b]; + const double ii = difference_i * lanes.ui[b]; + const double ri = difference_r * lanes.ui[b]; + const double ir = difference_i * lanes.ur[b]; + lanes.nr[b] = rr - ii; + lanes.ni[b] = ri + ir; + total_r[b] = weight0 * lanes.nr[b]; + total_i[b] = weight0 * lanes.ni[b]; + } + for (std::size_t t = 0; t < weights.order; ++t) { + const double weight = weights.comp_corrector[t + 1]; + for (std::size_t b = 0; b < count; ++b) { + const double real = weight * lanes.cr[t][b]; + const double imag = weight * lanes.ci[t][b]; + total_r[b] = total_r[b] + real; + total_i[b] = total_i[b] + imag; + } + } + for (std::size_t b = 0; b < count; ++b) { + const double real = dz * total_r[b]; + const double imag = dz * total_i[b]; + lanes.vr[b] = lanes.vr[b] + real; + lanes.vi[b] = lanes.vi[b] + imag; + } +} + +// The squared scaled norms of up to three factored defects of every lane into +// `defect`, as `adams_defect_block` forms them: `wr`/`wi` hold the new sample, +// `hr`/`hi` the history, `mr`/`mi` the moments, `nr`/`ni` the endpoint's and `cr`/`ci` +// the carried compensation, `scale` the divisors. Strict. +SOLPROP_NATIVE_STRICT_FUNCTION +inline void adams_lanes_defects(AdamsLanes& lanes, const ExponentialWeights& weights, + const std::size_t scored, const double dz) { + SOLPROP_NATIVE_STRICT_BLOCK + constexpr std::size_t B = AdamsLanes::B; + const std::size_t count = lanes.count; + for (std::size_t row = 0; row < scored; ++row) { + double difference_r[B]; + double difference_i[B]; + double weight_r[B]; + double weight_i[B]; + const double weight0 = weights.rows[row][0]; + for (std::size_t b = 0; b < count; ++b) { + difference_r[b] = weight0 * lanes.wr[b]; + difference_i[b] = weight0 * lanes.wi[b]; + weight_r[b] = 0.0; + weight_i[b] = 0.0; + } + for (std::size_t t = 1; t < weights.width; ++t) { + const double weight = weights.rows[row][t]; + for (std::size_t b = 0; b < count; ++b) { + const double real = weight * lanes.hr[t - 1][b]; + const double imag = weight * lanes.hi[t - 1][b]; + difference_r[b] = difference_r[b] + real; + difference_i[b] = difference_i[b] + imag; + } + } + for (std::size_t k = 0; k < weights.count; ++k) { + const double weight = weights.rows[row][weights.width + k]; + for (std::size_t b = 0; b < count; ++b) { + const double real = weight * lanes.mr[k][b]; + const double imag = weight * lanes.mi[k][b]; + weight_r[b] = weight_r[b] + real; + weight_i[b] = weight_i[b] + imag; + } + } + double defect_r[B]; + double defect_i[B]; + for (std::size_t b = 0; b < count; ++b) { + const double rr = difference_r[b] * weight_r[b]; + const double ii = difference_i[b] * weight_i[b]; + const double ri = difference_r[b] * weight_i[b]; + const double ir = difference_i[b] * weight_r[b]; + const double product_r = rr - ii; + const double product_i = ri + ir; + defect_r[b] = dz * product_r; + defect_i[b] = dz * product_i; + } + if (weights.compensated) { + double total_r[B]; + double total_i[B]; + const double first = weights.comp_rows[row][0]; + for (std::size_t b = 0; b < count; ++b) { + total_r[b] = first * lanes.nr[b]; + total_i[b] = first * lanes.ni[b]; + } + for (std::size_t t = 0; t + 1 < weights.width; ++t) { + const double weight = weights.comp_rows[row][t + 1]; + for (std::size_t b = 0; b < count; ++b) { + const double real = weight * lanes.cr[t][b]; + const double imag = weight * lanes.ci[t][b]; + total_r[b] = total_r[b] + real; + total_i[b] = total_i[b] + imag; + } + } + for (std::size_t b = 0; b < count; ++b) { + const double real = dz * total_r[b]; + const double imag = dz * total_i[b]; + defect_r[b] = defect_r[b] + real; + defect_i[b] = defect_i[b] + imag; + } + } + for (std::size_t b = 0; b < count; ++b) { + const double real = defect_r[b] / lanes.scale[b]; + const double imag = defect_i[b] / lanes.scale[b]; + const double rr = real * real; + const double ii = imag * imag; + lanes.defect[row][b] = rr + ii; + } + } +} +#endif + +#ifdef SOLPROP_NATIVE_ADAMS_LANES +// The whole step's phase increment `sum_t dPhi_t Phi_t` of every lane into `xr`/`xi`, +// as `adams_phase_sum` forms it, and with `Rate` a varying operator's endpoint rate +// `sum_t w_t Phi_t` into `dr`/`di`, as `adams_corrected_compensation` forms it, in the +// same pass over the terms. Strict. +template +SOLPROP_NATIVE_STRICT_FUNCTION inline void +adams_lanes_phase(AdamsLanes& lanes, const Terms& phi_terms, const Weights& phi_delta, + const RateWeights& rate_weights, const std::size_t offset, + const std::size_t n_phi, const std::size_t column, + const std::size_t first) { + SOLPROP_NATIVE_STRICT_BLOCK + const std::size_t count = lanes.count; + for (std::size_t b = 0; b < count; ++b) { + lanes.xr[b] = 0.0; + lanes.xi[b] = 0.0; + lanes.dr[b] = 0.0; + lanes.di[b] = 0.0; + } + for (std::size_t term = 0; term < n_phi; ++term) { + const double weight = phi_delta(offset + term); + const Complex* fixed = &phi_terms(term, column, first); + // A zero weight is skipped, exactly, as in `StageTerms::phase_sum`. + if constexpr (Rate) { + const double rate = rate_weights(term); + if (weight == 0.0) + for (std::size_t b = 0; b < count; ++b) { + lanes.dr[b] = lanes.dr[b] + rate * fixed[b].real(); + lanes.di[b] = lanes.di[b] + rate * fixed[b].imag(); + } + else + for (std::size_t b = 0; b < count; ++b) { + lanes.xr[b] = lanes.xr[b] + weight * fixed[b].real(); + lanes.xi[b] = lanes.xi[b] + weight * fixed[b].imag(); + lanes.dr[b] = lanes.dr[b] + rate * fixed[b].real(); + lanes.di[b] = lanes.di[b] + rate * fixed[b].imag(); + } + } else { + if (weight == 0.0) + continue; + for (std::size_t b = 0; b < count; ++b) { + lanes.xr[b] = lanes.xr[b] + weight * fixed[b].real(); + lanes.xi[b] = lanes.xi[b] + weight * fixed[b].imag(); + } + } + } +} +#endif + +#ifdef SOLPROP_NATIVE_ADAMS_LANES +/** + * The predict pass on a host block's lanes, the elements [first, last) of a column + * whose exponents are formed: the flow, the moments, the predictor's carried samples + * and the prediction, as the element-by-element form writes them. Returns the block's + * share of the accepted state's squared magnitude, summed in element order. + */ +template +inline double adams_lanes_predict( + const Terms& phi_terms, const Weights& phi_delta, const RateWeights& rate_weights, + const Points& endpoint_rate, const std::size_t offset, const std::size_t n_phi, + const ExponentialWeights& weights, const std::size_t formed, const double dz, + const double inverse_dz, const Points& exponent, const Points& flow, + const Points& current, const Points& predicted, const Points& lab, + const Cube& moments, const Cube& store, const Cube& carried, const Cube& states, + const Cube& rates, const Mask& in_norm, const std::size_t column, + const std::size_t first, const std::size_t last) { + constexpr std::size_t B = AdamsLanes::B; + AdamsLanes lanes; + const std::size_t n = last - first; + lanes.count = n; + if constexpr (Phase) { + // The corrector's endpoint rate reads the same terms: formed here, in the same + // pass, and read back by the correct pass. + if (weights.compensated) { + adams_lanes_phase(lanes, phi_terms, phi_delta, rate_weights, offset, n_phi, + column, first); + Complex* rate = &endpoint_rate(column, first); + for (std::size_t b = 0; b < n; ++b) + rate[b] = Complex(lanes.dr[b], lanes.di[b]); + } else { + adams_lanes_phase(lanes, phi_terms, phi_delta, rate_weights, offset, n_phi, + column, first); + } + Complex* x = &exponent(column, first); + for (std::size_t b = 0; b < n; ++b) + x[b] = Complex(lanes.xr[b], lanes.xi[b]); + } else { + const Complex* x = &exponent(column, first); + for (std::size_t b = 0; b < n; ++b) { + lanes.xr[b] = x[b].real(); + lanes.xi[b] = x[b].imag(); + } + } + // Formed before the vector maths writes them: clang-tidy's analyzer does not follow + // its stores (a few stores a block). + for (std::size_t b = 0; b < n; ++b) { + lanes.fr[b] = 0.0; + lanes.fi[b] = 0.0; + } + vmath::exp_complex(lanes.xr, lanes.xi, lanes.fr, lanes.fi, n); + Complex* flow_out = &flow(column, first); + for (std::size_t b = 0; b < n; ++b) + flow_out[b] = Complex(lanes.fr[b], lanes.fi[b]); + adams_lanes_classify(lanes, weights); + vmath::exp_complex(lanes.yr, lanes.yi, lanes.er, lanes.ei, n); + if (lanes.any_large) + adams_lanes_large_exponential(lanes, weights); + adams_lanes_phi(lanes, formed); + if (weights.compensated) + for (std::size_t t = 0; t + 1 < weights.width; ++t) { + const double span = weights.span[t]; + if (span == 1.0 || span == 0.0) + continue; + adams_lanes_span(lanes, span, t); + vmath::exp_complex(lanes.sr[t], lanes.si[t], lanes.sr[t], lanes.si[t], n); + } + // The moments M_k = k! phi_{k+1}. + const std::size_t moment_rows = formed; + for (std::size_t k = 0; k < moment_rows; ++k) { + const double factorial = adams_factorial(k); + for (std::size_t b = 0; b < n; ++b) { + lanes.mr[k][b] = factorial * lanes.pr[k + 1][b]; + lanes.mi[k][b] = factorial * lanes.pi[k + 1][b]; + } + } + for (std::size_t k = 0; k < weights.count; ++k) { + Complex* out = &moments(k, column, first); + for (std::size_t b = 0; b < n; ++b) + out[b] = Complex(lanes.mr[k][b], lanes.mi[k][b]); + } + adams_lanes_history(lanes, weights, store, weights.order, column, first); + // The prediction exp(x) u + dz sum_k M_k sum_j c_jk F_j. + const Complex* accepted = ¤t(column, first); + double total_r[B]; + double total_i[B]; + for (std::size_t b = 0; b < n; ++b) { + lanes.ur[b] = accepted[b].real(); + lanes.ui[b] = accepted[b].imag(); + total_r[b] = 0.0; + total_i[b] = 0.0; + } + for (std::size_t k = 0; k < weights.order; ++k) { + double combined_r[B]; + double combined_i[B]; + for (std::size_t b = 0; b < n; ++b) { + combined_r[b] = 0.0; + combined_i[b] = 0.0; + } + for (std::size_t t = 0; t < weights.order; ++t) { + const double weight = weights.predictor[t][k]; + for (std::size_t b = 0; b < n; ++b) { + combined_r[b] += weight * lanes.hr[t][b]; + combined_i[b] += weight * lanes.hi[t][b]; + } + } + for (std::size_t b = 0; b < n; ++b) { + total_r[b] += lanes.mr[k][b] * combined_r[b] - lanes.mi[k][b] * combined_i[b]; + total_i[b] += lanes.mr[k][b] * combined_i[b] + lanes.mi[k][b] * combined_r[b]; + } + } + for (std::size_t b = 0; b < n; ++b) { + lanes.vr[b] = + lanes.fr[b] * lanes.ur[b] - lanes.fi[b] * lanes.ui[b] + dz * total_r[b]; + lanes.vi[b] = + lanes.fr[b] * lanes.ui[b] + lanes.fi[b] * lanes.ur[b] + dz * total_i[b]; + } + double squares = 0.0; + for (std::size_t b = 0; b < n; ++b) + if (in_norm(first + b) != 0) + squares += lanes.ur[b] * lanes.ur[b] + lanes.ui[b] * lanes.ui[b]; + if (weights.compensated) { + adams_lanes_predicted_compensation(lanes, weights, inverse_dz, dz, rates, states, + column, first); + for (std::size_t t = 0; t + 1 < weights.width; ++t) { + Complex* out = &carried(t, column, first); + for (std::size_t b = 0; b < n; ++b) + out[b] = Complex(lanes.cr[t][b], lanes.ci[t][b]); + } + Complex* lab_out = &lab(column, first); + for (std::size_t b = 0; b < n; ++b) + lab_out[b] = Complex(lanes.vr[b], lanes.vi[b]); + } + Complex* out = &predicted(column, first); + for (std::size_t b = 0; b < n; ++b) + out[b] = Complex(lanes.vr[b], lanes.vi[b]); + return squares; +} +#endif + +// The squared scaled norms of up to three factored defects on the in-band elements of +// [first, last) of a column, added to `update`: a divided difference of the samples +// times one moment combination, and a varying operator's compensation term, over the +// divisor `scale[index - first]`. The defects and their compensation: strict (solprop +// SEP 25 section 4.2). +template +SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION void +adams_defect_block(const ExponentialWeights& weights, const std::size_t rows, + const Store& store, const Moments& moments, const Points& source, + const Carried& carried, const Points& endpoint, const Mask& in_norm, + const double dz, const double* scale, const std::size_t column, + const std::size_t first, const std::size_t last, double* update) { + SOLPROP_NATIVE_STRICT_BLOCK + // The block's sums, added to the reduction's once. A trial scores at most three + // defects (adams_exponential_trial refuses more); the bound says so here too. + constexpr std::size_t max_rows = 3; + const std::size_t scored = rows < max_rows ? rows : max_rows; + double sums[max_rows] = {0.0, 0.0, 0.0}; + for (std::size_t index = first; index < last; ++index) { + if (in_norm(index) == 0) + continue; + for (std::size_t row = 0; row < scored; ++row) { + Complex difference = strict_scale(weights.rows[row][0], source(column, index)); + for (std::size_t t = 1; t < weights.width; ++t) + difference = + strict_accumulate(difference, weights.rows[row][t], + adams_history(store, weights, t - 1, column, index)); + Complex weight{}; + for (std::size_t k = 0; k < weights.count; ++k) + weight = strict_accumulate(weight, weights.rows[row][weights.width + k], + moments(k, column, index)); + Complex defect = strict_scale(dz, strict_mul(difference, weight)); + if (weights.compensated) { + Complex compensation = + strict_scale(weights.comp_rows[row][0], endpoint(column, index)); + for (std::size_t t = 0; t + 1 < weights.width; ++t) + compensation = strict_accumulate(compensation, weights.comp_rows[row][t + 1], + carried(t, column, index)); + defect = strict_accumulate(defect, dz, compensation); + } + sums[row] += squared_abs(strict_divide(defect, scale[index - first])); + } + } + for (std::size_t row = 0; row < scored; ++row) + update[row] += sums[row]; +} + +/** + * The elementwise trial's last pass (solprop SEP 16 phase 3b), a block of a column a + * task: the new sample and the corrected field `exp(x) u + dz sum_k M_k sum_j c_jk + * F_j` with a varying operator's compensation, then the squared scaled norms of up to + * three factored defects, reduced in the same traversal. solprop's own controller's + * divisor is formed here from the accepted state's magnitude, as its + * PIController.scale forms it; any other controller's is read from `divisor`. + */ +struct ExponentialCorrection { + using value_type = double[]; + using size_type = std::size_t; + size_type value_count; + Kokkos::View store, moments, stages, carried; + Kokkos::View phi_terms; + Kokkos::View source, flow, current, advanced, + exponent; + Kokkos::View lab, endpoint, endpoint_rate; + Kokkos::View rate_weights; + Kokkos::View in_norm; + Kokkos::View divisor, scale_weights; + Kokkos::View magnitude_sum; + ExponentialWeights weights; + double dz, inverse_dz, rtol, atol; + std::size_t n, blocks, n_phi; + bool device_scale; + // The host lanes' predict pass formed the endpoint rate (off a separable route). + bool rate_formed; + +#ifdef SOLPROP_NATIVE_ADAMS_LANES + // The pass on a host block's lanes (`AdamsLanes`), as the element-by-element form + // below writes it; the norms' sums are taken in element order. + void lanes_block(const std::size_t column, const std::size_t first, + const std::size_t last, const double magnitude, + value_type update) const { + constexpr std::size_t B = AdamsLanes::B; + AdamsLanes lanes; + const std::size_t count = last - first; + lanes.count = count; + const Complex* sample = &stages(0, column, first); + Complex* sample_out = &source(column, first); + for (std::size_t b = 0; b < count; ++b) { + lanes.wr[b] = sample[b].real(); + lanes.wi[b] = sample[b].imag(); + sample_out[b] = sample[b]; + } + adams_lanes_history(lanes, weights, store, weights.width - 1, column, first); + for (std::size_t k = 0; k < weights.count; ++k) { + const Complex* moment = &moments(k, column, first); + for (std::size_t b = 0; b < count; ++b) { + lanes.mr[k][b] = moment[b].real(); + lanes.mi[k][b] = moment[b].imag(); + } + } + double total_r[B]; + double total_i[B]; + for (std::size_t b = 0; b < count; ++b) { + total_r[b] = 0.0; + total_i[b] = 0.0; + } + for (std::size_t k = 0; k < weights.count; ++k) { + double combined_r[B]; + double combined_i[B]; + const double weight0 = weights.corrector[0][k]; + for (std::size_t b = 0; b < count; ++b) { + combined_r[b] = weight0 * lanes.wr[b]; + combined_i[b] = weight0 * lanes.wi[b]; + } + for (std::size_t t = 1; t < weights.width; ++t) { + const double weight = weights.corrector[t][k]; + for (std::size_t b = 0; b < count; ++b) { + combined_r[b] += weight * lanes.hr[t - 1][b]; + combined_i[b] += weight * lanes.hi[t - 1][b]; + } + } + for (std::size_t b = 0; b < count; ++b) { + total_r[b] += lanes.mr[k][b] * combined_r[b] - lanes.mi[k][b] * combined_i[b]; + total_i[b] += lanes.mr[k][b] * combined_i[b] + lanes.mi[k][b] * combined_r[b]; + } + } + const Complex* propagate = &flow(column, first); + const Complex* accepted = ¤t(column, first); + for (std::size_t b = 0; b < count; ++b) { + const Complex value = + propagate[b] * accepted[b] + dz * Complex(total_r[b], total_i[b]); + lanes.vr[b] = value.real(); + lanes.vi[b] = value.imag(); + } + if (weights.compensated) { + const Complex* x = &exponent(column, first); + const Complex* field = &lab(column, first); + for (std::size_t b = 0; b < count; ++b) { + lanes.xr[b] = x[b].real(); + lanes.xi[b] = x[b].imag(); + lanes.ur[b] = field[b].real(); + lanes.ui[b] = field[b].imag(); + } + for (std::size_t t = 0; t + 1 < weights.width; ++t) { + const Complex* row = &carried(t, column, first); + for (std::size_t b = 0; b < count; ++b) { + lanes.cr[t][b] = row[b].real(); + lanes.ci[t][b] = row[b].imag(); + } + } + if (rate_formed) { + const Complex* rate = &endpoint_rate(column, first); + for (std::size_t b = 0; b < count; ++b) { + lanes.dr[b] = rate[b].real(); + lanes.di[b] = rate[b].imag(); + } + } else { + adams_lanes_rate(lanes, n_phi, rate_weights, phi_terms, endpoint_rate, column, + first); + } + adams_lanes_corrected_compensation(lanes, weights, inverse_dz, dz); + Complex* out = &endpoint(column, first); + for (std::size_t b = 0; b < count; ++b) + out[b] = Complex(lanes.nr[b], lanes.ni[b]); + } + Complex* out = &advanced(column, first); + for (std::size_t b = 0; b < count; ++b) + out[b] = Complex(lanes.vr[b], lanes.vi[b]); + if (device_scale) { + if (scale_weights.extent(0) != 0) + for (std::size_t b = 0; b < count; ++b) + lanes.scale[b] = rtol * scale_weights(first + b) * magnitude + atol; + else + for (std::size_t b = 0; b < count; ++b) + lanes.scale[b] = rtol * magnitude + atol; + } else { + for (std::size_t b = 0; b < count; ++b) + lanes.scale[b] = divisor(first + b); + } + constexpr std::size_t max_rows = 3; + const std::size_t scored = value_count < max_rows ? value_count : max_rows; + adams_lanes_defects(lanes, weights, scored, dz); + double sums[max_rows] = {0.0, 0.0, 0.0}; + for (std::size_t b = 0; b < count; ++b) { + if (in_norm(first + b) == 0) + continue; + for (std::size_t row = 0; row < scored; ++row) + sums[row] += lanes.defect[row][b]; + } + for (std::size_t row = 0; row < scored; ++row) + update[row] += sums[row]; + } +#endif + +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // One device element's pass, `Width` the trial's samples known at compile time: as + // the loops below and `adams_defect_block` form it, in the same operations and + // order, but with the history rows and moments read once into registers. Through + // the views, which the pass also writes, every use reloads them; on an AMD Instinct + // MI325X this pass is latency-bound on a single-mode trial's few thousand elements. + template + KOKKOS_INLINE_FUNCTION void + device_element(const std::size_t column, const std::size_t index, + const double magnitude, value_type update) const { + const Complex g = stages(0, column, index); + source(column, index) = g; + Complex history[Width]; + for (std::size_t t = 0; t + 1 < Width; ++t) + history[t] = adams_history(store, weights, t, column, index); + Complex moment[Width]; + for (std::size_t k = 0; k < Width; ++k) + if (k < weights.count) + moment[k] = moments(k, column, index); + Complex total{}; + for (std::size_t k = 0; k < Width; ++k) { + if (k >= weights.count) + break; + Complex combined = weights.corrector[0][k] * g; + for (std::size_t t = 1; t < Width; ++t) + combined += weights.corrector[t][k] * history[t - 1]; + total += moment[k] * combined; + } + Complex value = flow(column, index) * current(column, index) + dz * total; + if (weights.compensated) + value = adams_corrected_compensation(weights, n_phi, rate_weights, phi_terms, + exponent(column, index), inverse_dz, + lab(column, index), dz, value, endpoint_rate, + endpoint, carried, column, index); + advanced(column, index) = value; + if (in_norm(index) == 0) + return; + const double scale = device_scale + ? (scale_weights.extent(0) != 0 + ? rtol * scale_weights(index) * magnitude + atol + : rtol * magnitude + atol) + : divisor(index); + constexpr std::size_t max_rows = 3; + const std::size_t scored = value_count < max_rows ? value_count : max_rows; + for (std::size_t row = 0; row < max_rows; ++row) { + if (row >= scored) + break; + Complex difference = strict_scale(weights.rows[row][0], g); + for (std::size_t t = 1; t < Width; ++t) + difference = + strict_accumulate(difference, weights.rows[row][t], history[t - 1]); + Complex weight{}; + for (std::size_t k = 0; k < Width; ++k) { + if (k >= weights.count) + break; + weight = strict_accumulate(weight, weights.rows[row][Width + k], moment[k]); + } + Complex defect = strict_scale(dz, strict_mul(difference, weight)); + if (weights.compensated) { + Complex compensation = + strict_scale(weights.comp_rows[row][0], endpoint(column, index)); + for (std::size_t t = 0; t + 1 < Width; ++t) + compensation = strict_accumulate(compensation, weights.comp_rows[row][t + 1], + carried(t, column, index)); + defect = strict_accumulate(defect, dz, compensation); + } + update[row] += squared_abs(strict_divide(defect, scale)); + } + } +#endif + + KOKKOS_INLINE_FUNCTION void operator()(const std::size_t task, + value_type update) const { + const auto column = task / blocks; + const auto first = (task % blocks) * adams_block; + const auto last = first + adams_block < n ? first + adams_block : n; + const double magnitude = device_scale ? Kokkos::sqrt(magnitude_sum()) : 0.0; +#ifdef SOLPROP_NATIVE_ADAMS_LANES + if constexpr (vmath::vectorised) { + lanes_block(column, first, last, magnitude, update); + return; + } +#endif +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + static_assert(adams_block == 1, "a device thread takes one element"); + if (first < last) { + // A switch, as in `adams_device_predict`. + switch (weights.width) { + case 1: + return device_element<1>(column, first, magnitude, update); + case 2: + return device_element<2>(column, first, magnitude, update); + case 3: + return device_element<3>(column, first, magnitude, update); + case 4: + return device_element<4>(column, first, magnitude, update); + case 5: + return device_element<5>(column, first, magnitude, update); + case 6: + return device_element<6>(column, first, magnitude, update); + case 7: + return device_element<7>(column, first, magnitude, update); + case 8: + return device_element<8>(column, first, magnitude, update); + default: + break; + } + } +#endif + double scale[adams_block]{}; + for (std::size_t index = first; index < last; ++index) { + const Complex g = stages(0, column, index); + source(column, index) = g; + Complex total{}; + for (std::size_t k = 0; k < weights.count; ++k) { + Complex combined = weights.corrector[0][k] * g; + for (std::size_t t = 1; t < weights.width; ++t) + combined += weights.corrector[t][k] * + adams_history(store, weights, t - 1, column, index); + total += moments(k, column, index) * combined; + } + Complex value = flow(column, index) * current(column, index) + dz * total; + // The new sample's compensation at the predicted field, untransported, with the + // rate read on the side of the step's end the step came from. + if (weights.compensated) + value = adams_corrected_compensation( + weights, n_phi, rate_weights, phi_terms, exponent(column, index), + inverse_dz, lab(column, index), dz, value, endpoint_rate, endpoint, carried, + column, index); + advanced(column, index) = value; + if (in_norm(index) == 0) + continue; + if (device_scale) + scale[index - first] = scale_weights.extent(0) != 0 + ? rtol * scale_weights(index) * magnitude + atol + : rtol * magnitude + atol; + else + scale[index - first] = divisor(index); + } + adams_defect_block(weights, value_count, store, moments, source, carried, endpoint, + in_norm, dz, scale, column, first, last, update); + } + KOKKOS_INLINE_FUNCTION void join(value_type destination, + const value_type source_values) const { + for (size_type row = 0; row < value_count; ++row) + destination[row] += source_values[row]; + } + KOKKOS_INLINE_FUNCTION void init(value_type values) const { + for (size_type row = 0; row < value_count; ++row) + values[row] = 0.0; + } +}; + +} // namespace + +// The exponential trial without a coupled flow, every formula elementwise, in three +// passes on the trial's own width (solprop SEP 16 phase 3b). The first forms the +// step's exponent, flow and moments, a varying operator's carried samples and the +// prediction, and reduces the accepted state's magnitude for the norm's divisor; the +// second is the source at the prediction; the third is ExponentialCorrection. Each +// threaded pass costs a fork and a join, which on a host is most of what one of these +// passes costs on a few thousand elements. The prediction is written beside the +// accepted state and the two exchange places around the source, so the accepted +// state is never copied. +std::vector GeneralWorkspace::adams_elementwise_trial( + const double dz, const ExponentialWeights& weights, + const std::vector& divisor, const TrialScalars& step, + const TrialScalars& sample, const std::vector& rate) { + const auto columns = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + const bool compensated = weights.compensated; + // solprop's own controller's scale is formed from the magnitude below; any other + // controller's divisor crosses. + const bool device_scale = divisor.empty() && adams_device_scale_; + if (!device_scale) + adams_upload_divisor(divisor); + // The corrector forms a compensated trial's endpoint rate and compensation; only its + // weights cross, with the step's and the sample's scalars, in the trial's one copy. + upload_trial(step, &sample, compensated ? &rate : nullptr); + const std::size_t blocks = (n + adams_block - 1) / adams_block; +#ifdef KOKKOS_ENABLE_CUDA + // A device thread takes one element of a long FP64 chain, and Kokkos's CUDA range + // reduction gives each block 256 of them: a four-mode capillary's 10496 elements + // were 41 blocks on an NVIDIA A40's 84 multiprocessors. Blocks of 128 take them + // all. + using Tasks = Kokkos::RangePolicy>; +#else + using Tasks = Kokkos::RangePolicy; +#endif + const auto tasks = Tasks(adams_execution_space_, 0, columns * blocks); + const bool separable = problem_.separable_geometry(); + const auto spatial = spatial_; + const auto active = active_mask_; + const auto n_pol = problem_.dims.n_pol_state; + const auto n_stages = problem_.stepper.n_stages; + const auto phi_terms = phi_terms_; + const auto phi_delta = stage_phi_delta_; + const auto n_phi = problem_.linear.n_phi_terms; + const auto offset = n_stages * n_phi; + const auto exponent = adams_exponent_; + const auto flow = adams_propagator_; + const auto moments = adams_moments_; + const auto store = adams_store_; + const auto current = current_; + const auto predicted = adams_saved_; + const auto lab = adams_lab_; + const auto carried = adams_carried_; + const auto states = adams_states_; + const auto rates = adams_rates_; + const auto in_norm = norm_mask_; + const std::size_t count = weights.count; + const std::size_t formed = count > weights.order ? count : weights.order; + const double inverse_dz = 1.0 / dz; +#ifdef SOLPROP_NATIVE_ADAMS_LANES + // A host block's predict pass forms the corrector's endpoint rate beside the phase, + // from the weights the trial uploaded with its scalars. + const auto endpoint_rates = stage_rate_weights_; + const auto endpoint_rate = adams_endpoint_rate_; +#endif + // The step's exponent, on a separable route the step's length times each element's + // operator as the stage kernel forms the whole propagator, and its flow; its + // moments (kept for the corrector and the norms); a varying operator's carried + // samples; and the prediction, exp(x) u + dz sum_k M_k sum_j c_jk F_j and the + // compensation's predictor, the field the lab-frame source is evaluated at. + const auto predict = KOKKOS_LAMBDA(const std::size_t task, double& magnitude) { + const auto column = task / blocks; + const auto first = (task % blocks) * adams_block; + const auto last = first + adams_block < n ? first + adams_block : n; + // With vector maths a host block forms its exponentials in passes and the rest of + // the pass on its lanes (`adams_lanes_predict`); otherwise each element forms its + // own. +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + constexpr bool passes = vmath::vectorised; +#else + // Read only by the host's lanes (Clang's -Wunused-variable on HIP builds). + [[maybe_unused]] constexpr bool passes = false; +#endif + if (separable) + for (std::size_t index = first; index < last; ++index) { + const Complex x = + phi_delta(n_stages) * + spatial.value(column / n_pol, index, active(index) != 0U).linear; + exponent(column, index) = x; + // A device lambda may not first capture a view inside `if constexpr` (nvcc): + // devices never form passes, so the guard is the host's alone. +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + if constexpr (!passes) +#endif + flow(column, index) = exponential(x); + } +#ifdef SOLPROP_NATIVE_ADAMS_LANES + if constexpr (passes) { + magnitude += separable + ? adams_lanes_predict( + phi_terms, phi_delta, endpoint_rates, endpoint_rate, + offset, n_phi, weights, formed, dz, inverse_dz, exponent, + flow, current, predicted, lab, moments, store, carried, + states, rates, in_norm, column, first, last) + : adams_lanes_predict( + phi_terms, phi_delta, endpoint_rates, endpoint_rate, + offset, n_phi, weights, formed, dz, inverse_dz, exponent, + flow, current, predicted, lab, moments, store, carried, + states, rates, in_norm, column, first, last); + return; + } +#endif + if (!separable) + adams_phase_block(phi_terms, phi_delta, offset, n_phi, column, first, last, + exponent, flow); + double squares = 0.0; + for (std::size_t index = first; index < last; ++index) { + const Complex x = exponent(column, index); + const Complex propagate = flow(column, index); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const Complex accepted = current(column, index); + if (in_norm(index) != 0) + squares += squared_abs(accepted); + Complex value = adams_device_predict(weights, formed, x, propagate, accepted, dz, + moments, store, column, index); +#else + Complex phi[2 * adams_max_samples + 1]; + adams_moment_phi(weights, x, propagate, formed, phi); + for (std::size_t k = 0; k < count; ++k) + moments(k, column, index) = adams_factorial(k) * phi[k + 1]; + const Complex accepted = current(column, index); + if (in_norm(index) != 0) + squares += squared_abs(accepted); + Complex total{}; + for (std::size_t k = 0; k < weights.order; ++k) { + Complex combined{}; + for (std::size_t t = 0; t < weights.order; ++t) + combined += + weights.predictor[t][k] * adams_history(store, weights, t, column, index); + const Complex moment = adams_factorial(k) * phi[k + 1]; + total += moment * combined; + } + Complex value = propagate * accepted + dz * total; +#endif + if (compensated) { + value = + adams_predicted_compensation(weights, x, propagate, inverse_dz, dz, value, + rates, states, carried, column, index); + lab(column, index) = value; + } + predicted(column, index) = value; + } + magnitude += squares; + }; + if (device_scale) { + // A threaded reduction in whatever order measures fastest (solprop SEP 25). + launch::parallel_reduce("solprop_native.adams.exponential_predict", tasks, predict, + adams_magnitude_sum_); + } else { + double unused = 0.0; + launch::parallel_reduce("solprop_native.adams.exponential_predict", tasks, predict, + unused); + } + std::swap(current_, adams_saved_); + use_uploaded_sample(sample); + // On a device the source's launches replay as one graph while they repeat. + adams_source_graph_.run(execution_space_, [&] { adams_evaluate_source(sample); }); + std::swap(current_, adams_saved_); + const ExponentialCorrection reducer{.value_count = weights.n_rows, + .store = store, + .moments = moments, + .stages = stages_, + .carried = carried, + .phi_terms = phi_terms, + .source = adams_source_, + .flow = flow, + .current = current_, + .advanced = advanced_, + .exponent = exponent, + .lab = lab, + .endpoint = adams_endpoint_, + .endpoint_rate = adams_endpoint_rate_, + .rate_weights = stage_rate_weights_, + .in_norm = in_norm, + .divisor = adams_divisor_, + .scale_weights = adams_scale_weights_, + .magnitude_sum = adams_magnitude_sum_, + .weights = weights, + .dz = dz, + .inverse_dz = inverse_dz, + .rtol = adams_rtol_, + .atol = adams_atol_, + .n = n, + .blocks = blocks, + .n_phi = n_phi, + .device_scale = device_scale, + .rate_formed = !separable}; + auto norms = adams_norms("solprop_native.adams.exponential_correct", tasks, reducer, + weights.n_rows); + adams_trial_compensated_ = compensated; + adams_trial_coupled_ = false; + return norms; +} + +// Refuse a trial whose bins off the flow's eigenbasis route would need more squarings +// than solprop's reference admits for a flow action (solprop SEP 47 phase 4): the +// augmented exponential of each bin's shifted block for the moments, and the block's +// own `span dz T` for each carried sample. Before any pass of the trial moves the +// state, so a refusal leaves the accepted step as it was. +void GeneralWorkspace::adams_fallback_squarings(const std::size_t section, + const double dz, + const ExponentialWeights& weights) { + const auto slots = flow_->block_slots(); + const auto route = flow_->route(); + const auto triangular = flow_->fallback_triangular(); + const auto exponent = adams_exponent_; + const auto modes = problem_.dims.n_modes; + const std::size_t count = weights.count; + const auto policy = Kokkos::RangePolicy(flow_execution_space_, 0, + problem_.dims.n_omega); + // The moments' augmented exponent, and the carries' `span dz T`, apart, so that a + // refusal names which. + int moments = 0; + int carries = 0; + launch::parallel_reduce( + "solprop_native.adams.fallback_squarings", policy, + KOKKOS_LAMBDA(const std::size_t index, int& most, int& carried) { + const auto one = slots(index); + if (one == 0) + return; + const auto path = route(section, one - 1); + if (path == 0) + return; + const Complex argument = adams_moment_argument(weights, exponent(0, index)); + const int squarings = flow_functions::augmented_squarings( + argument, dz, triangular, path - 1, modes, count); + most = squarings > most ? squarings : most; + if (weights.compensated) + for (std::size_t t = 0; t + 1 < weights.width; ++t) { + const int carry = flow_functions::augmented_squarings( + Complex{}, weights.span[t] * dz, triangular, path - 1, modes, 0); + carried = carry > carried ? carry : carried; + } + }, + Kokkos::Max(moments), Kokkos::Max(carries)); + const std::string bin = + "native cannot take the flow and moments of a coupled flow's bin off its " + "eigenbasis route"; + if (moments == flow_functions::non_finite || carries == flow_functions::non_finite) + throw BackendRefusal(bin + ": the step's exponent there is not finite"); + if (moments <= flow_functions::max_squarings && + carries <= flow_functions::max_squarings) + return; + std::array length{}; + std::snprintf(length.data(), length.size(), "%.4g", dz); + const std::string limit = + concat(" would need more than ", flow_functions::max_squarings, + " squarings, the most this backend admits. Cap dz_max"); + if (moments > flow_functions::max_squarings) + refuse(bin, " over a step of ", length.data(), " m: its exponent", + limit); + refuse(bin, " over a step of ", length.data(), + " m: a history sample's carry to the step's end", limit); +} + +// NOLINTNEXTLINE(readability-function-size): hot path, split pending an A/B +std::vector GeneralWorkspace::adams_exponential_trial( + const double dz, const ExponentialWeights& weights, + const std::vector& divisor, const TrialScalars& step, + const TrialScalars& sample, const std::vector& rate) { + require_adams(); + if (weights.compensated && adams_states_.extent(0) == 0) + throw std::logic_error("the Adams history carries no compensation"); + // Beside a coupled flow the step's flow and moments are matrices on the block's bins + // (solprop SEP 16 phases 5 and 6); the sample is the material source alone. + const bool coupled = static_cast(flow_); + if (coupled && adams_block_.extent(0) == 0) + throw std::logic_error("the Adams history holds no coupled block"); + if (weights.n_rows == 0 || weights.n_rows > 3 || weights.width == 0 || + weights.width > adams_store_.extent(0) || weights.count > weights.width || + weights.order >= weights.width) + throw std::invalid_argument("Adams coefficient tables are out of range"); + for (std::size_t t = 0; t + 1 < weights.width; ++t) + if (weights.slot[t] >= adams_store_.extent(0)) + throw std::out_of_range("Adams history row"); + require_trial_scalars(step); + require_trial_scalars(sample); + if (!coupled) + return adams_elementwise_trial(dz, weights, divisor, step, sample, rate); + adams_upload_divisor(divisor); + // The step's exponent and flow, and its moments, once for every formula; the + // sample's scalars cross in the same copy. + upload_trial(step, &sample); + adams_phase(); + const std::size_t section = step.flow_section; + const bool residual = adams_applied_.extent(0) != 0; + if (residual) + upload_weights(execution_space_, adams_end_weights_, step.flow_weights); + const auto columns = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); + const auto bins = Kokkos::RangePolicy(flow_execution_space_, 0, n); + const auto exponent = adams_exponent_; + const auto flow = adams_propagator_; + const auto moments = adams_moments_; + const std::size_t count = weights.count; + const auto slots = flow_->block_slots(); + // The elementwise moments, off the block's bins: on them the block's matrices below + // take their place. + launch::parallel_for( + "solprop_native.adams.moments", policy, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + if (slots(index) != 0) + return; + Complex phi[2 * adams_max_samples + 1]; + adams_moment_phi(weights, exponent(column, index), flow(column, index), count, + phi); + for (std::size_t k = 0; k < count; ++k) + moments(k, column, index) = adams_factorial(k) * phi[k + 1]; + }); + // The block's flow and moments, W f(x + h lambda) Y from the section's eigenbasis on + // its route; a bin off it is formed from its Schur factors by the augmented + // exponential (`flow_functions`), on a host and a device alike. + const auto block = adams_block_; + // A host's channel vectors of a block wider than `adams_max_channels`. + [[maybe_unused]] const auto channels = adams_channels_; + const auto modes = problem_.dims.n_modes; + // The bins off the eigenbasis route (solprop SEP 47 phase 4). + const auto route = flow_->route(); + const auto fallback_vectors = flow_->fallback_vectors(); + const auto fallback_triangular = flow_->fallback_triangular(); + const auto fallback = adams_fallback_; + const bool falls_back = flow_->falls_back(section); + if (flow_->falls_back(section)) + adams_fallback_squarings(section, dz, weights); + const auto left = flow_->left(); + const auto right = flow_->right(); + const auto eigenvalues = flow_->eigenvalues(); + const auto terms = + residual ? flow_->residual_terms() : CoupledFlowAction::ComplexView4{}; + const auto frozen = residual ? flow_->frozen() : CoupledFlowAction::ComplexView4{}; + // A device takes a thread per (bin, channel) in the coupled passes below: its + // lanes outnumber the bins and a channel vector does not fit a thread's + // registers. A host keeps a task per bin, which forms a bin's shared terms once. +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + launch::parallel_for( + "solprop_native.adams.block_functions", + Range2(flow_execution_space_, {0, 0}, {n, modes * modes}), + KOKKOS_LAMBDA(const std::size_t index, const std::size_t pair) { + const auto one = slots(index); + if (one == 0) + return; + const auto bin = one - 1; + // A bin off the route is the next pass's. + if (falls_back && route(section, bin) != 0) + return; + const auto i = pair / modes; + const auto j = pair % modes; + const Complex shift = exponent(0, index); + // One entry (i, j) of every function's matrix, a thread each: the sum over + // the eigenvalues in their order, each function formed where it is needed. + Complex total[adams_max_samples + 1]; + for (std::size_t k = 0; k <= count; ++k) + total[k] = Complex{}; + Complex phi[2 * adams_max_samples + 1]; + for (std::size_t m = 0; m < modes; ++m) { + const Complex x = shift + dz * eigenvalues(section, bin, m); + const Complex exp_x = exponential(x); + // The flow is exp(h B); the moments are the shifted block's (solprop SEP + // 35 section 4.5): every eigenvalue moves by -sigma. + adams_moment_phi(weights, x, exp_x, count, phi); + const Complex l = left(section, bin, i, m); + const Complex r = right(section, bin, m, j); + total[0] += l * exp_x * r; + for (std::size_t k = 0; k < count; ++k) + total[k + 1] += l * (adams_factorial(k) * phi[k + 1]) * r; + } + for (std::size_t k = 0; k <= count; ++k) + block(k, bin, i, j) = total[k]; + }); + if (falls_back) + // Off the eigenbasis route, a thread per bin: the host task's passes, in its order + // (the first block row of the augmented exponential of the shifted block, carried + // back by Z), its matrices in the bin's scratch row rather than on a stack. + launch::parallel_for( + "solprop_native.adams.fallback_block_functions", bins, + KOKKOS_LAMBDA(const std::size_t index) { + const auto one = slots(index); + if (one == 0) + return; + const auto bin = one - 1; + const auto path = route(section, bin); + if (path == 0) + return; + const auto f = path - 1; + const Complex argument = adams_moment_argument(weights, exponent(0, index)); + flow_functions::augmented_row( + argument, dz, fallback_triangular, f, modes, count, + flow_functions::augmented_squarings(argument, dz, fallback_triangular, f, + modes, count), + fallback, bin); + const Complex factor = weights.shifted + ? exponential(Complex(0.0, weights.shift)) + : Complex(1.0, 0.0); + const auto product = count + 1; + for (std::size_t k = 0; k <= count; ++k) { + for (std::size_t i = 0; i < modes; ++i) + for (std::size_t j = 0; j < modes; ++j) { + Complex total{}; + for (std::size_t m = 0; m <= j; ++m) + total += fallback_vectors(f, i, m) * fallback(bin, k, m, j); + fallback(bin, product, i, j) = total; + } + const Complex scale = + k == 0 ? factor : Complex(adams_factorial(k - 1), 0.0); + for (std::size_t i = 0; i < modes; ++i) + for (std::size_t j = 0; j < modes; ++j) { + Complex total{}; + for (std::size_t m = 0; m < modes; ++m) + total += fallback(bin, product, i, m) * + Kokkos::conj(fallback_vectors(f, j, m)); + block(k, bin, i, j) = scale * total; + } + } + }); +#else + static_cast(falls_back); + launch::parallel_for( + "solprop_native.adams.block_functions", bins, + KOKKOS_LAMBDA(const std::size_t index) { + const auto one = slots(index); + if (one == 0) + return; + const auto bin = one - 1; + const auto path = route(section, bin); + if (path != 0) { + // Off the eigenbasis route: the first block row of the augmented + // exponential of the shifted block, carried back by Z, the moments + // scaled by k! and the flow by the carrier's exp(sigma) (solprop's + // `coupled_moments`). + const auto f = path - 1; + const Complex argument = adams_moment_argument(weights, exponent(0, index)); + flow_functions::augmented_row( + argument, dz, fallback_triangular, f, modes, count, + flow_functions::augmented_squarings(argument, dz, fallback_triangular, f, + modes, count), + fallback, bin); + const Complex factor = weights.shifted + ? exponential(Complex(0.0, weights.shift)) + : Complex(1.0, 0.0); + const auto product = count + 1; + for (std::size_t k = 0; k <= count; ++k) { + for (std::size_t i = 0; i < modes; ++i) + for (std::size_t j = 0; j < modes; ++j) { + Complex total{}; + for (std::size_t m = 0; m <= j; ++m) + total += fallback_vectors(f, i, m) * fallback(bin, k, m, j); + fallback(bin, product, i, j) = total; + } + const Complex scale = + k == 0 ? factor : Complex(adams_factorial(k - 1), 0.0); + for (std::size_t i = 0; i < modes; ++i) + for (std::size_t j = 0; j < modes; ++j) { + Complex total{}; + for (std::size_t m = 0; m < modes; ++m) + total += fallback(bin, product, i, m) * + Kokkos::conj(fallback_vectors(f, j, m)); + block(k, bin, i, j) = scale * total; + } + } + return; + } + const Complex shift = exponent(0, index); + // Eigenvalue m's functions at `values + m (adams_max_samples + 1)`. + constexpr std::size_t stride = adams_max_samples + 1; + Complex stack[adams_channel_stack]; + Complex* const values = adams_channel_rows(stack, channels, index); + Complex phi[2 * adams_max_samples + 1]; + for (std::size_t m = 0; m < modes; ++m) { + const Complex x = shift + dz * eigenvalues(section, bin, m); + const Complex exp_x = exponential(x); + values[m * stride] = exp_x; + adams_moment_phi(weights, x, exp_x, count, phi); + for (std::size_t k = 0; k < count; ++k) + values[m * stride + k + 1] = adams_factorial(k) * phi[k + 1]; + } + for (std::size_t k = 0; k <= count; ++k) + for (std::size_t i = 0; i < modes; ++i) + for (std::size_t j = 0; j < modes; ++j) { + Complex total{}; + for (std::size_t m = 0; m < modes; ++m) + total += left(section, bin, i, m) * values[m * stride + k] * + right(section, bin, m, j); + block(k, bin, i, j) = total; + } + }); +#endif + // A varying operator's samples carried to the step's end in the chord's flow: + // exp((1 - theta_j) x) (D(z_j) - x / dz) u_j, the difference of rates formed first. + // Beside a coupled flow a frozen-residual flow's R_s(z_j) u_j joins each, against + // the step's own section, and the block's bins are carried by its matrix flow. + const auto carried = adams_carried_; + const auto states = adams_states_; + const auto rates = adams_rates_; + const auto applied = adams_applied_; + const bool compensated = weights.compensated; + const double inverse_dz = 1.0 / dz; + if (compensated) +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + launch::parallel_for( + "solprop_native.adams.coupled_carry", + Range2(execution_space_, {0, 0}, {n, modes}), + KOKKOS_LAMBDA(const std::size_t index, const std::size_t i) { + const auto one = slots(index); + // A bin off the route is the next pass's. + if (falls_back && one != 0 && route(section, one - 1) != 0) + return; + // A sample's term on channel c, formed where it is needed: a thread holds no + // channel vectors. + const auto term = [&](const std::size_t slot, const std::size_t c) { + Complex value = + compensation_residual(rates(slot, c, index), exponent(c, index), + inverse_dz, states(slot, c, index)); + if (residual && one != 0) { + const auto bin = one - 1; + // R_s(z_j) u_j: the sample's own generator applied once, less the + // step's section's frozen block. + Complex extra = applied(slot, c, index); + for (std::size_t j = 0; j < modes; ++j) + extra = strict_sub(extra, strict_mul(frozen(section, bin, c, j), + states(slot, j, index))); + value = strict_add(value, extra); + } + return value; + }; + for (std::size_t t = 0; t + 1 < weights.width; ++t) { + const auto slot = weights.slot[t]; + const double span = weights.span[t]; + Complex result; + if (span == 0.0) { + result = term(slot, i); + } else if (one == 0) { + result = exponential(span * exponent(i, index)) * term(slot, i); + } else { + const auto bin = one - 1; + const Complex shift = exponent(0, index); + Complex total{}; + for (std::size_t m = 0; m < modes; ++m) { + Complex value{}; + for (std::size_t j = 0; j < modes; ++j) + value += right(section, bin, m, j) * term(slot, j); + total += + left(section, bin, i, m) * + (exponential(span * (shift + dz * eigenvalues(section, bin, m))) * + value); + } + result = total; + } + carried(t, i, index) = result; + } + }); + if (compensated && falls_back) + // Off the eigenbasis route, a thread per bin on the block functions' instance, + // whose scratch row it reuses: the host task's carry in its order, the bin's + // channel vectors in blocks 3 (the term) and 4 (its rotation) of the row. + launch::parallel_for( + "solprop_native.adams.fallback_carry", bins, + KOKKOS_LAMBDA(const std::size_t index) { + const auto one = slots(index); + if (one == 0) + return; + const auto bin = one - 1; + const auto path = route(section, bin); + if (path == 0) + return; + const auto f = path - 1; + constexpr std::size_t term = 3; + constexpr std::size_t rotated = 4; + for (std::size_t t = 0; t + 1 < weights.width; ++t) { + const auto slot = weights.slot[t]; + for (std::size_t i = 0; i < modes; ++i) + fallback(bin, term, 0, i) = + compensation_residual(rates(slot, i, index), exponent(i, index), + inverse_dz, states(slot, i, index)); + if (residual) + for (std::size_t i = 0; i < modes; ++i) { + // R_s(z_j) u_j: the sample's own generator applied once, less the + // step's section's frozen block. + Complex value = applied(slot, i, index); + for (std::size_t j = 0; j < modes; ++j) + value = strict_sub(value, strict_mul(frozen(section, bin, i, j), + states(slot, j, index))); + fallback(bin, term, 0, i) = + strict_add(fallback(bin, term, 0, i), value); + } + const double span = weights.span[t]; + if (span != 0.0) { + // The reference's flow matrix Z exp(span x) Z^H with x = shift I + dz T, + // the shift's exponential apart: exp(span shift) Z exp(span dz T) Z^H. + const double length = span * dz; + flow_functions::augmented_row( + Complex{}, length, fallback_triangular, f, modes, 0, + flow_functions::augmented_squarings(Complex{}, length, + fallback_triangular, f, modes, 0), + fallback, bin); + const Complex factor = exponential(span * exponent(0, index)); + for (std::size_t i = 0; i < modes; ++i) + for (std::size_t j = 0; j < modes; ++j) { + Complex total{}; + for (std::size_t m = 0; m <= j; ++m) + total += fallback_vectors(f, i, m) * fallback(bin, 0, m, j); + fallback(bin, 1, i, j) = total; + } + for (std::size_t i = 0; i < modes; ++i) + for (std::size_t j = 0; j < modes; ++j) { + Complex total{}; + for (std::size_t m = 0; m < modes; ++m) + total += fallback(bin, 1, i, m) * + Kokkos::conj(fallback_vectors(f, j, m)); + fallback(bin, 2, i, j) = factor * total; + } + for (std::size_t i = 0; i < modes; ++i) { + Complex value{}; + for (std::size_t j = 0; j < modes; ++j) + value += fallback(bin, 2, i, j) * fallback(bin, term, 0, j); + fallback(bin, rotated, 0, i) = value; + } + for (std::size_t i = 0; i < modes; ++i) + fallback(bin, term, 0, i) = fallback(bin, rotated, 0, i); + } + for (std::size_t i = 0; i < modes; ++i) + carried(t, i, index) = fallback(bin, term, 0, i); + } + }); +#else + launch::parallel_for( + "solprop_native.adams.coupled_carry", bins, + KOKKOS_LAMBDA(const std::size_t index) { + const auto one = slots(index); + Complex stack[adams_channel_stack]; + Complex* const term = adams_channel_rows(stack, channels, index); + Complex* const rotated = term + modes; + for (std::size_t t = 0; t + 1 < weights.width; ++t) { + const auto slot = weights.slot[t]; + for (std::size_t i = 0; i < modes; ++i) + term[i] = compensation_residual(rates(slot, i, index), exponent(i, index), + inverse_dz, states(slot, i, index)); + if (residual && one != 0) { + const auto bin = one - 1; + for (std::size_t i = 0; i < modes; ++i) { + // R_s(z_j) u_j: the sample's own generator applied once, less the + // step's section's frozen block. + Complex value = applied(slot, i, index); + for (std::size_t j = 0; j < modes; ++j) + value = strict_sub(value, strict_mul(frozen(section, bin, i, j), + states(slot, j, index))); + term[i] = strict_add(term[i], value); + } + } + const double span = weights.span[t]; + if (span != 0.0) { + if (one == 0) { + for (std::size_t i = 0; i < modes; ++i) + term[i] = exponential(span * exponent(i, index)) * term[i]; + } else if (const auto path = route(section, one - 1); path != 0) { + // Off the eigenbasis route, the reference's flow matrix + // Z exp(span x) Z^H with x = shift I + dz T, the shift's exponential + // apart: exp(span shift) Z exp(span dz T) Z^H, then applied. + const auto f = path - 1; + const auto bin = one - 1; + const double length = span * dz; + flow_functions::augmented_row( + Complex{}, length, fallback_triangular, f, modes, 0, + flow_functions::augmented_squarings( + Complex{}, length, fallback_triangular, f, modes, 0), + fallback, bin); + const Complex factor = exponential(span * exponent(0, index)); + for (std::size_t i = 0; i < modes; ++i) + for (std::size_t j = 0; j < modes; ++j) { + Complex total{}; + for (std::size_t m = 0; m <= j; ++m) + total += fallback_vectors(f, i, m) * fallback(bin, 0, m, j); + fallback(bin, 1, i, j) = total; + } + for (std::size_t i = 0; i < modes; ++i) + for (std::size_t j = 0; j < modes; ++j) { + Complex total{}; + for (std::size_t m = 0; m < modes; ++m) + total += fallback(bin, 1, i, m) * + Kokkos::conj(fallback_vectors(f, j, m)); + fallback(bin, 2, i, j) = factor * total; + } + for (std::size_t i = 0; i < modes; ++i) { + Complex value{}; + for (std::size_t j = 0; j < modes; ++j) + value += fallback(bin, 2, i, j) * term[j]; + rotated[i] = value; + } + for (std::size_t i = 0; i < modes; ++i) + term[i] = rotated[i]; + } else { + const auto bin = one - 1; + const Complex shift = exponent(0, index); + for (std::size_t m = 0; m < modes; ++m) { + Complex value{}; + for (std::size_t j = 0; j < modes; ++j) + value += right(section, bin, m, j) * term[j]; + rotated[m] = + exponential(span * (shift + dz * eigenvalues(section, bin, m))) * + value; + } + for (std::size_t i = 0; i < modes; ++i) { + Complex value{}; + for (std::size_t m = 0; m < modes; ++m) + value += left(section, bin, i, m) * rotated[m]; + term[i] = value; + } + } + } + for (std::size_t i = 0; i < modes; ++i) + carried(t, i, index) = term[i]; + } + }); +#endif + // The predicted lab-frame field, exp(x) u + dz sum_k M_k sum_j c_jk F_j and the + // compensation's predictor, is the field the lab-frame source is evaluated at; the + // accepted state is kept aside. + const auto store = adams_store_; + const auto current = current_; + const auto saved = adams_saved_; + const auto lab = adams_lab_; + graph::deep_copy(execution_space_, saved, current); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + launch::parallel_for( + "solprop_native.adams.coupled_predict", + Range2(execution_space_, {0, 0}, {n, modes}), + KOKKOS_LAMBDA(const std::size_t index, const std::size_t i) { + const auto one = slots(index); + // Channel c's combinations sum_t w_tk F_t, its samples read once. + const auto combine = [&](const std::size_t c, Complex* out) { + Complex samples[adams_max_samples]; + for (std::size_t t = 0; t < weights.order; ++t) + samples[t] = adams_history(store, weights, t, c, index); + for (std::size_t k = 0; k < weights.order; ++k) { + Complex total{}; + for (std::size_t t = 0; t < weights.order; ++t) + total += weights.predictor[t][k] * samples[t]; + out[k] = total; + } + }; + Complex combined[adams_max_samples]; + Complex value{}; + if (one == 0) { + combine(i, combined); + Complex total{}; + for (std::size_t k = 0; k < weights.order; ++k) + total += moments(k, i, index) * combined[k]; + value = flow(i, index) * saved(i, index) + dz * total; + } else { + const auto bin = one - 1; + Complex linear{}; + Complex total{}; + for (std::size_t j = 0; j < modes; ++j) { + linear += block(0, bin, i, j) * saved(j, index); + combine(j, combined); + for (std::size_t k = 0; k < weights.order; ++k) + total += block(k + 1, bin, i, j) * combined[k]; + } + value = linear + dz * total; + } + if (compensated) { + Complex compensation{}; + for (std::size_t t = 0; t < weights.order; ++t) + compensation = strict_accumulate(compensation, weights.comp_predictor[t], + carried(t, i, index)); + value = strict_accumulate(value, dz, compensation); + lab(i, index) = value; + } + current(i, index) = value; + }); +#else + launch::parallel_for( + "solprop_native.adams.coupled_predict", bins, + KOKKOS_LAMBDA(const std::size_t index) { + const auto one = slots(index); + // Combination k of channel i at `combined[k modes + i]`. + Complex stack[adams_channel_stack]; + Complex* const combined = adams_channel_rows(stack, channels, index); + for (std::size_t k = 0; k < weights.order; ++k) + for (std::size_t i = 0; i < modes; ++i) { + Complex total{}; + for (std::size_t t = 0; t < weights.order; ++t) + total += + weights.predictor[t][k] * adams_history(store, weights, t, i, index); + combined[k * modes + i] = total; + } + for (std::size_t i = 0; i < modes; ++i) { + Complex value{}; + if (one == 0) { + Complex total{}; + for (std::size_t k = 0; k < weights.order; ++k) + total += moments(k, i, index) * combined[k * modes + i]; + value = flow(i, index) * saved(i, index) + dz * total; + } else { + const auto bin = one - 1; + Complex linear{}; + Complex total{}; + for (std::size_t j = 0; j < modes; ++j) { + linear += block(0, bin, i, j) * saved(j, index); + for (std::size_t k = 0; k < weights.order; ++k) + total += block(k + 1, bin, i, j) * combined[k * modes + j]; + } + value = linear + dz * total; + } + if (compensated) { + Complex compensation{}; + for (std::size_t t = 0; t < weights.order; ++t) + compensation = strict_accumulate(compensation, weights.comp_predictor[t], + carried(t, i, index)); + value = strict_accumulate(value, dz, compensation); + lab(i, index) = value; + } + current(i, index) = value; + } + }); +#endif + use_uploaded_sample(sample); + { + const FlowAside aside(flow_); + adams_evaluate(sample); + } + // The new sample's compensation at the predicted field, untransported, with the + // rate read on the side of the step's end the step came from; beside a coupled flow + // a frozen-residual flow's residual at the step's end joins it. + const auto endpoint = adams_endpoint_; + const auto end_applied = adams_end_applied_; + if (compensated) { + adams_rate(rate, adams_endpoint_rate_); + const auto endpoint_rate = adams_endpoint_rate_; + launch::parallel_for( + "solprop_native.adams.endpoint", policy, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + endpoint(column, index) = compensation_residual( + endpoint_rate(column, index), exponent(column, index), inverse_dz, + lab(column, index)); + }); + if (residual) { + // The generator at the step's end applied to the predicted field: the new + // sample's G(z_n + h) u, kept for it on acceptance. + apply_generator(execution_space_, n, modes, slots, terms, adams_end_weights_, lab, + end_applied); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + launch::parallel_for( + "solprop_native.adams.endpoint_residual", + Range2(execution_space_, {0, 0}, {n, modes}), + KOKKOS_LAMBDA(const std::size_t index, const std::size_t i) { + const auto one = slots(index); + if (one == 0) + return; + const auto bin = one - 1; + Complex value = end_applied(i, index); + for (std::size_t j = 0; j < modes; ++j) + value = strict_sub(value, + strict_mul(frozen(section, bin, i, j), lab(j, index))); + endpoint(i, index) = strict_add(endpoint(i, index), value); + }); +#else + launch::parallel_for( + "solprop_native.adams.endpoint_residual", bins, + KOKKOS_LAMBDA(const std::size_t index) { + const auto one = slots(index); + if (one == 0) + return; + const auto bin = one - 1; + for (std::size_t i = 0; i < modes; ++i) { + Complex value = end_applied(i, index); + for (std::size_t j = 0; j < modes; ++j) + value = strict_sub( + value, strict_mul(frozen(section, bin, i, j), lab(j, index))); + endpoint(i, index) = strict_add(endpoint(i, index), value); + } + }); +#endif + } + } + const auto stages = stages_; + const auto source = adams_source_; + const auto advanced = advanced_; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + launch::parallel_for( + "solprop_native.adams.coupled_correct", + Range2(execution_space_, {0, 0}, {n, modes}), + KOKKOS_LAMBDA(const std::size_t index, const std::size_t i) { + const auto one = slots(index); + source(i, index) = stages(0, i, index); + // Channel c's combinations sum_t w_tk F_t with the new sample first. + const auto combine = [&](const std::size_t c, Complex* out) { + const Complex g = stages(0, c, index); + Complex samples[adams_max_samples]; + for (std::size_t t = 1; t < weights.width; ++t) + samples[t] = adams_history(store, weights, t - 1, c, index); + for (std::size_t k = 0; k < count; ++k) { + Complex total = weights.corrector[0][k] * g; + for (std::size_t t = 1; t < weights.width; ++t) + total += weights.corrector[t][k] * samples[t]; + out[k] = total; + } + }; + Complex combined[adams_max_samples]; + Complex value{}; + if (one == 0) { + combine(i, combined); + Complex total{}; + for (std::size_t k = 0; k < count; ++k) + total += moments(k, i, index) * combined[k]; + value = flow(i, index) * current(i, index) + dz * total; + } else { + const auto bin = one - 1; + Complex linear{}; + Complex total{}; + for (std::size_t j = 0; j < modes; ++j) { + linear += block(0, bin, i, j) * current(j, index); + combine(j, combined); + for (std::size_t k = 0; k < count; ++k) + total += block(k + 1, bin, i, j) * combined[k]; + } + value = linear + dz * total; + } + if (compensated) { + Complex compensation = + strict_scale(weights.comp_corrector[0], endpoint(i, index)); + for (std::size_t t = 0; t < weights.order; ++t) + compensation = strict_accumulate( + compensation, weights.comp_corrector[t + 1], carried(t, i, index)); + value = strict_accumulate(value, dz, compensation); + } + advanced(i, index) = value; + }); +#else + launch::parallel_for( + "solprop_native.adams.coupled_correct", bins, + KOKKOS_LAMBDA(const std::size_t index) { + const auto one = slots(index); + Complex stack[adams_channel_stack]; + Complex* const combined = adams_channel_rows(stack, channels, index); + for (std::size_t i = 0; i < modes; ++i) { + const Complex g = stages(0, i, index); + source(i, index) = g; + for (std::size_t k = 0; k < count; ++k) { + Complex total = weights.corrector[0][k] * g; + for (std::size_t t = 1; t < weights.width; ++t) + total += weights.corrector[t][k] * + adams_history(store, weights, t - 1, i, index); + combined[k * modes + i] = total; + } + } + for (std::size_t i = 0; i < modes; ++i) { + Complex value{}; + if (one == 0) { + Complex total{}; + for (std::size_t k = 0; k < count; ++k) + total += moments(k, i, index) * combined[k * modes + i]; + value = flow(i, index) * current(i, index) + dz * total; + } else { + const auto bin = one - 1; + Complex linear{}; + Complex total{}; + for (std::size_t j = 0; j < modes; ++j) { + linear += block(0, bin, i, j) * current(j, index); + for (std::size_t k = 0; k < count; ++k) + total += block(k + 1, bin, i, j) * combined[k * modes + j]; + } + value = linear + dz * total; + } + if (compensated) { + Complex compensation = + strict_scale(weights.comp_corrector[0], endpoint(i, index)); + for (std::size_t t = 0; t < weights.order; ++t) + compensation = strict_accumulate( + compensation, weights.comp_corrector[t + 1], carried(t, i, index)); + value = strict_accumulate(value, dz, compensation); + } + advanced(i, index) = value; + } + }); +#endif + const CoupledDefectNorms reducer{.value_count = weights.n_rows, + .store = store, + .moments = moments, + .source = source, + .carried = carried, + .endpoint = endpoint, + .block = block, + .slots = slots, + .in_norm = norm_mask_, + .divisor = adams_divisor_, + .weights = weights, + .dz = dz, + .modes = modes, + .channels = adams_channels_}; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // A device item is a (bin, channel) pair, a host item a bin. + const auto items = n * modes; +#else + const auto items = n; +#endif + auto norms = + adams_norms("solprop_native.adams.coupled_norms", + Kokkos::RangePolicy(flow_execution_space_, 0, items), + reducer, weights.n_rows); + adams_trial_compensated_ = compensated; + adams_trial_coupled_ = residual && compensated; + return norms; +} + +void GeneralWorkspace::adams_exponential_accept(const double dz, const std::size_t free, + const AdamsWeights& dense) { + require_adams(); + if (!adams_trial_ready_) + throw std::logic_error("no Adams trial is held"); + if (free >= adams_store_.extent(0)) + throw std::out_of_range("Adams history row"); + if (dense.width > 0 && adams_slopes_.extent(0) == 0) + throw std::logic_error("Adams dense observation storage is not allocated"); + adams_trial_ready_ = false; + const auto columns = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + const auto policy = Range2(execution_space_, {0, 0}, {columns, n}); + const auto store = adams_store_; + const auto source = adams_source_; + const auto slopes = adams_slopes_; + adams_dense_count_ = dense.width; + if (dense.width > 0) { + // The step's samples and the state it starts from, before the new sample takes a + // row: the accepted corrector's interpolant for dense observation. + graph::deep_copy(execution_space_, adams_dense_start_, current_); + graph::deep_copy(execution_space_, adams_dense_exponent_, adams_exponent_); + launch::parallel_for( + "solprop_native.adams.exponential_dense", policy, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + slopes(0, column, index) = source(column, index); + for (std::size_t t = 1; t < dense.width; ++t) + slopes(t, column, index) = store(dense.slot[t - 1], column, index); + }); + if (adams_trial_compensated_) { + // The interpolant's compensation terms, untransported: the endpoint's and + // (D(z_j) - x / dz) u_j of each history sample it reads. + const auto terms = adams_dense_terms_; + const auto endpoint = adams_endpoint_; + const auto exponent = adams_exponent_; + const auto states = adams_states_; + const auto rates = adams_rates_; + const double inverse_dz = 1.0 / dz; + launch::parallel_for( + "solprop_native.adams.dense_terms", policy, + // Compensation terms only: strict as one function (solprop SEP 25 4.2). + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) + SOLPROP_NATIVE_STRICT_FUNCTION { + SOLPROP_NATIVE_STRICT_BLOCK + terms(0, column, index) = endpoint(column, index); + for (std::size_t t = 1; t < dense.width; ++t) { + const auto slot = dense.slot[t - 1]; + terms(t, column, index) = compensation_residual( + rates(slot, column, index), exponent(column, index), inverse_dz, + states(slot, column, index)); + } + }); + } + } + adams_dense_compensated_ = dense.width > 0 && adams_trial_compensated_; + // The accepted field takes `finish`'s boundary treatment: a split-step absorber's + // half steps, or the band's mask in the pass below. Untransported samples: nothing + // moves; the new sample takes row `free`, and for a varying operator its state and + // rate the rows beside it. + const bool masked = !problem_.absorber.split_step; + if (!masked) + split_step(dz, advanced_); + const bool compensated = adams_trial_compensated_; + const auto active = active_mask_; + const auto advanced = advanced_; + const auto states = adams_states_; + const auto rates = adams_rates_; + const auto lab = adams_lab_; + const auto endpoint_rate = adams_endpoint_rate_; + const std::size_t blocks = (n + adams_block - 1) / adams_block; + launch::parallel_for( + "solprop_native.adams.exponential_accept", + Kokkos::RangePolicy(adams_copy_space_, 0, columns * blocks), + KOKKOS_LAMBDA(const std::size_t task) { + const auto column = task / blocks; + const auto first = (task % blocks) * adams_block; + const auto last = first + adams_block < n ? first + adams_block : n; + for (std::size_t index = first; index < last; ++index) { + if (masked && active(index) == 0U) + advanced(column, index) = Complex{}; + store(free, column, index) = source(column, index); + if (compensated) { + states(free, column, index) = lab(column, index); + rates(free, column, index) = endpoint_rate(column, index); + } + } + }); + std::swap(current_, advanced_); + // The last evaluation was at the prediction, so no response stash is the state's. + stash_is_current_ = false; + // A frozen-residual flow's weights at the new sample, the trial's end. + if (adams_trial_coupled_) + graph::deep_copy(execution_space_, + Kokkos::subview(adams_applied_, free, Kokkos::ALL, Kokkos::ALL), + adams_end_applied_); + adams_accepted_boundary(); +} + +// A host's execution instances are partitions of its thread pool, and the next +// trial's passes may run on another one than this step's, so the accepted step ends +// with a fence there. A device's instances are its one stream, which already orders +// the next trial's work after this step's: there the host goes on to the next trial's +// tables while the device finishes the step (on an AMD Instinct MI325X a trial's +// kernels take a few microseconds each, so the wait was a visible part of a trial). +void GeneralWorkspace::adams_accepted_boundary() const { + if constexpr (build::host) + execution_space_.fence("solprop_native Adams accepted-step boundary"); +} + +// The field at the fraction `xi` of the last accepted step: the accepted corrector's +// interpolant integrated exactly against the flow, exp(xi x) u_n + dz sum_k +// xi^(k+1) M_k(xi x) sum_j c_jk F_j (`weights.corrector`), masked into current_. +void GeneralWorkspace::adams_exponential_observe(const double xi, const double dz, + const ExponentialWeights& weights) { + require_adams(); + if (adams_dense_count_ == 0) + throw std::logic_error("no Adams step is held for dense observation"); + const auto columns = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + const std::size_t count = adams_dense_count_; + const auto start = adams_dense_start_; + const auto exponent = adams_dense_exponent_; + const auto slopes = adams_slopes_; + const auto current = current_; + const auto terms = adams_dense_terms_; + const bool compensated = adams_dense_compensated_; + launch::parallel_for( + "solprop_native.adams.exponential_observe", + Range2(execution_space_, {0, 0}, {columns, n}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + const Complex x = xi * exponent(column, index); + const Complex e = exponential(x); + Complex phi[2 * adams_max_samples + 1]; + if (weights.shifted) { + // The accepted step's own sigma (solprop SEP 35 section 4.1): the moments at + // xi (x - sigma), its rotated samples, and exp(-sigma (1 - xi)) on the sum. + const Complex residual = + xi * (exponent(column, index) - Complex(0.0, weights.shift)); + adams_phi(residual, Kokkos::abs(residual), exponential(residual), count, phi); + } else { + adams_phi(x, Kokkos::abs(x), e, count, phi); + } + Complex total{}; + double power = xi; + for (std::size_t k = 0; k < count; ++k) { + Complex combined{}; + for (std::size_t t = 0; t < count; ++t) { + Complex slope = slopes(t, column, index); + if (weights.shifted && t > 0) + slope = strict_mul( + Complex(weights.factor_re[t - 1], weights.factor_im[t - 1]), slope); + combined += weights.corrector[t][k] * slope; + } + total += (power * (adams_factorial(k) * phi[k + 1])) * combined; + power *= xi; + } + if (weights.shifted) + total = exponential(Complex(0.0, -weights.shift * (1.0 - xi))) * total; + Complex value = e * start(column, index) + dz * total; + if (compensated) { + // e^{(xi - theta_j) x} carries each term; for the endpoint it is an inverse + // flow, so a bin the step damps by more than an e-fold takes the anchor's. + const Complex chord = exponent(column, index); + value = adams_observed_compensation(weights, count, xi, dz, chord, value, + terms, column, index); + if (chord.real() < -1.0) + value = e * start(column, index); + } + current(column, index) = value; + }); + mask(current_); + stash_is_current_ = false; + execution_space_.fence("solprop_native Adams observation boundary"); +} + +// --- solprop SEP 47: a Runge-Kutta march's dense output in a native session --------- +// +// DOPRI5's continuous extension (Hairer, Norsett and Wanner, II.6) in the step's +// interaction picture, as solprop's `extension_coefficients`, `extension_value` and +// `HostDriver._observe_inside` form it: with `v0` the state at the step's anchor, `k_i` +// its stages in the anchor's frame and `v1 - v0 = h sum_i b_i k_i`, the coefficients +// `v0`, `v1 - v0`, `h k1 - (v1 - v0)`, `(v1 - v0) - h k7 - (h k1 - (v1 - v0))` and +// `h sum_i d_i k_i`, evaluated at the fraction theta as `r1 + theta (r2 + (1 - theta) +// (r3 + theta (r4 + (1 - theta) r5)))`. A bin the step's flow changes by more than an +// e-fold takes the anchor's value: no polynomial in theta follows its exponentially +// growing interaction-picture derivative. The value is carried from the anchor to the +// plane by `U(position, anchor)` and masked. Every product and sum is rounded on its +// own, in solprop's order. + +namespace { + +/** The most stages a dense extension reads. */ +constexpr std::size_t rk_dense_max_stages = 16; + +/** A tableau's solution and dense-output weights, captured by a kernel. */ +struct RungeKuttaDenseWeights { + double solution[rk_dense_max_stages]{}; + double dense[rk_dense_max_stages]{}; + std::size_t stages{}; +}; + +/** The step's phase increment at one element from the uploaded weights' last row: on a + separable route the step's length times the element's operator, otherwise `sum_t + dPhi_t Phi_t` in term order (`adams_phase`'s exponent). */ +struct StepExponent { + bool separable{}; + SpatialCoefficients spatial; + ByteView active; + RealView delta; + ComplexView3 phi_terms; + std::size_t n_pol{}; + std::size_t n_stages{}; + std::size_t n_phi{}; + KOKKOS_INLINE_FUNCTION Complex operator()(const std::size_t column, + const std::size_t index) const { + if (separable) { + const auto values = spatial.value(column / n_pol, index, active(index) != 0U); + return delta(n_stages) * values.linear; + } + return adams_phase_sum(phi_terms, delta, n_stages * n_phi, n_phi, column, index); + } +}; + +} // namespace + +void GeneralWorkspace::rk_dense_allocate(const std::vector& weights) { + if (problem_.stepper.low_storage || stages_.extent(0) == 0) + throw std::logic_error("Runge-Kutta dense output needs the Runge-Kutta stages"); + if (weights.size() != problem_.stepper.n_stages || + weights.size() > rk_dense_max_stages) + throw std::invalid_argument("the dense-output weights are one per stage"); + const auto columns = problem_.n_columns(); + const auto n = problem_.dims.n_omega; + rk_dense_weights_ = weights; + if (rk_dense_coefficients_.extent(0) == 0) { + rk_dense_coefficients_ = + ComplexView3("solprop_native.rk.dense_coefficients", 5, columns, n); + rk_dense_stiff_ = Kokkos::View( + "solprop_native.rk.dense_stiff", columns, n); + rk_dense_held_ = ComplexView2("solprop_native.rk.dense_held", columns, n); + } + rk_dense_ready_ = false; +} + +void GeneralWorkspace::rk_dense_hold(const double dz) { + if (rk_dense_coefficients_.extent(0) == 0) + throw std::logic_error("Runge-Kutta dense output storage is not allocated"); + if (!stages_ready_) + throw std::logic_error("no Runge-Kutta trial is held for a dense extension"); + RungeKuttaDenseWeights weights; + weights.stages = problem_.stepper.n_stages; + for (std::size_t stage = 0; stage < weights.stages; ++stage) { + weights.solution[stage] = problem_.stepper.solution_weights[stage]; + weights.dense[stage] = rk_dense_weights_[stage]; + } + // The trial's own scalars are bound: their last row is the whole step's. + const StepExponent step{.separable = problem_.separable_geometry(), + .spatial = spatial_, + .active = active_mask_, + .delta = stage_phi_delta_, + .phi_terms = phi_terms_, + .n_pol = problem_.dims.n_pol_state, + .n_stages = problem_.stepper.n_stages, + .n_phi = problem_.linear.n_phi_terms}; + const auto current = current_; + const auto stages = stages_; + const auto coefficients = rk_dense_coefficients_; + const auto stiff = rk_dense_stiff_; + launch::parallel_for( + "solprop_native.rk.dense_hold", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) + SOLPROP_NATIVE_STRICT_FUNCTION { + SOLPROP_NATIVE_STRICT_BLOCK + Complex span{}; + Complex dense{}; + for (std::size_t stage = 0; stage < weights.stages; ++stage) { + const Complex slope = stages(stage, column, index); + span = strict_accumulate(span, weights.solution[stage], slope); + dense = strict_accumulate(dense, weights.dense[stage], slope); + } + span = strict_scale(dz, span); + const Complex first = + strict_sub(strict_scale(dz, stages(0, column, index)), span); + const Complex last = strict_sub( + strict_sub(span, + strict_scale(dz, stages(weights.stages - 1, column, index))), + first); + coefficients(0, column, index) = current(column, index); + coefficients(1, column, index) = span; + coefficients(2, column, index) = first; + coefficients(3, column, index) = last; + coefficients(4, column, index) = strict_scale(dz, dense); + const Complex x = step(column, index); + stiff(column, index) = Kokkos::abs(x.real()) > 1.0 ? 1U : 0U; + }); + rk_dense_ready_ = true; +} + +void GeneralWorkspace::rk_dense_observe(const double theta, const TrialScalars& carry) { + if (!rk_dense_ready_) + throw std::logic_error("no Runge-Kutta step is held for a dense observation"); + require_trial_scalars(carry); + graph::deep_copy(execution_space_, rk_dense_held_, current_); + upload_trial(carry); + const StepExponent flow{.separable = problem_.separable_geometry(), + .spatial = spatial_, + .active = active_mask_, + .delta = stage_phi_delta_, + .phi_terms = phi_terms_, + .n_pol = problem_.dims.n_pol_state, + .n_stages = problem_.stepper.n_stages, + .n_phi = problem_.linear.n_phi_terms}; + const double rest = 1.0 - theta; + const auto coefficients = rk_dense_coefficients_; + const auto stiff = rk_dense_stiff_; + const auto current = current_; + launch::parallel_for( + "solprop_native.rk.dense_observe", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, + const std::size_t index) SOLPROP_NATIVE_STRICT_FUNCTION { + SOLPROP_NATIVE_STRICT_BLOCK + const Complex start = coefficients(0, column, index); + Complex value = start; + if (stiff(column, index) == 0U) { + Complex inner = strict_scale(rest, coefficients(4, column, index)); + inner = + strict_scale(theta, strict_add(coefficients(3, column, index), inner)); + inner = strict_scale(rest, strict_add(coefficients(2, column, index), inner)); + inner = + strict_scale(theta, strict_add(coefficients(1, column, index), inner)); + value = strict_add(start, inner); + } + current(column, index) = strict_mul(exponential(flow(column, index)), value); + }); + mask(current_); + stash_is_current_ = false; + // A plane inside a step already taken has no quadrature score of its own, as on + // the driver's path, which reloads the observed field. + quadrature_error_ = Kokkos::Experimental::quiet_NaN_v; + quadrature_stale_ = false; + execution_space_.fence("solprop_native Runge-Kutta observation boundary"); +} + +void GeneralWorkspace::rk_dense_restore() { + if (rk_dense_held_.extent(0) == 0) + throw std::logic_error("Runge-Kutta dense output storage is not allocated"); + graph::deep_copy(execution_space_, current_, rk_dense_held_); + // As the driver's march reloads the committed state (`_adams_restore`): the next + // step evaluates its first stage afresh rather than reuse the crossing step's last, + // so that the session's run is the driver's whatever FFT plans round each. + clear_fsal(); + execution_space_.fence("solprop_native Runge-Kutta restore boundary"); +} + +void GeneralWorkspace::rk_dense_release() noexcept { + rk_dense_coefficients_ = {}; + rk_dense_stiff_ = {}; + rk_dense_held_ = {}; + // A moved-in empty vector releases the storage and cannot throw (`= {}` may). + rk_dense_weights_ = std::vector(); + rk_dense_ready_ = false; +} + +std::vector GeneralWorkspace::collect_trial() { + // A coupled flow's last stage is carried back by the step's own inverse action, so + // it is not the next step's first stage; solprop's reference takes no FSAL there. + pending_fsal_ = + problem_.stepper.first_same_as_last && !problem_.absorber.split_step && !flow_; + graph::deep_copy(execution_space_, pinned_partials_, partials_); + execution_space_.fence("solprop_native trial boundary"); + const auto& host = pinned_partials_; + // Every column's slots, in order: the host's reduction is the caller's sum of a row. + const auto slots = host.extent(1); + std::vector result(3 * slots); + for (std::size_t row = 0; row < 3; ++row) + for (std::size_t slot = 0; slot < slots; ++slot) + result[row * slots + slot] = host(row, slot); + return result; +} + +void GeneralWorkspace::reuse_fsal() { + const auto stages = stages_; + const auto recycled = recycled_; + const auto solution = solution_; + const auto error = error_; + const auto whole = whole_propagator_; + const double solution_weight = problem_.stepper.solution_weights[0]; + const double error_weight = problem_.stepper.error_weights[0]; + const auto n = problem_.dims.n_omega; + launch::parallel_for( + "solprop_native.stage.reuse_fsal", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + const Complex value = recycled(column, index); + stages(0, column, index) = value; + solution(column, index) = solution_weight * value; + error(column, index) = error_weight * value; + }); +} + +TransverseChunk +GeneralWorkspace::transverse_chunk(const std::size_t first_unit) const noexcept { + const auto units = + std::min(transverse_units_per_chunk(), transverse_units() - first_unit); + if (problem_.slab_geometry()) + return {.first_row = first_unit * problem_.spatial.n_p, + .rows = units * problem_.spatial.n_p, + .first_node = first_unit * problem_.spatial.n_x, + .nodes = units * problem_.spatial.n_x}; + return { + .first_row = first_unit, .rows = units, .first_node = first_unit, .nodes = units}; +} + +void GeneralWorkspace::transverse_forward() { + if (slab_forward_) + graph::execute(*slab_forward_, transverse_, transverse_, + KokkosFFT::Normalization::none); + else + graph::execute(*transverse_forward_, transverse_, transverse_, + KokkosFFT::Normalization::none); +} + +void GeneralWorkspace::transverse_inverse() { transverse_inverse(transverse_); } + +// The in-place inverse transverse plan on another volume of the same extents: the +// plan's own scratch or the magnetic closure's (solprop SEP 34). +void GeneralWorkspace::transverse_inverse(const ComplexView3& scratch) { + if (slab_inverse_) + graph::execute(*slab_inverse_, scratch, scratch, KokkosFFT::Normalization::none); + else + graph::execute(*transverse_inverse_, scratch, scratch, + KokkosFFT::Normalization::none); +} + +void GeneralWorkspace::separable_stage_input(const std::size_t stage, const double dz) { + const auto current = current_; + const auto stages = stages_; + const auto tableau = tableau_; + const auto spatial = spatial_; + const auto delta = stage_phi_delta_; + const auto at = stage_phi_at_; + const auto active = active_mask_; + const auto noise = noise_; + const auto scratch = transverse_; + const auto whole = whole_propagator_; + const auto index = stage_index(); + const auto n_stages = problem_.stepper.n_stages; + const bool add_noise = problem_.stepper.modified_noise && has_noise_; + const double inverse_area_count = inverse_transverse_count(); + // The stage input at one element, given its spatial value and its propagator. + const auto input = KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, + const Complex linear, const Complex propagator) { + Complex accumulated{}; + for (std::size_t previous = 0; previous < stage; ++previous) + accumulated += + tableau(stage * n_stages + previous) * stages(previous, column, omega); + Complex value = propagator * (current(column, omega) + dz * accumulated); + if (add_noise) { + const double angle = at(stage) * linear.imag(); + value += noise(column, omega) * Complex(Kokkos::cos(angle), Kokkos::sin(angle)); + } + scratch.data()[index.cell(column, omega)] = value * inverse_area_count; + }; + // One element's propagators: devices, and a host whose vector maths is the system + // libm, where the per-element form is the faster (solprop ADR-059). + const auto element = + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + const auto values = spatial.value(index.mode(column), omega, active(omega) != 0U); + input(column, omega, values.linear, propagator_exp(delta(stage) * values.linear)); + if (stage + 1 == n_stages) + whole(column, omega) = propagator_exp(delta(n_stages) * values.linear); + }; +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + if constexpr (vmath::vectorised) { + // Each tile's propagators in one pass of the vector maths, then the elements in + // `crossing_for`'s order: the propagators' cosines and sines took a fifth of a + // one-thread `line` run one element at a time (solprop issue #338). + const bool last = stage + 1 == n_stages; + crossing_tiles( + "solprop_native.free.stage_input", execution_space_, problem_.n_columns(), + problem_.dims.n_omega, [=](const CrossingTile& tile) { + PropagatorTile step; + PropagatorTile finish; + Complex linear[PropagatorTile::capacity]; + tile.visit([&](const std::size_t column, const std::size_t omega) { + const auto k = tile.at(column, omega); + linear[k] = + spatial.value(index.mode(column), omega, active(omega) != 0U).linear; + const Complex argument = delta(stage) * linear[k]; + step.real[k] = argument.real(); + step.imag[k] = argument.imag(); + if (last) { + const Complex whole_argument = delta(n_stages) * linear[k]; + finish.real[k] = whole_argument.real(); + finish.imag[k] = whole_argument.imag(); + } + }); + step.form(tile.size()); + if (last) + finish.form(tile.size()); + tile.visit([&](const std::size_t column, const std::size_t omega) { + const auto k = tile.at(column, omega); + input(column, omega, linear[k], step[k]); + if (last) + whole(column, omega) = finish[k]; + }); + }); + return; + } +#endif + crossing_for("solprop_native.free.stage_input", execution_space_, + problem_.n_columns(), problem_.dims.n_omega, element); +} + +void GeneralWorkspace::separable_load_current() { + const auto current = current_; + const auto scratch = transverse_; + const auto index = register_index(); + const double scale = inverse_transverse_count(); + crossing_for( + "solprop_native.free.current_transverse_input", execution_space_, + problem_.n_columns(), problem_.dims.n_omega, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + scratch.data()[index.cell(column, omega)] = scale * current(column, omega); + }); + transverse_inverse(); +} + +void GeneralWorkspace::separable_pack_chunk(const TransverseChunk& chunk, + const bool material) { + const auto fine = fine_spectrum_; + // Every crop bin of the chunk's rows is written below; only the bins outside the + // crop need zeros, each chunk, since the inverse transform may destroy its input. + // Rows past a short last chunk keep earlier finite values nothing reads back. + if (!separable_gaps_.empty()) { + const auto gaps = separable_gaps_; + cartesian_for( + "solprop_native.free.zero_fine_gaps", execution_space_, + chunk.rows * problem_.dims.n_pol_state, gaps.extent(0), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t gap) { + fine(row, gaps(gap)) = Complex{}; + }); + } + const auto scratch = transverse_; + const auto crop = crop_index_; + const auto n_pol = problem_.dims.n_pol_state; + [[maybe_unused]] const auto layout = transverse_layout(); + const auto first = chunk.first_row; + const auto n = problem_.dims.n_omega; + const auto n_to = problem_.dims.n_to; + const bool analytic = problem_.dims.family == FieldFamily::field_analytic; + // Free space's material field carries the pad scale here; the hybrid planar kind's + // slab synthesis carries it (solprop's `Transform.synthesis`). + const double scale = 1.0 / static_cast(material && !problem_.slab_geometry() + ? problem_.dims.n_t + : problem_.dims.n_to); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + tiled_crossing( + "solprop_native.free.pack_temporal_chunk", execution_space_, chunk.rows * n_pol, + n, true, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + return layout.cell(scratch, omega * n_pol + row % n_pol, first + row / n_pol); + }, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega, + const Complex value) { + const double factor = + analytic && omega > 0 && 2 * crop(omega) < n_to ? 2.0 : 1.0; + fine(row, crop(omega)) = factor * scale * value; + }); +#else + // The chunk's row `row` is the state's column `first_column + row`. + const auto index = register_index(); + const auto first_column = first * n_pol; + crossing_for( + "solprop_native.free.pack_temporal_chunk", execution_space_, chunk.rows * n_pol, + n, KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + // The analytic signal doubles the bins strictly between zero and the fine + // grid's Nyquist bin, the same rule as the modal pad and the reference. + const double factor = + analytic && omega > 0 && 2 * crop(omega) < n_to ? 2.0 : 1.0; + fine(row, crop(omega)) = + factor * scale * scratch.data()[index.cell(first_column + row, omega)]; + }); +#endif +} + +// The chunk's physical nodes from its temporal rows: free space's are the rows +// themselves; the hybrid planar kind's are the slab synthesis of each y row. +void GeneralWorkspace::separable_nodes(const TransverseChunk& chunk) { + const bool real_field = problem_.dims.family == FieldFamily::field_real; + const auto n_to = problem_.dims.n_to; + const auto n_pol = problem_.dims.n_pol_state; + if (problem_.slab_geometry()) { + const auto units = chunk.nodes / problem_.spatial.n_x; + if (real_field) + slab_contract("solprop_native.slab.synthesis", execution_space_, synthesis_, + modal_real_, node_real_, units, n_to); + else + slab_contract("solprop_native.slab.synthesis", execution_space_, synthesis_, + modal_complex_, node_complex_, units, n_to); + return; + } + if (separable_aliased_) + return; + const auto time = modal_real_; + const auto nodes = node_real_; + const auto complex_time = modal_complex_; + const auto complex_nodes = node_complex_; + cartesian_for( + "solprop_native.free.physical_columns", execution_space_, chunk.rows * n_pol, + n_to, KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { + if (real_field) + nodes(row, index) = time(row, index); + else + complex_nodes(row, index) = complex_time(row, index); + }); +} + +// The hybrid planar kind's slab projection of a chunk's node source onto its y rows' +// columns: the polarisation, or one envelope-plasma current stream. +void GeneralWorkspace::slab_project(const TransverseChunk& chunk, + const EnvelopeStream* stream) { + const auto units = chunk.nodes / problem_.spatial.n_x; + const auto n_to = problem_.dims.n_to; + if (stream != nullptr) + slab_contract("solprop_native.slab.projection", execution_space_, projection_, + stream->node, stream->projected, units, n_to); + else if (problem_.dims.family == FieldFamily::envelope) + slab_contract("solprop_native.slab.projection", execution_space_, projection_, + polarisation_complex_, projected_complex_, units, n_to); + else + slab_contract("solprop_native.slab.projection", execution_space_, projection_, + polarisation_real_, projected_real_, units, n_to); +} + +void GeneralWorkspace::separable_finish_transform(const std::size_t stage, + const bool material) { + const auto scratch = transverse_; + const auto spatial = spatial_; + const auto stages = stages_; + const auto window = omega_window_; + const auto active = active_mask_; + const auto delta = stage_phi_delta_; + const auto index = stage_index(); + // The finish at one element, given its prefactor and its inverse propagator. + const auto finish = KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, + const Complex prefactor, const Complex inverse) { + Complex value = scratch.data()[index.cell(column, omega)]; + value *= + material ? prefactor * window(omega) : Complex(active(omega) != 0U ? 1.0 : 0.0); + value *= inverse; + if (material) + stages(stage, column, omega) = value; + else + stages(stage, column, omega) += value; + }; + // One element, on devices and without host vector maths (as the stage input). + const auto element = + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + const auto values = spatial.value(index.mode(column), omega, active(omega) != 0U); + finish(column, omega, values.prefactor, + propagator_exp(-delta(stage) * values.linear)); + }; +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + if constexpr (vmath::vectorised) { + // As the stage input: each tile's inverse propagators in one pass (issue #338). + crossing_tiles( + "solprop_native.free.finish_transform", execution_space_, problem_.n_columns(), + problem_.dims.n_omega, [=](const CrossingTile& tile) { + PropagatorTile inverse; + Complex prefactor[PropagatorTile::capacity]; + tile.visit([&](const std::size_t column, const std::size_t omega) { + const auto k = tile.at(column, omega); + const auto values = + spatial.value(index.mode(column), omega, active(omega) != 0U); + prefactor[k] = values.prefactor; + const Complex argument = -delta(stage) * values.linear; + inverse.real[k] = argument.real(); + inverse.imag[k] = argument.imag(); + }); + inverse.form(tile.size()); + tile.visit([&](const std::size_t column, const std::size_t omega) { + const auto k = tile.at(column, omega); + finish(column, omega, prefactor[k], inverse[k]); + }); + }); + return; + } +#endif + crossing_for("solprop_native.free.finish_transform", + execution_space_, problem_.n_columns(), + problem_.dims.n_omega, element); +} + +// Free space (solprop `NumpyRefBackend._nonlinear_free`) and the hybrid planar kind +// (`_nonlinear_slab`): the stage input to the transverse spectrum, the inverse +// transverse transform, then chunk by chunk the temporal rows to time, the material +// response at the physical nodes (for the planar kind, synthesised from and projected +// back onto the rows in time) or the in-stage absorber on the rows, and back; the +// forward transverse transform and each column's prefactor and propagator. +void GeneralWorkspace::evaluate_separable_stage(const std::size_t stage, + const double dz, + const bool accumulate) { + auto* const dispersive = separable_dispersive_.get(); + const bool nonlinear = problem_.kerr.nonlinear || problem_.plasma.enabled || + problem_.raman.enabled || dispersive != nullptr; + const auto stages = stages_; + const auto solution = solution_; + const auto error = error_; + const auto n = problem_.dims.n_omega; + // A material finish assigns the stage row; only an absorber-only or a linear stage + // accumulates into (or leaves) a cleared row. + if (!nonlinear) + cartesian_for( + "solprop_native.free.clear_stage", execution_space_, problem_.n_columns(), n, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + stages(stage, column, omega) = Complex{}; + }); + separable_stage_input(stage, dz); + for (const bool material : {true, false}) { + if ((material && !nonlinear) || (!material && !problem_.absorber.in_stage)) + continue; + if (!material) + separable_stage_input(stage, dz); + separable_core(stage, material); + separable_finish_transform(stage, material); + } + if (!accumulate) + return; + const double weight = problem_.stepper.solution_weights[stage]; + const double error_weight = problem_.stepper.error_weights[stage]; + cartesian_for( + "solprop_native.free.accumulate_stage", execution_space_, problem_.n_columns(), n, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + const Complex value = stages(stage, column, omega); + if (stage == 0) { + solution(column, omega) = weight * value; + error(column, omega) = error_weight * value; + } else { + solution(column, omega) += weight * value; + error(column, omega) += error_weight * value; + } + }); +} + +void GeneralWorkspace::separable_to_time(const TransverseChunk& chunk) { + const bool tail = temporal_tail_rows_ != 0 && + chunk.rows * problem_.dims.n_pol_state == temporal_tail_rows_; + if (problem_.dims.family == FieldFamily::field_real) { + if (tail) + temporal_tail_inverse_real_.execute(); + else + graph::execute(*inverse_real_plan_, fine_spectrum_, modal_real_, + KokkosFFT::Normalization::none); + } else { + if (tail) + temporal_tail_inverse_complex_.execute(); + else + graph::execute(*inverse_complex_plan_, fine_spectrum_, modal_complex_, + KokkosFFT::Normalization::none); + } +} + +void GeneralWorkspace::separable_to_frequency(const TransverseChunk& chunk) { + const bool tail = temporal_tail_rows_ != 0 && + chunk.rows * problem_.dims.n_pol_state == temporal_tail_rows_; + if (problem_.dims.family == FieldFamily::envelope) { + if (tail) + temporal_tail_forward_complex_.execute(); + else + graph::execute(*forward_complex_plan_, projected_complex_, projected_spectrum_, + KokkosFFT::Normalization::none); + } else { + if (tail) + temporal_tail_forward_real_.execute(); + else + graph::execute(*forward_real_plan_, projected_real_, projected_spectrum_, + KokkosFFT::Normalization::none); + } +} + +// `volume_kerr_`'s stage core: the scratch's (omega, y, x) volume to time and the +// plane at once, the Kerr polarisation in place sample by sample -- the device +// column's arithmetic (`device_response`), the pad scale 1/n_t that the chunk's pack +// would apply taken with the field -- and back. The rank-3 transform is the chunks' +// temporal transforms and the plane's together, the same DFT to rounding, without the +// pack's and the crop's transposes. +void GeneralWorkspace::separable_volume_core(const std::size_t stage) { + Kokkos::Profiling::pushRegion("solprop_native.free.volume_core"); + const auto scratch = volume_order_ ? volume_scratch_ : transverse_; + graph::execute(*volume_inverse_, scratch, scratch, KokkosFFT::Normalization::none); + const auto window = time_window_; + const auto kerr_density = stage_kerr_density_; + const double kerr_coefficient = problem_.kerr.coefficient; + const double self_factor = problem_.kerr.self_factor; + const double conjugate_factor = problem_.kerr.conjugate_factor; + const double scale = 1.0 / static_cast(problem_.dims.n_t); + const auto plane = transverse_.extent(1) * transverse_.extent(2); + const bool time_fastest = volume_order_; + const auto n_t = problem_.dims.n_t; + launch::parallel_for( + "solprop_native.free.volume_kerr", + Kokkos::RangePolicy(execution_space_, 0, scratch.size()), + KOKKOS_LAMBDA(const std::size_t item) { + const std::size_t time = time_fastest ? item % n_t : item / plane; + Complex& cell = scratch.data()[item]; + const Complex value = scale * cell; + const double modulus = squared_abs(value); + const Complex square = value * value; + const double factor = kerr_coefficient * kerr_density(stage); + const double weighted_real = conjugate_factor * square.real(); + const double weighted_imag = conjugate_factor * square.imag(); + const double weighted_self = self_factor * modulus; + const Complex result( + factor * (weighted_self * value.real() + + (weighted_real * value.real() + weighted_imag * value.imag())), + factor * (weighted_self * value.imag() + + (weighted_imag * value.real() - weighted_real * value.imag()))); + cell = result * window(time); + }); + graph::execute(*volume_forward_, scratch, scratch, KokkosFFT::Normalization::none); + Kokkos::Profiling::popRegion(); +} + +// The in-stage collar's term alone (solprop SEP 40 section 4.7): with no temporal +// rate beside it, the rate depends on the column alone and commutes with the temporal +// transforms, so the term is the rate times the scratch's spectrum, in the transverse +// physical space, and the pass takes only the transverse pair. The temporal pair +// would return the crop's bins to rounding; the bins it would drop lie outside the +// active band, which the finish masks. +void GeneralWorkspace::separable_spatial_core() { + Kokkos::Profiling::pushRegion("solprop_native.free.spatial_core"); + transverse_inverse(); + const auto index = register_index(); + const auto spatial = spatial_; + const auto scratch = transverse_; + const auto n_pol = problem_.dims.n_pol_state; + cartesian_for( + "solprop_native.free.absorber_spectrum", execution_space_, problem_.n_columns(), + problem_.dims.n_omega, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + scratch.data()[index.cell(column, omega)] *= + -0.5 * spatial.absorber(column / n_pol); + }); + transverse_forward(); + Kokkos::Profiling::popRegion(); +} + +// The separable stage's core (solprop SEP 24 Q5): from its input in the transverse +// scratch, the material response (or, `material` false, the in-stage absorber's term) +// back into the scratch, in frequency, chunk by chunk through the temporal transforms. +// Shared by the Runge-Kutta stage and the low-storage march's passes, which differ in +// how they form the input and fold the output. +void GeneralWorkspace::separable_core(const std::size_t stage, const bool material) { + if (material && volume_kerr_) { + separable_volume_core(stage); + return; + } + if (!material && spatial_only_absorber_) { + separable_spatial_core(); + return; + } + // SEP 6's dispersive response runs chunk by chunk beside the material's response + // column, or alone in its place (`SeparableDispersive`). + auto* const dispersive = separable_dispersive_.get(); + const bool alone = dispersive != nullptr && dispersive->alone(); + const auto n_pol = problem_.dims.n_pol_state; + // The host's scratch positions without a division (`RegisterIndex`). + [[maybe_unused]] const auto index = register_index(); + [[maybe_unused]] const auto layout = transverse_layout(); + const auto n = problem_.dims.n_omega; + const auto n_to = problem_.dims.n_to; + const auto real_time = modal_real_; + const auto complex_time = modal_complex_; + const auto polarisation_real = polarisation_real_; + const auto polarisation_complex = polarisation_complex_; + const auto projected_real = projected_real_; + const auto projected_complex = projected_complex_; + const auto temporal_rate = absorber_fine_; + const auto spatial = spatial_; + const auto spectrum = projected_spectrum_; + const auto crop = crop_index_; + const auto scratch = transverse_; + const bool real_field = problem_.dims.family == FieldFamily::field_real; + const bool envelope = problem_.dims.family == FieldFamily::envelope; + const bool slab = problem_.slab_geometry(); + // Profiling regions name the core's stages for a Kokkos Tools timer (SEP 28 phase + // 0); with no tool loaded each is one test of a flag. The kernels inside carry + // their own labels. + Kokkos::Profiling::pushRegion("solprop_native.free.core"); + if (material && dispersive != nullptr) + dispersive->begin(stage); + Kokkos::Profiling::pushRegion("solprop_native.free.transverse_inverse"); + transverse_inverse(); + Kokkos::Profiling::popRegion(); + for (std::size_t unit = 0; unit < transverse_units(); + unit += transverse_units_per_chunk()) { + const auto chunk = transverse_chunk(unit); + const auto first = chunk.first_row; + Kokkos::Profiling::pushRegion("solprop_native.free.pack"); + separable_pack_chunk(chunk, material); + Kokkos::Profiling::popRegion(); + // The response filters the packed spectrum first: the inverse transform destroys + // it. In place, that transform gives the filtered field. + if (material && dispersive != nullptr) + dispersive->prepare(chunk.nodes); + Kokkos::Profiling::pushRegion("solprop_native.free.to_time"); + separable_to_time(chunk); + Kokkos::Profiling::popRegion(); + // The projected rows are overwritten below by the polarisation (or are the + // polarisation, aliased) or by the absorber's term; only a planar projection or a + // response that projects nothing of the stage's needs them zeroed first. + const bool overwrites = + !slab && + (!material || !alone || (dispersive != nullptr && dispersive->borrowed())); + Kokkos::Profiling::pushRegion("solprop_native.free.clear_projected"); + if (!overwrites) { + if (envelope) + graph::deep_copy(execution_space_, projected_complex_, Complex{}); + else + graph::deep_copy(execution_space_, projected_real_, 0.0); + } + Kokkos::Profiling::popRegion(); + if (material) { + separable_nodes(chunk); + Kokkos::Profiling::pushRegion("solprop_native.free.response"); + if (!alone) + response(stage, chunk.first_node, chunk.nodes); + if (problem_.plasma.envelope) + envelope_response(stage, chunk.first_node, chunk.nodes, &chunk); + if (problem_.raman.enabled) + convolve_raman(stage, chunk.nodes); + if (dispersive != nullptr) + dispersive->respond(chunk.nodes); + if (held_plasma()) + held_response(stage, chunk.first_node, chunk.nodes, &chunk); + if (gated_plasma()) + gated_response(chunk.nodes); + Kokkos::Profiling::popRegion(); + // Alone with products of its own, nothing of the stage's is projected. + const bool stage_projects = !alone || dispersive->borrowed(); + if (stage_projects && slab) + slab_project(chunk, nullptr); + else if (stage_projects && !separable_aliased_) + cartesian_for( + "solprop_native.free.polarisation_columns", execution_space_, + chunk.rows * n_pol, n_to, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { + if (envelope) + projected_complex(row, time) = polarisation_complex(row, time); + else + projected_real(row, time) = polarisation_real(row, time); + }); + } else { + // The collar's rate depends on y alone on the planar kind, so it acts on the + // rows, whose time signals are the slab amplitudes' (solprop's reference). + cartesian_for( + "solprop_native.free.absorber_columns", execution_space_, chunk.rows * n_pol, + n_to, KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { + const double rate = + -0.5 * (spatial.absorber(first + row / n_pol) + temporal_rate(time)); + if (envelope) + projected_complex(row, time) = rate * complex_time(row, time); + else + projected_real(row, time) = + rate * + (real_field ? real_time(row, time) : complex_time(row, time).real()); + }); + } + const bool responds = material && dispersive != nullptr; + if (responds) + dispersive->finish(); + Kokkos::Profiling::pushRegion("solprop_native.free.to_frequency"); + if (!responds || !dispersive->replaces_transform()) { + separable_to_frequency(chunk); + } + Kokkos::Profiling::popRegion(); + // The response's spectrum joins the chunk's, before the envelope currents'. + if (responds) + dispersive->combine(); + if (material && problem_.plasma.envelope) + envelope_transform(); + if (material && held_plasma()) + held_transform(); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + tiled_crossing( + "solprop_native.free.crop_temporal_chunk", execution_space_, chunk.rows * n_pol, + n, false, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + return spectrum(row, crop(omega)); + }, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega, + const Complex value) { + layout.cell(scratch, omega * n_pol + row % n_pol, first + row / n_pol) = + value; + }); +#else + const auto first_column = first * n_pol; + crossing_for( + "solprop_native.free.crop_temporal_chunk", execution_space_, chunk.rows * n_pol, + n, KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + stream_store(scratch.data()[index.cell(first_column + row, omega)], + spectrum(row, crop(omega))); + }); +#endif + } + Kokkos::Profiling::pushRegion("solprop_native.free.transverse_forward"); + transverse_forward(); + Kokkos::Profiling::popRegion(); + Kokkos::Profiling::popRegion(); +} + +// The fine bins outside the crop as runs `(first bin, length)` in bin order. +std::vector> +GeneralWorkspace::fine_gap_runs() const { + std::vector filled(fine_spectrum_size(), false); + for (const auto index : problem_.grid.crop_index) + filled[index] = true; + std::vector> runs; + for (std::size_t index = 0; index < filled.size(); ++index) { + if (filled[index]) + continue; + if (!runs.empty() && runs.back().first + runs.back().second == index) + ++runs.back().second; + else + runs.emplace_back(index, 1); + } + return runs; +} + +StageTerms GeneralWorkspace::stage_terms(const std::size_t stage) const { + StageTerms terms; + terms.current = current_; + terms.fine = fine_spectrum_; + terms.whole = whole_propagator_; + terms.solution = solution_; + terms.error = error_; + terms.stage_phase = stage_phase_; + terms.stage_inverse = stage_inverse_; + terms.noise = noise_; + terms.flow = static_cast(flow_); + terms.carried = terms.flow && !flow_->identity(stage); + terms.keep_lab = terms.flow && flow_->residual(); + if (terms.flow) { + terms.flow_input = flow_->input(); + terms.flow_drive = flow_->drive(); + } + if (terms.keep_lab) + terms.flow_lab = flow_->lab(); + terms.general = static_cast(general_); + if (terms.general) { + terms.general_scaled = general_->scaled(); + terms.general_source = general_->source(); + } + terms.projected = projected_spectrum_; + terms.absorber = absorber_spectrum_; + terms.stages = stages_; + terms.phi_terms = phi_terms_; + terms.prefactor_terms = prefactor_terms_; + terms.tableau = tableau_; + terms.phi_delta = stage_phi_delta_; + terms.phi_at = stage_phi_at_; + terms.prefactor_weights = stage_prefactor_; + terms.window = omega_window_; + terms.crop = crop_index_; + terms.gaps = fine_gaps_; + terms.gap_ranges = fine_gaps_.extent(0) / 3; + terms.active = active_mask_; + terms.n = problem_.dims.n_omega; + terms.n_phi = problem_.linear.n_phi_terms; + terms.n_stages = problem_.stepper.n_stages; + terms.n_prefactor = problem_.linear.n_prefactor_terms; + terms.n_to = problem_.dims.n_to; + terms.inverse_n_to = 1.0 / static_cast(problem_.dims.n_to); + terms.general_pad = problem_.general.pad_scale; + terms.add_noise = problem_.stepper.modified_noise && has_noise_; + terms.analytic = problem_.dims.family == FieldFamily::field_analytic; + const bool general_active = + terms.general && (general_->responds() || problem_.absorber.in_stage); + terms.zero_response = + terms.general ? !general_active + : !problem_.kerr.nonlinear && !problem_.plasma.enabled && + !problem_.raman.enabled && !absorbing() && !dispersive_; + terms.in_stage_absorber = absorbing(); + terms.identity_crop = identity_crop_; + return terms; +} + +// The stage's entry kernel: the deferred finish of the stage before it (when a trial +// left one pending), the stage's phase increment and the whole step's at the last +// stage, the combination, and the zero pad of the fine spectrum, in one traversal of +// the bins. The finish and the combination are each the arithmetic of their own +// kernels, so fusing them moves no rounding. A lossless problem's kernel makes no +// `exp` calls, and a lossless stage repeating the stage before's phase (the last two +// stages of a FSAL tableau, and the whole step's at the last) reuses its pair +// (`StageTerms::combine`). +void GeneralWorkspace::stage_entry(const std::size_t stage, const double dz) { + const bool repeated = lossless_phase_ && stage > 0 && entered_stage_ == stage - 1 && + phase_repeats_[stage]; + entered_stage_ = stage; + if (repeated) + stage_entry_with(stage, dz); + else if (lossless_phase_) + stage_entry_with(stage, dz); + else + stage_entry_with(stage, dz); +} + +template +void GeneralWorkspace::stage_entry_with(const std::size_t stage, const double dz) { + constexpr bool lossless = Mode != EntryPhase::general; + const bool finish_previous = finish_pending_; + finish_pending_ = false; + if (finish_previous && (stage == 0 || pending_stage_ + 1 != stage)) + throw std::logic_error("a deferred stage finish lost its order"); + const std::size_t previous = finish_previous ? stage - 1 : 0; + const double previous_solution = problem_.stepper.solution_weights[previous]; + const double previous_error = problem_.stepper.error_weights[previous]; + const double solution_weight = problem_.stepper.solution_weights[stage]; + const double error_weight = problem_.stepper.error_weights[stage]; + const auto terms = stage_terms(stage); + const auto n = problem_.dims.n_omega; + const auto n_phi = problem_.linear.n_phi_terms; + const auto n_stages = problem_.stepper.n_stages; + const auto gaps = problem_.separable_geometry() ? 0 : fine_spectrum_size() - n; + const bool last = stage + 1 == n_stages; + // The whole step's propagator is the last stage's when their phases are equal. + const bool reuse_whole = last && lossless && whole_repeats_; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // On a device, one thread per bin: the host layout's blocks of 64 bins, each a work + // item holding two 64-element arrays, gave a GPU a few dozen threads walking every + // term, which made a varying capillary no faster on a GPU than on a CPU. + launch::parallel_for( + "solprop_native.stage.entry", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + if (finish_previous) + terms.finish(previous, previous_solution, previous_error, column, index); + const Complex phase = terms.phase_sum(stage * n_phi, column, index); + const Complex propagator = terms.template combine( + stage, dz, solution_weight, error_weight, column, index, phase, + terms.template entry_factors(stage, column, index, phase)); + if (reuse_whole) { + terms.whole(column, index) = propagator; + } else if (last) { + // At a last stage whose node is the step's end (DOPRI5's) the phase sums are + // equal on a lossy problem too, and the stage's propagator is the step's. + const Complex whole = terms.phase_sum(n_stages * n_phi, column, index); + terms.whole(column, index) = + whole == phase ? propagator : entry_exponential(whole); + } + for (std::size_t gap = index; gap < gaps; gap += n) + terms.zero_gaps(column, gap, gap + 1); + }); +#else + const std::size_t blocks = (n + stage_block - 1) / stage_block; + const bool block_forms = terms.blocks_vectorise(); + launch::parallel_for( + "solprop_native.stage.entry", + Kokkos::RangePolicy(execution_space_, 0, + problem_.n_columns() * blocks), + KOKKOS_LAMBDA(const std::size_t item) { + const std::size_t column = item / blocks; + const std::size_t block = item % blocks; + const std::size_t first = block * stage_block; + const std::size_t count = n - first < stage_block ? n - first : stage_block; + if (finish_previous && block_forms) + terms.finish_block(previous, previous_solution, previous_error, column, first, + count); + else if (finish_previous) + for (std::size_t bin = 0; bin < count; ++bin) + terms.finish(previous, previous_solution, previous_error, column, + first + bin); + double real[stage_block]; + double imag[stage_block]; + terms.phase_block(stage * n_phi, column, first, count, real, imag); + // With vector maths the block's transcendentals are passes of the layer; with + // the system libm each bin keeps its own calls, which measured faster there + // (the passes' buffers cost 1 to 3 % on an AMD EPYC 7763 with glibc's). + if constexpr (vmath::vectorised) { + EntryBlock factors{}; + terms.template entry_factors_block(stage, column, first, count, real, + imag, factors); + if (block_forms) + terms.template combine_block(stage, dz, column, first, count, factors, + reuse_whole); + else + for (std::size_t bin = 0; bin < count; ++bin) { + const Complex propagator = terms.template combine( + stage, dz, solution_weight, error_weight, column, first + bin, + Complex(real[bin], imag[bin]), factors.at(bin)); + if (reuse_whole) + terms.whole(column, first + bin) = propagator; + } + if (last && !reuse_whole) { + // The whole step's propagator, `entry_exponential` over the block. + terms.phase_block(n_stages * n_phi, column, first, count, real, imag); + // `e^{+-0} (cos b, sin b)` is `(cos b, sin b)` to the bit. + if (lossless || StageTerms::lossless_block(real, count)) + vmath::sincos(imag, factors.sine, factors.cosine, count); + else + vmath::exp_complex(real, imag, factors.cosine, factors.sine, count); + for (std::size_t bin = 0; bin < count; ++bin) + terms.whole(column, first + bin) = + Complex(factors.cosine[bin], factors.sine[bin]); + } + } else { + for (std::size_t bin = 0; bin < count; ++bin) { + const Complex phase(real[bin], imag[bin]); + const Complex propagator = terms.template combine( + stage, dz, solution_weight, error_weight, column, first + bin, phase, + terms.template entry_factors(stage, column, first + bin, phase)); + if (reuse_whole) + terms.whole(column, first + bin) = propagator; + } + if (last && !reuse_whole) { + terms.phase_block(n_stages * n_phi, column, first, count, real, imag); + for (std::size_t bin = 0; bin < count; ++bin) + terms.whole(column, first + bin) = + entry_exponential(Complex(real[bin], imag[bin])); + } + } + terms.zero_gaps(column, block * gaps / blocks, (block + 1) * gaps / blocks); + }); +#endif +} + +// A stage's finish, or (inside a trial, when the next stage's entry follows with +// nothing between) its deferral to that entry kernel. +void GeneralWorkspace::finish_or_defer(const std::size_t stage) { + if (defer_finish_) { + finish_pending_ = true; + pending_stage_ = stage; + return; + } + finish_stage(stage); +} + +void GeneralWorkspace::evaluate_stage(const std::size_t stage, const double dz) { + ++rk_source_evaluations_; + if (!flow_) { + evaluate_stage_source(stage, dz); + return; + } + // solprop `NumpyRefBackend._trial_step` on a coupled flow: the stage input is + // carried forward by the stage's action, the source (with a frozen-residual flow's + // explicit residual) is evaluated into the flow's drive, and carried back. + if (!flow_->identity(stage)) + flow_->carry_input(stage, dz, current_, stages_, tableau_); + evaluate_stage_source(stage, dz); + flow_->add_residual(stage, stage_phase_); + flow_->carry_back(stage, problem_.stepper.solution_weights[stage], + problem_.stepper.error_weights[stage], stages_, solution_, error_); +} + +void GeneralWorkspace::evaluate_stage_source(const std::size_t stage, const double dz, + const bool accumulate) { + if (!defer_quadrature_) + coarse_pending_ = false; + if (problem_.separable_geometry()) { + evaluate_separable_stage(stage, dz, accumulate); + return; + } + stage_entry(stage, dz); + if (modal_core(stage)) + finish_or_defer(stage); +} + +// The modal routes' stage core (solprop SEP 24 Q5): from the input in the fine spectrum +// (and the general route's scaled channels) to the source in the projected spectra, +// the in-stage absorber's beside it. Shared by the Runge-Kutta stage and the +// low-storage march's passes. Returns whether a source was evaluated: a stage without +// a response has none to finish. +bool GeneralWorkspace::modal_core(const std::size_t stage) { + if (general_) { + const bool general_active = general_->responds() || problem_.absorber.in_stage; + if (general_active) + general_core(stage); + return general_active; + } + if (!problem_.kerr.nonlinear && !problem_.plasma.enabled && !problem_.raman.enabled && + !absorbing() && !dispersive_) + return false; + // A device's forward rows go in one transform after the absorber's are formed. + const bool packed = !forward_pack_.empty(); + // SEP 6's dispersive response filters the padded spectrum first: the inverse + // transform destroys it. In place, that transform gives the filtered field. + if (dispersive_) + dispersive_->prepare(stage); + { + const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.inverse"); + if (problem_.dims.family == FieldFamily::field_real) + graph::execute(*inverse_real_plan_, fine_spectrum_, modal_real_, + KokkosFFT::Normalization::none); + else if (!retained_synthesis_) + graph::execute(*inverse_complex_plan_, fine_spectrum_, modal_complex_, + KokkosFFT::Normalization::none); + } + // A single node's synthesis and projection are folded into its response column. + const bool folded = folds_single_node(); + if (!folded) + scale_transform(stage, trial_taper_[stage]); + // Alone, the response is the stage's only source: no response column runs, and + // unless its product is the stage's polarisation nothing is projected or + // transformed but its own. + const bool alone = dispersive_ && dispersive_->alone(); + const bool stage_projects = !alone || dispersive_->borrowed(); + const bool stage_synthesises = + !folded && (!dispersive_ || !dispersive_->synthesises_nodes()); + // A projection writes every sample of its targets at the first chunk (a GEMM with + // a zero beta, the custom kernel by assignment, the folded column by its own + // product), so only a stage that does not project needs them zeroed. + if (!stage_projects) { + if (problem_.dims.family == FieldFamily::envelope) { + graph::deep_copy(execution_space_, projected_complex_, Complex{}); + if (!projected_coarse_complex_.empty()) + graph::deep_copy(execution_space_, projected_coarse_complex_, Complex{}); + } else { + graph::deep_copy(execution_space_, projected_real_, 0.0); + if (!projected_coarse_real_.empty()) + graph::deep_copy(execution_space_, projected_coarse_real_, 0.0); + } + } + if (dispersive_) + dispersive_->begin(); + if (raman_products_ != 0) + convolve_raman_products(); + bool first_chunk = true; + std::size_t chunk = 0; + for (std::size_t first_node = 0; first_node < problem_.dims.n_nodes; + first_node += problem_.dims.node_chunk) { + const auto node_count = + std::min(problem_.dims.node_chunk, problem_.dims.n_nodes - first_node); + if (stage_synthesises) + synthesise(first_node, node_count); + if (!alone) + response(stage, first_node, node_count); + if (raman_products_ != 0) + finish_raman_products(stage, first_node, node_count); + else if (problem_.raman.enabled) + convolve_raman(stage, node_count); + if (dispersive_) + dispersive_->respond(chunk, first_node, node_count); + if (held_plasma()) { + const Kokkos::Profiling::ScopedRegion region("solprop_native.held.response"); + held_response(stage, first_node, node_count); + } + if (gated_plasma()) + gated_response(node_count); + if (stage_projects && !folded && !held_radial()) { + const Kokkos::Profiling::ScopedRegion region("solprop_native.projection"); + project(first_node, node_count, first_chunk); + } + if (problem_.plasma.envelope) + envelope_response(stage, first_node, node_count); + first_chunk = false; + ++chunk; + } + if (dispersive_) + dispersive_->finish(); + if (!held_radial() && (!dispersive_ || !dispersive_->replaces_transform())) { + const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.forward"); + if (problem_.dims.family == FieldFamily::envelope) { + graph::execute(*forward_complex_plan_, projected_complex_, projected_spectrum_, + KokkosFFT::Normalization::none); + if (forward_coarse_complex_plan_ && !defer_quadrature_) + graph::execute(*forward_coarse_complex_plan_, projected_coarse_complex_, + projected_coarse_spectrum_, KokkosFFT::Normalization::none); + } else if (!packed) { + graph::execute(*forward_real_plan_, projected_real_, projected_spectrum_, + KokkosFFT::Normalization::none); + if (forward_coarse_real_plan_ && !defer_quadrature_) + graph::execute(*forward_coarse_real_plan_, projected_coarse_real_, + projected_coarse_spectrum_, KokkosFFT::Normalization::none); + } + } + // The response's spectrum joins the stage's, before the envelope currents'. + if (dispersive_) + dispersive_->combine(); + if (absorbing()) { + form_absorber(); + const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.absorber"); + if (problem_.dims.family == FieldFamily::envelope) + graph::execute(*absorber_complex_plan_, absorber_complex_, absorber_spectrum_, + KokkosFFT::Normalization::none); + else if (!packed) + graph::execute(*absorber_real_plan_, absorber_real_, absorber_spectrum_, + KokkosFFT::Normalization::none); + } + if (packed) + transform_forward_pack(); + if (problem_.plasma.envelope) + envelope_transform(); + if (held_plasma() && !held_radial()) + held_transform(false, packed); + if (defer_quadrature_ && project_coarse_) + coarse_pending_ = true; + return true; +} + +// The general modal route's stage core: the shared lab-frame input, the in-stage +// absorber on the modal time field as on every route, and the general source in place +// of the legacy node pipeline; the finish applies the shared normalisation and the +// return to the interaction picture (solprop `NumpyRefBackend._nonlinear_general`). +void GeneralWorkspace::general_core(const std::size_t stage) { + if (problem_.absorber.in_stage) { + graph::execute(*inverse_real_plan_, fine_spectrum_, modal_real_, + KokkosFFT::Normalization::none); + form_absorber(); + graph::execute(*absorber_real_plan_, absorber_real_, absorber_spectrum_, + KokkosFFT::Normalization::none); + } + // A guide whose every region is linear has an identically zero source, which + // solprop's reference short-circuits in the same way. + if (general_->responds()) { + general_->combine(stage); + if (general_->dispersive() != nullptr) + general_->dispersive()->begin(stage); + general_->evaluate(); + } else { + graph::deep_copy(execution_space_, general_->source(), Complex{}); + } +} + +// The transforms scaled by stage `stage`'s taper, whose value `taper` the host holds. +// They depend on that value alone: a stage whose taper equals the last one scaled (a +// guide without a taper has one throughout) keeps them, the same bits. +void GeneralWorkspace::scale_transform(const std::size_t stage, + const double taper_value) { + if (scaled_taper_ready_ && taper_value == scaled_taper_) + return; + scaled_taper_ready_ = true; + scaled_taper_ = taper_value; + const auto synthesis = synthesis_; + const auto projection = projection_; + const auto projection_coarse = projection_coarse_; + const auto scaled_synthesis = scaled_synthesis_; + const auto scaled_projection = scaled_projection_; + const auto scaled_projection_coarse = scaled_projection_coarse_; + const auto scaled_projection_stacked = scaled_projection_stacked_; + const auto scaled_synthesis_complex = scaled_synthesis_complex_; + const auto scaled_projection_complex = scaled_projection_complex_; + const auto scaled_projection_coarse_complex = scaled_projection_coarse_complex_; + const auto scaled_projection_stacked_complex = scaled_projection_stacked_complex_; + const auto taper = stage_taper_; + const auto rows = problem_.dims.n_nodes * problem_.dims.n_pol_transform; + const auto modes = problem_.dims.n_modes; + launch::parallel_for( + "solprop_native.transform.scale", + Range2(execution_space_, {0, 0}, {rows, problem_.dims.n_modes}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t mode) { + const double scale = taper(stage); + scaled_synthesis(row, mode) = synthesis(row, mode) * scale; + scaled_projection(mode, row) = projection(mode, row) / scale; + if (!scaled_synthesis_complex.empty()) { + scaled_synthesis_complex(row, mode) = + Complex(synthesis(row, mode) * scale, 0.0); + scaled_projection_complex(mode, row) = + Complex(projection(mode, row) / scale, 0.0); + } + if (!scaled_projection_coarse.empty()) + scaled_projection_coarse(mode, row) = projection_coarse(mode, row) / scale; + if (!scaled_projection_coarse_complex.empty()) + scaled_projection_coarse_complex(mode, row) = + Complex(projection_coarse(mode, row) / scale, 0.0); + if (!scaled_projection_stacked.empty()) { + scaled_projection_stacked(mode, row) = projection(mode, row) / scale; + scaled_projection_stacked(modes + mode, row) = + projection_coarse(mode, row) / scale; + } + if (!scaled_projection_stacked_complex.empty()) { + scaled_projection_stacked_complex(mode, row) = + Complex(projection(mode, row) / scale, 0.0); + scaled_projection_stacked_complex(modes + mode, row) = + Complex(projection_coarse(mode, row) / scale, 0.0); + } + }); +} + +void GeneralWorkspace::synthesise(const std::size_t first_node, + const std::size_t node_count) { + if (retained_synthesis_) { + synthesise_retained(first_node, node_count); + return; + } + const bool whole_transform = first_node == 0 && node_count == problem_.dims.n_nodes && + problem_.dims.n_pol_state == 1; + if (whole_transform && !radial_blas_input_.empty()) { + packed_radial_gemm(scaled_synthesis_, modal_complex_, node_complex_); + return; + } + // A radial analytic field can exceed the node budget even at 256 nodes. Its + // row chunks are contiguous in LayoutRight, including the short final chunk: + // alias those rows directly instead of falling through to scalar synthesis. A + // device takes the same GEMM wherever its shape guard admits the interleaved + // synthesis, which `synthesis_provider` reports. + const bool radial_analytic_chunk = + problem_.dims.family == FieldFamily::field_analytic && + !problem_.spatial.spectral_weights.empty(); + if ((whole_transform || radial_analytic_chunk) && use_interleaved_synthesis()) { + Kokkos::Profiling::pushRegion( + "solprop_native.transform.synthesis.kokkosblas_interleaved"); + using Interleaved = Kokkos::View>; + // Complex samples are pairs of doubles, real part first: the same rows read as + // twice as many real columns. + static_assert(sizeof(Complex) == 2 * sizeof(double)); + const Interleaved modal(reinterpret_cast(modal_complex_.data()), + problem_.dims.n_modes, 2 * problem_.dims.n_to); + const Interleaved synthesis(scaled_synthesis_.data() + + first_node * problem_.dims.n_modes, + node_count, problem_.dims.n_modes); + const Interleaved node(reinterpret_cast(node_complex_.data()), node_count, + 2 * problem_.dims.n_to); + engine_gemm(transform_space_for(node_count), synthesis, modal, node, 0.0); + Kokkos::Profiling::popRegion(); + return; + } + if (whole_transform && use_blas_for_synthesis()) { + Kokkos::Profiling::pushRegion("solprop_native.transform.synthesis.kokkosblas"); + if (problem_.dims.family == FieldFamily::envelope) + selected_gemm(transform_space_for(node_count), scaled_synthesis_complex_, + modal_complex_, node_complex_, Complex{}, true); + else if (one_mode_products_) + one_mode_synthesis(response_space_for(node_count), scaled_synthesis_, modal_real_, + node_real_); + else + selected_gemm(transform_space_for(node_count), scaled_synthesis_, modal_real_, + node_real_, 0.0, true); + Kokkos::Profiling::popRegion(); + return; + } + if (use_chunked_blas_for_synthesis()) { + Kokkos::Profiling::pushRegion( + "solprop_native.transform.synthesis.kokkosblas_chunked"); + const auto rows = node_count * problem_.dims.n_pol_transform; + const auto first_row = first_node * problem_.dims.n_pol_transform; + const bool tail = node_count != problem_.dims.node_chunk; + if (problem_.dims.family == FieldFamily::envelope) { + const auto packed = tail ? tail_synthesis_complex_ : chunk_synthesis_complex_; + const auto source = scaled_synthesis_complex_; + launch::parallel_for( + "solprop_native.transform.pack_synthesis_complex", + Range2(matrix_copy_space_for(node_count), {0, 0}, + {rows, problem_.dims.n_modes}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t mode) { + packed(row, mode) = source(first_row + row, mode); + }); + const auto output = Kokkos::subview( + node_complex_, std::make_pair(std::size_t{0}, rows), Kokkos::ALL()); + selected_gemm(transform_space_for(node_count), packed, modal_complex_, output, + Complex{}, true); + } else { + const auto packed = tail ? tail_synthesis_ : chunk_synthesis_; + const auto source = scaled_synthesis_; + launch::parallel_for( + "solprop_native.transform.pack_synthesis", + Range2(matrix_copy_space_for(node_count), {0, 0}, + {rows, problem_.dims.n_modes}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t mode) { + packed(row, mode) = source(first_row + row, mode); + }); + const auto output = Kokkos::subview( + node_real_, std::make_pair(std::size_t{0}, rows), Kokkos::ALL()); + selected_gemm(transform_space_for(node_count), packed, modal_real_, output, 0.0, + true); + } + Kokkos::Profiling::popRegion(); + return; + } + Kokkos::Profiling::pushRegion("solprop_native.transform.synthesis.custom"); + const auto synthesis = scaled_synthesis_; + const auto modal_real = modal_real_; + const auto modal_complex = modal_complex_; + const auto node_real = node_real_; + const auto node_complex = node_complex_; + const auto n_modes = problem_.dims.n_modes; + const auto n_pol_state = problem_.dims.n_pol_state; + const auto n_pol_transform = problem_.dims.n_pol_transform; + const auto n_to = problem_.dims.n_to; + const bool real_family = problem_.dims.family == FieldFamily::field_real; + const auto effective_rows = node_count * problem_.effective_node_pol(); + launch::parallel_for( + "solprop_native.transform.synthesis.custom", + Range2(transform_space_for(node_count), {0, 0}, {effective_rows, n_to}), + KOKKOS_LAMBDA(const std::size_t local_row, const std::size_t time) { + const std::size_t state_pol = local_row % n_pol_state; + const std::size_t transform_local = local_row / n_pol_state; + const std::size_t transform_row = + first_node * n_pol_transform + transform_local; + if (real_family) { + double value = 0.0; + for (std::size_t mode = 0; mode < n_modes; ++mode) + value += synthesis(transform_row, mode) * + modal_real(mode * n_pol_state + state_pol, time); + node_real(local_row, time) = value; + } else { + Complex value{}; + for (std::size_t mode = 0; mode < n_modes; ++mode) + value += synthesis(transform_row, mode) * + modal_complex(mode * n_pol_state + state_pol, time); + node_complex(local_row, time) = value; + } + }); + Kokkos::Profiling::popRegion(); +} + +template +void GeneralWorkspace::response_harmonic(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count) { + if constexpr (Nonlinear && Family != FieldFamily::field_real) { + if (problem_.kerr.harmonic_factor != 0.0) { + response_raman( + stage, first_node, node_count); + return; + } + } + response_raman(stage, first_node, + node_count); +} + +template +void GeneralWorkspace::response_raman(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count) { + if (problem_.raman.enabled) { + if (problem_.raman.thg) { + response_impl(stage, first_node, node_count); + } else { + response_impl(stage, first_node, node_count); + } + return; + } + response_impl(stage, first_node, node_count); +} + +template +void GeneralWorkspace::response_stash(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count) { + if constexpr (Plasma) { + // Clang's host loop is measurably faster when the maximum-rate accumulator is + // present. Keep that loop shape for every plasma problem, while the view and its + // final-stage stores remain conditional on requested periodic statistics. + response_harmonic(stage, first_node, + node_count); + return; + } + response_harmonic(stage, first_node, + node_count); +} + +template +void GeneralWorkspace::response_nonlinear(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count) { + if (problem_.kerr.nonlinear) { + response_stash(stage, first_node, node_count); + } else { + response_stash(stage, first_node, node_count); + } +} + +template +void GeneralWorkspace::response_plasma(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count) { + if (problem_.plasma.enabled && !problem_.plasma.envelope) { + if constexpr (Family != FieldFamily::envelope) { + // SEP 27: the band column writes the physical current separately. The + // ordinary traversal forms and collars only the Kerr/Raman polarisation. + if (problem_.plasma.band) { + if (band_tiled_) { + if (problem_.plasma.n_channels > 1) + tiled_band_plasma_column(stage, first_node, + node_count); + else + tiled_band_plasma_column(stage, first_node, + node_count); + } else if (problem_.plasma.n_channels > 1) { + team_band_plasma_column(stage, first_node, + node_count); + } else { + team_band_plasma_column(stage, first_node, + node_count); + } + response_nonlinear(stage, first_node, node_count); + return; + } + } + response_nonlinear(stage, first_node, node_count); + } else { + response_nonlinear(stage, first_node, node_count); + } +} + +void GeneralWorkspace::response(const std::size_t stage, const std::size_t first_node, + const std::size_t node_count) { + const auto components = problem_.effective_node_pol(); + switch (problem_.dims.family) { + case FieldFamily::field_real: + if (components == 1) + response_plasma(stage, first_node, node_count); + else + response_plasma(stage, first_node, node_count); + return; + case FieldFamily::field_analytic: + if (components == 1) + response_plasma(stage, first_node, node_count); + else + response_plasma(stage, first_node, node_count); + return; + case FieldFamily::envelope: + if (components == 1) + response_plasma(stage, first_node, node_count); + else + response_plasma(stage, first_node, node_count); + } +} + +template +void GeneralWorkspace::response_impl(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count) { + // A single node's transforms fold into the column (`folds_single_node`), compiled + // apart so that the other columns' arithmetic and layout are untouched. + if constexpr (Components == 1 && !Raman && !Folded) { + if (folds_single_node()) { + response_impl(stage, first_node, node_count); + return; + } + } + if constexpr (!Multiple) { + if (problem_.plasma.n_channels > 1 && Plasma) { + response_impl(stage, first_node, node_count); + return; + } + } + const auto node_real = node_real_; + const auto node_complex = node_complex_; + const auto out_real = polarisation_real_; + const auto held_current = held_current_; + const auto out_complex = polarisation_complex_; + const auto raman_drive = node_raman_drive_; + const auto raman_drives = problem_.raman_drives(); + // The modal products replace the nodes' drives (`raman_modal_products`). + const bool node_drives = raman_products_ == 0; + const auto window = time_window_; + const auto carrier = carrier_; + const auto rate_coefficients = rate_coefficients_; + const auto stash = stash_candidate_; + const auto density = stage_plasma_density_; + const auto kerr_density = stage_kerr_density_; + const auto channels = problem_.plasma.n_channels; + const auto channel_parameters = channel_parameters_; + const auto channel_coefficients = channel_coefficients_; + const auto channel_density = stage_channel_density_; + const auto channel_preionisation = stage_channel_preionisation_; + const auto scratch = channel_scratch_; + const auto preionisation = stage_preionisation_; + const auto n_to = problem_.dims.n_to; + const double kerr_coefficient = problem_.kerr.coefficient; + const double self_factor = problem_.kerr.self_factor; + const double conjugate_factor = problem_.kerr.conjugate_factor; + const double harmonic_factor = problem_.kerr.harmonic_factor; + const double charge_mass_ratio = problem_.plasma.charge_mass_ratio; + const double ionisation_potential = problem_.plasma.ionisation_potential; + const double dt = problem_.grid.dt_fine; + const double log_field_min = problem_.plasma.log_field_min; + const double d_log_field = problem_.plasma.d_log_field; + const double field_min = problem_.plasma.field_min; + const double field_max = problem_.plasma.field_max; + const double rate_max = problem_.plasma.rate_max; + const auto rate_intervals = problem_.plasma.rate_intervals; + const auto n_stages = problem_.stepper.n_stages; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // The device's form (`device_response`); a mixture of channels takes the generic + // column below. + if constexpr (!Multiple) { + device_response(stage, first_node, node_count); + return; + } +#endif + // A folded single node reads the modal field and writes the projected one, each + // through its one-element transform (`folds_single_node`); otherwise both scales + // are one, which changes no bit. + constexpr bool folded = Folded; + const auto input_real = folded ? modal_real_ : node_real_; + const auto input_complex = folded ? modal_complex_ : node_complex_; + const auto target_real = folded ? projected_real_ : polarisation_real_; + const auto target_complex = folded ? projected_complex_ : polarisation_complex_; + const auto synthesis = synthesis_; + const auto projection = projection_; + const auto taper = stage_taper_; + launch::parallel_for( + "solprop_native.response.column", + Kokkos::RangePolicy(response_space_for(node_count), 0, + node_count), + KOKKOS_LAMBDA(const std::size_t node) { + // CUDA extended-lambda restriction 12 requires the first capture outside + // if-constexpr blocks. Reference binding keeps every compile-time branch. + capture_anchor(channels, scratch, channel_preionisation, node_real, + node_complex, channel_coefficients, channel_parameters, dt, + channel_density, rate_coefficients, log_field_min, d_log_field, + field_min, field_max, rate_max, rate_intervals, + ionisation_potential, charge_mass_ratio, density, + conjugate_factor, self_factor, harmonic_factor, carrier, + raman_drive, raman_drives, node_drives, out_real, held_current, + window, out_complex, stash, n_stages, first_node); + const std::size_t base = node * Components; + const double input_scale = folded ? synthesis(0, 0) * taper(stage) : 1.0; + const double output_scale = folded ? projection(0, 0) / taper(stage) : 1.0; + double exposure = 0.0; + double last_rate = 0.0; + double largest_rate = 0.0; + double fraction = Plasma && !Multiple ? preionisation(stage) : 0.0; + double free_first = 0.0; + double free_second = 0.0; + double last_first = 0.0; + double last_second = 0.0; + if constexpr (Multiple) { + for (std::size_t channel = 0; channel < channels; ++channel) { + scratch(channel, node, 0) = 0.0; + scratch(channel, node, 1) = 0.0; + scratch(channel, node, 2) = channel_preionisation(stage, channel); + scratch(channel, node, 3) = 0.0; + } + } + // Cache contiguous pointwise addresses to avoid repeated View stride loads. + // Plasma retains its measured recurrence-loop layout; applying + // the same change there regresses HE plasma by 8-9% (kokkos-reopt report). + auto* const real_input = input_real.data(); + auto* const complex_input = input_complex.data(); + auto* const real_target = target_real.data(); + auto* const complex_target = target_complex.data(); + auto* const window_data = window.data(); + const double pointwise_factor = + Plasma ? 0.0 : kerr_coefficient * kerr_density(stage); + for (std::size_t time = 0; time < n_to; ++time) { + Complex first_value{}; + Complex second_value{}; + if constexpr (Family == FieldFamily::field_real) { + first_value = + Complex(input_scale * (Plasma ? input_real(base, time) + : real_input[base * n_to + time]), + 0.0); + if constexpr (Components == 2) + second_value = + Complex(input_scale * (Plasma ? input_real(base + 1, time) + : real_input[(base + 1) * n_to + time]), + 0.0); + } else { + first_value = input_scale * (Plasma ? input_complex(base, time) + : complex_input[base * n_to + time]); + if constexpr (Components == 2) + second_value = + input_scale * (Plasma ? input_complex(base + 1, time) + : complex_input[(base + 1) * n_to + time]); + } + const double modulus = squared_abs(first_value) + + (Components == 2 ? squared_abs(second_value) : 0.0); + const Complex square = + first_value * first_value + + (Components == 2 ? second_value * second_value : Complex{}); + if constexpr (Plasma) { + // IntelLLVM's precise model retains an unused hypot outside this branch. + const double magnitude = + Components == 2 ? Kokkos::hypot(first_value.real(), second_value.real()) + : Kokkos::abs(first_value.real()); + double guard = 0.0; + double weight = 0.0; + if constexpr (Multiple) { + double work = 0.0; + for (std::size_t channel = 0; channel < channels; ++channel) { + const double rate = ionisation_rate( + magnitude, channel_coefficients, channel_parameters(channel, 0), + channel_parameters(channel, 1), channel_parameters(channel, 2), + channel_parameters(channel, 3), channel_parameters(channel, 4), + static_cast(channel_parameters(channel, 5)), + static_cast(channel_parameters(channel, 6))); + double channel_exposure = scratch(channel, node, 0); + const double previous_exposure = channel_exposure; + if (time > 0) + channel_exposure += trapezoid(rate, scratch(channel, node, 1), dt); + double channel_fraction = scratch(channel, node, 2); + if (time == 0 || channel_exposure != previous_exposure) + channel_fraction = ionised_fraction( + channel_exposure, channel_preionisation(stage, channel)); + scratch(channel, node, 0) = channel_exposure; + scratch(channel, node, 1) = rate; + scratch(channel, node, 2) = channel_fraction; + if (rate > scratch(channel, node, 3)) + scratch(channel, node, 3) = rate; + const double partial = channel_density(stage, channel); + weight += partial * channel_fraction * channel_parameters(channel, 7); + work += partial * channel_parameters(channel, 8) * rate * + (1.0 - channel_fraction); + } + guard = work > 0.0 && magnitude * magnitude > 0.0 + ? work / (magnitude * magnitude) + : 0.0; + } else { + const double rate = ionisation_rate(magnitude, rate_coefficients, + log_field_min, d_log_field, field_min, + field_max, rate_max, rate_intervals); + const double previous_exposure = exposure; + if (time > 0) + exposure += trapezoid(rate, last_rate, dt); + last_rate = rate; + if constexpr (Stash) { + if (rate > largest_rate) + largest_rate = rate; + } + // Preserve the first subtraction's rounding even for pre-ionized + // input; afterwards an unchanged rounded exposure has the same exp. + if (time == 0 || exposure != previous_exposure) + fraction = ionised_fraction(exposure, preionisation(stage)); + guard = rate > 0.0 && magnitude * magnitude > 0.0 + ? ionisation_potential * rate * (1.0 - fraction) / + (magnitude * magnitude) + : 0.0; + weight = fraction * charge_mass_ratio; + } + double integrand = weight * first_value.real(); + if (time > 0) + free_first += trapezoid(integrand, last_first, dt); + double current = free_first + guard * first_value.real(); + last_first = integrand; + held_current(base, time) = Multiple ? current : density(stage) * current; + if constexpr (Components == 2) { + integrand = weight * second_value.real(); + if (time > 0) + free_second += trapezoid(integrand, last_second, dt); + current = free_second + guard * second_value.real(); + last_second = integrand; + held_current(base + 1, time) = + Multiple ? current : density(stage) * current; + } + } + const double factor = + Plasma ? kerr_coefficient * kerr_density(stage) : pointwise_factor; + Complex first{}; + Complex second{}; + if constexpr (Nonlinear) { + if constexpr (Family == FieldFamily::field_real) { + first += Complex(factor * modulus * first_value.real(), 0.0); + if constexpr (Components == 2) + second += Complex(factor * modulus * second_value.real(), 0.0); + } else { + const auto analytic = [&](const Complex value) { + const double weighted_real = conjugate_factor * square.real(); + const double weighted_imag = conjugate_factor * square.imag(); + const double weighted_self = self_factor * modulus; + Complex result(factor * (weighted_self * value.real() + + (weighted_real * value.real() + + weighted_imag * value.imag())), + factor * (weighted_self * value.imag() + + (weighted_imag * value.real() - + weighted_real * value.imag()))); + if constexpr (Harmonic) + result += + ((factor * harmonic_factor) * carrier(time)) * square * value; + return result; + }; + first += analytic(first_value); + if constexpr (Components == 2) + second += analytic(second_value); + } + } + if constexpr (Raman) { + if (node_drives) + write_raman_drive( + raman_drive, node, time, first_value, second_value, raman_drives); + } + if constexpr (Family != FieldFamily::envelope) { + (Plasma ? target_real(base, time) : real_target[base * n_to + time]) = + Raman ? first.real() + : output_scale * (first.real() * + (Plasma ? window(time) : window_data[time])); + if constexpr (Components == 2) + (Plasma ? target_real(base + 1, time) + : real_target[(base + 1) * n_to + time]) = + Raman ? second.real() + : output_scale * (second.real() * + (Plasma ? window(time) : window_data[time])); + } else { + (Plasma ? target_complex(base, time) : complex_target[base * n_to + time]) = + Raman ? first + : output_scale * + (first * (Plasma ? window(time) : window_data[time])); + if constexpr (Components == 2) + (Plasma ? target_complex(base + 1, time) + : complex_target[(base + 1) * n_to + time]) = + Raman ? second + : output_scale * + (second * (Plasma ? window(time) : window_data[time])); + } + } + if constexpr (Stash) { + if (stash.empty() || stage + 1 != n_stages) + return; + if constexpr (Multiple) { + for (std::size_t channel = 0; channel < channels; ++channel) { + stash(first_node + node, 2 * channel) = scratch(channel, node, 2); + stash(first_node + node, 2 * channel + 1) = scratch(channel, node, 3); + } + } else { + stash(first_node + node, 0) = fraction; + stash(first_node + node, 1) = largest_rate; + } + } + }); +} + +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +// The device's response with no team tied to a node, so that a one-node fibre fills the +// device: +// +// - the pointwise terms (Kerr, the harmonic, the Raman drive and the windowed +// polarisation) as one two-dimensional range over (node, time), in the column's +// arithmetic sample by sample; +// - the trapezoid plasma as two-level scans over tiles of `plasma_tile_samples`: +// each running integral is a kernel of one team per (row, tile) that sums the +// tile's increments into its total (`plasma_tile_totals_`), then one per (row, +// tile) that adds the row's earlier totals and scans the tile again from that +// offset, forming the exposure's fraction or the current from each prefix. +template +void GeneralWorkspace::device_response(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count) { + const auto node_real = node_real_; + const auto node_complex = node_complex_; + const auto out_real = polarisation_real_; + const auto out_complex = polarisation_complex_; + const auto raman_drive = node_raman_drive_; + const auto raman_drives = problem_.raman_drives(); + const auto window = time_window_; + const auto carrier = carrier_; + const auto kerr_density = stage_kerr_density_; + const auto n_to = problem_.dims.n_to; + const double kerr_coefficient = problem_.kerr.coefficient; + const double self_factor = problem_.kerr.self_factor; + const double conjugate_factor = problem_.kerr.conjugate_factor; + const double harmonic_factor = problem_.kerr.harmonic_factor; + if constexpr (Plasma) + device_trapezoid_plasma(stage, first_node, node_count); + launch::parallel_for( + Plasma ? "solprop_native.response.device_pointwise_after_plasma" + : "solprop_native.response.device_pointwise", + Range2(response_space_for(node_count), {0, 0}, {node_count, n_to}), + KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { + capture_anchor(self_factor, conjugate_factor, node_real, node_complex, out_real, + out_complex, raman_drive, raman_drives, window, carrier, + kerr_density, kerr_coefficient, harmonic_factor, stage); + const std::size_t base = node * Components; + const auto value_at = [&](const std::size_t row) { + if constexpr (Family == FieldFamily::field_real) + return Complex(node_real(row, time), 0.0); + else + return Complex(node_complex(row, time)); + }; + const Complex first_value = value_at(base); + Complex second_value{}; + if constexpr (Components == 2) + second_value = value_at(base + 1); + const double modulus = squared_abs(first_value) + + (Components == 2 ? squared_abs(second_value) : 0.0); + const Complex square = + first_value * first_value + + (Components == 2 ? second_value * second_value : Complex{}); + const double factor = kerr_coefficient * kerr_density(stage); + Complex first{}; + Complex second{}; + if constexpr (Nonlinear) { + if constexpr (Family == FieldFamily::field_real) { + first += Complex(factor * modulus * first_value.real(), 0.0); + if constexpr (Components == 2) + second += Complex(factor * modulus * second_value.real(), 0.0); + } else { + const auto analytic = [&](const Complex value) { + const double weighted_real = conjugate_factor * square.real(); + const double weighted_imag = conjugate_factor * square.imag(); + const double weighted_self = self_factor * modulus; + Complex result( + factor * + (weighted_self * value.real() + + (weighted_real * value.real() + weighted_imag * value.imag())), + factor * + (weighted_self * value.imag() + + (weighted_imag * value.real() - weighted_real * value.imag()))); + if constexpr (Harmonic) + result += ((factor * harmonic_factor) * carrier(time)) * square * value; + return result; + }; + first += analytic(first_value); + if constexpr (Components == 2) + second += analytic(second_value); + } + } + if constexpr (Raman) + write_raman_drive(raman_drive, node, time, first_value, + second_value, raman_drives); + if constexpr (Family != FieldFamily::envelope) { + out_real(base, time) = Raman ? first.real() : first.real() * window(time); + if constexpr (Components == 2) + out_real(base + 1, time) = + Raman ? second.real() : second.real() * window(time); + } else { + out_complex(base, time) = Raman ? first : first * window(time); + if constexpr (Components == 2) + out_complex(base + 1, time) = Raman ? second : second * window(time); + } + }); +} + +// The trapezoid plasma's two-level scans on a device (`device_response`). Rows of the +// rate, exposure and fraction views are the chunk's nodes; rows of the integrand and +// of the tile totals are its component rows. +template +void GeneralWorkspace::device_trapezoid_plasma(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count) { + using TeamPolicy = Kokkos::TeamPolicy; + using TeamMember = TeamPolicy::member_type; + const auto node_real = node_real_; + const auto node_complex = node_complex_; + const auto held_current = held_current_; + const auto rates = plasma_rate_; + const auto exposures = plasma_exposure_; + const auto fractions = plasma_fraction_; + const auto integrands = plasma_current_integrand_; + const auto totals = plasma_tile_totals_; + const auto stash = stash_candidate_; + const auto rate_coefficients = rate_coefficients_; + const auto density = stage_plasma_density_; + const auto preionisation = stage_preionisation_; + const double charge_mass_ratio = problem_.plasma.charge_mass_ratio; + const double ionisation_potential = problem_.plasma.ionisation_potential; + const double dt = problem_.grid.dt_fine; + const double log_field_min = problem_.plasma.log_field_min; + const double d_log_field = problem_.plasma.d_log_field; + const double field_min = problem_.plasma.field_min; + const double field_max = problem_.plasma.field_max; + const double rate_max = problem_.plasma.rate_max; + const auto rate_intervals = problem_.plasma.rate_intervals; + const auto n_to = problem_.dims.n_to; + const auto samples = static_cast(n_to); + const auto width = static_cast(plasma_tile_samples); + const auto tiles = static_cast(plasma_tiles_); + const auto last = n_to - 1; + const bool stashing = + Stash && !stash.empty() && stage + 1 == problem_.stepper.n_stages; + const std::size_t rows = node_count * Components; + const auto team_size = static_cast(response_team_size()); + // The field a row reads: a real row's value, or an analytic or envelope row's real + // part. + const auto raw = [=] KOKKOS_FUNCTION(const std::size_t row, const std::size_t time) { + capture_anchor(node_real, node_complex); + if constexpr (Family == FieldFamily::field_real) + return node_real(row, time); + else + return node_complex(row, time).real(); + }; + + // 1. The rates. + launch::parallel_for( + "solprop_native.response.device_plasma_rate", + Range2(response_space_for(node_count), {0, 0}, {node_count, n_to}), + KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { + const std::size_t base = node * Components; + const double first = raw(base, time); + double magnitude = 0.0; + if constexpr (Components == 2) + magnitude = Kokkos::hypot(first, raw(base + 1, time)); + else + magnitude = Kokkos::abs(first); + rates(node, time) = + ionisation_rate(magnitude, rate_coefficients, log_field_min, d_log_field, + field_min, field_max, rate_max, rate_intervals); + }); + // 2. The exposure: tile totals, then each tile's prefixes from its offset, and the + // fraction from each. + const auto exposure_local = KOKKOS_LAMBDA(const TeamMember& team) { + capture_anchor(rates, totals, samples, width, tiles, dt); + const auto node = static_cast(team.league_rank() / tiles); + const int tile = team.league_rank() % tiles; + const int first = tile * width; + const int end = Kokkos::min(first + width, samples); + const double total = team_trapezoid_total( + team, + [&](const int sample) { return rates(node, static_cast(sample)); }, + first, end, dt); + Kokkos::single(Kokkos::PerTeam(team), + [&] { totals(node, static_cast(tile)) = total; }); + }; + launch::parallel_for("solprop_native.response.device_plasma_exposure_tiles", + TeamPolicy(response_space_for(node_count), + static_cast(node_count) * tiles, team_size), + exposure_local); + const auto exposure_finish = KOKKOS_LAMBDA(const TeamMember& team) { + capture_anchor(rates, totals, exposures, fractions, preionisation, samples, width, + tiles, dt, stage); + const auto node = static_cast(team.league_rank() / tiles); + const int tile = team.league_rank() % tiles; + const int first = tile * width; + const int end = Kokkos::min(first + width, samples); + const double offset = team_tile_offset(team, totals, node, tile); + team_trapezoid_finish( + team, + [&](const int sample) { return rates(node, static_cast(sample)); }, + first, end, dt, offset, + [&](const int sample, const double prefix) { + const auto time = static_cast(sample); + exposures(node, time) = prefix; + fractions(node, time) = ionised_fraction(prefix, preionisation(stage)); + }); + }; + launch::parallel_for("solprop_native.response.device_plasma_exposure", + TeamPolicy(response_space_for(node_count), + static_cast(node_count) * tiles, team_size), + exposure_finish); + // 3. The free-current integrand, per component row. + launch::parallel_for( + "solprop_native.response.device_plasma_integrand", + Range2(response_space_for(node_count), {0, 0}, {rows, n_to}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { + integrands(row, time) = + fractions(row / Components, time) * charge_mass_ratio * raw(row, time); + }); + // 4. The free current: tile totals, then each tile's prefixes from its offset and + // the current from each. + const auto current_local = KOKKOS_LAMBDA(const TeamMember& team) { + capture_anchor(integrands, totals, samples, width, tiles, dt); + const auto row = static_cast(team.league_rank() / tiles); + const int tile = team.league_rank() % tiles; + const int first = tile * width; + const int end = Kokkos::min(first + width, samples); + const double total = team_trapezoid_total( + team, + [&](const int sample) { + return integrands(row, static_cast(sample)); + }, + first, end, dt); + Kokkos::single(Kokkos::PerTeam(team), + [&] { totals(row, static_cast(tile)) = total; }); + }; + launch::parallel_for("solprop_native.response.device_plasma_current_tiles", + TeamPolicy(response_space_for(node_count), + static_cast(rows) * tiles, team_size), + current_local); + const auto current_finish = KOKKOS_LAMBDA(const TeamMember& team) { + capture_anchor(integrands, totals, held_current, rates, fractions, density, samples, + width, tiles, dt, ionisation_potential, stage); + const auto row = static_cast(team.league_rank() / tiles); + const int tile = team.league_rank() % tiles; + const int first = tile * width; + const int end = Kokkos::min(first + width, samples); + const double offset = team_tile_offset(team, totals, row, tile); + team_trapezoid_finish( + team, + [&](const int sample) { + return integrands(row, static_cast(sample)); + }, + first, end, dt, offset, + [&](const int sample, const double prefix) { + const auto time = static_cast(sample); + const std::size_t node = row / Components; + const std::size_t base = node * Components; + const double first_field = raw(base, time); + const double second_field = Components == 2 ? raw(base + 1, time) : 0.0; + const double magnitude_squared = + first_field * first_field + second_field * second_field; + const double rate = rates(node, time); + const double guard = rate > 0.0 && magnitude_squared > 0.0 + ? ionisation_potential * rate * + (1.0 - fractions(node, time)) / + magnitude_squared + : 0.0; + held_current(row, time) = density(stage) * (prefix + guard * raw(row, time)); + }); + }; + launch::parallel_for("solprop_native.response.device_plasma_current", + TeamPolicy(response_space_for(node_count), + static_cast(rows) * tiles, team_size), + current_finish); + // 5. The stash: the last sample's fraction and the largest rate, per node. + if (stashing) { + const auto peak = KOKKOS_LAMBDA(const TeamMember& team) { + capture_anchor(rates, fractions, stash, samples, last, first_node); + const auto node = static_cast(team.league_rank()); + double largest = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, samples), + [&](const int sample, double& update) { + const double rate = rates(node, static_cast(sample)); + if (rate > update) + update = rate; + }, + Kokkos::Max(largest)); + Kokkos::single(Kokkos::PerTeam(team), [&] { + stash(first_node + node, 0) = fractions(node, last); + stash(first_node + node, 1) = largest; + }); + }; + launch::parallel_for("solprop_native.response.device_plasma_stash", + TeamPolicy(response_space_for(node_count), + static_cast(node_count), team_size), + peak); + } +} +#endif + +static_assert(band_max_half + 1 == static_cast(plasma_max_taps), + "the team band filter's tap bound is the problem's"); + +namespace { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +/** Whether the band column skips each row's quiet samples (`band_support`): host only. + */ +constexpr bool band_windows = false; +#else +constexpr bool band_windows = true; +/** + * The fewest samples a host thread of a band team takes from a row: below it a thread's + * share of a pass no longer repays the team's barriers between the passes. Measured on + * the mode-averaged helium RDW case (n_to = 8192, one node) on a loaded Apple M-series + * host: teams of two and four threads ran the band column faster than one, six slower; + * 2048 caps that row at four. To be re-measured on each host class (solprop SEP 25). + */ +constexpr std::size_t band_host_samples_per_thread = 2048; +#endif + +/** The first and last indices whose value is non-zero, as a Kokkos MinMax reduction. */ +using BandLive = Kokkos::MinMaxScalar; + +/** The band window of a row whose non-zero samples `live` found. */ +KOKKOS_INLINE_FUNCTION BandWindow band_window_of(const BandLive& live, + const int samples, const int half) { + if constexpr (band_windows) + return band_support(live.min_val, live.max_val, samples, half); + else + return {.first = 0, .last = samples - 1}; +} + +/** Count `index` into `live` when `value` is not exactly zero. */ +KOKKOS_INLINE_FUNCTION void band_note(BandLive& live, const int index, + const double value) { + if (value != 0.0) { + live.min_val = Kokkos::min(live.min_val, index); + live.max_val = Kokkos::max(live.max_val, index); + } +} + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +/** + * Count a host block's non-zero values into `live`, as `band_note` of each: + * `values[k]` is the sample `offset + k`. Only the first and the last non-zero value + * matter, so the block is searched from either end. + */ +inline void band_note_block(BandLive& live, const double* values, + const std::size_t count, const int offset) { + std::size_t low = 0; + while (low < count && values[low] == 0.0) + ++low; + if (low == count) + return; + std::size_t high = count - 1; + while (values[high] == 0.0) + --high; + live.min_val = std::min(live.min_val, offset + static_cast(low)); + live.max_val = std::max(live.max_val, offset + static_cast(high)); +} + +/** + * `ionisation_rate` of a host block's `count <= rate_block` magnitudes into `row`, + * with the table's limits `lower`, `upper` and `highest` as it takes them (a NaN + * magnitude is live); `within(values, live)` runs `ionisation_rates_within` on the + * live ones. A mostly live block runs every sample through the layer, a quiet one's + * magnitude replaced by the lower limit, and then keeps the limits' values: each value + * is the same function of its own magnitude, so the passes' lanes do not change it, + * and no sample is gathered. A mostly quiet block gathers its live magnitudes. + */ +template +void band_rates_block(const double* magnitudes, double* row, const std::size_t count, + const double lower, const double upper, const double highest, + const Within& within) { + double values[rate_block]; + std::size_t live = 0; + for (std::size_t k = 0; k < count; ++k) + live += ((magnitudes[k] > upper) | (magnitudes[k] < lower)) ? 0 : 1; + if (2 * live >= count && lower > 0.0) { + for (std::size_t k = 0; k < count; ++k) { + const double magnitude = magnitudes[k]; + values[k] = ((magnitude > upper) | (magnitude < lower)) ? lower : magnitude; + } + within(values, count); + for (std::size_t k = 0; k < count; ++k) { + const double magnitude = magnitudes[k]; + row[k] = magnitude > upper ? highest : magnitude < lower ? 0.0 : values[k]; + } + return; + } + if (live == 0) { + // A quiet block takes only the limits. + for (std::size_t k = 0; k < count; ++k) + row[k] = magnitudes[k] > upper ? highest : 0.0; + return; + } + std::size_t active[rate_block]; + std::size_t gathered = 0; + for (std::size_t k = 0; k < count; ++k) { + const double magnitude = magnitudes[k]; + row[k] = magnitude > upper ? highest : 0.0; + if (!(magnitude > upper) && !(magnitude < lower)) { + values[gathered] = magnitude; + active[gathered] = k; + ++gathered; + } + } + within(values, gathered); + for (std::size_t k = 0; k < gathered; ++k) + row[active[k]] = values[k]; +} +#endif +} // namespace + +int GeneralWorkspace::band_host_team_size(const std::size_t node_count) const { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + static_cast(node_count); + return 1; +#else + // solprop SEP 25 section 6.7.5: share the threads between the chunk's nodes, so a + // chunk of fewer nodes than threads runs each node's passes over chunks of t. + const auto threads = static_cast( + std::max(1, response_space_for(node_count).concurrency())); + const auto per_node = + std::max(1, threads / std::max(1, node_count)); + const auto by_samples = + std::max(1, problem_.dims.n_to / band_host_samples_per_thread); + return static_cast(std::min(per_node, by_samples)); +#endif +} + +std::size_t GeneralWorkspace::band_league(const std::size_t node_count) const { + return build::device ? node_count : std::min(node_count, band_slots_); +} + +template +Kokkos::TeamPolicy +GeneralWorkspace::band_team_policy(const std::size_t node_count, + const Functor& functor) const { + using Policy = Kokkos::TeamPolicy; + constexpr auto policy_limit = + static_cast(std::numeric_limits::max()); + if (node_count > policy_limit || problem_.dims.n_to > policy_limit) + throw std::length_error("band team column extent exceeds signed 32-bit range"); + const auto league = static_cast(band_league(node_count)); + const auto scratch = + Kokkos::PerTeam(band_staging_bytes()); + // The widest team this kernel can launch with its scratch: a device's registers per + // thread can hold the band column below the configured width. + Policy probe(response_space_for(node_count), league, 1); + probe.set_scratch_size(0, scratch); + const int widest = probe.team_size_max(functor, Kokkos::ParallelForTag()); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const int requested = static_cast(response_team_size()); +#else + const int requested = band_host_team_size(node_count); +#endif + Policy policy(response_space_for(node_count), league, + std::max(1, std::min(requested, widest))); + policy.set_scratch_size(0, scratch); + return policy; +} + +// solprop SEP 18's band rule as a team column in whole-row passes (solprop SEP 25 +// section 6.7.4; include/solprop_native/plasma.hpp): per channel the rates, their +// filter and scan give the exposure; the fraction (floored where it is formed), the +// loss guard and the free-current weight follow pointwise, a mixture adding each +// channel's share in channel order. Per component the free-current integrand is +// filtered and scanned, and the scan's store forms the current from it, the guard and +// the field. SEP 27 uses this current directly, without a third running integral. +// +// The rows, by slot (a team's own; the node on a device): `plasma_rate_` the raw rates, +// `plasma_fraction_` the filtered row, `plasma_exposure_` the exposure, +// `plasma_free_current_` the loss guard (a mixture's work until the last channel), +// `plasma_outer_current_` a mixture's weight, `plasma_current_integrand_` the +// free-current integrand. The separate `held_current_` stores the total current. Each +// filter and scan runs on its row's live window (`band_support`) and writes the rest: +// zero before it, the final prefix after it, and an unchanged exposure's fraction. +template +// NOLINTNEXTLINE(readability-function-size): hot path, split pending an A/B +void GeneralWorkspace::team_band_plasma_column(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count) { + if constexpr (Components == 1 && !Folded) { + if (folds_single_node()) { + team_band_plasma_column(stage, first_node, + node_count); + return; + } + } + static_assert(Family != FieldFamily::envelope, + "the band rule integrates a field-grid plasma column"); + using TeamMember = Kokkos::TeamPolicy::member_type; + // A folded single node reads the modal field through its one-element synthesis + // (`folds_single_node`). + constexpr bool folded = Folded; + const auto node_real = folded ? modal_real_ : node_real_; + const auto node_complex = folded ? modal_complex_ : node_complex_; + const auto synthesis = synthesis_; + const auto taper = stage_taper_; + const auto out = held_current_; + const auto rates = plasma_rate_; + const auto filtered = plasma_fraction_; + const auto exposures = plasma_exposure_; + const auto guards = plasma_free_current_; + const auto weights = plasma_outer_current_; + const auto integrands = plasma_current_integrand_; + const auto taps_view = plasma_taps_; + const auto stash = stash_candidate_; + const bool stashing = !stash.empty() && stage + 1 == problem_.stepper.n_stages; + const auto rate_coefficients = rate_coefficients_; + const auto channel_parameters = channel_parameters_; + const auto channel_coefficients = channel_coefficients_; + const auto channel_density = stage_channel_density_; + const auto channel_preionisation = stage_channel_preionisation_; + const auto density = stage_plasma_density_; + const auto preionisation = stage_preionisation_; + const double charge_mass_ratio = problem_.plasma.charge_mass_ratio; + const double ionisation_potential = problem_.plasma.ionisation_potential; + const double dt = problem_.grid.dt_fine; + const double log_field_min = problem_.plasma.log_field_min; + const double d_log_field = problem_.plasma.d_log_field; + const double field_min = problem_.plasma.field_min; + const double field_max = problem_.plasma.field_max; + const double rate_max = problem_.plasma.rate_max; + const auto rate_intervals = problem_.plasma.rate_intervals; + const std::size_t channels = Multiple ? problem_.plasma.n_channels : 1; + const auto samples = static_cast(problem_.dims.n_to); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + const std::size_t sample_count = problem_.dims.n_to; +#endif + const auto half = static_cast(problem_.plasma.half_width()); + const std::size_t last = problem_.dims.n_to - 1; + const std::size_t league = band_league(node_count); + const auto column = KOKKOS_LAMBDA(const TeamMember& team) { + // CUDA extended-lambda restriction 12 requires the first capture outside + // if-constexpr blocks. + capture_anchor(node_real, node_complex, out, rates, filtered, exposures, guards, + weights, integrands, taps_view, stash, stashing, rate_coefficients, + channel_parameters, channel_coefficients, channel_density, + channel_preionisation, density, preionisation, charge_mass_ratio, + ionisation_potential, dt, log_field_min, d_log_field, field_min, + field_max, rate_max, rate_intervals, channels, samples, half, last, + first_node, stage, league, synthesis, taper); + const auto slot = static_cast(team.league_rank()); + const std::size_t own = slot * Components; + double taps[plasma_max_taps] = {}; + for (int k = 0; k <= half; ++k) + taps[k] = taps_view(static_cast(k)); + const auto tile = band_tile(team); + const double input_scale = folded ? synthesis(0, 0) * taper(stage) : 1.0; + for (std::size_t node = slot; node < node_count; node += league) { + const std::size_t base = node * Components; + const auto field_at = [&](const std::size_t row, const std::size_t time) { + if constexpr (Family == FieldFamily::field_real) + return input_scale * node_real(row, time); + else + return input_scale * node_complex(row, time).real(); + }; + const auto magnitude_at = [&](const std::size_t time) { + const double first = field_at(base, time); + if constexpr (Components == 2) + return Kokkos::hypot(first, field_at(base + 1, time)); + else + return Kokkos::abs(first); + }; + [[maybe_unused]] const auto rate_of = [&](const double magnitude, + const std::size_t channel) { + if constexpr (Multiple) + return ionisation_rate( + magnitude, channel_coefficients, channel_parameters(channel, 0), + channel_parameters(channel, 1), channel_parameters(channel, 2), + channel_parameters(channel, 3), channel_parameters(channel, 4), + static_cast(channel_parameters(channel, 5)), + static_cast(channel_parameters(channel, 6))); + else + return ionisation_rate(magnitude, rate_coefficients, log_field_min, + d_log_field, field_min, field_max, rate_max, + rate_intervals); + }; + BandLive integrand_live; + integrand_live.min_val = samples; + integrand_live.max_val = -1; + +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // With vector maths a host team thread takes blocks of samples and gathers the + // live ones' transcendentals into passes of the layer (`ionisation_rates_within`, + // the decays below), writing the rest as it goes; with the system libm each + // sample keeps its own calls, as on a device. + [[maybe_unused]] const auto blocks = + static_cast((sample_count + rate_block - 1) / rate_block); + [[maybe_unused]] const auto rates_into = + [&](const std::size_t first, const std::size_t count, + const std::size_t channel, BandLive& update) { + const double lower = Multiple ? channel_parameters(channel, 2) : field_min; + const double upper = Multiple ? channel_parameters(channel, 3) : field_max; + const double highest = Multiple ? channel_parameters(channel, 4) : rate_max; + const auto within = [&](double* values, const std::size_t live) { + if constexpr (Multiple) + ionisation_rates_within( + values, live, channel_coefficients, channel_parameters(channel, 0), + channel_parameters(channel, 1), + static_cast(channel_parameters(channel, 5)), + static_cast(channel_parameters(channel, 6))); + else + ionisation_rates_within(values, live, rate_coefficients, log_field_min, + d_log_field, rate_intervals); + }; + double magnitudes[rate_block]; + for (std::size_t k = 0; k < count; ++k) + magnitudes[k] = magnitude_at(first + k); + double* const row = &rates(slot, first); + band_rates_block(magnitudes, row, count, lower, upper, highest, within); + band_note_block(update, row, count, static_cast(first)); + }; +#endif + + // 1. Per channel: rates -> filter -> scan -> exposure, then the fraction, the + // guard and the weight pointwise. + for (std::size_t channel = 0; channel < channels; ++channel) { + BandLive rate_live; +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + if constexpr (vmath::vectorised) { + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, blocks), + [&](const int block, BandLive& update) { + const auto first = static_cast(block) * rate_block; + rates_into(first, std::min(rate_block, sample_count - first), channel, + update); + }, + Kokkos::MinMax(rate_live)); + } else +#endif + { + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, samples), + [&](const int sample, BandLive& update) { + const auto time = static_cast(sample); + const double rate = rate_of(magnitude_at(time), channel); + rates(slot, time) = rate; + band_note(update, sample, rate); + }, + Kokkos::MinMax(rate_live)); + } + team.team_barrier(); + const BandWindow window = band_window_of(rate_live, samples, half); + if (window.first <= window.last) + team_band_filter(team, tile, rates, slot, filtered, slot, + samples, taps, half, window.first, + window.last + 1); + const double total = + team_trapezoid_window(team, filtered, slot, window, samples, dt, + [&](const int sample, const double prefix) { + exposures(slot, static_cast(sample)) = + prefix; + }); + const double initial = + Multiple ? channel_preionisation(stage, channel) : preionisation(stage); + // Outside [window.first, window.last + 1] the exposure is 0 before and `total` + // after, so the fraction there is one of two values (band_support). + const int changing_end = Kokkos::min(window.last + 2, samples); + const double before = band_fraction(0.0, initial); + const double after = band_fraction(total, initial); + [[maybe_unused]] const auto fraction_at = [&](const int sample) { + if (sample < window.first) + return before; + if (sample >= changing_end) + return after; + return band_fraction(exposures(slot, static_cast(sample)), + initial); + }; +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // A host block's fractions with vector maths, each handed to `apply` with its + // sample: the changing samples' decays `exp(-max(X, 0))` in one pass, each + // fraction then `band_fraction`'s strict arithmetic. + [[maybe_unused]] const auto fractions_into = + [&](const std::size_t first, const std::size_t count, const auto& apply) { + double decay[rate_block]; + std::size_t active[rate_block]; + std::size_t live = 0; + for (std::size_t time = first; time < first + count; ++time) { + const auto sample = static_cast(time); + if (sample < window.first) { + apply(time, before); + } else if (sample >= changing_end) { + apply(time, after); + } else { + const double exposure = exposures(slot, time); + decay[live] = -(exposure < 0.0 ? 0.0 : exposure); + active[live] = time; + ++live; + } + } + vmath::exp(decay, decay, live); + for (std::size_t k = 0; k < live; ++k) + apply(active[k], decayed_fraction(decay[k], initial)); + }; + // A single gas's host block of free-current integrands and loss guards as + // loops over the block's samples: the fractions as `fractions_into` forms + // them (the changing samples' decays in one pass, `decayed_fractions` in + // another), then `band_loss_scale` and the weights with their branch taken + // as a selection, and the integrand rows' live samples searched afterwards. + [[maybe_unused]] const auto integrands_into = [&](const std::size_t first, + const std::size_t count, + BandLive& update) { + double fraction[rate_block]; + const auto start = static_cast(first); + const auto stop = static_cast(first + count); + for (std::size_t k = 0; k < count; ++k) + fraction[k] = start + static_cast(k) < window.first ? before : after; + const int begin = Kokkos::max(window.first, start); + const int end = Kokkos::min(changing_end, stop); + if (begin < end) { + double* const changing = fraction + (begin - start); + const auto changing_count = static_cast(end - begin); + const double* const exposure = + &exposures(slot, static_cast(begin)); + for (std::size_t k = 0; k < changing_count; ++k) + changing[k] = -(exposure[k] < 0.0 ? 0.0 : exposure[k]); + vmath::exp(changing, changing, changing_count); + decayed_fractions(changing, changing_count, initial); + } + const double* const rate_row = &rates(slot, first); + double* const guard_row = &guards(slot, first); + for (std::size_t k = 0; k < count; ++k) { + const std::size_t time = first + k; + const double magnitude = magnitude_at(time); + const double squared = magnitude * magnitude; + const double rate = rate_row[k]; + const bool loses = rate > 0.0 && squared > 0.0; + const double scale = ionisation_potential * rate * (1.0 - fraction[k]) / + (loses ? squared : 1.0); + guard_row[k] = loses ? scale : 0.0; + const double weight = fraction[k] * charge_mass_ratio; + for (std::size_t component = 0; component < Components; ++component) + integrands(own + component, time) = + weight * field_at(base + component, time); + } + for (std::size_t component = 0; component < Components; ++component) + band_note_block(update, &integrands(own + component, first), count, start); + }; +#endif + if constexpr (Multiple) { + const double partial = channel_density(stage, channel); + const double charge = channel_parameters(channel, 7); + const double potential = channel_parameters(channel, 8); + const auto share_at = [&](const std::size_t time, const double fraction) { + const double rate = rates(slot, time); + const double share = partial * fraction * charge; + const double paid = partial * potential * rate * (1.0 - fraction); + const double weight = channel == 0 ? 0.0 : weights(slot, time); + const double work = channel == 0 ? 0.0 : guards(slot, time); + weights(slot, time) = weight + share; + guards(slot, time) = work + paid; + }; +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + if constexpr (vmath::vectorised) { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, blocks), [&](const int block) { + const auto first = static_cast(block) * rate_block; + fractions_into(first, std::min(rate_block, sample_count - first), + share_at); + }); + } else +#endif + { + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, samples), [&](const int sample) { + share_at(static_cast(sample), fraction_at(sample)); + }); + } + } else { + const auto integrand_at = [&](const std::size_t time, const double fraction, + BandLive& update) { + const double magnitude = magnitude_at(time); + const double squared = magnitude * magnitude; + guards(slot, time) = band_loss_scale(rates(slot, time), fraction, squared, + ionisation_potential); + const double weight = fraction * charge_mass_ratio; + for (std::size_t component = 0; component < Components; ++component) { + const double value = weight * field_at(base + component, time); + integrands(own + component, time) = value; + band_note(update, static_cast(time), value); + } + }; +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + if constexpr (vmath::vectorised) { + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, blocks), + [&](const int block, BandLive& update) { + const auto first = static_cast(block) * rate_block; + integrands_into(first, std::min(rate_block, sample_count - first), + update); + }, + Kokkos::MinMax(integrand_live)); + } else +#endif + { + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, samples), + [&](const int sample, BandLive& update) { + integrand_at(static_cast(sample), fraction_at(sample), + update); + }, + Kokkos::MinMax(integrand_live)); + } + } + if (stashing) { + double largest = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, samples), + [&](const int sample, double& update) { + const double rate = rates(slot, static_cast(sample)); + if (rate > update) + update = rate; + }, + Kokkos::Max(largest)); + Kokkos::single(Kokkos::PerTeam(team), [&] { + stash(first_node + node, 2 * channel) = + band_fraction(exposures(slot, last), initial); + stash(first_node + node, 2 * channel + 1) = largest > 0.0 ? largest : 0.0; + }); + } + team.team_barrier(); + } + if constexpr (Multiple) { + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, samples), + [&](const int sample, BandLive& update) { + const auto time = static_cast(sample); + const double magnitude = magnitude_at(time); + const double squared = magnitude * magnitude; + guards(slot, time) = + band_loss_scale(1.0, 0.0, squared, guards(slot, time)); + const double weight = weights(slot, time); + for (std::size_t component = 0; component < Components; ++component) { + const double value = weight * field_at(base + component, time); + integrands(own + component, time) = value; + band_note(update, sample, value); + } + }, + Kokkos::MinMax(integrand_live)); + } + team.team_barrier(); + + // 2. Per component: the free current, and from it the current. + const BandWindow free_window = band_window_of(integrand_live, samples, half); + // A single gas's density, read once: inside the store below the compiler must + // reload it after each store to the current, which might alias it. + const double stage_density = Multiple ? 0.0 : density(stage); + for (std::size_t component = 0; component < Components; ++component) { + const std::size_t row = base + component; + if (free_window.first <= free_window.last) + team_band_filter(team, tile, integrands, own + component, + filtered, slot, samples, taps, half, + free_window.first, free_window.last + 1); + team_trapezoid_window(team, filtered, slot, free_window, samples, dt, + [&](const int sample, const double prefix) { + const auto time = static_cast(sample); + const double drive = + guards(slot, time) * field_at(row, time); + out(row, time) = Multiple + ? prefix + drive + : stage_density * (prefix + drive); + }); + } + } + }; + launch::parallel_for("solprop_native.response.team_band_plasma_column", + band_team_policy(node_count, column), column); +} + +namespace { +/** + * A band column's node field: `scale` times the real row of a field-family node (or of + * the folded single node's modal row), and its magnitude over a node's components. + */ +template struct BandField { + RealView2 real; + ComplexView2 complex; + KOKKOS_INLINE_FUNCTION double at(const std::size_t row, const std::size_t time, + const double scale) const { + if constexpr (Family == FieldFamily::field_real) + return scale * real(row, time); + else + return scale * complex(row, time).real(); + } + KOKKOS_INLINE_FUNCTION double + magnitude(const std::size_t base, const std::size_t time, const double scale) const { + const double first = at(base, time, scale); + if constexpr (Components == 2) + return Kokkos::hypot(first, at(base + 1, time, scale)); + else + return Kokkos::abs(first); + } +}; + +/** A band column's ionisation rate: the single gas's table, or a mixture channel's. */ +template struct BandRate { + RealView coefficients; + RealView2 parameters; + RealView channel_coefficients; + double log_field_min{}, d_log_field{}, field_min{}, field_max{}, rate_max{}; + std::size_t intervals{}; + KOKKOS_INLINE_FUNCTION double operator()(const double magnitude, + const std::size_t channel) const { + if constexpr (Multiple) + return ionisation_rate(magnitude, channel_coefficients, parameters(channel, 0), + parameters(channel, 1), parameters(channel, 2), + parameters(channel, 3), parameters(channel, 4), + static_cast(parameters(channel, 5)), + static_cast(parameters(channel, 6))); + else + return ionisation_rate(magnitude, coefficients, log_field_min, d_log_field, + field_min, field_max, rate_max, intervals); + } +}; +} // namespace + +template +Kokkos::TeamPolicy +GeneralWorkspace::tiled_band_policy(const std::size_t rows, const bool scratch, + const Functor& functor) const { + using Policy = Kokkos::TeamPolicy; + constexpr auto policy_limit = + static_cast(std::numeric_limits::max()); + if (band_tile_count_ == 0 || rows > policy_limit / band_tile_count_ || + problem_.dims.n_to > policy_limit) + throw std::length_error("tiled band column extent exceeds signed 32-bit range"); + const auto league = static_cast(rows * band_tile_count_); + const auto bytes = scratch ? band_tile_bytes(band_tile_) : 0; + Policy probe(response_execution_space_, league, 1); + probe.set_scratch_size(0, Kokkos::PerTeam(bytes)); + const int widest = probe.team_size_max(functor, Kokkos::ParallelForTag()); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const auto requested = static_cast(std::min(response_team_size(), band_tile_)); +#else + const int requested = std::max(1, band_tile_threads_); +#endif + Policy policy(response_execution_space_, league, + std::max(1, std::min(requested, widest))); + policy.set_scratch_size(0, Kokkos::PerTeam(bytes)); + return policy; +} + +// The band column in tiles (solprop SEP 25 section 6.7.6; +// include/solprop_native/plasma.hpp): each running integral is a local kernel of one +// team per (row, tile), which loads, filters and scans its tile from zero +// (`team_band_tile`), then a finish kernel of one team per (row, tile), which adds the +// row's earlier tile totals (`team_tile_offset`) and forms the pass's product from the +// prefix, in the team column's arithmetic: +// +// 1. per channel: the local kernel forms the rates in its window (keeping its own +// tile's in `plasma_rate_`) and the exposure's local prefixes +// (`plasma_exposure_`); the finish forms the fraction, the loss guard +// (`plasma_free_current_`) and the free-current integrand +// (`plasma_current_integrand_`), a mixture accumulating its weight +// (`plasma_outer_current_`) and work channel by channel; +// 2. per component row: the integrand's local prefixes (`plasma_outer_current_`, +// free after the first pass), then the current (`held_current_`). +// +// Rows are by node within the chunk, the totals by row (`band_tile_totals_`). The +// stash's fraction comes from the last sample's finish and its largest rate from a +// reduction over the rate row. No window is skipped: every sample is formed. +template +void GeneralWorkspace::tiled_band_plasma_column(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count) { + if constexpr (Components == 1 && !Folded) { + if (folds_single_node()) { + tiled_band_plasma_column(stage, first_node, + node_count); + return; + } + } + static_assert(Family != FieldFamily::envelope, + "the band rule integrates a field-grid plasma column"); + using TeamMember = Kokkos::TeamPolicy::member_type; + constexpr bool folded = Folded; + const BandField field{folded ? modal_real_ : node_real_, + folded ? modal_complex_ : node_complex_}; + const BandRate rate{rate_coefficients_, + channel_parameters_, + channel_coefficients_, + problem_.plasma.log_field_min, + problem_.plasma.d_log_field, + problem_.plasma.field_min, + problem_.plasma.field_max, + problem_.plasma.rate_max, + problem_.plasma.rate_intervals}; + const auto synthesis = synthesis_; + const auto taper = stage_taper_; + const auto out = held_current_; + const auto rates = plasma_rate_; + const auto exposures = plasma_exposure_; + const auto guards = plasma_free_current_; + const auto weights = plasma_outer_current_; + const auto locals = plasma_outer_current_; + const auto integrands = plasma_current_integrand_; + const auto totals = band_tile_totals_; + const auto taps_view = plasma_taps_; + const auto stash = stash_candidate_; + const bool stashing = !stash.empty() && stage + 1 == problem_.stepper.n_stages; + const auto channel_parameters = channel_parameters_; + const auto channel_density = stage_channel_density_; + const auto channel_preionisation = stage_channel_preionisation_; + const auto density = stage_plasma_density_; + const auto preionisation = stage_preionisation_; + const double charge_mass_ratio = problem_.plasma.charge_mass_ratio; + const double ionisation_potential = problem_.plasma.ionisation_potential; + const double dt = problem_.grid.dt_fine; + const std::size_t channels = Multiple ? problem_.plasma.n_channels : 1; + const auto samples = static_cast(problem_.dims.n_to); + const auto half = static_cast(problem_.plasma.half_width()); + const auto width = static_cast(band_tile_); + const auto tiles = static_cast(band_tile_count_); + const int last = samples - 1; + const std::size_t rows = node_count * Components; + + // 1. Per channel: the exposure, then the fraction, the guard and the integrand. + for (std::size_t channel = 0; channel < channels; ++channel) { + const auto local = KOKKOS_LAMBDA(const TeamMember& team) { + capture_anchor(field, rate, rates, exposures, totals, taps_view, synthesis, taper, + stage, channel, samples, half, width, tiles, dt); + const auto node = static_cast(team.league_rank() / tiles); + const int index = team.league_rank() % tiles; + const int first = index * width; + const int end = Kokkos::min(first + width, samples); + double taps[plasma_max_taps] = {}; + for (int k = 0; k <= half; ++k) + taps[k] = taps_view(static_cast(k)); + const double scale = folded ? synthesis(0, 0) * taper(stage) : 1.0; + const std::size_t base = node * Components; + const double total = team_band_tile( + team, + [&](const int sample) { + const auto time = static_cast(sample); + const double value = rate(field.magnitude(base, time, scale), channel); + if (sample >= first && sample < end) + rates(node, time) = value; + return value; + }, + first, end, samples, taps, half, dt, + [&](const int sample, const double prefix) { + exposures(node, static_cast(sample)) = prefix; + }); + Kokkos::single(Kokkos::PerTeam(team), + [&] { totals(node, static_cast(index)) = total; }); + }; + launch::parallel_for("solprop_native.response.tiled_band_exposure", + tiled_band_policy(node_count, true, local), local); + const auto finish = KOKKOS_LAMBDA(const TeamMember& team) { + capture_anchor(field, rates, exposures, guards, weights, integrands, totals, + stash, stashing, synthesis, taper, channel_parameters, + channel_density, channel_preionisation, preionisation, + charge_mass_ratio, ionisation_potential, channels, channel, + samples, width, tiles, last, first_node, stage); + const auto node = static_cast(team.league_rank() / tiles); + const int index = team.league_rank() % tiles; + const int first = index * width; + const int end = Kokkos::min(first + width, samples); + const double offset = team_tile_offset(team, totals, node, index); + const double scale = folded ? synthesis(0, 0) * taper(stage) : 1.0; + const std::size_t base = node * Components; + const double initial = + Multiple ? channel_preionisation(stage, channel) : preionisation(stage); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, first, end), [&](const int sample) { + const auto time = static_cast(sample); + const double fraction = + band_fraction(offset + exposures(node, time), initial); + const double value = rates(node, time); + if constexpr (Multiple) { + const double partial = channel_density(stage, channel); + const double charge = channel_parameters(channel, 7); + const double potential = channel_parameters(channel, 8); + const double share = partial * fraction * charge; + const double paid = partial * potential * value * (1.0 - fraction); + const double weight = (channel == 0 ? 0.0 : weights(node, time)) + share; + const double work = (channel == 0 ? 0.0 : guards(node, time)) + paid; + if (channel + 1 == channels) { + const double magnitude = field.magnitude(base, time, scale); + guards(node, time) = + band_loss_scale(1.0, 0.0, magnitude * magnitude, work); + for (std::size_t component = 0; component < Components; ++component) + integrands(base + component, time) = + weight * field.at(base + component, time, scale); + } else { + weights(node, time) = weight; + guards(node, time) = work; + } + } else { + const double magnitude = field.magnitude(base, time, scale); + guards(node, time) = band_loss_scale( + value, fraction, magnitude * magnitude, ionisation_potential); + const double weight = fraction * charge_mass_ratio; + for (std::size_t component = 0; component < Components; ++component) + integrands(base + component, time) = + weight * field.at(base + component, time, scale); + } + if (stashing && sample == last) + stash(first_node + node, 2 * channel) = fraction; + }); + }; + launch::parallel_for("solprop_native.response.tiled_band_fraction", + tiled_band_policy(node_count, false, finish), finish); + if (stashing) { + const auto peak = KOKKOS_LAMBDA(const TeamMember& team) { + capture_anchor(rates, stash, channel, samples, first_node); + const auto node = static_cast(team.league_rank()); + double largest = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, samples), + [&](const int sample, double& update) { + const double value = rates(node, static_cast(sample)); + if (value > update) + update = value; + }, + Kokkos::Max(largest)); + Kokkos::single(Kokkos::PerTeam(team), [&] { + stash(first_node + node, 2 * channel + 1) = largest > 0.0 ? largest : 0.0; + }); + }; + launch::parallel_for("solprop_native.response.tiled_band_peak_rate", + Kokkos::TeamPolicy( + response_space_for(node_count), + static_cast(node_count), Kokkos::AUTO()), + peak); + } + } + + // 2. Per component row: the free current, followed by the pointwise loss current. + tiled_band_local(integrands, rows, "solprop_native.response.tiled_band_free_current"); + const auto current = KOKKOS_LAMBDA(const TeamMember& team) { + capture_anchor(field, guards, locals, out, density, totals, synthesis, taper, stage, + samples, width, tiles); + const auto row = static_cast(team.league_rank() / tiles); + const int index = team.league_rank() % tiles; + const int first = index * width; + const int end = Kokkos::min(first + width, samples); + const double offset = team_tile_offset(team, totals, row, index); + const double scale = folded ? synthesis(0, 0) * taper(stage) : 1.0; + const std::size_t node = row / Components; + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, first, end), [&](const int sample) { + const auto time = static_cast(sample); + const double prefix = offset + locals(row, time); + const double drive = guards(node, time) * field.at(row, time, scale); + out(row, time) = + Multiple ? prefix + drive : density(stage) * (prefix + drive); + }); + }; + launch::parallel_for("solprop_native.response.tiled_band_current", + tiled_band_policy(rows, false, current), current); +} + +// The tiled column's local kernel over component rows: `source`'s band-filtered tile +// prefixes from zero into the free-current rows (`plasma_outer_current_`), +// and each tile's total into `band_tile_totals_`. +void GeneralWorkspace::tiled_band_local(const RealView2& source, const std::size_t rows, + const char* label) { + using TeamMember = Kokkos::TeamPolicy::member_type; + const auto locals = plasma_outer_current_; + const auto totals = band_tile_totals_; + const auto taps_view = plasma_taps_; + const double dt = problem_.grid.dt_fine; + const auto samples = static_cast(problem_.dims.n_to); + const auto half = static_cast(problem_.plasma.half_width()); + const auto width = static_cast(band_tile_); + const auto tiles = static_cast(band_tile_count_); + const auto local = KOKKOS_LAMBDA(const TeamMember& team) { + const auto row = static_cast(team.league_rank() / tiles); + const int index = team.league_rank() % tiles; + const int first = index * width; + const int end = Kokkos::min(first + width, samples); + double taps[plasma_max_taps] = {}; + for (int k = 0; k <= half; ++k) + taps[k] = taps_view(static_cast(k)); + const double total = team_band_tile( + team, + [&](const int sample) { return source(row, static_cast(sample)); }, + first, end, samples, taps, half, dt, + [&](const int sample, const double prefix) { + locals(row, static_cast(sample)) = prefix; + }); + Kokkos::single(Kokkos::PerTeam(team), + [&] { totals(row, static_cast(index)) = total; }); + }; + launch::parallel_for(label, tiled_band_policy(rows, true, local), local); +} + +// The band rule's statistics scan: per channel the rates, their band filter and +// trapezoid scan, then team reductions over the rows. The floored final exposure and +// the largest rate always; with `Diagnose` the six parts of SEP 18's run diagnostics, +// the largest over the channels: the exposure's largest dip and rise (maxima of the +// scan's increments), the largest |paid - freed| and freed (the paid work a sum of the +// trapezoid's increments of the raw rate's owed work), and the largest |f E| in the +// first 2K samples and anywhere. Maxima combine exactly; the sums associate as the +// team's scan does. The rows are the band column's (`plasma_rate_`, +// `plasma_fraction_` filtered and then owed work, `plasma_exposure_` exposure and then +// fraction), by slot as the column walks the nodes. Not a step's hot path: it runs +// whole rows, without the column's live windows. +template +void GeneralWorkspace::team_band_statistics(const std::size_t first_node, + const std::size_t node_count, + const RealView2& output, + const double preionisation) { + using TeamMember = Kokkos::TeamPolicy::member_type; + const auto node_real = node_real_; + const auto node_complex = node_complex_; + const bool real_family = problem_.dims.family == FieldFamily::field_real; + const auto rates = plasma_rate_; + const auto filtered = plasma_fraction_; + const auto exposures = plasma_exposure_; + const auto taps_view = plasma_taps_; + const auto diagnostics = plasma_diagnostics_; + const auto coefficients = rate_coefficients_; + const auto parameters = channel_parameters_; + const auto channel_coefficients = channel_coefficients_; + const auto channel_preionisation = stage_channel_preionisation_; + const auto channels = problem_.plasma.n_channels; + const bool multiple = channels > 1; + const auto n_pol = problem_.effective_node_pol(); + const auto samples = static_cast(problem_.dims.n_to); + const auto intervals = problem_.plasma.rate_intervals; + const auto dt = problem_.grid.dt_fine; + const auto log_field_min = problem_.plasma.log_field_min; + const auto d_log_field = problem_.plasma.d_log_field; + const auto field_min = problem_.plasma.field_min; + const auto field_max = problem_.plasma.field_max; + const auto rate_max = problem_.plasma.rate_max; + const auto half = static_cast(problem_.plasma.half_width()); + const int reach = Kokkos::min(2 * half, samples); + const std::size_t last = problem_.dims.n_to - 1; + const std::size_t league = band_league(node_count); + const auto scan = KOKKOS_LAMBDA(const TeamMember& team) { + capture_anchor(node_real, node_complex, real_family, rates, filtered, exposures, + taps_view, diagnostics, coefficients, parameters, + channel_coefficients, channel_preionisation, channels, multiple, + n_pol, samples, intervals, dt, log_field_min, d_log_field, field_min, + field_max, rate_max, half, reach, last, first_node, output, + preionisation, league); + const auto slot = static_cast(team.league_rank()); + double taps[plasma_max_taps] = {}; + for (int k = 0; k <= half; ++k) + taps[k] = taps_view(static_cast(k)); + const auto tile = band_tile(team); + for (std::size_t node = slot; node < node_count; node += league) { + const auto base = node * n_pol; + const auto field_at = [&](const std::size_t row, const std::size_t time) { + return real_family ? node_real(row, time) : node_complex(row, time).real(); + }; + double parts[plasma_diagnostic_parts] = {}; + for (std::size_t channel = 0; channel < channels; ++channel) { + const double initial = + multiple ? channel_preionisation(0, channel) : preionisation; +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // With vector maths a host thread forms blocks of rates as the column does + // (`band_rates_block`). + if constexpr (vmath::vectorised) { + const double lower = multiple ? parameters(channel, 2) : field_min; + const double upper = multiple ? parameters(channel, 3) : field_max; + const double highest = multiple ? parameters(channel, 4) : rate_max; + const auto within = [&](double* values, const std::size_t live) { + if (multiple) + ionisation_rates_within(values, live, channel_coefficients, + parameters(channel, 0), parameters(channel, 1), + static_cast(parameters(channel, 5)), + static_cast(parameters(channel, 6))); + else + ionisation_rates_within(values, live, coefficients, log_field_min, + d_log_field, intervals); + }; + const auto sample_count = static_cast(samples); + const auto blocks = + static_cast((sample_count + rate_block - 1) / rate_block); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, blocks), [&](const int block) { + const auto start = static_cast(block) * rate_block; + const auto count = std::min(rate_block, sample_count - start); + double magnitudes[rate_block]; + for (std::size_t k = 0; k < count; ++k) { + const double first = field_at(base, start + k); + magnitudes[k] = + n_pol == 2 ? Kokkos::hypot(first, field_at(base + 1, start + k)) + : Kokkos::abs(first); + } + band_rates_block(magnitudes, &rates(slot, start), count, lower, upper, + highest, within); + }); + } else { +#endif + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, samples), [&](const int sample) { + const auto time = static_cast(sample); + const double first = field_at(base, time); + const double magnitude = + n_pol == 2 ? Kokkos::hypot(first, field_at(base + 1, time)) + : Kokkos::abs(first); + rates(slot, time) = + multiple + ? ionisation_rate( + magnitude, channel_coefficients, parameters(channel, 0), + parameters(channel, 1), parameters(channel, 2), + parameters(channel, 3), parameters(channel, 4), + static_cast(parameters(channel, 5)), + static_cast(parameters(channel, 6))) + : ionisation_rate(magnitude, coefficients, log_field_min, + d_log_field, field_min, field_max, rate_max, + intervals); + }); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + } +#endif + team.team_barrier(); + team_band_filter(team, tile, rates, slot, filtered, slot, samples, + taps, half); + team_trapezoid_prefix(team, filtered, slot, samples, dt, + [&](const int sample, const double prefix) { + exposures(slot, static_cast(sample)) = + prefix; + }); + double largest = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, samples), + [&](const int sample, double& update) { + const double rate = rates(slot, static_cast(sample)); + if (rate > update) + update = rate; + }, + Kokkos::Max(largest)); + const double exposure = exposures(slot, last); + Kokkos::single(Kokkos::PerTeam(team), [&] { + output(first_node + node, 2 * channel) = exposure > 0.0 ? exposure : 0.0; + output(first_node + node, 2 * channel + 1) = largest > 0.0 ? largest : 0.0; + }); + if constexpr (Diagnose) { + // The exposure's increments, as the scan formed them. + const auto increment = [&](const int sample) { + const auto time = static_cast(sample); + const double current = filtered(slot, time); + const double previous = filtered(slot, time - 1); + return dt * (current + previous) / 2.0; + }; + double dip = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, 1, samples), + [&](const int sample, double& update) { + const double fall = -increment(sample); + if (fall > update) + update = fall; + }, + Kokkos::Max(dip)); + double rise = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, 1, samples), + [&](const int sample, double& update) { + const double step = increment(sample); + if (step > update) + update = step; + }, + Kokkos::Max(rise)); + team.team_barrier(); + // The filtered row becomes the owed work, the exposure row the fraction. +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // With vector maths a host thread takes blocks: the decays in one pass, + // then `decayed_fractions`, as the column forms them. + if constexpr (vmath::vectorised) { + const auto sample_count = static_cast(samples); + const auto blocks = + static_cast((sample_count + rate_block - 1) / rate_block); + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, blocks), [&](const int block) { + const auto start = static_cast(block) * rate_block; + const auto count = std::min(rate_block, sample_count - start); + double* const exposed = &exposures(slot, start); + double* const owed = &filtered(slot, start); + const double* const rate = &rates(slot, start); + double fraction[rate_block]; + for (std::size_t k = 0; k < count; ++k) + fraction[k] = -(exposed[k] < 0.0 ? 0.0 : exposed[k]); + vmath::exp(fraction, fraction, count); + decayed_fractions(fraction, count, initial); + for (std::size_t k = 0; k < count; ++k) { + owed[k] = rate[k] * (1.0 - fraction[k]); + exposed[k] = fraction[k]; + } + }); + } else { +#endif + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, samples), [&](const int sample) { + const auto time = static_cast(sample); + const double fraction = band_fraction(exposures(slot, time), initial); + filtered(slot, time) = rates(slot, time) * (1.0 - fraction); + exposures(slot, time) = fraction; + }); +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + } +#endif + team.team_barrier(); + double paid = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, 1, samples), + [&](const int sample, double& sum) { sum += increment(sample); }, paid); + const auto level = [&](const int sample, double& update) { + const auto time = static_cast(sample); + const double fraction = exposures(slot, time); + for (std::size_t component = 0; component < n_pol; ++component) { + const double value = + Kokkos::abs(fraction * field_at(base + component, time)); + if (value > update) + update = value; + } + }; + double early = 0.0; + Kokkos::parallel_reduce(Kokkos::TeamThreadRange(team, reach), level, + Kokkos::Max(early)); + double peak = 0.0; + Kokkos::parallel_reduce(Kokkos::TeamThreadRange(team, samples), level, + Kokkos::Max(peak)); + const double freed = exposures(slot, last) - exposures(slot, 0); + const double values[plasma_diagnostic_parts] = { + dip > 0.0 ? dip : 0.0, rise > 0.0 ? rise : 0.0, + Kokkos::abs(paid - freed), freed, + early > 0.0 ? early : 0.0, peak > 0.0 ? peak : 0.0}; + for (std::size_t part = 0; part < plasma_diagnostic_parts; ++part) + parts[part] = Kokkos::max(parts[part], values[part]); + } + team.team_barrier(); + } + if constexpr (Diagnose) + Kokkos::single(Kokkos::PerTeam(team), [&] { + for (std::size_t part = 0; part < plasma_diagnostic_parts; ++part) + diagnostics(first_node + node, part) = parts[part]; + }); + team.team_barrier(); + } + }; + launch::parallel_for("solprop_native.plasma.team_band_statistics", + band_team_policy(node_count, scan), scan); +} + +void GeneralWorkspace::convolve_raman(const std::size_t stage, + const std::size_t node_count) { + const bool tail = node_count != problem_.dims.node_chunk; + Kokkos::Profiling::pushRegion("solprop_native.raman.drive_fft"); + execute_raman(tail ? *raman_tail_forward_plan_ : *raman_drive_forward_plan_, + tail ? raman_tail_drive_ : node_raman_drive_, + tail ? raman_tail_spectrum_ : node_raman_spectrum_, + KokkosFFT::Normalization::none); + Kokkos::Profiling::popRegion(); + + Kokkos::Profiling::pushRegion("solprop_native.raman.spectral_multiply"); + const auto node_spectrum = node_raman_spectrum_; + const auto kernel_spectrum = raman_kernel_spectrum_; + const auto drives = problem_.raman_drives(); + const double inverse_doubled = 1.0 / static_cast(2 * problem_.dims.n_to); + launch::parallel_for( + "solprop_native.raman.spectral_multiply", + Range2(tail ? raman_tail_multiply_space_ : raman_multiply_space_, {0, 0}, + {node_count * drives, problem_.dims.n_to + 1}), + KOKKOS_LAMBDA(const std::size_t node, const std::size_t index) { + node_spectrum(node, index) = + (node_spectrum(node, index) * + kernel_spectrum(node % drives == 0 ? 0 : 1, index)) * + inverse_doubled; + }); + Kokkos::Profiling::popRegion(); + + Kokkos::Profiling::pushRegion("solprop_native.raman.inverse_fft"); + execute_raman(tail ? *raman_tail_inverse_plan_ : *raman_convolution_inverse_plan_, + tail ? raman_tail_spectrum_ : node_raman_spectrum_, + tail ? raman_tail_convolved_ : node_raman_convolved_, + KokkosFFT::Normalization::none); + Kokkos::Profiling::popRegion(); + finish_raman(stage, node_count); +} + +void GeneralWorkspace::finish_raman(const std::size_t stage, + const std::size_t node_count) { + Kokkos::Profiling::pushRegion("solprop_native.raman.finish"); + const auto node_real = node_real_; + const auto node_complex = node_complex_; + const auto out_real = polarisation_real_; + const auto out_complex = polarisation_complex_; + const auto convolved = node_raman_convolved_; + const auto window = time_window_; + const auto density = stage_raman_density_; + const auto family = problem_.dims.family; + const auto components = problem_.effective_node_pol(); + const auto drives = problem_.raman_drives(); + const double epsilon0 = problem_.raman.epsilon0; + launch::parallel_for( + "solprop_native.raman.finish", + Range2(response_space_for(node_count), {0, 0}, {node_count, problem_.dims.n_to}), + KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { + if (components == 2) { + const auto base = 2 * node; + const auto row = drives * node; + const double u = convolved(row, time); + const double v = drives == 3 ? convolved(row + 1, time) : 0.0; + const double w = drives == 3 ? convolved(row + 2, time) : 0.0; + const double factor = epsilon0 * density(stage); + if (family == FieldFamily::envelope) { + const Complex x = node_complex(base, time); + const Complex y = node_complex(base + 1, time); + out_complex(base, time) = + (out_complex(base, time) + factor * ((u + v) * x + w * y)) * + window(time); + out_complex(base + 1, time) = + (out_complex(base + 1, time) + factor * (w * x + (u - v) * y)) * + window(time); + } else { + const double x = family == FieldFamily::field_real + ? node_real(base, time) + : node_complex(base, time).real(); + const double y = family == FieldFamily::field_real + ? node_real(base + 1, time) + : node_complex(base + 1, time).real(); + out_real(base, time) = + (out_real(base, time) + factor * ((u + v) * x + w * y)) * window(time); + out_real(base + 1, time) = + (out_real(base + 1, time) + factor * (w * x + (u - v) * y)) * + window(time); + } + return; + } + const double factor = epsilon0 * density(stage) * convolved(node, time); + if (family == FieldFamily::envelope) { + out_complex(node, time) = + (out_complex(node, time) + factor * node_complex(node, time)) * + window(time); + } else { + const double field = family == FieldFamily::field_real + ? node_real(node, time) + : node_complex(node, time).real(); + out_real(node, time) = (out_real(node, time) + factor * field) * window(time); + } + }); + Kokkos::Profiling::popRegion(); +} + +// The modal-product convolution (after the CPU experiment's Raman port). With a real +// static synthesis S, a scalar node field is x_j = sum_m S_jm a_m, so the Raman drive +// x_j^2 (or the cycle average |x_j|^2 / 2) is sum_{m <= n} c_mn S_jm S_jn P_mn with +// P_mn = Re(a_m) Re(a_n) (or (Re(a_m conj a_n)) / 2) and c_mn = 2 off the diagonal. +// The causal convolution is linear: convolving each of the M (M + 1) / 2 products once +// and reconstructing each node's convolved drive from them replaces one drive per node. +// It is chosen where it transforms fewer rows than the nodes and its rows fit the node +// drive's own scratch; the stage-dependent kernel, its support and the taper (inside +// S) are those of the node path. The reconstruction sums the products in a different +// order from squaring the synthesised field: the two agree to rounding of the drive's +// scale, sum_m |S_jm a_m| squared. Host builds only: a device keeps the node drives. +std::size_t GeneralWorkspace::raman_modal_products() const noexcept { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + return 0; +#else + if (!problem_.raman.enabled || problem_.general.enabled || + problem_.dispersive.enabled || problem_.separable_geometry() || + problem_.effective_node_pol() != 1 || problem_.dims.n_pol_state != 1 || + problem_.dims.n_pol_transform != 1 || problem_.raman_drives() != 1 || + problem_.dims.n_nodes < 2) + return 0; + const auto products = problem_.dims.n_modes * (problem_.dims.n_modes + 1) / 2; + return products < problem_.dims.n_nodes && products <= problem_.dims.node_chunk + ? products + : 0; +#endif +} + +void GeneralWorkspace::convolve_raman_products() { + Kokkos::Profiling::pushRegion("solprop_native.raman.products"); + const auto drive = raman_product_drive_; + const auto modal_real = modal_real_; + const auto modal_complex = modal_complex_; + const auto modes = problem_.dims.n_modes; + const auto n_to = problem_.dims.n_to; + const bool real_family = problem_.dims.family == FieldFamily::field_real; + const bool squared = problem_.raman.thg; + launch::parallel_for( + "solprop_native.raman.products", + Range2(raman_product_space_, {0, 0}, {raman_products_, 2 * n_to}), + KOKKOS_LAMBDA(const std::size_t product, const std::size_t time) { + if (time >= n_to) { + drive(product, time) = 0.0; + return; + } + // The unordered pair (first, second), first <= second, in row-major order. + std::size_t first = 0; + std::size_t second = product; + while (second >= modes - first) { + second -= modes - first; + ++first; + } + second += first; + const double multiplicity = first == second ? 1.0 : 2.0; + double value; + if (real_family) { + value = modal_real(first, time) * modal_real(second, time); + if (!squared) + value *= 0.5; + } else { + const Complex a = modal_complex(first, time); + const Complex b = modal_complex(second, time); + value = squared ? a.real() * b.real() + : 0.5 * (a.real() * b.real() + a.imag() * b.imag()); + } + drive(product, time) = multiplicity * value; + }); + execute_raman(*raman_product_forward_plan_, raman_product_drive_, + raman_product_spectrum_, KokkosFFT::Normalization::none); + const auto spectrum = raman_product_spectrum_; + const auto kernel_spectrum = raman_kernel_spectrum_; + const double inverse_doubled = 1.0 / static_cast(2 * n_to); + launch::parallel_for( + "solprop_native.raman.products_multiply", + Range2(raman_product_space_, {0, 0}, {raman_products_, n_to + 1}), + KOKKOS_LAMBDA(const std::size_t product, const std::size_t index) { + spectrum(product, index) = + (spectrum(product, index) * kernel_spectrum(0, index)) * inverse_doubled; + }); + execute_raman(*raman_product_inverse_plan_, raman_product_spectrum_, + raman_product_drive_, KokkosFFT::Normalization::none); + Kokkos::Profiling::popRegion(); +} + +// `finish_raman`'s scalar form with each node's convolved drive reconstructed from +// the convolved modal products. +void GeneralWorkspace::finish_raman_products(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count) { + Kokkos::Profiling::pushRegion("solprop_native.raman.finish_products"); + const auto node_real = node_real_; + const auto node_complex = node_complex_; + const auto out_real = polarisation_real_; + const auto out_complex = polarisation_complex_; + const auto convolved = raman_product_drive_; + const auto synthesis = scaled_synthesis_; + const auto window = time_window_; + const auto density = stage_raman_density_; + const auto family = problem_.dims.family; + const auto modes = problem_.dims.n_modes; + const double epsilon0 = problem_.raman.epsilon0; + const auto n_to = problem_.dims.n_to; + // Blocks of samples per node, the products accumulated in their order over each + // block as contiguous rows: one pass over the block per product, which vectorises, + // where a per-sample loop over the products strides between rows. + constexpr std::size_t block = 512; + const std::size_t blocks = (n_to + block - 1) / block; + const double* const products = convolved.data(); + const std::size_t stride = convolved.stride(0); + launch::parallel_for( + "solprop_native.raman.finish_products", + Kokkos::RangePolicy(response_space_for(node_count), 0, + node_count * blocks), + KOKKOS_LAMBDA(const std::size_t item) { + const std::size_t node = item / blocks; + const std::size_t begin = (item % blocks) * block; + const std::size_t count = n_to - begin < block ? n_to - begin : block; + const std::size_t row = first_node + node; + double drive[block]; + for (std::size_t offset = 0; offset < count; ++offset) + drive[offset] = 0.0; + std::size_t product = 0; + for (std::size_t first = 0; first < modes; ++first) { + const double weight = synthesis(row, first); + for (std::size_t second = first; second < modes; ++second, ++product) { + const double pair = weight * synthesis(row, second); + const double* const values = products + product * stride + begin; + for (std::size_t offset = 0; offset < count; ++offset) + drive[offset] += pair * values[offset]; + } + } + const double scale = epsilon0 * density(stage); + for (std::size_t offset = 0; offset < count; ++offset) { + const std::size_t time = begin + offset; + const double factor = scale * drive[offset]; + if (family == FieldFamily::envelope) { + out_complex(node, time) = + (out_complex(node, time) + factor * node_complex(node, time)) * + window(time); + } else { + const double field = family == FieldFamily::field_real + ? node_real(node, time) + : node_complex(node, time).real(); + out_real(node, time) = + (out_real(node, time) + factor * field) * window(time); + } + } + }); + Kokkos::Profiling::popRegion(); +} + +// One channel's cycle-averaged rates at each node and their running integral, from +// the polarisation ellipse of `field_rows`' `n_pol` rows per node: the node buffer +// with its components, or an envelope's on-axis row (`axis_populations`). +void GeneralWorkspace::envelope_cycle(const std::size_t channel, + const std::size_t node_count, + const ComplexView2& field_rows, + const std::size_t n_pol) { + // The kernels capture the view by value, never the caller's reference, which a + // device cannot read. + // NOLINTNEXTLINE(performance-unnecessary-copy-initialization) + const auto field = field_rows; + const auto parameters = channel_parameters_; + const auto coefficients = channel_coefficients_; + const auto nodes = + Kokkos::subview(envelope_rule_nodes_, channel, Kokkos::ALL(), Kokkos::ALL()); + const auto weights = + Kokkos::subview(envelope_rule_weights_, channel, Kokkos::ALL(), Kokkos::ALL()); + const auto moments = envelope_moments_; + const auto rates = envelope_rates_; + const auto failures = envelope_failures_; + const auto n_to = problem_.dims.n_to; + launch::parallel_for( + "solprop_native.envelope.cycle_moments", + Range2(response_space_for(node_count), {0, 0}, {node_count, n_to}), + KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { + const auto axes = + envelope_axes(field(node * n_pol, time), + n_pol == 2 ? field(node * n_pol + 1, time) : Complex{}); + const auto rate_at = [&](const double value) { + return ionisation_rate(value, coefficients, parameters(channel, 0), + parameters(channel, 1), parameters(channel, 2), + parameters(channel, 3), parameters(channel, 4), + static_cast(parameters(channel, 5)), + static_cast(parameters(channel, 6))); + }; + const auto pair = + envelope_moments(axes.major, axes.ratio, parameters(channel, 2), + parameters(channel, 3), rate_at, nodes, weights); + moments(0, node, time) = pair.cosine; + moments(1, node, time) = pair.sine; + rates(node, time) = pair.cosine + axes.ratio * axes.ratio * pair.sine; + failures(node * n_to + time) = pair.failed ? 1 : 0; + }); + const auto status = envelope_status_; + std::size_t no_failure = 0; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // A serial device scan dominated the short combined Raman/plasma profiles. + // Integer minimum preserves the first failing sample exactly; + // its identity denotes success without an additional normalization launch. + no_failure = std::numeric_limits::max(); + using StatusPolicy = Kokkos::TeamPolicy; + using StatusMember = StatusPolicy::member_type; + const auto samples = node_count * n_to; + // One team uses only launch-time shared memory; a global reduction could grow + // the execution-space scratch allocator inside the first trial. + launch::parallel_for( + "solprop_native.envelope.angular_status", + StatusPolicy(response_space_for(node_count), 1, + SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE), + KOKKOS_LAMBDA(const StatusMember& team) { + std::size_t first_failure = Kokkos::reduction_identity::min(); + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, samples), + [&](const std::size_t sample, std::size_t& minimum) { + if (failures(sample) != 0) + minimum = Kokkos::min(minimum, sample + 1); + }, + Kokkos::Min(first_failure)); + Kokkos::single(Kokkos::PerTeam(team), [&]() { status(0) = first_failure; }); + }); +#else + launch::parallel_for( + "solprop_native.envelope.angular_status", + Kokkos::RangePolicy(response_space_for(node_count), 0, 1), + KOKKOS_LAMBDA(const std::size_t) { + status(0) = 0; + for (std::size_t sample = 0; sample < node_count * n_to; ++sample) + if (failures(sample) != 0) { + status(0) = sample + 1; + break; + } + }); +#endif + // This planned scalar boundary rejects failed refinement before a current FFT. + graph::deep_copy(response_space_for(node_count), envelope_host_status_, + envelope_status_); + response_space_for(node_count).fence("solprop_native envelope refinement boundary"); + if (envelope_host_status_(0) != no_failure) + refuse("envelope plasma channel ", channel, + " exceeded the angular refinement budget"); + const auto rows = std::make_pair(std::size_t{0}, node_count); + portable_team_trapezoid_scan( + response_space_for(node_count), + Kokkos::subview(envelope_rates_, rows, Kokkos::ALL()), + Kokkos::subview(envelope_exposure_, rows, Kokkos::ALL()), problem_.grid.dt_fine); +} + +// One channel's exposure and peak cycle-averaged rate at each node of `field` +// (`envelope_cycle`), into the rows of `statistics` from `first_node`. +void GeneralWorkspace::envelope_statistics(const std::size_t channel, + const std::size_t first_node, + const std::size_t node_count, + const RealView2& statistics, + const ComplexView2& field, + const std::size_t n_pol) { + envelope_cycle(channel, node_count, field, n_pol); + const auto rates = envelope_rates_; + const auto exposure = envelope_exposure_; + const auto n_to = problem_.dims.n_to; + launch::parallel_for( + "solprop_native.envelope.population_statistics", + Kokkos::RangePolicy(response_space_for(node_count), 0, + node_count), + KOKKOS_LAMBDA(const std::size_t node) { + double peak = 0.0; + for (std::size_t time = 0; time < n_to; ++time) + peak = Kokkos::max(peak, rates(node, time)); + statistics(first_node + node, 2 * channel) = exposure(node, n_to - 1); + statistics(first_node + node, 2 * channel + 1) = peak; + }); +} + +void GeneralWorkspace::envelope_population(const std::size_t channel, + const std::size_t stage, + const std::size_t node_count) { + const auto n_pol = problem_.effective_node_pol(); + envelope_cycle(channel, node_count, node_complex_, n_pol); + const auto exposure = envelope_exposure_; + const auto n_to = problem_.dims.n_to; + const auto parameters = channel_parameters_; + const auto densities = stage_channel_density_; + const auto initials = stage_channel_preionisation_; + const auto moments = envelope_moments_; + const auto field = node_complex_; + const auto window = time_window_; + const auto drive = envelope_streams_[0].node; + const auto loss = envelope_streams_[1].node; + launch::parallel_for( + "solprop_native.envelope.currents", + Range2(response_space_for(node_count), {0, 0}, {node_count, n_to}), + KOKKOS_LAMBDA(const std::size_t node, const std::size_t time) { + const double initial = initials(stage, channel); + const double fraction = envelope_fraction(exposure(node, time), initial); + const double survival = (1.0 - initial) * Kokkos::exp(-exposure(node, time)); + const double density = densities(stage, channel); + const auto axes = + envelope_axes(field(node * n_pol, time), + n_pol == 2 ? field(node * n_pol + 1, time) : Complex{}); + const auto current = + envelope_loss(axes, moments(0, node, time), moments(1, node, time), + parameters(channel, 8) * density * survival); + const double drude = parameters(channel, 7) * density * fraction; + for (std::size_t pol = 0; pol < n_pol; ++pol) { + const auto row = node * n_pol + pol; + const auto drive_value = window(time) * drude * field(row, time); + const auto loss_value = window(time) * current[pol]; + if (channel == 0) { + drive(row, time) = drive_value; + loss(row, time) = loss_value; + } else { + drive(row, time) += drive_value; + loss(row, time) += loss_value; + } + } + }); +} + +void GeneralWorkspace::envelope_response(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count, + const TransverseChunk* chunk) { + for (std::size_t channel = 0; channel < problem_.plasma.n_channels; ++channel) + envelope_population(channel, stage, node_count); + for (auto& stream : envelope_streams_) { + if (problem_.slab_geometry()) { + // As the polarisation: the chunk's y rows, zero beyond a final partial chunk. + graph::deep_copy(response_space_for(node_count), stream.projected, Complex{}); + slab_project(*chunk, &stream); + } else if (problem_.free_geometry()) { + // The persistent maximum-chunk plan transforms zero padding in a final + // partial chunk; only its actual rows are ever copied to the spatial FFT. + graph::deep_copy(response_space_for(node_count), stream.projected, Complex{}); + const auto rows = + std::make_pair(std::size_t{0}, node_count * problem_.effective_node_pol()); + graph::deep_copy(response_space_for(node_count), + Kokkos::subview(stream.projected, rows, Kokkos::ALL()), + Kokkos::subview(stream.node, rows, Kokkos::ALL())); + } else { + project(first_node, node_count, first_node == 0, &stream); + } + } +} + +// SEP 27 forms two independently collared current products. Every maximum-chunk +// row is written, including a partial chunk's padding, before a persistent FFT plan +// can read it. Modal projection precedes the FFT; radial projection follows it. +void GeneralWorkspace::held_response(const std::size_t stage, + const std::size_t first_node, + const std::size_t node_count, + const TransverseChunk* chunk) { + const auto current = held_current_; + const auto window = time_window_; + const auto derivative = held_derivative_; + const auto product = held_streams_[0].node; + const auto edge = held_streams_[1].node; + const auto live = node_count * problem_.effective_node_pol(); + const auto n_to = problem_.dims.n_to; + cartesian_for( + "solprop_native.held.products", response_space_for(node_count), current.extent(0), + n_to, KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { + const double value = row < live ? current(row, time) : 0.0; + product(row, time) = window(time) * value; + edge(row, time) = derivative(time) * value; + }); + const bool folded = folds_single_node(); + const bool radial = held_radial(); + for (std::size_t index = 0; index < 2; ++index) { + auto& stream = held_streams_[index]; + if (folded) { + const auto source = stream.node; + const auto target = stream.projected; + const auto projection = projection_; + const auto taper = stage_taper_; + cartesian_for( + "solprop_native.held.folded_projection", response_space_for(node_count), 1, + n_to, KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { + target(row, time) = projection(0, 0) / taper(stage) * source(row, time); + }); + } else if (problem_.slab_geometry()) { + graph::deep_copy(response_space_for(node_count), stream.projected, 0.0); + slab_contract("solprop_native.held.slab_projection", + response_space_for(node_count), projection_, stream.node, + stream.projected, chunk->nodes / problem_.spatial.n_x, n_to); + } else if (problem_.free_geometry() || radial) { + graph::deep_copy(response_space_for(node_count), stream.projected, stream.node); + } else { + project(first_node, node_count, first_node == 0, nullptr, &stream); + } + } + if (radial) { + const auto source = polarisation_real_; + const auto target = held_streams_[2].projected; + cartesian_for( + "solprop_native.held.nonplasma", response_space_for(node_count), + target.extent(0), n_to, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { + target(row, time) = row < live ? source(row, time) : 0.0; + }); + held_radial_projection(first_node, node_count); + } +} + +// solprop's private gated benchmark (`NumpyRefBackend._column_sources`): the chunk's +// live current rows are integrated by the problem's running-integral rule, the band +// rule's filter then the trapezoid (solprop SEP 18), and w(t) times each prefix joins +// the local polarisation, which is then projected as the held current's streams would +// have been. The scans are the device column's portable primitives; no hot column +// changes, and a run without the benchmark never reaches this. +void GeneralWorkspace::gated_response(const std::size_t node_count) { + const Kokkos::Profiling::ScopedRegion region("solprop_native.gated.response"); + const auto live = std::pair( + 0, node_count * problem_.effective_node_pol()); + const auto current = Kokkos::subview(held_current_, live, Kokkos::ALL); + const auto prefix = Kokkos::subview(gated_prefix_, live, Kokkos::ALL); + const auto& space = response_space_for(node_count); + if (problem_.plasma.band) + portable_team_band_scan(space, current, + Kokkos::subview(gated_filtered_, live, Kokkos::ALL), prefix, + plasma_taps_, problem_.grid.dt_fine); + else + portable_team_trapezoid_scan(space, current, prefix, problem_.grid.dt_fine); + const auto window = time_window_; + const auto polarisation = polarisation_real_; + cartesian_for( + "solprop_native.gated.polarisation", space, live.second, problem_.dims.n_to, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t time) { + polarisation(row, time) += window(time) * prefix(row, time); + }); +} + +// A stage's forward rows in one transform: all of them when the absorber's are due, +// else those after the absorber's (or the projection's alone). A host deferring its +// quadrature transforms the fine block only; `finish_quadrature` takes the coarse. +void GeneralWorkspace::transform_forward_pack() { + const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.forward_pack"); + const bool fine = defer_quadrature_ && forward_pack_fine_plan_; + if (absorbing() || forward_pack_absorber_rows_ == 0) { + if (fine) + graph::execute(*forward_pack_fine_plan_, forward_pack_fine_, + forward_pack_fine_spectrum_, KokkosFFT::Normalization::none); + else + graph::execute(*forward_pack_plan_, forward_pack_, forward_pack_spectrum_, + KokkosFFT::Normalization::none); + } else if (fine ? static_cast(forward_pack_fine_tail_plan_) + : static_cast(forward_pack_tail_plan_)) { + if (fine) + graph::execute(*forward_pack_fine_tail_plan_, forward_pack_fine_tail_, + forward_pack_fine_tail_spectrum_, KokkosFFT::Normalization::none); + else + graph::execute(*forward_pack_tail_plan_, forward_pack_tail_, + forward_pack_tail_spectrum_, KokkosFFT::Normalization::none); + } else { + graph::execute(*forward_real_plan_, projected_real_, projected_spectrum_, + KokkosFFT::Normalization::none); + } +} + +void GeneralWorkspace::held_transform(const bool coarse_only, const bool transformed) { + const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.held_streams"); + // The packed rows are every transform this call takes when it takes the fine ones + // and, where there are coarse rows, those too. The stage's forward pack, when it + // took them, has transformed the streams' rows already. + const bool every_row = + !coarse_only && (!held_streams_[0].coarse_forward || !defer_quadrature_); + if (held_pack_forward_ && every_row) { + graph::execute(*held_pack_forward_, held_pack_, held_pack_spectrum_, + KokkosFFT::Normalization::none); + } else if (!transformed) { + for (std::size_t index = 0; index < 2; ++index) { + auto& stream = held_streams_[index]; + if (!coarse_only) + graph::execute(*stream.forward, stream.projected, stream.spectrum, + KokkosFFT::Normalization::none); + if (stream.coarse_forward && (coarse_only || !defer_quadrature_)) + graph::execute(*stream.coarse_forward, stream.coarse, stream.coarse_spectrum, + KokkosFFT::Normalization::none); + } + } + const auto product = held_streams_[0].spectrum; + const auto edge = held_streams_[1].spectrum; + const auto coarse_product = held_streams_[0].coarse_spectrum; + const auto coarse_edge = held_streams_[1].coarse_spectrum; + const auto projected = projected_spectrum_; + const auto coarse = projected_coarse_spectrum_; + const auto inverse = held_inverse_; + const auto active = active_mask_; + const auto crop = crop_index_; + const bool include_coarse = !coarse.empty() && (coarse_only || !defer_quadrature_); + cartesian_for( + "solprop_native.held.spectral_assembly", execution_space_, projected.extent(0), + problem_.dims.n_omega, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + const auto bin = crop(omega); + if (active(omega)) { + const double value = inverse(omega); + const Complex factor(0.0, -value); + if (!coarse_only) + projected(row, bin) += + factor * product(row, bin) + value * value * edge(row, bin); + if (include_coarse) + coarse(row, bin) += factor * coarse_product(row, bin) + + value * value * coarse_edge(row, bin); + } else { + if (!coarse_only) + projected(row, bin) = Complex{}; + if (include_coarse) + coarse(row, bin) = Complex{}; + } + }); +} + +void GeneralWorkspace::held_radial_projection(const std::size_t first_node, + const std::size_t node_count) { + if (held_pack_forward_) + graph::execute(*held_pack_forward_, held_pack_, held_pack_spectrum_, + KokkosFFT::Normalization::none); + else + for (auto& stream : held_streams_) + graph::execute(*stream.forward, stream.projected, stream.spectrum, + KokkosFFT::Normalization::none); + const auto product = held_streams_[0].spectrum; + const auto edge = held_streams_[1].spectrum; + const auto polarisation = held_streams_[2].spectrum; + const auto nodal = held_nodal_spectrum_; + const auto inverse = held_inverse_; + const auto active = active_mask_; + const auto crop = crop_index_; + const auto n = problem_.dims.n_omega; + const auto live = node_count * problem_.effective_node_pol(); + cartesian_for( + "solprop_native.held.radial_assembly", node_spectral_space_, nodal.extent(0), n, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + Complex value{}; + if (row < live && active(omega)) { + const auto bin = crop(omega); + const double inv = inverse(omega); + value = polarisation(row, bin) + Complex(0.0, -inv) * product(row, bin) + + inv * inv * edge(row, bin); + } + nodal(row, omega) = value; + }); + const auto packed = held_projection_; + const auto projection = scaled_projection_; + const auto first_row = first_node * problem_.dims.n_pol_transform; + const auto live_rows = node_count * problem_.dims.n_pol_transform; + cartesian_for( + "solprop_native.held.radial_projection_matrix", matrix_copy_space_, + packed.extent(0), packed.extent(1), + KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { + packed(mode, row) = row < live_rows ? projection(mode, first_row + row) : 0.0; + }); + // Interleaved real and imaginary parts share the scalar radial projection. This + // is one GEMM of the combined cropped source, independent of state components. + using Interleaved = Kokkos::View>; + static_assert(sizeof(Complex) == 2 * sizeof(double)); + const auto columns = 2 * problem_.dims.n_pol_state * n; + const Interleaved input(reinterpret_cast(nodal.data()), packed.extent(1), + columns); + const Interleaved output(reinterpret_cast(held_projected_spectrum_.data()), + problem_.dims.n_modes, columns); + selected_gemm(transform_space_for(node_count), packed, input, output, 0.0, true); + if (first_node == 0) + graph::deep_copy(temporal_execution_space_, projected_spectrum_, Complex{}); + const auto source = held_projected_spectrum_; + const auto target = projected_spectrum_; + cartesian_for( + "solprop_native.held.radial_accumulate", execution_space_, target.extent(0), n, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + if (active(omega)) + target(row, crop(omega)) += source(row, omega); + }); +} + +void GeneralWorkspace::envelope_transform() { + for (auto& stream : envelope_streams_) { + graph::execute(*stream.forward, stream.projected, stream.spectrum, + KokkosFFT::Normalization::none); + if (stream.coarse_forward) + graph::execute(*stream.coarse_forward, stream.coarse, stream.coarse_spectrum, + KokkosFFT::Normalization::none); + } + const auto drive = envelope_streams_[0].spectrum; + const auto loss = envelope_streams_[1].spectrum; + const auto coarse_drive = envelope_streams_[0].coarse_spectrum; + const auto coarse_loss = envelope_streams_[1].coarse_spectrum; + const auto projected = projected_spectrum_; + const auto coarse = projected_coarse_spectrum_; + const auto drude_factor = envelope_drude_factor_; + const auto loss_factor = envelope_loss_factor_; + const auto crop = crop_index_; + launch::parallel_for( + "solprop_native.envelope.spectral_assembly", + Range2(execution_space_, {0, 0}, {projected.extent(0), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + const auto index = crop(omega); + projected(column, index) += drude_factor(omega) * drive(column, index) + + loss_factor(omega) * loss(column, index); + if (!coarse.empty()) + coarse(column, index) += drude_factor(omega) * coarse_drive(column, index) + + loss_factor(omega) * coarse_loss(column, index); + }); +} + +void GeneralWorkspace::project(const std::size_t first_node, + const std::size_t node_count, const bool first_chunk, + const EnvelopeStream* stream, + const FieldCurrentStream* field_stream) { + const auto real_source = + field_stream == nullptr ? polarisation_real_ : field_stream->node; + const auto real_target = + field_stream == nullptr ? projected_real_ : field_stream->projected; + const auto real_coarse_target = + field_stream == nullptr ? projected_coarse_real_ : field_stream->coarse; + const auto real_stacked_target = + field_stream == nullptr ? projected_stacked_real_ : field_stream->stacked; + const auto node_source = stream == nullptr ? polarisation_complex_ : stream->node; + const auto projected_target = + stream == nullptr ? projected_complex_ : stream->projected; + const auto coarse_target = + stream == nullptr ? projected_coarse_complex_ : stream->coarse; + const auto stacked_target = + stream == nullptr ? projected_stacked_complex_ : stream->stacked; + const bool whole_transform = first_node == 0 && node_count == problem_.dims.n_nodes && + problem_.dims.n_pol_state == 1; + if (whole_transform && !radial_blas_input_.empty()) { + packed_radial_gemm(scaled_projection_, node_source, projected_target); + return; + } + if (whole_transform && use_interleaved_projection()) { + Kokkos::Profiling::pushRegion( + "solprop_native.transform.projection.kokkosblas_interleaved"); + using Interleaved = Kokkos::View>; + static_assert(sizeof(Complex) == 2 * sizeof(double)); + const Interleaved nodes(reinterpret_cast(node_source.data()), + problem_.dims.n_nodes, 2 * problem_.dims.n_to); + const Interleaved fine(reinterpret_cast(projected_target.data()), + problem_.dims.n_modes, 2 * problem_.dims.n_to); + engine_gemm(transform_space_for(node_count), scaled_projection_, nodes, fine, 0.0); + if (project_coarse_ && !problem_.transform.projection_coarse_matrix.empty()) { + const Interleaved coarse(reinterpret_cast(coarse_target.data()), + problem_.dims.n_modes, 2 * problem_.dims.n_to); + engine_gemm(transform_space_for(node_count), scaled_projection_coarse_, nodes, + coarse, 0.0); + } + Kokkos::Profiling::popRegion(); + return; + } + // The stacked projections' splits copy on the stage's instance: on the temporal + // copy's serial instance one took about 90 us at 16 threads on the four-mode + // capillary, beside full-width kernels (docs/dev/benchmarks.md, "Stacked projection + // splits"). + if (whole_transform && use_blas_for_projection()) { + Kokkos::Profiling::pushRegion("solprop_native.transform.projection.kokkosblas"); + if (problem_.dims.family == FieldFamily::envelope && project_coarse_ && + !scaled_projection_stacked_complex_.empty()) { + selected_gemm(transform_space_for(node_count), scaled_projection_stacked_complex_, + node_source, stacked_target, Complex{}, true); + const auto& stacked = stacked_target; + const auto& fine = projected_target; + const auto& coarse = coarse_target; + const auto columns = problem_.n_columns(); + launch::parallel_for( + "solprop_native.transform.projection.split_stacked_complex", + Range2(execution_space_, {0, 0}, {columns, problem_.dims.n_to}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { + fine(column, time) = stacked(column, time); + coarse(column, time) = stacked(columns + column, time); + }); + } else if (problem_.dims.family == FieldFamily::envelope) { + selected_gemm(transform_space_for(node_count), scaled_projection_complex_, + node_source, projected_target, Complex{}, true); + } else if (one_mode_products_) { + const bool stacked = project_coarse_ && !scaled_projection_stacked_.empty(); + one_mode_projection(response_space_for(node_count), + stacked ? scaled_projection_stacked_ : scaled_projection_, + real_source, real_target, real_coarse_target); + } else if (project_coarse_ && !scaled_projection_stacked_.empty()) { + selected_gemm(transform_space_for(node_count), scaled_projection_stacked_, + real_source, real_stacked_target, 0.0, true); + const auto& stacked = real_stacked_target; + const auto& fine = real_target; + const auto& coarse = real_coarse_target; + const auto columns = problem_.n_columns(); + launch::parallel_for( + "solprop_native.transform.projection.split_stacked", + Range2(execution_space_, {0, 0}, {columns, problem_.dims.n_to}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { + fine(column, time) = stacked(column, time); + coarse(column, time) = stacked(columns + column, time); + }); + } else { + selected_gemm(transform_space_for(node_count), scaled_projection_, real_source, + real_target, 0.0, true); + } + Kokkos::Profiling::popRegion(); + return; + } + if (use_chunked_blas_for_projection()) { + Kokkos::Profiling::pushRegion( + "solprop_native.transform.projection.kokkosblas_chunked"); + const auto rows = node_count * problem_.dims.n_pol_transform; + const auto first_row = first_node * problem_.dims.n_pol_transform; + const bool tail = node_count != problem_.dims.node_chunk; + const bool last_chunk = first_node + node_count == problem_.dims.n_nodes; + const bool stacked = + project_coarse_ && !problem_.transform.projection_coarse_matrix.empty(); + if (problem_.dims.family == FieldFamily::envelope) { + const auto polarisation = Kokkos::subview( + node_source, std::make_pair(std::size_t{0}, rows), Kokkos::ALL()); + if (stacked) { + const auto packed = + tail ? tail_projection_stacked_complex_ : chunk_projection_stacked_complex_; + const auto source = scaled_projection_stacked_complex_; + launch::parallel_for( + "solprop_native.transform.pack_projection_stacked_complex", + Range2(matrix_copy_space_for(node_count, true), {0, 0}, + {2 * problem_.dims.n_modes, rows}), + KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { + packed(mode, row) = source(mode, first_row + row); + }); + selected_gemm(transform_space_for(node_count), packed, polarisation, + stacked_target, first_chunk ? Complex{} : Complex{1.0, 0.0}, + true); + if (last_chunk) { + const auto& projected = stacked_target; + const auto& fine = projected_target; + const auto& coarse = coarse_target; + const auto columns = problem_.n_columns(); + launch::parallel_for( + "solprop_native.transform.projection.split_stacked_complex", + Range2(execution_space_, {0, 0}, {columns, problem_.dims.n_to}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { + fine(column, time) = projected(column, time); + coarse(column, time) = projected(columns + column, time); + }); + } + } else { + // The first half of the existing stacked scratch is a contiguous fine + // matrix. Alias it when this trial stage does not need the coarse rows. + const auto scratch = + tail ? tail_projection_stacked_complex_ : chunk_projection_stacked_complex_; + const ComplexView2 packed = + scratch.empty() + ? (tail ? tail_projection_complex_ : chunk_projection_complex_) + : ComplexView2(Kokkos::subview( + scratch, std::pair{std::size_t{0}, problem_.dims.n_modes}, + Kokkos::ALL())); + const auto source = scaled_projection_complex_; + launch::parallel_for( + "solprop_native.transform.pack_projection_complex", + Range2(matrix_copy_space_for(node_count), {0, 0}, + {problem_.dims.n_modes, rows}), + KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { + packed(mode, row) = source(mode, first_row + row); + }); + selected_gemm(transform_space_for(node_count), packed, polarisation, + projected_target, first_chunk ? Complex{} : Complex{1.0, 0.0}, + true); + } + } else { + const auto polarisation = Kokkos::subview( + real_source, std::make_pair(std::size_t{0}, rows), Kokkos::ALL()); + if (stacked) { + const auto packed = tail ? tail_projection_stacked_ : chunk_projection_stacked_; + const auto source = scaled_projection_stacked_; + launch::parallel_for( + "solprop_native.transform.pack_projection_stacked", + Range2(matrix_copy_space_for(node_count, true), {0, 0}, + {2 * problem_.dims.n_modes, rows}), + KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { + packed(mode, row) = source(mode, first_row + row); + }); + selected_gemm(transform_space_for(node_count), packed, polarisation, + real_stacked_target, first_chunk ? 0.0 : 1.0, true); + if (last_chunk) { + const auto& projected = real_stacked_target; + const auto& fine = real_target; + const auto& coarse = real_coarse_target; + const auto columns = problem_.n_columns(); + launch::parallel_for( + "solprop_native.transform.projection.split_stacked", + Range2(execution_space_, {0, 0}, {columns, problem_.dims.n_to}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { + fine(column, time) = projected(column, time); + coarse(column, time) = projected(columns + column, time); + }); + } + } else { + const auto scratch = + tail ? tail_projection_stacked_ : chunk_projection_stacked_; + const RealView2 packed = + scratch.empty() + ? (tail ? tail_projection_ : chunk_projection_) + : RealView2(Kokkos::subview( + scratch, std::pair{std::size_t{0}, problem_.dims.n_modes}, + Kokkos::ALL())); + const auto source = scaled_projection_; + launch::parallel_for( + "solprop_native.transform.pack_projection", + Range2(matrix_copy_space_for(node_count), {0, 0}, + {problem_.dims.n_modes, rows}), + KOKKOS_LAMBDA(const std::size_t mode, const std::size_t row) { + packed(mode, row) = source(mode, first_row + row); + }); + selected_gemm(transform_space_for(node_count), packed, polarisation, + real_target, first_chunk ? 0.0 : 1.0, true); + } + } + Kokkos::Profiling::popRegion(); + return; + } + Kokkos::Profiling::pushRegion("solprop_native.transform.projection.custom"); + const auto projection = scaled_projection_; + const auto projection_coarse = scaled_projection_coarse_; + const auto& polar_real = real_source; + const auto& polar_complex = node_source; + const auto& projected_real = real_target; + const auto& projected_complex = projected_target; + const auto& projected_coarse_real = real_coarse_target; + const auto& projected_coarse_complex = coarse_target; + const auto n_pol_state = problem_.dims.n_pol_state; + const auto n_pol_transform = problem_.dims.n_pol_transform; + const auto n_to = problem_.dims.n_to; + const bool envelope = problem_.dims.family == FieldFamily::envelope; + const bool coarse = project_coarse_ && !projection_coarse.empty(); + launch::parallel_for( + "solprop_native.transform.projection.custom", + Range2(transform_space_for(node_count), {0, 0}, {problem_.n_columns(), n_to}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { + const std::size_t mode = column / n_pol_state; + const std::size_t state_pol = column % n_pol_state; + if (envelope) { + Complex fine{}; + Complex rough{}; + for (std::size_t node = 0; node < node_count; ++node) { + for (std::size_t transform_pol = 0; transform_pol < n_pol_transform; + ++transform_pol) { + const std::size_t global_row = + (first_node + node) * n_pol_transform + transform_pol; + const std::size_t local_row = + (node * n_pol_transform + transform_pol) * n_pol_state + state_pol; + const Complex value = polar_complex(local_row, time); + fine += projection(mode, global_row) * value; + if (coarse) + rough += projection_coarse(mode, global_row) * value; + } + } + if (first_chunk) { + projected_complex(column, time) = fine; + if (coarse) + projected_coarse_complex(column, time) = rough; + } else { + projected_complex(column, time) += fine; + if (coarse) + projected_coarse_complex(column, time) += rough; + } + } else { + double fine = 0.0; + double rough = 0.0; + for (std::size_t node = 0; node < node_count; ++node) { + for (std::size_t transform_pol = 0; transform_pol < n_pol_transform; + ++transform_pol) { + const std::size_t global_row = + (first_node + node) * n_pol_transform + transform_pol; + const std::size_t local_row = + (node * n_pol_transform + transform_pol) * n_pol_state + state_pol; + const double value = polar_real(local_row, time); + fine += projection(mode, global_row) * value; + if (coarse) + rough += projection_coarse(mode, global_row) * value; + } + } + if (first_chunk) { + projected_real(column, time) = fine; + if (coarse) + projected_coarse_real(column, time) = rough; + } else { + projected_real(column, time) += fine; + if (coarse) + projected_coarse_real(column, time) += rough; + } + } + }); + Kokkos::Profiling::popRegion(); +} + +void GeneralWorkspace::form_absorber() { + const bool spatial = !problem_.spatial.absorber_matrix.empty(); + const auto modes = problem_.dims.n_modes; + const auto polarisations = problem_.dims.n_pol_state; + // Flatten (polarisation,time) together: the same scalar radial matrix acts + // independently on each component. The views alias existing FFT work buffers. + if (spatial && problem_.dims.family == FieldFamily::envelope) { + using Matrix = Kokkos::View>; + const Matrix input(modal_complex_.data(), modes, + polarisations * problem_.dims.n_to); + const Matrix output(absorber_complex_.data(), modes, + polarisations * problem_.dims.n_to); + selected_gemm(absorber_execution_space_, spatial_absorber_complex_, input, output, + Complex{}, true); + } else if (spatial && problem_.dims.family == FieldFamily::field_real) { + using Matrix = Kokkos::View>; + const Matrix input(modal_real_.data(), modes, polarisations * problem_.dims.n_to); + const Matrix output(absorber_real_.data(), modes, + polarisations * problem_.dims.n_to); + selected_gemm(absorber_execution_space_, spatial_absorber_real_, input, output, 0.0, + true); + } + const auto rate = absorber_fine_; + const auto spatial_matrix = spatial_absorber_real_; + const auto modal_real = modal_real_; + const auto modal_complex = modal_complex_; + const auto absorber_real = absorber_real_; + const auto absorber_complex = absorber_complex_; + const auto family = problem_.dims.family; + launch::parallel_for( + "solprop_native.absorber.nonlinear_term", + Range2(temporal_execution_space_, {0, 0}, + {problem_.n_columns(), problem_.dims.n_to}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t time) { + if (family == FieldFamily::envelope) { + absorber_complex(column, time) = + (spatial ? absorber_complex(column, time) : Complex{}) - + 0.5 * rate(time) * modal_complex(column, time); + } else if (family == FieldFamily::field_real) { + absorber_real(column, time) = (spatial ? absorber_real(column, time) : 0.0) - + 0.5 * rate(time) * modal_real(column, time); + } else { + double transverse = 0.0; + if (spatial) + for (std::size_t source = 0; source < modes; ++source) + transverse += + spatial_matrix(column / polarisations, source) * + modal_complex(source * polarisations + column % polarisations, time) + .real(); + absorber_real(column, time) = + transverse - 0.5 * rate(time) * modal_complex(column, time).real(); + } + }); +} + +void GeneralWorkspace::finish_stage(const std::size_t stage) { + const auto terms = stage_terms(stage); + const double solution_weight = problem_.stepper.solution_weights[stage]; + const double error_weight = problem_.stepper.error_weights[stage]; +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // The host's blocks of bins, as the stage's entry kernel takes them. + if (terms.blocks_vectorise()) { + const auto n = problem_.dims.n_omega; + const std::size_t blocks = (n + stage_block - 1) / stage_block; + launch::parallel_for( + "solprop_native.stage.finish_accumulate", + Kokkos::RangePolicy(execution_space_, 0, + problem_.n_columns() * blocks), + KOKKOS_LAMBDA(const std::size_t item) { + const std::size_t first = (item % blocks) * stage_block; + terms.finish_block(stage, solution_weight, error_weight, item / blocks, first, + n - first < stage_block ? n - first : stage_block); + }); + return; + } +#endif + launch::parallel_for( + "solprop_native.stage.finish_accumulate", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + terms.finish(stage, solution_weight, error_weight, column, index); + }); +} + +void GeneralWorkspace::set_flow_actions(const std::vector& forward, + const std::vector& inverse, + const std::vector& identity) { + if (!flow_) + throw std::logic_error("this state carries no coupled flow"); + flow_->set_actions(forward, inverse, identity); +} + +std::vector GeneralWorkspace::adams_block_to_host() const { + require_adams(); + if (adams_block_.extent(0) == 0) + throw std::logic_error("the Adams history holds no coupled block"); + std::vector values(adams_block_.size()); + const auto mirror = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, adams_block_); + std::copy(mirror.data(), mirror.data() + mirror.size(), values.begin()); + return values; +} + +std::vector GeneralWorkspace::adams_exponent_to_host() const { + require_adams(); + std::vector values(adams_exponent_.size()); + const auto mirror = + Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, adams_exponent_); + std::copy(mirror.data(), mirror.data() + mirror.size(), values.begin()); + return values; +} + +std::vector GeneralWorkspace::flow_actions_to_host(const bool inverse) const { + if (!flow_) + throw std::logic_error("this state carries no coupled flow"); + return flow_->actions_to_host(inverse); +} + +std::vector +GeneralWorkspace::form_flow_actions(const std::size_t section, const double dz, + const std::vector& fractions) { + if (!flow_) + throw std::logic_error("this state carries no coupled flow"); + return flow_->form_actions(section, dz, fractions); +} + +std::vector GeneralWorkspace::weighted_error(const std::vector& scale) { + if (!problem_.stepper.weighted_controller) + throw std::logic_error("this state has no weighted controller workspace"); + require_extent(scale.size(), problem_.dims.n_omega, "weighted controller scale"); + copy_vector(execution_space_, weighted_scale_, scale); + const auto residual = residual_; + const auto in_norm = norm_mask_; + const auto native_scale = weighted_scale_; + const auto partials = weighted_partials_; + const auto n = problem_.dims.n_omega; + using TeamPolicy = Kokkos::TeamPolicy; + using TeamMember = TeamPolicy::member_type; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const int team_size = SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE; +#else + // A host team per column, sharing the threads when the columns are fewer. + const int team_size = + host_reduction_team_size(execution_space_, problem_.n_columns(), n); +#endif + // On a device each column is `reduction_slots_` teams over its own bins; the + // caller sums the slots. + const auto slots = reduction_slots_; + const TeamPolicy policy(execution_space_, + static_cast(problem_.n_columns() * slots), team_size); + launch::parallel_for( + "solprop_native.step.weighted_error_partials", policy, + KOKKOS_LAMBDA(const TeamMember& team) { + const auto league = static_cast(team.league_rank()); + const auto column = league / slots; + std::size_t first = 0; + std::size_t end = 0; + slot_range(n, slots, league % slots, first, end); + double sum = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(first), + static_cast(end)), + [&](const int index_value, double& update) { + const auto index = static_cast(index_value); + if (in_norm(index) == 0U) + return; + const Complex value = residual(column, index) / native_scale(index); + update += squared_abs(value); + }, + sum); + Kokkos::single(Kokkos::PerTeam(team), [&] { partials(league) = sum; }); + }); + execution_space_.fence("solprop_native weighted-error boundary"); + const auto host = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, partials); + std::vector result(problem_.n_columns() * slots); + for (std::size_t slot = 0; slot < result.size(); ++slot) + result[slot] = host(slot); + return result; +} + +void GeneralWorkspace::scan_plasma_statistics(const std::size_t first_node, + const std::size_t node_count, + const RealView2& output, + const bool diagnose, + const double preionisation) { + if (problem_.plasma.envelope) { + if (diagnose) + throw std::invalid_argument("the plasma integration diagnostics read the " + "field-grid plasma column, and this problem " + "carries none"); + for (std::size_t channel = 0; channel < problem_.plasma.n_channels; ++channel) + envelope_statistics(channel, first_node, node_count, output, node_complex_, + problem_.effective_node_pol()); + return; + } + // solprop SEP 18: the band rule's statistics, and every problem's run diagnostics, + // come from the band scan; the trapezoid's own scan below is unchanged. + if (problem_.plasma.band || diagnose) { + if (diagnose) + team_band_statistics(first_node, node_count, output, preionisation); + else + team_band_statistics(first_node, node_count, output, preionisation); + return; + } + const auto node_real = node_real_; + const auto node_complex = node_complex_; + const bool real_family = problem_.dims.family == FieldFamily::field_real; + const auto coefficients = rate_coefficients_; + const auto n_pol = problem_.effective_node_pol(); + const auto n_to = problem_.dims.n_to; + const auto intervals = problem_.plasma.rate_intervals; + const auto dt = problem_.grid.dt_fine; + const auto log_field_min = problem_.plasma.log_field_min; + const auto d_log_field = problem_.plasma.d_log_field; + const auto field_min = problem_.plasma.field_min; + const auto field_max = problem_.plasma.field_max; + const auto rate_max = problem_.plasma.rate_max; + if (problem_.plasma.n_channels > 1) { + const auto parameters = channel_parameters_; + const auto channel_coefficients = channel_coefficients_; + const auto channels = problem_.plasma.n_channels; + launch::parallel_for( + "solprop_native.plasma.channel_statistics_scan", + Kokkos::RangePolicy(response_space_for(node_count), 0, + node_count), + KOKKOS_LAMBDA(const std::size_t node) { + for (std::size_t channel = 0; channel < channels; ++channel) { + double exposure = 0.0; + double previous_rate = 0.0; + double largest_rate = 0.0; + for (std::size_t time = 0; time < n_to; ++time) { + const double first = + (real_family ? node_real(node * n_pol, time) + : node_complex(node * n_pol, time).real()); + const double second = + n_pol == 2 + ? (real_family ? node_real(node * n_pol + 1, time) + : node_complex(node * n_pol + 1, time).real()) + : 0.0; + const double magnitude = + n_pol == 2 ? Kokkos::hypot(first, second) : Kokkos::abs(first); + const double rate = ionisation_rate( + magnitude, channel_coefficients, parameters(channel, 0), + parameters(channel, 1), parameters(channel, 2), + parameters(channel, 3), parameters(channel, 4), + static_cast(parameters(channel, 5)), + static_cast(parameters(channel, 6))); + if (time > 0) + exposure += trapezoid(rate, previous_rate, dt); + previous_rate = rate; + if (rate > largest_rate) + largest_rate = rate; + } + output(first_node + node, 2 * channel) = exposure; + output(first_node + node, 2 * channel + 1) = largest_rate; + } + }); + return; + } +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const auto rates = plasma_rate_; + const auto exposures = plasma_exposure_; + using TeamPolicy = Kokkos::TeamPolicy; + using TeamMember = TeamPolicy::member_type; + const TeamPolicy policy(response_space_for(node_count), static_cast(node_count), + static_cast(response_team_size())); + launch::parallel_for( + "solprop_native.plasma.device_statistics_scan", policy, + KOKKOS_LAMBDA(const TeamMember& team) { + const auto node = static_cast(team.league_rank()); + const auto base = node * n_pol; + Kokkos::parallel_for( + Kokkos::TeamThreadRange(team, static_cast(n_to)), + [&](const int time_value) { + const auto time = static_cast(time_value); + const double first = (real_family ? node_real(base, time) + : node_complex(base, time).real()); + const double second = + n_pol == 2 ? (real_family ? node_real(base + 1, time) + : node_complex(base + 1, time).real()) + : 0.0; + const double magnitude = + n_pol == 2 ? Kokkos::hypot(first, second) : Kokkos::abs(first); + rates(node, time) = + ionisation_rate(magnitude, coefficients, log_field_min, d_log_field, + field_min, field_max, rate_max, intervals); + }); + team.team_barrier(); + Kokkos::parallel_scan( + Kokkos::TeamThreadRange(team, static_cast(n_to)), + [&](const int time_value, double& update, const bool final) { + const auto time = static_cast(time_value); + if (time != 0) + update += trapezoid(rates(node, time), rates(node, time - 1), dt); + if (final) + exposures(node, time) = update; + }); + double largest_rate = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n_to)), + [&](const int time_value, double& largest) { + const auto time = static_cast(time_value); + if (rates(node, time) > largest) + largest = rates(node, time); + }, + Kokkos::Max(largest_rate)); + Kokkos::single(Kokkos::PerTeam(team), [&]() { + output(first_node + node, 0) = exposures(node, n_to - 1); + output(first_node + node, 1) = largest_rate; + }); + }); +#else + launch::parallel_for( + "solprop_native.plasma.statistics_scan", + Kokkos::RangePolicy(response_space_for(node_count), 0, + node_count), + KOKKOS_LAMBDA(const std::size_t node) { + const auto base = node * n_pol; + double exposure = 0.0; + double previous_rate = 0.0; + double largest_rate = 0.0; + for (std::size_t time = 0; time < n_to; ++time) { + const double first = + (real_family ? node_real(base, time) : node_complex(base, time).real()); + const double second = + n_pol == 2 ? (real_family ? node_real(base + 1, time) + : node_complex(base + 1, time).real()) + : 0.0; + const double magnitude = + n_pol == 2 ? Kokkos::hypot(first, second) : Kokkos::abs(first); + const double rate = + ionisation_rate(magnitude, coefficients, log_field_min, d_log_field, + field_min, field_max, rate_max, intervals); + if (time > 0) + exposure += trapezoid(rate, previous_rate, dt); + previous_rate = rate; + if (rate > largest_rate) + largest_rate = rate; + } + output(first_node + node, 0) = exposure; + output(first_node + node, 1) = largest_rate; + }); +#endif +} + +void GeneralWorkspace::finish(const double dz) { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + if (problem_.stepper.low_storage) { + // The march accepted the step in place (solprop SEP 24 section 6.2): the + // boundary treatments act on the state itself, and the stash a response pass + // left is the last evaluation's, at a predicted field, not this one's. + if (problem_.absorber.split_step) + split_step(dz, current_); + else + mask(current_); + stash_is_current_ = false; + execution_space_.fence("solprop_native accepted-step boundary"); + return; + } + if (problem_.absorber.split_step) + split_step(dz, advanced_); + else + mask(advanced_); + std::swap(current_, advanced_); + if (!stash_current_.empty()) + graph::deep_copy(execution_space_, stash_current_, stash_candidate_); + stash_is_current_ = !stash_current_.empty(); + execution_space_.fence("solprop_native accepted-step boundary"); +} + +void GeneralWorkspace::finish_quadrature() { + const Kokkos::Profiling::ScopedRegion region("solprop_native.quadrature.finish"); + if (!coarse_pending_) + return; + if (problem_.dims.family == FieldFamily::envelope) + graph::execute(*forward_coarse_complex_plan_, projected_coarse_complex_, + projected_coarse_spectrum_, KokkosFFT::Normalization::none); + else if (forward_pack_coarse_plan_) + graph::execute(*forward_pack_coarse_plan_, forward_pack_coarse_, + forward_pack_coarse_spectrum_, KokkosFFT::Normalization::none); + else + graph::execute(*forward_coarse_real_plan_, projected_coarse_real_, + projected_coarse_spectrum_, KokkosFFT::Normalization::none); + if (held_plasma()) + held_transform(true, static_cast(forward_pack_coarse_plan_)); + coarse_pending_ = false; +} + +void GeneralWorkspace::score_quadrature() { + quadrature_error_ = Kokkos::Experimental::quiet_NaN_v; + quadrature_stale_ = false; + if (quadrature_partials_.empty() || + (!problem_.kerr.nonlinear && !problem_.plasma.enabled && !dispersive_)) + return; + finish_quadrature(); + // SEP 6's dispersive products join the embedded rule's spectrum once, when scored. + if (dispersive_) + dispersive_->finish_coarse(); + + const auto fine = projected_spectrum_; + const auto coarse = projected_coarse_spectrum_; + const auto crop = crop_index_; + const auto window = omega_window_; + const auto prefactor_terms = prefactor_terms_; + const auto prefactor_weights = stage_prefactor_; + const auto partials = quadrature_partials_; + const auto last = problem_.stepper.n_stages - 1; + const auto n = problem_.dims.n_omega; + const auto n_prefactor = problem_.linear.n_prefactor_terms; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + using TeamPolicy = Kokkos::TeamPolicy; + using TeamMember = TeamPolicy::member_type; + const TeamPolicy policy(execution_space_, static_cast(problem_.n_columns()), + SOLPROP_NATIVE_DEVICE_REDUCTION_SIZE); + launch::parallel_for( + "solprop_native.transform.device_quadrature_partials", policy, + KOKKOS_LAMBDA(const TeamMember& team) { + const auto column = static_cast(team.league_rank()); + double difference_sum = 0.0; + double fine_sum = 0.0; + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n)), + [&](const int index_value, double& update) { + const auto index = static_cast(index_value); + Complex prefactor{}; + for (std::size_t term = 0; term < n_prefactor; ++term) + prefactor += prefactor_weights(last * n_prefactor + term) * + prefactor_terms(term, column, index); + const Complex weight = prefactor * window(index); + const Complex fine_value = weight * fine(column, crop(index)); + const Complex coarse_value = weight * coarse(column, crop(index)); + update += squared_abs(coarse_value - fine_value); + }, + difference_sum); + Kokkos::parallel_reduce( + Kokkos::TeamThreadRange(team, static_cast(n)), + [&](const int index_value, double& update) { + const auto index = static_cast(index_value); + Complex prefactor{}; + for (std::size_t term = 0; term < n_prefactor; ++term) + prefactor += prefactor_weights(last * n_prefactor + term) * + prefactor_terms(term, column, index); + const Complex weight = prefactor * window(index); + update += squared_abs(weight * fine(column, crop(index))); + }, + fine_sum); + Kokkos::single(Kokkos::PerTeam(team), [&]() { + partials(0, column) = difference_sum; + partials(1, column) = fine_sum; + }); + }); +#else + launch::parallel_for( + "solprop_native.transform.quadrature_partials", + Kokkos::RangePolicy(execution_space_, 0, problem_.n_columns()), + KOKKOS_LAMBDA(const std::size_t column) { + double difference_sum = 0.0; + double fine_sum = 0.0; + for (std::size_t index = 0; index < n; ++index) { + Complex prefactor{}; + for (std::size_t term = 0; term < n_prefactor; ++term) + prefactor += prefactor_weights(last * n_prefactor + term) * + prefactor_terms(term, column, index); + const Complex weight = prefactor * window(index); + const Complex fine_value = weight * fine(column, crop(index)); + const Complex coarse_value = weight * coarse(column, crop(index)); + fine_sum += squared_abs(fine_value); + difference_sum += squared_abs(coarse_value - fine_value); + } + partials(0, column) = difference_sum; + partials(1, column) = fine_sum; + }); +#endif + // The columns' sums in their order, on the device: the last score and the peak + // since the backend last read it. + const auto scores = quadrature_scores_; + const auto columns = problem_.n_columns(); + launch::parallel_for( + "solprop_native.transform.quadrature_score", + Kokkos::RangePolicy(execution_space_, 0, 1), + KOKKOS_LAMBDA(const std::size_t) { + double difference_sum = 0.0; + double fine_sum = 0.0; + for (std::size_t column = 0; column < columns; ++column) { + difference_sum += partials(0, column); + fine_sum += partials(1, column); + } + double value = Kokkos::Experimental::quiet_NaN_v; + if (fine_sum != 0.0) + value = Kokkos::sqrt(difference_sum / fine_sum); + scores(0) = value; + if (fine_sum != 0.0 && !(scores(1) >= value)) + scores(1) = value; + }); + quadrature_stale_ = true; +} + +double GeneralWorkspace::quadrature_error() const { + if (quadrature_stale_) { + const auto last = Kokkos::subview(quadrature_scores_, 0); + Kokkos::deep_copy(execution_space_, quadrature_error_, last); + execution_space_.fence("solprop_native quadrature score"); + quadrature_stale_ = false; + } + return quadrature_error_; +} + +double GeneralWorkspace::quadrature_peak() { + if (quadrature_scores_.extent(0) == 0) + return Kokkos::Experimental::quiet_NaN_v; + double peak = 0.0; + const auto stored = Kokkos::subview(quadrature_scores_, 1); + graph::deep_copy(execution_space_, peak, stored); + execution_space_.fence("solprop_native quadrature peak"); + graph::deep_copy(execution_space_, stored, Kokkos::Experimental::quiet_NaN_v); + return peak; +} + +void GeneralWorkspace::promote_fsal() { + score_quadrature(); + if (!pending_fsal_) + return; + const auto stages = stages_; + const auto whole = whole_propagator_; + const auto recycled = recycled_; + const auto last = problem_.stepper.n_stages - 1; + launch::parallel_for( + "solprop_native.stage.promote_fsal", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + recycled(column, index) = whole(column, index) * stages(last, column, index); + }); + has_recycled_ = true; + pending_fsal_ = false; +} + +void GeneralWorkspace::clear_fsal() { + has_recycled_ = false; + pending_fsal_ = false; + coarse_pending_ = false; + if (!stash_current_.empty()) + graph::deep_copy(execution_space_, stash_current_, 0.0); + stash_is_current_ = false; +} + +void GeneralWorkspace::mask(const ComplexView2& values) { + const auto active = active_mask_; + launch::parallel_for( + "solprop_native.mask.accepted", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + if (active(index) == 0U) + values(column, index) = Complex{}; + }); +} + +namespace { +// The absorber split's factor `exp(-rate h / 2)` on `rows` rows of `time`. The factor +// depends on the sample alone: a device forms it once a step for every sample, and a +// host task forms it once for a tile of samples, through the vector-maths layer +// (solprop ADR-059), and applies it to a block of rows, the same expression to the bit +// on the libm route. A device reads each sample's factor from `factor`, formed once a +// step by `form_split_factor` with the same expression: one binary64 exponential per +// sample rather than per element, which on a device with a reduced binary64 rate is +// most of the pass. +template +void split_apply(const char* label, const ExecutionSpace& execution, + const TimeView& time, const RealView& rate, const RealView& factors, + const double dz, const std::size_t rows, const std::size_t samples, + const double scale = 1.0) { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + static_cast(rate); + static_cast(dz); + launch::parallel_for( + label, Range2(execution, {0, 0}, {rows, samples}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t index) { + time(row, index) *= scale * factors(index); + }); +#else + static_cast(factors); + constexpr std::size_t tile = 256; + constexpr std::size_t row_block = 64; + const std::size_t tiles = (samples + tile - 1) / tile; + const std::size_t blocks = (rows + row_block - 1) / row_block; + launch::parallel_for( + label, Kokkos::RangePolicy(execution, 0, tiles * blocks), + KOKKOS_LAMBDA(const std::size_t task) { + const std::size_t first = (task % tiles) * tile; + const std::size_t count = samples - first < tile ? samples - first : tile; + const std::size_t row_first = (task / tiles) * row_block; + const std::size_t row_last = + rows - row_first < row_block ? rows : row_first + row_block; + double factor[tile]; + for (std::size_t sample = 0; sample < count; ++sample) + factor[sample] = -0.5 * rate(first + sample) * dz; + vmath::exp(factor, factor, count); + for (std::size_t sample = 0; sample < count; ++sample) + factor[sample] *= scale; + for (std::size_t row = row_first; row < row_last; ++row) + for (std::size_t sample = 0; sample < count; ++sample) + time(row, first + sample) *= factor[sample]; + }); +#endif +} +} // namespace + +void GeneralWorkspace::split_columns(const double dz, std::size_t active_rows, + const double parseval, + const std::size_t first_column) { + const bool tail = temporal_tail_rows_ != 0 && active_rows == temporal_tail_rows_; + if (active_rows == 0) + active_rows = coarse_spectrum_.extent(0); + if (problem_.dims.family == FieldFamily::envelope) { + if (tail) + coarse_tail_inverse_complex_.execute(); + else + graph::execute(*coarse_inverse_complex_plan_, coarse_spectrum_, + coarse_time_complex_, KokkosFFT::Normalization::none); + if (monitoring_split()) + add_split_removal(dz, active_rows, parseval, first_column, coarse_time_complex_, + 1.0); + split_apply("solprop_native.absorber.split_apply_complex", execution_space_, + coarse_time_complex_, absorber_coarse_, split_factor_, dz, active_rows, + problem_.dims.n_t); + if (tail) + coarse_tail_forward_complex_.execute(); + else + graph::execute(*coarse_forward_complex_plan_, coarse_time_complex_, + coarse_spectrum_, KokkosFFT::Normalization::none); + } else { + if (tail) + coarse_tail_inverse_real_.execute(); + else + graph::execute(*coarse_inverse_real_plan_, coarse_spectrum_, coarse_time_real_, + KokkosFFT::Normalization::none); + if (monitoring_split()) + add_split_removal(dz, active_rows, parseval, first_column, {}, 1.0); + split_apply("solprop_native.absorber.split_apply_real", execution_space_, + coarse_time_real_, absorber_coarse_, split_factor_, dz, active_rows, + problem_.dims.n_t); + if (tail) + coarse_tail_forward_real_.execute(); + else + graph::execute(*coarse_forward_real_plan_, coarse_time_real_, coarse_spectrum_, + KokkosFFT::Normalization::none); + } +} + +// Free space's complex rows: the split's factor depends on time alone, so it acts alike +// on every column of any transverse basis, and the temporal pair runs in place on the +// state's own rows, a block of them at a time. Nothing is copied to a coarse buffer and +// back, the transform's normalisation goes with the factor, and the band's mask is the +// one pass after the blocks. Equal to the reference's split in real space up to the +// transforms' rounding, at any block size. +void GeneralWorkspace::split_state_rows(const double dz, const ComplexView2& state) { + const double inverse_time = 1.0 / static_cast(problem_.dims.n_t); + const auto columns = problem_.n_columns(); + for (std::size_t first = 0; first < columns; first += split_rows_) { + const auto rows = std::min(split_rows_, columns - first); + const ComplexView2 block = + Kokkos::subview(state, std::make_pair(first, first + rows), Kokkos::ALL()); + const bool tail = rows != split_rows_; + graph::execute(tail ? *split_tail_inverse_plan_ : *split_inverse_plan_, block, + block, KokkosFFT::Normalization::none); + if (monitoring_split()) + add_split_removal(dz, rows, 0.0, first, block, inverse_time * inverse_time); + split_apply("solprop_native.absorber.split_apply_complex", execution_space_, block, + absorber_coarse_, split_factor_, dz, rows, problem_.dims.n_t, + inverse_time); + graph::execute(tail ? *split_tail_forward_plan_ : *split_forward_plan_, block, + block, KokkosFFT::Normalization::none); + } + mask(state); +} + +// What one split takes of the power it finds in the coarse time rows, `sum P (1 - +// exp(-alpha h))` with the boundary monitor's column weights, added to the run's +// accumulator in the memory space (solprop SEP 21, ADR-057): the monitor's reading +// drains it at a check, so every accepted step is counted, checked or not. The same +// pass adds to the step's `sum P'` and the guards' and the collar's sums of `P'`, the +// power found halfway through the split's factor, `P exp(-alpha h / 2)`, which the +// monitor judges the occupancies on: the symmetric splitting's state at the step's +// end. A separable route's rows are real-space rows, measured by `parseval`; zero +// means the rows are the stored columns from `first_column` on, weighted by their +// quadrature. `complex_time` holds a complex route's rows, whose power `power_scale` +// brings to the state's measure: the state's own rows carry no transform normalisation. +void GeneralWorkspace::add_split_removal(const double dz, const std::size_t active_rows, + const double parseval, + const std::size_t first_column, + const ComplexView2& complex_time, + const double power_scale) { + const auto rate = absorber_coarse_; + const auto share = monitor_share_; + const auto half = monitor_half_; + const auto weights = monitor_column_weights_; + const bool stored = parseval == 0.0; + // The share the split takes, and half its power factor, depend on time alone: + // formed once, not per row. + launch::parallel_for( + "solprop_native.monitor.split_share", + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_t), + KOKKOS_LAMBDA(const std::size_t index) { + share(index) = -Kokkos::expm1(-rate(index) * dz); + half(index) = Kokkos::exp(-0.5 * rate(index) * dz); + }); + const auto guards = monitor_weights_; + const auto samples = + Range2(execution_space_, {0, 0}, {active_rows, problem_.dims.n_t}); + if (problem_.dims.family == FieldFamily::envelope) { + launch::parallel_reduce( + "solprop_native.monitor.split_removal_complex", samples, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t index, + SplitSums& update) { + const double power = (stored ? weights(first_column + row) : parseval) * + (power_scale * squared_abs(complex_time(row, index))); + update.values[0] += power * share(index); + const double found = half(index) * power; + update.values[1] += found; + update.values[2] += guards(0, index) * found; + update.values[3] += guards(1, index) * found; + update.values[4] += guards(2, index) * found; + }, + Kokkos::Sum(monitor_split_sum_)); + } else { + const auto time = coarse_time_real_; + launch::parallel_reduce( + "solprop_native.monitor.split_removal_real", samples, + KOKKOS_LAMBDA(const std::size_t row, const std::size_t index, + SplitSums& update) { + const double value = time(row, index); + const double power = + (stored ? weights(first_column + row) : parseval) * value * value; + update.values[0] += power * share(index); + const double found = half(index) * power; + update.values[1] += found; + update.values[2] += guards(0, index) * found; + update.values[3] += guards(1, index) * found; + update.values[4] += guards(2, index) * found; + }, + Kokkos::Sum(monitor_split_sum_)); + } + const auto removed = monitor_removed_; + const auto split = monitor_split_sum_; + const auto found = monitor_found_; + launch::parallel_for( + "solprop_native.monitor.split_removal_add", + Kokkos::RangePolicy(execution_space_, 0, 1), + KOKKOS_LAMBDA(const std::size_t) { + removed() += split().values[0]; + found() += split(); + }); +} + +// The split-step temporal absorber on a separable geometry: the rate is the same at +// every node, so it acts on the temporal rows (free space's nodes, the planar kind's +// y rows of slab amplitudes) between the two transverse transforms. +void GeneralWorkspace::separable_split_step(const double dz, + const ComplexView2& advanced) { + if (split_rows_ != 0) { + split_state_rows(dz, advanced); + return; + } + if (problem_.free_geometry()) { + // The split's factor depends on time alone, so it acts alike on every column of + // any transverse basis (solprop's reference measures its removal on the stored + // columns for this reason): free space splits its stored transverse-spectrum + // columns directly, chunk by chunk of the temporal rows, as a modal route splits + // its modes, with no transverse transform pair. Equal to the reference's split in + // real space up to the transforms' rounding. + const auto coarse = coarse_spectrum_; + const auto active = active_mask_; + const auto n_pol = problem_.dims.n_pol_state; + const auto n = problem_.dims.n_omega; + const double inverse_time = 1.0 / static_cast(problem_.dims.n_t); + for (std::size_t unit = 0; unit < transverse_units(); + unit += transverse_units_per_chunk()) { + const auto chunk = transverse_chunk(unit); + const auto first = chunk.first_row * n_pol; + const auto rows = chunk.rows * n_pol; + launch::parallel_for( + "solprop_native.free.split_columns_input", + Range2(execution_space_, {0, 0}, {rows, n}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + coarse(row, omega) = inverse_time * advanced(first + row, omega); + }); + split_columns(dz, rows, 0.0, first); + launch::parallel_for( + "solprop_native.free.split_columns_output", + Range2(execution_space_, {0, 0}, {rows, n}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + advanced(first + row, omega) = + active(omega) != 0U ? coarse(row, omega) : Complex{}; + }); + } + return; + } + const auto scratch = transverse_; + const auto coarse = coarse_spectrum_; + const auto active = active_mask_; + const auto n_pol = problem_.dims.n_pol_state; + const auto layout = transverse_layout(); + const auto n = problem_.dims.n_omega; + const double inverse_nodes = inverse_transverse_count(); + const double inverse_time = 1.0 / static_cast(problem_.dims.n_t); + launch::parallel_for( + "solprop_native.free.split_input", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + const auto mode = row / n_pol; + layout.cell(scratch, omega * n_pol + row % n_pol, mode) = + inverse_nodes * advanced(row, omega); + }); + transverse_inverse(); + for (std::size_t unit = 0; unit < transverse_units(); + unit += transverse_units_per_chunk()) { + const auto chunk = transverse_chunk(unit); + const auto first = chunk.first_row; + const auto count = chunk.rows; + graph::deep_copy(execution_space_, coarse, Complex{}); + launch::parallel_for( + "solprop_native.free.split_temporal_input", + Range2(execution_space_, {0, 0}, {count * n_pol, n}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + const auto mode = first + row / n_pol; + coarse(row, omega) = + inverse_time * layout.cell(scratch, omega * n_pol + row % n_pol, mode); + }); + // The rows are real-space rows here: by Parseval their power is the stored + // columns' over the transverse count, which the monitor's measure is in. + split_columns(dz, count * n_pol, 1.0 / inverse_nodes); + launch::parallel_for( + "solprop_native.free.split_temporal_output", + Range2(execution_space_, {0, 0}, {count * n_pol, n}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + const auto mode = first + row / n_pol; + layout.cell(scratch, omega * n_pol + row % n_pol, mode) = coarse(row, omega); + }); + } + transverse_forward(); + launch::parallel_for( + "solprop_native.free.split_output", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), n}), + KOKKOS_LAMBDA(const std::size_t row, const std::size_t omega) { + const auto mode = row / n_pol; + advanced(row, omega) = + active(omega) != 0U + ? layout.cell(scratch, omega * n_pol + row % n_pol, mode) + : Complex{}; + }); +} + +void GeneralWorkspace::form_split_factor(const double dz) { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const auto rate = absorber_coarse_; + const auto factor = split_factor_; + launch::parallel_for( + "solprop_native.absorber.split_factor", + Kokkos::RangePolicy(execution_space_, 0, problem_.dims.n_t), + KOKKOS_LAMBDA(const std::size_t index) { + factor(index) = Kokkos::exp(-0.5 * rate(index) * dz); + }); +#else + static_cast(dz); +#endif +} + +void GeneralWorkspace::split_step(const double dz, const ComplexView2& advanced) { + form_split_factor(dz); + if (monitoring_split()) { + // The chunks below add what this step's split finds; the last step's is dropped. + const auto found = monitor_found_; + launch::parallel_for( + "solprop_native.monitor.split_found_clear", + Kokkos::RangePolicy(execution_space_, 0, 1), + KOKKOS_LAMBDA(const std::size_t) { found() = SplitSums{}; }); + } + if (problem_.separable_geometry()) { + separable_split_step(dz, advanced); + return; + } + const auto coarse = coarse_spectrum_; + const double inverse_n = 1.0 / static_cast(problem_.dims.n_t); + launch::parallel_for( + "solprop_native.absorber.split_pad", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + coarse(column, index) = inverse_n * advanced(column, index); + }); + split_columns(dz); + const auto active = active_mask_; + launch::parallel_for( + "solprop_native.absorber.split_commit", + Range2(execution_space_, {0, 0}, {problem_.n_columns(), problem_.dims.n_omega}), + KOKKOS_LAMBDA(const std::size_t column, const std::size_t index) { + advanced(column, index) = + active(index) != 0U ? coarse(column, index) : Complex{}; + }); +} + +} // namespace solprop_native diff --git a/native/src/general/workspace.hpp b/native/src/general/workspace.hpp new file mode 100644 index 000000000..f76c8ed94 --- /dev/null +++ b/native/src/general/workspace.hpp @@ -0,0 +1,1087 @@ +// SPDX-FileCopyrightText: 2026 John C. Travers and contributors +// +// SPDX-License-Identifier: GPL-3.0-or-later + +#pragma once + +// The general workspace (GeneralState's implementation), private to the units of +// src/general/: checks.cpp, setup.cpp, trial.cpp (the hot set, kept in one unit so +// that its inlining is the compiler's whole view), statistics.cpp and state.cpp. +#ifdef SOLPROP_NATIVE_STRICT_TU +#error "general/workspace.hpp contracts: a strict translation unit must not include it" +#endif + +#include "kernels.hpp" + +namespace solprop_native { + +// Refuse a problem the workspace cannot run, before it allocates (checks.cpp). +void require_general_problem(const Problem& problem); + +// What a Cartesian plane's widths read of its fluence (`plane_fluence_moments`). +inline constexpr std::size_t plane_moment_slots = 8; + +class GeneralWorkspace { +public: + explicit GeneralWorkspace(Problem problem); + // Every view, plan and execution instance is a member, released by destruction once + // the queued work has finished (state.cpp). + ~GeneralWorkspace(); + GeneralWorkspace(const GeneralWorkspace&) = delete; + GeneralWorkspace& operator=(const GeneralWorkspace&) = delete; + GeneralWorkspace(GeneralWorkspace&&) = delete; + GeneralWorkspace& operator=(GeneralWorkspace&&) = delete; + + void set_initial(std::span field, std::span noise); + std::vector trial_step(double dz, const TrialScalars& scalars); + void prepare_trial(double dz, const TrialScalars& scalars); + std::vector adams_source(const std::vector& input, + const TrialScalars& scalars); + void adopt(const std::vector& field, double dz); + double low_storage_step(double dz, const TrialScalars& scalars, bool rebuild_table); + void low_storage_flows_formed(bool rebuild_table); + void low_storage_separable_pass(std::size_t pass, double dz); + void low_storage_separable_input(std::size_t pass); + void low_storage_separable_fold(std::size_t pass, int mode, int term, double dz); + void low_storage_modal_pass(std::size_t pass, double dz); + void low_storage_modal_input(std::size_t pass); + void adams_allocate(std::size_t slots, bool compensated, bool interpolate); + void adams_compensation_start(std::size_t slot, const std::vector& rate); + void adams_start(std::size_t slot, const TrialScalars& scalars); + double adams_magnitude(); + // The controller's norm scale on the device: rtol * w_i * |u| + atol over the + // in-band magnitude, formed each trial without the magnitude crossing to the host. + void adams_scale(double rtol, double atol, const std::vector& weights); + void adams_upload_divisor(const std::vector& divisor); + std::vector adams_fused_trial(double dz, const AdamsWeights& weights, + const std::vector& divisor, + const TrialScalars& scalars); + void adams_fused_accept(double dz, const AdamsWeights& keep, std::size_t free, + const AdamsWeights& dense); + void adams_hold(); + void adams_restore(); + void adams_observe(const AdamsWeights& weights, const TrialScalars& scalars); + std::vector + adams_exponential_trial(double dz, const ExponentialWeights& weights, + const std::vector& divisor, const TrialScalars& step, + const TrialScalars& sample, const std::vector& rate); + void adams_exponential_accept(double dz, std::size_t free, const AdamsWeights& dense); + // The end of an accepted Adams step (a host fence; nothing on a device). + void adams_accepted_boundary() const; + void adams_exponential_observe(double xi, double dz, + const ExponentialWeights& weights); + // solprop SEP 47: a Runge-Kutta march's dense output in a native session. + void rk_dense_allocate(const std::vector& weights); + void rk_dense_hold(double dz); + void rk_dense_observe(double theta, const TrialScalars& carry); + void rk_dense_restore(); + void rk_dense_release() noexcept; + std::vector adams_row_names() const; + std::array adams_row_extents(const std::string& name) const; + std::vector adams_rows_to_host(const std::string& name) const; + void adams_rows_to_host(const std::string& name, std::complex* destination, + std::size_t size) const; + void adams_rows_from_host(const std::string& name, + const std::vector& values); + std::vector collect_trial(); + void table_upload(TableKind kind, std::vector edges, double lower, + double upper, const std::complex* coefficients, + std::size_t n_intervals, std::size_t n_terms, std::size_t n_values); + void table_evaluate(const std::vector& slots, + const std::vector& positions); + void table_rate(std::size_t slot, double z, bool before); + std::vector table_terms_to_host(TableKind kind, std::size_t slot) const; + std::vector phase_sum_to_host(const std::vector& weights); + std::vector weighted_error(const std::vector& scale); + void set_flow_actions(const std::vector& forward, + const std::vector& inverse, + const std::vector& identity); + std::vector form_flow_actions(std::size_t section, double dz, + const std::vector& fractions); + std::vector flow_actions_to_host(bool inverse) const; + std::vector adams_block_to_host() const; + std::vector adams_exponent_to_host() const; + void prepare_raman(const std::vector& rates, + const std::vector& densities); + std::vector spatial_statistics(std::uint64_t flags, double intensity_factor, + double z); + std::vector plane_fluence_trace(double center_x, double center_y, double dx, + double dy, double extent, double spacing, + std::size_t n_radii, + const std::vector& cosines, + const std::vector& sines); + std::vector axis_spectrum(const std::vector& matrix); + std::vector + axis_populations(const std::vector& matrix, + const std::vector& channel_preionisation); + std::vector aperture_statistics(std::uint64_t flags, + const std::vector& apertures, + int exponent, + const std::vector& regions, + double intensity_factor); + std::vector statistics(std::uint64_t flags, + const std::vector& windows, + const std::vector& on_axis, double density, + double taper, double preionisation, + double intensity_factor, + const std::vector& channel_preionisation, + const std::vector& power_weight); + std::vector whitened_statistics(const std::vector& root, + std::uint64_t flags, + const std::vector& windows, + double intensity_factor, bool spectral); + void finish(double dz); + void promote_fsal(); + void clear_fsal(); + // The boundary monitor found the temporal collar quiet (solprop SEP 21). + void set_absorber_quiet(const bool quiet) noexcept { absorber_quiet_ = quiet; } + void bind_monitor(const std::vector& weights, + const std::vector& column_weights); + void unbind_monitor() noexcept { monitor_bound_ = false; } + void bind_occupancy(const std::vector& buckets, std::size_t count); + std::vector monitor_reading(); + void bind_edge(const std::vector& faces, const std::vector& clip, + const std::vector& totals, + const std::vector& synthesis, std::size_t first_row); + void bind_edge_lines(const EdgeLines& lines); + EdgeNumbers edge_reading(bool faces, bool whole, bool totals, bool strip, + bool want_current); + // What reading the edge holds in the state's memory space (solprop SEP 44). + [[nodiscard]] std::size_t edge_bytes() const noexcept { + std::size_t total = + (edge_faces_.size() + edge_clip_.size() + edge_totals_.size() + + edge_synthesis_.size() + edge_columns_.size() + edge_scale_.size()) * + sizeof(double); + for (const auto& line : edge_lines_) + total += line.positions.size() * sizeof(std::size_t) + + line.window.size() * sizeof(double); + return total + edge_strip_.size() * sizeof(Complex); + } + // The bytes through this state's crossings since it was built, each way (solprop SEP + // 44). + [[nodiscard]] CrossedBytes crossed_bytes() const noexcept { return crossed_; } + std::vector field_to_host() const; + void field_to_host(std::span destination) const; + const Complex* host_field() const { + if (!initialized_) + throw std::logic_error("no initial field has been loaded"); + if constexpr (!static_cast( + Kokkos::SpaceAccessibility::accessible)) + return nullptr; + execution_space_.fence("solprop_native host_field"); + return current_.data(); + } + void count_to_host(const std::size_t bytes) const noexcept { + crossed_.to_host += bytes; + } + void cross_to_host(const ComplexView2& source, std::span destination) const; + void set_crossing_workers(const std::size_t workers) noexcept { + crossing_ring_.set_workers(workers); + } + std::vector stage_to_host(std::size_t stage) const; + void stage_to_host(std::size_t stage, std::span destination) const; + std::vector projected_to_host(bool coarse); + std::map dispersive_path() const; + std::map memory_ledger() const; + /** The team a compact problem's FFTW loops take (`CpuTeamScope`): its pools' + widest instance, which every threaded kernel and transform of it uses. Zero for + any other problem: the runtime's team. */ + [[nodiscard]] int cpu_team() const noexcept { + return compact_ ? std::max({kernel_pool_.widest(), fft_pool_.widest(), + raman_pool_.widest(), raman_tail_pool_.widest()}) + : 0; + } + const CpuExecution& cpu_execution() const noexcept { + return kernel_pool_.execution(); + } + CpuSizing execution_sizing() const { + auto result = kernel_pool_.sizing(); + result.merge(fft_pool_.sizing()); + result.merge(raman_pool_.sizing()); + result.merge(raman_tail_pool_.sizing()); + if (general_) + result.merge(general_->execution_sizing()); + return result; + } + std::map base_memory_ledger() const; + double planning_seconds() const noexcept { return planning_seconds_; } + double raman_planning_seconds() const noexcept { return raman_planning_seconds_; } + std::string raman_fft_provider() const { + return raman_drive_forward_plan_ ? raman_drive_forward_plan_->provider() : "none"; + } + std::map raman_fft_tasks() const { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + const int full = 0; + const int tail = 0; + const int kernel = 0; + const int products = 0; +#else + const int full = raman_execution_space_.concurrency(); + const int tail = raman_tail_execution_space_.concurrency(); + const int kernel = raman_kernel_execution_space_.concurrency(); + const int products = raman_product_execution_space_.concurrency(); +#endif + // `products` is nonzero when the modal products replace the node drives. + return { + {"full", + raman_drive_forward_plan_ ? raman_drive_forward_plan_->tasks(full) : 0}, + {"tail", raman_tail_forward_plan_ ? raman_tail_forward_plan_->tasks(tail) : 0}, + {"kernel", + raman_kernel_forward_plan_ ? raman_kernel_forward_plan_->tasks(kernel) : 0}, + {"products", raman_product_forward_plan_ + ? raman_product_forward_plan_->tasks(products) + : 0}}; + } + std::size_t raman_kernel_transforms() const noexcept { + return raman_kernel_transforms_; + } + // The last accepted step's score, read from the device when asked for. + double quadrature_error() const; + // The largest score since the last call, then reset (NaN when none was scored). + double quadrature_peak(); + std::string synthesis_provider() const; + std::string projection_provider() const; + static std::string_view gemm_shape_policy(); + std::size_t response_team_size() const noexcept; + bool fsal_ready() const noexcept { return has_recycled_; } + std::size_t rk_source_evaluations() const noexcept { return rk_source_evaluations_; } + std::size_t axis_scans() const noexcept { return axis_scans_; } + + // NVCC extended device lambdas require their enclosing member to be public. + // GeneralWorkspace is private to src/general/, so this does not widen the API. + std::size_t fine_spectrum_size() const { + return problem_.dims.family == FieldFamily::field_real ? problem_.dims.n_to / 2 + 1 + : problem_.dims.n_to; + } + std::size_t projected_spectrum_size() const { + return problem_.dims.family == FieldFamily::envelope ? problem_.dims.n_to + : problem_.dims.n_to / 2 + 1; + } + void require_trial_scalars(const TrialScalars& scalars) const; + void require_adams() const; + void adams_phase(); + void adams_evaluate_source(const TrialScalars& scalars); + void adams_evaluate(const TrialScalars& scalars); + std::vector + adams_elementwise_trial(double dz, const ExponentialWeights& weights, + const std::vector& divisor, const TrialScalars& step, + const TrialScalars& sample, const std::vector& rate); + void adams_rate(const std::vector& rate, const ComplexView2& out); + void adams_fallback_squarings(std::size_t section, double dz, + const ExponentialWeights& weights); + // A trial's `rows` norms: its reduction over `policy` into the pinned buffer, then + // the trial's one fence, on the reduction's own instance. A reduction into memory the + // device writes returns before it completes. + template + std::vector adams_norms(const char* label, const Policy& policy, + const Reducer& reducer, const std::size_t rows) { + const Kokkos::View> + result(adams_norms_.data(), rows); + launch::parallel_reduce(label, policy, reducer, result); + policy.space().fence("solprop_native Adams trial boundary"); + std::vector norms(rows); + for (std::size_t row = 0; row < rows; ++row) + norms[row] = std::sqrt(result(row)); + adams_trial_ready_ = true; + return norms; + } + void allocate(); + void allocate_separable(std::size_t n); + void allocate_low_storage(std::size_t columns, std::size_t n); + void allocate_statistics(std::size_t columns, std::size_t n, + std::size_t temporal_columns, std::size_t statistic_nodes, + std::size_t statistics_rows); + void allocate_raman(); + void allocate_transforms(std::size_t columns, std::size_t transform_rows, + std::size_t temporal_columns); + void allocate_forward_rows(std::size_t columns, std::size_t chunk_rows, + std::size_t temporal_columns); + void allocate_absorbers(); + void allocate_envelope_plasma(std::size_t chunk_rows, std::size_t temporal_columns); + void allocate_plasma(std::size_t chunk_rows); + void allocate_split(std::size_t n, std::size_t temporal_columns); + void allocate_parts(); + void upload(); + void upload_trial(const TrialScalars& scalars, const TrialScalars* sample = nullptr, + const std::vector* rate = nullptr); + void use_uploaded_sample(const TrialScalars& sample); + void bind_stage_scalars(std::size_t slot); + void note_trial_scalars(const TrialScalars& scalars); + void plan(); + void plan_transverse(); + void plan_raman(); + void plan_split(); + void reuse_fsal(); + void evaluate_stage(std::size_t stage, double dz); + // `accumulate` false: an Adams trial reads the stage row alone. + void evaluate_stage_source(std::size_t stage, double dz, bool accumulate = true); + bool modal_core(std::size_t stage); + void general_core(std::size_t stage); + StageTerms stage_terms(std::size_t stage) const; + std::vector> fine_gap_runs() const; + void stage_entry(std::size_t stage, double dz); + template void stage_entry_with(std::size_t stage, double dz); + void finish_or_defer(std::size_t stage); + // Free space and the hybrid planar kind (`Problem::separable_geometry`). + TransverseLayout transverse_layout() const noexcept { + const std::size_t plane = transverse_.extent(1) * transverse_.extent(2); + return problem_.slab_geometry() ? TransverseLayout{.inner = problem_.spatial.n_p, + .channel_fastest = true, + .plane = plane} + : TransverseLayout{.inner = problem_.spatial.n_x, + .channel_fastest = false, + .plane = plane}; + } + RegisterIndex register_index() const noexcept { + constexpr bool plane = build::host; + return {.layout = transverse_layout(), + .n_pol = problem_.dims.n_pol_state, + .n = problem_.dims.n_omega, + .plane = plane, + .scalar = problem_.dims.n_pol_state == 1}; + } + // A stage's scratch positions: the volume route's own order on a device, or the + // scratch's (`register_index`). Every other pass over the scratch forms its own input + // in `(omega, y, x)`, since the scratch is dead between them. + RegisterIndex stage_index() const noexcept { + auto index = register_index(); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + index.volume = volume_order_; +#endif + return index; + } + // The unscaled transverse DFT's sample count: the plane's, or the slab's y axis. + double inverse_transverse_count() const noexcept { + return 1.0 / static_cast(problem_.slab_geometry() ? problem_.spatial.n_y + : problem_.dims.n_modes); + } + std::size_t transverse_units() const noexcept { + return problem_.slab_geometry() ? problem_.spatial.n_y : problem_.dims.n_nodes; + } + std::size_t transverse_units_per_chunk() const noexcept { + return problem_.slab_geometry() ? problem_.dims.node_chunk / problem_.spatial.n_x + : problem_.dims.node_chunk; + } + TransverseChunk transverse_chunk(std::size_t first_unit) const noexcept; + void transverse_forward(); + void transverse_inverse(); + void transverse_inverse(const ComplexView3& scratch); + void flux_at(double z); + void require_flux_closure(); + void fill_statistics_closure(); + void evaluate_separable_stage(std::size_t stage, double dz, bool accumulate); + void seed_free_wisdom(); + void seed_modal_wisdom(); + void seed_stream_wisdom(); + void separable_core(std::size_t stage, bool material); + void separable_spatial_core(); + void separable_volume_core(std::size_t stage); + void separable_stage_input(std::size_t stage, double dz); + void separable_load_current(); + void separable_pack_chunk(const TransverseChunk& chunk, bool material); + void separable_nodes(const TransverseChunk& chunk); + void slab_project(const TransverseChunk& chunk, const EnvelopeStream* stream); + void separable_finish_transform(std::size_t stage, bool material); + void separable_time_statistics(std::size_t axis_peak, std::size_t peak, bool fluence, + double intensity_factor); + void separable_fluence(double intensity_factor); + FarFieldPass::Inputs far_field_inputs() const; + void plane_fluence_moments(double (&moments)[plane_moment_slots]); + void scale_transform(std::size_t stage, double taper); + void synthesise(std::size_t first_node, std::size_t node_count); + void response(std::size_t stage, std::size_t first_node, std::size_t node_count); + void prepare_raman_stage(std::size_t stage, const double* rates, + const double* densities = nullptr); + void convolve_raman(std::size_t stage, std::size_t node_count); + void finish_raman(std::size_t stage, std::size_t node_count); + // A scalar state's Raman drive is a quadratic form of its modal fields, so the + // causal convolution is applied once per unordered modal product and each node's + // convolved drive is reconstructed from them (`raman_modal_products`). + [[nodiscard]] std::size_t raman_modal_products() const noexcept; + void convolve_raman_products(); + void finish_raman_products(std::size_t stage, std::size_t first_node, + std::size_t node_count); + template + void response_plasma(std::size_t stage, std::size_t first_node, + std::size_t node_count); + template + void response_nonlinear(std::size_t stage, std::size_t first_node, + std::size_t node_count); + template + void response_stash(std::size_t stage, std::size_t first_node, + std::size_t node_count); + template + void response_harmonic(std::size_t stage, std::size_t first_node, + std::size_t node_count); + template + void response_raman(std::size_t stage, std::size_t first_node, + std::size_t node_count); + template + void response_impl(std::size_t stage, std::size_t first_node, std::size_t node_count); +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // The device's response (`device_response`) and its trapezoid plasma's two-level + // scans. + template + void device_response(std::size_t stage, std::size_t first_node, + std::size_t node_count); + template + void device_trapezoid_plasma(std::size_t stage, std::size_t first_node, + std::size_t node_count); +#endif + // solprop SEP 18's band rule as a team column (include/solprop_native/plasma.hpp): + // filter, then scan, for the population and free current, in whole-row passes. + // It writes the current separately from the collared non-plasma response. + template + void team_band_plasma_column(std::size_t stage, std::size_t first_node, + std::size_t node_count); + // The same column in tiles (solprop SEP 25 section 6.7.6, the device form): each + // running integral is a local kernel, one team per (row, tile) filtering and scanning + // its tile from zero, and a finish kernel adding the earlier tiles' totals and + // forming the pass's product. + template + void tiled_band_plasma_column(std::size_t stage, std::size_t first_node, + std::size_t node_count); + // The tiled column's teams: `rows` x tiles of them, as wide as the build's response + // team on a device (or `band_tile_threads_` on the host) and the kernel allows, with + // the tile's scratch when `scratch`. + template + [[nodiscard]] Kokkos::TeamPolicy + tiled_band_policy(std::size_t rows, bool scratch, const Functor& functor) const; + void tiled_band_local(const RealView2& source, std::size_t rows, const char* label); + // The band rule's first stage alone at every node of a chunk: the floored final + // exposure and largest rate per channel, and with `Diagnose` the six per-node parts + // of the run diagnostics; team scans with team reductions. + template + void team_band_statistics(std::size_t first_node, std::size_t node_count, + const RealView2& output, double preionisation); + // The band column's teams: on a device one per node, as wide as the build's team and + // the kernel's resources allow; on the host the threads shared between the nodes. + // Each team walks the nodes `slot, slot + league, ...` with its slot's rows. + template + [[nodiscard]] Kokkos::TeamPolicy + band_team_policy(std::size_t node_count, const Functor& functor) const; + [[nodiscard]] std::size_t band_league(std::size_t node_count) const; + [[nodiscard]] int band_host_team_size(std::size_t node_count) const; + void project(std::size_t first_node, std::size_t node_count, bool first_chunk, + const EnvelopeStream* stream = nullptr, + const FieldCurrentStream* field_stream = nullptr); + bool field_plasma() const noexcept { + return problem_.plasma.enabled && !problem_.plasma.envelope; + } + bool held_plasma() const noexcept { return field_plasma() && !problem_.plasma.gated; } + // solprop's private gated benchmark: the current's running integral, gated by the + // time window, joins the local polarisation before it is projected. + bool gated_plasma() const noexcept { return field_plasma() && problem_.plasma.gated; } + bool held_radial() const noexcept { + return held_plasma() && !problem_.separable_geometry() && + !problem_.spatial.spectral_weights.empty(); + } + // Whether a device takes a modal stage's forward transforms -- the projection's + // (fine and coarse), the in-stage absorber's and the held currents', each a real row + // of n_to samples -- as one batched transform at the end of the stage + // (`transform_forward_pack`). On an AMD Instinct MI325X each transform of a + // single-mode run was four rocFFT kernels of a few microseconds, launched one by one; + // on an NVIDIA A40 a transform of a four-mode capillary's rows took about 96 + // microseconds on 16 of its 84 multiprocessors, whatever the rows. Batched, the + // stage's rows share them. A device never defers the coarse quadrature + // (`deferred_quadrature_eligible`), so the coarse rows are projected at every stage. + // A host packs them too, unless the problem is compact or SOLPROP_NATIVE_FORWARD_PACK + // is 0 (solprop issue #584): one transform of every row, threaded by its shape, in + // place of one a transform over only the modes' rows, which left most of the + // threads idle: on 18 cores of an Intel Xeon Gold 6240 the pack ran rdw 0.90, + // argon_blueshift 0.88 and ctc 0.92 of their windows. A host keeps the coarse rows + // after the fine ones so that a deferred quadrature's stages transform the fine block + // alone. + bool packs_forward() const noexcept { +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // A compact problem keeps its separate transforms: its one row a transform runs + // serially below the FFT start, and the pack's few rows on the compact team ran + // averaged helium 1.35 times slower on 18 cores of an Intel Xeon Gold 6240 and + // 1.11 on 16 of an AMD EPYC 7763 (solprop issue #584). + if (!host_forward_pack() || compact_) + return false; +#endif + return problem_.dims.family == FieldFamily::field_real && + !problem_.separable_geometry() && !problem_.general.enabled && + !problem_.dispersive.enabled && !problem_.plasma.envelope && + !held_radial() && + (problem_.absorber.in_stage || held_plasma() || + !problem_.transform.projection_coarse_matrix.empty()); + } + static bool host_forward_pack() noexcept { + static const bool enabled = [] { + const char* text = std::getenv("SOLPROP_NATIVE_FORWARD_PACK"); + return text == nullptr || std::string_view(text) != "0"; + }(); + return enabled; + } + void transform_forward_pack(); + void held_response(std::size_t stage, std::size_t first_node, std::size_t node_count, + const TransverseChunk* chunk = nullptr); + void gated_response(std::size_t node_count); + void held_transform(bool coarse_only = false, bool transformed = false); + void finish_quadrature(); + bool deferred_quadrature_eligible() const noexcept { + if constexpr (build::device) + return false; + // The validated seven-stage FSAL tableau identifies the ordinary DOPRI5 + // route without another seam field. Other consumers retain eager spectra. + return problem_.stepper.first_same_as_last && problem_.stepper.n_stages == 7 && + !problem_.stepper.low_storage && !problem_.separable_geometry() && + !general_ && !flow_ && !dispersive_ && !problem_.plasma.envelope && + !held_radial() && !problem_.transform.projection_coarse_matrix.empty(); + } + void held_radial_projection(std::size_t first_node, std::size_t node_count); + void envelope_cycle(std::size_t channel, std::size_t node_count, + const ComplexView2& field_rows, std::size_t n_pol); + void envelope_statistics(std::size_t channel, std::size_t first_node, + std::size_t node_count, const RealView2& statistics, + const ComplexView2& field, std::size_t n_pol); + void envelope_population(std::size_t channel, std::size_t stage, + std::size_t node_count); + void envelope_response(std::size_t stage, std::size_t first_node, + std::size_t node_count, + const TransverseChunk* chunk = nullptr); + void envelope_transform(); + void finish_stage(std::size_t stage); + void score_quadrature(); + void fill_statistics_time(bool windowed, double lower, double upper, + bool weighted = false); + void fill_statistics_pair(bool windowed, double lower, double upper); + void pack_energy(RealView packed, RealView2 power_weights, std::size_t omega0_offset, + std::size_t energy_offset); + void statistics_window_energy(RealView packed, RealView2 power_weights, + std::size_t n_windows, std::size_t energy_window_offset, + const std::vector& windows); + void statistics_intensity(RealView packed, std::size_t intensity_offset, + const std::vector& on_axis, + double intensity_factor); + void statistics_plasma(RealView packed, std::size_t density_offset, + std::size_t rate_offset, std::size_t populations_offset, + std::size_t diagnostics_offset, bool diagnose, double density, + double taper, double preionisation); + void form_statistics_power(); + void form_statistics_total_power(); + void statistics_modal_time(); + void scan_plasma_statistics(std::size_t first_node, std::size_t node_count, + const RealView2& output, bool diagnose = false, + double preionisation = 0.0); + void form_absorber(); + void split_step(double dz, const ComplexView2& advanced); + void form_split_factor(double dz); + void split_columns(double dz, std::size_t active_rows = 0, double parseval = 0.0, + std::size_t first_column = 0); + void split_state_rows(double dz, const ComplexView2& state); + void add_split_removal(double dz, std::size_t active_rows, double parseval, + std::size_t first_column, const ComplexView2& complex_time, + double power_scale); + void separable_to_time(const TransverseChunk& chunk); + void separable_to_frequency(const TransverseChunk& chunk); + void separable_split_step(double dz, const ComplexView2& advanced); + void mask(const ComplexView2& values); + bool use_packed_radial_blas() const noexcept; + // A square radial analytic map synthesises the retained spectrum, before padding, + // and transforms each node (`retained_synthesis`). + bool retained_synthesis() const noexcept; + void synthesise_retained(std::size_t first_node, std::size_t node_count); + void packed_radial_gemm(const RealView2& matrix, const ComplexView2& source, + const ComplexView2& destination); + bool use_blas_for_synthesis() const noexcept; + bool use_interleaved_synthesis() const noexcept; + bool use_interleaved_projection() const noexcept; + bool hybrid_shape_eligible() const noexcept; + bool use_blas_for_projection() const noexcept; + bool use_chunked_blas_for_synthesis() const noexcept; + bool folds_single_node() const noexcept; + bool use_chunked_blas_for_projection() const noexcept; + [[nodiscard]] const ExecutionSpace& fft_space_for(std::size_t samples, + std::size_t length); + /** A batch's FFT instance, sized by its rows: extent 0 counts the rows, the + remaining extents make one row (a transverse plane counts as one row). */ + template + [[nodiscard]] const ExecutionSpace& fft_space_for_rows(const View& rows) { + return fft_space_for(rows.size(), row_length(rows)); + } + + Problem problem_; + // A compact problem's pools (`compact_problem`) and the team its FFTW loops take. + bool compact_{}; + // One real mode at several nodes in one chunk: synthesis and projection through + // the one-mode kernels on the response team (gemm.hpp). + bool one_mode_products_{}; + CpuExecutionPool kernel_pool_{CpuPool::kernels, problem_.cpu_execution}; + CpuExecutionPool fft_pool_{CpuPool::fft, problem_.cpu_execution}; + CpuExecutionPool raman_pool_{CpuPool::raman_fft, problem_.cpu_execution}; + CpuExecutionPool raman_tail_pool_{CpuPool::raman_tail_fft, problem_.cpu_execution}; + mutable ExecutionSpace execution_space_; + ExecutionSpace response_execution_space_, response_tail_execution_space_; + ExecutionSpace transform_execution_space_, transform_tail_execution_space_; + ExecutionSpace matrix_copy_space_, tail_matrix_copy_space_; + ExecutionSpace stacked_matrix_copy_space_, tail_stacked_matrix_copy_space_; + ExecutionSpace node_spectral_space_; + const ExecutionSpace& matrix_copy_space_for(std::size_t nodes, + bool stacked = false) const { + if (stacked) + return nodes == problem_.dims.node_chunk ? stacked_matrix_copy_space_ + : tail_stacked_matrix_copy_space_; + return nodes == problem_.dims.node_chunk ? matrix_copy_space_ + : tail_matrix_copy_space_; + } + ExecutionSpace fft_execution_space_; + ExecutionSpace temporal_execution_space_, flow_execution_space_, + absorber_execution_space_; + // The exponential Adams trial's passes and its accepted step's rows, sized by + // adams_allocate when the history's rows are known. + ExecutionSpace adams_execution_space_, adams_copy_space_; + ExecutionSpace raman_multiply_space_, raman_tail_multiply_space_, + raman_rebuild_space_, raman_scale_space_; + ExecutionSpace raman_execution_space_, raman_tail_execution_space_, + raman_kernel_execution_space_; + ExecutionSpace raman_product_execution_space_, raman_product_space_; + const ExecutionSpace& response_space_for(std::size_t nodes) const { + return nodes == problem_.dims.node_chunk ? response_execution_space_ + : response_tail_execution_space_; + } + const ExecutionSpace& transform_space_for(std::size_t nodes) const { + return nodes == problem_.dims.node_chunk ? transform_execution_space_ + : transform_tail_execution_space_; + } + bool initialized_{}; + // A constant operator's low-storage table is built (set on a rebuild, cleared by + // `set_initial` and `close`): a step that reuses a table never built refuses. + bool table_ready_{}; + bool stages_ready_{}; + bool has_noise_{}; + bool has_recycled_{}; + std::size_t rk_source_evaluations_{}; + // On-axis row scans (`axis_populations`), for the session's record (SEP 19). + std::size_t axis_scans_{}; + bool pending_fsal_{}; + // Only the final trial source supplies the accepted quadrature score. Its + // coarse time buffers survive until scoring, or until a new source replaces + // them. The stored accepted score and peak are independent of these flags. + bool defer_quadrature_{}; + bool project_coarse_{true}; + bool coarse_pending_{}; + // Set by a trial for a stage whose finish may run in the next stage's entry kernel; + // `finish_pending_` names the stage's finish still to run there. + bool defer_finish_{}; + bool finish_pending_{}; + // `problem_.grid.crop_index` is the identity (`StageTerms::identity_crop`). + bool identity_crop_{}; + // The bound trial's tapers on the host, and the one the scaled transforms hold + // (`scale_transform`). + std::vector trial_taper_; + double scaled_taper_{}; + bool scaled_taper_ready_{}; + std::size_t pending_stage_{}; + bool stash_is_current_{}; + std::size_t plan_heap_growth_{}, plan_device_growth_{}; + double planning_seconds_{}; + double raman_planning_seconds_{}; + // Every fixed Phi term is purely imaginary: the stage phases have no real part. + bool lossless_phase_ = false; + // Per stage of the current trial, whether its phase weights equal the stage + // before's, and whether the whole step's equal the last stage's (`upload_trial`); + // the last stage whose entry formed the pair in this trial. + std::vector phase_repeats_; + bool whole_repeats_ = false; + std::size_t entered_stage_ = std::numeric_limits::max(); + std::size_t raman_kernel_transforms_{}; + bool raman_kernel_ready_{}; + std::vector cached_raman_rates_, cached_raman_densities_; + mutable double quadrature_error_{Kokkos::Experimental::quiet_NaN_v}; + bool adams_device_scale_ = false; + double adams_rtol_ = 0.0, adams_atol_ = 0.0; + RealView adams_scale_weights_; + Kokkos::View adams_magnitude_sum_; + // The device's (last, peak) scores: an accepted step scores the quadrature on the + // device and nothing crosses until the score is read. + RealView quadrature_scores_; + mutable bool quadrature_stale_ = false; + SpatialCoefficients spatial_; + // solprop SEP 34: the plane measure's coefficients. A separable run's are its own + // operator's views (aliased, nothing more held); a radial run's are the entrance + // squares, the material square replaced by the material z table's value at the + // statistic's plane when the medium varies (`flux_table_`). + SpatialCoefficients flux_; + z_table::Table flux_table_; + // The magnetic closure's transverse volume on a separable run, beside + // `transverse_`, allocated when a local flux statistic is asked for. + ComplexView3 flux_transverse_; + ComplexView3 transverse_; + std::unique_ptr transverse_forward_, transverse_inverse_; + // On a device, a free-space envelope whose material response is the Kerr term alone, + // on a time grid the spectrum fills (n_t = n_to = n_omega, the crop the identity), + // with one polarisation: the stage core transforms the whole scratch volume in + // (omega, y, x) at once and forms the response there, in place of the chunks' + // transposes and temporal transforms (`separable_volume_core`). + bool volume_kerr_ = false; + // The volume route's stages hold the scratch as `(y, x, omega)`, `volume_scratch_`, + // on a device (solprop #365): its rank-3 transform then keeps frequency contiguous + // (cuFFT's `{N,N,T}`, rocFFT's `{T,N,N}`), and its input and fold no longer cross + // layouts. + bool volume_order_ = false; + ComplexView3 volume_scratch_; + std::unique_ptr volume_forward_, volume_inverse_; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + ComplexView transform_work_; +#endif + std::unique_ptr slab_forward_, slab_inverse_; + ComplexView2 current_, advanced_, solution_, error_, residual_, recycled_, noise_; + // solprop SEP 24 phase 3: the low-storage march's registers B and C (the state is + // A) and its flows. + ComplexView2 register_accumulator_, register_stage_; + ComplexView3 low_storage_flows_; + ComplexView2 whole_propagator_, fine_spectrum_, projected_spectrum_; + // The stage's `sum_t dPhi_t Phi_t`, formed by the combination for a coupled flow's + // residual, and its inverse propagator `exp(-sum)`, which the finish applies. + ComplexView2 stage_phase_, stage_inverse_; + ComplexView2 projected_coarse_spectrum_, absorber_spectrum_, modal_complex_; + ComplexView2 statistics_spectrum_, statistics_time_, statistics_weighted_time_; + RealView2 statistics_power_weight_; + bool statistics_unit_weight_ = true; + ComplexView2 node_complex_, polarisation_complex_, projected_complex_; + ComplexView2 projected_coarse_complex_, absorber_complex_, coarse_spectrum_; + ComplexView2 coarse_time_complex_, scaled_synthesis_complex_; + ComplexView2 scaled_projection_complex_, scaled_projection_coarse_complex_; + ComplexView2 scaled_projection_stacked_complex_, projected_stacked_complex_; + ComplexView2 chunk_synthesis_complex_, tail_synthesis_complex_; + ComplexView2 chunk_projection_complex_, tail_projection_complex_; + ComplexView2 chunk_projection_stacked_complex_, tail_projection_stacked_complex_; + ComplexView2 node_raman_spectrum_, raman_kernel_spectrum_; + ComplexView2 raman_tail_spectrum_; + ComplexView2 spatial_absorber_complex_; + ComplexView3 stages_, phi_terms_, prefactor_terms_; + // solprop SEP 29 G1: the general z tables' coefficients, uploaded once; their values + // at a trial's positions are written into the term slots above. + z_table::Table phi_table_, prefactor_table_, rate_table_; + RealView2 radial_blas_input_, radial_blas_output_; + // The retained synthesis's coarse modal and node spectra. + ComplexView2 retained_modal_, retained_node_; + bool retained_synthesis_ = false; + RealView2 modal_real_, node_real_, polarisation_real_, projected_real_; + RealView2 projected_coarse_real_, projected_stacked_real_, absorber_real_; + RealView2 coarse_time_real_; + RealView2 node_raman_drive_, node_raman_convolved_, raman_undamped_; + RealView2 raman_tail_drive_, raman_tail_convolved_; + // The modal products' rows, the leading rows of the node drive and its spectrum: + // the drive rows hold each product, then its convolution. + RealView2 raman_product_drive_; + ComplexView2 raman_product_spectrum_; + std::size_t raman_products_ = 0; + RealView2 raman_kernel_time_, stage_raman_rates_, stage_raman_densities_; + RealView2 raman_tensor_weights_; + RealView raman_row_component_, raman_component_fraction_; + RealView2 synthesis_, projection_, projection_coarse_; + RealView2 spatial_absorber_real_; + RealView2 scaled_synthesis_, scaled_projection_, scaled_projection_coarse_; + RealView2 scaled_projection_stacked_; + RealView2 chunk_synthesis_, tail_synthesis_; + RealView2 chunk_projection_, tail_projection_; + RealView2 chunk_projection_stacked_, tail_projection_stacked_; + RealView2 partials_, quadrature_partials_, stash_candidate_, stash_current_; + RealView2 statistics_power_, plasma_statistics_scratch_; + // solprop SEP 34: each node's `|E| |G|` beside its signed flux, which bounds a radial + // fluence's rounding; allocated with the closure's buffers. + RealView2 statistics_magnitude_; + // `spatial_statistics_partials_` holds a plane's scaled fluence for its trace. + bool plane_fluence_held_ = false; + RealView2 plasma_rate_, plasma_exposure_, plasma_fraction_; + RealView2 plasma_current_integrand_, plasma_free_current_; + // The device trapezoid plasma's tile totals, by component row, and its tile count + // (`device_trapezoid_plasma`). + RealView2 plasma_tile_totals_; + std::size_t plasma_tiles_{}; + RealView2 plasma_outer_current_; + RealView weighted_scale_, weighted_partials_; + RealView spatial_spectral_weights_; + ComplexView spatial_on_axis_; + RealView2 spatial_statistics_partials_; + RealView statistics_total_power_; + RealView statistics_on_axis_, statistics_packed_; + // An envelope's on-axis row on a one-component node scan, its x and y components + // (`axis_populations`), allocated by the first statistics call that needs it. + ComplexView2 statistics_axis_row_; + // solprop SEP 47: a general modal basis's whitened columns, the metric's square + // root at one position and their sums, allocated by the first statistics call that + // needs them (outside any trial step). + ComplexView2 statistics_whitened_; + ComplexView statistics_root_; + RealView statistics_whitened_sums_; + RealView tableau_, omega_, omega_window_, time_window_, absorber_fine_, + absorber_coarse_; + RealView stage_phi_delta_, stage_phi_at_, stage_prefactor_; + RealView stage_density_, stage_taper_, stage_preionisation_, rate_coefficients_; + RealView stage_kerr_density_, stage_plasma_density_, stage_raman_density_; + // Every per-stage scalar view above, a Raman run's stage rates and amplitude + // densities and a compensated Adams trial's rate weights live in one buffer of two + // slots, uploaded in one copy a trial from the next of a ring of pinned host buffers + // (`upload_trial`, solprop SEP 39 section 6.7): an Adams trial's step scalars in + // slot 0 and its sample's in slot 1. The views point into the slot in use. + RealView stage_scalars_; + using StagingView = decltype(Kokkos::create_mirror_view( + Kokkos::SharedHostPinnedSpace{}, std::declval())); + std::vector stage_staging_; + std::size_t stage_staging_next_ = 0; + std::size_t stage_slot_size_ = 0; + std::array stage_offsets_{}; + RealView stage_rate_weights_; + RealView2 channel_parameters_, stage_channel_density_, stage_channel_preionisation_; + RealView channel_coefficients_; + Kokkos::View channel_scratch_; + // solprop SEP 18 on a CPU build: the running-integral rule's taps, one row of ring + // scratch per node of a chunk (the band column's, or the statistics scan's alone + // under the trapezoid) and the run diagnostics' six parts per node. + RealView plasma_taps_; + RealView2 plasma_diagnostics_; + // The band column's scratch rows per kind: one per concurrently running team. + std::size_t band_slots_{}; + // Whether the band rule runs the tiled column (`tiled_band_plasma_column`), its + // tile's samples, the tiles of a row, a host tile team's threads, and the tiles' + // totals by row. + bool band_tiled_{band_tiled(thread_policy())}; + // Each column's norm reductions on a device are this many teams (`reduction_slots`): + // the norm partials hold `columns * reduction_slots_` slots, reduced on the host. + std::size_t reduction_slots_{ + reduction_slots(problem_.n_columns(), problem_.dims.n_omega)}; + std::size_t band_tile_{}; + std::size_t band_tile_count_{}; + int band_tile_threads_{band_tile_threads(thread_policy())}; + RealView2 band_tile_totals_; + ComplexView carrier_; + std::array envelope_streams_; + std::array held_streams_; + // A device's two held currents, fine and coarse rows of each, in one buffer and its + // spectrum, which one plan transforms (`held_transform`); empty on a host and on the + // radial route. + RealView2 held_pack_; + ComplexView2 held_pack_spectrum_; + std::unique_ptr held_pack_forward_; + // A device's forward rows of a stage, one buffer and its spectrum (`packs_forward`): + // the in-stage absorber's rows, then the projection's, then the held pack's, and the + // plans of all of them and of the rows after the absorber's (a quiet collar's stage). + RealView2 forward_pack_, forward_pack_tail_; + ComplexView2 forward_pack_spectrum_, forward_pack_tail_spectrum_; + std::unique_ptr forward_pack_plan_, forward_pack_tail_plan_; + std::size_t forward_pack_absorber_rows_ = 0; + // A host's deferred quadrature (`deferred_quadrature_eligible`): the pack's fine + // block (the absorber's, the projection's and the held currents' fine rows) with and + // without the absorber's, for the stages, and its coarse block for the finish. + RealView2 forward_pack_fine_, forward_pack_fine_tail_, forward_pack_coarse_; + ComplexView2 forward_pack_fine_spectrum_, forward_pack_fine_tail_spectrum_, + forward_pack_coarse_spectrum_; + std::unique_ptr forward_pack_fine_plan_, + forward_pack_fine_tail_plan_, forward_pack_coarse_plan_; + RealView2 held_current_, held_projection_; + // The gated benchmark's running integral of each current row, and the band rule's + // filtered rows (`gated_response`). + RealView2 gated_prefix_, gated_filtered_; + ComplexView2 held_nodal_spectrum_, held_projected_spectrum_; + RealView held_derivative_, held_inverse_; + RealView3 envelope_rule_nodes_, envelope_rule_weights_, envelope_moments_; + RealView2 envelope_rates_, envelope_exposure_; + IndexView envelope_failures_, envelope_status_; + Kokkos::View envelope_host_status_; + RealView envelope_drude_factor_; + ComplexView envelope_loss_factor_; + ByteView active_mask_, norm_mask_; + // solprop SEP 16 phase 3b: the resident Adams history and its trial's scratch. + ComplexView3 adams_store_, adams_slopes_, adams_moments_; + ComplexView2 adams_source_, adams_propagator_, adams_exponent_, adams_saved_; + ComplexView2 adams_held_, adams_dense_start_, adams_dense_end_, adams_dense_exponent_; + Kokkos::View adams_divisor_; + // A Runge-Kutta trial's norm partials cross through this mirror, pinned on a device + // (the view itself on a host build), so that no trial takes a fresh host buffer. The + // copy is asynchronous: it is read after the trial's fence. + decltype(Kokkos::create_mirror_view(Kokkos::SharedHostPinnedSpace{}, + std::declval())) pinned_partials_; + // A trial's squared norms, which its reduction writes and the host reads: in pinned + // host memory on a device, which the reduction's last block writes directly, so that + // no copy from the device follows it before the trial's one fence. + Kokkos::View adams_norms_; + // An elementwise trial's source evaluation, replayed as a device graph. + graph::Region adams_source_graph_; + // A varying operator's compensation: each sample's state and rate, the trial's + // carried rows, its predicted field, endpoint term and rate, and dense observation's + // terms. + ComplexView3 adams_states_, adams_rates_, adams_carried_, adams_dense_terms_; + ComplexView2 adams_lab_, adams_endpoint_, adams_endpoint_rate_; + Kokkos::View adams_rate_weights_; + // A coupled flow's exponential trial (solprop SEP 16 phase 6): the step's flow and + // moments on the block's bins, `(count + 1, n_active, n_modes, n_modes)` with the + // flow first; a frozen-residual flow's weights at each sample and at the trial's end. + Kokkos::View adams_block_; + // A host's channel vectors per bin, `(n_omega, (adams_max_samples + 1) n_modes)`, + // beside a block wider than `adams_max_channels` (solprop SEP 47 phase 4); empty + // otherwise, when a host task holds them on its stack. + ComplexView2 adams_channels_; + // Each build's functions of the bins off the flow's eigenbasis route, `(n_active, + // augmented_blocks(slots), n_modes, n_modes)` by active bin: the augmented + // exponential's first block row and its scratch for the trial's section (solprop + // SEP 47 phase 4); empty without a bin off the route. + Kokkos::View adams_fallback_; + // A frozen-residual flow's generator at each sample applied to its state, `G(z_j) + // u_j`, formed once per sample; the trial's at its end, and the weights it is formed + // from. A trial forms `R_s(z_j) u_j = G(z_j) u_j - G_s u_j` against its own section. + ComplexView3 adams_applied_; + ComplexView2 adams_end_applied_; + Kokkos::View adams_end_weights_; + std::size_t adams_dense_count_{}; + // solprop SEP 47: a Runge-Kutta step's continuous extension for dense output, its + // five coefficients `(5, columns, n_omega)`; the bins its flow changes by more than + // an e-fold; the committed state held across an observation; the tableau's + // dense-output weights; and whether a step is held. + ComplexView3 rk_dense_coefficients_; + Kokkos::View rk_dense_stiff_; + ComplexView2 rk_dense_held_; + std::vector rk_dense_weights_; + bool rk_dense_ready_{}; + bool adams_trial_ready_{}; + bool adams_trial_compensated_{}; + bool adams_trial_coupled_{}; + bool adams_dense_compensated_{}; + // The persistent row view a resume exports or installs, by name (SEP 29 G2). + ComplexView3 adams_row_view(const std::string& name) const; + IndexView crop_index_; + // The fine bins outside the crop, zeroed by the stage's entry kernel. + IndexView fine_gaps_; + // The separable core's fine bins outside the crop, zeroed per chunk instead of the + // whole fine spectrum (SEP 28); empty when the crop fills every fine bin. + IndexView separable_gaps_; + // Free space's node field and polarisation alias the chunk's time rows and their + // projected rows (SEP 28): the copies between them are identities. + bool separable_aliased_{}; + // The in-stage collar with no temporal rate beside it: its rate depends on the + // column alone, so its pass needs no temporal pair (solprop SEP 40 section 4.7). + bool spatial_only_absorber_{}; + std::size_t temporal_tail_rows_{}; + TailRowPlans temporal_tail_inverse_real_, + coarse_tail_inverse_real_; + TailRowPlans temporal_tail_forward_real_, + coarse_tail_forward_real_; + TailRowPlans + temporal_tail_inverse_complex_, temporal_tail_forward_complex_, + coarse_tail_inverse_complex_, coarse_tail_forward_complex_; + std::unique_ptr inverse_real_plan_; + std::unique_ptr inverse_complex_plan_; + std::unique_ptr forward_real_plan_; + std::unique_ptr forward_complex_plan_; + std::unique_ptr forward_coarse_real_plan_; + std::unique_ptr forward_coarse_complex_plan_; + std::unique_ptr statistics_inverse_plan_; + std::unique_ptr absorber_real_plan_; + std::unique_ptr absorber_complex_plan_; + std::unique_ptr coarse_inverse_real_plan_; + // A free-space envelope's split collar transforms the state's own rows in place, this + // many at a time, and holds no coarse buffers (`split_state_rows`). + std::size_t split_rows_{}; + std::unique_ptr split_inverse_plan_, split_forward_plan_; + std::unique_ptr split_tail_inverse_plan_, split_tail_forward_plan_; + std::unique_ptr coarse_inverse_complex_plan_; + std::unique_ptr coarse_forward_real_plan_; + std::unique_ptr coarse_forward_complex_plan_; + std::unique_ptr raman_drive_forward_plan_; + std::unique_ptr raman_convolution_inverse_plan_; + std::unique_ptr raman_tail_forward_plan_; + std::unique_ptr raman_tail_inverse_plan_; + std::unique_ptr raman_product_forward_plan_; + std::unique_ptr raman_product_inverse_plan_; + std::unique_ptr raman_kernel_forward_plan_; + // The retained synthesis's node transforms, full and tail chunks: from the leading + // rows of the modal time buffer, which then holds node spectra, to the node field. + std::unique_ptr retained_inverse_plan_, retained_tail_inverse_plan_; + // solprop's general modal route; null on every other route. + std::unique_ptr general_; + bool absorber_quiet_ = false; + // solprop's boundary monitor (SEP 21, ADR-057), bound for a run that watches its + // window: its four time weights `(4, n_t)` (low guard, high guard, the collar's + // 1 - W, the collar's rate), each state column's quadrature weight, and what the + // split-step collar took since the last reading beside one split's sums, the + // energies of what the last step's split found and the share it takes at each + // time. Allocated at the first bind, a few rows, and kept. + bool monitor_bound_ = false; + RealView2 monitor_weights_; + RealView monitor_column_weights_; + RealView monitor_share_, monitor_half_; + // A device's split factor per coarse sample, formed once an accepted step. + RealView split_factor_; + Kokkos::View monitor_removed_; + Kokkos::View monitor_split_sum_, monitor_found_; + // The split collar's walk-off bucket of each (mode, frequency) bin and the state's + // energy per bucket, which a reading returns for the crossing limit (solprop + // #440); empty unbound. + ByteView occupancy_buckets_; + std::size_t occupancy_count_ = 0; + mutable CrossedBytes crossed_; + // What the transverse edge's reading sums with, once a monitor is bound (SEP 44). + RealView2 edge_faces_, edge_clip_, edge_totals_, edge_synthesis_; + // A radial edge's node fields at the rows a read uses, real and imaginary parts as + // columns, when the state has one polarisation (`edge_reading`); empty otherwise. + RealView2 edge_nodes_; + RealView edge_columns_; + std::size_t edge_first_row_ = 0; + // A requested collar's lines, and one padded batch per distinct shape, kept across a + // runner's points. Native chooses none of what's in them. + struct EdgeLine { + std::size_t axis{}, nodes{}, rows{}, batch{}, first{}, last{}; + double factor{}, measure{}, step{}; + std::size_t onset{EdgeLines::no_onset}, before{}, after{}; + IndexView positions; + RealView window; + }; + struct EdgeBatch { + ComplexView2 buffer; + std::unique_ptr plan; + std::size_t heap_growth{}, device_growth{}; + }; + std::vector edge_lines_; + std::map, EdgeBatch> edge_batches_; + ComplexView edge_strip_; + RealView edge_scale_; + double edge_k_min_{}; + // A large crossing's pinned chunks, once one has been made (`crossing.hpp`). + mutable crossing::Ring crossing_ring_; + [[nodiscard]] bool monitoring_split() const noexcept { + return monitor_bound_ && problem_.absorber.split_step; + } + // Whether the stage adds the in-stage absorber's term. A quiet temporal collar is + // left out on the modal routes, as solprop's reference leaves it out, unless a + // real-space collar shares the term or the route is the general modal one or a + // coupled flow, which keep it (solprop SEP 21). + [[nodiscard]] bool absorbing() const noexcept { + return problem_.absorber.in_stage && + !(absorber_quiet_ && !general_ && !problem_.flow.enabled && + problem_.spatial.absorber_matrix.empty()); + } + // solprop's linear.coupled_flow; null otherwise. + std::unique_ptr flow_; + // solprop SEP 6's dispersive electronic response on the fused node path, and on the + // Cartesian free-space, line and planar routes; null otherwise. The general modal + // route's is its source's. + std::unique_ptr dispersive_; + // solprop SEP 23's far-field pass, on free space with spatial statistics. + std::unique_ptr far_field_; + std::unique_ptr separable_dispersive_; + // The response's layout and stage weights on whichever route runs it, or null. + DispersiveWeights* dispersive_weights() const noexcept { + if (dispersive_) + return &dispersive_->weights(); + if (separable_dispersive_) + return &separable_dispersive_->weights(); + if (general_ && general_->dispersive() != nullptr) + return &general_->dispersive()->weights(); + return nullptr; + } +}; + +} // namespace solprop_native diff --git a/native/src/general_modal.cpp b/native/src/general_modal.cpp index 8e1d8fad0..43547069b 100644 --- a/native/src/general_modal.cpp +++ b/native/src/general_modal.cpp @@ -4,8 +4,10 @@ #include "solprop_native/general_modal.hpp" +#include "solprop_native/dispatch.hpp" #include "solprop_native/fft_plan.hpp" #include "solprop_native/launch.hpp" +#include "solprop_native/refuse.hpp" #include #include @@ -66,9 +68,7 @@ std::size_t product(const std::size_t left, const std::size_t right, const char* void require_size(const std::size_t actual, const std::size_t expected, const char* name) { if (actual != expected) - throw std::invalid_argument(std::string("general modal ") + name + " has " + - std::to_string(actual) + " values, expected " + - std::to_string(expected)); + refuse("general modal ", name, " has ", actual, " values, expected ", expected); } template bool finite(const std::vector& values) { @@ -135,8 +135,8 @@ static_assert(symmetric_monomials(3) == 10 && symmetric_monomial(3, 2, 0, 1) == symmetric_monomial(2, 1, 0, 1) == 2); bool general_crop_is_contiguous(const Problem& problem) { - const auto& crop = problem.crop_index; - if (crop.size() != problem.n_omega || crop.empty()) + const auto& crop = problem.grid.crop_index; + if (crop.size() != problem.dims.n_omega || crop.empty()) return false; for (std::size_t omega = 0; omega < crop.size(); ++omega) if (crop[omega] != crop.front() + omega) @@ -159,31 +159,32 @@ int general_team_lanes() { } // namespace void require_general_modal(const Problem& problem) { - if (!problem.general_modal) { - if (!problem.general_synthesis.empty() || !problem.general_projection.empty() || - !problem.general_tensor.empty() || !problem.general_frequency_factor.empty() || - !problem.general_delayed_tensor.empty() || - !problem.general_delayed_spectrum.empty() || - !problem.general_node_response.empty()) + if (!problem.general.enabled) { + if (!problem.general.synthesis.empty() || !problem.general.projection.empty() || + !problem.general.tensor.empty() || !problem.general.frequency_factor.empty() || + !problem.general.delayed_tensor.empty() || + !problem.general.delayed_spectrum.empty() || + !problem.general.node_response.empty()) throw std::invalid_argument("general modal arrays require general_modal"); return; } - const auto components = problem.general_components; - const auto nodes = problem.general_nodes; + const auto components = problem.general.components; + const auto nodes = problem.general.nodes; if (components == 0 || components > max_components) throw std::invalid_argument("the general modal route resolves one to three " "Cartesian components"); - if (nodes == 0 || problem.general_node_chunk == 0 || - problem.general_node_chunk > nodes) + if (nodes == 0 || problem.general.node_chunk == 0 || + problem.general.node_chunk > nodes) throw std::invalid_argument("general_node_chunk must lie in [1, general_nodes]"); - if (problem.family != FieldFamily::field_real || problem.n_pol_state != 1 || - problem.n_pol_transform != 1 || problem.separable_geometry()) + if (problem.dims.family != FieldFamily::field_real || problem.dims.n_pol_state != 1 || + problem.dims.n_pol_transform != 1 || problem.separable_geometry()) throw std::invalid_argument("the general modal route runs one channel axis on a " "field grid with no polarisation axis"); - if (problem.nonlinear || problem.plasma || problem.raman || problem.envelope_plasma) + if (problem.kerr.nonlinear || problem.plasma.enabled || problem.raman.enabled || + problem.plasma.envelope) throw std::invalid_argument("the general modal route carries its own regional " "responses and no legacy nonlinear term"); - if (problem.n_to == 0 || problem.n_to % 2 != 0) + if (problem.dims.n_to == 0 || problem.dims.n_to % 2 != 0) throw std::invalid_argument("the general modal route needs an even fine grid"); // A field grid's coarse bins are the leading bins of the fine spectrum (solprop // `fourier.crop_index`), so the source reads and writes them as one contiguous @@ -191,41 +192,42 @@ void require_general_modal(const Problem& problem) { if (!general_crop_is_contiguous(problem)) throw std::invalid_argument("the general modal route needs the coarse bins to be " "one contiguous block of the fine spectrum"); - if (!std::isfinite(problem.general_pad_scale) || problem.general_pad_scale <= 0.0 || - !std::isfinite(problem.general_epsilon0) || problem.general_epsilon0 <= 0.0) + if (!std::isfinite(problem.general.pad_scale) || problem.general.pad_scale <= 0.0 || + !std::isfinite(problem.general.epsilon0) || problem.general.epsilon0 <= 0.0) throw std::invalid_argument( "general modal pad scale and vacuum permittivity must be finite and positive"); - if (problem.general_map_intervals == 0 || problem.general_map_n_deg == 0) + if (problem.general.map_intervals == 0 || problem.general.map_n_deg == 0) throw std::invalid_argument("general modal maps carry at least one section of at " "least one Chebyshev term along z"); const auto map_terms = - product(problem.general_map_intervals, problem.general_map_n_deg, "map terms"); - if (map_terms > 1 && problem.n_stages == 0) + product(problem.general.map_intervals, problem.general.map_n_deg, "map terms"); + if (map_terms > 1 && problem.stepper.n_stages == 0) throw std::invalid_argument("moving general modal maps need at least one stage"); const auto rows = product(nodes, components, "node rows"); - const auto maps = - product(product(rows, problem.n_modes, "maps"), problem.n_omega, "maps"); - require_size(problem.general_synthesis.size(), product(map_terms, maps, "map terms"), + const auto maps = product(product(rows, problem.dims.n_modes, "maps"), + problem.dims.n_omega, "maps"); + require_size(problem.general.synthesis.size(), product(map_terms, maps, "map terms"), "synthesis"); - require_size(problem.general_projection.size(), product(map_terms, maps, "map terms"), + require_size(problem.general.projection.size(), product(map_terms, maps, "map terms"), "projection"); if (map_terms > 1) - static_cast(product(product(problem.n_stages, map_terms, "map weights"), - sizeof(double), "map weight bytes")); - require_size(problem.general_tensor.size(), + static_cast( + product(product(problem.stepper.n_stages, map_terms, "map weights"), + sizeof(double), "map weight bytes")); + require_size(problem.general.tensor.size(), product(nodes, fourth_power(components), "tensor"), "tensor"); - require_size(problem.general_frequency_factor.size(), problem.n_omega, + require_size(problem.general.frequency_factor.size(), problem.dims.n_omega, "frequency factor"); - if (!finite(problem.general_synthesis) || !finite(problem.general_projection) || - !finite(problem.general_tensor) || !finite(problem.general_frequency_factor)) + if (!finite(problem.general.synthesis) || !finite(problem.general.projection) || + !finite(problem.general.tensor) || !finite(problem.general.frequency_factor)) throw std::invalid_argument("general modal maps must be finite"); - const auto chunk_rows = product(problem.general_node_chunk, components, "chunk rows"); - static_cast(product(product(chunk_rows, problem.n_to, "chunk buffers"), + const auto chunk_rows = product(problem.general.node_chunk, components, "chunk rows"); + static_cast(product(product(chunk_rows, problem.dims.n_to, "chunk buffers"), sizeof(Complex), "chunk bytes")); - if (problem.general_terms == 0) { - if (problem.general_responses != 0 || !problem.general_delayed_tensor.empty() || - !problem.general_delayed_spectrum.empty() || - !problem.general_node_response.empty()) + if (problem.general.terms == 0) { + if (problem.general.responses != 0 || !problem.general.delayed_tensor.empty() || + !problem.general.delayed_spectrum.empty() || + !problem.general.node_response.empty()) throw std::invalid_argument("a general modal problem with no delayed term " "carries no delayed arrays"); return; @@ -233,44 +235,46 @@ void require_general_modal(const Problem& problem) { if (components != max_components) throw std::invalid_argument( "a delayed general modal response needs the three Cartesian components"); - if (problem.general_responses == 0) + if (problem.general.responses == 0) throw std::invalid_argument("a delayed general modal term needs a response"); const auto channels = - product(problem.general_responses, problem.general_terms, "delayed channels"); - require_size(problem.general_delayed_tensor.size(), + product(problem.general.responses, problem.general.terms, "delayed channels"); + require_size(problem.general.delayed_tensor.size(), product(channels, fourth_power(components), "delayed tensor"), "delayed tensor"); - require_size(problem.general_delayed_spectrum.size(), - product(channels, problem.n_to + 1, "delayed spectrum"), + require_size(problem.general.delayed_spectrum.size(), + product(channels, problem.dims.n_to + 1, "delayed spectrum"), "delayed spectrum"); - require_size(problem.general_node_response.size(), nodes, "node response"); - if (!finite(problem.general_delayed_tensor) || - !finite(problem.general_delayed_spectrum)) + require_size(problem.general.node_response.size(), nodes, "node response"); + if (!finite(problem.general.delayed_tensor) || + !finite(problem.general.delayed_spectrum)) throw std::invalid_argument("general modal delayed data must be finite"); - for (const auto owner : problem.general_node_response) - if (owner > problem.general_responses) + for (const auto owner : problem.general.node_response) + if (owner > problem.general.responses) throw std::invalid_argument("general modal node response names no response"); static_cast(product( - product(product(problem.general_node_chunk, general_pair_histories, "pairs"), - 2 * problem.n_to, "pair buffers"), + product(product(problem.general.node_chunk, general_pair_histories, "pairs"), + 2 * problem.dims.n_to, "pair buffers"), sizeof(Complex), "pair bytes")); } GeneralModalSource::GeneralModalSource(const Problem& problem) - : n_modes_(problem.n_modes), n_omega_(problem.n_omega), n_to_(problem.n_to), - components_(problem.general_components), nodes_(problem.general_nodes), - chunk_(problem.general_node_chunk), terms_(problem.general_terms), - map_terms_(problem.general_map_intervals * problem.general_map_n_deg), - map_n_deg_(problem.general_map_n_deg), n_stages_(problem.n_stages), - crop_offset_(problem.crop_index.empty() ? 0 : problem.crop_index.front()), - real_form_(problem.general_real_form), epsilon0_(problem.general_epsilon0), + : n_modes_(problem.dims.n_modes), n_omega_(problem.dims.n_omega), + n_to_(problem.dims.n_to), components_(problem.general.components), + nodes_(problem.general.nodes), chunk_(problem.general.node_chunk), + terms_(problem.general.terms), + map_terms_(problem.general.map_intervals * problem.general.map_n_deg), + map_n_deg_(problem.general.map_n_deg), n_stages_(problem.stepper.n_stages), + crop_offset_(problem.grid.crop_index.empty() ? 0 + : problem.grid.crop_index.front()), + real_form_(problem.general.real_form), epsilon0_(problem.general.epsilon0), kernel_pool_(CpuPool::kernels, problem.cpu_execution), fft_pool_(CpuPool::fft, problem.cpu_execution), raman_pool_(CpuPool::raman_fft, problem.cpu_execution), raman_tail_pool_(CpuPool::raman_tail_fft, problem.cpu_execution) { require_general_modal(problem); require_cpu_execution(problem.cpu_execution); - if (!problem.general_modal) + if (!problem.general.enabled) throw std::invalid_argument("the problem does not declare the general modal route"); kernels_ = kernel_pool_.get(CpuOperation::stage, n_modes_ * n_omega_); response_ = @@ -287,20 +291,20 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) // zero; with no regional tensor and no delayed response the contraction is // identically zero, and it and its transform are skipped. const bool tensor = std::ranges::any_of( - problem.general_tensor, [](const double value) { return value != 0.0; }); + problem.general.tensor, [](const double value) { return value != 0.0; }); regional_ = tensor || terms_ != 0; - responds_ = tensor || problem.dispersive; + responds_ = tensor || problem.dispersive.enabled; const auto chunk_rows = chunk_ * components_; if (moving()) { // A moving guide holds every coefficient of both maps and one evaluated map of // each, which `combine` rewrites per stage; the weights and their host copy are // allocated here so that a trial allocates nothing. - const auto one_map = problem.general_synthesis.size() / map_terms_; + const auto one_map = problem.general.synthesis.size() / map_terms_; maps_ = kernel_pool_.get(CpuOperation::modal, 2 * one_map * map_n_deg_); synthesis_terms_ = ComplexView("solprop_native.general.synthesis_terms", - problem.general_synthesis.size()); + problem.general.synthesis.size()); projection_terms_ = ComplexView("solprop_native.general.projection_terms", - problem.general_projection.size()); + problem.general.projection.size()); synthesis_ = ComplexView("solprop_native.general.stage_synthesis", one_map); projection_ = ComplexView("solprop_native.general.stage_projection", one_map); map_weights_ = @@ -309,18 +313,19 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) stage_sections_.assign(n_stages_, 0); } else { synthesis_ = ComplexView("solprop_native.general.synthesis", - problem.general_synthesis.size()); + problem.general.synthesis.size()); projection_ = ComplexView("solprop_native.general.projection", - problem.general_projection.size()); + problem.general.projection.size()); } frequency_factor_ = ComplexView("solprop_native.general.frequency_factor", n_omega_); - tensor_ = RealView("solprop_native.general.tensor", problem.general_tensor.size()); + tensor_ = RealView("solprop_native.general.tensor", problem.general.tensor.size()); scaled_ = ComplexView2("solprop_native.general.scaled_state", n_modes_, n_omega_); source_ = ComplexView2("solprop_native.general.source", n_modes_, n_omega_); node_spectrum_ = ComplexView2("solprop_native.general.node_spectrum", chunk_rows, n_to_); node_field_ = ComplexView2("solprop_native.general.node_field", chunk_rows, n_to_); - real_signal_ = general_real_signal_enabled && real_form_ && !problem.dispersive; + real_signal_ = + general_real_signal_enabled && real_form_ && !problem.dispersive.enabled; if (real_signal_) { half_spectrum_ = AliasComplexView2(node_spectrum_.data(), chunk_rows, n_to_ / 2 + 1); @@ -334,9 +339,9 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) if (terms_ != 0) { const auto pair_rows = chunk_ * general_pair_histories; delayed_tensor_ = RealView("solprop_native.general.delayed_tensor", - problem.general_delayed_tensor.size()); + problem.general.delayed_tensor.size()); delayed_spectrum_ = ComplexView("solprop_native.general.delayed_spectrum", - problem.general_delayed_spectrum.size()); + problem.general.delayed_spectrum.size()); node_response_ = IndexView("solprop_native.general.node_response", nodes_); node_drive_ = RealView2("solprop_native.general.node_drive", pair_rows, 2 * n_to_); node_drive_spectrum_ = ComplexView2("solprop_native.general.node_drive_spectrum", @@ -346,16 +351,16 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) node_histories_ = RealView2("solprop_native.general.node_histories", pair_rows, 2 * n_to_); node_delayed_ = RealView2("solprop_native.general.node_delayed", chunk_rows, n_to_); - upload(kernels_, delayed_tensor_, problem.general_delayed_tensor); - upload(kernels_, delayed_spectrum_, problem.general_delayed_spectrum); - upload(kernels_, node_response_, problem.general_node_response); + upload(kernels_, delayed_tensor_, problem.general.delayed_tensor); + upload(kernels_, delayed_spectrum_, problem.general.delayed_spectrum); + upload(kernels_, node_response_, problem.general.node_response); } - upload(kernels_, moving() ? synthesis_terms_ : synthesis_, problem.general_synthesis); + upload(kernels_, moving() ? synthesis_terms_ : synthesis_, problem.general.synthesis); upload(kernels_, moving() ? projection_terms_ : projection_, - problem.general_projection); - upload(kernels_, frequency_factor_, problem.general_frequency_factor); - upload(kernels_, tensor_, problem.general_tensor); - if (problem.dispersive) + problem.general.projection); + upload(kernels_, frequency_factor_, problem.general.frequency_factor); + upload(kernels_, tensor_, problem.general.tensor); + if (problem.dispersive.enabled) dispersive_ = std::make_unique( problem, kernel_pool_, GeneralDispersive::StageViews{ @@ -363,7 +368,7 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) kernels_.fence("solprop_native general modal upload"); } -GeneralModalSource::~GeneralModalSource() { close(); } +GeneralModalSource::~GeneralModalSource() { kernels_.fence(); } void GeneralModalSource::plan() { transforms_ = @@ -454,13 +459,7 @@ void GeneralModalSource::plan() { } } -std::string GeneralModalSource::delayed_fft_provider() const { - return drive_plan_ ? drive_plan_->provider() : "none"; -} - void GeneralModalSource::evaluate() { - if (closed_) - throw std::logic_error("the general modal source is closed"); if (!inverse_plan_ && !signal_plan_) throw std::logic_error("the general modal source has not been planned"); graph::deep_copy(kernels_, source_, Complex{}); @@ -513,8 +512,6 @@ void GeneralModalSource::evaluate() { } void GeneralModalSource::set_weights(const std::vector& weights) { - if (closed_) - throw std::logic_error("the general modal source is closed"); if (!moving()) { if (!weights.empty()) throw std::invalid_argument("a stage carries transform_weights for general modal " @@ -552,8 +549,6 @@ void GeneralModalSource::set_weights(const std::vector& weights) { void GeneralModalSource::combine(const std::size_t stage) { if (!moving()) return; - if (closed_) - throw std::logic_error("the general modal source is closed"); if (!weights_ready_) throw std::logic_error("a moving guide's maps need the trial's transform_weights"); if (stage >= n_stages_) @@ -985,34 +980,9 @@ void GeneralModalSource::project(const std::size_t first, const std::size_t coun #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) project_device(first, count); #else - switch (projection_lanes_) { - case 2048: - project_lanes<2048>(first, count); - break; - case 1024: - project_lanes<1024>(first, count); - break; - case 512: - project_lanes<512>(first, count); - break; - case 256: - project_lanes<256>(first, count); - break; - case 128: - project_lanes<128>(first, count); - break; - case 64: - project_lanes<64>(first, count); - break; - case 32: - project_lanes<32>(first, count); - break; - case 16: - project_lanes<16>(first, count); - break; - default: - project_lanes(first, count); - } + dispatch_among<2048, 1024, 512, 256, 128, 64, 32, 16>( + projection_lanes_, [&] { project_lanes(first, count); }, + [&] { project_lanes(first, count); }); #endif } @@ -1222,71 +1192,4 @@ std::map GeneralModalSource::memory_ledger() const { }; } -void GeneralModalSource::close() noexcept { - if (closed_) - return; - kernels_.fence(); - if (dispersive_) - dispersive_->close(); - dispersive_.reset(); - tail_inverse_plan_.reset(); - tail_signal_plan_.reset(); - tail_forward_plan_.reset(); - tail_spectrum_ = {}; - tail_field_ = {}; - tail_source_ = {}; - tail_polarisation_ = {}; - inverse_plan_.reset(); - signal_plan_.reset(); - forward_plan_.reset(); - tail_drive_plan_.reset(); - tail_history_plan_.reset(); - tail_drive_ = {}; - tail_histories_ = {}; - tail_drive_spectrum_ = {}; - tail_product_ = {}; - drive_plan_.reset(); - history_plan_.reset(); - synthesis_ = {}; - projection_ = {}; - synthesis_terms_ = {}; - projection_terms_ = {}; - map_weights_ = {}; - host_map_weights_ = {}; - weights_ready_ = false; - frequency_factor_ = {}; - delayed_spectrum_ = {}; - tensor_ = {}; - delayed_tensor_ = {}; - node_response_ = {}; - scaled_ = {}; - source_ = {}; - half_spectrum_ = {}; - tail_half_spectrum_ = {}; - real_field_ = {}; - tail_real_field_ = {}; - node_spectrum_ = {}; - node_field_ = {}; - node_source_ = {}; - node_polarisation_ = {}; - node_drive_ = {}; - node_histories_ = {}; - node_delayed_ = {}; - node_drive_spectrum_ = {}; - node_product_ = {}; - kernels_ = ExecutionSpace(); - transforms_ = ExecutionSpace(); - delayed_transforms_ = ExecutionSpace(); - response_ = ExecutionSpace(); - tail_response_ = ExecutionSpace(); - maps_ = ExecutionSpace(); - dense_ = ExecutionSpace(); - tail_dense_ = ExecutionSpace(); - kernel_pool_.clear(); - fft_pool_.clear(); - raman_pool_.clear(); - raman_tail_pool_.clear(); - closed_ = true; -} - } // namespace solprop_native diff --git a/native/src/helpers.cpp b/native/src/helpers.cpp index 8b80c1ee3..cabb4f2c4 100644 --- a/native/src/helpers.cpp +++ b/native/src/helpers.cpp @@ -6,6 +6,7 @@ #include "solprop_native/chebyshev.hpp" #include "solprop_native/lagrange_basis.hpp" +#include "solprop_native/refuse.hpp" #include "solprop_native/vmath.hpp" #include @@ -40,7 +41,7 @@ template using Output = nb::ndarray, nb::c_contig, nb::device::cpu>; std::string shape_text(const std::size_t rows, const std::size_t columns) { - return "(" + std::to_string(rows) + ", " + std::to_string(columns) + ")"; + return concat("(", rows, ", ", columns, ")"); } template @@ -53,16 +54,14 @@ std::ptrdiff_t clenshaw_points(const Input& z, const Input throw std::invalid_argument( "clenshaw_points: edges needs at least two breakpoints"); if (c.shape(0) != n_edges - 1 || c.shape(1) < 1) - throw std::invalid_argument("clenshaw_points: coefficients of shape " + - shape_text(c.shape(0), c.shape(1)) + - "[...] do not match " + std::to_string(n_edges) + - " breakpoints with at least one term"); + refuse("clenshaw_points: coefficients of shape ", + shape_text(c.shape(0), c.shape(1)), "[...] do not match ", n_edges, + " breakpoints with at least one term"); const std::size_t n_points = z.shape(0); const std::size_t n_values = c.shape(2); if (out.shape(0) != n_points || out.shape(1) != n_values) - throw std::invalid_argument("clenshaw_points: out has shape " + - shape_text(out.shape(0), out.shape(1)) + ", expected " + - shape_text(n_points, n_values)); + refuse("clenshaw_points: out has shape ", shape_text(out.shape(0), out.shape(1)), + ", expected ", shape_text(n_points, n_values)); return chebyshev::clenshaw_points(z.data(), n_points, edges.data(), n_edges, lower, upper, c.data(), c.shape(1), n_values, out.data()); } @@ -72,14 +71,11 @@ void basis_rows(const Input& points, const double upper, const Output& weights) { const std::size_t n = points.shape(0); if (n > lagrange::max_nodes) - throw std::invalid_argument("basis_rows: " + std::to_string(n) + - " nodes exceed the " + - std::to_string(lagrange::max_nodes) + " supported"); + refuse("basis_rows: ", n, " nodes exceed the ", lagrange::max_nodes, " supported"); if (rows.shape(0) != n || rows.shape(1) != n || weights.shape(0) != n) - throw std::invalid_argument( - "basis_rows: rows " + shape_text(rows.shape(0), rows.shape(1)) + - " and weights (" + std::to_string(weights.shape(0)) + - ",) must be (n, n) and (n,) for n = " + std::to_string(n) + " nodes"); + refuse("basis_rows: rows ", shape_text(rows.shape(0), rows.shape(1)), + " and weights (", weights.shape(0), ",) must be (n, n) and (n,) for n = ", n, + " nodes"); lagrange::basis_rows(points.data(), n, upper, integrate, rows.data(), weights.data()); } @@ -96,10 +92,8 @@ void trial_tables(const Input& nodes, const std::size_t order, std::ranges::all_of(orders, [](const std::size_t value) { return value > 0; }); if (n_orders == 0 || n_orders > 3 || !scored || order == 0 || order > highest || highest > n || highest + 1 > lagrange::max_nodes) - throw std::invalid_argument("trial_tables: order " + std::to_string(order) + - " and " + std::to_string(n_orders) + - " scored orders up to " + std::to_string(highest) + - " do not fit " + std::to_string(n) + " nodes"); + refuse("trial_tables: order ", order, " and ", n_orders, " scored orders up to ", + highest, " do not fit ", n, " nodes"); const std::size_t width = highest + 1; const std::size_t carried_size = compensated ? order : 0; if (predictor.shape(0) != order || predictor.shape(1) != order || @@ -110,11 +104,10 @@ void trial_tables(const Input& nodes, const std::size_t order, endpoint.shape(0) != (compensated ? order + 1 : 0) || defects.shape(0) != (compensated ? n_orders : 0) || defects.shape(1) != (compensated ? width : 0)) - throw std::invalid_argument( - "trial_tables: the tables must be (order, order), (width, width) and " - "(n_orders, 2 width), and the compensation's (width - 1,), (order,), " - "(order + 1,) and (n_orders, width) when compensated, else empty; order " + - std::to_string(order) + ", width " + std::to_string(width)); + refuse("trial_tables: the tables must be (order, order), (width, width) and " + "(n_orders, 2 width), and the compensation's (width - 1,), (order,), " + "(order + 1,) and (n_orders, width) when compensated, else empty; order ", + order, ", width ", width); lagrange::trial_tables(nodes.data(), order, orders.data(), n_orders, width, compensated, predictor.data(), corrector.data(), factored.data(), spans.data(), carried.data(), endpoint.data(), @@ -124,9 +117,7 @@ void trial_tables(const Input& nodes, const std::size_t order, void require_length(const char* name, const std::size_t expected, const std::size_t actual) { if (actual != expected) - throw std::invalid_argument(std::string(name) + ": the arrays have lengths " + - std::to_string(expected) + " and " + - std::to_string(actual)); + refuse(name, ": the arrays have lengths ", expected, " and ", actual); } void vector_exp(const Input& x, const Output& out) { diff --git a/native/src/session.cpp b/native/src/session.cpp index a1bb88870..b5af2ce02 100644 --- a/native/src/session.cpp +++ b/native/src/session.cpp @@ -5,6 +5,7 @@ #include "solprop_native/session.hpp" #include "solprop_native/core.hpp" +#include "solprop_native/refuse.hpp" #include #include @@ -1000,9 +1001,9 @@ void Session::acknowledge(const std::uint64_t sequence) { const std::scoped_lock lock(mutex_); require_open(); if (!pending_ || sequence != sequence_) - throw LifecycleError("receipt " + std::to_string(sequence) + - " is not the pending batch's: a foreign, stale or repeated " - "acknowledgment"); + refuse("receipt ", sequence, + " is not the pending batch's: a foreign, stale or repeated " + "acknowledgment"); pending_ = false; state_.store(outcome_.load() != Outcome::none ? State::finished : State::ready, std::memory_order_release); diff --git a/native/src/session_statistics.cpp b/native/src/session_statistics.cpp index 9f11ff108..986dded29 100644 --- a/native/src/session_statistics.cpp +++ b/native/src/session_statistics.cpp @@ -5,6 +5,7 @@ #include "solprop_native/chebyshev.hpp" #include "solprop_native/core.hpp" #include "solprop_native/far_field.hpp" +#include "solprop_native/refuse.hpp" #include "solprop_native/statistics_plan.hpp" #include @@ -649,9 +650,8 @@ template std::vector Table::at(const double z) const chebyshev::clenshaw_points(&z, 1, edges.data(), edges.size(), lower, upper, coefficients.data(), n_terms, n_values, out.data()); if (refused >= 0) - throw std::out_of_range( - "z = " + std::to_string(z) + " m is outside a statistics table's span [" + - std::to_string(edges.front()) + ", " + std::to_string(edges.back()) + "] m"); + refuse("z = ", z, " m is outside a statistics table's span [", + edges.front(), ", ", edges.back(), "] m"); return out; } @@ -853,8 +853,8 @@ std::vector evaluate(GeneralState& state, const Plan& plan, const double for (const Stat stat : plan.outputs) { const auto& values = results[static_cast(stat)]; if (values.size() != plan.width(stat)) - throw std::logic_error("statistic " + std::to_string(static_cast(stat)) + - " is not evaluated on this route"); + refuse("statistic ", static_cast(stat), + " is not evaluated on this route"); packed.insert(packed.end(), values.begin(), values.end()); } return packed; diff --git a/native/src/session_widths.cpp b/native/src/session_widths.cpp index 968f99991..80318cf9f 100644 --- a/native/src/session_widths.cpp +++ b/native/src/session_widths.cpp @@ -17,6 +17,7 @@ #endif #include "solprop_native/core.hpp" +#include "solprop_native/refuse.hpp" #include "solprop_native/statistics_plan.hpp" #include @@ -61,11 +62,11 @@ std::string scientific(const double value) { /** `solprop.backends.spatial._negative_fluence`. */ std::invalid_argument negative_fluence(const double value, const std::size_t node) { - return std::invalid_argument( + return std::invalid_argument(concat( "beam_radius and fwhm_r need a nonnegative fluence profile, and the signed " - "axial flux is " + - scientific(value) + " J/m^2 at node " + std::to_string(node) + - ", beyond its rounding budget; the width is undefined for it"); + "axial flux is ", + scientific(value), " J/m^2 at node ", node, + ", beyond its rounding budget; the width is undefined for it")); } /** `fluence_budget`'s one value: eight rounding units a term of the largest diff --git a/native/src/solprop_native/_core.pyi b/native/src/solprop_native/_core.pyi index ad73940bb..a57486e4a 100644 --- a/native/src/solprop_native/_core.pyi +++ b/native/src/solprop_native/_core.pyi @@ -4,13 +4,250 @@ from __future__ import annotations +import enum from typing import Any, overload import numpy as np import numpy.typing as npt +# Generated by native/scripts/problem_stub.py from nanobind.stubgen. +# Array fields are copied in and out: assign a whole array, since an item +# assigned through a read copy changes nothing. +class FieldFamily(enum.Enum): + field_real = 0 + + field_analytic = 1 + + envelope = 2 + +class Problem: + def __init__(self) -> None: ... + class Dims: + def __init__(self) -> None: ... + n_omega: int + n_t: int + n_to: int + n_modes: int + n_pol_state: int + n_pol_transform: int + n_nodes: int + node_chunk: int + family: FieldFamily + + class Grid: + def __init__(self) -> None: ... + crop_index: npt.NDArray[np.uint64] + active_mask: npt.NDArray[np.bool_] + norm_mask: npt.NDArray[np.bool_] + omega_window: npt.NDArray[np.float64] + omega: npt.NDArray[np.float64] + power_weight: npt.NDArray[np.float64] + time_window: npt.NDArray[np.float64] + time_window_derivative: npt.NDArray[np.float64] + carrier: npt.NDArray[np.complex128] + dt_fine: float + dt: float + + class Spatial: + def __init__(self) -> None: ... + n_x: int + n_y: int + k_squared: npt.NDArray[np.complex128] + k_squared_low: npt.NDArray[np.complex128] + numerator: npt.NDArray[np.complex128] + clamped: npt.NDArray[np.complex128] + kperp_sq: npt.NDArray[np.float64] + kperp_sq_low: npt.NDArray[np.float64] + frame: npt.NDArray[np.float64] + spectral_rate: npt.NDArray[np.float64] + user_rate: npt.NDArray[np.float64] + absorber_rate: npt.NDArray[np.float64] + kperp_rate: npt.NDArray[np.float64] + evanescent_rate: float + k_x: npt.NDArray[np.float64] + k_y: npt.NDArray[np.float64] + far_field_slots: int + n_p: int + kappa_sq: npt.NDArray[np.complex128] + absorber_matrix: npt.NDArray[np.float64] + spectral_weights: npt.NDArray[np.float64] + on_axis: npt.NDArray[np.complex128] + + class Flux: + def __init__(self) -> None: ... + k_squared: npt.NDArray[np.complex128] + k_squared_low: npt.NDArray[np.complex128] + kperp_sq: npt.NDArray[np.float64] + kperp_sq_low: npt.NDArray[np.float64] + table: bool + closure: bool + + class Stepper: + def __init__(self) -> None: ... + n_stages: int + tableau_matrix: npt.NDArray[np.float64] + tableau_nodes: npt.NDArray[np.float64] + solution_weights: npt.NDArray[np.float64] + error_weights: npt.NDArray[np.float64] + first_same_as_last: bool + weighted_controller: bool + modified_noise: bool + low_storage: bool + low_storage_stage_register: bool + low_storage_flows: int + low_storage_tiled_input: bool + + class Linear: + def __init__(self) -> None: ... + n_phi_terms: int + n_prefactor_terms: int + phi_table: bool + prefactor_table: bool + phi_terms: npt.NDArray[np.complex128] + prefactor_terms: npt.NDArray[np.complex128] + + class Absorber: + def __init__(self) -> None: ... + in_stage: bool + split_step: bool + fine: npt.NDArray[np.float64] + coarse: npt.NDArray[np.float64] + + class Transform: + def __init__(self) -> None: ... + synthesis_matrix: npt.NDArray[np.float64] + projection_matrix: npt.NDArray[np.float64] + projection_coarse_matrix: npt.NDArray[np.float64] + synthesis: float + projection: float + + class Kerr: + def __init__(self) -> None: ... + nonlinear: bool + coefficient: float + self_factor: float + conjugate_factor: float + harmonic_factor: float + take_real: bool + + class Plasma: + def __init__(self) -> None: ... + enabled: bool + envelope: bool + gated: bool + stash: bool + envelope_rule_nodes: npt.NDArray[np.float64] + envelope_rule_weights: npt.NDArray[np.float64] + envelope_drude_factor: npt.NDArray[np.float64] + envelope_loss_factor: npt.NDArray[np.complex128] + charge_mass_ratio: float + ionisation_potential: float + log_field_min: float + d_log_field: float + field_min: float + field_max: float + rate_max: float + rate_intervals: int + rate_coefficients: npt.NDArray[np.float64] + n_channels: int + band: bool + taps: npt.NDArray[np.float64] + channel_parameters: npt.NDArray[np.float64] + channel_coefficients: npt.NDArray[np.float64] + + class Raman: + def __init__(self) -> None: ... + enabled: bool + thg: bool + tensor_capable: bool + n_terms: int + fraction: float + epsilon0: float + undamped: npt.NDArray[np.float64] + tensor_weights: npt.NDArray[np.float64] + n_components: int + row_component: npt.NDArray[np.float64] + component_fraction: npt.NDArray[np.float64] + + class General: + def __init__(self) -> None: ... + enabled: bool + real_form: bool + map_intervals: int + map_n_deg: int + components: int + nodes: int + node_chunk: int + responses: int + terms: int + pad_scale: float + epsilon0: float + synthesis: npt.NDArray[np.complex128] + projection: npt.NDArray[np.complex128] + frequency_factor: npt.NDArray[np.complex128] + delayed_spectrum: npt.NDArray[np.complex128] + tensor: npt.NDArray[np.float64] + delayed_tensor: npt.NDArray[np.float64] + node_response: npt.NDArray[np.uint64] + + class Flow: + def __init__(self) -> None: ... + enabled: bool + active: int + sections: int + terms: int + slot: npt.NDArray[np.uint64] + frozen: npt.NDArray[np.complex128] + residual_terms: npt.NDArray[np.complex128] + left: npt.NDArray[np.complex128] + right: npt.NDArray[np.complex128] + eigenvalues: npt.NDArray[np.complex128] + route: npt.NDArray[np.uint64] + fallback_vectors: npt.NDArray[np.complex128] + fallback_triangular: npt.NDArray[np.complex128] + + class Dispersive: + def __init__(self) -> None: ... + enabled: bool + symmetric: bool + form: int + inputs: npt.NDArray[np.float64] + term_multipliers: npt.NDArray[np.float64] + coefficients: npt.NDArray[np.float64] + pair_constants: npt.NDArray[np.float64] + quartics: npt.NDArray[np.float64] + input_scale: npt.NDArray[np.float64] + output_scale: npt.NDArray[np.float64] + term_rows: npt.NDArray[np.uint64] + term_channels: npt.NDArray[np.uint64] + group_offsets: npt.NDArray[np.uint64] + group_members: npt.NDArray[np.uint64] + group_keys: npt.NDArray[np.uint64] + node_region: npt.NDArray[np.uint64] + + dims: Problem.Dims + grid: Problem.Grid + spatial: Problem.Spatial + flux: Problem.Flux + stepper: Problem.Stepper + linear: Problem.Linear + absorber: Problem.Absorber + transform: Problem.Transform + kerr: Problem.Kerr + plasma: Problem.Plasma + raman: Problem.Raman + general: Problem.General + flow: Problem.Flow + dispersive: Problem.Dispersive + cpu_execution: dict[str, int | bool] + fft_planning: dict[str, object] + + def validate(self) -> None: ... + +# End of the generated problem stub. + class State: - # The payload is `backend._native_problem` of a validated + # The problem is `backend._lower` of a validated # `solprop.translate.problem.PropagationProblem`: its shared grid, operator, # response and source components, with the route variant the adapter visited # (solprop SEP 47 section 6.2); `n_modes` and `n_pol_state` are the state's @@ -18,7 +255,7 @@ class State: # `(1, n_modes, n_pol, n_omega)`: `set_initial`, `field_to_host` and the # Adams rows carry the leading axis of one, which the adapter adds and removes, # and a single trajectory crosses the Python seam without it. - def __init__(self, problem: dict[str, object]) -> None: ... + def __init__(self, problem: Problem) -> None: ... def set_initial( self, field: npt.NDArray[np.complex128], diff --git a/native/src/solprop_native/backend.py b/native/src/solprop_native/backend.py index 8715792fb..9416111b8 100644 --- a/native/src/solprop_native/backend.py +++ b/native/src/solprop_native/backend.py @@ -404,20 +404,17 @@ def __init__( self._execution_source = ( _package.EXECUTION_SOURCE if cpu_execution is None else "given" ) - self._native_state = _core.State( - _native_problem( - problem, - request, - self._terms, - self._flow, - self._z_tables, - node_chunk=self._node_chunk, - general_chunk=self._general_chunk, - ) - | _low_storage_payload(problem, stepper, device=_on_device()) - | {"cpu_execution": (cpu_execution or _package.EXECUTION).native()} - | _planning_payload(fft_planning, stored) - ) + plan = _LowerPlan( + terms=self._terms, + flow=self._flow, + tables=self._z_tables, + node_chunk=self._node_chunk, + general_chunk=self._general_chunk, + cpu_execution=cpu_execution, + fft_planning=_planning(fft_planning, stored), + device=_on_device(), + ) + self._native_state = _core.State(_lower(problem, request, plan)) self._applied_execution = ExecutionRecord.from_native( self._native_state.cpu_execution ) @@ -429,18 +426,15 @@ def __init__( stored |= _keep_plans(self._native_state, wisdom) self._native_state.close() self._native_state = _core.State( - _native_problem( + _lower( problem, request, - self._terms, - self._flow, - self._z_tables, - node_chunk=self._node_chunk, - general_chunk=self._general_chunk, + dataclasses.replace( + plan, + cpu_execution=refined[0], + fft_planning=_planning(fft_planning, stored), + ), ) - | _low_storage_payload(problem, stepper, device=_on_device()) - | {"cpu_execution": refined[0].native()} - | _planning_payload(fft_planning, stored) ) self._execution_source = refined[1] self._applied_execution = ExecutionRecord.from_native( @@ -2184,28 +2178,20 @@ def native_node_chunk( return chunk -def _low_storage_payload( +def _low_storage_flows( problem: PropagationProblem, stepper: StepperSettings, *, device: bool -) -> dict[str, object]: - """Return the native problem's low-storage rows (solprop kernel schema 32). +) -> int | None: + """Return the low-storage march's flow volumes, or None for another march. - The march's own allocation: its registers in place of the Runge-Kutta views, the - stage register elided where solprop's estimate elides it, and the flows solprop's - estimate counts -- except that a device build recomputes under ``"auto"``, where - the estimate's CPU count over-states it by the table. A device's input crosses the - state's layout through tiles (``docs/dev/benchmarks.md``); a CPU's has its own. + The flows solprop's estimate counts (solprop kernel schema 32) -- except that a + device build recomputes under ``"auto"``, where the estimate's CPU count + over-states it by the table. A device's input crosses the state's layout through + tiles (``docs/dev/benchmarks.md``); a CPU's has its own. """ if not isinstance(stepper.integrator, LowStorageRungeKutta): - return {} + return None flows = low_storage_flows(problem, stepper, "native") - if device and stepper.propagator_mode == "auto": - flows = 0 - return { - "low_storage": True, - "low_storage_stage_register": low_storage_stage_register(problem, "native"), - "low_storage_flows": flows, - "low_storage_tiled_input": device, - } + return 0 if device and stepper.propagator_mode == "auto" else flows _MARCH_COUNTERS = ( @@ -2643,8 +2629,9 @@ def __init__(self, backend: NativeBackend, method: LowStorageRungeKutta) -> None self.atomic = True problem = backend._problem # pyright: ignore[reportPrivateUsage] - the owner's problem stepper = backend._stepper # pyright: ignore[reportPrivateUsage] - the owner's policy - payload = _low_storage_payload(problem, stepper, device=_on_device()) - self._flows = int(cast(int, payload["low_storage_flows"])) + flows = _low_storage_flows(problem, stepper, device=_on_device()) + assert flows is not None # the march is the low-storage one + self._flows = flows schedule = stepper.schedule assert schedule is not None # the translation refused the march without one self._keys, self._edges = table_steps(schedule), schedule.edges @@ -3087,47 +3074,6 @@ def _flow_allowance(modes: int) -> float: return 1.0 + 64.0 * modes * float(np.finfo(float).eps) -def _flow_payload(flow: _HostFlow | None) -> dict[str, object]: - """Pack a coupled flow's bins and residual data; its actions follow per step.""" - if flow is None: - return {"coupled_flow": False} - slot = np.zeros(flow.active.size, dtype=np.uint64) - slot[flow.active] = np.arange(1, int(np.count_nonzero(flow.active)) + 1) - residual = flow.residual_terms - empty = np.empty(0, dtype=np.complex128) - # The bins formed by scaling and squaring are numbered 1, 2, ... in row-major - # (section, bin) order and carry their Schur factors; the rest take the - # eigenbasis, whose `W` and `Y` are zero where it is not used. - fallback = ~flow.route - route = np.zeros(flow.route.shape, dtype=np.uint64) - route[fallback] = np.arange(1, int(np.count_nonzero(fallback)) + 1) - # The residual's data are zero on the bins the error norm does not score, so the - # engine's residual there is an exact zero and the frozen flow alone carries them, - # as in solprop's reference. Every term is finite, so each - # product with a zero is a signed zero and the running sums stay +0: the source is - # what adding nothing leaves, but for the sign of a zero. - scored = flow.scored[None, :, None, None] - return { - "coupled_flow": True, - "flow_active": int(np.count_nonzero(flow.active)), - "flow_sections": int(flow.frozen.shape[0]), - "flow_terms": 0 if residual is None else int(residual.shape[0]), - "flow_slot": slot, - "flow_frozen": empty - if residual is None - else np.ascontiguousarray(np.where(scored, flow.frozen, 0.0)), - "flow_residual_terms": empty - if residual is None - else np.ascontiguousarray(np.where(scored, residual, 0.0)), - "flow_left": flow.left, - "flow_right": flow.right, - "flow_eigenvalues": flow.eigenvalues, - "flow_route": route.reshape(-1), - "flow_fallback_vectors": np.ascontiguousarray(flow.vectors[fallback]), - "flow_fallback_triangular": np.ascontiguousarray(flow.triangular[fallback]), - } - - _APERTURE_CONSTANTS = 11 """The constants before an aperture row's scale row (``far_field.hpp``).""" @@ -3206,21 +3152,22 @@ def _refined_record( return None -def _planning_payload( +def _planning( given: Mapping[str, object] | None, stored: Mapping[str, str] -) -> dict[str, object]: - """Return the problem's ``fft_planning``: the one given, or the run's own. +) -> dict[str, object] | None: + """Return the problem's FFT planning: the one given, or the run's own. A run's own is the profile's, or the plan policy's, offered the plans runs measured before (`stored`) and keeping those it measures (solprop ADR-076); one given, a - calibration's, plans on its own. + calibration's, plans on its own. None on a device, which plans with its vendor's + FFT. """ if _on_device(): - return {} + return None if given is not None: - return {"fft_planning": dict(given)} + return dict(given) chosen = _package.PLANNING or {"mode": "legacy"} - return {"fft_planning": dict(chosen) | {"stored": dict(stored), "keep": True}} + return dict(chosen) | {"stored": dict(stored), "keep": True} def _keep_plans(state: _core.State, path: Path | None) -> dict[str, str]: @@ -3280,144 +3227,326 @@ def _planning_record(state: _core.State) -> dict[str, object]: } -def _native_problem( - problem: PropagationProblem, - request: RunRequest, - fixed: StageTerms, - flow: _HostFlow | None = None, - tables: _ZTables | None = None, - *, - node_chunk: int | None = None, - general_chunk: int | None = None, -) -> dict[str, object]: +@dataclasses.dataclass(frozen=True) +class _LowerPlan: + """The backend's choices a lowering takes beside the problem and its request. + + `terms` are the fixed stage terms, `flow` a coupled flow's host factors and + `tables` the z tables; the chunks are the backend's lowered execution choices + (solprop SEP 47 section 6.5), the automatic plan's when absent. `cpu_execution` + is the workspace's CPU record (the process's when absent) and `fft_planning` its + FFT planning record (none on a device); `device` says the state is a device's. + """ + + terms: StageTerms + flow: _HostFlow | None = None + tables: _ZTables | None = None + node_chunk: int | None = None + general_chunk: int | None = None + cpu_execution: ExecutionRecord | None = None + fft_planning: Mapping[str, object] | None = None + device: bool = False + + +def _lower( + problem: PropagationProblem, request: RunRequest, plan: _LowerPlan +) -> _core.Problem: + """Fill the native problem's structs from solprop's problem, request and `plan`.""" stepper = request.stepper - # The constructor hands its lowered chunks; a direct caller takes the automatic - # plan's (solprop SEP 47 section 6.5). + node_chunk = plan.node_chunk if node_chunk is None: node_chunk = native_node_chunk( problem, plan_node_chunk(problem), device=_on_device() ) + general_chunk = plan.general_chunk if general_chunk is None and problem.transform.modal_tables is not None: general_chunk = general_planned_chunk(problem, plan_node_chunk(problem)) + native = _core.Problem() + native.cpu_execution = (plan.cpu_execution or _package.EXECUTION).native() + if plan.fft_planning is not None: + native.fft_planning = dict(plan.fft_planning) kerr = problem.responses.kerr_coefficients dispersive = problem.responses.dispersive_kerr + absorber = problem.responses.temporal_absorber + absorber_fine = problem.responses.temporal_absorber_fine + spatial_absorber = radial_absorber_matrix( + problem.transform, problem.grids.pad_scale + ) + temporal_in_stage = ( + absorber_fine is not None and problem.responses.absorber_scheme == "nonlinear" + ) + geometry = problem.transform.geometry + # Free space and schema 15's hybrid planar kind share the separable operator and + # the transverse FFT engine; the planar kind adds its slab rule in x. + free = geometry is not None and geometry.kind in ("free", "slab") + spatial_absorption = bool(spatial_absorber.size) or ( + free and geometry is not None and bool(np.any(geometry.absorber_rate)) + ) + + dims = native.dims + dims.n_omega = problem.shape[-1] + dims.n_t = problem.grids.n_t + dims.n_to = problem.grids.n_to + dims.n_modes = problem.shape[0] + dims.n_pol_state = problem.shape[1] + dims.n_pol_transform = problem.transform.n_pol + dims.n_nodes = problem.transform.n_nodes + # solprop SEP 47 section 6.5: the backend's execution choice, lowered by + # `native_node_chunk`. + dims.node_chunk = node_chunk # solprop SEP 6's number-conserving sector is formed on analytic signals, of the - # filtered fields and, for the polynomial, of the bare one; a Raman drive beside it - # is the cycle-averaged one, which reads the analytic signal too. - # The general modal route synthesises its own analytic signal on the real family. - family = ( - "envelope" + # filtered fields and, for the polynomial, of the bare one; a Raman drive beside + # it is the cycle-averaged one, which reads the analytic signal too. The general + # modal route synthesises its own analytic signal on the real family. + dims.family = ( + _core.FieldFamily.envelope if problem.grid_kind == "envelope" - else "field_analytic" + else _core.FieldFamily.field_analytic if (kerr is not None and kerr.analytic_signal) or ( dispersive is not None and dispersive.sector == "conserving" and problem.transform.modal_tables is None ) - else "field_real" + else _core.FieldFamily.field_real ) - absorber = problem.responses.temporal_absorber - absorber_fine = problem.responses.temporal_absorber_fine - spatial_absorber = radial_absorber_matrix( - problem.transform, problem.grids.pad_scale + + grids = problem.grids + carrier = grids.carrier_2w0 + grid = native.grid + grid.crop_index = np.ascontiguousarray(grids.crop_index, dtype=np.uint64) + grid.active_mask = np.ascontiguousarray(grids.active_mask) + grid.norm_mask = np.ascontiguousarray(grids.norm_mask) + grid.omega_window = np.ascontiguousarray(grids.omega_window) + grid.omega = np.ascontiguousarray(grids.omega) + grid.time_window = np.ascontiguousarray(grids.t_window_fine) + if problem.grid_kind == "field" and problem.responses.plasma_channels: + grid.time_window_derivative = np.ascontiguousarray( + grids.t_window_fine_derivative + ) + if geometry is None: + grid.power_weight = np.ascontiguousarray(power_weight_at(problem, 0.0)) + grid.carrier = np.ascontiguousarray( + np.ones(grids.n_to, dtype=np.complex128) if carrier is None else carrier ) - temporal_in_stage = ( - absorber_fine is not None and problem.responses.absorber_scheme == "nonlinear" + grid.dt_fine = grids.dt_fine + grid.dt = grids.dt + + _lower_linear(native, problem, plan) + coarse_matrix = _lower_spatial(native, problem, request) + _lower_stepper(native, problem, stepper, device=plan.device) + + native.absorber.in_stage = temporal_in_stage or spatial_absorption + native.absorber.split_step = ( + absorber is not None and problem.responses.absorber_scheme == "split_step" ) - carrier = problem.grids.carrier_2w0 - n_omega = problem.shape[-1] - phi_terms = fixed.phi - prefactor_terms = fixed.prefactor - spatial = fixed.separable - if spatial is not None: - phi = np.empty((1, 0, n_omega), dtype=np.complex128) - prefactor = np.empty((1, 0, n_omega), dtype=np.complex128) + if temporal_in_stage: + native.absorber.fine = np.ascontiguousarray(absorber_fine, dtype=np.float64) + elif spatial_absorption: + native.absorber.fine = np.zeros(grids.n_to) + if absorber is not None: + native.absorber.coarse = np.ascontiguousarray(absorber, dtype=np.float64) + + transform = native.transform + transform.synthesis_matrix = np.ascontiguousarray(problem.transform.synthesis) + transform.projection_matrix = np.ascontiguousarray(problem.transform.projection) + transform.projection_coarse_matrix = np.ascontiguousarray(coarse_matrix) + transform.synthesis = 0.0 if free else float(problem.transform.synthesis[0, 0]) + transform.projection = 0.0 if free else float(problem.transform.projection[0, 0]) + + if kerr is not None: + native.kerr.nonlinear = True + native.kerr.coefficient = kerr.coefficient + native.kerr.self_factor = kerr.self_factor + native.kerr.conjugate_factor = kerr.conjugate_factor + native.kerr.harmonic_factor = kerr.harmonic_factor + native.kerr.take_real = kerr.take_real + + _lower_plasma(native, problem, request) + _lower_raman(native, problem) + _lower_general(native, problem, general_chunk) + _lower_flow(native, plan.flow) + _lower_dispersive(native, problem) + _lower_flux(native, problem, request.observation.stats) + return native + + +def _lower_linear( + native: _core.Problem, problem: PropagationProblem, plan: _LowerPlan +) -> None: + """Fill the fixed terms, (n_terms, rows, n_omega): one empty term on a separable + route, a z table's slots.""" + del problem + tables = plan.tables + linear = native.linear + n_omega = native.dims.n_omega + linear.phi_table = tables is not None and tables.phi + linear.prefactor_table = tables is not None and tables.prefactor + if plan.terms.separable is not None: + linear.n_phi_terms = 1 + linear.n_prefactor_terms = 1 + return + # A z table's terms are slots the native evaluation fills (solprop SEP 29 G1): + # their count crosses in their place. + if tables is not None and tables.phi: + linear.n_phi_terms = tables.n_phi + elif plan.terms.phi is None: + raise BackendError("Kokkos requires a device-ready fixed-term problem") else: - # A z table's terms are slots the native evaluation fills (solprop SEP 29 G1): - # an empty array whose leading extent is their count crosses in their place. - if tables is not None and tables.phi: - phi = np.empty((tables.n_phi, 0), dtype=np.complex128) - elif phi_terms is None: - raise BackendError("Kokkos requires a device-ready fixed-term problem") - else: - phi = np.ascontiguousarray( - phi_terms.reshape(phi_terms.shape[0], -1, n_omega) - ) - if tables is not None and tables.prefactor: - prefactor = np.empty((tables.slots, 0), dtype=np.complex128) - elif prefactor_terms is None: - raise BackendError("Kokkos requires a device-ready fixed-term problem") - else: - prefactor = np.ascontiguousarray( - prefactor_terms.reshape(prefactor_terms.shape[0], -1, n_omega) - ) - channels = problem.responses.population_channels - raman_terms = problem.responses.raman_terms - plasma = channels[0] if channels else None - raman = raman_terms[0] if raman_terms else None - rate = None if plasma is None else plasma.rate + linear.n_phi_terms = plan.terms.phi.shape[0] + linear.phi_terms = np.ascontiguousarray( + plan.terms.phi.reshape(plan.terms.phi.shape[0], -1, n_omega) + ) + if tables is not None and tables.prefactor: + linear.n_prefactor_terms = tables.slots + elif plan.terms.prefactor is None: + raise BackendError("Kokkos requires a device-ready fixed-term problem") + else: + linear.n_prefactor_terms = plan.terms.prefactor.shape[0] + linear.prefactor_terms = np.ascontiguousarray( + plan.terms.prefactor.reshape(plan.terms.prefactor.shape[0], -1, n_omega) + ) + + +def _lower_spatial( + native: _core.Problem, problem: PropagationProblem, request: RunRequest +) -> np.ndarray: + """Fill the separable operator and a radial run's rows; return the coarse matrix. + + The quadrature alarm's embedded Kronrod projection is the transform's own unless + it is the plain projection or the route has one node, and the hybrid planar kind + packs none: the reference scores no alarm on it. + """ + transform = problem.transform coarse = ( np.empty(0, dtype=np.float64) - if problem.transform.n_nodes == 1 - or np.array_equal( - problem.transform.projection_coarse, problem.transform.projection - ) - else problem.transform.projection_coarse + if transform.n_nodes == 1 + or np.array_equal(transform.projection_coarse, transform.projection) + else transform.projection_coarse ) - geometry = problem.transform.geometry - # Free space and schema 15's hybrid planar kind share the separable operator and - # the transverse FFT engine; the planar kind adds its slab rule in x. - free = geometry is not None and geometry.kind in ("free", "slab") - spatial_absorption = bool(spatial_absorber.size) or ( - free and geometry is not None and bool(np.any(geometry.absorber_rate)) + geometry = transform.geometry + spatial = native.spatial + spatial.absorber_matrix = radial_absorber_matrix(transform, problem.grids.pad_scale) + if geometry is not None: + spatial.spectral_weights = geometry.spectral_weights + spatial.on_axis = geometry.on_axis + separable = problem.linear.separable + if geometry is None or geometry.kind not in ("free", "slab") or separable is None: + return coarse + spatial.n_x, spatial.n_y = geometry.shape + spatial.k_squared = separable.k_squared + spatial.k_squared_low = separable.k_squared_low + spatial.kperp_sq = separable.kperp_sq + spatial.kperp_sq_low = separable.kperp_sq_low + spatial.frame = separable.frame + spatial.spectral_rate = separable.spectral_rate + assert separable.user_rate is not None + assert separable.kperp_rate is not None + spatial.user_rate = separable.user_rate + spatial.numerator = separable.prefactor_numerator + spatial.clamped = separable.prefactor_clamped + spatial.evanescent_rate = separable.evanescent_rate + spatial.absorber_rate = geometry.absorber_rate + # A transverse collar's rate per column (solprop kernel schema 31). + spatial.kperp_rate = separable.kperp_rate + if geometry.kind == "free": + # solprop SEP 23: the stored axes and the slots of the problem's own + # far-field request, which size the native band partials. + k_x, k_y = geometry.spectral_axes + slots, _ = far_field_partials(problem, request.observation.stats) + spatial.k_x = np.ascontiguousarray(k_x, dtype=np.float64) + spatial.k_y = np.ascontiguousarray(k_y, dtype=np.float64) + spatial.far_field_slots = slots + return coarse + # Schema 15's `SeparableOperator.kappa_sq` is (n_channels, n_omega), column `c` + # belonging to channel `c % n_channels`; the geometry's collar rate depends on y + # alone (solprop checks it), so it is packed per column `p + n_p * iy`, the index + # of the temporal row it acts on. + assert separable.kappa_sq is not None + n_x, n_y = geometry.shape + n_p = separable.n_channels + spatial.n_p = n_p + spatial.kappa_sq = np.ascontiguousarray(separable.kappa_sq) + spatial.absorber_rate = np.ascontiguousarray( + np.repeat(geometry.absorber_rate.reshape(n_y, n_x)[:, 0], n_p) ) - coarse_matrix = coarse - spatial_data: dict[str, object] = {} - if free and geometry is not None and spatial is not None: - spatial_data = { - "n_x": geometry.shape[0], - "n_y": geometry.shape[1], - "spatial_k_squared": spatial.k_squared, - "spatial_k_squared_low": spatial.k_squared_low, - "spatial_kperp_sq": spatial.kperp_sq, - "spatial_kperp_sq_low": spatial.kperp_sq_low, - "spatial_frame": spatial.frame, - "spatial_spectral_rate": spatial.spectral_rate, - "spatial_user_rate": spatial.user_rate, - "spatial_numerator": spatial.prefactor_numerator, - "spatial_clamped": spatial.prefactor_clamped, - "spatial_evanescent_rate": spatial.evanescent_rate, - "spatial_absorber_rate": geometry.absorber_rate, - # A transverse collar's rate per column (solprop kernel schema 31). - "spatial_kperp_rate": spatial.kperp_rate, - } - if geometry.kind == "free": - # solprop SEP 23: the stored axes and the slots of the problem's own - # far-field request, which size the native band partials. - k_x, k_y = geometry.spectral_axes - slots, _ = far_field_partials(problem, request.observation.stats) - spatial_data |= { - "spatial_k_x": np.ascontiguousarray(k_x, dtype=np.float64), - "spatial_k_y": np.ascontiguousarray(k_y, dtype=np.float64), - "spatial_far_field_slots": slots, - } - if geometry.kind == "slab": - spatial_data |= _slab_payload(problem) - # The reference scores no quadrature alarm on this kind, so no embedded - # Kronrod projection is packed. - coarse_matrix = np.empty(0, dtype=np.float64) - # The accepted step's last stage leaves the plasma statistics behind only when - # that stage's input is the committed field: a first-same-as-last - # tableau, no split-step absorber (applied to the field after the last stage) - # and no coupled flow; and not under the modified noise model, where that input - # carries the noise draw while solprop defines the statistics on the committed - # field its reference scans. Otherwise they are scanned afresh. - stash_plasma = ( - plasma is not None + return np.empty(0, dtype=np.float64) + + +def _lower_stepper( + native: _core.Problem, + problem: PropagationProblem, + stepper: StepperSettings, + *, + device: bool, +) -> None: + """Fill the tableau and the low-storage march's rows (solprop kernel schema 32).""" + tableau = stepper.tableau + native_stepper = native.stepper + native_stepper.n_stages = tableau.n_stages + native_stepper.tableau_matrix = np.ascontiguousarray(tableau.matrix) + native_stepper.tableau_nodes = np.ascontiguousarray(tableau.nodes) + native_stepper.solution_weights = np.ascontiguousarray(tableau.solution_weights) + native_stepper.error_weights = np.ascontiguousarray(tableau.error_weights) + native_stepper.first_same_as_last = tableau.first_same_as_last + native_stepper.weighted_controller = stepper.controller.weights is not None + native_stepper.modified_noise = problem.initial.noise_model == "modified" + flows = _low_storage_flows(problem, stepper, device=device) + if flows is None: + return + native_stepper.low_storage = True + native_stepper.low_storage_stage_register = low_storage_stage_register( + problem, "native" + ) + native_stepper.low_storage_flows = flows + native_stepper.low_storage_tiled_input = device + + +def _lower_plasma( + native: _core.Problem, problem: PropagationProblem, request: RunRequest +) -> None: + """Fill the plasma columns: the first channel's rate, every channel's rows.""" + channels = problem.responses.population_channels + channel = channels[0] if channels else None + plasma = native.plasma + # solprop SEP 18: the running-integral rule, one per problem. + plasma.band = problem.responses.plasma_integral.policy == "band" + plasma.taps = problem.responses.plasma_integral.taps + if channel is not None: + rate = channel.rate + plasma.enabled = True + # The private gated benchmark (solprop's tests/gated_source.py), read where + # the backend is built, as the reference reads it. + plasma.gated = _BENCHMARK_GATED.get() + plasma.charge_mass_ratio = channel.charge_mass_ratio + plasma.ionisation_potential = channel.ionisation_potential + plasma.log_field_min = rate.log_field_min + plasma.d_log_field = rate.d_log_field + plasma.field_min = rate.field_min + plasma.field_max = rate.field_max + plasma.rate_max = rate.rate_max + plasma.rate_intervals = rate.coefficients.shape[1] + plasma.rate_coefficients = np.ascontiguousarray(rate.coefficients) + plasma.stash = _stash_plasma(problem, request) + _lower_channels(native, problem) + _lower_envelope(native, problem) + + +def _stash_plasma(problem: PropagationProblem, request: RunRequest) -> bool: + """Whether the accepted step's last stage leaves the plasma statistics behind. + + Only when that stage's input is the committed field: a first-same-as-last tableau, + no split-step absorber (applied to the field after the last stage) and no coupled + flow; and not under the modified noise model, where that input carries the noise + draw while solprop defines the statistics on the committed field its reference + scans. Otherwise they are scanned afresh. + """ + absorber = problem.responses.temporal_absorber + return ( + bool(problem.responses.population_channels) and problem.responses.envelope_plasma is None and problem.initial.noise_model != "modified" - and stepper.tableau.first_same_as_last + and request.stepper.tableau.first_same_as_last and not ( absorber is not None and problem.responses.absorber_scheme == "split_step" ) @@ -3432,133 +3561,35 @@ def _native_problem( & set(request.observation.stats.periodic) ) ) - return { - # solprop SEP 46: how the workspace's CPU operations thread, the process's - # record unless the backend was given one. - "cpu_execution": _package.EXECUTION.native(), - **spatial_data, - **_channel_payload(problem), - **_envelope_payload(problem), - **_general_payload(problem, general_chunk), - **_flow_payload(flow), - **_dispersive_payload(problem), - **_flux_payload(problem, request.observation.stats), - "n_omega": n_omega, - "n_t": problem.grids.n_t, - "n_to": problem.grids.n_to, - "n_stages": stepper.tableau.n_stages, - "n_modes": problem.shape[0], - "n_pol_state": problem.shape[1], - "n_pol_transform": problem.transform.n_pol, - "n_nodes": problem.transform.n_nodes, - # solprop SEP 47 section 6.5: the backend's execution choice, lowered by - # `native_node_chunk`. - "node_chunk": node_chunk, - "family": family, - "nonlinear": kerr is not None, - "modified_noise": problem.initial.noise_model == "modified", - "in_stage_absorber": temporal_in_stage or spatial_absorption, - "split_step_absorber": absorber is not None - and problem.responses.absorber_scheme == "split_step", - "first_same_as_last": stepper.tableau.first_same_as_last, - "crop_index": np.ascontiguousarray(problem.grids.crop_index, dtype=np.uint64), - "active_mask": np.ascontiguousarray(problem.grids.active_mask), - "norm_mask": np.ascontiguousarray(problem.grids.norm_mask), - "omega_window": np.ascontiguousarray(problem.grids.omega_window), - "omega": np.ascontiguousarray(problem.grids.omega), - "time_window": np.ascontiguousarray(problem.grids.t_window_fine), - "time_window_derivative": np.ascontiguousarray( - problem.grids.t_window_fine_derivative - if problem.grid_kind == "field" and problem.responses.plasma_channels - else np.empty(0) - ), - "absorber_fine": np.ascontiguousarray( - absorber_fine - if temporal_in_stage - else np.zeros(problem.grids.n_to) - if spatial_absorption - else np.empty(0), - dtype=np.float64, - ), - "spatial_absorber_matrix": spatial_absorber, - "spatial_spectral_weights": np.empty(0) - if geometry is None - else geometry.spectral_weights, - "spatial_on_axis": np.empty(0, dtype=np.complex128) - if geometry is None - else geometry.on_axis, - "absorber_coarse": np.ascontiguousarray( - np.empty(0) if absorber is None else absorber, dtype=np.float64 - ), - "carrier": np.ascontiguousarray( - np.ones(problem.grids.n_to, dtype=np.complex128) - if carrier is None - else carrier - ), - "synthesis_matrix": np.ascontiguousarray(problem.transform.synthesis), - "projection_matrix": np.ascontiguousarray(problem.transform.projection), - "projection_coarse_matrix": np.ascontiguousarray(coarse_matrix), - "phi_terms": phi, - "prefactor_terms": prefactor, - "phi_table": tables is not None and tables.phi, - "prefactor_table": tables is not None and tables.prefactor, - "power_weight": np.ascontiguousarray( - np.empty(0) if geometry is not None else power_weight_at(problem, 0.0) - ), - "tableau_matrix": np.ascontiguousarray(stepper.tableau.matrix), - "tableau_nodes": np.ascontiguousarray(stepper.tableau.nodes), - "solution_weights": np.ascontiguousarray(stepper.tableau.solution_weights), - "error_weights": np.ascontiguousarray(stepper.tableau.error_weights), - "synthesis": 0.0 if free else float(problem.transform.synthesis[0, 0]), - "projection": 0.0 if free else float(problem.transform.projection[0, 0]), - "kerr_coefficient": 0.0 if kerr is None else kerr.coefficient, - "self_factor": 0.0 if kerr is None else kerr.self_factor, - "conjugate_factor": 0.0 if kerr is None else kerr.conjugate_factor, - "harmonic_factor": 0.0 if kerr is None else kerr.harmonic_factor, - "take_real": kerr is not None and kerr.take_real, - "plasma": plasma is not None, - # The private gated benchmark (solprop's tests/gated_source.py), read where - # the backend is built, as the reference reads it. - "plasma_gated": plasma is not None and _BENCHMARK_GATED.get(), - "raman": raman is not None, - "raman_thg": raman is not None - and bool(raman.thg) - and problem.grid_kind == "field", - "raman_tensor_capable": bool(raman_terms) - and all(term.tensor_capable for term in raman_terms), - "raman_tensor_weights": np.ascontiguousarray( - np.concatenate([term.tensor_weights for term in raman_terms], axis=0) - if raman_terms - else np.empty((0, 2), dtype=np.float64) - ), - "stash_plasma": stash_plasma, - "weighted_controller": stepper.controller.weights is not None, - "dt_fine": problem.grids.dt_fine, - "dt": problem.grids.dt, - "raman_fraction": 0.0 if raman is None else float(raman.fraction), - "raman_epsilon0": EPS0 if raman is not None else 0.0, - "raman_undamped": np.ascontiguousarray( - np.empty((0, 2 * problem.grids.n_to), dtype=np.float64) - if raman is None - else np.concatenate([term.undamped for term in raman_terms], axis=0) - ), - "charge_mass_ratio": 0.0 if plasma is None else plasma.charge_mass_ratio, - "ionisation_potential": 0.0 if plasma is None else plasma.ionisation_potential, - "log_field_min": 0.0 if rate is None else rate.log_field_min, - "d_log_field": 0.0 if rate is None else rate.d_log_field, - "field_min": 0.0 if rate is None else rate.field_min, - "field_max": 0.0 if rate is None else rate.field_max, - "rate_max": 0.0 if rate is None else rate.rate_max, - "rate_coefficients": np.ascontiguousarray( - np.empty((4, 0), dtype=np.float64) if rate is None else rate.coefficients - ), - # solprop SEP 18: the running-integral rule, one per problem. - "plasma_band": problem.responses.plasma_integral.policy == "band", - "plasma_taps": problem.responses.plasma_integral.taps, - } -def _dispersive_payload(problem: PropagationProblem) -> dict[str, object]: +def _lower_raman(native: _core.Problem, problem: PropagationProblem) -> None: + """Fill the Raman terms, row-major (n_terms, 2 n_to) and (n_terms, 2).""" + terms = problem.responses.raman_terms + raman = native.raman + raman.n_components = len(terms) + raman.row_component = np.asarray( + [component for component, term in enumerate(terms) for _ in term.undamped], + dtype=np.float64, + ) + raman.component_fraction = np.asarray([term.fraction for term in terms]) + if not terms: + return + first = terms[0] + undamped = np.concatenate([term.undamped for term in terms], axis=0) + raman.enabled = True + raman.thg = bool(first.thg) and problem.grid_kind == "field" + raman.tensor_capable = all(term.tensor_capable for term in terms) + raman.n_terms = undamped.shape[0] + raman.fraction = float(first.fraction) + raman.epsilon0 = EPS0 + raman.undamped = np.ascontiguousarray(undamped) + raman.tensor_weights = np.ascontiguousarray( + np.concatenate([term.tensor_weights for term in terms], axis=0) + ) + + +def _lower_dispersive(native: _core.Problem, problem: PropagationProblem) -> None: """Lay out solprop SEP 6's dispersive response for the native engine. The structure the native response takes on every route: the channels grouped as the @@ -3577,7 +3608,7 @@ def _dispersive_payload(problem: PropagationProblem) -> dict[str, object]: """ leaf = problem.responses.dispersive_kerr if leaf is None: - return {"dispersive": False} + return tables = problem.transform.modal_tables general = tables is not None kind = "field" if general else problem.grid_kind @@ -3649,70 +3680,62 @@ def _dispersive_payload(problem: PropagationProblem) -> dict[str, object]: else: quartics = np.empty(0, dtype=np.float64) offsets = np.cumsum([0, *(len(group) for group in groups)]) - route: dict[str, object] = {} + lowered = native.dispersive if tables is not None: - route = { - "dispersive_group_keys": np.asarray(keys, dtype=np.uint64), - "dispersive_node_region": np.ascontiguousarray( - tables.region_index, dtype=np.uint64 - ), - } - return { - **route, - "dispersive": True, - "dispersive_symmetric": symmetric, - "dispersive_form": {"real": 0, "conserving": 1, "harmonic": 2}[form], - "dispersive_inputs": np.ascontiguousarray(stacked), - "dispersive_term_multipliers": np.ascontiguousarray( - np.array(multipliers, dtype=np.float64) - ), - "dispersive_term_rows": np.asarray(rows, dtype=np.uint64), - "dispersive_term_channels": np.asarray(channels, dtype=np.uint64), - "dispersive_coefficients": np.asarray(leaf.coefficients, dtype=np.float64), - "dispersive_pair_constants": np.ascontiguousarray(constants), - "dispersive_group_offsets": np.asarray(offsets, dtype=np.uint64), - "dispersive_group_members": np.asarray( - [channel for group in groups for channel in group], dtype=np.uint64 - ), - "dispersive_quartics": quartics, - "dispersive_input_scale": np.ascontiguousarray(input_scale), - "dispersive_output_scale": np.ascontiguousarray(scale), - } + lowered.group_keys = np.asarray(keys, dtype=np.uint64) + lowered.node_region = np.ascontiguousarray(tables.region_index, dtype=np.uint64) + lowered.enabled = True + lowered.symmetric = symmetric + lowered.form = {"real": 0, "conserving": 1, "harmonic": 2}[form] + lowered.inputs = np.ascontiguousarray(stacked) + lowered.term_multipliers = np.ascontiguousarray( + np.array(multipliers, dtype=np.float64) + ) + lowered.term_rows = np.asarray(rows, dtype=np.uint64) + lowered.term_channels = np.asarray(channels, dtype=np.uint64) + lowered.coefficients = np.asarray(leaf.coefficients, dtype=np.float64) + lowered.pair_constants = np.ascontiguousarray(constants) + lowered.group_offsets = np.asarray(offsets, dtype=np.uint64) + lowered.group_members = np.asarray( + [channel for group in groups for channel in group], dtype=np.uint64 + ) + lowered.quartics = quartics + lowered.input_scale = np.ascontiguousarray(input_scale) + lowered.output_scale = np.ascontiguousarray(scale) -def _flux_payload(problem: PropagationProblem, stats: StatsSpec) -> dict[str, object]: - """Pack the plane measure of solprop SEP 34 (kernel schema 35). +def _lower_flux( + native: _core.Problem, problem: PropagationProblem, stats: StatsSpec +) -> None: + """Fill the plane measure of solprop SEP 34 (kernel schema 35). A separable run (Cartesian, line, planar) is measured by its own operator's root, which the native state already holds: nothing more is packed, and a plan that is not that operator is refused. A radial run packs the entrance's material and transverse squares and whether a material z table follows - (:func:`_upload_flux_table`); the admission record is not read. ``flux_closure`` + (:func:`_upload_flux_table`); the admission record is not read. ``closure`` sizes the magnetic closure's buffers for the run's own local statistics, `stats`. """ flux = flux_of(problem.route) if flux is None: - return {} - closure = bool(set(stats.names) & LOCAL_FLUX_STATISTICS) + return + native.flux.closure = bool(set(stats.names) & LOCAL_FLUX_STATISTICS) if problem.linear.separable is not None: if flux.base is not problem.linear.separable or flux.material is not None: raise BackendError( "solprop-native measures a separable run by its own operator's root, " "and this plane measure is another" ) - return {"flux_closure": closure} + return base = flux.base - return { - "flux_k_squared": np.ascontiguousarray(base.k_squared, dtype=np.complex128), - "flux_k_squared_low": np.ascontiguousarray( - base.k_squared_low, dtype=np.complex128 - ), - "flux_kperp_sq": np.ascontiguousarray(base.kperp_sq, dtype=np.float64), - "flux_kperp_sq_low": np.ascontiguousarray(base.kperp_sq_low, dtype=np.float64), - "flux_table": flux.material is not None, - "flux_closure": closure, - } + native.flux.k_squared = np.ascontiguousarray(base.k_squared, dtype=np.complex128) + native.flux.k_squared_low = np.ascontiguousarray( + base.k_squared_low, dtype=np.complex128 + ) + native.flux.kperp_sq = np.ascontiguousarray(base.kperp_sq, dtype=np.float64) + native.flux.kperp_sq_low = np.ascontiguousarray(base.kperp_sq_low, dtype=np.float64) + native.flux.table = flux.material is not None def _upload_flux_table(native: _core.State, problem: PropagationProblem) -> None: @@ -3735,111 +3758,78 @@ def _upload_flux_table(native: _core.State, problem: PropagationProblem) -> None ) -def _slab_payload(problem: PropagationProblem) -> dict[str, object]: - """Pack the hybrid planar kind's channels, wall terms and per-column collar rate. - - Schema 15's ``SeparableOperator.kappa_sq`` is ``(n_channels, n_omega)``, column - ``c`` belonging to channel ``c % n_channels``; the geometry's collar rate depends on - y alone (solprop checks it), so it is packed per column ``p + n_p * iy``, the index - of the temporal row it acts on. - """ - geometry = problem.transform.geometry - separable = problem.linear.separable - assert geometry is not None - assert separable is not None - assert separable.kappa_sq is not None - n_x, n_y = geometry.shape - n_p = separable.n_channels - return { - "n_p": n_p, - "spatial_kappa_sq": np.ascontiguousarray(separable.kappa_sq), - "spatial_absorber_rate": np.ascontiguousarray( - np.repeat(geometry.absorber_rate.reshape(n_y, n_x)[:, 0], n_p) - ), - } - - -def _general_payload( - problem: PropagationProblem, chunk: int | None -) -> dict[str, object]: - """Pack the general modal route's maps, derived as the reference derives them. +def _lower_general( + native: _core.Problem, problem: PropagationProblem, chunk: int | None +) -> None: + """Fill the general modal route's maps, derived as the reference derives them. The per-node tensor, the form and the delayed maps come from solprop's own derivations (``solprop.translate.general``), so both backends execute the same arrays; the chunk is the route's own at the same budget (:func:`solprop.backends.node_chunk.general_planned_chunk`), given as `chunk`. - Nothing is packed for a problem without ``transform.modal_tables``. + Nothing is filled for a problem without ``transform.modal_tables``. """ tables = problem.transform.modal_tables if tables is None: - return {"general_modal": False} + return assert tables.synthesis is not None assert tables.projection is not None - regional = problem.responses.regional_responses - components = int(tables.electric.shape[2]) - nodes = int(tables.region_index.size) assert chunk is not None + regional = problem.responses.regional_responses owner, delayed_tensor, delayed_spectrum = node_delayed_maps(tables, regional) terms = 0 if delayed_tensor is None else int(delayed_tensor.shape[1]) - intervals, n_deg = (int(one) for one in tables.synthesis.shape[:2]) - return { - "general_modal": True, - "general_real_form": node_kerr_form(regional) == "real", - # A moving guide's maps are every Chebyshev coefficient along z, flattened - # over (n_intervals, n_deg) in the order of the stage bundle's - # transform_weights; a constant guide's are the one coefficient. - "general_map_intervals": intervals, - "general_map_n_deg": n_deg, - "general_components": components, - "general_nodes": nodes, - "general_node_chunk": chunk, - "general_responses": 0 if delayed_tensor is None else delayed_tensor.shape[0], - "general_terms": terms, - "general_pad_scale": float(problem.grids.pad_scale), - "general_epsilon0": EPS0, - "general_synthesis": np.ascontiguousarray(tables.synthesis), - "general_projection": np.ascontiguousarray(tables.projection), - "general_tensor": np.ascontiguousarray(node_kerr_tensor(tables, regional)), - "general_frequency_factor": np.ascontiguousarray( - tables.source_frequency_factor, dtype=np.complex128 - ), - "general_delayed_tensor": np.ascontiguousarray( - np.empty(0) if delayed_tensor is None else delayed_tensor - ), - "general_delayed_spectrum": np.ascontiguousarray( - np.empty(0, dtype=np.complex128) - if delayed_spectrum is None - else delayed_spectrum - ), - "general_node_response": np.ascontiguousarray( - np.empty(0, dtype=np.uint64) if not terms else owner.astype(np.int64) + 1, - dtype=np.uint64, - ), - } + general = native.general + general.enabled = True + general.real_form = node_kerr_form(regional) == "real" + # A moving guide's maps are every Chebyshev coefficient along z, flattened over + # (n_intervals, n_deg) in the order of the stage bundle's transform_weights; a + # constant guide's are the one coefficient. + general.map_intervals, general.map_n_deg = ( + int(one) for one in tables.synthesis.shape[:2] + ) + general.components = int(tables.electric.shape[2]) + general.nodes = int(tables.region_index.size) + general.node_chunk = chunk + general.responses = 0 if delayed_tensor is None else delayed_tensor.shape[0] + general.terms = terms + general.pad_scale = float(problem.grids.pad_scale) + general.epsilon0 = EPS0 + general.synthesis = np.ascontiguousarray(tables.synthesis) + general.projection = np.ascontiguousarray(tables.projection) + general.tensor = np.ascontiguousarray(node_kerr_tensor(tables, regional)) + general.frequency_factor = np.ascontiguousarray( + tables.source_frequency_factor, dtype=np.complex128 + ) + if delayed_tensor is not None: + general.delayed_tensor = np.ascontiguousarray(delayed_tensor) + if delayed_spectrum is not None: + general.delayed_spectrum = np.ascontiguousarray(delayed_spectrum) + if terms: + general.node_response = np.ascontiguousarray( + owner.astype(np.int64) + 1, dtype=np.uint64 + ) -def _envelope_payload(problem: PropagationProblem) -> dict[str, object]: - """Pack the accepted cycle rules and already masked spectral factors.""" +def _lower_envelope(native: _core.Problem, problem: PropagationProblem) -> None: + """Fill the accepted cycle rules, (n_channels, 4, 512), and masked factors.""" assembly = problem.responses.envelope_plasma if assembly is None: - return {"envelope_plasma": False} + return nodes = np.zeros((len(assembly.channels), 4, 512)) weights = np.zeros_like(nodes) for channel_index, channel in enumerate(assembly.channels): for level, rule in enumerate(channel.cycle.rules): nodes[channel_index, level, : len(rule.nodes)] = rule.nodes weights[channel_index, level, : len(rule.weights)] = rule.weights - return { - "envelope_plasma": True, - "envelope_rule_nodes": nodes, - "envelope_rule_weights": weights, - "envelope_drude_factor": assembly.factors.drude, - "envelope_loss_factor": assembly.factors.loss, - } + native.plasma.envelope = True + native.plasma.envelope_rule_nodes = nodes + native.plasma.envelope_rule_weights = weights + native.plasma.envelope_drude_factor = assembly.factors.drude + native.plasma.envelope_loss_factor = assembly.factors.loss -def _channel_payload(problem: PropagationProblem) -> dict[str, object]: - """Pack independent rate tables without resampling their coefficients.""" +def _lower_channels(native: _core.Problem, problem: PropagationProblem) -> None: + """Fill independent rate tables without resampling their coefficients.""" parameters: list[list[float]] = [] coefficients: list[np.ndarray] = [] offset = 0 @@ -3860,17 +3850,45 @@ def _channel_payload(problem: PropagationProblem) -> dict[str, object]: ) coefficients.append(rate.coefficients.ravel()) offset += rate.coefficients.size - terms = problem.responses.raman_terms - return { - "n_channels": len(parameters), - "channel_parameters": np.asarray(parameters, dtype=np.float64).reshape(-1, 9), - "channel_coefficients": np.concatenate(coefficients) - if coefficients - else np.empty(0), - "n_raman_components": len(terms), - "raman_row_component": np.asarray( - [component for component, term in enumerate(terms) for _ in term.undamped], - dtype=np.float64, - ), - "raman_component_fraction": np.asarray([term.fraction for term in terms]), - } + native.plasma.n_channels = len(parameters) + native.plasma.channel_parameters = np.asarray(parameters, dtype=np.float64).reshape( + -1, 9 + ) + if coefficients: + native.plasma.channel_coefficients = np.concatenate(coefficients) + + +def _lower_flow(native: _core.Problem, flow: _HostFlow | None) -> None: + """Fill a coupled flow's bins and residual data; its actions follow per step.""" + if flow is None: + return + slot = np.zeros(flow.active.size, dtype=np.uint64) + slot[flow.active] = np.arange(1, int(np.count_nonzero(flow.active)) + 1) + residual = flow.residual_terms + # The bins formed by scaling and squaring are numbered 1, 2, ... in row-major + # (section, bin) order and carry their Schur factors; the rest take the + # eigenbasis, whose `W` and `Y` are zero where it is not used. + fallback = ~flow.route + route = np.zeros(flow.route.shape, dtype=np.uint64) + route[fallback] = np.arange(1, int(np.count_nonzero(fallback)) + 1) + lowered = native.flow + lowered.enabled = True + lowered.active = int(np.count_nonzero(flow.active)) + lowered.sections = int(flow.frozen.shape[0]) + lowered.terms = 0 if residual is None else int(residual.shape[0]) + lowered.slot = slot + if residual is not None: + # The residual's data are zero on the bins the error norm does not score, so + # the engine's residual there is an exact zero and the frozen flow alone + # carries them, as in solprop's reference. Every term is finite, so each + # product with a zero is a signed zero and the running sums stay +0: the + # source is what adding nothing leaves, but for the sign of a zero. + scored = flow.scored[None, :, None, None] + lowered.frozen = np.ascontiguousarray(np.where(scored, flow.frozen, 0.0)) + lowered.residual_terms = np.ascontiguousarray(np.where(scored, residual, 0.0)) + lowered.left = flow.left + lowered.right = flow.right + lowered.eigenvalues = flow.eigenvalues + lowered.route = route + lowered.fallback_vectors = np.ascontiguousarray(flow.vectors[fallback]) + lowered.fallback_triangular = np.ascontiguousarray(flow.triangular[fallback]) diff --git a/native/src/solprop_native/session.py b/native/src/solprop_native/session.py index 1d7e1e60c..56a0341d0 100644 --- a/native/src/solprop_native/session.py +++ b/native/src/solprop_native/session.py @@ -152,7 +152,7 @@ _NO_SCHEDULE, # pyright: ignore[reportPrivateUsage] NativeBackend, _aperture_payload, # pyright: ignore[reportPrivateUsage] - _low_storage_payload, # pyright: ignore[reportPrivateUsage] + _low_storage_flows, # pyright: ignore[reportPrivateUsage] _on_device, # pyright: ignore[reportPrivateUsage] _raise_native, # pyright: ignore[reportPrivateUsage] ) @@ -604,11 +604,12 @@ def _prepare( kind = "low_storage" # The flows `_ResidentLowStorage` holds, and the operator's origin, from # which a constant operator's table is built. - payload = _low_storage_payload( + flows = _low_storage_flows( self._problem, self._stepper, device=_on_device() ) + assert flows is not None # the march is the low-storage one backend._native_state.low_storage_configure( # pyright: ignore[reportPrivateUsage] - int(cast("int", payload["low_storage_flows"])), + flows, driver._terms.separable is not None, float(self._problem.linear.z_span[0]), ) diff --git a/native/src/watch.cpp b/native/src/watch.cpp index 79e6d4bb7..785bcd8d1 100644 --- a/native/src/watch.cpp +++ b/native/src/watch.cpp @@ -4,6 +4,8 @@ #include "solprop_native/watch.hpp" +#include "solprop_native/refuse.hpp" + #include #include #include @@ -43,8 +45,8 @@ double py_min(const double a, const double b) { return b < a ? b : a; } /** NumPy's `maximum(x, 0.0)`: the larger, a NaN kept. */ double clip_low(const double x) { return std::isnan(x) || x >= 0.0 ? x : 0.0; } -[[noreturn]] void refuse(const std::string& field) { - throw std::invalid_argument("the boundary watch's policy has an invalid " + field); +[[noreturn]] void refuse_policy(const char* field) { + refuse("the boundary watch's policy has an invalid ", field); } /** `_settle`: the cadence and the quiet flag from a check's shares. */ @@ -111,15 +113,15 @@ Verdict judge_shares(const Policy& policy, State& state, const double* shares, void validate(const Policy& policy) { if (policy.cadence < 1) - refuse("cadence"); + refuse_policy("cadence"); if (!(policy.interval > 0.0)) - refuse("interval"); + refuse_policy("interval"); if (!(policy.warn > 0.0) || !(policy.stop >= policy.warn)) - refuse("warning or stop threshold"); + refuse_policy("warning or stop threshold"); if (!(policy.activity >= 0.0) || !(policy.launch_floor >= 0.0)) - refuse("activity threshold or launch floor"); + refuse_policy("activity threshold or launch floor"); if (policy.crossing && policy.bucket_limits.size() != buckets) - refuse("bucket limits"); + refuse_policy("bucket limits"); } bool due(const Policy& policy, const State& state, const double z) { @@ -214,11 +216,11 @@ Monitor::Monitor(Config config) : config_(std::move(config)) { policy.launch_floor = 0.0; validate(policy); if (!(config_.floor_share >= 0.0) || !(config_.floor_margin >= 0.0)) - refuse("floor share or margin"); + refuse_policy("floor share or margin"); if (!(config_.budget > 0.0)) - refuse("budget"); + refuse_policy("budget"); if (config_.rate.empty()) - refuse("collar rate: the coarse axis has no samples"); + refuse_policy("collar rate: the coarse axis has no samples"); // `_busy` starts true: every step is checked until the launch says otherwise. state_.busy = true; } diff --git a/native/tests/native/test_aperture.cpp b/native/tests/native/test_aperture.cpp index 325511dc8..e1fcb1097 100644 --- a/native/tests/native/test_aperture.cpp +++ b/native/tests/native/test_aperture.cpp @@ -68,17 +68,17 @@ Case make_case(const std::size_t n_x, const std::size_t n_y, const std::size_t n Case data; auto& problem = data.problem; const std::size_t n = 9; - problem.n_x = n_x; - problem.n_y = n_y; - problem.n_omega = n; - problem.n_t = envelope ? n : 2 * (n - 1); - problem.n_pol_state = n_pol; - problem.n_modes = n_x * n_y; - problem.dt = 1.5e-15; - problem.family = + problem.spatial.n_x = n_x; + problem.spatial.n_y = n_y; + problem.dims.n_omega = n; + problem.dims.n_t = envelope ? n : 2 * (n - 1); + problem.dims.n_pol_state = n_pol; + problem.dims.n_modes = n_x * n_y; + problem.grid.dt = 1.5e-15; + problem.dims.family = envelope ? native::FieldFamily::envelope : native::FieldFamily::field_real; - problem.spatial_k_x = fft_axis(n_x, n_y > 1 ? 1.5e6 : 1.0e5); - problem.spatial_k_y = n_y > 1 ? fft_axis(n_y, 1.2e6) : std::vector{0.0}; + problem.spatial.k_x = fft_axis(n_x, n_y > 1 ? 1.5e6 : 1.0e5); + problem.spatial.k_y = n_y > 1 ? fft_axis(n_y, 1.2e6) : std::vector{0.0}; const std::size_t columns = n_x * n_y; // NOLINTNEXTLINE(bugprone-random-generator-seed): a fixed seed keeps the test reproducible std::mt19937_64 random(7); @@ -91,8 +91,8 @@ Case make_case(const std::size_t n_x, const std::size_t n_y, const std::size_t n const double phase = 0.3 * static_cast(column); data.on_axis.emplace_back(std::cos(phase) / static_cast(columns), std::sin(phase) / static_cast(columns)); - const double kx = problem.spatial_k_x[column % n_x]; - const double ky = problem.spatial_k_y[column / n_x]; + const double kx = problem.spatial.k_x[column % n_x]; + const double ky = problem.spatial.k_y[column / n_x]; data.transverse.push_back(kx * kx + ky * ky); data.transverse_low.push_back(1e-3 * static_cast(column % 3)); data.transverse_rate.push_back(column % 2 == 0 ? 0.0 : 3.0); @@ -152,13 +152,13 @@ template auto device(const char* label, const std::vector& values) // One row's table entry, by its constant. double& constant_of(Case& data, const std::size_t index, const native::FarFieldConstant constant) { - const auto row = native::far_field_aperture_constants + data.problem.n_omega; + const auto row = native::far_field_aperture_constants + data.problem.dims.n_omega; return data.apertures[index * row + constant]; } std::vector run(Case& data, const std::uint64_t flags, const int profile, const int exponent) { - const auto n = data.problem.n_omega; + const auto n = data.problem.dims.n_omega; // Every row takes `profile`; a test of mixed rows passes kPerRow and sets its own. std::vector apertures = data.apertures; if (profile != kPerRow) { @@ -166,7 +166,8 @@ std::vector run(Case& data, const std::uint64_t flags, const int profile for (std::size_t index = 0; index * row < apertures.size(); ++index) apertures[index * row + native::far_field_profile] = static_cast(profile); } - const auto rows = data.problem.n_x * data.problem.n_y * data.problem.n_pol_state; + const auto rows = data.problem.spatial.n_x * data.problem.spatial.n_y * + data.problem.dims.n_pol_state; native::FarFieldPass::ComplexView2 field("test.field", rows, n); const auto host = Kokkos::create_mirror_view(field); for (std::size_t row = 0; row < rows; ++row) @@ -188,7 +189,6 @@ std::vector run(Case& data, const std::uint64_t flags, const int profile .active = device("test.active", data.active), .evanescent = data.evanescent}; auto values = pass.run(flags, apertures, exponent, data.regions, 0.5, field, inputs); - pass.close(); return values; } @@ -197,7 +197,7 @@ std::vector run(Case& data, const std::uint64_t flags, const int profile // in TheShapesAreTheirFormulas. double host_weight(const Case& data, const std::size_t index, const double kx, const double ky, const std::size_t bin, const int profile) { - const auto row = native::far_field_aperture_constants + data.problem.n_omega; + const auto row = native::far_field_aperture_constants + data.problem.dims.n_omega; const double* constants = &data.apertures[index * row]; const double scale = constants[native::far_field_aperture_constants + bin]; const double centre_x = constants[native::far_field_centre_x]; @@ -235,11 +235,11 @@ double host_weight(const Case& data, const std::size_t index, const double kx, std::vector reference(const Case& data, const int profile, const int exponent, const bool magnitude) { const auto& problem = data.problem; - const auto n = problem.n_omega; - const auto n_pol = problem.n_pol_state; - const auto columns = problem.n_x * problem.n_y; - const bool envelope = problem.family == native::FieldFamily::envelope; - const double dt = problem.dt; + const auto n = problem.dims.n_omega; + const auto n_pol = problem.dims.n_pol_state; + const auto columns = problem.spatial.n_x * problem.spatial.n_y; + const bool envelope = problem.dims.family == native::FieldFamily::envelope; + const double dt = problem.grid.dt; const double density = envelope ? dt * dt / (2.0 * std::numbers::pi) : 2.0 * dt * dt / std::numbers::pi; const std::size_t row = native::far_field_aperture_constants + n; @@ -258,8 +258,8 @@ std::vector reference(const Case& data, const int profile, const int exp std::vector> closed(n_pol); std::vector> closed_axial(n_apertures * n_pol); for (std::size_t column = 0; column < columns; ++column) { - const double kx = problem.spatial_k_x[column % problem.n_x]; - const double ky = problem.spatial_k_y[column / problem.n_x]; + const double kx = problem.spatial.k_x[column % problem.spatial.n_x]; + const double ky = problem.spatial.k_y[column / problem.spatial.n_x]; double power = 0.0; for (std::size_t pol = 0; pol < n_pol; ++pol) power += std::norm( @@ -284,7 +284,7 @@ std::vector reference(const Case& data, const int profile, const int exp const double weighted = data.weights[column] * power * closing.real(); beam[bin] += weighted; const double energy = (envelope || bin == 0 || bin + 1 == n ? 1.0 : 4.0) * dt / - static_cast(problem.n_t); + static_cast(problem.dims.n_t); absorbed[0] += energy * weighted * (data.spectral_rate[bin] - data.user_rate[bin]); absorbed[4] += energy * weighted * data.user_rate[bin]; @@ -387,7 +387,7 @@ TEST_P(FarFieldPassAgainstHost, EveryStatisticMatchesTheSerialReduction) { expect_matches(actual, expected, magnitude, variant.n_x * variant.n_y); // Every channel but the evanescent one, which carries no axial flux, and every // aperture take part on this fixture. - const auto n = data.problem.n_omega; + const auto n = data.problem.dims.n_omega; for (const std::size_t channel : {0U, 1U, 3U, 4U}) EXPECT_NE(actual[n + channel], 0.0) << channel; EXPECT_EQ(actual[n + 2], 0.0); @@ -412,7 +412,8 @@ TEST(FarFieldPass, EveryRadiusPathMatchesTheSerialReduction) { // and a second distance (neither): each against the host reduction. for (const int path : {0, 1, 2}) { auto data = make_case(5, 4, 2, true, 1); - const std::size_t row = native::far_field_aperture_constants + data.problem.n_omega; + const std::size_t row = + native::far_field_aperture_constants + data.problem.dims.n_omega; for (std::size_t index = 1; index < 3; ++index) { if (path == 0) { data.apertures[index * row + native::far_field_centre_x] = @@ -422,7 +423,7 @@ TEST(FarFieldPass, EveryRadiusPathMatchesTheSerialReduction) { } if (path == 2 && index == 2) { data.apertures[index * row + native::far_field_distance] = 0.3; - for (std::size_t bin = 0; bin < data.problem.n_omega; ++bin) + for (std::size_t bin = 0; bin < data.problem.dims.n_omega; ++bin) data.apertures[index * row + native::far_field_aperture_constants + bin] = 0.3 * native::far_field_speed_of_light / data.omega[bin]; } @@ -438,7 +439,7 @@ TEST(FarFieldPass, MixedShapesAndProfilesMatchTheSerialReduction) { // ellipse, one distance and then two, so the shaped branch runs under both origins. for (const bool one_distance : {true, false}) { auto data = make_case(5, 4, 2, true, 1); - const std::size_t n = data.problem.n_omega; + const std::size_t n = data.problem.dims.n_omega; const std::size_t row = native::far_field_aperture_constants + n; const auto scale_row = std::vector(data.apertures.begin() + @@ -486,7 +487,7 @@ TEST(FarFieldPass, AHalfPlaneAndItsComplementTileTheColumns) { // In a band whose outer columns the map still reaches, both rows at one edge sum to // the terms of every column inside the disc: exactly one row holds each, by one sign. auto data = make_case(5, 4, 1, true, 0); - const std::size_t n = data.problem.n_omega; + const std::size_t n = data.problem.dims.n_omega; for (const auto shape_pair : {std::array{native::far_field_half_plane_x_ge, native::far_field_half_plane_x_lt}, std::array{native::far_field_half_plane_y_ge, @@ -501,11 +502,11 @@ TEST(FarFieldPass, AHalfPlaneAndItsComplementTileTheColumns) { constant_of(rows, index, native::far_field_centre_y) = shape_pair[0] == native::far_field_half_plane_x_ge ? 0.0 : -0.02; } - const auto columns = rows.problem.n_x * rows.problem.n_y; + const auto columns = rows.problem.spatial.n_x * rows.problem.spatial.n_y; for (std::size_t column = 0; column < columns; ++column) for (std::size_t bin = 0; bin < n; ++bin) { - const double kx = rows.problem.spatial_k_x[column % rows.problem.n_x]; - const double ky = rows.problem.spatial_k_y[column / rows.problem.n_x]; + const double kx = rows.problem.spatial.k_x[column % rows.problem.spatial.n_x]; + const double ky = rows.problem.spatial.k_y[column / rows.problem.spatial.n_x]; const double first = host_weight(rows, 0, kx, ky, bin, 0); const double second = host_weight(rows, 1, kx, ky, bin, 0); const double scale = rows.apertures[native::far_field_aperture_constants + bin]; @@ -607,7 +608,7 @@ TEST(FarFieldPass, TheShapesAreTheirFormulas) { TEST(FarFieldPass, APartialRequestPacksOnlyItsRows) { auto data = make_case(5, 4, 1, true, 1); const auto every = run(data, kEvery, 1, 2); - const auto n = data.problem.n_omega; + const auto n = data.problem.dims.n_omega; // The beam spectrum and the apertures' on-axis spectra alone: the same values as // the full request's rows, within the bound (another slot count, the same sums). const auto partial = @@ -641,7 +642,7 @@ TEST(FarFieldPass, DeclaredAperturesCostNothingUnlessAsked) { data.apertures.clear(); const auto bare = run(data, flags, 2, 1); ASSERT_EQ(declared.size(), bare.size()); - const auto n = data.problem.n_omega; + const auto n = data.problem.dims.n_omega; constexpr auto absorbed = native::far_field_absorbed_channels; ASSERT_EQ(bare.size(), (1 + 4 + 1) * n + absorbed); const double relative = 3.0 * gamma(8 * 20 + 64) + 8.0 * kEpsilon; @@ -791,7 +792,7 @@ TEST(FarFieldPass, InvalidRequestsAreRefused) { EXPECT_THROW(static_cast(run(bare, native::far_field_collected, 1, 2)), std::invalid_argument); auto radial = data; - radial.problem.n_x = 0; + radial.problem.spatial.n_x = 0; EXPECT_THROW(static_cast(run(radial, kEvery, 1, 2)), std::invalid_argument); } diff --git a/native/tests/native/test_core.cpp b/native/tests/native/test_core.cpp index 235f173d8..9382fae59 100644 --- a/native/tests/native/test_core.cpp +++ b/native/tests/native/test_core.cpp @@ -93,52 +93,54 @@ std::vector real_time(const std::vector& spectrum, native::Problem base_problem(const native::FieldFamily family) { native::Problem problem; problem.cpu_execution = native::test::table(); - problem.n_t = 4; - problem.n_to = family == native::FieldFamily::envelope ? 4 : 8; - problem.n_omega = family == native::FieldFamily::envelope ? 4 : 3; - problem.n_stages = 1; - problem.n_phi_terms = 1; - problem.n_prefactor_terms = 1; - problem.family = family; - problem.crop_index = {0, 1, 2}; + problem.dims.n_t = 4; + problem.dims.n_to = family == native::FieldFamily::envelope ? 4 : 8; + problem.dims.n_omega = family == native::FieldFamily::envelope ? 4 : 3; + problem.stepper.n_stages = 1; + problem.linear.n_phi_terms = 1; + problem.linear.n_prefactor_terms = 1; + problem.dims.family = family; + problem.grid.crop_index = {0, 1, 2}; if (family == native::FieldFamily::envelope) - problem.crop_index.push_back(3); - problem.active_mask.assign(problem.n_omega, 1U); - problem.norm_mask.assign(problem.n_omega, 1U); - problem.omega_window.assign(problem.n_omega, 1.0); - for (std::size_t index = 0; index < problem.n_omega; ++index) - problem.omega.push_back(static_cast(index + 1)); - problem.dt = 0.25; - problem.time_window.assign(problem.n_to, 1.0); - problem.time_window_derivative.assign(problem.n_to, 0.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); - problem.phi_terms.assign(problem.n_omega, Complex{}); - problem.prefactor_terms.assign(problem.n_omega, Complex(1.0, 0.0)); - problem.tableau_matrix = {0.0}; - problem.tableau_nodes = {0.0}; - problem.solution_weights = {1.0}; - problem.error_weights = {0.25}; - problem.synthesis = 2.0; - problem.projection = 0.5; - problem.kerr_coefficient = 0.125; - problem.self_factor = 0.5; - problem.conjugate_factor = 0.25; - problem.harmonic_factor = 0.0; - problem.take_real = family != native::FieldFamily::envelope; + problem.grid.crop_index.push_back(3); + problem.grid.active_mask.assign(problem.dims.n_omega, 1U); + problem.grid.norm_mask.assign(problem.dims.n_omega, 1U); + problem.grid.omega_window.assign(problem.dims.n_omega, 1.0); + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) + problem.grid.omega.push_back(static_cast(index + 1)); + problem.grid.dt = 0.25; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.time_window_derivative.assign(problem.dims.n_to, 0.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); + problem.linear.phi_terms.assign(problem.dims.n_omega, Complex{}); + problem.linear.prefactor_terms.assign(problem.dims.n_omega, Complex(1.0, 0.0)); + problem.stepper.tableau_matrix = {0.0}; + problem.stepper.tableau_nodes = {0.0}; + problem.stepper.solution_weights = {1.0}; + problem.stepper.error_weights = {0.25}; + problem.transform.synthesis = 2.0; + problem.transform.projection = 0.5; + problem.kerr.coefficient = 0.125; + problem.kerr.self_factor = 0.5; + problem.kerr.conjugate_factor = 0.25; + problem.kerr.harmonic_factor = 0.0; + problem.kerr.take_real = family != native::FieldFamily::envelope; return problem; } native::TrialScalars trial(const native::Problem& problem, const double taper = 1.0) { return { - .phi_delta = - std::vector((problem.n_stages + 1) * problem.n_phi_terms, 0.0), - .phi_at = std::vector(problem.n_stages * problem.n_phi_terms, 0.0), - .prefactor = - std::vector(problem.n_stages * problem.n_prefactor_terms, 1.0), - .density = std::vector(problem.n_stages, 1.0), - .taper = std::vector(problem.n_stages, taper), + .phi_delta = std::vector( + (problem.stepper.n_stages + 1) * problem.linear.n_phi_terms, 0.0), + .phi_at = std::vector( + problem.stepper.n_stages * problem.linear.n_phi_terms, 0.0), + .prefactor = std::vector( + problem.stepper.n_stages * problem.linear.n_prefactor_terms, 1.0), + .density = std::vector(problem.stepper.n_stages, 1.0), + .taper = std::vector(problem.stepper.n_stages, taper), .preionisation = {}, - .raman_rates = std::vector(problem.n_stages * problem.n_raman_terms, 0.0), + .raman_rates = + std::vector(problem.stepper.n_stages * problem.raman.n_terms, 0.0), .kerr_density = {}, .plasma_densities = {}, .plasma_preionisation = {}, @@ -152,64 +154,66 @@ native::TrialScalars trial(const native::Problem& problem, const double taper = native::Problem multimode_problem(const std::size_t node_chunk = 2) { auto problem = base_problem(native::FieldFamily::field_real); - problem.n_modes = 2; - problem.n_pol_state = 1; - problem.n_pol_transform = 1; - problem.n_nodes = 2; - problem.node_chunk = node_chunk; - problem.nonlinear = true; - problem.synthesis_matrix = {1.0, 0.5, -0.25, 1.5}; - problem.projection_matrix = {0.75, -0.5, 0.25, 1.25}; - problem.projection_coarse_matrix = {0.5, -0.25, 0.125, 1.0}; - problem.phi_terms.assign(problem.n_modes * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(problem.n_modes * problem.n_omega, Complex(1.0, 0.0)); + problem.dims.n_modes = 2; + problem.dims.n_pol_state = 1; + problem.dims.n_pol_transform = 1; + problem.dims.n_nodes = 2; + problem.dims.node_chunk = node_chunk; + problem.kerr.nonlinear = true; + problem.transform.synthesis_matrix = {1.0, 0.5, -0.25, 1.5}; + problem.transform.projection_matrix = {0.75, -0.5, 0.25, 1.25}; + problem.transform.projection_coarse_matrix = {0.5, -0.25, 0.125, 1.0}; + problem.linear.phi_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex{}); + problem.linear.prefactor_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex(1.0, 0.0)); return problem; } native::Problem general_problem(const native::FieldFamily family) { auto problem = base_problem(family); - problem.nonlinear = true; - problem.synthesis_matrix = {problem.synthesis}; - problem.projection_matrix = {problem.projection}; + problem.kerr.nonlinear = true; + problem.transform.synthesis_matrix = {problem.transform.synthesis}; + problem.transform.projection_matrix = {problem.transform.projection}; return problem; } native::Problem plasma_problem() { auto problem = general_problem(native::FieldFamily::field_real); - problem.nonlinear = false; - problem.plasma = true; - problem.stash_plasma = true; - problem.dt_fine = 0.125; - problem.charge_mass_ratio = 2.0; - problem.ionisation_potential = 3.0; - problem.log_field_min = std::log(0.01); - problem.d_log_field = std::log(1000.0); - problem.field_min = 0.01; - problem.field_max = 10.0; - problem.rate_max = 0.5; - problem.rate_intervals = 1; - problem.rate_coefficients = {0.0, 0.0, 0.0, std::log(0.5)}; + problem.kerr.nonlinear = false; + problem.plasma.enabled = true; + problem.plasma.stash = true; + problem.grid.dt_fine = 0.125; + problem.plasma.charge_mass_ratio = 2.0; + problem.plasma.ionisation_potential = 3.0; + problem.plasma.log_field_min = std::log(0.01); + problem.plasma.d_log_field = std::log(1000.0); + problem.plasma.field_min = 0.01; + problem.plasma.field_max = 10.0; + problem.plasma.rate_max = 0.5; + problem.plasma.rate_intervals = 1; + problem.plasma.rate_coefficients = {0.0, 0.0, 0.0, std::log(0.5)}; return problem; } native::Problem raman_problem(const native::FieldFamily family) { auto problem = general_problem(family); - problem.nonlinear = false; - problem.raman = true; - problem.raman_thg = family == native::FieldFamily::field_real; - problem.n_raman_terms = 2; - problem.raman_tensor_weights = {1.0, 0.0, 1.0, 0.0}; - problem.raman_fraction = 0.4; - problem.raman_epsilon0 = 2.0; - problem.dt_fine = 0.125; - problem.raman_undamped.assign(problem.n_raman_terms * 2 * problem.n_to, 0.0); - problem.raman_undamped[1] = 1.0; - problem.raman_undamped[2] = 0.5; - problem.raman_undamped[3] = -0.25; - const auto second = 2 * problem.n_to; - problem.raman_undamped[second + 1] = 0.2; - problem.raman_undamped[second + 2] = 0.1; - problem.raman_undamped[second + 3] = 0.05; + problem.kerr.nonlinear = false; + problem.raman.enabled = true; + problem.raman.thg = family == native::FieldFamily::field_real; + problem.raman.n_terms = 2; + problem.raman.tensor_weights = {1.0, 0.0, 1.0, 0.0}; + problem.raman.fraction = 0.4; + problem.raman.epsilon0 = 2.0; + problem.grid.dt_fine = 0.125; + problem.raman.undamped.assign(problem.raman.n_terms * 2 * problem.dims.n_to, 0.0); + problem.raman.undamped[1] = 1.0; + problem.raman.undamped[2] = 0.5; + problem.raman.undamped[3] = -0.25; + const auto second = 2 * problem.dims.n_to; + problem.raman.undamped[second + 1] = 0.2; + problem.raman.undamped[second + 2] = 0.1; + problem.raman.undamped[second + 3] = 0.05; return problem; } @@ -217,94 +221,95 @@ std::vector raman_reference(const native::Problem& problem, const std::vector& spectrum, const std::vector& rates) { std::vector modal; - if (problem.family == native::FieldFamily::field_real) { - for (const double value : real_time(spectrum, problem.n_to)) + if (problem.dims.family == native::FieldFamily::field_real) { + for (const double value : real_time(spectrum, problem.dims.n_to)) modal.emplace_back(value, 0.0); - } else if (problem.family == native::FieldFamily::field_analytic) { - auto analytic = std::vector(problem.n_to); + } else if (problem.dims.family == native::FieldFamily::field_analytic) { + auto analytic = std::vector(problem.dims.n_to); analytic[0] = spectrum[0]; for (std::size_t index = 1; index < spectrum.size(); ++index) analytic[index] = 2.0 * spectrum[index]; - modal = envelope_time(analytic, problem.n_to); + modal = envelope_time(analytic, problem.dims.n_to); } else { - modal = envelope_time(spectrum, problem.n_to); + modal = envelope_time(spectrum, problem.dims.n_to); } - const auto doubled = 2 * problem.n_to; + const auto doubled = 2 * problem.dims.n_to; std::vector kernel(doubled); for (std::size_t time = 0; time < doubled; ++time) { double total = 0.0; - const double delay = static_cast(time) * problem.dt_fine; - for (std::size_t term = 0; term < problem.n_raman_terms; ++term) - total += problem.raman_undamped[term * doubled + time] * + const double delay = static_cast(time) * problem.grid.dt_fine; + for (std::size_t term = 0; term < problem.raman.n_terms; ++term) + total += problem.raman.undamped[term * doubled + time] * std::exp(-rates[term] * delay); - kernel[time] = problem.raman_fraction * total; + kernel[time] = problem.raman.fraction * total; } - std::vector projected(problem.n_to); - for (std::size_t node = 0; node < problem.n_nodes; ++node) { - const double synthesis = problem.synthesis_matrix[node]; - const double projection = problem.projection_matrix[node]; - std::vector field(problem.n_to); - std::vector drive(problem.n_to); - for (std::size_t time = 0; time < problem.n_to; ++time) { + std::vector projected(problem.dims.n_to); + for (std::size_t node = 0; node < problem.dims.n_nodes; ++node) { + const double synthesis = problem.transform.synthesis_matrix[node]; + const double projection = problem.transform.projection_matrix[node]; + std::vector field(problem.dims.n_to); + std::vector drive(problem.dims.n_to); + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { field[time] = synthesis * modal[time]; - drive[time] = problem.raman_thg ? field[time].real() * field[time].real() + drive[time] = problem.raman.thg ? field[time].real() * field[time].real() : 0.5 * (field[time].real() * field[time].real() + field[time].imag() * field[time].imag()); } - for (std::size_t time = 0; time < problem.n_to; ++time) { + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { double convolved = 0.0; for (std::size_t previous = 0; previous <= time; ++previous) convolved += drive[previous] * kernel[time - previous]; - convolved *= problem.dt_fine; - const Complex multiplier = problem.family == native::FieldFamily::envelope + convolved *= problem.grid.dt_fine; + const Complex multiplier = problem.dims.family == native::FieldFamily::envelope ? field[time] : Complex(field[time].real(), 0.0); - projected[time] += projection * problem.raman_epsilon0 * multiplier * convolved * - problem.time_window[time]; + projected[time] += projection * problem.raman.epsilon0 * multiplier * convolved * + problem.grid.time_window[time]; } } const auto transformed = dft(projected); return {transformed.begin(), - transformed.begin() + static_cast(problem.n_omega)}; + transformed.begin() + static_cast(problem.dims.n_omega)}; } std::vector multimode_reference(const native::Problem& problem, const std::vector& spectrum, const bool coarse = false) { - std::vector> modal(problem.n_modes); - for (std::size_t mode = 0; mode < problem.n_modes; ++mode) { + std::vector> modal(problem.dims.n_modes); + for (std::size_t mode = 0; mode < problem.dims.n_modes; ++mode) { const auto first = - spectrum.begin() + static_cast(mode * problem.n_omega); + spectrum.begin() + static_cast(mode * problem.dims.n_omega); const std::vector one( - first, first + static_cast(problem.n_omega)); - modal[mode] = real_time(one, problem.n_to); + first, first + static_cast(problem.dims.n_omega)); + modal[mode] = real_time(one, problem.dims.n_to); } - std::vector> polarisation(problem.n_nodes, - std::vector(problem.n_to)); - for (std::size_t node = 0; node < problem.n_nodes; ++node) { - for (std::size_t time = 0; time < problem.n_to; ++time) { + std::vector> polarisation(problem.dims.n_nodes, + std::vector(problem.dims.n_to)); + for (std::size_t node = 0; node < problem.dims.n_nodes; ++node) { + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { double value = 0.0; - for (std::size_t mode = 0; mode < problem.n_modes; ++mode) + for (std::size_t mode = 0; mode < problem.dims.n_modes; ++mode) value += - problem.synthesis_matrix[node * problem.n_modes + mode] * modal[mode][time]; - polarisation[node][time] = problem.kerr_coefficient * value * value * value; + problem.transform.synthesis_matrix[node * problem.dims.n_modes + mode] * + modal[mode][time]; + polarisation[node][time] = problem.kerr.coefficient * value * value * value; } } - std::vector result(problem.n_modes * problem.n_omega); - const auto& projection = - coarse ? problem.projection_coarse_matrix : problem.projection_matrix; - for (std::size_t mode = 0; mode < problem.n_modes; ++mode) { - std::vector projected(problem.n_to); - for (std::size_t time = 0; time < problem.n_to; ++time) { - for (std::size_t node = 0; node < problem.n_nodes; ++node) + std::vector result(problem.dims.n_modes * problem.dims.n_omega); + const auto& projection = coarse ? problem.transform.projection_coarse_matrix + : problem.transform.projection_matrix; + for (std::size_t mode = 0; mode < problem.dims.n_modes; ++mode) { + std::vector projected(problem.dims.n_to); + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { + for (std::size_t node = 0; node < problem.dims.n_nodes; ++node) projected[time] += - projection[mode * problem.n_nodes + node] * polarisation[node][time]; + projection[mode * problem.dims.n_nodes + node] * polarisation[node][time]; } const auto transformed = dft(projected); - for (std::size_t index = 0; index < problem.n_omega; ++index) - result[mode * problem.n_omega + index] = transformed[index]; + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) + result[mode * problem.dims.n_omega + index] = transformed[index]; } return result; } @@ -313,38 +318,40 @@ std::vector nonlinear_reference(const native::Problem& problem, const std::vector& spectrum, const double taper) { std::vector time; - if (problem.family == native::FieldFamily::envelope) { - time = envelope_time(spectrum, problem.n_to); - } else if (problem.family == native::FieldFamily::field_real) { - const auto real = real_time(spectrum, problem.n_to); + if (problem.dims.family == native::FieldFamily::envelope) { + time = envelope_time(spectrum, problem.dims.n_to); + } else if (problem.dims.family == native::FieldFamily::field_real) { + const auto real = real_time(spectrum, problem.dims.n_to); for (const auto value : real) time.emplace_back(value, 0.0); } else { - std::vector analytic(problem.n_to); + std::vector analytic(problem.dims.n_to); analytic[0] = spectrum[0]; for (std::size_t index = 1; index < spectrum.size(); ++index) { analytic[index] = 2.0 * spectrum[index]; } - time = envelope_time(analytic, problem.n_to); + time = envelope_time(analytic, problem.dims.n_to); } - std::vector projected(problem.n_to); - for (std::size_t index = 0; index < problem.n_to; ++index) { - const Complex node = problem.synthesis * taper * time[index]; + std::vector projected(problem.dims.n_to); + for (std::size_t index = 0; index < problem.dims.n_to; ++index) { + const Complex node = problem.transform.synthesis * taper * time[index]; Complex polarisation; - if (problem.family == native::FieldFamily::field_real) { - polarisation = problem.kerr_coefficient * node.real() * node.real() * + if (problem.dims.family == native::FieldFamily::field_real) { + polarisation = problem.kerr.coefficient * node.real() * node.real() * Complex(node.real(), 0.0); } else { const double modulus = node.real() * node.real() + node.imag() * node.imag(); - polarisation = problem.kerr_coefficient * - (problem.self_factor + problem.conjugate_factor) * modulus * node; + polarisation = problem.kerr.coefficient * + (problem.kerr.self_factor + problem.kerr.conjugate_factor) * + modulus * node; } - polarisation *= problem.projection / taper; + polarisation *= problem.transform.projection / taper; projected[index] = - problem.take_real ? Complex(polarisation.real(), 0.0) : polarisation; + problem.kerr.take_real ? Complex(polarisation.real(), 0.0) : polarisation; } const auto full = dft(projected); - return {full.begin(), full.begin() + static_cast(problem.n_omega)}; + return {full.begin(), + full.begin() + static_cast(problem.dims.n_omega)}; } void expect_complex_near(const Complex actual, const Complex expected, @@ -404,19 +411,19 @@ TEST(MultiplyPhase, AppliesQuarterTurnAndPreservesMagnitude) { TEST(GeneralState, MasksAndAppliesCombinedWholeStepPropagator) { auto problem = general_problem(native::FieldFamily::envelope); - problem.n_phi_terms = 2; - problem.phi_terms.clear(); + problem.linear.n_phi_terms = 2; + problem.linear.phi_terms.clear(); for (std::size_t term = 0; term < 2; ++term) { - for (std::size_t index = 0; index < problem.n_omega; ++index) { - problem.phi_terms.emplace_back(0.0, static_cast(term + 1) * - static_cast(index + 1)); + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) { + problem.linear.phi_terms.emplace_back(0.0, static_cast(term + 1) * + static_cast(index + 1)); } } - problem.nonlinear = false; - problem.active_mask[2] = 0U; + problem.kerr.nonlinear = false; + problem.grid.active_mask[2] = 0U; auto scalars = trial(problem); scalars.phi_delta = {0.1, 0.2, 0.3, 0.4}; - const std::vector initial(problem.n_omega, {1.0, -0.5}); + const std::vector initial(problem.dims.n_omega, {1.0, -0.5}); native::GeneralState state(problem); state.set_initial(initial, {}); const auto partials = state.trial_step(0.25, scalars); @@ -439,12 +446,12 @@ TEST(GeneralState, KerrFamiliesMatchStraightforwardHostTransforms) { native::FieldFamily::envelope}) { SCOPED_TRACE(static_cast(family)); auto problem = general_problem(family); - problem.nonlinear = true; - std::vector initial(problem.n_omega); + problem.kerr.nonlinear = true; + std::vector initial(problem.dims.n_omega); initial[0] = Complex(1.0, 0.0); initial[1] = Complex(0.25, -0.125); initial[2] = Complex(-0.1, 0.05); - if (problem.n_omega == 4) + if (problem.dims.n_omega == 4) initial[3] = Complex(0.05, 0.025); constexpr double taper = 1.5; constexpr double dz = 0.02; @@ -459,7 +466,7 @@ TEST(GeneralState, KerrFamiliesMatchStraightforwardHostTransforms) { const Complex expected = initial[index] + dz * derivative[index]; magnitude = std::max(magnitude, Kokkos::abs(expected)); const double tolerance = - 512.0 * kEpsilon * static_cast(problem.n_to) * magnitude; + 512.0 * kEpsilon * static_cast(problem.dims.n_to) * magnitude; expect_complex_near(result[index], expected, tolerance); } EXPECT_GT(partials[2], 0.0); @@ -477,7 +484,7 @@ TEST(GeneralState, MultimodeTransformMatchesStraightforwardHostImplementation) { EXPECT_THROW(static_cast(state.stage_to_host(0)), std::logic_error); const auto partials = state.trial_step(dz, trial(problem)); const auto stage = state.stage_to_host(0); - EXPECT_THROW(static_cast(state.stage_to_host(problem.n_stages)), + EXPECT_THROW(static_cast(state.stage_to_host(problem.stepper.n_stages)), std::out_of_range); ASSERT_EQ(stage.size(), derivative.size()); for (std::size_t index = 0; index < stage.size(); ++index) { @@ -487,7 +494,7 @@ TEST(GeneralState, MultimodeTransformMatchesStraightforwardHostImplementation) { } state.finish(dz); const auto result = state.field_to_host(); - ASSERT_EQ(partials.size(), 3 * problem.n_modes); + ASSERT_EQ(partials.size(), 3 * problem.dims.n_modes); ASSERT_EQ(result.size(), initial.size()); for (std::size_t index = 0; index < result.size(); ++index) { const Complex expected = initial[index] + dz * derivative[index]; @@ -518,7 +525,7 @@ TEST(GeneralState, NodeChunkingPreservesTheMultimodeResult) { TEST(GeneralState, QuadratureErrorUsesFineAndCoarseProjectionOfTheSameResponse) { auto problem = multimode_problem(); - problem.first_same_as_last = false; + problem.stepper.first_same_as_last = false; const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, {0.5, 0.0}, {-0.2, 0.1}, {0.05, -0.025}}; const auto fine = multimode_reference(problem, initial); @@ -536,18 +543,19 @@ TEST(GeneralState, QuadratureErrorUsesFineAndCoarseProjectionOfTheSameResponse) static_cast(state.trial_step(0.01, trial(problem))); state.promote_fsal(); EXPECT_NEAR(state.quadrature_error(), expected, - 512.0 * static_cast(problem.n_to) * kEpsilon); + 512.0 * static_cast(problem.dims.n_to) * kEpsilon); } TEST(GeneralState, WeightedResidualDividesEachBinBeforeReducing) { auto problem = multimode_problem(); - problem.weighted_controller = true; + problem.stepper.weighted_controller = true; const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, {0.5, 0.0}, {-0.2, 0.1}, {0.05, -0.025}}; native::GeneralState state(problem); state.set_initial(initial, {}); const auto ordinary = state.trial_step(0.01, trial(problem)); - const auto weighted = state.weighted_error(std::vector(problem.n_omega, 2.0)); + const auto weighted = + state.weighted_error(std::vector(problem.dims.n_omega, 2.0)); ASSERT_EQ(weighted.size(), problem.n_columns()); for (std::size_t column = 0; column < problem.n_columns(); ++column) EXPECT_NEAR(weighted[column], ordinary[2 * problem.n_columns() + column] / 4.0, @@ -556,19 +564,19 @@ TEST(GeneralState, WeightedResidualDividesEachBinBeforeReducing) { TEST(GeneralState, PackedStatisticsMatchStraightforwardHostReductions) { auto problem = multimode_problem(); - problem.family = native::FieldFamily::envelope; - problem.n_t = 4; - problem.n_to = 4; - problem.n_omega = 4; - problem.crop_index = {0, 1, 2, 3}; - problem.active_mask.assign(4, 1U); - problem.norm_mask.assign(4, 1U); - problem.omega_window.assign(4, 1.0); - problem.omega = {1.0, 2.0, 3.0, 4.0}; - problem.time_window.assign(4, 1.0); - problem.carrier.assign(4, Complex(1.0, 0.0)); - problem.phi_terms.assign(8, Complex{}); - problem.prefactor_terms.assign(8, Complex(1.0, 0.0)); + problem.dims.family = native::FieldFamily::envelope; + problem.dims.n_t = 4; + problem.dims.n_to = 4; + problem.dims.n_omega = 4; + problem.grid.crop_index = {0, 1, 2, 3}; + problem.grid.active_mask.assign(4, 1U); + problem.grid.norm_mask.assign(4, 1U); + problem.grid.omega_window.assign(4, 1.0); + problem.grid.omega = {1.0, 2.0, 3.0, 4.0}; + problem.grid.time_window.assign(4, 1.0); + problem.grid.carrier.assign(4, Complex(1.0, 0.0)); + problem.linear.phi_terms.assign(8, Complex{}); + problem.linear.prefactor_terms.assign(8, Complex(1.0, 0.0)); const std::vector initial{{1.0, 0.0}, {0.5, 0.25}, {0.0, 0.0}, {0.0, 0.0}, {0.25, 0.0}, {0.0, 0.5}, {0.0, 0.0}, {0.0, 0.0}}; constexpr auto flags = @@ -588,10 +596,10 @@ TEST(GeneralState, PackedStatisticsMatchStraightforwardHostReductions) { const double one = std::norm(std::complex( initial[mode * 4 + index].real(), initial[mode * 4 + index].imag())); power += one; - weighted += problem.omega[index] * one; + weighted += problem.grid.omega[index] * one; } EXPECT_NEAR(values[mode], weighted / power, 32.0 * kEpsilon); - EXPECT_NEAR(values[2 + mode], problem.dt / 4.0 * power, 32.0 * kEpsilon); + EXPECT_NEAR(values[2 + mode], problem.grid.dt / 4.0 * power, 32.0 * kEpsilon); } std::array, 2> time{ envelope_time(std::vector(initial.begin(), initial.begin() + 4), 4), @@ -625,11 +633,11 @@ TEST(GeneralState, EveryFieldFamilyMatchesTheStraightforwardHostResponse) { native::FieldFamily::envelope}) { SCOPED_TRACE(static_cast(family)); auto problem = general_problem(family); - std::vector initial(problem.n_omega); + std::vector initial(problem.dims.n_omega); initial[0] = Complex(1.0, 0.0); initial[1] = Complex(0.25, -0.125); initial[2] = Complex(-0.1, 0.05); - if (problem.n_omega == 4) + if (problem.dims.n_omega == 4) initial[3] = Complex(0.05, 0.025); constexpr double taper = 1.5; constexpr double dz = 0.02; @@ -641,7 +649,8 @@ TEST(GeneralState, EveryFieldFamilyMatchesTheStraightforwardHostResponse) { const auto result = state.field_to_host(); for (std::size_t index = 0; index < result.size(); ++index) { const Complex expected = initial[index] + dz * derivative[index]; - const double tolerance = 512.0 * kEpsilon * static_cast(problem.n_to) * + const double tolerance = 512.0 * kEpsilon * + static_cast(problem.dims.n_to) * std::max(1.0, Kokkos::abs(expected)); expect_complex_near(result[index], expected, tolerance); } @@ -655,11 +664,11 @@ TEST(GeneralState, RamanMatchesDirectCausalConvolutionForEveryDriveForm) { native::FieldFamily::envelope}) { SCOPED_TRACE(static_cast(family)); auto problem = raman_problem(family); - std::vector initial(problem.n_omega); + std::vector initial(problem.dims.n_omega); initial[0] = Complex(1.0, 0.0); initial[1] = Complex(0.25, -0.125); initial[2] = Complex(-0.1, 0.05); - if (problem.n_omega == 4) + if (problem.dims.n_omega == 4) initial[3] = Complex(0.05, 0.025); const auto derivative = raman_reference(problem, initial, rates); auto scalars = trial(problem); @@ -673,7 +682,7 @@ TEST(GeneralState, RamanMatchesDirectCausalConvolutionForEveryDriveForm) { for (std::size_t index = 0; index < result.size(); ++index) { const Complex expected = initial[index] + dz * derivative[index]; const double tolerance = 512.0 * kEpsilon * - std::log2(2.0 * static_cast(problem.n_to)) * + std::log2(2.0 * static_cast(problem.dims.n_to)) * std::max(1.0, Kokkos::abs(expected)); expect_complex_near(result[index], expected, tolerance); } @@ -682,10 +691,10 @@ TEST(GeneralState, RamanMatchesDirectCausalConvolutionForEveryDriveForm) { TEST(GeneralState, RamanMatchesDirectConvolutionForAShortFinalNodeChunk) { auto problem = raman_problem(native::FieldFamily::field_real); - problem.n_nodes = 3; - problem.node_chunk = 2; - problem.synthesis_matrix = {1.0, 0.75, -0.5}; - problem.projection_matrix = {0.5, 0.25, -0.75}; + problem.dims.n_nodes = 3; + problem.dims.node_chunk = 2; + problem.transform.synthesis_matrix = {1.0, 0.75, -0.5}; + problem.transform.projection_matrix = {0.5, 0.25, -0.75}; const std::vector rates{0.125, 0.25}; const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}}; const auto derivative = raman_reference(problem, initial, rates); @@ -700,7 +709,7 @@ TEST(GeneralState, RamanMatchesDirectConvolutionForAShortFinalNodeChunk) { for (std::size_t index = 0; index < result.size(); ++index) { const Complex expected = initial[index] + dz * derivative[index]; const double tolerance = 512.0 * kEpsilon * - std::log2(2.0 * static_cast(problem.n_to)) * + std::log2(2.0 * static_cast(problem.dims.n_to)) * std::max(1.0, Kokkos::abs(expected)); expect_complex_near(result[index], expected, tolerance); } @@ -725,44 +734,44 @@ std::vector vector_raman_reference(const native::Problem& problem, const bool isotropic_only) { std::array, 2> modal; for (std::size_t pol = 0; pol < 2; ++pol) { - const auto offset = static_cast(pol * problem.n_omega); + const auto offset = static_cast(pol * problem.dims.n_omega); std::vector input(spectrum.begin() + offset, spectrum.begin() + offset + - static_cast(problem.n_omega)); - if (problem.family == native::FieldFamily::field_real) { - for (const double value : real_time(input, problem.n_to)) + static_cast(problem.dims.n_omega)); + if (problem.dims.family == native::FieldFamily::field_real) { + for (const double value : real_time(input, problem.dims.n_to)) modal[pol].emplace_back(value, 0.0); } else { - if (problem.family == native::FieldFamily::field_analytic) { + if (problem.dims.family == native::FieldFamily::field_analytic) { for (std::size_t j = 1; j < input.size(); ++j) input[j] *= 2.0; } - modal[pol] = envelope_time(input, problem.n_to); + modal[pol] = envelope_time(input, problem.dims.n_to); } } - std::array, 2> projected{std::vector(problem.n_to), - std::vector(problem.n_to)}; - for (std::size_t node = 0; node < problem.n_nodes; ++node) { - const double synthesis = problem.synthesis_matrix[node]; - for (std::size_t time = 0; time < problem.n_to; ++time) { + std::array, 2> projected{ + std::vector(problem.dims.n_to), std::vector(problem.dims.n_to)}; + for (std::size_t node = 0; node < problem.dims.n_nodes; ++node) { + const double synthesis = problem.transform.synthesis_matrix[node]; + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { std::array field{synthesis * modal[0][time], synthesis * modal[1][time]}; - if (problem.family != native::FieldFamily::envelope) + if (problem.dims.family != native::FieldFamily::envelope) for (auto& value : field) value = Complex(value.real(), 0.0); for (std::size_t previous = 0; previous <= time; ++previous) { const Complex x = synthesis * modal[0][previous]; const Complex y = synthesis * modal[1][previous]; - const double half = problem.raman_thg ? 1.0 : 0.5; + const double half = problem.raman.thg ? 1.0 : 0.5; const double xx = half * (x.real() * x.real() + x.imag() * x.imag()); const double yy = half * (y.real() * y.real() + y.imag() * y.imag()); const double xy = half * (x.real() * y.real() + x.imag() * y.imag()); const std::array, 2> q{{{xx, xy}, {xy, yy}}}; const auto lag = time - previous; - const double delay = static_cast(lag) * problem.dt_fine; - double hv = problem.raman_undamped[2 * problem.n_to + lag] * + const double delay = static_cast(lag) * problem.grid.dt_fine; + double hv = problem.raman.undamped[2 * problem.dims.n_to + lag] * std::exp(-rates[1] * delay); - double hr = problem.raman_undamped[lag] * std::exp(-rates[0] * delay); + double hr = problem.raman.undamped[lag] * std::exp(-rates[0] * delay); if (isotropic_only) { hv += hr; hr = 0.0; @@ -772,9 +781,10 @@ std::vector vector_raman_reference(const native::Problem& problem, for (std::size_t other = 0; other < 2; ++other) value += 0.5 * hr * (3.0 * q[pol][other] - (pol == other ? xx + yy : 0.0)) * field[other]; - projected[pol][time] += problem.projection_matrix[node] * - problem.raman_epsilon0 * problem.raman_fraction * - problem.dt_fine * problem.time_window[time] * value; + projected[pol][time] += problem.transform.projection_matrix[node] * + problem.raman.epsilon0 * problem.raman.fraction * + problem.grid.dt_fine * + problem.grid.time_window[time] * value; } } } @@ -784,7 +794,7 @@ std::vector vector_raman_reference(const native::Problem& problem, const auto transformed = dft(component); derivative.insert(derivative.end(), transformed.begin(), transformed.begin() + - static_cast(problem.n_omega)); + static_cast(problem.dims.n_omega)); } return derivative; } @@ -796,29 +806,30 @@ TEST(GeneralState, VectorRamanMatchesPhysicalCausalSumsAndRepeatsWithTailPlans) for (const std::size_t count : {8U, 12U, 16U}) { for (const bool isotropic_only : {false, true}) { auto problem = raman_problem(family); - const auto old_count = problem.n_to; - const auto old_rows = problem.raman_undamped; - problem.n_to = count; - problem.time_window.assign(count, 1.0); - problem.carrier.assign(count, Complex(1.0, 0.0)); - problem.raman_undamped.assign(problem.n_raman_terms * 2 * count, 0.0); - for (std::size_t term = 0; term < problem.n_raman_terms; ++term) + const auto old_count = problem.dims.n_to; + const auto old_rows = problem.raman.undamped; + problem.dims.n_to = count; + problem.grid.time_window.assign(count, 1.0); + problem.grid.carrier.assign(count, Complex(1.0, 0.0)); + problem.raman.undamped.assign(problem.raman.n_terms * 2 * count, 0.0); + for (std::size_t term = 0; term < problem.raman.n_terms; ++term) for (std::size_t sample = 0; sample < old_count; ++sample) - problem.raman_undamped[term * 2 * count + sample] = + problem.raman.undamped[term * 2 * count + sample] = old_rows[term * 2 * old_count + sample]; - problem.n_pol_state = 2; - problem.raman_tensor_capable = true; - problem.raman_tensor_weights = isotropic_only + problem.dims.n_pol_state = 2; + problem.raman.tensor_capable = true; + problem.raman.tensor_weights = isotropic_only ? std::vector{1.0, 0.0, 1.0, 0.0} : std::vector{0.25, 0.75, 1.0, 0.0}; - problem.n_nodes = 3; - problem.node_chunk = 2; - problem.synthesis_matrix = {1.0, 0.75, -0.5}; - problem.projection_matrix = {0.5, 0.25, -0.75}; - problem.phi_terms.assign(2 * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(2 * problem.n_omega, Complex(1.0, 0.0)); + problem.dims.n_nodes = 3; + problem.dims.node_chunk = 2; + problem.transform.synthesis_matrix = {1.0, 0.75, -0.5}; + problem.transform.projection_matrix = {0.5, 0.25, -0.75}; + problem.linear.phi_terms.assign(2 * problem.dims.n_omega, Complex{}); + problem.linear.prefactor_terms.assign(2 * problem.dims.n_omega, + Complex(1.0, 0.0)); const std::vector rates{0.125, 0.25}; - std::vector initial(2 * problem.n_omega); + std::vector initial(2 * problem.dims.n_omega); for (std::size_t j = 0; j < initial.size(); ++j) initial[j] = Complex(std::cos(static_cast(j)), 0.25 * std::sin(0.7 * static_cast(j))); @@ -829,8 +840,8 @@ TEST(GeneralState, VectorRamanMatchesPhysicalCausalSumsAndRepeatsWithTailPlans) // cancellation in an individual spectral sample; real and imaginary input // components <= 1. const double tolerance = - (3.0 * 512.0 * std::log2(2.0 * static_cast(problem.n_to)) + - 64.0 * static_cast(problem.n_to * problem.n_nodes)) * + (3.0 * 512.0 * std::log2(2.0 * static_cast(problem.dims.n_to)) + + 64.0 * static_cast(problem.dims.n_to * problem.dims.n_nodes)) * kEpsilon; ASSERT_GT(tolerance, 0.0); native::GeneralState state(problem); @@ -850,11 +861,11 @@ TEST(GeneralState, VectorRamanMatchesPhysicalCausalSumsAndRepeatsWithTailPlans) for (std::size_t j = 0; j < actual.size(); ++j) expect_complex_near(repeated[j], actual[j], tolerance); const auto ledger = state.memory_ledger(); - EXPECT_EQ(ledger.at("node_raman_drive"), problem.node_chunk * - problem.raman_drives() * 2 * - problem.n_to * sizeof(double)); + EXPECT_EQ(ledger.at("node_raman_drive"), + problem.dims.node_chunk * problem.raman_drives() * 2 * + problem.dims.n_to * sizeof(double)); EXPECT_EQ(ledger.at("raman_tensor_weights"), - problem.n_raman_terms * 2 * sizeof(double)); + problem.raman.n_terms * 2 * sizeof(double)); } } } @@ -865,7 +876,7 @@ TEST(GeneralState, RamanTensorWeightsAreValidatedBeforeAllocation) { {std::vector{1.0}, std::vector{1.0, 0.5, 1.0, 0.0}, std::vector{1.0, 0.0, std::numeric_limits::quiet_NaN(), 0.0}}) { auto problem = raman_problem(native::FieldFamily::envelope); - problem.raman_tensor_weights = weights; + problem.raman.tensor_weights = weights; EXPECT_THROW(static_cast(native::GeneralState(std::move(problem))), std::invalid_argument); } @@ -873,20 +884,20 @@ TEST(GeneralState, RamanTensorWeightsAreValidatedBeforeAllocation) { TEST(GeneralState, RamanValidationPrecedesFieldAllocation) { auto vector = raman_problem(native::FieldFamily::field_real); - vector.n_pol_state = 2; - vector.phi_terms.assign(2 * vector.n_omega, Complex{}); - vector.prefactor_terms.assign(2 * vector.n_omega, Complex(1.0, 0.0)); + vector.dims.n_pol_state = 2; + vector.linear.phi_terms.assign(2 * vector.dims.n_omega, Complex{}); + vector.linear.prefactor_terms.assign(2 * vector.dims.n_omega, Complex(1.0, 0.0)); EXPECT_THROW(static_cast(native::GeneralState(std::move(vector))), std::invalid_argument); auto wrong = raman_problem(native::FieldFamily::field_real); - wrong.raman_undamped.pop_back(); + wrong.raman.undamped.pop_back(); EXPECT_THROW(static_cast(native::GeneralState(std::move(wrong))), std::invalid_argument); auto valid = raman_problem(native::FieldFamily::field_real); native::GeneralState state(valid); - state.set_initial(std::vector(valid.n_omega), {}); + state.set_initial(std::vector(valid.dims.n_omega), {}); auto rates = trial(valid); rates.raman_rates = {-1.0, 0.0}; EXPECT_THROW(static_cast(state.trial_step(0.01, rates)), std::invalid_argument); @@ -894,26 +905,27 @@ TEST(GeneralState, RamanValidationPrecedesFieldAllocation) { TEST(GeneralState, TwoStatePolarisationsUseTheSharedVectorModulus) { auto problem = general_problem(native::FieldFamily::field_real); - problem.n_pol_state = 2; - problem.phi_terms.assign(2 * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(2 * problem.n_omega, Complex(1.0, 0.0)); + problem.dims.n_pol_state = 2; + problem.linear.phi_terms.assign(2 * problem.dims.n_omega, Complex{}); + problem.linear.prefactor_terms.assign(2 * problem.dims.n_omega, Complex(1.0, 0.0)); const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, {0.5, 0.0}, {-0.2, 0.1}, {0.05, -0.025}}; std::array, 2> time{ real_time(std::vector(initial.begin(), initial.begin() + 3), - problem.n_to), + problem.dims.n_to), real_time(std::vector(initial.begin() + 3, initial.end()), - problem.n_to)}; + problem.dims.n_to)}; std::array, 2> derivative; for (std::size_t pol = 0; pol < 2; ++pol) { - std::vector response(problem.n_to); - for (std::size_t sample = 0; sample < problem.n_to; ++sample) { - const double first = problem.synthesis * time[0][sample]; - const double second = problem.synthesis * time[1][sample]; + std::vector response(problem.dims.n_to); + for (std::size_t sample = 0; sample < problem.dims.n_to; ++sample) { + const double first = problem.transform.synthesis * time[0][sample]; + const double second = problem.transform.synthesis * time[1][sample]; const double modulus = first * first + second * second; - response[sample] = Complex(problem.projection * problem.kerr_coefficient * - modulus * (pol == 0 ? first : second), - 0.0); + response[sample] = + Complex(problem.transform.projection * problem.kerr.coefficient * modulus * + (pol == 0 ? first : second), + 0.0); } const auto transformed = dft(response); derivative[pol] = {transformed.begin(), transformed.begin() + 3}; @@ -925,50 +937,56 @@ TEST(GeneralState, TwoStatePolarisationsUseTheSharedVectorModulus) { state.finish(dz); const auto result = state.field_to_host(); for (std::size_t pol = 0; pol < 2; ++pol) { - for (std::size_t index = 0; index < problem.n_omega; ++index) { - const auto flat = pol * problem.n_omega + index; + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) { + const auto flat = pol * problem.dims.n_omega + index; const Complex expected = initial[flat] + dz * derivative[pol][index]; expect_complex_near(result[flat], expected, - 1024.0 * static_cast(problem.n_to) * kEpsilon); + 1024.0 * static_cast(problem.dims.n_to) * kEpsilon); } } } TEST(GeneralState, HeldCurrentMatchesTheDiscreteSinusoidalDrift) { auto problem = plasma_problem(); - problem.stash_plasma = false; + problem.plasma.stash = false; const double theta = - 2.0 * std::numbers::pi_v / static_cast(problem.n_to); - problem.omega = {0.0, theta / problem.dt_fine, 2.0 * theta / problem.dt_fine}; - problem.active_mask[0] = 0; - problem.field_min = 1e6; - problem.field_max = 1e7; - problem.log_field_min = std::log(problem.field_min); - problem.d_log_field = std::numbers::ln10; + 2.0 * std::numbers::pi_v / static_cast(problem.dims.n_to); + problem.grid.omega = {0.0, theta / problem.grid.dt_fine, + 2.0 * theta / problem.grid.dt_fine}; + problem.grid.active_mask[0] = 0; + problem.plasma.field_min = 1e6; + problem.plasma.field_max = 1e7; + problem.plasma.log_field_min = std::log(problem.plasma.field_min); + problem.plasma.d_log_field = std::numbers::ln10; constexpr double population = 0.25; // The unit mode at bin one produces E_n = 2*synthesis/N*cos(n theta). // The exact finite trapezoid sum of cos is dt*cot(theta/2)*sin(n theta)/2. - const double amplitude = 2.0 * problem.synthesis / static_cast(problem.n_to); - const double current_scale = population * problem.charge_mass_ratio * amplitude * - problem.dt_fine / (2.0 * std::tan(theta / 2.0)); - std::vector product(problem.n_to); - std::vector edge(problem.n_to); + const double amplitude = + 2.0 * problem.transform.synthesis / static_cast(problem.dims.n_to); + const double current_scale = population * problem.plasma.charge_mass_ratio * + amplitude * problem.grid.dt_fine / + (2.0 * std::tan(theta / 2.0)); + std::vector product(problem.dims.n_to); + std::vector edge(problem.dims.n_to); double operand = 0.0; - for (std::size_t time = 0; time < problem.n_to; ++time) { + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { const double angle = theta * static_cast(time); - problem.time_window[time] = 0.5 * (1.0 - std::cos(angle)); - problem.time_window_derivative[time] = - 0.5 * theta / problem.dt_fine * std::sin(angle); + problem.grid.time_window[time] = 0.5 * (1.0 - std::cos(angle)); + problem.grid.time_window_derivative[time] = + 0.5 * theta / problem.grid.dt_fine * std::sin(angle); const double current = current_scale * std::sin(angle); - product[time] = problem.projection * problem.time_window[time] * current; - edge[time] = problem.projection * problem.time_window_derivative[time] * current; - operand += Kokkos::abs(product[time]) / problem.omega[1] + - Kokkos::abs(edge[time]) / (problem.omega[1] * problem.omega[1]); + product[time] = + problem.transform.projection * problem.grid.time_window[time] * current; + edge[time] = problem.transform.projection * + problem.grid.time_window_derivative[time] * current; + operand += + Kokkos::abs(product[time]) / problem.grid.omega[1] + + Kokkos::abs(edge[time]) / (problem.grid.omega[1] * problem.grid.omega[1]); } // Independent trigonometric sums, N trapezoids and three transforms. The bound // uses both absolute operands before their cancellation, fixed before execution. const double tolerance = - 512.0 * static_cast(problem.n_to) * kEpsilon * operand; + 512.0 * static_cast(problem.dims.n_to) * kEpsilon * operand; ASSERT_GT(tolerance, 0.0); const auto x = dft(product); const auto z = dft(edge); @@ -979,13 +997,13 @@ TEST(GeneralState, HeldCurrentMatchesTheDiscreteSinusoidalDrift) { static_cast(state.trial_step(0.0, scalars)); const auto actual = state.stage_to_host(0); EXPECT_EQ(actual[0], Complex{}); - for (std::size_t bin = 1; bin < problem.n_omega; ++bin) { - const double omega = problem.omega[bin]; + for (std::size_t bin = 1; bin < problem.dims.n_omega; ++bin) { + const double omega = problem.grid.omega[bin]; const Complex expected = Complex(0.0, -1.0 / omega) * x[bin] + z[bin] / (omega * omega); expect_complex_near(actual[bin], expected, tolerance); } - state.set_initial(std::vector(problem.n_omega), {}); + state.set_initial(std::vector(problem.dims.n_omega), {}); static_cast(state.trial_step(0.0, scalars)); for (const auto value : state.stage_to_host(0)) EXPECT_EQ(value, Complex{}); @@ -993,15 +1011,15 @@ TEST(GeneralState, HeldCurrentMatchesTheDiscreteSinusoidalDrift) { TEST(GeneralState, HeldCurrentRefusesMissingDerivativeAndNonpositiveActiveFrequency) { auto problem = plasma_problem(); - problem.time_window_derivative.clear(); + problem.grid.time_window_derivative.clear(); EXPECT_THROW(native::GeneralState state(problem), std::invalid_argument); problem = plasma_problem(); - problem.time_window_derivative[0] = std::numeric_limits::infinity(); + problem.grid.time_window_derivative[0] = std::numeric_limits::infinity(); EXPECT_THROW(native::GeneralState state(problem), std::invalid_argument); problem = plasma_problem(); - problem.omega[0] = 0.0; + problem.grid.omega[0] = 0.0; EXPECT_THROW(native::GeneralState state(problem), std::invalid_argument); - problem.active_mask[0] = 0; + problem.grid.active_mask[0] = 0; EXPECT_NO_THROW(native::GeneralState state(problem)); } @@ -1013,8 +1031,8 @@ TEST(GeneralState, RejectedPlasmaTrialCannotReplaceTheAcceptedStash) { auto rejected = trial(problem); rejected.preionisation = {0.7}; constexpr double density = 4.0; - const double exposure = - problem.rate_max * problem.dt_fine * static_cast(problem.n_to - 1); + const double exposure = problem.plasma.rate_max * problem.grid.dt_fine * + static_cast(problem.dims.n_to - 1); const auto final_fraction = [exposure](const double preionisation) { return 1.0 - (1.0 - preionisation) * std::exp(-exposure); }; @@ -1029,7 +1047,7 @@ TEST(GeneralState, RejectedPlasmaTrialCannotReplaceTheAcceptedStash) { const auto retained = state.statistics(flags, {}, {}, density, 1.0, 0.7, 1.0); ASSERT_EQ(retained.size(), 2); EXPECT_NEAR(retained[0], density * final_fraction(0.1), 64.0 * kEpsilon); - EXPECT_NEAR(retained[1], problem.rate_max, 32.0 * kEpsilon); + EXPECT_NEAR(retained[1], problem.plasma.rate_max, 32.0 * kEpsilon); state.clear_fsal(); const auto rescanned = state.statistics(flags, {}, {}, density, 1.0, 0.7, 1.0); EXPECT_NEAR(rescanned[0], density * final_fraction(0.7), 64.0 * kEpsilon); @@ -1040,23 +1058,24 @@ TEST(GeneralState, RejectedPlasmaTrialCannotReplaceTheAcceptedStash) { TEST(GeneralState, IndependentChannelStashesSurviveRejectedTrials) { auto problem = plasma_problem(); - problem.n_channels = 2; + problem.plasma.n_channels = 2; const std::array rates{0.125, 0.25}; for (std::size_t channel = 0; channel < 2; ++channel) { - const std::array parameters{problem.log_field_min, - problem.d_log_field, - problem.field_min, - problem.field_max, + const std::array parameters{problem.plasma.log_field_min, + problem.plasma.d_log_field, + problem.plasma.field_min, + problem.plasma.field_max, rates[channel], 1.0, 4.0 * static_cast(channel), - problem.charge_mass_ratio, - problem.ionisation_potential}; - problem.channel_parameters.insert(problem.channel_parameters.end(), - parameters.begin(), parameters.end()); + problem.plasma.charge_mass_ratio, + problem.plasma.ionisation_potential}; + problem.plasma.channel_parameters.insert(problem.plasma.channel_parameters.end(), + parameters.begin(), parameters.end()); const std::array coefficients{0.0, 0.0, 0.0, std::log(rates[channel])}; - problem.channel_coefficients.insert(problem.channel_coefficients.end(), - coefficients.begin(), coefficients.end()); + problem.plasma.channel_coefficients.insert( + problem.plasma.channel_coefficients.end(), coefficients.begin(), + coefficients.end()); } auto accepted = trial(problem); accepted.preionisation = {0.0}; @@ -1068,8 +1087,8 @@ TEST(GeneralState, IndependentChannelStashesSurviveRejectedTrials) { // operations enter each fraction, including exp, before statistics are read. constexpr double tolerance = 64.0 * kEpsilon; const auto expected = [&](const double initial, const std::size_t channel) { - return 1.0 - (1.0 - initial) * std::exp(-rates[channel] * problem.dt_fine * - static_cast(problem.n_to - 1)); + return 1.0 - (1.0 - initial) * std::exp(-rates[channel] * problem.grid.dt_fine * + static_cast(problem.dims.n_to - 1)); }; native::GeneralState state(problem); state.set_initial(std::vector{{2.0, 0.0}, {0.0, 0.0}, {0.0, 0.0}}, {}); @@ -1096,65 +1115,66 @@ TEST(GeneralState, IndependentChannelStashesSurviveRejectedTrials) { TEST(GeneralState, MemoryLedgerMatchesConditionalAllocationFormulae) { auto problem = multimode_problem(); - problem.weighted_controller = true; - problem.plasma = true; - problem.stash_plasma = true; - problem.dt_fine = 0.125; - problem.charge_mass_ratio = 2.0; - problem.ionisation_potential = 3.0; - problem.log_field_min = std::log(0.01); - problem.d_log_field = std::log(1000.0); - problem.field_min = 0.01; - problem.field_max = 10.0; - problem.rate_max = 0.5; - problem.rate_intervals = 1; - problem.rate_coefficients = {0.0, 0.0, 0.0, std::log(0.5)}; + problem.stepper.weighted_controller = true; + problem.plasma.enabled = true; + problem.plasma.stash = true; + problem.grid.dt_fine = 0.125; + problem.plasma.charge_mass_ratio = 2.0; + problem.plasma.ionisation_potential = 3.0; + problem.plasma.log_field_min = std::log(0.01); + problem.plasma.d_log_field = std::log(1000.0); + problem.plasma.field_min = 0.01; + problem.plasma.field_max = 10.0; + problem.plasma.rate_max = 0.5; + problem.plasma.rate_intervals = 1; + problem.plasma.rate_coefficients = {0.0, 0.0, 0.0, std::log(0.5)}; native::GeneralState state(problem); const auto ledger = state.memory_ledger(); const std::size_t real = sizeof(double); const std::size_t complex = sizeof(Complex); - EXPECT_EQ(ledger.at("state"), problem.n_columns() * problem.n_omega * complex); - EXPECT_EQ(ledger.at("stages"), - problem.n_stages * problem.n_columns() * problem.n_omega * complex); + EXPECT_EQ(ledger.at("state"), problem.n_columns() * problem.dims.n_omega * complex); + EXPECT_EQ(ledger.at("stages"), problem.stepper.n_stages * problem.n_columns() * + problem.dims.n_omega * complex); EXPECT_EQ(ledger.at("transform_matrices"), - (problem.synthesis_matrix.size() + problem.projection_matrix.size() + - problem.projection_coarse_matrix.size()) * + (problem.transform.synthesis_matrix.size() + + problem.transform.projection_matrix.size() + + problem.transform.projection_coarse_matrix.size()) * real); EXPECT_EQ(ledger.at("stacked_projection"), - 2 * problem.n_columns() * problem.n_to * real); + 2 * problem.n_columns() * problem.dims.n_to * real); EXPECT_EQ(ledger.at("scaled_transform_matrices"), - 5 * problem.n_modes * problem.n_nodes * real); + 5 * problem.dims.n_modes * problem.dims.n_nodes * real); EXPECT_EQ(ledger.at("plasma_stash"), - 2 * problem.n_nodes * (1 + 2 * problem.n_channels) * real); + 2 * problem.dims.n_nodes * (1 + 2 * problem.plasma.n_channels) * real); // The device column's rows; on the host the band statistics' three rows per team // slot, at most one slot per thread (a trapezoid problem keeps no band rows). #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - const auto chunk_rows = problem.node_chunk * problem.effective_node_pol(); + const auto chunk_rows = problem.dims.node_chunk * problem.effective_node_pol(); // With the trapezoid's binary64 tile totals, a row's tiles of 1,024 samples. - const auto tiles = (problem.n_to + 1023) / 1024; + const auto tiles = (problem.dims.n_to + 1023) / 1024; EXPECT_EQ(ledger.at("plasma_device_scan"), - (3 * problem.node_chunk * problem.n_to + 3 * chunk_rows * problem.n_to + - chunk_rows * tiles) * + (3 * problem.dims.node_chunk * problem.dims.n_to + + 3 * chunk_rows * problem.dims.n_to + chunk_rows * tiles) * real); #else // One slot per thread of the response operation's own width (problem-size policy). const auto response_elements = - problem.node_chunk * problem.effective_node_pol() * problem.n_to; + problem.dims.node_chunk * problem.effective_node_pol() * problem.dims.n_to; const int width = native::cpu_size_policy_enabled ? native::operation_threads( native::thread_policy(), native::test::table(), native::CpuPool::kernels, native::CpuOperation::response, - response_elements, problem.n_to) + response_elements, problem.dims.n_to) : native::ExecutionSpace().concurrency(); const auto slots = - std::min(problem.node_chunk, static_cast(std::max(1, width))); - EXPECT_EQ(ledger.at("plasma_device_scan"), 3 * slots * problem.n_to * real); + std::min(problem.dims.node_chunk, static_cast(std::max(1, width))); + EXPECT_EQ(ledger.at("plasma_device_scan"), 3 * slots * problem.dims.n_to * real); #endif EXPECT_EQ(ledger.at("weighted_control"), - (problem.n_omega + problem.n_columns()) * real); + (problem.dims.n_omega + problem.n_columns()) * real); EXPECT_EQ(ledger.at("error_partials"), 3 * problem.n_columns() * real); EXPECT_EQ(ledger.at("statistics_spectrum"), - problem.n_columns() * problem.n_t * complex); + problem.n_columns() * problem.dims.n_t * complex); } TEST(GeneralState, RamanMemoryLedgerMatchesTheConditionalViewFormulae) { @@ -1164,18 +1184,18 @@ TEST(GeneralState, RamanMemoryLedgerMatchesTheConditionalViewFormulae) { const auto real = sizeof(double); const auto complex = sizeof(Complex); EXPECT_EQ(ledger.at("node_raman_drive"), - problem.node_chunk * 2 * problem.n_to * real); + problem.dims.node_chunk * 2 * problem.dims.n_to * real); EXPECT_EQ(ledger.at("node_raman_spectrum"), - problem.node_chunk * (problem.n_to + 1) * complex); + problem.dims.node_chunk * (problem.dims.n_to + 1) * complex); EXPECT_EQ(ledger.at("node_raman_convolved"), - problem.node_chunk * 2 * problem.n_to * real); + problem.dims.node_chunk * 2 * problem.dims.n_to * real); EXPECT_EQ(ledger.at("raman_undamped"), - problem.n_raman_terms * 2 * problem.n_to * real); - EXPECT_EQ(ledger.at("raman_kernel_time"), 2 * problem.n_to * real); - EXPECT_EQ(ledger.at("raman_kernel_spectrum"), (problem.n_to + 1) * complex); + problem.raman.n_terms * 2 * problem.dims.n_to * real); + EXPECT_EQ(ledger.at("raman_kernel_time"), 2 * problem.dims.n_to * real); + EXPECT_EQ(ledger.at("raman_kernel_spectrum"), (problem.dims.n_to + 1) * complex); // Two slots of the packed stage scalars, the trial's step and its sample. EXPECT_EQ(ledger.at("stage_raman_rates"), - 2 * problem.n_stages * problem.n_raman_terms * real); + 2 * problem.stepper.n_stages * problem.raman.n_terms * real); native::GeneralState ordinary(multimode_problem()); const auto absent = ordinary.memory_ledger(); @@ -1422,35 +1442,39 @@ struct SynthesisCapture { // with the whole transform. void check_radial_analytic_chunks(const bool absorber, const char* expected_region) { auto problem = general_problem(native::FieldFamily::field_analytic); - problem.n_modes = 5; - problem.n_nodes = 5; - problem.node_chunk = 3; - problem.spatial_spectral_weights.assign(problem.n_modes, 1.0); - problem.spatial_on_axis.assign(problem.n_modes, 1.0); - problem.phi_terms.assign(problem.n_modes * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(problem.n_modes * problem.n_omega, Complex(1.0, 0.0)); - problem.in_stage_absorber = absorber; + problem.dims.n_modes = 5; + problem.dims.n_nodes = 5; + problem.dims.node_chunk = 3; + problem.spatial.spectral_weights.assign(problem.dims.n_modes, 1.0); + problem.spatial.on_axis.assign(problem.dims.n_modes, 1.0); + problem.linear.phi_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex{}); + problem.linear.prefactor_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex(1.0, 0.0)); + problem.absorber.in_stage = absorber; if (absorber) { - problem.absorber_fine.resize(problem.n_to); - for (std::size_t time = 0; time < problem.n_to; ++time) - problem.absorber_fine[time] = 0.01 * static_cast(time % 3); - } - problem.synthesis_matrix.resize(problem.n_nodes * problem.n_modes); - problem.projection_matrix.resize(problem.n_modes * problem.n_nodes); - for (std::size_t row = 0; row < problem.n_nodes; ++row) { - for (std::size_t column = 0; column < problem.n_modes; ++column) { - const auto index = row * problem.n_modes + column; - problem.synthesis_matrix[index] = + problem.absorber.fine.resize(problem.dims.n_to); + for (std::size_t time = 0; time < problem.dims.n_to; ++time) + problem.absorber.fine[time] = 0.01 * static_cast(time % 3); + } + problem.transform.synthesis_matrix.resize(problem.dims.n_nodes * + problem.dims.n_modes); + problem.transform.projection_matrix.resize(problem.dims.n_modes * + problem.dims.n_nodes); + for (std::size_t row = 0; row < problem.dims.n_nodes; ++row) { + for (std::size_t column = 0; column < problem.dims.n_modes; ++column) { + const auto index = row * problem.dims.n_modes + column; + problem.transform.synthesis_matrix[index] = row == column ? 1.0 : 0.125 * static_cast(row + 1); - problem.projection_matrix[index] = + problem.transform.projection_matrix[index] = row == column ? 0.75 : -0.0625 * static_cast(column + 1); } } - std::vector initial(problem.n_modes * problem.n_omega); + std::vector initial(problem.dims.n_modes * problem.dims.n_omega); for (std::size_t index = 0; index < initial.size(); ++index) initial[index] = Complex(0.01 * static_cast(index + 1), -0.02); auto whole_problem = problem; - whole_problem.node_chunk = whole_problem.n_nodes; + whole_problem.dims.node_chunk = whole_problem.dims.n_nodes; native::GeneralState whole(whole_problem); whole.set_initial(initial, {}); static_cast(whole.trial_step(0.01, trial(whole_problem))); @@ -1476,9 +1500,9 @@ void check_radial_analytic_chunks(const bool absorber, const char* expected_regi // Both computations have one inverse/forward FFT and two length-five products; // carry their rounding through the cubic and the stage's scaled increment. const double tolerance = - 512.0 * kEpsilon * static_cast(problem.n_to) + - 4.0 * (static_cast(problem.n_modes) * kEpsilon / - (1.0 - static_cast(problem.n_modes) * kEpsilon)); + 512.0 * kEpsilon * static_cast(problem.dims.n_to) + + 4.0 * (static_cast(problem.dims.n_modes) * kEpsilon / + (1.0 - static_cast(problem.dims.n_modes) * kEpsilon)); ASSERT_GT(tolerance, 0.0); for (std::size_t index = 0; index < actual.size(); ++index) expect_complex_near(actual[index], expected[index], @@ -1498,21 +1522,24 @@ TEST(GeneralState, SquareRadialAnalyticChunksSynthesiseRetainedSpectra) { TEST(GeneralState, ChunkedBlasBuffersHaveExactPersistentByteCost) { auto problem = multimode_problem(3); - problem.n_to = 1024; - problem.n_nodes = 8; - problem.time_window.assign(problem.n_to, 1.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); - problem.synthesis_matrix.assign(problem.n_nodes * problem.n_modes, 0.5); - problem.projection_matrix.assign(problem.n_modes * problem.n_nodes, 0.25); - problem.projection_coarse_matrix.assign(problem.n_modes * problem.n_nodes, 0.125); + problem.dims.n_to = 1024; + problem.dims.n_nodes = 8; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); + problem.transform.synthesis_matrix.assign(problem.dims.n_nodes * problem.dims.n_modes, + 0.5); + problem.transform.projection_matrix.assign( + problem.dims.n_modes * problem.dims.n_nodes, 0.25); + problem.transform.projection_coarse_matrix.assign( + problem.dims.n_modes * problem.dims.n_nodes, 0.125); native::GeneralState state(problem); const auto ledger = state.memory_ledger(); EXPECT_EQ(ledger.at("chunk_transform_buffers"), - (3U + 2U) * problem.n_modes * 3U * sizeof(double)); + (3U + 2U) * problem.dims.n_modes * 3U * sizeof(double)); auto whole_problem = problem; - whole_problem.node_chunk = whole_problem.n_nodes; + whole_problem.dims.node_chunk = whole_problem.dims.n_nodes; const std::vector initial{{1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, {0.5, 0.0}, {-0.2, 0.1}, {0.05, -0.025}}; native::GeneralState whole(whole_problem); @@ -1524,10 +1551,10 @@ TEST(GeneralState, ChunkedBlasBuffersHaveExactPersistentByteCost) { state.finish(0.01); const auto reference = whole.field_to_host(); const auto chunked = state.field_to_host(); - const auto dot_length = static_cast(problem.n_nodes); + const auto dot_length = static_cast(problem.dims.n_nodes); const double dot_gamma = dot_length * kEpsilon / (1.0 - dot_length * kEpsilon); const double tolerance = - 512.0 * kEpsilon * std::ceil(std::log2(static_cast(problem.n_to))) + + 512.0 * kEpsilon * std::ceil(std::log2(static_cast(problem.dims.n_to))) + 4.0 * dot_gamma; for (std::size_t index = 0; index < chunked.size(); ++index) expect_complex_near(chunked[index], reference[index], @@ -1537,56 +1564,62 @@ TEST(GeneralState, ChunkedBlasBuffersHaveExactPersistentByteCost) { #ifdef SOLPROP_NATIVE_TPL_BLAS_OPENBLAS TEST(GeneralState, OpenBlasEnvelopePolicyAllocatesAndExecutesComplexTransforms) { auto problem = multimode_problem(8); - problem.family = native::FieldFamily::envelope; - problem.n_t = 1024; - problem.n_to = 1024; - problem.n_omega = 1024; - problem.n_nodes = 8; - problem.crop_index.clear(); - for (std::size_t index = 0; index < problem.n_omega; ++index) - problem.crop_index.push_back(index); - problem.active_mask.assign(problem.n_omega, 1U); - problem.norm_mask.assign(problem.n_omega, 1U); - problem.omega_window.assign(problem.n_omega, 1.0); - problem.omega.assign(problem.n_omega, 1.0); - problem.time_window.assign(problem.n_to, 1.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); - problem.phi_terms.assign(problem.n_modes * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(problem.n_modes * problem.n_omega, Complex(1.0, 0.0)); - problem.synthesis_matrix.assign(problem.n_nodes * problem.n_modes, 0.5); - problem.projection_matrix.assign(problem.n_modes * problem.n_nodes, 0.25); - problem.projection_coarse_matrix.assign(problem.n_modes * problem.n_nodes, 0.125); + problem.dims.family = native::FieldFamily::envelope; + problem.dims.n_t = 1024; + problem.dims.n_to = 1024; + problem.dims.n_omega = 1024; + problem.dims.n_nodes = 8; + problem.grid.crop_index.clear(); + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) + problem.grid.crop_index.push_back(index); + problem.grid.active_mask.assign(problem.dims.n_omega, 1U); + problem.grid.norm_mask.assign(problem.dims.n_omega, 1U); + problem.grid.omega_window.assign(problem.dims.n_omega, 1.0); + problem.grid.omega.assign(problem.dims.n_omega, 1.0); + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); + problem.linear.phi_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex{}); + problem.linear.prefactor_terms.assign(problem.dims.n_modes * problem.dims.n_omega, + Complex(1.0, 0.0)); + problem.transform.synthesis_matrix.assign(problem.dims.n_nodes * problem.dims.n_modes, + 0.5); + problem.transform.projection_matrix.assign( + problem.dims.n_modes * problem.dims.n_nodes, 0.25); + problem.transform.projection_coarse_matrix.assign( + problem.dims.n_modes * problem.dims.n_nodes, 0.125); native::GeneralState state(problem); // Scalar envelopes use a real GEMM over the complex samples' interleaved view. EXPECT_EQ(state.synthesis_provider(), "kokkosblas-tpl-interleaved-real"); EXPECT_EQ(state.projection_provider(), "kokkosblas-tpl-interleaved-real"); const auto ledger = state.memory_ledger(); - const auto real_matrix_values = 3 * problem.n_modes * problem.n_nodes; - const auto complex_matrix_values = 5 * problem.n_modes * problem.n_nodes; + const auto real_matrix_values = 3 * problem.dims.n_modes * problem.dims.n_nodes; + const auto complex_matrix_values = 5 * problem.dims.n_modes * problem.dims.n_nodes; EXPECT_EQ(ledger.at("scaled_transform_matrices"), real_matrix_values * sizeof(double) + complex_matrix_values * sizeof(Complex)); EXPECT_EQ(ledger.at("stacked_projection"), - 2 * problem.n_modes * problem.n_to * sizeof(Complex)); + 2 * problem.dims.n_modes * problem.dims.n_to * sizeof(Complex)); - state.set_initial(std::vector(problem.n_modes * problem.n_omega), {}); + state.set_initial(std::vector(problem.dims.n_modes * problem.dims.n_omega), + {}); const auto partials = state.trial_step(0.01, trial(problem)); // Exact: a zero field has zero norms and a zero error. - EXPECT_EQ(partials, std::vector(3 * problem.n_modes, 0.0)); + EXPECT_EQ(partials, std::vector(3 * problem.dims.n_modes, 0.0)); } #endif TEST(GeneralState, RejectsOverflowingExtentsBeforeAllocation) { auto column_overflow = general_problem(native::FieldFamily::field_real); - column_overflow.n_modes = std::numeric_limits::max(); - column_overflow.n_pol_state = 2; + column_overflow.dims.n_modes = std::numeric_limits::max(); + column_overflow.dims.n_pol_state = 2; EXPECT_THROW(static_cast(native::GeneralState(std::move(column_overflow))), std::length_error); auto field_overflow = general_problem(native::FieldFamily::field_real); - field_overflow.n_modes = - std::numeric_limits::max() / field_overflow.n_omega + 1; + field_overflow.dims.n_modes = + std::numeric_limits::max() / field_overflow.dims.n_omega + 1; EXPECT_THROW(static_cast(native::GeneralState(std::move(field_overflow))), std::length_error); } @@ -1596,35 +1629,35 @@ TEST(GeneralState, RejectsOverflowingExtentsBeforeAllocation) { // (x, y) are ix + 3 iy, and each column's operator subtracts its channel's wall term. native::Problem planar_problem(const std::size_t rows_per_chunk) { auto problem = base_problem(native::FieldFamily::field_real); - problem.n_p = 2; - problem.n_x = 3; - problem.n_y = 4; - problem.n_modes = problem.n_p * problem.n_y; - problem.n_nodes = problem.n_x * problem.n_y; - problem.node_chunk = rows_per_chunk * problem.n_x; - problem.nonlinear = true; - problem.phi_terms.clear(); - problem.prefactor_terms.clear(); - problem.synthesis_matrix = {1.0, 0.5, -0.25, 1.5, 0.75, -0.5}; - problem.projection_matrix = {0.5, 0.25, -0.125, 0.375, 0.625, -0.25}; - problem.spatial_k_squared = {{40.0, 0.5}, {60.0, 1.0}, {90.0, 1.5}}; - problem.spatial_k_squared_low = {{1e-15, 0.0}, {-2e-15, 0.0}, {0.0, 1e-16}}; - problem.spatial_numerator = {{0.0, -1.0}, {0.0, -2.0}, {0.0, -3.0}}; - problem.spatial_clamped = {{0.0, -0.25}, {0.0, -0.5}, {0.0, -0.75}}; - problem.spatial_frame = {5.0, 6.0, 7.0}; - problem.spatial_spectral_rate = {0.0, 0.25, 0.5}; - problem.spatial_user_rate = {0.0, 0.0, 0.0}; + problem.spatial.n_p = 2; + problem.spatial.n_x = 3; + problem.spatial.n_y = 4; + problem.dims.n_modes = problem.spatial.n_p * problem.spatial.n_y; + problem.dims.n_nodes = problem.spatial.n_x * problem.spatial.n_y; + problem.dims.node_chunk = rows_per_chunk * problem.spatial.n_x; + problem.kerr.nonlinear = true; + problem.linear.phi_terms.clear(); + problem.linear.prefactor_terms.clear(); + problem.transform.synthesis_matrix = {1.0, 0.5, -0.25, 1.5, 0.75, -0.5}; + problem.transform.projection_matrix = {0.5, 0.25, -0.125, 0.375, 0.625, -0.25}; + problem.spatial.k_squared = {{40.0, 0.5}, {60.0, 1.0}, {90.0, 1.5}}; + problem.spatial.k_squared_low = {{1e-15, 0.0}, {-2e-15, 0.0}, {0.0, 1e-16}}; + problem.spatial.numerator = {{0.0, -1.0}, {0.0, -2.0}, {0.0, -3.0}}; + problem.spatial.clamped = {{0.0, -0.25}, {0.0, -0.5}, {0.0, -0.75}}; + problem.spatial.frame = {5.0, 6.0, 7.0}; + problem.spatial.spectral_rate = {0.0, 0.25, 0.5}; + problem.spatial.user_rate = {0.0, 0.0, 0.0}; // k_y^2 per column, the channel fastest; the last row is evanescent in the first // bin, so both branches of the operator are exercised. const std::vector ky_squared{0.0, 4.0, 45.0, 4.0}; - for (std::size_t column = 0; column < problem.n_modes; ++column) { - problem.spatial_kperp_sq.push_back(ky_squared[column / problem.n_p]); - problem.spatial_kperp_sq_low.push_back(column == 3 ? 1e-16 : 0.0); - } - problem.spatial_absorber_rate.assign(problem.n_modes, 0.0); - problem.spatial_kperp_rate.assign(problem.n_modes, 0.0); - problem.spatial_evanescent_rate = 200.0; - problem.spatial_kappa_sq = {{1.0, 0.25}, {2.0, 0.5}, {3.0, 0.75}, + for (std::size_t column = 0; column < problem.dims.n_modes; ++column) { + problem.spatial.kperp_sq.push_back(ky_squared[column / problem.spatial.n_p]); + problem.spatial.kperp_sq_low.push_back(column == 3 ? 1e-16 : 0.0); + } + problem.spatial.absorber_rate.assign(problem.dims.n_modes, 0.0); + problem.spatial.kperp_rate.assign(problem.dims.n_modes, 0.0); + problem.spatial.evanescent_rate = 200.0; + problem.spatial.kappa_sq = {{1.0, 0.25}, {2.0, 0.5}, {3.0, 0.75}, {9.0, 1.0}, {12.0, 1.5}, {15.0, 2.0}}; return problem; } @@ -1636,20 +1669,21 @@ struct PlanarElement { PlanarElement planar_element(const native::Problem& problem, const std::size_t column, const std::size_t omega) { const Complex square = - (problem.spatial_k_squared[omega] - problem.spatial_kperp_sq[column]) + - (problem.spatial_k_squared_low[omega] - problem.spatial_kperp_sq_low[column]) - - problem.spatial_kappa_sq[(column % problem.n_p) * problem.n_omega + omega]; + (problem.spatial.k_squared[omega] - problem.spatial.kperp_sq[column]) + + (problem.spatial.k_squared_low[omega] - problem.spatial.kperp_sq_low[column]) - + problem.spatial + .kappa_sq[(column % problem.spatial.n_p) * problem.dims.n_omega + omega]; const std::complex root = std::sqrt(std::complex(square.real(), square.imag())); const bool propagating = square.real() > 0.0; const double attenuation = propagating ? root.imag() - : std::min(std::abs(root.imag()), problem.spatial_evanescent_rate); + : std::min(std::abs(root.imag()), problem.spatial.evanescent_rate); return {.linear = - Complex(-attenuation, -(root.real() - problem.spatial_frame[omega])) - - 0.5 * problem.spatial_spectral_rate[omega], - .prefactor = propagating ? problem.spatial_numerator[omega] / root.real() - : problem.spatial_clamped[omega]}; + Complex(-attenuation, -(root.real() - problem.spatial.frame[omega])) - + 0.5 * problem.spatial.spectral_rate[omega], + .prefactor = propagating ? problem.spatial.numerator[omega] / root.real() + : problem.spatial.clamped[omega]}; } // The stage straight from the definitions: the inverse DFT in y, each row's real time @@ -1657,10 +1691,10 @@ PlanarElement planar_element(const native::Problem& problem, const std::size_t c // and each column's prefactor. std::vector planar_reference(const native::Problem& problem, const std::vector& columns) { - const auto n_p = problem.n_p; - const auto n_x = problem.n_x; - const auto n_y = problem.n_y; - const auto n = problem.n_omega; + const auto n_p = problem.spatial.n_p; + const auto n_x = problem.spatial.n_x; + const auto n_y = problem.spatial.n_y; + const auto n = problem.dims.n_omega; std::vector> rows(n_p * n_y); for (std::size_t iy = 0; iy < n_y; ++iy) for (std::size_t p = 0; p < n_p; ++p) { @@ -1672,25 +1706,26 @@ std::vector planar_reference(const native::Problem& problem, cis(2.0 * std::numbers::pi_v * static_cast(ky * iy) / static_cast(n_y)) / static_cast(n_y); - rows[p + n_p * iy] = real_time(spectrum, problem.n_to); + rows[p + n_p * iy] = real_time(spectrum, problem.dims.n_to); } std::vector projected(n_p * n_y * n); for (std::size_t iy = 0; iy < n_y; ++iy) { - std::vector> polarisation(n_x, - std::vector(problem.n_to)); + std::vector> polarisation( + n_x, std::vector(problem.dims.n_to)); for (std::size_t ix = 0; ix < n_x; ++ix) - for (std::size_t time = 0; time < problem.n_to; ++time) { + for (std::size_t time = 0; time < problem.dims.n_to; ++time) { double field = 0.0; for (std::size_t p = 0; p < n_p; ++p) - field += problem.synthesis_matrix[ix * n_p + p] * rows[p + n_p * iy][time]; - polarisation[ix][time] = problem.kerr_coefficient * field * field * field; + field += problem.transform.synthesis_matrix[ix * n_p + p] * + rows[p + n_p * iy][time]; + polarisation[ix][time] = problem.kerr.coefficient * field * field * field; } for (std::size_t p = 0; p < n_p; ++p) { - std::vector signal(problem.n_to); - for (std::size_t time = 0; time < problem.n_to; ++time) + std::vector signal(problem.dims.n_to); + for (std::size_t time = 0; time < problem.dims.n_to; ++time) for (std::size_t ix = 0; ix < n_x; ++ix) - signal[time] += - problem.projection_matrix[p * n_x + ix] * polarisation[ix][time]; + signal[time] += problem.transform.projection_matrix[p * n_x + ix] * + polarisation[ix][time]; const auto spectrum = dft(signal); for (std::size_t omega = 0; omega < n; ++omega) projected[(p + n_p * iy) * n + omega] = spectrum[omega]; @@ -1707,7 +1742,7 @@ std::vector planar_reference(const native::Problem& problem, static_cast(n_y)); const auto column = p + n_p * ky; result[column * n + omega] = planar_element(problem, column, omega).prefactor * - problem.omega_window[omega] * sum; + problem.grid.omega_window[omega] * sum; } return result; } @@ -1722,7 +1757,7 @@ TEST(GeneralState, HybridPlanarStageMatchesStraightforwardHostImplementation) { // One y row per chunk (a partial chunk nowhere), then three (a partial last one). for (const std::size_t rows_per_chunk : {std::size_t{1}, std::size_t{3}}) { const auto problem = planar_problem(rows_per_chunk); - ASSERT_EQ(initial.size(), problem.n_modes * problem.n_omega); + ASSERT_EQ(initial.size(), problem.dims.n_modes * problem.dims.n_omega); const auto derivative = planar_reference(problem, initial); constexpr double dz = 0.01; auto scalars = trial(problem); @@ -1740,9 +1775,9 @@ TEST(GeneralState, HybridPlanarStageMatchesStraightforwardHostImplementation) { // weighted update (one stage, unit weight), which carries the wall terms. state.finish(dz); const auto result = state.field_to_host(); - for (std::size_t column = 0; column < problem.n_modes; ++column) - for (std::size_t omega = 0; omega < problem.n_omega; ++omega) { - const auto index = column * problem.n_omega + omega; + for (std::size_t column = 0; column < problem.dims.n_modes; ++column) + for (std::size_t omega = 0; omega < problem.dims.n_omega; ++omega) { + const auto index = column * problem.dims.n_omega + omega; const Complex exponent = dz * planar_element(problem, column, omega).linear; const Complex expected = std::exp(exponent.real()) * cis(exponent.imag()) * (initial[index] + dz * derivative[index]); @@ -1754,22 +1789,22 @@ TEST(GeneralState, HybridPlanarStageMatchesStraightforwardHostImplementation) { TEST(GeneralState, HybridPlanarProblemsAreValidatedBeforeAllocation) { auto partial_row = planar_problem(1); - partial_row.node_chunk = 4; + partial_row.dims.node_chunk = 4; EXPECT_THROW(native::GeneralState state(partial_row), std::invalid_argument); auto short_kappa = planar_problem(1); - short_kappa.spatial_kappa_sq.pop_back(); + short_kappa.spatial.kappa_sq.pop_back(); EXPECT_THROW(native::GeneralState state(short_kappa), std::invalid_argument); auto coarse = planar_problem(1); - coarse.projection_coarse_matrix = coarse.projection_matrix; + coarse.transform.projection_coarse_matrix = coarse.transform.projection_matrix; EXPECT_THROW(native::GeneralState state(coarse), std::invalid_argument); auto vector = planar_problem(1); - vector.n_pol_state = 2; + vector.dims.n_pol_state = 2; EXPECT_THROW(native::GeneralState state(vector), std::invalid_argument); auto columns = planar_problem(1); - columns.n_p = 3; + columns.spatial.n_p = 3; EXPECT_THROW(native::GeneralState state(columns), std::invalid_argument); auto modal = general_problem(native::FieldFamily::field_real); - modal.spatial_kappa_sq = {{1.0, 0.0}}; + modal.spatial.kappa_sq = {{1.0, 0.0}}; EXPECT_THROW(native::GeneralState state(modal), std::invalid_argument); } @@ -2436,7 +2471,7 @@ TEST(PlasmaBandScan, FilteringPreservesShortRowsBlockTailsAndDynamicRange) { TEST(GeneralState, RepeatedTrialsOverwriteFullCropAndStageZeroAccumulators) { auto problem = general_problem(native::FieldFamily::envelope); - problem.nonlinear = true; + problem.kerr.nonlinear = true; const std::vector initial = { {1.0, 0.0}, {0.25, -0.125}, {-0.1, 0.05}, {0.05, 0.025}}; native::GeneralState state(problem); @@ -2453,7 +2488,7 @@ TEST(GeneralState, RepeatedTrialsOverwriteFullCropAndStageZeroAccumulators) { largest = std::max(largest, std::abs(value)); ASSERT_GT(largest, 0.0); const double tolerance = - 2.0 * 512.0 * kEpsilon * static_cast(problem.n_to) * largest; + 2.0 * 512.0 * kEpsilon * static_cast(problem.dims.n_to) * largest; for (std::size_t index = 0; index < first.size(); ++index) EXPECT_NEAR(second[index], first[index], tolerance) << index; // Exact: a trial never writes the accepted state. @@ -2462,16 +2497,16 @@ TEST(GeneralState, RepeatedTrialsOverwriteFullCropAndStageZeroAccumulators) { TEST(GeneralState, RejectsDuplicateCropBins) { auto problem = general_problem(native::FieldFamily::envelope); - problem.crop_index[1] = problem.crop_index[0]; + problem.grid.crop_index[1] = problem.grid.crop_index[0]; EXPECT_THROW(native::GeneralState state(problem), std::invalid_argument); } TEST(GeneralState, SplitStepAbsorberDampsOnlyAcceptedState) { auto problem = general_problem(native::FieldFamily::envelope); - problem.nonlinear = false; - problem.split_step_absorber = true; - problem.absorber_coarse.assign(problem.n_t, 2.0); - const std::vector initial(problem.n_omega, {1.0, -0.25}); + problem.kerr.nonlinear = false; + problem.absorber.split_step = true; + problem.absorber.coarse.assign(problem.dims.n_t, 2.0); + const std::vector initial(problem.dims.n_omega, {1.0, -0.25}); constexpr double dz = 0.1; native::GeneralState state(problem); state.set_initial(initial, {}); @@ -2491,11 +2526,11 @@ TEST(GeneralState, SplitStepAbsorberDampsOnlyAcceptedState) { TEST(GeneralState, InStageAbsorberPreservesFirstSameAsLastEligibility) { auto problem = general_problem(native::FieldFamily::envelope); - problem.nonlinear = false; - problem.in_stage_absorber = true; - problem.first_same_as_last = true; - problem.absorber_fine.assign(problem.n_to, 2.0); - const std::vector initial(problem.n_omega, {1.0, 0.0}); + problem.kerr.nonlinear = false; + problem.absorber.in_stage = true; + problem.stepper.first_same_as_last = true; + problem.absorber.fine.assign(problem.dims.n_to, 2.0); + const std::vector initial(problem.dims.n_omega, {1.0, 0.0}); constexpr double dz = 0.1; native::GeneralState state(problem); state.set_initial(initial, {}); @@ -2516,17 +2551,17 @@ TEST(GeneralState, InStageAbsorberPreservesFirstSameAsLastEligibility) { // source (`multimode_reference`) and the exact half-step flows. native::Problem low_storage_problem(const std::size_t flows) { auto problem = multimode_problem(); - problem.n_stages = 3; - problem.tableau_matrix.assign(9, 0.0); - problem.tableau_nodes = {0.0, 0.5, 1.0}; - problem.solution_weights = {1.0, 0.0, 0.0}; - problem.error_weights = {0.25, 0.0, -0.25}; - problem.projection_coarse_matrix.clear(); - problem.low_storage = true; - problem.low_storage_flows = flows; - for (std::size_t mode = 0; mode < problem.n_modes; ++mode) - for (std::size_t index = 0; index < problem.n_omega; ++index) - problem.phi_terms[mode * problem.n_omega + index] = + problem.stepper.n_stages = 3; + problem.stepper.tableau_matrix.assign(9, 0.0); + problem.stepper.tableau_nodes = {0.0, 0.5, 1.0}; + problem.stepper.solution_weights = {1.0, 0.0, 0.0}; + problem.stepper.error_weights = {0.25, 0.0, -0.25}; + problem.transform.projection_coarse_matrix.clear(); + problem.stepper.low_storage = true; + problem.stepper.low_storage_flows = flows; + for (std::size_t mode = 0; mode < problem.dims.n_modes; ++mode) + for (std::size_t index = 0; index < problem.dims.n_omega; ++index) + problem.linear.phi_terms[mode * problem.dims.n_omega + index] = Complex(-0.3 * static_cast(index), 2.0 + static_cast(mode)); return problem; } @@ -2546,7 +2581,7 @@ std::vector low_storage_reference(const native::Problem& problem, const auto size = state.size(); std::vector flow(size); for (std::size_t element = 0; element < size; ++element) { - const Complex exponent = 0.5 * dz * problem.phi_terms[element]; + const Complex exponent = 0.5 * dz * problem.linear.phi_terms[element]; flow[element] = std::exp(exponent.real()) * Complex(std::cos(exponent.imag()), std::sin(exponent.imag())); } @@ -2621,8 +2656,8 @@ TEST(GeneralState, LowStoragePassesMatchTheHostInteractionPictureStep) { TEST(GeneralState, LowStorageStepsAreRefusedOutsideTheirState) { auto runge_kutta = low_storage_problem(1); - runge_kutta.low_storage = false; - runge_kutta.low_storage_flows = 0; + runge_kutta.stepper.low_storage = false; + runge_kutta.stepper.low_storage_flows = 0; native::GeneralState state(runge_kutta); state.set_initial(std::vector(6), {}); EXPECT_THROW(static_cast(state.low_storage_step( @@ -2631,7 +2666,7 @@ TEST(GeneralState, LowStorageStepsAreRefusedOutsideTheirState) { auto three = low_storage_problem(3); EXPECT_THROW(native::GeneralState too_many(three), std::invalid_argument); auto elided = low_storage_problem(0); - elided.low_storage_stage_register = false; + elided.stepper.low_storage_stage_register = false; EXPECT_THROW(native::GeneralState modal(elided), std::invalid_argument); native::GeneralState march(low_storage_problem(0)); march.set_initial(std::vector(6), {}); @@ -2657,20 +2692,20 @@ std::string construction_refusal(const native::Problem& problem) { // with no slab channels. native::Problem low_storage_free_problem() { auto problem = planar_problem(1); - problem.n_p = 0; - problem.n_modes = problem.n_x * problem.n_y; - problem.n_nodes = problem.n_modes; - problem.synthesis_matrix.clear(); - problem.projection_matrix.clear(); - problem.spatial_kappa_sq.clear(); + problem.spatial.n_p = 0; + problem.dims.n_modes = problem.spatial.n_x * problem.spatial.n_y; + problem.dims.n_nodes = problem.dims.n_modes; + problem.transform.synthesis_matrix.clear(); + problem.transform.projection_matrix.clear(); + problem.spatial.kappa_sq.clear(); // Construction only: the unit transverse square is k_x = 1, k_y = 0. - problem.spatial_k_x.assign(problem.n_x, 1.0); - problem.spatial_k_y.assign(problem.n_y, 0.0); - problem.spatial_kperp_sq.assign(problem.n_modes, 1.0); - problem.spatial_kperp_sq_low.assign(problem.n_modes, 0.0); - problem.spatial_absorber_rate.assign(problem.n_modes, 0.0); - problem.spatial_kperp_rate.assign(problem.n_modes, 0.0); - problem.low_storage = true; + problem.spatial.k_x.assign(problem.spatial.n_x, 1.0); + problem.spatial.k_y.assign(problem.spatial.n_y, 0.0); + problem.spatial.kperp_sq.assign(problem.dims.n_modes, 1.0); + problem.spatial.kperp_sq_low.assign(problem.dims.n_modes, 0.0); + problem.spatial.absorber_rate.assign(problem.dims.n_modes, 0.0); + problem.spatial.kperp_rate.assign(problem.dims.n_modes, 0.0); + problem.stepper.low_storage = true; return problem; } @@ -2678,23 +2713,23 @@ TEST(GeneralState, LowStorageRefusesRegistersItsFoldsCannotRead) { // The exchange fold of an in-stage absorber re-forms its input from the stage // register, so it is never elided there. auto elided = low_storage_free_problem(); - elided.low_storage_stage_register = false; + elided.stepper.low_storage_stage_register = false; EXPECT_EQ(construction_refusal(elided), ""); - elided.in_stage_absorber = true; - elided.absorber_fine.assign(elided.n_to, 1.0); + elided.absorber.in_stage = true; + elided.absorber.fine.assign(elided.dims.n_to, 1.0); EXPECT_NE(construction_refusal(elided).find("beside an in-stage absorber"), std::string::npos); // A separable fold applies one flow volume, a constant operator's table. auto table = low_storage_free_problem(); - table.low_storage_flows = 1; + table.stepper.low_storage_flows = 1; EXPECT_EQ(construction_refusal(table), ""); auto pair = low_storage_free_problem(); - pair.low_storage_flows = 2; + pair.stepper.low_storage_flows = 2; EXPECT_NE(construction_refusal(pair).find("at most one flow volume"), std::string::npos); auto planar = planar_problem(1); - planar.low_storage = true; - planar.low_storage_flows = 2; + planar.stepper.low_storage = true; + planar.stepper.low_storage_flows = 2; EXPECT_NE(construction_refusal(planar).find("at most one flow volume"), std::string::npos); } @@ -3084,25 +3119,25 @@ TEST(CpuThreadPolicy, CompactProblemsHaveOneColumnOnOneNode) { const auto base = general_problem(native::FieldFamily::field_real); EXPECT_TRUE(native::compact_problem(base)); auto nodes = base; - nodes.n_nodes = 2; + nodes.dims.n_nodes = 2; EXPECT_FALSE(native::compact_problem(nodes)); auto modes = base; - modes.n_modes = 2; + modes.dims.n_modes = 2; EXPECT_FALSE(native::compact_problem(modes)); auto pairs = base; - pairs.n_pol_transform = 2; + pairs.dims.n_pol_transform = 2; EXPECT_FALSE(native::compact_problem(pairs)); auto long_column = base; - long_column.n_to = native::test::table().compact_max_fine_samples; + long_column.dims.n_to = native::test::table().compact_max_fine_samples; EXPECT_FALSE(native::compact_problem(long_column)); auto modal = base; - modal.general_modal = true; + modal.general.enabled = true; EXPECT_FALSE(native::compact_problem(modal)); auto flow = base; - flow.coupled_flow = true; + flow.flow.enabled = true; EXPECT_FALSE(native::compact_problem(flow)); auto free_space = base; - free_space.n_x = 2; + free_space.spatial.n_x = 2; EXPECT_FALSE(native::compact_problem(free_space)); } @@ -3123,15 +3158,15 @@ TEST(CpuThreadPolicy, MetadataDistinguishesEqualWorkWithDifferentResponseShapes) TEST(CpuThreadPolicy, MatrixPackingDoesNotInheritTheDenseWorkWidth) { auto problem = general_problem(native::FieldFamily::field_real); - problem.n_to = 32768; - problem.n_nodes = 35; - problem.node_chunk = 33; - problem.synthesis_matrix.assign(problem.n_nodes, 1.0); - problem.projection_matrix.assign(problem.n_nodes, 1.0 / 35.0); - problem.projection_coarse_matrix = problem.projection_matrix; - problem.time_window.assign(problem.n_to, 1.0); - problem.time_window_derivative.assign(problem.n_to, 0.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); + problem.dims.n_to = 32768; + problem.dims.n_nodes = 35; + problem.dims.node_chunk = 33; + problem.transform.synthesis_matrix.assign(problem.dims.n_nodes, 1.0); + problem.transform.projection_matrix.assign(problem.dims.n_nodes, 1.0 / 35.0); + problem.transform.projection_coarse_matrix = problem.transform.projection_matrix; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.time_window_derivative.assign(problem.dims.n_to, 0.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); native::GeneralState state(problem); const auto sizing = state.execution_sizing(); // Full and tail matrices contain 33 and 2 entries, or twice those counts @@ -3152,21 +3187,21 @@ TEST(CpuThreadPolicy, WorkspacesKeepIndependentChoicesAndReleaseTheirPools) { auto problem = general_problem(native::FieldFamily::envelope); native::GeneralState small(problem); // One long row, past the compact bound: its transforms thread by shape. - problem.n_to = 65536; - problem.time_window.assign(problem.n_to, 1.0); - problem.time_window_derivative.assign(problem.n_to, 0.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); + problem.dims.n_to = 65536; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.time_window_derivative.assign(problem.dims.n_to, 0.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); native::GeneralState large(problem); native::GeneralState another_small(general_problem(native::FieldFamily::envelope)); const auto small_sizing = small.execution_sizing(); EXPECT_EQ(small_sizing, another_small.execution_sizing()); - const auto key = "fft.fft." + std::to_string(problem.n_to) + ".samples." + - std::to_string(problem.n_to); - EXPECT_EQ(large.execution_sizing().at(key).at("elements"), problem.n_to); + const auto key = "fft.fft." + std::to_string(problem.dims.n_to) + ".samples." + + std::to_string(problem.dims.n_to); + EXPECT_EQ(large.execution_sizing().at(key).at("elements"), problem.dims.n_to); EXPECT_EQ(large.execution_sizing().at(key).at("threads"), static_cast(native::operation_threads( native::thread_policy(), native::test::table(), native::CpuPool::fft, - native::CpuOperation::fft, problem.n_to, problem.n_to))); + native::CpuOperation::fft, problem.dims.n_to, problem.dims.n_to))); large.close(); EXPECT_EQ(small.execution_sizing(), small_sizing); small.close(); diff --git a/native/tests/native/test_coupled_flow.cpp b/native/tests/native/test_coupled_flow.cpp index 112b7a9b3..a2b1c9afd 100644 --- a/native/tests/native/test_coupled_flow.cpp +++ b/native/tests/native/test_coupled_flow.cpp @@ -51,45 +51,45 @@ std::size_t block(const std::size_t count) { return count * kActive * kModes * k native::Problem flow_problem(const bool residual) { native::Problem problem; problem.cpu_execution = native::test::table(); - problem.n_t = 4; - problem.n_to = 8; - problem.n_omega = kOmega; - problem.n_stages = kStages; - problem.n_phi_terms = 1; - problem.n_prefactor_terms = 1; - problem.n_modes = kModes; - problem.family = native::FieldFamily::field_real; - problem.crop_index = {0, 1, 2, 3}; - problem.active_mask.assign(kOmega, 1U); - problem.norm_mask.assign(kOmega, 1U); - problem.omega_window.assign(kOmega, 1.0); - problem.omega = {1.0, 2.0, 3.0, 4.0}; - problem.dt = 0.25; - problem.time_window.assign(problem.n_to, 1.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); + problem.dims.n_t = 4; + problem.dims.n_to = 8; + problem.dims.n_omega = kOmega; + problem.stepper.n_stages = kStages; + problem.linear.n_phi_terms = 1; + problem.linear.n_prefactor_terms = 1; + problem.dims.n_modes = kModes; + problem.dims.family = native::FieldFamily::field_real; + problem.grid.crop_index = {0, 1, 2, 3}; + problem.grid.active_mask.assign(kOmega, 1U); + problem.grid.norm_mask.assign(kOmega, 1U); + problem.grid.omega_window.assign(kOmega, 1.0); + problem.grid.omega = {1.0, 2.0, 3.0, 4.0}; + problem.grid.dt = 0.25; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); // A mode-independent carrier, as the flow requires: exp(dPhi) is a scalar per bin. for (std::size_t mode = 0; mode < kModes; ++mode) for (std::size_t index = 0; index < kOmega; ++index) - problem.phi_terms.emplace_back(-0.3, 5.0 * static_cast(index)); - problem.prefactor_terms.assign(kModes * kOmega, Complex(1.0, 0.0)); + problem.linear.phi_terms.emplace_back(-0.3, 5.0 * static_cast(index)); + problem.linear.prefactor_terms.assign(kModes * kOmega, Complex(1.0, 0.0)); // Heun's pair with a third stage at c = 1: a zero first fraction and a repeated // whole one, the two cases the action table has. - problem.tableau_matrix = {0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.5, 0.5, 0.0}; - problem.tableau_nodes = {0.0, 1.0, 1.0}; - problem.solution_weights = {0.5, 0.5, 0.0}; - problem.error_weights = {0.5, 0.5, -1.0}; - problem.synthesis_matrix.assign(kModes, 0.0); - problem.projection_matrix.assign(kModes, 0.0); - problem.coupled_flow = true; - problem.flow_active = kActive; - problem.flow_sections = kSections; - problem.flow_slot.assign(kSlot.begin(), kSlot.end()); + problem.stepper.tableau_matrix = {0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.5, 0.5, 0.0}; + problem.stepper.tableau_nodes = {0.0, 1.0, 1.0}; + problem.stepper.solution_weights = {0.5, 0.5, 0.0}; + problem.stepper.error_weights = {0.5, 0.5, -1.0}; + problem.transform.synthesis_matrix.assign(kModes, 0.0); + problem.transform.projection_matrix.assign(kModes, 0.0); + problem.flow.enabled = true; + problem.flow.active = kActive; + problem.flow.sections = kSections; + problem.flow.slot.assign(kSlot.begin(), kSlot.end()); if (residual) { - problem.flow_terms = kTerms; + problem.flow.terms = kTerms; for (std::size_t index = 0; index < block(kSections); ++index) - problem.flow_frozen.emplace_back(fill(index + 3, 2.0), fill(index + 5, 1.5)); + problem.flow.frozen.emplace_back(fill(index + 3, 2.0), fill(index + 5, 1.5)); for (std::size_t index = 0; index < block(kTerms); ++index) - problem.flow_residual_terms.emplace_back(fill(index + 7, 2.5), + problem.flow.residual_terms.emplace_back(fill(index + 7, 2.5), fill(index + 11, 1.0)); } return problem; @@ -164,13 +164,13 @@ std::vector reference_step(const native::Problem& problem, const Action std::vector solution(current.size()); const auto phase = [&](const std::size_t stage, const std::size_t mode, const std::size_t index) { - return trial.phi_delta[stage] * problem.phi_terms[mode * kOmega + index]; + return trial.phi_delta[stage] * problem.linear.phi_terms[mode * kOmega + index]; }; for (std::size_t stage = 0; stage < kStages; ++stage) { std::vector sum(current.size()); for (std::size_t previous = 0; previous < stage; ++previous) for (std::size_t element = 0; element < sum.size(); ++element) - sum[element] += problem.tableau_matrix[stage * kStages + previous] * + sum[element] += problem.stepper.tableau_matrix[stage * kStages + previous] * stages[previous][element]; std::vector lab(current.size()); std::vector drive(current.size()); @@ -191,7 +191,7 @@ std::vector reference_step(const native::Problem& problem, const Action lab[row * kOmega + index] = cexp(phase(stage, row, index)) * (state + kStep * carried); } - if (slot == 0 || problem.flow_terms == 0) + if (slot == 0 || problem.flow.terms == 0) continue; for (std::size_t row = 0; row < kModes; ++row) { Complex residual{}; @@ -200,13 +200,13 @@ std::vector reference_step(const native::Problem& problem, const Action for (std::size_t term = 0; term < kTerms; ++term) combined += trial.flow_weights[stage * kTerms + term] * - problem.flow_residual_terms + problem.flow.residual_terms [((term * kActive + slot - 1) * kModes + row) * kModes + column]; combined -= - problem.flow_frozen[((trial.flow_section * kActive + slot - 1) * kModes + - row) * - kModes + - column]; + problem.flow + .frozen[((trial.flow_section * kActive + slot - 1) * kModes + row) * + kModes + + column]; residual += combined * lab[column * kOmega + index]; } drive[row * kOmega + index] = cexp(-phase(stage, row, index)) * residual; @@ -227,7 +227,8 @@ std::vector reference_step(const native::Problem& problem, const Action } } for (std::size_t element = 0; element < solution.size(); ++element) - solution[element] += problem.solution_weights[stage] * stage_value[element]; + solution[element] += + problem.stepper.solution_weights[stage] * stage_value[element]; stages.push_back(stage_value); } std::vector advanced(current.size()); @@ -291,9 +292,9 @@ TEST(CoupledFlow, ConstantBlockCarriesTheStepByItsWholeAction) { for (std::size_t row = 0; row < kModes; ++row) { const auto element = row * kOmega; EXPECT_NE(actual[element + 1], - start[element + 1] * cexp(kStep * problem.phi_terms[1])); + start[element + 1] * cexp(kStep * problem.linear.phi_terms[1])); EXPECT_LE(Kokkos::abs(actual[element] - - start[element] * cexp(kStep * problem.phi_terms[0])), + start[element] * cexp(kStep * problem.linear.phi_terms[0])), kBound * largest(expected)); } } @@ -326,7 +327,7 @@ TEST(CoupledFlow, RepeatedTrialsAgreeToRounding) { TEST(CoupledFlow, NoStageIsReusedAcrossSteps) { auto problem = flow_problem(false); - problem.first_same_as_last = true; + problem.stepper.first_same_as_last = true; const auto flow = actions(); native::GeneralState state(problem); state.set_initial(initial(), {}); @@ -339,25 +340,25 @@ TEST(CoupledFlow, NoStageIsReusedAcrossSteps) { TEST(CoupledFlow, MalformedDataAreRefusedBeforeAllocation) { auto slot = flow_problem(false); - slot.flow_slot = {0, 2, 0, 1}; + slot.flow.slot = {0, 2, 0, 1}; EXPECT_THROW(native::GeneralState{slot}, std::invalid_argument); auto count = flow_problem(false); - count.flow_active = 3; + count.flow.active = 3; EXPECT_THROW(native::GeneralState{count}, std::invalid_argument); auto pol = flow_problem(false); - pol.n_pol_state = 2; + pol.dims.n_pol_state = 2; EXPECT_THROW(native::GeneralState{pol}, std::invalid_argument); auto noise = flow_problem(false); - noise.modified_noise = true; + noise.stepper.modified_noise = true; EXPECT_THROW(native::GeneralState{noise}, std::invalid_argument); auto terms = flow_problem(true); - terms.flow_residual_terms.pop_back(); + terms.flow.residual_terms.pop_back(); EXPECT_THROW(native::GeneralState{terms}, std::invalid_argument); auto stray = flow_problem(false); - stray.coupled_flow = false; + stray.flow.enabled = false; EXPECT_THROW(native::GeneralState{stray}, std::invalid_argument); auto nonfinite = flow_problem(true); - nonfinite.flow_frozen[0] = Complex(std::numeric_limits::quiet_NaN(), 0.0); + nonfinite.flow.frozen[0] = Complex(std::numeric_limits::quiet_NaN(), 0.0); EXPECT_THROW(native::GeneralState{nonfinite}, std::invalid_argument); } @@ -369,9 +370,9 @@ TEST(CoupledFlow, ATwoComponentSynthesisLeavesTheFlowAsItIs) { for (const bool residual : {false, true}) { const auto scalar = flow_problem(residual); auto vector = scalar; - vector.n_pol_transform = 2; - vector.synthesis_matrix.assign(2 * kModes, 0.0); - vector.projection_matrix.assign(2 * kModes, 0.0); + vector.dims.n_pol_transform = 2; + vector.transform.synthesis_matrix.assign(2 * kModes, 0.0); + vector.transform.projection_matrix.assign(2 * kModes, 0.0); const auto flow = actions(); const auto trial = scalars(residual, 1); const auto expected = native_step(scalar, flow, trial); @@ -423,10 +424,10 @@ TEST(CoupledFlow, LedgerCountsTheFlowViewsAndIsZeroElsewhere) { "flow_carried_solution", "flow_carried_error"}) EXPECT_EQ(ledger.at(name), kModes * kOmega * complex_bytes) << name; auto plain = flow_problem(false); - plain.coupled_flow = false; - plain.flow_active = 0; - plain.flow_sections = 0; - plain.flow_slot.clear(); + plain.flow.enabled = false; + plain.flow.active = 0; + plain.flow.sections = 0; + plain.flow.slot.clear(); native::GeneralState other(plain); for (const auto& [name, bytes] : native::CoupledFlowAction::empty_ledger()) EXPECT_EQ(other.memory_ledger().at(name), 0U) << name; @@ -442,9 +443,9 @@ TEST(CoupledFlow, ZeroWeightTermsAreSkippedWithoutMovingABit) { const auto flow = actions(); const auto trial = scalars(true, 1); auto padded = problem; - padded.flow_terms = 2 * kTerms; + padded.flow.terms = 2 * kTerms; for (std::size_t index = 0; index < block(kTerms); ++index) - padded.flow_residual_terms.emplace_back(fill(index + 53, 9.0), + padded.flow.residual_terms.emplace_back(fill(index + 53, 9.0), fill(index + 59, -7.0)); auto padded_trial = trial; padded_trial.flow_weights.clear(); @@ -559,19 +560,19 @@ native::Problem forming_problem(const bool fallback) { const auto upper = triangular(section, bin); const auto [left, right] = eigenbasis(vectors, upper); for (std::size_t mode = 0; mode < kModes; ++mode) - problem.flow_eigenvalues.push_back(upper[mode * kModes + mode]); + problem.flow.eigenvalues.push_back(upper[mode * kModes + mode]); if (fallback) { - problem.flow_route.push_back(++route); - problem.flow_left.insert(problem.flow_left.end(), kModes * kModes, Complex{}); - problem.flow_right.insert(problem.flow_right.end(), kModes * kModes, Complex{}); - problem.flow_fallback_vectors.insert(problem.flow_fallback_vectors.end(), + problem.flow.route.push_back(++route); + problem.flow.left.insert(problem.flow.left.end(), kModes * kModes, Complex{}); + problem.flow.right.insert(problem.flow.right.end(), kModes * kModes, Complex{}); + problem.flow.fallback_vectors.insert(problem.flow.fallback_vectors.end(), vectors.begin(), vectors.end()); - problem.flow_fallback_triangular.insert(problem.flow_fallback_triangular.end(), + problem.flow.fallback_triangular.insert(problem.flow.fallback_triangular.end(), upper.begin(), upper.end()); } else { - problem.flow_route.push_back(0); - problem.flow_left.insert(problem.flow_left.end(), left.begin(), left.end()); - problem.flow_right.insert(problem.flow_right.end(), right.begin(), right.end()); + problem.flow.route.push_back(0); + problem.flow.left.insert(problem.flow.left.end(), left.begin(), left.end()); + problem.flow.right.insert(problem.flow.right.end(), right.begin(), right.end()); } } return problem; @@ -703,23 +704,23 @@ TEST(CoupledFlowFormation, FormationAndItsDataAreChecked) { static_cast(state.form_flow_actions(0, kLength, {0.0, 0.5, 1.0, 0.0})), std::invalid_argument); auto numbering = forming_problem(true); - numbering.flow_route[0] = 2; + numbering.flow.route[0] = 2; EXPECT_THROW(native::GeneralState{numbering}, std::invalid_argument); auto fallback = forming_problem(true); - fallback.flow_fallback_vectors.pop_back(); + fallback.flow.fallback_vectors.pop_back(); EXPECT_THROW(native::GeneralState{fallback}, std::invalid_argument); auto partial = forming_problem(false); - partial.flow_right.pop_back(); + partial.flow.right.pop_back(); EXPECT_THROW(native::GeneralState{partial}, std::invalid_argument); auto nonfinite = forming_problem(false); - nonfinite.flow_eigenvalues[0] = Complex(std::numeric_limits::infinity(), 0.0); + nonfinite.flow.eigenvalues[0] = Complex(std::numeric_limits::infinity(), 0.0); EXPECT_THROW(native::GeneralState{nonfinite}, std::invalid_argument); auto stray = flow_problem(false); - stray.coupled_flow = false; - stray.flow_active = 0; - stray.flow_sections = 0; - stray.flow_slot.clear(); - stray.flow_route = {0}; + stray.flow.enabled = false; + stray.flow.active = 0; + stray.flow.sections = 0; + stray.flow.slot.clear(); + stray.flow.route = {0}; EXPECT_THROW(native::GeneralState{stray}, std::invalid_argument); } diff --git a/native/tests/native/test_dispersive.cpp b/native/tests/native/test_dispersive.cpp index 6b07a3f3a..121d3918a 100644 --- a/native/tests/native/test_dispersive.cpp +++ b/native/tests/native/test_dispersive.cpp @@ -299,43 +299,43 @@ TEST(DispersiveMixture, TheRowsAreTheReferenceArithmetic) { native::Problem dispersive_problem() { native::Problem problem; problem.cpu_execution = native::test::table(); - problem.n_t = 4; - problem.n_to = 8; - problem.n_omega = 3; - problem.n_stages = 1; - problem.family = native::FieldFamily::field_real; - problem.crop_index = {0, 1, 2}; - problem.active_mask = {1, 1, 1}; - problem.norm_mask = {1, 1, 1}; - problem.omega_window = {1.0, 1.0, 1.0}; - problem.time_window.assign(problem.n_to, 1.0); - problem.carrier.assign(problem.n_to, Complex{1.0, 0.0}); - problem.phi_terms.assign(problem.n_omega, Complex{}); - problem.prefactor_terms.assign(problem.n_omega, Complex{1.0, 0.0}); - problem.n_phi_terms = 1; - problem.n_prefactor_terms = 1; - problem.tableau_matrix = {0.0}; - problem.tableau_nodes = {0.0}; - problem.solution_weights = {1.0}; - problem.error_weights = {0.0}; - problem.synthesis_matrix = {1.0}; - problem.projection_matrix = {1.0}; - problem.synthesis = 1.0; - problem.projection = 1.0; - const std::size_t bins = problem.n_to / 2 + 1; - problem.dispersive = true; - problem.dispersive_symmetric = true; - problem.dispersive_form = 0; - problem.dispersive_inputs.assign(bins, 1.0); - problem.dispersive_term_multipliers.assign(bins, 1.0); - problem.dispersive_term_rows = {0}; - problem.dispersive_term_channels = {0}; - problem.dispersive_coefficients = {2.0}; - problem.dispersive_pair_constants = {0.0, 0.0}; - problem.dispersive_group_offsets = {0, 1}; - problem.dispersive_group_members = {0}; - problem.dispersive_input_scale.assign(bins, 1.0); - problem.dispersive_output_scale.assign(bins, 1.0); + problem.dims.n_t = 4; + problem.dims.n_to = 8; + problem.dims.n_omega = 3; + problem.stepper.n_stages = 1; + problem.dims.family = native::FieldFamily::field_real; + problem.grid.crop_index = {0, 1, 2}; + problem.grid.active_mask = {1, 1, 1}; + problem.grid.norm_mask = {1, 1, 1}; + problem.grid.omega_window = {1.0, 1.0, 1.0}; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex{1.0, 0.0}); + problem.linear.phi_terms.assign(problem.dims.n_omega, Complex{}); + problem.linear.prefactor_terms.assign(problem.dims.n_omega, Complex{1.0, 0.0}); + problem.linear.n_phi_terms = 1; + problem.linear.n_prefactor_terms = 1; + problem.stepper.tableau_matrix = {0.0}; + problem.stepper.tableau_nodes = {0.0}; + problem.stepper.solution_weights = {1.0}; + problem.stepper.error_weights = {0.0}; + problem.transform.synthesis_matrix = {1.0}; + problem.transform.projection_matrix = {1.0}; + problem.transform.synthesis = 1.0; + problem.transform.projection = 1.0; + const std::size_t bins = problem.dims.n_to / 2 + 1; + problem.dispersive.enabled = true; + problem.dispersive.symmetric = true; + problem.dispersive.form = 0; + problem.dispersive.inputs.assign(bins, 1.0); + problem.dispersive.term_multipliers.assign(bins, 1.0); + problem.dispersive.term_rows = {0}; + problem.dispersive.term_channels = {0}; + problem.dispersive.coefficients = {2.0}; + problem.dispersive.pair_constants = {0.0, 0.0}; + problem.dispersive.group_offsets = {0, 1}; + problem.dispersive.group_members = {0}; + problem.dispersive.input_scale.assign(bins, 1.0); + problem.dispersive.output_scale.assign(bins, 1.0); return problem; } @@ -360,7 +360,7 @@ TEST(DispersiveResponse, AUnitFilterIsTheConstantCubicOfOneNode) { static_cast(state.trial_step(0.0, scalars)); const auto projected = state.projected_to_host(false); // The field in time, cubed, transformed back. - const std::size_t n_to = problem.n_to; + const std::size_t n_to = problem.dims.n_to; std::vector time(n_to, 0.0); for (std::size_t sample = 0; sample < n_to; ++sample) { double value = field[0].real() / static_cast(n_to); @@ -390,8 +390,9 @@ TEST(DispersiveResponse, AUnitFilterIsTheConstantCubicOfOneNode) { EXPECT_EQ(ledger.at("dispersive_product_spectra"), 0U); EXPECT_EQ(ledger.at("dispersive_node_fields"), 0U); EXPECT_GT(ledger.at("dispersive_multipliers"), 0U); + // Closing destroys the workspace: a closed state holds nothing and reads nothing. state.close(); - EXPECT_EQ(state.memory_ledger().at("dispersive_multipliers"), 0U); + EXPECT_THROW(static_cast(state.memory_ledger()), std::logic_error); } TEST(DispersiveResponse, ATrialNeedsEachStageSDensities) { @@ -414,45 +415,45 @@ TEST(DispersiveResponse, ATrialNeedsEachStageSDensities) { TEST(DispersiveResponse, MalformedLayoutsAreRefusedBeforeAllocation) { { auto problem = dispersive_problem(); - problem.dispersive_form = 1; // the conserving sector on the real family + problem.dispersive.form = 1; // the conserving sector on the real family EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } { auto problem = dispersive_problem(); - problem.dispersive_input_scale.pop_back(); + problem.dispersive.input_scale.pop_back(); EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } { auto problem = dispersive_problem(); - problem.dispersive_term_rows = {1}; // a product the layout does not have + problem.dispersive.term_rows = {1}; // a product the layout does not have EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } { auto problem = dispersive_problem(); - problem.dispersive_group_members = {1}; + problem.dispersive.group_members = {1}; EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } { auto problem = dispersive_problem(); - problem.dispersive = false; // data without the response + problem.dispersive.enabled = false; // data without the response EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } } TEST(DispersiveResponse, AProblemWithoutItNamesTheRowsAtZero) { auto problem = dispersive_problem(); - problem.dispersive = false; - problem.dispersive_inputs.clear(); - problem.dispersive_term_multipliers.clear(); - problem.dispersive_term_rows.clear(); - problem.dispersive_term_channels.clear(); - problem.dispersive_coefficients.clear(); - problem.dispersive_pair_constants.clear(); - problem.dispersive_group_offsets.clear(); - problem.dispersive_group_members.clear(); - problem.dispersive_input_scale.clear(); - problem.dispersive_output_scale.clear(); - problem.nonlinear = true; + problem.dispersive.enabled = false; + problem.dispersive.inputs.clear(); + problem.dispersive.term_multipliers.clear(); + problem.dispersive.term_rows.clear(); + problem.dispersive.term_channels.clear(); + problem.dispersive.coefficients.clear(); + problem.dispersive.pair_constants.clear(); + problem.dispersive.group_offsets.clear(); + problem.dispersive.group_members.clear(); + problem.dispersive.input_scale.clear(); + problem.dispersive.output_scale.clear(); + problem.kerr.nonlinear = true; native::GeneralState state(problem); const auto ledger = state.memory_ledger(); for (const auto& [name, bytes] : native::DispersiveResponse::empty_ledger()) @@ -465,39 +466,39 @@ native::Problem general_problem(const bool symmetric) { // krypton in region 5, as solprop's leaf groups them. native::Problem problem; problem.cpu_execution = native::test::table(); - problem.n_to = 8; - problem.n_stages = 1; - problem.family = native::FieldFamily::field_real; - problem.general_modal = true; - problem.general_nodes = 5; - problem.general_node_chunk = 2; - problem.general_components = 3; - const std::size_t bins = problem.n_to / 2 + 1; - problem.dispersive = true; - problem.dispersive_symmetric = symmetric; - problem.dispersive_form = 1; - problem.dispersive_coefficients = {1.0, 2.0, 3.0}; - problem.dispersive_pair_constants.assign(6, 0.5); - problem.dispersive_input_scale.assign(bins, 1.0); - problem.dispersive_output_scale.assign(bins, 0.125); - problem.dispersive_node_region = {5, 5, 3, 0, 3}; + problem.dims.n_to = 8; + problem.stepper.n_stages = 1; + problem.dims.family = native::FieldFamily::field_real; + problem.general.enabled = true; + problem.general.nodes = 5; + problem.general.node_chunk = 2; + problem.general.components = 3; + const std::size_t bins = problem.dims.n_to / 2 + 1; + problem.dispersive.enabled = true; + problem.dispersive.symmetric = symmetric; + problem.dispersive.form = 1; + problem.dispersive.coefficients = {1.0, 2.0, 3.0}; + problem.dispersive.pair_constants.assign(6, 0.5); + problem.dispersive.input_scale.assign(bins, 1.0); + problem.dispersive.output_scale.assign(bins, 0.125); + problem.dispersive.node_region = {5, 5, 3, 0, 3}; if (symmetric) { - problem.dispersive_group_offsets = {0, 1, 3}; - problem.dispersive_group_members = {0, 1, 2}; - problem.dispersive_group_keys = {3, 5}; - problem.dispersive_inputs.assign(2 * bins, 1.0); - problem.dispersive_term_rows = {0, 1}; - problem.dispersive_term_channels = {0, 4}; - problem.dispersive_term_multipliers.assign(2 * bins, 0.125); - problem.dispersive_quartics.assign(3 * bins, 1.0); + problem.dispersive.group_offsets = {0, 1, 3}; + problem.dispersive.group_members = {0, 1, 2}; + problem.dispersive.group_keys = {3, 5}; + problem.dispersive.inputs.assign(2 * bins, 1.0); + problem.dispersive.term_rows = {0, 1}; + problem.dispersive.term_channels = {0, 4}; + problem.dispersive.term_multipliers.assign(2 * bins, 0.125); + problem.dispersive.quartics.assign(3 * bins, 1.0); } else { - problem.dispersive_group_offsets = {0, 1, 2, 3}; - problem.dispersive_group_members = {0, 1, 2}; - problem.dispersive_group_keys = {3, 5, 5}; - problem.dispersive_inputs.assign(6 * bins, 1.0); - problem.dispersive_term_rows = {0, 2, 3, 1, 4, 5, 1, 6, 7}; - problem.dispersive_term_channels = {0, 0, 3, 1, 1, 3, 2, 2, 3}; - problem.dispersive_term_multipliers.assign(9 * bins, 0.125); + problem.dispersive.group_offsets = {0, 1, 2, 3}; + problem.dispersive.group_members = {0, 1, 2}; + problem.dispersive.group_keys = {3, 5, 5}; + problem.dispersive.inputs.assign(6 * bins, 1.0); + problem.dispersive.term_rows = {0, 2, 3, 1, 4, 5, 1, 6, 7}; + problem.dispersive.term_channels = {0, 0, 3, 1, 1, 3, 2, 2, 3}; + problem.dispersive.term_multipliers.assign(9 * bins, 0.125); } return problem; } @@ -524,28 +525,28 @@ TEST(DispersiveWeights, ThePolynomialFormsOneCubePerRegion) { TEST(DispersiveWeights, RegionsBelongToTheGeneralRouteAlone) { { auto problem = general_problem(false); - problem.dispersive_node_region.pop_back(); + problem.dispersive.node_region.pop_back(); EXPECT_THROW(native::require_dispersive(problem), std::invalid_argument); } { auto problem = general_problem(false); - problem.dispersive_group_keys.pop_back(); + problem.dispersive.group_keys.pop_back(); EXPECT_THROW(native::require_dispersive(problem), std::invalid_argument); } { auto problem = general_problem(false); - problem.dispersive_form = 2; // an envelope's harmonic groups + problem.dispersive.form = 2; // an envelope's harmonic groups EXPECT_THROW(native::require_dispersive(problem), std::invalid_argument); } { auto problem = general_problem(false); - problem.dispersive_term_rows.back() = 8; // one product past the layout + problem.dispersive.term_rows.back() = 8; // one product past the layout EXPECT_THROW(native::require_dispersive(problem), std::invalid_argument); } { // Off the general route every channel acts at every node. auto problem = dispersive_problem(); - problem.dispersive_group_keys = {0}; + problem.dispersive.group_keys = {0}; EXPECT_THROW(native::GeneralState{problem}, std::invalid_argument); } EXPECT_NO_THROW(native::require_dispersive(general_problem(true))); diff --git a/native/tests/native/test_general_modal.cpp b/native/tests/native/test_general_modal.cpp index 6bb2d7f0f..46f0e7bfc 100644 --- a/native/tests/native/test_general_modal.cpp +++ b/native/tests/native/test_general_modal.cpp @@ -61,68 +61,69 @@ native::Problem modal_problem(const bool real_form, const std::size_t chunk, const std::size_t n_to = 8) { native::Problem problem; problem.cpu_execution = native::test::table(); - problem.n_t = n_to / 2; - problem.n_to = n_to; - problem.n_omega = n_omega; - problem.n_stages = 1; - problem.n_phi_terms = 1; - problem.n_prefactor_terms = 1; - problem.n_modes = kModes; - problem.family = native::FieldFamily::field_real; - problem.crop_index.resize(n_omega); - std::iota(problem.crop_index.begin(), problem.crop_index.end(), offset); - problem.active_mask.assign(problem.n_omega, 1U); - problem.norm_mask.assign(problem.n_omega, 1U); - problem.omega_window.assign(problem.n_omega, 1.0); + problem.dims.n_t = n_to / 2; + problem.dims.n_to = n_to; + problem.dims.n_omega = n_omega; + problem.stepper.n_stages = 1; + problem.linear.n_phi_terms = 1; + problem.linear.n_prefactor_terms = 1; + problem.dims.n_modes = kModes; + problem.dims.family = native::FieldFamily::field_real; + problem.grid.crop_index.resize(n_omega); + std::iota(problem.grid.crop_index.begin(), problem.grid.crop_index.end(), offset); + problem.grid.active_mask.assign(problem.dims.n_omega, 1U); + problem.grid.norm_mask.assign(problem.dims.n_omega, 1U); + problem.grid.omega_window.assign(problem.dims.n_omega, 1.0); for (std::size_t w = 0; w < n_omega; ++w) - problem.omega.push_back(static_cast(w + 1)); - problem.dt = 0.25; - problem.time_window.assign(problem.n_to, 1.0); - problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); - problem.phi_terms.assign(kModes * problem.n_omega, Complex{}); - problem.prefactor_terms.assign(kModes * problem.n_omega, Complex(1.0, 0.0)); - problem.tableau_matrix = {0.0}; - problem.tableau_nodes = {0.0}; - problem.solution_weights = {1.0}; - problem.error_weights = {0.25}; - problem.synthesis_matrix = {0.0, 0.0}; - problem.projection_matrix = {0.0, 0.0}; - problem.general_modal = true; - problem.general_real_form = real_form; - problem.general_components = kComponents; - problem.general_nodes = kNodes; - problem.general_node_chunk = chunk; - problem.general_pad_scale = 2.0; - problem.general_epsilon0 = 1.0; - const auto maps = kNodes * kComponents * kModes * problem.n_omega; + problem.grid.omega.push_back(static_cast(w + 1)); + problem.grid.dt = 0.25; + problem.grid.time_window.assign(problem.dims.n_to, 1.0); + problem.grid.carrier.assign(problem.dims.n_to, Complex(1.0, 0.0)); + problem.linear.phi_terms.assign(kModes * problem.dims.n_omega, Complex{}); + problem.linear.prefactor_terms.assign(kModes * problem.dims.n_omega, + Complex(1.0, 0.0)); + problem.stepper.tableau_matrix = {0.0}; + problem.stepper.tableau_nodes = {0.0}; + problem.stepper.solution_weights = {1.0}; + problem.stepper.error_weights = {0.25}; + problem.transform.synthesis_matrix = {0.0, 0.0}; + problem.transform.projection_matrix = {0.0, 0.0}; + problem.general.enabled = true; + problem.general.real_form = real_form; + problem.general.components = kComponents; + problem.general.nodes = kNodes; + problem.general.node_chunk = chunk; + problem.general.pad_scale = 2.0; + problem.general.epsilon0 = 1.0; + const auto maps = kNodes * kComponents * kModes * problem.dims.n_omega; for (std::size_t index = 0; index < maps; ++index) { - problem.general_synthesis.emplace_back(fill(index, 1.0), fill(index + 97, 0.5)); - problem.general_projection.emplace_back(fill(index + 13, 0.8), + problem.general.synthesis.emplace_back(fill(index, 1.0), fill(index + 97, 0.5)); + problem.general.projection.emplace_back(fill(index + 13, 0.8), fill(index + 41, 0.6)); } for (std::size_t index = 0; index < kNodes * fourth(kComponents); ++index) - problem.general_tensor.push_back(fill(index + 7, 1.0)); + problem.general.tensor.push_back(fill(index + 7, 1.0)); for (std::size_t w = 0; w < n_omega; ++w) - problem.general_frequency_factor.emplace_back(0.0, -static_cast(w + 1)); + problem.general.frequency_factor.emplace_back(0.0, -static_cast(w + 1)); if (delayed) { - problem.general_responses = kResponses; - problem.general_terms = kTerms; + problem.general.responses = kResponses; + problem.general.terms = kTerms; for (std::size_t index = 0; index < kResponses * kTerms * fourth(kComponents); ++index) - problem.general_delayed_tensor.push_back(fill(index + 3, 0.5)); - for (std::size_t index = 0; index < kResponses * kTerms * (problem.n_to + 1); + problem.general.delayed_tensor.push_back(fill(index + 3, 0.5)); + for (std::size_t index = 0; index < kResponses * kTerms * (problem.dims.n_to + 1); ++index) - problem.general_delayed_spectrum.emplace_back(fill(index + 5, 1.0), + problem.general.delayed_spectrum.emplace_back(fill(index + 5, 1.0), fill(index + 11, 0.4)); - problem.general_node_response = {1, 2, 0, 1, 2}; + problem.general.node_response = {1, 2, 0, 1, 2}; } return problem; } std::vector scaled_state(const native::Problem& problem) { std::vector state; - state.reserve(kModes * problem.n_omega); - for (std::size_t index = 0; index < kModes * problem.n_omega; ++index) + state.reserve(kModes * problem.dims.n_omega); + for (std::size_t index = 0; index < kModes * problem.dims.n_omega; ++index) state.emplace_back(fill(index + 17, 1.0), fill(index + 23, 1.0)); return state; } @@ -130,8 +131,8 @@ std::vector scaled_state(const native::Problem& problem) { // solprop's `general_source`, written as sums over samples and bins. std::vector reference_source(const native::Problem& problem, const std::vector& scaled) { - const auto n_to = problem.n_to; - const auto n_omega = problem.n_omega; + const auto n_to = problem.dims.n_to; + const auto n_omega = problem.dims.n_omega; const auto components = kComponents; const auto two_pi = 2.0 * std::numbers::pi_v; std::vector source(kModes * n_omega); @@ -141,19 +142,17 @@ std::vector reference_source(const native::Problem& problem, for (std::size_t w = 0; w < n_omega; ++w) { Complex bin{}; for (std::size_t m = 0; m < kModes; ++m) - bin += - problem - .general_synthesis[((node * components + c) * kModes + m) * n_omega + - w] * - scaled[m * n_omega + w]; - const auto fine = problem.crop_index[w]; + bin += problem.general + .synthesis[((node * components + c) * kModes + m) * n_omega + w] * + scaled[m * n_omega + w]; + const auto fine = problem.grid.crop_index[w]; for (std::size_t t = 0; t < n_to; ++t) field[c][t] += bin * cis(two_pi * static_cast(fine * t) / static_cast(n_to)) / static_cast(n_to); } - if (problem.general_real_form) + if (problem.general.real_form) for (auto& component : field) for (auto& value : component) value = Complex(value.real(), 0.0); @@ -166,17 +165,18 @@ std::vector reference_source(const native::Problem& problem, for (std::size_t j = 0; j < components; ++j) for (std::size_t k = 0; k < components; ++k) for (std::size_t l = 0; l < components; ++l) - value += problem.general_tensor[base + - ((i * components + j) * components + k) * - components + - l] * - field[k][t] * Kokkos::conj(field[l][t]) * field[j][t]; - polarisation[i][t] = problem.general_epsilon0 * value.real(); + value += + problem.general + .tensor[base + + ((i * components + j) * components + k) * components + + l] * + field[k][t] * Kokkos::conj(field[l][t]) * field[j][t]; + polarisation[i][t] = problem.general.epsilon0 * value.real(); } - if (problem.general_terms != 0 && problem.general_node_response[node] != 0) { - const auto owner = problem.general_node_response[node] - 1; + if (problem.general.terms != 0 && problem.general.node_response[node] != 0) { + const auto owner = problem.general.node_response[node] - 1; const auto doubled = 2 * n_to; - for (std::size_t term = 0; term < problem.general_terms; ++term) { + for (std::size_t term = 0; term < problem.general.terms; ++term) { std::vector>> history( components, std::vector>(components, std::vector(n_to))); @@ -185,7 +185,7 @@ std::vector reference_source(const native::Problem& problem, std::vector drive(doubled); for (std::size_t t = 0; t < n_to; ++t) { const double pair = (field[k][t] * Kokkos::conj(field[l][t])).real(); - drive[t] = problem.general_real_form ? pair : 0.5 * pair; + drive[t] = problem.general.real_form ? pair : 0.5 * pair; } std::vector product(n_to + 1); for (std::size_t f = 0; f <= n_to; ++f) { @@ -194,8 +194,8 @@ std::vector reference_source(const native::Problem& problem, bin += drive[t] * cis(-two_pi * static_cast(f * t) / static_cast(doubled)); product[f] = - problem.general_delayed_spectrum - [(owner * problem.general_terms + term) * (n_to + 1) + f] * + problem.general.delayed_spectrum + [(owner * problem.general.terms + term) * (n_to + 1) + f] * bin; } for (std::size_t t = 0; t < n_to; ++t) { @@ -208,7 +208,7 @@ std::vector reference_source(const native::Problem& problem, history[k][l][t] = value / static_cast(doubled); } } - const auto tensor = (owner * problem.general_terms + term) * fourth(components); + const auto tensor = (owner * problem.general.terms + term) * fourth(components); for (std::size_t t = 0; t < n_to; ++t) for (std::size_t i = 0; i < components; ++i) { double value = 0.0; @@ -216,32 +216,31 @@ std::vector reference_source(const native::Problem& problem, for (std::size_t k = 0; k < components; ++k) for (std::size_t l = 0; l < components; ++l) value += - problem.general_delayed_tensor + problem.general.delayed_tensor [tensor + ((i * components + j) * components + k) * components + l] * history[k][l][t] * field[j][t].real(); - polarisation[i][t] += problem.general_epsilon0 * value; + polarisation[i][t] += problem.general.epsilon0 * value; } } } for (std::size_t c = 0; c < components; ++c) for (std::size_t w = 0; w < n_omega; ++w) { - const auto fine = problem.crop_index[w]; + const auto fine = problem.grid.crop_index[w]; Complex bin{}; for (std::size_t t = 0; t < n_to; ++t) bin += polarisation[c][t] * cis(-two_pi * static_cast(fine * t) / static_cast(n_to)); for (std::size_t m = 0; m < kModes; ++m) source[m * n_omega + w] += - problem - .general_projection[((m * kNodes + node) * components + c) * n_omega + - w] * + problem.general + .projection[((m * kNodes + node) * components + c) * n_omega + w] * bin; } } for (std::size_t m = 0; m < kModes; ++m) for (std::size_t w = 0; w < n_omega; ++w) - source[m * n_omega + w] *= problem.general_frequency_factor[w]; + source[m * n_omega + w] *= problem.general.frequency_factor[w]; return source; } @@ -251,15 +250,15 @@ std::vector evaluated(const native::Problem& problem, source.plan(); auto input = Kokkos::create_mirror_view(source.scaled()); for (std::size_t m = 0; m < kModes; ++m) - for (std::size_t w = 0; w < problem.n_omega; ++w) - input(m, w) = scaled[m * problem.n_omega + w]; + for (std::size_t w = 0; w < problem.dims.n_omega; ++w) + input(m, w) = scaled[m * problem.dims.n_omega + w]; Kokkos::deep_copy(source.scaled(), input); source.evaluate(); const auto output = Kokkos::create_mirror_view_and_copy(Kokkos::HostSpace{}, source.source()); std::vector result; for (std::size_t m = 0; m < kModes; ++m) - for (std::size_t w = 0; w < problem.n_omega; ++w) + for (std::size_t w = 0; w < problem.dims.n_omega; ++w) result.push_back(output(m, w)); return result; } @@ -376,8 +375,8 @@ TEST(GeneralModal, RepeatedEvaluationAgreesToRounding) { auto input = Kokkos::create_mirror(source.scaled()); for (int repeat = 0; repeat < 3; ++repeat) { for (std::size_t m = 0; m < kModes; ++m) - for (std::size_t w = 0; w < problem.n_omega; ++w) - input(m, w) = scaled[m * problem.n_omega + w]; + for (std::size_t w = 0; w < problem.dims.n_omega; ++w) + input(m, w) = scaled[m * problem.dims.n_omega + w]; Kokkos::deep_copy(source.scaled(), input); source.evaluate(); const auto output = @@ -392,38 +391,36 @@ TEST(GeneralModal, RepeatedEvaluationAgreesToRounding) { EXPECT_EQ(zero.data()[index], Complex{}); EXPECT_EQ(source.memory_ledger(), ledger); } - source.close(); - source.close(); } } TEST(GeneralModal, MalformedDataIsRefusedBeforeAllocation) { auto short_map = modal_problem(true, 2, false); - short_map.general_synthesis.pop_back(); + short_map.general.synthesis.pop_back(); EXPECT_THROW(native::require_general_modal(short_map), std::invalid_argument); auto chunk = modal_problem(true, 2, false); - chunk.general_node_chunk = kNodes + 1; + chunk.general.node_chunk = kNodes + 1; EXPECT_THROW(native::require_general_modal(chunk), std::invalid_argument); auto owner = modal_problem(true, 2, true); - owner.general_node_response[0] = kResponses + 1; + owner.general.node_response[0] = kResponses + 1; EXPECT_THROW(native::require_general_modal(owner), std::invalid_argument); auto two = modal_problem(true, 2, true); - two.general_components = 2; + two.general.components = 2; EXPECT_THROW(native::require_general_modal(two), std::invalid_argument); auto legacy = modal_problem(true, 2, false); - legacy.nonlinear = true; + legacy.kerr.nonlinear = true; EXPECT_THROW(native::require_general_modal(legacy), std::invalid_argument); auto envelope = modal_problem(true, 2, false); - envelope.family = native::FieldFamily::envelope; + envelope.dims.family = native::FieldFamily::envelope; EXPECT_THROW(native::require_general_modal(envelope), std::invalid_argument); auto scattered = modal_problem(true, 2, false); - scattered.crop_index = {0, 2, 1}; + scattered.grid.crop_index = {0, 2, 1}; EXPECT_THROW(native::require_general_modal(scattered), std::invalid_argument); auto stray = modal_problem(true, 2, false); - stray.general_modal = false; + stray.general.enabled = false; EXPECT_THROW(native::require_general_modal(stray), std::invalid_argument); auto infinite = modal_problem(true, 2, false); - infinite.general_tensor[3] = std::numeric_limits::infinity(); + infinite.general.tensor[3] = std::numeric_limits::infinity(); EXPECT_THROW(native::GeneralState state(infinite), std::invalid_argument); } @@ -433,8 +430,8 @@ TEST(GeneralModal, FullAndTailTransformsUseTheDedicatedPolicy) { auto problem = modal_problem(true, 3, false, 4096, 0, 16384); native::GeneralModalSource source(problem); const auto sizing = source.execution_sizing(); - const auto full = 3 * kComponents * kModes * problem.n_omega; - const auto tail = 2 * kComponents * kModes * problem.n_omega; + const auto full = 3 * kComponents * kModes * problem.dims.n_omega; + const auto tail = 2 * kComponents * kModes * problem.dims.n_omega; const auto capacity = static_cast(native::ExecutionSpace().concurrency()); EXPECT_EQ(sizing.at("kernels.modal." + std::to_string(full)).at("threads"), capacity); @@ -456,11 +453,11 @@ TEST(GeneralModal, LedgerRowsAreTheAllocatedViews) { const auto real = sizeof(double); const auto rows = 2 * kComponents; const auto pairs = 2 * native::general_pair_histories; - const auto n_to = problem.n_to; - EXPECT_EQ(ledger.at("general_synthesis"), problem.general_synthesis.size() * complex); + const auto n_to = problem.dims.n_to; + EXPECT_EQ(ledger.at("general_synthesis"), problem.general.synthesis.size() * complex); EXPECT_EQ(ledger.at("general_projection"), - problem.general_projection.size() * complex); - EXPECT_EQ(ledger.at("general_tensor"), problem.general_tensor.size() * real); + problem.general.projection.size() * complex); + EXPECT_EQ(ledger.at("general_tensor"), problem.general.tensor.size() * real); EXPECT_EQ(ledger.at("general_node_spectrum"), rows * n_to * complex); EXPECT_EQ(ledger.at("general_node_field"), rows * n_to * complex); EXPECT_EQ(ledger.at("general_node_polarisation"), rows * n_to * real); @@ -471,7 +468,7 @@ TEST(GeneralModal, LedgerRowsAreTheAllocatedViews) { EXPECT_EQ(ledger.at("general_node_product"), pairs * (n_to + 1) * complex); EXPECT_EQ(ledger.at("general_node_delayed"), rows * n_to * real); EXPECT_EQ(ledger.at("general_delayed_spectrum"), - problem.general_delayed_spectrum.size() * complex); + problem.general.delayed_spectrum.size() * complex); EXPECT_EQ(ledger.at("general_node_response"), kNodes * sizeof(std::size_t)); } @@ -480,16 +477,16 @@ TEST(GeneralModal, TheStateCarriesTheSourceAndItsLedger) { native::GeneralState state(problem); const auto ledger = state.memory_ledger(); EXPECT_EQ(ledger.at("general_synthesis"), - problem.general_synthesis.size() * sizeof(Complex)); + problem.general.synthesis.size() * sizeof(Complex)); // The host maps are released once uploaded; the retained problem is small. const auto host_maps = - (problem.general_synthesis.size() + problem.general_projection.size() + - problem.general_delayed_spectrum.size()) * + (problem.general.synthesis.size() + problem.general.projection.size() + + problem.general.delayed_spectrum.size()) * sizeof(Complex) + - (problem.general_tensor.size() + problem.general_delayed_tensor.size()) * + (problem.general.tensor.size() + problem.general.delayed_tensor.size()) * sizeof(double); EXPECT_LT(ledger.at("host_problem_arrays"), host_maps); - std::vector field(kModes * problem.n_omega); + std::vector field(kModes * problem.dims.n_omega); for (std::size_t index = 0; index < field.size(); ++index) field[index] = Complex(fill(index, 1.0), fill(index + 1, 1.0)); state.set_initial(field, {}); @@ -526,7 +523,7 @@ TEST(GeneralModal, TheStateCarriesTheSourceAndItsLedger) { TEST(GeneralModal, AnEdgeLineReadsAPlainTransformAndASlope) { const auto problem = modal_problem(true, 2, false); native::GeneralState state(problem); - const auto n = problem.n_omega; + const auto n = problem.dims.n_omega; // Rows flowing both ways, so that rectifying a sum would read something else. std::vector field(kModes * n); for (std::size_t row = 0; row < n; ++row) { @@ -627,16 +624,16 @@ TEST(GeneralModal, AnEdgeLineThatDoesNotAccountForItsNodesIsRefused) { TEST(GeneralModal, TheAdamsRowsRoundTripExactly) { auto problem = modal_problem(true, 2, false); native::GeneralState state(problem); - std::vector field(kModes * problem.n_omega, Complex(1.0, 0.5)); + std::vector field(kModes * problem.dims.n_omega, Complex(1.0, 0.5)); state.set_initial(field, {}); constexpr std::size_t kSlots = 4; state.adams_allocate(kSlots, true, false); EXPECT_EQ(state.adams_row_names(), (std::vector{"store", "states", "rates"})); - const std::array extents{kSlots, kModes, problem.n_omega}; + const std::array extents{kSlots, kModes, problem.dims.n_omega}; for (const std::string name : {"store", "states", "rates"}) { EXPECT_EQ(state.adams_row_extents(name), extents); - std::vector rows(kSlots * kModes * problem.n_omega); + std::vector rows(kSlots * kModes * problem.dims.n_omega); for (std::size_t index = 0; index < rows.size(); ++index) rows[index] = Complex(fill(index, 1.0), fill(index + name.size(), 1e-300)); state.adams_rows_from_host(name, rows); @@ -669,7 +666,7 @@ TEST(GeneralModal, TheAdamsRowsRoundTripExactly) { TEST(GeneralModal, TheAdamsRowsNeedTheHistory) { auto problem = modal_problem(true, 2, false); native::GeneralState state(problem); - std::vector field(kModes * problem.n_omega, Complex(1.0, 0.5)); + std::vector field(kModes * problem.dims.n_omega, Complex(1.0, 0.5)); state.set_initial(field, {}); EXPECT_THROW(static_cast(state.adams_row_names()), std::logic_error); state.adams_allocate(3, false, false); @@ -679,9 +676,9 @@ TEST(GeneralModal, TheAdamsRowsNeedTheHistory) { TEST(GeneralModal, ALinearGuideLeavesTheStageZero) { auto problem = modal_problem(true, 2, false); - std::ranges::fill(problem.general_tensor, 0.0); + std::ranges::fill(problem.general.tensor, 0.0); native::GeneralState state(problem); - std::vector field(kModes * problem.n_omega, Complex(1.0, 0.5)); + std::vector field(kModes * problem.dims.n_omega, Complex(1.0, 0.5)); state.set_initial(field, {}); const native::TrialScalars scalars{.phi_delta = {0.0, 0.0}, .phi_at = {0.0}, @@ -725,23 +722,23 @@ constexpr double kCombinationBound = 2 * kDegree * kEpsilon; // other terms fall off as a Chebyshev series' do. native::Problem moving_problem(const bool real_form, const std::size_t stages = 2) { auto problem = modal_problem(real_form, 2, false); - problem.n_stages = stages; - problem.general_map_intervals = kSections; - problem.general_map_n_deg = kDegree; - const auto synthesis = problem.general_synthesis; - const auto projection = problem.general_projection; + problem.stepper.n_stages = stages; + problem.general.map_intervals = kSections; + problem.general.map_n_deg = kDegree; + const auto synthesis = problem.general.synthesis; + const auto projection = problem.general.projection; const auto maps = synthesis.size(); - problem.general_synthesis.clear(); - problem.general_projection.clear(); + problem.general.synthesis.clear(); + problem.general.projection.clear(); for (std::size_t term = 0; term < kMapTerms; ++term) { const double scale = 1.0 / static_cast(1 + term % kDegree); for (std::size_t index = 0; index < maps; ++index) { const auto shift = 31 * term; - problem.general_synthesis.push_back( + problem.general.synthesis.push_back( term == 0 ? synthesis[index] : Complex(fill(index + shift, scale), fill(index + shift + 5, scale))); - problem.general_projection.push_back( + problem.general.projection.push_back( term == 0 ? projection[index] : Complex(fill(index + shift + 9, scale), fill(index + shift + 2, scale))); @@ -788,8 +785,8 @@ void load_scaled(native::GeneralModalSource& source, const native::Problem& prob const std::vector& scaled) { auto input = Kokkos::create_mirror_view(source.scaled()); for (std::size_t m = 0; m < kModes; ++m) - for (std::size_t w = 0; w < problem.n_omega; ++w) - input(m, w) = scaled[m * problem.n_omega + w]; + for (std::size_t w = 0; w < problem.dims.n_omega; ++w) + input(m, w) = scaled[m * problem.dims.n_omega + w]; Kokkos::deep_copy(source.scaled(), input); } @@ -803,7 +800,7 @@ TEST(GeneralModal, MovingMapsAreEachStageSWeightedSumOfItsSectionSTerms) { source.combine(stage); for (const bool synthesis : {true, false}) { const auto& terms = - synthesis ? problem.general_synthesis : problem.general_projection; + synthesis ? problem.general.synthesis : problem.general.projection; const auto actual = to_host(synthesis ? source.synthesis() : source.projection()); const auto expected = host_map(terms, weights, stage, false); const auto magnitude = host_map(terms, weights, stage, true); @@ -834,7 +831,7 @@ TEST(GeneralModal, OneSelectedTermIsTheConstantGuideOfThatTerm) { source.combine(0); for (const bool synthesis : {true, false}) { const auto& terms = - synthesis ? problem.general_synthesis : problem.general_projection; + synthesis ? problem.general.synthesis : problem.general.projection; const auto maps = to_host(synthesis ? source.synthesis() : source.projection()); const auto selected = host_map(terms, weights, 0, false); ASSERT_EQ(maps.size(), selected.size()); @@ -858,10 +855,10 @@ TEST(GeneralModal, AMovingStageSSourceIsTheSourceOfItsCombinedMaps) { const auto weights = moving_weights(); for (std::size_t stage = 0; stage < 2; ++stage) { auto combined = modal_problem(false, 2, false); - combined.general_synthesis = - host_map(problem.general_synthesis, weights, stage, false); - combined.general_projection = - host_map(problem.general_projection, weights, stage, false); + combined.general.synthesis = + host_map(problem.general.synthesis, weights, stage, false); + combined.general.projection = + host_map(problem.general.projection, weights, stage, false); const auto scaled = scaled_state(combined); native::GeneralModalSource source(problem); source.plan(); @@ -902,13 +899,13 @@ TEST(GeneralModal, MovingMapsRefuseMalformedWeightsAndAMissingTrial) { fixed.set_weights({}); fixed.combine(0); auto short_terms = moving_problem(true); - short_terms.general_projection.resize(short_terms.general_projection.size() - 1); + short_terms.general.projection.resize(short_terms.general.projection.size() - 1); EXPECT_THROW(native::require_general_modal(short_terms), std::invalid_argument); auto no_sections = moving_problem(true); - no_sections.general_map_intervals = 0; + no_sections.general.map_intervals = 0; EXPECT_THROW(native::require_general_modal(no_sections), std::invalid_argument); auto no_stages = moving_problem(true); - no_stages.n_stages = 0; + no_stages.stepper.n_stages = 0; EXPECT_THROW(native::require_general_modal(no_stages), std::invalid_argument); } @@ -917,7 +914,7 @@ TEST(GeneralModal, MovingLedgerRowsAreTheTermsTheStageMapsAndTheWeights) { native::GeneralModalSource source(problem); const auto ledger = source.memory_ledger(); const auto complex = sizeof(Complex); - const auto one = problem.general_synthesis.size() / kMapTerms; + const auto one = problem.general.synthesis.size() / kMapTerms; EXPECT_EQ(ledger.at("general_synthesis"), kMapTerms * one * complex); EXPECT_EQ(ledger.at("general_projection"), kMapTerms * one * complex); EXPECT_EQ(ledger.at("general_stage_maps"), 2 * one * complex); @@ -934,8 +931,8 @@ TEST(GeneralModal, TheStateCombinesAMovingGuideSMapsPerTrial) { const auto problem = moving_problem(true, 1); native::GeneralState state(problem); EXPECT_EQ(state.memory_ledger().at("general_stage_maps"), - 2 * problem.general_synthesis.size() / kMapTerms * sizeof(Complex)); - std::vector field(kModes * problem.n_omega); + 2 * problem.general.synthesis.size() / kMapTerms * sizeof(Complex)); + std::vector field(kModes * problem.dims.n_omega); for (std::size_t index = 0; index < field.size(); ++index) field[index] = Complex(fill(index, 1.0), fill(index + 1, 1.0)); state.set_initial(field, {}); diff --git a/native/tests/test_backend.py b/native/tests/test_backend.py index 88a44ce71..950c6a9e3 100644 --- a/native/tests/test_backend.py +++ b/native/tests/test_backend.py @@ -45,7 +45,8 @@ import solprop_native from solprop_native import NativeBackend, _core, build_info, check_build from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] _ResidentLowStorage, # pyright: ignore[reportPrivateUsage] ) from solprop_native.session import NativeSession @@ -106,7 +107,7 @@ def test_native_modal_payload_requires_both_fixed_coefficient_arrays( prefactor=None if missing == "prefactor" else terms.prefactor, ) with pytest.raises(BackendError, match="device-ready fixed-term problem"): - _native_problem(problem, run_request, damaged) + _lower(problem, run_request, _LowerPlan(terms=damaged)) finally: backend.close() @@ -646,14 +647,14 @@ def test_raman_metadata_memoization_and_conditional_ledger( backend.close() -def _raman_native_payload(raman_case: RamanCase) -> dict[str, object]: +def _raman_native_payload(raman_case: RamanCase) -> _core.Problem: problem, run_request, grid = raman_case backend = NativeBackend(problem, run_request, grid=grid) try: - return _native_problem( # pyright: ignore[reportPrivateUsage] + return _lower( problem, run_request, - backend._terms, # pyright: ignore[reportPrivateUsage] + _LowerPlan(terms=backend._terms), # pyright: ignore[reportPrivateUsage] ) finally: backend.close() @@ -665,25 +666,77 @@ def test_native_raman_rows_refuse_hidden_array_conversions( ) -> None: """The private seam accepts exact arrays and never copies a near miss.""" payload = _raman_native_payload(raman_case) - rows = cast(np.ndarray, payload["raman_undamped"]) - if spoiling == "dtype": - payload["raman_undamped"] = rows.astype(np.float32) - elif spoiling == "rank": - payload["raman_undamped"] = rows.ravel() - else: - payload["raman_undamped"] = np.zeros( - (rows.shape[0], 2 * rows.shape[1]), dtype=np.float64 - )[:, ::2] + raman = payload.raman + rows = raman.undamped.reshape(raman.n_terms, -1) + + def _spoil() -> None: + if spoiling == "dtype": + raman.undamped = rows.astype(np.float32) # pyright: ignore[reportAttributeAccessIssue] + elif spoiling == "rank": + raman.undamped = rows.ravel() + else: + raman.undamped = np.zeros( + (rows.shape[0], 2 * rows.shape[1]), dtype=np.float64 + )[:, ::2] + _core.State(payload) + with pytest.raises((TypeError, ValueError)): + _spoil() + + +@pytest.mark.parametrize("spoiling", ["undamped", "tensor_weights"]) +def test_native_raman_tables_refuse_a_transposed_layout( + raman_case: RamanCase, spoiling: str +) -> None: + """A table with the right element count but its axes swapped is refused by name.""" + payload = _raman_native_payload(raman_case) + raman = payload.raman + if spoiling == "undamped": + raman.undamped = raman.undamped.reshape(-1, raman.n_terms) + else: + raman.tensor_weights = raman.tensor_weights.reshape(2, raman.n_terms) + with pytest.raises(ValueError, match=f"raman_{spoiling} must have shape"): + _core.State(payload) + + +@pytest.mark.parametrize("spoiling", ["flat", "rank_four", "count"]) +def test_native_fixed_terms_refuse_a_wrong_layout( + phase_a_case: PhaseACase, spoiling: str +) -> None: + """The fixed terms are (n_phi_terms, ...) arrays of rank two or three.""" + problem, run_request, grid = phase_a_case + backend = NativeBackend(problem, run_request, grid=grid) + try: + terms = backend._terms # pyright: ignore[reportPrivateUsage] + payload = _lower(problem, run_request, _LowerPlan(terms=terms)) + finally: + backend.close() + assert terms.phi is not None + # As the adapter lays them out: (n_phi_terms, rows, n_omega). + phi = np.ascontiguousarray(terms.phi).reshape( + terms.phi.shape[0], -1, problem.shape[-1] + ) + if spoiling == "flat": + payload.linear.phi_terms = phi.reshape(-1) + message = "rank-two or rank-three" + elif spoiling == "rank_four": + payload.linear.phi_terms = phi.reshape(1, *phi.shape) + message = "rank-two or rank-three" + else: + # The same elements, the leading axis no longer the term count. + assert phi.shape[::-1][0] != payload.linear.n_phi_terms + payload.linear.phi_terms = phi.reshape(phi.shape[::-1]) + message = "phi_terms must hold n_phi_terms rows" + with pytest.raises(ValueError, match=message): _core.State(payload) def test_native_raman_rows_accept_read_only_input(raman_case: RamanCase) -> None: """Construction reads Raman rows but does not require write access to them.""" payload = _raman_native_payload(raman_case) - rows = np.array(cast(np.ndarray, payload["raman_undamped"]), copy=True) + rows = payload.raman.undamped.reshape(payload.raman.n_terms, -1) rows.setflags(write=False) - payload["raman_undamped"] = rows + payload.raman.undamped = rows state = _core.State(payload) state.close() @@ -928,12 +981,12 @@ def test_the_plasma_stash_is_off_under_modified_noise( ) problem = simulation.translate() assert problem.responses.plasma_channels - payload = _native_problem( # pyright: ignore[reportPrivateUsage] + payload = _lower( problem, simulation._run_request(), # pyright: ignore[reportPrivateUsage] - terms(problem), + _LowerPlan(terms=terms(problem)), ) - assert payload["stash_plasma"] is stashed + assert payload.plasma.stash is stashed def test_the_plasma_stash_is_off_under_the_split_step_absorber() -> None: @@ -942,7 +995,7 @@ def test_the_plasma_stash_is_off_under_the_split_step_absorber() -> None: The last stage's input is then not the committed field, so the plasma statistics are scanned afresh (the stash needs a first-same-as-last step). """ - payloads = {} + payloads: dict[str, _core.Problem] = {} for scheme in ("nonlinear", "split_step"): simulation = Simulation.from_capillary( 125e-6, @@ -968,11 +1021,11 @@ def test_the_plasma_stash_is_off_under_the_split_step_absorber() -> None: simulation.settings, absorber_scheme=scheme ) problem = simulation.translate() - payloads[scheme] = _native_problem( # pyright: ignore[reportPrivateUsage] + payloads[scheme] = _lower( problem, simulation._run_request(), # pyright: ignore[reportPrivateUsage] - terms(problem), + _LowerPlan(terms=terms(problem)), ) - assert payloads["nonlinear"]["stash_plasma"] is True - assert payloads["split_step"]["split_step_absorber"] is True - assert payloads["split_step"]["stash_plasma"] is False + assert payloads["nonlinear"].plasma.stash is True + assert payloads["split_step"].absorber.split_step is True + assert payloads["split_step"].plasma.stash is False diff --git a/native/tests/test_band_pathological.py b/native/tests/test_band_pathological.py index d996254a9..dd635c4cc 100644 --- a/native/tests/test_band_pathological.py +++ b/native/tests/test_band_pathological.py @@ -22,7 +22,7 @@ first knot to zero. - **An index past either end of the table.** A table whose stored ``log_field_min`` is moved, so that every live sample's index is negative, or past the last interval, while - its fields stay within the table's range: ``general.cpp``'s two clamps. + its fields stay within the table's range: ``src/general/kernels.hpp``'s two clamps. **Where this is weaker than the original.** A 1e-170 sample beside samples of order one is out of reach, because a transform would round it away: the tiny sample here diff --git a/native/tests/test_band_plasma.py b/native/tests/test_band_plasma.py index d9da0efcc..0a72b278e 100644 --- a/native/tests/test_band_plasma.py +++ b/native/tests/test_band_plasma.py @@ -62,7 +62,8 @@ from plasma_bounds import assert_regression_guard, plasma_stage_bound, rate_roundoff from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] - corrupt native payloads + _lower, # pyright: ignore[reportPrivateUsage] - corrupt native payloads + _LowerPlan, # pyright: ignore[reportPrivateUsage] - corrupt native payloads ) INFO = solprop_native.build_info() @@ -756,28 +757,50 @@ def test_native_taps_are_validated(damage: str) -> None: grid, problem, run_request = modal("mode_averaged", "band") backend = NativeBackend(problem, run_request, grid=grid) try: - payload = _native_problem( + payload = _lower( problem, run_request, - backend._terms, # pyright: ignore[reportPrivateUsage] - fixed payload + _LowerPlan(terms=backend._terms), # pyright: ignore[reportPrivateUsage] - fixed payload ) finally: backend.close() - taps = np.array(payload["plasma_taps"], dtype=float, copy=True) + plasma = payload.plasma + taps = plasma.taps if damage == "too_many": - payload["plasma_taps"] = np.ones(10) + plasma.taps = np.ones(10) elif damage == "nonfinite": taps[1] = np.nan - payload["plasma_taps"] = taps + plasma.taps = taps elif damage == "empty": - payload["plasma_taps"] = np.empty(0) + plasma.taps = np.empty(0) else: # The trapezoid is the one identity tap and nothing else. - payload["plasma_band"] = False + plasma.band = False with pytest.raises(ValueError, match=r"taps|trapezoid|band"): _core.State(payload) +def test_native_rate_table_refuses_a_transposed_layout() -> None: + """(n_intervals, 4) holds the right elements but is refused: (4, n_intervals).""" + grid, problem, run_request = modal("mode_averaged", "band") + backend = NativeBackend(problem, run_request, grid=grid) + try: + payload = _lower( + problem, + run_request, + _LowerPlan(terms=backend._terms), # pyright: ignore[reportPrivateUsage] - fixed payload + ) + finally: + backend.close() + plasma = payload.plasma + assert plasma.rate_intervals != 4 + plasma.rate_coefficients = plasma.rate_coefficients.reshape( + plasma.rate_intervals, 4 + ) + with pytest.raises(ValueError, match=r"rate_coefficients must have shape"): + _core.State(payload) + + def test_the_diagnostics_need_a_field_plasma_column(phase_a_case: PhaseACase) -> None: problem, run_request, grid = phase_a_case backend = NativeBackend(problem, run_request, grid=grid) diff --git a/native/tests/test_coupled_flow.py b/native/tests/test_coupled_flow.py index a917518e0..73209941b 100644 --- a/native/tests/test_coupled_flow.py +++ b/native/tests/test_coupled_flow.py @@ -23,7 +23,7 @@ import math import warnings from collections.abc import Iterator -from typing import cast +from typing import Any, cast import numpy as np import numpy.typing as npt @@ -47,10 +47,10 @@ from solprop.translate.problem import PropagationProblem from solprop.translate.run_request import RunRequest -from solprop_native import NativeBackend +from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _flow_payload, # pyright: ignore[reportPrivateUsage] _host_flow, # pyright: ignore[reportPrivateUsage] + _lower_flow, # pyright: ignore[reportPrivateUsage] ) from solprop_native.group import refusal as group_refusal @@ -58,6 +58,13 @@ type RealArray = npt.NDArray[np.float64] +def _flow_rows(flow: Any) -> _core.Problem.Flow: + """The native problem's coupled flow group as the adapter fills it.""" + native = _core.Problem() + _lower_flow(native, flow) + return native.flow + + def _norms(stack: ComplexArray | RealArray, *, spectral: bool = True) -> RealArray: """Return the 2-norm (or Frobenius norm) of each matrix of a stack.""" if spectral: @@ -460,9 +467,9 @@ def test_the_residual_s_data_are_zero_exactly_outside_the_requested_band() -> No scored = problem.grids.inband_mask[flow.active] assert np.any(~scored) assert np.any(scored) - payload = _flow_payload(flow) - frozen = cast("ComplexArray", payload["flow_frozen"]) - terms = cast("ComplexArray", payload["flow_residual_terms"]) + payload = _flow_rows(flow) + frozen = payload.frozen.reshape(flow.frozen.shape) + terms = payload.residual_terms.reshape(flow.residual_terms.shape) assert np.all(frozen[:, ~scored] == 0.0) assert np.all(terms[:, ~scored] == 0.0) assert np.array_equal(frozen[:, scored], flow.frozen[:, scored]) @@ -475,8 +482,8 @@ def test_the_residual_s_data_are_zero_exactly_outside_the_requested_band() -> No ) full = _host_flow(everywhere) assert full is not None - unmasked = _flow_payload(full) - assert np.array_equal(cast("ComplexArray", unmasked["flow_frozen"]), flow.frozen) + unmasked = _flow_rows(full) + assert np.array_equal(unmasked.frozen.reshape(flow.frozen.shape), flow.frozen) assert np.array_equal( - cast("ComplexArray", unmasked["flow_residual_terms"]), flow.residual_terms + unmasked.residual_terms.reshape(flow.residual_terms.shape), flow.residual_terms ) diff --git a/native/tests/test_dispersive.py b/native/tests/test_dispersive.py index ead61de74..c0166f76e 100644 --- a/native/tests/test_dispersive.py +++ b/native/tests/test_dispersive.py @@ -4,7 +4,7 @@ """The adapter of solprop SEP 6's dispersive response. -The native response executes the layout the adapter packs (``_dispersive_payload``): +The native response executes the layout the adapter packs (``_lower_dispersive``): these tests hold the layout to the leaf (the general modal route's regions among it), the field family to the sector and one evaluation to solprop's reference. The full parity ladders are solprop's @@ -41,12 +41,21 @@ from solprop.translate.responses import dispersive_terms from solprop.translate.run_request import RunRequest -from solprop_native import NativeBackend +from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _dispersive_payload, # pyright: ignore[reportPrivateUsage] - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _lower_dispersive, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] ) + +def _dispersive_payload(problem: PropagationProblem) -> _core.Problem.Dispersive: + """The native problem's dispersive group as the adapter fills it.""" + native = _core.Problem() + _lower_dispersive(native, problem) + return native.dispersive + + EPS = float(np.finfo(float).eps) LENGTH = 2e-3 POLICIES = {"symmetric": "symmetric-kerr", "polynomial": "gas-polynomial"} @@ -120,25 +129,28 @@ def test_a_symmetric_mixture_is_one_product_under_its_filter() -> None: leaf = problem.responses.dispersive_kerr assert leaf is not None assert leaf.n_channels == 2 - payload = cast(dict[str, Any], _dispersive_payload(problem)) + payload = _dispersive_payload(problem) bins = leaf.positive.size - assert payload["dispersive_symmetric"] is True - assert payload["dispersive_form"] == 1 - assert payload["dispersive_inputs"].shape == (1, bins) + assert payload.symmetric is True + assert payload.form == 1 + assert payload.inputs.shape == (bins,) # Copies, indices, masks and power-of-two scalings of a mask are exact; the # quartics are three products of the filters', within gamma(3) each way. - np.testing.assert_array_equal(payload["dispersive_inputs"][0], leaf.filters[0]) - np.testing.assert_array_equal(payload["dispersive_term_rows"], [0]) - np.testing.assert_array_equal(payload["dispersive_term_channels"], [3]) - np.testing.assert_array_equal(payload["dispersive_group_offsets"], [0, 2]) - np.testing.assert_array_equal(payload["dispersive_group_members"], [0, 1]) + np.testing.assert_array_equal(payload.inputs, leaf.filters[0]) + np.testing.assert_array_equal(payload.term_rows, [0]) + np.testing.assert_array_equal(payload.term_channels, [3]) + np.testing.assert_array_equal(payload.group_offsets, [0, 2]) + np.testing.assert_array_equal(payload.group_members, [0, 1]) squares = leaf.filters * leaf.filters np.testing.assert_allclose( - payload["dispersive_quartics"], squares * squares, rtol=2 * gamma(3), atol=0.0 + payload.quartics.reshape(squares.shape), + squares * squares, + rtol=2 * gamma(3), + atol=0.0, ) positive = leaf.positive.astype(float) - np.testing.assert_array_equal(payload["dispersive_output_scale"], positive / 4.0) - np.testing.assert_array_equal(payload["dispersive_input_scale"], np.ones(bins)) + np.testing.assert_array_equal(payload.output_scale, positive / 4.0) + np.testing.assert_array_equal(payload.input_scale, np.ones(bins)) def test_the_polynomial_groups_species_of_one_leg_bound() -> None: @@ -154,34 +166,34 @@ def test_the_polynomial_groups_species_of_one_leg_bound() -> None: assert leaf is not None assert leaf.groups() == ((0, 1),) assert leaf.outputs is not None - payload = cast(dict[str, Any], _dispersive_payload(problem)) + payload = _dispersive_payload(problem) positive = leaf.positive.astype(float) # The field grid's conserving sector: the analytic family, Pi_+ / 8 on the output. - assert payload["dispersive_form"] == 1 + assert payload.form == 1 # The masks, indices and power-of-two scalings are exact; a product of filters # or of a multiplier and the mask is within gamma(2) each way. - np.testing.assert_array_equal(payload["dispersive_input_scale"], positive) + np.testing.assert_array_equal(payload.input_scale, positive) np.testing.assert_allclose( - payload["dispersive_inputs"], + payload.inputs.reshape(2, -1), np.stack([leaf.filters[0], leaf.filters[0] * leaf.filters[0]]) * leaf.positive, rtol=2 * gamma(2), atol=0.0, ) - np.testing.assert_array_equal(payload["dispersive_term_rows"], [0, 1, 0, 1, 2]) - np.testing.assert_array_equal(payload["dispersive_term_channels"], [0, 0, 1, 1, 2]) + np.testing.assert_array_equal(payload.term_rows, [0, 1, 0, 1, 2]) + np.testing.assert_array_equal(payload.term_channels, [0, 0, 1, 1, 2]) np.testing.assert_allclose( - payload["dispersive_term_multipliers"][2], + payload.term_multipliers.reshape(-1, positive.size)[2], leaf.outputs[1, 0] * positive / 8.0, rtol=2 * gamma(2), atol=0.0, ) np.testing.assert_array_equal( - payload["dispersive_term_multipliers"][4], positive / 8.0 + payload.term_multipliers.reshape(-1, positive.size)[4], positive / 8.0 ) np.testing.assert_array_equal( - payload["dispersive_pair_constants"], np.asarray(leaf.products) + payload.pair_constants.reshape(-1, 2), np.asarray(leaf.products) ) - assert payload["dispersive_quartics"].size == 0 + assert payload.quartics.size == 0 @pytest.mark.parametrize( @@ -200,17 +212,19 @@ def test_the_sector_chooses_the_field_family( problem, run_request = capillary("gas-polynomial", envelope=envelope, thg=thg) backend = NativeBackend(problem, run_request) try: - payload = _native_problem(problem, run_request, backend._terms) # pyright: ignore[reportPrivateUsage] + payload = _lower(problem, run_request, _LowerPlan(terms=backend._terms)) # pyright: ignore[reportPrivateUsage] finally: backend.close() - assert payload["family"] == family - assert payload["dispersive_form"] == form - assert payload["nonlinear"] is False + assert payload.dims.family == getattr(_core.FieldFamily, family) + assert payload.dispersive.form == form + assert payload.kerr.nonlinear is False def test_a_problem_without_the_leaf_packs_nothing() -> None: problem, _ = capillary("carrier-kerr", envelope=False, thg=True) - assert _dispersive_payload(problem) == {"dispersive": False} + payload = _dispersive_payload(problem) + assert payload.enabled is False + assert payload.inputs.size == 0 def general_leaf(evaluator: str) -> tuple[Any, FieldGrid]: @@ -247,23 +261,19 @@ def test_the_general_route_lays_out_its_regions(evaluator: str) -> None: ), grid_kind="field", ) - payload = cast( - dict[str, Any], _dispersive_payload(cast(PropagationProblem, problem)) - ) - np.testing.assert_array_equal(payload["dispersive_node_region"], region_index) - assert payload["dispersive_form"] == 1 + payload = _dispersive_payload(cast(PropagationProblem, problem)) + np.testing.assert_array_equal(payload.node_region, region_index) + assert payload.form == 1 if evaluator == "symmetric": assert leaf.groups() == ((0,), (1, 2)) - np.testing.assert_array_equal(payload["dispersive_group_keys"], [3, 5]) - np.testing.assert_array_equal(payload["dispersive_term_rows"], [0, 1]) - np.testing.assert_array_equal(payload["dispersive_term_channels"], [0, 4]) + np.testing.assert_array_equal(payload.group_keys, [3, 5]) + np.testing.assert_array_equal(payload.term_rows, [0, 1]) + np.testing.assert_array_equal(payload.term_channels, [0, 4]) return assert leaf.groups() == ((0,), (1,), (2,)) - np.testing.assert_array_equal(payload["dispersive_group_keys"], [3, 5, 5]) + np.testing.assert_array_equal(payload.group_keys, [3, 5, 5]) # Two cubes, by region in the order the groups name them; then each group's pair. - np.testing.assert_array_equal( - payload["dispersive_term_rows"], [0, 2, 3, 1, 4, 5, 1, 6, 7] - ) + np.testing.assert_array_equal(payload.term_rows, [0, 2, 3, 1, 4, 5, 1, 6, 7]) def test_a_trial_without_the_densities_is_refused() -> None: diff --git a/native/tests/test_envelope_plasma.py b/native/tests/test_envelope_plasma.py index 1e73c1a85..ed3b9bfd2 100644 --- a/native/tests/test_envelope_plasma.py +++ b/native/tests/test_envelope_plasma.py @@ -15,7 +15,6 @@ from typing import Any, cast import numpy as np -import numpy.typing as npt import pytest from legacy_geometry import LEGACY_GRID from session_runs import ( @@ -49,7 +48,8 @@ from solprop_native import NativeBackend, _core, build_info from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] - validate the native boundary + _lower, # pyright: ignore[reportPrivateUsage] - validate the native boundary + _LowerPlan, # pyright: ignore[reportPrivateUsage] - validate the native boundary ) # The FFT collar extends beyond the neutral-gas index fit. These are current @@ -371,42 +371,55 @@ def test_envelope_currents_steps_statistics_and_repeatability( @pytest.mark.parametrize( "corruption", - ["field", "shape", "nan", "weight", "inactive", "sign", "thg", "stash"], + ["field", "shape", "layout", "nan", "weight", "inactive", "sign", "thg", "stash"], ) def test_envelope_native_payload_rejects_inconsistent_inputs(corruption: str) -> None: problem, run_request, _ = envelope_case("modal", 1, mixture=False) - payload = _native_problem( + payload = _lower( problem, run_request, - terms(problem), - node_chunk=envelope_options("modal").chunk, + _LowerPlan(terms=terms(problem), node_chunk=envelope_options("modal").chunk), ) - if corruption in ("field", "thg", "stash"): - key, value = { - "field": ("family", "field_real"), - "thg": ("harmonic_factor", 1.0), - "stash": ("stash_plasma", True), - }[corruption] - payload[key] = value - elif corruption in ("shape", "nan", "weight"): - key = ( - "envelope_rule_nodes" if corruption != "weight" else "envelope_rule_weights" - ) - array = cast(npt.NDArray[np.float64], payload[key]).copy() - if corruption == "shape": - payload[key] = array.ravel() + plasma = payload.plasma + shape = (plasma.n_channels, 4, 512) + + def _spoil() -> None: + if corruption == "field": + payload.dims.family = _core.FieldFamily.field_real + elif corruption == "thg": + payload.kerr.harmonic_factor = 1.0 + elif corruption == "stash": + plasma.stash = True + elif corruption in ("shape", "layout", "nan", "weight"): + weight = corruption == "weight" + rule = ( + plasma.envelope_rule_weights if weight else plasma.envelope_rule_nodes + ) + if corruption == "shape": + corrupted = rule + elif corruption == "layout": + # The rule's elements, its (n_channels, 4, 512) axes reversed. + corrupted = rule.reshape(shape[::-1]) + else: + corrupted = rule.reshape(shape) + corrupted.flat[0] = np.nan if corruption == "nan" else 0.0 + if weight: + plasma.envelope_rule_weights = corrupted + else: + plasma.envelope_rule_nodes = corrupted else: - array.flat[0] = np.nan if corruption == "nan" else 0.0 - payload[key] = array - else: - array = cast(npt.NDArray[np.float64], payload["envelope_drude_factor"]).copy() - mask = problem.grids.active_mask & (problem.grids.omega > 0) - index = int(np.flatnonzero(mask if corruption == "sign" else ~mask)[0]) - array[index] = 1.0 - payload["envelope_drude_factor"] = array - with pytest.raises(ValueError, match="envelope"): + array = plasma.envelope_drude_factor + mask = problem.grids.active_mask & (problem.grids.omega > 0) + index = int(np.flatnonzero(mask if corruption == "sign" else ~mask)[0]) + array[index] = 1.0 + plasma.envelope_drude_factor = array _core.State(payload) + # A reordered rule keeps its elements: only its layout can refuse it. + match = "envelope rules require" if corruption == "layout" else "envelope" + with pytest.raises(ValueError, match=match): + _spoil() + @pytest.mark.parametrize( ("geometry", "n_pol"), [("modal", 1), ("radial", 2), ("free", 2)] @@ -424,24 +437,23 @@ def test_envelope_angular_failure_is_a_solprop_convergence_error( # No statistics: the failure is the first step's, inside the native march, as the # driver's was (the launch plane's populations would meet the bad rule first). run_request = run_request.for_run(stats=StatsSpec()) - payload = _native_problem( + payload = _lower( problem, run_request, - terms(problem), - node_chunk=envelope_options(geometry).chunk, + _LowerPlan(terms=terms(problem), node_chunk=envelope_options(geometry).chunk), ) - nodes = cast(npt.NDArray[np.float64], payload["envelope_rule_nodes"]).copy() + nodes = payload.plasma.envelope_rule_nodes.reshape(-1, 4, 512) # Sorted interior nodes and normalized positive weights still satisfy native # shape/domain checks. Alternating incompatible rules must fail refinement. for level in (1, 3): order = 64 << level nodes[:, level, :order] = 0.5 * (nodes[:, level, :order] + 1) - payload["envelope_rule_nodes"] = nodes + payload.plasma.envelope_rule_nodes = nodes - def corrupt_payload(*args: object, **keywords: object) -> dict[str, object]: + def corrupt_payload(*args: object, **keywords: object) -> _core.Problem: return payload - monkeypatch.setattr(adapter, "_native_problem", corrupt_payload) + monkeypatch.setattr(adapter, "_lower", corrupt_payload) owner = NativeBackend( problem, run_request, grid=grid, options=envelope_options(geometry) ) diff --git a/native/tests/test_envelope_statistics_refusal.py b/native/tests/test_envelope_statistics_refusal.py index 0ee929ddc..9116b28a5 100644 --- a/native/tests/test_envelope_statistics_refusal.py +++ b/native/tests/test_envelope_statistics_refusal.py @@ -11,10 +11,7 @@ from __future__ import annotations -from typing import cast - import numpy as np -import numpy.typing as npt import pytest from session_runs import drain, run from solprop.backends.events import Outcome @@ -24,9 +21,10 @@ from test_envelope_plasma import envelope_case, envelope_options, terms import solprop_native.backend as adapter -from solprop_native import NativeBackend +from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] ) SPEC = StatsSpec(names=("electron_density", "peak_ionisation_rate")) @@ -40,23 +38,22 @@ def test_the_statistics_refuse_and_a_quiet_field_recovers( monkeypatch: pytest.MonkeyPatch, geometry: str ) -> None: problem, run_request, grid = envelope_case(geometry, 1, mixture=False) - payload = _native_problem( + payload = _lower( problem, run_request, - terms(problem), - node_chunk=envelope_options(geometry).chunk, + _LowerPlan(terms=terms(problem), node_chunk=envelope_options(geometry).chunk), ) - nodes = cast(npt.NDArray[np.float64], payload["envelope_rule_nodes"]).copy() + nodes = payload.plasma.envelope_rule_nodes.reshape(-1, 4, 512) # As the source's test: alternating incompatible rules must fail refinement. for level in (1, 3): order = 64 << level nodes[:, level, :order] = 0.5 * (nodes[:, level, :order] + 1) - payload["envelope_rule_nodes"] = nodes + payload.plasma.envelope_rule_nodes = nodes - def corrupt_payload(*args: object, **keywords: object) -> dict[str, object]: + def corrupt_payload(*args: object, **keywords: object) -> _core.Problem: return payload - monkeypatch.setattr(adapter, "_native_problem", corrupt_payload) + monkeypatch.setattr(adapter, "_lower", corrupt_payload) # The launch plane's statistics meet the bad rule before any step. run_request = run_request.for_run(stats=SPEC) owner = NativeBackend( diff --git a/native/tests/test_execution.py b/native/tests/test_execution.py index 474661da8..705ea3243 100644 --- a/native/tests/test_execution.py +++ b/native/tests/test_execution.py @@ -24,7 +24,8 @@ from solprop_native._wait import SPIN_MILLISECONDS from solprop_native.backend import ( NativeBackend, - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] ) # The constants of core.hpp before SEP 46 moved them here (solprop PR #576), Linux and @@ -175,27 +176,59 @@ def test_records_round_trip_and_name_their_differences() -> None: assert table.differences(table) == {} -def _payload(case: PhaseACase) -> dict[str, object]: +_MISSING = object() +"""No record: the state is built without one.""" + + +def _payload(case: PhaseACase) -> _core.Problem: problem, run_request, grid = case backend = NativeBackend(problem, run_request, grid=grid) try: - return _native_problem(problem, run_request, backend._terms) # pyright: ignore[reportPrivateUsage] + return _lower(problem, run_request, _LowerPlan(terms=backend._terms)) # pyright: ignore[reportPrivateUsage] finally: backend.close() +def _with_record(payload: _core.Problem, record: object = _MISSING) -> _core.Problem: + """A copy of `payload` with its CPU execution record replaced, or without one.""" + changed = _core.Problem() + for group in _GROUPS: + setattr(changed, group, getattr(payload, group)) + if record is not _MISSING: + changed.cpu_execution = record # pyright: ignore[reportAttributeAccessIssue] + return changed + + +_GROUPS = ( + "dims", + "grid", + "spatial", + "flux", + "stepper", + "linear", + "absorber", + "transform", + "kerr", + "plasma", + "raman", + "general", + "flow", + "dispersive", +) + + def test_a_state_reads_back_the_record_it_was_built_with( phase_a_case: PhaseACase, ) -> None: payload = _payload(phase_a_case) - assert payload["cpu_execution"] == native.BUILTIN_EXECUTION.native() + assert payload.cpu_execution == native.BUILTIN_EXECUTION.native() serial = dataclasses.replace( native.BUILTIN_EXECUTION, **{f"{name}_threshold": 1 << 40 for name in ("stage", "copy", "adams")}, **{f"{name}_full": 1 << 40 for name in ("stage", "copy", "adams")}, ) first = _core.State(payload) - second = _core.State(payload | {"cpu_execution": serial.native()}) + second = _core.State(_with_record(payload, serial.native())) try: # Two live workspaces hold their own records (solprop SEP 46 section 7 item 7). assert first.cpu_execution == native.BUILTIN_EXECUTION.native() @@ -223,13 +256,16 @@ def test_a_state_refuses_a_missing_or_malformed_record( phase_a_case: PhaseACase, change: dict[str, object] | str, message: str ) -> None: payload = _payload(phase_a_case) - if change == "missing": - del payload["cpu_execution"] - else: - assert isinstance(change, dict) - payload |= change + + def _spoil() -> None: + if change == "missing": + _core.State(_with_record(payload)) + else: + assert isinstance(change, dict) + _core.State(_with_record(payload, change["cpu_execution"])) + with pytest.raises(ValueError, match=message): - _core.State(payload) + _spoil() def test_a_state_refuses_a_record_the_native_core_refuses( @@ -238,12 +274,12 @@ def test_a_state_refuses_a_record_the_native_core_refuses( payload = _payload(phase_a_case) record = native.BUILTIN_EXECUTION.native() with pytest.raises(ValueError, match="positive threshold"): - _core.State(payload | {"cpu_execution": record | {"dense_full": 1}}) + _core.State(_with_record(payload, record | {"dense_full": 1})) with pytest.raises(ValueError, match="positive response bounds"): - _core.State(payload | {"cpu_execution": record | {"medium_threads": 0}}) + _core.State(_with_record(payload, record | {"medium_threads": 0})) renamed = {k: v for k, v in record.items() if k != "adams_full"} with pytest.raises(ValueError, match="misses a field: adams_full"): - _core.State(payload | {"cpu_execution": renamed | {"adams_fulll": 1}}) + _core.State(_with_record(payload, renamed | {"adams_fulll": 1})) def test_a_backend_given_a_record_builds_with_it_and_records_it( diff --git a/native/tests/test_free.py b/native/tests/test_free.py index a05a10f79..ea8fd6535 100644 --- a/native/tests/test_free.py +++ b/native/tests/test_free.py @@ -6,7 +6,7 @@ from __future__ import annotations -from typing import cast +from typing import Any import numpy as np import pytest @@ -26,12 +26,42 @@ from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] ) +_GROUP = { + "n_x": "spatial", + "n_y": "spatial", + "n_p": "spatial", + "n_omega": "dims", + "n_to": "dims", + "node_chunk": "dims", + "n_pol_state": "dims", + "projection_matrix": "transform", + "projection_coarse_matrix": "transform", +} + + +def _path(name: str) -> tuple[str, str]: + """The native problem's group and field of a flat payload name.""" + if name.startswith("spatial_"): + return "spatial", name.removeprefix("spatial_") + return _GROUP[name], name + + +def _field(payload: _core.Problem, name: str) -> Any: + group, field = _path(name) + return getattr(getattr(payload, group), field) + + +def _set(payload: _core.Problem, name: str, value: object) -> None: + group, field = _path(name) + setattr(getattr(payload, group), field, value) + @pytest.fixture -def free_payload(phase_a_case: PhaseACase) -> dict[str, object]: +def free_payload(phase_a_case: PhaseACase) -> _core.Problem: _, base_request, grid = phase_a_case space = FreeGrid(1e-3, 4, 1.5e-3, 6, collar_fraction=0) problem, run_request = assemble( @@ -50,11 +80,13 @@ def free_payload(phase_a_case: PhaseACase) -> dict[str, object]: problem, run_request, grid=grid, options=ExecutionOptions(node_chunk=5) ) try: - return _native_problem( + return _lower( problem, run_request, - backend._terms, # pyright: ignore[reportPrivateUsage] - node_chunk=backend.node_chunk, + _LowerPlan( + terms=backend._terms, # pyright: ignore[reportPrivateUsage] + node_chunk=backend.node_chunk, + ), ) finally: backend.close() @@ -80,19 +112,23 @@ def free_payload(phase_a_case: PhaseACase) -> dict[str, object]: ) @pytest.mark.parametrize("spoiling", ["length", "nonfinite", "dtype"]) def test_free_native_vectors_reject_invalid_data( - free_payload: dict[str, object], name: str, spoiling: str + free_payload: _core.Problem, name: str, spoiling: str ) -> None: - values = np.array(cast(np.ndarray, free_payload[name]), copy=True) + values = np.array(_field(free_payload, name), copy=True) if spoiling == "length": values = values[:-1] elif spoiling == "nonfinite": values[0] = np.nan else: values = values.astype(np.complex64 if np.iscomplexobj(values) else np.float32) - free_payload[name] = values - with pytest.raises((TypeError, ValueError)): + + def _spoil() -> None: + _set(free_payload, name, values) _core.State(free_payload) + with pytest.raises((TypeError, ValueError)): + _spoil() + @pytest.mark.parametrize( ("name", "value"), @@ -106,33 +142,37 @@ def test_free_native_vectors_reject_invalid_data( ], ) def test_free_native_axes_and_clamp_are_checked( - free_payload: dict[str, object], name: str, value: float + free_payload: _core.Problem, name: str, value: float ) -> None: - free_payload[name] = value - with pytest.raises((TypeError, ValueError)): + def _spoil() -> None: + _set(free_payload, name, value) _core.State(free_payload) + with pytest.raises((TypeError, ValueError)): + _spoil() + def test_the_far_field_payload_belongs_to_free_space( - free_payload: dict[str, object], planar_payload: dict[str, object] + free_payload: _core.Problem, planar_payload: _core.Problem ) -> None: """solprop SEP 23: the axes and the slot count are free space's alone.""" - assert free_payload["spatial_far_field_slots"] == 0 - assert cast(np.ndarray, free_payload["spatial_k_x"]).shape == (4,) - assert cast(np.ndarray, free_payload["spatial_k_y"]).shape == (6,) + assert free_payload.spatial.far_field_slots == 0 + assert free_payload.spatial.k_x.shape == (4,) + assert free_payload.spatial.k_y.shape == (6,) + planar_payload.spatial.far_field_slots = 3 with pytest.raises(ValueError, match="belong to free space"): - _core.State(planar_payload | {"spatial_far_field_slots": 3}) + _core.State(planar_payload) def test_free_native_ledger_retains_one_transverse_volume_and_chunked_time( - free_payload: dict[str, object], + free_payload: _core.Problem, ) -> None: state = _core.State(free_payload) try: ledger = state.memory_ledger() omega, time = ( - int(cast(int, free_payload["n_omega"])), - int(cast(int, free_payload["n_to"])), + free_payload.dims.n_omega, + free_payload.dims.n_to, ) assert ledger["transverse_scratch"] == 24 * omega * 16 assert ledger["modal_time"] == 5 * time * 16 @@ -141,7 +181,7 @@ def test_free_native_ledger_retains_one_transverse_volume_and_chunked_time( assert ledger["fixed_phi_terms"] == ledger["fixed_prefactor_terms"] == 0 assert ledger["transform_matrices"] == 0 expected = sum( - cast(np.ndarray, free_payload[name]).nbytes + _field(free_payload, name).nbytes for name in ( "spatial_k_squared", "spatial_k_squared_low", @@ -163,12 +203,12 @@ def test_free_native_ledger_retains_one_transverse_volume_and_chunked_time( @pytest.mark.parametrize("slots", [3, 7, 8]) def test_adams_dense_storage_is_optional_and_reallocation_is_idempotent( - free_payload: dict[str, object], slots: int + free_payload: _core.Problem, slots: int ) -> None: state = _core.State(free_payload) try: field_bytes = state.memory_ledger()["state"] - omega = int(cast(int, free_payload["n_omega"])) + omega = free_payload.dims.n_omega initial = np.zeros(field_bytes // 16, dtype=np.complex128) state.set_initial(initial, np.empty(0, dtype=np.complex128)) for dense in (False, True, True, False): @@ -192,7 +232,7 @@ def test_adams_dense_storage_is_optional_and_reallocation_is_idempotent( def test_adams_rows_cross_to_the_host_and_back_exactly( - free_payload: dict[str, object], + free_payload: _core.Problem, ) -> None: # solprop SEP 29 G2: the resident history's rows are exported for a resume and # installed again; a copy, so exact, and refused by name, extent, dtype and order @@ -200,7 +240,7 @@ def test_adams_rows_cross_to_the_host_and_back_exactly( state = _core.State(free_payload) try: columns = state.memory_ledger()["state"] // 16 - omega = int(cast(int, free_payload["n_omega"])) + omega = free_payload.dims.n_omega state.set_initial( np.zeros(columns, dtype=np.complex128), np.empty(0, dtype=np.complex128) ) @@ -387,18 +427,20 @@ def test_the_hybrid_planar_kind_matches_the_reference(phase_a_case: PhaseACase) @pytest.fixture -def planar_payload(phase_a_case: PhaseACase) -> dict[str, object]: +def planar_payload(phase_a_case: PhaseACase) -> _core.Problem: _, base_request, grid = phase_a_case space, problem, run_request = _planar_problem(grid, base_request.stepper.controller) backend = NativeBackend( problem, run_request, grid=grid, options=_planar_options(space) ) try: - return _native_problem( + return _lower( problem, run_request, - backend._terms, # pyright: ignore[reportPrivateUsage] - node_chunk=backend.node_chunk, + _LowerPlan( + terms=backend._terms, # pyright: ignore[reportPrivateUsage] + node_chunk=backend.node_chunk, + ), ) finally: backend.close() @@ -418,32 +460,34 @@ def planar_payload(phase_a_case: PhaseACase) -> dict[str, object]: ], ) def test_the_planar_native_payload_is_checked( - planar_payload: dict[str, object], name: str, value: int | str + planar_payload: _core.Problem, name: str, value: int | str ) -> None: """The planar axes, whole-row chunks, scalar state and wall terms are validated.""" - kappa = np.array(cast(np.ndarray, planar_payload["spatial_kappa_sq"]), copy=True) - if value == "present": - planar_payload[name] = np.array( - cast(np.ndarray, planar_payload["projection_matrix"]), copy=True - ) - elif value == "short": - planar_payload[name] = kappa[:-1] - elif value == "nonfinite": - kappa[0] = np.nan - planar_payload[name] = kappa - else: - planar_payload[name] = value - with pytest.raises((TypeError, ValueError)): + kappa = planar_payload.spatial.kappa_sq + + def _spoil() -> None: + if value == "present": + _set(planar_payload, name, planar_payload.transform.projection_matrix) + elif value == "short": + _set(planar_payload, name, kappa[:-1]) + elif value == "nonfinite": + kappa[0] = np.nan + _set(planar_payload, name, kappa) + else: + _set(planar_payload, name, value) _core.State(planar_payload) + with pytest.raises((TypeError, ValueError)): + _spoil() + def test_a_free_payload_with_slab_channels_is_refused( - free_payload: dict[str, object], + free_payload: _core.Problem, ) -> None: """Slab channels turn a plane into the planar kind, whose shapes it lacks.""" - free_payload["n_p"] = 1 - free_payload["spatial_kappa_sq"] = np.zeros( - int(cast(int, free_payload["n_omega"])), dtype=np.complex128 + free_payload.spatial.n_p = 1 + free_payload.spatial.kappa_sq = np.zeros( + free_payload.dims.n_omega, dtype=np.complex128 ) with pytest.raises(ValueError, match="hybrid planar"): _core.State(free_payload) diff --git a/native/tests/test_mixtures.py b/native/tests/test_mixtures.py index 248e54583..aedda1934 100644 --- a/native/tests/test_mixtures.py +++ b/native/tests/test_mixtures.py @@ -22,7 +22,8 @@ from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] - corrupt native payloads + _lower, # pyright: ignore[reportPrivateUsage] - corrupt native payloads + _LowerPlan, # pyright: ignore[reportPrivateUsage] - corrupt native payloads ) @@ -96,14 +97,15 @@ def test_native_channel_payload_refusals( problem, run_request, grid = mixture_case backend = NativeBackend(problem, run_request, grid=grid) try: - payload = _native_problem( + payload = _lower( problem, run_request, - backend._terms, # pyright: ignore[reportPrivateUsage] - fixed payload + _LowerPlan(terms=backend._terms), # pyright: ignore[reportPrivateUsage] - fixed payload ) - parameters = np.array(payload["channel_parameters"], dtype=float, copy=True) + plasma = payload.plasma + parameters = plasma.channel_parameters.reshape(plasma.n_channels, 9) if damage == "zero_channels": - payload["n_channels"] = 0 + plasma.n_channels = 0 elif damage == "parameter_extent": parameters = parameters[:, :-1].copy() elif damage == "nonfinite": @@ -113,8 +115,8 @@ def test_native_channel_payload_refusals( elif damage == "interval": parameters[1, 5] = -1 else: - payload["channel_coefficients"] = np.empty(0) - payload["channel_parameters"] = parameters + plasma.channel_coefficients = np.empty(0) + plasma.channel_parameters = parameters with pytest.raises(ValueError, match=r"channel|Channel|plasma"): _core.State(payload) finally: diff --git a/native/tests/test_session_failure.py b/native/tests/test_session_failure.py index 3197d57f9..a3fd0e517 100644 --- a/native/tests/test_session_failure.py +++ b/native/tests/test_session_failure.py @@ -13,10 +13,7 @@ from __future__ import annotations -from typing import cast - import numpy as np -import numpy.typing as npt import pytest from solprop.backends.events import Terminal from solprop.backends.session import Start @@ -24,9 +21,10 @@ from test_envelope_plasma import envelope_case, envelope_options, terms import solprop_native.backend as adapter -from solprop_native import NativeBackend +from solprop_native import NativeBackend, _core from solprop_native.backend import ( - _native_problem, # pyright: ignore[reportPrivateUsage] + _lower, # pyright: ignore[reportPrivateUsage] + _LowerPlan, # pyright: ignore[reportPrivateUsage] ) @@ -35,23 +33,22 @@ def test_an_unresolved_launch_statistic_ends_the_run_as_a_convergence_error( monkeypatch: pytest.MonkeyPatch, geometry: str ) -> None: problem, run_request, grid = envelope_case(geometry, 1, mixture=False) - payload = _native_problem( + payload = _lower( problem, run_request, - terms(problem), - node_chunk=envelope_options(geometry).chunk, + _LowerPlan(terms=terms(problem), node_chunk=envelope_options(geometry).chunk), ) - nodes = cast(npt.NDArray[np.float64], payload["envelope_rule_nodes"]).copy() + nodes = payload.plasma.envelope_rule_nodes.reshape(-1, 4, 512) # Alternating incompatible rules fail refinement (`test_envelope_plasma.py`). for level in (1, 3): order = 64 << level nodes[:, level, :order] = 0.5 * (nodes[:, level, :order] + 1) - payload["envelope_rule_nodes"] = nodes + payload.plasma.envelope_rule_nodes = nodes - def corrupt_payload(*args: object, **keywords: object) -> dict[str, object]: + def corrupt_payload(*args: object, **keywords: object) -> _core.Problem: return payload - monkeypatch.setattr(adapter, "_native_problem", corrupt_payload) + monkeypatch.setattr(adapter, "_lower", corrupt_payload) owner = NativeBackend( problem, run_request, grid=grid, options=envelope_options(geometry) ) diff --git a/tests/backends/test_native_far_field_measure.py b/tests/backends/test_native_far_field_measure.py index e6e25f4d5..1c48af486 100644 --- a/tests/backends/test_native_far_field_measure.py +++ b/tests/backends/test_native_far_field_measure.py @@ -5,7 +5,7 @@ """A plane measure the far-field pass can't read is refused on ``native`` (SEP 43). ``numba_cpu``'s pass read the operator's root as the plane's, and any other measure was -refused. Native has that guard at the payload (``_flux_payload``), and no test held it. +refused. Native has that guard at the payload (``_lower_flux``), and no test held it. """ import copy diff --git a/tests/backends/test_native_floating_point_licence.py b/tests/backends/test_native_floating_point_licence.py index e91aaa99b..1b6cb1a85 100644 --- a/tests/backends/test_native_floating_point_licence.py +++ b/tests/backends/test_native_floating_point_licence.py @@ -42,9 +42,9 @@ PROTECTED = [ # The ionised fraction (Numba's fraction_at, envelope_population). - ("src/general.cpp", "decayed_fraction"), - ("src/general.cpp", "decayed_fractions"), - ("src/general.cpp", "envelope_fraction"), + ("src/general/kernels.hpp", "decayed_fraction"), + ("src/general/kernels.hpp", "decayed_fractions"), + ("src/general/kernels.hpp", "envelope_fraction"), # The small-argument phi-functions (multistep._phi, _weights). ("include/solprop_native/adams_phi.hpp", "adams_phi_fixed"), ("include/solprop_native/adams_phi.hpp", "adams_lanes_classify"), @@ -53,22 +53,22 @@ ("include/solprop_native/lagrange_basis.hpp", "basis_rows"), ("include/solprop_native/lagrange_basis.hpp", "trial_tables"), # The stable phase increments. - ("src/general.cpp", "phase_sum"), - ("src/general.cpp", "phase_block"), - ("src/general.cpp", "adams_phase_sum"), - ("src/general.cpp", "adams_phase_block"), - ("src/general.cpp", "adams_lanes_phase"), + ("src/general/kernels.hpp", "phase_sum"), + ("src/general/kernels.hpp", "phase_block"), + ("src/general/trial.cpp", "adams_phase_sum"), + ("src/general/trial.cpp", "adams_phase_block"), + ("src/general/trial.cpp", "adams_lanes_phase"), # The multistep defects and compensation (correct, exponential_correct, carry, # _coupled_correct, _carry_flows). - ("src/general.cpp", "compensation_residual"), - ("src/general.cpp", "adams_predicted_compensation"), - ("src/general.cpp", "adams_corrected_compensation"), - ("src/general.cpp", "adams_observed_compensation"), - ("src/general.cpp", "adams_lanes_predicted_compensation"), - ("src/general.cpp", "adams_lanes_corrected_compensation"), - ("src/general.cpp", "adams_lanes_defects"), - ("src/general.cpp", "adams_lanes_rate"), - ("src/general.cpp", "adams_defect_block"), + ("src/general/kernels.hpp", "compensation_residual"), + ("src/general/trial.cpp", "adams_predicted_compensation"), + ("src/general/trial.cpp", "adams_corrected_compensation"), + ("src/general/trial.cpp", "adams_observed_compensation"), + ("src/general/trial.cpp", "adams_lanes_predicted_compensation"), + ("src/general/trial.cpp", "adams_lanes_corrected_compensation"), + ("src/general/trial.cpp", "adams_lanes_defects"), + ("src/general/trial.cpp", "adams_lanes_rate"), + ("src/general/trial.cpp", "adams_defect_block"), # The far field's plane radius (mask_plane_radius). ("include/solprop_native/far_field.hpp", "far_field_radius"), ("include/solprop_native/far_field.hpp", "far_field_offset"), @@ -175,14 +175,22 @@ def test_the_build_system_asks_for_none_of_the_refused_flags(): assert flag not in text, flag assert "-fno-fast-math" in text assert "-ffp-contract=fast" in text - # The strict translation units round as Python's do, on every compiler. + # The strict translation units round as Python's do, on every compiler, and refuse + # the general workspace's contracting headers. strict = re.search( - r"set_source_files_properties\(((?:src/\w+\.cpp\s+)+)PROPERTIES\s+" - r'COMPILE_OPTIONS\s+"-ffp-contract=off"\)', - text, + r"set\(SOLPROP_NATIVE_STRICT_SOURCES\s+((?:src/\w+\.cpp\s*)+)\)", text ) assert strict is not None assert tuple(strict.group(1).split()) == STRICT_SOURCES + for options in ( + r'COMPILE_OPTIONS\s+"-ffp-contract=off"', + r"COMPILE_DEFINITIONS\s+SOLPROP_NATIVE_STRICT_TU", + ): + assert re.search( + r"set_source_files_properties\(\$\{SOLPROP_NATIVE_STRICT_SOURCES\}\s+" + r"PROPERTIES\s+" + options + r"\)", + text, + ), options def test_the_installed_build_records_none_of_the_refused_flags(): diff --git a/tests/backends/test_native_phi_moments.py b/tests/backends/test_native_phi_moments.py index 24c3a7408..67c5703c3 100644 --- a/tests/backends/test_native_phi_moments.py +++ b/tests/backends/test_native_phi_moments.py @@ -91,16 +91,19 @@ def read_moments(module, grid, values): try: # The payload and the step's resident rows are the backend's own; the state # they open is this test's, so nothing is loaded into the backend's. - payload = backend_module._native_problem( - problem, run_request, backend._terms, None, None + payload = backend_module._lower( + problem, run_request, backend_module._LowerPlan(terms=backend._terms) ) arrays, material = backend._resident_arrays(0.0, 1e-3) finally: backend.close() - terms = np.zeros_like(payload["phi_terms"]) + terms = np.zeros_like(payload.linear.phi_terms).reshape( + payload.linear.n_phi_terms, -1, payload.dims.n_omega + ) assert terms.shape[1] == 1 terms[0, 0, bins] = values - state = backend_module._core.State(payload | {"phi_terms": terms}) + payload.linear.phi_terms = terms + state = backend_module._core.State(payload) try: (phi_delta, *rest) = arrays step = np.zeros_like(phi_delta) diff --git a/tests/backends/test_node_chunk.py b/tests/backends/test_node_chunk.py index b5ac30c1f..71d56dc17 100644 --- a/tests/backends/test_node_chunk.py +++ b/tests/backends/test_node_chunk.py @@ -552,14 +552,16 @@ def test_a_general_guide_takes_the_plan_s_budget(native_backend): assert general_planned_chunk(problem, forced) == 2 whole = _general_maps(problem, automatic) assert whole is not None and whole.node_chunk > 1 - payload = native_backend._native_problem( - problem, run_request, terms(problem), general_chunk=2 + payload = native_backend._lower( + problem, + run_request, + native_backend._LowerPlan(terms=terms(problem), general_chunk=2), ) - assert payload["general_node_chunk"] == 2 + assert payload.general.node_chunk == 2 assert ( - native_backend._native_problem(problem, run_request, terms(problem))[ - "general_node_chunk" - ] + native_backend._lower( + problem, run_request, native_backend._LowerPlan(terms=terms(problem)) + ).general.node_chunk == whole.node_chunk )