diff --git a/changelog.d/361-device-split-collar.changed.md b/changelog.d/361-device-split-collar.changed.md new file mode 100644 index 000000000..c3212caa1 --- /dev/null +++ b/changelog.d/361-device-split-collar.changed.md @@ -0,0 +1 @@ +On a CUDA or HIP build, a free-space envelope's split-step temporal collar transforms every row of the state at once, in place, rather than a node chunk at a time, whatever `node_budget` says. At 768 x 768 x 600 a step takes 14 kernels and 10 transforms where it took 119 and 220, and on an AMD Instinct MI300X a point takes 1.6 % less time. Results are unchanged to rounding. diff --git a/changelog.d/365-volume-order.changed.md b/changelog.d/365-volume-order.changed.md new file mode 100644 index 000000000..ef59b64d5 --- /dev/null +++ b/changelog.d/365-volume-order.changed.md @@ -0,0 +1 @@ +On a CUDA build, a Kerr-only free-space envelope's stage holds its transverse scratch in the state's order, so its rank-3 transform keeps frequency contiguous and its input and fold no longer cross layouts. A step on an NVIDIA H200 takes 204 ms at 768 x 768 x 600 where it took 225 ms, and 36 ms at 384 x 384 x 600 where it took 51 ms, with 1.3 GiB less memory there. Results are unchanged to rounding. diff --git a/changelog.d/529-shared-work-area.changed.md b/changelog.d/529-shared-work-area.changed.md new file mode 100644 index 000000000..c3b4e068a --- /dev/null +++ b/changelog.d/529-shared-work-area.changed.md @@ -0,0 +1 @@ +On a CUDA or HIP build, a free-space run's transforms of the transverse scratch share one work area instead of holding one each. At 768 x 768 x 600 a run holds about 16 GiB less, 22.6 GiB against 38.5 GiB on an NVIDIA A40, and 23.4 GiB on an AMD Instinct MI300X. Results are unchanged. diff --git a/docs/dev/accuracy.md b/docs/dev/accuracy.md index cf25be008..95e4d1c70 100644 --- a/docs/dev/accuracy.md +++ b/docs/dev/accuracy.md @@ -501,6 +501,23 @@ which that target does not bound. accuracy_figures.backend_equivalence() ``` +**A hard aperture edge** decides exactly as `numpy_ref` decides on every backend, a +stronger contract than the ladder's. SEP 23's mask-plane radius rounds each product and +sum on its own and takes a correctly rounded square root. On a CPU and in a CUDA build +that root is IEEE 754's (`std::sqrt`, `__dsqrt_rn`). In a HIP build it is the +compiler's. HIP's `sqrt(double)` is `__builtin_sqrt`, which becomes `llvm.sqrt.f64`. +LLVM defines that intrinsic to match a conforming IEEE 754 library unless the call +carries the approximate-functions flag `afn`, and solprop's HIP build sets no fast-math +option. The AMDGPU backend has expanded it that way since 2023 (LLVM `e3fd8f8`): for +`gfx942` under ROCm 7.2.4 a kernel's double `sqrt` compiles to a reciprocal-root +estimate refined by fused multiply-adds, not the bare `v_sqrt_f64`. AMD's documentation of HIP's +math functions promises less. It lists `__dsqrt_rn` as round-to-nearest-even and `sqrt` +at 1 ulp, both measured over `[0, 100]` rather than guaranteed. The device tests' +hard-edge fuzz in `tests/backends/test_native_far_field.py` holds the contract on each +card it runs on. On an AMD Instinct MI300X it fails 31 of its 120 cases with the far +field's contraction pragma removed +([#504](https://github.com/LupoLab/solprop/issues/504)). + ## 8. Where Luna is not converged, and where the two codes still differ diff --git a/docs/dev/backends.md b/docs/dev/backends.md index cdc42c08f..ce06778a9 100644 --- a/docs/dev/backends.md +++ b/docs/dev/backends.md @@ -1449,7 +1449,11 @@ free-space envelope whose material response is the Kerr term alone, with one polarisation, on a time grid the spectrum fills (`n_t = n_to = n_omega`, the crop the identity) takes its stage core on a device through one rank-3 transform of the scratch volume each way, forming the polarisation in place between them, instead of the chunks' -transposing pack and crop and their temporal transforms. The state crosses the seam once +transposing pack and crop and their temporal transforms. On CUDA the stage holds that +scratch as `(y, x, omega)`, the state's own order, so the transform keeps frequency +contiguous and the stage's input and fold stream beside the registers. Every other pass +over the scratch forms its own input in `(omega, y, x)`, and an in-stage absorber keeps +that order for the stage too. The state crosses the seam once at a load and once at an observation, straight between the memory space and the NumPy buffer. diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index a45575203..d895fb8cd 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -897,6 +897,8 @@ An NVIDIA H200 (driver 580.196.07, `nvcc` 13.1, an Intel Xeon Platinum 8592+ hos The crossings are native's own byte count in `tests/backends/test_native_edge_reading.py`, which ran on both cards, and the point's profile, which lists no crossing. No separate Kokkos trace was taken on these cards. At 51 steps the temporal monitor's sums take most of what the check still costs: on the H200 0.38 s of 0.58 s, and the edge's 18 reads 0.11 s. +**At the head, on an AMD Instinct MI300X** ([#541](https://github.com/LupoLab/solprop/issues/541)). This ran the later commits the MI325X above did not: ROCm 10.0.0, `gfx942`, an Intel Xeon Platinum 8568Y+ host, 20 cores, at `c0bd23f2`, on 2026-11-06. `pytest -m gpu` passed over `tests`, 251 with 1 skipped, and over `native/tests`, 31. The modules of the edge, the collar, the split rows, the transverse backends, the scans, the runner and the stop passed too, 270 with 3 skipped. The same probe at 768 x 768 x 600, 8 steps, 4 planes, takes 2.87 s with the check on and 2.13 s with it off, x1.35. At 384 x 384 x 600, 51 steps, 17 planes, the check costs x1.19. At 768 a step takes 241 ms and an edge read 32 ms. `field_to_host` takes 168 ms, `energy` 6.7, `on_axis_intensity` 14.5, `beam_radius` and `fwhm_r` 48, and `peak_intensity` 78. Every aim holds. These figures are the MI300X's own, not the MI325X's at a later commit. A whole `Local` scan of 5 points through `with_input`, check on, ran two rounds each way, and every point finished. Keeping the backend took 768 x 768 x 600, 8 steps, from 37.4 and 36.2 s to 29.2 and 28.9 s, x0.79, and 384 x 384 x 600, 51 steps, from 27.7 and 27.0 s to 23.7 and 23.7 s, x0.87. With the far field's `clang fp contract(off)` removed, as on #497's base, 31 of the hard-edge fuzz's 120 cases fail (12 of the circle's 40 seeds, 19 of the ellipse's, none of the half-plane's), and so does the older hard-edge test. So the fuzz sees that bug on HIP too. + `python -m benchmarks ab --device` against `main` at `8ed26dd7`, 3 rounds, the same work: | | H200 | Fields | MI325X | Fields | @@ -955,6 +957,19 @@ So a `Local` scan on a device now keeps the backend while each point is the last | 864 | 6.67 | 26.70 | 48.80 | 2.515 | 0.377 | | 900 | 7.24 | 14.48 | 38.30 | 2.373 | 0.328 | +**One work area.** A device's plans of the scratch now share one, as cuFFT's guide allows for plans launched one after another, so the table's Plans column is what each size held before. On an NVIDIA A40, CUDA 12.8.1, 8 steps, check off, two interleaved rounds of three runs each, against the commit before: + +| N x N x 600 | Plans, GiB | Total, GiB | Point, s | +|---:|---:|---:|---:| +| 384 | 2.64 to 1.32 | 8.04 to 6.73 | 4.84, 4.94 to 4.86, 4.79 | +| 750 | 10.06 to 5.03 | 26.68 to 21.66 | 20.24, 20.19 to 20.28, 20.10 | +| 768 | 21.09 to 5.28 | 38.46 to 22.64 | 21.37, 21.41 to 21.37, 21.46 | +| 900 | 14.48 to 7.24 | 37.80 to 30.56 | 28.74, 28.61 to 28.64, 28.50 | + +`python -m benchmarks ab --device --length 0.3`, 3 rounds, the same steps and final fields with no difference: `kerr_beam_384` x0.994 (0.984–1.048), `kerr_beam_192` x1.030 (1.001–1.035, a window of two steps). So a size no longer costs more memory for its rank-2 pair, and 768 x 768 x 600 now holds less than 900 x 900 x 600. The size still sets the seconds per GiB. + +rocFFT's plans share it as well. On an AMD Instinct MI300X (ROCm 10.0.0, `gfx942`) at `c0bd23f2`, 8 steps, check off, the plans grow by 1.35, 5.07, 5.76 and 7.28 GiB at 384, 750, 768 and 900 x 900 x 600, and a run holds 7.01, 21.95, 23.37 and 30.85 GiB. Asked of rocFFT directly, in place, Z2Z, a rank-3 plan's work area is one state at every N from 320 to 1200 and every T from 512 to 1024. A rank-2 plan batched over T takes one state at 192 and 384, none at 256 and 512, and 1.10, 1.08 and 1.06 states at 640, 768 and 1024. A rank-1 plan takes none. So at 768 x 768 x 600 four areas of their own would hold 22.0 GiB, as the MI325X's 22.02 GiB at `9a5d2346` did, and the shared one holds the largest of them. + ### The transverse edge against 0.1.0 [#453](https://github.com/LupoLab/solprop/issues/453) found the tracking set's `kerr_beam` @@ -2435,6 +2450,8 @@ node chunk, and `Transform` needs no field to tell a stated budget from the defa cards, and cuFFT's work area is small (at 589,824 rows of 600 points each plan holds 9 MiB, in place or not, 0.1 MiB at the node chunk). A CPU keeps the chunk for its own reason: on an Intel Xeon Gold 6240, nine cores, at 384 x 384 x 600, FFTW's threaded plan of a whole state took 2.5 s a step in transforms against 1.5 s. +**A whole state again.** #361's 10 % is asked of a change that costs default memory, and this one costs none, so the maintainer asked for it back if an interleaved run on an AMD card shows a consistent gain. A device now takes every row at once whatever the node budget, with no seam field: in place, the block's only memory is its plans' work area. On an NVIDIA A40, CUDA 12.8.1, `python -m benchmarks ab --device --length 0.3`, 3 rounds, against the same tree at the node chunk, the same steps and final fields with no difference: `kerr_beam_384` x0.989 (0.958–0.994), `kerr_beam_192` x0.971 (0.951–1.065), and every route without the collar between x0.998 and x1.004. On an AMD Instinct MI300X, ROCm 10.0.0, at `c0bd23f2` against the same tree at the node chunk: a field-free point of 8 steps with the check off, the median of a runner's second and third runs, three interleaved rounds of fresh processes. At 384 x 384 x 600 the rounds give x0.990, x0.988 and x0.990 (0.576 to 0.569 s), and at 768 x 768 x 600 x0.984, x0.983 and x0.986 (2.172 to 2.138 s), with the plans' memory the same. `ab --device --length 0.3`, 3 rounds, gives `kerr_beam_384` x0.952 (0.912–0.999), with the final fields identical. Every round gained, so it stays, though at about 1.5 % of a point it is far from #361's 10 %. + **The fold.** A flow's magnitude on a lossless bin is `exp(+-0)`, exactly one, and took a binary64 `exp` per element: 13.4 ms a pass on the A40, whose reduced binary64 rate overstates it. Runge-Kutta's `stage_input` and `finish_transform` fall by 13 % the same way. The flowless pass forms no exponent (64.3 to 61.5 ms). The first input crosses the plane's stride through tiles of team scratch (42.5 to 24.9 ms). On a host the branch is out of line: inlined there, it took the tabulated fold's kernel past GCC's inlining limit, @@ -2458,11 +2475,28 @@ against the build before them: `he_rdw_averaged` 0.988 (0.986–1.007), `rdw` 0. **Scratch-order registers on the volume route**, tried on the A40 and not adopted, take the fold from 407 to 387 ms and the step to 2,538 ms, with the fold's tile staging the state rather than the scratch. A layout change needs its own issue. +**The scratch in the registers' order** ([#365](https://github.com/LupoLab/solprop/issues/365)), on CUDA only. A stage of the volume route views the scratch as `(y, x, omega)`, the state's and the +registers' order, so its rank-3 pair is cuFFT's frequency-fastest `{N,N,T}`, and its input and fold stream instead of crossing. A probe of the two orders on an NVIDIA H200, CUDA 13.1, +T = 600, took `{N,N,T}` 2.49, 11.08, 14.71, 17.62 and 13.81 ms a transform at N = 384, 750, 768, 800 and 900, against 3.74, 14.97, 17.63, 17.97 and 18.96 ms for `{T,N,N}`, and `{N,N,T}` +needs no work area at those sizes but 768. A step, from a field-free point with the check off, (24 steps - 8 steps) / 16, three interleaved rounds against the build before it +(`c0bd23f2` against `8e9b4e76`): + +| N x N x 600, NVIDIA H200 | Before, ms | After, ms | Ratio | A run's memory, GiB | +|---:|---:|---:|---:|---:| +| 384 | 50.96–51.17 | 36.22–36.49 | 0.713 | 7.23 → 5.91 | +| 768 | 225.21–225.78 | 203.88–204.13 | 0.905 | 23.14 → 23.14 | + +At 768 the transforms account for it: eight of them are 23.4 ms apart in the probe, and the step gains 21.6 ms. `python -m benchmarks ab --device --length 0.3`, 3 rounds: +`kerr_beam_384` x0.971 (0.928–1.003), fields to 3.2e-17. On the NVIDIA A40, CUDA 12.8.1, where the orders are within 1 % at 768, two rounds against `29e9e156`, whose native code is +the same: 2,535 and 2,578 ms a step before, 2,556 and 2,597 after, at 768; 559 and 560 ms before, 513 and 513 after, at 384; `ab --device` geometric mean 0.998, with `kerr_beam_384` +x1.010 (0.997–1.042) and `kerr_beam_192` x0.977 (0.950–1.070). The device tests pass on both cards. With the stage's window read in the old order, nine of them fail across the three +marches, and with the fold reading the old order, three low-storage ones do. HIP keeps the scratch's order until rocFFT's orders are measured. + On a CPU, `python -m benchmarks ab` against the merge base on an Intel Xeon Gold 6240 (nine cores of one socket, bound passive threads, on a machine others shared; three rounds, the same steps): `kerr_beam_192` 0.982 (0.978–0.996) at one thread and 0.974 (0.973–0.987) at nine; `kerr_beam_384` 0.981 (0.981–0.982) at nine, fields to 6.4e-16. `rdw` under `rk` and `multistep-interaction`, one and nine threads, without the host's fold fix, which they do not reach: 1.011, 0.974, 0.996 and 0.999. Before the fold's fix, `radial_filament_256` 0.993 and `planar` 0.980 at nine threads, five rounds, took neither path. -A data-centre card has to confirm the step on an H200 and an MI325X, what rocFFT's plans hold, and what the launches are worth. +The H200 has since measured the step in the scratch's order, above. An AMD Instinct MI300X has measured what rocFFT's plans hold and what a whole state's collar is worth. No MI325X has run the head. ## Reference tables diff --git a/docs/guide/performance/tuning.md b/docs/guide/performance/tuning.md index 17c896a91..870c3d65b 100644 --- a/docs/guide/performance/tuning.md +++ b/docs/guide/performance/tuning.md @@ -104,7 +104,8 @@ plane's columns in node chunks, each paying a fixed cost for its launches, its t transforms and its gather and scatter. Their default budget is therefore 1 GiB, against 64 MiB on modal routes; `SimulationSettings.node_budget` sets it explicitly. On a laptop short of memory a smaller budget saves memory at some speed; on a large device a larger -one can still help. +one can still help. A device takes an envelope's split collar every row at once, whatever +the budget: it transforms the state's own rows in place and holds no buffer for them. **Where a free-space step goes.** On the native backend the stages of a free-space evaluation are profiling regions (`solprop_native.free.*`) that any Kokkos Tools timer diff --git a/native/include/solprop_native/far_field.hpp b/native/include/solprop_native/far_field.hpp index c8713ad72..0e7b5b20d 100644 --- a/native/include/solprop_native/far_field.hpp +++ b/native/include/solprop_native/far_field.hpp @@ -162,8 +162,9 @@ KOKKOS_INLINE_FUNCTION std::size_t far_field_band_edge(const std::size_t band, * input (an exact hard-mask contract). A device compiler keeps its own contraction * default, so a CUDA build spells each operation with a round-to-nearest intrinsic and * a HIP build, whose intrinsics are no protection, takes the block above. A HIP - * device's square root is the compiler's own, which decided every edge the device tests - * tried; its correct rounding is observed there, not guaranteed by its documentation. + * device's square root is `__builtin_sqrt`, so `llvm.sqrt.f64`, which LLVM defines as a + * conforming IEEE 754 square root, correctly rounded, unless the call carries `afn`; + * AMD's own math documentation promises less (docs/dev/accuracy.md section 7). */ SOLPROP_NATIVE_STRICT_FUNCTION KOKKOS_INLINE_FUNCTION double far_field_radius(const double scale, const double k_x, diff --git a/native/src/general.cpp b/native/src/general.cpp index 430eccbc5..260938716 100644 --- a/native/src/general.cpp +++ b/native/src/general.cpp @@ -258,10 +258,17 @@ using ForwardRealPlan = KokkosFFT::Plan using RamanForwardPlan = RamanFftPlan; using RamanInversePlan = RamanFftPlan; using ComplexPlan = KokkosFFT::Plan; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +// A device's plans of the transverse scratch run one after another, so they share one +// work area (`transform_work_`) in place of holding one each (solprop #529). A +// Kerr-only free-space envelope takes the stage's time axis with the plane's in one +// rank-3 transform (`volume_kerr_`). +using TransversePlan = KokkosFFT::DynPlan; +using VolumePlan = TransversePlan; +#else using TransversePlan = KokkosFFT::Plan; -// A device's Kerr-only free-space envelope takes the stage's time axis with the plane's -// in one rank-3 transform (`volume_kerr_`). using VolumePlan = KokkosFFT::Plan; +#endif // The hybrid planar kind transforms y alone, the trailing axis of (bins, n_p, n_y). using SlabPlan = KokkosFFT::Plan; // A requested collar's padded transform, along the last axis of a batch (solprop SEP @@ -605,6 +612,11 @@ struct RegisterIndex { // One polarisation: a column is its mode, without a division (a flag of its own, // which the compiler cannot fold back into `column / 1`). bool scalar{}; +#if defined(KOKKOS_ENABLE_CUDA) + // A stage of the volume route on CUDA (`volume_order_`): the scratch is `(y, x, + // omega)`, the state's and the registers' own order, so nothing crosses. + bool volume{}; +#endif KOKKOS_INLINE_FUNCTION std::size_t mode(const std::size_t column) const { return scalar ? column : column / n_pol; } @@ -614,6 +626,10 @@ struct RegisterIndex { // The transverse scratch's position of the element, on any build. KOKKOS_INLINE_FUNCTION std::size_t cell(const std::size_t column, const std::size_t omega) const { +#if defined(KOKKOS_ENABLE_CUDA) + if (volume) + return column * n + omega; +#endif return (omega * n_pol + pol(column)) * layout.plane + layout.offset(mode(column)); } KOKKOS_INLINE_FUNCTION std::size_t operator()(const std::size_t column, @@ -2358,6 +2374,16 @@ class GeneralWorkspace { return {transverse_layout(), problem_.n_pol_state, problem_.n_omega, plane, problem_.n_pol_state == 1}; } + // A stage's scratch positions: the volume route's own order on CUDA, or the scratch's + // (`register_index`). Every other pass over the scratch forms its own input in + // `(omega, y, x)`, since the scratch is dead between them. + RegisterIndex stage_index() const noexcept { + auto index = register_index(); +#if defined(KOKKOS_ENABLE_CUDA) + index.volume = volume_order_; +#endif + return index; + } // The unscaled transverse DFT's sample count: the plane's, or the slab's y axis. double inverse_transverse_count() const noexcept { return 1.0 / static_cast(problem_.slab_geometry() ? problem_.n_y @@ -2687,7 +2713,15 @@ class GeneralWorkspace { // (omega, y, x) at once and forms the response there, in place of the chunks' // transposes and temporal transforms (`separable_volume_core`). bool volume_kerr_ = false; + // The volume route's stages hold the scratch as `(y, x, omega)`, `volume_scratch_`, + // on CUDA (solprop #365): its rank-3 transform is then cuFFT's frequency-fastest + // `{N,N,T}`, and its input and fold no longer cross layouts. + bool volume_order_ = false; + ComplexView3 volume_scratch_; std::unique_ptr volume_forward_, volume_inverse_; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + ComplexView transform_work_; +#endif std::unique_ptr slab_forward_, slab_inverse_; ComplexView2 current_, advanced_, solution_, error_, residual_, recycled_, noise_; // solprop SEP 24 phase 3: the low-storage march's registers B and C (the state is @@ -4199,7 +4233,16 @@ void GeneralWorkspace::allocate() { split_factor_ = RealView("solprop_native.absorber.split_factor", problem_.n_t); #endif if (problem_.free_geometry() && problem_.family == FieldFamily::envelope) { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + // In place on the state's rows, a block holds no buffer of its own, only its + // plans' work area (9 MiB on cuFFT for every row at 768 x 768 x 600), so a device + // takes every row at once whatever the node budget: 14 kernels and 10 transforms + // a step there, against 119 and 220 at the node chunk (solprop #361). A CPU keeps + // the chunk, where FFTW's threaded plan of a whole state was much the slower. + split_rows_ = problem_.n_columns(); +#else split_rows_ = temporal_columns; +#endif } else { coarse_spectrum_ = ComplexView2("solprop_native.absorber.coarse_spectrum", temporal_columns, n); @@ -4516,7 +4559,11 @@ void GeneralWorkspace::plan() { slab_inverse_ = std::make_unique(space, transverse_, transverse_, KokkosFFT::Direction::backward, y_axis); } else if (problem_.free_geometry()) { +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + constexpr std::size_t axes = 2; +#else const KokkosFFT::axis_type<2> axes{-2, -1}; +#endif transverse_forward_ = std::make_unique( fft_space_for(transverse_.size()), transverse_, transverse_, KokkosFFT::Direction::forward, axes); @@ -4534,15 +4581,41 @@ void GeneralWorkspace::plan() { !problem_.raman && !problem_.envelope_plasma && !problem_.dispersive && separable_dispersive_ == nullptr && problem_.harmonic_factor == 0.0 && separable_gaps_.empty(); +#if defined(KOKKOS_ENABLE_CUDA) + // At 768 x 768 x 600 on an NVIDIA H200, cuFFT's `{N,N,T}` takes 14.7 ms a transform + // against 17.6 ms for `{T,N,N}` (solprop #365). An in-stage absorber evaluates its + // term from the same input in the scratch's order, so it keeps that order. + volume_order_ = volume_kerr_ && !problem_.in_stage_absorber; +#endif + if (volume_order_) + volume_scratch_ = ComplexView3(transverse_.data(), problem_.n_y, problem_.n_x, + problem_.n_omega); if (volume_kerr_) { - const KokkosFFT::axis_type<3> volume_axes{-3, -2, -1}; - volume_forward_ = - std::make_unique(execution_space_, transverse_, transverse_, - KokkosFFT::Direction::forward, volume_axes); + constexpr std::size_t volume_axes = 3; + const auto& volume = volume_order_ ? volume_scratch_ : transverse_; + volume_forward_ = std::make_unique( + execution_space_, volume, volume, KokkosFFT::Direction::forward, volume_axes); volume_inverse_ = - std::make_unique(execution_space_, transverse_, transverse_, + std::make_unique(execution_space_, volume, volume, KokkosFFT::Direction::backward, volume_axes); } + std::size_t work = KokkosFFT::compute_required_workspace_size( + *transverse_forward_, *transverse_inverse_); + if (volume_kerr_) + work = std::max(work, KokkosFFT::compute_required_workspace_size( + *volume_forward_, *volume_inverse_)); + if (work != 0) { + transform_work_ = + ComplexView(Kokkos::view_alloc("solprop_native.free.transform_work", + Kokkos::WithoutInitializing), + work); + transverse_forward_->set_work_area(transform_work_); + transverse_inverse_->set_work_area(transform_work_); + if (volume_kerr_) { + volume_forward_->set_work_area(transform_work_); + volume_inverse_->set_work_area(transform_work_); + } + } #endif } constexpr int axis = -1; @@ -5653,7 +5726,7 @@ void column_norms(const char* label, const ExecutionSpace& execution, void GeneralWorkspace::low_storage_separable_input(const std::size_t pass) { const auto state = current_; const auto stage_register = register_stage_; - const auto index = register_index(); + const auto index = stage_index(); const auto spatial = spatial_; const auto at = stage_phi_at_; const auto active = active_mask_; @@ -5673,7 +5746,8 @@ void GeneralWorkspace::low_storage_separable_input(const std::size_t pass) { } return value * scale; }; - if (problem_.low_storage_tiled_input) { + // In the volume route's order the scratch is the state's, so the input streams. + if (problem_.low_storage_tiled_input && !volume_order_) { // Read along the state's bins and written along the scratch's rows, a tile at a // time. tiled_crossing( @@ -5712,7 +5786,7 @@ void GeneralWorkspace::low_storage_separable_fold(const std::size_t pass, // A device gathers and scatters the scratch through its own tiles. [[maybe_unused]] const auto n_pol = problem_.n_pol_state; [[maybe_unused]] const auto layout = transverse_layout(); - const auto index = register_index(); + const auto index = stage_index(); const bool add_noise = problem_.modified_noise && has_noise_; const double scale = inverse_transverse_count(); const int stage = static_cast(pass); @@ -5791,19 +5865,32 @@ void GeneralWorkspace::low_storage_separable_fold(const std::size_t pass, return final && in_norm(omega) != 0U ? squared_abs(a) : 0.0; }; #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) - tiled_update( - "solprop_native.low_storage.separable_fold", execution_space_, - problem_.n_columns(), problem_.n_omega, exchange || forward, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { - return layout.cell(scratch, omega * n_pol + column % n_pol, column / n_pol); - }, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, Complex& cell) { - static_cast(body(column, omega, cell, state(column, omega))); - }, - KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, - const Complex value) { - layout.cell(scratch, omega * n_pol + column % n_pol, column / n_pol) = value; - }); + // In the volume route's order, the scratch's cell sits beside the registers' and the + // state's, so the fold streams all three. + if (volume_order_) + cartesian_for( + "solprop_native.low_storage.separable_fold", execution_space_, + problem_.n_columns(), problem_.n_omega, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + static_cast(body(column, omega, + scratch.data()[index.cell(column, omega)], + state(column, omega))); + }); + else + tiled_update( + "solprop_native.low_storage.separable_fold", execution_space_, + problem_.n_columns(), problem_.n_omega, exchange || forward, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega) { + return layout.cell(scratch, omega * n_pol + column % n_pol, column / n_pol); + }, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, + Complex& cell) { + static_cast(body(column, omega, cell, state(column, omega))); + }, + KOKKOS_LAMBDA(const std::size_t column, const std::size_t omega, + const Complex value) { + layout.cell(scratch, omega * n_pol + column % n_pol, column / n_pol) = value; + }); #else // The host's registers sit at the scratch cell's own position (`RegisterIndex`), so a // block of adjacent columns reads them, and the scratch, in contiguous runs at every @@ -8945,7 +9032,7 @@ void GeneralWorkspace::separable_stage_input(const std::size_t stage, const doub const auto noise = noise_; const auto scratch = transverse_; const auto whole = whole_propagator_; - const auto index = register_index(); + const auto index = stage_index(); const auto n_stages = problem_.n_stages; const bool add_noise = problem_.modified_noise && has_noise_; const double inverse_area_count = inverse_transverse_count(); @@ -9101,7 +9188,7 @@ void GeneralWorkspace::separable_finish_transform(const std::size_t stage, const auto window = omega_window_; const auto active = active_mask_; const auto delta = stage_phi_delta_; - const auto index = register_index(); + const auto index = stage_index(); crossing_for( "solprop_native.free.finish_transform", execution_space_, problem_.n_columns(), problem_.n_omega, @@ -9214,9 +9301,8 @@ void GeneralWorkspace::separable_to_frequency(const TransverseChunk& chunk) { // pack's and the crop's transposes. void GeneralWorkspace::separable_volume_core(const std::size_t stage) { Kokkos::Profiling::pushRegion("solprop_native.free.volume_core"); - graph::execute(*volume_inverse_, transverse_, transverse_, - KokkosFFT::Normalization::none); - const auto scratch = transverse_; + const auto scratch = volume_order_ ? volume_scratch_ : transverse_; + graph::execute(*volume_inverse_, scratch, scratch, KokkosFFT::Normalization::none); const auto window = time_window_; const auto kerr_density = stage_kerr_density_; const double kerr_coefficient = problem_.kerr_coefficient; @@ -9224,11 +9310,13 @@ void GeneralWorkspace::separable_volume_core(const std::size_t stage) { const double conjugate_factor = problem_.conjugate_factor; const double scale = 1.0 / static_cast(problem_.n_t); const auto plane = transverse_.extent(1) * transverse_.extent(2); + const bool time_fastest = volume_order_; + const auto n_t = problem_.n_t; launch::parallel_for( "solprop_native.free.volume_kerr", - Kokkos::RangePolicy(execution_space_, 0, transverse_.size()), + Kokkos::RangePolicy(execution_space_, 0, scratch.size()), KOKKOS_LAMBDA(const std::size_t item) { - const std::size_t time = item / plane; + const std::size_t time = time_fastest ? item % n_t : item / plane; Complex& cell = scratch.data()[item]; const Complex value = scale * cell; const double modulus = squared_abs(value); @@ -9244,8 +9332,7 @@ void GeneralWorkspace::separable_volume_core(const std::size_t stage) { (weighted_imag * value.real() - weighted_real * value.imag()))); cell = result * window(time); }); - graph::execute(*volume_forward_, transverse_, transverse_, - KokkosFFT::Normalization::none); + graph::execute(*volume_forward_, scratch, scratch, KokkosFFT::Normalization::none); Kokkos::Profiling::popRegion(); } @@ -16237,9 +16324,13 @@ void GeneralWorkspace::close() noexcept { radial_blas_input_ = {}; radial_blas_output_ = {}; transverse_forward_.reset(); + transverse_inverse_.reset(); volume_forward_.reset(); volume_inverse_.reset(); - transverse_inverse_.reset(); + volume_scratch_ = {}; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + transform_work_ = {}; +#endif slab_forward_.reset(); slab_inverse_.reset(); transverse_ = {}; diff --git a/tests/backends/test_free_split_rows.py b/tests/backends/test_free_split_rows.py index c427662aa..e36b81ed0 100644 --- a/tests/backends/test_free_split_rows.py +++ b/tests/backends/test_free_split_rows.py @@ -177,6 +177,8 @@ def test_a_device_block_of_any_size_takes_the_reference_s_steps( ) -> None: if device_backend("native") == "none": pytest.skip("the installed solprop-native build has no accelerator backend") + # A device takes the collar's rows all at once whatever the chunk, so this holds + # that block to the reference beside the chunk's other passes. _check_block(reference, chunk) diff --git a/tests/backends/test_native_gpu.py b/tests/backends/test_native_gpu.py index f48085660..7e2debefa 100644 --- a/tests/backends/test_native_gpu.py +++ b/tests/backends/test_native_gpu.py @@ -8,7 +8,7 @@ import numpy as np import pytest -from benchmarks.device_split import _simulation +from benchmarks.device_split import STEP, _simulation from bounds import assert_within, ladder_bound from solprop.backends.numpy_ref import NumpyRefBackend @@ -123,6 +123,35 @@ def test_a_large_state_crosses_both_ways_through_the_ring_bit_for_bit( engine.close() +@pytest.mark.filterwarnings("ignore::solprop.exceptions.MaterialRangeWarning") +def test_the_plans_of_the_scratch_hold_one_work_area(accelerator_backend): + # A forward and an inverse rank-3 plan take a state's work area each at this size, + # on their own (solprop #529). Run a step on it, so that a shared area can't be + # one no transform has used, and read the field the transforms left. + del accelerator_backend + simulation = _simulation(512, 2, 1, None) + problem = simulation.translate() + engine = cast( + "Any", + select( + "native", + problem, + grid=simulation.grid, + modes=simulation.modes, + profiles=simulation.profiles, + ), + ) + try: + state = int(np.prod(problem.shape)) * 16 + engine.set_initial(problem.initial.field0, None, z=problem.initial.z) + engine.propagate_to(2 * STEP) + field = engine.observe(field=True).require_field() + assert np.isfinite(field).all() and np.any(field != 0.0) + assert engine.memory_ledger()["fft_plan_device_growth"] < 1.5 * state + finally: + engine.close() + + @pytest.mark.filterwarnings("ignore::solprop.exceptions.MaterialRangeWarning") def test_a_small_state_holds_no_pinned_ring(accelerator_backend): del accelerator_backend