diff --git a/profiling/examples/CurlCurl/cube_strong_scaling/params_curl_curl.py b/profiling/examples/CurlCurl/cube_strong_scaling/params_curl_curl.py index 8dcbadfe0..ea7ec5508 100644 --- a/profiling/examples/CurlCurl/cube_strong_scaling/params_curl_curl.py +++ b/profiling/examples/CurlCurl/cube_strong_scaling/params_curl_curl.py @@ -4,13 +4,50 @@ # Please fill in a verbal description of the simulation. # It will be printed at the beginning of the simulation and can be used to keep track of the different runs. -name = "Curl-curl strong scaling on 3D cube, no preconditioner" -description = """ +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# `--precond` selects the preconditioner (the variant), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--precond", + default="None", + choices=["None", "HiptmairXu", "MultiGrid", "StiffnessPreconditioner"], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 64, 64], help="Number of elements per direction.") +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +# preconditioner -> (name suffix, linear solver) for the name and the description +PRECONDS = { + None: ("no preconditioner", "unpreconditioned CG"), + "HiptmairXu": ( + "Hiptmair-Xu preconditioner", + "CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner)", + ), + "MultiGrid": ( + "multigrid preconditioner", + "CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair", + ), + "StiffnessPreconditioner": ( + "stiffness preconditioner", + "CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner)", + ), +} + +name = f"Curl-curl strong scaling on 3D cube, {PRECONDS[precond][0]}" +description = f""" Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a 3D cube (Cuboid). The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with unpreconditioned CG. +The linear system is solved with {PRECONDS[precond][1]}. """ import logging @@ -19,7 +56,6 @@ set_logging_level(logging.INFO) -import argparse # ------------------ # Import Struphy API @@ -55,13 +91,6 @@ # -------------------------- # Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - env = EnvironmentOptions( sim_folder=f"sim_{args.id:02d}", restart=False, @@ -80,7 +109,7 @@ equil = None # Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) +grid = grids.TensorProductGrid(num_elements=tuple(args.num_elements), mpi_dims_mask=(True, True, True)) # Derham options derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) @@ -114,11 +143,11 @@ # coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition sigma = 1e-2 -solver_params = SolverParameters(tol=1e-8, maxiter=10000, info=True, recycle=False) +solver_params = SolverParameters(tol=1e-8, maxiter=10000 if precond is None else 3000, info=True, recycle=False) model.propagators.curl_curl.options = model.propagators.curl_curl.Options( sigma=sigma, solver="pcg", - precond=None, + precond=precond, solver_params=solver_params, ) diff --git a/profiling/examples/CurlCurl/cube_strong_scaling_multigrid/params_curl_curl.py b/profiling/examples/CurlCurl/cube_strong_scaling_multigrid/params_curl_curl.py deleted file mode 100644 index acb6d3610..000000000 --- a/profiling/examples/CurlCurl/cube_strong_scaling_multigrid/params_curl_curl.py +++ /dev/null @@ -1,253 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Curl-curl strong scaling on 3D cube, multigrid preconditioner" -description = """ -Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a 3D cube (Cuboid). -The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant -vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. -Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair. -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - ProfilingOptions, - Simulation, - Time, - domains, - grids, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import CurlCurl - -# Units -base_units = BaseUnits() - -# Model instance -model = CurlCurl(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.e_field.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -Lx = 2.0 -Ly = 3.0 -Lz = 4.0 -domain = domains.Cuboid(r1=Lx, l2=-Ly / 2, r2=Ly / 2, r3=Lz) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -# coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition -sigma = 1e-2 - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.curl_curl.options = model.propagators.curl_curl.Options( - sigma=sigma, - solver="pcg", - precond="MultiGrid", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np -import sympy as sym - -from struphy.initial.base import GenericPerturbation - -# The exact solution is the vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a -# localized Gaussian g (below). It is negligible at the boundary, hence satisfies the boundary conditions -# up to that accuracy. Being localized, it excites the whole spectrum of the curl-curl operator, including -# gradients, so CG has to work for convergence. -# The right-hand side J = curl curl E + sigma E is computed symbolically. Both are given in Cartesian -# components (given_in_basis="physical"). -x_, y_, z_ = sym.symbols("x y z", real=True) -# Gaussian of width 0.15 centered at (1, 0, 2) -g = sym.exp(-((x_ - 1.0) ** 2 + y_**2 + (z_ - 2.0) ** 2) / 0.15**2) -c = np.array([1.0, 2.0, -1.0]) / np.sqrt(6.0) - - -def curl(F): - """Curl of a Cartesian vector field (list of sympy expressions).""" - return [ - sym.diff(F[2], y_) - sym.diff(F[1], z_), - sym.diff(F[0], z_) - sym.diff(F[2], x_), - sym.diff(F[1], x_) - sym.diff(F[0], y_), - ] - - -E_sym = [ck * g for ck in c] -J_sym = [cc + sigma * e for cc, e in zip(curl(curl(E_sym)), E_sym)] -E_fun = [sym.lambdify((x_, y_, z_), e, "numpy") for e in E_sym] -J_fun = [sym.lambdify((x_, y_, z_), j, "numpy") for j in J_sym] - - -def exact_solution(comp): - return lambda x, y, z: E_fun[comp](x, y, z) + 0.0 * x - - -def rhs(comp): - return lambda x, y, z: J_fun[comp](x, y, z) + 0.0 * x - - -for comp in range(3): - model.em_fields.source.add_perturbation(GenericPerturbation(rhs(comp), given_in_basis="physical", comp=comp)) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True, physical=True) - - from matplotlib import pyplot as plt - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error of the magnitude on the logical slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(data, shading="gouraud") - plt.colorbar() - plt.xlabel(f"eta{[2, 1, 1][k]} index") - plt.ylabel(f"eta{[3, 3, 2][k]} index") - plt.title(f"|{name}| {what}, slice at eta{k + 1} = 0.5") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Cartesian components of the fields at the cell centres of the simulation grid - e_xyz = out.evaluate("em_fields/e_field_xyz", t=-1) - j_xyz = out.evaluate("em_fields/source_xyz", t=0) - X, Y, Z = (e_xyz.coords[c_].values for c_ in ("X", "Y", "Z")) - - e = np.stack([e_xyz.isel(component=k).values for k in range(3)]) - j = np.stack([j_xyz.isel(component=k).values for k in range(3)]) - e_exact = np.stack([exact_solution(k)(X, Y, Z) for k in range(3)]) - j_exact = np.stack([rhs(k)(X, Y, Z) for k in range(3)]) - - fig_rhs = plot_slices(np.linalg.norm(j, axis=0), np.linalg.norm(j_exact, axis=0), "J") - fig_e = plot_slices(np.linalg.norm(e, axis=0), np.linalg.norm(e_exact, axis=0), "E") - - rel_err_rhs = np.max(np.abs(j - j_exact)) / np.max(np.abs(j_exact)) - rel_err_e = np.max(np.abs(e - e_exact)) / np.max(np.abs(e_exact)) - - print(f"Max relative error in J: {rel_err_rhs:.2e}") - print(f"Max relative error in E: {rel_err_e:.2e}") - - assert rel_err_rhs < 2.5e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_e < 1.7e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_e}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_e.npy"), rel_err_e) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_e.savefig(os.path.join(results_dir, "e_slices.png")) - else: - plt.show() diff --git a/profiling/examples/CurlCurl/cube_strong_scaling_stiffness/params_curl_curl.py b/profiling/examples/CurlCurl/cube_strong_scaling_stiffness/params_curl_curl.py deleted file mode 100644 index b41b4ee11..000000000 --- a/profiling/examples/CurlCurl/cube_strong_scaling_stiffness/params_curl_curl.py +++ /dev/null @@ -1,253 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Curl-curl strong scaling on 3D cube, stiffness preconditioner" -description = """ -Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a 3D cube (Cuboid). -The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant -vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. -Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner). -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - ProfilingOptions, - Simulation, - Time, - domains, - grids, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import CurlCurl - -# Units -base_units = BaseUnits() - -# Model instance -model = CurlCurl(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.e_field.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -Lx = 2.0 -Ly = 3.0 -Lz = 4.0 -domain = domains.Cuboid(r1=Lx, l2=-Ly / 2, r2=Ly / 2, r3=Lz) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -# coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition -sigma = 1e-2 - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.curl_curl.options = model.propagators.curl_curl.Options( - sigma=sigma, - solver="pcg", - precond="StiffnessPreconditioner", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np -import sympy as sym - -from struphy.initial.base import GenericPerturbation - -# The exact solution is the vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a -# localized Gaussian g (below). It is negligible at the boundary, hence satisfies the boundary conditions -# up to that accuracy. Being localized, it excites the whole spectrum of the curl-curl operator, including -# gradients, so CG has to work for convergence. -# The right-hand side J = curl curl E + sigma E is computed symbolically. Both are given in Cartesian -# components (given_in_basis="physical"). -x_, y_, z_ = sym.symbols("x y z", real=True) -# Gaussian of width 0.15 centered at (1, 0, 2) -g = sym.exp(-((x_ - 1.0) ** 2 + y_**2 + (z_ - 2.0) ** 2) / 0.15**2) -c = np.array([1.0, 2.0, -1.0]) / np.sqrt(6.0) - - -def curl(F): - """Curl of a Cartesian vector field (list of sympy expressions).""" - return [ - sym.diff(F[2], y_) - sym.diff(F[1], z_), - sym.diff(F[0], z_) - sym.diff(F[2], x_), - sym.diff(F[1], x_) - sym.diff(F[0], y_), - ] - - -E_sym = [ck * g for ck in c] -J_sym = [cc + sigma * e for cc, e in zip(curl(curl(E_sym)), E_sym)] -E_fun = [sym.lambdify((x_, y_, z_), e, "numpy") for e in E_sym] -J_fun = [sym.lambdify((x_, y_, z_), j, "numpy") for j in J_sym] - - -def exact_solution(comp): - return lambda x, y, z: E_fun[comp](x, y, z) + 0.0 * x - - -def rhs(comp): - return lambda x, y, z: J_fun[comp](x, y, z) + 0.0 * x - - -for comp in range(3): - model.em_fields.source.add_perturbation(GenericPerturbation(rhs(comp), given_in_basis="physical", comp=comp)) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True, physical=True) - - from matplotlib import pyplot as plt - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error of the magnitude on the logical slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(data, shading="gouraud") - plt.colorbar() - plt.xlabel(f"eta{[2, 1, 1][k]} index") - plt.ylabel(f"eta{[3, 3, 2][k]} index") - plt.title(f"|{name}| {what}, slice at eta{k + 1} = 0.5") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Cartesian components of the fields at the cell centres of the simulation grid - e_xyz = out.evaluate("em_fields/e_field_xyz", t=-1) - j_xyz = out.evaluate("em_fields/source_xyz", t=0) - X, Y, Z = (e_xyz.coords[c_].values for c_ in ("X", "Y", "Z")) - - e = np.stack([e_xyz.isel(component=k).values for k in range(3)]) - j = np.stack([j_xyz.isel(component=k).values for k in range(3)]) - e_exact = np.stack([exact_solution(k)(X, Y, Z) for k in range(3)]) - j_exact = np.stack([rhs(k)(X, Y, Z) for k in range(3)]) - - fig_rhs = plot_slices(np.linalg.norm(j, axis=0), np.linalg.norm(j_exact, axis=0), "J") - fig_e = plot_slices(np.linalg.norm(e, axis=0), np.linalg.norm(e_exact, axis=0), "E") - - rel_err_rhs = np.max(np.abs(j - j_exact)) / np.max(np.abs(j_exact)) - rel_err_e = np.max(np.abs(e - e_exact)) / np.max(np.abs(e_exact)) - - print(f"Max relative error in J: {rel_err_rhs:.2e}") - print(f"Max relative error in E: {rel_err_e:.2e}") - - assert rel_err_rhs < 2.5e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_e < 1.7e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_e}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_e.npy"), rel_err_e) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_e.savefig(os.path.join(results_dir, "e_slices.png")) - else: - plt.show() diff --git a/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling/params_curl_curl.py b/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling/params_curl_curl.py index 59d830c1e..02d3e5808 100644 --- a/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling/params_curl_curl.py +++ b/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling/params_curl_curl.py @@ -4,13 +4,50 @@ # Please fill in a verbal description of the simulation. # It will be printed at the beginning of the simulation and can be used to keep track of the different runs. -name = "Curl-curl strong scaling on hollow cylinder, no preconditioner" -description = """ +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# `--precond` selects the preconditioner (the variant), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--precond", + default="None", + choices=["None", "HiptmairXu", "MultiGrid", "StiffnessPreconditioner"], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 64, 64], help="Number of elements per direction.") +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +# preconditioner -> (name suffix, linear solver) for the name and the description +PRECONDS = { + None: ("no preconditioner", "unpreconditioned CG"), + "HiptmairXu": ( + "Hiptmair-Xu preconditioner", + "CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner)", + ), + "MultiGrid": ( + "multigrid preconditioner", + "CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair", + ), + "StiffnessPreconditioner": ( + "stiffness preconditioner", + "CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner)", + ), +} + +name = f"Curl-curl strong scaling on hollow cylinder, {PRECONDS[precond][0]}" +description = f""" Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4). The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with unpreconditioned CG. +The linear system is solved with {PRECONDS[precond][1]}. """ import logging @@ -19,7 +56,6 @@ set_logging_level(logging.INFO) -import argparse # ------------------ # Import Struphy API @@ -55,13 +91,6 @@ # -------------------------- # Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - env = EnvironmentOptions( sim_folder=f"sim_{args.id:02d}", restart=False, @@ -80,7 +109,7 @@ equil = None # Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) +grid = grids.TensorProductGrid(num_elements=tuple(args.num_elements), mpi_dims_mask=(True, True, True)) # Derham options derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) @@ -114,11 +143,11 @@ # coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition sigma = 1e-2 -solver_params = SolverParameters(tol=1e-8, maxiter=10000, info=True, recycle=False) +solver_params = SolverParameters(tol=1e-8, maxiter=10000 if precond is None else 3000, info=True, recycle=False) model.propagators.curl_curl.options = model.propagators.curl_curl.Options( sigma=sigma, solver="pcg", - precond=None, + precond=precond, solver_params=solver_params, ) diff --git a/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling_hiptmair_xu/params_curl_curl.py b/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling_hiptmair_xu/params_curl_curl.py deleted file mode 100644 index e2bc2a892..000000000 --- a/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling_hiptmair_xu/params_curl_curl.py +++ /dev/null @@ -1,253 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Curl-curl strong scaling on hollow cylinder, Hiptmair-Xu preconditioner" -description = """ -Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4). -The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant -vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. -Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner). -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - ProfilingOptions, - Simulation, - Time, - domains, - grids, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import CurlCurl - -# Units -base_units = BaseUnits() - -# Model instance -model = CurlCurl(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.e_field.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -a1 = 1e-2 -a2 = 1.0 -Lz = 4.0 -domain = domains.HollowCylinder(a1=a1, a2=a2, Lz=Lz) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -# coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition -sigma = 1e-2 - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.curl_curl.options = model.propagators.curl_curl.Options( - sigma=sigma, - solver="pcg", - precond="HiptmairXu", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np -import sympy as sym - -from struphy.initial.base import GenericPerturbation - -# The exact solution is the vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a -# localized Gaussian g (below). It is negligible at the boundary, hence satisfies the boundary conditions -# up to that accuracy. Being localized, it excites the whole spectrum of the curl-curl operator, including -# gradients, so CG has to work for convergence. -# The right-hand side J = curl curl E + sigma E is computed symbolically. Both are given in Cartesian -# components (given_in_basis="physical"). -x_, y_, z_ = sym.symbols("x y z", real=True) -# Gaussian of width 0.15 centered at mid radius, (0.5, 0, 2) -g = sym.exp(-((x_ - 0.5) ** 2 + y_**2 + (z_ - 2.0) ** 2) / 0.15**2) -c = np.array([1.0, 2.0, -1.0]) / np.sqrt(6.0) - - -def curl(F): - """Curl of a Cartesian vector field (list of sympy expressions).""" - return [ - sym.diff(F[2], y_) - sym.diff(F[1], z_), - sym.diff(F[0], z_) - sym.diff(F[2], x_), - sym.diff(F[1], x_) - sym.diff(F[0], y_), - ] - - -E_sym = [ck * g for ck in c] -J_sym = [cc + sigma * e for cc, e in zip(curl(curl(E_sym)), E_sym)] -E_fun = [sym.lambdify((x_, y_, z_), e, "numpy") for e in E_sym] -J_fun = [sym.lambdify((x_, y_, z_), j, "numpy") for j in J_sym] - - -def exact_solution(comp): - return lambda x, y, z: E_fun[comp](x, y, z) + 0.0 * x - - -def rhs(comp): - return lambda x, y, z: J_fun[comp](x, y, z) + 0.0 * x - - -for comp in range(3): - model.em_fields.source.add_perturbation(GenericPerturbation(rhs(comp), given_in_basis="physical", comp=comp)) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True, physical=True) - - from matplotlib import pyplot as plt - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error of the magnitude on the logical slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(data, shading="gouraud") - plt.colorbar() - plt.xlabel(f"eta{[2, 1, 1][k]} index") - plt.ylabel(f"eta{[3, 3, 2][k]} index") - plt.title(f"|{name}| {what}, slice at eta{k + 1} = 0.5") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Cartesian components of the fields at the cell centres of the simulation grid - e_xyz = out.evaluate("em_fields/e_field_xyz", t=-1) - j_xyz = out.evaluate("em_fields/source_xyz", t=0) - X, Y, Z = (e_xyz.coords[c_].values for c_ in ("X", "Y", "Z")) - - e = np.stack([e_xyz.isel(component=k).values for k in range(3)]) - j = np.stack([j_xyz.isel(component=k).values for k in range(3)]) - e_exact = np.stack([exact_solution(k)(X, Y, Z) for k in range(3)]) - j_exact = np.stack([rhs(k)(X, Y, Z) for k in range(3)]) - - fig_rhs = plot_slices(np.linalg.norm(j, axis=0), np.linalg.norm(j_exact, axis=0), "J") - fig_e = plot_slices(np.linalg.norm(e, axis=0), np.linalg.norm(e_exact, axis=0), "E") - - rel_err_rhs = np.max(np.abs(j - j_exact)) / np.max(np.abs(j_exact)) - rel_err_e = np.max(np.abs(e - e_exact)) / np.max(np.abs(e_exact)) - - print(f"Max relative error in J: {rel_err_rhs:.2e}") - print(f"Max relative error in E: {rel_err_e:.2e}") - - assert rel_err_rhs < 3e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_e < 3e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_e}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_e.npy"), rel_err_e) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_e.savefig(os.path.join(results_dir, "e_slices.png")) - else: - plt.show() diff --git a/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling_multigrid/params_curl_curl.py b/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling_multigrid/params_curl_curl.py deleted file mode 100644 index be8b03c46..000000000 --- a/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling_multigrid/params_curl_curl.py +++ /dev/null @@ -1,253 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Curl-curl strong scaling on hollow cylinder, multigrid preconditioner" -description = """ -Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4). -The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant -vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. -Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair. -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - ProfilingOptions, - Simulation, - Time, - domains, - grids, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import CurlCurl - -# Units -base_units = BaseUnits() - -# Model instance -model = CurlCurl(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.e_field.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -a1 = 1e-2 -a2 = 1.0 -Lz = 4.0 -domain = domains.HollowCylinder(a1=a1, a2=a2, Lz=Lz) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -# coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition -sigma = 1e-2 - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.curl_curl.options = model.propagators.curl_curl.Options( - sigma=sigma, - solver="pcg", - precond="MultiGrid", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np -import sympy as sym - -from struphy.initial.base import GenericPerturbation - -# The exact solution is the vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a -# localized Gaussian g (below). It is negligible at the boundary, hence satisfies the boundary conditions -# up to that accuracy. Being localized, it excites the whole spectrum of the curl-curl operator, including -# gradients, so CG has to work for convergence. -# The right-hand side J = curl curl E + sigma E is computed symbolically. Both are given in Cartesian -# components (given_in_basis="physical"). -x_, y_, z_ = sym.symbols("x y z", real=True) -# Gaussian of width 0.15 centered at mid radius, (0.5, 0, 2) -g = sym.exp(-((x_ - 0.5) ** 2 + y_**2 + (z_ - 2.0) ** 2) / 0.15**2) -c = np.array([1.0, 2.0, -1.0]) / np.sqrt(6.0) - - -def curl(F): - """Curl of a Cartesian vector field (list of sympy expressions).""" - return [ - sym.diff(F[2], y_) - sym.diff(F[1], z_), - sym.diff(F[0], z_) - sym.diff(F[2], x_), - sym.diff(F[1], x_) - sym.diff(F[0], y_), - ] - - -E_sym = [ck * g for ck in c] -J_sym = [cc + sigma * e for cc, e in zip(curl(curl(E_sym)), E_sym)] -E_fun = [sym.lambdify((x_, y_, z_), e, "numpy") for e in E_sym] -J_fun = [sym.lambdify((x_, y_, z_), j, "numpy") for j in J_sym] - - -def exact_solution(comp): - return lambda x, y, z: E_fun[comp](x, y, z) + 0.0 * x - - -def rhs(comp): - return lambda x, y, z: J_fun[comp](x, y, z) + 0.0 * x - - -for comp in range(3): - model.em_fields.source.add_perturbation(GenericPerturbation(rhs(comp), given_in_basis="physical", comp=comp)) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True, physical=True) - - from matplotlib import pyplot as plt - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error of the magnitude on the logical slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(data, shading="gouraud") - plt.colorbar() - plt.xlabel(f"eta{[2, 1, 1][k]} index") - plt.ylabel(f"eta{[3, 3, 2][k]} index") - plt.title(f"|{name}| {what}, slice at eta{k + 1} = 0.5") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Cartesian components of the fields at the cell centres of the simulation grid - e_xyz = out.evaluate("em_fields/e_field_xyz", t=-1) - j_xyz = out.evaluate("em_fields/source_xyz", t=0) - X, Y, Z = (e_xyz.coords[c_].values for c_ in ("X", "Y", "Z")) - - e = np.stack([e_xyz.isel(component=k).values for k in range(3)]) - j = np.stack([j_xyz.isel(component=k).values for k in range(3)]) - e_exact = np.stack([exact_solution(k)(X, Y, Z) for k in range(3)]) - j_exact = np.stack([rhs(k)(X, Y, Z) for k in range(3)]) - - fig_rhs = plot_slices(np.linalg.norm(j, axis=0), np.linalg.norm(j_exact, axis=0), "J") - fig_e = plot_slices(np.linalg.norm(e, axis=0), np.linalg.norm(e_exact, axis=0), "E") - - rel_err_rhs = np.max(np.abs(j - j_exact)) / np.max(np.abs(j_exact)) - rel_err_e = np.max(np.abs(e - e_exact)) / np.max(np.abs(e_exact)) - - print(f"Max relative error in J: {rel_err_rhs:.2e}") - print(f"Max relative error in E: {rel_err_e:.2e}") - - assert rel_err_rhs < 3e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_e < 3e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_e}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_e.npy"), rel_err_e) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_e.savefig(os.path.join(results_dir, "e_slices.png")) - else: - plt.show() diff --git a/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling_stiffness/params_curl_curl.py b/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling_stiffness/params_curl_curl.py deleted file mode 100644 index 5cf463f18..000000000 --- a/profiling/examples/CurlCurl/hollow_cylinder_strong_scaling_stiffness/params_curl_curl.py +++ /dev/null @@ -1,253 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Curl-curl strong scaling on hollow cylinder, stiffness preconditioner" -description = """ -Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4). -The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant -vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. -Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner). -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - ProfilingOptions, - Simulation, - Time, - domains, - grids, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import CurlCurl - -# Units -base_units = BaseUnits() - -# Model instance -model = CurlCurl(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.e_field.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -a1 = 1e-2 -a2 = 1.0 -Lz = 4.0 -domain = domains.HollowCylinder(a1=a1, a2=a2, Lz=Lz) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -# coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition -sigma = 1e-2 - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.curl_curl.options = model.propagators.curl_curl.Options( - sigma=sigma, - solver="pcg", - precond="StiffnessPreconditioner", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np -import sympy as sym - -from struphy.initial.base import GenericPerturbation - -# The exact solution is the vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a -# localized Gaussian g (below). It is negligible at the boundary, hence satisfies the boundary conditions -# up to that accuracy. Being localized, it excites the whole spectrum of the curl-curl operator, including -# gradients, so CG has to work for convergence. -# The right-hand side J = curl curl E + sigma E is computed symbolically. Both are given in Cartesian -# components (given_in_basis="physical"). -x_, y_, z_ = sym.symbols("x y z", real=True) -# Gaussian of width 0.15 centered at mid radius, (0.5, 0, 2) -g = sym.exp(-((x_ - 0.5) ** 2 + y_**2 + (z_ - 2.0) ** 2) / 0.15**2) -c = np.array([1.0, 2.0, -1.0]) / np.sqrt(6.0) - - -def curl(F): - """Curl of a Cartesian vector field (list of sympy expressions).""" - return [ - sym.diff(F[2], y_) - sym.diff(F[1], z_), - sym.diff(F[0], z_) - sym.diff(F[2], x_), - sym.diff(F[1], x_) - sym.diff(F[0], y_), - ] - - -E_sym = [ck * g for ck in c] -J_sym = [cc + sigma * e for cc, e in zip(curl(curl(E_sym)), E_sym)] -E_fun = [sym.lambdify((x_, y_, z_), e, "numpy") for e in E_sym] -J_fun = [sym.lambdify((x_, y_, z_), j, "numpy") for j in J_sym] - - -def exact_solution(comp): - return lambda x, y, z: E_fun[comp](x, y, z) + 0.0 * x - - -def rhs(comp): - return lambda x, y, z: J_fun[comp](x, y, z) + 0.0 * x - - -for comp in range(3): - model.em_fields.source.add_perturbation(GenericPerturbation(rhs(comp), given_in_basis="physical", comp=comp)) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True, physical=True) - - from matplotlib import pyplot as plt - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error of the magnitude on the logical slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(data, shading="gouraud") - plt.colorbar() - plt.xlabel(f"eta{[2, 1, 1][k]} index") - plt.ylabel(f"eta{[3, 3, 2][k]} index") - plt.title(f"|{name}| {what}, slice at eta{k + 1} = 0.5") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Cartesian components of the fields at the cell centres of the simulation grid - e_xyz = out.evaluate("em_fields/e_field_xyz", t=-1) - j_xyz = out.evaluate("em_fields/source_xyz", t=0) - X, Y, Z = (e_xyz.coords[c_].values for c_ in ("X", "Y", "Z")) - - e = np.stack([e_xyz.isel(component=k).values for k in range(3)]) - j = np.stack([j_xyz.isel(component=k).values for k in range(3)]) - e_exact = np.stack([exact_solution(k)(X, Y, Z) for k in range(3)]) - j_exact = np.stack([rhs(k)(X, Y, Z) for k in range(3)]) - - fig_rhs = plot_slices(np.linalg.norm(j, axis=0), np.linalg.norm(j_exact, axis=0), "J") - fig_e = plot_slices(np.linalg.norm(e, axis=0), np.linalg.norm(e_exact, axis=0), "E") - - rel_err_rhs = np.max(np.abs(j - j_exact)) / np.max(np.abs(j_exact)) - rel_err_e = np.max(np.abs(e - e_exact)) / np.max(np.abs(e_exact)) - - print(f"Max relative error in J: {rel_err_rhs:.2e}") - print(f"Max relative error in E: {rel_err_e:.2e}") - - assert rel_err_rhs < 3e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_e < 3e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_e}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_e.npy"), rel_err_e) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_e.savefig(os.path.join(results_dir, "e_slices.png")) - else: - plt.show() diff --git a/profiling/examples/CurlCurl/hollow_torus_strong_scaling/params_curl_curl.py b/profiling/examples/CurlCurl/hollow_torus_strong_scaling/params_curl_curl.py index b409275e1..593d400b3 100644 --- a/profiling/examples/CurlCurl/hollow_torus_strong_scaling/params_curl_curl.py +++ b/profiling/examples/CurlCurl/hollow_torus_strong_scaling/params_curl_curl.py @@ -4,13 +4,50 @@ # Please fill in a verbal description of the simulation. # It will be printed at the beginning of the simulation and can be used to keep track of the different runs. -name = "Curl-curl strong scaling on hollow torus, no preconditioner" -description = """ +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# `--precond` selects the preconditioner (the variant), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--precond", + default="None", + choices=["None", "HiptmairXu", "MultiGrid", "StiffnessPreconditioner"], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 64, 64], help="Number of elements per direction.") +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +# preconditioner -> (name suffix, linear solver) for the name and the description +PRECONDS = { + None: ("no preconditioner", "unpreconditioned CG"), + "HiptmairXu": ( + "Hiptmair-Xu preconditioner", + "CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner)", + ), + "MultiGrid": ( + "multigrid preconditioner", + "CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair", + ), + "StiffnessPreconditioner": ( + "stiffness preconditioner", + "CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner)", + ), +} + +name = f"Curl-curl strong scaling on hollow torus, {PRECONDS[precond][0]}" +description = f""" Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4). The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with unpreconditioned CG. +The linear system is solved with {PRECONDS[precond][1]}. """ import logging @@ -19,7 +56,6 @@ set_logging_level(logging.INFO) -import argparse # ------------------ # Import Struphy API @@ -55,13 +91,6 @@ # -------------------------- # Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - env = EnvironmentOptions( sim_folder=f"sim_{args.id:02d}", restart=False, @@ -81,7 +110,7 @@ equil = None # Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) +grid = grids.TensorProductGrid(num_elements=tuple(args.num_elements), mpi_dims_mask=(True, True, True)) # Derham options derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) @@ -115,11 +144,11 @@ # coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition sigma = 1e-2 -solver_params = SolverParameters(tol=1e-8, maxiter=10000, info=True, recycle=False) +solver_params = SolverParameters(tol=1e-8, maxiter=10000 if precond is None else 3000, info=True, recycle=False) model.propagators.curl_curl.options = model.propagators.curl_curl.Options( sigma=sigma, solver="pcg", - precond=None, + precond=precond, solver_params=solver_params, ) diff --git a/profiling/examples/CurlCurl/hollow_torus_strong_scaling_multigrid/params_curl_curl.py b/profiling/examples/CurlCurl/hollow_torus_strong_scaling_multigrid/params_curl_curl.py deleted file mode 100644 index bbd1ff3c8..000000000 --- a/profiling/examples/CurlCurl/hollow_torus_strong_scaling_multigrid/params_curl_curl.py +++ /dev/null @@ -1,258 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Curl-curl strong scaling on hollow torus, multigrid preconditioner" -description = """ -Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4). -The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant -vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. -Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair. -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - ProfilingOptions, - Simulation, - Time, - domains, - grids, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import CurlCurl - -# Units -base_units = BaseUnits() - -# Model instance -model = CurlCurl(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.e_field.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -a1 = 1e-2 -a2 = 1.0 -R0 = 4.0 -tor_period = 1 -domain = domains.HollowTorus(a1=a1, a2=a2, R0=R0, sfl=False, pol_period=1, tor_period=tor_period) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -# coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition -sigma = 1e-2 - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.curl_curl.options = model.propagators.curl_curl.Options( - sigma=sigma, - solver="pcg", - precond="MultiGrid", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np -import sympy as sym - -from struphy.initial.base import GenericPerturbation - -# The exact solution is the vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a -# localized Gaussian g (below). It is negligible at the boundary, hence satisfies the boundary conditions -# up to that accuracy. Being localized, it excites the whole spectrum of the curl-curl operator, including -# gradients, so CG has to work for convergence. -# The right-hand side J = curl curl E + sigma E is computed symbolically. Both are given in Cartesian -# components (given_in_basis="physical"). -x_, y_, z_ = sym.symbols("x y z", real=True) -# Gaussian following the torus, centered at mid radius on the outboard side (theta = phi = 0, R = R0 + 0.5): -# width 0.15 in the poloidal plane and 1.5 along the toroidal arc length (Rc * phi, with phi**2 replaced by -# the smooth periodic 2 * (1 - cos(phi))), where the elements are longest (2*pi*R / 64 ~ 0.44) -Rc = R0 + 0.5 -R_ = sym.sqrt(x_**2 + y_**2) -g = sym.exp(-((R_ - Rc) ** 2 + z_**2) / 0.15**2 - 2 * Rc**2 * (1 - x_ / R_) / 1.5**2) -c = np.array([1.0, 2.0, -1.0]) / np.sqrt(6.0) - - -def curl(F): - """Curl of a Cartesian vector field (list of sympy expressions).""" - return [ - sym.diff(F[2], y_) - sym.diff(F[1], z_), - sym.diff(F[0], z_) - sym.diff(F[2], x_), - sym.diff(F[1], x_) - sym.diff(F[0], y_), - ] - - -E_sym = [ck * g for ck in c] -J_sym = [cc + sigma * e for cc, e in zip(curl(curl(E_sym)), E_sym)] -E_fun = [sym.lambdify((x_, y_, z_), e, "numpy") for e in E_sym] -J_fun = [sym.lambdify((x_, y_, z_), j, "numpy") for j in J_sym] - - -def exact_solution(comp): - return lambda x, y, z: E_fun[comp](x, y, z) + 0.0 * x - - -def rhs(comp): - return lambda x, y, z: J_fun[comp](x, y, z) + 0.0 * x - - -for comp in range(3): - model.em_fields.source.add_perturbation(GenericPerturbation(rhs(comp), given_in_basis="physical", comp=comp)) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True, physical=True) - - from matplotlib import pyplot as plt - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error of the magnitude on the logical slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(data, shading="gouraud") - plt.colorbar() - plt.xlabel(f"eta{[2, 1, 1][k]} index") - plt.ylabel(f"eta{[3, 3, 2][k]} index") - plt.title(f"|{name}| {what}, slice at eta{k + 1} = 0.5") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Cartesian components of the fields at the cell centres of the simulation grid - e_xyz = out.evaluate("em_fields/e_field_xyz", t=-1) - j_xyz = out.evaluate("em_fields/source_xyz", t=0) - X, Y, Z = (e_xyz.coords[c_].values for c_ in ("X", "Y", "Z")) - - e = np.stack([e_xyz.isel(component=k).values for k in range(3)]) - j = np.stack([j_xyz.isel(component=k).values for k in range(3)]) - e_exact = np.stack([exact_solution(k)(X, Y, Z) for k in range(3)]) - j_exact = np.stack([rhs(k)(X, Y, Z) for k in range(3)]) - - fig_rhs = plot_slices(np.linalg.norm(j, axis=0), np.linalg.norm(j_exact, axis=0), "J") - fig_e = plot_slices(np.linalg.norm(e, axis=0), np.linalg.norm(e_exact, axis=0), "E") - - rel_err_rhs = np.max(np.abs(j - j_exact)) / np.max(np.abs(j_exact)) - rel_err_e = np.max(np.abs(e - e_exact)) / np.max(np.abs(e_exact)) - - print(f"Max relative error in J: {rel_err_rhs:.2e}") - print(f"Max relative error in E: {rel_err_e:.2e}") - - assert rel_err_rhs < 8e-3, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_e < 2e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_e}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_e.npy"), rel_err_e) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_e.savefig(os.path.join(results_dir, "e_slices.png")) - else: - plt.show() diff --git a/profiling/examples/CurlCurl/hollow_torus_strong_scaling_stiffness/params_curl_curl.py b/profiling/examples/CurlCurl/hollow_torus_strong_scaling_stiffness/params_curl_curl.py deleted file mode 100644 index ea14727d4..000000000 --- a/profiling/examples/CurlCurl/hollow_torus_strong_scaling_stiffness/params_curl_curl.py +++ /dev/null @@ -1,258 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Curl-curl strong scaling on hollow torus, stiffness preconditioner" -description = """ -Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4). -The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant -vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. -Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner). -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - ProfilingOptions, - Simulation, - Time, - domains, - grids, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import CurlCurl - -# Units -base_units = BaseUnits() - -# Model instance -model = CurlCurl(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.e_field.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -a1 = 1e-2 -a2 = 1.0 -R0 = 4.0 -tor_period = 1 -domain = domains.HollowTorus(a1=a1, a2=a2, R0=R0, sfl=False, pol_period=1, tor_period=tor_period) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -# coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition -sigma = 1e-2 - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.curl_curl.options = model.propagators.curl_curl.Options( - sigma=sigma, - solver="pcg", - precond="StiffnessPreconditioner", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np -import sympy as sym - -from struphy.initial.base import GenericPerturbation - -# The exact solution is the vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a -# localized Gaussian g (below). It is negligible at the boundary, hence satisfies the boundary conditions -# up to that accuracy. Being localized, it excites the whole spectrum of the curl-curl operator, including -# gradients, so CG has to work for convergence. -# The right-hand side J = curl curl E + sigma E is computed symbolically. Both are given in Cartesian -# components (given_in_basis="physical"). -x_, y_, z_ = sym.symbols("x y z", real=True) -# Gaussian following the torus, centered at mid radius on the outboard side (theta = phi = 0, R = R0 + 0.5): -# width 0.15 in the poloidal plane and 1.5 along the toroidal arc length (Rc * phi, with phi**2 replaced by -# the smooth periodic 2 * (1 - cos(phi))), where the elements are longest (2*pi*R / 64 ~ 0.44) -Rc = R0 + 0.5 -R_ = sym.sqrt(x_**2 + y_**2) -g = sym.exp(-((R_ - Rc) ** 2 + z_**2) / 0.15**2 - 2 * Rc**2 * (1 - x_ / R_) / 1.5**2) -c = np.array([1.0, 2.0, -1.0]) / np.sqrt(6.0) - - -def curl(F): - """Curl of a Cartesian vector field (list of sympy expressions).""" - return [ - sym.diff(F[2], y_) - sym.diff(F[1], z_), - sym.diff(F[0], z_) - sym.diff(F[2], x_), - sym.diff(F[1], x_) - sym.diff(F[0], y_), - ] - - -E_sym = [ck * g for ck in c] -J_sym = [cc + sigma * e for cc, e in zip(curl(curl(E_sym)), E_sym)] -E_fun = [sym.lambdify((x_, y_, z_), e, "numpy") for e in E_sym] -J_fun = [sym.lambdify((x_, y_, z_), j, "numpy") for j in J_sym] - - -def exact_solution(comp): - return lambda x, y, z: E_fun[comp](x, y, z) + 0.0 * x - - -def rhs(comp): - return lambda x, y, z: J_fun[comp](x, y, z) + 0.0 * x - - -for comp in range(3): - model.em_fields.source.add_perturbation(GenericPerturbation(rhs(comp), given_in_basis="physical", comp=comp)) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True, physical=True) - - from matplotlib import pyplot as plt - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error of the magnitude on the logical slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(data, shading="gouraud") - plt.colorbar() - plt.xlabel(f"eta{[2, 1, 1][k]} index") - plt.ylabel(f"eta{[3, 3, 2][k]} index") - plt.title(f"|{name}| {what}, slice at eta{k + 1} = 0.5") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Cartesian components of the fields at the cell centres of the simulation grid - e_xyz = out.evaluate("em_fields/e_field_xyz", t=-1) - j_xyz = out.evaluate("em_fields/source_xyz", t=0) - X, Y, Z = (e_xyz.coords[c_].values for c_ in ("X", "Y", "Z")) - - e = np.stack([e_xyz.isel(component=k).values for k in range(3)]) - j = np.stack([j_xyz.isel(component=k).values for k in range(3)]) - e_exact = np.stack([exact_solution(k)(X, Y, Z) for k in range(3)]) - j_exact = np.stack([rhs(k)(X, Y, Z) for k in range(3)]) - - fig_rhs = plot_slices(np.linalg.norm(j, axis=0), np.linalg.norm(j_exact, axis=0), "J") - fig_e = plot_slices(np.linalg.norm(e, axis=0), np.linalg.norm(e_exact, axis=0), "E") - - rel_err_rhs = np.max(np.abs(j - j_exact)) / np.max(np.abs(j_exact)) - rel_err_e = np.max(np.abs(e - e_exact)) / np.max(np.abs(e_exact)) - - print(f"Max relative error in J: {rel_err_rhs:.2e}") - print(f"Max relative error in E: {rel_err_e:.2e}") - - assert rel_err_rhs < 8e-3, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_e < 2e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_e}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_e.npy"), rel_err_e) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_e.savefig(os.path.join(results_dir, "e_slices.png")) - else: - plt.show() diff --git a/profiling/examples/CurlCurl/cube_strong_scaling_hiptmair_xu/params_curl_curl.py b/profiling/examples/CurlCurl/polar_cylinder_strong_scaling/params_curl_curl.py similarity index 52% rename from profiling/examples/CurlCurl/cube_strong_scaling_hiptmair_xu/params_curl_curl.py rename to profiling/examples/CurlCurl/polar_cylinder_strong_scaling/params_curl_curl.py index ee3e65753..ceb518280 100644 --- a/profiling/examples/CurlCurl/cube_strong_scaling_hiptmair_xu/params_curl_curl.py +++ b/profiling/examples/CurlCurl/polar_cylinder_strong_scaling/params_curl_curl.py @@ -4,13 +4,59 @@ # Please fill in a verbal description of the simulation. # It will be printed at the beginning of the simulation and can be used to keep track of the different runs. -name = "Curl-curl strong scaling on 3D cube, Hiptmair-Xu preconditioner" -description = """ -Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a 3D cube (Cuboid). -The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant -vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. -Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner). +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# The other flags select the variant (spline space and preconditioner), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--splines", + default="polar", + choices=["polar", "tensor"], + help="C1 polar splines on a disc (IGAPolarCylinder) or tensor-product splines on a cylinder with a hole (HollowCylinder).", +) +parser.add_argument( + "--precond", + default="None", + choices=[ + "None", + "MassMatrixPreconditioner", + "PolarMassMatrixPreconditioner", + "StiffnessPreconditioner", + "MultiGrid", + "HiptmairXu", + ], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument( + "--poloidal-solver", + default="schur", + choices=["schur", "lu", "ilu"], + help="Poloidal solver of the polar preconditioners and projectors.", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 66, 64], help="Number of elements per direction.") +parser.add_argument( + "--hole-bc", + default="dirichlet", + choices=["free", "dirichlet"], + help="Boundary condition at the hole of the tensor-product domain (default dirichlet; the natural boundary condition contradicts the solution, whose curl is largest on the axis, and the error is amplified by 1/sigma).", +) +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +name = f"Curl-curl strong scaling on a cylinder, {args.splines} splines, preconditioner {precond}" +description = f""" +Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a cylinder of radius 1 and length 4, +comparing C1 polar splines (IGAPolarCylinder, no hole) with tensor-product splines on a cylinder with a hole +(HollowCylinder, a1=1e-2). The manufactured solution E = g c (constant Cartesian vector c) has a Gaussian g whose +gradient is largest on the axis r=0. +Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set at r=1 (boundary condition {args.hole_bc} at the hole). +The linear system is solved with CG, preconditioner {precond} (poloidal solver {args.poloidal_solver}). """ import logging @@ -19,8 +65,6 @@ set_logging_level(logging.INFO) -import argparse - # ------------------ # Import Struphy API # ------------------ @@ -55,13 +99,6 @@ # -------------------------- # Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - env = EnvironmentOptions( sim_folder=f"sim_{args.id:02d}", restart=False, @@ -70,22 +107,32 @@ # Time stepping time_opts = Time() -# Geometry -Lx = 2.0 -Ly = 3.0 +# Grid (polar splines need a multiple of 3 elements in eta2) +num_elements = tuple(args.num_elements) +degree = (3, 3, 3) +grid = grids.TensorProductGrid(num_elements=num_elements, mpi_dims_mask=(True, True, True)) + +# Geometry and Derham options +a = 1.0 Lz = 4.0 -domain = domains.Cuboid(r1=Lx, l2=-Ly / 2, r2=Ly / 2, r3=Lz) +if args.splines == "polar": + # the spline mapping has the resolution of the grid in the poloidal plane + domain = domains.IGAPolarCylinder(num_elements=num_elements[:2], degree=degree[:2], a=a, Lz=Lz) + derham_opts = DerhamOptions( + degree=degree, + bcs=(("free", "dirichlet"), None, None), + polar_splines=True, + polar_poloidal_solver=args.poloidal_solver, + ) +else: + a1 = 1e-2 + domain = domains.HollowCylinder(a1=a1, a2=a, Lz=Lz) + derham_opts = DerhamOptions(degree=degree, bcs=((args.hole_bc, "dirichlet"), None, None)) # Fluid equilibrium (can be used as part of initial conditions) equil = None -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options +# Profiling options profiling_opts = ProfilingOptions( use_line_profiler=True, ) @@ -114,11 +161,17 @@ # coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition sigma = 1e-2 -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) +# options of the polar preconditioners +precond_params = {} +if args.splines == "polar" and precond in ("PolarMassMatrixPreconditioner", "StiffnessPreconditioner"): + precond_params["poloidal_solver"] = args.poloidal_solver + +solver_params = SolverParameters(tol=1e-8, maxiter=10000, info=True, recycle=False) model.propagators.curl_curl.options = model.propagators.curl_curl.Options( sigma=sigma, solver="pcg", - precond="HiptmairXu", + precond=precond, + precond_params=precond_params, solver_params=solver_params, ) @@ -130,15 +183,18 @@ from struphy.initial.base import GenericPerturbation -# The exact solution is the vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a -# localized Gaussian g (below). It is negligible at the boundary, hence satisfies the boundary conditions -# up to that accuracy. Being localized, it excites the whole spectrum of the curl-curl operator, including -# gradients, so CG has to work for convergence. +# The exact solution is the vector field E = g c in Cartesian coordinates, with a constant vector c and +# g(x, y, z) = Gaussian of width w in the cross-section, centered at distance w / sqrt(2) from the axis, where its +# gradient is largest, so the gradient is largest on the axis r = 0, times a periodic modulation along the axis. +# It is negligible at r = 1, hence satisfies the boundary conditions up to that accuracy. +# Tensor-product splines cannot represent non-zero fields and gradients across the axis (they live on a domain +# with a hole and a boundary condition there, see --hole-bc); polar splines can. # The right-hand side J = curl curl E + sigma E is computed symbolically. Both are given in Cartesian # components (given_in_basis="physical"). x_, y_, z_ = sym.symbols("x y z", real=True) -# Gaussian of width 0.15 centered at (1, 0, 2) -g = sym.exp(-((x_ - 1.0) ** 2 + y_**2 + (z_ - 2.0) ** 2) / 0.15**2) +w = 0.2 +x0 = w / np.sqrt(2.0) +g = sym.exp(-((x_ - x0) ** 2 + y_**2) / w**2) * (1 + sym.cos(2 * sym.pi * z_ / Lz) / 2) c = np.array([1.0, 2.0, -1.0]) / np.sqrt(6.0) @@ -165,6 +221,10 @@ def rhs(comp): return lambda x, y, z: J_fun[comp](x, y, z) + 0.0 * x +def distance_to_axis(x, y, z): + return np.sqrt(x**2 + y**2) + + for comp in range(3): model.em_fields.source.add_perturbation(GenericPerturbation(rhs(comp), given_in_basis="physical", comp=comp)) @@ -192,19 +252,23 @@ def rhs(comp): from matplotlib import pyplot as plt def plot_slices(num, exact, name): - """Numerical solution, exact solution and error of the magnitude on the logical slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) + """Numerical solution, exact solution and error of the magnitude on the cross-section at eta3 = 0.5 and the + half plane at eta2 = 0.5, as functions of (x, y) and (x, z).""" + fig = plt.figure(figsize=(12, 12)) + idx3 = (slice(None), slice(None), num.shape[2] // 2) + idx2 = (slice(None), num.shape[1] // 2, slice(None)) + for col, (idx, h, v, labels) in enumerate( + [(idx3, X, Y, ("x", "y")), (idx2, X, Z, ("x", "z"))], + ): for row, (data, what) in enumerate( [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(data, shading="gouraud") + plt.subplot(3, 2, 2 * row + col + 1) + plt.pcolormesh(h[idx], v[idx], np.nan_to_num(data), shading="gouraud") plt.colorbar() - plt.xlabel(f"eta{[2, 1, 1][k]} index") - plt.ylabel(f"eta{[3, 3, 2][k]} index") - plt.title(f"|{name}| {what}, slice at eta{k + 1} = 0.5") + plt.xlabel(labels[0]) + plt.ylabel(labels[1]) + plt.title(f"|{name}| {what}") fig.tight_layout() return fig @@ -222,17 +286,19 @@ def plot_slices(num, exact, name): fig_rhs = plot_slices(np.linalg.norm(j, axis=0), np.linalg.norm(j_exact, axis=0), "J") fig_e = plot_slices(np.linalg.norm(e, axis=0), np.linalg.norm(e_exact, axis=0), "E") - rel_err_rhs = np.max(np.abs(j - j_exact)) / np.max(np.abs(j_exact)) - rel_err_e = np.max(np.abs(e - e_exact)) / np.max(np.abs(e_exact)) + # the Cartesian components are not defined on the pole (singular mapping), which is left out + valid = np.isfinite(e).all(axis=0) & np.isfinite(j).all(axis=0) + rel_err_rhs = np.max(np.abs(j - j_exact)[:, valid]) / np.max(np.abs(j_exact)) + rel_err_e = np.max(np.abs(e - e_exact)[:, valid]) / np.max(np.abs(e_exact)) + # error close to the axis, where the gradient of the solution is largest + near_axis = valid & (distance_to_axis(X, Y, Z) < 0.1) + rel_err_e_axis = np.max(np.abs(e - e_exact)[:, near_axis]) / np.max(np.abs(e_exact)) print(f"Max relative error in J: {rel_err_rhs:.2e}") print(f"Max relative error in E: {rel_err_e:.2e}") + print(f"Max relative error in E close to the axis (r < 0.1): {rel_err_e_axis:.2e}") - assert rel_err_rhs < 2.5e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_e < 1.7e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_e}." - ) - + import json import os # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare @@ -243,8 +309,19 @@ def plot_slices(num, exact, name): np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) np.save(os.path.join(results_dir, "rel_err_e.npy"), rel_err_e) + np.save(os.path.join(results_dir, "rel_err_e_axis.npy"), rel_err_e_axis) np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) + with open(os.path.join(results_dir, "variant.json"), "w") as f: + json.dump( + { + "splines": args.splines, + "precond": args.precond, + "poloidal_solver": args.poloidal_solver, + "hole_bc": args.hole_bc, + }, + f, + ) if save_figs: fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) diff --git a/profiling/examples/CurlCurl/hollow_torus_strong_scaling_hiptmair_xu/params_curl_curl.py b/profiling/examples/CurlCurl/polar_torus_strong_scaling/params_curl_curl.py similarity index 52% rename from profiling/examples/CurlCurl/hollow_torus_strong_scaling_hiptmair_xu/params_curl_curl.py rename to profiling/examples/CurlCurl/polar_torus_strong_scaling/params_curl_curl.py index 5b72450e0..d7f968b5f 100644 --- a/profiling/examples/CurlCurl/hollow_torus_strong_scaling_hiptmair_xu/params_curl_curl.py +++ b/profiling/examples/CurlCurl/polar_torus_strong_scaling/params_curl_curl.py @@ -4,13 +4,59 @@ # Please fill in a verbal description of the simulation. # It will be printed at the beginning of the simulation and can be used to keep track of the different runs. -name = "Curl-curl strong scaling on hollow torus, Hiptmair-Xu preconditioner" -description = """ -Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4). -The manufactured solution is a Gaussian vector field E = g(x) c in Cartesian coordinates, with a constant -vector c and a Gaussian g centered at x0 inside the domain, exciting the full spectrum. -Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set in direction eta1. -The linear system is solved with CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner). +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# The other flags select the variant (spline space and preconditioner), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--splines", + default="polar", + choices=["polar", "tensor"], + help="C1 polar splines on a full torus (IGAPolarTorus) or tensor-product splines on a torus with a hole (HollowTorus).", +) +parser.add_argument( + "--precond", + default="None", + choices=[ + "None", + "MassMatrixPreconditioner", + "PolarMassMatrixPreconditioner", + "StiffnessPreconditioner", + "MultiGrid", + "HiptmairXu", + ], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument( + "--poloidal-solver", + default="schur", + choices=["schur", "lu", "ilu"], + help="Poloidal solver of the polar preconditioners and projectors.", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 66, 64], help="Number of elements per direction.") +parser.add_argument( + "--hole-bc", + default="dirichlet", + choices=["free", "dirichlet"], + help="Boundary condition at the hole of the tensor-product domain (default dirichlet; the natural boundary condition contradicts the solution, whose curl is largest on the axis, and the error is amplified by 1/sigma).", +) +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +name = f"Curl-curl strong scaling on a torus, {args.splines} splines, preconditioner {precond}" +description = f""" +Strong scaling test for the curl-curl problem curl curl E + sigma E = J on a full torus of minor radius 1 and major +radius 4, comparing C1 polar splines (IGAPolarTorus, no hole) with tensor-product splines on a torus with a hole +(HollowTorus, a1=1e-2). The manufactured solution E = g c (constant Cartesian vector c) has a Gaussian ring g whose +gradient is largest on the magnetic axis r=0. +Homogeneous Dirichlet boundary conditions (vanishing tangential E) are set at r=1 (boundary condition {args.hole_bc} at the hole). +The linear system is solved with CG, preconditioner {precond} (poloidal solver {args.poloidal_solver}). """ import logging @@ -19,8 +65,6 @@ set_logging_level(logging.INFO) -import argparse - # ------------------ # Import Struphy API # ------------------ @@ -55,13 +99,6 @@ # -------------------------- # Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - env = EnvironmentOptions( sim_folder=f"sim_{args.id:02d}", restart=False, @@ -70,23 +107,34 @@ # Time stepping time_opts = Time() -# Geometry -a1 = 1e-2 -a2 = 1.0 +# Grid (polar splines need a multiple of 3 elements in eta2) +num_elements = tuple(args.num_elements) +degree = (3, 3, 3) +grid = grids.TensorProductGrid(num_elements=num_elements, mpi_dims_mask=(True, True, True)) + +# Geometry and Derham options +a = 1.0 R0 = 4.0 -tor_period = 1 -domain = domains.HollowTorus(a1=a1, a2=a2, R0=R0, sfl=False, pol_period=1, tor_period=tor_period) +if args.splines == "polar": + # the spline mapping has the resolution of the grid in the poloidal plane + domain = domains.IGAPolarTorus( + num_elements=num_elements[:2], degree=degree[:2], a=a, R0=R0, sfl=False, tor_period=1 + ) + derham_opts = DerhamOptions( + degree=degree, + bcs=(("free", "dirichlet"), None, None), + polar_splines=True, + polar_poloidal_solver=args.poloidal_solver, + ) +else: + a1 = 1e-2 + domain = domains.HollowTorus(a1=a1, a2=a, R0=R0, sfl=False, pol_period=1, tor_period=1) + derham_opts = DerhamOptions(degree=degree, bcs=((args.hole_bc, "dirichlet"), None, None)) # Fluid equilibrium (can be used as part of initial conditions) equil = None -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(2, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options +# Profiling options profiling_opts = ProfilingOptions( use_line_profiler=True, ) @@ -115,11 +163,17 @@ # coefficient of the mass term; small values make the kernel of the curl (the gradients) hard to precondition sigma = 1e-2 -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) +# options of the polar preconditioners +precond_params = {} +if args.splines == "polar" and precond in ("PolarMassMatrixPreconditioner", "StiffnessPreconditioner"): + precond_params["poloidal_solver"] = args.poloidal_solver + +solver_params = SolverParameters(tol=1e-8, maxiter=10000, info=True, recycle=False) model.propagators.curl_curl.options = model.propagators.curl_curl.Options( sigma=sigma, solver="pcg", - precond="HiptmairXu", + precond=precond, + precond_params=precond_params, solver_params=solver_params, ) @@ -131,19 +185,20 @@ from struphy.initial.base import GenericPerturbation -# The exact solution is the vector field E = g(x) c in Cartesian coordinates, with a constant vector c and a -# localized Gaussian g (below). It is negligible at the boundary, hence satisfies the boundary conditions -# up to that accuracy. Being localized, it excites the whole spectrum of the curl-curl operator, including -# gradients, so CG has to work for convergence. +# The exact solution is the vector field E = g c in Cartesian coordinates, with a constant vector c and +# g = Gaussian ring of width w in the poloidal plane (R = sqrt(x**2 + y**2)), centered at distance w / sqrt(2) from +# the magnetic axis (R0, 0), where its gradient is largest, so the gradient is largest on the axis r = 0, times +# a toroidal modulation 1 + x / (2 R). +# It is negligible at r = 1, hence satisfies the boundary conditions up to that accuracy. +# Tensor-product splines cannot represent non-zero fields and gradients across the axis (they live on a domain +# with a hole and a boundary condition there, see --hole-bc); polar splines can. # The right-hand side J = curl curl E + sigma E is computed symbolically. Both are given in Cartesian # components (given_in_basis="physical"). x_, y_, z_ = sym.symbols("x y z", real=True) -# Gaussian following the torus, centered at mid radius on the outboard side (theta = phi = 0, R = R0 + 0.5): -# width 0.15 in the poloidal plane and 1.5 along the toroidal arc length (Rc * phi, with phi**2 replaced by -# the smooth periodic 2 * (1 - cos(phi))), where the elements are longest (2*pi*R / 64 ~ 0.44) -Rc = R0 + 0.5 +w = 0.2 +x0 = w / np.sqrt(2.0) R_ = sym.sqrt(x_**2 + y_**2) -g = sym.exp(-((R_ - Rc) ** 2 + z_**2) / 0.15**2 - 2 * Rc**2 * (1 - x_ / R_) / 1.5**2) +g = sym.exp(-((R_ - R0 - x0) ** 2 + z_**2) / w**2) * (1 + x_ / (2 * R_)) c = np.array([1.0, 2.0, -1.0]) / np.sqrt(6.0) @@ -170,6 +225,10 @@ def rhs(comp): return lambda x, y, z: J_fun[comp](x, y, z) + 0.0 * x +def distance_to_axis(x, y, z): + return np.sqrt((np.sqrt(x**2 + y**2) - R0) ** 2 + z**2) + + for comp in range(3): model.em_fields.source.add_perturbation(GenericPerturbation(rhs(comp), given_in_basis="physical", comp=comp)) @@ -197,19 +256,23 @@ def rhs(comp): from matplotlib import pyplot as plt def plot_slices(num, exact, name): - """Numerical solution, exact solution and error of the magnitude on the logical slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) + """Numerical solution, exact solution and error of the magnitude on the poloidal plane at eta3 = 0.5 and on + the surface eta2 = 0.5 seen from above, as functions of (R, z) and (x, y).""" + fig = plt.figure(figsize=(12, 12)) + idx3 = (slice(None), slice(None), num.shape[2] // 2) + idx2 = (slice(None), num.shape[1] // 2, slice(None)) + for col, (idx, h, v, labels) in enumerate( + [(idx3, np.sqrt(X**2 + Y**2), Z, ("R", "z")), (idx2, X, Y, ("x", "y"))], + ): for row, (data, what) in enumerate( [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(data, shading="gouraud") + plt.subplot(3, 2, 2 * row + col + 1) + plt.pcolormesh(h[idx], v[idx], np.nan_to_num(data), shading="gouraud") plt.colorbar() - plt.xlabel(f"eta{[2, 1, 1][k]} index") - plt.ylabel(f"eta{[3, 3, 2][k]} index") - plt.title(f"|{name}| {what}, slice at eta{k + 1} = 0.5") + plt.xlabel(labels[0]) + plt.ylabel(labels[1]) + plt.title(f"|{name}| {what}") fig.tight_layout() return fig @@ -227,17 +290,19 @@ def plot_slices(num, exact, name): fig_rhs = plot_slices(np.linalg.norm(j, axis=0), np.linalg.norm(j_exact, axis=0), "J") fig_e = plot_slices(np.linalg.norm(e, axis=0), np.linalg.norm(e_exact, axis=0), "E") - rel_err_rhs = np.max(np.abs(j - j_exact)) / np.max(np.abs(j_exact)) - rel_err_e = np.max(np.abs(e - e_exact)) / np.max(np.abs(e_exact)) + # the Cartesian components are not defined on the pole (singular mapping), which is left out + valid = np.isfinite(e).all(axis=0) & np.isfinite(j).all(axis=0) + rel_err_rhs = np.max(np.abs(j - j_exact)[:, valid]) / np.max(np.abs(j_exact)) + rel_err_e = np.max(np.abs(e - e_exact)[:, valid]) / np.max(np.abs(e_exact)) + # error close to the axis, where the gradient of the solution is largest + near_axis = valid & (distance_to_axis(X, Y, Z) < 0.1) + rel_err_e_axis = np.max(np.abs(e - e_exact)[:, near_axis]) / np.max(np.abs(e_exact)) print(f"Max relative error in J: {rel_err_rhs:.2e}") print(f"Max relative error in E: {rel_err_e:.2e}") + print(f"Max relative error in E close to the axis (r < 0.1): {rel_err_e_axis:.2e}") - assert rel_err_rhs < 8e-3, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_e < 2e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_e}." - ) - + import json import os # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare @@ -248,8 +313,19 @@ def plot_slices(num, exact, name): np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) np.save(os.path.join(results_dir, "rel_err_e.npy"), rel_err_e) + np.save(os.path.join(results_dir, "rel_err_e_axis.npy"), rel_err_e_axis) np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) + with open(os.path.join(results_dir, "variant.json"), "w") as f: + json.dump( + { + "splines": args.splines, + "precond": args.precond, + "poloidal_solver": args.poloidal_solver, + "hole_bc": args.hole_bc, + }, + f, + ) if save_figs: fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) diff --git a/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling.py b/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling.py index 3103ed110..5e789bd8f 100644 --- a/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling.py +++ b/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling.py @@ -1,4 +1,4 @@ -"""Curl-curl strong scaling profiling case on 3D cube, no preconditioner. +"""Curl-curl strong scaling profiling case on 3D cube. This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a @@ -7,6 +7,8 @@ each run as soon as its own job finishes. Each generated script runs the simulation itself by invoking `params_curl_curl.py` directly (its `__main__` block is the worker). + +The variant (preconditioner) is chosen with `--variant`; run this script once per variant. """ import argparse @@ -18,6 +20,35 @@ from profiling_job import ProfilingCase # noqa: E402 +# variant -> (label, name, description, flags of the params file); the labels identify the +# cases in the profiling-data repo and are kept from the former one-case-per-script layout +VARIANTS = { + "none": ( + "curl_curl_cube_strong_scaling", + "Curl-curl on 3D cube strong scaling test, no preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a 3D cube (Cuboid), solved with unpreconditioned CG.", + [], + ), + "hiptmair_xu": ( + "curl_curl_cube_strong_scaling_hiptmair_xu", + "Curl-curl on 3D cube strong scaling test, Hiptmair-Xu preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a 3D cube (Cuboid), solved with CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner).", + ["--precond", "HiptmairXu"], + ), + "multigrid": ( + "curl_curl_cube_strong_scaling_multigrid", + "Curl-curl on 3D cube strong scaling test, multigrid preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a 3D cube (Cuboid), solved with CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair.", + ["--precond", "MultiGrid"], + ), + "stiffness": ( + "curl_curl_cube_strong_scaling_stiffness", + "Curl-curl on 3D cube strong scaling test, stiffness preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a 3D cube (Cuboid), solved with CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner).", + ["--precond", "StiffnessPreconditioner"], + ), +} + def main() -> None: @@ -25,6 +56,12 @@ def main() -> None: parser = argparse.ArgumentParser( description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), ) + parser.add_argument( + "--variant", + default="none", + choices=list(VARIANTS), + help="Preconditioner of the runs.", + ) parser.add_argument( "--upload", action="store_true", @@ -42,15 +79,16 @@ def main() -> None: help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', ) args = parser.parse_args() + label, name, description, param_flags = VARIANTS[args.variant] # Paths relative to this script's location, so it can be run from anywhere. script_dir = Path(__file__).resolve().parent params_dir = script_dir / "cube_strong_scaling" profiling_case = ProfilingCase( - label="curl_curl_cube_strong_scaling", - name="Curl-curl on 3D cube strong scaling test, no preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a 3D cube (Cuboid), solved with unpreconditioned CG.", + label=label, + name=name, + description=description, physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", struphy_model_used="CurlCurl", params_source=params_dir / "params_curl_curl.py", @@ -61,7 +99,7 @@ def main() -> None: # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) + profiling_case.launch(num_tasks, param_flags=param_flags) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling_hiptmair_xu.py b/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling_hiptmair_xu.py deleted file mode 100644 index 2a18c93ba..000000000 --- a/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling_hiptmair_xu.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Curl-curl strong scaling profiling case on 3D cube, Hiptmair-Xu preconditioner. - -This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_curl_curl.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "cube_strong_scaling_hiptmair_xu" - - profiling_case = ProfilingCase( - label="curl_curl_cube_strong_scaling_hiptmair_xu", - name="Curl-curl on 3D cube strong scaling test, Hiptmair-Xu preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a 3D cube (Cuboid), solved with CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner).", - physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", - struphy_model_used="CurlCurl", - params_source=params_dir / "params_curl_curl.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling_multigrid.py b/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling_multigrid.py deleted file mode 100644 index 447870619..000000000 --- a/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling_multigrid.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Curl-curl strong scaling profiling case on 3D cube, multigrid preconditioner. - -This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_curl_curl.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "cube_strong_scaling_multigrid" - - profiling_case = ProfilingCase( - label="curl_curl_cube_strong_scaling_multigrid", - name="Curl-curl on 3D cube strong scaling test, multigrid preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a 3D cube (Cuboid), solved with CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair.", - physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", - struphy_model_used="CurlCurl", - params_source=params_dir / "params_curl_curl.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling_stiffness.py b/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling_stiffness.py deleted file mode 100644 index 00c0bf27a..000000000 --- a/profiling/examples/CurlCurl/submit_curl_curl_cube_strong_scaling_stiffness.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Curl-curl strong scaling profiling case on 3D cube, stiffness preconditioner. - -This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_curl_curl.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "cube_strong_scaling_stiffness" - - profiling_case = ProfilingCase( - label="curl_curl_cube_strong_scaling_stiffness", - name="Curl-curl on 3D cube strong scaling test, stiffness preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a 3D cube (Cuboid), solved with CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner).", - physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", - struphy_model_used="CurlCurl", - params_source=params_dir / "params_curl_curl.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling.py b/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling.py index 438fb8adc..bce7d037f 100644 --- a/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling.py +++ b/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling.py @@ -1,4 +1,4 @@ -"""Curl-curl strong scaling profiling case on hollow cylinder, no preconditioner. +"""Curl-curl strong scaling profiling case on hollow cylinder. This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a @@ -7,6 +7,8 @@ each run as soon as its own job finishes. Each generated script runs the simulation itself by invoking `params_curl_curl.py` directly (its `__main__` block is the worker). + +The variant (preconditioner) is chosen with `--variant`; run this script once per variant. """ import argparse @@ -18,6 +20,35 @@ from profiling_job import ProfilingCase # noqa: E402 +# variant -> (label, name, description, flags of the params file); the labels identify the +# cases in the profiling-data repo and are kept from the former one-case-per-script layout +VARIANTS = { + "none": ( + "curl_curl_hollow_cylinder_strong_scaling", + "Curl-curl on hollow cylinder strong scaling test, no preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with unpreconditioned CG.", + [], + ), + "hiptmair_xu": ( + "curl_curl_hollow_cylinder_strong_scaling_hiptmair_xu", + "Curl-curl on hollow cylinder strong scaling test, Hiptmair-Xu preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner).", + ["--precond", "HiptmairXu"], + ), + "multigrid": ( + "curl_curl_hollow_cylinder_strong_scaling_multigrid", + "Curl-curl on hollow cylinder strong scaling test, multigrid preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair.", + ["--precond", "MultiGrid"], + ), + "stiffness": ( + "curl_curl_hollow_cylinder_strong_scaling_stiffness", + "Curl-curl on hollow cylinder strong scaling test, stiffness preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner).", + ["--precond", "StiffnessPreconditioner"], + ), +} + def main() -> None: @@ -25,6 +56,12 @@ def main() -> None: parser = argparse.ArgumentParser( description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), ) + parser.add_argument( + "--variant", + default="none", + choices=list(VARIANTS), + help="Preconditioner of the runs.", + ) parser.add_argument( "--upload", action="store_true", @@ -42,15 +79,16 @@ def main() -> None: help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', ) args = parser.parse_args() + label, name, description, param_flags = VARIANTS[args.variant] # Paths relative to this script's location, so it can be run from anywhere. script_dir = Path(__file__).resolve().parent params_dir = script_dir / "hollow_cylinder_strong_scaling" profiling_case = ProfilingCase( - label="curl_curl_hollow_cylinder_strong_scaling", - name="Curl-curl on hollow cylinder strong scaling test, no preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with unpreconditioned CG.", + label=label, + name=name, + description=description, physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", struphy_model_used="CurlCurl", params_source=params_dir / "params_curl_curl.py", @@ -61,7 +99,7 @@ def main() -> None: # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) + profiling_case.launch(num_tasks, param_flags=param_flags) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling_hiptmair_xu.py b/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling_hiptmair_xu.py deleted file mode 100644 index 1aa8264a4..000000000 --- a/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling_hiptmair_xu.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Curl-curl strong scaling profiling case on hollow cylinder, Hiptmair-Xu preconditioner. - -This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_curl_curl.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_cylinder_strong_scaling_hiptmair_xu" - - profiling_case = ProfilingCase( - label="curl_curl_hollow_cylinder_strong_scaling_hiptmair_xu", - name="Curl-curl on hollow cylinder strong scaling test, Hiptmair-Xu preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner).", - physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", - struphy_model_used="CurlCurl", - params_source=params_dir / "params_curl_curl.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling_multigrid.py b/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling_multigrid.py deleted file mode 100644 index 1549a5429..000000000 --- a/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling_multigrid.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Curl-curl strong scaling profiling case on hollow cylinder, multigrid preconditioner. - -This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_curl_curl.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_cylinder_strong_scaling_multigrid" - - profiling_case = ProfilingCase( - label="curl_curl_hollow_cylinder_strong_scaling_multigrid", - name="Curl-curl on hollow cylinder strong scaling test, multigrid preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair.", - physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", - struphy_model_used="CurlCurl", - params_source=params_dir / "params_curl_curl.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling_stiffness.py b/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling_stiffness.py deleted file mode 100644 index 4eff4404f..000000000 --- a/profiling/examples/CurlCurl/submit_curl_curl_hollow_cylinder_strong_scaling_stiffness.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Curl-curl strong scaling profiling case on hollow cylinder, stiffness preconditioner. - -This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_curl_curl.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_cylinder_strong_scaling_stiffness" - - profiling_case = ProfilingCase( - label="curl_curl_hollow_cylinder_strong_scaling_stiffness", - name="Curl-curl on hollow cylinder strong scaling test, stiffness preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner).", - physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", - struphy_model_used="CurlCurl", - params_source=params_dir / "params_curl_curl.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling.py b/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling.py index 8443a19da..a034ae4b8 100644 --- a/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling.py +++ b/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling.py @@ -1,4 +1,4 @@ -"""Curl-curl strong scaling profiling case on hollow torus, no preconditioner. +"""Curl-curl strong scaling profiling case on hollow torus. This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a @@ -7,6 +7,8 @@ each run as soon as its own job finishes. Each generated script runs the simulation itself by invoking `params_curl_curl.py` directly (its `__main__` block is the worker). + +The variant (preconditioner) is chosen with `--variant`; run this script once per variant. """ import argparse @@ -18,6 +20,35 @@ from profiling_job import ProfilingCase # noqa: E402 +# variant -> (label, name, description, flags of the params file); the labels identify the +# cases in the profiling-data repo and are kept from the former one-case-per-script layout +VARIANTS = { + "none": ( + "curl_curl_hollow_torus_strong_scaling", + "Curl-curl on hollow torus strong scaling test, no preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with unpreconditioned CG.", + [], + ), + "hiptmair_xu": ( + "curl_curl_hollow_torus_strong_scaling_hiptmair_xu", + "Curl-curl on hollow torus strong scaling test, Hiptmair-Xu preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner).", + ["--precond", "HiptmairXu"], + ), + "multigrid": ( + "curl_curl_hollow_torus_strong_scaling_multigrid", + "Curl-curl on hollow torus strong scaling test, multigrid preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair.", + ["--precond", "MultiGrid"], + ), + "stiffness": ( + "curl_curl_hollow_torus_strong_scaling_stiffness", + "Curl-curl on hollow torus strong scaling test, stiffness preconditioner", + "Strong scaling of the CurlCurl model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner).", + ["--precond", "StiffnessPreconditioner"], + ), +} + def main() -> None: @@ -25,6 +56,12 @@ def main() -> None: parser = argparse.ArgumentParser( description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), ) + parser.add_argument( + "--variant", + default="none", + choices=list(VARIANTS), + help="Preconditioner of the runs.", + ) parser.add_argument( "--upload", action="store_true", @@ -42,15 +79,16 @@ def main() -> None: help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', ) args = parser.parse_args() + label, name, description, param_flags = VARIANTS[args.variant] # Paths relative to this script's location, so it can be run from anywhere. script_dir = Path(__file__).resolve().parent params_dir = script_dir / "hollow_torus_strong_scaling" profiling_case = ProfilingCase( - label="curl_curl_hollow_torus_strong_scaling", - name="Curl-curl on hollow torus strong scaling test, no preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with unpreconditioned CG.", + label=label, + name=name, + description=description, physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", struphy_model_used="CurlCurl", params_source=params_dir / "params_curl_curl.py", @@ -61,7 +99,7 @@ def main() -> None: # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) + profiling_case.launch(num_tasks, param_flags=param_flags) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling_hiptmair_xu.py b/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling_hiptmair_xu.py deleted file mode 100644 index 3801f28f1..000000000 --- a/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling_hiptmair_xu.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Curl-curl strong scaling profiling case on hollow torus, Hiptmair-Xu preconditioner. - -This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_curl_curl.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_torus_strong_scaling_hiptmair_xu" - - profiling_case = ProfilingCase( - label="curl_curl_hollow_torus_strong_scaling_hiptmair_xu", - name="Curl-curl on hollow torus strong scaling test, Hiptmair-Xu preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with CG preconditioned by the auxiliary space preconditioner of Hiptmair and Xu (HiptmairXuPreconditioner).", - physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", - struphy_model_used="CurlCurl", - params_source=params_dir / "params_curl_curl.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling_multigrid.py b/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling_multigrid.py deleted file mode 100644 index be31a18b5..000000000 --- a/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling_multigrid.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Curl-curl strong scaling profiling case on hollow torus, multigrid preconditioner. - -This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_curl_curl.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_torus_strong_scaling_multigrid" - - profiling_case = ProfilingCase( - label="curl_curl_hollow_torus_strong_scaling_multigrid", - name="Curl-curl on hollow torus strong scaling test, multigrid preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with CG preconditioned by geometric multigrid with the hybrid smoother of Hiptmair.", - physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", - struphy_model_used="CurlCurl", - params_source=params_dir / "params_curl_curl.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling_stiffness.py b/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling_stiffness.py deleted file mode 100644 index bcd359ba4..000000000 --- a/profiling/examples/CurlCurl/submit_curl_curl_hollow_torus_strong_scaling_stiffness.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Curl-curl strong scaling profiling case on hollow torus, stiffness preconditioner. - -This file defines the curl-curl strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_curl_curl.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_torus_strong_scaling_stiffness" - - profiling_case = ProfilingCase( - label="curl_curl_hollow_torus_strong_scaling_stiffness", - name="Curl-curl on hollow torus strong scaling test, stiffness preconditioner", - description="Strong scaling of the CurlCurl model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with CG preconditioned by the Kronecker approximation of the inverse curl-curl operator with gradient correction (StiffnessPreconditioner).", - physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", - struphy_model_used="CurlCurl", - params_source=params_dir / "params_curl_curl.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_polar_cylinder_strong_scaling.py b/profiling/examples/CurlCurl/submit_curl_curl_polar_cylinder_strong_scaling.py new file mode 100644 index 000000000..8ff2176ae --- /dev/null +++ b/profiling/examples/CurlCurl/submit_curl_curl_polar_cylinder_strong_scaling.py @@ -0,0 +1,123 @@ +"""CurlCurl polar cylinder strong scaling profiling case: polar splines versus tensor-product splines on a cylinder with a hole. + +This file defines one variant of the CurlCurl strong scaling profiling case on a cylinder (radius 1, length 4) (the +`ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a +SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch system, runs directly +on this machine. `finalize_run` then packages and uploads each run as soon as its own job finishes. +Each generated script runs the simulation itself by invoking `params_curl_curl.py` directly (its `__main__` +block is the worker), with the flags of the variant. + +The variant (spline space and preconditioner) is chosen with `--variant`; run this script once per variant. +""" + +import argparse +import sys +from pathlib import Path + +# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from profiling_job import ProfilingCase # noqa: E402 + +# variant -> (description, flags of params_curl_curl.py) +VARIANTS = { + "tensor_none": ( + "tensor-product splines, no preconditioner", + ["--splines", "tensor", "--precond", "None"], + ), + "tensor_stiffness": ( + "tensor-product splines, stiffness preconditioner", + ["--splines", "tensor", "--precond", "StiffnessPreconditioner"], + ), + "tensor_hiptmair_xu": ( + "tensor-product splines, Hiptmair-Xu preconditioner", + ["--splines", "tensor", "--precond", "HiptmairXu"], + ), + "tensor_multigrid": ( + "tensor-product splines, multigrid preconditioner", + ["--splines", "tensor", "--precond", "MultiGrid"], + ), + "polar_none": ( + "polar splines, no preconditioner", + ["--splines", "polar", "--precond", "None"], + ), + "polar_tensor_mass": ( + "polar splines, tensor-product mass matrix preconditioner", + ["--splines", "polar", "--precond", "MassMatrixPreconditioner"], + ), + "polar_mass": ( + "polar splines, polar mass matrix preconditioner (Schur)", + ["--splines", "polar", "--precond", "PolarMassMatrixPreconditioner"], + ), + "polar_stiffness": ( + "polar splines, polar stiffness preconditioner (Schur)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner"], + ), + "polar_stiffness_lu": ( + "polar splines, polar stiffness preconditioner (LU)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner", "--poloidal-solver", "lu"], + ), +} + + +def main() -> None: + + # Parse arguments, do not remove --upload + parser = argparse.ArgumentParser( + description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), + ) + parser.add_argument( + "--variant", + required=True, + choices=list(VARIANTS), + help="Spline space and preconditioner of the runs.", + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload the packaged profiling results to the profiling-data repo.", + ) + parser.add_argument( + "--language", + default="fortran", + choices=["fortran", "c"], + help="Pyccel language to compile the Struphy kernels with.", + ) + parser.add_argument( + "--compiler", + default="GNU", + help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', + ) + args = parser.parse_args() + variant_description, param_flags = VARIANTS[args.variant] + + # Paths relative to this script's location, so it can be run from anywhere. + script_dir = Path(__file__).resolve().parent + params_dir = script_dir / "polar_cylinder_strong_scaling" + + profiling_case = ProfilingCase( + label=f"curl_curl_polar_cylinder_strong_scaling_{args.variant}", + name=f"CurlCurl on a cylinder strong scaling test, {variant_description}", + description=( + "Strong scaling of the CurlCurl model with a manufactured solution whose gradient is largest on the axis, " + "comparing C1 polar splines (IGAPolarCylinder) with tensor-product splines on a cylinder with a hole " + f"(HollowCylinder, a1=1e-2): {variant_description}." + ), + physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", + struphy_model_used="CurlCurl", + params_source=params_dir / "params_curl_curl.py", + language=args.language, + compiler=args.compiler, + upload=args.upload, + ) + + # Launch one run per rank count + for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): + profiling_case.launch(num_tasks, param_flags=param_flags) + + # Package and push each run as its own job finishes. + profiling_case.finalize_run() + + +if __name__ == "__main__": + main() diff --git a/profiling/examples/CurlCurl/submit_curl_curl_polar_torus_strong_scaling.py b/profiling/examples/CurlCurl/submit_curl_curl_polar_torus_strong_scaling.py new file mode 100644 index 000000000..bb0b9315d --- /dev/null +++ b/profiling/examples/CurlCurl/submit_curl_curl_polar_torus_strong_scaling.py @@ -0,0 +1,123 @@ +"""CurlCurl polar torus strong scaling profiling case: polar splines versus tensor-product splines on a torus with a hole. + +This file defines one variant of the CurlCurl strong scaling profiling case on a full torus (minor radius 1, major radius 4) (the +`ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a +SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch system, runs directly +on this machine. `finalize_run` then packages and uploads each run as soon as its own job finishes. +Each generated script runs the simulation itself by invoking `params_curl_curl.py` directly (its `__main__` +block is the worker), with the flags of the variant. + +The variant (spline space and preconditioner) is chosen with `--variant`; run this script once per variant. +""" + +import argparse +import sys +from pathlib import Path + +# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from profiling_job import ProfilingCase # noqa: E402 + +# variant -> (description, flags of params_curl_curl.py) +VARIANTS = { + "tensor_none": ( + "tensor-product splines, no preconditioner", + ["--splines", "tensor", "--precond", "None"], + ), + "tensor_stiffness": ( + "tensor-product splines, stiffness preconditioner", + ["--splines", "tensor", "--precond", "StiffnessPreconditioner"], + ), + "tensor_hiptmair_xu": ( + "tensor-product splines, Hiptmair-Xu preconditioner", + ["--splines", "tensor", "--precond", "HiptmairXu"], + ), + "tensor_multigrid": ( + "tensor-product splines, multigrid preconditioner", + ["--splines", "tensor", "--precond", "MultiGrid"], + ), + "polar_none": ( + "polar splines, no preconditioner", + ["--splines", "polar", "--precond", "None"], + ), + "polar_tensor_mass": ( + "polar splines, tensor-product mass matrix preconditioner", + ["--splines", "polar", "--precond", "MassMatrixPreconditioner"], + ), + "polar_mass": ( + "polar splines, polar mass matrix preconditioner (Schur)", + ["--splines", "polar", "--precond", "PolarMassMatrixPreconditioner"], + ), + "polar_stiffness": ( + "polar splines, polar stiffness preconditioner (Schur)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner"], + ), + "polar_stiffness_lu": ( + "polar splines, polar stiffness preconditioner (LU)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner", "--poloidal-solver", "lu"], + ), +} + + +def main() -> None: + + # Parse arguments, do not remove --upload + parser = argparse.ArgumentParser( + description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), + ) + parser.add_argument( + "--variant", + required=True, + choices=list(VARIANTS), + help="Spline space and preconditioner of the runs.", + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload the packaged profiling results to the profiling-data repo.", + ) + parser.add_argument( + "--language", + default="fortran", + choices=["fortran", "c"], + help="Pyccel language to compile the Struphy kernels with.", + ) + parser.add_argument( + "--compiler", + default="GNU", + help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', + ) + args = parser.parse_args() + variant_description, param_flags = VARIANTS[args.variant] + + # Paths relative to this script's location, so it can be run from anywhere. + script_dir = Path(__file__).resolve().parent + params_dir = script_dir / "polar_torus_strong_scaling" + + profiling_case = ProfilingCase( + label=f"curl_curl_polar_torus_strong_scaling_{args.variant}", + name=f"CurlCurl on a torus strong scaling test, {variant_description}", + description=( + "Strong scaling of the CurlCurl model with a manufactured solution whose gradient is largest on the axis, " + "comparing C1 polar splines (IGAPolarTorus) with tensor-product splines on a torus with a hole " + f"(HollowTorus, a1=1e-2): {variant_description}." + ), + physics_problem="Occurs in implicit time stepping of Maxwell's equations and in magnetostatics.", + struphy_model_used="CurlCurl", + params_source=params_dir / "params_curl_curl.py", + language=args.language, + compiler=args.compiler, + upload=args.upload, + ) + + # Launch one run per rank count + for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): + profiling_case.launch(num_tasks, param_flags=param_flags) + + # Package and push each run as its own job finishes. + profiling_case.finalize_run() + + +if __name__ == "__main__": + main() diff --git a/profiling/examples/GyrokineticPoisson/hollow_cylinder_strong_scaling/params_gyrokinetic_poisson.py b/profiling/examples/GyrokineticPoisson/hollow_cylinder_strong_scaling/params_gyrokinetic_poisson.py index 1fbcc031a..111f1cff5 100644 --- a/profiling/examples/GyrokineticPoisson/hollow_cylinder_strong_scaling/params_gyrokinetic_poisson.py +++ b/profiling/examples/GyrokineticPoisson/hollow_cylinder_strong_scaling/params_gyrokinetic_poisson.py @@ -4,14 +4,44 @@ # Please fill in a verbal description of the simulation. # It will be printed at the beginning of the simulation and can be used to keep track of the different runs. -name = "GyrokineticPoisson strong scaling on hollow cylinder, no preconditioner" -description = """ +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# `--precond` selects the preconditioner (the variant), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--precond", + default="None", + choices=["None", "MultiGrid", "StiffnessPreconditioner"], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 64, 64], help="Number of elements per direction.") +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +# preconditioner -> (name suffix, linear solver) for the name and the description +PRECONDS = { + None: ("no preconditioner", "unpreconditioned CG"), + "MultiGrid": ("multigrid preconditioner", "CG preconditioned by geometric multigrid"), + "StiffnessPreconditioner": ( + "stiffness preconditioner", + "CG preconditioned by the Kronecker approximation of the inverse operator (StiffnessPreconditioner)", + ), +} + +name = f"GyrokineticPoisson strong scaling on hollow cylinder, {PRECONDS[precond][0]}" +description = f""" Strong scaling test for the gyrokinetic Poisson equation with adiabatic electrons on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=2*pi) in a screw pinch equilibrium (ScrewPinch, a=1, R0=1). The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), exciting the full spectrum. Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with unpreconditioned CG. +The linear system is solved with {PRECONDS[precond][1]}. """ import logging @@ -20,7 +50,6 @@ set_logging_level(logging.INFO) -import argparse import numpy as np @@ -61,13 +90,6 @@ # -------------------------- # Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - env = EnvironmentOptions( sim_folder=f"sim_{args.id:02d}", restart=False, @@ -95,7 +117,7 @@ equil = equils.ScrewPinch(a=a2, R0=R0, B0=B0, q0=q0, q1=q1, n1=n1, n2=n2, na=na, p0=p0) # Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) +grid = grids.TensorProductGrid(num_elements=tuple(args.num_elements), mpi_dims_mask=(True, True, True)) # Derham options derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) @@ -130,7 +152,7 @@ model.propagators.gyrokinetic_poisson.options = model.propagators.gyrokinetic_poisson.Options( which_geometry="cylindrical", solver="pcg", - precond=None, + precond=precond, solver_params=solver_params, ) diff --git a/profiling/examples/GyrokineticPoisson/hollow_cylinder_strong_scaling_multigrid/params_gyrokinetic_poisson.py b/profiling/examples/GyrokineticPoisson/hollow_cylinder_strong_scaling_multigrid/params_gyrokinetic_poisson.py deleted file mode 100644 index 018f0c947..000000000 --- a/profiling/examples/GyrokineticPoisson/hollow_cylinder_strong_scaling_multigrid/params_gyrokinetic_poisson.py +++ /dev/null @@ -1,348 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "GyrokineticPoisson strong scaling on hollow cylinder, multigrid preconditioner" -description = """ -Strong scaling test for the gyrokinetic Poisson equation with adiabatic electrons on a hollow cylinder -(HollowCylinder, a1=1e-2, a2=1, Lz=2*pi) in a screw pinch equilibrium (ScrewPinch, a=1, R0=1). -The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), -exciting the full spectrum. -Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with CG preconditioned by geometric multigrid. -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -import numpy as np - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - ProfilingOptions, - Simulation, - Time, - domains, - equils, - grids, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import GyrokineticPoisson - -# Units -base_units = BaseUnits() - -# Model instance -epsilon = 1.0 -charge_number = 1 -model = GyrokineticPoisson(base_units=base_units, epsilon=epsilon, Z=charge_number) - -# List all variables and decide whether to save their data -model.em_fields.phi.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry (the screw pinch requires Lz = 2*pi*R0) -a1 = 1e-2 -a2 = 1.0 -R0 = 1.0 -Lz = 2 * np.pi * R0 -domain = domains.HollowCylinder(a1=a1, a2=a2, Lz=Lz) - -# MHD equilibrium: a non-flat density and the pressure offset p0 keep n0/T0 = n0**2/p and n0/|B0|**2 -# of order one and varying in r (the default p0 = 1e-8 would give n0/T0 ~ 1e7 at the boundary) -B0 = 1.0 -q0 = 1.05 -q1 = 1.80 -n1 = 2.0 -n2 = 1.0 -na = 0.2 -p0 = 0.1 -equil = equils.ScrewPinch(a=a2, R0=R0, B0=B0, q0=q0, q1=q1, n1=n1, n2=n2, na=na, p0=p0) - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profiling options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.multigrid.preconditioner import MultiGridOptions -from struphy.linear_algebra.solver import SolverParameters - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.gyrokinetic_poisson.options = model.propagators.gyrokinetic_poisson.Options( - which_geometry="cylindrical", - solver="pcg", - precond="MultiGrid", - multigrid=MultiGridOptions(), - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -from scipy.special import erf - -from struphy.initial.base import GenericPerturbation - -# The exact solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5). -# Its value at the Dirichlet boundary eta1 = 0, 1 is exp(-0.25 / w**2) ~ 1e-11, and it is periodic -# in eta2 and eta3 up to the same accuracy. Being localized, it excites the whole spectrum of the -# operator, so CG has to work for convergence. Both the solution and the right-hand side are 0-forms -# given as functions of the logical coordinates (given_in_basis="0"); the pushed-forward solution -# at x = F(eta) is phi(eta). -# -# The model solves (see GyrokineticPoisson.doc_pde) -# -# 1/(charge_number*epsilon**2) * n0/T0 * (phi - ) - div(n0/|B0|**2 * (I - b0 b0^T) grad(phi)) = rho / epsilon, -# -# where is the average over eta3 weighted with n0/T0 * sqrt(g). In logical coordinates the -# diffusion term reads -1/sqrt(g) * d_i(C^ij d_j phi), with C^ij = sqrt(g) * n0/|B0|**2 * (G^ij - b^i b^j), -# the inverse metric G^ij and the contravariant components b^i of b0. -w = 0.1 -eta0 = (0.5, 0.5, 0.5) - - -def gaussian_derivatives(e1, e2, e3): - """The Gaussian, its first and (pure) second derivatives and the mixed derivative d2 d3.""" - eta = (e1, e2, e3) - g = np.exp(-sum((e - c) ** 2 for e, c in zip(eta, eta0)) / w**2) - d = [-2 * (e - c) / w**2 * g for e, c in zip(eta, eta0)] - dd = [(4 * (e - c) ** 2 / w**4 - 2 / w**2) * g for e, c in zip(eta, eta0)] - d23 = 4 * (e2 - eta0[1]) * (e3 - eta0[2]) / w**4 * g - return g, d, dd, d23 - - -def exact_solution(e1, e2, e3): - return gaussian_derivatives(e1, e2, e3)[0] - - -def equilibrium_coefficients(e1, e2): - """sqrt(g), n0/T0 and C^11, C^22, C^23, C^33 (C^12 = C^13 = 0) as functions of (eta1, eta2). - - HollowCylinder: r = a1 + (a2 - a1) * eta1, theta = 2*pi*eta2, z = Lz * eta3, with the scale factors - h1 = a2 - a1, h2 = 2*pi*r, h3 = Lz. ScrewPinch: B0 = B0 * (e_z + r / (q R0) e_theta), - q = q0 + (q1 - q0) * r**2 / a**2, n0 = na + (1 - na) * (1 - (r/a)**n1)**n2 and - p = p0 + B0**2 * (a/R0)**2 * q0 / (2 * (q1 - q0)) * (1/q**2 - 1/q1**2), T0 = p/n0. - All coefficients depend on eta1 only. Only operations defined for complex numbers are used, - such that the derivatives can be computed with the complex step. - """ - da = a2 - a1 - r = a1 + da * e1 - q = q0 + (q1 - q0) * r**2 / a2**2 - n = na + (1 - na) * (1 - (r / a2) ** n1) ** n2 - p = p0 + B0**2 * (a2 / R0) ** 2 * q0 / (2 * (q1 - q0)) * (1 / q**2 - 1 / q1**2) - b_theta = B0 * r / (R0 * q) - b_z = B0 + 0 * r - abs_b_sq = b_theta**2 + b_z**2 - - h1, h2, h3 = da, 2 * np.pi * r, Lz - sqrt_g = h1 * h2 * h3 - # contravariant components of the unit vector b0 = B0 / |B0| (b^1 = 0) - b2 = b_theta / np.sqrt(abs_b_sq) / h2 - b3 = b_z / np.sqrt(abs_b_sq) / h3 - - weight = sqrt_g * n / abs_b_sq - c11 = weight / h1**2 - c22 = weight * (1 / h2**2 - b2**2) - c23 = -weight * b2 * b3 - c33 = weight * (1 / h3**2 - b3**2) - return sqrt_g, n**2 / p, c11, c22, c23, c33 - - -def complex_step(f, x, h=1e-30): - """Derivative f'(x) with the complex step, exact up to machine precision for analytic f.""" - return f(x + 1j * h).imag / h - - -def weighted_average(e1, e2, e3): - """Average of the exact solution over eta3, weighted with n0/T0 * sqrt(g). - - The weight depends on eta1 only, hence it is the plain integral of the Gaussian over eta3. - """ - g1 = np.exp(-((e1 - eta0[0]) ** 2) / w**2) - g2 = np.exp(-((e2 - eta0[1]) ** 2) / w**2) - int3 = w * np.sqrt(np.pi) * erf(0.5 / w) - return g1 * g2 * int3 + 0 * e3 - - -def gyrokinetic_operator(e1, e2, e3): - """Left-hand side of the gyrokinetic Poisson equation applied to the exact solution.""" - g, d, dd, d23 = gaussian_derivatives(e1, e2, e3) - sqrt_g, n_over_t, c11, c22, c23, c33 = equilibrium_coefficients(e1, e2) - dc11 = complex_step(lambda x: equilibrium_coefficients(x, e2)[2], e1) - dc22 = complex_step(lambda x: equilibrium_coefficients(e1, x)[3], e2) - dc23 = complex_step(lambda x: equilibrium_coefficients(e1, x)[4], e2) - div = dc11 * d[0] + c11 * dd[0] + dc22 * d[1] + c22 * dd[1] + dc23 * d[2] + 2 * c23 * d23 + c33 * dd[2] - return n_over_t / (charge_number * epsilon**2) * (g - weighted_average(e1, e2, e3)) - div / sqrt_g - - -def rhs_fun(e1, e2, e3): - return epsilon * gyrokinetic_operator(e1, e2, e3) - - -rhs_perturbation = GenericPerturbation(rhs_fun, given_in_basis="0") - -model.em_fields.source.add_perturbation(rhs_perturbation) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True) - - from matplotlib import pyplot as plt - - def slice_axes(k, X, Y, Z, E1, E2, E3): - """Axes (horizontal, vertical, labels) for plotting the slice eta_k = 0.5: the cross-section (x, y) - at eta3 = 0.5, the half plane (r, z) at eta2 = 0.5 and the logical (eta2, eta3) at eta1 = 0.5.""" - if k == 0: - return E2, E3, "eta2", "eta3" - elif k == 1: - return np.sqrt(X**2 + Y**2), Z, "r", "z" - else: - return X, Y, "x", "y" - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error on the slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - h, v, hlabel, vlabel = slice_axes(k, X[idx], Y[idx], Z[idx], E1[idx], E2[idx], E3[idx]) - eta_k = (E1, E2, E3)[k][idx].flat[0] - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(h, v, data, shading="gouraud") - plt.colorbar() - plt.xlabel(hlabel) - plt.ylabel(vlabel) - plt.title(f"{name} {what}, slice at eta{k + 1} = {eta_k:.2f}") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Raw FEEC fields are evaluated directly from the saved spline coefficients at the - # cell centres of the simulation grid (serial Derham, safe on rank 0 only). - rhs = out.evaluate("em_fields/source", t=0).values - - phi_data = out.evaluate("em_fields/phi", t=-1) - phi = phi_data.values - - # logical coordinates of the evaluation points and their images x = F(eta) - E1, E2, E3 = np.meshgrid( - *(phi_data.coords[c].values for c in ("eta1", "eta2", "eta3")), - indexing="ij", - ) - X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) - - # pushed-forward exact solutions: the 0-forms at x = F(eta) are their values at eta - rhs_exact = rhs_fun(E1, E2, E3) - phi_exact = exact_solution(E1, E2, E3) - - fig_rhs = plot_slices(rhs, rhs_exact, "RHS") - fig_phi = plot_slices(phi, phi_exact, "Phi") - - rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) - rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) - - print(f"Max relative error in RHS: {rel_err_rhs:.2e}") - print(f"Max relative error in Phi: {rel_err_phi:.2e}") - - assert rel_err_rhs < 2.2e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_phi < 1.3e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_phi}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) - else: - plt.show() diff --git a/profiling/examples/GyrokineticPoisson/hollow_cylinder_strong_scaling_stiffness/params_gyrokinetic_poisson.py b/profiling/examples/GyrokineticPoisson/hollow_cylinder_strong_scaling_stiffness/params_gyrokinetic_poisson.py deleted file mode 100644 index 15fb08c8e..000000000 --- a/profiling/examples/GyrokineticPoisson/hollow_cylinder_strong_scaling_stiffness/params_gyrokinetic_poisson.py +++ /dev/null @@ -1,346 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "GyrokineticPoisson strong scaling on hollow cylinder, stiffness preconditioner" -description = """ -Strong scaling test for the gyrokinetic Poisson equation with adiabatic electrons on a hollow cylinder -(HollowCylinder, a1=1e-2, a2=1, Lz=2*pi) in a screw pinch equilibrium (ScrewPinch, a=1, R0=1). -The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), -exciting the full spectrum. -Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with CG preconditioned by the Kronecker approximation of the inverse operator (StiffnessPreconditioner). -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -import numpy as np - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - ProfilingOptions, - Simulation, - Time, - domains, - equils, - grids, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import GyrokineticPoisson - -# Units -base_units = BaseUnits() - -# Model instance -epsilon = 1.0 -charge_number = 1 -model = GyrokineticPoisson(base_units=base_units, epsilon=epsilon, Z=charge_number) - -# List all variables and decide whether to save their data -model.em_fields.phi.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry (the screw pinch requires Lz = 2*pi*R0) -a1 = 1e-2 -a2 = 1.0 -R0 = 1.0 -Lz = 2 * np.pi * R0 -domain = domains.HollowCylinder(a1=a1, a2=a2, Lz=Lz) - -# MHD equilibrium: a non-flat density and the pressure offset p0 keep n0/T0 = n0**2/p and n0/|B0|**2 -# of order one and varying in r (the default p0 = 1e-8 would give n0/T0 ~ 1e7 at the boundary) -B0 = 1.0 -q0 = 1.05 -q1 = 1.80 -n1 = 2.0 -n2 = 1.0 -na = 0.2 -p0 = 0.1 -equil = equils.ScrewPinch(a=a2, R0=R0, B0=B0, q0=q0, q1=q1, n1=n1, n2=n2, na=na, p0=p0) - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profiling options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.gyrokinetic_poisson.options = model.propagators.gyrokinetic_poisson.Options( - which_geometry="cylindrical", - solver="pcg", - precond="StiffnessPreconditioner", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -from scipy.special import erf - -from struphy.initial.base import GenericPerturbation - -# The exact solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5). -# Its value at the Dirichlet boundary eta1 = 0, 1 is exp(-0.25 / w**2) ~ 1e-11, and it is periodic -# in eta2 and eta3 up to the same accuracy. Being localized, it excites the whole spectrum of the -# operator, so CG has to work for convergence. Both the solution and the right-hand side are 0-forms -# given as functions of the logical coordinates (given_in_basis="0"); the pushed-forward solution -# at x = F(eta) is phi(eta). -# -# The model solves (see GyrokineticPoisson.doc_pde) -# -# 1/(charge_number*epsilon**2) * n0/T0 * (phi - ) - div(n0/|B0|**2 * (I - b0 b0^T) grad(phi)) = rho / epsilon, -# -# where is the average over eta3 weighted with n0/T0 * sqrt(g). In logical coordinates the -# diffusion term reads -1/sqrt(g) * d_i(C^ij d_j phi), with C^ij = sqrt(g) * n0/|B0|**2 * (G^ij - b^i b^j), -# the inverse metric G^ij and the contravariant components b^i of b0. -w = 0.1 -eta0 = (0.5, 0.5, 0.5) - - -def gaussian_derivatives(e1, e2, e3): - """The Gaussian, its first and (pure) second derivatives and the mixed derivative d2 d3.""" - eta = (e1, e2, e3) - g = np.exp(-sum((e - c) ** 2 for e, c in zip(eta, eta0)) / w**2) - d = [-2 * (e - c) / w**2 * g for e, c in zip(eta, eta0)] - dd = [(4 * (e - c) ** 2 / w**4 - 2 / w**2) * g for e, c in zip(eta, eta0)] - d23 = 4 * (e2 - eta0[1]) * (e3 - eta0[2]) / w**4 * g - return g, d, dd, d23 - - -def exact_solution(e1, e2, e3): - return gaussian_derivatives(e1, e2, e3)[0] - - -def equilibrium_coefficients(e1, e2): - """sqrt(g), n0/T0 and C^11, C^22, C^23, C^33 (C^12 = C^13 = 0) as functions of (eta1, eta2). - - HollowCylinder: r = a1 + (a2 - a1) * eta1, theta = 2*pi*eta2, z = Lz * eta3, with the scale factors - h1 = a2 - a1, h2 = 2*pi*r, h3 = Lz. ScrewPinch: B0 = B0 * (e_z + r / (q R0) e_theta), - q = q0 + (q1 - q0) * r**2 / a**2, n0 = na + (1 - na) * (1 - (r/a)**n1)**n2 and - p = p0 + B0**2 * (a/R0)**2 * q0 / (2 * (q1 - q0)) * (1/q**2 - 1/q1**2), T0 = p/n0. - All coefficients depend on eta1 only. Only operations defined for complex numbers are used, - such that the derivatives can be computed with the complex step. - """ - da = a2 - a1 - r = a1 + da * e1 - q = q0 + (q1 - q0) * r**2 / a2**2 - n = na + (1 - na) * (1 - (r / a2) ** n1) ** n2 - p = p0 + B0**2 * (a2 / R0) ** 2 * q0 / (2 * (q1 - q0)) * (1 / q**2 - 1 / q1**2) - b_theta = B0 * r / (R0 * q) - b_z = B0 + 0 * r - abs_b_sq = b_theta**2 + b_z**2 - - h1, h2, h3 = da, 2 * np.pi * r, Lz - sqrt_g = h1 * h2 * h3 - # contravariant components of the unit vector b0 = B0 / |B0| (b^1 = 0) - b2 = b_theta / np.sqrt(abs_b_sq) / h2 - b3 = b_z / np.sqrt(abs_b_sq) / h3 - - weight = sqrt_g * n / abs_b_sq - c11 = weight / h1**2 - c22 = weight * (1 / h2**2 - b2**2) - c23 = -weight * b2 * b3 - c33 = weight * (1 / h3**2 - b3**2) - return sqrt_g, n**2 / p, c11, c22, c23, c33 - - -def complex_step(f, x, h=1e-30): - """Derivative f'(x) with the complex step, exact up to machine precision for analytic f.""" - return f(x + 1j * h).imag / h - - -def weighted_average(e1, e2, e3): - """Average of the exact solution over eta3, weighted with n0/T0 * sqrt(g). - - The weight depends on eta1 only, hence it is the plain integral of the Gaussian over eta3. - """ - g1 = np.exp(-((e1 - eta0[0]) ** 2) / w**2) - g2 = np.exp(-((e2 - eta0[1]) ** 2) / w**2) - int3 = w * np.sqrt(np.pi) * erf(0.5 / w) - return g1 * g2 * int3 + 0 * e3 - - -def gyrokinetic_operator(e1, e2, e3): - """Left-hand side of the gyrokinetic Poisson equation applied to the exact solution.""" - g, d, dd, d23 = gaussian_derivatives(e1, e2, e3) - sqrt_g, n_over_t, c11, c22, c23, c33 = equilibrium_coefficients(e1, e2) - dc11 = complex_step(lambda x: equilibrium_coefficients(x, e2)[2], e1) - dc22 = complex_step(lambda x: equilibrium_coefficients(e1, x)[3], e2) - dc23 = complex_step(lambda x: equilibrium_coefficients(e1, x)[4], e2) - div = dc11 * d[0] + c11 * dd[0] + dc22 * d[1] + c22 * dd[1] + dc23 * d[2] + 2 * c23 * d23 + c33 * dd[2] - return n_over_t / (charge_number * epsilon**2) * (g - weighted_average(e1, e2, e3)) - div / sqrt_g - - -def rhs_fun(e1, e2, e3): - return epsilon * gyrokinetic_operator(e1, e2, e3) - - -rhs_perturbation = GenericPerturbation(rhs_fun, given_in_basis="0") - -model.em_fields.source.add_perturbation(rhs_perturbation) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True) - - from matplotlib import pyplot as plt - - def slice_axes(k, X, Y, Z, E1, E2, E3): - """Axes (horizontal, vertical, labels) for plotting the slice eta_k = 0.5: the cross-section (x, y) - at eta3 = 0.5, the half plane (r, z) at eta2 = 0.5 and the logical (eta2, eta3) at eta1 = 0.5.""" - if k == 0: - return E2, E3, "eta2", "eta3" - elif k == 1: - return np.sqrt(X**2 + Y**2), Z, "r", "z" - else: - return X, Y, "x", "y" - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error on the slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - h, v, hlabel, vlabel = slice_axes(k, X[idx], Y[idx], Z[idx], E1[idx], E2[idx], E3[idx]) - eta_k = (E1, E2, E3)[k][idx].flat[0] - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(h, v, data, shading="gouraud") - plt.colorbar() - plt.xlabel(hlabel) - plt.ylabel(vlabel) - plt.title(f"{name} {what}, slice at eta{k + 1} = {eta_k:.2f}") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Raw FEEC fields are evaluated directly from the saved spline coefficients at the - # cell centres of the simulation grid (serial Derham, safe on rank 0 only). - rhs = out.evaluate("em_fields/source", t=0).values - - phi_data = out.evaluate("em_fields/phi", t=-1) - phi = phi_data.values - - # logical coordinates of the evaluation points and their images x = F(eta) - E1, E2, E3 = np.meshgrid( - *(phi_data.coords[c].values for c in ("eta1", "eta2", "eta3")), - indexing="ij", - ) - X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) - - # pushed-forward exact solutions: the 0-forms at x = F(eta) are their values at eta - rhs_exact = rhs_fun(E1, E2, E3) - phi_exact = exact_solution(E1, E2, E3) - - fig_rhs = plot_slices(rhs, rhs_exact, "RHS") - fig_phi = plot_slices(phi, phi_exact, "Phi") - - rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) - rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) - - print(f"Max relative error in RHS: {rel_err_rhs:.2e}") - print(f"Max relative error in Phi: {rel_err_phi:.2e}") - - assert rel_err_rhs < 2.2e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_phi < 1.3e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_phi}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) - else: - plt.show() diff --git a/profiling/examples/GyrokineticPoisson/hollow_torus_strong_scaling/params_gyrokinetic_poisson.py b/profiling/examples/GyrokineticPoisson/hollow_torus_strong_scaling/params_gyrokinetic_poisson.py index beda25694..32f544ab2 100644 --- a/profiling/examples/GyrokineticPoisson/hollow_torus_strong_scaling/params_gyrokinetic_poisson.py +++ b/profiling/examples/GyrokineticPoisson/hollow_torus_strong_scaling/params_gyrokinetic_poisson.py @@ -4,14 +4,44 @@ # Please fill in a verbal description of the simulation. # It will be printed at the beginning of the simulation and can be used to keep track of the different runs. -name = "GyrokineticPoisson strong scaling on hollow torus, no preconditioner" -description = """ +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# `--precond` selects the preconditioner (the variant), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--precond", + default="None", + choices=["None", "MultiGrid", "StiffnessPreconditioner"], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 64, 64], help="Number of elements per direction.") +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +# preconditioner -> (name suffix, linear solver) for the name and the description +PRECONDS = { + None: ("no preconditioner", "unpreconditioned CG"), + "MultiGrid": ("multigrid preconditioner", "CG preconditioned by geometric multigrid"), + "StiffnessPreconditioner": ( + "stiffness preconditioner", + "CG preconditioned by the Kronecker approximation of the inverse operator (StiffnessPreconditioner)", + ), +} + +name = f"GyrokineticPoisson strong scaling on hollow torus, {PRECONDS[precond][0]}" +description = f""" Strong scaling test for the gyrokinetic Poisson equation with adiabatic electrons on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=3) in an ad hoc tokamak equilibrium (AdhocTorus, a=1, R0=3). The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), exciting the full spectrum. Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with unpreconditioned CG. +The linear system is solved with {PRECONDS[precond][1]}. """ import logging @@ -20,7 +50,6 @@ set_logging_level(logging.INFO) -import argparse import numpy as np @@ -61,13 +90,6 @@ # -------------------------- # Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - env = EnvironmentOptions( sim_folder=f"sim_{args.id:02d}", restart=False, @@ -113,7 +135,7 @@ ) # Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) +grid = grids.TensorProductGrid(num_elements=tuple(args.num_elements), mpi_dims_mask=(True, True, True)) # Derham options derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) @@ -148,7 +170,7 @@ model.propagators.gyrokinetic_poisson.options = model.propagators.gyrokinetic_poisson.Options( which_geometry="toroidal", solver="pcg", - precond=None, + precond=precond, solver_params=solver_params, ) diff --git a/profiling/examples/GyrokineticPoisson/hollow_torus_strong_scaling_multigrid/params_gyrokinetic_poisson.py b/profiling/examples/GyrokineticPoisson/hollow_torus_strong_scaling_multigrid/params_gyrokinetic_poisson.py deleted file mode 100644 index 511b68513..000000000 --- a/profiling/examples/GyrokineticPoisson/hollow_torus_strong_scaling_multigrid/params_gyrokinetic_poisson.py +++ /dev/null @@ -1,374 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "GyrokineticPoisson strong scaling on hollow torus, multigrid preconditioner" -description = """ -Strong scaling test for the gyrokinetic Poisson equation with adiabatic electrons on a full hollow torus -(HollowTorus, a1=1e-2, a2=1, R0=3) in an ad hoc tokamak equilibrium (AdhocTorus, a=1, R0=3). -The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), -exciting the full spectrum. -Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with CG preconditioned by geometric multigrid. -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -import numpy as np - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - ProfilingOptions, - Simulation, - Time, - domains, - equils, - grids, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import GyrokineticPoisson - -# Units -base_units = BaseUnits() - -# Model instance -epsilon = 1.0 -charge_number = 1 -model = GyrokineticPoisson(base_units=base_units, epsilon=epsilon, Z=charge_number) - -# List all variables and decide whether to save their data -model.em_fields.phi.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -a1 = 1e-2 -a2 = 1.0 -R0 = 3.0 -tor_period = 1 -domain = domains.HollowTorus(a1=a1, a2=a2, R0=R0, sfl=False, pol_period=1, tor_period=tor_period) - -# MHD equilibrium with circular concentric flux surfaces matching the domain (a = a2, same R0); -# the non-flat density and the ad hoc pressure keep n0/T0 = n0**2/p and n0/|B0|**2 of order one -B0 = 2.0 -q0 = 1.71 -q1 = 1.87 -n1 = 2.0 -n2 = 1.0 -na = 0.2 -beta = 0.179 -p0 = 1.0 -p1 = 0.1 -p2 = 0.1 -equil = equils.AdhocTorus( - a=a2, - R0=R0, - B0=B0, - q_kind=0, - q0=q0, - q1=q1, - n1=n1, - n2=n2, - na=na, - p_kind=1, - p0=p0, - p1=p1, - p2=p2, - beta=beta, -) - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profiling options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.multigrid.preconditioner import MultiGridOptions -from struphy.linear_algebra.solver import SolverParameters - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.gyrokinetic_poisson.options = model.propagators.gyrokinetic_poisson.Options( - which_geometry="toroidal", - solver="pcg", - precond="MultiGrid", - multigrid=MultiGridOptions(), - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -from scipy.special import erf - -from struphy.initial.base import GenericPerturbation - -# The exact solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5). -# Its value at the Dirichlet boundary eta1 = 0, 1 is exp(-0.25 / w**2) ~ 1e-11, and it is periodic -# in eta2 and eta3 up to the same accuracy. Being localized, it excites the whole spectrum of the -# operator, so CG has to work for convergence. Both the solution and the right-hand side are 0-forms -# given as functions of the logical coordinates (given_in_basis="0"); the pushed-forward solution -# at x = F(eta) is phi(eta). -# -# The model solves (see GyrokineticPoisson.doc_pde) -# -# 1/(charge_number*epsilon**2) * n0/T0 * (phi - ) - div(n0/|B0|**2 * (I - b0 b0^T) grad(phi)) = rho / epsilon, -# -# where is the average over eta2 and eta3 weighted with n0/T0 * sqrt(g). In logical coordinates the -# diffusion term reads -1/sqrt(g) * d_i(C^ij d_j phi), with C^ij = sqrt(g) * n0/|B0|**2 * (G^ij - b^i b^j), -# the inverse metric G^ij and the contravariant components b^i of b0. -w = 0.1 -eta0 = (0.5, 0.5, 0.5) - - -def gaussian_derivatives(e1, e2, e3): - """The Gaussian, its first and (pure) second derivatives and the mixed derivative d2 d3.""" - eta = (e1, e2, e3) - g = np.exp(-sum((e - c) ** 2 for e, c in zip(eta, eta0)) / w**2) - d = [-2 * (e - c) / w**2 * g for e, c in zip(eta, eta0)] - dd = [(4 * (e - c) ** 2 / w**4 - 2 / w**2) * g for e, c in zip(eta, eta0)] - d23 = 4 * (e2 - eta0[1]) * (e3 - eta0[2]) / w**4 * g - return g, d, dd, d23 - - -def exact_solution(e1, e2, e3): - return gaussian_derivatives(e1, e2, e3)[0] - - -def equilibrium_coefficients(e1, e2): - """sqrt(g), n0/T0 and C^11, C^22, C^23, C^33 (C^12 = C^13 = 0) as functions of (eta1, eta2). - - HollowTorus (sfl=False): r = a1 + (a2 - a1) * eta1, theta = 2*pi*eta2, phi = 2*pi*eta3 / tor_period, - R = R0 + r*cos(theta), with the scale factors h1 = a2 - a1, h2 = 2*pi*r, h3 = 2*pi*R / tor_period. - AdhocTorus (q_kind=0, p_kind=1): B0 = grad(psi) x grad(phi) - B0*R0 grad(phi) with - dpsi/dr = B0 * r / (q * sqrt(1 - r**2/R0**2)), hence |B_pol| = dpsi/dr / R and |B_tor| = B0*R0 / R, - q = q0 + (q1 - q0) * r**2 / a**2, n0 = na + (1 - na) * (1 - (r/a)**n1)**n2 and - p = beta * B0**2 / 2 * (p0 - p1 * r**2/a**2 - p2 * r**4/a**4), T0 = p/n0. - Only operations defined for complex numbers are used, such that the derivatives can be computed - with the complex step. - """ - da = a2 - a1 - r = a1 + da * e1 - theta = 2 * np.pi * e2 - R = R0 + r * np.cos(theta) - q = q0 + (q1 - q0) * r**2 / a2**2 - n = na + (1 - na) * (1 - (r / a2) ** n1) ** n2 - p = beta * B0**2 / 2 * (p0 - p1 * r**2 / a2**2 - p2 * r**4 / a2**4) - b_pol = B0 * r / (q * np.sqrt(1 - r**2 / R0**2)) / R - b_tor = B0 * R0 / R - abs_b_sq = b_pol**2 + b_tor**2 - - h1, h2, h3 = da, 2 * np.pi * r, 2 * np.pi * R / tor_period - sqrt_g = h1 * h2 * h3 - # contravariant components of the unit vector b0 = B0 / |B0| (b^1 = 0), signs from the - # orientation of the mapping (only the sign of b^2 * b^3 enters) - b2 = b_pol / np.sqrt(abs_b_sq) / h2 - b3 = b_tor / np.sqrt(abs_b_sq) / h3 - - weight = sqrt_g * n / abs_b_sq - c11 = weight / h1**2 - c22 = weight * (1 / h2**2 - b2**2) - c23 = -weight * b2 * b3 - c33 = weight * (1 / h3**2 - b3**2) - return sqrt_g, n**2 / p, c11, c22, c23, c33 - - -def complex_step(f, x, h=1e-30): - """Derivative f'(x) with the complex step, exact up to machine precision for analytic f.""" - return f(x + 1j * h).imag / h - - -def weighted_average(e1, e2, e3): - """Average of the exact solution over eta2 and eta3, weighted with n0/T0 * sqrt(g). - - The weight is proportional to f(eta1) * R = f(eta1) * (R0 + r*cos(2*pi*eta2)), hence the average is - g1 * int3 * (R0 * int2 + r * int_cos) / R0 with the integrals over [0, 1] of the Gaussians - g2 and g3 (int2 = int3) and of g2 * cos(2*pi*eta2) (int_cos). - """ - r = a1 + (a2 - a1) * e1 - g1 = np.exp(-((e1 - eta0[0]) ** 2) / w**2) - int3 = w * np.sqrt(np.pi) * erf(0.5 / w) - int_cos = -w * np.sqrt(np.pi) * np.exp(-((np.pi * w) ** 2)) * erf(0.5 / w - 1j * np.pi * w).real - return g1 * int3 * (R0 * int3 + r * int_cos) / R0 + 0 * e2 + 0 * e3 - - -def gyrokinetic_operator(e1, e2, e3): - """Left-hand side of the gyrokinetic Poisson equation applied to the exact solution.""" - g, d, dd, d23 = gaussian_derivatives(e1, e2, e3) - sqrt_g, n_over_t, c11, c22, c23, c33 = equilibrium_coefficients(e1, e2) - dc11 = complex_step(lambda x: equilibrium_coefficients(x, e2)[2], e1) - dc22 = complex_step(lambda x: equilibrium_coefficients(e1, x)[3], e2) - dc23 = complex_step(lambda x: equilibrium_coefficients(e1, x)[4], e2) - div = dc11 * d[0] + c11 * dd[0] + dc22 * d[1] + c22 * dd[1] + dc23 * d[2] + 2 * c23 * d23 + c33 * dd[2] - return n_over_t / (charge_number * epsilon**2) * (g - weighted_average(e1, e2, e3)) - div / sqrt_g - - -def rhs_fun(e1, e2, e3): - return epsilon * gyrokinetic_operator(e1, e2, e3) - - -rhs_perturbation = GenericPerturbation(rhs_fun, given_in_basis="0") - -model.em_fields.source.add_perturbation(rhs_perturbation) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True) - - from matplotlib import pyplot as plt - - def slice_axes(k, X, Y, Z, E1, E2, E3): - """Axes (horizontal, vertical, labels) for plotting the slice eta_k = 0.5: the poloidal plane (R, z) - at eta3 = 0.5 and the logical planes at eta1 = 0.5 and eta2 = 0.5.""" - if k == 0: - return E2, E3, "eta2", "eta3" - elif k == 1: - return E1, E3, "eta1", "eta3" - else: - return np.sqrt(X**2 + Y**2), Z, "R", "z" - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error on the slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - h, v, hlabel, vlabel = slice_axes(k, X[idx], Y[idx], Z[idx], E1[idx], E2[idx], E3[idx]) - eta_k = (E1, E2, E3)[k][idx].flat[0] - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(h, v, data, shading="gouraud") - plt.colorbar() - plt.xlabel(hlabel) - plt.ylabel(vlabel) - plt.title(f"{name} {what}, slice at eta{k + 1} = {eta_k:.2f}") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Raw FEEC fields are evaluated directly from the saved spline coefficients at the - # cell centres of the simulation grid (serial Derham, safe on rank 0 only). - rhs = out.evaluate("em_fields/source", t=0).values - - phi_data = out.evaluate("em_fields/phi", t=-1) - phi = phi_data.values - - # logical coordinates of the evaluation points and their images x = F(eta) - E1, E2, E3 = np.meshgrid( - *(phi_data.coords[c].values for c in ("eta1", "eta2", "eta3")), - indexing="ij", - ) - X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) - - # pushed-forward exact solutions: the 0-forms at x = F(eta) are their values at eta - rhs_exact = rhs_fun(E1, E2, E3) - phi_exact = exact_solution(E1, E2, E3) - - fig_rhs = plot_slices(rhs, rhs_exact, "RHS") - fig_phi = plot_slices(phi, phi_exact, "Phi") - - rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) - rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) - - print(f"Max relative error in RHS: {rel_err_rhs:.2e}") - print(f"Max relative error in Phi: {rel_err_phi:.2e}") - - assert rel_err_rhs < 2.2e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_phi < 1.3e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_phi}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) - else: - plt.show() diff --git a/profiling/examples/GyrokineticPoisson/hollow_torus_strong_scaling_stiffness/params_gyrokinetic_poisson.py b/profiling/examples/GyrokineticPoisson/hollow_torus_strong_scaling_stiffness/params_gyrokinetic_poisson.py deleted file mode 100644 index a59151e96..000000000 --- a/profiling/examples/GyrokineticPoisson/hollow_torus_strong_scaling_stiffness/params_gyrokinetic_poisson.py +++ /dev/null @@ -1,372 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "GyrokineticPoisson strong scaling on hollow torus, stiffness preconditioner" -description = """ -Strong scaling test for the gyrokinetic Poisson equation with adiabatic electrons on a full hollow torus -(HollowTorus, a1=1e-2, a2=1, R0=3) in an ad hoc tokamak equilibrium (AdhocTorus, a=1, R0=3). -The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), -exciting the full spectrum. -Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with CG preconditioned by the Kronecker approximation of the inverse operator (StiffnessPreconditioner). -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -import numpy as np - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - ProfilingOptions, - Simulation, - Time, - domains, - equils, - grids, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import GyrokineticPoisson - -# Units -base_units = BaseUnits() - -# Model instance -epsilon = 1.0 -charge_number = 1 -model = GyrokineticPoisson(base_units=base_units, epsilon=epsilon, Z=charge_number) - -# List all variables and decide whether to save their data -model.em_fields.phi.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -a1 = 1e-2 -a2 = 1.0 -R0 = 3.0 -tor_period = 1 -domain = domains.HollowTorus(a1=a1, a2=a2, R0=R0, sfl=False, pol_period=1, tor_period=tor_period) - -# MHD equilibrium with circular concentric flux surfaces matching the domain (a = a2, same R0); -# the non-flat density and the ad hoc pressure keep n0/T0 = n0**2/p and n0/|B0|**2 of order one -B0 = 2.0 -q0 = 1.71 -q1 = 1.87 -n1 = 2.0 -n2 = 1.0 -na = 0.2 -beta = 0.179 -p0 = 1.0 -p1 = 0.1 -p2 = 0.1 -equil = equils.AdhocTorus( - a=a2, - R0=R0, - B0=B0, - q_kind=0, - q0=q0, - q1=q1, - n1=n1, - n2=n2, - na=na, - p_kind=1, - p0=p0, - p1=p1, - p2=p2, - beta=beta, -) - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profiling options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.gyrokinetic_poisson.options = model.propagators.gyrokinetic_poisson.Options( - which_geometry="toroidal", - solver="pcg", - precond="StiffnessPreconditioner", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -from scipy.special import erf - -from struphy.initial.base import GenericPerturbation - -# The exact solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5). -# Its value at the Dirichlet boundary eta1 = 0, 1 is exp(-0.25 / w**2) ~ 1e-11, and it is periodic -# in eta2 and eta3 up to the same accuracy. Being localized, it excites the whole spectrum of the -# operator, so CG has to work for convergence. Both the solution and the right-hand side are 0-forms -# given as functions of the logical coordinates (given_in_basis="0"); the pushed-forward solution -# at x = F(eta) is phi(eta). -# -# The model solves (see GyrokineticPoisson.doc_pde) -# -# 1/(charge_number*epsilon**2) * n0/T0 * (phi - ) - div(n0/|B0|**2 * (I - b0 b0^T) grad(phi)) = rho / epsilon, -# -# where is the average over eta2 and eta3 weighted with n0/T0 * sqrt(g). In logical coordinates the -# diffusion term reads -1/sqrt(g) * d_i(C^ij d_j phi), with C^ij = sqrt(g) * n0/|B0|**2 * (G^ij - b^i b^j), -# the inverse metric G^ij and the contravariant components b^i of b0. -w = 0.1 -eta0 = (0.5, 0.5, 0.5) - - -def gaussian_derivatives(e1, e2, e3): - """The Gaussian, its first and (pure) second derivatives and the mixed derivative d2 d3.""" - eta = (e1, e2, e3) - g = np.exp(-sum((e - c) ** 2 for e, c in zip(eta, eta0)) / w**2) - d = [-2 * (e - c) / w**2 * g for e, c in zip(eta, eta0)] - dd = [(4 * (e - c) ** 2 / w**4 - 2 / w**2) * g for e, c in zip(eta, eta0)] - d23 = 4 * (e2 - eta0[1]) * (e3 - eta0[2]) / w**4 * g - return g, d, dd, d23 - - -def exact_solution(e1, e2, e3): - return gaussian_derivatives(e1, e2, e3)[0] - - -def equilibrium_coefficients(e1, e2): - """sqrt(g), n0/T0 and C^11, C^22, C^23, C^33 (C^12 = C^13 = 0) as functions of (eta1, eta2). - - HollowTorus (sfl=False): r = a1 + (a2 - a1) * eta1, theta = 2*pi*eta2, phi = 2*pi*eta3 / tor_period, - R = R0 + r*cos(theta), with the scale factors h1 = a2 - a1, h2 = 2*pi*r, h3 = 2*pi*R / tor_period. - AdhocTorus (q_kind=0, p_kind=1): B0 = grad(psi) x grad(phi) - B0*R0 grad(phi) with - dpsi/dr = B0 * r / (q * sqrt(1 - r**2/R0**2)), hence |B_pol| = dpsi/dr / R and |B_tor| = B0*R0 / R, - q = q0 + (q1 - q0) * r**2 / a**2, n0 = na + (1 - na) * (1 - (r/a)**n1)**n2 and - p = beta * B0**2 / 2 * (p0 - p1 * r**2/a**2 - p2 * r**4/a**4), T0 = p/n0. - Only operations defined for complex numbers are used, such that the derivatives can be computed - with the complex step. - """ - da = a2 - a1 - r = a1 + da * e1 - theta = 2 * np.pi * e2 - R = R0 + r * np.cos(theta) - q = q0 + (q1 - q0) * r**2 / a2**2 - n = na + (1 - na) * (1 - (r / a2) ** n1) ** n2 - p = beta * B0**2 / 2 * (p0 - p1 * r**2 / a2**2 - p2 * r**4 / a2**4) - b_pol = B0 * r / (q * np.sqrt(1 - r**2 / R0**2)) / R - b_tor = B0 * R0 / R - abs_b_sq = b_pol**2 + b_tor**2 - - h1, h2, h3 = da, 2 * np.pi * r, 2 * np.pi * R / tor_period - sqrt_g = h1 * h2 * h3 - # contravariant components of the unit vector b0 = B0 / |B0| (b^1 = 0), signs from the - # orientation of the mapping (only the sign of b^2 * b^3 enters) - b2 = b_pol / np.sqrt(abs_b_sq) / h2 - b3 = b_tor / np.sqrt(abs_b_sq) / h3 - - weight = sqrt_g * n / abs_b_sq - c11 = weight / h1**2 - c22 = weight * (1 / h2**2 - b2**2) - c23 = -weight * b2 * b3 - c33 = weight * (1 / h3**2 - b3**2) - return sqrt_g, n**2 / p, c11, c22, c23, c33 - - -def complex_step(f, x, h=1e-30): - """Derivative f'(x) with the complex step, exact up to machine precision for analytic f.""" - return f(x + 1j * h).imag / h - - -def weighted_average(e1, e2, e3): - """Average of the exact solution over eta2 and eta3, weighted with n0/T0 * sqrt(g). - - The weight is proportional to f(eta1) * R = f(eta1) * (R0 + r*cos(2*pi*eta2)), hence the average is - g1 * int3 * (R0 * int2 + r * int_cos) / R0 with the integrals over [0, 1] of the Gaussians - g2 and g3 (int2 = int3) and of g2 * cos(2*pi*eta2) (int_cos). - """ - r = a1 + (a2 - a1) * e1 - g1 = np.exp(-((e1 - eta0[0]) ** 2) / w**2) - int3 = w * np.sqrt(np.pi) * erf(0.5 / w) - int_cos = -w * np.sqrt(np.pi) * np.exp(-((np.pi * w) ** 2)) * erf(0.5 / w - 1j * np.pi * w).real - return g1 * int3 * (R0 * int3 + r * int_cos) / R0 + 0 * e2 + 0 * e3 - - -def gyrokinetic_operator(e1, e2, e3): - """Left-hand side of the gyrokinetic Poisson equation applied to the exact solution.""" - g, d, dd, d23 = gaussian_derivatives(e1, e2, e3) - sqrt_g, n_over_t, c11, c22, c23, c33 = equilibrium_coefficients(e1, e2) - dc11 = complex_step(lambda x: equilibrium_coefficients(x, e2)[2], e1) - dc22 = complex_step(lambda x: equilibrium_coefficients(e1, x)[3], e2) - dc23 = complex_step(lambda x: equilibrium_coefficients(e1, x)[4], e2) - div = dc11 * d[0] + c11 * dd[0] + dc22 * d[1] + c22 * dd[1] + dc23 * d[2] + 2 * c23 * d23 + c33 * dd[2] - return n_over_t / (charge_number * epsilon**2) * (g - weighted_average(e1, e2, e3)) - div / sqrt_g - - -def rhs_fun(e1, e2, e3): - return epsilon * gyrokinetic_operator(e1, e2, e3) - - -rhs_perturbation = GenericPerturbation(rhs_fun, given_in_basis="0") - -model.em_fields.source.add_perturbation(rhs_perturbation) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True) - - from matplotlib import pyplot as plt - - def slice_axes(k, X, Y, Z, E1, E2, E3): - """Axes (horizontal, vertical, labels) for plotting the slice eta_k = 0.5: the poloidal plane (R, z) - at eta3 = 0.5 and the logical planes at eta1 = 0.5 and eta2 = 0.5.""" - if k == 0: - return E2, E3, "eta2", "eta3" - elif k == 1: - return E1, E3, "eta1", "eta3" - else: - return np.sqrt(X**2 + Y**2), Z, "R", "z" - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error on the slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - h, v, hlabel, vlabel = slice_axes(k, X[idx], Y[idx], Z[idx], E1[idx], E2[idx], E3[idx]) - eta_k = (E1, E2, E3)[k][idx].flat[0] - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(h, v, data, shading="gouraud") - plt.colorbar() - plt.xlabel(hlabel) - plt.ylabel(vlabel) - plt.title(f"{name} {what}, slice at eta{k + 1} = {eta_k:.2f}") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Raw FEEC fields are evaluated directly from the saved spline coefficients at the - # cell centres of the simulation grid (serial Derham, safe on rank 0 only). - rhs = out.evaluate("em_fields/source", t=0).values - - phi_data = out.evaluate("em_fields/phi", t=-1) - phi = phi_data.values - - # logical coordinates of the evaluation points and their images x = F(eta) - E1, E2, E3 = np.meshgrid( - *(phi_data.coords[c].values for c in ("eta1", "eta2", "eta3")), - indexing="ij", - ) - X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) - - # pushed-forward exact solutions: the 0-forms at x = F(eta) are their values at eta - rhs_exact = rhs_fun(E1, E2, E3) - phi_exact = exact_solution(E1, E2, E3) - - fig_rhs = plot_slices(rhs, rhs_exact, "RHS") - fig_phi = plot_slices(phi, phi_exact, "Phi") - - rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) - rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) - - print(f"Max relative error in RHS: {rel_err_rhs:.2e}") - print(f"Max relative error in Phi: {rel_err_phi:.2e}") - - assert rel_err_rhs < 2.2e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_phi < 1.3e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_phi}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) - else: - plt.show() diff --git a/profiling/examples/GyrokineticPoisson/polar_cylinder_strong_scaling/params_gyrokinetic_poisson.py b/profiling/examples/GyrokineticPoisson/polar_cylinder_strong_scaling/params_gyrokinetic_poisson.py new file mode 100644 index 000000000..5394666ca --- /dev/null +++ b/profiling/examples/GyrokineticPoisson/polar_cylinder_strong_scaling/params_gyrokinetic_poisson.py @@ -0,0 +1,363 @@ +# ----------------------------- +# Description of the simulation +# ----------------------------- +# Please fill in a verbal description of the simulation. +# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. + +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# The other flags select the variant (spline space and preconditioner), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--splines", + default="polar", + choices=["polar", "tensor"], + help="C1 polar splines on a disc (IGAPolarCylinder) or tensor-product splines on a cylinder with a hole (HollowCylinder).", +) +parser.add_argument( + "--precond", + default="None", + choices=[ + "None", + "MassMatrixPreconditioner", + "PolarMassMatrixPreconditioner", + "StiffnessPreconditioner", + "MultiGrid", + ], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument( + "--poloidal-solver", + default="schur", + choices=["schur", "lu", "ilu"], + help="Poloidal solver of the polar preconditioners and projectors.", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 66, 64], help="Number of elements per direction.") +parser.add_argument( + "--hole-bc", + default="free", + choices=["free", "dirichlet"], + help="Boundary condition at the hole of the tensor-product domain (default free; the solution does not vanish there).", +) +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +name = f"GyrokineticPoisson strong scaling on a cylinder, {args.splines} splines, preconditioner {precond}" +description = f""" +Strong scaling test for the gyrokinetic Poisson equation with adiabatic electrons on a cylinder of radius 1 and +length 2*pi in a screw pinch equilibrium (ScrewPinch, a=1, R0=1), comparing C1 polar splines (IGAPolarCylinder, +no hole) with tensor-product splines on a cylinder with a hole (HollowCylinder, a1=1e-2). +The manufactured solution is a Gaussian whose gradient is largest on the axis r=0, with a non-zero average over eta3. +Homogeneous Dirichlet boundary conditions are set at r=1 (boundary condition {args.hole_bc} at the hole). +The linear system is solved with CG, preconditioner {precond} (poloidal solver {args.poloidal_solver}). +""" + +import logging + +from struphy import set_logging_level + +set_logging_level(logging.INFO) + +# ------------------ +# Import Struphy API +# ------------------ +import numpy as np + +from struphy import ( + BaseUnits, + DerhamOptions, + EnvironmentOptions, + ProfilingOptions, + Simulation, + Time, + domains, + equils, + grids, +) + +# --------------------- +# Instance of the model +# --------------------- +from struphy.models import GyrokineticPoisson + +# Units +base_units = BaseUnits() + +# Model instance +epsilon = 1.0 +charge_number = 1 +model = GyrokineticPoisson(base_units=base_units, epsilon=epsilon, Z=charge_number) + +# List all variables and decide whether to save their data +model.em_fields.phi.save_data = True +model.em_fields.source.save_data = True + +# -------------------------- +# Instance of the simulation +# -------------------------- + +# Environment options +env = EnvironmentOptions( + sim_folder=f"sim_{args.id:02d}", + restart=False, +) + +# Time stepping +time_opts = Time() + +# Grid (polar splines need a multiple of 3 elements in eta2) +num_elements = tuple(args.num_elements) +degree = (3, 3, 3) +grid = grids.TensorProductGrid(num_elements=num_elements, mpi_dims_mask=(True, True, True)) + +# Geometry and Derham options (the screw pinch requires Lz = 2*pi*R0) +a = 1.0 +R0 = 1.0 +Lz = 2 * np.pi * R0 +if args.splines == "polar": + # the spline mapping has the resolution of the grid in the poloidal plane + domain = domains.IGAPolarCylinder(num_elements=num_elements[:2], degree=degree[:2], a=a, Lz=Lz) + derham_opts = DerhamOptions( + degree=degree, + bcs=(("free", "dirichlet"), None, None), + polar_splines=True, + polar_poloidal_solver=args.poloidal_solver, + ) +else: + a1 = 1e-2 + domain = domains.HollowCylinder(a1=a1, a2=a, Lz=Lz) + derham_opts = DerhamOptions(degree=degree, bcs=((args.hole_bc, "dirichlet"), None, None)) + +# MHD equilibrium: a non-flat density and the pressure offset p0 keep n0/T0 = n0**2/p and n0/|B0|**2 +# of order one and varying in r +B0 = 1.0 +q0 = 1.05 +q1 = 1.80 +n1 = 2.0 +n2 = 1.0 +na = 0.2 +p0 = 0.1 +equil = equils.ScrewPinch(a=a, R0=R0, B0=B0, q0=q0, q1=q1, n1=n1, n2=n2, na=na, p0=p0) + +# Profiling options +profiling_opts = ProfilingOptions( + use_line_profiler=True, +) + +# Simulation object +sim = Simulation( + model=model, + name=name, + description=description, + params_path=__file__, + env=env, + time_opts=time_opts, + domain=domain, + equil=equil, + grid=grid, + derham_opts=derham_opts, + profiling_opts=profiling_opts, +) + +# ------------------ +# Propagator options +# ------------------ + +from struphy.linear_algebra.solver import SolverParameters + +# options of the polar preconditioners +precond_params = {} +if args.splines == "polar" and precond in ("PolarMassMatrixPreconditioner", "StiffnessPreconditioner"): + precond_params["poloidal_solver"] = args.poloidal_solver + +solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) +model.propagators.gyrokinetic_poisson.options = model.propagators.gyrokinetic_poisson.Options( + which_geometry="cylindrical", + solver="pcg", + precond=precond, + precond_params=precond_params, + solver_params=solver_params, +) + +# ------------------ +# Initial conditions +# ------------------ +import sympy as sym + +from struphy.initial.base import GenericPerturbation + +# The model solves (see GyrokineticPoisson.doc_pde) +# +# 1/(charge_number*epsilon**2) * n0/T0 * (phi - ) - div(n0/|B0|**2 * (I - b0 b0^T) grad(phi)) = rho / epsilon, +# +# where is the average over eta3 (the axis direction z = Lz * eta3) weighted with n0/T0 * sqrt(g). +# The exact solution is phi = g(x, y) * h(z): a Gaussian g of width w in the cross-section, centered at distance +# w / sqrt(2) from the axis, where its gradient is largest, so the gradient is largest on the axis r = 0, and the +# modulation h = 1 + cos(2*pi*z/Lz)/2 along the axis. The weight does not depend on z, hence = g exactly. +# phi is negligible at r = 1 (Dirichlet boundary). Tensor-product splines cannot represent a non-zero gradient +# across the axis (they live on a domain with a hole and a boundary condition there, see --hole-bc); polar +# splines can. The right-hand side rho is computed symbolically from the equilibrium in Cartesian coordinates +# (checked against the struphy equilibrium below); phi and rho are given as functions of the Cartesian +# coordinates (given_in_basis="physical"). +x_, y_, z_ = sym.symbols("x y z", real=True) +w = 0.2 +x0 = w / np.sqrt(2.0) +g_sym = sym.exp(-((x_ - x0) ** 2 + y_**2) / w**2) +phi_sym = g_sym * (1 + sym.cos(2 * sym.pi * z_ / Lz) / 2) +avg_sym = g_sym + +# ScrewPinch in Cartesian coordinates: B0 = B0 * (e_z + r / (q R0) e_theta), q = q0 + (q1 - q0) * r**2 / a**2, +# n0 = na + (1 - na) * (1 - (r/a)**n1)**n2, p = p0 + B0**2 * (a/R0)**2 * q0 / (2 * (q1 - q0)) * (1/q**2 - 1/q1**2) +r2 = x_**2 + y_**2 +q = q0 + (q1 - q0) * r2 / a**2 +n_sym = na + (1 - na) * (1 - (r2 / a**2) ** (n1 / 2)) ** n2 +p_sym = p0 + B0**2 * (a / R0) ** 2 * q0 / (2 * (q1 - q0)) * (1 / q**2 - 1 / q1**2) +B_sym = [-B0 * y_ / (R0 * q), B0 * x_ / (R0 * q), B0 + 0 * x_] + + +def gyrokinetic_rho(phi_sym, avg_sym, n_sym, p_sym, B_sym): + """rho = epsilon * (n0/T0 / (Z eps**2) * (phi - ) - div(n0/|B0|**2 * (I - b0 b0^T) grad(phi))).""" + X = (x_, y_, z_) + abs_B_sq = sum(b**2 for b in B_sym) + grad = [sym.diff(phi_sym, xi) for xi in X] + b_dot_grad = sum(b * gr for b, gr in zip(B_sym, grad)) / abs_B_sq + flux = [n_sym / abs_B_sq * (gr - b * b_dot_grad) for b, gr in zip(B_sym, grad)] + div = sum(sym.diff(f, xi) for f, xi in zip(flux, X)) + return epsilon * (n_sym**2 / p_sym / (charge_number * epsilon**2) * (phi_sym - avg_sym) - div) + + +rho_sym = gyrokinetic_rho(phi_sym, avg_sym, n_sym, p_sym, B_sym) +phi_lam = sym.lambdify((x_, y_, z_), phi_sym, "numpy") +rho_lam = sym.lambdify((x_, y_, z_), rho_sym, "numpy") + +# the symbolic equilibrium must match the struphy equilibrium +_rng = np.random.default_rng(0) +_r, _t = _rng.uniform(1e-3, 0.99, 50), _rng.uniform(0.0, 2 * np.pi, 50) +_pts = (_r * np.cos(_t), _r * np.sin(_t), _rng.uniform(0.0, Lz, 50)) +_check = [ + (sym.lambdify((x_, y_, z_), n_sym, "numpy"), equil.n_xyz), + (sym.lambdify((x_, y_, z_), p_sym, "numpy"), equil.p_xyz), +] + [(sym.lambdify((x_, y_, z_), B_sym[k], "numpy"), lambda x, y, z, k=k: equil.b_xyz(x, y, z)[k]) for k in range(3)] +for _sym_fun, _equil_fun in _check: + assert np.allclose(_sym_fun(*_pts), _equil_fun(*_pts), rtol=1e-12, atol=1e-12), "Symbolic equilibrium mismatch." + + +def exact_solution(x, y, z): + return phi_lam(x, y, z) + 0.0 * x + + +def rhs_fun(x, y, z): + return rho_lam(x, y, z) + 0.0 * x + + +def distance_to_axis(x, y, z): + return np.sqrt(x**2 + y**2) + + +model.em_fields.source.add_perturbation(GenericPerturbation(rhs_fun, given_in_basis="physical")) + + +if __name__ == "__main__": + # switches for debugging and testing + estimate_mem = False + run = True + pproc = True + save_figs = True + + if estimate_mem: + sim.estimate_mem(print_report=True) + exit() + + if run: + out = sim.run(profiling_activated=True, one_time_step=True) + else: + # use this for working with existing sim data from a previous run + out = sim.output + + if pproc: + out.pproc(create_vtk=True, parallel=True) + + from matplotlib import pyplot as plt + + def plot_slices(num, exact, name): + """Numerical solution, exact solution and error on the cross-section at eta3 = 0.5 and the + half plane at eta2 = 0.5, as functions of (x, y) and (x, z).""" + fig = plt.figure(figsize=(12, 12)) + idx3 = (slice(None), slice(None), num.shape[2] // 2) + idx2 = (slice(None), num.shape[1] // 2, slice(None)) + for col, (idx, h, v, labels) in enumerate( + [(idx3, X, Y, ("x", "y")), (idx2, X, Z, ("x", "z"))], + ): + for row, (data, what) in enumerate( + [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] + ): + plt.subplot(3, 2, 2 * row + col + 1) + plt.pcolormesh(h[idx], v[idx], data, shading="gouraud") + plt.colorbar() + plt.xlabel(labels[0]) + plt.ylabel(labels[1]) + plt.title(f"{name} {what}") + fig.tight_layout() + return fig + + if sim.rank == 0: + # Raw FEEC fields are evaluated directly from the saved spline coefficients at the + # cell centres of the simulation grid (serial Derham, safe on rank 0 only). + rhs = out.evaluate("em_fields/source", t=0).values + + phi_data = out.evaluate("em_fields/phi", t=-1) + phi = phi_data.values + X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) + + rhs_exact = rhs_fun(X, Y, Z) + phi_exact = exact_solution(X, Y, Z) + + fig_rhs = plot_slices(rhs, rhs_exact, "RHS") + fig_phi = plot_slices(phi, phi_exact, "Phi") + + rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) + rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) + # error close to the axis, where the gradient of the solution is largest + near_axis = distance_to_axis(X, Y, Z) < 0.1 + rel_err_phi_axis = np.max(np.abs(phi - phi_exact)[near_axis]) / np.max(np.abs(phi_exact)) + + print(f"Max relative error in RHS: {rel_err_rhs:.2e}") + print(f"Max relative error in Phi: {rel_err_phi:.2e}") + print(f"Max relative error in Phi close to the axis (r < 0.1): {rel_err_phi_axis:.2e}") + + import json + import os + + # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare + # name resolved against the CWD. The profiling packaging picks these files up from + # here and uploads them as `results-out`. + results_dir = os.path.join(out.path_out, "results") + os.makedirs(results_dir, exist_ok=True) + + np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) + np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) + np.save(os.path.join(results_dir, "rel_err_phi_axis.npy"), rel_err_phi_axis) + np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) + np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) + with open(os.path.join(results_dir, "variant.json"), "w") as f: + json.dump( + { + "splines": args.splines, + "precond": args.precond, + "poloidal_solver": args.poloidal_solver, + "hole_bc": args.hole_bc, + }, + f, + ) + + if save_figs: + fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) + fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) + else: + plt.show() diff --git a/profiling/examples/GyrokineticPoisson/polar_torus_strong_scaling/params_gyrokinetic_poisson.py b/profiling/examples/GyrokineticPoisson/polar_torus_strong_scaling/params_gyrokinetic_poisson.py new file mode 100644 index 000000000..4b9563c90 --- /dev/null +++ b/profiling/examples/GyrokineticPoisson/polar_torus_strong_scaling/params_gyrokinetic_poisson.py @@ -0,0 +1,409 @@ +# ----------------------------- +# Description of the simulation +# ----------------------------- +# Please fill in a verbal description of the simulation. +# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. + +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# The other flags select the variant (spline space and preconditioner), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--splines", + default="polar", + choices=["polar", "tensor"], + help="C1 polar splines on a full torus (IGAPolarTorus) or tensor-product splines on a torus with a hole (HollowTorus).", +) +parser.add_argument( + "--precond", + default="None", + choices=[ + "None", + "MassMatrixPreconditioner", + "PolarMassMatrixPreconditioner", + "StiffnessPreconditioner", + "MultiGrid", + ], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument( + "--poloidal-solver", + default="schur", + choices=["schur", "lu", "ilu"], + help="Poloidal solver of the polar preconditioners and projectors.", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 66, 64], help="Number of elements per direction.") +parser.add_argument( + "--hole-bc", + default="free", + choices=["free", "dirichlet"], + help="Boundary condition at the hole of the tensor-product domain (default free; the solution does not vanish there).", +) +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +name = f"GyrokineticPoisson strong scaling on a torus, {args.splines} splines, preconditioner {precond}" +description = f""" +Strong scaling test for the gyrokinetic Poisson equation with adiabatic electrons on a full torus of minor radius 1 +and major radius 3 in an ad hoc tokamak equilibrium (AdhocTorus, a=1, R0=3), comparing C1 polar splines +(IGAPolarTorus, no hole) with tensor-product splines on a torus with a hole (HollowTorus, a1=1e-2). +The manufactured solution is a Gaussian ring whose gradient is largest on the magnetic axis r=0, with a non-zero +flux-surface average. +Homogeneous Dirichlet boundary conditions are set at r=1 (boundary condition {args.hole_bc} at the hole). +The linear system is solved with CG, preconditioner {precond} (poloidal solver {args.poloidal_solver}). +""" + +import logging + +from struphy import set_logging_level + +set_logging_level(logging.INFO) + +# ------------------ +# Import Struphy API +# ------------------ +import numpy as np + +from struphy import ( + BaseUnits, + DerhamOptions, + EnvironmentOptions, + ProfilingOptions, + Simulation, + Time, + domains, + equils, + grids, +) + +# --------------------- +# Instance of the model +# --------------------- +from struphy.models import GyrokineticPoisson + +# Units +base_units = BaseUnits() + +# Model instance +epsilon = 1.0 +charge_number = 1 +model = GyrokineticPoisson(base_units=base_units, epsilon=epsilon, Z=charge_number) + +# List all variables and decide whether to save their data +model.em_fields.phi.save_data = True +model.em_fields.source.save_data = True + +# -------------------------- +# Instance of the simulation +# -------------------------- + +# Environment options +env = EnvironmentOptions( + sim_folder=f"sim_{args.id:02d}", + restart=False, +) + +# Time stepping +time_opts = Time() + +# Grid (polar splines need a multiple of 3 elements in eta2) +num_elements = tuple(args.num_elements) +degree = (3, 3, 3) +grid = grids.TensorProductGrid(num_elements=num_elements, mpi_dims_mask=(True, True, True)) + +# Geometry and Derham options +a = 1.0 +R0 = 3.0 +if args.splines == "polar": + # the spline mapping has the resolution of the grid in the poloidal plane + domain = domains.IGAPolarTorus( + num_elements=num_elements[:2], degree=degree[:2], a=a, R0=R0, sfl=False, tor_period=1 + ) + derham_opts = DerhamOptions( + degree=degree, + bcs=(("free", "dirichlet"), None, None), + polar_splines=True, + polar_poloidal_solver=args.poloidal_solver, + ) +else: + a1 = 1e-2 + domain = domains.HollowTorus(a1=a1, a2=a, R0=R0, sfl=False, pol_period=1, tor_period=1) + derham_opts = DerhamOptions(degree=degree, bcs=((args.hole_bc, "dirichlet"), None, None)) + +# MHD equilibrium with circular concentric flux surfaces matching the domain (same a and R0); +# the non-flat density and the ad hoc pressure keep n0/T0 = n0**2/p and n0/|B0|**2 of order one +B0 = 2.0 +q0 = 1.71 +q1 = 1.87 +n1 = 2.0 +n2 = 1.0 +na = 0.2 +beta = 0.179 +p0 = 1.0 +p1 = 0.1 +p2 = 0.1 +equil = equils.AdhocTorus( + a=a, + R0=R0, + B0=B0, + q_kind=0, + q0=q0, + q1=q1, + n1=n1, + n2=n2, + na=na, + p_kind=1, + p0=p0, + p1=p1, + p2=p2, + beta=beta, +) + +# Profiling options +profiling_opts = ProfilingOptions( + use_line_profiler=True, +) + +# Simulation object +sim = Simulation( + model=model, + name=name, + description=description, + params_path=__file__, + env=env, + time_opts=time_opts, + domain=domain, + equil=equil, + grid=grid, + derham_opts=derham_opts, + profiling_opts=profiling_opts, +) + +# ------------------ +# Propagator options +# ------------------ + +from struphy.linear_algebra.solver import SolverParameters + +# options of the polar preconditioners +precond_params = {} +if args.splines == "polar" and precond in ("PolarMassMatrixPreconditioner", "StiffnessPreconditioner"): + precond_params["poloidal_solver"] = args.poloidal_solver + +solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) +model.propagators.gyrokinetic_poisson.options = model.propagators.gyrokinetic_poisson.Options( + which_geometry="toroidal", + solver="pcg", + precond=precond, + precond_params=precond_params, + solver_params=solver_params, +) + +# ------------------ +# Initial conditions +# ------------------ +import sympy as sym + +from struphy.initial.base import GenericPerturbation + +# The model solves (see GyrokineticPoisson.doc_pde) +# +# 1/(charge_number*epsilon**2) * n0/T0 * (phi - ) - div(n0/|B0|**2 * (I - b0 b0^T) grad(phi)) = rho / epsilon, +# +# where is the average over eta2 and eta3 (the poloidal angle theta and the toroidal angle) weighted with +# n0/T0 * sqrt(g), sqrt(g) ~ r * R. The exact solution is phi = g(R, z) * h: a Gaussian ring g of width w in the +# poloidal plane (R = sqrt(x**2 + y**2)), centered at distance w / sqrt(2) from the magnetic axis (R0, 0), where its +# gradient is largest, so the gradient is largest on the axis r = 0, and the toroidal modulation h = 1 + x / (2 R). +# The weight is a function of r only, hence (r) = int g R dtheta / int R dtheta (computed by quadrature in theta; +# h averages to 1). +# phi is negligible at r = 1 (Dirichlet boundary). Tensor-product splines cannot represent a non-zero gradient +# across the axis (they live on a domain with a hole and a boundary condition there, see --hole-bc); polar +# splines can. The right-hand side rho is computed symbolically from the equilibrium in Cartesian coordinates +# (checked against the struphy equilibrium below); phi and rho are given as functions of the Cartesian +# coordinates (given_in_basis="physical"). +x_, y_, z_ = sym.symbols("x y z", real=True) +w = 0.2 +x0 = w / np.sqrt(2.0) +R_ = sym.sqrt(x_**2 + y_**2) +g_sym = sym.exp(-((R_ - R0 - x0) ** 2 + z_**2) / w**2) +phi_sym = g_sym * (1 + x_ / (2 * R_)) + +# AdhocTorus (q_kind=0, p_kind=1) in Cartesian coordinates: B0 = grad(psi) x grad(phi) - B0*R0 grad(phi), with +# dpsi/dr = B0 * r / (q * sqrt(1 - r**2/R0**2)), r = sqrt((R - R0)**2 + z**2), q = q0 + (q1 - q0) * r**2 / a**2, +# i.e. B_R = -psi_z / R, B_phi = -B0 * R0 / R, B_z = psi_R / R; n0 = na + (1 - na) * (1 - (r/a)**n1)**n2 and +# p = B0**2 * beta / 2 * (p0 - p1 * r**2/a**2 - p2 * r**4/a**4) +r2 = (R_ - R0) ** 2 + z_**2 +q = q0 + (q1 - q0) * r2 / a**2 +dpsi_over_r = B0 / (q * sym.sqrt(1 - r2 / R0**2)) +B_R = -dpsi_over_r * z_ / R_ +B_phi = -B0 * R0 / R_ +B_z = dpsi_over_r * (R_ - R0) / R_ +n_sym = na + (1 - na) * (1 - (r2 / a**2) ** (n1 / 2)) ** n2 +p_sym = B0**2 * beta / 2 * (p0 - p1 * r2 / a**2 - p2 * r2**2 / a**4) +B_sym = [B_R * x_ / R_ - B_phi * y_ / R_, B_R * y_ / R_ + B_phi * x_ / R_, B_z] + + +def gyrokinetic_rho(phi_sym, avg_sym, n_sym, p_sym, B_sym): + """rho = epsilon * (n0/T0 / (Z eps**2) * (phi - ) - div(n0/|B0|**2 * (I - b0 b0^T) grad(phi))).""" + X = (x_, y_, z_) + abs_B_sq = sum(b**2 for b in B_sym) + grad = [sym.diff(phi_sym, xi) for xi in X] + b_dot_grad = sum(b * gr for b, gr in zip(B_sym, grad)) / abs_B_sq + flux = [n_sym / abs_B_sq * (gr - b * b_dot_grad) for b, gr in zip(B_sym, grad)] + div = sum(sym.diff(f, xi) for f, xi in zip(flux, X)) + return epsilon * (n_sym**2 / p_sym / (charge_number * epsilon**2) * (phi_sym - avg_sym) - div) + + +# rho without the average, which is added numerically below +rho_sym = gyrokinetic_rho(phi_sym, 0, n_sym, p_sym, B_sym) +phi_lam = sym.lambdify((x_, y_, z_), phi_sym, "numpy") +rho_lam = sym.lambdify((x_, y_, z_), rho_sym, "numpy") +n_over_t_lam = sym.lambdify((x_, y_, z_), n_sym**2 / p_sym, "numpy") + +# depends on r only: tabulated once (trapezoidal rule in theta, exponentially accurate for periodic +# integrands) and interpolated +_theta = np.linspace(0.0, 2 * np.pi, 512, endpoint=False) +_r_tab = np.linspace(0.0, a, 2001) +_R_tab = R0 + _r_tab[:, None] * np.cos(_theta) +_g_tab = np.exp(-((_r_tab[:, None] * np.cos(_theta) - x0) ** 2 + (_r_tab[:, None] * np.sin(_theta)) ** 2) / w**2) +_avg_tab = np.sum(_g_tab * _R_tab, axis=1) / np.sum(_R_tab, axis=1) + + +def average(x, y, z): + """Flux-surface average (r) = int g R dtheta / int R dtheta at the points (x, y, z).""" + r = np.sqrt((np.sqrt(x**2 + y**2) - R0) ** 2 + z**2) + return np.interp(r, _r_tab, _avg_tab) + + +# the symbolic equilibrium must match the struphy equilibrium (points in the torus) +_rng = np.random.default_rng(0) +_r, _t, _p = _rng.uniform(1e-3, 0.99, 50), _rng.uniform(0.0, 2 * np.pi, 50), _rng.uniform(0.0, 2 * np.pi, 50) +_pts = ((R0 + _r * np.cos(_t)) * np.cos(_p), (R0 + _r * np.cos(_t)) * np.sin(_p), _r * np.sin(_t)) +_check = [ + (sym.lambdify((x_, y_, z_), n_sym, "numpy"), equil.n_xyz), + (sym.lambdify((x_, y_, z_), p_sym, "numpy"), equil.p_xyz), +] + [(sym.lambdify((x_, y_, z_), B_sym[k], "numpy"), lambda x, y, z, k=k: equil.b_xyz(x, y, z)[k]) for k in range(3)] +for _sym_fun, _equil_fun in _check: + assert np.allclose(_sym_fun(*_pts), _equil_fun(*_pts), rtol=1e-12, atol=1e-12), "Symbolic equilibrium mismatch." + + +def exact_solution(x, y, z): + return phi_lam(x, y, z) + 0.0 * x + + +def rhs_fun(x, y, z): + avg = average(x, y, z) + return rho_lam(x, y, z) - epsilon * n_over_t_lam(x, y, z) / (charge_number * epsilon**2) * avg + 0.0 * x + + +def distance_to_axis(x, y, z): + return np.sqrt((np.sqrt(x**2 + y**2) - R0) ** 2 + z**2) + + +model.em_fields.source.add_perturbation(GenericPerturbation(rhs_fun, given_in_basis="physical")) + + +if __name__ == "__main__": + # switches for debugging and testing + estimate_mem = False + run = True + pproc = True + save_figs = True + + if estimate_mem: + sim.estimate_mem(print_report=True) + exit() + + if run: + out = sim.run(profiling_activated=True, one_time_step=True) + else: + # use this for working with existing sim data from a previous run + out = sim.output + + if pproc: + out.pproc(create_vtk=True, parallel=True) + + from matplotlib import pyplot as plt + + def plot_slices(num, exact, name): + """Numerical solution, exact solution and error on the poloidal plane at eta3 = 0.5 and on the + surface eta2 = 0.5 seen from above, as functions of (R, z) and (x, y).""" + fig = plt.figure(figsize=(12, 12)) + idx3 = (slice(None), slice(None), num.shape[2] // 2) + idx2 = (slice(None), num.shape[1] // 2, slice(None)) + for col, (idx, h, v, labels) in enumerate( + [(idx3, np.sqrt(X**2 + Y**2), Z, ("R", "z")), (idx2, X, Y, ("x", "y"))], + ): + for row, (data, what) in enumerate( + [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] + ): + plt.subplot(3, 2, 2 * row + col + 1) + plt.pcolormesh(h[idx], v[idx], data, shading="gouraud") + plt.colorbar() + plt.xlabel(labels[0]) + plt.ylabel(labels[1]) + plt.title(f"{name} {what}") + fig.tight_layout() + return fig + + if sim.rank == 0: + # Raw FEEC fields are evaluated directly from the saved spline coefficients at the + # cell centres of the simulation grid (serial Derham, safe on rank 0 only). + rhs = out.evaluate("em_fields/source", t=0).values + + phi_data = out.evaluate("em_fields/phi", t=-1) + phi = phi_data.values + X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) + + rhs_exact = rhs_fun(X, Y, Z) + phi_exact = exact_solution(X, Y, Z) + + fig_rhs = plot_slices(rhs, rhs_exact, "RHS") + fig_phi = plot_slices(phi, phi_exact, "Phi") + + rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) + rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) + # error close to the axis, where the gradient of the solution is largest + near_axis = distance_to_axis(X, Y, Z) < 0.1 + rel_err_phi_axis = np.max(np.abs(phi - phi_exact)[near_axis]) / np.max(np.abs(phi_exact)) + + print(f"Max relative error in RHS: {rel_err_rhs:.2e}") + print(f"Max relative error in Phi: {rel_err_phi:.2e}") + print(f"Max relative error in Phi close to the axis (r < 0.1): {rel_err_phi_axis:.2e}") + + import json + import os + + # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare + # name resolved against the CWD. The profiling packaging picks these files up from + # here and uploads them as `results-out`. + results_dir = os.path.join(out.path_out, "results") + os.makedirs(results_dir, exist_ok=True) + + np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) + np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) + np.save(os.path.join(results_dir, "rel_err_phi_axis.npy"), rel_err_phi_axis) + np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) + np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) + with open(os.path.join(results_dir, "variant.json"), "w") as f: + json.dump( + { + "splines": args.splines, + "precond": args.precond, + "poloidal_solver": args.poloidal_solver, + "hole_bc": args.hole_bc, + }, + f, + ) + + if save_figs: + fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) + fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) + else: + plt.show() diff --git a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_cylinder_strong_scaling.py b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_cylinder_strong_scaling.py index 075c52c12..5d4fa6a3d 100644 --- a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_cylinder_strong_scaling.py +++ b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_cylinder_strong_scaling.py @@ -1,4 +1,4 @@ -"""GyrokineticPoisson strong scaling profiling case on hollow cylinder, no preconditioner. +"""GyrokineticPoisson strong scaling profiling case on hollow cylinder. This file defines the GyrokineticPoisson strong scaling profiling case (the `ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a @@ -7,6 +7,8 @@ each run as soon as its own job finishes. Each generated script runs the simulation itself by invoking `params_gyrokinetic_poisson.py` directly (its `__main__` block is the worker). + +The variant (preconditioner) is chosen with `--variant`; run this script once per variant. """ import argparse @@ -18,6 +20,29 @@ from profiling_job import ProfilingCase # noqa: E402 +# variant -> (label, name, description, flags of the params file); the labels identify the +# cases in the profiling-data repo and are kept from the former one-case-per-script layout +VARIANTS = { + "none": ( + "gyrokinetic_poisson_hollow_cylinder_strong_scaling", + "GyrokineticPoisson on hollow cylinder strong scaling test, no preconditioner", + "Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a hollow cylinder in a screw pinch equilibrium (HollowCylinder, a1=1e-2, a2=1, Lz=2*pi; ScrewPinch, a=1, R0=1), solved with unpreconditioned CG.", + [], + ), + "multigrid": ( + "gyrokinetic_poisson_hollow_cylinder_strong_scaling_multigrid", + "GyrokineticPoisson on hollow cylinder strong scaling test, multigrid preconditioner", + "Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a hollow cylinder in a screw pinch equilibrium (HollowCylinder, a1=1e-2, a2=1, Lz=2*pi; ScrewPinch, a=1, R0=1), solved with CG preconditioned by geometric multigrid (MultiGridPreconditioner).", + ["--precond", "MultiGrid"], + ), + "stiffness": ( + "gyrokinetic_poisson_hollow_cylinder_strong_scaling_stiffness", + "GyrokineticPoisson on hollow cylinder strong scaling test, stiffness preconditioner", + "Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a hollow cylinder in a screw pinch equilibrium (HollowCylinder, a1=1e-2, a2=1, Lz=2*pi; ScrewPinch, a=1, R0=1), solved with CG preconditioned by the Kronecker approximation of the inverse operator (StiffnessPreconditioner).", + ["--precond", "StiffnessPreconditioner"], + ), +} + def main() -> None: @@ -25,6 +50,12 @@ def main() -> None: parser = argparse.ArgumentParser( description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), ) + parser.add_argument( + "--variant", + default="none", + choices=list(VARIANTS), + help="Preconditioner of the runs.", + ) parser.add_argument( "--upload", action="store_true", @@ -42,15 +73,16 @@ def main() -> None: help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', ) args = parser.parse_args() + label, name, description, param_flags = VARIANTS[args.variant] # Paths relative to this script's location, so it can be run from anywhere. script_dir = Path(__file__).resolve().parent params_dir = script_dir / "hollow_cylinder_strong_scaling" profiling_case = ProfilingCase( - label="gyrokinetic_poisson_hollow_cylinder_strong_scaling", - name="GyrokineticPoisson on hollow cylinder strong scaling test, no preconditioner", - description="Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a hollow cylinder in a screw pinch equilibrium (HollowCylinder, a1=1e-2, a2=1, Lz=2*pi; ScrewPinch, a=1, R0=1), solved with unpreconditioned CG.", + label=label, + name=name, + description=description, physics_problem="Field equation of gyrokinetic simulations with adiabatic electrons.", struphy_model_used="GyrokineticPoisson", params_source=params_dir / "params_gyrokinetic_poisson.py", @@ -61,7 +93,7 @@ def main() -> None: # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) + profiling_case.launch(num_tasks, param_flags=param_flags) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_cylinder_strong_scaling_multigrid.py b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_cylinder_strong_scaling_multigrid.py deleted file mode 100644 index 2e5607161..000000000 --- a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_cylinder_strong_scaling_multigrid.py +++ /dev/null @@ -1,71 +0,0 @@ -"""GyrokineticPoisson strong scaling profiling case on hollow cylinder, multigrid preconditioner. - -This file defines the GyrokineticPoisson strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_gyrokinetic_poisson.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_cylinder_strong_scaling_multigrid" - - profiling_case = ProfilingCase( - label="gyrokinetic_poisson_hollow_cylinder_strong_scaling_multigrid", - name="GyrokineticPoisson on hollow cylinder strong scaling test, multigrid preconditioner", - description="Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a hollow cylinder in a screw pinch equilibrium (HollowCylinder, a1=1e-2, a2=1, Lz=2*pi; ScrewPinch, a=1, R0=1), solved with CG preconditioned by geometric multigrid (MultiGridPreconditioner).", - physics_problem="Field equation of gyrokinetic simulations with adiabatic electrons.", - struphy_model_used="GyrokineticPoisson", - params_source=params_dir / "params_gyrokinetic_poisson.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_cylinder_strong_scaling_stiffness.py b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_cylinder_strong_scaling_stiffness.py deleted file mode 100644 index bf6069a1b..000000000 --- a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_cylinder_strong_scaling_stiffness.py +++ /dev/null @@ -1,71 +0,0 @@ -"""GyrokineticPoisson strong scaling profiling case on hollow cylinder, stiffness preconditioner. - -This file defines the GyrokineticPoisson strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_gyrokinetic_poisson.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_cylinder_strong_scaling_stiffness" - - profiling_case = ProfilingCase( - label="gyrokinetic_poisson_hollow_cylinder_strong_scaling_stiffness", - name="GyrokineticPoisson on hollow cylinder strong scaling test, stiffness preconditioner", - description="Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a hollow cylinder in a screw pinch equilibrium (HollowCylinder, a1=1e-2, a2=1, Lz=2*pi; ScrewPinch, a=1, R0=1), solved with CG preconditioned by the Kronecker approximation of the inverse operator (StiffnessPreconditioner).", - physics_problem="Field equation of gyrokinetic simulations with adiabatic electrons.", - struphy_model_used="GyrokineticPoisson", - params_source=params_dir / "params_gyrokinetic_poisson.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_torus_strong_scaling.py b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_torus_strong_scaling.py index fbb91a832..07d57fde6 100644 --- a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_torus_strong_scaling.py +++ b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_torus_strong_scaling.py @@ -1,4 +1,4 @@ -"""GyrokineticPoisson strong scaling profiling case on hollow torus, no preconditioner. +"""GyrokineticPoisson strong scaling profiling case on hollow torus. This file defines the GyrokineticPoisson strong scaling profiling case (the `ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a @@ -7,6 +7,8 @@ each run as soon as its own job finishes. Each generated script runs the simulation itself by invoking `params_gyrokinetic_poisson.py` directly (its `__main__` block is the worker). + +The variant (preconditioner) is chosen with `--variant`; run this script once per variant. """ import argparse @@ -18,6 +20,29 @@ from profiling_job import ProfilingCase # noqa: E402 +# variant -> (label, name, description, flags of the params file); the labels identify the +# cases in the profiling-data repo and are kept from the former one-case-per-script layout +VARIANTS = { + "none": ( + "gyrokinetic_poisson_hollow_torus_strong_scaling", + "GyrokineticPoisson on hollow torus strong scaling test, no preconditioner", + "Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a full hollow torus in an ad hoc tokamak equilibrium (HollowTorus, a1=1e-2, a2=1, R0=3; AdhocTorus, a=1, R0=3), solved with unpreconditioned CG.", + [], + ), + "multigrid": ( + "gyrokinetic_poisson_hollow_torus_strong_scaling_multigrid", + "GyrokineticPoisson on hollow torus strong scaling test, multigrid preconditioner", + "Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a full hollow torus in an ad hoc tokamak equilibrium (HollowTorus, a1=1e-2, a2=1, R0=3; AdhocTorus, a=1, R0=3), solved with CG preconditioned by geometric multigrid (MultiGridPreconditioner).", + ["--precond", "MultiGrid"], + ), + "stiffness": ( + "gyrokinetic_poisson_hollow_torus_strong_scaling_stiffness", + "GyrokineticPoisson on hollow torus strong scaling test, stiffness preconditioner", + "Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a full hollow torus in an ad hoc tokamak equilibrium (HollowTorus, a1=1e-2, a2=1, R0=3; AdhocTorus, a=1, R0=3), solved with CG preconditioned by the Kronecker approximation of the inverse operator (StiffnessPreconditioner).", + ["--precond", "StiffnessPreconditioner"], + ), +} + def main() -> None: @@ -25,6 +50,12 @@ def main() -> None: parser = argparse.ArgumentParser( description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), ) + parser.add_argument( + "--variant", + default="none", + choices=list(VARIANTS), + help="Preconditioner of the runs.", + ) parser.add_argument( "--upload", action="store_true", @@ -42,15 +73,16 @@ def main() -> None: help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', ) args = parser.parse_args() + label, name, description, param_flags = VARIANTS[args.variant] # Paths relative to this script's location, so it can be run from anywhere. script_dir = Path(__file__).resolve().parent params_dir = script_dir / "hollow_torus_strong_scaling" profiling_case = ProfilingCase( - label="gyrokinetic_poisson_hollow_torus_strong_scaling", - name="GyrokineticPoisson on hollow torus strong scaling test, no preconditioner", - description="Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a full hollow torus in an ad hoc tokamak equilibrium (HollowTorus, a1=1e-2, a2=1, R0=3; AdhocTorus, a=1, R0=3), solved with unpreconditioned CG.", + label=label, + name=name, + description=description, physics_problem="Field equation of gyrokinetic simulations with adiabatic electrons.", struphy_model_used="GyrokineticPoisson", params_source=params_dir / "params_gyrokinetic_poisson.py", @@ -61,7 +93,7 @@ def main() -> None: # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) + profiling_case.launch(num_tasks, param_flags=param_flags) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_torus_strong_scaling_multigrid.py b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_torus_strong_scaling_multigrid.py deleted file mode 100644 index e0771350b..000000000 --- a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_torus_strong_scaling_multigrid.py +++ /dev/null @@ -1,71 +0,0 @@ -"""GyrokineticPoisson strong scaling profiling case on hollow torus, multigrid preconditioner. - -This file defines the GyrokineticPoisson strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_gyrokinetic_poisson.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_torus_strong_scaling_multigrid" - - profiling_case = ProfilingCase( - label="gyrokinetic_poisson_hollow_torus_strong_scaling_multigrid", - name="GyrokineticPoisson on hollow torus strong scaling test, multigrid preconditioner", - description="Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a full hollow torus in an ad hoc tokamak equilibrium (HollowTorus, a1=1e-2, a2=1, R0=3; AdhocTorus, a=1, R0=3), solved with CG preconditioned by geometric multigrid (MultiGridPreconditioner).", - physics_problem="Field equation of gyrokinetic simulations with adiabatic electrons.", - struphy_model_used="GyrokineticPoisson", - params_source=params_dir / "params_gyrokinetic_poisson.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_torus_strong_scaling_stiffness.py b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_torus_strong_scaling_stiffness.py deleted file mode 100644 index 5fecc1025..000000000 --- a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_hollow_torus_strong_scaling_stiffness.py +++ /dev/null @@ -1,71 +0,0 @@ -"""GyrokineticPoisson strong scaling profiling case on hollow torus, stiffness preconditioner. - -This file defines the GyrokineticPoisson strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_gyrokinetic_poisson.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_torus_strong_scaling_stiffness" - - profiling_case = ProfilingCase( - label="gyrokinetic_poisson_hollow_torus_strong_scaling_stiffness", - name="GyrokineticPoisson on hollow torus strong scaling test, stiffness preconditioner", - description="Strong scaling of the GyrokineticPoisson model (adiabatic electrons, epsilon=1, Z=1) with manufactured solution on a full hollow torus in an ad hoc tokamak equilibrium (HollowTorus, a1=1e-2, a2=1, R0=3; AdhocTorus, a=1, R0=3), solved with CG preconditioned by the Kronecker approximation of the inverse operator (StiffnessPreconditioner).", - physics_problem="Field equation of gyrokinetic simulations with adiabatic electrons.", - struphy_model_used="GyrokineticPoisson", - params_source=params_dir / "params_gyrokinetic_poisson.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_polar_cylinder_strong_scaling.py b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_polar_cylinder_strong_scaling.py new file mode 100644 index 000000000..87f2c832e --- /dev/null +++ b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_polar_cylinder_strong_scaling.py @@ -0,0 +1,123 @@ +"""GyrokineticPoisson polar cylinder strong scaling profiling case: polar splines versus tensor-product splines on a cylinder with a hole. + +This file defines one variant of the GyrokineticPoisson strong scaling profiling case on a cylinder (radius 1, length 2*pi) in a screw pinch equilibrium (the +`ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a +SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch system, runs directly +on this machine. `finalize_run` then packages and uploads each run as soon as its own job finishes. +Each generated script runs the simulation itself by invoking `params_gyrokinetic_poisson.py` directly (its `__main__` +block is the worker), with the flags of the variant. + +The variant (spline space and preconditioner) is chosen with `--variant`; run this script once per variant. +""" + +import argparse +import sys +from pathlib import Path + +# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from profiling_job import ProfilingCase # noqa: E402 + +# variant -> (description, flags of params_gyrokinetic_poisson.py) +VARIANTS = { + "tensor_none": ( + "tensor-product splines, no preconditioner", + ["--splines", "tensor", "--precond", "None"], + ), + "tensor_stiffness": ( + "tensor-product splines, stiffness preconditioner", + ["--splines", "tensor", "--precond", "StiffnessPreconditioner"], + ), + "tensor_multigrid": ( + "tensor-product splines, multigrid preconditioner", + ["--splines", "tensor", "--precond", "MultiGrid"], + ), + "polar_none": ( + "polar splines, no preconditioner", + ["--splines", "polar", "--precond", "None"], + ), + "polar_tensor_mass": ( + "polar splines, tensor-product mass matrix preconditioner", + ["--splines", "polar", "--precond", "MassMatrixPreconditioner"], + ), + "polar_mass": ( + "polar splines, polar mass matrix preconditioner (Schur)", + ["--splines", "polar", "--precond", "PolarMassMatrixPreconditioner"], + ), + "polar_mass_ilu": ( + "polar splines, polar mass matrix preconditioner (ILU)", + ["--splines", "polar", "--precond", "PolarMassMatrixPreconditioner", "--poloidal-solver", "ilu"], + ), + "polar_stiffness": ( + "polar splines, polar stiffness preconditioner (Schur)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner"], + ), + "polar_stiffness_ilu": ( + "polar splines, polar stiffness preconditioner (ILU)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner", "--poloidal-solver", "ilu"], + ), +} + + +def main() -> None: + + # Parse arguments, do not remove --upload + parser = argparse.ArgumentParser( + description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), + ) + parser.add_argument( + "--variant", + required=True, + choices=list(VARIANTS), + help="Spline space and preconditioner of the runs.", + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload the packaged profiling results to the profiling-data repo.", + ) + parser.add_argument( + "--language", + default="fortran", + choices=["fortran", "c"], + help="Pyccel language to compile the Struphy kernels with.", + ) + parser.add_argument( + "--compiler", + default="GNU", + help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', + ) + args = parser.parse_args() + variant_description, param_flags = VARIANTS[args.variant] + + # Paths relative to this script's location, so it can be run from anywhere. + script_dir = Path(__file__).resolve().parent + params_dir = script_dir / "polar_cylinder_strong_scaling" + + profiling_case = ProfilingCase( + label=f"gyrokinetic_poisson_polar_cylinder_strong_scaling_{args.variant}", + name=f"GyrokineticPoisson on a cylinder strong scaling test, {variant_description}", + description=( + "Strong scaling of the GyrokineticPoisson model with a manufactured solution whose gradient is largest on the axis, " + "comparing C1 polar splines (IGAPolarCylinder) with tensor-product splines on a cylinder with a hole " + f"(HollowCylinder, a1=1e-2): {variant_description}." + ), + physics_problem="Field equation of gyrokinetic simulations with adiabatic electrons.", + struphy_model_used="GyrokineticPoisson", + params_source=params_dir / "params_gyrokinetic_poisson.py", + language=args.language, + compiler=args.compiler, + upload=args.upload, + ) + + # Launch one run per rank count + for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): + profiling_case.launch(num_tasks, param_flags=param_flags) + + # Package and push each run as its own job finishes. + profiling_case.finalize_run() + + +if __name__ == "__main__": + main() diff --git a/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_polar_torus_strong_scaling.py b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_polar_torus_strong_scaling.py new file mode 100644 index 000000000..74570ce17 --- /dev/null +++ b/profiling/examples/GyrokineticPoisson/submit_gyrokinetic_poisson_polar_torus_strong_scaling.py @@ -0,0 +1,123 @@ +"""GyrokineticPoisson polar torus strong scaling profiling case: polar splines versus tensor-product splines on a torus with a hole. + +This file defines one variant of the GyrokineticPoisson strong scaling profiling case on a full torus (minor radius 1, major radius 3) in an ad hoc tokamak equilibrium (the +`ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a +SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch system, runs directly +on this machine. `finalize_run` then packages and uploads each run as soon as its own job finishes. +Each generated script runs the simulation itself by invoking `params_gyrokinetic_poisson.py` directly (its `__main__` +block is the worker), with the flags of the variant. + +The variant (spline space and preconditioner) is chosen with `--variant`; run this script once per variant. +""" + +import argparse +import sys +from pathlib import Path + +# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from profiling_job import ProfilingCase # noqa: E402 + +# variant -> (description, flags of params_gyrokinetic_poisson.py) +VARIANTS = { + "tensor_none": ( + "tensor-product splines, no preconditioner", + ["--splines", "tensor", "--precond", "None"], + ), + "tensor_stiffness": ( + "tensor-product splines, stiffness preconditioner", + ["--splines", "tensor", "--precond", "StiffnessPreconditioner"], + ), + "tensor_multigrid": ( + "tensor-product splines, multigrid preconditioner", + ["--splines", "tensor", "--precond", "MultiGrid"], + ), + "polar_none": ( + "polar splines, no preconditioner", + ["--splines", "polar", "--precond", "None"], + ), + "polar_tensor_mass": ( + "polar splines, tensor-product mass matrix preconditioner", + ["--splines", "polar", "--precond", "MassMatrixPreconditioner"], + ), + "polar_mass": ( + "polar splines, polar mass matrix preconditioner (Schur)", + ["--splines", "polar", "--precond", "PolarMassMatrixPreconditioner"], + ), + "polar_mass_ilu": ( + "polar splines, polar mass matrix preconditioner (ILU)", + ["--splines", "polar", "--precond", "PolarMassMatrixPreconditioner", "--poloidal-solver", "ilu"], + ), + "polar_stiffness": ( + "polar splines, polar stiffness preconditioner (Schur)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner"], + ), + "polar_stiffness_ilu": ( + "polar splines, polar stiffness preconditioner (ILU)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner", "--poloidal-solver", "ilu"], + ), +} + + +def main() -> None: + + # Parse arguments, do not remove --upload + parser = argparse.ArgumentParser( + description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), + ) + parser.add_argument( + "--variant", + required=True, + choices=list(VARIANTS), + help="Spline space and preconditioner of the runs.", + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload the packaged profiling results to the profiling-data repo.", + ) + parser.add_argument( + "--language", + default="fortran", + choices=["fortran", "c"], + help="Pyccel language to compile the Struphy kernels with.", + ) + parser.add_argument( + "--compiler", + default="GNU", + help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', + ) + args = parser.parse_args() + variant_description, param_flags = VARIANTS[args.variant] + + # Paths relative to this script's location, so it can be run from anywhere. + script_dir = Path(__file__).resolve().parent + params_dir = script_dir / "polar_torus_strong_scaling" + + profiling_case = ProfilingCase( + label=f"gyrokinetic_poisson_polar_torus_strong_scaling_{args.variant}", + name=f"GyrokineticPoisson on a torus strong scaling test, {variant_description}", + description=( + "Strong scaling of the GyrokineticPoisson model with a manufactured solution whose gradient is largest on the axis, " + "comparing C1 polar splines (IGAPolarTorus) with tensor-product splines on a torus with a hole " + f"(HollowTorus, a1=1e-2): {variant_description}." + ), + physics_problem="Field equation of gyrokinetic simulations with adiabatic electrons.", + struphy_model_used="GyrokineticPoisson", + params_source=params_dir / "params_gyrokinetic_poisson.py", + language=args.language, + compiler=args.compiler, + upload=args.upload, + ) + + # Launch one run per rank count + for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): + profiling_case.launch(num_tasks, param_flags=param_flags) + + # Package and push each run as its own job finishes. + profiling_case.finalize_run() + + +if __name__ == "__main__": + main() diff --git a/profiling/examples/Poisson/cube_strong_scaling/params_poisson.py b/profiling/examples/Poisson/cube_strong_scaling/params_poisson.py index 40f4962fb..9e89775f6 100644 --- a/profiling/examples/Poisson/cube_strong_scaling/params_poisson.py +++ b/profiling/examples/Poisson/cube_strong_scaling/params_poisson.py @@ -4,13 +4,43 @@ # Please fill in a verbal description of the simulation. # It will be printed at the beginning of the simulation and can be used to keep track of the different runs. -name = "Poisson strong scaling on 3D cube, no preconditioner" -description = """ +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# `--precond` selects the preconditioner (the variant), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--precond", + default="None", + choices=["None", "MultiGrid", "StiffnessPreconditioner"], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 64, 64], help="Number of elements per direction.") +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +# preconditioner -> (name suffix, linear solver) for the name and the description +PRECONDS = { + None: ("no preconditioner", "unpreconditioned CG"), + "MultiGrid": ("multigrid preconditioner", "CG preconditioned by geometric multigrid"), + "StiffnessPreconditioner": ( + "stiffness preconditioner", + "CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner)", + ), +} + +name = f"Poisson strong scaling on 3D cube, {PRECONDS[precond][0]}" +description = f""" Strong scaling test for Poisson equation on a 3D cube (Cuboid). The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), exciting the full spectrum. Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with unpreconditioned CG. +The linear system is solved with {PRECONDS[precond][1]}. """ import logging @@ -19,7 +49,6 @@ set_logging_level(logging.INFO) -import argparse # ------------------ # Import Struphy API @@ -58,13 +87,6 @@ # -------------------------- # Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - env = EnvironmentOptions( sim_folder=f"sim_{args.id:02d}", restart=False, @@ -83,7 +105,7 @@ equil = None # Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) +grid = grids.TensorProductGrid(num_elements=tuple(args.num_elements), mpi_dims_mask=(True, True, True)) # Derham options derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) @@ -118,7 +140,7 @@ model.propagators.poisson.options = model.propagators.poisson.Options( stab_eps=0.0, solver="pcg", - precond=None, + precond=precond, solver_params=solver_params, ) diff --git a/profiling/examples/Poisson/cube_strong_scaling_multigrid/params_poisson.py b/profiling/examples/Poisson/cube_strong_scaling_multigrid/params_poisson.py deleted file mode 100644 index ff030f266..000000000 --- a/profiling/examples/Poisson/cube_strong_scaling_multigrid/params_poisson.py +++ /dev/null @@ -1,272 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Poisson strong scaling on 3D cube, multigrid preconditioner" -description = """ -Strong scaling test for Poisson equation on a 3D cube (Cuboid). -The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), -exciting the full spectrum. -Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with CG preconditioned by geometric multigrid. -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - FieldsBackground, - MultiGridOptions, - ProfilingOptions, - Simulation, - Time, - domains, - equils, - grids, - perturbations, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import Poisson - -# Units -base_units = BaseUnits() - -# Model instance -model = Poisson(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.phi.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -Lx = 2.0 -Ly = 3.0 -Lz = 4.0 -domain = domains.Cuboid(r1=Lx, l2=-Ly / 2, r2=Ly / 2, r3=Lz) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.poisson.options = model.propagators.poisson.Options( - stab_eps=0.0, - solver="pcg", - precond="MultiGrid", - multigrid=MultiGridOptions(), - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np - -from struphy.initial.base import GenericPerturbation - -# The exact solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5). -# Its value at the Dirichlet boundary eta1 = 0, 1 is exp(-0.25 / w**2) ~ 1e-11, and it is periodic -# in eta2 and eta3 up to the same accuracy. Being localized, it excites the whole spectrum of the -# Laplacian, so CG has to work for convergence. Both the solution and the right-hand side -# -Laplace(phi) are 0-forms given as functions of the logical coordinates (given_in_basis="0"); -# the pushed-forward solution at x = F(eta) is phi(eta). -w = 0.1 -eta0 = (0.5, 0.5, 0.5) - - -def gaussian_derivatives(e1, e2, e3): - """The Gaussian and its first and second derivatives with respect to eta1, eta2, eta3.""" - eta = (e1, e2, e3) - g = np.exp(-sum((e - c) ** 2 for e, c in zip(eta, eta0)) / w**2) - d = [-2 * (e - c) / w**2 * g for e, c in zip(eta, eta0)] - dd = [(4 * (e - c) ** 2 / w**4 - 2 / w**2) * g for e, c in zip(eta, eta0)] - return g, d, dd - - -def exact_solution(e1, e2, e3): - return gaussian_derivatives(e1, e2, e3)[0] - - -def laplacian(e1, e2, e3): - """Physical Laplacian of the exact solution, in logical coordinates (Cuboid: x_i = l_i + L_i * eta_i).""" - g, d, dd = gaussian_derivatives(e1, e2, e3) - return dd[0] / Lx**2 + dd[1] / Ly**2 + dd[2] / Lz**2 - - -def rhs_fun(e1, e2, e3): - return -laplacian(e1, e2, e3) - - -rhs_perturbation = GenericPerturbation(rhs_fun, given_in_basis="0") - -model.em_fields.source.add_perturbation(rhs_perturbation) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True) - - from matplotlib import pyplot as plt - - def slice_axes(k, X, Y, Z, E1, E2, E3): - """Axes (horizontal, vertical, labels) for plotting the slice eta_k = 0.5 in physical coordinates.""" - if k == 0: - return Y, Z, "y", "z" - elif k == 1: - return X, Z, "x", "z" - else: - return X, Y, "x", "y" - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error on the slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - h, v, hlabel, vlabel = slice_axes(k, X[idx], Y[idx], Z[idx], E1[idx], E2[idx], E3[idx]) - eta_k = (E1, E2, E3)[k][idx].flat[0] - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(h, v, data, shading="gouraud") - plt.colorbar() - plt.xlabel(hlabel) - plt.ylabel(vlabel) - plt.title(f"{name} {what}, slice at eta{k + 1} = {eta_k:.2f}") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Raw FEEC fields are evaluated directly from the saved spline coefficients at the - # cell centres of the simulation grid (serial Derham, safe on rank 0 only). - rhs = out.evaluate("em_fields/source", t=0).values - - phi_data = out.evaluate("em_fields/phi", t=-1) - phi = phi_data.values - - # logical coordinates of the evaluation points and their images x = F(eta) - E1, E2, E3 = np.meshgrid( - *(phi_data.coords[c].values for c in ("eta1", "eta2", "eta3")), - indexing="ij", - ) - X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) - - # pushed-forward exact solutions: the 0-forms at x = F(eta) are their values at eta - rhs_exact = rhs_fun(E1, E2, E3) - phi_exact = exact_solution(E1, E2, E3) - - fig_rhs = plot_slices(rhs, rhs_exact, "RHS") - fig_phi = plot_slices(phi, phi_exact, "Phi") - - rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) - rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) - - print(f"Max relative error in RHS: {rel_err_rhs:.2e}") - print(f"Max relative error in Phi: {rel_err_phi:.2e}") - - assert rel_err_rhs < 1.7e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_phi < 1.2e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_phi}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) - else: - plt.show() diff --git a/profiling/examples/Poisson/cube_strong_scaling_stiffness/params_poisson.py b/profiling/examples/Poisson/cube_strong_scaling_stiffness/params_poisson.py deleted file mode 100644 index 4fe500f7e..000000000 --- a/profiling/examples/Poisson/cube_strong_scaling_stiffness/params_poisson.py +++ /dev/null @@ -1,270 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Poisson strong scaling on 3D cube, stiffness preconditioner" -description = """ -Strong scaling test for Poisson equation on a 3D cube (Cuboid). -The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), -exciting the full spectrum. -Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner). -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - FieldsBackground, - ProfilingOptions, - Simulation, - Time, - domains, - equils, - grids, - perturbations, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import Poisson - -# Units -base_units = BaseUnits() - -# Model instance -model = Poisson(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.phi.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -Lx = 2.0 -Ly = 3.0 -Lz = 4.0 -domain = domains.Cuboid(r1=Lx, l2=-Ly / 2, r2=Ly / 2, r3=Lz) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.poisson.options = model.propagators.poisson.Options( - stab_eps=0.0, - solver="pcg", - precond="StiffnessPreconditioner", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np - -from struphy.initial.base import GenericPerturbation - -# The exact solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5). -# Its value at the Dirichlet boundary eta1 = 0, 1 is exp(-0.25 / w**2) ~ 1e-11, and it is periodic -# in eta2 and eta3 up to the same accuracy. Being localized, it excites the whole spectrum of the -# Laplacian, so CG has to work for convergence. Both the solution and the right-hand side -# -Laplace(phi) are 0-forms given as functions of the logical coordinates (given_in_basis="0"); -# the pushed-forward solution at x = F(eta) is phi(eta). -w = 0.1 -eta0 = (0.5, 0.5, 0.5) - - -def gaussian_derivatives(e1, e2, e3): - """The Gaussian and its first and second derivatives with respect to eta1, eta2, eta3.""" - eta = (e1, e2, e3) - g = np.exp(-sum((e - c) ** 2 for e, c in zip(eta, eta0)) / w**2) - d = [-2 * (e - c) / w**2 * g for e, c in zip(eta, eta0)] - dd = [(4 * (e - c) ** 2 / w**4 - 2 / w**2) * g for e, c in zip(eta, eta0)] - return g, d, dd - - -def exact_solution(e1, e2, e3): - return gaussian_derivatives(e1, e2, e3)[0] - - -def laplacian(e1, e2, e3): - """Physical Laplacian of the exact solution, in logical coordinates (Cuboid: x_i = l_i + L_i * eta_i).""" - g, d, dd = gaussian_derivatives(e1, e2, e3) - return dd[0] / Lx**2 + dd[1] / Ly**2 + dd[2] / Lz**2 - - -def rhs_fun(e1, e2, e3): - return -laplacian(e1, e2, e3) - - -rhs_perturbation = GenericPerturbation(rhs_fun, given_in_basis="0") - -model.em_fields.source.add_perturbation(rhs_perturbation) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True) - - from matplotlib import pyplot as plt - - def slice_axes(k, X, Y, Z, E1, E2, E3): - """Axes (horizontal, vertical, labels) for plotting the slice eta_k = 0.5 in physical coordinates.""" - if k == 0: - return Y, Z, "y", "z" - elif k == 1: - return X, Z, "x", "z" - else: - return X, Y, "x", "y" - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error on the slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - h, v, hlabel, vlabel = slice_axes(k, X[idx], Y[idx], Z[idx], E1[idx], E2[idx], E3[idx]) - eta_k = (E1, E2, E3)[k][idx].flat[0] - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(h, v, data, shading="gouraud") - plt.colorbar() - plt.xlabel(hlabel) - plt.ylabel(vlabel) - plt.title(f"{name} {what}, slice at eta{k + 1} = {eta_k:.2f}") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Raw FEEC fields are evaluated directly from the saved spline coefficients at the - # cell centres of the simulation grid (serial Derham, safe on rank 0 only). - rhs = out.evaluate("em_fields/source", t=0).values - - phi_data = out.evaluate("em_fields/phi", t=-1) - phi = phi_data.values - - # logical coordinates of the evaluation points and their images x = F(eta) - E1, E2, E3 = np.meshgrid( - *(phi_data.coords[c].values for c in ("eta1", "eta2", "eta3")), - indexing="ij", - ) - X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) - - # pushed-forward exact solutions: the 0-forms at x = F(eta) are their values at eta - rhs_exact = rhs_fun(E1, E2, E3) - phi_exact = exact_solution(E1, E2, E3) - - fig_rhs = plot_slices(rhs, rhs_exact, "RHS") - fig_phi = plot_slices(phi, phi_exact, "Phi") - - rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) - rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) - - print(f"Max relative error in RHS: {rel_err_rhs:.2e}") - print(f"Max relative error in Phi: {rel_err_phi:.2e}") - - assert rel_err_rhs < 1.7e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_phi < 1.2e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_phi}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) - else: - plt.show() diff --git a/profiling/examples/Poisson/hollow_cylinder_strong_scaling/params_poisson.py b/profiling/examples/Poisson/hollow_cylinder_strong_scaling/params_poisson.py index 4af93b5d5..78c1a3508 100644 --- a/profiling/examples/Poisson/hollow_cylinder_strong_scaling/params_poisson.py +++ b/profiling/examples/Poisson/hollow_cylinder_strong_scaling/params_poisson.py @@ -4,13 +4,43 @@ # Please fill in a verbal description of the simulation. # It will be printed at the beginning of the simulation and can be used to keep track of the different runs. -name = "Poisson strong scaling on hollow cylinder, no preconditioner" -description = """ +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# `--precond` selects the preconditioner (the variant), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--precond", + default="None", + choices=["None", "MultiGrid", "StiffnessPreconditioner"], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 64, 64], help="Number of elements per direction.") +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +# preconditioner -> (name suffix, linear solver) for the name and the description +PRECONDS = { + None: ("no preconditioner", "unpreconditioned CG"), + "MultiGrid": ("multigrid preconditioner", "CG preconditioned by geometric multigrid"), + "StiffnessPreconditioner": ( + "stiffness preconditioner", + "CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner)", + ), +} + +name = f"Poisson strong scaling on hollow cylinder, {PRECONDS[precond][0]}" +description = f""" Strong scaling test for Poisson equation on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4). The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), exciting the full spectrum. Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with unpreconditioned CG. +The linear system is solved with {PRECONDS[precond][1]}. """ import logging @@ -19,7 +49,6 @@ set_logging_level(logging.INFO) -import argparse # ------------------ # Import Struphy API @@ -58,13 +87,6 @@ # -------------------------- # Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - env = EnvironmentOptions( sim_folder=f"sim_{args.id:02d}", restart=False, @@ -83,7 +105,7 @@ equil = None # Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) +grid = grids.TensorProductGrid(num_elements=tuple(args.num_elements), mpi_dims_mask=(True, True, True)) # Derham options derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) @@ -118,7 +140,7 @@ model.propagators.poisson.options = model.propagators.poisson.Options( stab_eps=0.0, solver="pcg", - precond=None, + precond=precond, solver_params=solver_params, ) diff --git a/profiling/examples/Poisson/hollow_cylinder_strong_scaling_multigrid/params_poisson.py b/profiling/examples/Poisson/hollow_cylinder_strong_scaling_multigrid/params_poisson.py deleted file mode 100644 index a43e58040..000000000 --- a/profiling/examples/Poisson/hollow_cylinder_strong_scaling_multigrid/params_poisson.py +++ /dev/null @@ -1,279 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Poisson strong scaling on hollow cylinder, multigrid preconditioner" -description = """ -Strong scaling test for Poisson equation on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4). -The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), -exciting the full spectrum. -Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with CG preconditioned by geometric multigrid. -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - FieldsBackground, - MultiGridOptions, - ProfilingOptions, - Simulation, - Time, - domains, - equils, - grids, - perturbations, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import Poisson - -# Units -base_units = BaseUnits() - -# Model instance -model = Poisson(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.phi.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -a1 = 1e-2 -a2 = 1.0 -Lz = 4.0 -domain = domains.HollowCylinder(a1=a1, a2=a2, Lz=Lz) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.poisson.options = model.propagators.poisson.Options( - stab_eps=0.0, - solver="pcg", - precond="MultiGrid", - multigrid=MultiGridOptions(), - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np - -from struphy.initial.base import GenericPerturbation - -# The exact solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5). -# Its value at the Dirichlet boundary eta1 = 0, 1 is exp(-0.25 / w**2) ~ 1e-11, and it is periodic -# in eta2 and eta3 up to the same accuracy. Being localized, it excites the whole spectrum of the -# Laplacian, so CG has to work for convergence. Both the solution and the right-hand side -# -Laplace(phi) are 0-forms given as functions of the logical coordinates (given_in_basis="0"); -# the pushed-forward solution at x = F(eta) is phi(eta). -w = 0.1 -eta0 = (0.5, 0.5, 0.5) - - -def gaussian_derivatives(e1, e2, e3): - """The Gaussian and its first and second derivatives with respect to eta1, eta2, eta3.""" - eta = (e1, e2, e3) - g = np.exp(-sum((e - c) ** 2 for e, c in zip(eta, eta0)) / w**2) - d = [-2 * (e - c) / w**2 * g for e, c in zip(eta, eta0)] - dd = [(4 * (e - c) ** 2 / w**4 - 2 / w**2) * g for e, c in zip(eta, eta0)] - return g, d, dd - - -def exact_solution(e1, e2, e3): - return gaussian_derivatives(e1, e2, e3)[0] - - -def laplacian(e1, e2, e3): - """Physical Laplacian of the exact solution, in logical coordinates. - - HollowCylinder: r = a1 + (a2 - a1) * eta1, theta = 2*pi*eta2, z = Lz * eta3, and - Laplace(phi) = phi_rr + phi_r / r + phi_theta,theta / r**2 + phi_zz. - """ - g, d, dd = gaussian_derivatives(e1, e2, e3) - da = a2 - a1 - r = a1 + da * e1 - return dd[0] / da**2 + d[0] / (da * r) + dd[1] / (2 * np.pi * r) ** 2 + dd[2] / Lz**2 - - -def rhs_fun(e1, e2, e3): - return -laplacian(e1, e2, e3) - - -rhs_perturbation = GenericPerturbation(rhs_fun, given_in_basis="0") - -model.em_fields.source.add_perturbation(rhs_perturbation) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True) - - from matplotlib import pyplot as plt - - def slice_axes(k, X, Y, Z, E1, E2, E3): - """Axes (horizontal, vertical, labels) for plotting the slice eta_k = 0.5: the cross-section (x, y) - at eta3 = 0.5, the half plane (r, z) at eta2 = 0.5 and the logical (eta2, eta3) at eta1 = 0.5.""" - if k == 0: - return E2, E3, "eta2", "eta3" - elif k == 1: - return np.sqrt(X**2 + Y**2), Z, "r", "z" - else: - return X, Y, "x", "y" - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error on the slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - h, v, hlabel, vlabel = slice_axes(k, X[idx], Y[idx], Z[idx], E1[idx], E2[idx], E3[idx]) - eta_k = (E1, E2, E3)[k][idx].flat[0] - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(h, v, data, shading="gouraud") - plt.colorbar() - plt.xlabel(hlabel) - plt.ylabel(vlabel) - plt.title(f"{name} {what}, slice at eta{k + 1} = {eta_k:.2f}") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Raw FEEC fields are evaluated directly from the saved spline coefficients at the - # cell centres of the simulation grid (serial Derham, safe on rank 0 only). - rhs = out.evaluate("em_fields/source", t=0).values - - phi_data = out.evaluate("em_fields/phi", t=-1) - phi = phi_data.values - - # logical coordinates of the evaluation points and their images x = F(eta) - E1, E2, E3 = np.meshgrid( - *(phi_data.coords[c].values for c in ("eta1", "eta2", "eta3")), - indexing="ij", - ) - X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) - - # pushed-forward exact solutions: the 0-forms at x = F(eta) are their values at eta - rhs_exact = rhs_fun(E1, E2, E3) - phi_exact = exact_solution(E1, E2, E3) - - fig_rhs = plot_slices(rhs, rhs_exact, "RHS") - fig_phi = plot_slices(phi, phi_exact, "Phi") - - rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) - rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) - - print(f"Max relative error in RHS: {rel_err_rhs:.2e}") - print(f"Max relative error in Phi: {rel_err_phi:.2e}") - - assert rel_err_rhs < 2.2e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_phi < 1.3e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_phi}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) - else: - plt.show() diff --git a/profiling/examples/Poisson/hollow_cylinder_strong_scaling_stiffness/params_poisson.py b/profiling/examples/Poisson/hollow_cylinder_strong_scaling_stiffness/params_poisson.py deleted file mode 100644 index 20d04710c..000000000 --- a/profiling/examples/Poisson/hollow_cylinder_strong_scaling_stiffness/params_poisson.py +++ /dev/null @@ -1,277 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Poisson strong scaling on hollow cylinder, stiffness preconditioner" -description = """ -Strong scaling test for Poisson equation on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4). -The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), -exciting the full spectrum. -Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner). -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - FieldsBackground, - ProfilingOptions, - Simulation, - Time, - domains, - equils, - grids, - perturbations, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import Poisson - -# Units -base_units = BaseUnits() - -# Model instance -model = Poisson(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.phi.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -a1 = 1e-2 -a2 = 1.0 -Lz = 4.0 -domain = domains.HollowCylinder(a1=a1, a2=a2, Lz=Lz) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.poisson.options = model.propagators.poisson.Options( - stab_eps=0.0, - solver="pcg", - precond="StiffnessPreconditioner", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np - -from struphy.initial.base import GenericPerturbation - -# The exact solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5). -# Its value at the Dirichlet boundary eta1 = 0, 1 is exp(-0.25 / w**2) ~ 1e-11, and it is periodic -# in eta2 and eta3 up to the same accuracy. Being localized, it excites the whole spectrum of the -# Laplacian, so CG has to work for convergence. Both the solution and the right-hand side -# -Laplace(phi) are 0-forms given as functions of the logical coordinates (given_in_basis="0"); -# the pushed-forward solution at x = F(eta) is phi(eta). -w = 0.1 -eta0 = (0.5, 0.5, 0.5) - - -def gaussian_derivatives(e1, e2, e3): - """The Gaussian and its first and second derivatives with respect to eta1, eta2, eta3.""" - eta = (e1, e2, e3) - g = np.exp(-sum((e - c) ** 2 for e, c in zip(eta, eta0)) / w**2) - d = [-2 * (e - c) / w**2 * g for e, c in zip(eta, eta0)] - dd = [(4 * (e - c) ** 2 / w**4 - 2 / w**2) * g for e, c in zip(eta, eta0)] - return g, d, dd - - -def exact_solution(e1, e2, e3): - return gaussian_derivatives(e1, e2, e3)[0] - - -def laplacian(e1, e2, e3): - """Physical Laplacian of the exact solution, in logical coordinates. - - HollowCylinder: r = a1 + (a2 - a1) * eta1, theta = 2*pi*eta2, z = Lz * eta3, and - Laplace(phi) = phi_rr + phi_r / r + phi_theta,theta / r**2 + phi_zz. - """ - g, d, dd = gaussian_derivatives(e1, e2, e3) - da = a2 - a1 - r = a1 + da * e1 - return dd[0] / da**2 + d[0] / (da * r) + dd[1] / (2 * np.pi * r) ** 2 + dd[2] / Lz**2 - - -def rhs_fun(e1, e2, e3): - return -laplacian(e1, e2, e3) - - -rhs_perturbation = GenericPerturbation(rhs_fun, given_in_basis="0") - -model.em_fields.source.add_perturbation(rhs_perturbation) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True) - - from matplotlib import pyplot as plt - - def slice_axes(k, X, Y, Z, E1, E2, E3): - """Axes (horizontal, vertical, labels) for plotting the slice eta_k = 0.5: the cross-section (x, y) - at eta3 = 0.5, the half plane (r, z) at eta2 = 0.5 and the logical (eta2, eta3) at eta1 = 0.5.""" - if k == 0: - return E2, E3, "eta2", "eta3" - elif k == 1: - return np.sqrt(X**2 + Y**2), Z, "r", "z" - else: - return X, Y, "x", "y" - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error on the slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - h, v, hlabel, vlabel = slice_axes(k, X[idx], Y[idx], Z[idx], E1[idx], E2[idx], E3[idx]) - eta_k = (E1, E2, E3)[k][idx].flat[0] - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(h, v, data, shading="gouraud") - plt.colorbar() - plt.xlabel(hlabel) - plt.ylabel(vlabel) - plt.title(f"{name} {what}, slice at eta{k + 1} = {eta_k:.2f}") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Raw FEEC fields are evaluated directly from the saved spline coefficients at the - # cell centres of the simulation grid (serial Derham, safe on rank 0 only). - rhs = out.evaluate("em_fields/source", t=0).values - - phi_data = out.evaluate("em_fields/phi", t=-1) - phi = phi_data.values - - # logical coordinates of the evaluation points and their images x = F(eta) - E1, E2, E3 = np.meshgrid( - *(phi_data.coords[c].values for c in ("eta1", "eta2", "eta3")), - indexing="ij", - ) - X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) - - # pushed-forward exact solutions: the 0-forms at x = F(eta) are their values at eta - rhs_exact = rhs_fun(E1, E2, E3) - phi_exact = exact_solution(E1, E2, E3) - - fig_rhs = plot_slices(rhs, rhs_exact, "RHS") - fig_phi = plot_slices(phi, phi_exact, "Phi") - - rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) - rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) - - print(f"Max relative error in RHS: {rel_err_rhs:.2e}") - print(f"Max relative error in Phi: {rel_err_phi:.2e}") - - assert rel_err_rhs < 2.2e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_phi < 1.3e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_phi}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) - else: - plt.show() diff --git a/profiling/examples/Poisson/hollow_torus_strong_scaling/params_poisson.py b/profiling/examples/Poisson/hollow_torus_strong_scaling/params_poisson.py index 5a8584d6f..6a03cd598 100644 --- a/profiling/examples/Poisson/hollow_torus_strong_scaling/params_poisson.py +++ b/profiling/examples/Poisson/hollow_torus_strong_scaling/params_poisson.py @@ -4,13 +4,43 @@ # Please fill in a verbal description of the simulation. # It will be printed at the beginning of the simulation and can be used to keep track of the different runs. -name = "Poisson strong scaling on hollow torus, no preconditioner" -description = """ +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# `--precond` selects the preconditioner (the variant), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--precond", + default="None", + choices=["None", "MultiGrid", "StiffnessPreconditioner"], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 64, 64], help="Number of elements per direction.") +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +# preconditioner -> (name suffix, linear solver) for the name and the description +PRECONDS = { + None: ("no preconditioner", "unpreconditioned CG"), + "MultiGrid": ("multigrid preconditioner", "CG preconditioned by geometric multigrid"), + "StiffnessPreconditioner": ( + "stiffness preconditioner", + "CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner)", + ), +} + +name = f"Poisson strong scaling on hollow torus, {PRECONDS[precond][0]}" +description = f""" Strong scaling test for Poisson equation on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4). The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), exciting the full spectrum. Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with unpreconditioned CG. +The linear system is solved with {PRECONDS[precond][1]}. """ import logging @@ -19,7 +49,6 @@ set_logging_level(logging.INFO) -import argparse # ------------------ # Import Struphy API @@ -58,13 +87,6 @@ # -------------------------- # Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - env = EnvironmentOptions( sim_folder=f"sim_{args.id:02d}", restart=False, @@ -84,7 +106,7 @@ equil = None # Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) +grid = grids.TensorProductGrid(num_elements=tuple(args.num_elements), mpi_dims_mask=(True, True, True)) # Derham options derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) @@ -119,7 +141,7 @@ model.propagators.poisson.options = model.propagators.poisson.Options( stab_eps=0.0, solver="pcg", - precond=None, + precond=precond, solver_params=solver_params, ) diff --git a/profiling/examples/Poisson/hollow_torus_strong_scaling_multigrid/params_poisson.py b/profiling/examples/Poisson/hollow_torus_strong_scaling_multigrid/params_poisson.py deleted file mode 100644 index 54c8f42d3..000000000 --- a/profiling/examples/Poisson/hollow_torus_strong_scaling_multigrid/params_poisson.py +++ /dev/null @@ -1,291 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Poisson strong scaling on hollow torus, multigrid preconditioner" -description = """ -Strong scaling test for Poisson equation on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4). -The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), -exciting the full spectrum. -Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with CG preconditioned by geometric multigrid. -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - FieldsBackground, - MultiGridOptions, - ProfilingOptions, - Simulation, - Time, - domains, - equils, - grids, - perturbations, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import Poisson - -# Units -base_units = BaseUnits() - -# Model instance -model = Poisson(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.phi.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -a1 = 1e-2 -a2 = 1.0 -R0 = 4.0 -tor_period = 1 -domain = domains.HollowTorus(a1=a1, a2=a2, R0=R0, sfl=False, pol_period=1, tor_period=tor_period) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.poisson.options = model.propagators.poisson.Options( - stab_eps=0.0, - solver="pcg", - precond="MultiGrid", - multigrid=MultiGridOptions(), - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np - -from struphy.initial.base import GenericPerturbation - -# The exact solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5). -# Its value at the Dirichlet boundary eta1 = 0, 1 is exp(-0.25 / w**2) ~ 1e-11, and it is periodic -# in eta2 and eta3 up to the same accuracy. Being localized, it excites the whole spectrum of the -# Laplacian, so CG has to work for convergence. Both the solution and the right-hand side -# -Laplace(phi) are 0-forms given as functions of the logical coordinates (given_in_basis="0"); -# the pushed-forward solution at x = F(eta) is phi(eta). -w = 0.1 -eta0 = (0.5, 0.5, 0.5) - - -def gaussian_derivatives(e1, e2, e3): - """The Gaussian and its first and second derivatives with respect to eta1, eta2, eta3.""" - eta = (e1, e2, e3) - g = np.exp(-sum((e - c) ** 2 for e, c in zip(eta, eta0)) / w**2) - d = [-2 * (e - c) / w**2 * g for e, c in zip(eta, eta0)] - dd = [(4 * (e - c) ** 2 / w**4 - 2 / w**2) * g for e, c in zip(eta, eta0)] - return g, d, dd - - -def exact_solution(e1, e2, e3): - return gaussian_derivatives(e1, e2, e3)[0] - - -def laplacian(e1, e2, e3): - """Physical Laplacian of the exact solution, in logical coordinates. - - HollowTorus (sfl=False): r = a1 + (a2 - a1) * eta1, theta = 2*pi*eta2, phi = 2*pi*eta3 / tor_period, - R = R0 + r*cos(theta), metric dr**2 + r**2 dtheta**2 + R**2 dphi**2, and - Laplace(u) = u_rr + (R + r*cos(theta)) / (r*R) * u_r + u_theta,theta / r**2 - sin(theta) / (r*R) * u_theta - + u_phi,phi / R**2. - """ - g, d, dd = gaussian_derivatives(e1, e2, e3) - da = a2 - a1 - r = a1 + da * e1 - theta = 2 * np.pi * e2 - R = R0 + r * np.cos(theta) - dphi = 2 * np.pi / tor_period - return ( - dd[0] / da**2 - + (R + r * np.cos(theta)) / (r * R) * d[0] / da - + dd[1] / (2 * np.pi * r) ** 2 - - np.sin(theta) / (r * R) * d[1] / (2 * np.pi) - + dd[2] / (dphi * R) ** 2 - ) - - -def rhs_fun(e1, e2, e3): - return -laplacian(e1, e2, e3) - - -rhs_perturbation = GenericPerturbation(rhs_fun, given_in_basis="0") - -model.em_fields.source.add_perturbation(rhs_perturbation) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True) - - from matplotlib import pyplot as plt - - def slice_axes(k, X, Y, Z, E1, E2, E3): - """Axes (horizontal, vertical, labels) for plotting the slice eta_k = 0.5: the poloidal plane (R, z) - at eta3 = 0.5 and the logical planes at eta1 = 0.5 and eta2 = 0.5.""" - if k == 0: - return E2, E3, "eta2", "eta3" - elif k == 1: - return E1, E3, "eta1", "eta3" - else: - return np.sqrt(X**2 + Y**2), Z, "R", "z" - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error on the slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - h, v, hlabel, vlabel = slice_axes(k, X[idx], Y[idx], Z[idx], E1[idx], E2[idx], E3[idx]) - eta_k = (E1, E2, E3)[k][idx].flat[0] - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(h, v, data, shading="gouraud") - plt.colorbar() - plt.xlabel(hlabel) - plt.ylabel(vlabel) - plt.title(f"{name} {what}, slice at eta{k + 1} = {eta_k:.2f}") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Raw FEEC fields are evaluated directly from the saved spline coefficients at the - # cell centres of the simulation grid (serial Derham, safe on rank 0 only). - rhs = out.evaluate("em_fields/source", t=0).values - - phi_data = out.evaluate("em_fields/phi", t=-1) - phi = phi_data.values - - # logical coordinates of the evaluation points and their images x = F(eta) - E1, E2, E3 = np.meshgrid( - *(phi_data.coords[c].values for c in ("eta1", "eta2", "eta3")), - indexing="ij", - ) - X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) - - # pushed-forward exact solutions: the 0-forms at x = F(eta) are their values at eta - rhs_exact = rhs_fun(E1, E2, E3) - phi_exact = exact_solution(E1, E2, E3) - - fig_rhs = plot_slices(rhs, rhs_exact, "RHS") - fig_phi = plot_slices(phi, phi_exact, "Phi") - - rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) - rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) - - print(f"Max relative error in RHS: {rel_err_rhs:.2e}") - print(f"Max relative error in Phi: {rel_err_phi:.2e}") - - assert rel_err_rhs < 2.2e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_phi < 1.3e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_phi}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) - else: - plt.show() diff --git a/profiling/examples/Poisson/hollow_torus_strong_scaling_stiffness/params_poisson.py b/profiling/examples/Poisson/hollow_torus_strong_scaling_stiffness/params_poisson.py deleted file mode 100644 index 784face9c..000000000 --- a/profiling/examples/Poisson/hollow_torus_strong_scaling_stiffness/params_poisson.py +++ /dev/null @@ -1,289 +0,0 @@ -# ----------------------------- -# Description of the simulation -# ----------------------------- -# Please fill in a verbal description of the simulation. -# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. - -name = "Poisson strong scaling on hollow torus, stiffness preconditioner" -description = """ -Strong scaling test for Poisson equation on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4). -The manufactured solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5), -exciting the full spectrum. -Homogeneous Dirichlet boundary conditions are set in direction eta1. -The linear system is solved with CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner). -""" - -import logging - -from struphy import set_logging_level - -set_logging_level(logging.INFO) - -import argparse - -# ------------------ -# Import Struphy API -# ------------------ -from struphy import ( - BaseUnits, - DerhamOptions, - EnvironmentOptions, - FieldsBackground, - ProfilingOptions, - Simulation, - Time, - domains, - equils, - grids, - perturbations, -) - -# --------------------- -# Instance of the model -# --------------------- -from struphy.models import Poisson - -# Units -base_units = BaseUnits() - -# Model instance -model = Poisson(base_units=base_units) - -# List all variables and decide whether to save their data -model.em_fields.phi.save_data = True -model.em_fields.source.save_data = True - -# -------------------------- -# Instance of the simulation -# -------------------------- - -# Environment options -# `--id` distinguishes runs that share a rank count but differ in something else; the -# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). -# Unknown flags are ignored so the driver can forward other parameters as well. -parser = argparse.ArgumentParser() -parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") -args, _ = parser.parse_known_args() - -env = EnvironmentOptions( - sim_folder=f"sim_{args.id:02d}", - restart=False, -) - -# Time stepping -time_opts = Time() - -# Geometry -a1 = 1e-2 -a2 = 1.0 -R0 = 4.0 -tor_period = 1 -domain = domains.HollowTorus(a1=a1, a2=a2, R0=R0, sfl=False, pol_period=1, tor_period=tor_period) - -# Fluid equilibrium (can be used as part of initial conditions) -equil = None - -# Grid -grid = grids.TensorProductGrid(num_elements=(64, 64, 64), mpi_dims_mask=(True, True, True)) - -# Derham options -derham_opts = DerhamOptions(degree=(1, 2, 3), bcs=(("dirichlet", "dirichlet"), None, None)) - -# Profilinig options -profiling_opts = ProfilingOptions( - use_line_profiler=True, -) - -# Simulation object -sim = Simulation( - model=model, - name=name, - description=description, - params_path=__file__, - env=env, - time_opts=time_opts, - domain=domain, - equil=equil, - grid=grid, - derham_opts=derham_opts, - profiling_opts=profiling_opts, -) - -# ------------------ -# Propagator options -# ------------------ - -from struphy.linear_algebra.solver import SolverParameters - -solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) -model.propagators.poisson.options = model.propagators.poisson.Options( - stab_eps=0.0, - solver="pcg", - precond="StiffnessPreconditioner", - solver_params=solver_params, -) - -# ------------------ -# Initial conditions -# ------------------ -import numpy as np - -from struphy.initial.base import GenericPerturbation - -# The exact solution is a Gaussian 0-form on the logical domain, centered at eta = (0.5, 0.5, 0.5). -# Its value at the Dirichlet boundary eta1 = 0, 1 is exp(-0.25 / w**2) ~ 1e-11, and it is periodic -# in eta2 and eta3 up to the same accuracy. Being localized, it excites the whole spectrum of the -# Laplacian, so CG has to work for convergence. Both the solution and the right-hand side -# -Laplace(phi) are 0-forms given as functions of the logical coordinates (given_in_basis="0"); -# the pushed-forward solution at x = F(eta) is phi(eta). -w = 0.1 -eta0 = (0.5, 0.5, 0.5) - - -def gaussian_derivatives(e1, e2, e3): - """The Gaussian and its first and second derivatives with respect to eta1, eta2, eta3.""" - eta = (e1, e2, e3) - g = np.exp(-sum((e - c) ** 2 for e, c in zip(eta, eta0)) / w**2) - d = [-2 * (e - c) / w**2 * g for e, c in zip(eta, eta0)] - dd = [(4 * (e - c) ** 2 / w**4 - 2 / w**2) * g for e, c in zip(eta, eta0)] - return g, d, dd - - -def exact_solution(e1, e2, e3): - return gaussian_derivatives(e1, e2, e3)[0] - - -def laplacian(e1, e2, e3): - """Physical Laplacian of the exact solution, in logical coordinates. - - HollowTorus (sfl=False): r = a1 + (a2 - a1) * eta1, theta = 2*pi*eta2, phi = 2*pi*eta3 / tor_period, - R = R0 + r*cos(theta), metric dr**2 + r**2 dtheta**2 + R**2 dphi**2, and - Laplace(u) = u_rr + (R + r*cos(theta)) / (r*R) * u_r + u_theta,theta / r**2 - sin(theta) / (r*R) * u_theta - + u_phi,phi / R**2. - """ - g, d, dd = gaussian_derivatives(e1, e2, e3) - da = a2 - a1 - r = a1 + da * e1 - theta = 2 * np.pi * e2 - R = R0 + r * np.cos(theta) - dphi = 2 * np.pi / tor_period - return ( - dd[0] / da**2 - + (R + r * np.cos(theta)) / (r * R) * d[0] / da - + dd[1] / (2 * np.pi * r) ** 2 - - np.sin(theta) / (r * R) * d[1] / (2 * np.pi) - + dd[2] / (dphi * R) ** 2 - ) - - -def rhs_fun(e1, e2, e3): - return -laplacian(e1, e2, e3) - - -rhs_perturbation = GenericPerturbation(rhs_fun, given_in_basis="0") - -model.em_fields.source.add_perturbation(rhs_perturbation) - - -if __name__ == "__main__": - # switches for debugging and testing - estimate_mem = False - run = True - pproc = True - save_figs = True - - if estimate_mem: - sim.estimate_mem(print_report=True) - exit() - - if run: - out = sim.run(profiling_activated=True, one_time_step=True) - else: - # use this for working with existing sim data from a previous run - out = sim.output - - if pproc: - out.pproc(create_vtk=True, parallel=True) - - from matplotlib import pyplot as plt - - def slice_axes(k, X, Y, Z, E1, E2, E3): - """Axes (horizontal, vertical, labels) for plotting the slice eta_k = 0.5: the poloidal plane (R, z) - at eta3 = 0.5 and the logical planes at eta1 = 0.5 and eta2 = 0.5.""" - if k == 0: - return E2, E3, "eta2", "eta3" - elif k == 1: - return E1, E3, "eta1", "eta3" - else: - return np.sqrt(X**2 + Y**2), Z, "R", "z" - - def plot_slices(num, exact, name): - """Numerical solution, exact solution and error on the slices eta_k = 0.5 (k = 1, 2, 3).""" - fig = plt.figure(figsize=(16, 12)) - for k in range(3): - idx = tuple(num.shape[k] // 2 if i == k else slice(None) for i in range(3)) - h, v, hlabel, vlabel = slice_axes(k, X[idx], Y[idx], Z[idx], E1[idx], E2[idx], E3[idx]) - eta_k = (E1, E2, E3)[k][idx].flat[0] - for row, (data, what) in enumerate( - [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] - ): - plt.subplot(3, 3, 3 * row + k + 1) - plt.pcolormesh(h, v, data, shading="gouraud") - plt.colorbar() - plt.xlabel(hlabel) - plt.ylabel(vlabel) - plt.title(f"{name} {what}, slice at eta{k + 1} = {eta_k:.2f}") - fig.tight_layout() - return fig - - if sim.rank == 0: - # Raw FEEC fields are evaluated directly from the saved spline coefficients at the - # cell centres of the simulation grid (serial Derham, safe on rank 0 only). - rhs = out.evaluate("em_fields/source", t=0).values - - phi_data = out.evaluate("em_fields/phi", t=-1) - phi = phi_data.values - - # logical coordinates of the evaluation points and their images x = F(eta) - E1, E2, E3 = np.meshgrid( - *(phi_data.coords[c].values for c in ("eta1", "eta2", "eta3")), - indexing="ij", - ) - X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) - - # pushed-forward exact solutions: the 0-forms at x = F(eta) are their values at eta - rhs_exact = rhs_fun(E1, E2, E3) - phi_exact = exact_solution(E1, E2, E3) - - fig_rhs = plot_slices(rhs, rhs_exact, "RHS") - fig_phi = plot_slices(phi, phi_exact, "Phi") - - rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) - rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) - - print(f"Max relative error in RHS: {rel_err_rhs:.2e}") - print(f"Max relative error in Phi: {rel_err_phi:.2e}") - - assert rel_err_rhs < 2.2e-2, f"The computed RHS does not match the exact RHS, max rel error = {rel_err_rhs}." - assert rel_err_phi < 1.3e-2, ( - f"The computed solution does not match the exact solution, max rel error = {rel_err_phi}." - ) - - import os - - # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare - # name resolved against the CWD. The profiling packaging picks these files up from - # here and uploads them as `results-out`. - results_dir = os.path.join(out.path_out, "results") - os.makedirs(results_dir, exist_ok=True) - - np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) - np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) - np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) - np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) - - if save_figs: - fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) - fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) - else: - plt.show() diff --git a/profiling/examples/Poisson/polar_cylinder_strong_scaling/params_poisson.py b/profiling/examples/Poisson/polar_cylinder_strong_scaling/params_poisson.py new file mode 100644 index 000000000..b89405d48 --- /dev/null +++ b/profiling/examples/Poisson/polar_cylinder_strong_scaling/params_poisson.py @@ -0,0 +1,309 @@ +# ----------------------------- +# Description of the simulation +# ----------------------------- +# Please fill in a verbal description of the simulation. +# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. + +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# The other flags select the variant (spline space and preconditioner), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--splines", + default="polar", + choices=["polar", "tensor"], + help="C1 polar splines on a disc (IGAPolarCylinder) or tensor-product splines on a cylinder with a hole (HollowCylinder).", +) +parser.add_argument( + "--precond", + default="None", + choices=[ + "None", + "MassMatrixPreconditioner", + "PolarMassMatrixPreconditioner", + "StiffnessPreconditioner", + "MultiGrid", + ], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument( + "--poloidal-solver", + default="schur", + choices=["schur", "lu", "ilu"], + help="Poloidal solver of the polar preconditioners and projectors.", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 66, 64], help="Number of elements per direction.") +parser.add_argument( + "--hole-bc", + default="free", + choices=["free", "dirichlet"], + help="Boundary condition at the hole of the tensor-product domain (default free; the solution does not vanish there).", +) +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +name = f"Poisson strong scaling on a cylinder, {args.splines} splines, preconditioner {precond}" +description = f""" +Strong scaling test for the Poisson equation on a cylinder of radius 1 and length 4, comparing C1 polar splines +(IGAPolarCylinder, no hole) with tensor-product splines on a cylinder with a hole (HollowCylinder, a1=1e-2). +The manufactured solution is a Gaussian whose gradient is largest on the axis r=0. +Homogeneous Dirichlet boundary conditions are set at r=1 (boundary condition {args.hole_bc} at the hole). +The linear system is solved with CG, preconditioner {precond} (poloidal solver {args.poloidal_solver}). +""" + +import logging + +from struphy import set_logging_level + +set_logging_level(logging.INFO) + +# ------------------ +# Import Struphy API +# ------------------ +from struphy import ( + BaseUnits, + DerhamOptions, + EnvironmentOptions, + ProfilingOptions, + Simulation, + Time, + domains, + grids, +) + +# --------------------- +# Instance of the model +# --------------------- +from struphy.models import Poisson + +# Units +base_units = BaseUnits() + +# Model instance +model = Poisson(base_units=base_units) + +# List all variables and decide whether to save their data +model.em_fields.phi.save_data = True +model.em_fields.source.save_data = True + +# -------------------------- +# Instance of the simulation +# -------------------------- + +# Environment options +env = EnvironmentOptions( + sim_folder=f"sim_{args.id:02d}", + restart=False, +) + +# Time stepping +time_opts = Time() + +# Grid (polar splines need a multiple of 3 elements in eta2) +num_elements = tuple(args.num_elements) +degree = (3, 3, 3) +grid = grids.TensorProductGrid(num_elements=num_elements, mpi_dims_mask=(True, True, True)) + +# Geometry and Derham options +a = 1.0 +Lz = 4.0 +if args.splines == "polar": + # the spline mapping has the resolution of the grid in the poloidal plane + domain = domains.IGAPolarCylinder(num_elements=num_elements[:2], degree=degree[:2], a=a, Lz=Lz) + derham_opts = DerhamOptions( + degree=degree, + bcs=(("free", "dirichlet"), None, None), + polar_splines=True, + polar_poloidal_solver=args.poloidal_solver, + ) +else: + a1 = 1e-2 + domain = domains.HollowCylinder(a1=a1, a2=a, Lz=Lz) + derham_opts = DerhamOptions(degree=degree, bcs=((args.hole_bc, "dirichlet"), None, None)) + +# Fluid equilibrium (can be used as part of initial conditions) +equil = None + +# Profiling options +profiling_opts = ProfilingOptions( + use_line_profiler=True, +) + +# Simulation object +sim = Simulation( + model=model, + name=name, + description=description, + params_path=__file__, + env=env, + time_opts=time_opts, + domain=domain, + equil=equil, + grid=grid, + derham_opts=derham_opts, + profiling_opts=profiling_opts, +) + +# ------------------ +# Propagator options +# ------------------ + +from struphy.linear_algebra.solver import SolverParameters + +# options of the polar preconditioners +precond_params = {} +if args.splines == "polar" and precond in ("PolarMassMatrixPreconditioner", "StiffnessPreconditioner"): + precond_params["poloidal_solver"] = args.poloidal_solver + +solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) +model.propagators.poisson.options = model.propagators.poisson.Options( + stab_eps=0.0, + solver="pcg", + precond=precond, + precond_params=precond_params, + solver_params=solver_params, +) + +# ------------------ +# Initial conditions +# ------------------ +import numpy as np +import sympy as sym + +from struphy.initial.base import GenericPerturbation + +# The exact solution is phi = g(x, y) * h(z): a Gaussian g of width w in the cross-section, centered at distance +# w / sqrt(2) from the axis, where its gradient is largest, so the gradient is largest on the axis r = 0, +# and a periodic modulation h along the axis. It is negligible at r = 1 (Dirichlet boundary). +# Tensor-product splines cannot represent a non-zero gradient across the axis (they live on a domain +# with a hole and a boundary condition there, see --hole-bc); polar splines can. +# The right-hand side -Laplace(phi) is computed symbolically. Both are given as functions of +# the Cartesian coordinates (given_in_basis="physical"). +x_, y_, z_ = sym.symbols("x y z", real=True) +w = 0.2 +x0 = w / np.sqrt(2.0) +phi_sym = sym.exp(-((x_ - x0) ** 2 + y_**2) / w**2) * (1 + sym.cos(2 * sym.pi * z_ / Lz) / 2) +rhs_sym = -(sym.diff(phi_sym, x_, 2) + sym.diff(phi_sym, y_, 2) + sym.diff(phi_sym, z_, 2)) +phi_fun = sym.lambdify((x_, y_, z_), phi_sym, "numpy") +rhs_lam = sym.lambdify((x_, y_, z_), rhs_sym, "numpy") + + +def exact_solution(x, y, z): + return phi_fun(x, y, z) + 0.0 * x + + +def rhs_fun(x, y, z): + return rhs_lam(x, y, z) + 0.0 * x + + +def distance_to_axis(x, y, z): + return np.sqrt(x**2 + y**2) + + +model.em_fields.source.add_perturbation(GenericPerturbation(rhs_fun, given_in_basis="physical")) + + +if __name__ == "__main__": + # switches for debugging and testing + estimate_mem = False + run = True + pproc = True + save_figs = True + + if estimate_mem: + sim.estimate_mem(print_report=True) + exit() + + if run: + out = sim.run(profiling_activated=True, one_time_step=True) + else: + # use this for working with existing sim data from a previous run + out = sim.output + + if pproc: + out.pproc(create_vtk=True, parallel=True) + + from matplotlib import pyplot as plt + + def plot_slices(num, exact, name): + """Numerical solution, exact solution and error on the cross-section at eta3 = 0.5 and the + half plane at eta2 = 0.5, as functions of (x, y) and (x, z).""" + fig = plt.figure(figsize=(12, 12)) + idx3 = (slice(None), slice(None), num.shape[2] // 2) + idx2 = (slice(None), num.shape[1] // 2, slice(None)) + for col, (idx, h, v, labels) in enumerate( + [(idx3, X, Y, ("x", "y")), (idx2, X, Z, ("x", "z"))], + ): + for row, (data, what) in enumerate( + [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] + ): + plt.subplot(3, 2, 2 * row + col + 1) + plt.pcolormesh(h[idx], v[idx], data, shading="gouraud") + plt.colorbar() + plt.xlabel(labels[0]) + plt.ylabel(labels[1]) + plt.title(f"{name} {what}") + fig.tight_layout() + return fig + + if sim.rank == 0: + # Raw FEEC fields are evaluated directly from the saved spline coefficients at the + # cell centres of the simulation grid (serial Derham, safe on rank 0 only). + rhs = out.evaluate("em_fields/source", t=0).values + + phi_data = out.evaluate("em_fields/phi", t=-1) + phi = phi_data.values + X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) + + rhs_exact = rhs_fun(X, Y, Z) + phi_exact = exact_solution(X, Y, Z) + + fig_rhs = plot_slices(rhs, rhs_exact, "RHS") + fig_phi = plot_slices(phi, phi_exact, "Phi") + + rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) + rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) + # error close to the axis, where the gradient of the solution is largest + near_axis = distance_to_axis(X, Y, Z) < 0.1 + rel_err_phi_axis = np.max(np.abs(phi - phi_exact)[near_axis]) / np.max(np.abs(phi_exact)) + + print(f"Max relative error in RHS: {rel_err_rhs:.2e}") + print(f"Max relative error in Phi: {rel_err_phi:.2e}") + print(f"Max relative error in Phi close to the axis (r < 0.1): {rel_err_phi_axis:.2e}") + + import json + import os + + # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare + # name resolved against the CWD. The profiling packaging picks these files up from + # here and uploads them as `results-out`. + results_dir = os.path.join(out.path_out, "results") + os.makedirs(results_dir, exist_ok=True) + + np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) + np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) + np.save(os.path.join(results_dir, "rel_err_phi_axis.npy"), rel_err_phi_axis) + np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) + np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) + with open(os.path.join(results_dir, "variant.json"), "w") as f: + json.dump( + { + "splines": args.splines, + "precond": args.precond, + "poloidal_solver": args.poloidal_solver, + "hole_bc": args.hole_bc, + }, + f, + ) + + if save_figs: + fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) + fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) + else: + plt.show() diff --git a/profiling/examples/Poisson/polar_torus_strong_scaling/params_poisson.py b/profiling/examples/Poisson/polar_torus_strong_scaling/params_poisson.py new file mode 100644 index 000000000..6ed323730 --- /dev/null +++ b/profiling/examples/Poisson/polar_torus_strong_scaling/params_poisson.py @@ -0,0 +1,312 @@ +# ----------------------------- +# Description of the simulation +# ----------------------------- +# Please fill in a verbal description of the simulation. +# It will be printed at the beginning of the simulation and can be used to keep track of the different runs. + +import argparse + +# Command line options +# `--id` distinguishes runs that share a rank count but differ in something else; the +# profiling driver passes its launch counter (see `ProfilingJob.build_commands`). +# The other flags select the variant (spline space and preconditioner), see the submit script. +# Unknown flags are ignored so the driver can forward other parameters as well. +parser = argparse.ArgumentParser() +parser.add_argument("--id", type=int, default=0, help="Run id, used to name the output folder.") +parser.add_argument( + "--splines", + default="polar", + choices=["polar", "tensor"], + help="C1 polar splines on a full torus (IGAPolarTorus) or tensor-product splines on a torus with a hole (HollowTorus).", +) +parser.add_argument( + "--precond", + default="None", + choices=[ + "None", + "MassMatrixPreconditioner", + "PolarMassMatrixPreconditioner", + "StiffnessPreconditioner", + "MultiGrid", + ], + help="Name of the preconditioner (None for no preconditioner).", +) +parser.add_argument( + "--poloidal-solver", + default="schur", + choices=["schur", "lu", "ilu"], + help="Poloidal solver of the polar preconditioners and projectors.", +) +parser.add_argument("--num-elements", type=int, nargs=3, default=[64, 66, 64], help="Number of elements per direction.") +parser.add_argument( + "--hole-bc", + default="free", + choices=["free", "dirichlet"], + help="Boundary condition at the hole of the tensor-product domain (default free; the solution does not vanish there).", +) +args, _ = parser.parse_known_args() + +precond = None if args.precond == "None" else args.precond + +name = f"Poisson strong scaling on a torus, {args.splines} splines, preconditioner {precond}" +description = f""" +Strong scaling test for the Poisson equation on a full torus of minor radius 1 and major radius 4, comparing C1 polar +splines (IGAPolarTorus, no hole) with tensor-product splines on a torus with a hole (HollowTorus, a1=1e-2). +The manufactured solution is a Gaussian ring whose gradient is largest on the magnetic axis r=0. +Homogeneous Dirichlet boundary conditions are set at r=1 (boundary condition {args.hole_bc} at the hole). +The linear system is solved with CG, preconditioner {precond} (poloidal solver {args.poloidal_solver}). +""" + +import logging + +from struphy import set_logging_level + +set_logging_level(logging.INFO) + +# ------------------ +# Import Struphy API +# ------------------ +from struphy import ( + BaseUnits, + DerhamOptions, + EnvironmentOptions, + ProfilingOptions, + Simulation, + Time, + domains, + grids, +) + +# --------------------- +# Instance of the model +# --------------------- +from struphy.models import Poisson + +# Units +base_units = BaseUnits() + +# Model instance +model = Poisson(base_units=base_units) + +# List all variables and decide whether to save their data +model.em_fields.phi.save_data = True +model.em_fields.source.save_data = True + +# -------------------------- +# Instance of the simulation +# -------------------------- + +# Environment options +env = EnvironmentOptions( + sim_folder=f"sim_{args.id:02d}", + restart=False, +) + +# Time stepping +time_opts = Time() + +# Grid (polar splines need a multiple of 3 elements in eta2) +num_elements = tuple(args.num_elements) +degree = (3, 3, 3) +grid = grids.TensorProductGrid(num_elements=num_elements, mpi_dims_mask=(True, True, True)) + +# Geometry and Derham options +a = 1.0 +R0 = 4.0 +if args.splines == "polar": + # the spline mapping has the resolution of the grid in the poloidal plane + domain = domains.IGAPolarTorus( + num_elements=num_elements[:2], degree=degree[:2], a=a, R0=R0, sfl=False, tor_period=1 + ) + derham_opts = DerhamOptions( + degree=degree, + bcs=(("free", "dirichlet"), None, None), + polar_splines=True, + polar_poloidal_solver=args.poloidal_solver, + ) +else: + a1 = 1e-2 + domain = domains.HollowTorus(a1=a1, a2=a, R0=R0, sfl=False, pol_period=1, tor_period=1) + derham_opts = DerhamOptions(degree=degree, bcs=((args.hole_bc, "dirichlet"), None, None)) + +# Fluid equilibrium (can be used as part of initial conditions) +equil = None + +# Profiling options +profiling_opts = ProfilingOptions( + use_line_profiler=True, +) + +# Simulation object +sim = Simulation( + model=model, + name=name, + description=description, + params_path=__file__, + env=env, + time_opts=time_opts, + domain=domain, + equil=equil, + grid=grid, + derham_opts=derham_opts, + profiling_opts=profiling_opts, +) + +# ------------------ +# Propagator options +# ------------------ + +from struphy.linear_algebra.solver import SolverParameters + +# options of the polar preconditioners +precond_params = {} +if args.splines == "polar" and precond in ("PolarMassMatrixPreconditioner", "StiffnessPreconditioner"): + precond_params["poloidal_solver"] = args.poloidal_solver + +solver_params = SolverParameters(tol=1e-8, maxiter=3000, info=True, recycle=False) +model.propagators.poisson.options = model.propagators.poisson.Options( + stab_eps=0.0, + solver="pcg", + precond=precond, + precond_params=precond_params, + solver_params=solver_params, +) + +# ------------------ +# Initial conditions +# ------------------ +import numpy as np +import sympy as sym + +from struphy.initial.base import GenericPerturbation + +# The exact solution is phi = g(R, z) * h: a Gaussian ring g of width w in the poloidal plane (R = sqrt(x**2 + y**2)), +# centered at distance w / sqrt(2) from the magnetic axis (R0, 0), where its gradient is largest, so the gradient is +# largest on the axis r = 0, and a toroidal modulation h = 1 + x / (2 R). It is negligible at r = 1 (Dirichlet boundary). +# Tensor-product splines cannot represent a non-zero gradient across the axis (they live on a domain +# with a hole and a boundary condition there, see --hole-bc); polar splines can. +# The right-hand side -Laplace(phi) is computed symbolically. Both are given as functions of +# the Cartesian coordinates (given_in_basis="physical"). +x_, y_, z_ = sym.symbols("x y z", real=True) +w = 0.2 +x0 = w / np.sqrt(2.0) +R_ = sym.sqrt(x_**2 + y_**2) +phi_sym = sym.exp(-((R_ - R0 - x0) ** 2 + z_**2) / w**2) * (1 + x_ / (2 * R_)) +rhs_sym = -(sym.diff(phi_sym, x_, 2) + sym.diff(phi_sym, y_, 2) + sym.diff(phi_sym, z_, 2)) +phi_fun = sym.lambdify((x_, y_, z_), phi_sym, "numpy") +rhs_lam = sym.lambdify((x_, y_, z_), rhs_sym, "numpy") + + +def exact_solution(x, y, z): + return phi_fun(x, y, z) + 0.0 * x + + +def rhs_fun(x, y, z): + return rhs_lam(x, y, z) + 0.0 * x + + +def distance_to_axis(x, y, z): + return np.sqrt((np.sqrt(x**2 + y**2) - R0) ** 2 + z**2) + + +model.em_fields.source.add_perturbation(GenericPerturbation(rhs_fun, given_in_basis="physical")) + + +if __name__ == "__main__": + # switches for debugging and testing + estimate_mem = False + run = True + pproc = True + save_figs = True + + if estimate_mem: + sim.estimate_mem(print_report=True) + exit() + + if run: + out = sim.run(profiling_activated=True, one_time_step=True) + else: + # use this for working with existing sim data from a previous run + out = sim.output + + if pproc: + out.pproc(create_vtk=True, parallel=True) + + from matplotlib import pyplot as plt + + def plot_slices(num, exact, name): + """Numerical solution, exact solution and error on the poloidal plane at eta3 = 0.5 and on the + surface eta2 = 0.5 seen from above, as functions of (R, z) and (x, y).""" + fig = plt.figure(figsize=(12, 12)) + idx3 = (slice(None), slice(None), num.shape[2] // 2) + idx2 = (slice(None), num.shape[1] // 2, slice(None)) + for col, (idx, h, v, labels) in enumerate( + [(idx3, np.sqrt(X**2 + Y**2), Z, ("R", "z")), (idx2, X, Y, ("x", "y"))], + ): + for row, (data, what) in enumerate( + [(num[idx], "from struphy"), (exact[idx], "exact"), (np.abs(num[idx] - exact[idx]), "error")] + ): + plt.subplot(3, 2, 2 * row + col + 1) + plt.pcolormesh(h[idx], v[idx], data, shading="gouraud") + plt.colorbar() + plt.xlabel(labels[0]) + plt.ylabel(labels[1]) + plt.title(f"{name} {what}") + fig.tight_layout() + return fig + + if sim.rank == 0: + # Raw FEEC fields are evaluated directly from the saved spline coefficients at the + # cell centres of the simulation grid (serial Derham, safe on rank 0 only). + rhs = out.evaluate("em_fields/source", t=0).values + + phi_data = out.evaluate("em_fields/phi", t=-1) + phi = phi_data.values + X, Y, Z = (phi_data.coords[c].values for c in ("X", "Y", "Z")) + + rhs_exact = rhs_fun(X, Y, Z) + phi_exact = exact_solution(X, Y, Z) + + fig_rhs = plot_slices(rhs, rhs_exact, "RHS") + fig_phi = plot_slices(phi, phi_exact, "Phi") + + rel_err_rhs = np.max(np.abs(rhs - rhs_exact)) / np.max(np.abs(rhs_exact)) + rel_err_phi = np.max(np.abs(phi - phi_exact)) / np.max(np.abs(phi_exact)) + # error close to the axis, where the gradient of the solution is largest + near_axis = distance_to_axis(X, Y, Z) < 0.1 + rel_err_phi_axis = np.max(np.abs(phi - phi_exact)[near_axis]) / np.max(np.abs(phi_exact)) + + print(f"Max relative error in RHS: {rel_err_rhs:.2e}") + print(f"Max relative error in Phi: {rel_err_phi:.2e}") + print(f"Max relative error in Phi close to the axis (r < 0.1): {rel_err_phi_axis:.2e}") + + import json + import os + + # `out.path_out` is the out's (absolute) output folder; `sim_folder` alone is a bare + # name resolved against the CWD. The profiling packaging picks these files up from + # here and uploads them as `results-out`. + results_dir = os.path.join(out.path_out, "results") + os.makedirs(results_dir, exist_ok=True) + + np.save(os.path.join(results_dir, "rel_err_rhs.npy"), rel_err_rhs) + np.save(os.path.join(results_dir, "rel_err_phi.npy"), rel_err_phi) + np.save(os.path.join(results_dir, "rel_err_phi_axis.npy"), rel_err_phi_axis) + np.save(os.path.join(results_dir, "resolution.npy"), sim.grid.num_elements) + np.save(os.path.join(results_dir, "spline_degree.npy"), sim.derham_opts.degree) + with open(os.path.join(results_dir, "variant.json"), "w") as f: + json.dump( + { + "splines": args.splines, + "precond": args.precond, + "poloidal_solver": args.poloidal_solver, + "hole_bc": args.hole_bc, + }, + f, + ) + + if save_figs: + fig_rhs.savefig(os.path.join(results_dir, "rhs_slices.png")) + fig_phi.savefig(os.path.join(results_dir, "phi_slices.png")) + else: + plt.show() diff --git a/profiling/examples/Poisson/submit_poisson_cube_strong_scaling.py b/profiling/examples/Poisson/submit_poisson_cube_strong_scaling.py index b41fc652a..6a73610bf 100644 --- a/profiling/examples/Poisson/submit_poisson_cube_strong_scaling.py +++ b/profiling/examples/Poisson/submit_poisson_cube_strong_scaling.py @@ -1,4 +1,4 @@ -"""Poisson strong scaling profiling case on 3D cube, no preconditioner. +"""Poisson strong scaling profiling case on 3D cube. This file defines the Poisson strong scaling profiling case (the `ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a @@ -7,6 +7,8 @@ each run as soon as its own job finishes. Each generated script runs the simulation itself by invoking `params_poisson.py` directly (its `__main__` block is the worker). + +The variant (preconditioner) is chosen with `--variant`; run this script once per variant. """ import argparse @@ -18,6 +20,29 @@ from profiling_job import ProfilingCase # noqa: E402 +# variant -> (label, name, description, flags of the params file); the labels identify the +# cases in the profiling-data repo and are kept from the former one-case-per-script layout +VARIANTS = { + "none": ( + "poisson_cube_strong_scaling", + "Poisson on 3D cube strong scaling test, no preconditioner", + "Strong scaling of the Poisson model with manufactured solution on a 3D cube (Cuboid), solved with unpreconditioned CG.", + [], + ), + "multigrid": ( + "poisson_cube_strong_scaling_multigrid", + "Poisson on 3D cube strong scaling test, multigrid preconditioner", + "Strong scaling of the Poisson model with manufactured solution on a 3D cube (Cuboid), solved with CG preconditioned by geometric multigrid.", + ["--precond", "MultiGrid"], + ), + "stiffness": ( + "poisson_cube_strong_scaling_stiffness", + "Poisson on 3D cube strong scaling test, stiffness preconditioner", + "Strong scaling of the Poisson model with manufactured solution on a 3D cube (Cuboid), solved with CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner).", + ["--precond", "StiffnessPreconditioner"], + ), +} + def main() -> None: @@ -25,6 +50,12 @@ def main() -> None: parser = argparse.ArgumentParser( description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), ) + parser.add_argument( + "--variant", + default="none", + choices=list(VARIANTS), + help="Preconditioner of the runs.", + ) parser.add_argument( "--upload", action="store_true", @@ -42,15 +73,16 @@ def main() -> None: help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', ) args = parser.parse_args() + label, name, description, param_flags = VARIANTS[args.variant] # Paths relative to this script's location, so it can be run from anywhere. script_dir = Path(__file__).resolve().parent params_dir = script_dir / "cube_strong_scaling" profiling_case = ProfilingCase( - label="poisson_cube_strong_scaling", - name="Poisson on 3D cube strong scaling test, no preconditioner", - description="Strong scaling of the Poisson model with manufactured solution on a 3D cube (Cuboid), solved with unpreconditioned CG.", + label=label, + name=name, + description=description, physics_problem="Occurs in many plasma applications.", struphy_model_used="Poisson", params_source=params_dir / "params_poisson.py", @@ -61,7 +93,7 @@ def main() -> None: # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) + profiling_case.launch(num_tasks, param_flags=param_flags) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/Poisson/submit_poisson_cube_strong_scaling_multigrid.py b/profiling/examples/Poisson/submit_poisson_cube_strong_scaling_multigrid.py deleted file mode 100644 index 684ba2c41..000000000 --- a/profiling/examples/Poisson/submit_poisson_cube_strong_scaling_multigrid.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Poisson strong scaling profiling case on 3D cube, multigrid preconditioner. - -This file defines the Poisson strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_poisson.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "cube_strong_scaling_multigrid" - - profiling_case = ProfilingCase( - label="poisson_cube_strong_scaling_multigrid", - name="Poisson on 3D cube strong scaling test, multigrid preconditioner", - description="Strong scaling of the Poisson model with manufactured solution on a 3D cube (Cuboid), solved with CG preconditioned by geometric multigrid.", - physics_problem="Occurs in many plasma applications.", - struphy_model_used="Poisson", - params_source=params_dir / "params_poisson.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/Poisson/submit_poisson_cube_strong_scaling_stiffness.py b/profiling/examples/Poisson/submit_poisson_cube_strong_scaling_stiffness.py deleted file mode 100644 index 82909ad33..000000000 --- a/profiling/examples/Poisson/submit_poisson_cube_strong_scaling_stiffness.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Poisson strong scaling profiling case on 3D cube, stiffness preconditioner. - -This file defines the Poisson strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_poisson.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "cube_strong_scaling_stiffness" - - profiling_case = ProfilingCase( - label="poisson_cube_strong_scaling_stiffness", - name="Poisson on 3D cube strong scaling test, stiffness preconditioner", - description="Strong scaling of the Poisson model with manufactured solution on a 3D cube (Cuboid), solved with CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner).", - physics_problem="Occurs in many plasma applications.", - struphy_model_used="Poisson", - params_source=params_dir / "params_poisson.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/Poisson/submit_poisson_hollow_cylinder_strong_scaling.py b/profiling/examples/Poisson/submit_poisson_hollow_cylinder_strong_scaling.py index d98ad53a3..61f8356f1 100644 --- a/profiling/examples/Poisson/submit_poisson_hollow_cylinder_strong_scaling.py +++ b/profiling/examples/Poisson/submit_poisson_hollow_cylinder_strong_scaling.py @@ -1,4 +1,4 @@ -"""Poisson strong scaling profiling case on hollow cylinder, no preconditioner. +"""Poisson strong scaling profiling case on hollow cylinder. This file defines the Poisson strong scaling profiling case (the `ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a @@ -7,6 +7,8 @@ each run as soon as its own job finishes. Each generated script runs the simulation itself by invoking `params_poisson.py` directly (its `__main__` block is the worker). + +The variant (preconditioner) is chosen with `--variant`; run this script once per variant. """ import argparse @@ -18,6 +20,29 @@ from profiling_job import ProfilingCase # noqa: E402 +# variant -> (label, name, description, flags of the params file); the labels identify the +# cases in the profiling-data repo and are kept from the former one-case-per-script layout +VARIANTS = { + "none": ( + "poisson_hollow_cylinder_strong_scaling", + "Poisson on hollow cylinder strong scaling test, no preconditioner", + "Strong scaling of the Poisson model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with unpreconditioned CG.", + [], + ), + "multigrid": ( + "poisson_hollow_cylinder_strong_scaling_multigrid", + "Poisson on hollow cylinder strong scaling test, multigrid preconditioner", + "Strong scaling of the Poisson model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with CG preconditioned by geometric multigrid.", + ["--precond", "MultiGrid"], + ), + "stiffness": ( + "poisson_hollow_cylinder_strong_scaling_stiffness", + "Poisson on hollow cylinder strong scaling test, stiffness preconditioner", + "Strong scaling of the Poisson model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner).", + ["--precond", "StiffnessPreconditioner"], + ), +} + def main() -> None: @@ -25,6 +50,12 @@ def main() -> None: parser = argparse.ArgumentParser( description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), ) + parser.add_argument( + "--variant", + default="none", + choices=list(VARIANTS), + help="Preconditioner of the runs.", + ) parser.add_argument( "--upload", action="store_true", @@ -42,15 +73,16 @@ def main() -> None: help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', ) args = parser.parse_args() + label, name, description, param_flags = VARIANTS[args.variant] # Paths relative to this script's location, so it can be run from anywhere. script_dir = Path(__file__).resolve().parent params_dir = script_dir / "hollow_cylinder_strong_scaling" profiling_case = ProfilingCase( - label="poisson_hollow_cylinder_strong_scaling", - name="Poisson on hollow cylinder strong scaling test, no preconditioner", - description="Strong scaling of the Poisson model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with unpreconditioned CG.", + label=label, + name=name, + description=description, physics_problem="Occurs in many plasma applications.", struphy_model_used="Poisson", params_source=params_dir / "params_poisson.py", @@ -61,7 +93,7 @@ def main() -> None: # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) + profiling_case.launch(num_tasks, param_flags=param_flags) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/Poisson/submit_poisson_hollow_cylinder_strong_scaling_multigrid.py b/profiling/examples/Poisson/submit_poisson_hollow_cylinder_strong_scaling_multigrid.py deleted file mode 100644 index 85b895335..000000000 --- a/profiling/examples/Poisson/submit_poisson_hollow_cylinder_strong_scaling_multigrid.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Poisson strong scaling profiling case on hollow cylinder, multigrid preconditioner. - -This file defines the Poisson strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_poisson.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_cylinder_strong_scaling_multigrid" - - profiling_case = ProfilingCase( - label="poisson_hollow_cylinder_strong_scaling_multigrid", - name="Poisson on hollow cylinder strong scaling test, multigrid preconditioner", - description="Strong scaling of the Poisson model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with CG preconditioned by geometric multigrid.", - physics_problem="Occurs in many plasma applications.", - struphy_model_used="Poisson", - params_source=params_dir / "params_poisson.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/Poisson/submit_poisson_hollow_cylinder_strong_scaling_stiffness.py b/profiling/examples/Poisson/submit_poisson_hollow_cylinder_strong_scaling_stiffness.py deleted file mode 100644 index d9ce5c9f3..000000000 --- a/profiling/examples/Poisson/submit_poisson_hollow_cylinder_strong_scaling_stiffness.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Poisson strong scaling profiling case on hollow cylinder, stiffness preconditioner. - -This file defines the Poisson strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_poisson.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_cylinder_strong_scaling_stiffness" - - profiling_case = ProfilingCase( - label="poisson_hollow_cylinder_strong_scaling_stiffness", - name="Poisson on hollow cylinder strong scaling test, stiffness preconditioner", - description="Strong scaling of the Poisson model with manufactured solution on a hollow cylinder (HollowCylinder, a1=1e-2, a2=1, Lz=4), solved with CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner).", - physics_problem="Occurs in many plasma applications.", - struphy_model_used="Poisson", - params_source=params_dir / "params_poisson.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/Poisson/submit_poisson_hollow_torus_strong_scaling.py b/profiling/examples/Poisson/submit_poisson_hollow_torus_strong_scaling.py index 3d1b1c605..39792aba8 100644 --- a/profiling/examples/Poisson/submit_poisson_hollow_torus_strong_scaling.py +++ b/profiling/examples/Poisson/submit_poisson_hollow_torus_strong_scaling.py @@ -1,4 +1,4 @@ -"""Poisson strong scaling profiling case on hollow torus, no preconditioner. +"""Poisson strong scaling profiling case on hollow torus. This file defines the Poisson strong scaling profiling case (the `ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a @@ -7,6 +7,8 @@ each run as soon as its own job finishes. Each generated script runs the simulation itself by invoking `params_poisson.py` directly (its `__main__` block is the worker). + +The variant (preconditioner) is chosen with `--variant`; run this script once per variant. """ import argparse @@ -18,6 +20,29 @@ from profiling_job import ProfilingCase # noqa: E402 +# variant -> (label, name, description, flags of the params file); the labels identify the +# cases in the profiling-data repo and are kept from the former one-case-per-script layout +VARIANTS = { + "none": ( + "poisson_hollow_torus_strong_scaling", + "Poisson on hollow torus strong scaling test, no preconditioner", + "Strong scaling of the Poisson model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with unpreconditioned CG.", + [], + ), + "multigrid": ( + "poisson_hollow_torus_strong_scaling_multigrid", + "Poisson on hollow torus strong scaling test, multigrid preconditioner", + "Strong scaling of the Poisson model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with CG preconditioned by geometric multigrid.", + ["--precond", "MultiGrid"], + ), + "stiffness": ( + "poisson_hollow_torus_strong_scaling_stiffness", + "Poisson on hollow torus strong scaling test, stiffness preconditioner", + "Strong scaling of the Poisson model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner).", + ["--precond", "StiffnessPreconditioner"], + ), +} + def main() -> None: @@ -25,6 +50,12 @@ def main() -> None: parser = argparse.ArgumentParser( description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), ) + parser.add_argument( + "--variant", + default="none", + choices=list(VARIANTS), + help="Preconditioner of the runs.", + ) parser.add_argument( "--upload", action="store_true", @@ -42,15 +73,16 @@ def main() -> None: help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', ) args = parser.parse_args() + label, name, description, param_flags = VARIANTS[args.variant] # Paths relative to this script's location, so it can be run from anywhere. script_dir = Path(__file__).resolve().parent params_dir = script_dir / "hollow_torus_strong_scaling" profiling_case = ProfilingCase( - label="poisson_hollow_torus_strong_scaling", - name="Poisson on hollow torus strong scaling test, no preconditioner", - description="Strong scaling of the Poisson model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with unpreconditioned CG.", + label=label, + name=name, + description=description, physics_problem="Occurs in many plasma applications.", struphy_model_used="Poisson", params_source=params_dir / "params_poisson.py", @@ -61,7 +93,7 @@ def main() -> None: # Launch one run per rank count for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) + profiling_case.launch(num_tasks, param_flags=param_flags) # Package and push each run as its own job finishes. profiling_case.finalize_run() diff --git a/profiling/examples/Poisson/submit_poisson_hollow_torus_strong_scaling_multigrid.py b/profiling/examples/Poisson/submit_poisson_hollow_torus_strong_scaling_multigrid.py deleted file mode 100644 index 2ebde34dd..000000000 --- a/profiling/examples/Poisson/submit_poisson_hollow_torus_strong_scaling_multigrid.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Poisson strong scaling profiling case on hollow torus, multigrid preconditioner. - -This file defines the Poisson strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_poisson.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_torus_strong_scaling_multigrid" - - profiling_case = ProfilingCase( - label="poisson_hollow_torus_strong_scaling_multigrid", - name="Poisson on hollow torus strong scaling test, multigrid preconditioner", - description="Strong scaling of the Poisson model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with CG preconditioned by geometric multigrid.", - physics_problem="Occurs in many plasma applications.", - struphy_model_used="Poisson", - params_source=params_dir / "params_poisson.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/Poisson/submit_poisson_hollow_torus_strong_scaling_stiffness.py b/profiling/examples/Poisson/submit_poisson_hollow_torus_strong_scaling_stiffness.py deleted file mode 100644 index 288630e20..000000000 --- a/profiling/examples/Poisson/submit_poisson_hollow_torus_strong_scaling_stiffness.py +++ /dev/null @@ -1,71 +0,0 @@ -"""Poisson strong scaling profiling case on hollow torus, stiffness preconditioner. - -This file defines the Poisson strong scaling profiling case (the `ProfilingCase`) -and submits it: for each rank count, `ProfilingCase.launch` builds and submits a -SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch -system, runs directly on this machine. `finalize_run` then packages and uploads -each run as soon as its own job finishes. -Each generated script runs the simulation itself by invoking `params_poisson.py` -directly (its `__main__` block is the worker). -""" - -import argparse -import sys -from pathlib import Path - -# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up -sys.path.insert(0, str(Path(__file__).resolve().parents[2])) - -from profiling_job import ProfilingCase # noqa: E402 - - -def main() -> None: - - # Parse arguments, do not remove --upload - parser = argparse.ArgumentParser( - description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), - ) - parser.add_argument( - "--upload", - action="store_true", - help="Upload the packaged profiling results to the profiling-data repo.", - ) - parser.add_argument( - "--language", - default="fortran", - choices=["fortran", "c"], - help="Pyccel language to compile the Struphy kernels with.", - ) - parser.add_argument( - "--compiler", - default="GNU", - help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', - ) - args = parser.parse_args() - - # Paths relative to this script's location, so it can be run from anywhere. - script_dir = Path(__file__).resolve().parent - params_dir = script_dir / "hollow_torus_strong_scaling_stiffness" - - profiling_case = ProfilingCase( - label="poisson_hollow_torus_strong_scaling_stiffness", - name="Poisson on hollow torus strong scaling test, stiffness preconditioner", - description="Strong scaling of the Poisson model with manufactured solution on a full hollow torus (HollowTorus, a1=1e-2, a2=1, R0=4), solved with CG preconditioned by the Kronecker approximation of the inverse Laplacian (StiffnessPreconditioner).", - physics_problem="Occurs in many plasma applications.", - struphy_model_used="Poisson", - params_source=params_dir / "params_poisson.py", - language=args.language, - compiler=args.compiler, - upload=args.upload, - ) - - # Launch one run per rank count - for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): - profiling_case.launch(num_tasks) - - # Package and push each run as its own job finishes. - profiling_case.finalize_run() - - -if __name__ == "__main__": - main() diff --git a/profiling/examples/Poisson/submit_poisson_polar_cylinder_strong_scaling.py b/profiling/examples/Poisson/submit_poisson_polar_cylinder_strong_scaling.py new file mode 100644 index 000000000..ebcd74cbb --- /dev/null +++ b/profiling/examples/Poisson/submit_poisson_polar_cylinder_strong_scaling.py @@ -0,0 +1,123 @@ +"""Poisson polar cylinder strong scaling profiling case: polar splines versus tensor-product splines on a cylinder with a hole. + +This file defines one variant of the Poisson strong scaling profiling case on a cylinder (radius 1, length 4) (the +`ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a +SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch system, runs directly +on this machine. `finalize_run` then packages and uploads each run as soon as its own job finishes. +Each generated script runs the simulation itself by invoking `params_poisson.py` directly (its `__main__` +block is the worker), with the flags of the variant. + +The variant (spline space and preconditioner) is chosen with `--variant`; run this script once per variant. +""" + +import argparse +import sys +from pathlib import Path + +# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from profiling_job import ProfilingCase # noqa: E402 + +# variant -> (description, flags of params_poisson.py) +VARIANTS = { + "tensor_none": ( + "tensor-product splines, no preconditioner", + ["--splines", "tensor", "--precond", "None"], + ), + "tensor_stiffness": ( + "tensor-product splines, stiffness preconditioner", + ["--splines", "tensor", "--precond", "StiffnessPreconditioner"], + ), + "tensor_multigrid": ( + "tensor-product splines, multigrid preconditioner", + ["--splines", "tensor", "--precond", "MultiGrid"], + ), + "polar_none": ( + "polar splines, no preconditioner", + ["--splines", "polar", "--precond", "None"], + ), + "polar_tensor_mass": ( + "polar splines, tensor-product mass matrix preconditioner", + ["--splines", "polar", "--precond", "MassMatrixPreconditioner"], + ), + "polar_mass": ( + "polar splines, polar mass matrix preconditioner (Schur)", + ["--splines", "polar", "--precond", "PolarMassMatrixPreconditioner"], + ), + "polar_mass_ilu": ( + "polar splines, polar mass matrix preconditioner (ILU)", + ["--splines", "polar", "--precond", "PolarMassMatrixPreconditioner", "--poloidal-solver", "ilu"], + ), + "polar_stiffness": ( + "polar splines, polar stiffness preconditioner (Schur)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner"], + ), + "polar_stiffness_ilu": ( + "polar splines, polar stiffness preconditioner (ILU)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner", "--poloidal-solver", "ilu"], + ), +} + + +def main() -> None: + + # Parse arguments, do not remove --upload + parser = argparse.ArgumentParser( + description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), + ) + parser.add_argument( + "--variant", + required=True, + choices=list(VARIANTS), + help="Spline space and preconditioner of the runs.", + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload the packaged profiling results to the profiling-data repo.", + ) + parser.add_argument( + "--language", + default="fortran", + choices=["fortran", "c"], + help="Pyccel language to compile the Struphy kernels with.", + ) + parser.add_argument( + "--compiler", + default="GNU", + help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', + ) + args = parser.parse_args() + variant_description, param_flags = VARIANTS[args.variant] + + # Paths relative to this script's location, so it can be run from anywhere. + script_dir = Path(__file__).resolve().parent + params_dir = script_dir / "polar_cylinder_strong_scaling" + + profiling_case = ProfilingCase( + label=f"poisson_polar_cylinder_strong_scaling_{args.variant}", + name=f"Poisson on a cylinder strong scaling test, {variant_description}", + description=( + "Strong scaling of the Poisson model with a manufactured solution whose gradient is largest on the axis, " + "comparing C1 polar splines (IGAPolarCylinder) with tensor-product splines on a cylinder with a hole " + f"(HollowCylinder, a1=1e-2): {variant_description}." + ), + physics_problem="Occurs in many plasma applications.", + struphy_model_used="Poisson", + params_source=params_dir / "params_poisson.py", + language=args.language, + compiler=args.compiler, + upload=args.upload, + ) + + # Launch one run per rank count + for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): + profiling_case.launch(num_tasks, param_flags=param_flags) + + # Package and push each run as its own job finishes. + profiling_case.finalize_run() + + +if __name__ == "__main__": + main() diff --git a/profiling/examples/Poisson/submit_poisson_polar_torus_strong_scaling.py b/profiling/examples/Poisson/submit_poisson_polar_torus_strong_scaling.py new file mode 100644 index 000000000..23f7bceff --- /dev/null +++ b/profiling/examples/Poisson/submit_poisson_polar_torus_strong_scaling.py @@ -0,0 +1,123 @@ +"""Poisson polar torus strong scaling profiling case: polar splines versus tensor-product splines on a torus with a hole. + +This file defines one variant of the Poisson strong scaling profiling case on a full torus (minor radius 1, major radius 4) (the +`ProfilingCase`) and submits it: for each rank count, `ProfilingCase.launch` builds and submits a +SLURM script (using `clusters.SLURM_PRESETS` by default), or, without a batch system, runs directly +on this machine. `finalize_run` then packages and uploads each run as soon as its own job finishes. +Each generated script runs the simulation itself by invoking `params_poisson.py` directly (its `__main__` +block is the worker), with the flags of the variant. + +The variant (spline space and preconditioner) is chosen with `--variant`; run this script once per variant. +""" + +import argparse +import sys +from pathlib import Path + +# the profiling driver (profiling_job.py, clusters.py, ...) lives in profiling/, two levels up +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + +from profiling_job import ProfilingCase # noqa: E402 + +# variant -> (description, flags of params_poisson.py) +VARIANTS = { + "tensor_none": ( + "tensor-product splines, no preconditioner", + ["--splines", "tensor", "--precond", "None"], + ), + "tensor_stiffness": ( + "tensor-product splines, stiffness preconditioner", + ["--splines", "tensor", "--precond", "StiffnessPreconditioner"], + ), + "tensor_multigrid": ( + "tensor-product splines, multigrid preconditioner", + ["--splines", "tensor", "--precond", "MultiGrid"], + ), + "polar_none": ( + "polar splines, no preconditioner", + ["--splines", "polar", "--precond", "None"], + ), + "polar_tensor_mass": ( + "polar splines, tensor-product mass matrix preconditioner", + ["--splines", "polar", "--precond", "MassMatrixPreconditioner"], + ), + "polar_mass": ( + "polar splines, polar mass matrix preconditioner (Schur)", + ["--splines", "polar", "--precond", "PolarMassMatrixPreconditioner"], + ), + "polar_mass_ilu": ( + "polar splines, polar mass matrix preconditioner (ILU)", + ["--splines", "polar", "--precond", "PolarMassMatrixPreconditioner", "--poloidal-solver", "ilu"], + ), + "polar_stiffness": ( + "polar splines, polar stiffness preconditioner (Schur)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner"], + ), + "polar_stiffness_ilu": ( + "polar splines, polar stiffness preconditioner (ILU)", + ["--splines", "polar", "--precond", "StiffnessPreconditioner", "--poloidal-solver", "ilu"], + ), +} + + +def main() -> None: + + # Parse arguments, do not remove --upload + parser = argparse.ArgumentParser( + description=("Submit profiling jobs to a SLURM cluster and package the results for upload."), + ) + parser.add_argument( + "--variant", + required=True, + choices=list(VARIANTS), + help="Spline space and preconditioner of the runs.", + ) + parser.add_argument( + "--upload", + action="store_true", + help="Upload the packaged profiling results to the profiling-data repo.", + ) + parser.add_argument( + "--language", + default="fortran", + choices=["fortran", "c"], + help="Pyccel language to compile the Struphy kernels with.", + ) + parser.add_argument( + "--compiler", + default="GNU", + help='Pyccel compiler family ("GNU", "intel", "PGI", "nvidia" or "LLVM").', + ) + args = parser.parse_args() + variant_description, param_flags = VARIANTS[args.variant] + + # Paths relative to this script's location, so it can be run from anywhere. + script_dir = Path(__file__).resolve().parent + params_dir = script_dir / "polar_torus_strong_scaling" + + profiling_case = ProfilingCase( + label=f"poisson_polar_torus_strong_scaling_{args.variant}", + name=f"Poisson on a torus strong scaling test, {variant_description}", + description=( + "Strong scaling of the Poisson model with a manufactured solution whose gradient is largest on the axis, " + "comparing C1 polar splines (IGAPolarTorus) with tensor-product splines on a torus with a hole " + f"(HollowTorus, a1=1e-2): {variant_description}." + ), + physics_problem="Occurs in many plasma applications.", + struphy_model_used="Poisson", + params_source=params_dir / "params_poisson.py", + language=args.language, + compiler=args.compiler, + upload=args.upload, + ) + + # Launch one run per rank count + for num_tasks in (1, 2, 4, 8, 16, 32, 64, 128, 256): + profiling_case.launch(num_tasks, param_flags=param_flags) + + # Package and push each run as its own job finishes. + profiling_case.finalize_run() + + +if __name__ == "__main__": + main() diff --git a/src/struphy/post_processing/output.py b/src/struphy/post_processing/output.py index 82612fccc..e8b68472d 100644 --- a/src/struphy/post_processing/output.py +++ b/src/struphy/post_processing/output.py @@ -24,6 +24,7 @@ from struphy.models.species import ParticleSpecies from struphy.models.variables import PICVariable, SPHVariable from struphy.pic.base import Particles +from struphy.polar.basic import PolarVector from struphy.post_processing import store from struphy.post_processing.arrays import ( BINNED_LABELS, @@ -316,6 +317,8 @@ def _reset(self): self._seconds = None self._spline_derham = None self._spline_snapshots = {} + # polar splines: operators recovering the polar coefficients from the saved ones, per (derham, space) + self._polar_restart_ops = {} def evaluate( self, @@ -1471,6 +1474,7 @@ def _load_femfields(self, fields: dict, files: list, n: int, step: int = 1): step : int Time-step stride of the saved snapshots. """ + polar_fields = {} for file in files: for species, dset in file["feec"].items(): for var, ddset in dset.items(): @@ -1483,7 +1487,11 @@ def _load_femfields(self, fields: dict, files: list, n: int, step: int = 1): assert gl_e.shape == (3,) or gl_e.shape == (3, 3) assert pads.shape == (3,) or pads.shape == (3, 3) - vector = fields[species][var].vector + # the saved data are the tensor-product coefficients (for polar splines: the + # extracted coefficients E^T v, from which v is recovered below) + field = fields[species][var] + is_polar = isinstance(field.vector, PolarVector) + vector = field.vector_stencil if is_polar else field.vector # scalar field if gl_s.shape == (3,): @@ -1511,6 +1519,16 @@ def _load_femfields(self, fields: dict, files: list, n: int, step: int = 1): ] = ddset[str(comp + 1)][n * step, p1:-p1, p2:-p2, p3:-p3] vector.update_ghost_regions() + if is_polar: + polar_fields[(species, var)] = field + + # polar splines: recover the polar coefficients once all files are loaded + for field in polar_fields.values(): + key = (id(field.derham), field.space_id) + if key not in self._polar_restart_ops: + self._polar_restart_ops[key] = field._restart_extraction_op() + self._polar_restart_ops[key].dot(field.vector_stencil, out=field.vector) + field.vector.update_ghost_regions() def _create_eval_grids(self, celldivide: int | Sequence[int] = (1, 1, 1)): """Build the logical evaluation grids and distribute them over the MPI ranks. diff --git a/src/struphy/post_processing/tests/test_pproc.py b/src/struphy/post_processing/tests/test_pproc.py index b369d8b1a..d3745d788 100644 --- a/src/struphy/post_processing/tests/test_pproc.py +++ b/src/struphy/post_processing/tests/test_pproc.py @@ -98,5 +98,64 @@ def test_products_are_processed_in_parallel_on_first_use(): comm.Barrier() +def test_pproc_polar(tmp_path): + """Fields on polar splines are loaded from the saved (extracted) coefficients: the Poisson + solution on a disc evaluates to the exact solution.""" + import sympy as sym + + from struphy import BaseUnits, DerhamOptions, EnvironmentOptions, Time, domains, grids + from struphy.initial.base import GenericPerturbation + from struphy.models import Poisson + + model = Poisson(base_units=BaseUnits()) + model.em_fields.phi.save_data = True + model.em_fields.source.save_data = True + + num_elements, degree = (12, 18, 4), (3, 3, 2) + domain = domains.IGAPolarCylinder(num_elements=num_elements[:2], degree=degree[:2], a=1.0, Lz=4.0) + sim = Simulation( + model=model, + params_path=__file__, + # the same output folder on all processes + env=EnvironmentOptions(out_folders=MPI.COMM_WORLD.bcast(str(tmp_path), root=0), sim_folder="sim_polar"), + time_opts=Time(), + domain=domain, + grid=grids.TensorProductGrid(num_elements=num_elements), + derham_opts=DerhamOptions(degree=degree, bcs=(("free", "dirichlet"), None, None), polar_splines=True), + ) + model.propagators.poisson.options = model.propagators.poisson.Options( + solver="pcg", precond="StiffnessPreconditioner" + ) + + # exact solution with non-zero gradient on the axis, vanishing at r = 1 + x, y, z = sym.symbols("x y z", real=True) + phi_sym = (1 - x**2 - y**2) * (1 + x + y / 2) + rhs_sym = -(sym.diff(phi_sym, x, 2) + sym.diff(phi_sym, y, 2) + sym.diff(phi_sym, z, 2)) + phi_fun = sym.lambdify((x, y, z), phi_sym, "numpy") + rhs_fun = sym.lambdify((x, y, z), rhs_sym, "numpy") + model.em_fields.source.add_perturbation( + GenericPerturbation(lambda x, y, z: rhs_fun(x, y, z) + 0.0 * x, given_in_basis="physical") + ) + + sim.run(one_time_step=True) + run = Output(sim.env.path_out) + + # serial pproc + run.pproc(parallel=False) + if sim.rank == 0: + phi = run.evaluate("em_fields/phi", t=-1) + X, Y, Z = (phi.coords[c].values for c in ("X", "Y", "Z")) + exact = phi_fun(X, Y, Z) + # (the spline mapping only approximates the disc; before loading the polar coefficients, the error was 1) + assert np.max(np.abs(phi.values - exact)) < 1e-2 * np.max(np.abs(exact)) + serial = np.asarray(run.fields.em_fields.phi) + + # parallel pproc (one data file per process) gives the same fields + run.pproc(parallel=True, force=True) + if sim.rank == 0: + assert np.allclose(np.asarray(run.fields.em_fields.phi), serial, atol=1e-7) + MPI.COMM_WORLD.Barrier() + + if __name__ == "__main__": test_pproc_mpi(show_plot=True)