diff --git a/FabNESO/__init__.py b/FabNESO/__init__.py index c5bfed2..9e82704 100644 --- a/FabNESO/__init__.py +++ b/FabNESO/__init__.py @@ -1,9 +1,23 @@ """Neptune Exploratory SOftware (NESO) plugin for FabSim3.""" try: - from .tasks import neso, neso_ensemble, neso_vbmc, neso_write_field + from .tasks import ( + neso, + neso_ensemble, + neso_vbmc, + neso_vvuq_campaign, + neso_write_field, + single_run_vvuq, + ) - __all__ = ["neso", "neso_ensemble", "neso_vbmc", "neso_write_field"] + __all__ = [ + "neso", + "neso_ensemble", + "neso_vbmc", + "neso_write_field", + "neso_vvuq_campaign", + "single_run_vvuq", + ] except ImportError: import warnings diff --git a/FabNESO/run_vvuq_instance.sh b/FabNESO/run_vvuq_instance.sh new file mode 100755 index 0000000..5209d6e --- /dev/null +++ b/FabNESO/run_vvuq_instance.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +echo fabsim $1 single_run_vvuq:$PWD,mesh_file=$3,solver=$4,processes=$5,nodes=$6,cpus_per_process=$7,wall_time=$8,neso_outfile=$9 + +fabsim $1 single_run_vvuq:$PWD,mesh_file=$3,solver=$4,processes=$5,nodes=$6,cpus_per_process=$7,wall_time=$8,neso_outfile=$9 diff --git a/FabNESO/tasks.py b/FabNESO/tasks.py index 611441a..b023c30 100644 --- a/FabNESO/tasks.py +++ b/FabNESO/tasks.py @@ -7,11 +7,14 @@ import re import shutil import time +from ast import literal_eval from contextlib import nullcontext from pathlib import Path from tempfile import TemporaryDirectory from typing import Any +import chaospy as cp +import easyvvuq as uq import numpy as np import pyvbmc @@ -579,3 +582,199 @@ def log_density( (config_dict["initial_run"] - observed_results["field_value"]) ** 2 / (2 * config_dict["observation_noise_std"] ** 2) ).sum() + + +def _parse_vvuq_parameters( + vvuq_param_file: Path, +) -> tuple[dict[Any, dict[str, float]], dict[str, Any], list[str]]: + # read the inpurt file + with Path.open(vvuq_param_file) as f: + data = f.read() + d = literal_eval(data) + vvuq_params = {} + parameter_variations = {} + for parameter, param_values in d["parameters"].items(): + vvuq_params[parameter] = { + "type": "float", + "default": param_values[0], + } + vary_length = 3 + if len(param_values) == vary_length: + # Make the chaospy distribution. Possibly to be able to edit this later? + parameter_variations[parameter] = cp.Uniform( + param_values[1], param_values[2] + ) + output_columns = d["output_columns"] + return (vvuq_params, parameter_variations, output_columns) + + +@fab.task +@fab.load_plugin_env_vars("FabNESO") +def single_run_vvuq( + config: str, + mesh_file: str = "mesh.xml", + solver: str = "Electrostatic2D3V", + processes: str | int = 4, + nodes: str | int = 1, + cpus_per_process: str | int = 1, + wall_time: str = "00:15:00", + neso_outfile: str = "Electrostatic2D3V_line_field_evaluations.h5part", +) -> None: + """ + Run a single instance of the VVUQ campaign. + + Run an instance of the VVUQ campaign using configurations in the designated solver. + The conditions and mesh are, by design, the defaults. When the iteration is done, + copy the NESO output to the local directory for later decoding through VVUQ. + + + Args: + config: Directory with single run configuration information. + + Keyword Args: + solver: Which NESO solver to use. + mesh_file_name: Name of mesh XML in configuration directory. + processes: Number of processes to run. + nodes: Number of nodes to run on. Only applicable when running on a multi-node + system. + cpus_per_process: Number of processing units to use per process. Only + applicable when running on a multi-node system. + wall_time: Maximum time to allow job to run for. Only applicable when submitting + to a job scheduler. + neso_outfile: Name of the output file the NESO solver produces, to copy across + to the local directory. + + """ + # If we're running remotely, tell the submission to wait until done + if "archer2" in fab.env.remote: + fab.update_environment( + { + "job_dispatch": "cd /work/$project/$project/$username ; sbatch --wait", + } + ) + + # Make a temporary directory to store the configs in - they can be deleted when the + # run is done + with TemporaryDirectory(dir=config) as temporary_config_dir: + config_path = Path(temporary_config_dir) + + # Copy the mesh and conditions files here for FabSIM to read it properly + shutil.copyfile(mesh_file, config_path / Path(mesh_file).name) + shutil.copyfile("conditions.xml", config_path / "conditions.xml") + + # Run an instance of NESO + neso( + temporary_config_dir, + solver=solver, + mesh_file_name=Path(mesh_file).name, + processes=processes, + nodes=nodes, + cpus_per_process=cpus_per_process, + wall_time=wall_time, + ) + + # Retrieve the results + fab.fetch_results() + + # Move the results to the local VVUQ directory for further processing + local_results_dir = Path(fab.env.job_results_local) / template( + fab.env.job_name_template + ) + shutil.copyfile(local_results_dir / neso_outfile, neso_outfile) + + +@fab.task +@fab.load_plugin_env_vars("FabNESO") +def neso_vvuq_campaign( # noqa: PLR0913 + vvuq_config: str, + conditions_template: str = "conditions.template", + vvuq_parameters: str = "vvuq_parameters.txt", + vvuq_script: str = "FabNESO/run_vvuq_instance.sh", + neso_mesh_file: str = "mesh.xml", + output_dir: str = "neso_pce/", + neso_outfile: str = "Electrostatic2D3V_line_field_evaluations.h5part", + solver: str = "Electrostatic2D3V", + processes: str | int = 4, + nodes: str | int = 1, + cpus_per_process: str | int = 1, + wall_time: str = "00:15:00", + *, + run_vvuq_sequential: bool = True, +) -> None: + """ + Run a VVUQ campaign on a NESO solver. + + Args: + vvuq_config: Directory with VVUQ parameters, conditions template file, and mesh + configuration. + + Keyword Args: + conditions_template: A template NESO conditions file. Parameters that are to be + varied are indicated with a $ sign. + vvuq_parameters: A file in the vvuq_config directory containing a dict with + "parameters" that correspond to the parameters to replace in template file, + "output_columns" that are the NESO outputs that VVUQ should read for its + analysis + neso_mesh_file: Name of mesh XML in vvuq_config directory. + output_dir: Directory in which to carry out the VVUQ campaign. + neso_outfile: Name of the output file the NESO solver produces, to copy across + to the local directory. + solver: Which NESO solver to use. + processes: Number of processes to run. + nodes: Number of nodes to run on. Only applicable when running on a multi-node + system. + cpus_per_process: Number of processing units to use per process. Only + applicable when running on a multi-node system. + wall_time: Maximum time to allow job to run for. Only applicable when submitting + to a job scheduler. + run_vvuq_sequential: Set to False to parallelise. Default is True because this + can cause issues if unchecked. + + """ + path_to_vvuq_configs = Path(fab.find_config_file_path(vvuq_config)) + + # Extract the parameters for the conditions template and variations from the config + params, vary, outputs_to_read = _parse_vvuq_parameters( + path_to_vvuq_configs / vvuq_parameters + ) + + # Make an encoder + encoder = uq.encoders.GenericEncoder( + path_to_vvuq_configs / conditions_template, + delimiter="$", + target_filename="conditions.xml", + ) + + # Make the VVUQ decoder - this should be more generic! + decoder = uq.decoders.HDF5( + target_filename=neso_outfile, + output_columns=outputs_to_read, + ) + + # Get the path for this plugin, + vvuq_run_path = Path(fab.get_plugin_path("FabNESO")) / vvuq_script + + # The script that runs the individual NESO instances + execute_local = uq.actions.ExecuteLocal( + f"{vvuq_run_path} {fab.env.remote} {vvuq_config} " + f"{path_to_vvuq_configs / neso_mesh_file} {solver} {processes} " + f"{nodes} {cpus_per_process} {wall_time} {neso_outfile}" + ) + + # Put together the actions required for the VVUQ campaign + actions = uq.actions.Actions( + uq.actions.CreateRunDirectory("/tmp", flatten=True), # noqa: S108 + uq.actions.Encode(encoder), + execute_local, + uq.actions.Decode(decoder), + ) + + # Make the VVUQ campaign + campaign = uq.Campaign(name=output_dir, params=params, actions=actions) + + # Make and attach a sampler to the campaign + my_sampler = uq.sampling.PCESampler(vary) + campaign.set_sampler(my_sampler) + + # Run VVUQ + campaign.execute(sequential=run_vvuq_sequential).collate() diff --git a/config_files/vvuq_example/conditions.template b/config_files/vvuq_example/conditions.template new file mode 100644 index 0000000..9f68042 --- /dev/null +++ b/config_files/vvuq_example/conditions.template @@ -0,0 +1,77 @@ + + + + + + + + + + + + + + + + + + + + + + + +

Lambda = 0.0

+

epsilon = -1.0

+

num_particles_total = $NUMBER_PARTICLES

+

num_particles_per_cell = -1

+

particle_time_step = 0.001

+

particle_num_time_steps = $TIME_STEPS

+

particle_num_write_particle_steps = 0

+

particle_num_write_field_energy_steps = 20

+

particle_num_write_field_steps = $WRITE_FIELD_STEPS

+

particle_num_print_steps = 40

+

particle_distribution_position = 2

+

particle_initial_velocity = $INIT_VELOCITY

+

particle_charge_density = $CHARGE_DENSITY

+

particle_number_density = $NUMBER_DENSITY

+

line_field_deriv_evaluations_step = $LINE_STEP

+

line_field_deriv_evaluations_numx = $LINE_NUMX

+

line_field_deriv_evaluations_numy = $LINE_NUMY

+
+ + + u + rho + + + + C[100] + C[200] + C[300] + C[400] + + + + +

+

+ + +

+

+ + +

+

+ + +

+

+ + + + + + diff --git a/config_files/vvuq_example/mesh.xml b/config_files/vvuq_example/mesh.xml new file mode 100644 index 0000000..dba3c9f --- /dev/null +++ b/config_files/vvuq_example/mesh.xml @@ -0,0 +1,155 @@ + + + + + 0.00000000e+00 0.00000000e+00 0.00000000e+00 + 5.00000000e-02 0.00000000e+00 0.00000000e+00 + 5.00000000e-02 1.00000000e-02 0.00000000e+00 + 0.00000000e+00 1.00000000e-02 0.00000000e+00 + 1.00000000e-01 0.00000000e+00 0.00000000e+00 + 1.00000000e-01 1.00000000e-02 0.00000000e+00 + 1.50000000e-01 0.00000000e+00 0.00000000e+00 + 1.50000000e-01 1.00000000e-02 0.00000000e+00 + 2.00000000e-01 0.00000000e+00 0.00000000e+00 + 2.00000000e-01 1.00000000e-02 0.00000000e+00 + 2.50000000e-01 0.00000000e+00 0.00000000e+00 + 2.50000000e-01 1.00000000e-02 0.00000000e+00 + 3.00000000e-01 0.00000000e+00 0.00000000e+00 + 3.00000000e-01 1.00000000e-02 0.00000000e+00 + 3.50000000e-01 0.00000000e+00 0.00000000e+00 + 3.50000000e-01 1.00000000e-02 0.00000000e+00 + 4.00000000e-01 0.00000000e+00 0.00000000e+00 + 4.00000000e-01 1.00000000e-02 0.00000000e+00 + 4.50000000e-01 0.00000000e+00 0.00000000e+00 + 4.50000000e-01 1.00000000e-02 0.00000000e+00 + 5.00000000e-01 0.00000000e+00 0.00000000e+00 + 5.00000000e-01 1.00000000e-02 0.00000000e+00 + 5.50000000e-01 0.00000000e+00 0.00000000e+00 + 5.50000000e-01 1.00000000e-02 0.00000000e+00 + 6.00000000e-01 0.00000000e+00 0.00000000e+00 + 6.00000000e-01 1.00000000e-02 0.00000000e+00 + 6.50000000e-01 0.00000000e+00 0.00000000e+00 + 6.50000000e-01 1.00000000e-02 0.00000000e+00 + 7.00000000e-01 0.00000000e+00 0.00000000e+00 + 7.00000000e-01 1.00000000e-02 0.00000000e+00 + 7.50000000e-01 0.00000000e+00 0.00000000e+00 + 7.50000000e-01 1.00000000e-02 0.00000000e+00 + 8.00000000e-01 0.00000000e+00 0.00000000e+00 + 8.00000000e-01 1.00000000e-02 0.00000000e+00 + 8.50000000e-01 0.00000000e+00 0.00000000e+00 + 8.50000000e-01 1.00000000e-02 0.00000000e+00 + 9.00000000e-01 0.00000000e+00 0.00000000e+00 + 9.00000000e-01 1.00000000e-02 0.00000000e+00 + 9.50000000e-01 0.00000000e+00 0.00000000e+00 + 9.50000000e-01 1.00000000e-02 0.00000000e+00 + 1.00000000e+00 0.00000000e+00 0.00000000e+00 + 1.00000000e+00 1.00000000e-02 0.00000000e+00 + + + 0 1 + 1 2 + 2 3 + 3 0 + 1 4 + 4 5 + 5 2 + 4 6 + 6 7 + 7 5 + 6 8 + 8 9 + 9 7 + 8 10 + 10 11 + 11 9 + 10 12 + 12 13 + 13 11 + 12 14 + 14 15 + 15 13 + 14 16 + 16 17 + 17 15 + 16 18 + 18 19 + 19 17 + 18 20 + 20 21 + 21 19 + 20 22 + 22 23 + 23 21 + 22 24 + 24 25 + 25 23 + 24 26 + 26 27 + 27 25 + 26 28 + 28 29 + 29 27 + 28 30 + 30 31 + 31 29 + 30 32 + 32 33 + 33 31 + 32 34 + 34 35 + 35 33 + 34 36 + 36 37 + 37 35 + 36 38 + 38 39 + 39 37 + 38 40 + 40 41 + 41 39 + + + 0 1 2 3 + 4 5 6 1 + 7 8 9 5 + 10 11 12 8 + 13 14 15 11 + 16 17 18 14 + 19 20 21 17 + 22 23 24 20 + 25 26 27 23 + 28 29 30 26 + 31 32 33 29 + 34 35 36 32 + 37 38 39 35 + 40 41 42 38 + 43 44 45 41 + 46 47 48 44 + 49 50 51 47 + 52 53 54 50 + 55 56 57 53 + 58 59 60 56 + + + + Q[0-19] + E[0,4,7,10,13,16,19,22,25,28,31,34,37,40,43,46,49,52,55,58] + E[59] + E[2,6,9,12,15,18,21,24,27,30,33,36,39,42,45,48,51,54,57,60] + E[3] + + + C[1] + + + + + + f1598d5e39f175acf388b90df392f76ff29d7f9d + L0211-XU + 5.2.0 + 23-Nov-2022 10:54:06 + + -v -m peralign:surf1=100:surf2=300:dir=y -m peralign:surf1=200:surf2=400:dir=x periodic_structured_cartesian_rectangle.msh periodic_structured_cartesian_rectangle.xml:xml:uncompress + + diff --git a/config_files/vvuq_example/vvuq_parameters.txt b/config_files/vvuq_example/vvuq_parameters.txt new file mode 100644 index 0000000..8ca9311 --- /dev/null +++ b/config_files/vvuq_example/vvuq_parameters.txt @@ -0,0 +1,17 @@ +{ + "parameters": + {"NUMBER_PARTICLES" : [40000], + "INIT_VELOCITY" : [1.0,0.9,1.1], + "CHARGE_DENSITY" : [105,101,109], + "NUMBER_DENSITY" : [105], + "TIME_STEPS" : [180], + "WRITE_FIELD_STEPS": [100], + "LINE_STEP": [20], + "LINE_NUMX": [100], + "LINE_NUMY": [1], + }, + "output_columns": [ + "Step#160/FIELD_EVALUATION_0", + "Step#160/x", + ] +} diff --git a/requirements.txt b/requirements.txt index 167df30..eae1bf5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ h5py>=3.10.0 PyVBMC>=1.0.1 +easyvvuq>=1.2 +chaospy>=4.3.2