diff --git a/core/opengate_core/opengate_lib/digitizer/GateDigitizerProjectionActor.cpp b/core/opengate_core/opengate_lib/digitizer/GateDigitizerProjectionActor.cpp index f31643a28b..e827faad1d 100644 --- a/core/opengate_core/opengate_lib/digitizer/GateDigitizerProjectionActor.cpp +++ b/core/opengate_core/opengate_lib/digitizer/GateDigitizerProjectionActor.cpp @@ -51,10 +51,11 @@ void GateDigitizerProjectionActor::EnableSquaredImage(const bool b) { // FIXME check if weight exists ? } -// Called when the simulation starts +// Resolve the input collections once. Their contents change for each event, +// but the collection objects and their attribute schema are simulation-wide. void GateDigitizerProjectionActor::StartSimulationAction() { - // Get the input hits collection auto *hcm = GateDigiCollectionManager::GetInstance(); + fInputDigiCollections.clear(); for (const auto &name : fInputDigiCollectionNames) { auto *hc = hcm->GetDigiCollection(name); fInputDigiCollections.push_back(hc); @@ -62,7 +63,8 @@ void GateDigitizerProjectionActor::StartSimulationAction() { } } -void GateDigitizerProjectionActor::BeginOfRunActionMasterThread(int run_id) { +void GateDigitizerProjectionActor::BeginOfRunActionMasterThread( + int /*run_id*/) { // Set the image to the correct position/orientation AttachImageToVolume(fImage, fPhysicalVolumeName, G4ThreeVector(), fDetectorOrientationMatrix); @@ -153,7 +155,7 @@ void GateDigitizerProjectionActor::BeginOfRunAction(const G4Run *run) { void GateDigitizerProjectionActor::EndOfEventAction(const G4Event * /*event*/) { const auto run = G4RunManager::GetRunManager()->GetCurrentRun()->GetRunID(); for (size_t channel = 0; channel < fInputDigiCollections.size(); channel++) { - const auto slice = channel + run * fInputDigiCollections.size(); + const auto slice = channel; ProcessSlice(slice, channel); } } @@ -233,11 +235,10 @@ void GateDigitizerProjectionActor::ScoreSquaredValue( } GateDigitizerProjectionActor::ImageType::RegionType -GateDigitizerProjectionActor::GetRunRegion(const int run_id) const { +GateDigitizerProjectionActor::GetRunRegion(const int /*run_id*/) const { auto region = fImage->GetLargestPossibleRegion(); auto index = region.GetIndex(); auto size = region.GetSize(); - index[2] += run_id * fInputDigiCollections.size(); size[2] = fInputDigiCollections.size(); region.SetIndex(index); region.SetSize(size); diff --git a/core/opengate_core/opengate_lib/digitizer/pyGateDigitizerProjectionActor.cpp b/core/opengate_core/opengate_lib/digitizer/pyGateDigitizerProjectionActor.cpp index 5ec6e29ee8..33de59b946 100644 --- a/core/opengate_core/opengate_lib/digitizer/pyGateDigitizerProjectionActor.cpp +++ b/core/opengate_core/opengate_lib/digitizer/pyGateDigitizerProjectionActor.cpp @@ -9,12 +9,32 @@ #include #include +class PyDigitizerProjectionActor : public GateDigitizerProjectionActor { +public: + // Inherit the constructors + using GateDigitizerProjectionActor::GateDigitizerProjectionActor; + + void BeginOfRunActionMasterThread(int run_id) override { + PYBIND11_OVERLOAD(void, GateDigitizerProjectionActor, + BeginOfRunActionMasterThread, run_id); + } + + int EndOfRunActionMasterThread(int run_id) override { + PYBIND11_OVERLOAD(int, GateDigitizerProjectionActor, + EndOfRunActionMasterThread, run_id); + } +}; + void init_GateDigitizerProjectionActor(py::module &m) { - py::class_, GateVActor>(m, "GateDigitizerProjectionActor") .def(py::init()) + .def("BeginOfRunActionMasterThread", + &GateDigitizerProjectionActor::BeginOfRunActionMasterThread) + .def("EndOfRunActionMasterThread", + &GateDigitizerProjectionActor::EndOfRunActionMasterThread) .def_readwrite("fImage", &GateDigitizerProjectionActor::fImage) .def_readwrite("fSquaredImage", &GateDigitizerProjectionActor::fSquaredImage) diff --git a/docs/source/user_guide/user_guide_chemistry.rst b/docs/source/user_guide/user_guide_chemistry.rst index 2380937107..f825030665 100644 --- a/docs/source/user_guide/user_guide_chemistry.rst +++ b/docs/source/user_guide/user_guide_chemistry.rst @@ -536,7 +536,7 @@ This combines the main pieces in one small script: source.position.translation = [0, 0, 0] source.direction.type = "momentum" source.direction.momentum = [0, 0, 1] - source.n = 1 + source.number_of_primaries = 1 chem_actor = sim.add_actor("ChemicalCountingActor", "chem_actor") chem_actor.attached_to = target diff --git a/docs/source/user_guide/user_guide_dynamic_parametrisations.rst b/docs/source/user_guide/user_guide_dynamic_parametrisations.rst index 9535fc370f..e416b8d27d 100644 --- a/docs/source/user_guide/user_guide_dynamic_parametrisations.rst +++ b/docs/source/user_guide/user_guide_dynamic_parametrisations.rst @@ -171,7 +171,7 @@ Example: source.image = "activity_0.mhd" source.direction.type = "iso" source.energy.mono = 1 * MeV - source.n = [2000, 2000, 2000] + source.number_of_primaries = [2000, 2000, 2000] source.add_dynamic_parametrisation( image=[ diff --git a/docs/source/user_guide/user_guide_how_to_convert_example_1.rst b/docs/source/user_guide/user_guide_how_to_convert_example_1.rst index be979220a0..3a9f7aae37 100644 --- a/docs/source/user_guide/user_guide_how_to_convert_example_1.rst +++ b/docs/source/user_guide/user_guide_how_to_convert_example_1.rst @@ -209,7 +209,7 @@ GATE 10 source.position.translation = [1060 * mm, 0 * mm, 0 * mm] source.direction.type = "focused" source.direction.focus_point = [1000 * mm, 0 * mm, 0 * mm] - source.n = 720000 / sim.number_of_threads + source.number_of_primaries = 720000 source.visualization.count = 2000 source.visualization.color = "red" source.visualization.size = 5 diff --git a/docs/source/user_guide/user_guide_multijobs.rst b/docs/source/user_guide/user_guide_multijobs.rst index 17c6dd7a45..ea59e06800 100644 --- a/docs/source/user_guide/user_guide_multijobs.rst +++ b/docs/source/user_guide/user_guide_multijobs.rst @@ -7,7 +7,7 @@ their output again. This is useful in two main situations: - local execution on a workstation, where several jobs can run in parallel on the same machine - server execution, where the split jobs are submitted to a scheduler such as - Slurm or HTCondor + Slurm, HTCondor or PBS. This section focuses on the current user-facing workflow in GATE 10. @@ -186,8 +186,8 @@ Server-based execution Server-based multi-job execution is intended to be file-based: 1. prepare a campaign folder containing ``simulation.json`` and the required - input files, probably locally -2. transfer the campaign folder to the server + input files, probably locally. To create the ```simulation.json`` file, you can use ``sim.to_json_file()`` in your script instead of ``sim.run(...)``. +2. transfer the campaign folder to the server. Be aware of the paths used in the .json file for material database or other input files. They should be valid on the server. 3. split the simulation on the server 4. submit the child jobs through a scheduler 5. inspect job status @@ -246,6 +246,9 @@ As a convenience, the command also looks for a default file named ``jobs_backend_options.json`` inside the campaign folder when no ``--backend-options-json`` or ``--backend`` argument is provided. +You can find examples you can adapt in the ``contrib/cluster`` folder for slurm and pbs backends. +The backend options are usually site-specific and should be adapted to the target server. + .. Example: Slurm .. ~~~~~~~~~~~~~~ diff --git a/docs/source/user_guide/user_guide_optical_photons.rst b/docs/source/user_guide/user_guide_optical_photons.rst index f4074d0914..463b430264 100644 --- a/docs/source/user_guide/user_guide_optical_photons.rst +++ b/docs/source/user_guide/user_guide_optical_photons.rst @@ -16,7 +16,7 @@ Optical photon polarization is not yet implemented. A random polarization is the source.particle = 'opticalphoton' source.position.type = 'point' source.position.translation = [0, 0, 0] - source.n = 100 + source.number_of_primaries = 100 source.energy.mono = 1.9*eV source.mother = 'world source.direction.type = "iso" diff --git a/docs/source/user_guide/user_guide_reference_sources_common_params.rst b/docs/source/user_guide/user_guide_reference_sources_common_params.rst index 14c987bb8b..378ae2c2b5 100644 --- a/docs/source/user_guide/user_guide_reference_sources_common_params.rst +++ b/docs/source/user_guide/user_guide_reference_sources_common_params.rst @@ -11,13 +11,13 @@ a source description to the simulation, you do: .. code:: python source1 = sim.add_source('GenericSource', 'source1') - source1.n = 100 + source1.number_of_primaries = 100 source2 = sim.add_source('VoxelSource', 'source2') source2.activity = 10 * g4_units.Bq There are several source types, each one with different parameter. In -this example, ``source1.n`` indicates that this source will generate 100 +this example, ``source1.number_of_primaries`` indicates that this source will generate 100 Events. The second source manages the time and will generate 10 Events per second, so according to the simulation run timing, a different number of Events will be generated. @@ -59,11 +59,11 @@ A minimal working example would be: .. code:: python source = sim.add_source("GenericSource", "mySource") - source.n = [10000, 150000, 36] + source.number_of_primaries = [10000, 150000, 36] sim.run_timing_intervals = [[0, 1 * sec], [2 * sec, 2.5 * sec], [4 * sec, 5 * sec]] -Please note that sim.run_timing_intervals must have the same length as source.n. +Please note that ``sim.run_timing_intervals`` must have the same length as ``source.number_of_primaries``. By default, all particles of a given run will be emitted at a timestamp equal to the minimum time of the corresponding timing interval. Coordinate system diff --git a/docs/source/user_guide/user_guide_reference_sources_generic_source.rst b/docs/source/user_guide/user_guide_reference_sources_generic_source.rst index 8357e4b5d5..3e3b55ca62 100644 --- a/docs/source/user_guide/user_guide_reference_sources_generic_source.rst +++ b/docs/source/user_guide/user_guide_reference_sources_generic_source.rst @@ -34,7 +34,7 @@ All parameters are stored into a dict-like structure (a Box). Particle can be ‘gamma’, ‘e+’, ‘e-’, ‘proton’ (all Geant4 names). The number of particles that will be generated by the source can be described by an activity ``source.activity = 10 * MBq`` or by a number of particle -``source.n = 100``. +``source.number_of_primaries = 100``. The positions from where the particles will be generated are defined by a shape (‘box’, ‘sphere’, ‘point’, ‘disc’), defined by several parameters diff --git a/docs/source/user_guide/user_guide_reference_sources_ion_pencil_beam_source.rst b/docs/source/user_guide/user_guide_reference_sources_ion_pencil_beam_source.rst index e35d695782..79c0f07fe0 100644 --- a/docs/source/user_guide/user_guide_reference_sources_ion_pencil_beam_source.rst +++ b/docs/source/user_guide/user_guide_reference_sources_ion_pencil_beam_source.rst @@ -41,7 +41,7 @@ a 120 MeV/n carbon ion beam. source.energy.mono = 1440 * MeV source.particle = "ion 6 12" # carbon source.position.translation = [100 * mm, 0 * mm, 0 * cm] - source.n = 20000 + source.number_of_primaries = 20000 source.direction.partPhSp_x = [ 2.3335754 * mm, 2.3335754 * mrad, diff --git a/docs/source/user_guide/user_guide_reference_sources_treatment_plan_pencil_beam_source.rst b/docs/source/user_guide/user_guide_reference_sources_treatment_plan_pencil_beam_source.rst index a3475d0ec4..b513fc9ea9 100644 --- a/docs/source/user_guide/user_guide_reference_sources_treatment_plan_pencil_beam_source.rst +++ b/docs/source/user_guide/user_guide_reference_sources_treatment_plan_pencil_beam_source.rst @@ -85,7 +85,7 @@ simulation: # tps n_sim = 80000 # particles to simulate per beam tps = sim.add_source("TreatmentPlanPBSource", "TPSource") - tps.n = n_sim + tps.number_of_primaries = n_sim tps.beam_model = beamline tps.plan_path = ref_path / "TreatmentPlan2Spots.txt" tps.beam_nr = 1 diff --git a/docs/source/user_guide/user_guide_sources.rst b/docs/source/user_guide/user_guide_sources.rst index 6d0a726400..49416aaa62 100644 --- a/docs/source/user_guide/user_guide_sources.rst +++ b/docs/source/user_guide/user_guide_sources.rst @@ -49,7 +49,7 @@ Alternatively, you can specify the fixed number of primary particles to be gener .. code-block:: python - source1.n = 100 + source1.number_of_primaries = 100 Note, however, that certain functionality regarding simulations with multiple runs is currently not supported when sources specify the fixed number of particles as above. diff --git a/opengate/actors/biasingactors.py b/opengate/actors/biasingactors.py index 28f2e001b3..5667a05694 100644 --- a/opengate/actors/biasingactors.py +++ b/opengate/actors/biasingactors.py @@ -418,11 +418,20 @@ class ScatterSplittingFreeFlightActor( False, {"doc": "Print debug information during the stepping action of the actor."}, ), + "particles": ( + [ + "gamma", + ], + { + "doc": "For scatter splitting free flight, only gamma is supported", + "override": True, + "read_only": True, + }, + ), } # Do NOT work with GammaGeneralProc processes = ["compt", "phot", "conv", "Rayl"] - particles = ["gamma"] user_output_config = { "info": { diff --git a/opengate/actors/digitizers.py b/opengate/actors/digitizers.py index 67c5455a06..975b00466f 100644 --- a/opengate/actors/digitizers.py +++ b/opengate/actors/digitizers.py @@ -1,18 +1,20 @@ from typing import List + import numpy as np -from scipy.spatial.transform import Rotation import opengate_core as g4 +from scipy.spatial.transform import Rotation + from ..base import process_cls -from .base import ActorBase -from ..exception import fatal from ..definitions import fwhm_to_sigma -from ..utility import g4_units +from ..exception import fatal from ..image import ( align_image_with_physical_volume, - update_image_py_to_cpp, get_py_image_from_cpp_image, + update_image_py_to_cpp, ) +from ..utility import g4_units from .actoroutput import ActorOutputRoot, ActorOutputSingleImage +from .base import ActorBase def ene_win_peak(name, energy, energy_width_percent): @@ -466,12 +468,6 @@ class DigitizerBlurringActor(DigitizerWithRootOutput, g4.GateDigitizerBlurringAc "doc": "FIXME", }, ), - "blur_fwhm": ( - None, - { - "doc": "FIXME", - }, - ), "blur_reference_value": ( 0, { @@ -503,6 +499,14 @@ def __initcpp__(self): g4.GateDigitizerBlurringActor.__init__(self, self.user_info) self.AddActions({"StartSimulationAction", "EndSimulationAction"}) + @property + def blur_fwhm(self): + return self.blur_sigma / fwhm_to_sigma + + @blur_fwhm.setter + def blur_fwhm(self, value): + self.blur_sigma = value * fwhm_to_sigma + def resolve_and_validate_config(self, context=None): super().resolve_and_validate_config(context=context) self.initialize_blurring_parameters() @@ -523,13 +527,6 @@ def initialize_blurring_parameters(self): self.set_param_linear() def set_param_gauss(self): - if self.blur_fwhm is not None and self.blur_sigma is not None: - fatal( - f"Error, use blur_sigma or blur_fwhm, not both " - f"(there are: {self.blur_sigma} and {self.blur_fwhm}" - ) - if self.blur_fwhm is not None: - self.blur_sigma = self.blur_fwhm * fwhm_to_sigma if self.blur_sigma is None: fatal(f"Error, use blur_sigma or blur_fwhm") self.blur_reference_value = -1 @@ -547,7 +544,6 @@ def set_param_inverse_square(self): f"Error, use positive blur_resolution " f"(current value = {self.blur_resolution}" ) - self.blur_fwhm = -1 self.blur_sigma = -1 if self.blur_slope is None: self.blur_slope = 0 @@ -804,7 +800,6 @@ class DigitizerSpatialBlurringActor( skip_attributes: List clear_every: int blur_attribute: str - blur_fwhm: float blur_sigma: float keep_in_solid_limits: bool use_truncated_Gaussian: bool @@ -840,12 +835,6 @@ class DigitizerSpatialBlurringActor( "doc": "Which attribute to blur, e.g. PostPosition.", }, ), - "blur_fwhm": ( - None, - { - "doc": "FWHM for the blurring.", - }, - ), "blur_sigma": ( None, { @@ -876,18 +865,19 @@ def __initcpp__(self): g4.GateDigitizerSpatialBlurringActor.__init__(self, self.user_info) self.AddActions({"StartSimulationAction", "EndSimulationAction"}) + @property + def blur_fwhm(self): + return self.blur_sigma / fwhm_to_sigma + + @blur_fwhm.setter + def blur_fwhm(self, value): + if not hasattr(value, "__len__"): + value = [value] * 3 + self.blur_sigma = np.array(value) * fwhm_to_sigma + def initialize_blurring_parameters(self): - if self.blur_fwhm is not None and self.blur_sigma is not None: - fatal( - f"Error, use blur_sigma or blur_fwhm, not both " - f"(there are: {self.blur_sigma} and {self.blur_fwhm}" - ) if not hasattr(self.blur_sigma, "__len__"): self.blur_sigma = [self.blur_sigma] * 3 - if not hasattr(self.blur_fwhm, "__len__"): - self.blur_fwhm = [self.blur_fwhm] * 3 - if self.blur_fwhm is not None: - self.blur_sigma = np.array(self.blur_fwhm) * fwhm_to_sigma if self.blur_sigma is None: fatal(f"Error, use blur_sigma or blur_fwhm") @@ -1185,7 +1175,7 @@ def __init__(self, *args, **kwargs): def __initcpp__(self): g4.GateDigitizerProjectionActor.__init__(self, self.user_info) - self.AddActions({"StartSimulationAction", "EndSimulationAction"}) + self.AddActions({"StartSimulationAction"}) def resolve_and_validate_config(self, context=None): super().resolve_and_validate_config(context=context) @@ -1256,6 +1246,12 @@ def compute_thickness(self, volume, channels): def StartSimulationAction(self): DigitizerBase.StartSimulationAction(self) + for actor_output in self.user_output.values(): + if actor_output.get_active(item="any"): + actor_output.start_of_simulation() + g4.GateDigitizerProjectionActor.StartSimulationAction(self) + + def BeginOfRunActionMasterThread(self, run_index): # for the moment, we cannot use this actor with several volumes if hasattr(self.attached_to, "__len__") and not isinstance( self.attached_to, str @@ -1264,20 +1260,14 @@ def StartSimulationAction(self): f"Sorry, cannot (yet) use several mothers volumes for " f"DigitizerProjectionActor {self.name}" ) - # define the new size and spacing according to the nb of channels # and according to the volume shape size = self.output_size spacing = self.output_spacing - size[2] = len(self.input_digi_collections) * len( - self.simulation.run_timing_intervals - ) + size[2] = len(self.input_digi_collections) spacing[2] = self.compute_thickness(self.attached_to, size[2]) - # we use the image associated with run 0 for the entire simulation - # in the future, this actor should implement a BeginOfRunActionMasterThread - # to be able to work on a per-run basis - self.user_output.counts.create_empty_image(0, size, spacing) + self.user_output.counts.create_empty_image(run_index, size, spacing) # check physical_volume_index and number of repeating n = len(self.attached_to_volume.g4_physical_volumes) @@ -1303,24 +1293,25 @@ def StartSimulationAction(self): ) pv = None # avoid warning from IDE align_image_with_physical_volume( - self.attached_to_volume, self.user_output.counts.data_per_run[0].image + self.attached_to_volume, + self.user_output.counts.data_per_run[run_index].image, ) self.SetPhysicalVolumeName(str(pv.GetName())) # update the cpp image and start update_image_py_to_cpp( - self.user_output.counts.data_per_run[0].image, self.fImage, True + self.user_output.counts.data_per_run[run_index].image, self.fImage, True ) # uncertainty ? if self.user_output.squared_counts.get_active(): - self.user_output.squared_counts.create_empty_image(0, size, spacing) + self.user_output.squared_counts.create_empty_image(run_index, size, spacing) align_image_with_physical_volume( self.attached_to_volume, - self.user_output.squared_counts.data_per_run[0].image, + self.user_output.squared_counts.data_per_run[run_index].image, ) update_image_py_to_cpp( - self.user_output.squared_counts.data_per_run[0].image, + self.user_output.squared_counts.data_per_run[run_index].image, self.fSquaredImage, True, ) @@ -1328,62 +1319,45 @@ def StartSimulationAction(self): # keep the initial origin self.start_output_origin = list( - self.user_output.counts.data_per_run[0].get_image_properties().origin + self.user_output.counts.data_per_run[run_index] + .get_image_properties() + .origin ) - g4.GateDigitizerProjectionActor.StartSimulationAction(self) - - def EndSimulationAction(self): - # Keep this trampoline thin: do not put functional Python logic here. - # Concrete Python-side finalization belongs in finalize_simulation(). - g4.GateDigitizerProjectionActor.EndSimulationAction(self) + g4.GateDigitizerProjectionActor.BeginOfRunActionMasterThread(self, run_index) - def finalize_simulation(self): - """Transfer the projection image from C++ to Python and finalize outputs.""" - - # retrieve the image + def EndOfRunActionMasterThread(self, run_index): + # Transfer the completed C++ images into their run-specific output slots + # before ActorOutput performs cumulative aggregation or discards them. self.user_output.counts.store_data( - "merged", get_py_image_from_cpp_image(self.fImage) + run_index, get_py_image_from_cpp_image(self.fImage, view=False) ) - # set its properties - info = self.user_output.counts.data_per_run[0].get_image_properties() - spacing = info.spacing + # Projection files use a neutral channel-axis spacing and origin. The + # physical detector thickness is only needed while scoring in C++. + info = self.user_output.counts.data_per_run[run_index].get_image_properties() + spacing = np.array(info.spacing, copy=True) if self.origin_as_image_center: origin = -info.size * spacing / 2.0 + spacing / 2.0 else: - origin = self.start_output_origin + origin = np.array(self.start_output_origin, copy=True) origin[2] = 0 spacing[2] = 1 - self.user_output.counts.merged_data.image.SetSpacing(list(spacing)) - self.user_output.counts.merged_data.image.SetOrigin(list(origin)) - - # FIXME: DigitizerProjectionActor currently uses ``data_per_run[0]`` as - # temporary internal scaffolding for image properties/runtime setup and - # later moves the actual result into ``merged_data`` before deleting - # the run-0 slot. This works pragmatically but blurs the actor-output - # semantics: per-run slots should represent true user-facing per-run - # output, not internal scratch state. Revisit this actor so temporary - # runtime bookkeeping is structurally separate from actual output slots. - # - # Related concern: this method still writes certain outputs explicitly - # instead of relying purely on the generic actor-output finalization - # path below, which suggests the projection actor does not yet align - # cleanly with the standard actor-output workflow. - # remove the image for run 0 as the result is in merged_data - self.user_output.counts.data_per_run.pop(0) - self.user_output.counts.write_data_if_requested(which="merged") - - # squared ? + self.user_output.counts.set_image_properties( + run_index, spacing=list(spacing), origin=list(origin) + ) + if self.user_output.squared_counts.get_active(): self.user_output.squared_counts.store_data( - "merged", get_py_image_from_cpp_image(self.fSquaredImage) + run_index, get_py_image_from_cpp_image(self.fSquaredImage, view=False) + ) + self.user_output.squared_counts.set_image_properties( + run_index, spacing=list(spacing), origin=list(origin) ) - self.user_output.squared_counts.merged_data.image.SetSpacing(list(spacing)) - self.user_output.squared_counts.merged_data.image.SetOrigin(list(origin)) - self.user_output.squared_counts.data_per_run.pop(0) - self.user_output.squared_counts.write_data_if_requested(which="merged") - DigitizerBase.finalize_simulation(self) + for actor_output in self.user_output.values(): + if actor_output.get_active(item="all"): + actor_output.end_of_run(run_index) + return 0 class CoincidenceSorterActor(DigitizerWithRootOutput, g4.GateCoincidenceSorterActor): diff --git a/opengate/bin/opengate_jobs_run.py b/opengate/bin/opengate_jobs_run.py index 1a0b20c965..224611af80 100644 --- a/opengate/bin/opengate_jobs_run.py +++ b/opengate/bin/opengate_jobs_run.py @@ -60,6 +60,10 @@ def _resolve_backend_configuration(campaign_dir, backend, backend_options_json): @click.option( "--backend", default=None, + type=click.Choice( + ["local_pool", "local_sequential", "htcondor", "slurm", "pbs"], + case_sensitive=False, + ), help="Execution backend to use. If omitted, the command looks for jobs_backend_options.json in CAMPAIGN_DIR.", ) @click.option( diff --git a/opengate/contrib/cluster/pbs/opengate_job.pbs b/opengate/contrib/cluster/pbs/opengate_job.pbs new file mode 100644 index 0000000000..06c00542d2 --- /dev/null +++ b/opengate/contrib/cluster/pbs/opengate_job.pbs @@ -0,0 +1,40 @@ +#!/bin/bash -x +#PBS -l walltime=10:00:00 +#PBS -N opengate +#PBS -l nodes=1:ppn=1 +#PBS -l mem=10gb +#PBS -o opengate.out +#PBS -e opengate.err + +function error { + echo "ERROR: $1" + exit 1 +} + +function warning { + echo "WARNING: $1" +} + +date +test -f ${HOME}/.bashrc && echo "Sourcing bashrc" && source ${HOME}/.bashrc +set -u + +echo "Checking" +uname -a +echo "JOBDIR=${JOBDIR:?"unknown JOBDIR"}" + +echo Using $(which python) +test -d "${JOBDIR}" || error "invalid macro" + + +# Enforce one thread +ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1 + +echo "Lauching macro" +date +cd ${JOBDIR} +cd .. +opengate_job_runner --backend "slurm" ${JOBDIR} || error "gate failed" +date + +echo "Success!!!" diff --git a/opengate/contrib/cluster/pbs/pbs.json b/opengate/contrib/cluster/pbs/pbs.json new file mode 100644 index 0000000000..5cdab19ee0 --- /dev/null +++ b/opengate/contrib/cluster/pbs/pbs.json @@ -0,0 +1,10 @@ +{ + "backend": "pbs", + "backend_options": { + "submit_binary": "qsub", + "submit_filename": "opengate_job.pbs", + "command_line_args": [ + "-V" + ] + } +} diff --git a/opengate/contrib/cluster/slurm/opengate_job.slurm b/opengate/contrib/cluster/slurm/opengate_job.slurm new file mode 100644 index 0000000000..d5f9ae2612 --- /dev/null +++ b/opengate/contrib/cluster/slurm/opengate_job.slurm @@ -0,0 +1,41 @@ +#!/bin/bash -x +#SBATCH --mem=20000 # Memory in MB per default +#SBATCH --time=10:00:00 +#SBATCH --ntasks=1 +#SBATCH --output=Seq%j.out # nom du fichier de sortie +#SBATCH --error=Seq%j.out # nom du fichier d'erreur (ici en commun avec la sortie) + + + +function error { + echo "ERROR: $1" + exit 1 +} + +function warning { + echo "WARNING: $1" +} + +date +test -f ${HOME}/.bashrc && echo "Sourcing bashrc" && source ${HOME}/.bashrc +set -u + +echo "Checking" +uname -a +echo "JOBDIR=${JOBDIR:?"unknown JOBDIR"}" + +echo Using $(which python) +test -d "${JOBDIR}" || error "invalid macro" + + +# Enforce one thread +ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1 + +echo "Lauching macro" +date +cd ${JOBDIR} +cd .. +opengate_job_runner --backend "slurm" ${JOBDIR} || error "gate failed" +date + +echo "Success!!!" diff --git a/opengate/contrib/cluster/slurm/slurm.json b/opengate/contrib/cluster/slurm/slurm.json new file mode 100644 index 0000000000..10c7aafe2a --- /dev/null +++ b/opengate/contrib/cluster/slurm/slurm.json @@ -0,0 +1,10 @@ +{ + "backend": "slurm", + "backend_options": { + "submit_binary": "sbatch", + "submit_filename": "opengate_job.slurm", + "command_line_args": [ + "--export=ALL" + ] + } +} diff --git a/opengate/contrib/linacs/elektaversa.py b/opengate/contrib/linacs/elektaversa.py index df9b5409f7..f0c94fe209 100644 --- a/opengate/contrib/linacs/elektaversa.py +++ b/opengate/contrib/linacs/elektaversa.py @@ -258,7 +258,7 @@ def add_cone(sim, p): ff_base.color = yellow ff_base.material = "flattening_filter_material_stain_steel" - cons_1 = volumes.ConsVolume(name=f"{linac_name}_flattening_filter_cons_1") + cons_1 = volumes.ConsVolume(name=f"{linac_name}_flattening_filter_cons_1_bis") cons_1.rmin1 = 77 / 2 * mm cons_1.rmax1 = 93 / 2 * mm cons_1.rmin2 = 77 / 2 * mm diff --git a/opengate/geometry/solids.py b/opengate/geometry/solids.py index 6ac94be2b1..869b868999 100644 --- a/opengate/geometry/solids.py +++ b/opengate/geometry/solids.py @@ -132,10 +132,13 @@ def from_dictionary(self, d): "The provided dictionary does not contain an entry 'creator_volumes'." ) for i, cv in enumerate(creator_volumes): - try: - vol = self.volume_manager.volumes[cv["user_info"]["name"]] - except KeyError: + if self.volume_manager is None: vol = create_gate_object_from_dict(cv) + else: + try: + vol = self.volume_manager.volumes[cv["user_info"]["name"]] + except KeyError: + vol = create_gate_object_from_dict(cv) self.creator_volumes[i] = vol self.creator_volumes[i].from_dictionary(cv) diff --git a/opengate/geometry/volumes.py b/opengate/geometry/volumes.py index 537f0a355d..1a4ad41a17 100644 --- a/opengate/geometry/volumes.py +++ b/opengate/geometry/volumes.py @@ -775,6 +775,10 @@ class RepeatParametrisedVolume(VolumeBase): [1, 1, 1], {"doc": "FIXME"}, ), + "repeated_volume_name": ( + None, + {"doc": "Name of the repeated volume."}, + ), "offset": ( [0, 0, 0], {"doc": "3 component vector or list."}, @@ -783,21 +787,34 @@ class RepeatParametrisedVolume(VolumeBase): "start": ("auto", {"doc": "FIXME"}), } - def __init__(self, repeated_volume, *args, **kwargs): + def __init__(self, repeated_volume=None, *args, **kwargs): # FIXME: This should probably be a user_info self.repeated_volume = repeated_volume - if "name" not in kwargs: - kwargs["name"] = f"{repeated_volume.name}_param" - kwargs["mother"] = repeated_volume.mother + if repeated_volume is not None: + if "name" not in kwargs: + kwargs["name"] = f"{repeated_volume.name}_param" + kwargs["mother"] = repeated_volume.mother super().__init__(*args, **kwargs) - if repeated_volume.build_physical_volume is True: - repeated_volume.build_physical_volume = False + if repeated_volume is not None: + self.user_info["repeated_volume_name"] = repeated_volume.name + if repeated_volume.build_physical_volume is True: + repeated_volume.build_physical_volume = False self.g4_repeat_parametrisation = None def close(self): self.repeated_volume.close() super().close() + def resolve_references(self): + if self.repeated_volume is not None: + return + if self.user_info["repeated_volume_name"] is None: + return + self.repeated_volume = self.volume_manager.volumes[ + self.user_info["repeated_volume_name"] + ] + self.repeated_volume.build_physical_volume = False + def release_g4_references(self): super().release_g4_references() # FIXME: unsure. If not set to None, we get the following error: diff --git a/opengate/jobs.py b/opengate/jobs.py index 317a064733..455e5d9041 100644 --- a/opengate/jobs.py +++ b/opengate/jobs.py @@ -49,7 +49,7 @@ JOB_EXECUTION_ALLOWED_STATUSES = ("running", "completed", "failed", "skipped") HTCONDOR_SUBMIT_FILENAME = "htcondor_jobs.submit" SLURM_SUBMIT_FILENAME = "slurm_jobs.sh" -SLURM_JOB_FOLDERS_FILENAME = "slurm_job_folders.txt" +PBS_SUBMIT_FILENAME = "pbs_jobs.sh" def _prepare_package_root(path, overwrite=False): @@ -1547,69 +1547,6 @@ def _extract_htcondor_cluster_id(submit_stdout): return match.group(1) -def _write_slurm_job_folders_file(split_root_folder, job_folders, backend_options): - job_folders_path = Path(split_root_folder) / backend_options["job_folders_filename"] - job_folders_path.parent.mkdir(parents=True, exist_ok=True) - with open(job_folders_path, "w") as output_file: - for job_folder in job_folders: - output_file.write(f"{Path(job_folder).resolve()}\n") - return job_folders_path - - -def _example_render_slurm_submit_script_lines(job_folders_file_path, **kwargs): - """Example Slurm submit-script renderer for users and tests. - - The public Slurm backend expects the user to provide a renderer via - backend_options["submit_script_renderer"]. This helper remains as a - reference template and for internal tests. - """ - script_commands = { - "output": "opengate_job_runner.%A_%a.out", - "error": "opengate_job_runner.%A_%a.err", - } - script_commands.update(kwargs.get("script_commands", {})) - job_runner_command = kwargs.get("job_runner_command", "opengate_job_runner") - lines = ["#!/bin/sh"] - lines.extend([f"#SBATCH --{key}={value}" for key, value in script_commands.items()]) - lines.extend( - [ - "", - "set -eu", - f'JOB_FOLDERS_FILE="{job_folders_file_path}"', - 'JOB_FOLDER="$(sed -n "$((SLURM_ARRAY_TASK_ID + 1))p" "$JOB_FOLDERS_FILE")"', - 'cd "$JOB_FOLDER"', - f"exec {job_runner_command} . --backend slurm", - "", - ] - ) - return lines - - -def _write_slurm_submit_script( - split_root_folder, job_folders_file_path, backend_options -): - submit_file_path = Path(split_root_folder) / backend_options["submit_filename"] - submit_file_path.parent.mkdir(parents=True, exist_ok=True) - submit_script_lines = backend_options["submit_script_renderer"]( - job_folders_file_path, - **backend_options.get("submit_script_renderer_kwargs", {}), - ) - submit_file_content = "\n".join([str(line) for line in submit_script_lines]) - with open(submit_file_path, "w") as output_file: - output_file.write(submit_file_content) - os.chmod(submit_file_path, 0o755) - return submit_file_path - - -def _extract_slurm_job_id(submit_stdout): - match = re.search( - r"Submitted batch job\s+(\d+)", submit_stdout, flags=re.IGNORECASE - ) - if match is None: - return None - return match.group(1) - - def _submit_job_folders_to_htcondor( split_root_folder, job_folders, @@ -1674,61 +1611,73 @@ def _submit_job_folders_to_slurm( backend_options, skipped_completed_jobs, ): - job_folders_file_path = _write_slurm_job_folders_file( - split_root_folder, job_folders, backend_options - ) - submit_file_path = _write_slurm_submit_script( - split_root_folder, job_folders_file_path, backend_options - ) - command = [backend_options["submit_binary"]] - command.extend(backend_options.get("command_line_args", [])) - command.append(f"--array=0-{len(job_folders) - 1}") - command.append(str(submit_file_path)) - try: - completed_process = subprocess.run( - command, - cwd=split_root_folder, - capture_output=True, - text=True, - check=False, - ) - except FileNotFoundError as error: - raise GateJobsBackendError( - f"Slurm submission command not found: {backend_options['submit_binary']}." - ) from error + for job_folder in job_folders: + command = [backend_options["submit_binary"]] # -> sbatch + command.extend( + backend_options.get("command_line_args", []) + ) # + set the path to the job folder + command[-1] += ",JOBDIR=" + str(job_folder) + command.append(str(backend_options["submit_filename"])) # ->slurm filename + try: + print(command) + completed_process = subprocess.run( + command, + cwd=split_root_folder, + capture_output=True, + text=True, + check=False, + ) + except FileNotFoundError as error: + raise GateJobsBackendError( + f"Slurm submission command not found: {backend_options['submit_binary']}." + ) from error - if completed_process.returncode != 0: - raise GateJobsBackendError( - "Slurm submission failed.\n" - f"Command: {' '.join(command)}\n" - f"Return code: {completed_process.returncode}\n" - f"stdout:\n{completed_process.stdout}\n" - f"stderr:\n{completed_process.stderr}" - ) + if completed_process.returncode != 0: + raise GateJobsBackendError( + "Slurm submission failed.\n" + f"Command: {' '.join(command)}\n" + f"Return code: {completed_process.returncode}\n" + f"stdout:\n{completed_process.stdout}\n" + f"stderr:\n{completed_process.stderr}" + ) - status_data = _write_jobs_backend_status( - split_root_folder, - backend="slurm", - status="submitted", - submitted_jobs=len(job_folders), - skipped_completed_jobs=skipped_completed_jobs, - submitted_at=_now_isoformat(), - scheduler_job_id=_extract_slurm_job_id(completed_process.stdout), - submit_file_path=str(submit_file_path), - submit_command=command, - submission_stdout=completed_process.stdout, - submission_stderr=completed_process.stderr, - ) - return { - "submit_file_path": str(submit_file_path), - "job_folders_file_path": str(job_folders_file_path), - "submission_command": command, - "submission_stdout": completed_process.stdout, - "submission_stderr": completed_process.stderr, - "scheduler_job_id": status_data["scheduler_job_id"], - "backend_status_path": str(_get_jobs_backend_status_path(split_root_folder)), - } +def _submit_job_folders_to_pbs( + split_root_folder, + job_folders, + backend_options, + skipped_completed_jobs, +): + for job_folder in job_folders: + command = [backend_options["submit_binary"]] # -> qsub + command.extend( + backend_options.get("command_line_args", []) + ) # + set the path to the job folder + command.append("-v") + command.append("JOBDIR=" + str(job_folder)) + command.append(str(backend_options["submit_filename"])) # ->pbs filename + try: + print(command) + completed_process = subprocess.run( + command, + cwd=split_root_folder, + capture_output=True, + text=True, + check=False, + ) + except FileNotFoundError as error: + raise GateJobsBackendError( + f"PBS submission command not found: {backend_options['submit_binary']}." + ) from error + + if completed_process.returncode != 0: + raise GateJobsBackendError( + "qsub submission failed.\n" + f"Command: {' '.join(command)}\n" + f"Return code: {completed_process.returncode}\n" + f"stdout:\n{completed_process.stdout}\n" + f"stderr:\n{completed_process.stderr}" + ) def _run_jobs_campaign(job_folders, backend, backend_options): @@ -1814,52 +1763,45 @@ def _validate_jobs_backend_options(backend, backend_options): validated_options["submit_binary"] = str(validated_options["submit_binary"]) return validated_options - if backend == "slurm": + if backend == "slurm" or backend == "pbs": allowed_top_level_keys = { - "submit_script_renderer", - "submit_script_renderer_kwargs", "command_line_args", "submit_filename", - "job_folders_filename", "submit_binary", } unknown_keys = set(backend_options.keys()).difference(allowed_top_level_keys) if len(unknown_keys) > 0: - raise GateJobsBackendError( - f"The slurm backend received unknown option groups: {sorted(unknown_keys)}." - ) + if backend == "slurm": + raise GateJobsBackendError( + f"The slurm backend received unknown option groups: {sorted(unknown_keys)}." + ) + elif backend == "pbs": + raise GateJobsBackendError( + f"The pbs backend received unknown option groups: {sorted(unknown_keys)}." + ) validated_options = dict(backend_options) - validated_options.setdefault("submit_script_renderer_kwargs", {}) validated_options.setdefault("command_line_args", []) - validated_options.setdefault("submit_filename", SLURM_SUBMIT_FILENAME) - validated_options.setdefault("job_folders_filename", SLURM_JOB_FOLDERS_FILENAME) + if backend == "slurm": + validated_options.setdefault("submit_filename", SLURM_SUBMIT_FILENAME) + elif backend == "pbs": + validated_options.setdefault("submit_filename", PBS_SUBMIT_FILENAME) validated_options.setdefault("submit_binary", "sbatch") - if "submit_script_renderer" not in validated_options: - raise GateJobsBackendError( - "The slurm backend requires a submit_script_renderer callable." - ) - if not callable(validated_options["submit_script_renderer"]): - raise GateJobsBackendError( - "The slurm backend requires submit_script_renderer to be callable." - ) - if not isinstance(validated_options["submit_script_renderer_kwargs"], dict): - raise GateJobsBackendError( - "The slurm backend requires submit_script_renderer_kwargs to be a dictionary." - ) if not isinstance(validated_options["command_line_args"], (list, tuple)): - raise GateJobsBackendError( - "The slurm backend requires command_line_args to be a list or tuple." - ) + if backend == "slurm": + raise GateJobsBackendError( + "The slurm backend requires command_line_args to be a list or tuple." + ) + elif backend == "pbs": + raise GateJobsBackendError( + "The pbs backend requires command_line_args to be a list or tuple." + ) validated_options["command_line_args"] = [ str(argument) for argument in validated_options["command_line_args"] ] validated_options["submit_filename"] = str(validated_options["submit_filename"]) - validated_options["job_folders_filename"] = str( - validated_options["job_folders_filename"] - ) validated_options["submit_binary"] = str(validated_options["submit_binary"]) return validated_options @@ -1977,7 +1919,27 @@ def jobs_run( raise GateJobsBackendError( "detach=False is only supported for local jobs backends." ) - submission_summary = _submit_job_folders_to_slurm( + _submit_job_folders_to_slurm( + split_root_folder, + selected_job_folders, + backend_options, + len(skipped_completed_jobs), + ) + return { + "backend": backend, + "manifest_path": str(manifest_path), + "campaign_dir": str(split_root_folder), + "submitted_jobs": len(selected_job_folders), + "skipped_completed_jobs": len(skipped_completed_jobs), + "campaign_process_pid": None, + } + + if backend == "pbs": + if detach is False: + raise GateJobsBackendError( + "detach=False is only supported for local jobs backends." + ) + _submit_job_folders_to_pbs( split_root_folder, selected_job_folders, backend_options, @@ -1990,7 +1952,6 @@ def jobs_run( "submitted_jobs": len(selected_job_folders), "skipped_completed_jobs": len(skipped_completed_jobs), "campaign_process_pid": None, - **submission_summary, } if backend in ("local_sequential", "local_pool"): @@ -3447,10 +3408,13 @@ def jobs_status(campaign_dir): if not folder_exists: job_status = "missing_folder" + warning(f"Job {folder_name}: missing job folder at '{job_folder}'") elif not metadata_exists: job_status = "missing_metadata" + warning(f"Job {folder_name}: missing metadata file at '{metadata_file}'") elif missing_input_files: job_status = "missing_input_file" + warning(f"Job {folder_name}: missing input files: {missing_input_files}") elif sim_exists: job_status = "ready" else: diff --git a/opengate/managers.py b/opengate/managers.py index 3232ba197c..baa693f7f1 100644 --- a/opengate/managers.py +++ b/opengate/managers.py @@ -1555,6 +1555,9 @@ def from_dictionary(self, d): field = field_types[field_type](name=v["user_info"]["name"]) field.from_dictionary(v) self.fields[field.name] = field + # Before, create the parrallel world volumes, so that they are available for the volumes to be created + for k in d["parallel_world_volumes"]: + self.add_parallel_world(k) # First create all volumes for k, v in d["volumes"].items(): # the world volume is always created in __init__ @@ -1564,6 +1567,10 @@ def from_dictionary(self, d): # to a volume in the volumes dictionary is satisfied for k, v in d["volumes"].items(): self.volumes[k].from_dictionary(v) + # and check that any reference to a volume in the volumes dictionary is satisfied (for RepeatParametrisedVolume) + for volume in self.volumes.values(): + if hasattr(volume, "resolve_references"): + volume.resolve_references() @property def world_volume(self): diff --git a/opengate/serialization.py b/opengate/serialization.py index 0a5a34bf25..8d6cd3dcb4 100644 --- a/opengate/serialization.py +++ b/opengate/serialization.py @@ -18,6 +18,8 @@ def default(self, obj): "__dtype__": str(obj.dtype), "__shape__": obj.shape, } + elif isinstance(obj, np.generic): + return obj.item() elif isinstance(obj, Path): path_parts = PurePath(obj).parts if len(path_parts) == 0: diff --git a/opengate/sources/phspsources.py b/opengate/sources/phspsources.py index 5ad2db0554..5fb786de69 100644 --- a/opengate/sources/phspsources.py +++ b/opengate/sources/phspsources.py @@ -81,7 +81,7 @@ def read_phsp_and_keys(self): def get_entry_start(self, entry_start): if not g4.IsMultithreadedApplication(): if not isinstance(entry_start, numbers.Number): - fatal("entry_start must be a simple number is mono-thread mode") + fatal("entry_start must be a simple number in mono-thread mode") n = int(entry_start % self.num_entries) if entry_start > self.num_entries: warning( diff --git a/opengate/tests/src/actors/test036_adder_depth_helpers.py b/opengate/tests/src/actors/test036_adder_depth_helpers.py index 63ea374444..a54ce57458 100644 --- a/opengate/tests/src/actors/test036_adder_depth_helpers.py +++ b/opengate/tests/src/actors/test036_adder_depth_helpers.py @@ -143,8 +143,13 @@ def create_simulation(geom, paths, version): sec = gate.g4_units.second sim.running_verbose_level = gate.logger.RUN - # sim.run_timing_intervals = [[0, 0.33 * sec], [0.33 * sec, 0.66 * sec], [0.66 * sec, 1 * sec]] - sim.run_timing_intervals = [[0, 1 * sec]] + sim.run_timing_intervals = [ + [0, 0.33 * sec], + [0.33 * sec, 0.66 * sec], + [0.66 * sec, 1 * sec], + [1 * sec, 1.5 * sec], + ] + # sim.run_timing_intervals = [[0, 1 * sec]] # print cuts print(sim.physics_manager.dump_production_cuts()) diff --git a/opengate/tests/src/actors/test036_proj_param_1.py b/opengate/tests/src/actors/test036_proj_param_1.py index 90988ed448..aa7893893d 100755 --- a/opengate/tests/src/actors/test036_proj_param_1.py +++ b/opengate/tests/src/actors/test036_proj_param_1.py @@ -33,9 +33,15 @@ source.activity = source.activity / 1000 # add a proj actor + keV = g4_units.keV + channels = [ + {"name": f"peak222", "min": 210 * keV, "max": 230 * keV}, + {"name": f"peak333", "min": 310 * keV, "max": 340 * keV}, + ] proj = sim.add_actor("DigitizerProjectionActor", "proj") proj.attached_to = "crystal" fname = "proj1.mha" + proj.input_digi_collections = [x["name"] for x in channels] proj.output_filename = fname.replace(".mha", "-1.mha") proj.size = [128, 128] proj.spacing = [5 * mm, 5 * mm] diff --git a/opengate/tests/src/actors/test041_dose_actor_mt_cp_images_n_threads.py b/opengate/tests/src/actors/test041_dose_actor_mt_cp_images_n_threads.py index e6d5a32f8d..faeb9ecd47 100755 --- a/opengate/tests/src/actors/test041_dose_actor_mt_cp_images_n_threads.py +++ b/opengate/tests/src/actors/test041_dose_actor_mt_cp_images_n_threads.py @@ -20,7 +20,6 @@ def run_sim(N_events: int, N_threads: int, N_voxels: int, paths): sim.output_dir = paths.output Ntotal = N_events - N_per_trhead = int(np.round(Ntotal)) # units cm = gate.g4_units.cm @@ -58,7 +57,7 @@ def run_sim(N_events: int, N_threads: int, N_voxels: int, paths): source.position.translation = [0, 0, 0] source.direction.type = "momentum" source.direction.momentum = [-1, 0, 0] - source.number_of_primaries = N_per_trhead + source.number_of_primaries = Ntotal # define actors: dose_size = [N_voxels, 1, 1] diff --git a/opengate/tests/src/actors/test041_dose_actor_mt_standard_error_of_mean_WIP.py b/opengate/tests/src/actors/test041_dose_actor_mt_standard_error_of_mean_WIP.py index 7f7520cb11..79b2e3e784 100755 --- a/opengate/tests/src/actors/test041_dose_actor_mt_standard_error_of_mean_WIP.py +++ b/opengate/tests/src/actors/test041_dose_actor_mt_standard_error_of_mean_WIP.py @@ -22,7 +22,6 @@ def run_sim(n_thr, c4_ref=None, paths=None): sim.number_of_threads = n_thr sim.output_dir = paths.output Ntotal = 10000 * (30 / n_thr) ** 2 - N_per_trhead = Ntotal # units cm = gate.g4_units.cm @@ -59,7 +58,7 @@ def run_sim(n_thr, c4_ref=None, paths=None): source.position.translation = [0, 0, 0] source.direction.type = "momentum" source.direction.momentum = [-1, 0, 0] - source.number_of_primaries = N_per_trhead + source.number_of_primaries = Ntotal dose_size = [100, 1, 1] dose_spacing = [0.5 * mm, 100.0 * mm, 100.0 * mm] diff --git a/opengate/tests/src/misc/test111_jobs_run_slurm.py b/opengate/tests/src/misc/test111_jobs_run_slurm.py index 805a15c66b..be14631e97 100644 --- a/opengate/tests/src/misc/test111_jobs_run_slurm.py +++ b/opengate/tests/src/misc/test111_jobs_run_slurm.py @@ -32,7 +32,7 @@ def render_test_slurm_submit_script(job_folders_file_path, **kwargs): if __name__ == "__main__": - paths = utility.get_default_test_paths(__file__, output_folder="test113_slurm") + paths = utility.get_default_test_paths(__file__, output_folder="test111_slurm") shutil.rmtree(paths.output, ignore_errors=True) is_ok = True @@ -84,14 +84,14 @@ def render_test_slurm_submit_script(job_folders_file_path, **kwargs): "import pathlib, sys; print(pathlib.Path(sys.argv[-1]).name)", ], "submit_filename": "gate_slurm.sh", - "job_folders_filename": "gate_slurm_job_folders.txt", - "submit_script_renderer": render_test_slurm_submit_script, - "submit_script_renderer_kwargs": { - "job_runner_command": "opengate_job_runner", - "partition": "cpu", - "cpus_per_task": "4", - "mem": "8G", - }, + # "job_folders_filename": "gate_slurm_job_folders.txt", + # "submit_script_renderer": render_test_slurm_submit_script, + # "submit_script_renderer_kwargs": { + # "job_runner_command": "opengate_job_runner", + # "partition": "cpu", + # "cpus_per_task": "4", + # "mem": "8G", + # }, }, ) diff --git a/opengate_job.slurm b/opengate_job.slurm new file mode 100644 index 0000000000..d5f9ae2612 --- /dev/null +++ b/opengate_job.slurm @@ -0,0 +1,41 @@ +#!/bin/bash -x +#SBATCH --mem=20000 # Memory in MB per default +#SBATCH --time=10:00:00 +#SBATCH --ntasks=1 +#SBATCH --output=Seq%j.out # nom du fichier de sortie +#SBATCH --error=Seq%j.out # nom du fichier d'erreur (ici en commun avec la sortie) + + + +function error { + echo "ERROR: $1" + exit 1 +} + +function warning { + echo "WARNING: $1" +} + +date +test -f ${HOME}/.bashrc && echo "Sourcing bashrc" && source ${HOME}/.bashrc +set -u + +echo "Checking" +uname -a +echo "JOBDIR=${JOBDIR:?"unknown JOBDIR"}" + +echo Using $(which python) +test -d "${JOBDIR}" || error "invalid macro" + + +# Enforce one thread +ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1 + +echo "Lauching macro" +date +cd ${JOBDIR} +cd .. +opengate_job_runner --backend "slurm" ${JOBDIR} || error "gate failed" +date + +echo "Success!!!" diff --git a/slurm.json b/slurm.json new file mode 100644 index 0000000000..ce4a7d5e28 --- /dev/null +++ b/slurm.json @@ -0,0 +1,10 @@ +{ + "backend": "slurm", + "backend_options": { + "submit_binary": "sbatch", + "submit_filename": "/sps/creatis/tbaudier/ane/opengate_veriton/opengate_job.slurm", + "command_line_args": [ + "--export=ALL" + ] + } +}