diff --git a/docs/changes/2190.feature.md b/docs/changes/2190.feature.md new file mode 100644 index 0000000000..0bbcd13eb7 --- /dev/null +++ b/docs/changes/2190.feature.md @@ -0,0 +1,2 @@ +Major improvements to simulation production configuration preparation with clear separation introduced between observation grid generation and backend submission code. +Allow NSHOW to follow a pre-defined power law for simulation production grid definition. diff --git a/docs/source/api-reference/production_configuration.md b/docs/source/api-reference/production_configuration.md index 7f1744570f..b8cf346fb2 100644 --- a/docs/source/api-reference/production_configuration.md +++ b/docs/source/api-reference/production_configuration.md @@ -43,15 +43,16 @@ the calculation of the number of events to be simulated given a pre-determined m :members: ``` -(generate-production-grid)= +(corsika-limits-lookup)= -## generate_production_grid +## corsika_limits_lookup ```{eval-rst} -.. automodule:: production_configuration.generate_production_grid +.. automodule:: production_configuration.corsika_limits_lookup :members: ``` + (interpolation-handler)= ## interpolation_handler @@ -61,6 +62,35 @@ the calculation of the number of events to be simulated given a pre-determined m :members: ``` +(job-grid-io)= + +## job_grid_io + +```{eval-rst} +.. automodule:: production_configuration.job_grid_io + :members: +``` + +(observation-grid)= + +## observation_grid + +```{eval-rst} +.. automodule:: production_configuration.observation_grid + :members: +``` + +(simulation-jobs)= + +## simulation_jobs + +```{eval-rst} +.. automodule:: production_configuration.simulation_jobs + :members: +``` + +(merge-corsika-limits)= + ## merge_corsika_limits ```{eval-rst} diff --git a/docs/source/api-reference/visualization.md b/docs/source/api-reference/visualization.md index a4cb14f40b..6d66c8139b 100644 --- a/docs/source/api-reference/visualization.md +++ b/docs/source/api-reference/visualization.md @@ -131,8 +131,6 @@ the visualization module. :members: ``` -(merge-corsika-limits)= - (plot-corsika-limits)= diff --git a/src/simtools/applications/plot_production_grid.py b/src/simtools/applications/plot_production_grid.py index f1500dcae6..73165b4f65 100644 --- a/src/simtools/applications/plot_production_grid.py +++ b/src/simtools/applications/plot_production_grid.py @@ -17,7 +17,7 @@ Model version used to read the site reference coordinates. observation_time (str, optional) Observation time in UTC ISO format used for Alt/Az <-> RA/Dec transformations. - If omitted, the application uses ``metadata.observing_time_utc`` from the + If omitted, the application uses ``metadata.time_of_observation_utc`` from the grid file when available. plot_ra_dec_tracks (flag, optional) If provided, plot RA/Dec guide tracks on top of the sky projection. When native diff --git a/src/simtools/applications/production_generate_grid.py b/src/simtools/applications/production_generate_grid.py index 7331d3f36f..65684e01cd 100644 --- a/src/simtools/applications/production_generate_grid.py +++ b/src/simtools/applications/production_generate_grid.py @@ -1,46 +1,30 @@ #!/usr/bin/python3 r""" -Generate a grid of simulation points using flexible axes definitions. - -This application generates a grid of simulation points based on the provided axes -definitions. The axes definitions (range, binning) are specified in a file. -The viewcone, radius and energy thresholds are provided as a lookup table and -are interpolated based on the generated grid points. The generated grid points are -filtered based on the specified telescope IDs and the limits from the lookup table. -The generated grid points are saved to a file. -It can also convert the generated points to RA/Dec coordinates if the selected -coordinate system is 'ra_dec'. - -For ``coordinate_system='ra_dec'``, the underlying grid generation supports -declination-line sampling with hour-angle spacing and applies zenith-angle -filtering based on the configured zenith range in that mode. -When explicit ``ra`` / ``dec`` axes are provided, all YAML-defined grid points are -preserved in the serialized output. +Generate simulation job grid for production configurations. + +This application expands simulation job rows and writes them to disk as ECSV files. +It supports both: + +- explicit cartesian job-grid configuration (primary, zenith, energy range, etc.), and +- axes-based production-grid configuration with optional ``ra_dec`` coordinate handling + and lookup-table interpolation. Command line arguments ---------------------- -axes (str, required) - Path to a YAML or JSON file defining the axes of the grid. -coordinate_system (str, optional, default='zenith_azimuth') - The coordinate system for the grid generation ('zenith_azimuth' or 'ra_dec'). - In ``ra_dec`` mode, observing location/time are used to build sky directions and - derive corresponding zenith/azimuth values for interpolation (ICRS/J2000 frame). -observing_time (str, optional) +axis (repeatable) + Compact axis definition in the form + ``--axis [scaling]``. + Example: ``--axis azimuth 310 deg 20 deg 3 linear``. +time_of_observation (str, optional) Time of the observation in UTC (format: 'YYYY-MM-DD HH:MM:SS'). - Used only in ``ra_dec`` mode (for coordinate transforms and sidereal-time - sampling). Ignored in ``zenith_azimuth`` mode. -lookup_table (str, required) + Used only if RA/Dec axes are provided (for coordinate transforms and sidereal-time + sampling). Ignored otherwise. +corsika_limits (str, optional) Path to the lookup table for simulation limits. The table should contain - varying azimuth and/or zenith angles. -telescope_ids (list of str, optional) - List of telescope names used to filter the lookup table rows - (e.g. ``MSTN-15``). -simtel_file (str, optional) - Path to a sim_telarray file used only when lookup-table telescope selections - are stored as numeric telescope IDs. -output_file (str, optional, default='grid_output.ecsv') - Output file for the generated grid points (default: 'grid_output.ecsv'). + varying azimuth and/or zenith angles for the selected array layout. +output_file (str, optional, default='job_grid.ecsv') + Output file for the generated executable job grid. Example @@ -50,11 +34,12 @@ .. code-block:: console simtools-production-generate-grid --site North --model_version 6.0.2 \ - --axes tests/resources/production_grid_generation_axes_definition.yml \ - --coordinate_system zenith_azimuth \ - --lookup_table tests/resources/corsika_simulation_limits/ - merged_corsika_limits_for_test.ecsv \ - --telescope_ids MSTN-15 + --array_layout_name alpha \ + --axis azimuth 310 deg 20 deg 3 linear \ + --axis zenith 30 deg 40 deg 2 linear \ + --axis nsb 4 MHz 5 MHz 2 linear \ + --axis offset 0 deg 10 deg 2 linear \ + --corsika_limits tests/resources/corsika_simulation_limits/merged_corsika_limits.ecsv To generate an all-sky RA/Dec direction grid and serialize output in RA/Dec, execute: @@ -62,44 +47,37 @@ .. code-block:: console simtools-production-generate-grid --site North --model_version 6.0.2 \ - --axes tests/resources/production_grid_generation_axes_definition_radec.yml \ - --coordinate_system ra_dec --observing_time "2017-09-16 00:00:00" \ - --lookup_table tests/resources/corsika_simulation_limits/ - merged_corsika_limits_for_test.ecsv \ - --telescope_ids MSTN-15 + --array_layout_name alpha \ + --axis ra 0 deg 360 deg 36 linear \ + --axis dec -90 deg 90 deg 18 linear \ + --axis nsb 4 MHz 4 MHz 1 linear \ + --axis offset 0 deg 10 deg 2 linear \ + --time_of_observation "2017-09-16 00:00:00" \ + --corsika_limits tests/resources/corsika_simulation_limits/merged_corsika_limits.ecsv """ -from pathlib import Path - -from astropy.coordinates import EarthLocation -from astropy.time import Time - from simtools.application_control import build_application -from simtools.io.ascii_handler import collect_data_from_file -from simtools.model.site_model import SiteModel -from simtools.production_configuration.generate_production_grid import GridGeneration +from simtools.production_configuration.job_grid_io import serialize_job_grid +from simtools.production_configuration.simulation_jobs import ( + build_job_grid_metadata, + build_simulation_jobs, +) def _add_arguments(parser): """Register application-specific command line arguments.""" parser.add_argument( - "--axes", - type=str, - required=True, - help="Path to a file defining the grid axes.", - ) - parser.add_argument( - "--coordinate_system", - type=str, - default="zenith_azimuth", + "--axis", + action="append", + nargs="+", + required=False, help=( - "Coordinate system ('zenith_azimuth' or 'ra_dec'). " - "In 'ra_dec' mode, sky directions are generated using observing" - " location/time and converted to zenith/azimuth for interpolation." + "Compact axis definition: --axis " + "[scaling]. May be repeated." ), ) parser.add_argument( - "--observing_time", + "--time_of_observation", type=str, required=False, help=( @@ -109,103 +87,63 @@ def _add_arguments(parser): parser.add_argument( "--output_file", type=str, - default="grid_output.ecsv", - help="Output file for the generated grid points (default: 'grid_output.ecsv').", + default="job_grid.ecsv", + help="Output file for the generated executable job grid.", ) parser.add_argument( - "--telescope_ids", + "--corsika_limits", type=str, - nargs="*", - default=None, - help=( - "List of telescope names used to get specific limits from the lookup table " - "(e.g. MSTN-15)." - ), - ) - parser.add_argument( - "--lookup_table", - type=str, - required=True, + required=False, help="Path to the lookup table for simulation limits. " "Table required with varying azimuth and or zenith angle. ", ) parser.add_argument( - "--simtel_file", - type=str, + "--number_of_runs", + help="Number of runs to be simulated.", + type=int, + required=False, + default=1, + ) + parser.add_argument( + "--nshow_power_index", + help=( + "Power-law index used to scale the baseline nshow with the geometric-mean energy " + "of each energy_range entry." + ), + type=float, required=False, + default=None, + ) + parser.add_argument( + "--nshow_reference_energy", help=( - "Optional path to a sim_telarray file used to map sim_telarray telescope IDs " - "to telescope names when lookup-table selections are numeric IDs." + "Reference energy for nshow power-law scaling (for example: '100 GeV'). " + "Required together with --nshow_power_index." ), + type=str, + required=False, + default=None, ) -def load_axes(file_path: str): - """ - Load axes definitions from a YAML or JSON file. - - Parameters - ---------- - file_path : str - Path to the axes YAML or JSON file. - - Returns - ------- - list[dict] - List of axes definitions with Quantity values. - """ - if not Path(file_path).exists(): - raise FileNotFoundError(f"Axes file {file_path} not found.") - - return collect_data_from_file(file_path) - - def main(): """See CLI description.""" app_context = build_application( initialization_kwargs={ "db_config": True, - "simulation_model": ["version", "site", "model_version"], + "preserve_by_version_keys": ["array_layout_name"], + "simulation_model": ["site", "layout", "telescope", "model_version"], + "simulation_configuration": {"software": None, "corsika_configuration": ["all"]}, }, ) - output_filepath = app_context.io_handler.get_output_file(app_context.args["output_file"]) - - axes = load_axes(app_context.args["axes"]) - site_model = SiteModel( - model_version=app_context.args["model_version"], - site=app_context.args["site"], + job_rows = build_simulation_jobs(app_context.args) + serialize_job_grid( + job_rows=job_rows, + output_file=app_context.io_handler.get_output_file(app_context.args["output_file"]), + metadata=build_job_grid_metadata(app_context.args), ) - ref_lat = site_model.get_parameter_value_with_unit("reference_point_latitude") - ref_long = site_model.get_parameter_value_with_unit("reference_point_longitude") - altitude = site_model.get_parameter_value_with_unit("reference_point_altitude") - - observing_location = EarthLocation(lat=ref_lat, lon=ref_long, height=altitude) - - coordinate_system = app_context.args["coordinate_system"] - observing_time = None - if app_context.args.get("observing_time"): - observing_time = Time(app_context.args["observing_time"], scale="utc") - elif coordinate_system == "ra_dec": - observing_time = Time.now() - - grid_gen = GridGeneration( - axes=axes, - coordinate_system=coordinate_system, - observing_location=observing_location, - observing_time=observing_time, - lookup_table=app_context.args["lookup_table"], - telescope_ids=app_context.args["telescope_ids"], - simtel_file=app_context.args.get("simtel_file"), - ) - - grid_points = grid_gen.generate_grid() - - if coordinate_system == "ra_dec": - grid_points = grid_gen.convert_coordinates(grid_points) - grid_gen.serialize_grid_points(grid_points, output_file=output_filepath) - if __name__ == "__main__": main() diff --git a/src/simtools/applications/simulate_prod_htcondor_generator.py b/src/simtools/applications/simulate_prod_htcondor_generator.py index e4dc97f76b..1a7ff0372e 100644 --- a/src/simtools/applications/simulate_prod_htcondor_generator.py +++ b/src/simtools/applications/simulate_prod_htcondor_generator.py @@ -3,7 +3,7 @@ r""" Generate a run script and submit file for HT Condor job submission of a simulation production. -This tool generates HTCondor submission files for one or more simulation production grids and +This tool generates HTCondor submission files from a pre-generated executable job grid and supports either a single Apptainer image or a label-to-image mapping for multi-image submissions. For each image label, it writes a dedicated ``simulate_prod.submit.