Generalize simulation production grid generation - less dependency on backend

docs/changes/2190.feature.md

@@ -0,0 +1,2 @@
+Major improvements to simulation production configuration preparation with clear separation introduced between observation grid generation and backend submission code.
+Allow NSHOW to follow a pre-defined power law for simulation production grid definition.

docs/source/api-reference/production_configuration.md

@@ -43,15 +43,16 @@ the calculation of the number of events to be simulated given a pre-determined m
    :members:
 ```
 
-(generate-production-grid)=
+(corsika-limits-lookup)=
 
-## generate_production_grid
+## corsika_limits_lookup
 
 ```{eval-rst}
-.. automodule:: production_configuration.generate_production_grid
+.. automodule:: production_configuration.corsika_limits_lookup
    :members:
 ```
 
+
 (interpolation-handler)=
 
 ## interpolation_handler
@@ -61,6 +62,35 @@ the calculation of the number of events to be simulated given a pre-determined m
    :members:
 ```
 
+(job-grid-io)=
+
+## job_grid_io
+
+```{eval-rst}
+.. automodule:: production_configuration.job_grid_io
+   :members:
+```
+
+(observation-grid)=
+
+## observation_grid
+
+```{eval-rst}
+.. automodule:: production_configuration.observation_grid
+   :members:
+```
+
+(simulation-jobs)=
+
+## simulation_jobs
+
+```{eval-rst}
+.. automodule:: production_configuration.simulation_jobs
+   :members:
+```
+
+(merge-corsika-limits)=
+
 ## merge_corsika_limits
 
 ```{eval-rst}

docs/source/api-reference/visualization.md

@@ -131,8 +131,6 @@ the visualization module.
    :members:
 ```
 
-(merge-corsika-limits)=
-
 
 (plot-corsika-limits)=
 

src/simtools/applications/plot_production_grid.py

@@ -17,7 +17,7 @@
     Model version used to read the site reference coordinates.
 observation_time (str, optional)
     Observation time in UTC ISO format used for Alt/Az <-> RA/Dec transformations.
-    If omitted, the application uses ``metadata.observing_time_utc`` from the
+    If omitted, the application uses ``metadata.time_of_observation_utc`` from the
     grid file when available.
 plot_ra_dec_tracks (flag, optional)
     If provided, plot RA/Dec guide tracks on top of the sky projection. When native

src/simtools/applications/production_generate_grid.py

@@ -1,46 +1,30 @@
 #!/usr/bin/python3
 
 r"""
-Generate a grid of simulation points using flexible axes definitions.
-
-This application generates a grid of simulation points based on the provided axes
-definitions. The axes definitions (range, binning) are specified in a file.
-The viewcone, radius and energy thresholds are provided as a lookup table and
-are interpolated based on the generated grid points. The generated grid points are
-filtered based on the specified telescope IDs and the limits from the lookup table.
-The generated grid points are saved to a file.
-It can also convert the generated points to RA/Dec coordinates if the selected
-coordinate system is 'ra_dec'.
-
-For ``coordinate_system='ra_dec'``, the underlying grid generation supports
-declination-line sampling with hour-angle spacing and applies zenith-angle
-filtering based on the configured zenith range in that mode.
-When explicit ``ra`` / ``dec`` axes are provided, all YAML-defined grid points are
-preserved in the serialized output.
+Generate simulation job grid for production configurations.
+
+This application expands simulation job rows and writes them to disk as ECSV files.
+It supports both:
+
+- explicit cartesian job-grid configuration (primary, zenith, energy range, etc.), and
+- axes-based production-grid configuration with optional ``ra_dec`` coordinate handling
+  and lookup-table interpolation.
 
 Command line arguments
 ----------------------
-axes (str, required)
-    Path to a YAML or JSON file defining the axes of the grid.
-coordinate_system (str, optional, default='zenith_azimuth')
-    The coordinate system for the grid generation ('zenith_azimuth' or 'ra_dec').
-    In ``ra_dec`` mode, observing location/time are used to build sky directions and
-    derive corresponding zenith/azimuth values for interpolation (ICRS/J2000 frame).
-observing_time (str, optional)
+axis (repeatable)
+    Compact axis definition in the form
+    ``--axis <name> <min> <unit> <max> <unit> <binning> [scaling]``.
+    Example: ``--axis azimuth 310 deg 20 deg 3 linear``.
+time_of_observation (str, optional)
     Time of the observation in UTC (format: 'YYYY-MM-DD HH:MM:SS').
-    Used only in ``ra_dec`` mode (for coordinate transforms and sidereal-time
-    sampling). Ignored in ``zenith_azimuth`` mode.
-lookup_table (str, required)
+    Used only if RA/Dec axes are provided (for coordinate transforms and sidereal-time
+    sampling). Ignored otherwise.
+corsika_limits (str, optional)
     Path to the lookup table for simulation limits. The table should contain
-    varying azimuth and/or zenith angles.
-telescope_ids (list of str, optional)
-    List of telescope names used to filter the lookup table rows
-    (e.g. ``MSTN-15``).
-simtel_file (str, optional)
-    Path to a sim_telarray file used only when lookup-table telescope selections
-    are stored as numeric telescope IDs.
-output_file (str, optional, default='grid_output.ecsv')
-    Output file for the generated grid points (default: 'grid_output.ecsv').
+    varying azimuth and/or zenith angles for the selected array layout.
+output_file (str, optional, default='job_grid.ecsv')
+    Output file for the generated executable job grid.
 
 
 Example
@@ -50,56 +34,50 @@
 .. code-block:: console
 
         simtools-production-generate-grid --site North --model_version 6.0.2 \
-            --axes tests/resources/production_grid_generation_axes_definition.yml \
-            --coordinate_system zenith_azimuth \
-            --lookup_table tests/resources/corsika_simulation_limits/
-                merged_corsika_limits_for_test.ecsv \
-            --telescope_ids MSTN-15
+            --array_layout_name alpha \
+            --axis azimuth 310 deg 20 deg 3 linear \
+            --axis zenith 30 deg 40 deg 2 linear \
+            --axis nsb 4 MHz 5 MHz 2 linear \
+            --axis offset 0 deg 10 deg 2 linear \
+            --corsika_limits tests/resources/corsika_simulation_limits/merged_corsika_limits.ecsv
 
 To generate an all-sky RA/Dec direction grid and serialize output in RA/Dec,
 execute:
 
 .. code-block:: console
 
         simtools-production-generate-grid --site North --model_version 6.0.2 \
-            --axes tests/resources/production_grid_generation_axes_definition_radec.yml \
-            --coordinate_system ra_dec --observing_time "2017-09-16 00:00:00" \
-            --lookup_table tests/resources/corsika_simulation_limits/
-                merged_corsika_limits_for_test.ecsv \
-            --telescope_ids MSTN-15
+            --array_layout_name alpha \
+            --axis ra 0 deg 360 deg 36 linear \
+            --axis dec -90 deg 90 deg 18 linear \
+            --axis nsb 4 MHz 4 MHz 1 linear \
+            --axis offset 0 deg 10 deg 2 linear \
+            --time_of_observation "2017-09-16 00:00:00" \
+            --corsika_limits tests/resources/corsika_simulation_limits/merged_corsika_limits.ecsv
 """
 
-from pathlib import Path
-
-from astropy.coordinates import EarthLocation
-from astropy.time import Time
-
 from simtools.application_control import build_application
-from simtools.io.ascii_handler import collect_data_from_file
-from simtools.model.site_model import SiteModel
-from simtools.production_configuration.generate_production_grid import GridGeneration
+from simtools.production_configuration.job_grid_io import serialize_job_grid
+from simtools.production_configuration.simulation_jobs import (
+    build_job_grid_metadata,
+    build_simulation_jobs,
+)
 
 
 def _add_arguments(parser):
     """Register application-specific command line arguments."""
     parser.add_argument(
-        "--axes",
-        type=str,
-        required=True,
-        help="Path to a file defining the grid axes.",
-    )
-    parser.add_argument(
-        "--coordinate_system",
-        type=str,
-        default="zenith_azimuth",
+        "--axis",
+        action="append",
+        nargs="+",
+        required=False,
         help=(
-            "Coordinate system ('zenith_azimuth' or 'ra_dec'). "
-            "In 'ra_dec' mode, sky directions are generated using observing"
-            " location/time and converted to zenith/azimuth for interpolation."
+            "Compact axis definition: --axis <name> <min> <unit> <max> <unit> <binning> "
+            "[scaling]. May be repeated."
         ),
     )
     parser.add_argument(
-        "--observing_time",
+        "--time_of_observation",
         type=str,
         required=False,
         help=(
@@ -109,103 +87,63 @@ def _add_arguments(parser):
     parser.add_argument(
         "--output_file",
         type=str,
-        default="grid_output.ecsv",
-        help="Output file for the generated grid points (default: 'grid_output.ecsv').",
+        default="job_grid.ecsv",
+        help="Output file for the generated executable job grid.",
     )
     parser.add_argument(
-        "--telescope_ids",
+        "--corsika_limits",
         type=str,
-        nargs="*",
-        default=None,
-        help=(
-            "List of telescope names used to get specific limits from the lookup table "
-            "(e.g. MSTN-15)."
-        ),
-    )
-    parser.add_argument(
-        "--lookup_table",
-        type=str,
-        required=True,
+        required=False,
         help="Path to the lookup table for simulation limits. "
         "Table required with varying azimuth and or zenith angle. ",
     )
     parser.add_argument(
-        "--simtel_file",
-        type=str,
+        "--number_of_runs",
+        help="Number of runs to be simulated.",
+        type=int,
+        required=False,
+        default=1,
+    )
+    parser.add_argument(
+        "--nshow_power_index",
+        help=(
+            "Power-law index used to scale the baseline nshow with the geometric-mean energy "
+            "of each energy_range entry."
+        ),
+        type=float,
         required=False,
+        default=None,
+    )
+    parser.add_argument(
+        "--nshow_reference_energy",
         help=(
-            "Optional path to a sim_telarray file used to map sim_telarray telescope IDs "
-            "to telescope names when lookup-table selections are numeric IDs."
+            "Reference energy for nshow power-law scaling (for example: '100 GeV'). "
+            "Required together with --nshow_power_index."
         ),
+        type=str,
+        required=False,
+        default=None,
     )
 
 
-def load_axes(file_path: str):
-    """
-    Load axes definitions from a YAML or JSON file.
-
-    Parameters
-    ----------
-    file_path : str
-        Path to the axes YAML or JSON file.
-
-    Returns
-    -------
-    list[dict]
-        List of axes definitions with Quantity values.
-    """
-    if not Path(file_path).exists():
-        raise FileNotFoundError(f"Axes file {file_path} not found.")
-
-    return collect_data_from_file(file_path)
-
-
 def main():
     """See CLI description."""
     app_context = build_application(
         initialization_kwargs={
             "db_config": True,
-            "simulation_model": ["version", "site", "model_version"],
+            "preserve_by_version_keys": ["array_layout_name"],
+            "simulation_model": ["site", "layout", "telescope", "model_version"],
+            "simulation_configuration": {"software": None, "corsika_configuration": ["all"]},
         },
     )
 
-    output_filepath = app_context.io_handler.get_output_file(app_context.args["output_file"])
-
-    axes = load_axes(app_context.args["axes"])
-    site_model = SiteModel(
-        model_version=app_context.args["model_version"],
-        site=app_context.args["site"],
+    job_rows = build_simulation_jobs(app_context.args)
+    serialize_job_grid(
+        job_rows=job_rows,
+        output_file=app_context.io_handler.get_output_file(app_context.args["output_file"]),
+        metadata=build_job_grid_metadata(app_context.args),
     )
 
-    ref_lat = site_model.get_parameter_value_with_unit("reference_point_latitude")
-    ref_long = site_model.get_parameter_value_with_unit("reference_point_longitude")
-    altitude = site_model.get_parameter_value_with_unit("reference_point_altitude")
-
-    observing_location = EarthLocation(lat=ref_lat, lon=ref_long, height=altitude)
-
-    coordinate_system = app_context.args["coordinate_system"]
-    observing_time = None
-    if app_context.args.get("observing_time"):
-        observing_time = Time(app_context.args["observing_time"], scale="utc")
-    elif coordinate_system == "ra_dec":
-        observing_time = Time.now()
-
-    grid_gen = GridGeneration(
-        axes=axes,
-        coordinate_system=coordinate_system,
-        observing_location=observing_location,
-        observing_time=observing_time,
-        lookup_table=app_context.args["lookup_table"],
-        telescope_ids=app_context.args["telescope_ids"],
-        simtel_file=app_context.args.get("simtel_file"),
-    )
-
-    grid_points = grid_gen.generate_grid()
-
-    if coordinate_system == "ra_dec":
-        grid_points = grid_gen.convert_coordinates(grid_points)
-    grid_gen.serialize_grid_points(grid_points, output_file=output_filepath)
-
 
 if __name__ == "__main__":
     main()