From 6a288599f013d5775b4fc1242db1d32423841fec Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 12:24:33 +0200 Subject: [PATCH 01/44] Improved grid job submission --- .../api-reference/production_configuration.md | 11 + .../simulate_prod_htcondor_generator.py | 14 + .../htcondor_script_generator.py | 199 +------------- .../job_spec_builder.py | 258 ++++++++++++++++++ .../test_simulate_prod_htcondor_generator.py | 12 + .../test_htcondor_script_generator.py | 87 +----- .../test_job_spec_builder.py | 174 ++++++++++++ 7 files changed, 482 insertions(+), 273 deletions(-) create mode 100644 src/simtools/production_configuration/job_spec_builder.py create mode 100644 tests/unit_tests/production_configuration/test_job_spec_builder.py diff --git a/docs/source/api-reference/production_configuration.md b/docs/source/api-reference/production_configuration.md index a1f7aef886..acb6b17b7f 100644 --- a/docs/source/api-reference/production_configuration.md +++ b/docs/source/api-reference/production_configuration.md @@ -52,6 +52,17 @@ the calculation of the number of events to be simulated given a pre-determined m :members: ``` + +(job-spec-builder)= + +## job_spec_builder + +```{eval-rst} +.. automodule:: production_configuration.job_spec_builder + :members: +``` + + (interpolation-handler)= ## interpolation_handler diff --git a/src/simtools/applications/simulate_prod_htcondor_generator.py b/src/simtools/applications/simulate_prod_htcondor_generator.py index e4dc97f76b..e2441fef35 100644 --- a/src/simtools/applications/simulate_prod_htcondor_generator.py +++ b/src/simtools/applications/simulate_prod_htcondor_generator.py @@ -51,6 +51,10 @@ Base directory for simulation output packages passed through as ``pack_for_grid_register``. priority (int, optional) Job priority (default: 1). +nshow_power_index (float, optional) + Power-law index used to scale the baseline ``nshow`` for each ``energy_range`` entry. + The scaling uses the geometric-mean energy of each entry and the first configured + ``energy_range`` entry as the reference energy. (all other command line arguments are identical to those of :ref:`simulate_prod`). @@ -97,6 +101,16 @@ def _add_arguments(parser): required=False, default=None, ) + parser.add_argument( + "--nshow_power_index", + help=( + "Power-law index used to scale the baseline nshow with the geometric-mean energy " + "of each energy_range entry." + ), + type=float, + required=False, + default=None, + ) def main(): diff --git a/src/simtools/job_execution/htcondor_script_generator.py b/src/simtools/job_execution/htcondor_script_generator.py index 705ee7868d..164a0ea74c 100644 --- a/src/simtools/job_execution/htcondor_script_generator.py +++ b/src/simtools/job_execution/htcondor_script_generator.py @@ -8,33 +8,19 @@ """ -import ast -import itertools import logging import re from pathlib import Path import astropy.units as u -import simtools.version as simtools_version -from simtools.configuration import defaults +from simtools.production_configuration.job_spec_builder import ( + build_job_specs, + resolve_array_layout_name, +) _logger = logging.getLogger(__name__) -_GRID_AXES = [ - "primary", - "azimuth_angle", - "zenith_angle", - "model_version", - "corsika_le_interaction", - "corsika_he_interaction", -] - -_GRID_AXIS_DEFAULTS = { - "corsika_le_interaction": defaults.CORSIKA_LE_INTERACTION, - "corsika_he_interaction": defaults.CORSIKA_HE_INTERACTION, -} - _PARAMS_FIELDS = [ "apptainer_label", "primary", @@ -56,45 +42,6 @@ ] -def _normalize_to_list(value): - """Normalize scalar values to lists of length one.""" - if isinstance(value, list): - return value - if isinstance(value, tuple): - return list(value) - return [value] - - -def _normalize_grid_axes(args_dict): - """Return normalized grid axes for cartesian product expansion.""" - return { - axis: ( - _normalize_to_list(args_dict[axis]) - if axis in args_dict and args_dict[axis] is not None - else [_GRID_AXIS_DEFAULTS[axis]] - if axis in _GRID_AXIS_DEFAULTS - else [None] - ) - for axis in _GRID_AXES - } - - -def _normalize_energy_ranges(energy_range): - """Normalize energy range argument to a list of (e_min, e_max) pairs.""" - if isinstance(energy_range, tuple) and len(energy_range) == 2: - return [energy_range] - - if isinstance(energy_range, list): - if len(energy_range) == 2 and all(hasattr(item, "to") for item in energy_range): - return [(energy_range[0], energy_range[1])] - if all(isinstance(item, (list, tuple)) and len(item) == 2 for item in energy_range): - return [tuple(item) for item in energy_range] - - raise ValueError( - "energy_range must be one pair (e_min, e_max) or a list of (e_min, e_max) pairs." - ) - - def _resolve_apptainer_images(apptainer_image_arg): """ Resolve and validate apptainer image configuration. @@ -179,141 +126,11 @@ def _sanitize_label_for_params(label): return re.sub(r"\s+", "_", str(label).strip()) -def _resolve_array_layout_name(array_layout_name, model_version): - """Resolve array layout configuration for a specific model version.""" - if isinstance(array_layout_name, list) and len(array_layout_name) == 1: - array_layout_name = array_layout_name[0] - - # Configurator/_fill_config stringifies dict values when rebuilding argparse arguments. - if isinstance(array_layout_name, str) and array_layout_name.strip().startswith("{"): - try: - parsed_layout = ast.literal_eval(array_layout_name) - if isinstance(parsed_layout, dict): - array_layout_name = parsed_layout - except (SyntaxError, ValueError): - return array_layout_name - - if not isinstance(array_layout_name, dict) or list(array_layout_name) != ["by_version"]: - return array_layout_name - - resolved = simtools_version.resolve_by_version( - {"array_layout_name": array_layout_name}, model_version - ) - return resolved["array_layout_name"] - - -def _get_energy_range_for_zenith_angle(zenith_angle, energy_range_pair, corsika_limits): - """ - Return a zenith-dependent energy range pair or None to skip the simulation step. - - Dummy function - to be implemented. - """ - _ = (zenith_angle, corsika_limits) - return energy_range_pair - - -def _get_core_scatter_max_for_zenith_angle(zenith_angle, core_scatter, corsika_limits): - """Return zenith-dependent max core-scatter value. - - Dummy function - to be implemented. - """ - _ = (zenith_angle, corsika_limits) - return core_scatter[1] - - -def _get_nshow_for_energy_range_and_zenith_angle( - energy_range_pair, zenith_angle, nshow, corsika_limits -): - """Return nshow that may depend on energy range and zenith angle. - - Dummy function - to be implemented. - """ - _ = (energy_range_pair, zenith_angle, corsika_limits) - return nshow - - -def _build_job_specs(args_dict, image_labels): - """Build backend-agnostic job specs from comparison and production grids.""" - grid_axes = _normalize_grid_axes(args_dict) - energy_ranges = _normalize_energy_ranges(args_dict["energy_range"]) - base_pack_dir = args_dict.get("simulation_output") or "simtools-output" - corsika_limits = args_dict.get("corsika_limits") - core_scatter = args_dict["core_scatter"] - nshow = args_dict["nshow"] - - combinations = list( - itertools.product( - grid_axes["primary"], - grid_axes["azimuth_angle"], - grid_axes["zenith_angle"], - grid_axes["model_version"], - grid_axes["corsika_le_interaction"], - grid_axes["corsika_he_interaction"], - energy_ranges, - ) - ) - - number_of_runs = args_dict.get("number_of_runs", 1) - run_number = int(args_dict.get("run_number") or 1) - - job_specs = [] - for label in image_labels: - params_label = _sanitize_label_for_params(label) - row_index = 0 - for ( - primary, - azimuth, - zenith, - model_version, - corsika_le, - corsika_he, - energy_range_pair, - ) in combinations: - selected_energy_range_pair = energy_range_pair - selected_core_scatter_max = core_scatter[1] - selected_nshow = nshow - - if corsika_limits is not None: - selected_energy_range_pair = _get_energy_range_for_zenith_angle( - zenith, energy_range_pair, corsika_limits - ) - if selected_energy_range_pair is None: - continue - selected_core_scatter_max = _get_core_scatter_max_for_zenith_angle( - zenith, core_scatter, corsika_limits - ) - selected_nshow = _get_nshow_for_energy_range_and_zenith_angle( - selected_energy_range_pair, zenith, nshow, corsika_limits - ) - - for _ in range(number_of_runs): - job_specs.append( - { - "apptainer_label": str(label), - "primary": primary, - "azimuth_angle": azimuth, - "zenith_angle": zenith, - "model_version": model_version, - "array_layout_name": args_dict.get("array_layout_name"), - "corsika_le_interaction": corsika_le, - "corsika_he_interaction": corsika_he, - "energy_min": selected_energy_range_pair[0], - "energy_max": selected_energy_range_pair[1], - "core_scatter_max": selected_core_scatter_max, - "nshow": selected_nshow, - "pack_for_grid_register": f"{base_pack_dir}/{params_label}", - "run_number": run_number + row_index, - } - ) - row_index += 1 - return job_specs - - def _group_job_specs_by_label(job_specs): """Group job specs by apptainer image label.""" grouped = {} for job_spec in job_specs: - label = job_spec["apptainer_label"] + label = job_spec["image_label"] grouped.setdefault(label, []).append(job_spec) return grouped @@ -322,7 +139,7 @@ def _write_params_file(params_file_path, label_job_specs): """Write parameter file consumed by HTCondor queue-from syntax.""" with open(params_file_path, "w", encoding="utf-8") as params_file_handle: for job_spec in label_job_specs: - array_layout_name = _resolve_array_layout_name( + array_layout_name = resolve_array_layout_name( job_spec["array_layout_name"], job_spec["model_version"] ) @@ -337,7 +154,7 @@ def _write_params_file(params_file_path, label_job_specs): ) row = [ - _format_param_value(job_spec["apptainer_label"], "apptainer_label"), + _format_param_value(job_spec["image_label"], "apptainer_label"), _format_param_value(job_spec["primary"], "primary"), _format_param_value(job_spec["azimuth_angle"], "azimuth_angle"), _format_param_value(job_spec["zenith_angle"], "zenith_angle"), @@ -368,7 +185,7 @@ def generate_submission_script(args_dict): Arguments dictionary. """ apptainer_images = _resolve_apptainer_images(args_dict["apptainer_image"]) - job_specs = _build_job_specs(args_dict, list(apptainer_images.keys())) + job_specs = build_job_specs(args_dict, list(apptainer_images.keys())) grouped_job_specs = _group_job_specs_by_label(job_specs) work_dir = Path(args_dict["output_path"]) diff --git a/src/simtools/production_configuration/job_spec_builder.py b/src/simtools/production_configuration/job_spec_builder.py new file mode 100644 index 0000000000..8c7e48118e --- /dev/null +++ b/src/simtools/production_configuration/job_spec_builder.py @@ -0,0 +1,258 @@ +"""Build backend-agnostic job specifications for production submissions.""" + +import ast +import itertools + +import numpy as np +from astropy import units as u + +import simtools.version as simtools_version +from simtools.configuration import defaults + +_GRID_AXES = [ + "primary", + "azimuth_angle", + "zenith_angle", + "model_version", + "corsika_le_interaction", + "corsika_he_interaction", +] + +_GRID_AXIS_DEFAULTS = { + "corsika_le_interaction": defaults.CORSIKA_LE_INTERACTION, + "corsika_he_interaction": defaults.CORSIKA_HE_INTERACTION, +} + + +def normalize_to_list(value): + """Normalize scalar values to lists of length one.""" + if isinstance(value, list): + return value + if isinstance(value, tuple): + return list(value) + return [value] + + +def normalize_grid_axes(args_dict): + """Return normalized grid axes for cartesian product expansion.""" + return { + axis: ( + normalize_to_list(args_dict[axis]) + if axis in args_dict and args_dict[axis] is not None + else [_GRID_AXIS_DEFAULTS[axis]] + if axis in _GRID_AXIS_DEFAULTS + else [None] + ) + for axis in _GRID_AXES + } + + +def normalize_energy_ranges(energy_range): + """Normalize energy range argument to a list of ``(e_min, e_max)`` pairs.""" + if isinstance(energy_range, tuple) and len(energy_range) == 2: + return [energy_range] + + if isinstance(energy_range, list): + if len(energy_range) == 2 and all(hasattr(item, "to") for item in energy_range): + return [(energy_range[0], energy_range[1])] + if all(isinstance(item, (list, tuple)) and len(item) == 2 for item in energy_range): + return [tuple(item) for item in energy_range] + + raise ValueError( + "energy_range must be one pair (e_min, e_max) or a list of (e_min, e_max) pairs." + ) + + +def resolve_array_layout_name(array_layout_name, model_version): + """Resolve array layout configuration for a specific model version.""" + if isinstance(array_layout_name, list) and len(array_layout_name) == 1: + array_layout_name = array_layout_name[0] + + if isinstance(array_layout_name, str) and array_layout_name.strip().startswith("{"): + try: + parsed_layout = ast.literal_eval(array_layout_name) + if isinstance(parsed_layout, dict): + array_layout_name = parsed_layout + except (SyntaxError, ValueError): + return array_layout_name + + if not isinstance(array_layout_name, dict) or list(array_layout_name) != ["by_version"]: + return array_layout_name + + resolved = simtools_version.resolve_by_version( + {"array_layout_name": array_layout_name}, model_version + ) + return resolved["array_layout_name"] + + +def get_energy_range_for_zenith_angle(zenith_angle, energy_range_pair, corsika_limits): + """Return a zenith-dependent energy range pair or None to skip the simulation step.""" + _ = (zenith_angle, corsika_limits) + return energy_range_pair + + +def get_core_scatter_max_for_zenith_angle(zenith_angle, core_scatter, corsika_limits): + """Return zenith-dependent max core-scatter value.""" + _ = (zenith_angle, corsika_limits) + return core_scatter[1] + + +def calculate_log_energy_midpoint(energy_range_pair): + """Return the geometric-mean energy for an energy range pair.""" + energy_min, energy_max = energy_range_pair + + if not isinstance(energy_min, u.Quantity) or not isinstance(energy_max, u.Quantity): + raise TypeError("energy_range_pair must contain astropy Quantity values.") + + energy_min_tev = energy_min.to(u.TeV) + energy_max_tev = energy_max.to(u.TeV) + + if energy_min_tev <= 0 * u.TeV or energy_max_tev <= 0 * u.TeV: + raise ValueError("Energy range values must be strictly positive.") + + mean_log_energy = np.mean( + [ + np.log10(energy_min_tev.value), + np.log10(energy_max_tev.value), + ] + ) + return 10**mean_log_energy * u.TeV + + +def get_nshow_scaling_reference_energy(energy_ranges): + """Return the default reference energy for nshow scaling.""" + if len(energy_ranges) == 0: + raise ValueError("At least one energy range is required to derive a reference energy.") + + return calculate_log_energy_midpoint(energy_ranges[0]) + + +def calculate_scaled_nshow( + energy_range_pair, + baseline_nshow, + nshow_power_index=None, + reference_energy=None, +): + """Return an energy-dependent nshow value.""" + if baseline_nshow < 1: + raise ValueError("baseline_nshow must be a positive integer.") + + if nshow_power_index is None: + return baseline_nshow + + if reference_energy is None: + raise ValueError("reference_energy is required when nshow_power_index is configured.") + + midpoint_energy = calculate_log_energy_midpoint(energy_range_pair) + scaling_factor = (midpoint_energy / reference_energy.to(midpoint_energy.unit)).to_value( + u.dimensionless_unscaled + ) ** nshow_power_index + scaled_nshow = int(np.ceil(baseline_nshow * scaling_factor)) + + if scaled_nshow < 1: + raise ValueError("Scaled nshow must be at least 1.") + + return scaled_nshow + + +def get_nshow_for_energy_range_and_zenith_angle( + energy_range_pair, + zenith_angle, + nshow, + corsika_limits, + nshow_power_index=None, + reference_energy=None, +): + """Return nshow that may depend on energy range and zenith angle.""" + _ = (zenith_angle, corsika_limits) + return calculate_scaled_nshow( + energy_range_pair, + nshow, + nshow_power_index=nshow_power_index, + reference_energy=reference_energy, + ) + + +def build_job_specs(args_dict, image_labels): + """Build backend-agnostic job specs from comparison and production grids.""" + grid_axes = normalize_grid_axes(args_dict) + energy_ranges = normalize_energy_ranges(args_dict["energy_range"]) + base_pack_dir = args_dict.get("simulation_output") or "simtools-output" + corsika_limits = args_dict.get("corsika_limits") + core_scatter = args_dict["core_scatter"] + nshow = args_dict["nshow"] + nshow_power_index = args_dict.get("nshow_power_index") + reference_energy = ( + get_nshow_scaling_reference_energy(energy_ranges) if nshow_power_index is not None else None + ) + + combinations = list( + itertools.product( + grid_axes["primary"], + grid_axes["azimuth_angle"], + grid_axes["zenith_angle"], + grid_axes["model_version"], + grid_axes["corsika_le_interaction"], + grid_axes["corsika_he_interaction"], + energy_ranges, + ) + ) + + number_of_runs = args_dict.get("number_of_runs", 1) + run_number = int(args_dict.get("run_number") or 1) + + job_specs = [] + for label in image_labels: + row_index = 0 + for ( + primary, + azimuth, + zenith, + model_version, + corsika_le, + corsika_he, + energy_range_pair, + ) in combinations: + selected_energy_range_pair = energy_range_pair + selected_core_scatter_max = core_scatter[1] + + if corsika_limits is not None: + selected_energy_range_pair = get_energy_range_for_zenith_angle( + zenith, energy_range_pair, corsika_limits + ) + if selected_energy_range_pair is None: + continue + selected_core_scatter_max = get_core_scatter_max_for_zenith_angle( + zenith, core_scatter, corsika_limits + ) + + selected_nshow = get_nshow_for_energy_range_and_zenith_angle( + selected_energy_range_pair, + zenith, + nshow, + corsika_limits, + nshow_power_index=nshow_power_index, + reference_energy=reference_energy, + ) + + for _ in range(number_of_runs): + job_specs.append( + { + "image_label": str(label), + "primary": primary, + "azimuth_angle": azimuth, + "zenith_angle": zenith, + "model_version": model_version, + "array_layout_name": args_dict.get("array_layout_name"), + "corsika_le_interaction": corsika_le, + "corsika_he_interaction": corsika_he, + "energy_min": selected_energy_range_pair[0], + "energy_max": selected_energy_range_pair[1], + "core_scatter_max": selected_core_scatter_max, + "nshow": selected_nshow, + "pack_for_grid_register": f"{base_pack_dir}/{label!s}", + "run_number": run_number + row_index, + } + ) + row_index += 1 + return job_specs diff --git a/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py b/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py index bfa76c9bdb..0affa35728 100644 --- a/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py +++ b/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py @@ -1,8 +1,11 @@ #!/usr/bin/python3 +import argparse from types import SimpleNamespace from unittest.mock import patch +import pytest + import simtools.applications.simulate_prod_htcondor_generator as app @@ -24,3 +27,12 @@ def test_main_uses_standard_build_application( "simulation_configuration": {"software": None, "corsika_configuration": ["all"]}, } mock_generate_submission_script.assert_called_once_with({"output_path": "htcondor_submit"}) + + +def test_add_arguments_registers_nshow_power_index(): + parser = argparse.ArgumentParser() + + app._add_arguments(parser) + args = parser.parse_args(["--number_of_runs", "1", "--nshow_power_index", "-0.5"]) + + assert args.nshow_power_index == pytest.approx(-0.5) diff --git a/tests/unit_tests/job_execution/test_htcondor_script_generator.py b/tests/unit_tests/job_execution/test_htcondor_script_generator.py index ffef52b577..278b876513 100644 --- a/tests/unit_tests/job_execution/test_htcondor_script_generator.py +++ b/tests/unit_tests/job_execution/test_htcondor_script_generator.py @@ -5,7 +5,6 @@ import pytest from simtools.job_execution.htcondor_script_generator import ( - _build_job_specs, _format_quantity, _get_submit_file, _get_submit_script, @@ -14,6 +13,7 @@ _write_params_file, generate_submission_script, ) +from simtools.production_configuration.job_spec_builder import build_job_specs @pytest.fixture @@ -229,54 +229,6 @@ def test_sanitize_label_for_filename(): assert _sanitize_label_for_filename(42) == "42" -def test_build_job_specs_expands_model_version_list(args_dict): - args_dict["model_version"] = ["6.3.0", "7.0.0"] - - job_specs = _build_job_specs(args_dict, ["7.0.0"]) - model_versions = {job_spec["model_version"] for job_spec in job_specs} - - assert model_versions == {"6.3.0", "7.0.0"} - assert len(job_specs) == 2 * args_dict["number_of_runs"] - - -def test_build_job_specs_expands_energy_range_list_of_pairs(args_dict): - args_dict["number_of_runs"] = 1 - args_dict["energy_range"] = [ - (30 * u.GeV, 30 * u.GeV), - (300 * u.GeV, 300 * u.GeV), - ] - - job_specs = _build_job_specs(args_dict, ["7.0.0"]) - energy_pairs = {(job_spec["energy_min"], job_spec["energy_max"]) for job_spec in job_specs} - - assert len(job_specs) == 2 - assert energy_pairs == { - (30 * u.GeV, 30 * u.GeV), - (300 * u.GeV, 300 * u.GeV), - } - - -def test_build_job_specs_uses_default_interaction_models_when_missing(args_dict): - args_dict.pop("corsika_le_interaction") - args_dict.pop("corsika_he_interaction") - - job_specs = _build_job_specs(args_dict, ["7.0.0"]) - - assert {job_spec["corsika_le_interaction"] for job_spec in job_specs} == {"urqmd"} - assert {job_spec["corsika_he_interaction"] for job_spec in job_specs} == {"epos"} - - -def test_build_job_specs_increments_run_number(args_dict): - args_dict["number_of_runs"] = 2 - args_dict["run_number"] = 10 - args_dict["model_version"] = ["6.3.0", "7.0.0"] - - job_specs = _build_job_specs(args_dict, ["7.0.0"]) - run_numbers = [job_spec["run_number"] for job_spec in job_specs] - - assert run_numbers == [10, 11, 12, 13] - - def test_write_params_file_resolves_array_layout_name_by_model_version( args_dict, tmp_test_directory ): @@ -290,7 +242,7 @@ def test_write_params_file_resolves_array_layout_name_by_model_version( } params_file_path = Path(tmp_test_directory) / "params.txt" - job_specs = _build_job_specs(args_dict, ["7.0.0"]) + job_specs = build_job_specs(args_dict, ["7.0.0"]) _write_params_file(params_file_path, job_specs) @@ -313,7 +265,7 @@ def test_write_params_file_resolves_stringified_by_version_layout(args_dict, tmp ) params_file_path = Path(tmp_test_directory) / "params.txt" - job_specs = _build_job_specs(args_dict, ["7.0.0"]) + job_specs = build_job_specs(args_dict, ["7.0.0"]) _write_params_file(params_file_path, job_specs) @@ -327,7 +279,7 @@ def test_write_params_file_keeps_energy_units(tmp_test_directory): params_file_path = Path(tmp_test_directory) / "params.txt" label_job_specs = [ { - "apptainer_label": "7.0.0", + "image_label": "7.0.0", "primary": "gamma", "azimuth_angle": 0 * u.deg, "zenith_angle": 20 * u.deg, @@ -361,7 +313,7 @@ def test_write_params_file_replaces_whitespace_in_apptainer_label(tmp_test_direc params_file_path = Path(tmp_test_directory) / "params.txt" label_job_specs = [ { - "apptainer_label": "grid label 7.0.0", + "image_label": "grid label 7.0.0", "primary": "gamma", "azimuth_angle": 0 * u.deg, "zenith_angle": 20 * u.deg, @@ -384,32 +336,3 @@ def test_write_params_file_replaces_whitespace_in_apptainer_label(tmp_test_direc "grid_label_7.0.0 gamma 0.0 20.0 30.0 GeV 10.0 TeV 200.0 m " "1000 7.0.0 CTAO-North-Alpha urqmd epos 10 simtools-output/grid_label_7.0.0\n" ) - - -@mock.patch( - "simtools.job_execution.htcondor_script_generator._get_energy_range_for_zenith_angle", - return_value=None, -) -def test_build_job_specs_skips_entries_when_energy_range_is_none(mock_energy_range, args_dict): - args_dict["corsika_limits"] = "limits.ecsv" - args_dict["number_of_runs"] = 1 - - job_specs = _build_job_specs(args_dict, ["7.0.0"]) - - assert job_specs == [] - mock_energy_range.assert_called_once() - - -@mock.patch( - "simtools.job_execution.htcondor_script_generator._get_nshow_for_energy_range_and_zenith_angle", - return_value=777, -) -def test_build_job_specs_uses_dummy_nshow_when_corsika_limits_set(mock_nshow, args_dict): - args_dict["corsika_limits"] = "limits.ecsv" - args_dict["number_of_runs"] = 1 - - job_specs = _build_job_specs(args_dict, ["7.0.0"]) - - assert len(job_specs) == 1 - assert job_specs[0]["nshow"] == 777 - mock_nshow.assert_called_once() diff --git a/tests/unit_tests/production_configuration/test_job_spec_builder.py b/tests/unit_tests/production_configuration/test_job_spec_builder.py new file mode 100644 index 0000000000..51a4da546b --- /dev/null +++ b/tests/unit_tests/production_configuration/test_job_spec_builder.py @@ -0,0 +1,174 @@ +from unittest import mock + +import astropy.units as u +import pytest + +from simtools.production_configuration.job_spec_builder import ( + build_job_specs, + calculate_log_energy_midpoint, + calculate_scaled_nshow, + get_nshow_scaling_reference_energy, + normalize_energy_ranges, + resolve_array_layout_name, +) + + +@pytest.fixture +def args_dict(): + return { + "azimuth_angle": 45 * u.deg, + "zenith_angle": 20 * u.deg, + "energy_range": [1 * u.GeV, 10 * u.GeV], + "core_scatter": [0, 100 * u.m], + "model_version": "v1.0", + "array_layout_name": "test_layout", + "primary": "gamma", + "nshow": 1000, + "run_number": 1, + "number_of_runs": 10, + "corsika_le_interaction": "urqmd", + "corsika_he_interaction": "epos", + } + + +def test_normalize_energy_ranges_expands_list_of_pairs(): + energy_ranges = normalize_energy_ranges( + [ + (30 * u.GeV, 30 * u.GeV), + (300 * u.GeV, 300 * u.GeV), + ] + ) + + assert energy_ranges == [ + (30 * u.GeV, 30 * u.GeV), + (300 * u.GeV, 300 * u.GeV), + ] + + +def test_calculate_log_energy_midpoint(): + midpoint_energy = calculate_log_energy_midpoint((1 * u.GeV, 100 * u.GeV)) + + assert midpoint_energy.to_value(u.GeV) == pytest.approx(10.0) + + +def test_get_nshow_scaling_reference_energy_uses_first_range(): + reference_energy = get_nshow_scaling_reference_energy( + [(10 * u.GeV, 10 * u.GeV), (100 * u.GeV, 100 * u.GeV)] + ) + + assert reference_energy.to_value(u.GeV) == pytest.approx(10.0) + + +def test_calculate_scaled_nshow_returns_baseline_without_power_index(): + scaled_nshow = calculate_scaled_nshow((10 * u.GeV, 100 * u.GeV), 50) + + assert scaled_nshow == 50 + + +def test_calculate_scaled_nshow_scales_against_reference_energy(): + scaled_nshow = calculate_scaled_nshow( + (100 * u.GeV, 100 * u.GeV), + 100, + nshow_power_index=-1.0, + reference_energy=10 * u.GeV, + ) + + assert scaled_nshow == 10 + + +def test_calculate_scaled_nshow_uses_ceil_for_fractional_result(): + scaled_nshow = calculate_scaled_nshow( + (100 * u.GeV, 100 * u.GeV), + 10, + nshow_power_index=-2.0, + reference_energy=10 * u.GeV, + ) + + assert scaled_nshow == 1 + + +def test_build_job_specs_expands_model_version_list(args_dict): + args_dict["model_version"] = ["6.3.0", "7.0.0"] + + job_specs = build_job_specs(args_dict, ["7.0.0"]) + model_versions = {job_spec["model_version"] for job_spec in job_specs} + + assert model_versions == {"6.3.0", "7.0.0"} + assert len(job_specs) == 2 * args_dict["number_of_runs"] + + +def test_build_job_specs_scales_nshow_by_energy_range(args_dict): + args_dict["number_of_runs"] = 1 + args_dict["nshow"] = 100 + args_dict["nshow_power_index"] = -1.0 + args_dict["energy_range"] = [ + (10 * u.GeV, 10 * u.GeV), + (100 * u.GeV, 100 * u.GeV), + ] + + job_specs = build_job_specs(args_dict, ["7.0.0"]) + + assert [job_spec["nshow"] for job_spec in job_specs] == [100, 10] + + +def test_build_job_specs_uses_default_interaction_models_when_missing(args_dict): + args_dict.pop("corsika_le_interaction") + args_dict.pop("corsika_he_interaction") + + job_specs = build_job_specs(args_dict, ["7.0.0"]) + + assert {job_spec["corsika_le_interaction"] for job_spec in job_specs} == {"urqmd"} + assert {job_spec["corsika_he_interaction"] for job_spec in job_specs} == {"epos"} + + +def test_build_job_specs_increments_run_number(args_dict): + args_dict["number_of_runs"] = 2 + args_dict["run_number"] = 10 + args_dict["model_version"] = ["6.3.0", "7.0.0"] + + job_specs = build_job_specs(args_dict, ["7.0.0"]) + run_numbers = [job_spec["run_number"] for job_spec in job_specs] + + assert run_numbers == [10, 11, 12, 13] + + +@mock.patch( + "simtools.production_configuration.job_spec_builder.get_energy_range_for_zenith_angle", + return_value=None, +) +def test_build_job_specs_skips_entries_when_energy_range_is_none(mock_energy_range, args_dict): + args_dict["corsika_limits"] = "limits.ecsv" + args_dict["number_of_runs"] = 1 + + job_specs = build_job_specs(args_dict, ["7.0.0"]) + + assert job_specs == [] + mock_energy_range.assert_called_once() + + +@mock.patch( + "simtools.production_configuration.job_spec_builder.get_nshow_for_energy_range_and_zenith_angle", + return_value=777, +) +def test_build_job_specs_uses_dummy_nshow_when_corsika_limits_set(mock_nshow, args_dict): + args_dict["corsika_limits"] = "limits.ecsv" + args_dict["number_of_runs"] = 1 + + job_specs = build_job_specs(args_dict, ["7.0.0"]) + + assert len(job_specs) == 1 + assert job_specs[0]["nshow"] == 777 + mock_nshow.assert_called_once() + + +def test_resolve_array_layout_name_resolves_stringified_by_version_layout(): + array_layout_name = str( + { + "by_version": { + "<7.0.0": "alpha", + ">=7.0.0": "CTAO-North-Alpha", + } + } + ) + + assert resolve_array_layout_name(array_layout_name, "7.0.0") == "CTAO-North-Alpha" From 10f73e07c6a95a0217f57d2d19f010684ab2fc99 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 12:37:01 +0200 Subject: [PATCH 02/44] unit tests --- .../test_htcondor_script_generator.py | 55 ++++++++++++++ .../test_job_spec_builder.py | 73 +++++++++++++++++++ 2 files changed, 128 insertions(+) diff --git a/tests/unit_tests/job_execution/test_htcondor_script_generator.py b/tests/unit_tests/job_execution/test_htcondor_script_generator.py index 278b876513..3b94547ddb 100644 --- a/tests/unit_tests/job_execution/test_htcondor_script_generator.py +++ b/tests/unit_tests/job_execution/test_htcondor_script_generator.py @@ -5,6 +5,7 @@ import pytest from simtools.job_execution.htcondor_script_generator import ( + _format_param_value, _format_quantity, _get_submit_file, _get_submit_script, @@ -66,6 +67,44 @@ def test_generate_submission_script(mock_is_file, mock_chmod, mock_open, mock_mk mock_chmod.assert_called_once_with(0o755) +@mock.patch("simtools.job_execution.htcondor_script_generator.Path.mkdir") +@mock.patch("simtools.job_execution.htcondor_script_generator.open", new_callable=mock.mock_open) +@mock.patch("simtools.job_execution.htcondor_script_generator.Path.chmod") +@mock.patch("simtools.job_execution.htcondor_script_generator.Path.is_file", return_value=True) +def test_generate_submission_script_writes_label_specific_files( + mock_is_file, mock_chmod, mock_open, mock_mkdir, args_dict +): + args_dict["output_path"] = "/test_output" + args_dict["htcondor_log_path"] = "/custom_logs" + args_dict["number_of_runs"] = 1 + args_dict["apptainer_image"] = { + "prod 7.0.0": "/path/to/prod.sif", + "beta": "/path/to/beta.sif", + } + + generate_submission_script(args_dict) + + work_dir = Path(args_dict["output_path"]) + + mock_is_file.assert_has_calls( + [mock.call(), mock.call()], + any_order=False, + ) + mock_open.assert_any_call( + work_dir / "simulate_prod.submit.prod_7.0.0.condor", "w", encoding="utf-8" + ) + mock_open.assert_any_call( + work_dir / "simulate_prod.submit.prod_7.0.0.params.txt", "w", encoding="utf-8" + ) + mock_open.assert_any_call(work_dir / "simulate_prod.submit.beta.condor", "w", encoding="utf-8") + mock_open.assert_any_call( + work_dir / "simulate_prod.submit.beta.params.txt", "w", encoding="utf-8" + ) + mock_open.assert_any_call(work_dir / "simulate_prod.submit.sh", "w", encoding="utf-8") + mock_mkdir.assert_any_call(parents=True, exist_ok=True) + mock_chmod.assert_called_once_with(0o755) + + @mock.patch("simtools.job_execution.htcondor_script_generator.Path.is_file", return_value=False) @mock.patch("simtools.job_execution.htcondor_script_generator.open", new_callable=mock.mock_open) def test_generate_submission_script_raises_for_missing_apptainer_image( @@ -191,6 +230,17 @@ def test_resolve_apptainer_images_empty_dict(): _resolve_apptainer_images({}) +def test_resolve_apptainer_images_raises_for_truthy_empty_mapping(): + class TruthyEmptyDict(dict): + def __bool__(self): + return True + + with pytest.raises( + ValueError, match="At least one apptainer image label/path must be configured" + ): + _resolve_apptainer_images(TruthyEmptyDict()) + + def test_resolve_apptainer_images_invalid_type(tmp_test_directory): with pytest.raises( TypeError, match="apptainer_image must be a string path or a label-to-path dictionary" @@ -223,6 +273,11 @@ def test_format_quantity_full_coverage(): assert unit is None +def test_format_param_value_raises_for_missing_required_value(): + with pytest.raises(ValueError, match="Missing required value for field 'primary'"): + _format_param_value(None, "primary") + + def test_sanitize_label_for_filename(): assert _sanitize_label_for_filename(" my label:7/0*0? ") == "my_label_7_0_0_" assert _sanitize_label_for_filename("v7.0.0-beta_1") == "v7.0.0-beta_1" diff --git a/tests/unit_tests/production_configuration/test_job_spec_builder.py b/tests/unit_tests/production_configuration/test_job_spec_builder.py index 51a4da546b..3603098e65 100644 --- a/tests/unit_tests/production_configuration/test_job_spec_builder.py +++ b/tests/unit_tests/production_configuration/test_job_spec_builder.py @@ -9,6 +9,8 @@ calculate_scaled_nshow, get_nshow_scaling_reference_energy, normalize_energy_ranges, + normalize_grid_axes, + normalize_to_list, resolve_array_layout_name, ) @@ -45,12 +47,48 @@ def test_normalize_energy_ranges_expands_list_of_pairs(): ] +def test_normalize_to_list_converts_tuple_values(): + assert normalize_to_list((1, 2)) == [1, 2] + + +def test_normalize_grid_axes_uses_defaults_and_none_for_missing_axes(): + grid_axes = normalize_grid_axes({"primary": "gamma"}) + + assert grid_axes["primary"] == ["gamma"] + assert grid_axes["azimuth_angle"] == [None] + assert grid_axes["zenith_angle"] == [None] + assert grid_axes["model_version"] == [None] + assert grid_axes["corsika_le_interaction"] == ["urqmd"] + assert grid_axes["corsika_he_interaction"] == ["epos"] + + +def test_normalize_energy_ranges_accepts_single_tuple_pair(): + energy_ranges = normalize_energy_ranges((30 * u.GeV, 300 * u.GeV)) + + assert energy_ranges == [(30 * u.GeV, 300 * u.GeV)] + + +def test_normalize_energy_ranges_raises_for_invalid_shape(): + with pytest.raises(ValueError, match="energy_range must be one pair"): + normalize_energy_ranges([30 * u.GeV, 300]) + + def test_calculate_log_energy_midpoint(): midpoint_energy = calculate_log_energy_midpoint((1 * u.GeV, 100 * u.GeV)) assert midpoint_energy.to_value(u.GeV) == pytest.approx(10.0) +def test_calculate_log_energy_midpoint_raises_for_non_quantity_values(): + with pytest.raises(TypeError, match="energy_range_pair must contain astropy Quantity values"): + calculate_log_energy_midpoint((1, 100 * u.GeV)) + + +def test_calculate_log_energy_midpoint_raises_for_non_positive_values(): + with pytest.raises(ValueError, match="Energy range values must be strictly positive"): + calculate_log_energy_midpoint((0 * u.GeV, 100 * u.GeV)) + + def test_get_nshow_scaling_reference_energy_uses_first_range(): reference_energy = get_nshow_scaling_reference_energy( [(10 * u.GeV, 10 * u.GeV), (100 * u.GeV, 100 * u.GeV)] @@ -59,6 +97,11 @@ def test_get_nshow_scaling_reference_energy_uses_first_range(): assert reference_energy.to_value(u.GeV) == pytest.approx(10.0) +def test_get_nshow_scaling_reference_energy_raises_for_empty_input(): + with pytest.raises(ValueError, match="At least one energy range is required"): + get_nshow_scaling_reference_energy([]) + + def test_calculate_scaled_nshow_returns_baseline_without_power_index(): scaled_nshow = calculate_scaled_nshow((10 * u.GeV, 100 * u.GeV), 50) @@ -87,6 +130,26 @@ def test_calculate_scaled_nshow_uses_ceil_for_fractional_result(): assert scaled_nshow == 1 +def test_calculate_scaled_nshow_raises_for_invalid_baseline(): + with pytest.raises(ValueError, match="baseline_nshow must be a positive integer"): + calculate_scaled_nshow((10 * u.GeV, 100 * u.GeV), 0) + + +def test_calculate_scaled_nshow_requires_reference_energy_when_scaled(): + with pytest.raises(ValueError, match="reference_energy is required"): + calculate_scaled_nshow((10 * u.GeV, 100 * u.GeV), 10, nshow_power_index=-1.0) + + +def test_calculate_scaled_nshow_raises_when_scaled_result_drops_below_one(): + with pytest.raises(ValueError, match="Scaled nshow must be at least 1"): + calculate_scaled_nshow( + (10 * u.GeV, 10 * u.GeV), + 1, + nshow_power_index=1.0, + reference_energy=-100 * u.GeV, + ) + + def test_build_job_specs_expands_model_version_list(args_dict): args_dict["model_version"] = ["6.3.0", "7.0.0"] @@ -172,3 +235,13 @@ def test_resolve_array_layout_name_resolves_stringified_by_version_layout(): ) assert resolve_array_layout_name(array_layout_name, "7.0.0") == "CTAO-North-Alpha" + + +def test_resolve_array_layout_name_unwraps_single_item_list(): + assert resolve_array_layout_name(["CTAO-North-Alpha"], "7.0.0") == "CTAO-North-Alpha" + + +def test_resolve_array_layout_name_keeps_invalid_stringified_dict(): + invalid_layout = "{not valid" + + assert resolve_array_layout_name(invalid_layout, "7.0.0") == invalid_layout From 84bea4a2b598c196d9a073a7d28db3f457ff8043 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 12:46:33 +0200 Subject: [PATCH 03/44] changelog --- docs/changes/2188.feature.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/2188.feature.md diff --git a/docs/changes/2188.feature.md b/docs/changes/2188.feature.md new file mode 100644 index 0000000000..d4521ed8a4 --- /dev/null +++ b/docs/changes/2188.feature.md @@ -0,0 +1 @@ +Allow NSHOW to follow a pre-defined power law (until now, the number of simulated showers was the same for all energies) for simulation production grid definition. From 53028442c69a02b40ff0287babdcdaf82573522d Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 13:13:51 +0200 Subject: [PATCH 04/44] reference energy --- .../simulate_prod_htcondor_generator.py | 15 +++- .../job_spec_builder.py | 75 ++++++++----------- .../test_simulate_prod_htcondor_generator.py | 14 +++- .../test_job_spec_builder.py | 26 +++---- 4 files changed, 67 insertions(+), 63 deletions(-) diff --git a/src/simtools/applications/simulate_prod_htcondor_generator.py b/src/simtools/applications/simulate_prod_htcondor_generator.py index e2441fef35..8933b79a97 100644 --- a/src/simtools/applications/simulate_prod_htcondor_generator.py +++ b/src/simtools/applications/simulate_prod_htcondor_generator.py @@ -53,8 +53,9 @@ Job priority (default: 1). nshow_power_index (float, optional) Power-law index used to scale the baseline ``nshow`` for each ``energy_range`` entry. - The scaling uses the geometric-mean energy of each entry and the first configured - ``energy_range`` entry as the reference energy. +nshow_reference_energy (quantity, optional) + Reference energy used for ``nshow`` power-law scaling (e.g. ``"100 GeV"``). + Required when ``nshow_power_index`` is configured. (all other command line arguments are identical to those of :ref:`simulate_prod`). @@ -111,6 +112,16 @@ def _add_arguments(parser): required=False, default=None, ) + parser.add_argument( + "--nshow_reference_energy", + help=( + "Reference energy for nshow power-law scaling (for example: '100 GeV'). " + "Required together with --nshow_power_index." + ), + type=str, + required=False, + default=None, + ) def main(): diff --git a/src/simtools/production_configuration/job_spec_builder.py b/src/simtools/production_configuration/job_spec_builder.py index 8c7e48118e..341ee913f1 100644 --- a/src/simtools/production_configuration/job_spec_builder.py +++ b/src/simtools/production_configuration/job_spec_builder.py @@ -119,14 +119,6 @@ def calculate_log_energy_midpoint(energy_range_pair): return 10**mean_log_energy * u.TeV -def get_nshow_scaling_reference_energy(energy_ranges): - """Return the default reference energy for nshow scaling.""" - if len(energy_ranges) == 0: - raise ValueError("At least one energy range is required to derive a reference energy.") - - return calculate_log_energy_midpoint(energy_ranges[0]) - - def calculate_scaled_nshow( energy_range_pair, baseline_nshow, @@ -155,22 +147,26 @@ def calculate_scaled_nshow( return scaled_nshow -def get_nshow_for_energy_range_and_zenith_angle( - energy_range_pair, - zenith_angle, - nshow, - corsika_limits, - nshow_power_index=None, - reference_energy=None, +def _select_energy_and_core_scatter_for_job( + zenith, energy_range_pair, core_scatter, corsika_limits ): - """Return nshow that may depend on energy range and zenith angle.""" - _ = (zenith_angle, corsika_limits) - return calculate_scaled_nshow( - energy_range_pair, - nshow, - nshow_power_index=nshow_power_index, - reference_energy=reference_energy, + """Return selected energy range and core scatter maximum for a job spec row.""" + selected_energy_range_pair = energy_range_pair + selected_core_scatter_max = core_scatter[1] + + if corsika_limits is None: + return selected_energy_range_pair, selected_core_scatter_max + + selected_energy_range_pair = get_energy_range_for_zenith_angle( + zenith, energy_range_pair, corsika_limits ) + if selected_energy_range_pair is None: + return None, None + + selected_core_scatter_max = get_core_scatter_max_for_zenith_angle( + zenith, core_scatter, corsika_limits + ) + return selected_energy_range_pair, selected_core_scatter_max def build_job_specs(args_dict, image_labels): @@ -182,9 +178,9 @@ def build_job_specs(args_dict, image_labels): core_scatter = args_dict["core_scatter"] nshow = args_dict["nshow"] nshow_power_index = args_dict.get("nshow_power_index") - reference_energy = ( - get_nshow_scaling_reference_energy(energy_ranges) if nshow_power_index is not None else None - ) + reference_energy = args_dict.get("nshow_reference_energy") + if nshow_power_index is not None and reference_energy is not None: + reference_energy = u.Quantity(reference_energy) combinations = list( itertools.product( @@ -213,26 +209,17 @@ def build_job_specs(args_dict, image_labels): corsika_he, energy_range_pair, ) in combinations: - selected_energy_range_pair = energy_range_pair - selected_core_scatter_max = core_scatter[1] - - if corsika_limits is not None: - selected_energy_range_pair = get_energy_range_for_zenith_angle( - zenith, energy_range_pair, corsika_limits - ) - if selected_energy_range_pair is None: - continue - selected_core_scatter_max = get_core_scatter_max_for_zenith_angle( - zenith, core_scatter, corsika_limits - ) - - selected_nshow = get_nshow_for_energy_range_and_zenith_angle( + ( selected_energy_range_pair, - zenith, - nshow, - corsika_limits, - nshow_power_index=nshow_power_index, - reference_energy=reference_energy, + selected_core_scatter_max, + ) = _select_energy_and_core_scatter_for_job( + zenith, energy_range_pair, core_scatter, corsika_limits + ) + if selected_energy_range_pair is None: + continue + + selected_nshow = calculate_scaled_nshow( + selected_energy_range_pair, nshow, nshow_power_index, reference_energy ) for _ in range(number_of_runs): diff --git a/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py b/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py index 0affa35728..38288feef8 100644 --- a/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py +++ b/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py @@ -29,10 +29,20 @@ def test_main_uses_standard_build_application( mock_generate_submission_script.assert_called_once_with({"output_path": "htcondor_submit"}) -def test_add_arguments_registers_nshow_power_index(): +def test_add_arguments_registers_nshow_scaling_arguments(): parser = argparse.ArgumentParser() app._add_arguments(parser) - args = parser.parse_args(["--number_of_runs", "1", "--nshow_power_index", "-0.5"]) + args = parser.parse_args( + [ + "--number_of_runs", + "1", + "--nshow_power_index", + "-0.5", + "--nshow_reference_energy", + "100 GeV", + ] + ) assert args.nshow_power_index == pytest.approx(-0.5) + assert args.nshow_reference_energy == "100 GeV" diff --git a/tests/unit_tests/production_configuration/test_job_spec_builder.py b/tests/unit_tests/production_configuration/test_job_spec_builder.py index 3603098e65..fdf6a84002 100644 --- a/tests/unit_tests/production_configuration/test_job_spec_builder.py +++ b/tests/unit_tests/production_configuration/test_job_spec_builder.py @@ -7,7 +7,6 @@ build_job_specs, calculate_log_energy_midpoint, calculate_scaled_nshow, - get_nshow_scaling_reference_energy, normalize_energy_ranges, normalize_grid_axes, normalize_to_list, @@ -89,19 +88,6 @@ def test_calculate_log_energy_midpoint_raises_for_non_positive_values(): calculate_log_energy_midpoint((0 * u.GeV, 100 * u.GeV)) -def test_get_nshow_scaling_reference_energy_uses_first_range(): - reference_energy = get_nshow_scaling_reference_energy( - [(10 * u.GeV, 10 * u.GeV), (100 * u.GeV, 100 * u.GeV)] - ) - - assert reference_energy.to_value(u.GeV) == pytest.approx(10.0) - - -def test_get_nshow_scaling_reference_energy_raises_for_empty_input(): - with pytest.raises(ValueError, match="At least one energy range is required"): - get_nshow_scaling_reference_energy([]) - - def test_calculate_scaled_nshow_returns_baseline_without_power_index(): scaled_nshow = calculate_scaled_nshow((10 * u.GeV, 100 * u.GeV), 50) @@ -164,6 +150,7 @@ def test_build_job_specs_scales_nshow_by_energy_range(args_dict): args_dict["number_of_runs"] = 1 args_dict["nshow"] = 100 args_dict["nshow_power_index"] = -1.0 + args_dict["nshow_reference_energy"] = 10 * u.GeV args_dict["energy_range"] = [ (10 * u.GeV, 10 * u.GeV), (100 * u.GeV, 100 * u.GeV), @@ -174,6 +161,15 @@ def test_build_job_specs_scales_nshow_by_energy_range(args_dict): assert [job_spec["nshow"] for job_spec in job_specs] == [100, 10] +def test_build_job_specs_requires_reference_energy_for_nshow_scaling(args_dict): + args_dict["number_of_runs"] = 1 + args_dict["nshow_power_index"] = -1.0 + args_dict["nshow_reference_energy"] = None + + with pytest.raises(ValueError, match="reference_energy is required"): + build_job_specs(args_dict, ["7.0.0"]) + + def test_build_job_specs_uses_default_interaction_models_when_missing(args_dict): args_dict.pop("corsika_le_interaction") args_dict.pop("corsika_he_interaction") @@ -210,7 +206,7 @@ def test_build_job_specs_skips_entries_when_energy_range_is_none(mock_energy_ran @mock.patch( - "simtools.production_configuration.job_spec_builder.get_nshow_for_energy_range_and_zenith_angle", + "simtools.production_configuration.job_spec_builder.calculate_scaled_nshow", return_value=777, ) def test_build_job_specs_uses_dummy_nshow_when_corsika_limits_set(mock_nshow, args_dict): From 4f5bc15f1adc04e65e76b0fffa25cb8a8adc8478 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 14:13:53 +0200 Subject: [PATCH 05/44] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- docs/changes/2188.feature.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changes/2188.feature.md b/docs/changes/2188.feature.md index d4521ed8a4..c33cd13fcf 100644 --- a/docs/changes/2188.feature.md +++ b/docs/changes/2188.feature.md @@ -1 +1 @@ -Allow NSHOW to follow a pre-defined power law (until now, the number of simulated showers was the same for all energies) for simulation production grid definition. +Allow NSHOW to follow a pre-defined power law for simulation production grid definition. From c070387ab82e82b680dc1909b2c8ea8c81ca45e7 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 14:14:50 +0200 Subject: [PATCH 06/44] copilot --- .../test_job_spec_builder.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/unit_tests/production_configuration/test_job_spec_builder.py b/tests/unit_tests/production_configuration/test_job_spec_builder.py index fdf6a84002..1e5321df13 100644 --- a/tests/unit_tests/production_configuration/test_job_spec_builder.py +++ b/tests/unit_tests/production_configuration/test_job_spec_builder.py @@ -2,6 +2,7 @@ import astropy.units as u import pytest +from astropy.tests.helper import assert_quantity_allclose from simtools.production_configuration.job_spec_builder import ( build_job_specs, @@ -40,10 +41,16 @@ def test_normalize_energy_ranges_expands_list_of_pairs(): ] ) - assert energy_ranges == [ - (30 * u.GeV, 30 * u.GeV), - (300 * u.GeV, 300 * u.GeV), - ] + assert len(energy_ranges) == 2 + for actual, expected in zip( + energy_ranges, + [ + (30 * u.GeV, 30 * u.GeV), + (300 * u.GeV, 300 * u.GeV), + ], + ): + assert_quantity_allclose(actual[0], expected[0]) + assert_quantity_allclose(actual[1], expected[1]) def test_normalize_to_list_converts_tuple_values(): @@ -63,8 +70,9 @@ def test_normalize_grid_axes_uses_defaults_and_none_for_missing_axes(): def test_normalize_energy_ranges_accepts_single_tuple_pair(): energy_ranges = normalize_energy_ranges((30 * u.GeV, 300 * u.GeV)) - - assert energy_ranges == [(30 * u.GeV, 300 * u.GeV)] + assert len(energy_ranges) == 1 + assert_quantity_allclose(energy_ranges[0][0], 30 * u.GeV) + assert_quantity_allclose(energy_ranges[0][1], 300 * u.GeV) def test_normalize_energy_ranges_raises_for_invalid_shape(): @@ -157,8 +165,9 @@ def test_build_job_specs_scales_nshow_by_energy_range(args_dict): ] job_specs = build_job_specs(args_dict, ["7.0.0"]) - - assert [job_spec["nshow"] for job_spec in job_specs] == [100, 10] + nshow_values = [job_spec["nshow"] for job_spec in job_specs] + assert nshow_values[0] == pytest.approx(100) + assert nshow_values[1] == pytest.approx(10) def test_build_job_specs_requires_reference_energy_for_nshow_scaling(args_dict): From d011ff17e7a2e104801e046a45e293ee8cd71bd5 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 20:44:54 +0200 Subject: [PATCH 07/44] ensure list instead of ensure iterable --- .../applications/plot_simtel_events.py | 2 +- src/simtools/applications/simulate_flasher.py | 2 +- .../configuration/commandline_parser.py | 2 +- src/simtools/corsika/corsika_config.py | 2 +- src/simtools/io/ascii_handler.py | 4 ++-- src/simtools/layout/array_layout_utils.py | 2 +- src/simtools/runners/corsika_simtel_runner.py | 2 +- src/simtools/sim_events/output_validator.py | 2 +- src/simtools/simtel/simtel_event_reader.py | 2 +- .../simtel/simtel_output_validator.py | 6 +++--- src/simtools/simulator.py | 8 ++++---- src/simtools/utils/general.py | 20 +++++++++---------- src/simtools/visualization/visualize.py | 2 +- tests/unit_tests/utils/test_general.py | 18 ++++++++--------- 14 files changed, 37 insertions(+), 37 deletions(-) diff --git a/src/simtools/applications/plot_simtel_events.py b/src/simtools/applications/plot_simtel_events.py index afe3f8aca7..7b0cbb31c7 100644 --- a/src/simtools/applications/plot_simtel_events.py +++ b/src/simtools/applications/plot_simtel_events.py @@ -167,7 +167,7 @@ def main(): }, ) - plots = list(gen.ensure_iterable(app_context.args.get("plots"))) + plots = list(gen.ensure_list(app_context.args.get("plots"))) generate_and_save_plots(plots=plots, args=app_context.args, ioh=app_context.io_handler) diff --git a/src/simtools/applications/simulate_flasher.py b/src/simtools/applications/simulate_flasher.py index b7d87ba63b..0586e798fa 100644 --- a/src/simtools/applications/simulate_flasher.py +++ b/src/simtools/applications/simulate_flasher.py @@ -168,7 +168,7 @@ def main(): telescopes = ( get_array_elements_for_layout(app_context.args["array_layout_name"]) if app_context.args.get("array_layout_name") is not None - else general.ensure_iterable(app_context.args["telescopes"]) + else general.ensure_list(app_context.args["telescopes"]) ) for telescope in telescopes: light_source = SimulatorLightEmission( diff --git a/src/simtools/configuration/commandline_parser.py b/src/simtools/configuration/commandline_parser.py index 2a11072e6a..cd558be51d 100644 --- a/src/simtools/configuration/commandline_parser.py +++ b/src/simtools/configuration/commandline_parser.py @@ -586,7 +586,7 @@ def telescope(value): str or list of str Validated telescope name(s) """ - values = general.ensure_iterable(value) + values = general.ensure_list(value) for v in values: names.validate_array_element_name(str(v)) return values if len(values) > 1 else values[0] diff --git a/src/simtools/corsika/corsika_config.py b/src/simtools/corsika/corsika_config.py index 2d524c23b2..6469018987 100644 --- a/src/simtools/corsika/corsika_config.py +++ b/src/simtools/corsika/corsika_config.py @@ -143,7 +143,7 @@ def _fill_corsika_configuration(self, args): config["USER_INPUT"] = self._corsika_configuration_from_user_input(args) config.update( - self._fill_corsika_configuration_from_db(gen.ensure_iterable(args.get("model_version"))) + self._fill_corsika_configuration_from_db(gen.ensure_list(args.get("model_version"))) ) return config diff --git a/src/simtools/io/ascii_handler.py b/src/simtools/io/ascii_handler.py index bd2d1c1ada..d796eb4c17 100644 --- a/src/simtools/io/ascii_handler.py +++ b/src/simtools/io/ascii_handler.py @@ -10,7 +10,7 @@ import numpy as np import yaml -from simtools.utils.general import ensure_iterable, is_url +from simtools.utils.general import ensure_list, is_url _logger = logging.getLogger(__name__) @@ -288,7 +288,7 @@ def _write_to_text_file(data, output_file, unique_lines): """ def iter_lines(data): - for entry in ensure_iterable(data): + for entry in ensure_list(data): yield from entry.splitlines() lines_to_write = ( diff --git a/src/simtools/layout/array_layout_utils.py b/src/simtools/layout/array_layout_utils.py index 670b49077c..e09104774b 100644 --- a/src/simtools/layout/array_layout_utils.py +++ b/src/simtools/layout/array_layout_utils.py @@ -339,7 +339,7 @@ def get_array_layouts_from_db( """ layout_names = [] if layout_name: - layout_names = gen.ensure_iterable(layout_name) + layout_names = gen.ensure_list(layout_name) else: site_model = SiteModel( site=site, diff --git a/src/simtools/runners/corsika_simtel_runner.py b/src/simtools/runners/corsika_simtel_runner.py index 90b45909bb..b13a9e672f 100644 --- a/src/simtools/runners/corsika_simtel_runner.py +++ b/src/simtools/runners/corsika_simtel_runner.py @@ -36,7 +36,7 @@ def __init__( curved_atmosphere_min_zenith_angle=None, ): self._logger = logging.getLogger(__name__) - self.corsika_config = gen.ensure_iterable(corsika_config) + self.corsika_config = gen.ensure_list(corsika_config) # the base corsika config is the one used to define the CORSIKA specific parameters. # The others are used for the array configurations. self.base_corsika_config = self.corsika_config[0] diff --git a/src/simtools/sim_events/output_validator.py b/src/simtools/sim_events/output_validator.py index 943f04b23e..90813c1ba0 100644 --- a/src/simtools/sim_events/output_validator.py +++ b/src/simtools/sim_events/output_validator.py @@ -19,7 +19,7 @@ def validate_sim_events(data_files, expected_mc_events): expected_mc_events: int Expected number of simulated MC events. """ - data_files = general.ensure_iterable(data_files) + data_files = general.ensure_list(data_files) validate_event_numbers(data_files, expected_mc_events) diff --git a/src/simtools/simtel/simtel_event_reader.py b/src/simtools/simtel/simtel_event_reader.py index 4a8ab47ef0..40a78c38ae 100644 --- a/src/simtools/simtel/simtel_event_reader.py +++ b/src/simtools/simtel/simtel_event_reader.py @@ -47,7 +47,7 @@ def read_events(file_name, telescope, event_ids, max_events=1, verbose=False): _logger.warning(f"Telescope type '{telescope}' not found in file '{file_name}'.") return None, None, None - event_ids = gen.ensure_iterable(event_ids) if event_ids is not None else [] + event_ids = gen.ensure_list(event_ids) if event_ids is not None else [] ids_with_data, events = [], [] with SimTelFile(file_name, skip_calibration=False) as f: diff --git a/src/simtools/simtel/simtel_output_validator.py b/src/simtools/simtel/simtel_output_validator.py index f181a549e9..e140a17568 100644 --- a/src/simtools/simtel/simtel_output_validator.py +++ b/src/simtools/simtel/simtel_output_validator.py @@ -55,8 +55,8 @@ def validate_sim_telarray( ValueError If the sim_telarray output files or metadata are not consistent with the array models. """ - data_files = general.ensure_iterable(data_files) - log_files = general.ensure_iterable(log_files) + data_files = general.ensure_list(data_files) + log_files = general.ensure_list(log_files) if array_models: validate_metadata(data_files, array_models, allow_for_changes) @@ -425,7 +425,7 @@ def validate_event_numbers(data_files, expected_mc_events, expected_shower_event If the number of simulated events does not match the expected number. """ event_errors = [] - for file in general.ensure_iterable(data_files): + for file in general.ensure_list(data_files): shower_events, mc_events = file_info.get_simulated_events(file) if (shower_events, mc_events) != (expected_shower_events, expected_mc_events): diff --git a/src/simtools/simulator.py b/src/simtools/simulator.py index d545c1f233..a1b1e73296 100644 --- a/src/simtools/simulator.py +++ b/src/simtools/simulator.py @@ -108,7 +108,7 @@ def _initialize_array_models(self): list, list List of ArrayModel and CorsikaConfig objects. """ - versions = general.ensure_iterable(self.model_version) + versions = general.ensure_list(self.model_version) array_model = [] corsika_configurations = [] @@ -118,7 +118,7 @@ def _initialize_array_models(self): label=self.label, site=self.site, layout_name=settings.config.args.get("array_layout_name"), - array_elements=general.ensure_iterable(settings.config.args.get("telescopes", [])), + array_elements=general.ensure_list(settings.config.args.get("telescopes", [])), model_version=version, calibration_device_types=self._get_calibration_device_types(self.run_mode), overwrite_model_parameters=settings.config.args.get("overwrite_model_parameters"), @@ -458,7 +458,7 @@ def pack_for_register(self, directory_for_grid_upload=None): model_logs + [f for f in histogram_files if model_version in str(f)] + [str(self.get_files(file_type="corsika_log"))] - + list(general.ensure_iterable(model.pack_model_files())) + + list(general.ensure_list(model.pack_model_files())) ) # simtools log file duplicated for each model version if simtools_log_file and Path(simtools_log_file).exists(): @@ -498,7 +498,7 @@ def _overwrite_flasher_photons_for_direct_injection(self): if self.run_mode != "direct_injection" or flasher_photons is None: return - for array_model in general.ensure_iterable(self.array_models): + for array_model in general.ensure_list(self.array_models): for calibration_models in array_model.calibration_models.values(): for calibration_model in calibration_models.values(): calibration_model.overwrite_model_parameter( diff --git a/src/simtools/utils/general.py b/src/simtools/utils/general.py index 8ae1e58366..0700dc8160 100644 --- a/src/simtools/utils/general.py +++ b/src/simtools/utils/general.py @@ -198,27 +198,27 @@ def get_log_level_from_user(log_level): return possible_levels[log_level_lower] -def ensure_iterable(value): +def ensure_list(value): """ - Return input value as iterable. - - - Single values will return as a list with a single element. - - None values will return as empty list. - - Values of list or tuple type are not changed. + Return input value as list. Parameters ---------- value: any - Input value to be converted to a iterable. + Input value to be converted to a list. Returns ------- - list or tuple - Converted value as list or tuple. + list + Converted value as list. """ if value is None: return [] - return value if isinstance(value, list | tuple) else [value] + if isinstance(value, list): + return value + if isinstance(value, tuple): + return list(value) + return [value] def parse_typed_sequence(value, cast=float): diff --git a/src/simtools/visualization/visualize.py b/src/simtools/visualization/visualize.py index 676fc1d5c1..19c6925f8a 100644 --- a/src/simtools/visualization/visualize.py +++ b/src/simtools/visualization/visualize.py @@ -640,7 +640,7 @@ def save_figure(fig, output_file, figure_format=None, log_title="", dpi="figure" figure_format = figure_format or configured_formats or ["png"] - for fmt in gen.ensure_iterable(figure_format): + for fmt in gen.ensure_list(figure_format): _file = Path(output_file).with_suffix(f".{fmt}") fig.savefig(_file, format=fmt, bbox_inches="tight", dpi=dpi) logging.info(f"Saved plot {log_title} to {_file}") diff --git a/tests/unit_tests/utils/test_general.py b/tests/unit_tests/utils/test_general.py index 2f9445180b..22bad46978 100644 --- a/tests/unit_tests/utils/test_general.py +++ b/tests/unit_tests/utils/test_general.py @@ -830,16 +830,16 @@ def test_find_differences_in_json_objects(): ] -def test_ensure_iterable(): - assert gen.ensure_iterable(None) == [] - assert gen.ensure_iterable([1, 2, 3]) == [1, 2, 3] - assert gen.ensure_iterable(5) == [5] - assert gen.ensure_iterable((1, 2, 3)) == (1, 2, 3) +def test_ensure_list(): + assert gen.ensure_list(None) == [] + assert gen.ensure_list([1, 2, 3]) == [1, 2, 3] + assert gen.ensure_list(5) == [5] # Test falsy values are correctly wrapped (not treated as None) - assert gen.ensure_iterable(0) == [0] - assert gen.ensure_iterable(0.0) == [0.0] - assert gen.ensure_iterable("") == [""] - assert gen.ensure_iterable(False) == [False] + assert gen.ensure_list(0) == [0] + assert gen.ensure_list(0.0) == [0.0] + assert gen.ensure_list("") == [""] + assert gen.ensure_list(False) == [False] + assert gen.ensure_list("abc") == ["abc"] def test_parse_typed_sequence(): From 5d5c0bd437c813952accc1ccb142f5464646678d Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 20:55:54 +0200 Subject: [PATCH 08/44] simplify --- src/simtools/layout/array_layout_utils.py | 58 +++++++++++-------- .../layout/test_array_layout_utils.py | 42 +++++++++----- 2 files changed, 63 insertions(+), 37 deletions(-) diff --git a/src/simtools/layout/array_layout_utils.py b/src/simtools/layout/array_layout_utils.py index e09104774b..b934f4d90d 100644 --- a/src/simtools/layout/array_layout_utils.py +++ b/src/simtools/layout/array_layout_utils.py @@ -1,5 +1,6 @@ """Retrieve, merge, and write layout dictionaries.""" +import ast import logging from pathlib import Path @@ -7,6 +8,7 @@ from astropy.table import QTable, Table import simtools.utils.general as gen +import simtools.version as simtools_version from simtools.data_model import data_reader from simtools.data_model.model_data_writer import ModelDataWriter from simtools.io import ascii_handler, io_handler @@ -564,30 +566,6 @@ def read_layouts(args_dict): return [], background_layout -def _get_array_name(array_name): - """ - Return telescope size and number of telescopes from regular array name. - - Finetuned to array names like "4MST", "1LST", etc. - - Parameters - ---------- - array_name : str - Name of the regular array (e.g. "4MST"). - - Returns - ------- - tel_size : str - Telescope size (e.g. "MST"). - n_tel : int - Number of telescopes (e.g. 4). - """ - if len(array_name) < 2 or not array_name[0].isdigit(): - raise ValueError(f"Invalid array_name: '{array_name}'") - - return array_name[1:], int(array_name[0]) - - def _create_star_array(tel_name, pos_x, pos_y, pos_z, n_telescopes, tel_type, site, distance): """Create star-shaped array positions along x and y axes.""" axis_sequence = ["x", "y", "-x", "-y"] @@ -818,3 +796,35 @@ def write_array_elements_info_yaml( } ascii_handler.write_data_to_file(data, output_file) + + +def resolve_array_layout_name(array_layout_name, model_version): + """ + Resolve array layout configuration for a specific model version. + + Parameters + ---------- + array_layout_name : str or dict or list of str or list of dict + Array layout name(s) or configuration(s) to resolve. + model_version : str + Model version to resolve for. + + """ + if isinstance(array_layout_name, list) and len(array_layout_name) == 1: + array_layout_name = array_layout_name[0] + + if isinstance(array_layout_name, str) and array_layout_name.strip().startswith("{"): + try: + parsed_layout = ast.literal_eval(array_layout_name) + if isinstance(parsed_layout, dict): + array_layout_name = parsed_layout + except (SyntaxError, ValueError): + return array_layout_name + + if not isinstance(array_layout_name, dict) or list(array_layout_name) != ["by_version"]: + return array_layout_name + + resolved = simtools_version.resolve_by_version( + {"array_layout_name": array_layout_name}, model_version + ) + return resolved["array_layout_name"] diff --git a/tests/unit_tests/layout/test_array_layout_utils.py b/tests/unit_tests/layout/test_array_layout_utils.py index e454cf55fa..cc67ffddeb 100644 --- a/tests/unit_tests/layout/test_array_layout_utils.py +++ b/tests/unit_tests/layout/test_array_layout_utils.py @@ -1008,19 +1008,6 @@ def test_create_regular_array_metadata(): assert list(table["telescope_name"]) == sorted(table["telescope_name"]) -def test_get_array_name_valid(): - assert array_layout_utils._get_array_name("4MST") == ("MST", 4) - assert array_layout_utils._get_array_name("1LST") == ("LST", 1) - assert array_layout_utils._get_array_name("2SST") == ("SST", 2) - - -def test_get_array_name_invalid(): - with pytest.raises(ValueError, match="Invalid array_name: 'MST'"): - array_layout_utils._get_array_name("MST") - with pytest.raises(ValueError, match="Invalid array_name: 'A4MST'"): - array_layout_utils._get_array_name("A4MST") - - def test_write_array_elements_from_file_to_repository_utm(tmp_test_directory): array_layout_utils.write_array_elements_from_file_to_repository( coordinate_system="utm", @@ -1325,3 +1312,32 @@ def test_get_array_elements_from_db_for_layouts_all_with_no_layouts(mocker): assert result == {} instance.get_list_of_array_layouts.assert_called_once() instance.get_array_elements_for_layout.assert_not_called() + + +def test_resolve_array_layout_name_resolves_stringified_by_version_layout(): + array_layout_name = str( + { + "by_version": { + "<7.0.0": "alpha", + ">=7.0.0": "CTAO-North-Alpha", + } + } + ) + + assert ( + array_layout_utils.resolve_array_layout_name(array_layout_name, "7.0.0") + == "CTAO-North-Alpha" + ) + + +def test_resolve_array_layout_name_unwraps_single_item_list(): + assert ( + array_layout_utils.resolve_array_layout_name(["CTAO-North-Alpha"], "7.0.0") + == "CTAO-North-Alpha" + ) + + +def test_resolve_array_layout_name_keeps_invalid_stringified_dict(): + invalid_layout = "{not valid" + + assert array_layout_utils.resolve_array_layout_name(invalid_layout, "7.0.0") == invalid_layout From 3eab7113d1888fdbe5ee22d9d477d53fc1f31e4f Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 21:09:57 +0200 Subject: [PATCH 09/44] tmp improvements --- .../api-reference/production_configuration.md | 60 ++ .../corsika/corsika_output_validator.py | 4 +- .../htcondor_script_generator.py | 6 +- .../corsika_limits_lookup.py | 319 ++++++++ .../generate_production_grid.py | 730 +----------------- .../job_spec_builder.py | 196 +---- .../production_grid_engine.py | 523 +++++++++++++ .../production_grid_helpers.py | 60 ++ .../production_grid_job_rows.py | 175 +++++ .../production_grid_serialization.py | 136 ++++ src/simtools/testing/log_inspector.py | 2 +- .../visualization/plot_simtel_events.py | 4 +- .../test_generate_production_grid.py | 2 +- .../test_job_spec_builder.py | 77 +- 14 files changed, 1368 insertions(+), 926 deletions(-) create mode 100644 src/simtools/production_configuration/corsika_limits_lookup.py create mode 100644 src/simtools/production_configuration/production_grid_engine.py create mode 100644 src/simtools/production_configuration/production_grid_helpers.py create mode 100644 src/simtools/production_configuration/production_grid_job_rows.py create mode 100644 src/simtools/production_configuration/production_grid_serialization.py diff --git a/docs/source/api-reference/production_configuration.md b/docs/source/api-reference/production_configuration.md index acb6b17b7f..f0478092eb 100644 --- a/docs/source/api-reference/production_configuration.md +++ b/docs/source/api-reference/production_configuration.md @@ -63,6 +63,66 @@ the calculation of the number of events to be simulated given a pre-determined m ``` +(corsika-limits-lookup)= + +## corsika_limits_lookup + +```{eval-rst} +.. automodule:: production_configuration.corsika_limits_lookup + :members: +``` + + +(production-grid-engine)= + +## production_grid_engine + +```{eval-rst} +.. automodule:: production_configuration.production_grid_engine + :members: +``` + + +(production-grid-helpers)= + +## production_grid_helpers + +```{eval-rst} +.. automodule:: production_configuration.production_grid_helpers + :members: +``` + + +(production-grid-job-rows)= + +## production_grid_job_rows + +```{eval-rst} +.. automodule:: production_configuration.production_grid_job_rows + :members: +``` + + +(production-grid)= + +## production_grid + +```{eval-rst} +.. automodule:: production_configuration.production_grid + :members: +``` + + +(production-grid-serialization)= + +## production_grid_serialization + +```{eval-rst} +.. automodule:: production_configuration.production_grid_serialization + :members: +``` + + (interpolation-handler)= ## interpolation_handler diff --git a/src/simtools/corsika/corsika_output_validator.py b/src/simtools/corsika/corsika_output_validator.py index 2315ae1907..0aa98045ac 100644 --- a/src/simtools/corsika/corsika_output_validator.py +++ b/src/simtools/corsika/corsika_output_validator.py @@ -24,8 +24,8 @@ def validate_corsika_output(data_files, log_files, expected_shower_events=None, curved_atmo: bool, optional Whether the CORSIKA simulation was run with the curved atmosphere option. """ - data_files = general.ensure_iterable(data_files) if data_files is not None else [] - log_files = general.ensure_iterable(log_files) + data_files = general.ensure_list(data_files) if data_files is not None else [] + log_files = general.ensure_list(log_files) validate_event_numbers(data_files, expected_shower_events) validate_log_files( diff --git a/src/simtools/job_execution/htcondor_script_generator.py b/src/simtools/job_execution/htcondor_script_generator.py index 164a0ea74c..2dd6bbdfb3 100644 --- a/src/simtools/job_execution/htcondor_script_generator.py +++ b/src/simtools/job_execution/htcondor_script_generator.py @@ -14,10 +14,8 @@ import astropy.units as u -from simtools.production_configuration.job_spec_builder import ( - build_job_specs, - resolve_array_layout_name, -) +from simtools.layout.array_layout_utils import resolve_array_layout_name +from simtools.production_configuration.job_spec_builder import build_job_specs _logger = logging.getLogger(__name__) diff --git a/src/simtools/production_configuration/corsika_limits_lookup.py b/src/simtools/production_configuration/corsika_limits_lookup.py new file mode 100644 index 0000000000..a0d7859ef9 --- /dev/null +++ b/src/simtools/production_configuration/corsika_limits_lookup.py @@ -0,0 +1,319 @@ +"""Lookup-table access and interpolation for CORSIKA production limits.""" + +import json + +import numpy as np +from astropy.table import Table +from scipy.interpolate import LinearNDInterpolator, griddata +from scipy.spatial import QhullError # pylint: disable=no-name-in-module + +from simtools.simtel.simtel_io_metadata import ( + get_sim_telarray_telescope_id_to_telescope_name_mapping, +) + + +def _value_in_unit(value, unit=None): + """Return a scalar value converted to the requested unit when possible.""" + if hasattr(value, "to_value"): + return value.to_value(unit) if unit is not None else value.value + return value + + +class CorsikaLimitsLookup: + """ + Read and interpolate CORSIKA limits for production grids. + + The lookup table is shared by both grid serialization and backend-specific + job-spec generation. + """ + + def __init__(self, lookup_table, telescope_ids=None, simtel_file=None): + """ + Initialize lookup-table access. + + Parameters + ---------- + lookup_table : str or Path + Path to the lookup-table ECSV file. + telescope_ids : list or str, optional + Telescope selection used to filter lookup-table rows. + simtel_file : str, optional + Path to a sim_telarray file used to resolve numeric telescope IDs. + """ + self.lookup_table = lookup_table + self.telescope_ids = telescope_ids + self.simtel_file = simtel_file + self._simtel_id_to_name = ( + get_sim_telarray_telescope_id_to_telescope_name_mapping(simtel_file) + if simtel_file + else {} + ) + self.lookup_points_for_interpolation = None + self.lookup_values_for_interpolation = None + self.lookup_interpolators_for_point = None + + @staticmethod + def _coerce_identifier_container(value): + """Coerce identifier input into a list.""" + if value is None: + return [] + if isinstance(value, str): + stripped = value.strip() + return json.loads(stripped) if stripped.startswith("[") else [stripped] + if isinstance(value, (list, tuple, set)): + return list(value) + return [value] + + @staticmethod + def coerce_identifier_container(value): + """Coerce identifier input into a list.""" + return CorsikaLimitsLookup._coerce_identifier_container(value) + + def _normalize_lookup_identifier(self, identifier): + """Normalize one telescope identifier and report if it is numeric.""" + if isinstance(identifier, (int, np.integer)): + return self._simtel_id_to_name.get(int(identifier), str(int(identifier))), True + + text = str(identifier).strip() + if text.lstrip("+-").isdigit(): + return self._simtel_id_to_name.get(int(text), text), True + return text, False + + def normalize_lookup_identifier(self, identifier): + """Normalize one telescope identifier and report if it is numeric.""" + return self._normalize_lookup_identifier(identifier) + + def _normalized_identifier_set(self, identifiers): + """Return normalized telescope identifiers as a set.""" + return { + self._normalize_lookup_identifier(identifier)[0] + for identifier in self._coerce_identifier_container(identifiers) + } + + def normalized_identifier_set(self, identifiers): + """Return normalized telescope identifiers as a set.""" + return self._normalized_identifier_set(identifiers) + + def _lookup_contains_numeric_telescope_ids(self, lookup_table): + """Return True when any lookup-table telescope identifier is numeric.""" + return any( + any( + self._normalize_lookup_identifier(identifier)[1] + for identifier in self._coerce_identifier_container(row["telescope_ids"]) + ) + for row in lookup_table + ) + + def lookup_contains_numeric_telescope_ids(self, lookup_table): + """Return True when any lookup-table telescope identifier is numeric.""" + return self._lookup_contains_numeric_telescope_ids(lookup_table) + + @property + def simtel_id_to_name(self): + """Return the sim_telarray telescope-ID mapping.""" + return self._simtel_id_to_name + + @simtel_id_to_name.setter + def simtel_id_to_name(self, value): + """Set the sim_telarray telescope-ID mapping.""" + self._simtel_id_to_name = value + + def load_matching_lookup_arrays(self): + """ + Load and filter lookup-table arrays for the selected telescope IDs. + + Returns + ------- + dict + Lookup arrays for interpolation. + """ + lookup_table = Table.read(self.lookup_table, format="ascii.ecsv") + selected_telescope_ids = self._normalized_identifier_set(self.telescope_ids) + + matching_rows = [ + row + for row in lookup_table + if selected_telescope_ids == self._normalized_identifier_set(row["telescope_ids"]) + ] + + if not matching_rows: + has_numeric_lookup_ids = self._lookup_contains_numeric_telescope_ids(lookup_table) + + if has_numeric_lookup_ids and not self.simtel_file: + raise ValueError( + "Lookup table telescope selections contain numeric IDs. " + "Provide --simtel_file to map those IDs to telescope names." + ) + + raise ValueError( + f"No matching rows in the lookup table for telescope_ids: {self.telescope_ids}" + ) + + def extract_array(field, transform=lambda x: x): + return np.array([transform(row[field]) for row in matching_rows]) + + zeniths = extract_array("zenith") + azimuths = extract_array("azimuth", lambda x: x % 360) + nsb_values = extract_array("nsb_level", float) + + return { + "points": np.column_stack((zeniths, azimuths, nsb_values)), + "lower_energy_threshold": extract_array("lower_energy_limit"), + "upper_scatter_radius": extract_array("upper_radius_limit"), + "viewcone_radius": extract_array("viewcone_radius"), + } + + def prepare_point_interpolators(self): + """ + Prepare lookup arrays for per-point interpolation. + + Returns + ------- + dict + Interpolators keyed by lookup quantity. + """ + lookup_arrays = self.load_matching_lookup_arrays() + points = lookup_arrays["points"] + azimuths = points[:, 1] + azimuths_wrapped = np.concatenate([azimuths + shift for shift in (0, 360, -360)]) + + def repeat_3(arr): + """Repeat an array three times.""" + return np.tile(arr, 3) + + self.lookup_points_for_interpolation = np.column_stack( + ( + repeat_3(points[:, 0]), + azimuths_wrapped, + repeat_3(points[:, 2]), + ) + ) + self.lookup_values_for_interpolation = { + "lower_energy_threshold": repeat_3(lookup_arrays["lower_energy_threshold"]), + "upper_scatter_radius": repeat_3(lookup_arrays["upper_scatter_radius"]), + "viewcone_radius": repeat_3(lookup_arrays["viewcone_radius"]), + } + try: + self.lookup_interpolators_for_point = { + key: LinearNDInterpolator( + self.lookup_points_for_interpolation, + self.lookup_values_for_interpolation[key], + fill_value=np.nan, + ) + for key in ("lower_energy_threshold", "upper_scatter_radius", "viewcone_radius") + } + except QhullError as exc: + raise ValueError( + "Lookup table does not contain enough unique points for 3D interpolation " + "(zenith, azimuth, nsb_level). Provide a denser lookup table " + "or run without --lookup_table if limits are not required for this use case." + ) from exc + return self.lookup_interpolators_for_point + + def interpolate_grid_limits(self, target_values): + """ + Interpolate lookup values on a regular zenith/azimuth/NSB grid. + + Parameters + ---------- + target_values : dict + Generated target-axis values. + + Returns + ------- + dict + Interpolated lookup values on the target grid. + """ + lookup_arrays = self.load_matching_lookup_arrays() + points_base = lookup_arrays["points"] + lower_energy_thresholds = lookup_arrays["lower_energy_threshold"] + upper_scatter_radii = lookup_arrays["upper_scatter_radius"] + viewcone_radii = lookup_arrays["viewcone_radius"] + + zeniths = points_base[:, 0] + azimuths = points_base[:, 1] + nsb_values = points_base[:, 2] + azimuths_wrapped = np.concatenate([azimuths + shift for shift in (0, 360, -360)]) + + def repeat_3(arr): + """Repeat an array three times.""" + return np.tile(arr, 3) + + points = np.column_stack( + ( + repeat_3(zeniths), + azimuths_wrapped, + repeat_3(nsb_values), + ) + ) + + target_grid = ( + np.array( + np.meshgrid( + target_values["zenith_angle"].value, + target_values["azimuth"].value, + target_values["nsb_level"].value, + indexing="ij", + ) + ) + .reshape(3, -1) + .T + ) + + def interpolate(values): + return griddata( + points, repeat_3(values), target_grid, method="linear", fill_value=np.nan + ).reshape( + len(target_values["zenith_angle"]), + len(target_values["azimuth"]), + len(target_values["nsb_level"]), + ) + + return { + "lower_energy_threshold": interpolate(lower_energy_thresholds), + "upper_scatter_radius": interpolate(upper_scatter_radii), + "viewcone_radius": interpolate(viewcone_radii), + } + + def interpolate_point(self, zenith, azimuth, nsb=1.0): + """ + Interpolate lookup-table limits for a single point. + + Parameters + ---------- + zenith : float or Quantity + Zenith angle. + azimuth : float or Quantity + Azimuth angle. + nsb : float or Quantity, optional + Night-sky background level. + + Returns + ------- + dict + Interpolated lower-energy threshold, scatter radius, and viewcone radius. + """ + if self.lookup_interpolators_for_point is None: + self.prepare_point_interpolators() + + target = np.array( + [ + [ + _value_in_unit(zenith), + _value_in_unit(azimuth) % 360.0, + _value_in_unit(nsb), + ] + ], + dtype=float, + ) + return { + "lower_energy_threshold": float( + self.lookup_interpolators_for_point["lower_energy_threshold"](target)[0] + ), + "upper_scatter_radius": float( + self.lookup_interpolators_for_point["upper_scatter_radius"](target)[0] + ), + "viewcone_radius": float( + self.lookup_interpolators_for_point["viewcone_radius"](target)[0] + ), + } diff --git a/src/simtools/production_configuration/generate_production_grid.py b/src/simtools/production_configuration/generate_production_grid.py index 0d747ed1b7..fb776d6e9f 100644 --- a/src/simtools/production_configuration/generate_production_grid.py +++ b/src/simtools/production_configuration/generate_production_grid.py @@ -1,729 +1,19 @@ """ -Module defines the `GridGeneration` class. +Thin adapter exposing the legacy production-grid API. -Used to generate a grid of simulation points based on flexible axes definitions such -azimuth, zenith angle, night-sky background, and camera offset. -The module handles axis binning, scaling and interpolation of energy thresholds, viewcone, -and radius limits from a lookup table. -Additionally, it allows for converting between Altitude/Azimuth and Right Ascension -Declination coordinates. The resulting grid points are saved to a file. +The backend-independent implementation lives in +``simtools.production_configuration.production_grid``. """ -import json -import logging -from pathlib import Path +from simtools.production_configuration.production_grid_engine import ProductionGridEngine -import numpy as np -from astropy import units as u -from astropy.coordinates import AltAz, EarthLocation, SkyCoord -from astropy.table import Table -from astropy.units import Quantity -from scipy.interpolate import LinearNDInterpolator, griddata -from scipy.spatial import QhullError # pylint: disable=no-name-in-module -from simtools.simtel.simtel_io_metadata import ( - get_sim_telarray_telescope_id_to_telescope_name_mapping, -) - -DEFAULT_SERIALIZATION_ROUND_DECIMALS = 6 - - -class GridGeneration: +class GridGeneration(ProductionGridEngine): + # pylint: disable=too-few-public-methods """ - Defines and generates a grid of simulation points based on flexible axes definitions. + Backward-compatible adapter for production-grid generation. - This class generates a grid of points for a simulation based on parameters such as - azimuth, zenith angle, night-sky background, and camera offset, - taking into account axis definitions, scaling, and units and interpolating values - for simulations from a lookup table. + This class preserves the legacy public API while delegating the full + implementation to + :class:`~simtools.production_configuration.production_grid.ProductionGridEngine`. """ - - def __init__( - self, - axes, - coordinate_system="zenith_azimuth", - observing_location=None, - observing_time=None, - lookup_table=None, - telescope_ids=None, - simtel_file=None, - ): - """ - Initialize the grid with the given axes and coordinate system. - - Parameters - ---------- - axes : dict - Dictionary where each key is the axis name and the value is a dictionary - defining the axis properties (range, binning, scaling, etc.). - coordinate_system : str, optional - The coordinate system for the grid generation (default is 'zenith_azimuth'). - observing_location : EarthLocation, optional - The location of the observation (latitude, longitude, height). - observing_time : Time, optional - The time of the observation. If None, coordinate conversion to RA/Dec not working. - lookup_table : str, optional - Path to the lookup table file (ECSV format). - telescope_ids : list of str, optional - List of telescope IDs to get the limits for. - simtel_file : str, optional - Path to a sim_telarray file used to map sim_telarray telescope IDs to - telescope names when matching lookup-table telescope selections. - """ - self._logger = logging.getLogger(__name__) - self.axes = axes["axes"] if "axes" in axes else axes - self.coordinate_system = coordinate_system - self.observing_location = ( - observing_location - if observing_location is not None - else EarthLocation(lat=0.0 * u.deg, lon=0.0 * u.deg, height=0 * u.m) - ) - self.observing_time = observing_time - self.lookup_table = lookup_table - self.telescope_ids = telescope_ids - self.simtel_file = simtel_file - self.interpolated_limits = {} - self.serialization_round_decimals = DEFAULT_SERIALIZATION_ROUND_DECIMALS - self._simtel_id_to_name = ( - get_sim_telarray_telescope_id_to_telescope_name_mapping(simtel_file) - if simtel_file - else {} - ) - - # Store target values for each axis - self.target_values = self._generate_target_values() - - if self.lookup_table: - if self.coordinate_system == "ra_dec": - self._prepare_lookup_table_limits_for_point_interpolation() - else: - self._apply_lookup_table_limits() - - @staticmethod - def _coerce_identifier_container(value): - """Coerce identifier input into a list.""" - if value is None: - return [] - if isinstance(value, str): - stripped = value.strip() - return json.loads(stripped) if stripped.startswith("[") else [stripped] - if isinstance(value, (list, tuple, set)): - return list(value) - return [value] - - def _normalize_lookup_identifier(self, identifier): - """Normalize one telescope identifier and report if it is numeric.""" - if isinstance(identifier, (int, np.integer)): - return self._simtel_id_to_name.get(int(identifier), str(int(identifier))), True - - text = str(identifier).strip() - if text.lstrip("+-").isdigit(): - return self._simtel_id_to_name.get(int(text), text), True - return text, False - - def _normalized_identifier_set(self, identifiers): - """Return normalized telescope identifiers as a set.""" - return { - self._normalize_lookup_identifier(identifier)[0] - for identifier in self._coerce_identifier_container(identifiers) - } - - def _lookup_contains_numeric_telescope_ids(self, lookup_table): - """Return True when any lookup-table telescope identifier is numeric.""" - return any( - any( - self._normalize_lookup_identifier(identifier)[1] - for identifier in self._coerce_identifier_container(row["telescope_ids"]) - ) - for row in lookup_table - ) - - def _load_matching_lookup_arrays(self): - """Load and filter lookup-table arrays for selected telescope IDs.""" - lookup_table = Table.read(self.lookup_table, format="ascii.ecsv") - selected_telescope_ids = self._normalized_identifier_set(self.telescope_ids) - - matching_rows = [ - row - for row in lookup_table - if selected_telescope_ids == self._normalized_identifier_set(row["telescope_ids"]) - ] - - if not matching_rows: - has_numeric_lookup_ids = self._lookup_contains_numeric_telescope_ids(lookup_table) - - if has_numeric_lookup_ids and not self.simtel_file: - raise ValueError( - "Lookup table telescope selections contain numeric IDs. " - "Provide --simtel_file to map those IDs to telescope names." - ) - - raise ValueError( - f"No matching rows in the lookup table for telescope_ids: {self.telescope_ids}" - ) - - def extract_array(field, transform=lambda x: x): - return np.array([transform(row[field]) for row in matching_rows]) - - zeniths = extract_array("zenith") - azimuths = extract_array("azimuth", lambda x: x % 360) - nsb_values = extract_array("nsb_level", float) - - return { - "points": np.column_stack((zeniths, azimuths, nsb_values)), - "lower_energy_threshold": extract_array("lower_energy_limit"), - "upper_scatter_radius": extract_array("upper_radius_limit"), - "viewcone_radius": extract_array("viewcone_radius"), - } - - def _require_observing_time(self): - """Return observing time if available, else raise a clear error.""" - if self.observing_time is None: - raise ValueError("Observing time is required for ra_dec grid generation.") - return self.observing_time - - def _get_max_zenith_for_radec_mode(self): - """Read maximum zenith from axes for RA/Dec direction sampling.""" - zenith_axis = self.axes.get("zenith_angle") - if not zenith_axis or "range" not in zenith_axis or len(zenith_axis["range"]) != 2: - raise ValueError( - "RA/Dec direction sampling requires 'zenith_angle' axis with a valid " - "two-element 'range' in the axes definition." - ) - return float(zenith_axis["range"][1]) - - def _prepare_lookup_table_limits_for_point_interpolation(self): - """Prepare lookup arrays for per-point interpolation in RA/Dec grid mode.""" - lookup_arrays = self._load_matching_lookup_arrays() - - points = lookup_arrays["points"] - azimuths = points[:, 1] - azimuths_wrapped = np.concatenate([azimuths + shift for shift in (0, 360, -360)]) - - def repeat_3(arr): - """Repeat an array three times.""" - return np.tile(arr, 3) - - self.lookup_points_for_interpolation = np.column_stack( - ( - repeat_3(points[:, 0]), - azimuths_wrapped, - repeat_3(points[:, 2]), - ) - ) - self.lookup_values_for_interpolation = { - "lower_energy_threshold": repeat_3(lookup_arrays["lower_energy_threshold"]), - "upper_scatter_radius": repeat_3(lookup_arrays["upper_scatter_radius"]), - "viewcone_radius": repeat_3(lookup_arrays["viewcone_radius"]), - } - try: - self.lookup_interpolators_for_point = { - key: LinearNDInterpolator( - self.lookup_points_for_interpolation, - self.lookup_values_for_interpolation[key], - fill_value=np.nan, - ) - for key in ("lower_energy_threshold", "upper_scatter_radius", "viewcone_radius") - } - except QhullError as exc: - raise ValueError( - "Lookup table does not contain enough unique points for 3D interpolation " - "(zenith, azimuth, nsb_level). Provide a denser lookup table " - "or run without --lookup_table if limits are not required for this use case." - ) from exc - - def _has_radec_axes(self): - """Return True if axes define a native RA/Dec grid.""" - return "ra" in self.axes and "dec" in self.axes - - def _generate_target_values(self): - """ - Generate target axis values and store them as Quantities. - - Returns - ------- - dict - Dictionary of target values for each axis, stored as Quantity objects. - """ - target_values = {} - for axis_name, axis in self.axes.items(): - axis_range = axis["range"] - binning = axis["binning"] - scaling = axis.get("scaling", "linear") - units = axis.get("units", None) - - if axis_name == "azimuth": - # Use circular binning for azimuth - values = self.create_circular_binning(axis_range, binning) - elif scaling == "log": - # Log scaling - values = np.logspace(np.log10(axis_range[0]), np.log10(axis_range[1]), binning) - elif scaling == "1/cos": - # 1/cos scaling - cos_min = np.cos(np.radians(axis_range[0])) - cos_max = np.cos(np.radians(axis_range[1])) - inv_cos_values = np.linspace(1 / cos_min, 1 / cos_max, binning) - values = np.degrees(np.arccos(1 / inv_cos_values)) - else: - # Linear scaling - values = np.linspace(axis_range[0], axis_range[1], binning) - - if units: - values = values * u.Unit(units) - - target_values[axis_name] = values - - return target_values - - def _apply_lookup_table_limits(self): - """Apply limits from the lookup table and interpolate values.""" - lookup_arrays = self._load_matching_lookup_arrays() - points_base = lookup_arrays["points"] - lower_energy_thresholds = lookup_arrays["lower_energy_threshold"] - upper_scatter_radii = lookup_arrays["upper_scatter_radius"] - viewcone_radii = lookup_arrays["viewcone_radius"] - - zeniths = points_base[:, 0] - azimuths = points_base[:, 1] - nsb_values = points_base[:, 2] - - # Wrap azimuths and repeat others - azimuths_wrapped = np.concatenate([azimuths + shift for shift in (0, 360, -360)]) - - def repeat_3(arr): - """Repeat an array three times.""" - return np.tile(arr, 3) - - points = np.column_stack( - ( - repeat_3(zeniths), - azimuths_wrapped, - repeat_3(nsb_values), - ) - ) - - target_grid = ( - np.array( - np.meshgrid( - self.target_values["zenith_angle"].value, - self.target_values["azimuth"].value, - self.target_values["nsb_level"].value, - indexing="ij", - ) - ) - .reshape(3, -1) - .T - ) - - def interpolate(values): - return griddata( - points, repeat_3(values), target_grid, method="linear", fill_value=np.nan - ).reshape( - len(self.target_values["zenith_angle"]), - len(self.target_values["azimuth"]), - len(self.target_values["nsb_level"]), - ) - - self.interpolated_limits = { - "lower_energy_threshold": interpolate(lower_energy_thresholds), - "upper_scatter_radius": interpolate(upper_scatter_radii), - "viewcone_radius": interpolate(viewcone_radii), - } - - def _generate_radec_grid_direction_points(self): - """Generate direction points from declination lines and hour-angle spacing.""" - observing_time = self._require_observing_time() - max_zenith = self._get_max_zenith_for_radec_mode() - lst_deg = observing_time.sidereal_time( - "apparent", longitude=self.observing_location.lon - ).deg - - direction_points = [] - for declination in np.arange(-90.0, 91.0, 1.0): - cos_dec = np.cos(np.deg2rad(declination)) - step_ha = 1.0 / cos_dec if cos_dec > 1e-6 else 360.0 - n_ha = max(1, int(np.ceil(360.0 / step_ha))) - hour_angles = np.linspace(-180.0, 180.0, n_ha, endpoint=False) - ra_values = (lst_deg - hour_angles) % 360.0 - - skycoord = SkyCoord( - ra=ra_values * u.deg, - dec=np.full_like(ra_values, declination) * u.deg, - frame="icrs", - ) - altaz = skycoord.transform_to( - AltAz(location=self.observing_location, obstime=observing_time) - ) - - zenith_values = (90.0 * u.deg - altaz.alt).to(u.deg).value - mask = (zenith_values >= 0.0) & (zenith_values <= max_zenith) - - for idx in np.nonzero(mask)[0]: - direction_points.append( - { - "zenith_angle": zenith_values[idx] * u.deg, - "azimuth": altaz.az.deg[idx] * u.deg, - } - ) - return direction_points - - def _generate_extra_axis_combinations(self, excluded_keys): - """Generate combinations for all axes except the excluded ones.""" - extra_axes = { - key: value for key, value in self.target_values.items() if key not in excluded_keys - } - if not extra_axes: - return list(extra_axes.keys()), [], [np.array([])] - - extra_value_arrays = [value.value for value in extra_axes.values()] - extra_units = [value.unit for value in extra_axes.values()] - extra_grid = np.meshgrid(*extra_value_arrays, indexing="ij") - extra_combinations = np.vstack(list(map(np.ravel, extra_grid))).T - return list(extra_axes.keys()), extra_units, extra_combinations - - def _add_lookup_limits_to_point(self, point, zenith, azimuth): - """Interpolate and attach lookup-table limits to a grid point.""" - if not self.lookup_table: - return - - nsb_value = point.get("nsb_level", 1) - if isinstance(nsb_value, Quantity): - nsb_value = nsb_value.value - limits = self._interpolate_limits_for_point( - zenith=zenith, - azimuth=azimuth, - nsb=float(nsb_value), - ) - point["lower_energy_threshold"] = limits["lower_energy_threshold"] * u.TeV - point["scatter_radius"] = limits["upper_scatter_radius"] * u.m - point["viewcone_radius"] = limits["viewcone_radius"] * u.deg - - def _generate_grid_from_radec_axes(self): - """Generate grid points from explicit RA/Dec axes definitions. - - All explicit RA/Dec combinations defined by the input axes are preserved, - even when their transformed Alt/Az coordinates fall below the local horizon. - """ - observing_time = self._require_observing_time() - - axis_keys = [key for key in self.target_values if key not in ("zenith_angle", "azimuth")] - value_arrays = [self.target_values[key].value for key in axis_keys] - units = [self.target_values[key].unit for key in axis_keys] - grid = np.meshgrid(*value_arrays, indexing="ij") - combinations = np.vstack(list(map(np.ravel, grid))).T - - grid_points = [] - for combination in combinations: - grid_point = { - key: Quantity(combination[i], units[i]) for i, key in enumerate(axis_keys) - } - - skycoord = SkyCoord( - ra=grid_point["ra"].to(u.deg), - dec=grid_point["dec"].to(u.deg), - frame="icrs", - ) - altaz = skycoord.transform_to( - AltAz(location=self.observing_location, obstime=observing_time) - ) - zenith = (90.0 * u.deg - altaz.alt).to(u.deg).value - - self._add_lookup_limits_to_point(grid_point, zenith=zenith, azimuth=altaz.az.deg) - - grid_points.append(grid_point) - - return grid_points - - def _interpolate_limits_for_point(self, zenith, azimuth, nsb): - """Interpolate lookup-table limits for a single point.""" - target = np.array([[zenith, azimuth % 360.0, nsb]], dtype=float) - return { - "lower_energy_threshold": float( - self.lookup_interpolators_for_point["lower_energy_threshold"](target)[0] - ), - "upper_scatter_radius": float( - self.lookup_interpolators_for_point["upper_scatter_radius"](target)[0] - ), - "viewcone_radius": float( - self.lookup_interpolators_for_point["viewcone_radius"](target)[0] - ), - } - - def _generate_grid_radec_mode(self): - """Generate grid points for RA/Dec mode. - - If explicit ``ra``/``dec`` axes are provided, use those points directly. - Otherwise, sample directions along declination lines with hour-angle spacing - and keep only points that satisfy the configured zenith-angle limits. - """ - if self._has_radec_axes(): - return self._generate_grid_from_radec_axes() - - direction_points = self._generate_radec_grid_direction_points() - extra_keys, extra_units, extra_combinations = self._generate_extra_axis_combinations( - excluded_keys=("zenith_angle", "azimuth") - ) - - grid_points = [] - for direction_point in direction_points: - for extra_combination in extra_combinations: - point = dict(direction_point) - for i, key in enumerate(extra_keys): - point[key] = Quantity(extra_combination[i], extra_units[i]) - - self._add_lookup_limits_to_point( - point, - zenith=point["zenith_angle"].value, - azimuth=point["azimuth"].value, - ) - - grid_points.append(point) - - return grid_points - - def create_circular_binning(self, azimuth_range, num_bins): - """ - Create bin centers for azimuth angles, handling circular wraparound (0 deg to 360 deg). - - Parameters - ---------- - azimuth_range : tuple - (min_azimuth, max_azimuth), can wrap around 0 deg. - num_bins : int - Number of bins. - - Returns - ------- - np.ndarray - Array of bin centers. - """ - azimuth_min, azimuth_max = azimuth_range - azimuth_min %= 360 # Normalize to [0, 360) - azimuth_max %= 360 - - clockwise_distance = (azimuth_max - azimuth_min) % 360 - counterclockwise_distance = (azimuth_min - azimuth_max) % 360 - - if clockwise_distance <= counterclockwise_distance: - bin_centers = ( - np.linspace(azimuth_min, azimuth_min + clockwise_distance, num_bins, endpoint=True) - % 360 - ) - else: - bin_centers = ( - np.linspace( - azimuth_min, azimuth_min - counterclockwise_distance, num_bins, endpoint=True - ) - % 360 - ) - - return bin_centers - - def generate_grid(self): - """ - Generate the grid based on the required axes and include interpolated limits. - - Takes energy threshold, viewcone, and radius from the interpolated lookup table. - - Returns - ------- - list of dict - A list of grid points, each represented as a dictionary with axis names - as keys and axis values as values. Axis values may include units where defined. - """ - if self.coordinate_system == "ra_dec": - return self._generate_grid_radec_mode() - - value_arrays = [value.value for value in self.target_values.values()] - units = [value.unit for value in self.target_values.values()] - grid = np.meshgrid(*value_arrays, indexing="ij") - combinations = np.vstack(list(map(np.ravel, grid))).T - grid_points = [] - for combination in combinations: - grid_point = { - key: Quantity(combination[i], units[i]) - for i, key in enumerate(self.target_values.keys()) - } - - if "lower_energy_threshold" in self.interpolated_limits: - zenith_idx = np.searchsorted( - self.target_values["zenith_angle"].value, grid_point["zenith_angle"].value - ) - azimuth_idx = np.searchsorted( - self.target_values["azimuth"].value, grid_point["azimuth"].value - ) - nsb_idx = np.searchsorted( - self.target_values["nsb_level"].value, - grid_point["nsb_level"].value, - ) - energy_lower = self.interpolated_limits["lower_energy_threshold"][ - zenith_idx, azimuth_idx, nsb_idx - ] - grid_point["lower_energy_threshold"] = energy_lower * u.TeV - - if "upper_scatter_radius" in self.interpolated_limits: - radius_value = self.interpolated_limits["upper_scatter_radius"][ - zenith_idx, azimuth_idx, nsb_idx - ] - grid_point["scatter_radius"] = radius_value * u.m - - if "viewcone_radius" in self.interpolated_limits: - viewcone_value = self.interpolated_limits["viewcone_radius"][ - zenith_idx, azimuth_idx, nsb_idx - ] - grid_point["viewcone_radius"] = viewcone_value * u.deg - - grid_points.append(grid_point) - - return grid_points - - def convert_altaz_to_radec(self, alt, az): - """ - Convert Altitude/Azimuth (AltAz) coordinates to Right Ascension/Declination (RA/Dec). - - Parameters - ---------- - alt : float - Altitude angle in degrees. - az : float - Azimuth angle in degrees. - - Returns - ------- - SkyCoord - SkyCoord object containing the RA/Dec coordinates. - - Raises - ------ - ValueError - If observing_time is not set. - """ - if self.observing_time is None: - raise ValueError( - "Observing time is not set. " - "Please provide an observing_time to convert coordinates." - ) - - alt_rad = alt.to(u.rad) - az_rad = az.to(u.rad) - aa = AltAz( - alt=alt_rad, - az=az_rad, - location=self.observing_location, - obstime=self.observing_time, - ) - skycoord = SkyCoord(aa) - return skycoord.icrs # Return RA/Dec in ICRS frame - - def convert_coordinates(self, grid_points): - """ - Convert the grid points to RA/Dec coordinates if necessary. - - Parameters - ---------- - grid_points : list of dict - List of grid points, each represented as a dictionary with axis - names as keys and values. - - Returns - ------- - list of dict - The grid points with converted RA/Dec coordinates. - """ - if self.coordinate_system == "ra_dec": - for point in grid_points: - if "zenith_angle" in point and "azimuth" in point: - alt = (90.0 * u.deg) - point.pop("zenith_angle") - az = point.pop("azimuth") - radec = self.convert_altaz_to_radec(alt, az) - point["ra"] = radec.ra.deg * u.deg - point["dec"] = radec.dec.deg * u.deg - return grid_points - - def serialize_grid_points(self, grid_points, output_file): - """Serialize the grid output and save to an ECSV table file.""" - if Path(output_file).suffix.lower() != ".ecsv": - raise ValueError("Grid output file must use '.ecsv' extension.") - - all_keys = self._collect_point_keys(grid_points) - rows, units = self._build_serialized_rows(grid_points, all_keys) - - output_table = Table(rows=rows, names=all_keys) - for column_name, unit in units.items(): - output_table[column_name].unit = u.Unit(unit) - - output_table.meta = self._build_grid_metadata() - - output_table.write(output_file, format="ascii.ecsv", overwrite=True) - self._logger.info(f"Output saved to {output_file}") - - @staticmethod - def _collect_point_keys(grid_points): - """Collect all grid-point keys while preserving first-seen order.""" - all_keys = [] - for point in grid_points: - for key in point: - if key not in all_keys: - all_keys.append(key) - return all_keys - - def _serialize_grid_value(self, value): - """Serialize one grid value and return (value, unit).""" - if isinstance(value, u.Quantity): - serialized = round(float(value.value), self.serialization_round_decimals) - return serialized, str(value.unit) - - if isinstance(value, dict) and "value" in value: - return value["value"], value.get("unit") - - if value is None: - return np.nan, None - - if isinstance(value, (np.floating, float)): - return round(float(value), self.serialization_round_decimals), None - - if isinstance(value, (np.integer, int)): - return int(value), None - - return value, None - - def _build_serialized_rows(self, grid_points, all_keys): - """Build serialized row dictionaries and collect units.""" - rows = [] - units = {} - - for point in grid_points: - row = {} - for key in all_keys: - serialized_value, unit = self._serialize_grid_value(point.get(key)) - row[key] = serialized_value - if unit is not None: - units.setdefault(key, unit) - rows.append(row) - - return rows, units - - def _build_grid_metadata(self): - """Build metadata for the output grid table.""" - return { - "coordinate_system": self.coordinate_system, - "reference_frame": "ICRS (J2000)", - "observing_time_utc": self.observing_time.isot if self.observing_time else None, - "observing_time_scale": self.observing_time.scale if self.observing_time else None, - "telescope_ids": self.telescope_ids, - "lookup_table": str(Path(self.lookup_table)) if self.lookup_table else None, - } - - def serialize_quantity(self, value): - """Serialize Quantity.""" - if isinstance(value, u.Quantity): - serialized_value = float(value.value) - rounded_value = round(serialized_value, self.serialization_round_decimals) - return {"value": rounded_value, "unit": str(value.unit)} - if isinstance(value, float): - return round(value, self.serialization_round_decimals) - if isinstance(value, np.floating): - return round(float(value), self.serialization_round_decimals) - if isinstance(value, np.integer): - return int(value) - return value diff --git a/src/simtools/production_configuration/job_spec_builder.py b/src/simtools/production_configuration/job_spec_builder.py index 341ee913f1..df301c239f 100644 --- a/src/simtools/production_configuration/job_spec_builder.py +++ b/src/simtools/production_configuration/job_spec_builder.py @@ -1,100 +1,13 @@ """Build backend-agnostic job specifications for production submissions.""" -import ast -import itertools - import numpy as np from astropy import units as u -import simtools.version as simtools_version -from simtools.configuration import defaults - -_GRID_AXES = [ - "primary", - "azimuth_angle", - "zenith_angle", - "model_version", - "corsika_le_interaction", - "corsika_he_interaction", -] - -_GRID_AXIS_DEFAULTS = { - "corsika_le_interaction": defaults.CORSIKA_LE_INTERACTION, - "corsika_he_interaction": defaults.CORSIKA_HE_INTERACTION, -} - - -def normalize_to_list(value): - """Normalize scalar values to lists of length one.""" - if isinstance(value, list): - return value - if isinstance(value, tuple): - return list(value) - return [value] - - -def normalize_grid_axes(args_dict): - """Return normalized grid axes for cartesian product expansion.""" - return { - axis: ( - normalize_to_list(args_dict[axis]) - if axis in args_dict and args_dict[axis] is not None - else [_GRID_AXIS_DEFAULTS[axis]] - if axis in _GRID_AXIS_DEFAULTS - else [None] - ) - for axis in _GRID_AXES - } - - -def normalize_energy_ranges(energy_range): - """Normalize energy range argument to a list of ``(e_min, e_max)`` pairs.""" - if isinstance(energy_range, tuple) and len(energy_range) == 2: - return [energy_range] - - if isinstance(energy_range, list): - if len(energy_range) == 2 and all(hasattr(item, "to") for item in energy_range): - return [(energy_range[0], energy_range[1])] - if all(isinstance(item, (list, tuple)) and len(item) == 2 for item in energy_range): - return [tuple(item) for item in energy_range] - - raise ValueError( - "energy_range must be one pair (e_min, e_max) or a list of (e_min, e_max) pairs." - ) - - -def resolve_array_layout_name(array_layout_name, model_version): - """Resolve array layout configuration for a specific model version.""" - if isinstance(array_layout_name, list) and len(array_layout_name) == 1: - array_layout_name = array_layout_name[0] - - if isinstance(array_layout_name, str) and array_layout_name.strip().startswith("{"): - try: - parsed_layout = ast.literal_eval(array_layout_name) - if isinstance(parsed_layout, dict): - array_layout_name = parsed_layout - except (SyntaxError, ValueError): - return array_layout_name - - if not isinstance(array_layout_name, dict) or list(array_layout_name) != ["by_version"]: - return array_layout_name - - resolved = simtools_version.resolve_by_version( - {"array_layout_name": array_layout_name}, model_version - ) - return resolved["array_layout_name"] - - -def get_energy_range_for_zenith_angle(zenith_angle, energy_range_pair, corsika_limits): - """Return a zenith-dependent energy range pair or None to skip the simulation step.""" - _ = (zenith_angle, corsika_limits) - return energy_range_pair - - -def get_core_scatter_max_for_zenith_angle(zenith_angle, core_scatter, corsika_limits): - """Return zenith-dependent max core-scatter value.""" - _ = (zenith_angle, corsika_limits) - return core_scatter[1] +from simtools.production_configuration.production_grid_job_rows import ( + build_backend_agnostic_job_rows, + get_core_scatter_max_for_zenith_angle, + get_energy_range_for_zenith_angle, +) def calculate_log_energy_midpoint(energy_range_pair): @@ -147,99 +60,24 @@ def calculate_scaled_nshow( return scaled_nshow -def _select_energy_and_core_scatter_for_job( - zenith, energy_range_pair, core_scatter, corsika_limits -): - """Return selected energy range and core scatter maximum for a job spec row.""" - selected_energy_range_pair = energy_range_pair - selected_core_scatter_max = core_scatter[1] - - if corsika_limits is None: - return selected_energy_range_pair, selected_core_scatter_max - - selected_energy_range_pair = get_energy_range_for_zenith_angle( - zenith, energy_range_pair, corsika_limits - ) - if selected_energy_range_pair is None: - return None, None - - selected_core_scatter_max = get_core_scatter_max_for_zenith_angle( - zenith, core_scatter, corsika_limits - ) - return selected_energy_range_pair, selected_core_scatter_max - - def build_job_specs(args_dict, image_labels): """Build backend-agnostic job specs from comparison and production grids.""" - grid_axes = normalize_grid_axes(args_dict) - energy_ranges = normalize_energy_ranges(args_dict["energy_range"]) base_pack_dir = args_dict.get("simulation_output") or "simtools-output" - corsika_limits = args_dict.get("corsika_limits") - core_scatter = args_dict["core_scatter"] - nshow = args_dict["nshow"] - nshow_power_index = args_dict.get("nshow_power_index") - reference_energy = args_dict.get("nshow_reference_energy") - if nshow_power_index is not None and reference_energy is not None: - reference_energy = u.Quantity(reference_energy) - - combinations = list( - itertools.product( - grid_axes["primary"], - grid_axes["azimuth_angle"], - grid_axes["zenith_angle"], - grid_axes["model_version"], - grid_axes["corsika_le_interaction"], - grid_axes["corsika_he_interaction"], - energy_ranges, - ) + normalized_rows = build_backend_agnostic_job_rows( + args_dict, + calculate_scaled_nshow, + get_energy_range_for_zenith_angle_function=get_energy_range_for_zenith_angle, + get_core_scatter_max_for_zenith_angle_function=get_core_scatter_max_for_zenith_angle, ) - number_of_runs = args_dict.get("number_of_runs", 1) - run_number = int(args_dict.get("run_number") or 1) - job_specs = [] for label in image_labels: - row_index = 0 - for ( - primary, - azimuth, - zenith, - model_version, - corsika_le, - corsika_he, - energy_range_pair, - ) in combinations: - ( - selected_energy_range_pair, - selected_core_scatter_max, - ) = _select_energy_and_core_scatter_for_job( - zenith, energy_range_pair, core_scatter, corsika_limits + for row in normalized_rows: + job_specs.append( + { + "image_label": str(label), + **row, + "pack_for_grid_register": f"{base_pack_dir}/{label!s}", + } ) - if selected_energy_range_pair is None: - continue - - selected_nshow = calculate_scaled_nshow( - selected_energy_range_pair, nshow, nshow_power_index, reference_energy - ) - - for _ in range(number_of_runs): - job_specs.append( - { - "image_label": str(label), - "primary": primary, - "azimuth_angle": azimuth, - "zenith_angle": zenith, - "model_version": model_version, - "array_layout_name": args_dict.get("array_layout_name"), - "corsika_le_interaction": corsika_le, - "corsika_he_interaction": corsika_he, - "energy_min": selected_energy_range_pair[0], - "energy_max": selected_energy_range_pair[1], - "core_scatter_max": selected_core_scatter_max, - "nshow": selected_nshow, - "pack_for_grid_register": f"{base_pack_dir}/{label!s}", - "run_number": run_number + row_index, - } - ) - row_index += 1 return job_specs diff --git a/src/simtools/production_configuration/production_grid_engine.py b/src/simtools/production_configuration/production_grid_engine.py new file mode 100644 index 0000000000..5da96005b8 --- /dev/null +++ b/src/simtools/production_configuration/production_grid_engine.py @@ -0,0 +1,523 @@ +"""Backend-independent engine for simulation production-grid generation.""" + +import logging + +import numpy as np +from astropy import units as u +from astropy.coordinates import AltAz, EarthLocation, SkyCoord +from astropy.units import Quantity + +from simtools.production_configuration.corsika_limits_lookup import CorsikaLimitsLookup +from simtools.production_configuration.production_grid_helpers import ( + DEFAULT_SERIALIZATION_ROUND_DECIMALS, +) +from simtools.production_configuration.production_grid_serialization import ( + build_grid_metadata, + build_serialized_rows, + collect_point_keys, + serialize_grid_points, + serialize_grid_value, +) + + +class ProductionGridEngine: + """ + Backend-independent engine for simulation production-grid generation. + + This engine handles axis expansion, coordinate-system specific grid + generation, lookup-table interpolation, coordinate conversion, and ECSV + serialization helpers used by backend adapters. + """ + + def __init__( + self, + axes, + coordinate_system="zenith_azimuth", + observing_location=None, + observing_time=None, + lookup_table=None, + telescope_ids=None, + simtel_file=None, + ): + """ + Initialize the production-grid engine. + + Parameters + ---------- + axes : dict + Dictionary where each key is the axis name and the value is a dictionary + defining the axis properties (range, binning, scaling, etc.). + coordinate_system : str, optional + The coordinate system for the grid generation. + observing_location : EarthLocation, optional + The location of the observation. + observing_time : Time, optional + The observing time used for RA/Dec transforms. + lookup_table : str, optional + Path to the lookup table file (ECSV format). + telescope_ids : list of str, optional + Telescope selection used to filter lookup rows. + simtel_file : str, optional + Path to a sim_telarray file used to map numeric telescope IDs. + """ + self._logger = logging.getLogger(__name__) + self.axes = axes["axes"] if "axes" in axes else axes + self.coordinate_system = coordinate_system + self.observing_location = ( + observing_location + if observing_location is not None + else EarthLocation(lat=0.0 * u.deg, lon=0.0 * u.deg, height=0 * u.m) + ) + self.observing_time = observing_time + self.lookup_table = lookup_table + self.telescope_ids = telescope_ids + self.simtel_file = simtel_file + self.interpolated_limits = {} + self.serialization_round_decimals = DEFAULT_SERIALIZATION_ROUND_DECIMALS + self._limits_lookup = CorsikaLimitsLookup( + lookup_table=lookup_table, + telescope_ids=telescope_ids, + simtel_file=simtel_file, + ) + self._simtel_id_to_name = self._limits_lookup.simtel_id_to_name + self.target_values = self._generate_target_values() + + if self.lookup_table: + if self.coordinate_system == "ra_dec": + self._prepare_lookup_table_limits_for_point_interpolation() + else: + self._apply_lookup_table_limits() + + @staticmethod + def _coerce_identifier_container(value): + """Coerce identifier input into a list.""" + return CorsikaLimitsLookup.coerce_identifier_container(value) + + def _normalize_lookup_identifier(self, identifier): + """Normalize one telescope identifier and report if it is numeric.""" + return self._limits_lookup.normalize_lookup_identifier(identifier) + + def _normalized_identifier_set(self, identifiers): + """Return normalized telescope identifiers as a set.""" + return self._limits_lookup.normalized_identifier_set(identifiers) + + def _lookup_contains_numeric_telescope_ids(self, lookup_table): + """Return True when any lookup-table telescope identifier is numeric.""" + return self._limits_lookup.lookup_contains_numeric_telescope_ids(lookup_table) + + def _sync_limits_lookup(self): + """Synchronize mutable lookup settings with the shared lookup helper.""" + self._limits_lookup.lookup_table = self.lookup_table + self._limits_lookup.telescope_ids = self.telescope_ids + self._limits_lookup.simtel_file = self.simtel_file + self._limits_lookup.simtel_id_to_name = self._simtel_id_to_name + + def _load_matching_lookup_arrays(self): + """Load and filter lookup-table arrays for selected telescope IDs.""" + self._sync_limits_lookup() + return self._limits_lookup.load_matching_lookup_arrays() + + def _require_observing_time(self): + """Return observing time if available, else raise a clear error.""" + if self.observing_time is None: + raise ValueError("Observing time is required for ra_dec grid generation.") + return self.observing_time + + def _get_max_zenith_for_radec_mode(self): + """Read maximum zenith from axes for RA/Dec direction sampling.""" + zenith_axis = self.axes.get("zenith_angle") + if not zenith_axis or "range" not in zenith_axis or len(zenith_axis["range"]) != 2: + raise ValueError( + "RA/Dec direction sampling requires 'zenith_angle' axis with a valid " + "two-element 'range' in the axes definition." + ) + return float(zenith_axis["range"][1]) + + def _prepare_lookup_table_limits_for_point_interpolation(self): + """Prepare lookup arrays for per-point interpolation in RA/Dec grid mode.""" + self._sync_limits_lookup() + self.lookup_interpolators_for_point = self._limits_lookup.prepare_point_interpolators() + self.lookup_points_for_interpolation = self._limits_lookup.lookup_points_for_interpolation + self.lookup_values_for_interpolation = self._limits_lookup.lookup_values_for_interpolation + + def _has_radec_axes(self): + """Return True if axes define a native RA/Dec grid.""" + return "ra" in self.axes and "dec" in self.axes + + def _generate_target_values(self): + """ + Generate target axis values and store them as Quantities. + + Returns + ------- + dict + Dictionary of target values for each axis, stored as Quantity objects. + """ + target_values = {} + for axis_name, axis in self.axes.items(): + axis_range = axis["range"] + binning = axis["binning"] + scaling = axis.get("scaling", "linear") + units = axis.get("units", None) + + if axis_name == "azimuth": + values = self.create_circular_binning(axis_range, binning) + elif scaling == "log": + values = np.logspace(np.log10(axis_range[0]), np.log10(axis_range[1]), binning) + elif scaling == "1/cos": + cos_min = np.cos(np.radians(axis_range[0])) + cos_max = np.cos(np.radians(axis_range[1])) + inv_cos_values = np.linspace(1 / cos_min, 1 / cos_max, binning) + values = np.degrees(np.arccos(1 / inv_cos_values)) + else: + values = np.linspace(axis_range[0], axis_range[1], binning) + + if units: + values = values * u.Unit(units) + + target_values[axis_name] = values + + return target_values + + def _apply_lookup_table_limits(self): + """Apply limits from the lookup table and interpolate values.""" + self._sync_limits_lookup() + self.interpolated_limits = self._limits_lookup.interpolate_grid_limits(self.target_values) + + def _generate_radec_grid_direction_points(self): + """Generate direction points from declination lines and hour-angle spacing.""" + observing_time = self._require_observing_time() + max_zenith = self._get_max_zenith_for_radec_mode() + lst_deg = observing_time.sidereal_time( + "apparent", longitude=self.observing_location.lon + ).deg + + direction_points = [] + for declination in np.arange(-90.0, 91.0, 1.0): + cos_dec = np.cos(np.deg2rad(declination)) + step_ha = 1.0 / cos_dec if cos_dec > 1e-6 else 360.0 + n_ha = max(1, int(np.ceil(360.0 / step_ha))) + hour_angles = np.linspace(-180.0, 180.0, n_ha, endpoint=False) + ra_values = (lst_deg - hour_angles) % 360.0 + + skycoord = SkyCoord( + ra=ra_values * u.deg, + dec=np.full_like(ra_values, declination) * u.deg, + frame="icrs", + ) + altaz = skycoord.transform_to( + AltAz(location=self.observing_location, obstime=observing_time) + ) + + zenith_values = (90.0 * u.deg - altaz.alt).to(u.deg).value + mask = (zenith_values >= 0.0) & (zenith_values <= max_zenith) + + for idx in np.nonzero(mask)[0]: + direction_points.append( + { + "zenith_angle": zenith_values[idx] * u.deg, + "azimuth": altaz.az.deg[idx] * u.deg, + } + ) + return direction_points + + def _generate_extra_axis_combinations(self, excluded_keys): + """Generate combinations for all axes except the excluded ones.""" + extra_axes = { + key: value for key, value in self.target_values.items() if key not in excluded_keys + } + if not extra_axes: + return list(extra_axes.keys()), [], [np.array([])] + + extra_value_arrays = [value.value for value in extra_axes.values()] + extra_units = [value.unit for value in extra_axes.values()] + extra_grid = np.meshgrid(*extra_value_arrays, indexing="ij") + extra_combinations = np.vstack(list(map(np.ravel, extra_grid))).T + return list(extra_axes.keys()), extra_units, extra_combinations + + def _interpolate_limits_for_point(self, zenith, azimuth, nsb): + """Interpolate lookup-table limits for a single point.""" + self._sync_limits_lookup() + return self._limits_lookup.interpolate_point(zenith, azimuth, nsb) + + def _add_lookup_limits_to_point(self, point, zenith, azimuth): + """Interpolate and attach lookup-table limits to a grid point.""" + if not self.lookup_table: + return + + nsb_value = point.get("nsb_level", 1) + if isinstance(nsb_value, Quantity): + nsb_value = nsb_value.value + limits = self._interpolate_limits_for_point( + zenith=zenith, + azimuth=azimuth, + nsb=float(nsb_value), + ) + point["lower_energy_threshold"] = limits["lower_energy_threshold"] * u.TeV + point["scatter_radius"] = limits["upper_scatter_radius"] * u.m + point["viewcone_radius"] = limits["viewcone_radius"] * u.deg + + def _generate_grid_from_radec_axes(self): + """Generate grid points from explicit RA/Dec axes definitions.""" + observing_time = self._require_observing_time() + + axis_keys = [key for key in self.target_values if key not in ("zenith_angle", "azimuth")] + value_arrays = [self.target_values[key].value for key in axis_keys] + units = [self.target_values[key].unit for key in axis_keys] + grid = np.meshgrid(*value_arrays, indexing="ij") + combinations = np.vstack(list(map(np.ravel, grid))).T + + grid_points = [] + for combination in combinations: + grid_point = { + key: Quantity(combination[i], units[i]) for i, key in enumerate(axis_keys) + } + + skycoord = SkyCoord( + ra=grid_point["ra"].to(u.deg), + dec=grid_point["dec"].to(u.deg), + frame="icrs", + ) + altaz = skycoord.transform_to( + AltAz(location=self.observing_location, obstime=observing_time) + ) + zenith = (90.0 * u.deg - altaz.alt).to(u.deg).value + + self._add_lookup_limits_to_point(grid_point, zenith=zenith, azimuth=altaz.az.deg) + grid_points.append(grid_point) + + return grid_points + + def _generate_grid_radec_mode(self): + """Generate grid points for RA/Dec mode.""" + if self._has_radec_axes(): + return self._generate_grid_from_radec_axes() + + direction_points = self._generate_radec_grid_direction_points() + extra_keys, extra_units, extra_combinations = self._generate_extra_axis_combinations( + excluded_keys=("zenith_angle", "azimuth") + ) + + grid_points = [] + for direction_point in direction_points: + for extra_combination in extra_combinations: + point = dict(direction_point) + for i, key in enumerate(extra_keys): + point[key] = Quantity(extra_combination[i], extra_units[i]) + + self._add_lookup_limits_to_point( + point, + zenith=point["zenith_angle"].value, + azimuth=point["azimuth"].value, + ) + grid_points.append(point) + + return grid_points + + def create_circular_binning(self, azimuth_range, num_bins): + """ + Create bin centers for azimuth angles, handling circular wraparound. + + Parameters + ---------- + azimuth_range : tuple + (min_azimuth, max_azimuth), can wrap around 0 deg. + num_bins : int + Number of bins. + + Returns + ------- + np.ndarray + Array of bin centers. + """ + azimuth_min, azimuth_max = azimuth_range + azimuth_min %= 360 + azimuth_max %= 360 + + clockwise_distance = (azimuth_max - azimuth_min) % 360 + counterclockwise_distance = (azimuth_min - azimuth_max) % 360 + + if clockwise_distance <= counterclockwise_distance: + return ( + np.linspace(azimuth_min, azimuth_min + clockwise_distance, num_bins, endpoint=True) + % 360 + ) + return ( + np.linspace( + azimuth_min, azimuth_min - counterclockwise_distance, num_bins, endpoint=True + ) + % 360 + ) + + def generate_grid(self): + """ + Generate the grid based on the required axes and include interpolated limits. + + Returns + ------- + list of dict + A list of generated grid points. + """ + if self.coordinate_system == "ra_dec": + return self._generate_grid_radec_mode() + + value_arrays = [value.value for value in self.target_values.values()] + units = [value.unit for value in self.target_values.values()] + grid = np.meshgrid(*value_arrays, indexing="ij") + combinations = np.vstack(list(map(np.ravel, grid))).T + grid_points = [] + + for combination in combinations: + grid_point = { + key: Quantity(combination[i], units[i]) + for i, key in enumerate(self.target_values.keys()) + } + + if "lower_energy_threshold" in self.interpolated_limits: + zenith_idx = np.searchsorted( + self.target_values["zenith_angle"].value, grid_point["zenith_angle"].value + ) + azimuth_idx = np.searchsorted( + self.target_values["azimuth"].value, grid_point["azimuth"].value + ) + nsb_idx = np.searchsorted( + self.target_values["nsb_level"].value, + grid_point["nsb_level"].value, + ) + grid_point["lower_energy_threshold"] = ( + self.interpolated_limits["lower_energy_threshold"][ + zenith_idx, azimuth_idx, nsb_idx + ] + * u.TeV + ) + + if "upper_scatter_radius" in self.interpolated_limits: + grid_point["scatter_radius"] = ( + self.interpolated_limits["upper_scatter_radius"][ + zenith_idx, azimuth_idx, nsb_idx + ] + * u.m + ) + + if "viewcone_radius" in self.interpolated_limits: + grid_point["viewcone_radius"] = ( + self.interpolated_limits["viewcone_radius"][zenith_idx, azimuth_idx, nsb_idx] + * u.deg + ) + + grid_points.append(grid_point) + + return grid_points + + def convert_altaz_to_radec(self, alt, az): + """ + Convert Altitude/Azimuth (AltAz) coordinates to RA/Dec. + + Parameters + ---------- + alt : float + Altitude angle in degrees. + az : float + Azimuth angle in degrees. + + Returns + ------- + SkyCoord + SkyCoord object containing the RA/Dec coordinates. + """ + if self.observing_time is None: + raise ValueError( + "Observing time is not set. " + "Please provide an observing_time to convert coordinates." + ) + + alt_rad = alt.to(u.rad) + az_rad = az.to(u.rad) + aa = AltAz( + alt=alt_rad, + az=az_rad, + location=self.observing_location, + obstime=self.observing_time, + ) + skycoord = SkyCoord(aa) + return skycoord.icrs + + def convert_coordinates(self, grid_points): + """ + Convert the grid points to RA/Dec coordinates if necessary. + + Parameters + ---------- + grid_points : list of dict + List of grid points. + + Returns + ------- + list of dict + The grid points with converted RA/Dec coordinates. + """ + if self.coordinate_system == "ra_dec": + for point in grid_points: + if "zenith_angle" in point and "azimuth" in point: + alt = (90.0 * u.deg) - point.pop("zenith_angle") + az = point.pop("azimuth") + radec = self.convert_altaz_to_radec(alt, az) + point["ra"] = radec.ra.deg * u.deg + point["dec"] = radec.dec.deg * u.deg + return grid_points + + def serialize_grid_points(self, grid_points, output_file): + """Serialize the grid output and save to an ECSV table file.""" + serialize_grid_points( + grid_points=grid_points, + output_file=output_file, + coordinate_system=self.coordinate_system, + observing_time=self.observing_time, + telescope_ids=self.telescope_ids, + lookup_table=self.lookup_table, + serialization_round_decimals=self.serialization_round_decimals, + ) + self._logger.info(f"Output saved to {output_file}") + + @staticmethod + def _collect_point_keys(grid_points): + """Collect all grid-point keys while preserving first-seen order.""" + return collect_point_keys(grid_points) + + def _serialize_grid_value(self, value): + """Serialize one grid value and return (value, unit).""" + return serialize_grid_value( + value, + serialization_round_decimals=self.serialization_round_decimals, + ) + + def _build_serialized_rows(self, grid_points, all_keys): + """Build serialized row dictionaries and collect units.""" + return build_serialized_rows( + grid_points, + all_keys, + serialization_round_decimals=self.serialization_round_decimals, + ) + + def _build_grid_metadata(self): + """Build metadata for the output grid table.""" + return build_grid_metadata( + coordinate_system=self.coordinate_system, + observing_time=self.observing_time, + telescope_ids=self.telescope_ids, + lookup_table=self.lookup_table, + ) + + def serialize_quantity(self, value): + """Serialize Quantity.""" + if isinstance(value, u.Quantity): + serialized_value = float(value.value) + rounded_value = round(serialized_value, self.serialization_round_decimals) + return {"value": rounded_value, "unit": str(value.unit)} + if isinstance(value, float): + return round(value, self.serialization_round_decimals) + if isinstance(value, np.floating): + return round(float(value), self.serialization_round_decimals) + if isinstance(value, np.integer): + return int(value) + return value diff --git a/src/simtools/production_configuration/production_grid_helpers.py b/src/simtools/production_configuration/production_grid_helpers.py new file mode 100644 index 0000000000..2779cccdfe --- /dev/null +++ b/src/simtools/production_configuration/production_grid_helpers.py @@ -0,0 +1,60 @@ +"""Helper functions and constants for production-grid generation.""" + +from astropy import units as u + +from simtools.configuration import defaults +from simtools.utils.general import ensure_list + +DEFAULT_SERIALIZATION_ROUND_DECIMALS = 6 + +_GRID_AXES = [ + "primary", + "azimuth_angle", + "zenith_angle", + "model_version", + "corsika_le_interaction", + "corsika_he_interaction", +] + +_GRID_AXIS_DEFAULTS = { + "corsika_le_interaction": defaults.CORSIKA_LE_INTERACTION, + "corsika_he_interaction": defaults.CORSIKA_HE_INTERACTION, +} + + +def normalize_grid_axes(args_dict): + """Return normalized grid axes for cartesian product expansion.""" + return { + axis: ( + ensure_list(args_dict[axis]) + if axis in args_dict and args_dict[axis] is not None + else [_GRID_AXIS_DEFAULTS[axis]] + if axis in _GRID_AXIS_DEFAULTS + else [None] + ) + for axis in _GRID_AXES + } + + +def normalize_energy_ranges(energy_range): + """Normalize energy range argument to a list of ``(e_min, e_max)`` pairs.""" + if isinstance(energy_range, tuple) and len(energy_range) == 2: + return [energy_range] + + if isinstance(energy_range, list): + if len(energy_range) == 2 and all(hasattr(item, "to") for item in energy_range): + return [(energy_range[0], energy_range[1])] + if all(isinstance(item, (list, tuple)) and len(item) == 2 for item in energy_range): + return [tuple(item) for item in energy_range] + + raise ValueError( + "energy_range must be one pair (e_min, e_max) or a list of (e_min, e_max) pairs." + ) + + +def normalize_azimuth_angle(azimuth_angle): + """Return an azimuth angle with degree units and modulo-360 normalization.""" + # TODO - duplication? + if isinstance(azimuth_angle, u.Quantity): + return azimuth_angle.to(u.deg) % (360 * u.deg) + return azimuth_angle % 360.0 diff --git a/src/simtools/production_configuration/production_grid_job_rows.py b/src/simtools/production_configuration/production_grid_job_rows.py new file mode 100644 index 0000000000..dfd1c200cb --- /dev/null +++ b/src/simtools/production_configuration/production_grid_job_rows.py @@ -0,0 +1,175 @@ +""" +Builds backend-agnostic job row dictionaries for simulation production grids. + +A job row is a configuration dictionary for a single simulation job/run, containing all +parameters needed to launch that simulation (e.g., primary, zenith, azimuth, energy range, +model version, etc.). +""" + +import itertools + +from astropy import units as u + +from simtools.production_configuration.corsika_limits_lookup import CorsikaLimitsLookup +from simtools.production_configuration.production_grid_helpers import ( + normalize_energy_ranges, + normalize_grid_axes, +) + + +def get_energy_range_for_zenith_angle( + zenith_angle, energy_range_pair, corsika_limits, azimuth_angle=None, nsb_level=1.0 +): + """ + Return a zenith-dependent energy range pair or ``None`` to skip the step. + + The lower energy bound is clipped to the lookup-table threshold for the + requested direction. If the threshold exceeds the configured upper bound, + the simulation step is skipped. + """ + if corsika_limits is None: + return energy_range_pair + + if not isinstance(corsika_limits, CorsikaLimitsLookup): + corsika_limits = CorsikaLimitsLookup(corsika_limits) + + azimuth_angle = 0.0 * u.deg if azimuth_angle is None else azimuth_angle + interpolated_limits = corsika_limits.interpolate_point(zenith_angle, azimuth_angle, nsb_level) + lower_energy_threshold = interpolated_limits["lower_energy_threshold"] * u.TeV + + energy_min, energy_max = energy_range_pair + if lower_energy_threshold > energy_max.to(lower_energy_threshold.unit): + return None + if lower_energy_threshold <= energy_min.to(lower_energy_threshold.unit): + return energy_range_pair + return lower_energy_threshold.to(energy_min.unit), energy_max + + +def get_core_scatter_max_for_zenith_angle( + zenith_angle, core_scatter, corsika_limits, azimuth_angle=None, nsb_level=1.0 +): + """ + Return zenith-dependent max core-scatter value. + + The lookup-table scatter radius is treated as an upper limit and therefore + clipped against the user-provided maximum value. + """ + if corsika_limits is None: + return core_scatter[1] + + if not isinstance(corsika_limits, CorsikaLimitsLookup): + corsika_limits = CorsikaLimitsLookup(corsika_limits) + + azimuth_angle = 0.0 * u.deg if azimuth_angle is None else azimuth_angle + interpolated_limits = corsika_limits.interpolate_point(zenith_angle, azimuth_angle, nsb_level) + lookup_scatter_max = interpolated_limits["upper_scatter_radius"] * u.m + configured_scatter_max = core_scatter[1] + return min(configured_scatter_max, lookup_scatter_max.to(configured_scatter_max.unit)) + + +def build_backend_agnostic_job_rows( + args_dict, + calculate_scaled_nshow, + get_energy_range_for_zenith_angle_function=get_energy_range_for_zenith_angle, + get_core_scatter_max_for_zenith_angle_function=get_core_scatter_max_for_zenith_angle, +): + """ + Build normalized production-grid rows for backend consumers. + + Parameters + ---------- + args_dict : dict + Production-job configuration. + calculate_scaled_nshow : callable + Callback used to compute the per-row ``nshow`` value. + get_energy_range_for_zenith_angle_function : callable, optional + Callback used to derive the direction-dependent energy range. + get_core_scatter_max_for_zenith_angle_function : callable, optional + Callback used to derive the direction-dependent core-scatter radius. + + Returns + ------- + list[dict] + Backend-independent row dictionaries. + """ + grid_axes = normalize_grid_axes(args_dict) + energy_ranges = normalize_energy_ranges(args_dict["energy_range"]) + corsika_limits = args_dict.get("corsika_limits") + if corsika_limits is not None: + corsika_limits = CorsikaLimitsLookup( + corsika_limits, + telescope_ids=args_dict.get("telescope_ids"), + simtel_file=args_dict.get("simtel_file"), + ) + + core_scatter = args_dict["core_scatter"] + nshow = args_dict["nshow"] + nshow_power_index = args_dict.get("nshow_power_index") + reference_energy = args_dict.get("nshow_reference_energy") + if nshow_power_index is not None and reference_energy is not None: + reference_energy = u.Quantity(reference_energy) + + combinations = list( + itertools.product( + grid_axes["primary"], + grid_axes["azimuth_angle"], + grid_axes["zenith_angle"], + grid_axes["model_version"], + grid_axes["corsika_le_interaction"], + grid_axes["corsika_he_interaction"], + energy_ranges, + ) + ) + + number_of_runs = args_dict.get("number_of_runs", 1) + run_number = int(args_dict.get("run_number") or 1) + + rows = [] + row_index = 0 + for ( + primary, + azimuth, + zenith, + model_version, + corsika_le, + corsika_he, + energy_range_pair, + ) in combinations: + selected_energy_range_pair = get_energy_range_for_zenith_angle_function( + zenith, + energy_range_pair, + corsika_limits, + azimuth_angle=azimuth, + ) + if selected_energy_range_pair is None: + continue + + selected_core_scatter_max = get_core_scatter_max_for_zenith_angle_function( + zenith, + core_scatter, + corsika_limits, + azimuth_angle=azimuth, + ) + selected_nshow = calculate_scaled_nshow( + selected_energy_range_pair, nshow, nshow_power_index, reference_energy + ) + + for _ in range(number_of_runs): + rows.append( + { + "primary": primary, + "azimuth_angle": azimuth, + "zenith_angle": zenith, + "model_version": model_version, + "array_layout_name": args_dict.get("array_layout_name"), + "corsika_le_interaction": corsika_le, + "corsika_he_interaction": corsika_he, + "energy_min": selected_energy_range_pair[0], + "energy_max": selected_energy_range_pair[1], + "core_scatter_max": selected_core_scatter_max, + "nshow": selected_nshow, + "run_number": run_number + row_index, + } + ) + row_index += 1 + return rows diff --git a/src/simtools/production_configuration/production_grid_serialization.py b/src/simtools/production_configuration/production_grid_serialization.py new file mode 100644 index 0000000000..6553c2243b --- /dev/null +++ b/src/simtools/production_configuration/production_grid_serialization.py @@ -0,0 +1,136 @@ +"""Serialization helpers for production-grid rows.""" + +from pathlib import Path + +import numpy as np +from astropy import units as u +from astropy.table import Table + +from simtools.production_configuration.production_grid_helpers import ( + DEFAULT_SERIALIZATION_ROUND_DECIMALS, +) + + +def collect_point_keys(grid_points): + """Collect all grid-point keys while preserving first-seen order.""" + all_keys = [] + for point in grid_points: + for key in point: + if key not in all_keys: + all_keys.append(key) + return all_keys + + +def serialize_grid_value(value, serialization_round_decimals=DEFAULT_SERIALIZATION_ROUND_DECIMALS): + """Serialize one grid value and return ``(value, unit)``.""" + if isinstance(value, u.Quantity): + serialized = round(float(value.value), serialization_round_decimals) + return serialized, str(value.unit) + + if isinstance(value, dict) and "value" in value: + return value["value"], value.get("unit") + + if value is None: + return np.nan, None + + if isinstance(value, (np.floating, float)): + return round(float(value), serialization_round_decimals), None + + if isinstance(value, (np.integer, int)): + return int(value), None + + return value, None + + +def build_serialized_rows( + grid_points, all_keys, serialization_round_decimals=DEFAULT_SERIALIZATION_ROUND_DECIMALS +): + """Build serialized row dictionaries and collect units.""" + rows = [] + units = {} + + for point in grid_points: + row = {} + for key in all_keys: + serialized_value, unit = serialize_grid_value( + point.get(key), + serialization_round_decimals=serialization_round_decimals, + ) + row[key] = serialized_value + if unit is not None: + units.setdefault(key, unit) + rows.append(row) + + return rows, units + + +def build_grid_metadata( + coordinate_system, observing_time=None, telescope_ids=None, lookup_table=None +): + """Build metadata for a serialized production grid.""" + return { + "coordinate_system": coordinate_system, + "reference_frame": "ICRS (J2000)", + "observing_time_utc": observing_time.isot if observing_time else None, + "observing_time_scale": observing_time.scale if observing_time else None, + "telescope_ids": telescope_ids, + "lookup_table": str(Path(lookup_table)) if lookup_table else None, + } + + +def serialize_grid_points( + grid_points, + output_file, + coordinate_system, + observing_time=None, + telescope_ids=None, + lookup_table=None, + serialization_round_decimals=DEFAULT_SERIALIZATION_ROUND_DECIMALS, +): + """ + Serialize grid points to an ECSV table file. + + Parameters + ---------- + grid_points : list[dict] + Grid rows to serialize. + output_file : str or Path + Output ECSV file path. + coordinate_system : str + Coordinate-system label stored in metadata. + observing_time : Time, optional + Observing time stored in metadata. + telescope_ids : list, optional + Telescope selection stored in metadata. + lookup_table : str or Path, optional + Lookup-table path stored in metadata. + serialization_round_decimals : int, optional + Number of decimal places used for scalar serialization. + + Returns + ------- + astropy.table.Table + Serialized output table. + """ + if Path(output_file).suffix.lower() != ".ecsv": + raise ValueError("Grid output file must use '.ecsv' extension.") + + all_keys = collect_point_keys(grid_points) + rows, units = build_serialized_rows( + grid_points, + all_keys, + serialization_round_decimals=serialization_round_decimals, + ) + + output_table = Table(rows=rows, names=all_keys) + for column_name, unit in units.items(): + output_table[column_name].unit = u.Unit(unit) + + output_table.meta = build_grid_metadata( + coordinate_system=coordinate_system, + observing_time=observing_time, + telescope_ids=telescope_ids, + lookup_table=lookup_table, + ) + output_table.write(output_file, format="ascii.ecsv", overwrite=True) + return output_table diff --git a/src/simtools/testing/log_inspector.py b/src/simtools/testing/log_inspector.py index 16085085fb..058f56affd 100644 --- a/src/simtools/testing/log_inspector.py +++ b/src/simtools/testing/log_inspector.py @@ -127,7 +127,7 @@ def check_plain_logs(log_files, file_test): if wanted is None and forbidden is None: return True - log_files = gen.ensure_iterable(log_files) + log_files = gen.ensure_list(log_files) def file_open(file): if file.suffix == ".gz": diff --git a/src/simtools/visualization/plot_simtel_events.py b/src/simtools/visualization/plot_simtel_events.py index 703b1cb41d..7b4bcd4a64 100644 --- a/src/simtools/visualization/plot_simtel_events.py +++ b/src/simtools/visualization/plot_simtel_events.py @@ -59,7 +59,7 @@ def generate_and_save_plots(plots, args, ioh): plotter = PlotSimtelEvent( file_name=args.get("simtel_file", None), telescope=args.get("telescope", None), - event_ids=gen.ensure_iterable(args.get("event_id", None)), + event_ids=gen.ensure_list(args.get("event_id", None)), max_events=args.get("max_events", None), ) output_file = plotter.make_output_paths(ioh, args.get("output_file")) @@ -244,7 +244,7 @@ def _plots_to_run(self, plots): """Generate list of plots to run based on user input.""" if "all" in plots: return list(self._plot_definitions.keys()) - return gen.ensure_iterable(plots) + return gen.ensure_list(plots) @property def _plot_definitions(self): diff --git a/tests/unit_tests/production_configuration/test_generate_production_grid.py b/tests/unit_tests/production_configuration/test_generate_production_grid.py index fd5ea7ca55..31d49a9a16 100644 --- a/tests/unit_tests/production_configuration/test_generate_production_grid.py +++ b/tests/unit_tests/production_configuration/test_generate_production_grid.py @@ -355,7 +355,7 @@ def _raise_qhull(*args, **kwargs): raise QhullError("mocked sparse triangulation failure") monkeypatch.setattr( - "simtools.production_configuration.generate_production_grid.LinearNDInterpolator", + "simtools.production_configuration.corsika_limits_lookup.LinearNDInterpolator", _raise_qhull, ) diff --git a/tests/unit_tests/production_configuration/test_job_spec_builder.py b/tests/unit_tests/production_configuration/test_job_spec_builder.py index 1e5321df13..8a378decfb 100644 --- a/tests/unit_tests/production_configuration/test_job_spec_builder.py +++ b/tests/unit_tests/production_configuration/test_job_spec_builder.py @@ -3,15 +3,17 @@ import astropy.units as u import pytest from astropy.tests.helper import assert_quantity_allclose +from simtools.production_configuration.production_grid import CorsikaLimitsLookup from simtools.production_configuration.job_spec_builder import ( build_job_specs, calculate_log_energy_midpoint, calculate_scaled_nshow, + get_core_scatter_max_for_zenith_angle, + get_energy_range_for_zenith_angle, normalize_energy_ranges, normalize_grid_axes, normalize_to_list, - resolve_array_layout_name, ) @@ -33,6 +35,11 @@ def args_dict(): } +@pytest.fixture +def corsika_limits(): + return "tests/resources/corsika_simulation_limits/merged_corsika_limits_for_test.ecsv" + + def test_normalize_energy_ranges_expands_list_of_pairs(): energy_ranges = normalize_energy_ranges( [ @@ -218,8 +225,11 @@ def test_build_job_specs_skips_entries_when_energy_range_is_none(mock_energy_ran "simtools.production_configuration.job_spec_builder.calculate_scaled_nshow", return_value=777, ) -def test_build_job_specs_uses_dummy_nshow_when_corsika_limits_set(mock_nshow, args_dict): - args_dict["corsika_limits"] = "limits.ecsv" +def test_build_job_specs_uses_dummy_nshow_when_corsika_limits_set( + mock_nshow, args_dict, corsika_limits +): + args_dict["corsika_limits"] = corsika_limits + args_dict["telescope_ids"] = ["LSTN-01"] args_dict["number_of_runs"] = 1 job_specs = build_job_specs(args_dict, ["7.0.0"]) @@ -229,24 +239,57 @@ def test_build_job_specs_uses_dummy_nshow_when_corsika_limits_set(mock_nshow, ar mock_nshow.assert_called_once() -def test_resolve_array_layout_name_resolves_stringified_by_version_layout(): - array_layout_name = str( - { - "by_version": { - "<7.0.0": "alpha", - ">=7.0.0": "CTAO-North-Alpha", - } - } +def test_get_energy_range_for_zenith_angle_clips_to_lookup_threshold(corsika_limits): + lookup = CorsikaLimitsLookup(corsika_limits, telescope_ids=["LSTN-01"]) + + selected_energy_range = get_energy_range_for_zenith_angle( + 20 * u.deg, + (1 * u.GeV, 10 * u.GeV), + lookup, + azimuth_angle=0 * u.deg, ) - assert resolve_array_layout_name(array_layout_name, "7.0.0") == "CTAO-North-Alpha" + assert_quantity_allclose(selected_energy_range[0], 7 * u.GeV) + assert_quantity_allclose(selected_energy_range[1], 10 * u.GeV) -def test_resolve_array_layout_name_unwraps_single_item_list(): - assert resolve_array_layout_name(["CTAO-North-Alpha"], "7.0.0") == "CTAO-North-Alpha" +def test_get_energy_range_for_zenith_angle_returns_none_if_threshold_exceeds_max(corsika_limits): + lookup = CorsikaLimitsLookup(corsika_limits, telescope_ids=["LSTN-01"]) + selected_energy_range = get_energy_range_for_zenith_angle( + 20 * u.deg, + (1 * u.GeV, 5 * u.GeV), + lookup, + azimuth_angle=0 * u.deg, + ) + + assert selected_energy_range is None -def test_resolve_array_layout_name_keeps_invalid_stringified_dict(): - invalid_layout = "{not valid" - assert resolve_array_layout_name(invalid_layout, "7.0.0") == invalid_layout +def test_get_core_scatter_max_for_zenith_angle_clips_to_lookup_radius(corsika_limits): + lookup = CorsikaLimitsLookup(corsika_limits, telescope_ids=["LSTN-01"]) + + selected_core_scatter_max = get_core_scatter_max_for_zenith_angle( + 20 * u.deg, + (10, 1000 * u.m), + lookup, + azimuth_angle=0 * u.deg, + ) + + assert_quantity_allclose(selected_core_scatter_max, 925 * u.m) + + +def test_build_job_specs_reads_limits_from_corsika_limits_file(args_dict, corsika_limits): + args_dict["number_of_runs"] = 1 + args_dict["azimuth_angle"] = 0 * u.deg + args_dict["energy_range"] = (1 * u.GeV, 10 * u.GeV) + args_dict["core_scatter"] = (10, 1000 * u.m) + args_dict["corsika_limits"] = corsika_limits + args_dict["telescope_ids"] = ["LSTN-01"] + + job_specs = build_job_specs(args_dict, ["7.0.0"]) + + assert len(job_specs) == 1 + assert_quantity_allclose(job_specs[0]["energy_min"], 7 * u.GeV) + assert_quantity_allclose(job_specs[0]["energy_max"], 10 * u.GeV) + assert_quantity_allclose(job_specs[0]["core_scatter_max"], 925 * u.m) From 2db96ff827587661fbf59a4925969e2b3623cbd1 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 21:11:57 +0200 Subject: [PATCH 10/44] obsolete --- .../applications/production_generate_grid.py | 4 +- .../generate_production_grid.py | 19 - .../test_generate_production_grid.py | 667 ------------------ 3 files changed, 2 insertions(+), 688 deletions(-) delete mode 100644 src/simtools/production_configuration/generate_production_grid.py delete mode 100644 tests/unit_tests/production_configuration/test_generate_production_grid.py diff --git a/src/simtools/applications/production_generate_grid.py b/src/simtools/applications/production_generate_grid.py index 7331d3f36f..3c3a529780 100644 --- a/src/simtools/applications/production_generate_grid.py +++ b/src/simtools/applications/production_generate_grid.py @@ -77,7 +77,7 @@ from simtools.application_control import build_application from simtools.io.ascii_handler import collect_data_from_file from simtools.model.site_model import SiteModel -from simtools.production_configuration.generate_production_grid import GridGeneration +from simtools.production_configuration.production_grid_engine import ProductionGridEngine def _add_arguments(parser): @@ -190,7 +190,7 @@ def main(): elif coordinate_system == "ra_dec": observing_time = Time.now() - grid_gen = GridGeneration( + grid_gen = ProductionGridEngine( axes=axes, coordinate_system=coordinate_system, observing_location=observing_location, diff --git a/src/simtools/production_configuration/generate_production_grid.py b/src/simtools/production_configuration/generate_production_grid.py deleted file mode 100644 index fb776d6e9f..0000000000 --- a/src/simtools/production_configuration/generate_production_grid.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -Thin adapter exposing the legacy production-grid API. - -The backend-independent implementation lives in -``simtools.production_configuration.production_grid``. -""" - -from simtools.production_configuration.production_grid_engine import ProductionGridEngine - - -class GridGeneration(ProductionGridEngine): - # pylint: disable=too-few-public-methods - """ - Backward-compatible adapter for production-grid generation. - - This class preserves the legacy public API while delegating the full - implementation to - :class:`~simtools.production_configuration.production_grid.ProductionGridEngine`. - """ diff --git a/tests/unit_tests/production_configuration/test_generate_production_grid.py b/tests/unit_tests/production_configuration/test_generate_production_grid.py deleted file mode 100644 index 31d49a9a16..0000000000 --- a/tests/unit_tests/production_configuration/test_generate_production_grid.py +++ /dev/null @@ -1,667 +0,0 @@ -import logging -import warnings -from pathlib import Path - -import numpy as np -import pytest -import yaml -from astropy import units as u -from astropy.coordinates import EarthLocation, SkyCoord -from astropy.table import Table -from astropy.time import Time -from astropy.units import Quantity -from astropy.utils import iers -from astropy.utils.iers import IERSWarning -from scipy.spatial import QhullError - -from simtools.production_configuration.generate_production_grid import GridGeneration - - -@pytest.fixture(autouse=True, scope="module") -def disable_iers_auto_download(): - """Disable IERS auto-download during tests to avoid network dependency.""" - previous_auto_download = iers.conf.auto_download - iers.conf.auto_download = False - try: - yield - finally: - iers.conf.auto_download = previous_auto_download - - -def _create_grid_generation( - axes, - coordinate_system, - observing_location, - observing_time, - lookup_table, - simtel_file=None, -): - """Create a GridGeneration instance with a standard telescope selection.""" - return GridGeneration( - axes=axes, - coordinate_system=coordinate_system, - observing_location=observing_location, - observing_time=observing_time, - lookup_table=lookup_table, - telescope_ids=["LSTN-01"], - simtel_file=simtel_file, - ) - - -def _build_single_point_radec_axes_definition(source_radec): - """Build one-bin RA/Dec axes around a source coordinate.""" - return { - "axes": { - "ra": { - "range": [source_radec.ra.deg, source_radec.ra.deg], - "binning": 1, - "scaling": "linear", - "units": "deg", - }, - "dec": { - "range": [source_radec.dec.deg, source_radec.dec.deg], - "binning": 1, - "scaling": "linear", - "units": "deg", - }, - "nsb_level": { - "range": [4, 4], - "binning": 1, - "scaling": "linear", - "units": "MHz", - }, - } - } - - -@pytest.fixture -def axes_definition(): - """Load the axes definition from the YAML file.""" - axes_file = Path("tests/resources/production_grid_generation_axes_definition.yml") - with open(axes_file) as f: - return yaml.safe_load(f) - - -@pytest.fixture -def lookup_table(): - """Load the lookup table from the resources directory.""" - return str( - Path("tests/resources/corsika_simulation_limits/merged_corsika_limits_for_test.ecsv") - ) - - -@pytest.fixture -def observing_location(): - """Return a mock observing location.""" - latitude = 28.7622 # degrees - longitude = -17.8920 # degrees - return EarthLocation(lon=longitude * u.deg, lat=latitude * u.deg, height=2000 * u.m) - - -@pytest.fixture -def observing_time(): - """Return a mock observing time.""" - return Time("2017-09-16 00:00:00") - - -@pytest.fixture -def grid_gen(axes_definition, lookup_table, observing_location, observing_time): - """Create a GridGeneration object with the provided fixtures.""" - return _create_grid_generation( - axes=axes_definition, - coordinate_system="zenith_azimuth", - observing_location=observing_location, - observing_time=observing_time, - lookup_table=lookup_table, - ) - - -def test_generate_grid(grid_gen): - grid_points = grid_gen.generate_grid() - assert isinstance(grid_points, list) - assert len(grid_points) > 0 - assert all(isinstance(point, dict) for point in grid_points) - assert all("zenith_angle" in point for point in grid_points) - assert all("azimuth" in point for point in grid_points) - assert all("nsb_level" in point for point in grid_points) - - -def test_generate_grid_log_scaling( - axes_definition, lookup_table, observing_location, observing_time -): - """Test grid generation with logarithmic scaling for nsb_level axis.""" - axes_definition["axes"]["nsb_level"] = { - "range": [2, 5], - "binning": 4, - "scaling": "log", - "units": "MHz", - } - - grid_gen = _create_grid_generation( - axes=axes_definition, - coordinate_system="zenith_azimuth", - observing_location=observing_location, - observing_time=observing_time, - lookup_table=lookup_table, - ) - - grid_points = grid_gen.generate_grid() - nsb_values = [point["nsb_level"].value for point in grid_points] - unique_nsb_values = np.unique(nsb_values) - - expected_values = np.logspace( - np.log10(axes_definition["axes"]["nsb_level"]["range"][0]), - np.log10(axes_definition["axes"]["nsb_level"]["range"][1]), - axes_definition["axes"]["nsb_level"]["binning"], - ) - - assert len(unique_nsb_values) == len(expected_values) - assert np.allclose(unique_nsb_values, expected_values, rtol=1e-4) - - -def test_generate_grid_1_over_cos_scaling( - axes_definition, lookup_table, observing_location, observing_time -): - """Test grid generation with 1/cos scaling for zenith_angle axis.""" - axes_definition["axes"]["zenith_angle"] = { - "range": [30, 60], - "binning": 5, - "scaling": "1/cos", - "units": "deg", - } - - grid_gen = _create_grid_generation( - axes=axes_definition, - coordinate_system="zenith_azimuth", - observing_location=observing_location, - observing_time=observing_time, - lookup_table=lookup_table, - ) - - grid_points = grid_gen.generate_grid() - zenith_values = [point["zenith_angle"].value for point in grid_points] - unique_zenith_values = np.unique(zenith_values) - - cos_min = np.cos(np.radians(axes_definition["axes"]["zenith_angle"]["range"][0])) - cos_max = np.cos(np.radians(axes_definition["axes"]["zenith_angle"]["range"][1])) - cos_values = np.linspace( - 1 / cos_min, 1 / cos_max, axes_definition["axes"]["zenith_angle"]["binning"] - ) - expected_values = np.degrees(np.arccos(1 / cos_values)) - - assert len(unique_zenith_values) == len(expected_values) - assert np.allclose(unique_zenith_values, expected_values, rtol=1e-4) - - -def test_generate_grid_radec_mode_minimal(observing_location, observing_time): - """Generate a minimal RA/Dec-native grid and apply zenith cut.""" - axes_definition = { - "axes": { - "zenith_angle": { - "range": [0, 10], - "binning": 2, - "scaling": "linear", - "units": "deg", - }, - "azimuth": { - "range": [0, 10], - "binning": 2, - "scaling": "linear", - "units": "deg", - }, - "nsb_level": { - "range": [4, 4], - "binning": 1, - "scaling": "linear", - "units": "MHz", - }, - } - } - - grid_gen = _create_grid_generation( - axes=axes_definition, - coordinate_system="ra_dec", - observing_location=observing_location, - observing_time=observing_time, - lookup_table=None, - ) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore", IERSWarning) - grid_points = grid_gen.generate_grid() - - assert len(grid_points) > 0 - assert all("zenith_angle" in point and "azimuth" in point for point in grid_points) - assert all(point["zenith_angle"].value <= 10 for point in grid_points) - - -def test_generate_grid_radec_axes_mode(observing_location, observing_time): - """Generate a grid directly from explicit RA/Dec axes.""" - source_altaz = SkyCoord( - alt=65.0 * u.deg, - az=210.0 * u.deg, - frame="altaz", - obstime=observing_time, - location=observing_location, - ) - source_radec = source_altaz.icrs - - axes_definition = _build_single_point_radec_axes_definition(source_radec) - - grid_gen = _create_grid_generation( - axes=axes_definition, - coordinate_system="ra_dec", - observing_location=observing_location, - observing_time=observing_time, - lookup_table=None, - ) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore", IERSWarning) - grid_points = grid_gen.generate_grid() - - assert len(grid_points) == 1 - assert "ra" in grid_points[0] - assert "dec" in grid_points[0] - assert "zenith_angle" not in grid_points[0] - assert "azimuth" not in grid_points[0] - assert grid_points[0]["ra"].value == pytest.approx(source_radec.ra.deg, abs=0.05) - assert grid_points[0]["dec"].value == pytest.approx(source_radec.dec.deg, abs=0.05) - - -def test_generate_grid_radec_axes_mode_keeps_below_horizon_points( - observing_location, observing_time -): - """Keep explicit RA/Dec YAML points even when they are below the horizon.""" - source_altaz = SkyCoord( - alt=-20.0 * u.deg, - az=45.0 * u.deg, - frame="altaz", - obstime=observing_time, - location=observing_location, - ) - source_radec = source_altaz.icrs - - axes_definition = _build_single_point_radec_axes_definition(source_radec) - - grid_gen = _create_grid_generation( - axes=axes_definition, - coordinate_system="ra_dec", - observing_location=observing_location, - observing_time=observing_time, - lookup_table=None, - ) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore", IERSWarning) - grid_points = grid_gen.generate_grid() - - assert len(grid_points) == 1 - assert grid_points[0]["ra"].value == pytest.approx(source_radec.ra.deg, abs=0.05) - assert grid_points[0]["dec"].value == pytest.approx(source_radec.dec.deg, abs=0.05) - - -def test_generate_grid_radec_axes_mode_with_lookup( - lookup_table, observing_location, observing_time -): - """Interpolate production limits for explicit RA/Dec axes points.""" - source_altaz = SkyCoord( - alt=50.0 * u.deg, - az=0.0 * u.deg, - frame="altaz", - obstime=observing_time, - location=observing_location, - ) - source_radec = source_altaz.icrs - axes_definition = _build_single_point_radec_axes_definition(source_radec) - axes_definition["axes"]["nsb_level"]["range"] = [1, 1] - - grid_gen = _create_grid_generation( - axes=axes_definition, - coordinate_system="ra_dec", - observing_location=observing_location, - observing_time=observing_time, - lookup_table=lookup_table, - ) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore", IERSWarning) - grid_points = grid_gen.generate_grid() - - assert len(grid_points) == 1 - assert "lower_energy_threshold" in grid_points[0] - assert "scatter_radius" in grid_points[0] - assert "viewcone_radius" in grid_points[0] - assert grid_points[0]["lower_energy_threshold"].value == pytest.approx(0.007, rel=1e-2) - assert grid_points[0]["scatter_radius"].value == pytest.approx(1100.0, rel=1e-2) - assert grid_points[0]["viewcone_radius"].value == pytest.approx(7.0, rel=1e-2) - - -def test_generate_grid_radec_axes_mode_with_sparse_lookup_raises_clear_error( - lookup_table, observing_location, observing_time, monkeypatch -): - """Raise a user-facing error when lookup points are too sparse for 3D interpolation.""" - source_altaz = SkyCoord( - alt=50.0 * u.deg, - az=0.0 * u.deg, - frame="altaz", - obstime=observing_time, - location=observing_location, - ) - source_radec = source_altaz.icrs - axes_definition = _build_single_point_radec_axes_definition(source_radec) - - def _raise_qhull(*args, **kwargs): - raise QhullError("mocked sparse triangulation failure") - - monkeypatch.setattr( - "simtools.production_configuration.corsika_limits_lookup.LinearNDInterpolator", - _raise_qhull, - ) - - with pytest.raises(ValueError, match="does not contain enough unique points"): - _create_grid_generation( - axes=axes_definition, - coordinate_system="ra_dec", - observing_location=observing_location, - observing_time=observing_time, - lookup_table=lookup_table, - ) - - -def test_interpolated_limits(grid_gen): - grid_gen.interpolated_limits = { - "lower_energy_threshold": np.array( - [[[0.1, 0.2], [0.3, 0.4], [0.5, 0.6]], [[0.7, 0.8], [0.9, 1.0], [1.1, 1.2]]] - ), - "upper_scatter_radius": np.array( - [[[100, 200], [300, 400], [500, 600]], [[700, 800], [900, 1000], [1100, 1200]]] - ), - "viewcone_radius": np.array([[[1, 2], [3, 4], [5, 6]], [[7, 8], [9, 10], [11, 12]]]), - "target_zeniths": np.array([30, 40]), - "target_azimuths": np.array([310, 345, 20]), - "target_nsb": np.array([4, 5]), - } - - grid_points = grid_gen.generate_grid() - - # Check that interpolated values are correctly assigned - for point in grid_points: - assert "lower_energy_threshold" in point - assert "scatter_radius" in point - assert "viewcone_radius" in point - assert isinstance(point["lower_energy_threshold"], Quantity) - assert isinstance(point["scatter_radius"], Quantity) - assert isinstance(point["viewcone_radius"], Quantity) - - -def test_load_matching_lookup_arrays_with_simtel_id_mapping(grid_gen, tmp_test_directory): - """Match lookup rows when telescope IDs are stored as sim_telarray numeric IDs.""" - lookup_table_path = tmp_test_directory / "lookup_simtel_ids.ecsv" - Table( - { - "telescope_ids": ["[1]"], - "zenith": [20.0], - "azimuth": [0.0], - "nsb_level": [4.0], - "lower_energy_limit": [0.01], - "upper_radius_limit": [1200.0], - "viewcone_radius": [5.0], - } - ).write(lookup_table_path, format="ascii.ecsv", overwrite=True) - - grid_gen.lookup_table = str(lookup_table_path) - grid_gen.telescope_ids = ["LSTN-01"] - grid_gen._simtel_id_to_name = {1: "LSTN-01"} - - lookup_arrays = grid_gen._load_matching_lookup_arrays() - - assert lookup_arrays["points"].shape[0] == 1 - assert lookup_arrays["lower_energy_threshold"][0] == pytest.approx(0.01) - - -def test_load_matching_lookup_arrays_numeric_ids_require_simtel_file(grid_gen, tmp_test_directory): - """Raise a clear error when numeric lookup telescope IDs are present without mapping input.""" - lookup_table_path = tmp_test_directory / "lookup_simtel_ids_no_mapping.ecsv" - Table( - { - "telescope_ids": ["[1]"], - "zenith": [20.0], - "azimuth": [0.0], - "nsb_level": [4.0], - "lower_energy_limit": [0.01], - "upper_radius_limit": [1200.0], - "viewcone_radius": [5.0], - } - ).write(lookup_table_path, format="ascii.ecsv", overwrite=True) - - grid_gen.lookup_table = str(lookup_table_path) - grid_gen.telescope_ids = ["LSTN-01"] - grid_gen.simtel_file = None - grid_gen._simtel_id_to_name = {} - - with pytest.raises(ValueError, match="Provide --simtel_file"): - grid_gen._load_matching_lookup_arrays() - - -def test_serialize_grid_points_with_output_file(grid_gen, tmp_test_directory, caplog): - """Test serialize_grid_points when an output file is provided.""" - grid_points = [ - { - "zenith_angle": 30 * u.deg, - "azimuth": 310 * u.deg, - "nsb_level": 4, - "lower_energy_threshold": 0.1 * u.TeV, - "scatter_radius": 100 * u.m, - "viewcone_radius": 1 * u.deg, - }, - { - "zenith_angle": 40 * u.deg, - "azimuth": 345 * u.deg, - "nsb_level": 5, - "lower_energy_threshold": 0.2 * u.TeV, - "scatter_radius": 200 * u.m, - "viewcone_radius": 2 * u.deg, - }, - ] - - output_file = tmp_test_directory / "grid_output.ecsv" - with caplog.at_level(logging.INFO): - grid_gen.serialize_grid_points(grid_points, output_file=output_file) - assert output_file.exists() - - output_data = Table.read(output_file, format="ascii.ecsv") - assert "zenith_angle" in output_data.colnames - assert "lower_energy_threshold" in output_data.colnames - assert output_data.meta["coordinate_system"] == grid_gen.coordinate_system - assert output_data.meta["reference_frame"] == "ICRS (J2000)" - - assert f"Output saved to {output_file}" in caplog.text - - -def test_serialize_quantity(grid_gen): - # Case 1: Value is a Quantity (single value) - quantity = 5 * u.m - serialized = grid_gen.serialize_quantity(quantity) - assert serialized == {"value": 5, "unit": "m"} - - # Case 2: Value is not a Quantity (single value) - value = 5 - serialized_value = grid_gen.serialize_quantity(value) - assert serialized_value == value - - -@pytest.mark.xfail(reason="May fail due to IERS data download timeout", strict=False) -def test_convert_altaz_to_radec_and_coordinates(grid_gen): - warnings.simplefilter("error", IERSWarning) - # Case 1: Valid AltAz to RA/Dec conversion - alt, az = 45.0 * u.deg, 30.0 * u.deg - radec = grid_gen.convert_altaz_to_radec(alt, az) - assert isinstance(radec, SkyCoord) - assert radec.ra.unit == u.deg - assert radec.dec.unit == u.deg - - # Case 2: Valid coordinate conversion - grid_gen.coordinate_system = "ra_dec" - grid_points = [ - {"zenith_angle": 30 * u.deg, "azimuth": 45 * u.deg}, - {"zenith_angle": 20 * u.deg, "azimuth": 60 * u.deg}, - ] - converted_points = grid_gen.convert_coordinates(grid_points) - assert "ra" in converted_points[0] - assert "dec" in converted_points[0] - - # Case 3: Missing zenith_angle or azimuth - grid_points = [{"azimuth": 45 * u.deg}] - converted_points = grid_gen.convert_coordinates(grid_points) - assert "ra" not in converted_points[0] - assert "dec" not in converted_points[0] - - -def test_create_circular_binning(grid_gen): - # Case 1: No wraparound - bins = grid_gen.create_circular_binning((0, 150), 6) - assert len(bins) == 6 - assert bins[0] == 0 - assert bins[-1] == 150 - assert bins[1] == 30 - assert bins[2] == 60 - - # Case 2: Wraparound - bins = grid_gen.create_circular_binning((300, 20), 5) - assert len(bins) == 5 - assert bins[0] == 300 - assert bins[-1] == 20 - assert bins[1] == 320 - - # Case 3: Single bin - bins = grid_gen.create_circular_binning((0, 360), 1) - assert len(bins) == 1 - assert bins[0] == 0 - - -def test_create_circular_binning_with_shortest_path(grid_gen): - # Case 1: Clockwise path (shortest distance) - bins = grid_gen.create_circular_binning((350, 10), 3) - expected_bins = [350, 0, 10] - assert np.allclose(bins, expected_bins) - - # Case 2: Counterclockwise path (shortest distance) - bins = grid_gen.create_circular_binning((10, 350), 3) - expected_bins = [10, 0, 350] - assert np.allclose(bins, expected_bins) - - -def test_apply_lookup_table_limits(lookup_table, observing_location, observing_time): - axes_definition = { - "axes": { - "azimuth": { - "range": [0, 180], - "binning": 2, - "scaling": "linear", - "units": "deg", - }, - "zenith_angle": { - "range": [20, 40], - "binning": 2, - "scaling": "linear", - "units": "deg", - }, - "nsb_level": { - "range": [1, 1], - "binning": 1, - "scaling": "linear", - "units": "MHz", - }, - } - } - grid_gen = _create_grid_generation( - axes=axes_definition, - coordinate_system="zenith_azimuth", - observing_location=observing_location, - observing_time=observing_time, - lookup_table=lookup_table, - ) - - assert "lower_energy_threshold" in grid_gen.interpolated_limits - assert "upper_scatter_radius" in grid_gen.interpolated_limits - assert "viewcone_radius" in grid_gen.interpolated_limits - assert np.isclose( - grid_gen.interpolated_limits["lower_energy_threshold"][0][0][0], 0.007, rtol=1e-2 - ) - assert np.isclose( - grid_gen.interpolated_limits["upper_scatter_radius"][0][0][0], 925.0, rtol=1e-2 - ) - assert np.isclose(grid_gen.interpolated_limits["viewcone_radius"][0][0][0], 9.25, rtol=1e-2) - - assert np.shape(grid_gen.interpolated_limits["lower_energy_threshold"]) == (2, 2, 1) - assert np.shape(grid_gen.interpolated_limits["upper_scatter_radius"]) == (2, 2, 1) - assert np.shape(grid_gen.interpolated_limits["viewcone_radius"]) == (2, 2, 1) - - -def test_no_matching_rows_in_lookup_table(axes_definition, observing_location, observing_time): - """Test behavior when no matching rows are found in the lookup table.""" - with pytest.raises( - ValueError, match=r"No matching rows in the lookup table for telescope_ids: \[999\]" - ): - GridGeneration( - axes=axes_definition, - coordinate_system="zenith_azimuth", - observing_location=observing_location, - observing_time=observing_time, - lookup_table="tests/resources/corsika_simulation_limits/merged_corsika_limits_for_test.ecsv", - telescope_ids=[999], - ) - - -def test_matching_rows_with_string_telescope_id( - axes_definition, lookup_table, observing_location, observing_time -): - """Match legacy numeric lookup-table rows using string telescope IDs.""" - grid_gen = GridGeneration( - axes=axes_definition, - coordinate_system="zenith_azimuth", - observing_location=observing_location, - observing_time=observing_time, - lookup_table=lookup_table, - telescope_ids=["LSTN-01"], - ) - - lookup_arrays = grid_gen._load_matching_lookup_arrays() - assert lookup_arrays["points"].shape[0] > 0 - - -def test_missing_observing_time(grid_gen): - """Test behavior when observing_time is not set.""" - grid_gen.observing_time = None - - with pytest.raises(ValueError, match="Observing time is not set"): - grid_gen.convert_altaz_to_radec(45 * u.deg, 30 * u.deg) - - -def test_iers_not_modified_without_env(monkeypatch): - from simtools.application_control import _configure_iers_from_env - - iers.conf.auto_download = True - - monkeypatch.delenv("SIMTOOLS_OFFLINE_IERS", raising=False) - - _configure_iers_from_env() - - GridGeneration(axes={"axes": {}}) - - assert iers.conf.auto_download is True - - -def test_iers_disabled_with_env(monkeypatch): - from simtools.application_control import _configure_iers_from_env - - iers.conf.auto_download = True - - monkeypatch.setenv("SIMTOOLS_OFFLINE_IERS", "1") - - _configure_iers_from_env() - - GridGeneration(axes={"axes": {}}) - - assert iers.conf.auto_download is False From d7cb00cf46ca4695a7461c682881c35b86fb4bdd Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 21:17:22 +0200 Subject: [PATCH 11/44] simplification --- .../htcondor_script_generator.py | 30 +- .../job_spec_builder.py | 83 ----- .../production_grid_helpers.py | 51 +++ .../test_job_spec_builder.py | 295 ------------------ 4 files changed, 80 insertions(+), 379 deletions(-) delete mode 100644 src/simtools/production_configuration/job_spec_builder.py delete mode 100644 tests/unit_tests/production_configuration/test_job_spec_builder.py diff --git a/src/simtools/job_execution/htcondor_script_generator.py b/src/simtools/job_execution/htcondor_script_generator.py index 2dd6bbdfb3..725587c280 100644 --- a/src/simtools/job_execution/htcondor_script_generator.py +++ b/src/simtools/job_execution/htcondor_script_generator.py @@ -15,7 +15,12 @@ import astropy.units as u from simtools.layout.array_layout_utils import resolve_array_layout_name -from simtools.production_configuration.job_spec_builder import build_job_specs +from simtools.production_configuration.production_grid_helpers import calculate_scaled_nshow +from simtools.production_configuration.production_grid_job_rows import ( + build_backend_agnostic_job_rows, + get_core_scatter_max_for_zenith_angle, + get_energy_range_for_zenith_angle, +) _logger = logging.getLogger(__name__) @@ -377,3 +382,26 @@ def _get_submit_script(args_dict): --log_level {args_dict["log_level"]} \\ --pack_for_grid_register "$pack_for_grid_register" """ + + +def build_job_specs(args_dict, image_labels): + """Build backend-agnostic job specs from comparison and production grids.""" + base_pack_dir = args_dict.get("simulation_output") or "simtools-output" + normalized_rows = build_backend_agnostic_job_rows( + args_dict, + calculate_scaled_nshow, + get_energy_range_for_zenith_angle_function=get_energy_range_for_zenith_angle, + get_core_scatter_max_for_zenith_angle_function=get_core_scatter_max_for_zenith_angle, + ) + + job_specs = [] + for label in image_labels: + for row in normalized_rows: + job_specs.append( + { + "image_label": str(label), + **row, + "pack_for_grid_register": f"{base_pack_dir}/{label!s}", + } + ) + return job_specs diff --git a/src/simtools/production_configuration/job_spec_builder.py b/src/simtools/production_configuration/job_spec_builder.py deleted file mode 100644 index df301c239f..0000000000 --- a/src/simtools/production_configuration/job_spec_builder.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Build backend-agnostic job specifications for production submissions.""" - -import numpy as np -from astropy import units as u - -from simtools.production_configuration.production_grid_job_rows import ( - build_backend_agnostic_job_rows, - get_core_scatter_max_for_zenith_angle, - get_energy_range_for_zenith_angle, -) - - -def calculate_log_energy_midpoint(energy_range_pair): - """Return the geometric-mean energy for an energy range pair.""" - energy_min, energy_max = energy_range_pair - - if not isinstance(energy_min, u.Quantity) or not isinstance(energy_max, u.Quantity): - raise TypeError("energy_range_pair must contain astropy Quantity values.") - - energy_min_tev = energy_min.to(u.TeV) - energy_max_tev = energy_max.to(u.TeV) - - if energy_min_tev <= 0 * u.TeV or energy_max_tev <= 0 * u.TeV: - raise ValueError("Energy range values must be strictly positive.") - - mean_log_energy = np.mean( - [ - np.log10(energy_min_tev.value), - np.log10(energy_max_tev.value), - ] - ) - return 10**mean_log_energy * u.TeV - - -def calculate_scaled_nshow( - energy_range_pair, - baseline_nshow, - nshow_power_index=None, - reference_energy=None, -): - """Return an energy-dependent nshow value.""" - if baseline_nshow < 1: - raise ValueError("baseline_nshow must be a positive integer.") - - if nshow_power_index is None: - return baseline_nshow - - if reference_energy is None: - raise ValueError("reference_energy is required when nshow_power_index is configured.") - - midpoint_energy = calculate_log_energy_midpoint(energy_range_pair) - scaling_factor = (midpoint_energy / reference_energy.to(midpoint_energy.unit)).to_value( - u.dimensionless_unscaled - ) ** nshow_power_index - scaled_nshow = int(np.ceil(baseline_nshow * scaling_factor)) - - if scaled_nshow < 1: - raise ValueError("Scaled nshow must be at least 1.") - - return scaled_nshow - - -def build_job_specs(args_dict, image_labels): - """Build backend-agnostic job specs from comparison and production grids.""" - base_pack_dir = args_dict.get("simulation_output") or "simtools-output" - normalized_rows = build_backend_agnostic_job_rows( - args_dict, - calculate_scaled_nshow, - get_energy_range_for_zenith_angle_function=get_energy_range_for_zenith_angle, - get_core_scatter_max_for_zenith_angle_function=get_core_scatter_max_for_zenith_angle, - ) - - job_specs = [] - for label in image_labels: - for row in normalized_rows: - job_specs.append( - { - "image_label": str(label), - **row, - "pack_for_grid_register": f"{base_pack_dir}/{label!s}", - } - ) - return job_specs diff --git a/src/simtools/production_configuration/production_grid_helpers.py b/src/simtools/production_configuration/production_grid_helpers.py index 2779cccdfe..dc3cacc8f4 100644 --- a/src/simtools/production_configuration/production_grid_helpers.py +++ b/src/simtools/production_configuration/production_grid_helpers.py @@ -1,5 +1,6 @@ """Helper functions and constants for production-grid generation.""" +import numpy as np from astropy import units as u from simtools.configuration import defaults @@ -58,3 +59,53 @@ def normalize_azimuth_angle(azimuth_angle): if isinstance(azimuth_angle, u.Quantity): return azimuth_angle.to(u.deg) % (360 * u.deg) return azimuth_angle % 360.0 + + +def calculate_log_energy_midpoint(energy_range_pair): + """Return the geometric-mean energy for an energy range pair.""" + energy_min, energy_max = energy_range_pair + + if not isinstance(energy_min, u.Quantity) or not isinstance(energy_max, u.Quantity): + raise TypeError("energy_range_pair must contain astropy Quantity values.") + + energy_min_tev = energy_min.to(u.TeV) + energy_max_tev = energy_max.to(u.TeV) + + if energy_min_tev <= 0 * u.TeV or energy_max_tev <= 0 * u.TeV: + raise ValueError("Energy range values must be strictly positive.") + + mean_log_energy = np.mean( + [ + np.log10(energy_min_tev.value), + np.log10(energy_max_tev.value), + ] + ) + return 10**mean_log_energy * u.TeV + + +def calculate_scaled_nshow( + energy_range_pair, + baseline_nshow, + nshow_power_index=None, + reference_energy=None, +): + """Return an energy-dependent nshow value.""" + if baseline_nshow < 1: + raise ValueError("baseline_nshow must be a positive integer.") + + if nshow_power_index is None: + return baseline_nshow + + if reference_energy is None: + raise ValueError("reference_energy is required when nshow_power_index is configured.") + + midpoint_energy = calculate_log_energy_midpoint(energy_range_pair) + scaling_factor = (midpoint_energy / reference_energy.to(midpoint_energy.unit)).to_value( + u.dimensionless_unscaled + ) ** nshow_power_index + scaled_nshow = int(np.ceil(baseline_nshow * scaling_factor)) + + if scaled_nshow < 1: + raise ValueError("Scaled nshow must be at least 1.") + + return scaled_nshow diff --git a/tests/unit_tests/production_configuration/test_job_spec_builder.py b/tests/unit_tests/production_configuration/test_job_spec_builder.py deleted file mode 100644 index 8a378decfb..0000000000 --- a/tests/unit_tests/production_configuration/test_job_spec_builder.py +++ /dev/null @@ -1,295 +0,0 @@ -from unittest import mock - -import astropy.units as u -import pytest -from astropy.tests.helper import assert_quantity_allclose -from simtools.production_configuration.production_grid import CorsikaLimitsLookup - -from simtools.production_configuration.job_spec_builder import ( - build_job_specs, - calculate_log_energy_midpoint, - calculate_scaled_nshow, - get_core_scatter_max_for_zenith_angle, - get_energy_range_for_zenith_angle, - normalize_energy_ranges, - normalize_grid_axes, - normalize_to_list, -) - - -@pytest.fixture -def args_dict(): - return { - "azimuth_angle": 45 * u.deg, - "zenith_angle": 20 * u.deg, - "energy_range": [1 * u.GeV, 10 * u.GeV], - "core_scatter": [0, 100 * u.m], - "model_version": "v1.0", - "array_layout_name": "test_layout", - "primary": "gamma", - "nshow": 1000, - "run_number": 1, - "number_of_runs": 10, - "corsika_le_interaction": "urqmd", - "corsika_he_interaction": "epos", - } - - -@pytest.fixture -def corsika_limits(): - return "tests/resources/corsika_simulation_limits/merged_corsika_limits_for_test.ecsv" - - -def test_normalize_energy_ranges_expands_list_of_pairs(): - energy_ranges = normalize_energy_ranges( - [ - (30 * u.GeV, 30 * u.GeV), - (300 * u.GeV, 300 * u.GeV), - ] - ) - - assert len(energy_ranges) == 2 - for actual, expected in zip( - energy_ranges, - [ - (30 * u.GeV, 30 * u.GeV), - (300 * u.GeV, 300 * u.GeV), - ], - ): - assert_quantity_allclose(actual[0], expected[0]) - assert_quantity_allclose(actual[1], expected[1]) - - -def test_normalize_to_list_converts_tuple_values(): - assert normalize_to_list((1, 2)) == [1, 2] - - -def test_normalize_grid_axes_uses_defaults_and_none_for_missing_axes(): - grid_axes = normalize_grid_axes({"primary": "gamma"}) - - assert grid_axes["primary"] == ["gamma"] - assert grid_axes["azimuth_angle"] == [None] - assert grid_axes["zenith_angle"] == [None] - assert grid_axes["model_version"] == [None] - assert grid_axes["corsika_le_interaction"] == ["urqmd"] - assert grid_axes["corsika_he_interaction"] == ["epos"] - - -def test_normalize_energy_ranges_accepts_single_tuple_pair(): - energy_ranges = normalize_energy_ranges((30 * u.GeV, 300 * u.GeV)) - assert len(energy_ranges) == 1 - assert_quantity_allclose(energy_ranges[0][0], 30 * u.GeV) - assert_quantity_allclose(energy_ranges[0][1], 300 * u.GeV) - - -def test_normalize_energy_ranges_raises_for_invalid_shape(): - with pytest.raises(ValueError, match="energy_range must be one pair"): - normalize_energy_ranges([30 * u.GeV, 300]) - - -def test_calculate_log_energy_midpoint(): - midpoint_energy = calculate_log_energy_midpoint((1 * u.GeV, 100 * u.GeV)) - - assert midpoint_energy.to_value(u.GeV) == pytest.approx(10.0) - - -def test_calculate_log_energy_midpoint_raises_for_non_quantity_values(): - with pytest.raises(TypeError, match="energy_range_pair must contain astropy Quantity values"): - calculate_log_energy_midpoint((1, 100 * u.GeV)) - - -def test_calculate_log_energy_midpoint_raises_for_non_positive_values(): - with pytest.raises(ValueError, match="Energy range values must be strictly positive"): - calculate_log_energy_midpoint((0 * u.GeV, 100 * u.GeV)) - - -def test_calculate_scaled_nshow_returns_baseline_without_power_index(): - scaled_nshow = calculate_scaled_nshow((10 * u.GeV, 100 * u.GeV), 50) - - assert scaled_nshow == 50 - - -def test_calculate_scaled_nshow_scales_against_reference_energy(): - scaled_nshow = calculate_scaled_nshow( - (100 * u.GeV, 100 * u.GeV), - 100, - nshow_power_index=-1.0, - reference_energy=10 * u.GeV, - ) - - assert scaled_nshow == 10 - - -def test_calculate_scaled_nshow_uses_ceil_for_fractional_result(): - scaled_nshow = calculate_scaled_nshow( - (100 * u.GeV, 100 * u.GeV), - 10, - nshow_power_index=-2.0, - reference_energy=10 * u.GeV, - ) - - assert scaled_nshow == 1 - - -def test_calculate_scaled_nshow_raises_for_invalid_baseline(): - with pytest.raises(ValueError, match="baseline_nshow must be a positive integer"): - calculate_scaled_nshow((10 * u.GeV, 100 * u.GeV), 0) - - -def test_calculate_scaled_nshow_requires_reference_energy_when_scaled(): - with pytest.raises(ValueError, match="reference_energy is required"): - calculate_scaled_nshow((10 * u.GeV, 100 * u.GeV), 10, nshow_power_index=-1.0) - - -def test_calculate_scaled_nshow_raises_when_scaled_result_drops_below_one(): - with pytest.raises(ValueError, match="Scaled nshow must be at least 1"): - calculate_scaled_nshow( - (10 * u.GeV, 10 * u.GeV), - 1, - nshow_power_index=1.0, - reference_energy=-100 * u.GeV, - ) - - -def test_build_job_specs_expands_model_version_list(args_dict): - args_dict["model_version"] = ["6.3.0", "7.0.0"] - - job_specs = build_job_specs(args_dict, ["7.0.0"]) - model_versions = {job_spec["model_version"] for job_spec in job_specs} - - assert model_versions == {"6.3.0", "7.0.0"} - assert len(job_specs) == 2 * args_dict["number_of_runs"] - - -def test_build_job_specs_scales_nshow_by_energy_range(args_dict): - args_dict["number_of_runs"] = 1 - args_dict["nshow"] = 100 - args_dict["nshow_power_index"] = -1.0 - args_dict["nshow_reference_energy"] = 10 * u.GeV - args_dict["energy_range"] = [ - (10 * u.GeV, 10 * u.GeV), - (100 * u.GeV, 100 * u.GeV), - ] - - job_specs = build_job_specs(args_dict, ["7.0.0"]) - nshow_values = [job_spec["nshow"] for job_spec in job_specs] - assert nshow_values[0] == pytest.approx(100) - assert nshow_values[1] == pytest.approx(10) - - -def test_build_job_specs_requires_reference_energy_for_nshow_scaling(args_dict): - args_dict["number_of_runs"] = 1 - args_dict["nshow_power_index"] = -1.0 - args_dict["nshow_reference_energy"] = None - - with pytest.raises(ValueError, match="reference_energy is required"): - build_job_specs(args_dict, ["7.0.0"]) - - -def test_build_job_specs_uses_default_interaction_models_when_missing(args_dict): - args_dict.pop("corsika_le_interaction") - args_dict.pop("corsika_he_interaction") - - job_specs = build_job_specs(args_dict, ["7.0.0"]) - - assert {job_spec["corsika_le_interaction"] for job_spec in job_specs} == {"urqmd"} - assert {job_spec["corsika_he_interaction"] for job_spec in job_specs} == {"epos"} - - -def test_build_job_specs_increments_run_number(args_dict): - args_dict["number_of_runs"] = 2 - args_dict["run_number"] = 10 - args_dict["model_version"] = ["6.3.0", "7.0.0"] - - job_specs = build_job_specs(args_dict, ["7.0.0"]) - run_numbers = [job_spec["run_number"] for job_spec in job_specs] - - assert run_numbers == [10, 11, 12, 13] - - -@mock.patch( - "simtools.production_configuration.job_spec_builder.get_energy_range_for_zenith_angle", - return_value=None, -) -def test_build_job_specs_skips_entries_when_energy_range_is_none(mock_energy_range, args_dict): - args_dict["corsika_limits"] = "limits.ecsv" - args_dict["number_of_runs"] = 1 - - job_specs = build_job_specs(args_dict, ["7.0.0"]) - - assert job_specs == [] - mock_energy_range.assert_called_once() - - -@mock.patch( - "simtools.production_configuration.job_spec_builder.calculate_scaled_nshow", - return_value=777, -) -def test_build_job_specs_uses_dummy_nshow_when_corsika_limits_set( - mock_nshow, args_dict, corsika_limits -): - args_dict["corsika_limits"] = corsika_limits - args_dict["telescope_ids"] = ["LSTN-01"] - args_dict["number_of_runs"] = 1 - - job_specs = build_job_specs(args_dict, ["7.0.0"]) - - assert len(job_specs) == 1 - assert job_specs[0]["nshow"] == 777 - mock_nshow.assert_called_once() - - -def test_get_energy_range_for_zenith_angle_clips_to_lookup_threshold(corsika_limits): - lookup = CorsikaLimitsLookup(corsika_limits, telescope_ids=["LSTN-01"]) - - selected_energy_range = get_energy_range_for_zenith_angle( - 20 * u.deg, - (1 * u.GeV, 10 * u.GeV), - lookup, - azimuth_angle=0 * u.deg, - ) - - assert_quantity_allclose(selected_energy_range[0], 7 * u.GeV) - assert_quantity_allclose(selected_energy_range[1], 10 * u.GeV) - - -def test_get_energy_range_for_zenith_angle_returns_none_if_threshold_exceeds_max(corsika_limits): - lookup = CorsikaLimitsLookup(corsika_limits, telescope_ids=["LSTN-01"]) - - selected_energy_range = get_energy_range_for_zenith_angle( - 20 * u.deg, - (1 * u.GeV, 5 * u.GeV), - lookup, - azimuth_angle=0 * u.deg, - ) - - assert selected_energy_range is None - - -def test_get_core_scatter_max_for_zenith_angle_clips_to_lookup_radius(corsika_limits): - lookup = CorsikaLimitsLookup(corsika_limits, telescope_ids=["LSTN-01"]) - - selected_core_scatter_max = get_core_scatter_max_for_zenith_angle( - 20 * u.deg, - (10, 1000 * u.m), - lookup, - azimuth_angle=0 * u.deg, - ) - - assert_quantity_allclose(selected_core_scatter_max, 925 * u.m) - - -def test_build_job_specs_reads_limits_from_corsika_limits_file(args_dict, corsika_limits): - args_dict["number_of_runs"] = 1 - args_dict["azimuth_angle"] = 0 * u.deg - args_dict["energy_range"] = (1 * u.GeV, 10 * u.GeV) - args_dict["core_scatter"] = (10, 1000 * u.m) - args_dict["corsika_limits"] = corsika_limits - args_dict["telescope_ids"] = ["LSTN-01"] - - job_specs = build_job_specs(args_dict, ["7.0.0"]) - - assert len(job_specs) == 1 - assert_quantity_allclose(job_specs[0]["energy_min"], 7 * u.GeV) - assert_quantity_allclose(job_specs[0]["energy_max"], 10 * u.GeV) - assert_quantity_allclose(job_specs[0]["core_scatter_max"], 925 * u.m) From 536b53aa8adeaeb2fd9ba7a480dc255b2784d333 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 21:19:17 +0200 Subject: [PATCH 12/44] unit tests --- .../unit_tests/job_execution/test_htcondor_script_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/job_execution/test_htcondor_script_generator.py b/tests/unit_tests/job_execution/test_htcondor_script_generator.py index 3b94547ddb..60e92c146d 100644 --- a/tests/unit_tests/job_execution/test_htcondor_script_generator.py +++ b/tests/unit_tests/job_execution/test_htcondor_script_generator.py @@ -12,9 +12,9 @@ _resolve_apptainer_images, _sanitize_label_for_filename, _write_params_file, + build_job_specs, generate_submission_script, ) -from simtools.production_configuration.job_spec_builder import build_job_specs @pytest.fixture From 1104342c14807270440cc8a7d179107b2cd5396a Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 18 May 2026 21:24:35 +0200 Subject: [PATCH 13/44] simplify --- .../htcondor_script_generator.py | 33 ++++++++----------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/src/simtools/job_execution/htcondor_script_generator.py b/src/simtools/job_execution/htcondor_script_generator.py index 725587c280..728525f12c 100644 --- a/src/simtools/job_execution/htcondor_script_generator.py +++ b/src/simtools/job_execution/htcondor_script_generator.py @@ -197,11 +197,13 @@ def generate_submission_script(args_dict): if args_dict.get("htcondor_log_path") else work_dir / "htcondor_logs" ) - log_dir = htcondor_log_path / "log" - error_dir = htcondor_log_path / "error" - output_dir = htcondor_log_path / "output" + htcondor_dirs = { + "log": htcondor_log_path / "log", + "error": htcondor_log_path / "error", + "output": htcondor_log_path / "output", + } work_dir.mkdir(parents=True, exist_ok=True) - for subdir in (log_dir, error_dir, output_dir): + for subdir in htcondor_dirs.values(): subdir.mkdir(parents=True, exist_ok=True) submit_file_name = "simulate_prod.submit" _logger.info(f"Generating HT Condor submission scripts (path: {work_dir})") @@ -224,9 +226,7 @@ def generate_submission_script(args_dict): apptainer_images[label], args_dict["priority"], params_file_name, - log_dir=log_dir, - error_dir=error_dir, - output_dir=output_dir, + htcondor_dirs=htcondor_dirs, ) ) @@ -236,9 +236,7 @@ def generate_submission_script(args_dict): Path(work_dir / f"{submit_file_name}.sh").chmod(0o755) -def _get_submit_file( - executable, apptainer_image, priority, params_file_name, log_dir, error_dir, output_dir -): +def _get_submit_file(executable, apptainer_image, priority, params_file_name, htcondor_dirs): """ Return HTCondor submit file. @@ -254,12 +252,9 @@ def _get_submit_file( Priority of the job. params_file_name: str Name of the params file for queue-from submission. - log_dir: Path - Directory for HTCondor log files. - error_dir: Path - Directory for HTCondor error files. - output_dir: Path - Directory for HTCondor output files. + htcondor_dirs: dict + Directory mapping with HTCondor files locations. Expected keys are + ``log``, ``error``, and ``output``. Returns ------- @@ -274,9 +269,9 @@ def _get_submit_file( transfer_container = false executable = {executable} -error = {error_dir}/err.$(cluster)_$(process) -output = {output_dir}/out.$(cluster)_$(process) -log = {log_dir}/log.$(cluster)_$(process) +error = {htcondor_dirs["error"]}/err.$(cluster)_$(process) +output = {htcondor_dirs["output"]}/out.$(cluster)_$(process) +log = {htcondor_dirs["log"]}/log.$(cluster)_$(process) priority = {priority} arguments = "{arguments_string}" From 15f8140730e9fa7dfc703a0ccd6110ecc755d9d8 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 19 May 2026 07:58:21 +0200 Subject: [PATCH 14/44] fix dirs --- .../job_execution/test_htcondor_script_generator.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/job_execution/test_htcondor_script_generator.py b/tests/unit_tests/job_execution/test_htcondor_script_generator.py index 60e92c146d..71d2bf4f44 100644 --- a/tests/unit_tests/job_execution/test_htcondor_script_generator.py +++ b/tests/unit_tests/job_execution/test_htcondor_script_generator.py @@ -171,14 +171,13 @@ def test_get_submit_file_uses_queue_from_params(tmp_test_directory): log_dir = Path(tmp_test_directory) / "htcondor_logs" / "log" error_dir = Path(tmp_test_directory) / "htcondor_logs" / "error" output_dir = Path(tmp_test_directory) / "htcondor_logs" / "output" + htcondor_dirs = {"log": log_dir, "error": error_dir, "output": output_dir} content = _get_submit_file( executable="simulate_prod.submit.sh", apptainer_image=apptainer_image, priority=1, params_file_name="simulate_prod.submit.params.txt", - log_dir=log_dir, - error_dir=error_dir, - output_dir=output_dir, + htcondor_dirs=htcondor_dirs, ) assert "queue apptainer_label,primary" in content From 5d63522a2b597fc73846681fd7ded5d69f644647 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 19 May 2026 09:27:14 +0200 Subject: [PATCH 15/44] simplifications --- .../applications/production_generate_grid.py | 3 - .../corsika_limits_lookup.py | 57 +------ .../production_grid_engine.py | 145 +++++------------- src/simtools/utils/general.py | 11 +- src/simtools/utils/value_conversion.py | 22 +++ 5 files changed, 79 insertions(+), 159 deletions(-) diff --git a/src/simtools/applications/production_generate_grid.py b/src/simtools/applications/production_generate_grid.py index 3c3a529780..973af3ec7d 100644 --- a/src/simtools/applications/production_generate_grid.py +++ b/src/simtools/applications/production_generate_grid.py @@ -201,9 +201,6 @@ def main(): ) grid_points = grid_gen.generate_grid() - - if coordinate_system == "ra_dec": - grid_points = grid_gen.convert_coordinates(grid_points) grid_gen.serialize_grid_points(grid_points, output_file=output_filepath) diff --git a/src/simtools/production_configuration/corsika_limits_lookup.py b/src/simtools/production_configuration/corsika_limits_lookup.py index a0d7859ef9..c5dbb3b388 100644 --- a/src/simtools/production_configuration/corsika_limits_lookup.py +++ b/src/simtools/production_configuration/corsika_limits_lookup.py @@ -1,7 +1,5 @@ """Lookup-table access and interpolation for CORSIKA production limits.""" -import json - import numpy as np from astropy.table import Table from scipy.interpolate import LinearNDInterpolator, griddata @@ -10,22 +8,12 @@ from simtools.simtel.simtel_io_metadata import ( get_sim_telarray_telescope_id_to_telescope_name_mapping, ) - - -def _value_in_unit(value, unit=None): - """Return a scalar value converted to the requested unit when possible.""" - if hasattr(value, "to_value"): - return value.to_value(unit) if unit is not None else value.value - return value +from simtools.utils.general import ensure_list +from simtools.utils.value_conversion import get_value_in_unit class CorsikaLimitsLookup: - """ - Read and interpolate CORSIKA limits for production grids. - - The lookup table is shared by both grid serialization and backend-specific - job-spec generation. - """ + """Read and interpolate CORSIKA limits for production grids.""" def __init__(self, lookup_table, telescope_ids=None, simtel_file=None): """ @@ -52,23 +40,6 @@ def __init__(self, lookup_table, telescope_ids=None, simtel_file=None): self.lookup_values_for_interpolation = None self.lookup_interpolators_for_point = None - @staticmethod - def _coerce_identifier_container(value): - """Coerce identifier input into a list.""" - if value is None: - return [] - if isinstance(value, str): - stripped = value.strip() - return json.loads(stripped) if stripped.startswith("[") else [stripped] - if isinstance(value, (list, tuple, set)): - return list(value) - return [value] - - @staticmethod - def coerce_identifier_container(value): - """Coerce identifier input into a list.""" - return CorsikaLimitsLookup._coerce_identifier_container(value) - def _normalize_lookup_identifier(self, identifier): """Normalize one telescope identifier and report if it is numeric.""" if isinstance(identifier, (int, np.integer)): @@ -79,35 +50,23 @@ def _normalize_lookup_identifier(self, identifier): return self._simtel_id_to_name.get(int(text), text), True return text, False - def normalize_lookup_identifier(self, identifier): - """Normalize one telescope identifier and report if it is numeric.""" - return self._normalize_lookup_identifier(identifier) - def _normalized_identifier_set(self, identifiers): """Return normalized telescope identifiers as a set.""" return { self._normalize_lookup_identifier(identifier)[0] - for identifier in self._coerce_identifier_container(identifiers) + for identifier in ensure_list(identifiers) } - def normalized_identifier_set(self, identifiers): - """Return normalized telescope identifiers as a set.""" - return self._normalized_identifier_set(identifiers) - def _lookup_contains_numeric_telescope_ids(self, lookup_table): """Return True when any lookup-table telescope identifier is numeric.""" return any( any( self._normalize_lookup_identifier(identifier)[1] - for identifier in self._coerce_identifier_container(row["telescope_ids"]) + for identifier in ensure_list(row["telescope_ids"]) ) for row in lookup_table ) - def lookup_contains_numeric_telescope_ids(self, lookup_table): - """Return True when any lookup-table telescope identifier is numeric.""" - return self._lookup_contains_numeric_telescope_ids(lookup_table) - @property def simtel_id_to_name(self): """Return the sim_telarray telescope-ID mapping.""" @@ -299,9 +258,9 @@ def interpolate_point(self, zenith, azimuth, nsb=1.0): target = np.array( [ [ - _value_in_unit(zenith), - _value_in_unit(azimuth) % 360.0, - _value_in_unit(nsb), + get_value_in_unit(zenith, "deg"), + get_value_in_unit(azimuth, "deg") % 360.0, + get_value_in_unit(nsb), ] ], dtype=float, diff --git a/src/simtools/production_configuration/production_grid_engine.py b/src/simtools/production_configuration/production_grid_engine.py index 5da96005b8..c10702ae7f 100644 --- a/src/simtools/production_configuration/production_grid_engine.py +++ b/src/simtools/production_configuration/production_grid_engine.py @@ -1,4 +1,15 @@ -"""Backend-independent engine for simulation production-grid generation.""" +""" +Generate simulation production grids for configurable axes, coordinate systems, and lookup tables. + +This module builds simulation grids from configurable axis definitions (for example, +azimuth, zenith angle, night-sky background, and camera offset). It supports +multiple binning and scaling modes, applies CORSIKA limit interpolation from lookup +tables, and handles coordinate conversions between horizontal (Alt/Az) and +equatorial (RA/Dec) systems. + +The engine also provides serialization helpers used by backend adapters to export +generated grid points and metadata. +""" import logging @@ -8,25 +19,16 @@ from astropy.units import Quantity from simtools.production_configuration.corsika_limits_lookup import CorsikaLimitsLookup -from simtools.production_configuration.production_grid_helpers import ( - DEFAULT_SERIALIZATION_ROUND_DECIMALS, -) -from simtools.production_configuration.production_grid_serialization import ( - build_grid_metadata, - build_serialized_rows, - collect_point_keys, - serialize_grid_points, - serialize_grid_value, -) +from simtools.production_configuration.production_grid_serialization import serialize_grid_points class ProductionGridEngine: """ - Backend-independent engine for simulation production-grid generation. + Generate simulation production grids. - This engine handles axis expansion, coordinate-system specific grid - generation, lookup-table interpolation, coordinate conversion, and ECSV - serialization helpers used by backend adapters. + This engine handles axis expansion (e.g. azimuth, zenith angle, night-sky background), + coordinate-system specific grid generation, lookup-table interpolation, coordinate conversion, + and ECSV serialization helpers used by backend adapters (e.g. HTCondor). """ def __init__( @@ -50,9 +52,9 @@ def __init__( coordinate_system : str, optional The coordinate system for the grid generation. observing_location : EarthLocation, optional - The location of the observation. + The location of the observation (latitude, longitude, height). observing_time : Time, optional - The observing time used for RA/Dec transforms. + The time of observation required for RA/Dec transforms. lookup_table : str, optional Path to the lookup table file (ECSV format). telescope_ids : list of str, optional @@ -73,7 +75,6 @@ def __init__( self.telescope_ids = telescope_ids self.simtel_file = simtel_file self.interpolated_limits = {} - self.serialization_round_decimals = DEFAULT_SERIALIZATION_ROUND_DECIMALS self._limits_lookup = CorsikaLimitsLookup( lookup_table=lookup_table, telescope_ids=telescope_ids, @@ -88,23 +89,6 @@ def __init__( else: self._apply_lookup_table_limits() - @staticmethod - def _coerce_identifier_container(value): - """Coerce identifier input into a list.""" - return CorsikaLimitsLookup.coerce_identifier_container(value) - - def _normalize_lookup_identifier(self, identifier): - """Normalize one telescope identifier and report if it is numeric.""" - return self._limits_lookup.normalize_lookup_identifier(identifier) - - def _normalized_identifier_set(self, identifiers): - """Return normalized telescope identifiers as a set.""" - return self._limits_lookup.normalized_identifier_set(identifiers) - - def _lookup_contains_numeric_telescope_ids(self, lookup_table): - """Return True when any lookup-table telescope identifier is numeric.""" - return self._limits_lookup.lookup_contains_numeric_telescope_ids(lookup_table) - def _sync_limits_lookup(self): """Synchronize mutable lookup settings with the shared lookup helper.""" self._limits_lookup.lookup_table = self.lookup_table @@ -112,11 +96,6 @@ def _sync_limits_lookup(self): self._limits_lookup.simtel_file = self.simtel_file self._limits_lookup.simtel_id_to_name = self._simtel_id_to_name - def _load_matching_lookup_arrays(self): - """Load and filter lookup-table arrays for selected telescope IDs.""" - self._sync_limits_lookup() - return self._limits_lookup.load_matching_lookup_arrays() - def _require_observing_time(self): """Return observing time if available, else raise a clear error.""" if self.observing_time is None: @@ -136,13 +115,7 @@ def _get_max_zenith_for_radec_mode(self): def _prepare_lookup_table_limits_for_point_interpolation(self): """Prepare lookup arrays for per-point interpolation in RA/Dec grid mode.""" self._sync_limits_lookup() - self.lookup_interpolators_for_point = self._limits_lookup.prepare_point_interpolators() - self.lookup_points_for_interpolation = self._limits_lookup.lookup_points_for_interpolation - self.lookup_values_for_interpolation = self._limits_lookup.lookup_values_for_interpolation - - def _has_radec_axes(self): - """Return True if axes define a native RA/Dec grid.""" - return "ra" in self.axes and "dec" in self.axes + self._limits_lookup.prepare_point_interpolators() def _generate_target_values(self): """ @@ -290,7 +263,7 @@ def _generate_grid_from_radec_axes(self): def _generate_grid_radec_mode(self): """Generate grid points for RA/Dec mode.""" - if self._has_radec_axes(): + if "ra" in self.axes and "dec" in self.axes: return self._generate_grid_from_radec_axes() direction_points = self._generate_radec_grid_direction_points() @@ -407,7 +380,7 @@ def generate_grid(self): grid_points.append(grid_point) - return grid_points + return self.convert_coordinates(grid_points) def convert_altaz_to_radec(self, alt, az): """ @@ -426,25 +399,20 @@ def convert_altaz_to_radec(self, alt, az): SkyCoord object containing the RA/Dec coordinates. """ if self.observing_time is None: - raise ValueError( - "Observing time is not set. " - "Please provide an observing_time to convert coordinates." - ) + raise ValueError("Conversion to RA/dec requires observing_time to be set. ") - alt_rad = alt.to(u.rad) - az_rad = az.to(u.rad) aa = AltAz( - alt=alt_rad, - az=az_rad, + alt=alt.to(u.rad), + az=az.to(u.rad), location=self.observing_location, obstime=self.observing_time, ) - skycoord = SkyCoord(aa) - return skycoord.icrs + sky_coord = SkyCoord(aa) + return sky_coord.icrs # Return RA/Dec in ICRS frame def convert_coordinates(self, grid_points): """ - Convert the grid points to RA/Dec coordinates if necessary. + Convert the grid points RA/Dec coordinates if necessary. Parameters ---------- @@ -467,7 +435,16 @@ def convert_coordinates(self, grid_points): return grid_points def serialize_grid_points(self, grid_points, output_file): - """Serialize the grid output and save to an ECSV table file.""" + """ + Serialize grid points to an ECSV table file. + + Parameters + ---------- + grid_points : list of dict + List of grid points to serialize. + output_file : str + Path to the output ECSV file. + """ serialize_grid_points( grid_points=grid_points, output_file=output_file, @@ -475,49 +452,5 @@ def serialize_grid_points(self, grid_points, output_file): observing_time=self.observing_time, telescope_ids=self.telescope_ids, lookup_table=self.lookup_table, - serialization_round_decimals=self.serialization_round_decimals, - ) - self._logger.info(f"Output saved to {output_file}") - - @staticmethod - def _collect_point_keys(grid_points): - """Collect all grid-point keys while preserving first-seen order.""" - return collect_point_keys(grid_points) - - def _serialize_grid_value(self, value): - """Serialize one grid value and return (value, unit).""" - return serialize_grid_value( - value, - serialization_round_decimals=self.serialization_round_decimals, - ) - - def _build_serialized_rows(self, grid_points, all_keys): - """Build serialized row dictionaries and collect units.""" - return build_serialized_rows( - grid_points, - all_keys, - serialization_round_decimals=self.serialization_round_decimals, - ) - - def _build_grid_metadata(self): - """Build metadata for the output grid table.""" - return build_grid_metadata( - coordinate_system=self.coordinate_system, - observing_time=self.observing_time, - telescope_ids=self.telescope_ids, - lookup_table=self.lookup_table, ) - - def serialize_quantity(self, value): - """Serialize Quantity.""" - if isinstance(value, u.Quantity): - serialized_value = float(value.value) - rounded_value = round(serialized_value, self.serialization_round_decimals) - return {"value": rounded_value, "unit": str(value.unit)} - if isinstance(value, float): - return round(value, self.serialization_round_decimals) - if isinstance(value, np.floating): - return round(float(value), self.serialization_round_decimals) - if isinstance(value, np.integer): - return int(value) - return value + self._logger.info(f"Simulation grid saved to {output_file}") diff --git a/src/simtools/utils/general.py b/src/simtools/utils/general.py index 0700dc8160..ed51c6b0fa 100644 --- a/src/simtools/utils/general.py +++ b/src/simtools/utils/general.py @@ -2,6 +2,7 @@ import datetime import glob +import json import logging import os import tarfile @@ -216,8 +217,16 @@ def ensure_list(value): return [] if isinstance(value, list): return value - if isinstance(value, tuple): + if isinstance(value, (tuple, set)): return list(value) + if isinstance(value, str): + stripped = value.strip() + if stripped.startswith("["): + try: + return json.loads(stripped) + except json.JSONDecodeError: + pass + return [stripped] return [value] diff --git a/src/simtools/utils/value_conversion.py b/src/simtools/utils/value_conversion.py index e12bbedd49..506fd64f6b 100644 --- a/src/simtools/utils/value_conversion.py +++ b/src/simtools/utils/value_conversion.py @@ -231,3 +231,25 @@ def _unit_as_string(unit): unit = [unit] unit = [str(element) if element is not None else None for element in unit] return unit[0] if len(set(unit)) == 1 else unit + + +def get_value_in_unit(value, unit=None): + """ + Return the numerical value of a Quantity in the requested unit. + + Parameters + ---------- + value: astropy.units.Quantity or any + Value to be converted. + unit: astropy.units.Unit or str or None + Unit to convert the value to. + + Returns + ------- + float or any + Numerical value of the Quantity in the requested unit, + or the original value if it is not a Quantity. + """ + if isinstance(value, u.Quantity): + return value.to_value(unit) if unit is not None else value.value + return value From 5173395b7247cfc9b995cb204d82ba539674ee24 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 19 May 2026 09:41:26 +0200 Subject: [PATCH 16/44] cleanup --- .../htcondor_script_generator.py | 12 +-- .../production_grid_helpers.py | 51 ------------ .../production_grid_job_rows.py | 79 +++++++++++++++---- 3 files changed, 64 insertions(+), 78 deletions(-) diff --git a/src/simtools/job_execution/htcondor_script_generator.py b/src/simtools/job_execution/htcondor_script_generator.py index 728525f12c..b365be1ded 100644 --- a/src/simtools/job_execution/htcondor_script_generator.py +++ b/src/simtools/job_execution/htcondor_script_generator.py @@ -15,11 +15,8 @@ import astropy.units as u from simtools.layout.array_layout_utils import resolve_array_layout_name -from simtools.production_configuration.production_grid_helpers import calculate_scaled_nshow from simtools.production_configuration.production_grid_job_rows import ( - build_backend_agnostic_job_rows, - get_core_scatter_max_for_zenith_angle, - get_energy_range_for_zenith_angle, + build_simulation_execution_rows, ) _logger = logging.getLogger(__name__) @@ -382,12 +379,7 @@ def _get_submit_script(args_dict): def build_job_specs(args_dict, image_labels): """Build backend-agnostic job specs from comparison and production grids.""" base_pack_dir = args_dict.get("simulation_output") or "simtools-output" - normalized_rows = build_backend_agnostic_job_rows( - args_dict, - calculate_scaled_nshow, - get_energy_range_for_zenith_angle_function=get_energy_range_for_zenith_angle, - get_core_scatter_max_for_zenith_angle_function=get_core_scatter_max_for_zenith_angle, - ) + normalized_rows = build_simulation_execution_rows(args_dict) job_specs = [] for label in image_labels: diff --git a/src/simtools/production_configuration/production_grid_helpers.py b/src/simtools/production_configuration/production_grid_helpers.py index dc3cacc8f4..2779cccdfe 100644 --- a/src/simtools/production_configuration/production_grid_helpers.py +++ b/src/simtools/production_configuration/production_grid_helpers.py @@ -1,6 +1,5 @@ """Helper functions and constants for production-grid generation.""" -import numpy as np from astropy import units as u from simtools.configuration import defaults @@ -59,53 +58,3 @@ def normalize_azimuth_angle(azimuth_angle): if isinstance(azimuth_angle, u.Quantity): return azimuth_angle.to(u.deg) % (360 * u.deg) return azimuth_angle % 360.0 - - -def calculate_log_energy_midpoint(energy_range_pair): - """Return the geometric-mean energy for an energy range pair.""" - energy_min, energy_max = energy_range_pair - - if not isinstance(energy_min, u.Quantity) or not isinstance(energy_max, u.Quantity): - raise TypeError("energy_range_pair must contain astropy Quantity values.") - - energy_min_tev = energy_min.to(u.TeV) - energy_max_tev = energy_max.to(u.TeV) - - if energy_min_tev <= 0 * u.TeV or energy_max_tev <= 0 * u.TeV: - raise ValueError("Energy range values must be strictly positive.") - - mean_log_energy = np.mean( - [ - np.log10(energy_min_tev.value), - np.log10(energy_max_tev.value), - ] - ) - return 10**mean_log_energy * u.TeV - - -def calculate_scaled_nshow( - energy_range_pair, - baseline_nshow, - nshow_power_index=None, - reference_energy=None, -): - """Return an energy-dependent nshow value.""" - if baseline_nshow < 1: - raise ValueError("baseline_nshow must be a positive integer.") - - if nshow_power_index is None: - return baseline_nshow - - if reference_energy is None: - raise ValueError("reference_energy is required when nshow_power_index is configured.") - - midpoint_energy = calculate_log_energy_midpoint(energy_range_pair) - scaling_factor = (midpoint_energy / reference_energy.to(midpoint_energy.unit)).to_value( - u.dimensionless_unscaled - ) ** nshow_power_index - scaled_nshow = int(np.ceil(baseline_nshow * scaling_factor)) - - if scaled_nshow < 1: - raise ValueError("Scaled nshow must be at least 1.") - - return scaled_nshow diff --git a/src/simtools/production_configuration/production_grid_job_rows.py b/src/simtools/production_configuration/production_grid_job_rows.py index dfd1c200cb..b815fe2148 100644 --- a/src/simtools/production_configuration/production_grid_job_rows.py +++ b/src/simtools/production_configuration/production_grid_job_rows.py @@ -1,5 +1,5 @@ """ -Builds backend-agnostic job row dictionaries for simulation production grids. +Build simulation execution rows based on the production configuration and lookup tables. A job row is a configuration dictionary for a single simulation job/run, containing all parameters needed to launch that simulation (e.g., primary, zenith, azimuth, energy range, @@ -8,6 +8,7 @@ import itertools +import numpy as np from astropy import units as u from simtools.production_configuration.corsika_limits_lookup import CorsikaLimitsLookup @@ -67,25 +68,70 @@ def get_core_scatter_max_for_zenith_angle( return min(configured_scatter_max, lookup_scatter_max.to(configured_scatter_max.unit)) -def build_backend_agnostic_job_rows( - args_dict, - calculate_scaled_nshow, - get_energy_range_for_zenith_angle_function=get_energy_range_for_zenith_angle, - get_core_scatter_max_for_zenith_angle_function=get_core_scatter_max_for_zenith_angle, +def calculate_log_energy_midpoint(energy_range_pair): + """Return the geometric-mean energy for an energy range pair.""" + energy_min, energy_max = energy_range_pair + + if not isinstance(energy_min, u.Quantity) or not isinstance(energy_max, u.Quantity): + raise TypeError("energy_range_pair must contain astropy Quantity values.") + + energy_min_tev = energy_min.to(u.TeV) + energy_max_tev = energy_max.to(u.TeV) + + if energy_min_tev <= 0 * u.TeV or energy_max_tev <= 0 * u.TeV: + raise ValueError("Energy range values must be strictly positive.") + + mean_log_energy = np.mean( + [ + np.log10(energy_min_tev.value), + np.log10(energy_max_tev.value), + ] + ) + return 10**mean_log_energy * u.TeV + + +def calculate_scaled_nshow( + energy_range_pair, + baseline_nshow, + nshow_power_index=None, + reference_energy=None, ): + """Return an energy-dependent nshow value.""" + if baseline_nshow < 1: + raise ValueError("baseline_nshow must be a positive integer.") + + if nshow_power_index is None: + return baseline_nshow + + if reference_energy is None: + raise ValueError("reference_energy is required when nshow_power_index is configured.") + + midpoint_energy = calculate_log_energy_midpoint(energy_range_pair) + scaling_factor = (midpoint_energy / reference_energy.to(midpoint_energy.unit)).to_value( + u.dimensionless_unscaled + ) ** nshow_power_index + scaled_nshow = int(np.ceil(baseline_nshow * scaling_factor)) + + if scaled_nshow < 1: + raise ValueError("Scaled nshow must be at least 1.") + + return scaled_nshow + + +def build_simulation_execution_rows(args_dict): """ - Build normalized production-grid rows for backend consumers. + Build simulation execution rows based on the production configuration and lookup tables. + + Each row corresponds to a single simulation job/run and contains all parameters needed to + launch that simulation (e.g., primary, zenith, azimuth, energy range, model version, etc.). + The energy range and core-scatter radius can be adjusted based on the direction-dependent + lookup-table limits. The number of events (``nshow``) can be scaled with energy according + to a configurable power law. Parameters ---------- args_dict : dict Production-job configuration. - calculate_scaled_nshow : callable - Callback used to compute the per-row ``nshow`` value. - get_energy_range_for_zenith_angle_function : callable, optional - Callback used to derive the direction-dependent energy range. - get_core_scatter_max_for_zenith_angle_function : callable, optional - Callback used to derive the direction-dependent core-scatter radius. Returns ------- @@ -135,7 +181,7 @@ def build_backend_agnostic_job_rows( corsika_he, energy_range_pair, ) in combinations: - selected_energy_range_pair = get_energy_range_for_zenith_angle_function( + selected_energy_range_pair = get_energy_range_for_zenith_angle( zenith, energy_range_pair, corsika_limits, @@ -144,7 +190,7 @@ def build_backend_agnostic_job_rows( if selected_energy_range_pair is None: continue - selected_core_scatter_max = get_core_scatter_max_for_zenith_angle_function( + selected_core_scatter_max = get_core_scatter_max_for_zenith_angle( zenith, core_scatter, corsika_limits, @@ -154,7 +200,7 @@ def build_backend_agnostic_job_rows( selected_energy_range_pair, nshow, nshow_power_index, reference_energy ) - for _ in range(number_of_runs): + for row_index in range(number_of_runs): rows.append( { "primary": primary, @@ -171,5 +217,4 @@ def build_backend_agnostic_job_rows( "run_number": run_number + row_index, } ) - row_index += 1 return rows From e48ca5f065400a26b44beccaf69be830d3356b30 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 19 May 2026 09:47:22 +0200 Subject: [PATCH 17/44] simplification --- .../production_grid_helpers.py | 60 ------------------- .../production_grid_job_rows.py | 50 ++++++++++++++-- .../production_grid_serialization.py | 4 +- 3 files changed, 47 insertions(+), 67 deletions(-) delete mode 100644 src/simtools/production_configuration/production_grid_helpers.py diff --git a/src/simtools/production_configuration/production_grid_helpers.py b/src/simtools/production_configuration/production_grid_helpers.py deleted file mode 100644 index 2779cccdfe..0000000000 --- a/src/simtools/production_configuration/production_grid_helpers.py +++ /dev/null @@ -1,60 +0,0 @@ -"""Helper functions and constants for production-grid generation.""" - -from astropy import units as u - -from simtools.configuration import defaults -from simtools.utils.general import ensure_list - -DEFAULT_SERIALIZATION_ROUND_DECIMALS = 6 - -_GRID_AXES = [ - "primary", - "azimuth_angle", - "zenith_angle", - "model_version", - "corsika_le_interaction", - "corsika_he_interaction", -] - -_GRID_AXIS_DEFAULTS = { - "corsika_le_interaction": defaults.CORSIKA_LE_INTERACTION, - "corsika_he_interaction": defaults.CORSIKA_HE_INTERACTION, -} - - -def normalize_grid_axes(args_dict): - """Return normalized grid axes for cartesian product expansion.""" - return { - axis: ( - ensure_list(args_dict[axis]) - if axis in args_dict and args_dict[axis] is not None - else [_GRID_AXIS_DEFAULTS[axis]] - if axis in _GRID_AXIS_DEFAULTS - else [None] - ) - for axis in _GRID_AXES - } - - -def normalize_energy_ranges(energy_range): - """Normalize energy range argument to a list of ``(e_min, e_max)`` pairs.""" - if isinstance(energy_range, tuple) and len(energy_range) == 2: - return [energy_range] - - if isinstance(energy_range, list): - if len(energy_range) == 2 and all(hasattr(item, "to") for item in energy_range): - return [(energy_range[0], energy_range[1])] - if all(isinstance(item, (list, tuple)) and len(item) == 2 for item in energy_range): - return [tuple(item) for item in energy_range] - - raise ValueError( - "energy_range must be one pair (e_min, e_max) or a list of (e_min, e_max) pairs." - ) - - -def normalize_azimuth_angle(azimuth_angle): - """Return an azimuth angle with degree units and modulo-360 normalization.""" - # TODO - duplication? - if isinstance(azimuth_angle, u.Quantity): - return azimuth_angle.to(u.deg) % (360 * u.deg) - return azimuth_angle % 360.0 diff --git a/src/simtools/production_configuration/production_grid_job_rows.py b/src/simtools/production_configuration/production_grid_job_rows.py index b815fe2148..3c90f4e3a6 100644 --- a/src/simtools/production_configuration/production_grid_job_rows.py +++ b/src/simtools/production_configuration/production_grid_job_rows.py @@ -11,11 +11,53 @@ import numpy as np from astropy import units as u +from simtools.configuration import defaults from simtools.production_configuration.corsika_limits_lookup import CorsikaLimitsLookup -from simtools.production_configuration.production_grid_helpers import ( - normalize_energy_ranges, - normalize_grid_axes, -) +from simtools.utils.general import ensure_list + +_GRID_AXES = [ + "primary", + "azimuth_angle", + "zenith_angle", + "model_version", + "corsika_le_interaction", + "corsika_he_interaction", +] + +_GRID_AXIS_DEFAULTS = { + "corsika_le_interaction": defaults.CORSIKA_LE_INTERACTION, + "corsika_he_interaction": defaults.CORSIKA_HE_INTERACTION, +} + + +def normalize_grid_axes(args_dict): + """Return normalized grid axes for cartesian product expansion.""" + return { + axis: ( + ensure_list(args_dict[axis]) + if axis in args_dict and args_dict[axis] is not None + else [_GRID_AXIS_DEFAULTS[axis]] + if axis in _GRID_AXIS_DEFAULTS + else [None] + ) + for axis in _GRID_AXES + } + + +def normalize_energy_ranges(energy_range): + """Normalize energy range argument to a list of ``(e_min, e_max)`` pairs.""" + if isinstance(energy_range, tuple) and len(energy_range) == 2: + return [energy_range] + + if isinstance(energy_range, list): + if len(energy_range) == 2 and all(hasattr(item, "to") for item in energy_range): + return [(energy_range[0], energy_range[1])] + if all(isinstance(item, (list, tuple)) and len(item) == 2 for item in energy_range): + return [tuple(item) for item in energy_range] + + raise ValueError( + "energy_range must be one pair (e_min, e_max) or a list of (e_min, e_max) pairs." + ) def get_energy_range_for_zenith_angle( diff --git a/src/simtools/production_configuration/production_grid_serialization.py b/src/simtools/production_configuration/production_grid_serialization.py index 6553c2243b..6441d1627b 100644 --- a/src/simtools/production_configuration/production_grid_serialization.py +++ b/src/simtools/production_configuration/production_grid_serialization.py @@ -6,9 +6,7 @@ from astropy import units as u from astropy.table import Table -from simtools.production_configuration.production_grid_helpers import ( - DEFAULT_SERIALIZATION_ROUND_DECIMALS, -) +DEFAULT_SERIALIZATION_ROUND_DECIMALS = 6 def collect_point_keys(grid_points): From adb3a0b57c3af8f8f9625555198343f4518e4228 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 19 May 2026 10:18:26 +0200 Subject: [PATCH 18/44] improved naming --- ...duction_grid_job_rows.py => build_grid.py} | 16 +- .../production_configuration/grid_engine.py | 456 ++++++++++++++++++ ...serialization.py => grid_serialization.py} | 0 .../production_grid_engine.py | 2 +- 4 files changed, 462 insertions(+), 12 deletions(-) rename src/simtools/production_configuration/{production_grid_job_rows.py => build_grid.py} (94%) create mode 100644 src/simtools/production_configuration/grid_engine.py rename src/simtools/production_configuration/{production_grid_serialization.py => grid_serialization.py} (100%) diff --git a/src/simtools/production_configuration/production_grid_job_rows.py b/src/simtools/production_configuration/build_grid.py similarity index 94% rename from src/simtools/production_configuration/production_grid_job_rows.py rename to src/simtools/production_configuration/build_grid.py index 3c90f4e3a6..e0ed6ca05f 100644 --- a/src/simtools/production_configuration/production_grid_job_rows.py +++ b/src/simtools/production_configuration/build_grid.py @@ -1,10 +1,4 @@ -""" -Build simulation execution rows based on the production configuration and lookup tables. - -A job row is a configuration dictionary for a single simulation job/run, containing all -parameters needed to launch that simulation (e.g., primary, zenith, azimuth, energy range, -model version, etc.). -""" +"""Build simulation execution grid based on the production configuration and lookup tables.""" import itertools @@ -160,11 +154,11 @@ def calculate_scaled_nshow( return scaled_nshow -def build_simulation_execution_rows(args_dict): +def build_simulation_jobs(args_dict): """ - Build simulation execution rows based on the production configuration and lookup tables. + Build simulation job parameters based on the production configuration and lookup tables. - Each row corresponds to a single simulation job/run and contains all parameters needed to + Each entry corresponds to a single simulation job/run and contains all parameters needed to launch that simulation (e.g., primary, zenith, azimuth, energy range, model version, etc.). The energy range and core-scatter radius can be adjusted based on the direction-dependent lookup-table limits. The number of events (``nshow``) can be scaled with energy according @@ -178,7 +172,7 @@ def build_simulation_execution_rows(args_dict): Returns ------- list[dict] - Backend-independent row dictionaries. + Simulation job parameter dictionaries for all combinations of configured axes. """ grid_axes = normalize_grid_axes(args_dict) energy_ranges = normalize_energy_ranges(args_dict["energy_range"]) diff --git a/src/simtools/production_configuration/grid_engine.py b/src/simtools/production_configuration/grid_engine.py new file mode 100644 index 0000000000..d77c3be59c --- /dev/null +++ b/src/simtools/production_configuration/grid_engine.py @@ -0,0 +1,456 @@ +""" +Generate simulation production grids for configurable axes, coordinate systems, and lookup tables. + +This module builds simulation grids from configurable axis definitions (for example, +azimuth, zenith angle, night-sky background, and camera offset). It supports +multiple binning and scaling modes, applies CORSIKA limit interpolation from lookup +tables, and handles coordinate conversions between horizontal (Alt/Az) and +equatorial (RA/Dec) systems. + +The engine also provides serialization helpers used by backend adapters to export +generated grid points and metadata. +""" + +import logging + +import numpy as np +from astropy import units as u +from astropy.coordinates import AltAz, EarthLocation, SkyCoord +from astropy.units import Quantity + +from simtools.production_configuration.corsika_limits_lookup import CorsikaLimitsLookup +from simtools.production_configuration.grid_serialization import serialize_grid_points + + +class ProductionGridEngine: + """ + Generate simulation production grids. + + This engine handles axis expansion (e.g. azimuth, zenith angle, night-sky background), + coordinate-system specific grid generation, lookup-table interpolation, coordinate conversion, + and ECSV serialization helpers used by backend adapters (e.g. HTCondor). + """ + + def __init__( + self, + axes, + coordinate_system="zenith_azimuth", + observing_location=None, + observing_time=None, + lookup_table=None, + telescope_ids=None, + simtel_file=None, + ): + """ + Initialize the production-grid engine. + + Parameters + ---------- + axes : dict + Dictionary where each key is the axis name and the value is a dictionary + defining the axis properties (range, binning, scaling, etc.). + coordinate_system : str, optional + The coordinate system for the grid generation. + observing_location : EarthLocation, optional + The location of the observation (latitude, longitude, height). + observing_time : Time, optional + The time of observation required for RA/Dec transforms. + lookup_table : str, optional + Path to the lookup table file (ECSV format). + telescope_ids : list of str, optional + Telescope selection used to filter lookup rows. + simtel_file : str, optional + Path to a sim_telarray file used to map numeric telescope IDs. + """ + self._logger = logging.getLogger(__name__) + self.axes = axes["axes"] if "axes" in axes else axes + self.coordinate_system = coordinate_system + self.observing_location = ( + observing_location + if observing_location is not None + else EarthLocation(lat=0.0 * u.deg, lon=0.0 * u.deg, height=0 * u.m) + ) + self.observing_time = observing_time + self.lookup_table = lookup_table + self.telescope_ids = telescope_ids + self.simtel_file = simtel_file + self.interpolated_limits = {} + self._limits_lookup = CorsikaLimitsLookup( + lookup_table=lookup_table, + telescope_ids=telescope_ids, + simtel_file=simtel_file, + ) + self._simtel_id_to_name = self._limits_lookup.simtel_id_to_name + self.target_values = self._generate_target_values() + + if self.lookup_table: + if self.coordinate_system == "ra_dec": + self._prepare_lookup_table_limits_for_point_interpolation() + else: + self._apply_lookup_table_limits() + + def _sync_limits_lookup(self): + """Synchronize mutable lookup settings with the shared lookup helper.""" + self._limits_lookup.lookup_table = self.lookup_table + self._limits_lookup.telescope_ids = self.telescope_ids + self._limits_lookup.simtel_file = self.simtel_file + self._limits_lookup.simtel_id_to_name = self._simtel_id_to_name + + def _require_observing_time(self): + """Return observing time if available, else raise a clear error.""" + if self.observing_time is None: + raise ValueError("Observing time is required for ra_dec grid generation.") + return self.observing_time + + def _get_max_zenith_for_radec_mode(self): + """Read maximum zenith from axes for RA/Dec direction sampling.""" + zenith_axis = self.axes.get("zenith_angle") + if not zenith_axis or "range" not in zenith_axis or len(zenith_axis["range"]) != 2: + raise ValueError( + "RA/Dec direction sampling requires 'zenith_angle' axis with a valid " + "two-element 'range' in the axes definition." + ) + return float(zenith_axis["range"][1]) + + def _prepare_lookup_table_limits_for_point_interpolation(self): + """Prepare lookup arrays for per-point interpolation in RA/Dec grid mode.""" + self._sync_limits_lookup() + self._limits_lookup.prepare_point_interpolators() + + def _generate_target_values(self): + """ + Generate target axis values and store them as Quantities. + + Returns + ------- + dict + Dictionary of target values for each axis, stored as Quantity objects. + """ + target_values = {} + for axis_name, axis in self.axes.items(): + axis_range = axis["range"] + binning = axis["binning"] + scaling = axis.get("scaling", "linear") + units = axis.get("units", None) + + if axis_name == "azimuth": + values = self.create_circular_binning(axis_range, binning) + elif scaling == "log": + values = np.logspace(np.log10(axis_range[0]), np.log10(axis_range[1]), binning) + elif scaling == "1/cos": + cos_min = np.cos(np.radians(axis_range[0])) + cos_max = np.cos(np.radians(axis_range[1])) + inv_cos_values = np.linspace(1 / cos_min, 1 / cos_max, binning) + values = np.degrees(np.arccos(1 / inv_cos_values)) + else: + values = np.linspace(axis_range[0], axis_range[1], binning) + + if units: + values = values * u.Unit(units) + + target_values[axis_name] = values + + return target_values + + def _apply_lookup_table_limits(self): + """Apply limits from the lookup table and interpolate values.""" + self._sync_limits_lookup() + self.interpolated_limits = self._limits_lookup.interpolate_grid_limits(self.target_values) + + def _generate_radec_grid_direction_points(self): + """Generate direction points from declination lines and hour-angle spacing.""" + observing_time = self._require_observing_time() + max_zenith = self._get_max_zenith_for_radec_mode() + lst_deg = observing_time.sidereal_time( + "apparent", longitude=self.observing_location.lon + ).deg + + direction_points = [] + for declination in np.arange(-90.0, 91.0, 1.0): + cos_dec = np.cos(np.deg2rad(declination)) + step_ha = 1.0 / cos_dec if cos_dec > 1e-6 else 360.0 + n_ha = max(1, int(np.ceil(360.0 / step_ha))) + hour_angles = np.linspace(-180.0, 180.0, n_ha, endpoint=False) + ra_values = (lst_deg - hour_angles) % 360.0 + + skycoord = SkyCoord( + ra=ra_values * u.deg, + dec=np.full_like(ra_values, declination) * u.deg, + frame="icrs", + ) + altaz = skycoord.transform_to( + AltAz(location=self.observing_location, obstime=observing_time) + ) + + zenith_values = (90.0 * u.deg - altaz.alt).to(u.deg).value + mask = (zenith_values >= 0.0) & (zenith_values <= max_zenith) + + for idx in np.nonzero(mask)[0]: + direction_points.append( + { + "zenith_angle": zenith_values[idx] * u.deg, + "azimuth": altaz.az.deg[idx] * u.deg, + } + ) + return direction_points + + def _generate_extra_axis_combinations(self, excluded_keys): + """Generate combinations for all axes except the excluded ones.""" + extra_axes = { + key: value for key, value in self.target_values.items() if key not in excluded_keys + } + if not extra_axes: + return list(extra_axes.keys()), [], [np.array([])] + + extra_value_arrays = [value.value for value in extra_axes.values()] + extra_units = [value.unit for value in extra_axes.values()] + extra_grid = np.meshgrid(*extra_value_arrays, indexing="ij") + extra_combinations = np.vstack(list(map(np.ravel, extra_grid))).T + return list(extra_axes.keys()), extra_units, extra_combinations + + def _interpolate_limits_for_point(self, zenith, azimuth, nsb): + """Interpolate lookup-table limits for a single point.""" + self._sync_limits_lookup() + return self._limits_lookup.interpolate_point(zenith, azimuth, nsb) + + def _add_lookup_limits_to_point(self, point, zenith, azimuth): + """Interpolate and attach lookup-table limits to a grid point.""" + if not self.lookup_table: + return + + nsb_value = point.get("nsb_level", 1) + if isinstance(nsb_value, Quantity): + nsb_value = nsb_value.value + limits = self._interpolate_limits_for_point( + zenith=zenith, + azimuth=azimuth, + nsb=float(nsb_value), + ) + point["lower_energy_threshold"] = limits["lower_energy_threshold"] * u.TeV + point["scatter_radius"] = limits["upper_scatter_radius"] * u.m + point["viewcone_radius"] = limits["viewcone_radius"] * u.deg + + def _generate_grid_from_radec_axes(self): + """Generate grid points from explicit RA/Dec axes definitions.""" + observing_time = self._require_observing_time() + + axis_keys = [key for key in self.target_values if key not in ("zenith_angle", "azimuth")] + value_arrays = [self.target_values[key].value for key in axis_keys] + units = [self.target_values[key].unit for key in axis_keys] + grid = np.meshgrid(*value_arrays, indexing="ij") + combinations = np.vstack(list(map(np.ravel, grid))).T + + grid_points = [] + for combination in combinations: + grid_point = { + key: Quantity(combination[i], units[i]) for i, key in enumerate(axis_keys) + } + + skycoord = SkyCoord( + ra=grid_point["ra"].to(u.deg), + dec=grid_point["dec"].to(u.deg), + frame="icrs", + ) + altaz = skycoord.transform_to( + AltAz(location=self.observing_location, obstime=observing_time) + ) + zenith = (90.0 * u.deg - altaz.alt).to(u.deg).value + + self._add_lookup_limits_to_point(grid_point, zenith=zenith, azimuth=altaz.az.deg) + grid_points.append(grid_point) + + return grid_points + + def _generate_grid_radec_mode(self): + """Generate grid points for RA/Dec mode.""" + if "ra" in self.axes and "dec" in self.axes: + return self._generate_grid_from_radec_axes() + + direction_points = self._generate_radec_grid_direction_points() + extra_keys, extra_units, extra_combinations = self._generate_extra_axis_combinations( + excluded_keys=("zenith_angle", "azimuth") + ) + + grid_points = [] + for direction_point in direction_points: + for extra_combination in extra_combinations: + point = dict(direction_point) + for i, key in enumerate(extra_keys): + point[key] = Quantity(extra_combination[i], extra_units[i]) + + self._add_lookup_limits_to_point( + point, + zenith=point["zenith_angle"].value, + azimuth=point["azimuth"].value, + ) + grid_points.append(point) + + return grid_points + + def create_circular_binning(self, azimuth_range, num_bins): + """ + Create bin centers for azimuth angles, handling circular wraparound. + + Parameters + ---------- + azimuth_range : tuple + (min_azimuth, max_azimuth), can wrap around 0 deg. + num_bins : int + Number of bins. + + Returns + ------- + np.ndarray + Array of bin centers. + """ + azimuth_min, azimuth_max = azimuth_range + azimuth_min %= 360 + azimuth_max %= 360 + + clockwise_distance = (azimuth_max - azimuth_min) % 360 + counterclockwise_distance = (azimuth_min - azimuth_max) % 360 + + if clockwise_distance <= counterclockwise_distance: + return ( + np.linspace(azimuth_min, azimuth_min + clockwise_distance, num_bins, endpoint=True) + % 360 + ) + return ( + np.linspace( + azimuth_min, azimuth_min - counterclockwise_distance, num_bins, endpoint=True + ) + % 360 + ) + + def generate_grid(self): + """ + Generate the grid based on the required axes and include interpolated limits. + + Returns + ------- + list of dict + A list of generated grid points. + """ + if self.coordinate_system == "ra_dec": + return self._generate_grid_radec_mode() + + value_arrays = [value.value for value in self.target_values.values()] + units = [value.unit for value in self.target_values.values()] + grid = np.meshgrid(*value_arrays, indexing="ij") + combinations = np.vstack(list(map(np.ravel, grid))).T + grid_points = [] + + for combination in combinations: + grid_point = { + key: Quantity(combination[i], units[i]) + for i, key in enumerate(self.target_values.keys()) + } + + if "lower_energy_threshold" in self.interpolated_limits: + zenith_idx = np.searchsorted( + self.target_values["zenith_angle"].value, grid_point["zenith_angle"].value + ) + azimuth_idx = np.searchsorted( + self.target_values["azimuth"].value, grid_point["azimuth"].value + ) + nsb_idx = np.searchsorted( + self.target_values["nsb_level"].value, + grid_point["nsb_level"].value, + ) + grid_point["lower_energy_threshold"] = ( + self.interpolated_limits["lower_energy_threshold"][ + zenith_idx, azimuth_idx, nsb_idx + ] + * u.TeV + ) + + if "upper_scatter_radius" in self.interpolated_limits: + grid_point["scatter_radius"] = ( + self.interpolated_limits["upper_scatter_radius"][ + zenith_idx, azimuth_idx, nsb_idx + ] + * u.m + ) + + if "viewcone_radius" in self.interpolated_limits: + grid_point["viewcone_radius"] = ( + self.interpolated_limits["viewcone_radius"][zenith_idx, azimuth_idx, nsb_idx] + * u.deg + ) + + grid_points.append(grid_point) + + return self.convert_coordinates(grid_points) + + def convert_altaz_to_radec(self, alt, az): + """ + Convert Altitude/Azimuth (AltAz) coordinates to RA/Dec. + + Parameters + ---------- + alt : float + Altitude angle in degrees. + az : float + Azimuth angle in degrees. + + Returns + ------- + SkyCoord + SkyCoord object containing the RA/Dec coordinates. + """ + if self.observing_time is None: + raise ValueError("Conversion to RA/dec requires observing_time to be set. ") + + aa = AltAz( + alt=alt.to(u.rad), + az=az.to(u.rad), + location=self.observing_location, + obstime=self.observing_time, + ) + sky_coord = SkyCoord(aa) + return sky_coord.icrs # Return RA/Dec in ICRS frame + + def convert_coordinates(self, grid_points): + """ + Convert the grid points RA/Dec coordinates if necessary. + + Parameters + ---------- + grid_points : list of dict + List of grid points. + + Returns + ------- + list of dict + The grid points with converted RA/Dec coordinates. + """ + if self.coordinate_system == "ra_dec": + for point in grid_points: + if "zenith_angle" in point and "azimuth" in point: + alt = (90.0 * u.deg) - point.pop("zenith_angle") + az = point.pop("azimuth") + radec = self.convert_altaz_to_radec(alt, az) + point["ra"] = radec.ra.deg * u.deg + point["dec"] = radec.dec.deg * u.deg + return grid_points + + def serialize_grid_points(self, grid_points, output_file): + """ + Serialize grid points to an ECSV table file. + + Parameters + ---------- + grid_points : list of dict + List of grid points to serialize. + output_file : str + Path to the output ECSV file. + """ + serialize_grid_points( + grid_points=grid_points, + output_file=output_file, + coordinate_system=self.coordinate_system, + observing_time=self.observing_time, + telescope_ids=self.telescope_ids, + lookup_table=self.lookup_table, + ) + self._logger.info(f"Simulation grid saved to {output_file}") diff --git a/src/simtools/production_configuration/production_grid_serialization.py b/src/simtools/production_configuration/grid_serialization.py similarity index 100% rename from src/simtools/production_configuration/production_grid_serialization.py rename to src/simtools/production_configuration/grid_serialization.py diff --git a/src/simtools/production_configuration/production_grid_engine.py b/src/simtools/production_configuration/production_grid_engine.py index c10702ae7f..d77c3be59c 100644 --- a/src/simtools/production_configuration/production_grid_engine.py +++ b/src/simtools/production_configuration/production_grid_engine.py @@ -19,7 +19,7 @@ from astropy.units import Quantity from simtools.production_configuration.corsika_limits_lookup import CorsikaLimitsLookup -from simtools.production_configuration.production_grid_serialization import serialize_grid_points +from simtools.production_configuration.grid_serialization import serialize_grid_points class ProductionGridEngine: From a415860d042c69961cf9e7424a7622bca0649541 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 19 May 2026 10:19:42 +0200 Subject: [PATCH 19/44] cleanup [skip ci] --- .../api-reference/production_configuration.md | 8 +- .../applications/production_generate_grid.py | 2 +- .../htcondor_script_generator.py | 6 +- .../production_grid_engine.py | 456 ------------------ 4 files changed, 7 insertions(+), 465 deletions(-) delete mode 100644 src/simtools/production_configuration/production_grid_engine.py diff --git a/docs/source/api-reference/production_configuration.md b/docs/source/api-reference/production_configuration.md index f0478092eb..b91c617965 100644 --- a/docs/source/api-reference/production_configuration.md +++ b/docs/source/api-reference/production_configuration.md @@ -75,10 +75,10 @@ the calculation of the number of events to be simulated given a pre-determined m (production-grid-engine)= -## production_grid_engine +## grid_engine ```{eval-rst} -.. automodule:: production_configuration.production_grid_engine +.. automodule:: production_configuration.grid_engine :members: ``` @@ -115,10 +115,10 @@ the calculation of the number of events to be simulated given a pre-determined m (production-grid-serialization)= -## production_grid_serialization +## grid_serialization ```{eval-rst} -.. automodule:: production_configuration.production_grid_serialization +.. automodule:: production_configuration.grid_serialization :members: ``` diff --git a/src/simtools/applications/production_generate_grid.py b/src/simtools/applications/production_generate_grid.py index 973af3ec7d..4cd057630a 100644 --- a/src/simtools/applications/production_generate_grid.py +++ b/src/simtools/applications/production_generate_grid.py @@ -77,7 +77,7 @@ from simtools.application_control import build_application from simtools.io.ascii_handler import collect_data_from_file from simtools.model.site_model import SiteModel -from simtools.production_configuration.production_grid_engine import ProductionGridEngine +from simtools.production_configuration.grid_engine import ProductionGridEngine def _add_arguments(parser): diff --git a/src/simtools/job_execution/htcondor_script_generator.py b/src/simtools/job_execution/htcondor_script_generator.py index b365be1ded..c53d202ce3 100644 --- a/src/simtools/job_execution/htcondor_script_generator.py +++ b/src/simtools/job_execution/htcondor_script_generator.py @@ -15,9 +15,7 @@ import astropy.units as u from simtools.layout.array_layout_utils import resolve_array_layout_name -from simtools.production_configuration.production_grid_job_rows import ( - build_simulation_execution_rows, -) +from simtools.production_configuration.build_grid import build_simulation_jobs _logger = logging.getLogger(__name__) @@ -379,7 +377,7 @@ def _get_submit_script(args_dict): def build_job_specs(args_dict, image_labels): """Build backend-agnostic job specs from comparison and production grids.""" base_pack_dir = args_dict.get("simulation_output") or "simtools-output" - normalized_rows = build_simulation_execution_rows(args_dict) + normalized_rows = build_simulation_jobs(args_dict) job_specs = [] for label in image_labels: diff --git a/src/simtools/production_configuration/production_grid_engine.py b/src/simtools/production_configuration/production_grid_engine.py deleted file mode 100644 index d77c3be59c..0000000000 --- a/src/simtools/production_configuration/production_grid_engine.py +++ /dev/null @@ -1,456 +0,0 @@ -""" -Generate simulation production grids for configurable axes, coordinate systems, and lookup tables. - -This module builds simulation grids from configurable axis definitions (for example, -azimuth, zenith angle, night-sky background, and camera offset). It supports -multiple binning and scaling modes, applies CORSIKA limit interpolation from lookup -tables, and handles coordinate conversions between horizontal (Alt/Az) and -equatorial (RA/Dec) systems. - -The engine also provides serialization helpers used by backend adapters to export -generated grid points and metadata. -""" - -import logging - -import numpy as np -from astropy import units as u -from astropy.coordinates import AltAz, EarthLocation, SkyCoord -from astropy.units import Quantity - -from simtools.production_configuration.corsika_limits_lookup import CorsikaLimitsLookup -from simtools.production_configuration.grid_serialization import serialize_grid_points - - -class ProductionGridEngine: - """ - Generate simulation production grids. - - This engine handles axis expansion (e.g. azimuth, zenith angle, night-sky background), - coordinate-system specific grid generation, lookup-table interpolation, coordinate conversion, - and ECSV serialization helpers used by backend adapters (e.g. HTCondor). - """ - - def __init__( - self, - axes, - coordinate_system="zenith_azimuth", - observing_location=None, - observing_time=None, - lookup_table=None, - telescope_ids=None, - simtel_file=None, - ): - """ - Initialize the production-grid engine. - - Parameters - ---------- - axes : dict - Dictionary where each key is the axis name and the value is a dictionary - defining the axis properties (range, binning, scaling, etc.). - coordinate_system : str, optional - The coordinate system for the grid generation. - observing_location : EarthLocation, optional - The location of the observation (latitude, longitude, height). - observing_time : Time, optional - The time of observation required for RA/Dec transforms. - lookup_table : str, optional - Path to the lookup table file (ECSV format). - telescope_ids : list of str, optional - Telescope selection used to filter lookup rows. - simtel_file : str, optional - Path to a sim_telarray file used to map numeric telescope IDs. - """ - self._logger = logging.getLogger(__name__) - self.axes = axes["axes"] if "axes" in axes else axes - self.coordinate_system = coordinate_system - self.observing_location = ( - observing_location - if observing_location is not None - else EarthLocation(lat=0.0 * u.deg, lon=0.0 * u.deg, height=0 * u.m) - ) - self.observing_time = observing_time - self.lookup_table = lookup_table - self.telescope_ids = telescope_ids - self.simtel_file = simtel_file - self.interpolated_limits = {} - self._limits_lookup = CorsikaLimitsLookup( - lookup_table=lookup_table, - telescope_ids=telescope_ids, - simtel_file=simtel_file, - ) - self._simtel_id_to_name = self._limits_lookup.simtel_id_to_name - self.target_values = self._generate_target_values() - - if self.lookup_table: - if self.coordinate_system == "ra_dec": - self._prepare_lookup_table_limits_for_point_interpolation() - else: - self._apply_lookup_table_limits() - - def _sync_limits_lookup(self): - """Synchronize mutable lookup settings with the shared lookup helper.""" - self._limits_lookup.lookup_table = self.lookup_table - self._limits_lookup.telescope_ids = self.telescope_ids - self._limits_lookup.simtel_file = self.simtel_file - self._limits_lookup.simtel_id_to_name = self._simtel_id_to_name - - def _require_observing_time(self): - """Return observing time if available, else raise a clear error.""" - if self.observing_time is None: - raise ValueError("Observing time is required for ra_dec grid generation.") - return self.observing_time - - def _get_max_zenith_for_radec_mode(self): - """Read maximum zenith from axes for RA/Dec direction sampling.""" - zenith_axis = self.axes.get("zenith_angle") - if not zenith_axis or "range" not in zenith_axis or len(zenith_axis["range"]) != 2: - raise ValueError( - "RA/Dec direction sampling requires 'zenith_angle' axis with a valid " - "two-element 'range' in the axes definition." - ) - return float(zenith_axis["range"][1]) - - def _prepare_lookup_table_limits_for_point_interpolation(self): - """Prepare lookup arrays for per-point interpolation in RA/Dec grid mode.""" - self._sync_limits_lookup() - self._limits_lookup.prepare_point_interpolators() - - def _generate_target_values(self): - """ - Generate target axis values and store them as Quantities. - - Returns - ------- - dict - Dictionary of target values for each axis, stored as Quantity objects. - """ - target_values = {} - for axis_name, axis in self.axes.items(): - axis_range = axis["range"] - binning = axis["binning"] - scaling = axis.get("scaling", "linear") - units = axis.get("units", None) - - if axis_name == "azimuth": - values = self.create_circular_binning(axis_range, binning) - elif scaling == "log": - values = np.logspace(np.log10(axis_range[0]), np.log10(axis_range[1]), binning) - elif scaling == "1/cos": - cos_min = np.cos(np.radians(axis_range[0])) - cos_max = np.cos(np.radians(axis_range[1])) - inv_cos_values = np.linspace(1 / cos_min, 1 / cos_max, binning) - values = np.degrees(np.arccos(1 / inv_cos_values)) - else: - values = np.linspace(axis_range[0], axis_range[1], binning) - - if units: - values = values * u.Unit(units) - - target_values[axis_name] = values - - return target_values - - def _apply_lookup_table_limits(self): - """Apply limits from the lookup table and interpolate values.""" - self._sync_limits_lookup() - self.interpolated_limits = self._limits_lookup.interpolate_grid_limits(self.target_values) - - def _generate_radec_grid_direction_points(self): - """Generate direction points from declination lines and hour-angle spacing.""" - observing_time = self._require_observing_time() - max_zenith = self._get_max_zenith_for_radec_mode() - lst_deg = observing_time.sidereal_time( - "apparent", longitude=self.observing_location.lon - ).deg - - direction_points = [] - for declination in np.arange(-90.0, 91.0, 1.0): - cos_dec = np.cos(np.deg2rad(declination)) - step_ha = 1.0 / cos_dec if cos_dec > 1e-6 else 360.0 - n_ha = max(1, int(np.ceil(360.0 / step_ha))) - hour_angles = np.linspace(-180.0, 180.0, n_ha, endpoint=False) - ra_values = (lst_deg - hour_angles) % 360.0 - - skycoord = SkyCoord( - ra=ra_values * u.deg, - dec=np.full_like(ra_values, declination) * u.deg, - frame="icrs", - ) - altaz = skycoord.transform_to( - AltAz(location=self.observing_location, obstime=observing_time) - ) - - zenith_values = (90.0 * u.deg - altaz.alt).to(u.deg).value - mask = (zenith_values >= 0.0) & (zenith_values <= max_zenith) - - for idx in np.nonzero(mask)[0]: - direction_points.append( - { - "zenith_angle": zenith_values[idx] * u.deg, - "azimuth": altaz.az.deg[idx] * u.deg, - } - ) - return direction_points - - def _generate_extra_axis_combinations(self, excluded_keys): - """Generate combinations for all axes except the excluded ones.""" - extra_axes = { - key: value for key, value in self.target_values.items() if key not in excluded_keys - } - if not extra_axes: - return list(extra_axes.keys()), [], [np.array([])] - - extra_value_arrays = [value.value for value in extra_axes.values()] - extra_units = [value.unit for value in extra_axes.values()] - extra_grid = np.meshgrid(*extra_value_arrays, indexing="ij") - extra_combinations = np.vstack(list(map(np.ravel, extra_grid))).T - return list(extra_axes.keys()), extra_units, extra_combinations - - def _interpolate_limits_for_point(self, zenith, azimuth, nsb): - """Interpolate lookup-table limits for a single point.""" - self._sync_limits_lookup() - return self._limits_lookup.interpolate_point(zenith, azimuth, nsb) - - def _add_lookup_limits_to_point(self, point, zenith, azimuth): - """Interpolate and attach lookup-table limits to a grid point.""" - if not self.lookup_table: - return - - nsb_value = point.get("nsb_level", 1) - if isinstance(nsb_value, Quantity): - nsb_value = nsb_value.value - limits = self._interpolate_limits_for_point( - zenith=zenith, - azimuth=azimuth, - nsb=float(nsb_value), - ) - point["lower_energy_threshold"] = limits["lower_energy_threshold"] * u.TeV - point["scatter_radius"] = limits["upper_scatter_radius"] * u.m - point["viewcone_radius"] = limits["viewcone_radius"] * u.deg - - def _generate_grid_from_radec_axes(self): - """Generate grid points from explicit RA/Dec axes definitions.""" - observing_time = self._require_observing_time() - - axis_keys = [key for key in self.target_values if key not in ("zenith_angle", "azimuth")] - value_arrays = [self.target_values[key].value for key in axis_keys] - units = [self.target_values[key].unit for key in axis_keys] - grid = np.meshgrid(*value_arrays, indexing="ij") - combinations = np.vstack(list(map(np.ravel, grid))).T - - grid_points = [] - for combination in combinations: - grid_point = { - key: Quantity(combination[i], units[i]) for i, key in enumerate(axis_keys) - } - - skycoord = SkyCoord( - ra=grid_point["ra"].to(u.deg), - dec=grid_point["dec"].to(u.deg), - frame="icrs", - ) - altaz = skycoord.transform_to( - AltAz(location=self.observing_location, obstime=observing_time) - ) - zenith = (90.0 * u.deg - altaz.alt).to(u.deg).value - - self._add_lookup_limits_to_point(grid_point, zenith=zenith, azimuth=altaz.az.deg) - grid_points.append(grid_point) - - return grid_points - - def _generate_grid_radec_mode(self): - """Generate grid points for RA/Dec mode.""" - if "ra" in self.axes and "dec" in self.axes: - return self._generate_grid_from_radec_axes() - - direction_points = self._generate_radec_grid_direction_points() - extra_keys, extra_units, extra_combinations = self._generate_extra_axis_combinations( - excluded_keys=("zenith_angle", "azimuth") - ) - - grid_points = [] - for direction_point in direction_points: - for extra_combination in extra_combinations: - point = dict(direction_point) - for i, key in enumerate(extra_keys): - point[key] = Quantity(extra_combination[i], extra_units[i]) - - self._add_lookup_limits_to_point( - point, - zenith=point["zenith_angle"].value, - azimuth=point["azimuth"].value, - ) - grid_points.append(point) - - return grid_points - - def create_circular_binning(self, azimuth_range, num_bins): - """ - Create bin centers for azimuth angles, handling circular wraparound. - - Parameters - ---------- - azimuth_range : tuple - (min_azimuth, max_azimuth), can wrap around 0 deg. - num_bins : int - Number of bins. - - Returns - ------- - np.ndarray - Array of bin centers. - """ - azimuth_min, azimuth_max = azimuth_range - azimuth_min %= 360 - azimuth_max %= 360 - - clockwise_distance = (azimuth_max - azimuth_min) % 360 - counterclockwise_distance = (azimuth_min - azimuth_max) % 360 - - if clockwise_distance <= counterclockwise_distance: - return ( - np.linspace(azimuth_min, azimuth_min + clockwise_distance, num_bins, endpoint=True) - % 360 - ) - return ( - np.linspace( - azimuth_min, azimuth_min - counterclockwise_distance, num_bins, endpoint=True - ) - % 360 - ) - - def generate_grid(self): - """ - Generate the grid based on the required axes and include interpolated limits. - - Returns - ------- - list of dict - A list of generated grid points. - """ - if self.coordinate_system == "ra_dec": - return self._generate_grid_radec_mode() - - value_arrays = [value.value for value in self.target_values.values()] - units = [value.unit for value in self.target_values.values()] - grid = np.meshgrid(*value_arrays, indexing="ij") - combinations = np.vstack(list(map(np.ravel, grid))).T - grid_points = [] - - for combination in combinations: - grid_point = { - key: Quantity(combination[i], units[i]) - for i, key in enumerate(self.target_values.keys()) - } - - if "lower_energy_threshold" in self.interpolated_limits: - zenith_idx = np.searchsorted( - self.target_values["zenith_angle"].value, grid_point["zenith_angle"].value - ) - azimuth_idx = np.searchsorted( - self.target_values["azimuth"].value, grid_point["azimuth"].value - ) - nsb_idx = np.searchsorted( - self.target_values["nsb_level"].value, - grid_point["nsb_level"].value, - ) - grid_point["lower_energy_threshold"] = ( - self.interpolated_limits["lower_energy_threshold"][ - zenith_idx, azimuth_idx, nsb_idx - ] - * u.TeV - ) - - if "upper_scatter_radius" in self.interpolated_limits: - grid_point["scatter_radius"] = ( - self.interpolated_limits["upper_scatter_radius"][ - zenith_idx, azimuth_idx, nsb_idx - ] - * u.m - ) - - if "viewcone_radius" in self.interpolated_limits: - grid_point["viewcone_radius"] = ( - self.interpolated_limits["viewcone_radius"][zenith_idx, azimuth_idx, nsb_idx] - * u.deg - ) - - grid_points.append(grid_point) - - return self.convert_coordinates(grid_points) - - def convert_altaz_to_radec(self, alt, az): - """ - Convert Altitude/Azimuth (AltAz) coordinates to RA/Dec. - - Parameters - ---------- - alt : float - Altitude angle in degrees. - az : float - Azimuth angle in degrees. - - Returns - ------- - SkyCoord - SkyCoord object containing the RA/Dec coordinates. - """ - if self.observing_time is None: - raise ValueError("Conversion to RA/dec requires observing_time to be set. ") - - aa = AltAz( - alt=alt.to(u.rad), - az=az.to(u.rad), - location=self.observing_location, - obstime=self.observing_time, - ) - sky_coord = SkyCoord(aa) - return sky_coord.icrs # Return RA/Dec in ICRS frame - - def convert_coordinates(self, grid_points): - """ - Convert the grid points RA/Dec coordinates if necessary. - - Parameters - ---------- - grid_points : list of dict - List of grid points. - - Returns - ------- - list of dict - The grid points with converted RA/Dec coordinates. - """ - if self.coordinate_system == "ra_dec": - for point in grid_points: - if "zenith_angle" in point and "azimuth" in point: - alt = (90.0 * u.deg) - point.pop("zenith_angle") - az = point.pop("azimuth") - radec = self.convert_altaz_to_radec(alt, az) - point["ra"] = radec.ra.deg * u.deg - point["dec"] = radec.dec.deg * u.deg - return grid_points - - def serialize_grid_points(self, grid_points, output_file): - """ - Serialize grid points to an ECSV table file. - - Parameters - ---------- - grid_points : list of dict - List of grid points to serialize. - output_file : str - Path to the output ECSV file. - """ - serialize_grid_points( - grid_points=grid_points, - output_file=output_file, - coordinate_system=self.coordinate_system, - observing_time=self.observing_time, - telescope_ids=self.telescope_ids, - lookup_table=self.lookup_table, - ) - self._logger.info(f"Simulation grid saved to {output_file}") From 343f6f17e5aa2de74486447fba1de2d4a6c420f3 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 19 May 2026 10:57:58 +0200 Subject: [PATCH 20/44] production grid --- .../applications/production_generate_grid.py | 59 +----- .../simulate_prod_htcondor_generator.py | 55 ++++- .../htcondor_script_generator.py | 65 +++--- .../production_configuration/build_grid.py | 195 ++++++++++++++++++ .../production_configuration/grid_engine.py | 96 ++++++--- ...ate_prod_htcondor_generator_grid_radec.yml | 55 +++++ .../test_production_generate_grid.py | 28 +++ .../test_simulate_prod_htcondor_generator.py | 34 ++- .../test_htcondor_script_generator.py | 28 +-- .../test_build_grid.py | 105 ++++++++++ .../test_grid_engine.py | 33 +++ 11 files changed, 620 insertions(+), 133 deletions(-) create mode 100644 tests/integration_tests/config/simulate_prod_htcondor_generator_grid_radec.yml create mode 100644 tests/unit_tests/applications/test_production_generate_grid.py create mode 100644 tests/unit_tests/production_configuration/test_build_grid.py create mode 100644 tests/unit_tests/production_configuration/test_grid_engine.py diff --git a/src/simtools/applications/production_generate_grid.py b/src/simtools/applications/production_generate_grid.py index 4cd057630a..c5220fad5e 100644 --- a/src/simtools/applications/production_generate_grid.py +++ b/src/simtools/applications/production_generate_grid.py @@ -69,15 +69,8 @@ --telescope_ids MSTN-15 """ -from pathlib import Path - -from astropy.coordinates import EarthLocation -from astropy.time import Time - from simtools.application_control import build_application -from simtools.io.ascii_handler import collect_data_from_file -from simtools.model.site_model import SiteModel -from simtools.production_configuration.grid_engine import ProductionGridEngine +from simtools.production_configuration.build_grid import build_production_grid_engine def _add_arguments(parser): @@ -140,26 +133,6 @@ def _add_arguments(parser): ) -def load_axes(file_path: str): - """ - Load axes definitions from a YAML or JSON file. - - Parameters - ---------- - file_path : str - Path to the axes YAML or JSON file. - - Returns - ------- - list[dict] - List of axes definitions with Quantity values. - """ - if not Path(file_path).exists(): - raise FileNotFoundError(f"Axes file {file_path} not found.") - - return collect_data_from_file(file_path) - - def main(): """See CLI description.""" app_context = build_application( @@ -170,35 +143,7 @@ def main(): ) output_filepath = app_context.io_handler.get_output_file(app_context.args["output_file"]) - - axes = load_axes(app_context.args["axes"]) - site_model = SiteModel( - model_version=app_context.args["model_version"], - site=app_context.args["site"], - ) - - ref_lat = site_model.get_parameter_value_with_unit("reference_point_latitude") - ref_long = site_model.get_parameter_value_with_unit("reference_point_longitude") - altitude = site_model.get_parameter_value_with_unit("reference_point_altitude") - - observing_location = EarthLocation(lat=ref_lat, lon=ref_long, height=altitude) - - coordinate_system = app_context.args["coordinate_system"] - observing_time = None - if app_context.args.get("observing_time"): - observing_time = Time(app_context.args["observing_time"], scale="utc") - elif coordinate_system == "ra_dec": - observing_time = Time.now() - - grid_gen = ProductionGridEngine( - axes=axes, - coordinate_system=coordinate_system, - observing_location=observing_location, - observing_time=observing_time, - lookup_table=app_context.args["lookup_table"], - telescope_ids=app_context.args["telescope_ids"], - simtel_file=app_context.args.get("simtel_file"), - ) + grid_gen = build_production_grid_engine(app_context.args) grid_points = grid_gen.generate_grid() grid_gen.serialize_grid_points(grid_points, output_file=output_filepath) diff --git a/src/simtools/applications/simulate_prod_htcondor_generator.py b/src/simtools/applications/simulate_prod_htcondor_generator.py index 8933b79a97..0ba1dcf9da 100644 --- a/src/simtools/applications/simulate_prod_htcondor_generator.py +++ b/src/simtools/applications/simulate_prod_htcondor_generator.py @@ -37,6 +37,10 @@ For the single-image default case, use ``condor_submit simulate_prod.submit.condor``. Simulation data products are written to the directory controlled by ``--simulation_output``. +When ``--axes`` is provided, the generator reuses the shared production-grid engine from +``simtools-production-generate-grid``. This supports both ``zenith_azimuth`` and ``ra_dec`` +grid definitions, while the HTCondor writer keeps serializing rows as submit parameters for +``simtools-simulate-prod``. Command line arguments ---------------------- @@ -49,6 +53,18 @@ Directory for HTCondor log files. Defaults to ``output_path/htcondor_logs``. simulation_output (str, optional) Base directory for simulation output packages passed through as ``pack_for_grid_register``. +axes (str, optional) + Path to a YAML or JSON file defining a shared production grid. +coordinate_system (str, optional, default='zenith_azimuth') + Coordinate system used for axis-defined grids ('zenith_azimuth' or 'ra_dec'). +observing_time (str, optional) + Observation time in UTC (format: 'YYYY-MM-DD HH:MM:SS'). Used only in ``ra_dec`` mode. +lookup_table (str, optional) + Lookup table used for axis-defined grids. +telescope_ids (list of str, optional) + Telescope selection used to filter lookup-table rows for axis-defined grids. +simtel_file (str, optional) + Optional sim_telarray file used to map numeric telescope IDs in lookup tables. priority (int, optional) Job priority (default: 1). nshow_power_index (float, optional) @@ -102,6 +118,43 @@ def _add_arguments(parser): required=False, default=None, ) + parser.add_argument( + "--axes", + type=str, + required=False, + help="Path to a YAML or JSON file defining a shared production grid.", + ) + parser.add_argument( + "--coordinate_system", + type=str, + default="zenith_azimuth", + help="Coordinate system ('zenith_azimuth' or 'ra_dec') for axis-defined grids.", + ) + parser.add_argument( + "--observing_time", + type=str, + required=False, + help="Observation time in UTC (format: 'YYYY-MM-DD HH:MM:SS') for 'ra_dec' grids.", + ) + parser.add_argument( + "--lookup_table", + type=str, + required=False, + help="Lookup table used for axis-defined production grids.", + ) + parser.add_argument( + "--telescope_ids", + type=str, + nargs="*", + default=None, + help="Telescope names used to filter lookup-table rows for axis-defined grids.", + ) + parser.add_argument( + "--simtel_file", + type=str, + required=False, + help="Optional sim_telarray file used to map numeric telescope IDs in lookup tables.", + ) parser.add_argument( "--nshow_power_index", help=( @@ -128,7 +181,7 @@ def main(): """See CLI description.""" app_context = build_application( initialization_kwargs={ - "db_config": False, + "db_config": True, "preserve_by_version_keys": ["array_layout_name"], "simulation_model": ["site", "layout", "telescope", "model_version"], "simulation_configuration": {"software": None, "corsika_configuration": ["all"]}, diff --git a/src/simtools/job_execution/htcondor_script_generator.py b/src/simtools/job_execution/htcondor_script_generator.py index c53d202ce3..5e186c37bf 100644 --- a/src/simtools/job_execution/htcondor_script_generator.py +++ b/src/simtools/job_execution/htcondor_script_generator.py @@ -30,6 +30,8 @@ "energy_max_unit", "core_scatter_max_value", "core_scatter_max_unit", + "view_cone_max_value", + "view_cone_max_unit", "nshow", "model_version", "array_layout_name", @@ -105,6 +107,13 @@ def _format_param_value(value, field_name): convert_to=u.m, ) + if field_name == "view_cone_max_value": + return _format_quantity( + value, + default_unit=u.deg, + convert_to=u.deg, + ) + if field_name in ("azimuth_angle", "zenith_angle"): if isinstance(value, u.Quantity): value = value.to(u.deg).value @@ -150,6 +159,9 @@ def _write_params_file(params_file_path, label_job_specs): core_scatter_max_value, core_scatter_max_unit = _format_param_value( job_spec["core_scatter_max"], "core_scatter_max_value" ) + view_cone_max_value, view_cone_max_unit = _format_param_value( + job_spec["view_cone_max"], "view_cone_max_value" + ) row = [ _format_param_value(job_spec["image_label"], "apptainer_label"), @@ -162,6 +174,8 @@ def _write_params_file(params_file_path, label_job_specs): energy_max_unit, core_scatter_max_value, core_scatter_max_unit, + view_cone_max_value, + view_cone_max_unit, _format_param_value(job_spec["nshow"], "nshow"), _format_param_value(job_spec["model_version"], "model_version"), _format_param_value(array_layout_name, "array_layout_name"), @@ -299,38 +313,27 @@ def _get_submit_script(args_dict): core_scatter = args_dict["core_scatter"] n_core_scatter = core_scatter[0] view_cone = args_dict["view_cone"] - view_cone_string = f'"{view_cone[0].to(u.deg)} {view_cone[1].to(u.deg)}"' + view_cone_min = view_cone[0].to(u.deg).value label = args_dict["label"] if args_dict["label"] else "simulate-prod" run_number_offset_arg = args_dict["run_number_offset"] run_number_offset = 0 if run_number_offset_arg is None else run_number_offset_arg - azimuth_angle_idx = bash_indices["azimuth_angle"] - zenith_angle_idx = bash_indices["zenith_angle"] - energy_min_value_idx = bash_indices["energy_min_value"] - energy_min_unit_idx = bash_indices["energy_min_unit"] - energy_max_value_idx = bash_indices["energy_max_value"] - energy_max_unit_idx = bash_indices["energy_max_unit"] - core_scatter_max_value_idx = bash_indices["core_scatter_max_value"] - core_scatter_max_unit_idx = bash_indices["core_scatter_max_unit"] - nshow_idx = bash_indices["nshow"] - model_version_idx = bash_indices["model_version"] - array_layout_name_idx = bash_indices["array_layout_name"] - corsika_le_interaction_idx = bash_indices["corsika_le_interaction"] - corsika_he_interaction_idx = bash_indices["corsika_he_interaction"] - run_number_idx = bash_indices["run_number"] - pack_for_grid_register_idx = bash_indices["pack_for_grid_register"] - energy_range_string = ( - f'"{energy_min_value_idx} {energy_min_unit_idx} ' - f'{energy_max_value_idx} {energy_max_unit_idx}"' + f'"{bash_indices["energy_min_value"]} {bash_indices["energy_min_unit"]} ' + f'{bash_indices["energy_max_value"]} {bash_indices["energy_max_unit"]}"' ) core_scatter_string = ( - f'"{n_core_scatter} {core_scatter_max_value_idx} {core_scatter_max_unit_idx}"' + f'"{n_core_scatter} {bash_indices["core_scatter_max_value"]} ' + f'{bash_indices["core_scatter_max_unit"]}"' + ) + view_cone_string = ( + f'"{view_cone_min} deg {bash_indices["view_cone_max_value"]} ' + f'{bash_indices["view_cone_max_unit"]}"' ) energy_range_tag = ( - f"erange-{energy_min_value_idx}{energy_min_unit_idx}-" - f"{energy_max_value_idx}{energy_max_unit_idx}" + f"erange-{bash_indices['energy_min_value']}{bash_indices['energy_min_unit']}-" + f"{bash_indices['energy_max_value']}{bash_indices['energy_max_unit']}" ) return f"""#!/usr/bin/env bash @@ -341,12 +344,12 @@ def _get_submit_script(args_dict): set -a; source "$2" apptainer_label="{bash_indices["apptainer_label"]}" primary="{bash_indices["primary"]}" -model_version="{model_version_idx}" -array_layout_name="{array_layout_name_idx}" -corsika_le_interaction="{corsika_le_interaction_idx}" -corsika_he_interaction="{corsika_he_interaction_idx}" -run_number="{run_number_idx}" -pack_for_grid_register="{pack_for_grid_register_idx}" +model_version="{bash_indices["model_version"]}" +array_layout_name="{bash_indices["array_layout_name"]}" +corsika_le_interaction="{bash_indices["corsika_le_interaction"]}" +corsika_he_interaction="{bash_indices["corsika_he_interaction"]}" +run_number="{bash_indices["run_number"]}" +pack_for_grid_register="{bash_indices["pack_for_grid_register"]}" energy_range_tag="{energy_range_tag}" job_label="{label}_${{corsika_he_interaction}}-${{corsika_le_interaction}}_${{energy_range_tag}}" @@ -357,9 +360,9 @@ def _get_submit_script(args_dict): --site {args_dict["site"]} \\ --array_layout_name "$array_layout_name" \\ --primary "$primary" \\ - --azimuth_angle "{azimuth_angle_idx}" \\ - --zenith_angle "{zenith_angle_idx}" \\ - --nshow "{nshow_idx}" \\ + --azimuth_angle "{bash_indices["azimuth_angle"]}" \\ + --zenith_angle "{bash_indices["zenith_angle"]}" \\ + --nshow "{bash_indices["nshow"]}" \\ --energy_range {energy_range_string} \\ --core_scatter {core_scatter_string} \\ --view_cone {view_cone_string} \\ diff --git a/src/simtools/production_configuration/build_grid.py b/src/simtools/production_configuration/build_grid.py index e0ed6ca05f..27cd4b4c52 100644 --- a/src/simtools/production_configuration/build_grid.py +++ b/src/simtools/production_configuration/build_grid.py @@ -1,12 +1,18 @@ """Build simulation execution grid based on the production configuration and lookup tables.""" import itertools +from pathlib import Path import numpy as np from astropy import units as u +from astropy.coordinates import EarthLocation +from astropy.time import Time from simtools.configuration import defaults +from simtools.io.ascii_handler import collect_data_from_file +from simtools.model.site_model import SiteModel from simtools.production_configuration.corsika_limits_lookup import CorsikaLimitsLookup +from simtools.production_configuration.grid_engine import ProductionGridEngine from simtools.utils.general import ensure_list _GRID_AXES = [ @@ -24,6 +30,64 @@ } +def load_axes(file_path): + """Load axes definitions from a YAML or JSON file.""" + if not Path(file_path).exists(): + raise FileNotFoundError(f"Axes file {file_path} not found.") + + return collect_data_from_file(file_path) + + +def resolve_observing_time(observing_time, coordinate_system): + """Resolve observing time from CLI arguments.""" + if observing_time: + return Time(observing_time, scale="utc") + if coordinate_system == "ra_dec": + return Time.now() + return None + + +def resolve_single_model_version(model_version): + """Resolve one model version for helpers that require a scalar version.""" + if isinstance(model_version, list): + return model_version[0] + return model_version + + +def build_observing_location(site, model_version): + """Build observing location from the site model.""" + site_model = SiteModel(model_version=resolve_single_model_version(model_version), site=site) + return EarthLocation( + lat=site_model.get_parameter_value_with_unit("reference_point_latitude"), + lon=site_model.get_parameter_value_with_unit("reference_point_longitude"), + height=site_model.get_parameter_value_with_unit("reference_point_altitude"), + ) + + +def build_production_grid_engine(args_dict): + """Build a production-grid engine from application arguments.""" + coordinate_system = args_dict.get("coordinate_system", "zenith_azimuth") + observing_location = None + if coordinate_system == "ra_dec": + observing_location = build_observing_location( + site=args_dict["site"], + model_version=args_dict["model_version"], + ) + + return ProductionGridEngine( + axes=load_axes(args_dict["axes"]), + coordinate_system=coordinate_system, + observing_location=observing_location, + observing_time=resolve_observing_time( + args_dict.get("observing_time"), + coordinate_system, + ), + lookup_table=args_dict.get("lookup_table"), + telescope_ids=args_dict.get("telescope_ids"), + simtel_file=args_dict.get("simtel_file"), + ) + + def normalize_grid_axes(args_dict): """Return normalized grid axes for cartesian product expansion.""" return { @@ -104,6 +168,23 @@ def get_core_scatter_max_for_zenith_angle( return min(configured_scatter_max, lookup_scatter_max.to(configured_scatter_max.unit)) +def get_viewcone_max_for_zenith_angle( + zenith_angle, view_cone, corsika_limits, azimuth_angle=None, nsb_level=1.0 +): + """Return zenith-dependent max viewcone value.""" + if corsika_limits is None: + return view_cone[1] + + if not isinstance(corsika_limits, CorsikaLimitsLookup): + corsika_limits = CorsikaLimitsLookup(corsika_limits) + + azimuth_angle = 0.0 * u.deg if azimuth_angle is None else azimuth_angle + interpolated_limits = corsika_limits.interpolate_point(zenith_angle, azimuth_angle, nsb_level) + lookup_viewcone_max = interpolated_limits["viewcone_radius"] * u.deg + configured_viewcone_max = view_cone[1] + return min(configured_viewcone_max, lookup_viewcone_max.to(configured_viewcone_max.unit)) + + def calculate_log_energy_midpoint(energy_range_pair): """Return the geometric-mean energy for an energy range pair.""" energy_min, energy_max = energy_range_pair @@ -154,6 +235,95 @@ def calculate_scaled_nshow( return scaled_nshow +def _clip_energy_range_from_threshold(energy_range_pair, lower_energy_threshold): + """Clip the lower energy bound of a configured energy range.""" + if lower_energy_threshold is None: + return energy_range_pair + + energy_min, energy_max = energy_range_pair + lower_energy_threshold = lower_energy_threshold.to(energy_min.unit) + if lower_energy_threshold > energy_max.to(lower_energy_threshold.unit): + return None + if lower_energy_threshold <= energy_min: + return energy_range_pair + return lower_energy_threshold, energy_max + + +def _clip_max_quantity(configured_max, lookup_max): + """Clip a configured maximum value against an interpolated lookup value.""" + if lookup_max is None: + return configured_max + return min(configured_max, lookup_max.to(configured_max.unit)) + + +def _build_simulation_jobs_from_production_grid( + args_dict, + energy_ranges, + grid_axes, + number_of_runs, + run_number, + nshow, + nshow_power_index, + reference_energy, +): + """Build simulation jobs from a shared production-grid definition.""" + grid_points = build_production_grid_engine(args_dict).generate_simulation_grid() + rows = [] + + for ( + primary, + model_version, + corsika_le, + corsika_he, + ) in itertools.product( + grid_axes["primary"], + grid_axes["model_version"], + grid_axes["corsika_le_interaction"], + grid_axes["corsika_he_interaction"], + ): + for point in grid_points: + selected_core_scatter_max = _clip_max_quantity( + args_dict["core_scatter"][1], + point.get("scatter_radius"), + ) + selected_viewcone_max = _clip_max_quantity( + args_dict["view_cone"][1], + point.get("viewcone_radius"), + ) + + for energy_range_pair in energy_ranges: + selected_energy_range_pair = _clip_energy_range_from_threshold( + energy_range_pair, + point.get("lower_energy_threshold"), + ) + if selected_energy_range_pair is None: + continue + + selected_nshow = calculate_scaled_nshow( + selected_energy_range_pair, nshow, nshow_power_index, reference_energy + ) + + for row_index in range(number_of_runs): + rows.append( + { + "primary": primary, + "azimuth_angle": point["azimuth"], + "zenith_angle": point["zenith"], + "model_version": model_version, + "array_layout_name": args_dict.get("array_layout_name"), + "corsika_le_interaction": corsika_le, + "corsika_he_interaction": corsika_he, + "energy_min": selected_energy_range_pair[0], + "energy_max": selected_energy_range_pair[1], + "core_scatter_max": selected_core_scatter_max, + "view_cone_max": selected_viewcone_max, + "nshow": selected_nshow, + "run_number": run_number + row_index, + } + ) + return rows + + def build_simulation_jobs(args_dict): """ Build simulation job parameters based on the production configuration and lookup tables. @@ -176,6 +346,23 @@ def build_simulation_jobs(args_dict): """ grid_axes = normalize_grid_axes(args_dict) energy_ranges = normalize_energy_ranges(args_dict["energy_range"]) + if args_dict.get("axes"): + nshow_power_index = args_dict.get("nshow_power_index") + reference_energy = args_dict.get("nshow_reference_energy") + if nshow_power_index is not None and reference_energy is not None: + reference_energy = u.Quantity(reference_energy) + + return _build_simulation_jobs_from_production_grid( + args_dict=args_dict, + energy_ranges=energy_ranges, + grid_axes=grid_axes, + number_of_runs=int(args_dict.get("number_of_runs", 1)), + run_number=int(args_dict.get("run_number") or 1), + nshow=args_dict["nshow"], + nshow_power_index=nshow_power_index, + reference_energy=reference_energy, + ) + corsika_limits = args_dict.get("corsika_limits") if corsika_limits is not None: corsika_limits = CorsikaLimitsLookup( @@ -185,6 +372,7 @@ def build_simulation_jobs(args_dict): ) core_scatter = args_dict["core_scatter"] + view_cone = args_dict["view_cone"] nshow = args_dict["nshow"] nshow_power_index = args_dict.get("nshow_power_index") reference_energy = args_dict.get("nshow_reference_energy") @@ -232,6 +420,12 @@ def build_simulation_jobs(args_dict): corsika_limits, azimuth_angle=azimuth, ) + selected_viewcone_max = get_viewcone_max_for_zenith_angle( + zenith, + view_cone, + corsika_limits, + azimuth_angle=azimuth, + ) selected_nshow = calculate_scaled_nshow( selected_energy_range_pair, nshow, nshow_power_index, reference_energy ) @@ -249,6 +443,7 @@ def build_simulation_jobs(args_dict): "energy_min": selected_energy_range_pair[0], "energy_max": selected_energy_range_pair[1], "core_scatter_max": selected_core_scatter_max, + "view_cone_max": selected_viewcone_max, "nshow": selected_nshow, "run_number": run_number + row_index, } diff --git a/src/simtools/production_configuration/grid_engine.py b/src/simtools/production_configuration/grid_engine.py index d77c3be59c..ab49ac100b 100644 --- a/src/simtools/production_configuration/grid_engine.py +++ b/src/simtools/production_configuration/grid_engine.py @@ -104,10 +104,10 @@ def _require_observing_time(self): def _get_max_zenith_for_radec_mode(self): """Read maximum zenith from axes for RA/Dec direction sampling.""" - zenith_axis = self.axes.get("zenith_angle") + zenith_axis = self.axes.get("zenith") if not zenith_axis or "range" not in zenith_axis or len(zenith_axis["range"]) != 2: raise ValueError( - "RA/Dec direction sampling requires 'zenith_angle' axis with a valid " + "RA/Dec direction sampling requires 'zenith' axis with a valid " "two-element 'range' in the axes definition." ) return float(zenith_axis["range"][1]) @@ -188,7 +188,7 @@ def _generate_radec_grid_direction_points(self): for idx in np.nonzero(mask)[0]: direction_points.append( { - "zenith_angle": zenith_values[idx] * u.deg, + "zenith": zenith_values[idx] * u.deg, "azimuth": altaz.az.deg[idx] * u.deg, } ) @@ -230,11 +230,11 @@ def _add_lookup_limits_to_point(self, point, zenith, azimuth): point["scatter_radius"] = limits["upper_scatter_radius"] * u.m point["viewcone_radius"] = limits["viewcone_radius"] * u.deg - def _generate_grid_from_radec_axes(self): + def _generate_grid_from_radec_axes(self, include_horizontal_coordinates=False): """Generate grid points from explicit RA/Dec axes definitions.""" observing_time = self._require_observing_time() - axis_keys = [key for key in self.target_values if key not in ("zenith_angle", "azimuth")] + axis_keys = [key for key in self.target_values if key not in ("zenith", "azimuth")] value_arrays = [self.target_values[key].value for key in axis_keys] units = [self.target_values[key].unit for key in axis_keys] grid = np.meshgrid(*value_arrays, indexing="ij") @@ -254,21 +254,32 @@ def _generate_grid_from_radec_axes(self): altaz = skycoord.transform_to( AltAz(location=self.observing_location, obstime=observing_time) ) - zenith = (90.0 * u.deg - altaz.alt).to(u.deg).value + zenith = (90.0 * u.deg - altaz.alt).to(u.deg) + azimuth = altaz.az.to(u.deg) - self._add_lookup_limits_to_point(grid_point, zenith=zenith, azimuth=altaz.az.deg) + if include_horizontal_coordinates: + grid_point["zenith"] = zenith + grid_point["azimuth"] = azimuth + + self._add_lookup_limits_to_point( + grid_point, + zenith=zenith.value, + azimuth=azimuth.value, + ) grid_points.append(grid_point) return grid_points - def _generate_grid_radec_mode(self): + def _generate_grid_radec_mode(self, include_horizontal_coordinates=False): """Generate grid points for RA/Dec mode.""" if "ra" in self.axes and "dec" in self.axes: - return self._generate_grid_from_radec_axes() + return self._generate_grid_from_radec_axes( + include_horizontal_coordinates=include_horizontal_coordinates + ) direction_points = self._generate_radec_grid_direction_points() extra_keys, extra_units, extra_combinations = self._generate_extra_axis_combinations( - excluded_keys=("zenith_angle", "azimuth") + excluded_keys=("zenith", "azimuth") ) grid_points = [] @@ -280,12 +291,15 @@ def _generate_grid_radec_mode(self): self._add_lookup_limits_to_point( point, - zenith=point["zenith_angle"].value, + zenith=point["zenith"].value, azimuth=point["azimuth"].value, ) grid_points.append(point) - return grid_points + return self.convert_coordinates( + grid_points, + keep_horizontal_coordinates=include_horizontal_coordinates, + ) def create_circular_binning(self, azimuth_range, num_bins): """ @@ -322,18 +336,8 @@ def create_circular_binning(self, azimuth_range, num_bins): % 360 ) - def generate_grid(self): - """ - Generate the grid based on the required axes and include interpolated limits. - - Returns - ------- - list of dict - A list of generated grid points. - """ - if self.coordinate_system == "ra_dec": - return self._generate_grid_radec_mode() - + def _generate_zenith_azimuth_grid(self): + """Generate grid points for zenith/azimuth mode.""" value_arrays = [value.value for value in self.target_values.values()] units = [value.unit for value in self.target_values.values()] grid = np.meshgrid(*value_arrays, indexing="ij") @@ -348,7 +352,7 @@ def generate_grid(self): if "lower_energy_threshold" in self.interpolated_limits: zenith_idx = np.searchsorted( - self.target_values["zenith_angle"].value, grid_point["zenith_angle"].value + self.target_values["zenith"].value, grid_point["zenith"].value ) azimuth_idx = np.searchsorted( self.target_values["azimuth"].value, grid_point["azimuth"].value @@ -380,7 +384,36 @@ def generate_grid(self): grid_points.append(grid_point) - return self.convert_coordinates(grid_points) + return grid_points + + def generate_simulation_grid(self): + """Generate grid points while retaining horizontal coordinates for execution backends.""" + if self.coordinate_system == "ra_dec": + return self._generate_grid_radec_mode(include_horizontal_coordinates=True) + + return self._generate_zenith_azimuth_grid() + + @staticmethod + def _strip_horizontal_coordinates(grid_points): + """Remove horizontal coordinates from serialized RA/Dec grid points.""" + for point in grid_points: + point.pop("zenith", None) + point.pop("azimuth", None) + return grid_points + + def generate_grid(self): + """ + Generate the grid based on the required axes and include interpolated limits. + + Returns + ------- + list of dict + A list of generated grid points. + """ + grid_points = self.generate_simulation_grid() + if self.coordinate_system == "ra_dec": + return self._strip_horizontal_coordinates(grid_points) + return grid_points def convert_altaz_to_radec(self, alt, az): """ @@ -410,7 +443,7 @@ def convert_altaz_to_radec(self, alt, az): sky_coord = SkyCoord(aa) return sky_coord.icrs # Return RA/Dec in ICRS frame - def convert_coordinates(self, grid_points): + def convert_coordinates(self, grid_points, keep_horizontal_coordinates=False): """ Convert the grid points RA/Dec coordinates if necessary. @@ -426,10 +459,13 @@ def convert_coordinates(self, grid_points): """ if self.coordinate_system == "ra_dec": for point in grid_points: - if "zenith_angle" in point and "azimuth" in point: - alt = (90.0 * u.deg) - point.pop("zenith_angle") - az = point.pop("azimuth") + if "zenith" in point and "azimuth" in point: + alt = (90.0 * u.deg) - point["zenith"] + az = point["azimuth"] radec = self.convert_altaz_to_radec(alt, az) + if not keep_horizontal_coordinates: + point.pop("zenith") + point.pop("azimuth") point["ra"] = radec.ra.deg * u.deg point["dec"] = radec.dec.deg * u.deg return grid_points diff --git a/tests/integration_tests/config/simulate_prod_htcondor_generator_grid_radec.yml b/tests/integration_tests/config/simulate_prod_htcondor_generator_grid_radec.yml new file mode 100644 index 0000000000..31b4a5d122 --- /dev/null +++ b/tests/integration_tests/config/simulate_prod_htcondor_generator_grid_radec.yml @@ -0,0 +1,55 @@ +--- +applications: +- application: simtools-simulate-prod-htcondor-generator + configuration: + apptainer_image: + 0.30.0: tests/resources/dummy_apptainer_image.sif + 0.29.0: tests/resources/dummy_apptainer_image.sif + array_layout_name: + by_version: + "<7.0.0": alpha + ">=7.0.0": CTAO-North-Alpha + axes: tests/resources/production_grid_generation_axes_definition_radec.yml + coordinate_system: ra_dec + core_scatter: 10 500 m + corsika_he_interaction: + - epos + - qgs3 + corsika_le_interaction: urqmd + energy_range: + - 30 GeV 30 GeV + - 300 GeV 300 GeV + view_cone: 0 deg 10 deg + eslope: -2.0 + label: test-production-grid-radec + log_level: DEBUG + model_version: + - 6.3.0 + - 7.0.0 + nshow: 5 + number_of_runs: 2 + observing_time: '2017-09-16 00:00:00' + output_path: htcondor_submit + primary: + - gamma + - proton + priority: 5 + run_number: 10 + simulation_output: ./simulation_output_path + simulation_software: corsika_sim_telarray + site: North + integration_tests: + - test_output_files: + - file: simulate_prod.submit.0.30.0.condor + path_descriptor: output_path + - file: simulate_prod.submit.0.30.0.params.txt + path_descriptor: output_path + - file: simulate_prod.submit.0.29.0.condor + path_descriptor: output_path + - file: simulate_prod.submit.0.29.0.params.txt + path_descriptor: output_path + - file: simulate_prod.submit.sh + path_descriptor: output_path + test_name: htcondor-grid-radec +schema_name: application_workflow.metaschema +schema_version: 0.4.0 diff --git a/tests/unit_tests/applications/test_production_generate_grid.py b/tests/unit_tests/applications/test_production_generate_grid.py new file mode 100644 index 0000000000..5093f79fa3 --- /dev/null +++ b/tests/unit_tests/applications/test_production_generate_grid.py @@ -0,0 +1,28 @@ +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock, patch + +import simtools.applications.production_generate_grid as app + + +@patch("simtools.applications.production_generate_grid.build_production_grid_engine") +@patch("simtools.applications.production_generate_grid.build_application") +def test_main_uses_shared_grid_builder(mock_build_application, mock_build_production_grid_engine): + io_handler = Mock() + io_handler.get_output_file.return_value = Path("grid_output.ecsv") + args = { + "axes": "grid.yml", + "output_file": "grid_output.ecsv", + } + mock_build_application.return_value = SimpleNamespace(args=args, io_handler=io_handler) + mock_grid_engine = Mock() + mock_grid_engine.generate_grid.return_value = [{"ra": 1}] + mock_build_production_grid_engine.return_value = mock_grid_engine + + app.main() + + mock_build_production_grid_engine.assert_called_once_with(args) + mock_grid_engine.serialize_grid_points.assert_called_once_with( + [{"ra": 1}], + output_file=Path("grid_output.ecsv"), + ) diff --git a/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py b/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py index 38288feef8..5107161b2c 100644 --- a/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py +++ b/tests/unit_tests/applications/test_simulate_prod_htcondor_generator.py @@ -21,7 +21,7 @@ def test_main_uses_standard_build_application( app.main() assert mock_build_application.call_args.kwargs["initialization_kwargs"] == { - "db_config": False, + "db_config": True, "preserve_by_version_keys": ["array_layout_name"], "simulation_model": ["site", "layout", "telescope", "model_version"], "simulation_configuration": {"software": None, "corsika_configuration": ["all"]}, @@ -46,3 +46,35 @@ def test_add_arguments_registers_nshow_scaling_arguments(): assert args.nshow_power_index == pytest.approx(-0.5) assert args.nshow_reference_energy == "100 GeV" + + +def test_add_arguments_registers_axis_defined_grid_arguments(): + parser = argparse.ArgumentParser() + + app._add_arguments(parser) + args = parser.parse_args( + [ + "--number_of_runs", + "1", + "--axes", + "grid.yml", + "--coordinate_system", + "ra_dec", + "--observing_time", + "2017-09-16 00:00:00", + "--lookup_table", + "limits.ecsv", + "--telescope_ids", + "MSTN-15", + "LSTN-01", + "--simtel_file", + "events.simtel.zst", + ] + ) + + assert args.axes == "grid.yml" + assert args.coordinate_system == "ra_dec" + assert args.observing_time == "2017-09-16 00:00:00" + assert args.lookup_table == "limits.ecsv" + assert args.telescope_ids == ["MSTN-15", "LSTN-01"] + assert args.simtel_file == "events.simtel.zst" diff --git a/tests/unit_tests/job_execution/test_htcondor_script_generator.py b/tests/unit_tests/job_execution/test_htcondor_script_generator.py index 71d2bf4f44..97fbb0ceea 100644 --- a/tests/unit_tests/job_execution/test_htcondor_script_generator.py +++ b/tests/unit_tests/job_execution/test_htcondor_script_generator.py @@ -120,7 +120,6 @@ def test_generate_submission_script_raises_for_missing_apptainer_image( def test_get_submit_script(args_dict): n_core_scatter = args_dict["core_scatter"][0] view_cone_low = args_dict["view_cone"][0].to(u.deg).value - view_cone_high = args_dict["view_cone"][1].to(u.deg).value container_output_path = str(Path("/", "tmp", "simtools-output")) expected_script = f"""#!/usr/bin/env bash @@ -131,12 +130,12 @@ def test_get_submit_script(args_dict): set -a; source "$2" apptainer_label="${{3}}" primary="${{4}}" -model_version="${{14}}" -array_layout_name="${{15}}" -corsika_le_interaction="${{16}}" -corsika_he_interaction="${{17}}" -run_number="${{18}}" -pack_for_grid_register="${{19}}" +model_version="${{16}}" +array_layout_name="${{17}}" +corsika_le_interaction="${{18}}" +corsika_he_interaction="${{19}}" +run_number="${{20}}" +pack_for_grid_register="${{21}}" energy_range_tag="erange-${{7}}${{8}}-${{9}}${{10}}" job_label="{args_dict["label"]}_${{corsika_he_interaction}}-${{corsika_le_interaction}}_${{energy_range_tag}}" @@ -149,10 +148,10 @@ def test_get_submit_script(args_dict): --primary "$primary" \\ --azimuth_angle "${{5}}" \\ --zenith_angle "${{6}}" \\ - --nshow "${{13}}" \\ + --nshow "${{15}}" \\ --energy_range "${{7}} ${{8}} ${{9}} ${{10}}" \\ --core_scatter "{n_core_scatter} ${{11}} ${{12}}" \\ - --view_cone "{view_cone_low} deg {view_cone_high} deg" \\ + --view_cone "{view_cone_low} deg ${{13}} ${{14}}" \\ --corsika_le_interaction "$corsika_le_interaction" \\ --corsika_he_interaction "$corsika_he_interaction" \\ --run_number "$run_number" \\ @@ -183,6 +182,7 @@ def test_get_submit_file_uses_queue_from_params(tmp_test_directory): assert "queue apptainer_label,primary" in content assert "energy_min_value,energy_min_unit,energy_max_value,energy_max_unit" in content assert "core_scatter_max_value,core_scatter_max_unit" in content + assert "view_cone_max_value,view_cone_max_unit" in content assert "nshow,model_version,array_layout_name" in content assert "model_version,array_layout_name,corsika_le_interaction" in content assert "from simulate_prod.submit.params.txt" in content @@ -303,7 +303,7 @@ def test_write_params_file_resolves_array_layout_name_by_model_version( params_lines = params_file_path.read_text(encoding="utf-8").splitlines() assert "6.3.0 alpha urqmd epos 1 simtools-output/7.0.0" in params_lines[0] - assert "7.0.0 CTAO-North-Alpha urqmd epos 2 simtools-output/7.0.0" in params_lines[1] + assert "7.0.0 CTAO-North-Alpha urqmd epos 1 simtools-output/7.0.0" in params_lines[1] def test_write_params_file_resolves_stringified_by_version_layout(args_dict, tmp_test_directory): @@ -326,7 +326,7 @@ def test_write_params_file_resolves_stringified_by_version_layout(args_dict, tmp params_lines = params_file_path.read_text(encoding="utf-8").splitlines() assert "6.3.0 alpha urqmd epos 1 simtools-output/7.0.0" in params_lines[0] - assert "7.0.0 CTAO-North-Alpha urqmd epos 2 simtools-output/7.0.0" in params_lines[1] + assert "7.0.0 CTAO-North-Alpha urqmd epos 1 simtools-output/7.0.0" in params_lines[1] def test_write_params_file_keeps_energy_units(tmp_test_directory): @@ -340,6 +340,7 @@ def test_write_params_file_keeps_energy_units(tmp_test_directory): "energy_min": 30 * u.GeV, "energy_max": 10 * u.TeV, "core_scatter_max": 200 * u.m, + "view_cone_max": 5 * u.deg, "nshow": 1000, "model_version": "7.0.0", "array_layout_name": { @@ -358,7 +359,7 @@ def test_write_params_file_keeps_energy_units(tmp_test_directory): _write_params_file(params_file_path, label_job_specs) assert params_file_path.read_text(encoding="utf-8") == ( - "7.0.0 gamma 0.0 20.0 30.0 GeV 10.0 TeV 200.0 m " + "7.0.0 gamma 0.0 20.0 30.0 GeV 10.0 TeV 200.0 m 5.0 deg " "1000 7.0.0 CTAO-North-Alpha urqmd epos 10 simtools-output/7.0.0\n" ) @@ -374,6 +375,7 @@ def test_write_params_file_replaces_whitespace_in_apptainer_label(tmp_test_direc "energy_min": 30 * u.GeV, "energy_max": 10 * u.TeV, "core_scatter_max": 200 * u.m, + "view_cone_max": 5 * u.deg, "nshow": 1000, "model_version": "7.0.0", "array_layout_name": "CTAO-North-Alpha", @@ -387,6 +389,6 @@ def test_write_params_file_replaces_whitespace_in_apptainer_label(tmp_test_direc _write_params_file(params_file_path, label_job_specs) assert params_file_path.read_text(encoding="utf-8") == ( - "grid_label_7.0.0 gamma 0.0 20.0 30.0 GeV 10.0 TeV 200.0 m " + "grid_label_7.0.0 gamma 0.0 20.0 30.0 GeV 10.0 TeV 200.0 m 5.0 deg " "1000 7.0.0 CTAO-North-Alpha urqmd epos 10 simtools-output/grid_label_7.0.0\n" ) diff --git a/tests/unit_tests/production_configuration/test_build_grid.py b/tests/unit_tests/production_configuration/test_build_grid.py new file mode 100644 index 0000000000..0d382a68bb --- /dev/null +++ b/tests/unit_tests/production_configuration/test_build_grid.py @@ -0,0 +1,105 @@ +from unittest.mock import Mock, patch + +import astropy.units as u +import pytest + +from simtools.production_configuration.build_grid import ( + build_simulation_jobs, + get_viewcone_max_for_zenith_angle, + resolve_single_model_version, +) + + +def test_get_viewcone_max_for_zenith_angle_without_lookup_table(): + max_view_cone = get_viewcone_max_for_zenith_angle(20 * u.deg, [0 * u.deg, 10 * u.deg], None) + + assert max_view_cone == 10 * u.deg + + +def test_resolve_single_model_version_uses_first_list_entry(): + assert resolve_single_model_version(["7.0.0", "7.1.0"]) == "7.0.0" + assert resolve_single_model_version("7.0.0") == "7.0.0" + + +@patch("simtools.production_configuration.build_grid.build_production_grid_engine") +def test_build_simulation_jobs_uses_shared_axis_defined_grid(mock_build_production_grid_engine): + mock_grid_engine = Mock() + mock_grid_engine.generate_simulation_grid.return_value = [ + { + "ra": 12 * u.deg, + "dec": -20 * u.deg, + "azimuth": 180 * u.deg, + "zenith_angle": 30 * u.deg, + "lower_energy_threshold": 50 * u.GeV, + "scatter_radius": 250 * u.m, + "viewcone_radius": 3 * u.deg, + } + ] + mock_build_production_grid_engine.return_value = mock_grid_engine + args_dict = { + "axes": "grid.yml", + "coordinate_system": "ra_dec", + "site": "North", + "model_version": ["7.0.0"], + "primary": ["gamma"], + "azimuth_angle": 0 * u.deg, + "zenith_angle": 20 * u.deg, + "energy_range": [[30 * u.GeV, 100 * u.GeV]], + "core_scatter": [10, 500 * u.m], + "view_cone": [0 * u.deg, 5 * u.deg], + "nshow": 5, + "number_of_runs": 2, + "run_number": 11, + "array_layout_name": "layout", + "corsika_le_interaction": "urqmd", + "corsika_he_interaction": "epos", + "lookup_table": "limits.ecsv", + "telescope_ids": ["MSTN-15"], + "simtel_file": None, + } + + rows = build_simulation_jobs(args_dict) + + assert len(rows) == 2 + assert rows[0]["azimuth_angle"] == 180 * u.deg + assert rows[0]["zenith_angle"] == 30 * u.deg + assert rows[0]["energy_min"] == 50 * u.GeV + assert rows[0]["energy_max"] == 100 * u.GeV + assert rows[0]["core_scatter_max"] == 250 * u.m + assert rows[0]["view_cone_max"] == 3 * u.deg + assert rows[0]["run_number"] == 11 + assert rows[1]["run_number"] == 12 + + +@patch("simtools.production_configuration.build_grid.get_core_scatter_max_for_zenith_angle") +@patch("simtools.production_configuration.build_grid.get_viewcone_max_for_zenith_angle") +@patch("simtools.production_configuration.build_grid.get_energy_range_for_zenith_angle") +def test_build_simulation_jobs_adds_viewcone_limit_from_lookup( + mock_get_energy_range_for_zenith_angle, + mock_get_viewcone_max_for_zenith_angle, + mock_get_core_scatter_max_for_zenith_angle, +): + mock_get_energy_range_for_zenith_angle.return_value = (30 * u.GeV, 100 * u.GeV) + mock_get_core_scatter_max_for_zenith_angle.return_value = 150 * u.m + mock_get_viewcone_max_for_zenith_angle.return_value = 2.5 * u.deg + args_dict = { + "primary": ["gamma"], + "azimuth_angle": [180 * u.deg], + "zenith_angle": [30 * u.deg], + "model_version": ["7.0.0"], + "corsika_le_interaction": "urqmd", + "corsika_he_interaction": "epos", + "energy_range": [[30 * u.GeV, 100 * u.GeV]], + "core_scatter": [10, 500 * u.m], + "view_cone": [0 * u.deg, 5 * u.deg], + "nshow": 5, + "number_of_runs": 1, + "run_number": 1, + "corsika_limits": None, + "telescope_ids": None, + "simtel_file": None, + } + + rows = build_simulation_jobs(args_dict) + + assert rows[0]["view_cone_max"].to_value(u.deg) == pytest.approx(2.5) diff --git a/tests/unit_tests/production_configuration/test_grid_engine.py b/tests/unit_tests/production_configuration/test_grid_engine.py new file mode 100644 index 0000000000..4cd2189d8f --- /dev/null +++ b/tests/unit_tests/production_configuration/test_grid_engine.py @@ -0,0 +1,33 @@ +from astropy import units as u +from astropy.coordinates import EarthLocation +from astropy.time import Time + +from simtools.production_configuration.grid_engine import ProductionGridEngine + + +def test_generate_simulation_grid_keeps_horizontal_coordinates_for_radec_axes(): + axes = { + "axes": { + "ra": {"range": [0, 0], "binning": 1, "scaling": "linear", "units": "deg"}, + "dec": {"range": [0, 0], "binning": 1, "scaling": "linear", "units": "deg"}, + } + } + engine = ProductionGridEngine( + axes=axes, + coordinate_system="ra_dec", + observing_location=EarthLocation(lat=28.76 * u.deg, lon=-17.89 * u.deg, height=2200 * u.m), + observing_time=Time("2017-09-16 00:00:00", scale="utc"), + lookup_table=None, + ) + + simulation_grid = engine.generate_simulation_grid() + serialized_grid = engine.generate_grid() + + assert "zenith_angle" in simulation_grid[0] + assert "azimuth" in simulation_grid[0] + assert "ra" in simulation_grid[0] + assert "dec" in simulation_grid[0] + assert "zenith_angle" not in serialized_grid[0] + assert "azimuth" not in serialized_grid[0] + assert "ra" in serialized_grid[0] + assert "dec" in serialized_grid[0] From cbf4bbe87004b108012c43317d87c477bbd790c9 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 19 May 2026 12:40:00 +0200 Subject: [PATCH 21/44] generalize grid specifications --- .../api-reference/production_configuration.md | 10 - .../applications/production_generate_grid.py | 87 +++++--- .../simulate_prod_htcondor_generator.py | 107 +--------- .../htcondor_script_generator.py | 69 +++---- .../production_configuration/build_grid.py | 38 +++- .../production_configuration/grid_engine.py | 44 ++--- .../grid_serialization.py | 134 ------------- .../production_configuration/job_grid_io.py | 145 ++++++++++++++ .../config/production_generate_grid.yml | 14 +- .../production_generate_grid_explicit.yml | 43 ++++ .../config/production_generate_grid_radec.yml | 15 +- ..._htcondor_generator_gamma_20_deg_north.yml | 20 +- ...e_prod_htcondor_generator_grid_example.yml | 32 +-- ...ate_prod_htcondor_generator_grid_radec.yml | 29 +-- .../production_job_grid_example.ecsv | 33 ++++ .../resources/production_job_grid_radec.ecsv | 34 ++++ .../resources/production_job_grid_single.ecsv | 32 +++ .../test_production_generate_grid.py | 30 +-- .../test_simulate_prod_htcondor_generator.py | 54 +---- .../test_htcondor_script_generator.py | 187 +++++++++--------- .../test_build_grid.py | 24 +++ .../test_job_grid_io.py | 65 ++++++ 22 files changed, 666 insertions(+), 580 deletions(-) delete mode 100644 src/simtools/production_configuration/grid_serialization.py create mode 100644 src/simtools/production_configuration/job_grid_io.py create mode 100644 tests/integration_tests/config/production_generate_grid_explicit.yml create mode 100644 tests/resources/production_job_grid_example.ecsv create mode 100644 tests/resources/production_job_grid_radec.ecsv create mode 100644 tests/resources/production_job_grid_single.ecsv create mode 100644 tests/unit_tests/production_configuration/test_job_grid_io.py diff --git a/docs/source/api-reference/production_configuration.md b/docs/source/api-reference/production_configuration.md index b91c617965..1b5eefe15c 100644 --- a/docs/source/api-reference/production_configuration.md +++ b/docs/source/api-reference/production_configuration.md @@ -113,16 +113,6 @@ the calculation of the number of events to be simulated given a pre-determined m ``` -(production-grid-serialization)= - -## grid_serialization - -```{eval-rst} -.. automodule:: production_configuration.grid_serialization - :members: -``` - - (interpolation-handler)= ## interpolation_handler diff --git a/src/simtools/applications/production_generate_grid.py b/src/simtools/applications/production_generate_grid.py index c5220fad5e..e53cdb6590 100644 --- a/src/simtools/applications/production_generate_grid.py +++ b/src/simtools/applications/production_generate_grid.py @@ -1,26 +1,18 @@ #!/usr/bin/python3 r""" -Generate a grid of simulation points using flexible axes definitions. - -This application generates a grid of simulation points based on the provided axes -definitions. The axes definitions (range, binning) are specified in a file. -The viewcone, radius and energy thresholds are provided as a lookup table and -are interpolated based on the generated grid points. The generated grid points are -filtered based on the specified telescope IDs and the limits from the lookup table. -The generated grid points are saved to a file. -It can also convert the generated points to RA/Dec coordinates if the selected -coordinate system is 'ra_dec'. - -For ``coordinate_system='ra_dec'``, the underlying grid generation supports -declination-line sampling with hour-angle spacing and applies zenith-angle -filtering based on the configured zenith range in that mode. -When explicit ``ra`` / ``dec`` axes are provided, all YAML-defined grid points are -preserved in the serialized output. +Generate executable simulation job grids. + +This application expands executable simulation job rows and writes them to disk +as ECSV files. It supports both: + +- explicit cartesian job-grid configuration (primary, zenith, energy range, etc.), and +- axes-based production-grid configuration with optional ``ra_dec`` coordinate handling + and lookup-table interpolation. Command line arguments ---------------------- -axes (str, required) +axes (str, optional) Path to a YAML or JSON file defining the axes of the grid. coordinate_system (str, optional, default='zenith_azimuth') The coordinate system for the grid generation ('zenith_azimuth' or 'ra_dec'). @@ -30,7 +22,7 @@ Time of the observation in UTC (format: 'YYYY-MM-DD HH:MM:SS'). Used only in ``ra_dec`` mode (for coordinate transforms and sidereal-time sampling). Ignored in ``zenith_azimuth`` mode. -lookup_table (str, required) +lookup_table (str, optional) Path to the lookup table for simulation limits. The table should contain varying azimuth and/or zenith angles. telescope_ids (list of str, optional) @@ -39,8 +31,8 @@ simtel_file (str, optional) Path to a sim_telarray file used only when lookup-table telescope selections are stored as numeric telescope IDs. -output_file (str, optional, default='grid_output.ecsv') - Output file for the generated grid points (default: 'grid_output.ecsv'). +output_file (str, optional, default='job_grid.ecsv') + Output file for the generated executable job grid. Example @@ -70,7 +62,11 @@ """ from simtools.application_control import build_application -from simtools.production_configuration.build_grid import build_production_grid_engine +from simtools.production_configuration.build_grid import ( + build_job_grid_metadata, + build_simulation_jobs, +) +from simtools.production_configuration.job_grid_io import serialize_job_grid def _add_arguments(parser): @@ -78,7 +74,7 @@ def _add_arguments(parser): parser.add_argument( "--axes", type=str, - required=True, + required=False, help="Path to a file defining the grid axes.", ) parser.add_argument( @@ -102,8 +98,8 @@ def _add_arguments(parser): parser.add_argument( "--output_file", type=str, - default="grid_output.ecsv", - help="Output file for the generated grid points (default: 'grid_output.ecsv').", + default="job_grid.ecsv", + help="Output file for the generated executable job grid.", ) parser.add_argument( "--telescope_ids", @@ -118,7 +114,7 @@ def _add_arguments(parser): parser.add_argument( "--lookup_table", type=str, - required=True, + required=False, help="Path to the lookup table for simulation limits. " "Table required with varying azimuth and or zenith angle. ", ) @@ -131,6 +127,33 @@ def _add_arguments(parser): "to telescope names when lookup-table selections are numeric IDs." ), ) + parser.add_argument( + "--number_of_runs", + help="Number of runs to be simulated.", + type=int, + required=False, + default=1, + ) + parser.add_argument( + "--nshow_power_index", + help=( + "Power-law index used to scale the baseline nshow with the geometric-mean energy " + "of each energy_range entry." + ), + type=float, + required=False, + default=None, + ) + parser.add_argument( + "--nshow_reference_energy", + help=( + "Reference energy for nshow power-law scaling (for example: '100 GeV'). " + "Required together with --nshow_power_index." + ), + type=str, + required=False, + default=None, + ) def main(): @@ -138,15 +161,19 @@ def main(): app_context = build_application( initialization_kwargs={ "db_config": True, - "simulation_model": ["version", "site", "model_version"], + "preserve_by_version_keys": ["array_layout_name"], + "simulation_model": ["site", "layout", "telescope", "model_version"], + "simulation_configuration": {"software": None, "corsika_configuration": ["all"]}, }, ) output_filepath = app_context.io_handler.get_output_file(app_context.args["output_file"]) - grid_gen = build_production_grid_engine(app_context.args) - - grid_points = grid_gen.generate_grid() - grid_gen.serialize_grid_points(grid_points, output_file=output_filepath) + job_rows = build_simulation_jobs(app_context.args) + serialize_job_grid( + job_rows=job_rows, + output_file=output_filepath, + metadata=build_job_grid_metadata(app_context.args), + ) if __name__ == "__main__": diff --git a/src/simtools/applications/simulate_prod_htcondor_generator.py b/src/simtools/applications/simulate_prod_htcondor_generator.py index 0ba1dcf9da..1a7ff0372e 100644 --- a/src/simtools/applications/simulate_prod_htcondor_generator.py +++ b/src/simtools/applications/simulate_prod_htcondor_generator.py @@ -3,7 +3,7 @@ r""" Generate a run script and submit file for HT Condor job submission of a simulation production. -This tool generates HTCondor submission files for one or more simulation production grids and +This tool generates HTCondor submission files from a pre-generated executable job grid and supports either a single Apptainer image or a label-to-image mapping for multi-image submissions. For each image label, it writes a dedicated ``simulate_prod.submit.