From 0cddcf7d43816be069d219a2e5b72b81a1f6abe1 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 13 Jan 2025 13:59:58 +0100 Subject: [PATCH 001/124] Upload scripts and tools for new simulation model repository --- .../upload_from_model_repository_to_db.sh | 26 +++++++++---------- ..._model_parameters_from_repository_to_db.py | 7 +++-- 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/database_scripts/upload_from_model_repository_to_db.sh b/database_scripts/upload_from_model_repository_to_db.sh index fe8f815466..f87ecf6a16 100755 --- a/database_scripts/upload_from_model_repository_to_db.sh +++ b/database_scripts/upload_from_model_repository_to_db.sh @@ -1,11 +1,12 @@ #!/bin/bash # Upload model parameter from repository to a local or remote mongoDB. # +# Execute this scripts from the ./database_scripts directory. # Cover 'source .env': the script ensure that this file exists: # shellcheck disable=SC1091 -DB_SIMULATION_MODEL_URL="https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters.git" -DB_SIMULATION_MODEL_BRANCH="main" +DB_SIMULATION_MODEL_URL="https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/simulation-models" +DB_SIMULATION_MODEL_BRANCH="v1.0.0-parameters" # Check that this script is not sourced but executed if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then @@ -27,9 +28,9 @@ echo "Cloning model parameters from $DB_SIMULATION_MODEL_URL" rm -rf ./tmp_model_parameters git clone --depth=1 -b $DB_SIMULATION_MODEL_BRANCH $DB_SIMULATION_MODEL_URL ./tmp_model_parameters -CURRENTDIR=$(pwd) +CURRENT_DIR=$(pwd) cd ./tmp_model_parameters/ || exit -cp -f "$CURRENTDIR"/../.env .env +cp -f "$CURRENT_DIR"/../.env .env # ask for confirmation before uploading to remote DB source .env @@ -44,15 +45,12 @@ if [[ $SIMTOOLS_DB_SERVER =~ $regex ]]; then fi # upload files to DB -model_directory="./model_versions/" -for dir in "${model_directory}"*/; do - simtools-db-add-model-parameters-from-repository-to-db \ - --model_version "$(basename "${dir}")" \ - --input_path "${dir}" \ - --db_name "$DB_SIMULATION_MODEL" \ - --type "model_parameters" -done - -cd "$CURRENTDIR" || exit +model_directory="./simulation-models/model_parameters/" +simtools-db-add-model-parameters-from-repository-to-db \ +--input_path "${model_directory}" \ +--db_name "$DB_SIMULATION_MODEL" \ +--type "model_parameters" + +cd "$CURRENT_DIR" || exit rm -rf ./tmp_model_parameters diff --git a/src/simtools/applications/db_add_model_parameters_from_repository_to_db.py b/src/simtools/applications/db_add_model_parameters_from_repository_to_db.py index 898c7f1a03..7983dad639 100644 --- a/src/simtools/applications/db_add_model_parameters_from_repository_to_db.py +++ b/src/simtools/applications/db_add_model_parameters_from_repository_to_db.py @@ -76,9 +76,7 @@ def _parse(label=None, description=None): choices=["model_parameters"], ) - args_dict, db_config = config.initialize( - output=True, require_command_line=True, db_config=True, simulation_model="version" - ) + args_dict, db_config = config.initialize(output=True, require_command_line=True, db_config=True) db_config["db_simulation_model"] = args_dict["db_name"] # overwrite explicitly DB configuration return args_dict, db_config @@ -138,6 +136,7 @@ def _add_model_parameters_to_db(args_dict, db, logger): """ input_path = Path(args_dict["input_path"]) + logger.info(f"Reading model parameters from repository path {input_path}") array_elements = [d for d in input_path.iterdir() if d.is_dir()] for element in array_elements: try: @@ -148,7 +147,7 @@ def _add_model_parameters_to_db(args_dict, db, logger): elif element.name in {"configuration_sim_telarray", "configuration_corsika"}: collection = element.name elif element.name == "Files": - logger.info("Files are uploaded with the corresponding model parameters") + logger.info("Files (tables) are uploaded with the corresponding model parameters") continue logger.info(f"Reading model parameters for {element.name} into collection {collection}") files_to_insert = list(Path(element).rglob("*json")) From 2139957a1c918e0ef2f59f8e47c9a3e760efb9de Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 13 Jan 2025 17:52:03 +0100 Subject: [PATCH 002/124] integration tests --- ... db_get_parameter_from_db_telescope_model_version.yml} | 2 +- ..._get_parameter_from_db_telescope_parameter_version.yml | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) rename tests/integration_tests/config/{db_get_parameter_from_db_telescope_parameter.yml => db_get_parameter_from_db_telescope_model_version.yml} (76%) create mode 100644 tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml diff --git a/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter.yml b/tests/integration_tests/config/db_get_parameter_from_db_telescope_model_version.yml similarity index 76% rename from tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter.yml rename to tests/integration_tests/config/db_get_parameter_from_db_telescope_model_version.yml index 063a1999a6..19f674dc1b 100644 --- a/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter.yml +++ b/tests/integration_tests/config/db_get_parameter_from_db_telescope_model_version.yml @@ -1,6 +1,6 @@ CTA_SIMPIPE: APPLICATION: simtools-db-get-parameter-from-db - TEST_NAME: telescope_parameter + TEST_NAME: telescope_parameter_parameter_version CONFIGURATION: SITE: North TELESCOPE: LSTN-01 diff --git a/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml b/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml new file mode 100644 index 0000000000..19c9a278a6 --- /dev/null +++ b/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml @@ -0,0 +1,8 @@ +CTA_SIMPIPE: + APPLICATION: simtools-db-get-parameter-from-db + TEST_NAME: telescope_parameter_parameter_version + CONFIGURATION: + SITE: North + TELESCOPE: LSTN-01 + PARAMETER: mirror_list + PARAMETER_VERSION: 1.0.0 From 5c910c3c486c4d97400475cf8931ab63dc8608ac Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 13 Jan 2025 17:57:56 +0100 Subject: [PATCH 003/124] docs --- docs/source/user-guide/applications.md | 2 +- ...model-parameters-from-repository-to-db.rst | 6 - ...simulation-model-from-repository-to-db.rst | 6 + docs/source/user-guide/databases.md | 2 +- pyproject.toml | 2 +- ..._model_parameters_from_repository_to_db.py | 183 ------------------ ..._simulation_model_from_repository_to_db.py | 100 ++++++++++ 7 files changed, 109 insertions(+), 192 deletions(-) delete mode 100644 docs/source/user-guide/applications/simtools-db-add-model-parameters-from-repository-to-db.rst create mode 100644 docs/source/user-guide/applications/simtools-db-add-simulation-model-from-repository-to-db.rst delete mode 100644 src/simtools/applications/db_add_model_parameters_from_repository_to_db.py create mode 100644 src/simtools/applications/db_add_simulation_model_from_repository_to_db.py diff --git a/docs/source/user-guide/applications.md b/docs/source/user-guide/applications.md index 54c21ba99e..212198f711 100644 --- a/docs/source/user-guide/applications.md +++ b/docs/source/user-guide/applications.md @@ -28,7 +28,7 @@ simtools-convert-all-model-parameters-from-simtel simtools-convert-model-parameter-from-simtel simtools-db-add-file-to-db -simtools-db-add-model-parameters-from-repository-to-db +simtools-db-add-simulation-model-from-repository-to-db simtools-db-add-value-from-json-to-db simtools-db-get-array-layouts-from-db simtools-db-get-file-from-db diff --git a/docs/source/user-guide/applications/simtools-db-add-model-parameters-from-repository-to-db.rst b/docs/source/user-guide/applications/simtools-db-add-model-parameters-from-repository-to-db.rst deleted file mode 100644 index 6fd7aa995d..0000000000 --- a/docs/source/user-guide/applications/simtools-db-add-model-parameters-from-repository-to-db.rst +++ /dev/null @@ -1,6 +0,0 @@ - -simtools-db-add-model-parameters-from-repository-to-db -====================================================== - -.. automodule:: db_add_model_parameters_from_repository_to_db - :members: diff --git a/docs/source/user-guide/applications/simtools-db-add-simulation-model-from-repository-to-db.rst b/docs/source/user-guide/applications/simtools-db-add-simulation-model-from-repository-to-db.rst new file mode 100644 index 0000000000..7049214841 --- /dev/null +++ b/docs/source/user-guide/applications/simtools-db-add-simulation-model-from-repository-to-db.rst @@ -0,0 +1,6 @@ + +simtools-db-add-simulation-model-from-repository-to-db +====================================================== + +.. automodule:: db_add_simulation_model_from_repository_to_db + :members: diff --git a/docs/source/user-guide/databases.md b/docs/source/user-guide/databases.md index 9f3d1b50c7..d19035e7a3 100644 --- a/docs/source/user-guide/databases.md +++ b/docs/source/user-guide/databases.md @@ -75,7 +75,7 @@ The following applications are important: * update or define a single model parameter from a json file (as defined in the model parameter repository): [db_add_value_from_json_to_db.py](db_add_value_from_json_to_db) * upload a model parameter file: [db_add_file_to_db.py](db_add_file_to_db) -* upload all model parameters and files from the model parameter repository: [db_add_model_parameters_from_repository_to_db.py](db_add_model_parameters_from_repository_to_db) +* upload all model parameters and files from the model parameter repository: [db_add_simulation_model_from_repository_to_db.py](db_add_simulation_model_from_repository_to_db) ## Configure and use a local copy of the model parameter database diff --git a/pyproject.toml b/pyproject.toml index 37ad437792..d3f3b71a68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ scripts.simtools-convert-all-model-parameters-from-simtel = "simtools.applicatio scripts.simtools-convert-geo-coordinates-of-array-elements = "simtools.applications.convert_geo_coordinates_of_array_elements:main" scripts.simtools-convert-model-parameter-from-simtel = "simtools.applications.convert_model_parameter_from_simtel:main" scripts.simtools-db-add-file-to-db = "simtools.applications.db_add_file_to_db:main" -scripts.simtools-db-add-model-parameters-from-repository-to-db = "simtools.applications.db_add_model_parameters_from_repository_to_db:main" +scripts.simtools-db-add-simulation-model-from-repository-to-db = "simtools.applications.db_add_simulation_model_from_repository_to_db:main" scripts.simtools-db-add-value-from-json-to-db = "simtools.applications.db_add_value_from_json_to_db:main" scripts.simtools-db-get-array-layouts-from-db = "simtools.applications.db_get_array_layouts_from_db:main" scripts.simtools-db-get-file-from-db = "simtools.applications.db_get_file_from_db:main" diff --git a/src/simtools/applications/db_add_model_parameters_from_repository_to_db.py b/src/simtools/applications/db_add_model_parameters_from_repository_to_db.py deleted file mode 100644 index 7983dad639..0000000000 --- a/src/simtools/applications/db_add_model_parameters_from_repository_to_db.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/python3 -r""" - Add parameters found in a model parameter repository to a new database. - - Generates a new database with all required collections. - Follows the structure of the CTAO gitlab model parameters repository. - file as input. - - This is an application for experts and should not be used by the general user. - - Command line arguments - - input_path (str, required) - Path of local copy of model parameter repository. - db_name (str, required) - Name of new DB to be created. - type (str, optional) - Type of data to be uploaded to the DB. Options are: model_parameters - - Examples - -------- - Upload model data repository to the DB: - - .. code-block:: console - - simtools-db-add_model-parameters-from-repository-to-db \ - --input_path /path/to/repository \ - --db_name new_db_name \ - --type model_parameters -""" - -import logging -from pathlib import Path - -import simtools.utils.general as gen -from simtools.configuration import configurator -from simtools.db import db_handler -from simtools.utils import names - - -def _parse(label=None, description=None): - """ - Parse command line configuration. - - Parameters - ---------- - label : str - Label describing application. - description : str - Description of application. - - Returns - ------- - CommandLineParser - Command line parser object. - """ - config = configurator.Configurator(label=label, description=description) - config.parser.add_argument( - "--input_path", - help="Path to model parameter repository.", - type=Path, - required=True, - ) - config.parser.add_argument( - "--db_name", - help="Name of the new model parameter database to be created.", - type=str.strip, - required=True, - ) - config.parser.add_argument( - "--type", - help="Type of data to be uploaded to the database.", - type=str, - required=False, - default="model_parameters", - choices=["model_parameters"], - ) - - args_dict, db_config = config.initialize(output=True, require_command_line=True, db_config=True) - db_config["db_simulation_model"] = args_dict["db_name"] # overwrite explicitly DB configuration - return args_dict, db_config - - -def add_values_from_json_to_db(file, collection, db, db_name, file_prefix, logger): - """ - Upload data from json files to db. - - Parameters - ---------- - file : list - Json file to be uploaded to the DB. - collection : str - The DB collection to which to add the file. - db : DatabaseHandler - Database handler object. - db_name : str - Name of the database to be created. - file_prefix : str - Path to location of all additional files to be uploaded. - logger : logging.Logger - Logger object. - """ - par_dict = gen.collect_data_from_file(file_name=file) - logger.info( - f"Adding the following parameter to the DB: {par_dict['parameter']} " - f"(collection {collection} in database {db_name})" - ) - db.add_new_parameter( - db_name=db_name, - array_element_name=par_dict["instrument"], - parameter=par_dict["parameter"], - version=par_dict["version"], - value=par_dict["value"], - site=par_dict["site"], - type=par_dict["type"], - collection_name=collection, - applicable=par_dict["applicable"], - file=par_dict["file"], - unit=par_dict.get("unit", None), - file_prefix=file_prefix, - ) - - -def _add_model_parameters_to_db(args_dict, db, logger): - """ - Add model parameters to the DB. - - Parameters - ---------- - args_dict : dict - Command line arguments. - db : DatabaseHandler - Database handler object. - logger : logging.Logger - Logger object. - - """ - input_path = Path(args_dict["input_path"]) - logger.info(f"Reading model parameters from repository path {input_path}") - array_elements = [d for d in input_path.iterdir() if d.is_dir()] - for element in array_elements: - try: - collection = names.get_collection_name_from_array_element_name(element.name) - except ValueError: - if element.name.startswith("OBS"): - collection = "sites" - elif element.name in {"configuration_sim_telarray", "configuration_corsika"}: - collection = element.name - elif element.name == "Files": - logger.info("Files (tables) are uploaded with the corresponding model parameters") - continue - logger.info(f"Reading model parameters for {element.name} into collection {collection}") - files_to_insert = list(Path(element).rglob("*json")) - for file in files_to_insert: - if collection == "files": - logger.info("Not yet implemented files") - else: - add_values_from_json_to_db( - file=file, - collection=collection, - db=db, - db_name=args_dict["db_name"], - file_prefix=input_path / "Files", - logger=logger, - ) - - -def main(): - """Application main.""" - label = Path(__file__).stem - args_dict, db_config = _parse( - label, description="Add or update a model parameter database to the DB" - ) - logger = logging.getLogger() - logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) - - db = db_handler.DatabaseHandler(mongo_db_config=db_config) - - _add_model_parameters_to_db(args_dict, db, logger) - - -if __name__ == "__main__": - main() diff --git a/src/simtools/applications/db_add_simulation_model_from_repository_to_db.py b/src/simtools/applications/db_add_simulation_model_from_repository_to_db.py new file mode 100644 index 0000000000..434816035e --- /dev/null +++ b/src/simtools/applications/db_add_simulation_model_from_repository_to_db.py @@ -0,0 +1,100 @@ +#!/usr/bin/python3 +r""" + Add parameters and production tables found in a model parameter repository to a new database. + + Generates a new database with all required collections. + Follows the structure of the CTAO gitlab model parameters repository. + + This is an application for experts and should not be used by the general user. + + Command line arguments + + input_path (str, required) + Path of local copy of model parameter repository. + db_name (str, required) + Name of new DB to be created. + type (str, optional) + Type of data to be uploaded to the DB. Options are: model_parameters, production_tables. + + Examples + -------- + Upload model data repository to the DB: + + .. code-block:: console + + simtools-db-simulation-model-from-repository-to-db \ + --input_path /path/to/repository \ + --db_name new_db_name \ + --type model_parameters +""" + +import logging +from pathlib import Path + +import simtools.utils.general as gen +from simtools.configuration import configurator +from simtools.db import db_handler, db_model_upload + + +def _parse(label=None, description=None): + """ + Parse command line configuration. + + Parameters + ---------- + label : str + Label describing application. + description : str + Description of application. + + Returns + ------- + CommandLineParser + Command line parser object. + """ + config = configurator.Configurator(label=label, description=description) + config.parser.add_argument( + "--input_path", + help="Path to model parameter repository.", + type=Path, + required=True, + ) + config.parser.add_argument( + "--db_name", + help="Name of the new model parameter database to be created.", + type=str.strip, + required=True, + ) + config.parser.add_argument( + "--type", + help="Type of data to be uploaded to the database.", + type=str, + required=False, + default="model_parameters", + choices=["model_parameters", "production_tables"], + ) + + args_dict, db_config = config.initialize(output=True, require_command_line=True, db_config=True) + db_config["db_simulation_model"] = args_dict["db_name"] # overwrite explicitly DB configuration + return args_dict, db_config + + +def main(): + """Application main.""" + label = Path(__file__).stem + args_dict, db_config = _parse( + label, description="Add or update a model parameter database to the DB" + ) + logger = logging.getLogger() + logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) + + db = db_handler.DatabaseHandler(mongo_db_config=db_config) + + if args_dict.get("type") == "model_parameters": + db_model_upload.add_model_parameters_to_db(args_dict, db) + elif args_dict.get("type") == "production_tables": + db_model_upload.add_production_tables_to_db(args_dict, db) + + +if __name__ == "__main__": + main() From 95f2a306b51147efde679e685fe08674921869ae Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 13 Jan 2025 17:59:52 +0100 Subject: [PATCH 004/124] upload production tables --- .../upload_from_model_repository_to_db.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/database_scripts/upload_from_model_repository_to_db.sh b/database_scripts/upload_from_model_repository_to_db.sh index f87ecf6a16..00b08cb88f 100755 --- a/database_scripts/upload_from_model_repository_to_db.sh +++ b/database_scripts/upload_from_model_repository_to_db.sh @@ -47,9 +47,16 @@ fi # upload files to DB model_directory="./simulation-models/model_parameters/" simtools-db-add-model-parameters-from-repository-to-db \ ---input_path "${model_directory}" \ ---db_name "$DB_SIMULATION_MODEL" \ ---type "model_parameters" + --input_path "${model_directory}" \ + --db_name "$DB_SIMULATION_MODEL" \ + --type "model_parameters" + +# upload production tables to DB +production_directory="./simulation-models/productions" +simtools-db-add-model-parameters-from-repository-to-db \ + --input_path "${production_directory}" \ + --db_name "$DB_SIMULATION_MODEL" \ + --type "production_tables" cd "$CURRENT_DIR" || exit From db7ef4dcd03066cd191c0e8640dad6dcc85d2390 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 13 Jan 2025 18:00:08 +0100 Subject: [PATCH 005/124] add parameter and production model version --- src/simtools/configuration/commandline_parser.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/simtools/configuration/commandline_parser.py b/src/simtools/configuration/commandline_parser.py index 4e01be250a..d06389ab6a 100644 --- a/src/simtools/configuration/commandline_parser.py +++ b/src/simtools/configuration/commandline_parser.py @@ -245,10 +245,17 @@ def initialize_simulation_model_arguments(self, model_options): _job_group = self.add_argument_group("simulation model") _job_group.add_argument( "--model_version", - help="model version", + help="production model version", type=str, default=None, ) + if "parameter_version" in model_options: + _job_group.add_argument( + "--parameter_version", + help="model parameter version", + type=str, + default=None, + ) if any( option in model_options for option in ["site", "telescope", "layout", "layout_file"] ): From 8275d9458fd4dd1c714c08778d09104fc9aa2a44 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 13 Jan 2025 18:01:35 +0100 Subject: [PATCH 006/124] New model to upload from repository --- src/simtools/db/db_model_upload.py | 131 +++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 src/simtools/db/db_model_upload.py diff --git a/src/simtools/db/db_model_upload.py b/src/simtools/db/db_model_upload.py new file mode 100644 index 0000000000..783c1cdb97 --- /dev/null +++ b/src/simtools/db/db_model_upload.py @@ -0,0 +1,131 @@ +"""Module for uploading a simulation model (parameters and production tables) to the database.""" + +import logging +from pathlib import Path + +import simtools.utils.general as gen +from simtools.utils import names + +logger = logging.getLogger(__name__) + + +def add_values_from_json_to_db(file, collection, db, db_name, file_prefix): + """ + Upload new model parameter from json files to db. + + Parameters + ---------- + file : list + Json file to be uploaded to the DB. + collection : str + The DB collection to which to add the file. + db : DatabaseHandler + Database handler object. + db_name : str + Name of the database to be created. + file_prefix : str + Path to location of all additional files to be uploaded. + """ + par_dict = gen.collect_data_from_file(file_name=file) + logger.info( + f"Adding the following parameter to the DB: {par_dict['parameter']} " + f"(collection {collection} in database {db_name})" + ) + db.add_new_parameter( + db_name=db_name, + array_element_name=par_dict["instrument"], + parameter=par_dict["parameter"], + parameter_version=par_dict["parameter_version"], + value=par_dict["value"], + site=par_dict["site"], + type=par_dict["type"], + collection_name=collection, + applicable=par_dict["applicable"], + file=par_dict["file"], + unit=par_dict.get("unit", None), + file_prefix=file_prefix, + ) + + +def add_model_parameters_to_db(args_dict, db): + """ + Read model parameters from a directory and upload them to the database. + + Parameters + ---------- + args_dict : dict + Command line arguments. + db : DatabaseHandler + Database handler object. + """ + input_path = Path(args_dict["input_path"]) + logger.info(f"Reading model parameters from repository path {input_path}") + array_elements = [d for d in input_path.iterdir() if d.is_dir()] + for element in array_elements: + try: + collection = names.get_collection_name_from_array_element_name(element.name) + except ValueError: + if element.name.startswith("OBS"): + collection = "sites" + elif element.name in {"configuration_sim_telarray", "configuration_corsika"}: + collection = element.name + elif element.name == "Files": + logger.info("Files (tables) are uploaded with the corresponding model parameters") + continue + logger.info(f"Reading model parameters for {element.name} into collection {collection}") + files_to_insert = list(Path(element).rglob("*json")) + for file in files_to_insert: + if collection == "files": + logger.info("Not yet implemented files") + else: + add_values_from_json_to_db( + file=file, + collection=collection, + db=db, + db_name=args_dict["db_name"], + file_prefix=input_path / "Files", + ) + + +def add_production_tables_to_db(args_dict, db): + """ + Read production tables from a directory and upload them to the database. + + A single dictionary is prepared for each model version, containing the complete + table of all array elements, sites, and parameters. + + Parameters + ---------- + args_dict : dict + Command line arguments. + db : DatabaseHandler + Database handler object. + """ + input_path = Path(args_dict["input_path"]) + logger.info(f"Reading production tables from repository path {input_path}") + model_versions = [d for d in input_path.iterdir() if d.is_dir()] + + for model in model_versions: + logger.info(f"Reading production tables for model version {model.name}") + files_to_insert = sorted(Path(model).rglob("*json")) + model_dict = { + "model_version": None, + "parameters": {}, + } + for file in files_to_insert: + array_element = file.stem + parameter_dict = gen.collect_data_from_file(file_name=file) + logger.info(f"Reading production table for {array_element}") + try: + model_dict["model_version"] = parameter_dict["model_version"] + model_dict["parameters"][array_element] = parameter_dict["parameters"][ + array_element + ] + except KeyError as exc: + logger.error(f"KeyError: {exc}") + raise + + db.add_production_table( + db_name=args_dict["db_name"], + production_table=model_dict, + ) From 41d834b2fc121159e633a498a72d4b717160f655 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 13 Jan 2025 21:20:34 +0100 Subject: [PATCH 007/124] upload --- database_scripts/upload_from_model_repository_to_db.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database_scripts/upload_from_model_repository_to_db.sh b/database_scripts/upload_from_model_repository_to_db.sh index 00b08cb88f..1bc9811952 100755 --- a/database_scripts/upload_from_model_repository_to_db.sh +++ b/database_scripts/upload_from_model_repository_to_db.sh @@ -46,14 +46,14 @@ fi # upload files to DB model_directory="./simulation-models/model_parameters/" -simtools-db-add-model-parameters-from-repository-to-db \ +simtools-db-add-simulation-model-from-repository-to-db \ --input_path "${model_directory}" \ --db_name "$DB_SIMULATION_MODEL" \ --type "model_parameters" # upload production tables to DB production_directory="./simulation-models/productions" -simtools-db-add-model-parameters-from-repository-to-db \ +simtools-db-add-simulation-model-from-repository-to-db \ --input_path "${production_directory}" \ --db_name "$DB_SIMULATION_MODEL" \ --type "production_tables" From 67cc9e75e014b447757225cdd43fe45f6545fcf0 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 14 Jan 2025 09:38:17 +0100 Subject: [PATCH 008/124] upload one production table per collection --- src/simtools/db/db_model_upload.py | 56 ++++++++++++++++-------------- src/simtools/utils/names.py | 18 ++++++++-- 2 files changed, 45 insertions(+), 29 deletions(-) diff --git a/src/simtools/db/db_model_upload.py b/src/simtools/db/db_model_upload.py index 783c1cdb97..d6e272eebb 100644 --- a/src/simtools/db/db_model_upload.py +++ b/src/simtools/db/db_model_upload.py @@ -62,16 +62,10 @@ def add_model_parameters_to_db(args_dict, db): logger.info(f"Reading model parameters from repository path {input_path}") array_elements = [d for d in input_path.iterdir() if d.is_dir()] for element in array_elements: - try: - collection = names.get_collection_name_from_array_element_name(element.name) - except ValueError: - if element.name.startswith("OBS"): - collection = "sites" - elif element.name in {"configuration_sim_telarray", "configuration_corsika"}: - collection = element.name - elif element.name == "Files": - logger.info("Files (tables) are uploaded with the corresponding model parameters") - continue + collection = names.get_collection_name_from_array_element_name(element.name, False) + if collection == "Files": + logger.info("Files (tables) are uploaded with the corresponding model parameters") + continue logger.info(f"Reading model parameters for {element.name} into collection {collection}") files_to_insert = list(Path(element).rglob("*json")) for file in files_to_insert: @@ -91,8 +85,8 @@ def add_production_tables_to_db(args_dict, db): """ Read production tables from a directory and upload them to the database. - A single dictionary is prepared for each model version, containing the complete - table of all array elements, sites, and parameters. + One dictionary per collection is prepared for each model version, containing + tables of all array elements, sites, and configuration parameters. Parameters ---------- @@ -103,29 +97,37 @@ def add_production_tables_to_db(args_dict, db): """ input_path = Path(args_dict["input_path"]) logger.info(f"Reading production tables from repository path {input_path}") - model_versions = [d for d in input_path.iterdir() if d.is_dir()] - for model in model_versions: + for model in filter(Path.is_dir, input_path.iterdir()): logger.info(f"Reading production tables for model version {model.name}") - files_to_insert = sorted(Path(model).rglob("*json")) - model_dict = { - "model_version": None, - "parameters": {}, - } - for file in files_to_insert: + model_dict = {} + for file in sorted(model.rglob("*json")): array_element = file.stem + collection = names.get_collection_name_from_array_element_name(array_element, False) + model_dict.setdefault( + collection, + { + "collection": collection, + "model_version": model.name, + "parameters": {}, + }, + ) parameter_dict = gen.collect_data_from_file(file_name=file) - logger.info(f"Reading production table for {array_element}") + logger.info(f"Reading production table for {array_element} (collection {collection})") try: - model_dict["model_version"] = parameter_dict["model_version"] - model_dict["parameters"][array_element] = parameter_dict["parameters"][ + model_dict[collection]["parameters"][array_element] = parameter_dict["parameters"][ array_element ] except KeyError as exc: logger.error(f"KeyError: {exc}") raise - db.add_production_table( - db_name=args_dict["db_name"], - production_table=model_dict, - ) + for collection, data in model_dict.items(): + if not data["parameters"]: + logger.info(f"No production table for {collection} in model version {model.name}") + continue + logger.info(f"Adding production table for {collection} to the database") + db.add_production_table( + db_name=args_dict["db_name"], + production_table=data, + ) diff --git a/src/simtools/utils/names.py b/src/simtools/utils/names.py index 51caac9c2e..b63594a8ce 100644 --- a/src/simtools/utils/names.py +++ b/src/simtools/utils/names.py @@ -293,7 +293,7 @@ def get_site_from_array_element_name(name): return array_elements()[get_array_element_type_from_name(name)]["site"] -def get_collection_name_from_array_element_name(name): +def get_collection_name_from_array_element_name(name, array_elements_only=True): """ Get collection name (e.g., telescopes, calibration_devices, sites) of array element from name. @@ -301,6 +301,8 @@ def get_collection_name_from_array_element_name(name): ---------- name: str Array element name. + array_elements_only: bool + If True, only array elements are considered. Returns ------- @@ -315,7 +317,19 @@ def get_collection_name_from_array_element_name(name): validate_site_name(name) return "sites" except ValueError as exc: - raise ValueError(f"Invalid array element name {name}: {exc}") from exc + if array_elements_only: + raise ValueError(f"Invalid array element name {name}: {exc}") from exc + if name.startswith("OBS"): + return "sites" + if name in ( + "configuration_sim_telarray", + "configuration_corsika", + "Files", + "Dummy-Telescope", + ): + return name + + raise ValueError(f"Invalid array element name {name}") def get_simulation_software_name_from_parameter_name( From 364f86ba7b17bb699d22e74ce192257dafe1f727 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 14 Jan 2025 14:19:21 +0100 Subject: [PATCH 009/124] explicit function statements --- .../applications/db_get_parameter_from_db.py | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index f1878b04c8..5e2a2546a8 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -83,7 +83,7 @@ def _parse(): required=False, ) - return config.initialize(db_config=True, simulation_model="telescope") + return config.initialize(db_config=True, simulation_model=["telescope", "parameter_version"]) def main(): # noqa: D103 @@ -92,26 +92,30 @@ def main(): # noqa: D103 logger = logging.getLogger() logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) + # TODO read parameter_version + db = db_handler.DatabaseHandler(mongo_db_config=db_config) if args_dict["db_collection"] == "configuration_sim_telarray": pars = db.get_model_parameters( - args_dict["site"], - args_dict["telescope"], - args_dict["model_version"], + site=args_dict["site"], + array_element_name=args_dict["telescope"], + model_version=args_dict["model_version"], collection="configuration_sim_telarray", ) elif args_dict["db_collection"] == "configuration_corsika": - pars = db.get_corsika_configuration_parameters(args_dict["model_version"]) + pars = db.get_corsika_configuration_parameters(model_version=args_dict["model_version"]) elif args_dict["telescope"] is not None: pars = db.get_model_parameters( - args_dict["site"], - args_dict["telescope"], - args_dict["model_version"], + site=args_dict["site"], + array_element_name=args_dict["telescope"], + model_version=args_dict["model_version"], collection="telescopes", ) else: - pars = db.get_site_parameters(args_dict["site"], args_dict["model_version"]) + pars = db.get_site_parameters( + site=args_dict["site"], model_version=args_dict["model_version"] + ) if args_dict["parameter"] not in pars: raise KeyError(f"The requested parameter, {args_dict['parameter']}, does not exist.") if args_dict["output_file"] is not None: From 0acbff4bbcaf96e5af530cd1ed654abc2517e78c Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 14 Jan 2025 14:19:33 +0100 Subject: [PATCH 010/124] todo --- src/simtools/model/model_parameter.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index c3bff5aec3..3abad15a1c 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -44,6 +44,8 @@ class ModelParameter: label: str Instance label. Important for output file naming. + TODO - make sure that model_version is to correct version to be used here + """ def __init__( From cab451be36c3d7aca77a3ca8bad79a0aced61fbe Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 14 Jan 2025 14:25:00 +0100 Subject: [PATCH 011/124] read production tables --- src/simtools/db/db_handler.py | 143 +++++++++++++++++++------- src/simtools/model/model_parameter.py | 4 +- 2 files changed, 105 insertions(+), 42 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index 4ab25d8ff9..81d008dbc7 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -9,7 +9,7 @@ import jsonschema from bson.objectid import ObjectId from packaging.version import Version -from pymongo import ASCENDING, MongoClient +from pymongo import ASCENDING, DESCENDING, MongoClient from pymongo.errors import BulkWriteError from simtools.db import db_array_elements, db_from_repo_handler @@ -72,6 +72,7 @@ class DatabaseHandler: DB_CTA_SIMULATION_MODEL_DESCRIPTIONS = "CTA-Simulation-Model-Descriptions" # DB collection with updates field names DB_DERIVED_VALUES = "Staging-CTA-Simulation-Model-Derived-Values" + DB_PRODUCTION_TABLES = "production_tables" ALLOWED_FILE_EXTENSIONS = [".dat", ".txt", ".lis", ".cfg", ".yml", ".yaml", ".ecsv"] @@ -216,17 +217,17 @@ def get_model_parameters( ------- dict containing the parameters + TODO - update with model_version and parameter_version """ - _site, _array_element_name, _model_version = self._validate_model_input( - site, array_element_name, model_version - ) + _site, _array_element_name = self._validate_model_input(site, array_element_name) + _model_version = None array_element_list = db_array_elements.get_array_element_list_for_db_query( _array_element_name, self, _model_version, collection ) pars = {} for array_element in array_element_list: _array_elements_cache_key = self._parameter_cache_key( - site, array_element, model_version, collection + _site, array_element, model_version, collection ) try: pars.update(DatabaseHandler.model_parameters_cached[_array_elements_cache_key]) @@ -382,6 +383,8 @@ def read_mongo_db( ValueError if query returned zero results. + TODO - model_version vs parameter_version + """ collection = self.get_collection(db_name, collection_name) _parameters = {} @@ -414,7 +417,6 @@ def get_site_parameters( self, site, model_version, - only_applicable=False, ): """ Get parameters from either MongoDB or simulation model repository for a specific site. @@ -425,17 +427,19 @@ def get_site_parameters( Site name. model_version: str Version of the model. - only_applicable: bool - If True, only applicable parameters will be read. Returns ------- dict containing the parameters + TODO - model_version vs parameter_version """ - _site, _, _model_version = self._validate_model_input(site, None, model_version) + _site, _ = self._validate_model_input(site, None) + _production_table = self.get_production_table_from_mongo_db("sites", model_version) _db_name = self._get_db_name() - _site_cache_key = self._parameter_cache_key(site, None, model_version) + _site_cache_key = self._parameter_cache_key( + site, None, _production_table.get("model_version") + ) try: return DatabaseHandler.site_parameters_cached[_site_cache_key] except KeyError: @@ -444,8 +448,7 @@ def get_site_parameters( _pars = self._get_site_parameters_mongo_db( _db_name, _site, - _model_version, - only_applicable, + _production_table, ) # update simulation model using repository if self.mongo_db_config.get("db_simulation_model_url", None) is not None: @@ -454,14 +457,14 @@ def get_site_parameters( site=_site, array_element_name=None, parameter_collection="site", - model_version=_model_version, + model_version=model_version, db_simulation_model_url=self.mongo_db_config.get("db_simulation_model_url", None), ) DatabaseHandler.site_parameters_cached[_site_cache_key] = _pars return DatabaseHandler.site_parameters_cached[_site_cache_key] - def _get_site_parameters_mongo_db(self, db_name, site, model_version, only_applicable=False): + def _get_site_parameters_mongo_db(self, db_name, site, production_table): """ Get parameters from MongoDB for a specific site. @@ -471,10 +474,8 @@ def _get_site_parameters_mongo_db(self, db_name, site, model_version, only_appli The name of the DB. site: str Site name. - model_version: str - Version of the model. - only_applicable: bool - If True, only applicable parameters will be read. + production_table: dict + Table with parameter versions. Returns ------- @@ -484,22 +485,25 @@ def _get_site_parameters_mongo_db(self, db_name, site, model_version, only_appli ------ ValueError if query returned zero results. - """ collection = self.get_collection(db_name, "sites") - _parameters = {} - + try: + parameter_query = production_table["parameters"][f"OBS-{site}"] + except KeyError as exc: + raise ValueError(f"Site {site} not found in the production table") from exc query = { "site": site, - "version": model_version, + "$or": [ + {"parameter": param, "parameter_version": version} + for param, version in parameter_query.items() + ], } - if only_applicable: - query["applicable"] = True if collection.count_documents(query) < 1: raise ValueError( "The following query returned zero results! Check the input data and rerun.\n", query, ) + _parameters = {} for post in collection.find(query).sort("parameter", ASCENDING): par_now = post["parameter"] _parameters[par_now] = post @@ -509,6 +513,37 @@ def _get_site_parameters_mongo_db(self, db_name, site, model_version, only_appli return _parameters + def get_production_table_from_mongo_db(self, collection_name, model_version): + """ + Get production table from MongoDB. + + Parameters + ---------- + collection_name: str + Name of the collection. + model_version: str + Version of the model. + """ + collection = self.get_collection(self._get_db_name(), "production_tables") + + query = { + "model_version": model_version, + "collection": collection_name, + } + post = collection.find_one(query, sort=[("_id", DESCENDING)]) + try: + return { + "collection": post["collection"], + "model_version": post["model_version"], + "parameters": post["parameters"], + "entry_date": ObjectId(post["_id"]).generation_time, + } + except TypeError as exc: + raise ValueError( + "The following query returned zero results:\n", + query, + ) from exc + def get_derived_values(self, site, array_element_name, model_version): """ Get all derived values from the DB for a specific array element. @@ -527,9 +562,8 @@ def get_derived_values(self, site, array_element_name, model_version): dict containing the parameters """ - _, _array_element_name, _model_version = self._validate_model_input( - site, array_element_name, model_version - ) + _, _array_element_name = self._validate_model_input(site, array_element_name) + _model_version = model_version return self.read_mongo_db( DatabaseHandler.DB_DERIVED_VALUES, @@ -565,6 +599,8 @@ def get_simulation_configuration_parameters( ------ ValueError if simulation_software is not valid. + + TODO - model_version vs parameter_version """ if simulation_software == "corsika": return self.get_corsika_configuration_parameters(model_version) @@ -589,6 +625,8 @@ def get_corsika_configuration_parameters(self, model_version): ------- dict Configuration parameters for CORSIKA + + TODO - model_version vs parameter_version """ _corsika_cache_key = self._parameter_cache_key(None, None, model_version) try: @@ -607,7 +645,7 @@ def get_corsika_configuration_parameters(self, model_version): ) return DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key] - def _validate_model_input(self, site, array_element_name, model_version): + def _validate_model_input(self, site, array_element_name): """ Validate input for model parameter queries. @@ -615,14 +653,10 @@ def _validate_model_input(self, site, array_element_name, model_version): Site name. array_element_name: str Name of the array element model (e.g. LSTN-01, MSTS-design) - model_version: str - Version of the model. - """ return ( names.validate_site_name(site), names.validate_array_element_name(array_element_name) if array_element_name else None, - self.model_version(model_version), ) @staticmethod @@ -711,6 +745,8 @@ def copy_array_element( ------ BulkWriteError + TODO - copy uses 'version' - can this function be removed? + """ db_name = self._get_db_name(db_name) if db_to_copy_to is None: @@ -776,6 +812,8 @@ def copy_documents(self, db_name, collection, query, db_to_copy_to, collection_t ------ BulkWriteError + TODO - remove this function? + """ db_name = self._get_db_name(db_name) @@ -821,6 +859,8 @@ def delete_query(self, db_name, collection, query): "version": "6.0.0", } + TODO - remove this function? + """ _collection = self.get_collection(db_name, collection) @@ -876,6 +916,8 @@ def update_parameter_field( ValueError if field not in allowed fields + TODO - remove? + """ db_name = self._get_db_name(db_name) allowed_fields = ["applicable", "unit", "type", "items", "minimum", "maximum"] @@ -937,10 +979,28 @@ def update_parameter_field( model_version=_model_version, ) + def add_production_table(self, db_name, production_table): + """ + Add a production table for a given model version to the DB. + + Parameters + ---------- + db_name: str + the name of the DB. + production_table: dict + The production table to add to the DB. + """ + db_name = self._get_db_name(db_name) + collection = self.get_collection(db_name, "production_tables") + self._logger.info(f"Adding production for {production_table.get('collection')} to to DB") + collection.insert_one(production_table) + + # TODO - reset some cache for production tables? + def add_new_parameter( self, db_name, - version, + parameter_version, parameter, value, array_element_name=None, @@ -959,7 +1019,7 @@ def add_new_parameter( ---------- db_name: str the name of the DB - version: str + parameter_version: str The version of the new parameter value parameter: str Which parameter to add @@ -999,7 +1059,7 @@ def add_new_parameter( else: raise ValueError(f"Cannot add parameter to collection {collection_name}") - db_entry["version"] = version + db_entry["parameter_version"] = parameter_version db_entry["parameter"] = parameter if site is not None: db_entry["site"] = names.validate_site_name(site) @@ -1024,9 +1084,6 @@ def add_new_parameter( file_path = Path(file_prefix).joinpath(value) files_to_add_to_db.add(f"{file_path}") - kwargs.pop("type", None) - db_entry.update(kwargs) - self._logger.info( f"Will add the following entry to DB {db_name} and collection {db_name}:\n{db_entry}" ) @@ -1036,7 +1093,7 @@ def add_new_parameter( self._logger.info(f"Will also add the file {file_to_insert_now} to the DB") self.insert_file_to_db(file_to_insert_now, db_name) - self._reset_parameter_cache(site, array_element_name, version) + self._reset_parameter_cache(site, array_element_name, parameter_version) def _get_db_name(self, db_name=None): """ @@ -1078,6 +1135,8 @@ def model_version(self, version, db_name=None): ValueError if version not valid. + # TODO - needs to be adjusted? + """ _all_versions = self.get_all_versions(db_name=db_name) if version in _all_versions: @@ -1167,6 +1226,8 @@ def get_all_versions( ValueError If key to collection_name is not valid. + TODO - this should be model or parameter versions? Need to be adjusted. + """ db_name = self._get_db_name() if db_name is None else db_name if not db_name: @@ -1220,6 +1281,8 @@ def _parameter_cache_key(self, site, array_element_name, model_version, collecti ------- str Cache key. + + # TODO - understand if model_version is correct here. """ parts = [] if site: @@ -1243,6 +1306,8 @@ def _reset_parameter_cache(self, site, array_element_name, model_version): Array element name. model_version: str Model version. + + # TODO - understand if model_version is correct here. """ self._logger.debug(f"Resetting cache for {site} {array_element_name} {model_version}") _cache_key = self._parameter_cache_key(site, array_element_name, model_version) diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index 3abad15a1c..78276ad8df 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -340,9 +340,7 @@ def _load_parameters_from_db(self): ) if self.site is not None: - _site_pars = self.db.get_site_parameters( - self.site, self.model_version, only_applicable=True - ) + _site_pars = self.db.get_site_parameters(self.site, self.model_version) self._parameters.update(_site_pars) self._load_simulation_software_parameter() From 6efae01e0a383bb22ad2a4ebc7257a15a89e33d2 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 14 Jan 2025 15:08:12 +0100 Subject: [PATCH 012/124] Add production table --- src/simtools/db/db_handler.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index b9a8cb2413..481b794bfd 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -753,6 +753,23 @@ def copy_array_element( except BulkWriteError as exc: raise BulkWriteError(str(exc.details)) from exc + def add_production_table(self, db_name, production_table): + """ + Add a production table for a given model version to the DB. + + Parameters + ---------- + db_name: str + the name of the DB. + production_table: dict + The production table to add to the DB. + """ + db_name = self._get_db_name(db_name) + collection = self.get_collection(db_name, "production_tables") + self._logger.info(f"Adding production for {production_table.get('collection')} to to DB") + collection.insert_one(production_table) + # TODO - reset some cache for production tables? + def add_new_parameter( self, db_name, From a13c87461cfde3baf0190ce4686323ed19a40c68 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 14 Jan 2025 16:55:55 +0100 Subject: [PATCH 013/124] simplification --- .../applications/db_get_parameter_from_db.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 5e2a2546a8..4841d2f854 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -96,22 +96,19 @@ def main(): # noqa: D103 db = db_handler.DatabaseHandler(mongo_db_config=db_config) - if args_dict["db_collection"] == "configuration_sim_telarray": + if args_dict["telescope"] is not None: pars = db.get_model_parameters( site=args_dict["site"], array_element_name=args_dict["telescope"], model_version=args_dict["model_version"], - collection="configuration_sim_telarray", + collection=( + "configuration_sim_telarray" + if args_dict["db_collection"] == "configuration_sim_telarray" + else "telescopes" + ), ) elif args_dict["db_collection"] == "configuration_corsika": pars = db.get_corsika_configuration_parameters(model_version=args_dict["model_version"]) - elif args_dict["telescope"] is not None: - pars = db.get_model_parameters( - site=args_dict["site"], - array_element_name=args_dict["telescope"], - model_version=args_dict["model_version"], - collection="telescopes", - ) else: pars = db.get_site_parameters( site=args_dict["site"], model_version=args_dict["model_version"] From 8257d279ac3060f74d6912cc8d8d635b4c40c2ad Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 14 Jan 2025 16:56:43 +0100 Subject: [PATCH 014/124] unit tests for _cache_key --- tests/unit_tests/db/test_db_handler.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index 10bc7d5590..b80abdc042 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -276,10 +276,7 @@ def test_adding_new_parameter_db(db, random_id, io_handler, model_version): assert pars["new_test_parameter_quantity_str"]["unit"] == "cm" # make sure that cache has been emptied after updating - assert ( - db._parameter_cache_key("North", "LSTN-test", test_model_version) - not in db.model_parameters_cached - ) + assert db._cache_key("North", "LSTN-test", test_model_version) not in db.model_parameters_cached # site parameters db.add_new_parameter( @@ -456,11 +453,11 @@ def test_get_all_versions(db, mocker, caplog): assert "No database name defined to determine" in caplog.text -def test_parameter_cache_key(db, model_version_prod5): +def test_cache_key(db, model_version_prod5): - assert db._parameter_cache_key("North", "LSTN-01", model_version_prod5) == "North-LSTN-01-5.0.0" - assert db._parameter_cache_key("North", None, model_version_prod5) == "North-5.0.0" - assert db._parameter_cache_key(None, None, model_version_prod5) == "5.0.0" + assert db._cache_key("North", "LSTN-01", model_version_prod5) == "North-LSTN-01-5.0.0" + assert db._cache_key("North", None, model_version_prod5) == "North-5.0.0" + assert db._cache_key(None, None, model_version_prod5) == "5.0.0" def test_model_version(db): From 34464234e9ba39984598db4cf3ff8e88ca80ca34 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 14 Jan 2025 17:01:18 +0100 Subject: [PATCH 015/124] remove obsolute code --- src/simtools/db/db_array_elements.py | 78 ------------------- tests/unit_tests/db/test_db_array_elements.py | 28 ------- 2 files changed, 106 deletions(-) diff --git a/src/simtools/db/db_array_elements.py b/src/simtools/db/db_array_elements.py index 4a0c2eff06..7e70ea0a9a 100644 --- a/src/simtools/db/db_array_elements.py +++ b/src/simtools/db/db_array_elements.py @@ -4,8 +4,6 @@ from pymongo import ASCENDING -from simtools.utils import names - @lru_cache def get_array_elements(db_collection, model_version): @@ -52,79 +50,3 @@ def get_array_elements(db_collection, model_version): raise ValueError(f"No array elements found in DB collection {db_collection}.") return _all_available_array_elements - - -def get_array_element_list_for_db_query(array_element_name, db, model_version, collection): - """ - Get array element name and design model for querying the database. - - Return a list of array element names to be used for querying the database for a given array - element. This is in most cases the array element itself and its design model. - In cases of no design model available, the design model of the array element is returned. - - Parameters - ---------- - array_element_name: str - Name of the array element model (e.g. MSTN-01). - db: DBHandler - Instance of the database handler - model_version: str - Model version. - collection: str - DB collection to get the array elements from (e.g., telescopes, calibration_devices) - - Returns - ------- - list - List of array element model names as used in the DB. - - """ - try: - _available_array_elements = get_array_elements( - db.get_collection(db_name=None, collection_name=collection), - db.model_version(model_version), - ) - except ValueError: - return [names.get_array_element_type_from_name(array_element_name) + "-design"] - try: - return [_available_array_elements[array_element_name], array_element_name] - except KeyError: - pass - - if array_element_name in _available_array_elements.values(): - return [array_element_name] - - raise ValueError(f"Array element {array_element_name} not found in DB.") - - -def get_array_elements_of_type(array_element_type, db, model_version, collection): - """ - Get all array elements of a certain type in the specified collection in the DB. - - Return e.g. for array_element_type='MSTS' all MSTS array elements found in the collection. - - Parameters - ---------- - array_element_type : str - Type of the array element (e.g. LSTN, MSTS) - model_version : str - Which version to get the array elements of - collection : str - Which collection to get the array elements from: - i.e. telescopes, calibration_devices - db_name : str - Database name - - Returns - ------- - list - Sorted list of all array element names found in collection - - """ - _available_array_elements = get_array_elements( - db.get_collection(db_name=None, collection_name=collection), - db.model_version(model_version), - ) - return sorted( - [entry for entry in _available_array_elements if entry.startswith(array_element_type)] - ) diff --git a/tests/unit_tests/db/test_db_array_elements.py b/tests/unit_tests/db/test_db_array_elements.py index 8d2a0f43e9..5508a29294 100644 --- a/tests/unit_tests/db/test_db_array_elements.py +++ b/tests/unit_tests/db/test_db_array_elements.py @@ -56,34 +56,6 @@ def test_get_array_elements(db, model_version): ) -def test_get_array_element_list_for_db_query(db, model_version): - - assert db_array_elements.get_array_element_list_for_db_query( - "LSTN-01", db=db, model_version=model_version, collection="telescopes" - ) == ["LSTN-design", "LSTN-01"] - - assert db_array_elements.get_array_element_list_for_db_query( - "MSTS-10", db=db, model_version=model_version, collection="telescopes" - ) == ["MSTS-design", "MSTS-10"] - - assert db_array_elements.get_array_element_list_for_db_query( - "MSTS-301", db=db, model_version=model_version, collection="telescopes" - ) == ["MSTN-design", "MSTS-301"] - - assert db_array_elements.get_array_element_list_for_db_query( - "MSTS-design", db=db, model_version=model_version, collection="telescopes" - ) == ["MSTS-design"] - - with pytest.raises(ValueError, match=r"^Array element MSTS-301 not found in DB."): - db_array_elements.get_array_element_list_for_db_query( - "MSTS-301", db=db, model_version=model_version, collection="calibration_devices" - ) - - assert db_array_elements.get_array_element_list_for_db_query( - "LSTN-02", db=db, model_version=model_version, collection="configuration_sim_telarray" - ) == ["LSTN-design"] - - def test_get_array_elements_of_type(db, model_version): available_telescopes = db_array_elements.get_array_elements_of_type( array_element_type="LSTN", db=db, model_version=model_version, collection="telescopes" From 0fd6713faee50408a6737b6003b43bf214a209c9 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 14 Jan 2025 17:06:32 +0100 Subject: [PATCH 016/124] read parameters --- src/simtools/db/db_handler.py | 139 +++++++++++++++++----------------- 1 file changed, 70 insertions(+), 69 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index 481b794bfd..bc1282e1ad 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -72,6 +72,7 @@ class DatabaseHandler: ALLOWED_FILE_EXTENSIONS = [".dat", ".txt", ".lis", ".cfg", ".yml", ".yaml", ".ecsv"] db_client = None + production_table_cached = {} site_parameters_cached = {} model_parameters_cached = {} model_versions_cached = {} @@ -185,7 +186,6 @@ def get_model_parameters( array_element_name, model_version, collection="telescope", - only_applicable=False, ): """ Get parameters from MongoDB or simulation model repository for an array element. @@ -211,33 +211,39 @@ def get_model_parameters( Returns ------- dict containing the parameters - - TODO - update with model_version and parameter_version """ - _site, _array_element_name = self._validate_model_input(site, array_element_name) - _model_version = None - array_element_list = db_array_elements.get_array_element_list_for_db_query( - _array_element_name, self, _model_version, collection - ) + _site, _ = self._validate_model_input(site, array_element_name) + _production_table = self.get_production_table_from_mongo_db(collection, model_version) + if "-design" in array_element_name: + array_element_list = [array_element_name] + else: + array_element_list = [ + names.get_array_element_type_from_name(array_element_name) + "-design", + array_element_name, + ] pars = {} - for array_element in array_element_list: - _array_elements_cache_key = self._parameter_cache_key( + for array_element in array_element_list: # design model must be read first + _array_elements_cache_key = self._cache_key( _site, array_element, model_version, collection ) try: pars.update(DatabaseHandler.model_parameters_cached[_array_elements_cache_key]) except KeyError: + pass + try: pars.update( self.read_mongo_db( - self.mongo_db_config.get("db_simulation_model", None), + db_name=self.mongo_db_config.get("db_simulation_model", None), + parameter_version_table=_production_table["parameters"][array_element], array_element_name=array_element, - model_version=_model_version, collection_name=collection, run_location=None, write_files=False, - only_applicable=only_applicable, ) ) + except KeyError as exc: + self._logger.error(f"Could not read parameters for {array_element}: {exc}") + raise exc DatabaseHandler.model_parameters_cached[_array_elements_cache_key] = pars return pars @@ -327,12 +333,11 @@ def _is_file(value): def read_mongo_db( self, db_name, + parameter_version_table, array_element_name, - model_version, - run_location, collection_name, + run_location=None, write_files=True, - only_applicable=False, ): """ Build and execute query to Read the MongoDB for a specific array element. @@ -343,18 +348,16 @@ def read_mongo_db( ---------- db_name: str the name of the DB + parameter_version_table: dict + Dict with parameter names vs parameter versions. array_element_name: str Name of the array element model (e.g. MSTN-design ...) - model_version: str - Version of the model. run_location: Path or str The sim_telarray run location to write the tabulated data files into. collection_name: str The name of the collection to read from. write_files: bool If true, write the files to the run_location. - only_applicable: bool - If True, only applicable parameters will be read. Returns ------- @@ -363,21 +366,17 @@ def read_mongo_db( Raises ------ ValueError - if query returned zero results. - - TODO - model_version vs parameter_version - + if query returned no results or if the collection is not found in the production table. """ collection = self.get_collection(db_name, collection_name) _parameters = {} - query = { "instrument": array_element_name, - "version": self.model_version(model_version, db_name), + "$or": [ + {"parameter": param, "parameter_version": version} + for param, version in parameter_version_table.items() + ], } - - if only_applicable: - query["applicable"] = True if collection.count_documents(query) < 1: raise ValueError( "The following query returned zero results! Check the input data and rerun.\n", @@ -387,7 +386,6 @@ def read_mongo_db( par_now = post["parameter"] _parameters[par_now] = post _parameters[par_now].pop("parameter", None) - _parameters[par_now].pop("instrument", None) _parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time if _parameters[par_now]["file"] and write_files: file = self._get_file_mongo_db(db_name, _parameters[par_now]["value"]) @@ -395,13 +393,9 @@ def read_mongo_db( return _parameters - def get_site_parameters( - self, - site, - model_version, - ): + def get_site_parameters(self, site, model_version): """ - Get parameters from either MongoDB or simulation model repository for a specific site. + Get site parameters from MongoDB. Parameters ---------- @@ -413,15 +407,11 @@ def get_site_parameters( Returns ------- dict containing the parameters - - TODO - model_version vs parameter_version """ _site, _ = self._validate_model_input(site, None) _production_table = self.get_production_table_from_mongo_db("sites", model_version) _db_name = self._get_db_name() - _site_cache_key = self._parameter_cache_key( - site, None, _production_table.get("model_version") - ) + _site_cache_key = self._cache_key(site, None, _production_table.get("model_version")) try: return DatabaseHandler.site_parameters_cached[_site_cache_key] except KeyError: @@ -496,12 +486,15 @@ def get_production_table_from_mongo_db(self, collection_name, model_version): model_version: str Version of the model. """ - collection = self.get_collection(self._get_db_name(), "production_tables") + try: + return DatabaseHandler.production_table_cached[ + self._cache_key(None, None, model_version, collection_name) + ] + except KeyError: + pass - query = { - "model_version": model_version, - "collection": collection_name, - } + query = {"model_version": model_version, "collection": collection_name} + collection = self.get_collection(self._get_db_name(), "production_tables") post = collection.find_one(query, sort=[("_id", DESCENDING)]) try: return { @@ -571,8 +564,6 @@ def get_simulation_configuration_parameters( ------ ValueError if simulation_software is not valid. - - TODO - model_version vs parameter_version """ if simulation_software == "corsika": return self.get_corsika_configuration_parameters(model_version) @@ -597,19 +588,22 @@ def get_corsika_configuration_parameters(self, model_version): ------- dict Configuration parameters for CORSIKA - - TODO - model_version vs parameter_version """ - _corsika_cache_key = self._parameter_cache_key(None, None, model_version) + _production_table = self.get_production_table_from_mongo_db( + "configuration_corsika", model_version + ) + _corsika_cache_key = self._cache_key(None, None, _production_table.get("model_version")) + try: return DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key] except KeyError: pass + DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key] = ( self.read_mongo_db( db_name=self._get_db_name(), + parameter_version_table=_production_table["parameters"]["configuration_corsika"], array_element_name=None, - model_version=model_version, run_location=None, collection_name="configuration_corsika", write_files=False, @@ -625,6 +619,8 @@ def _validate_model_input(self, site, array_element_name): Site name. array_element_name: str Name of the array element model (e.g. LSTN-01, MSTS-design) + + TODO - really needed?? """ return ( names.validate_site_name(site), @@ -716,9 +712,6 @@ def copy_array_element( Raises ------ BulkWriteError - - TODO - copy uses 'version' - can this function be removed? - """ db_name = self._get_db_name(db_name) if db_to_copy_to is None: @@ -768,7 +761,9 @@ def add_production_table(self, db_name, production_table): collection = self.get_collection(db_name, "production_tables") self._logger.info(f"Adding production for {production_table.get('collection')} to to DB") collection.insert_one(production_table) - # TODO - reset some cache for production tables? + self._reset_production_table_cache( + production_table.get("collection"), production_table.get("model_version") + ) def add_new_parameter( self, @@ -1035,7 +1030,7 @@ def get_all_versions( return DatabaseHandler.model_versions_cached[_cache_key] - def _parameter_cache_key(self, site, array_element_name, model_version, collection=None): + def _cache_key(self, site=None, array_element_name=None, model_version=None, collection=None): """ Create a cache key for the parameter cache dictionaries. @@ -1054,18 +1049,25 @@ def _parameter_cache_key(self, site, array_element_name, model_version, collecti ------- str Cache key. + """ + return "-".join( + part for part in [model_version, collection, site, array_element_name] if part + ) + + def _reset_production_table_cache(self, collection_name, model_version): + """ + Reset the cache for the production tables. - # TODO - understand if model_version is correct here. + Parameters + ---------- + collection_name: str + Collection name. + model_version: str + Model version. """ - parts = [] - if site: - parts.append(site) - if array_element_name: - parts.append(array_element_name) - parts.append(model_version) - if collection: - parts.append(collection) - return "-".join(parts) + DatabaseHandler.production_table_cached.pop( + self._cache_key(model_version=model_version, collection=collection_name), None + ) def _reset_parameter_cache(self, site, array_element_name, model_version): """ @@ -1080,10 +1082,9 @@ def _reset_parameter_cache(self, site, array_element_name, model_version): model_version: str Model version. - # TODO - understand if model_version is correct here. """ self._logger.debug(f"Resetting cache for {site} {array_element_name} {model_version}") - _cache_key = self._parameter_cache_key(site, array_element_name, model_version) + _cache_key = self._cache_key(site, array_element_name, model_version) DatabaseHandler.site_parameters_cached.pop(_cache_key, None) DatabaseHandler.model_parameters_cached.pop(_cache_key, None) db_array_elements.get_array_elements.cache_clear() From b62d9bf14e47f569d341c595f28cdba68f455b11 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 14 Jan 2025 21:10:37 +0100 Subject: [PATCH 017/124] correct handling of configuration production tables --- src/simtools/db/db_model_upload.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/simtools/db/db_model_upload.py b/src/simtools/db/db_model_upload.py index d6e272eebb..b3270d8e1d 100644 --- a/src/simtools/db/db_model_upload.py +++ b/src/simtools/db/db_model_upload.py @@ -115,9 +115,12 @@ def add_production_tables_to_db(args_dict, db): parameter_dict = gen.collect_data_from_file(file_name=file) logger.info(f"Reading production table for {array_element} (collection {collection})") try: - model_dict[collection]["parameters"][array_element] = parameter_dict["parameters"][ - array_element - ] + if array_element in ("configuration_corsika", "configuration_sim_telarray"): + model_dict[collection]["parameters"] = parameter_dict["parameters"] + else: + model_dict[collection]["parameters"][array_element] = parameter_dict[ + "parameters" + ][array_element] except KeyError as exc: logger.error(f"KeyError: {exc}") raise From 7fcce3ff4447b53f94b8bc2910bf2741d08b8706 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 14 Jan 2025 22:04:58 +0100 Subject: [PATCH 018/124] simplifications --- src/simtools/db/db_handler.py | 332 ++++++++------------------ src/simtools/model/model_parameter.py | 2 +- 2 files changed, 94 insertions(+), 240 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index bc1282e1ad..9e4e20b893 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -150,12 +150,13 @@ def _find_latest_simulation_model_db(self): """ try: - if not self.mongo_db_config["db_simulation_model"].endswith("LATEST"): + db_simulation_model = self.mongo_db_config["db_simulation_model"] + if not db_simulation_model.endswith("LATEST"): return - except TypeError: + except TypeError: # if db_simulation_model is None return - prefix = self.mongo_db_config["db_simulation_model"].replace("LATEST", "") + prefix = db_simulation_model.replace("LATEST", "") list_of_db_names = self.db_client.list_database_names() filtered_list_of_db_names = [s for s in list_of_db_names if s.startswith(prefix)] versioned_strings = [] @@ -180,71 +181,60 @@ def _find_latest_simulation_model_db(self): else: raise ValueError("Found LATEST in the DB name but no matching versions found in DB.") - def get_model_parameters( - self, - site, - array_element_name, - model_version, - collection="telescope", - ): + def get_model_parameters(self, site, array_element_name, model_version, collection): """ - Get parameters from MongoDB or simulation model repository for an array element. + Get model parameters from MongoDB. An array element can be e.g., a telescope or a calibration device. - Read parameters for design and for the specified array element (if necessary). This allows - to overwrite design parameters with specific parameters without having to copy - all model parameters when changing only a few. + Always queries parameters for design and for the specified array element (if necessary). Parameters ---------- site: str Site name. array_element_name: str - Name of the array element model (e.g. LSTN-01, MSTS-design) + Name of the array element model (e.g. LSTN-01, MSTS-design). model_version: str Version of the model. collection: str - collection of array element (e.g. telescopes, calibration_devices) - only_applicable: bool - If True, only applicable parameters will be read. + collection of array element (e.g. telescopes, calibration_devices). Returns ------- dict containing the parameters """ - _site, _ = self._validate_model_input(site, array_element_name) - _production_table = self.get_production_table_from_mongo_db(collection, model_version) - if "-design" in array_element_name: - array_element_list = [array_element_name] - else: - array_element_list = [ - names.get_array_element_type_from_name(array_element_name) + "-design", - array_element_name, - ] + production_table = self.get_production_table_from_mongo_db(collection, model_version) + design_model = f"{names.get_array_element_type_from_name(array_element_name)}-design" + array_element_list = ( + [array_element_name] + if "-design" in array_element_name + else [design_model, array_element_name] + ) + pars = {} for array_element in array_element_list: # design model must be read first - _array_elements_cache_key = self._cache_key( - _site, array_element, model_version, collection + cache_key = self._cache_key( + names.validate_site_name(site), array_element, model_version, collection ) + pars.update(DatabaseHandler.model_parameters_cached.get(cache_key, {})) try: - pars.update(DatabaseHandler.model_parameters_cached[_array_elements_cache_key]) - except KeyError: - pass - try: - pars.update( - self.read_mongo_db( - db_name=self.mongo_db_config.get("db_simulation_model", None), - parameter_version_table=_production_table["parameters"][array_element], - array_element_name=array_element, - collection_name=collection, - run_location=None, - write_files=False, - ) - ) + parameter_version_table = production_table["parameters"][array_element] except KeyError as exc: - self._logger.error(f"Could not read parameters for {array_element}: {exc}") - raise exc - DatabaseHandler.model_parameters_cached[_array_elements_cache_key] = pars + if array_element == design_model: + self._logger.error(f"Parameters for {array_element} could not be found.") + raise exc + # non-design model not defined (e.g. in collection 'configuration_sim_telarray') + continue + pars.update( + self.read_mongo_db( + db_name=self.mongo_db_config.get("db_simulation_model", None), + parameter_version_table=parameter_version_table, + array_element_name=array_element, + collection_name=collection, + write_files=False, + ) + ) + DatabaseHandler.model_parameters_cached[cache_key] = pars return pars @@ -369,7 +359,6 @@ def read_mongo_db( if query returned no results or if the collection is not found in the production table. """ collection = self.get_collection(db_name, collection_name) - _parameters = {} query = { "instrument": array_element_name, "$or": [ @@ -377,21 +366,23 @@ def read_mongo_db( for param, version in parameter_version_table.items() ], } - if collection.count_documents(query) < 1: + posts = list(collection.find(query).sort("parameter", ASCENDING)) + if not posts: raise ValueError( "The following query returned zero results! Check the input data and rerun.\n", query, ) - for post in collection.find(query).sort("parameter", ASCENDING): + parameters = {} + for post in posts: par_now = post["parameter"] - _parameters[par_now] = post - _parameters[par_now].pop("parameter", None) - _parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time - if _parameters[par_now]["file"] and write_files: - file = self._get_file_mongo_db(db_name, _parameters[par_now]["value"]) + parameters[par_now] = post + parameters[par_now].pop("parameter", None) + parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time + if parameters[par_now]["file"] and write_files: + file = self._get_file_mongo_db(db_name, parameters[par_now]["value"]) self._write_file_from_mongo_to_disk(db_name, run_location, file) - return _parameters + return parameters def get_site_parameters(self, site, model_version): """ @@ -408,23 +399,19 @@ def get_site_parameters(self, site, model_version): ------- dict containing the parameters """ - _site, _ = self._validate_model_input(site, None) - _production_table = self.get_production_table_from_mongo_db("sites", model_version) - _db_name = self._get_db_name() - _site_cache_key = self._cache_key(site, None, _production_table.get("model_version")) + site = names.validate_site_name(site) + production_table = self.get_production_table_from_mongo_db("sites", model_version) + cache_key = self._cache_key(site, None, production_table.get("model_version")) try: - return DatabaseHandler.site_parameters_cached[_site_cache_key] + return DatabaseHandler.site_parameters_cached[cache_key] except KeyError: pass - DatabaseHandler.site_parameters_cached[_site_cache_key] = ( - self._get_site_parameters_mongo_db( - _db_name, - _site, - _production_table, - ) + db_name = self._get_db_name() + DatabaseHandler.site_parameters_cached[cache_key] = self._get_site_parameters_mongo_db( + db_name, site, production_table ) - return DatabaseHandler.site_parameters_cached[_site_cache_key] + return DatabaseHandler.site_parameters_cached[cache_key] def _get_site_parameters_mongo_db(self, db_name, site, production_table): """ @@ -460,13 +447,14 @@ def _get_site_parameters_mongo_db(self, db_name, site, production_table): for param, version in parameter_query.items() ], } - if collection.count_documents(query) < 1: + posts = list(collection.find(query).sort("parameter", ASCENDING)) + if not posts: raise ValueError( "The following query returned zero results! Check the input data and rerun.\n", query, ) _parameters = {} - for post in collection.find(query).sort("parameter", ASCENDING): + for post in posts: par_now = post["parameter"] _parameters[par_now] = post _parameters[par_now].pop("parameter", None) @@ -496,27 +484,22 @@ def get_production_table_from_mongo_db(self, collection_name, model_version): query = {"model_version": model_version, "collection": collection_name} collection = self.get_collection(self._get_db_name(), "production_tables") post = collection.find_one(query, sort=[("_id", DESCENDING)]) - try: - return { - "collection": post["collection"], - "model_version": post["model_version"], - "parameters": post["parameters"], - "entry_date": ObjectId(post["_id"]).generation_time, - } - except TypeError as exc: - raise ValueError( - "The following query returned zero results:\n", - query, - ) from exc + if not post: + raise ValueError(f"The following query returned zero results: {query}") + + return { + "collection": post["collection"], + "model_version": post["model_version"], + "parameters": post["parameters"], + "entry_date": ObjectId(post["_id"]).generation_time, + } - def get_derived_values(self, site, array_element_name, model_version): + def get_derived_values(self, array_element_name, model_version): """ Get all derived values from the DB for a specific array element. Parameters ---------- - site: str - Site name. array_element_name: str Name of the array element model (e.g. MSTN, SSTS). model_version: str @@ -527,13 +510,14 @@ def get_derived_values(self, site, array_element_name, model_version): dict containing the parameters """ - _, _array_element_name = self._validate_model_input(site, array_element_name) - _model_version = model_version + array_element_name = ( + names.validate_array_element_name(array_element_name) if array_element_name else None + ) return self.read_mongo_db( DatabaseHandler.DB_DERIVED_VALUES, - _array_element_name, - _model_version, + array_element_name, + model_version, run_location=None, collection_name="derived_values", write_files=False, @@ -568,11 +552,13 @@ def get_simulation_configuration_parameters( if simulation_software == "corsika": return self.get_corsika_configuration_parameters(model_version) if simulation_software == "simtel": - if site and array_element_name: - return self.get_model_parameters( + return ( + self.get_model_parameters( site, array_element_name, model_version, collection="configuration_sim_telarray" ) - return {} + if site and array_element_name + else {} + ) raise ValueError(f"Unknown simulation software: {simulation_software}") def get_corsika_configuration_parameters(self, model_version): @@ -592,40 +578,21 @@ def get_corsika_configuration_parameters(self, model_version): _production_table = self.get_production_table_from_mongo_db( "configuration_corsika", model_version ) - _corsika_cache_key = self._cache_key(None, None, _production_table.get("model_version")) + cache_key = self._cache_key(None, None, _production_table.get("model_version")) try: - return DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key] + return DatabaseHandler.corsika_configuration_parameters_cached[cache_key] except KeyError: pass - DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key] = ( - self.read_mongo_db( - db_name=self._get_db_name(), - parameter_version_table=_production_table["parameters"]["configuration_corsika"], - array_element_name=None, - run_location=None, - collection_name="configuration_corsika", - write_files=False, - ) - ) - return DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key] - - def _validate_model_input(self, site, array_element_name): - """ - Validate input for model parameter queries. - - site: str - Site name. - array_element_name: str - Name of the array element model (e.g. LSTN-01, MSTS-design) - - TODO - really needed?? - """ - return ( - names.validate_site_name(site), - names.validate_array_element_name(array_element_name) if array_element_name else None, + DatabaseHandler.corsika_configuration_parameters_cached[cache_key] = self.read_mongo_db( + db_name=self._get_db_name(), + parameter_version_table=_production_table["parameters"], + array_element_name=None, + collection_name="configuration_corsika", + write_files=False, ) + return DatabaseHandler.corsika_configuration_parameters_cached[cache_key] @staticmethod def _get_file_mongo_db(db_name, file_name): @@ -728,11 +695,9 @@ def copy_array_element( collection = self.get_collection(db_name, collection_name) db_entries = [] - _version_to_copy = self.model_version(version_to_copy) - query = { "instrument": element_to_copy, - "version": _version_to_copy, + "version": version_to_copy, } for post in collection.find(query): post["instrument"] = new_array_element_name @@ -879,44 +844,6 @@ def _get_db_name(self, db_name=None): """ return self.mongo_db_config["db_simulation_model"] if db_name is None else db_name - def model_version(self, version, db_name=None): - """ - Return model version and check that it is valid. - - Queries the database for all available model versions and check if the - requested version is valid. - - Parameters - ---------- - version : str - Model version. - db_name : str - Database name. - - Returns - ------- - str - Model version. - - Raises - ------ - ValueError - if version not valid. - - # TODO - needs to be adjusted? - - """ - _all_versions = self.get_all_versions(db_name=db_name) - if version in _all_versions: - return version - if len(_all_versions) == 0: - return None - - raise ValueError( - f"Invalid model version {version} in DB {self._get_db_name(db_name)} " - f"(allowed are {_all_versions})" - ) - def insert_file_to_db(self, file_name, db_name=None, **kwargs): """ Insert a file to the DB. @@ -940,15 +867,11 @@ def insert_file_to_db(self, file_name, db_name=None, **kwargs): """ db_name = self._get_db_name(db_name) - db = DatabaseHandler.db_client[db_name] file_system = gridfs.GridFS(db) - if "content_type" not in kwargs: - kwargs["content_type"] = "ascii/dat" - - if "filename" not in kwargs: - kwargs["filename"] = Path(file_name).name + kwargs.setdefault("content_type", "ascii/dat") + kwargs.setdefault("filename", Path(file_name).name) if file_system.exists({"filename": kwargs["filename"]}): self._logger.warning( @@ -960,76 +883,6 @@ def insert_file_to_db(self, file_name, db_name=None, **kwargs): with open(file_name, "rb") as data_file: return file_system.put(data_file, **kwargs) - def get_all_versions( - self, - parameter=None, - array_element_name=None, - site=None, - db_name=None, - collection=None, - ): - """ - Get all version entries in the DB of collection and/or a specific parameter. - - Parameters - ---------- - parameter: str - Which parameter to get the versions of - array_element_name: str - Which array element to get the versions of (in case "collection_name" is not "sites") - site: str - Site name. - db_name: str - Database name. - collection_name: str - The name of the collection in which to update the parameter. - - Returns - ------- - all_versions: list - List of all versions found - - Raises - ------ - ValueError - If key to collection_name is not valid. - - TODO - this should be model or parameter versions? Need to be adjusted. - - """ - db_name = self._get_db_name() if db_name is None else db_name - if not db_name: - self._logger.warning("No database name defined to determine list of model versions") - return [] - _cache_key = f"model_versions_{db_name}-{collection}" - - query = {} - if parameter is not None: - query["parameter"] = parameter - _cache_key = f"{_cache_key}-{parameter}" - if collection in ["telescopes", "calibration_devices"] and array_element_name is not None: - query["instrument"] = names.validate_array_element_name(array_element_name) - _cache_key = f"{_cache_key}-{query['instrument']}" - elif collection == "sites" and site is not None: - query["site"] = names.validate_site_name(site) - _cache_key = f"{_cache_key}-{query['site']}" - - if _cache_key not in DatabaseHandler.model_versions_cached: - all_versions = set() - collections_to_query = ( - [collection] if collection else self.get_collections(db_name, True) - ) - for collection_name in collections_to_query: - db_collection = self.get_collection(db_name, collection_name) - sorted_posts = db_collection.find(query).sort("version", ASCENDING) - all_versions.update(post["version"] for post in sorted_posts) - DatabaseHandler.model_versions_cached[_cache_key] = list(all_versions) - - if len(DatabaseHandler.model_versions_cached[_cache_key]) == 0: - self._logger.warning(f"The query {query} did not return any results. No versions found") - - return DatabaseHandler.model_versions_cached[_cache_key] - def _cache_key(self, site=None, array_element_name=None, model_version=None, collection=None): """ Create a cache key for the parameter cache dictionaries. @@ -1106,15 +959,16 @@ def get_collections(self, db_name=None, model_collections_only=False): If True, only return model collections (i.e. exclude fs.files, fs.chunks, metadata) """ - db_name = self._get_db_name() if db_name is None else db_name + db_name = db_name or self._get_db_name() if db_name not in self.list_of_collections: self.list_of_collections[db_name] = DatabaseHandler.db_client[ db_name ].list_collection_names() + collections = self.list_of_collections[db_name] if model_collections_only: return [ collection - for collection in self.list_of_collections[db_name] + for collection in collections if not collection.startswith("fs.") and collection != "metadata" ] - return self.list_of_collections[db_name] + return collections diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index 78276ad8df..2ce68f78f5 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -336,7 +336,7 @@ def _load_parameters_from_db(self): if self.name is not None: self._parameters = self.db.get_model_parameters( - self.site, self.name, self.model_version, self.collection, only_applicable=True + self.site, self.name, self.model_version, self.collection ) if self.site is not None: From 877200f2b0764757f314fe918965e536501eec24 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 15 Jan 2025 11:11:18 +0100 Subject: [PATCH 019/124] remove db_array_elements --- .../applications/db_get_parameter_from_db.py | 24 ++++--- src/simtools/db/db_array_elements.py | 52 -------------- src/simtools/db/db_handler.py | 31 +++++++- src/simtools/model/array_model.py | 5 +- tests/unit_tests/db/test_db_array_elements.py | 71 ------------------- 5 files changed, 44 insertions(+), 139 deletions(-) delete mode 100644 src/simtools/db/db_array_elements.py delete mode 100644 tests/unit_tests/db/test_db_array_elements.py diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 4841d2f854..73f0c044ac 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -92,11 +92,13 @@ def main(): # noqa: D103 logger = logging.getLogger() logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) - # TODO read parameter_version + # TODO: Implement parameter version + if args_dict["parameter_version"] is not None: + raise NotImplementedError("Parameter version is not yet implemented.") db = db_handler.DatabaseHandler(mongo_db_config=db_config) - if args_dict["telescope"] is not None: + if args_dict["telescope"]: pars = db.get_model_parameters( site=args_dict["site"], array_element_name=args_dict["telescope"], @@ -113,19 +115,19 @@ def main(): # noqa: D103 pars = db.get_site_parameters( site=args_dict["site"], model_version=args_dict["model_version"] ) - if args_dict["parameter"] not in pars: + param = args_dict["parameter"] + if param not in pars: raise KeyError(f"The requested parameter, {args_dict['parameter']}, does not exist.") if args_dict["output_file"] is not None: - _io_handler = io_handler.IOHandler() - pars[args_dict["parameter"]].pop("_id") - pars[args_dict["parameter"]].pop("entry_date") - _output_file = Path(_io_handler.get_output_directory()) / args_dict["output_file"] + _output_file = ( + Path(io_handler.IOHandler().get_output_directory()) / args_dict["output_file"] + ) + pars[param].pop("_id") + pars[param].pop("entry_date") with open(_output_file, "w", encoding="utf-8") as json_file: - json.dump(pars[args_dict["parameter"]], json_file, indent=4) + json.dump(param, json_file, indent=4) else: - print() - pprint(pars[args_dict["parameter"]]) - print() + pprint(pars[param]) if __name__ == "__main__": diff --git a/src/simtools/db/db_array_elements.py b/src/simtools/db/db_array_elements.py deleted file mode 100644 index 7e70ea0a9a..0000000000 --- a/src/simtools/db/db_array_elements.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Retrieval of array elements from the database.""" - -from functools import lru_cache - -from pymongo import ASCENDING - - -@lru_cache -def get_array_elements(db_collection, model_version): - """ - Get all array element names and their design model for a given DB collection. - - Uses the 'design_model' parameter to determine the design model of the array element. - Assumes that a design model is defined for every array element. - - Parameters - ---------- - db_collection: - pymongo.collection.Collection - model_version: str - Model version. - - Returns - ------- - dict - Dict with array element names found and their design model - - Raises - ------ - ValueError - If query for collection name not implemented. - KeyError - If array element entry in the database is incomplete. - - """ - query = {"version": model_version} - results = db_collection.find(query, {"instrument": 1, "value": 1, "parameter": 1}).sort( - "instrument", ASCENDING - ) - - _all_available_array_elements = {} - for doc in results: - try: - if doc["parameter"] == "design_model": - _all_available_array_elements[doc["instrument"]] = doc["value"] - except KeyError as exc: - raise KeyError(f"Incomplete array element entry in the database: {doc}.") from exc - - if len(_all_available_array_elements) == 0: - raise ValueError(f"No array elements found in DB collection {db_collection}.") - - return _all_available_array_elements diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index e52b968323..9c50185044 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -14,7 +14,6 @@ from pymongo.errors import BulkWriteError from simtools.data_model import validate_data -from simtools.db import db_array_elements from simtools.io_operations import io_handler from simtools.utils import names, value_conversion @@ -496,6 +495,35 @@ def get_production_table_from_mongo_db(self, collection_name, model_version): "entry_date": ObjectId(post["_id"]).generation_time, } + def get_array_elements_of_type(self, array_element_type, model_version, collection): + """ + Get all array elements of a certain type (e.g. 'LSTN') from a collection in the DB. + + Parameters + ---------- + array_element_type: str + Type of the array element (e.g. LSTN, MSTS). + model_version: str + Version of the model. + collection: str + Which collection to get the array elements from: + i.e. telescopes, calibration_devices. + + Returns + ------- + list + Sorted list of all array element names found in collection + """ + production_table = self.get_production_table_from_mongo_db(collection, model_version) + all_array_elements = production_table["parameters"] + return sorted( + [ + entry + for entry in all_array_elements + if entry.startswith(array_element_type) and "-design" not in entry + ] + ) + def get_derived_values(self, array_element_name, model_version): """ Get all derived values from the DB for a specific array element. @@ -905,7 +933,6 @@ def _reset_parameter_cache(self, site, array_element_name, model_version): _cache_key = self._cache_key(site, array_element_name, model_version) DatabaseHandler.site_parameters_cached.pop(_cache_key, None) DatabaseHandler.model_parameters_cached.pop(_cache_key, None) - db_array_elements.get_array_elements.cache_clear() def get_collections(self, db_name=None, model_collections_only=False): """ diff --git a/src/simtools/model/array_model.py b/src/simtools/model/array_model.py index 672b4fa4a0..643d5d2648 100644 --- a/src/simtools/model/array_model.py +++ b/src/simtools/model/array_model.py @@ -7,7 +7,7 @@ from astropy.table import QTable from simtools.data_model import data_reader -from simtools.db import db_array_elements, db_handler +from simtools.db import db_handler from simtools.io_operations import io_handler from simtools.model.site_model import SiteModel from simtools.model.telescope_model import TelescopeModel @@ -367,9 +367,8 @@ def _get_all_array_elements_of_type(self, array_element_type: str) -> dict: dict Dict with array elements. """ - all_elements = db_array_elements.get_array_elements_of_type( + all_elements = self.db.get_array_elements_of_type( array_element_type=array_element_type, - db=self.db, model_version=self.model_version, collection="telescopes", ) diff --git a/tests/unit_tests/db/test_db_array_elements.py b/tests/unit_tests/db/test_db_array_elements.py deleted file mode 100644 index 5508a29294..0000000000 --- a/tests/unit_tests/db/test_db_array_elements.py +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/python3 - -import time - -import pytest - -from simtools.db import db_array_elements - - -def test_get_array_elements(db, model_version): - - time_1 = time.time() - db_array_elements.get_array_elements( - db.get_collection(db_name=None, collection_name="telescopes"), - db.model_version(model_version), - ) - time_2 = time.time() - available_telescopes = db_array_elements.get_array_elements( - db.get_collection(db_name=None, collection_name="telescopes"), - db.model_version(model_version), - ) - time_3 = time.time() - - # check that the second call is much faster than the first one - assert (time_2 - time_1) > 0.1 * (time_3 - time_2) - - expected_telescope_names = { - "LSTN-01": "LSTN-design", - "LSTN-02": "LSTN-design", - "LSTN-03": "LSTN-design", - "LSTN-04": "LSTN-design", - "MSTN-15": "MSTN-design", - "MSTS-10": "MSTS-design", - "MSTS-301": "MSTN-design", - } - for _t in expected_telescope_names: - assert _t in available_telescopes - assert expected_telescope_names[_t] in available_telescopes[_t] - - available_calibration_devices = db_array_elements.get_array_elements( - db.get_collection(db_name=None, collection_name="calibration_devices"), - db.model_version(model_version), - ) - expected_calibration_devices = { - "ILLN-01": "ILLN-design", - "ILLS-02": "ILLS-design", - } - for _d in expected_calibration_devices: - assert _d in available_calibration_devices - assert expected_calibration_devices[_d] in available_calibration_devices[_d] - - with pytest.raises(ValueError, match=r"^No array elements found in DB collection"): - db_array_elements.get_array_elements( - db.get_collection(db_name=None, collection_name="wrong_collection"), - db.model_version(model_version), - ) - - -def test_get_array_elements_of_type(db, model_version): - available_telescopes = db_array_elements.get_array_elements_of_type( - array_element_type="LSTN", db=db, model_version=model_version, collection="telescopes" - ) - assert available_telescopes == ["LSTN-01", "LSTN-02", "LSTN-03", "LSTN-04"] - - available_calibration_devices = db_array_elements.get_array_elements_of_type( - array_element_type="ILLS", - db=db, - model_version=model_version, - collection="calibration_devices", - ) - assert available_calibration_devices == ["ILLS-01", "ILLS-02", "ILLS-03", "ILLS-04"] From fd6b788355bf6e3d540dac3e8cffa28b0a0b8db1 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 15 Jan 2025 17:19:42 +0100 Subject: [PATCH 020/124] fix cachte reset --- src/simtools/db/db_handler.py | 14 ++++++++++---- src/simtools/db/db_model_upload.py | 10 +--------- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index 9c50185044..8f46404f91 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -819,7 +819,7 @@ def add_new_parameter( self._logger.info(f"Will also add the file {file_to_insert_now} to the DB") self.insert_file_to_db(file_to_insert_now, db_name) - self._reset_parameter_cache(par_dict["site"], par_dict["instrument"], par_dict["version"]) + self._reset_parameter_cache(par_dict["site"], par_dict["instrument"], None) def _get_db_name(self, db_name=None): """ @@ -919,6 +919,8 @@ def _reset_parameter_cache(self, site, array_element_name, model_version): """ Reset the cache for the parameters. + A value of 'None' for any of the parameters will reset the entire cache. + Parameters ---------- site: str @@ -930,9 +932,13 @@ def _reset_parameter_cache(self, site, array_element_name, model_version): """ self._logger.debug(f"Resetting cache for {site} {array_element_name} {model_version}") - _cache_key = self._cache_key(site, array_element_name, model_version) - DatabaseHandler.site_parameters_cached.pop(_cache_key, None) - DatabaseHandler.model_parameters_cached.pop(_cache_key, None) + if None in [site, array_element_name, model_version]: + DatabaseHandler.site_parameters_cached.clear() + DatabaseHandler.model_parameters_cached.clear() + else: + _cache_key = self._cache_key(site, array_element_name, model_version) + DatabaseHandler.site_parameters_cached.pop(_cache_key, None) + DatabaseHandler.model_parameters_cached.pop(_cache_key, None) def get_collections(self, db_name=None, model_collections_only=False): """ diff --git a/src/simtools/db/db_model_upload.py b/src/simtools/db/db_model_upload.py index b3270d8e1d..ce80e26d7a 100644 --- a/src/simtools/db/db_model_upload.py +++ b/src/simtools/db/db_model_upload.py @@ -33,16 +33,8 @@ def add_values_from_json_to_db(file, collection, db, db_name, file_prefix): ) db.add_new_parameter( db_name=db_name, - array_element_name=par_dict["instrument"], - parameter=par_dict["parameter"], - parameter_version=par_dict["parameter_version"], - value=par_dict["value"], - site=par_dict["site"], - type=par_dict["type"], + par_dict=par_dict, collection_name=collection, - applicable=par_dict["applicable"], - file=par_dict["file"], - unit=par_dict.get("unit", None), file_prefix=file_prefix, ) From 208313eca5ee5151c88862d4dd51a959e97847c0 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 15 Jan 2025 20:14:13 +0100 Subject: [PATCH 021/124] new file naming with versioning --- src/simtools/data_model/validate_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simtools/data_model/validate_data.py b/src/simtools/data_model/validate_data.py index c5a161f694..a414d979d6 100644 --- a/src/simtools/data_model/validate_data.py +++ b/src/simtools/data_model/validate_data.py @@ -108,7 +108,7 @@ def validate_data_file(self): def validate_parameter_and_file_name(self): """Validate that file name and key 'parameter_name' in data dict are the same.""" - if self.data_dict.get("parameter") != Path(self.data_file_name).stem: + if not str(Path(self.data_file_name).stem).startswith(self.data_dict.get("parameter")): raise ValueError( f"Parameter name in data dict {self.data_dict.get('parameter')} and " f"file name {Path(self.data_file_name).stem} do not match." From aa727473f57d2ae6eaa7dcd7c15be6802b3f6690 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 15 Jan 2025 20:30:13 +0100 Subject: [PATCH 022/124] bug fix in validating dict (e.g. array layouts) --- src/simtools/data_model/validate_data.py | 3 ++- src/simtools/db/db_model_upload.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/simtools/data_model/validate_data.py b/src/simtools/data_model/validate_data.py index a414d979d6..6ea2bdc08b 100644 --- a/src/simtools/data_model/validate_data.py +++ b/src/simtools/data_model/validate_data.py @@ -193,7 +193,8 @@ def _get_value_and_units_as_lists(self): ] try: return [ - v * c if not isinstance(v, bool) else v for v, c in zip(value, conversion_factor) + v * c if not isinstance(v, bool) and not isinstance(v, dict) else v + for v, c in zip(value, conversion_factor) ], target_unit except TypeError: return [None], target_unit diff --git a/src/simtools/db/db_model_upload.py b/src/simtools/db/db_model_upload.py index ce80e26d7a..83890d4bcf 100644 --- a/src/simtools/db/db_model_upload.py +++ b/src/simtools/db/db_model_upload.py @@ -29,8 +29,10 @@ def add_values_from_json_to_db(file, collection, db, db_name, file_prefix): par_dict = gen.collect_data_from_file(file_name=file) logger.info( f"Adding the following parameter to the DB: {par_dict['parameter']} " + f"version {par_dict['parameter_version']} " f"(collection {collection} in database {db_name})" ) + db.add_new_parameter( db_name=db_name, par_dict=par_dict, From 220696fcdb37583285034b037fefddbf4c95ca3c Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 08:33:08 +0100 Subject: [PATCH 023/124] fix printout --- src/simtools/applications/db_get_parameter_from_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 73f0c044ac..5d70506457 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -125,7 +125,7 @@ def main(): # noqa: D103 pars[param].pop("_id") pars[param].pop("entry_date") with open(_output_file, "w", encoding="utf-8") as json_file: - json.dump(param, json_file, indent=4) + json.dump(pars[param], json_file, indent=4) else: pprint(pars[param]) From f2261b7a90d0728ac1f93312b68eb332d5b07bb7 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 08:33:34 +0100 Subject: [PATCH 024/124] remove duplicated log statement --- src/simtools/corsika/corsika_config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/simtools/corsika/corsika_config.py b/src/simtools/corsika/corsika_config.py index 718b809a95..5e109d5b11 100644 --- a/src/simtools/corsika/corsika_config.py +++ b/src/simtools/corsika/corsika_config.py @@ -111,8 +111,6 @@ def fill_corsika_configuration(self, args_dict, db_config=None): if args_dict is None: return {} - self._logger.debug("Setting CORSIKA parameters ") - self._is_file_updated = False self.azimuth_angle = int(args_dict["azimuth_angle"].to("deg").value) self.zenith_angle = args_dict["zenith_angle"].to("deg").value From 8ec92adcb3315b437a0ffbf4d789019b39531119 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 08:34:14 +0100 Subject: [PATCH 025/124] ensure string format in database --- src/simtools/data_model/validate_data.py | 42 +++++++++++++++++++----- src/simtools/db/db_handler.py | 4 ++- 2 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/simtools/data_model/validate_data.py b/src/simtools/data_model/validate_data.py index 6ea2bdc08b..759b2ccdc4 100644 --- a/src/simtools/data_model/validate_data.py +++ b/src/simtools/data_model/validate_data.py @@ -57,7 +57,7 @@ def __init__( self.data_table = data_table self.check_exact_data_type = check_exact_data_type - def validate_and_transform(self, is_model_parameter=False): + def validate_and_transform(self, is_model_parameter=False, lists_as_strings=False): """ Validate data and data file. @@ -65,6 +65,8 @@ def validate_and_transform(self, is_model_parameter=False): ---------- is_model_parameter: bool This is a model parameter (add some data preparation) + lists_as_strings: bool + Convert lists to strings (as needed for model parameters) Returns ------- @@ -80,13 +82,9 @@ def validate_and_transform(self, is_model_parameter=False): if self.data_file_name: self.validate_data_file() if isinstance(self.data_dict, dict): - if is_model_parameter: - self._prepare_model_parameter() - self._validate_data_dict() - return self.data_dict + return self._validate_data_dict(is_model_parameter, lists_as_strings) if isinstance(self.data_table, Table): - self._validate_data_table() - return self.data_table + return self._validate_data_table() self._logger.error("No data or data table to validate") raise TypeError @@ -114,19 +112,29 @@ def validate_parameter_and_file_name(self): f"file name {Path(self.data_file_name).stem} do not match." ) - def _validate_data_dict(self): + def _validate_data_dict(self, is_model_parameter=False, lists_as_strings=False): """ Validate values in a dictionary. Handles different types of naming in data dicts (using 'name' or 'parameter' keys for name fields). + Parameters + ---------- + is_model_parameter: bool + This is a model parameter (add some data preparation) + lists_as_strings: bool + Convert lists to strings (as needed for model parameters) + Raises ------ KeyError if data dict does not contain a 'name' or 'parameter' key. """ + if is_model_parameter: + self._prepare_model_parameter() + if not (_name := self.data_dict.get("name") or self.data_dict.get("parameter")): raise KeyError("Data dict does not contain a 'name' or 'parameter' key.") self._data_description = self._read_validation_schema(self.schema_file_name, _name) @@ -145,6 +153,11 @@ def _validate_data_dict(self): self._check_version_string(self.data_dict.get("version")) + if lists_as_strings: + self._convert_results_to_model_format() + + return self.data_dict + def _validate_value_and_unit(self, value, unit, index): """ Validate value, unit, and perform type checking and conversions. @@ -234,6 +247,7 @@ def _validate_data_table(self): self._validate_data_columns() self._check_data_for_duplicates() self._sort_data() + return self.data_table def _validate_data_columns(self): """ @@ -779,6 +793,18 @@ def _prepare_model_parameter(self): if self.data_dict["unit"] is not None: self.data_dict["unit"] = gen.convert_string_to_list(self.data_dict["unit"]) + def _convert_results_to_model_format(self): + """ + Convert results to model format. + + Convert lists to strings (as needed for model parameters). + """ + value = self.data_dict["value"] + if isinstance(value, list): + self.data_dict["value"] = gen.convert_list_to_string(value) + if isinstance(self.data_dict["unit"], list): + self.data_dict["unit"] = gen.convert_list_to_string(self.data_dict["unit"]) + def _check_version_string(self, version): """ Check that version string follows semantic versioning. diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index 8f46404f91..1dd9bda2b3 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -790,7 +790,9 @@ def add_new_parameter( data_dict=par_dict, check_exact_data_type=False, ) - par_dict = data_validator.validate_and_transform(is_model_parameter=True) + par_dict = data_validator.validate_and_transform( + is_model_parameter=True, lists_as_strings=True + ) db_name = self._get_db_name(db_name) collection = self.get_collection(db_name, collection_name) From 24f4089aecabfc0e33c2440687cae44c6734541c Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 09:02:14 +0100 Subject: [PATCH 026/124] Lists as lists and not as strings in model parameters --- src/simtools/corsika/corsika_config.py | 5 ++--- src/simtools/db/db_handler.py | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/simtools/corsika/corsika_config.py b/src/simtools/corsika/corsika_config.py index 5e109d5b11..d6997c1ba9 100644 --- a/src/simtools/corsika/corsika_config.py +++ b/src/simtools/corsika/corsika_config.py @@ -6,7 +6,6 @@ import numpy as np from astropy import units as u -import simtools.utils.general as gen from simtools.corsika.primary_particle import PrimaryParticle from simtools.io_operations import io_handler from simtools.model.model_parameter import ModelParameter @@ -241,7 +240,7 @@ def _input_config_corsika_starting_grammage(self, entry): def _input_config_corsika_particle_kinetic_energy_cutoff(self, entry): """Return ECUTS parameter CORSIKA format.""" - e_cuts = gen.convert_string_to_list(entry["value"]) + e_cuts = entry["value"] return [ f"{e_cuts[0]*u.Unit(entry['unit']).to('GeV')} " f"{e_cuts[1]*u.Unit(entry['unit']).to('GeV')} " @@ -278,7 +277,7 @@ def _corsika_configuration_cherenkov_parameters(self, parameters_from_db): def _input_config_corsika_cherenkov_wavelength(self, entry): """Return CWAVLG parameter CORSIKA format.""" - wavelength_range = gen.convert_string_to_list(entry["value"]) + wavelength_range = entry["value"] return [ f"{wavelength_range[0]*u.Unit(entry['unit']).to('nm')}", f"{wavelength_range[1]*u.Unit(entry['unit']).to('nm')}", diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index 1dd9bda2b3..8f46404f91 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -790,9 +790,7 @@ def add_new_parameter( data_dict=par_dict, check_exact_data_type=False, ) - par_dict = data_validator.validate_and_transform( - is_model_parameter=True, lists_as_strings=True - ) + par_dict = data_validator.validate_and_transform(is_model_parameter=True) db_name = self._get_db_name(db_name) collection = self.get_collection(db_name, collection_name) From d892a95ddb2b239b660f75dd2d20a50bf6de768d Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 09:07:46 +0100 Subject: [PATCH 027/124] new model parameter --- .../resources/array_element_position_ground.json | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/resources/array_element_position_ground.json b/tests/resources/array_element_position_ground.json index 6636ab0eeb..cda15316d1 100644 --- a/tests/resources/array_element_position_ground.json +++ b/tests/resources/array_element_position_ground.json @@ -1,9 +1,16 @@ { - "version": "6.0.0", + "schema_version": "0.1.0", + "instrument": "MSTN-09", "site": "North", - "value": "221.68 -355.66 49.0", + "parameter_version": "2.0.0", + "unique_id": null, + "value": [ + 221.68, + -355.66, + 49.0 + ], "unit": "m", "type": "float64", - "file": false, - "applicable": true + "applicable": true, + "file": false } From 4aa2cd16022cf4143060d94d5a3f5fdd69a2745c Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 10:03:50 +0100 Subject: [PATCH 028/124] ensure comparisions lists exist --- src/simtools/testing/validate_output.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/simtools/testing/validate_output.py b/src/simtools/testing/validate_output.py index c8dcfd5d44..6d90453bf0 100644 --- a/src/simtools/testing/validate_output.py +++ b/src/simtools/testing/validate_output.py @@ -168,9 +168,13 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2): if data1 == data2: return True + value_list_1 = data1 if isinstance(data1, list) else None + value_list_2 = data2 if isinstance(data2, list) else None if "value" in data1 and isinstance(data1["value"], str): value_list_1 = gen.convert_string_to_list(data1.pop("value")) + if "value" in data2 and isinstance(data2["value"], str): value_list_2 = gen.convert_string_to_list(data2.pop("value")) + if value_list_1 is not None and value_list_2 is not None: return np.allclose(value_list_1, value_list_2, rtol=tolerance) return data1 == data2 From cbb3bf62fee12e9ce336cfaaee01f5b5e1498298 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 10:04:14 +0100 Subject: [PATCH 029/124] allow for missing elements in sim_telarray configuration --- src/simtools/db/db_handler.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index 8f46404f91..edd9277efa 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -182,7 +182,14 @@ def _find_latest_simulation_model_db(self): else: raise ValueError("Found LATEST in the DB name but no matching versions found in DB.") - def get_model_parameters(self, site, array_element_name, model_version, collection): + def get_model_parameters( + self, + site, + array_element_name, + model_version, + collection, + allow_missing_array_elements=False, + ): """ Get model parameters from MongoDB. @@ -199,6 +206,8 @@ def get_model_parameters(self, site, array_element_name, model_version, collecti Version of the model. collection: str collection of array element (e.g. telescopes, calibration_devices). + allow_missing_array_elements: bool + Allow missing array elements in the DB without raising an error. Returns ------- @@ -221,7 +230,7 @@ def get_model_parameters(self, site, array_element_name, model_version, collecti try: parameter_version_table = production_table["parameters"][array_element] except KeyError as exc: - if array_element == design_model: + if array_element == design_model and not allow_missing_array_elements: self._logger.error(f"Parameters for {array_element} could not be found.") raise exc # non-design model not defined (e.g. in collection 'configuration_sim_telarray') @@ -583,8 +592,13 @@ def get_simulation_configuration_parameters( return self.get_corsika_configuration_parameters(model_version) if simulation_software == "simtel": return ( + # not all array elements are present in the DB (allow for missing elements) self.get_model_parameters( - site, array_element_name, model_version, collection="configuration_sim_telarray" + site, + array_element_name, + model_version, + collection="configuration_sim_telarray", + allow_missing_array_elements=True, ) if site and array_element_name else {} From b8e2f55d41bf76aee36e75476525afdf405c5b26 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 10:04:50 +0100 Subject: [PATCH 030/124] log level --- .../config/db_get_array_layouts_from_db_list_arrays.yml | 1 + .../db_get_parameter_from_db_array_element_position_ground.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/tests/integration_tests/config/db_get_array_layouts_from_db_list_arrays.yml b/tests/integration_tests/config/db_get_array_layouts_from_db_list_arrays.yml index 31c031e9f3..67d5f64cbd 100644 --- a/tests/integration_tests/config/db_get_array_layouts_from_db_list_arrays.yml +++ b/tests/integration_tests/config/db_get_array_layouts_from_db_list_arrays.yml @@ -5,3 +5,4 @@ CTA_SIMPIPE: SITE: North MODEL_VERSION: "6.0.0" LIST_AVAILABLE_LAYOUTS: True + LOG_LEVEL: DEBUG diff --git a/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml b/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml index d2eda1e9e9..143dfe9e14 100644 --- a/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml +++ b/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml @@ -8,6 +8,7 @@ CTA_SIMPIPE: MODEL_VERSION: 6.0.0 OUTPUT_FILE: test.json OUTPUT_PATH: simtools-output + LOG_LEVEL: DEBUG INTEGRATION_TESTS: - REFERENCE_OUTPUT_FILE: | ./tests/resources/array_element_position_ground.json From 02272e8d06fafc5b6bee5bc74dfd6adadcf2d203 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 11:03:20 +0100 Subject: [PATCH 031/124] simplified model parameter validation --- src/simtools/data_model/validate_data.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/simtools/data_model/validate_data.py b/src/simtools/data_model/validate_data.py index 759b2ccdc4..64addc5fd6 100644 --- a/src/simtools/data_model/validate_data.py +++ b/src/simtools/data_model/validate_data.py @@ -3,6 +3,7 @@ import logging import os import re +from importlib.resources import files from pathlib import Path import jsonschema @@ -112,6 +113,29 @@ def validate_parameter_and_file_name(self): f"file name {Path(self.data_file_name).stem} do not match." ) + @staticmethod + def validate_model_parameter(par_dict): + """ + Validate a simulation model parameter (static method). + + Parameters + ---------- + par_dict: dict + Data dictionary + + Returns + ------- + dict + Validated data dictionary + """ + data_validator = DataValidator( + schema_file=files("simtools") + / f"schemas/model_parameters/{par_dict['parameter']}.schema.yml", + data_dict=par_dict, + check_exact_data_type=False, + ) + return data_validator.validate_and_transform(is_model_parameter=True) + def _validate_data_dict(self, is_model_parameter=False, lists_as_strings=False): """ Validate values in a dictionary. From d801acd0128d350880c1d6cae87d84a815c1ff61 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 11:03:37 +0100 Subject: [PATCH 032/124] correct version --- .../db_get_parameter_from_db_telescope_parameter_version.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml b/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml index 19c9a278a6..ea57845b0f 100644 --- a/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml +++ b/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml @@ -5,4 +5,4 @@ CTA_SIMPIPE: SITE: North TELESCOPE: LSTN-01 PARAMETER: mirror_list - PARAMETER_VERSION: 1.0.0 + PARAMETER_VERSION: 2.0.0 From d608740551960935a3036550aef2137cbdb77e8d Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 11:11:13 +0100 Subject: [PATCH 033/124] simplify data validation --- src/simtools/db/db_handler.py | 143 ++++++++++++++++------------------ 1 file changed, 68 insertions(+), 75 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index edd9277efa..cd14ebb03b 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -2,7 +2,6 @@ import logging import re -from importlib.resources import files from pathlib import Path from threading import Lock @@ -68,7 +67,6 @@ class DatabaseHandler: DB_CTA_SIMULATION_MODEL_DESCRIPTIONS = "CTA-Simulation-Model-Descriptions" # DB collection with updates field names DB_DERIVED_VALUES = "Staging-CTA-Simulation-Model-Derived-Values" - DB_PRODUCTION_TABLES = "production_tables" ALLOWED_FILE_EXTENSIONS = [".dat", ".txt", ".lis", ".cfg", ".yml", ".yaml", ".ecsv"] @@ -182,6 +180,49 @@ def _find_latest_simulation_model_db(self): else: raise ValueError("Found LATEST in the DB name but no matching versions found in DB.") + def get_model_parameter( + self, + parameter, + parameter_version, + site, + array_element_name, + collection, + ): + """ + Get a single model parameter (using the parameter version). + + Parameters + ---------- + parameter: str + Name of the parameter. + parameter_version: str + Version of the parameter. + site: str + Site name. + array_element_name: str + Name of the array element model (e.g. MSTN, SSTS). + collection: str + Collection of array element (e.g. telescopes, calibration_devices). + + Returns + ------- + dict containing the parameter + + """ + query = { + "parameter_version": parameter_version, + "parameter": parameter, + } + if array_element_name is not None: + query["instrument"] = array_element_name + if site is not None: + query["site"] = site + return self.read_mongo_db( + query=query, + collection_name=collection, + write_files=False, + ) + def get_model_parameters( self, site, @@ -237,9 +278,9 @@ def get_model_parameters( continue pars.update( self.read_mongo_db( - db_name=self.mongo_db_config.get("db_simulation_model", None), - parameter_version_table=parameter_version_table, - array_element_name=array_element, + query=self._get_query_from_parameter_version_table( + parameter_version_table, array_element + ), collection_name=collection, write_files=False, ) @@ -330,11 +371,19 @@ def _is_file(value): """Verify if a parameter value is a file name.""" return any(ext in str(value) for ext in DatabaseHandler.ALLOWED_FILE_EXTENSIONS) + def _get_query_from_parameter_version_table(self, parameter_version_table, array_element_name): + """Return query based on parameter version table.""" + return { + "instrument": array_element_name, + "$or": [ + {"parameter": param, "parameter_version": version} + for param, version in parameter_version_table.items() + ], + } + def read_mongo_db( self, - db_name, - parameter_version_table, - array_element_name, + query, collection_name, run_location=None, write_files=True, @@ -346,12 +395,8 @@ def read_mongo_db( Parameters ---------- - db_name: str - the name of the DB - parameter_version_table: dict - Dict with parameter names vs parameter versions. - array_element_name: str - Name of the array element model (e.g. MSTN-design ...) + query: dict + Dictionary describing the query to execute. run_location: Path or str The sim_telarray run location to write the tabulated data files into. collection_name: str @@ -368,14 +413,8 @@ def read_mongo_db( ValueError if query returned no results or if the collection is not found in the production table. """ + db_name = self._get_db_name() collection = self.get_collection(db_name, collection_name) - query = { - "instrument": array_element_name, - "$or": [ - {"parameter": param, "parameter_version": version} - for param, version in parameter_version_table.items() - ], - } posts = list(collection.find(query).sort("parameter", ASCENDING)) if not posts: raise ValueError( @@ -417,35 +456,6 @@ def get_site_parameters(self, site, model_version): except KeyError: pass - db_name = self._get_db_name() - DatabaseHandler.site_parameters_cached[cache_key] = self._get_site_parameters_mongo_db( - db_name, site, production_table - ) - return DatabaseHandler.site_parameters_cached[cache_key] - - def _get_site_parameters_mongo_db(self, db_name, site, production_table): - """ - Get parameters from MongoDB for a specific site. - - Parameters - ---------- - db_name: str - The name of the DB. - site: str - Site name. - production_table: dict - Table with parameter versions. - - Returns - ------- - dict containing the parameters - - Raises - ------ - ValueError - if query returned zero results. - """ - collection = self.get_collection(db_name, "sites") try: parameter_query = production_table["parameters"][f"OBS-{site}"] except KeyError as exc: @@ -457,21 +467,10 @@ def _get_site_parameters_mongo_db(self, db_name, site, production_table): for param, version in parameter_query.items() ], } - posts = list(collection.find(query).sort("parameter", ASCENDING)) - if not posts: - raise ValueError( - "The following query returned zero results! Check the input data and rerun.\n", - query, - ) - _parameters = {} - for post in posts: - par_now = post["parameter"] - _parameters[par_now] = post - _parameters[par_now].pop("parameter", None) - _parameters[par_now].pop("site", None) - _parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time - - return _parameters + DatabaseHandler.site_parameters_cached[cache_key] = self.read_mongo_db( + query=query, collection_name="sites", write_files=False + ) + return DatabaseHandler.site_parameters_cached[cache_key] def get_production_table_from_mongo_db(self, collection_name, model_version): """ @@ -630,9 +629,9 @@ def get_corsika_configuration_parameters(self, model_version): pass DatabaseHandler.corsika_configuration_parameters_cached[cache_key] = self.read_mongo_db( - db_name=self._get_db_name(), - parameter_version_table=_production_table["parameters"], - array_element_name=None, + query=self._get_query_from_parameter_version_table( + _production_table["parameters"], None + ), collection_name="configuration_corsika", write_files=False, ) @@ -798,13 +797,7 @@ def add_new_parameter( file_prefix: str or Path where to find files to upload to the DB """ - data_validator = validate_data.DataValidator( - schema_file=files("simtools") - / f"schemas/model_parameters/{par_dict['parameter']}.schema.yml", - data_dict=par_dict, - check_exact_data_type=False, - ) - par_dict = data_validator.validate_and_transform(is_model_parameter=True) + par_dict = validate_data.DataValidator.validate_model_parameter(par_dict) db_name = self._get_db_name(db_name) collection = self.get_collection(db_name, collection_name) From 99a99a004661900e65f021fe1162aca3457148bc Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 11:11:30 +0100 Subject: [PATCH 034/124] add get parameter using its version --- .../applications/db_get_parameter_from_db.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 5d70506457..8bdb747194 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -92,13 +92,18 @@ def main(): # noqa: D103 logger = logging.getLogger() logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) - # TODO: Implement parameter version - if args_dict["parameter_version"] is not None: - raise NotImplementedError("Parameter version is not yet implemented.") - db = db_handler.DatabaseHandler(mongo_db_config=db_config) - if args_dict["telescope"]: + if args_dict["parameter_version"] is not None: + pars = db.get_model_parameter( + parameter=args_dict["parameter"], + parameter_version=args_dict["parameter_version"], + site=args_dict["site"], + array_element_name=args_dict["telescope"], + collection=(args_dict["db_collection"] if args_dict["db_collection"] else "telescopes"), + ) + + elif args_dict["telescope"]: pars = db.get_model_parameters( site=args_dict["site"], array_element_name=args_dict["telescope"], From 2142ef8ac59f4bbf484789effa48f961552989e3 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 12:44:26 +0100 Subject: [PATCH 035/124] remove derived --- database_scripts/upload_dump_to_local_db.sh | 1 - docs/source/user-guide/databases.md | 11 +------ .../applications/db_get_file_from_db.py | 2 -- src/simtools/db/db_handler.py | 33 ------------------- src/simtools/model/model_parameter.py | 32 ------------------ tests/unit_tests/db/test_db_handler.py | 17 ---------- .../unit_tests/model/test_model_parameter.py | 16 --------- .../unit_tests/model/test_telescope_model.py | 2 -- tests/unit_tests/test_camera_efficiency.py | 1 - 9 files changed, 1 insertion(+), 114 deletions(-) diff --git a/database_scripts/upload_dump_to_local_db.sh b/database_scripts/upload_dump_to_local_db.sh index b13f372dce..c6941b9883 100755 --- a/database_scripts/upload_dump_to_local_db.sh +++ b/database_scripts/upload_dump_to_local_db.sh @@ -7,7 +7,6 @@ SIMTOOLS_NETWORK="simtools-mongo-network" CONTAINER_NAME="simtools-mongodb" SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-v0-3-0' # Name of the database to be created -SIMTOOLS_DB_SIMULATION_MODEL='Staging-CTA-Simulation-Model-Derived-Values' # Check if podman is available, if not use docker if command -v podman &> /dev/null; then diff --git a/docs/source/user-guide/databases.md b/docs/source/user-guide/databases.md index d19035e7a3..d286779db1 100644 --- a/docs/source/user-guide/databases.md +++ b/docs/source/user-guide/databases.md @@ -1,6 +1,6 @@ # Databases -The simtools package uses a prototype MongoDB database to store the simulation model parameters and derived data products. +The simtools package uses a prototype MongoDB database to store the simulation model parameters. Access to the DB is handled via a dedicated API module ([db_handler](#dbhandler)). Simulation model parameters are stored in databases (see the [Simulation Model](model_parameters.md#simulation-model) section) and synced with the [CTAO model parameter repository](https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters). @@ -27,15 +27,6 @@ Collections: * `metadata` containing tables describing the model versions * `fs.files` with all file type entries for the model parameters (e.g., the quantum-efficiency tables) -### Derived values DB - -Database with derived values DB (e.g., `Staging-CTA-Simulation-Model-Derived-Values` defined in `db_handler.DB_DERIVED_VALUES`). - -Collections are: - -* `derived_values` with the derived values for each telescope or site -* `fs.files` with file type derived results - ### Other databases All other currently available databases are not in use and kept for historical reasons. diff --git a/src/simtools/applications/db_get_file_from_db.py b/src/simtools/applications/db_get_file_from_db.py index f1796d6924..21dc75e371 100644 --- a/src/simtools/applications/db_get_file_from_db.py +++ b/src/simtools/applications/db_get_file_from_db.py @@ -66,8 +66,6 @@ def main(): # noqa: D103 db = db_handler.DatabaseHandler(mongo_db_config=db_config) available_dbs = [ db_config["db_simulation_model"], - db.DB_CTA_SIMULATION_MODEL_DESCRIPTIONS, - db.DB_DERIVED_VALUES, "sandbox", ] file_id = None diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index cd14ebb03b..d9bf4383d2 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -64,10 +64,6 @@ class DatabaseHandler: Dictionary with the MongoDB configuration (see jsonschema_db_dict for details). """ - DB_CTA_SIMULATION_MODEL_DESCRIPTIONS = "CTA-Simulation-Model-Descriptions" - # DB collection with updates field names - DB_DERIVED_VALUES = "Staging-CTA-Simulation-Model-Derived-Values" - ALLOWED_FILE_EXTENSIONS = [".dat", ".txt", ".lis", ".cfg", ".yml", ".yaml", ".ecsv"] db_client = None @@ -532,35 +528,6 @@ def get_array_elements_of_type(self, array_element_type, model_version, collecti ] ) - def get_derived_values(self, array_element_name, model_version): - """ - Get all derived values from the DB for a specific array element. - - Parameters - ---------- - array_element_name: str - Name of the array element model (e.g. MSTN, SSTS). - model_version: str - Version of the model. - - Returns - ------- - dict containing the parameters - - """ - array_element_name = ( - names.validate_array_element_name(array_element_name) if array_element_name else None - ) - - return self.read_mongo_db( - DatabaseHandler.DB_DERIVED_VALUES, - array_element_name, - model_version, - run_location=None, - collection_name="derived_values", - write_files=False, - ) - def get_simulation_configuration_parameters( self, simulation_software, site, array_element_name, model_version ): diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index 2ce68f78f5..ec1faf04dd 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -67,7 +67,6 @@ def __init__( self._parameters = {} self._simulation_config_parameters = {"corsika": {}, "simtel": {}} - self._derived = None self.collection = collection self.label = label self.model_version = model_version @@ -111,10 +110,6 @@ def _get_parameter_dict(self, par_name): """ try: return self._parameters[par_name] - except KeyError: - pass - try: - return self.derived[par_name] except (KeyError, ValueError) as e: msg = f"Parameter {par_name} was not found in the model" self._logger.error(msg) @@ -241,33 +236,6 @@ def get_parameter_file_flag(self, par_name): self._logger.debug(f"Parameter {par_name} does not have a file associated with it.") return False - @property - def derived(self): - """Load the derived values and export them if the class instance hasn't done it yet.""" - if self._derived is None: - self._load_derived_values() - self._export_derived_files() - return self._derived - - def _load_derived_values(self): - """Load derived values from the DB.""" - self._logger.debug("Reading derived values from DB") - self._derived = self.db.get_derived_values( - self.site, - self.name, - self.model_version, - ) - - def _export_derived_files(self): - """Write to disk a file from the derived values DB.""" - for par_now in self.derived.values(): - if par_now.get("File") or par_now.get("file"): - self.db.export_file_db( - db_name=self.db.DB_DERIVED_VALUES, - dest=self.config_file_directory, - file_name=(par_now.get("value") or par_now.get("Value")), - ) - def print_parameters(self): """Print parameters and their values for debugging purposes.""" for par in self._parameters: diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index 716704f7d5..d54e689bb3 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -139,23 +139,6 @@ def test_reading_db_sst(db, model_version): assert pars["camera_pixels"] == 2048 -@pytest.mark.xfail(reason="Test requires Derived-Values Database") -def test_get_derived_values(db, model_version_prod5): - logger.info("----Testing reading derived values-----") - try: - pars = db.get_derived_values("North", "LSTN-01", model_version_prod5) - assert ( - pars["ray_tracing"]["value"] - == "ray-tracing-North-LST-1-d10.0-za20.0_validate_optics.ecsv" - ) - except ValueError: - logger.error("Derived DB not updated for new telescope names. Expect failure") - raise AssertionError - - with pytest.raises(ValueError, match=r"^abc"): - pars = db.get_derived_values("North", None, model_version_prod5) - - def test_get_sim_telarray_configuration_parameters(db, model_version): _pars = db.get_model_parameters( diff --git a/tests/unit_tests/model/test_model_parameter.py b/tests/unit_tests/model/test_model_parameter.py index db981806d5..b48239138e 100644 --- a/tests/unit_tests/model/test_model_parameter.py +++ b/tests/unit_tests/model/test_model_parameter.py @@ -294,22 +294,6 @@ def test_updating_export_model_files(db_config, io_handler, model_version): assert False is tel._is_exported_model_files_up_to_date -@pytest.mark.xfail(reason="Test requires Derived-Values Database") -def test_export_derived_files(io_handler, db_config, model_version_prod5): - tel_model = TelescopeModel( - site="North", - telescope_name="LSTN-01", - model_version=model_version_prod5, - mongo_db_config=db_config, - label="test-telescope-model-lst", - ) - - _ = tel_model.derived - assert tel_model.config_file_directory.joinpath( - "ray-tracing-North-LST-1-d10.0-za20.0_validate_optics.ecsv" - ).exists() - - def test_export_parameter_file(telescope_model_lst, mocker): parameter = "array_coordinates_UTM" file_path = "tests/resources/telescope_positions-North-ground.ecsv" diff --git a/tests/unit_tests/model/test_telescope_model.py b/tests/unit_tests/model/test_telescope_model.py index 26e77a1618..066f9177a1 100644 --- a/tests/unit_tests/model/test_telescope_model.py +++ b/tests/unit_tests/model/test_telescope_model.py @@ -55,7 +55,6 @@ def test_read_incidence_angle_distribution(telescope_model_sst): def test_calc_average_curve(telescope_model_sst_prod5): tel_model = telescope_model_sst_prod5 tel_model.export_config_file() - _ = tel_model.derived two_dim_file = tel_model.get_parameter_value("camera_filter") two_dim_dist = tel_model.read_two_dim_wavelength_angle(two_dim_file) @@ -72,7 +71,6 @@ def test_calc_average_curve(telescope_model_sst_prod5): def test_export_table_to_model_directory(telescope_model_sst_prod5): tel_model = telescope_model_sst_prod5 tel_model.export_config_file() - _ = tel_model.derived two_dim_file = tel_model.get_parameter_value("camera_filter") two_dim_dist = tel_model.read_two_dim_wavelength_angle(two_dim_file) diff --git a/tests/unit_tests/test_camera_efficiency.py b/tests/unit_tests/test_camera_efficiency.py index 0e161ae5c5..c92b58bf40 100644 --- a/tests/unit_tests/test_camera_efficiency.py +++ b/tests/unit_tests/test_camera_efficiency.py @@ -144,7 +144,6 @@ def test_calc_reflectivity(camera_efficiency_lst, prepare_results_file): ) # Value for Prod5 LST-1 -@pytest.mark.xfail(reason="Missing ray_tracing for prod6 in Derived-DB") def test_calc_nsb_rate(camera_efficiency_lst, prepare_results_file): camera_efficiency_lst._read_results() camera_efficiency_lst.export_model_files() From a6aee682a324cc6ed576ff1e8cde94f6f3ae7c10 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 13:22:56 +0100 Subject: [PATCH 036/124] adjust model parameter validation to new file naming --- .../validate_file_using_schema.py | 25 ++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/simtools/applications/validate_file_using_schema.py b/src/simtools/applications/validate_file_using_schema.py index a587ef9546..a02217d3cc 100644 --- a/src/simtools/applications/validate_file_using_schema.py +++ b/src/simtools/applications/validate_file_using_schema.py @@ -33,6 +33,7 @@ """ import logging +import re from importlib.resources import files from pathlib import Path @@ -116,19 +117,24 @@ def _get_schema_file_name(args_dict, data_dict=None): def validate_schema(args_dict, logger): """ - Validate a schema file given in yaml or json format. + Validate a schema file (or several files) given in yaml or json format. Schema is either given as command line argument, read from the meta_schema_url or from the metadata section of the data dictionary. """ - try: - data = gen.collect_data_from_file(file_name=args_dict["file_name"]) - except FileNotFoundError as exc: - logger.error(f"Error reading schema file from {args_dict['file_name']}") - raise exc - metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data)) - logger.info(f"Successful validation of schema file {args_dict['file_name']}") + if args_dict.get("model_parameters_directory") is not None: + file_list = list(Path(args_dict["model_parameters_directory"]).rglob("*.json")) + else: + file_list = [args_dict["file_name"]] + for file_name in file_list: + try: + data = gen.collect_data_from_file(file_name=file_name) + except FileNotFoundError as exc: + logger.error(f"Error reading schema file from {file_name}") + raise exc + metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data)) + logger.info(f"Successful validation of schema file {file_name}") def validate_data_files(args_dict, logger): @@ -138,8 +144,9 @@ def validate_data_files(args_dict, logger): tmp_args_dict = {} for file_name in Path(model_parameters_directory).rglob("*.json"): tmp_args_dict["file_name"] = file_name + parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem) schema_file = ( - files("simtools") / "schemas/model_parameters" / f"{file_name.stem}.schema.yml" + files("simtools") / "schemas/model_parameters" / f"{parameter_name}.schema.yml" ) tmp_args_dict["schema"] = schema_file tmp_args_dict["data_type"] = "model_parameter" From e9bd2e52c15b3286689bc806b19ff0f05c6920a3 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 16:00:27 +0100 Subject: [PATCH 037/124] remove applicable field --- docs/source/user-guide/model_parameters.md | 5 +- .../applications/db_get_parameter_from_db.py | 13 --- src/simtools/data_model/model_data_writer.py | 58 +++--------- .../schemas/model_parameter.metaschema.yml | 92 ++++++++++++++++++- .../data_model/test_model_data_writer.py | 49 ++++------ tests/unit_tests/model/test_array_model.py | 5 +- 6 files changed, 126 insertions(+), 96 deletions(-) diff --git a/docs/source/user-guide/model_parameters.md b/docs/source/user-guide/model_parameters.md index 3dd4d6cbd8..65425be19b 100644 --- a/docs/source/user-guide/model_parameters.md +++ b/docs/source/user-guide/model_parameters.md @@ -189,14 +189,15 @@ A typical model parameter file looks like: ```json { + "schema_version": "0.1.0", "parameter": "num_gains", "instrument": "LSTN-01", "site": "North", - "version": "6.0.0", + "parameter_version": "6.0.0", + "unique_id": null, "value": 2, "unit": null, "type": "int64", - "applicable": true, "file": false } ``` diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 8bdb747194..e5f8a169a1 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -34,19 +34,6 @@ --site North --telescope LSTN-01 \\ --model_version 5.0.0 - Expected final print-out message: - - .. code-block:: console - - {'Applicable': True, - 'File': True, - 'Type': 'str', - 'Value': 'mirror_CTA-N-LST1_v2019-03-31.dat', - 'Version': '5.0.0', - '_id': ObjectId('608834f257df2db2531b8e78'), - 'entry_date': datetime.datetime(2021, 4, 27, 15, 59, 46, tzinfo=)} - """ import json diff --git a/src/simtools/data_model/model_data_writer.py b/src/simtools/data_model/model_data_writer.py index f29352b01e..507e348119 100644 --- a/src/simtools/data_model/model_data_writer.py +++ b/src/simtools/data_model/model_data_writer.py @@ -122,7 +122,7 @@ def dump_model_parameter( parameter_name, value, instrument, - model_version, + parameter_version, output_file, output_path=None, use_plain_output_path=False, @@ -139,8 +139,8 @@ def dump_model_parameter( Value of the parameter. instrument: str Name of the instrument. - model_version: str - Version of the model. + parameter_version: str + Version of the parameter. output_file: str Name of output file. output_path: str or Path @@ -163,7 +163,7 @@ def dump_model_parameter( use_plain_output_path=use_plain_output_path, ) _json_dict = writer.get_validated_parameter_dict( - parameter_name, value, instrument, model_version + parameter_name, value, instrument, parameter_version ) writer.write_dict_to_model_parameter_json(output_file, _json_dict) if metadata_input_dict is not None: @@ -175,7 +175,9 @@ def dump_model_parameter( ) return _json_dict - def get_validated_parameter_dict(self, parameter_name, value, instrument, model_version): + def get_validated_parameter_dict( + self, parameter_name, value, instrument, parameter_version, schema_version="0.1.0" + ): """ Get validated parameter dictionary. @@ -187,8 +189,10 @@ def get_validated_parameter_dict(self, parameter_name, value, instrument, model_ Value of the parameter. instrument: str Name of the instrument. - model_version: str - Version of the model. + parameter_version: str + Version of the parameter. + schema_version: str + Version of the schema. Returns ------- @@ -203,22 +207,18 @@ def get_validated_parameter_dict(self, parameter_name, value, instrument, model_ except ValueError: # e.g. instrument is 'LSTN-01' site = names.get_site_from_array_element_name(instrument) - try: - applicable = self._get_parameter_applicability(instrument) - except ValueError: - applicable = True # Default to True (expect that this field goes in future) - value, unit = value_conversion.split_value_and_unit(value) data_dict = { + "schema_version": schema_version, "parameter": parameter_name, "instrument": instrument, "site": site, - "version": model_version, + "parameter_version": parameter_version, + "unique_id": None, "value": value, "unit": unit, "type": self._get_parameter_type(), - "applicable": applicable, "file": self._parameter_is_a_file(), } return self.validate_and_transform( @@ -273,36 +273,6 @@ def _parameter_is_a_file(self): pass return False - def _get_parameter_applicability(self, telescope_name): - """ - Check if a parameter is applicable for a given telescope using schema files. - - First check for exact telescope name (e.g., LSTN-01), if not listed in the schema - use telescope type (LSTN). - - Parameters - ---------- - telescope_name: str - Telescope name (e.g., LSTN-01) - - Returns - ------- - bool - True if parameter is applicable to telescope. - - """ - try: - if telescope_name in self.schema_dict["instrument"]["type"]: - return True - except KeyError as exc: - self._logger.error("Schema file does not contain 'instrument:type' key.") - raise exc - - return ( - names.get_array_element_type_from_name(telescope_name) - in self.schema_dict["instrument"]["type"] - ) - def _get_unit_from_schema(self): """ Return unit(s) from schema dict. diff --git a/src/simtools/schemas/model_parameter.metaschema.yml b/src/simtools/schemas/model_parameter.metaschema.yml index 9428c876c3..ee8c1ac8d2 100644 --- a/src/simtools/schemas/model_parameter.metaschema.yml +++ b/src/simtools/schemas/model_parameter.metaschema.yml @@ -2,12 +2,99 @@ $schema: http://json-schema.org/draft-06/schema# $ref: '#/definitions/SimtoolsModelParameters' title: SimPipe DB Model Parameter Metaschema -description: YAML representation of db model parameter metaschema -version: 0.1.0 +description: | + YAML representation of db model parameter metaschema + (based on simulation model DB). +version: 0.2.0 name: modelparameter.metaschema type: object additionalProperties: false +definitions: + SimtoolsModelParameters: + description: "" + type: object + properties: + _id: + type: string + description: "DB unique identifier" + entry_date: + type: string + description: "Value entry data" + file: + type: boolean + description: "This parameter is a file." + instrument: + type: string + description: "Associated instrument." + site: + type: string + description: "Associated CTAO site." + enum: + - North + - South + type: + type: string + description: "Data type" + enum: + - boolean + - dict + - double + - file + - float64 + - int + - int64 + - string + - uint + - uint32 + - uint64 + unit: + anyOf: + - type: string + - type: "null" + description: "Unit of the parameter." + value: + anyOf: + - type: boolean + - type: number + - type: string + description: "Value of the parameter." + parameter_version: + anyOf: + - type: string + - type: "null" + description: "Parameter version." + schema_version: + anyOf: + - type: string + - type: "null" + description: "Metaschema version." + unique_id: + anyOf: + - type: string + - type: "null" + description: "Unique ID of parameter definition." + required: + - file + - instrument + - parameter_version + - schema_version + - site + - type + - unit + - value +... +--- +$schema: http://json-schema.org/draft-06/schema# +$ref: '#/definitions/SimtoolsModelParameters' +title: SimPipe DB Model Parameter Metaschema +description: | + YAML representation of db model parameter metaschema + (based on model_parameters DB). +version: 0.1.0 +name: modelparameter.metaschema +type: object +additionalProperties: false definitions: SimtoolsModelParameters: @@ -48,6 +135,7 @@ definitions: - boolean - double - int + - int64 - string - uint - file diff --git a/tests/unit_tests/data_model/test_model_data_writer.py b/tests/unit_tests/data_model/test_model_data_writer.py index 55e1090b02..0a89b3cd1a 100644 --- a/tests/unit_tests/data_model/test_model_data_writer.py +++ b/tests/unit_tests/data_model/test_model_data_writer.py @@ -208,14 +208,14 @@ def test_json_numpy_encoder(): def test_dump_model_parameter(tmp_test_directory): - model_version = "6.0.0" + parameter_version = "1.1.0" instrument = "LSTN-01" # single value, no unit num_gains_dict = writer.ModelDataWriter.dump_model_parameter( parameter_name="num_gains", value=2, instrument=instrument, - model_version=model_version, + model_version=parameter_version, output_file="num_gains.json", output_path=tmp_test_directory, use_plain_output_path=True, @@ -230,7 +230,7 @@ def test_dump_model_parameter(tmp_test_directory): parameter_name="array_element_position_utm", value=[217.6596 * u.km, 3184.9951 * u.km, 218500.0 * u.cm], instrument=instrument, - model_version=model_version, + model_version=parameter_version, output_file="array_element_position_utm.json", output_path=tmp_test_directory, use_plain_output_path=True, @@ -248,7 +248,7 @@ def test_dump_model_parameter(tmp_test_directory): parameter_name="focus_offset", value=[6.55 * u.cm, 0.0 * u.deg, 0.0, 0.0], instrument="LSTN-01", - model_version="6.0.0", + model_version=parameter_version, output_file="focus_offset.json", output_path=tmp_test_directory, use_plain_output_path=True, @@ -264,16 +264,17 @@ def test_get_validated_parameter_dict(): w1 = writer.ModelDataWriter() assert w1.get_validated_parameter_dict( - parameter_name="num_gains", value=2, instrument="MSTN-01", model_version="0.0.1" + parameter_name="num_gains", value=2, instrument="MSTN-01", parameter_version="0.0.1" ) == { + "schema_version": "0.1.0", "parameter": "num_gains", "instrument": "MSTN-01", "site": "North", - "version": "0.0.1", + "parameter_version": "0.0.1", + "unique_id": None, "value": 2, "unit": u.Unit(""), "type": "int", - "applicable": True, "file": False, } @@ -281,16 +282,17 @@ def test_get_validated_parameter_dict(): parameter_name="transit_time_error", value=5.0 * u.ns, instrument="LSTN-01", - model_version="0.0.1", + parameter_version="0.0.1", ) == { + "schema_version": "0.1.0", "parameter": "transit_time_error", "instrument": "LSTN-01", "site": "North", - "version": "0.0.1", + "parameter_version": "0.0.1", + "unique_id": None, "value": 5, "unit": u.Unit("ns"), "type": "double", - "applicable": True, "file": False, } @@ -298,40 +300,21 @@ def test_get_validated_parameter_dict(): parameter_name="reference_point_altitude", value=2.7 * u.km, instrument="North", - model_version="0.0.1", + parameter_version="0.0.1", ) == { + "schema_version": "0.1.0", "parameter": "reference_point_altitude", "instrument": "North", "site": "North", - "version": "0.0.1", + "parameter_version": "0.0.1", + "unique_id": None, "value": 2700.0, "unit": u.Unit("m"), "type": "double", - "applicable": True, "file": False, } -def test_get_parameter_applicability(num_gains_schema): - - w1 = writer.ModelDataWriter() - w1.schema_dict = num_gains_schema - - assert w1._get_parameter_applicability("LSTN-01") - - # illuminator does not have gains - assert not w1._get_parameter_applicability("ILLN-01") - - # change schema dict - w1.schema_dict["instrument"]["type"].append("LSTN-55") - assert w1._get_parameter_applicability("LSTN-55") - - # change schema dict - w1.schema_dict["instrument"].pop("type") - with pytest.raises(KeyError): - w1._get_parameter_applicability("LSTN-01") - - def test_prepare_data_dict_for_writing(): data_dict_1 = {} diff --git a/tests/unit_tests/model/test_array_model.py b/tests/unit_tests/model/test_array_model.py index 1c884c7c24..cd347f8cd9 100644 --- a/tests/unit_tests/model/test_array_model.py +++ b/tests/unit_tests/model/test_array_model.py @@ -110,14 +110,15 @@ def test_get_telescope_position_parameter(array_model, io_handler): assert am._get_telescope_position_parameter( "LSTN-01", "North", 10.0 * u.m, 200.0 * u.cm, 30.0 * u.m ) == { + "schema_version": "0.1.0", "parameter": "array_element_position_ground", "instrument": "LSTN-01", "site": "North", - "version": "6.0.0", + "parameter_version": "2.0.0", + "unique_id": None, "value": "10.0 2.0 30.0", "unit": "m", "type": "float64", - "applicable": True, "file": False, } From f1602a145f4fc14ee4771917d15ebdaad4ba2210 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 16 Jan 2025 17:11:58 +0100 Subject: [PATCH 038/124] parameter version --- .../convert_all_model_parameters_from_simtel.py | 9 +++++---- ...onvert_all_model_parameters_from_simtel_num_gains.yml | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/simtools/applications/convert_all_model_parameters_from_simtel.py b/src/simtools/applications/convert_all_model_parameters_from_simtel.py index c3a47f68e9..c910d54d78 100644 --- a/src/simtools/applications/convert_all_model_parameters_from_simtel.py +++ b/src/simtools/applications/convert_all_model_parameters_from_simtel.py @@ -6,6 +6,8 @@ ready to be submitted to the model database. Prints out parameters which are not found in simtel configuration file and parameters which are not found in simtools schema files. + Note that all parameter are assigned the same parameter version. + Command line arguments ---------------------- simtel_cfg_file (str) @@ -30,7 +32,7 @@ --simtel_cfg_file all_telescope_config_la_palma.cfg\\ --simtel_telescope_name CT1\\ --telescope LSTN-01\\ - --model_version "2024-03-06" + --parameter_version "0.0.1" The export of the model parameters from sim_telarray for 6.0.0 can be done e.g., as follows: @@ -103,7 +105,7 @@ def _parse(label=None, description=None): type=str, required=True, ) - return config.initialize(simulation_model="telescope") + return config.initialize(simulation_model=["telescope", "parameter_version"]) def get_list_of_parameters_and_schema_files(schema_directory): @@ -219,7 +221,6 @@ def read_and_export_parameters(args_dict, logger): """ Read and export parameters from simtel configuration file to json files. - Only applicable parameters are exported to json. Provide extensive logging information on the parameters found in the simtel configuration file. @@ -266,7 +267,7 @@ def read_and_export_parameters(args_dict, logger): parameter_name=_parameter, value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]), instrument=args_dict["telescope"], - model_version=args_dict["model_version"], + parameter_version=args_dict["parameter_version"], output_file=io_handler.get_output_file(f"{_parameter}.json"), ) diff --git a/tests/integration_tests/config/convert_all_model_parameters_from_simtel_num_gains.yml b/tests/integration_tests/config/convert_all_model_parameters_from_simtel_num_gains.yml index 46aa1c55a0..0095d610fa 100644 --- a/tests/integration_tests/config/convert_all_model_parameters_from_simtel_num_gains.yml +++ b/tests/integration_tests/config/convert_all_model_parameters_from_simtel_num_gains.yml @@ -5,7 +5,7 @@ CTA_SIMPIPE: SIMTEL_CFG_FILE: ./tests/resources/simtel_config_test_la_palma.cfg SIMTEL_TELESCOPE_NAME: CT1 TELESCOPE: LSTN-01 - MODEL_VERSION: "6.0.0" + PARAMETER_VERSION: "0.0.1" SCHEMA_DIRECTORY: ./tests/resources/ OUTPUT_PATH: simtools-output INTEGRATION_TESTS: From d26e7ef5da33692b64ba3b181273d3155d84ba20 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Fri, 17 Jan 2025 11:06:17 +0100 Subject: [PATCH 039/124] added schema for production table --- .../schemas/production_tables.schema.yml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 src/simtools/schemas/production_tables.schema.yml diff --git a/src/simtools/schemas/production_tables.schema.yml b/src/simtools/schemas/production_tables.schema.yml new file mode 100644 index 0000000000..630d277923 --- /dev/null +++ b/src/simtools/schemas/production_tables.schema.yml @@ -0,0 +1,35 @@ +--- +$schema: http://json-schema.org/draft-06/schema# +$ref: '#/definitions/ProductionModelTable' +title: SimPipe DB Production Model Metaschema +description: | + YAML representation of DB production model metaschema + (based on simulation model DB). +version: 0.1.0 +name: production_table.metaschema +type: object +additionalProperties: false + +definitions: + ProductionModelTable: + properties: + model_version: + type: string + description: Model version. + pattern: '^\d+\.\d+\.\d+$' + parameters: + type: object + description: Model parameters + additionalProperties: + type: object + description: Model parameter. + additionalProperties: + type: string + description: Parameter version (semantical versioning). + pattern: '^\d+\.\d+\.\d+$' + propertyNames: + description: Allowed parameter name patterns. + pattern: '^([A-Za-z](ST|LL)[N,S,x]-\d{2,3}|[A-Za-z](ST|LL)[N,S,x]-design|OBS-(North|South)|Dummy-Telescope)$' + required: + - model_version + - parameters From bd0a479eff94e4a3fa583b9c3062a37f527147c9 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Fri, 17 Jan 2025 11:06:28 +0100 Subject: [PATCH 040/124] simplified command line --- .../validate_file_using_schema.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/src/simtools/applications/validate_file_using_schema.py b/src/simtools/applications/validate_file_using_schema.py index a02217d3cc..b36252ef42 100644 --- a/src/simtools/applications/validate_file_using_schema.py +++ b/src/simtools/applications/validate_file_using_schema.py @@ -63,13 +63,12 @@ def _parse(label, description): group = config.parser.add_mutually_exclusive_group(required=True) group.add_argument("--file_name", help="File to be validated") group.add_argument( - "--model_parameters_directory", + "--file_directory", help=( - "Directory with json files with model parameters to be validated." - "All *.json files in the directory will be validated." - "Schema files will be taken from simtools/schemas/model_parameters/." - "Note that in this case the data_type argument is ignored" - "and data_type=model_parameter is always used." + "Directory with json files to be validated. " + "If no schema file is provided, the assumption is that model " + "parameters are validated and the schema files are taken from " + "simtools/schemas/model_parameters/." ), ) config.parser.add_argument("--schema", help="Json schema file", required=False) @@ -123,8 +122,8 @@ def validate_schema(args_dict, logger): the metadata section of the data dictionary. """ - if args_dict.get("model_parameters_directory") is not None: - file_list = list(Path(args_dict["model_parameters_directory"]).rglob("*.json")) + if args_dict.get("file_directory") is not None: + file_list = list(Path(args_dict["file_directory"]).rglob("*.json")) else: file_list = [args_dict["file_name"]] for file_name in file_list: @@ -139,10 +138,10 @@ def validate_schema(args_dict, logger): def validate_data_files(args_dict, logger): """Validate data files.""" - model_parameters_directory = args_dict.get("model_parameters_directory") - if model_parameters_directory is not None: + file_directory = args_dict.get("file_directory") + if file_directory is not None: tmp_args_dict = {} - for file_name in Path(model_parameters_directory).rglob("*.json"): + for file_name in Path(file_directory).rglob("*.json"): tmp_args_dict["file_name"] = file_name parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem) schema_file = ( From 35dae597ff94d2099155f25a2fc46943fc6512ea Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Fri, 17 Jan 2025 11:38:31 +0100 Subject: [PATCH 041/124] design model --- src/simtools/db/db_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index d9bf4383d2..9e8f3bb552 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -597,7 +597,8 @@ def get_corsika_configuration_parameters(self, model_version): DatabaseHandler.corsika_configuration_parameters_cached[cache_key] = self.read_mongo_db( query=self._get_query_from_parameter_version_table( - _production_table["parameters"], None + _production_table["parameters"]["xSTx-design"], # design model for all telescopes + None, ), collection_name="configuration_corsika", write_files=False, From 6e3c6af3599c9ea18dbeb229794b7e80d9f01aa1 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Fri, 17 Jan 2025 12:08:41 +0100 Subject: [PATCH 042/124] integration tests prod6 --- .../convert_geo_coordinates_of_array_elements.py | 5 ++++- .../convert_model_parameter_from_simtel.py | 4 ++-- .../applications/db_get_array_layouts_from_db.py | 4 +++- .../applications/db_get_parameter_from_db.py | 4 +++- src/simtools/applications/derive_mirror_rnda.py | 4 +++- src/simtools/applications/derive_psf_parameters.py | 2 +- src/simtools/applications/generate_array_config.py | 2 +- .../applications/generate_regular_arrays.py | 4 +++- src/simtools/applications/plot_array_layout.py | 4 +++- .../applications/simulate_light_emission.py | 2 +- src/simtools/applications/simulate_prod.py | 2 +- .../simulate_prod_htcondor_generator.py | 2 +- .../submit_model_parameter_from_external.py | 14 ++++++++------ .../applications/validate_camera_efficiency.py | 2 +- src/simtools/applications/validate_camera_fov.py | 2 +- .../applications/validate_cumulative_psf.py | 2 +- src/simtools/applications/validate_optics.py | 2 +- src/simtools/configuration/commandline_parser.py | 13 +++++++------ ...nvert_model_parameter_from_simtel_num_gains.yml | 2 +- ...parameter_from_external_submit_focus_offset.yml | 2 +- ..._parameter_from_external_submit_mirror_list.yml | 2 +- ...el_parameter_from_external_submit_num_gains.yml | 2 +- ...om_external_submit_reference_point_altitude.yml | 2 +- ...hema_validate_directory_of_model_parameters.yml | 2 +- tests/resources/array_element_position_ground.json | 3 +-- 25 files changed, 52 insertions(+), 37 deletions(-) diff --git a/src/simtools/applications/convert_geo_coordinates_of_array_elements.py b/src/simtools/applications/convert_geo_coordinates_of_array_elements.py index a4d94be638..f103ba6140 100644 --- a/src/simtools/applications/convert_geo_coordinates_of_array_elements.py +++ b/src/simtools/applications/convert_geo_coordinates_of_array_elements.py @@ -130,7 +130,10 @@ def _parse(label=None, description=None): action="store_true", ) return config.initialize( - output=True, require_command_line=True, db_config=True, simulation_model=["version", "site"] + output=True, + require_command_line=True, + db_config=True, + simulation_model=["model_version", "site"], ) diff --git a/src/simtools/applications/convert_model_parameter_from_simtel.py b/src/simtools/applications/convert_model_parameter_from_simtel.py index ed8ddb8c7c..ec6e064a03 100644 --- a/src/simtools/applications/convert_model_parameter_from_simtel.py +++ b/src/simtools/applications/convert_model_parameter_from_simtel.py @@ -78,7 +78,7 @@ def _parse(label=None, description=None): type=str, required=True, ) - return config.initialize(simulation_model="telescope", output=True) + return config.initialize(simulation_model=["telescope", "parameter_version"], output=True) def main(): # noqa: D103 @@ -107,7 +107,7 @@ def main(): # noqa: D103 parameter_name=simtel_config_reader.parameter_name, value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]), instrument=args_dict["telescope"], - model_version=args_dict["model_version"], + parameter_version=args_dict["parameter_version"], output_file=args_dict["output_file"], output_path=args_dict.get("output_path"), use_plain_output_path=args_dict.get("use_plain_output_path"), diff --git a/src/simtools/applications/db_get_array_layouts_from_db.py b/src/simtools/applications/db_get_array_layouts_from_db.py index ac7ca1da8d..78157e1a0c 100644 --- a/src/simtools/applications/db_get_array_layouts_from_db.py +++ b/src/simtools/applications/db_get_array_layouts_from_db.py @@ -94,7 +94,9 @@ def _parse(label, description): default="ground", choices=["ground", "utm"], ) - return config.initialize(db_config=True, simulation_model=["site", "layout"], output=True) + return config.initialize( + db_config=True, simulation_model=["site", "layout", "model_version"], output=True + ) def _layout_from_db(args_dict, db_config): diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index e5f8a169a1..2eceeddb55 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -70,7 +70,9 @@ def _parse(): required=False, ) - return config.initialize(db_config=True, simulation_model=["telescope", "parameter_version"]) + return config.initialize( + db_config=True, simulation_model=["telescope", "parameter_version", "model_version"] + ) def main(): # noqa: D103 diff --git a/src/simtools/applications/derive_mirror_rnda.py b/src/simtools/applications/derive_mirror_rnda.py index 2cbec10d47..58695d94ba 100644 --- a/src/simtools/applications/derive_mirror_rnda.py +++ b/src/simtools/applications/derive_mirror_rnda.py @@ -207,7 +207,9 @@ def _parse(label): action="store_true", required=False, ) - return config.initialize(db_config=True, output=True, simulation_model="telescope") + return config.initialize( + db_config=True, output=True, simulation_model=["telescope", "model_version"] + ) def main(): # noqa: D103 diff --git a/src/simtools/applications/derive_psf_parameters.py b/src/simtools/applications/derive_psf_parameters.py index 1123b057bc..43f0d4445b 100644 --- a/src/simtools/applications/derive_psf_parameters.py +++ b/src/simtools/applications/derive_psf_parameters.py @@ -150,7 +150,7 @@ def _parse(): help=("Keep the first entry of mirror_reflection_random_angle fixed."), action="store_true", ) - return config.initialize(db_config=True, simulation_model="telescope") + return config.initialize(db_config=True, simulation_model=["telescope", "model_version"]) def add_parameters( diff --git a/src/simtools/applications/generate_array_config.py b/src/simtools/applications/generate_array_config.py index 6ba883712e..38ecf95861 100644 --- a/src/simtools/applications/generate_array_config.py +++ b/src/simtools/applications/generate_array_config.py @@ -50,7 +50,7 @@ def _parse(label, description): Command line parser object. """ config = configurator.Configurator(label=label, description=description) - return config.initialize(db_config=True, simulation_model=["site", "layout"]) + return config.initialize(db_config=True, simulation_model=["site", "layout", "model_version"]) def main(): diff --git a/src/simtools/applications/generate_regular_arrays.py b/src/simtools/applications/generate_regular_arrays.py index 883f7536fd..d53e4911f9 100644 --- a/src/simtools/applications/generate_regular_arrays.py +++ b/src/simtools/applications/generate_regular_arrays.py @@ -51,7 +51,9 @@ def _parse(): f" SST: {telescope_distance['SST']}\n" ), ) - return config.initialize(db_config=False, simulation_model="site", output=True) + return config.initialize( + db_config=False, simulation_model=["site", "model_version"], output=True + ) def main(): diff --git a/src/simtools/applications/plot_array_layout.py b/src/simtools/applications/plot_array_layout.py index d6fd28d410..2f66c43948 100644 --- a/src/simtools/applications/plot_array_layout.py +++ b/src/simtools/applications/plot_array_layout.py @@ -146,7 +146,9 @@ def _parse(label, description, usage): required=False, default=None, ) - return config.initialize(db_config=True, simulation_model=["site", "layout", "layout_file"]) + return config.initialize( + db_config=True, simulation_model=["site", "model_version", "layout", "layout_file"] + ) def _get_site_from_telescope_list_name(telescope_list_file): diff --git a/src/simtools/applications/simulate_light_emission.py b/src/simtools/applications/simulate_light_emission.py index e488add70c..091b4df868 100644 --- a/src/simtools/applications/simulate_light_emission.py +++ b/src/simtools/applications/simulate_light_emission.py @@ -235,7 +235,7 @@ def _parse(label): ) return config.initialize( db_config=True, - simulation_model="telescope", + simulation_model=["telescope", "model_version"], require_command_line=True, ) diff --git a/src/simtools/applications/simulate_prod.py b/src/simtools/applications/simulate_prod.py index 3aea5f9029..665a8a1c01 100644 --- a/src/simtools/applications/simulate_prod.py +++ b/src/simtools/applications/simulate_prod.py @@ -134,7 +134,7 @@ def _parse(description=None): return config.initialize( db_config=True, job_submission=True, - simulation_model=["site", "layout", "telescope"], + simulation_model=["site", "layout", "telescope", "model_version"], simulation_configuration={"software": None, "corsika_configuration": ["all"]}, ) diff --git a/src/simtools/applications/simulate_prod_htcondor_generator.py b/src/simtools/applications/simulate_prod_htcondor_generator.py index 9f2f908c5d..0f8ef4127e 100644 --- a/src/simtools/applications/simulate_prod_htcondor_generator.py +++ b/src/simtools/applications/simulate_prod_htcondor_generator.py @@ -77,7 +77,7 @@ def _parse(description=None): return config.initialize( db_config=False, job_submission=False, - simulation_model=["site", "layout", "telescope"], + simulation_model=["site", "layout", "telescope", "model_version"], simulation_configuration={"software": None, "corsika_configuration": ["all"]}, ) diff --git a/src/simtools/applications/submit_model_parameter_from_external.py b/src/simtools/applications/submit_model_parameter_from_external.py index c0874998b8..0c89c8e636 100644 --- a/src/simtools/applications/submit_model_parameter_from_external.py +++ b/src/simtools/applications/submit_model_parameter_from_external.py @@ -16,8 +16,8 @@ instrument name. site (str) site location. - model_version (str) - Model version. + parameter_version (str) + Parameter version. input_meta (str, optional) input meta data file (yml format) @@ -33,7 +33,7 @@ --value 2 \\ --instrument LSTN-design \\ --site North \\ - --model_version 6.0.0 \\ + --parameter_version 0.1.0 \\ --input_meta num_gains.metadata.yml """ @@ -70,7 +70,9 @@ def _parse(label, description): ) config.parser.add_argument("--instrument", type=str, required=True, help="Instrument name") config.parser.add_argument("--site", type=str, required=True, help="Site location") - config.parser.add_argument("--model_version", type=str, required=True, help="Model version") + config.parser.add_argument( + "--parameter_version", type=str, required=True, help="Parameter version" + ) config.parser.add_argument( "--value", @@ -102,7 +104,7 @@ def main(): # noqa: D103 logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) output_path = ( - Path(args_dict["output_path"]) / args_dict["model_version"] / args_dict["instrument"] + Path(args_dict["output_path"]) / args_dict["parameter_version"] / args_dict["instrument"] if args_dict.get("output_path") else None ) @@ -110,7 +112,7 @@ def main(): # noqa: D103 parameter_name=args_dict["parameter"], value=args_dict["value"], instrument=args_dict["instrument"], - model_version=args_dict["model_version"], + parameter_version=args_dict["parameter_version"], output_file=Path(args_dict["parameter"]).with_suffix(".json"), output_path=output_path, use_plain_output_path=args_dict.get("use_plain_output_path"), diff --git a/src/simtools/applications/validate_camera_efficiency.py b/src/simtools/applications/validate_camera_efficiency.py index 6a8ceef56b..b7b7fcf962 100644 --- a/src/simtools/applications/validate_camera_efficiency.py +++ b/src/simtools/applications/validate_camera_efficiency.py @@ -86,7 +86,7 @@ def _parse(label): ) _args_dict, _db_config = config.initialize( db_config=True, - simulation_model="telescope", + simulation_model=["telescope", "model_version"], simulation_configuration={"corsika_configuration": ["zenith_angle", "azimuth_angle"]}, ) if _args_dict["site"] is None or _args_dict["telescope"] is None: diff --git a/src/simtools/applications/validate_camera_fov.py b/src/simtools/applications/validate_camera_fov.py index c2bbb929b1..a7fed1e2d8 100644 --- a/src/simtools/applications/validate_camera_fov.py +++ b/src/simtools/applications/validate_camera_fov.py @@ -83,7 +83,7 @@ def _parse(): ), default=50, ) - return config.initialize(db_config=True, simulation_model="telescope") + return config.initialize(db_config=True, simulation_model=["telescope", "model_version"]) def main(): # noqa: D103 diff --git a/src/simtools/applications/validate_cumulative_psf.py b/src/simtools/applications/validate_cumulative_psf.py index 3c0b8728c6..d6d42e4c2f 100644 --- a/src/simtools/applications/validate_cumulative_psf.py +++ b/src/simtools/applications/validate_cumulative_psf.py @@ -110,7 +110,7 @@ def _parse(label): help="Data file name with the measured PSF vs radius [cm]", type=str, ) - return config.initialize(db_config=True, simulation_model="telescope") + return config.initialize(db_config=True, simulation_model=["telescope", "model_version"]) def load_data(datafile): diff --git a/src/simtools/applications/validate_optics.py b/src/simtools/applications/validate_optics.py index 02539c4993..0f157bfc6e 100644 --- a/src/simtools/applications/validate_optics.py +++ b/src/simtools/applications/validate_optics.py @@ -114,7 +114,7 @@ def _parse(label): help="Produce a multiple pages pdf file with the image plots.", action="store_true", ) - return config.initialize(db_config=True, simulation_model=["telescope"]) + return config.initialize(db_config=True, simulation_model=["telescope", "model_version"]) def main(): # noqa: D103 diff --git a/src/simtools/configuration/commandline_parser.py b/src/simtools/configuration/commandline_parser.py index 6ff72daf5b..4e5b26945d 100644 --- a/src/simtools/configuration/commandline_parser.py +++ b/src/simtools/configuration/commandline_parser.py @@ -236,12 +236,13 @@ def initialize_simulation_model_arguments(self, model_options): return _job_group = self.add_argument_group("simulation model") - _job_group.add_argument( - "--model_version", - help="production model version", - type=str, - default=None, - ) + if "model_version" in model_options: + _job_group.add_argument( + "--model_version", + help="production model version", + type=str, + default=None, + ) if "parameter_version" in model_options: _job_group.add_argument( "--parameter_version", diff --git a/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml b/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml index 5d69f70fa1..234afc4f78 100644 --- a/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml +++ b/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml @@ -5,7 +5,7 @@ CTA_SIMPIPE: SIMTEL_CFG_FILE: ./tests/resources/simtel_config_test_la_palma.cfg SIMTEL_TELESCOPE_NAME: CT1 TELESCOPE: LSTN-01 - MODEL_VERSION: "6.0.0" + PARAMETER_VERSION: "0.1.0" SCHEMA: ./tests/resources/num_gains.schema.yml OUTPUT_PATH: simtools-output OUTPUT_FILE: num_gains.json diff --git a/tests/integration_tests/config/submit_model_parameter_from_external_submit_focus_offset.yml b/tests/integration_tests/config/submit_model_parameter_from_external_submit_focus_offset.yml index 5c03c7679f..37d5ac75a6 100644 --- a/tests/integration_tests/config/submit_model_parameter_from_external_submit_focus_offset.yml +++ b/tests/integration_tests/config/submit_model_parameter_from_external_submit_focus_offset.yml @@ -6,7 +6,7 @@ CTA_SIMPIPE: VALUE: "6.55 cm, 0.0 deg, 0.0, 0.0" INSTRUMENT: LSTN-01 SITE: North - MODEL_VERSION: 6.0.0 + PARAMETER_VERSION: 0.1.0 INPUT_META: null OUTPUT_PATH: simtools-output USE_PLAIN_OUTPUT_PATH: true diff --git a/tests/integration_tests/config/submit_model_parameter_from_external_submit_mirror_list.yml b/tests/integration_tests/config/submit_model_parameter_from_external_submit_mirror_list.yml index 2090c964c1..fcd48ed4ab 100644 --- a/tests/integration_tests/config/submit_model_parameter_from_external_submit_mirror_list.yml +++ b/tests/integration_tests/config/submit_model_parameter_from_external_submit_mirror_list.yml @@ -6,7 +6,7 @@ CTA_SIMPIPE: VALUE: mirror_CTA-100_1.20-86-0.04.dat INSTRUMENT: MSTS-design SITE: South - MODEL_VERSION: 6.0.0 + PARAMETER_VERSION: 0.1.0 INPUT_META: null OUTPUT_PATH: simtools-output USE_PLAIN_OUTPUT_PATH: true diff --git a/tests/integration_tests/config/submit_model_parameter_from_external_submit_num_gains.yml b/tests/integration_tests/config/submit_model_parameter_from_external_submit_num_gains.yml index 82ca438853..b01258086e 100644 --- a/tests/integration_tests/config/submit_model_parameter_from_external_submit_num_gains.yml +++ b/tests/integration_tests/config/submit_model_parameter_from_external_submit_num_gains.yml @@ -6,7 +6,7 @@ CTA_SIMPIPE: VALUE: 2 INSTRUMENT: LSTN-design SITE: North - MODEL_VERSION: 6.0.0 + PARAMETER_VERSION: 0.1.0 INPUT_META: ./tests/resources/num_gains.meta.yml OUTPUT_PATH: simtools-output USE_PLAIN_OUTPUT_PATH: true diff --git a/tests/integration_tests/config/submit_model_parameter_from_external_submit_reference_point_altitude.yml b/tests/integration_tests/config/submit_model_parameter_from_external_submit_reference_point_altitude.yml index a734f65a7c..10584dd611 100644 --- a/tests/integration_tests/config/submit_model_parameter_from_external_submit_reference_point_altitude.yml +++ b/tests/integration_tests/config/submit_model_parameter_from_external_submit_reference_point_altitude.yml @@ -6,7 +6,7 @@ CTA_SIMPIPE: VALUE: "2.177 km" INSTRUMENT: North SITE: North - MODEL_VERSION: 6.0.0 + PARAMETER_VERSION: 0.1.0 INPUT_META: null OUTPUT_PATH: simtools-output USE_PLAIN_OUTPUT_PATH: true diff --git a/tests/integration_tests/config/validate_file_using_schema_validate_directory_of_model_parameters.yml b/tests/integration_tests/config/validate_file_using_schema_validate_directory_of_model_parameters.yml index a4a8977b26..7bcbbd38f1 100644 --- a/tests/integration_tests/config/validate_file_using_schema_validate_directory_of_model_parameters.yml +++ b/tests/integration_tests/config/validate_file_using_schema_validate_directory_of_model_parameters.yml @@ -2,4 +2,4 @@ CTA_SIMPIPE: APPLICATION: simtools-validate-file-using-schema TEST_NAME: validate_directory_of_model_parameters CONFIGURATION: - MODEL_PARAMETERS_DIRECTORY: tests/resources/model_parameters + FILE_DIRECTORY: tests/resources/model_parameters diff --git a/tests/resources/array_element_position_ground.json b/tests/resources/array_element_position_ground.json index cda15316d1..84972f150e 100644 --- a/tests/resources/array_element_position_ground.json +++ b/tests/resources/array_element_position_ground.json @@ -1,5 +1,5 @@ { - "schema_version": "0.1.0", + "schema_version": "0.2.0", "instrument": "MSTN-09", "site": "North", "parameter_version": "2.0.0", @@ -11,6 +11,5 @@ ], "unit": "m", "type": "float64", - "applicable": true, "file": false } From 1aacd2fc724a7983a9c92b4de6079b3c4c8f5d08 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Fri, 17 Jan 2025 12:10:56 +0100 Subject: [PATCH 043/124] db name in unit and integration tests --- .env_template | 2 +- .github/workflows/CI-integrationtests.yml | 2 +- .github/workflows/CI-unittests.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.env_template b/.env_template index 99fe884107..2a8a55752c 100644 --- a/.env_template +++ b/.env_template @@ -6,7 +6,7 @@ SIMTOOLS_DB_SERVER='cta-simpipe-protodb.zeuthen.desy.de' # MongoDB server SIMTOOLS_DB_API_USER=YOUR_USERNAME # username for MongoDB: ask the responsible person SIMTOOLS_DB_API_PW=YOUR_PASSWORD # Password for MongoDB: ask the responsible person SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin' -SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-LATEST' +SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-LATEST' SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray' # The dashboards to monitor the MongoDB instance are in (accessible only from within DESY) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index a56aaaeba9..833f7ed6d3 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -7,7 +7,7 @@ env: SIMTOOLS_DB_API_USER: ${{ secrets.DB_API_USER }} SIMTOOLS_DB_API_PW: ${{ secrets.DB_API_PW }} SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }} - SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-Model-LATEST" + SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-ModelParameters-LATEST" SIMTOOLS_SIMTEL_PATH: "/workdir/sim_telarray/" on: diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml index dc01be539d..a73c9593e4 100644 --- a/.github/workflows/CI-unittests.yml +++ b/.github/workflows/CI-unittests.yml @@ -7,7 +7,7 @@ env: SIMTOOLS_DB_API_USER: ${{ secrets.DB_API_USER }} SIMTOOLS_DB_API_PW: ${{ secrets.DB_API_PW }} SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }} - SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-Model-LATEST" + SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-ModelParameters-LATEST" SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} on: From c2e1cee71087106e054d95c579c9dfe9ed582d44 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Fri, 17 Jan 2025 14:25:49 +0100 Subject: [PATCH 044/124] improved docstring --- src/simtools/applications/db_get_parameter_from_db.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 2eceeddb55..4df2976726 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -26,7 +26,7 @@ Example ------- - Get the mirror_list parameter from the DB. + Get the mirror_list parameter used for a given model_version from the DB. .. code-block:: console @@ -34,6 +34,14 @@ --site North --telescope LSTN-01 \\ --model_version 5.0.0 + Get the mirror_list parameter using the parameter_version from the DB. + + .. code-block:: console + + simtools-db-get-parameter-from-db --parameter mirror_list \\ + --site North --telescope LSTN-01 \\ + --parameter_version 5.0.0 + """ import json From 4ac29e7c870e6437283907958255ecd2cfbd770b Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Fri, 17 Jan 2025 14:33:20 +0100 Subject: [PATCH 045/124] parameter vs model version --- .../write_array_elements_positions_to_repository.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py b/src/simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py index 1ea65ee87b..1dc4aa3670 100644 --- a/src/simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +++ b/src/simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py @@ -11,8 +11,8 @@ File containing a table of array element positions. repository_path : str Path of local copy of model parameter repository. - model_version : str - Model version. + parameter_version : str + Parameter version. site : str Observatory site. coordinate_system : str @@ -27,7 +27,7 @@ simtools-write-array-element-positions-to-repository \ --input /path/to/positions.txt \ --repository_path /path/to/repository \ - --model_version 1.0.0 \ + --parameter_version 0.1.0 \ --coordinate_system ground \ --site North @@ -82,7 +82,7 @@ def _parse(label=None, description=None): choices=["ground", "utm"], ) - return config.initialize(db_config=True, simulation_model="site") + return config.initialize(db_config=True, simulation_model=["site", "parameter_version"]) def write_utm_array_elements_to_repository(args_dict, logger): @@ -115,7 +115,7 @@ def write_utm_array_elements_to_repository(args_dict, logger): parameter_name="array_element_position_utm", instrument=instrument, value=f"{row['utm_east']} {row['utm_north']} {row['altitude']}", - model_version=args_dict["model_version"], + parameter_version=args_dict["parameter_version"], output_path=output_path, output_file="array_element_position_utm.json", ) @@ -137,7 +137,7 @@ def write_ground_array_elements_to_repository(args_dict, db_config, logger): """ array_model = ArrayModel( mongo_db_config=db_config, - model_version=args_dict["model_version"], + model_version=None, site=args_dict["site"], array_elements=args_dict["input"], ) From 7fb4079283c1b72265340eb6b98c14a0f175ce5a Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Fri, 17 Jan 2025 14:35:59 +0100 Subject: [PATCH 046/124] fix db modules --- docs/source/api-reference/db_handler.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/api-reference/db_handler.md b/docs/source/api-reference/db_handler.md index 6fc06286eb..47b7a52255 100644 --- a/docs/source/api-reference/db_handler.md +++ b/docs/source/api-reference/db_handler.md @@ -13,11 +13,11 @@ Modules for database access. See the databases sections for details. :members: ``` -## db_array_elements +## db_model_upload -(db-array-elements)= +(db-module-upload)= ```{eval-rst} -.. automodule:: db.db_array_elements +.. automodule:: db.db_model_upload :members: ``` From 71a3e5d16a9754055078bc85cd1b1316d6481bf2 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sat, 18 Jan 2025 15:57:53 +0100 Subject: [PATCH 047/124] integration test fix --- ...ter_from_db_array_element_position_ground.yml | 4 ++-- .../resources/array_element_position_ground.json | 15 --------------- .../array_element_position_ground.json | 16 ++++++++++------ 3 files changed, 12 insertions(+), 23 deletions(-) delete mode 100644 tests/resources/array_element_position_ground.json diff --git a/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml b/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml index 64ce490091..0af1b84807 100644 --- a/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml +++ b/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml @@ -2,8 +2,8 @@ CTA_SIMPIPE: APPLICATION: simtools-db-get-parameter-from-db TEST_NAME: telescope_parameter_array_element_position_ground CONFIGURATION: - SITE: South - TELESCOPE: SSTS-09 + SITE: North + TELESCOPE: MSTN-09 PARAMETER: array_element_position_ground MODEL_VERSION: 6.0.0 OUTPUT_FILE: test.json diff --git a/tests/resources/array_element_position_ground.json b/tests/resources/array_element_position_ground.json deleted file mode 100644 index 84972f150e..0000000000 --- a/tests/resources/array_element_position_ground.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "schema_version": "0.2.0", - "instrument": "MSTN-09", - "site": "North", - "parameter_version": "2.0.0", - "unique_id": null, - "value": [ - 221.68, - -355.66, - 49.0 - ], - "unit": "m", - "type": "float64", - "file": false -} diff --git a/tests/resources/model_parameters/array_element_position_ground.json b/tests/resources/model_parameters/array_element_position_ground.json index 1d2ac2a396..84972f150e 100644 --- a/tests/resources/model_parameters/array_element_position_ground.json +++ b/tests/resources/model_parameters/array_element_position_ground.json @@ -1,11 +1,15 @@ { - "parameter": "array_element_position_ground", - "instrument": "SSTS-09", - "site": "South", - "version": "6.0.0", - "value": "-4.79 -499.24 39.75", + "schema_version": "0.2.0", + "instrument": "MSTN-09", + "site": "North", + "parameter_version": "2.0.0", + "unique_id": null, + "value": [ + 221.68, + -355.66, + 49.0 + ], "unit": "m", "type": "float64", - "applicable": true, "file": false } From 2b4985081720478ea5189ddac066ee71a75e632c Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sat, 18 Jan 2025 16:43:26 +0100 Subject: [PATCH 048/124] re-enable cache_key --- src/simtools/db/db_handler.py | 74 +++++++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index 9e8f3bb552..092d3638ba 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -254,16 +254,25 @@ def get_model_parameters( design_model = f"{names.get_array_element_type_from_name(array_element_name)}-design" array_element_list = ( [array_element_name] - if "-design" in array_element_name + if "-design" in array_element_name # design model must be first in list else [design_model, array_element_name] ) pars = {} - for array_element in array_element_list: # design model must be read first - cache_key = self._cache_key( - names.validate_site_name(site), array_element, model_version, collection + for array_element in array_element_list: + cache_key, cache_dict = self._fill_from_cache( + DatabaseHandler.model_parameters_cached, + names.validate_site_name(site), + array_element, + model_version, + collection, ) - pars.update(DatabaseHandler.model_parameters_cached.get(cache_key, {})) + if cache_dict: + self._logger.debug(f"Found {array_element} in cache (key: {cache_key})") + pars.update(cache_dict) + continue + self._logger.debug(f"Did not find {array_element} in cache (key: {cache_key})") + try: parameter_version_table = production_table["parameters"][array_element] except KeyError as exc: @@ -446,11 +455,12 @@ def get_site_parameters(self, site, model_version): """ site = names.validate_site_name(site) production_table = self.get_production_table_from_mongo_db("sites", model_version) - cache_key = self._cache_key(site, None, production_table.get("model_version")) - try: - return DatabaseHandler.site_parameters_cached[cache_key] - except KeyError: - pass + cache_key, cache_dict = self._fill_from_cache( + DatabaseHandler.site_parameters_cached, + self._cache_key(site, None, production_table.get("model_version")), + ) + if cache_dict: + return cache_dict try: parameter_query = production_table["parameters"][f"OBS-{site}"] @@ -588,12 +598,14 @@ def get_corsika_configuration_parameters(self, model_version): _production_table = self.get_production_table_from_mongo_db( "configuration_corsika", model_version ) - cache_key = self._cache_key(None, None, _production_table.get("model_version")) - - try: - return DatabaseHandler.corsika_configuration_parameters_cached[cache_key] - except KeyError: - pass + cache_key, cache_dict = self._fill_from_cache( + DatabaseHandler.corsika_configuration_parameters_cached, + None, + None, + _production_table.get("model_version"), + ) + if cache_dict: + return cache_dict DatabaseHandler.corsika_configuration_parameters_cached[cache_key] = self.read_mongo_db( query=self._get_query_from_parameter_version_table( @@ -875,6 +887,36 @@ def _cache_key(self, site=None, array_element_name=None, model_version=None, col part for part in [model_version, collection, site, array_element_name] if part ) + def _fill_from_cache( + self, cache_dict, site=None, array_element_name=None, model_version=None, collection=None + ): + """ + Fill a parameter dict from cache. + + Parameters + ---------- + cache_dict: dict + Cache dictionary. + site: str + Site name. + array_element_name: str + Array element name. + model_version: str + Model version. + collection: str + DB collection name. + + Returns + ------- + str + Cache key. + """ + cache_key = self._cache_key(site, array_element_name, model_version, collection) + try: + return cache_key, cache_dict[cache_key] + except KeyError: + return cache_key, None + def _reset_production_table_cache(self, collection_name, model_version): """ Reset the cache for the production tables. From 2cea5ffc41a69cca569b49f782ce0bcdc38aeb30 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sat, 18 Jan 2025 17:01:04 +0100 Subject: [PATCH 049/124] remove TODO --- src/simtools/model/model_parameter.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index ec1faf04dd..e8a8251efb 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -44,8 +44,6 @@ class ModelParameter: label: str Instance label. Important for output file naming. - TODO - make sure that model_version is to correct version to be used here - """ def __init__( From 0848c46ac3813f484166212737bc501c6340beb7 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sat, 18 Jan 2025 17:42:56 +0100 Subject: [PATCH 050/124] unit tests --- .../configuration/test_commandline_parser.py | 6 +++--- tests/unit_tests/corsika/test_corsika_config.py | 13 ++++++++----- .../unit_tests/data_model/test_model_data_writer.py | 6 +++--- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/tests/unit_tests/configuration/test_commandline_parser.py b/tests/unit_tests/configuration/test_commandline_parser.py index cf9dcf424b..38e684a996 100644 --- a/tests/unit_tests/configuration/test_commandline_parser.py +++ b/tests/unit_tests/configuration/test_commandline_parser.py @@ -169,7 +169,7 @@ def test_simulation_model(): # Model version only, no site or telescope _parser_v = parser.CommandLineParser() - _parser_v.initialize_default_arguments(simulation_model=["version"]) + _parser_v.initialize_default_arguments(simulation_model=["model_version"]) job_groups = _parser_v._action_groups assert SIMULATION_MODEL_STRING in [str(group.title) for group in job_groups] @@ -181,7 +181,7 @@ def test_simulation_model(): # Site model can exist without a telescope model _parser_s = parser.CommandLineParser() - _parser_s.initialize_default_arguments(simulation_model=["site"]) + _parser_s.initialize_default_arguments(simulation_model=["site", "model_version"]) job_groups = _parser_s._action_groups assert SIMULATION_MODEL_STRING in [str(group.title) for group in job_groups] for group in job_groups: @@ -192,7 +192,7 @@ def test_simulation_model(): # No telescope model without site model _parser_t = parser.CommandLineParser() - _parser_t.initialize_default_arguments(simulation_model=["telescope", "site"]) + _parser_t.initialize_default_arguments(simulation_model=["telescope", "site", "model_version"]) job_groups = _parser_t._action_groups assert SIMULATION_MODEL_STRING in [str(group.title) for group in job_groups] for group in job_groups: diff --git a/tests/unit_tests/corsika/test_corsika_config.py b/tests/unit_tests/corsika/test_corsika_config.py index b0aa400674..c35f69fba5 100644 --- a/tests/unit_tests/corsika/test_corsika_config.py +++ b/tests/unit_tests/corsika/test_corsika_config.py @@ -29,9 +29,12 @@ def corsika_configuration_parameters(gcm2): return { "corsika_iact_max_bunches": {"value": 1000000, "unit": None}, "corsika_cherenkov_photon_bunch_size": {"value": 5.0, "unit": None}, - "corsika_cherenkov_photon_wavelength_range": {"value": "240. 700.", "unit": "nm"}, + "corsika_cherenkov_photon_wavelength_range": {"value": [240.0, 700.0], "unit": "nm"}, "corsika_first_interaction_height": {"value": 0.0, "unit": "cm"}, - "corsika_particle_kinetic_energy_cutoff": {"value": "0.3 0.1 0.020 0.020", "unit": "GeV"}, + "corsika_particle_kinetic_energy_cutoff": { + "value": [0.3, 0.1, 0.020, 0.020], + "unit": "GeV", + }, "corsika_longitudinal_shower_development": {"value": 20.0, "unit": gcm2}, "corsika_iact_split_auto": {"value": 15000000, "unit": None}, "corsika_starting_grammage": {"value": 0.0, "unit": gcm2}, @@ -125,10 +128,10 @@ def test_input_config_corsika_starting_grammage(corsika_config_mock_array_model, def test_input_config_corsika_particle_kinetic_energy_cutoff(corsika_config_mock_array_model): assert corsika_config_mock_array_model._input_config_corsika_particle_kinetic_energy_cutoff( - {"value": "0.3 0.1 0.020 0.020", "unit": "GeV"} + {"value": [0.3, 0.1, 0.020, 0.020], "unit": "GeV"} ) == ["0.3 0.1 0.02 0.02"] assert corsika_config_mock_array_model._input_config_corsika_particle_kinetic_energy_cutoff( - {"value": "0.3 0.1 0.020 0.020", "unit": "TeV"} + {"value": [0.3, 0.1, 0.020, 0.020], "unit": "TeV"} ) == ["300.0 100.0 20.0 20.0"] @@ -154,7 +157,7 @@ def test_corsika_configuration_cherenkov_parameters( def test_input_config_corsika_cherenkov_wavelength(corsika_config_mock_array_model): assert corsika_config_mock_array_model._input_config_corsika_cherenkov_wavelength( - {"value": "240. 700.", "unit": "nm"} + {"value": [240.0, 700.0], "unit": "nm"} ) == ["240.0", "700.0"] diff --git a/tests/unit_tests/data_model/test_model_data_writer.py b/tests/unit_tests/data_model/test_model_data_writer.py index 0a89b3cd1a..580d1622a9 100644 --- a/tests/unit_tests/data_model/test_model_data_writer.py +++ b/tests/unit_tests/data_model/test_model_data_writer.py @@ -215,7 +215,7 @@ def test_dump_model_parameter(tmp_test_directory): parameter_name="num_gains", value=2, instrument=instrument, - model_version=parameter_version, + parameter_version=parameter_version, output_file="num_gains.json", output_path=tmp_test_directory, use_plain_output_path=True, @@ -230,7 +230,7 @@ def test_dump_model_parameter(tmp_test_directory): parameter_name="array_element_position_utm", value=[217.6596 * u.km, 3184.9951 * u.km, 218500.0 * u.cm], instrument=instrument, - model_version=parameter_version, + parameter_version=parameter_version, output_file="array_element_position_utm.json", output_path=tmp_test_directory, use_plain_output_path=True, @@ -248,7 +248,7 @@ def test_dump_model_parameter(tmp_test_directory): parameter_name="focus_offset", value=[6.55 * u.cm, 0.0 * u.deg, 0.0, 0.0], instrument="LSTN-01", - model_version=parameter_version, + parameter_version=parameter_version, output_file="focus_offset.json", output_path=tmp_test_directory, use_plain_output_path=True, From 42dcbbaf326213bc0c18f0b293f3b38584112d60 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sat, 18 Jan 2025 18:05:13 +0100 Subject: [PATCH 051/124] unit tests --- src/simtools/layout/array_layout.py | 2 +- src/simtools/model/model_parameter.py | 5 ++++- .../array_element_position_ground.json | 1 + .../array_element_position_utm.json | 15 ++++++++++----- tests/unit_tests/layout/test_array_layout.py | 2 +- tests/unit_tests/model/test_array_model.py | 1 - 6 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/simtools/layout/array_layout.py b/src/simtools/layout/array_layout.py index 03f7dca5b2..7658a09b9e 100644 --- a/src/simtools/layout/array_layout.py +++ b/src/simtools/layout/array_layout.py @@ -386,7 +386,7 @@ def _read_table_from_json_file(self, file_name): with Path(file_name).open("r", encoding="utf-8") as file: data = json.load(file) - position = gen.convert_string_to_list(data["value"]) + position = data["value"] self.site = data.get("site", None) table = QTable() diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index e8a8251efb..66392a7ca1 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -173,7 +173,10 @@ def get_parameter_value_with_unit(self, par_name): _value = self.get_parameter_value(par_name, _parameter) try: - _unit = [item.strip() for item in _parameter.get("unit").split(",")] + if isinstance(_parameter.get("unit"), str): # TODO - check if still needed + _unit = [item.strip() for item in _parameter.get("unit").split(",")] + else: + _unit = _parameter.get("unit") # if there is only one value or the values share one unit if (isinstance(_value, (int | float))) or (len(_value) > len(_unit)): diff --git a/tests/resources/model_parameters/array_element_position_ground.json b/tests/resources/model_parameters/array_element_position_ground.json index 84972f150e..63082c00e7 100644 --- a/tests/resources/model_parameters/array_element_position_ground.json +++ b/tests/resources/model_parameters/array_element_position_ground.json @@ -1,5 +1,6 @@ { "schema_version": "0.2.0", + "parameter": "array_element_position_ground", "instrument": "MSTN-09", "site": "North", "parameter_version": "2.0.0", diff --git a/tests/resources/model_parameters/array_element_position_utm.json b/tests/resources/model_parameters/array_element_position_utm.json index 8cc265657c..159635e62c 100644 --- a/tests/resources/model_parameters/array_element_position_utm.json +++ b/tests/resources/model_parameters/array_element_position_utm.json @@ -1,11 +1,16 @@ { + "schema_version": "0.2.0", "parameter": "array_element_position_utm", - "instrument": "SSTS-09", - "site": "South", - "version": "6.0.0", - "value": "367321.0 7269466.0 2183.5", + "instrument": "MSTN-09", + "site": "North", + "parameter_version": "2.0.0", + "unique_id": null, + "value": [ + 217970.0, + 3185280.3, + 2196.0 + ], "unit": "m", "type": "float64", - "applicable": true, "file": false } diff --git a/tests/unit_tests/layout/test_array_layout.py b/tests/unit_tests/layout/test_array_layout.py index fd0f0b7d9f..2b25f896b5 100644 --- a/tests/unit_tests/layout/test_array_layout.py +++ b/tests/unit_tests/layout/test_array_layout.py @@ -455,7 +455,7 @@ def test_export_one_telescope_as_json(db_config, model_version, telescope_north_ ground_dict = layout.export_one_telescope_as_json(crs_name="ground") assert isinstance(ground_dict, dict) - assert ground_dict["instrument"] == "SSTS-09" + assert ground_dict["instrument"] == "MSTN-09" assert ground_dict["parameter"] == "array_element_position_ground" utm_dict = layout.export_one_telescope_as_json(crs_name="utm") diff --git a/tests/unit_tests/model/test_array_model.py b/tests/unit_tests/model/test_array_model.py index cd347f8cd9..8c6714ed86 100644 --- a/tests/unit_tests/model/test_array_model.py +++ b/tests/unit_tests/model/test_array_model.py @@ -94,7 +94,6 @@ def test_exporting_config_files(db_config, io_handler, model_version): ] for model_file in list_of_export_files: - logger.info("Checking file: %s", model_file) assert Path(am.get_config_directory()).joinpath(model_file).exists() From 5eb4d58e95b0abfa8e1f865e87b4d3676300c783 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 19 Jan 2025 16:45:43 +0100 Subject: [PATCH 052/124] add design model --- src/simtools/db/db_model_upload.py | 5 +++++ src/simtools/schemas/production_tables.schema.yml | 8 +++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/simtools/db/db_model_upload.py b/src/simtools/db/db_model_upload.py index 83890d4bcf..01e0c2bebb 100644 --- a/src/simtools/db/db_model_upload.py +++ b/src/simtools/db/db_model_upload.py @@ -104,6 +104,7 @@ def add_production_tables_to_db(args_dict, db): "collection": collection, "model_version": model.name, "parameters": {}, + "design_model": {}, }, ) parameter_dict = gen.collect_data_from_file(file_name=file) @@ -118,6 +119,10 @@ def add_production_tables_to_db(args_dict, db): except KeyError as exc: logger.error(f"KeyError: {exc}") raise + if "design_model" in parameter_dict: + model_dict[collection]["design_model"][array_element] = parameter_dict[ + "design_model" + ] for collection, data in model_dict.items(): if not data["parameters"]: diff --git a/src/simtools/schemas/production_tables.schema.yml b/src/simtools/schemas/production_tables.schema.yml index 630d277923..a12900e102 100644 --- a/src/simtools/schemas/production_tables.schema.yml +++ b/src/simtools/schemas/production_tables.schema.yml @@ -19,7 +19,7 @@ definitions: pattern: '^\d+\.\d+\.\d+$' parameters: type: object - description: Model parameters + description: Model parameters. additionalProperties: type: object description: Model parameter. @@ -30,6 +30,12 @@ definitions: propertyNames: description: Allowed parameter name patterns. pattern: '^([A-Za-z](ST|LL)[N,S,x]-\d{2,3}|[A-Za-z](ST|LL)[N,S,x]-design|OBS-(North|South)|Dummy-Telescope)$' + design_model: + type: object + description: Design models. + additionalProperties: + type: string + description: Design model of a telescope required: - model_version - parameters From f4e20c99268629ccc0301f5032da3c07a723cb95 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 19 Jan 2025 17:39:00 +0100 Subject: [PATCH 053/124] simplifications --- .../applications/db_get_file_from_db.py | 10 +- src/simtools/db/db_handler.py | 181 ++++++++---------- src/simtools/db/db_model_upload.py | 6 +- 3 files changed, 88 insertions(+), 109 deletions(-) diff --git a/src/simtools/applications/db_get_file_from_db.py b/src/simtools/applications/db_get_file_from_db.py index 21dc75e371..17a22f9566 100644 --- a/src/simtools/applications/db_get_file_from_db.py +++ b/src/simtools/applications/db_get_file_from_db.py @@ -68,11 +68,13 @@ def main(): # noqa: D103 db_config["db_simulation_model"], "sandbox", ] - file_id = None + file_id = {} for db_name in available_dbs: try: - file_id = db.export_file_db( - db_name, _io_handler.get_output_directory(), args_dict["file_name"] + file_id = db.export_model_files( + db_name=db_name, + dest=_io_handler.get_output_directory(), + file_names=args_dict["file_name"], ) logger.info( f"Got file {args_dict['file_name']} from DB {db_name} " @@ -82,7 +84,7 @@ def main(): # noqa: D103 except FileNotFoundError: continue - if file_id is None: + if file_id.get(args_dict["file_name"]) is None: logger.error( f"The file {args_dict['file_name']} was not found in any of the available DBs." ) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index 092d3638ba..8d89e9f104 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -213,11 +213,7 @@ def get_model_parameter( query["instrument"] = array_element_name if site is not None: query["site"] = site - return self.read_mongo_db( - query=query, - collection_name=collection, - write_files=False, - ) + return self.read_mongo_db(query=query, collection_name=collection) def get_model_parameters( self, @@ -225,7 +221,6 @@ def get_model_parameters( array_element_name, model_version, collection, - allow_missing_array_elements=False, ): """ Get model parameters from MongoDB. @@ -243,24 +238,17 @@ def get_model_parameters( Version of the model. collection: str collection of array element (e.g. telescopes, calibration_devices). - allow_missing_array_elements: bool - Allow missing array elements in the DB without raising an error. Returns ------- dict containing the parameters """ production_table = self.get_production_table_from_mongo_db(collection, model_version) - design_model = f"{names.get_array_element_type_from_name(array_element_name)}-design" - array_element_list = ( - [array_element_name] - if "-design" in array_element_name # design model must be first in list - else [design_model, array_element_name] - ) + array_element_list = self._get_array_element_list(array_element_name, production_table) pars = {} for array_element in array_element_list: - cache_key, cache_dict = self._fill_from_cache( + cache_key, cache_dict = self._read_cache( DatabaseHandler.model_parameters_cached, names.validate_site_name(site), array_element, @@ -275,11 +263,7 @@ def get_model_parameters( try: parameter_version_table = production_table["parameters"][array_element] - except KeyError as exc: - if array_element == design_model and not allow_missing_array_elements: - self._logger.error(f"Parameters for {array_element} could not be found.") - raise exc - # non-design model not defined (e.g. in collection 'configuration_sim_telarray') + except KeyError: continue pars.update( self.read_mongo_db( @@ -287,7 +271,6 @@ def get_model_parameters( parameter_version_table, array_element ), collection_name=collection, - write_files=False, ) ) DatabaseHandler.model_parameters_cached[cache_key] = pars @@ -314,62 +297,54 @@ def get_collection(self, db_name, collection_name): db_name = self._get_db_name(db_name) return DatabaseHandler.db_client[db_name][collection_name] - def export_file_db(self, db_name, dest, file_name): - """ - Get file from the DB and write to disk. - - Parameters - ---------- - db_name: str - Name of the DB to search in. - dest: str or Path - Location where to write the file to. - file_name: str - Name of the file to get. - - Returns - ------- - file_id: GridOut._id - the database ID the file was assigned when it was inserted to the DB. - - Raises - ------ - FileNotFoundError - If the desired file is not found. - + def export_model_files(self, parameters=None, file_names=None, dest=None, db_name=None): """ - db_name = self._get_db_name(db_name) + Export files from the DB to the model directory. - self._logger.debug(f"Getting {file_name} from {db_name} and writing it to {dest}") - file_path_instance = self._get_file_mongo_db(db_name, file_name) - self._write_file_from_mongo_to_disk(db_name, dest, file_path_instance) - return file_path_instance._id # pylint: disable=protected-access; - - def export_model_files(self, parameters, dest): - """ - Export all the files in a model from the DB and write them to disk. + The files to be exported can be specified by file_name or retrieved from the database + using the parameters dictionary. Parameters ---------- parameters: dict Dict of model parameters + file_names: list, str + List (or string) of file names to export dest: str or Path Location where to write the files to. + Returns + ------- + file_id: dict of GridOut._id + Dict of database IDs of files. + Raises ------ FileNotFoundError if a file in parameters.values is not found """ - if self.mongo_db_config: - for info in parameters.values(): - if not info or not info.get("file") or info["value"] is None: - continue - if Path(dest).joinpath(info["value"]).exists(): - continue - file = self._get_file_mongo_db(self._get_db_name(), info["value"]) - self._write_file_from_mongo_to_disk(self._get_db_name(), dest, file) + db_name = self._get_db_name(db_name) + + if file_names: + file_names = [file_names] if not isinstance(file_names, list) else file_names + elif parameters: + file_names = [ + info["value"] + for info in parameters.values() + if info and info.get("file") and info["value"] is not None + ] + file_names = [] + + instance_ids = {} + for file_name in file_names: + if Path(dest).joinpath(file_name).exists(): + instance_ids[file_name] = "file exits" + else: + file_path_instance = self._get_file_mongo_db(self._get_db_name(), file_name) + self._write_file_from_mongo_to_disk(self._get_db_name(), dest, file_path_instance) + instance_ids[file_name] = file_path_instance._id # pylint: disable=protected-access + return instance_ids @staticmethod def _is_file(value): @@ -390,8 +365,6 @@ def read_mongo_db( self, query, collection_name, - run_location=None, - write_files=True, ): """ Build and execute query to Read the MongoDB for a specific array element. @@ -402,12 +375,8 @@ def read_mongo_db( ---------- query: dict Dictionary describing the query to execute. - run_location: Path or str - The sim_telarray run location to write the tabulated data files into. collection_name: str The name of the collection to read from. - write_files: bool - If true, write the files to the run_location. Returns ------- @@ -432,9 +401,6 @@ def read_mongo_db( parameters[par_now] = post parameters[par_now].pop("parameter", None) parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time - if parameters[par_now]["file"] and write_files: - file = self._get_file_mongo_db(db_name, parameters[par_now]["value"]) - self._write_file_from_mongo_to_disk(db_name, run_location, file) return parameters @@ -455,7 +421,7 @@ def get_site_parameters(self, site, model_version): """ site = names.validate_site_name(site) production_table = self.get_production_table_from_mongo_db("sites", model_version) - cache_key, cache_dict = self._fill_from_cache( + cache_key, cache_dict = self._read_cache( DatabaseHandler.site_parameters_cached, self._cache_key(site, None, production_table.get("model_version")), ) @@ -474,7 +440,7 @@ def get_site_parameters(self, site, model_version): ], } DatabaseHandler.site_parameters_cached[cache_key] = self.read_mongo_db( - query=query, collection_name="sites", write_files=False + query=query, collection_name="sites" ) return DatabaseHandler.site_parameters_cached[cache_key] @@ -506,6 +472,7 @@ def get_production_table_from_mongo_db(self, collection_name, model_version): "collection": post["collection"], "model_version": post["model_version"], "parameters": post["parameters"], + "design_model": post.get("design_model", {}), "entry_date": ObjectId(post["_id"]).generation_time, } @@ -568,13 +535,11 @@ def get_simulation_configuration_parameters( return self.get_corsika_configuration_parameters(model_version) if simulation_software == "simtel": return ( - # not all array elements are present in the DB (allow for missing elements) self.get_model_parameters( site, array_element_name, model_version, collection="configuration_sim_telarray", - allow_missing_array_elements=True, ) if site and array_element_name else {} @@ -598,7 +563,7 @@ def get_corsika_configuration_parameters(self, model_version): _production_table = self.get_production_table_from_mongo_db( "configuration_corsika", model_version ) - cache_key, cache_dict = self._fill_from_cache( + cache_key, cache_dict = self._read_cache( DatabaseHandler.corsika_configuration_parameters_cached, None, None, @@ -613,7 +578,6 @@ def get_corsika_configuration_parameters(self, model_version): None, ), collection_name="configuration_corsika", - write_files=False, ) return DatabaseHandler.corsika_configuration_parameters_cached[cache_key] @@ -806,7 +770,7 @@ def add_new_parameter( self._logger.info(f"Will also add the file {file_to_insert_now} to the DB") self.insert_file_to_db(file_to_insert_now, db_name) - self._reset_parameter_cache(par_dict["site"], par_dict["instrument"], None) + self._reset_parameter_cache() def _get_db_name(self, db_name=None): """ @@ -887,11 +851,11 @@ def _cache_key(self, site=None, array_element_name=None, model_version=None, col part for part in [model_version, collection, site, array_element_name] if part ) - def _fill_from_cache( + def _read_cache( self, cache_dict, site=None, array_element_name=None, model_version=None, collection=None ): """ - Fill a parameter dict from cache. + Read parameters from cache. Parameters ---------- @@ -932,30 +896,10 @@ def _reset_production_table_cache(self, collection_name, model_version): self._cache_key(model_version=model_version, collection=collection_name), None ) - def _reset_parameter_cache(self, site, array_element_name, model_version): - """ - Reset the cache for the parameters. - - A value of 'None' for any of the parameters will reset the entire cache. - - Parameters - ---------- - site: str - Site name. - array_element_name: str - Array element name. - model_version: str - Model version. - - """ - self._logger.debug(f"Resetting cache for {site} {array_element_name} {model_version}") - if None in [site, array_element_name, model_version]: - DatabaseHandler.site_parameters_cached.clear() - DatabaseHandler.model_parameters_cached.clear() - else: - _cache_key = self._cache_key(site, array_element_name, model_version) - DatabaseHandler.site_parameters_cached.pop(_cache_key, None) - DatabaseHandler.model_parameters_cached.pop(_cache_key, None) + def _reset_parameter_cache(self): + """Reset the cache for the parameters.""" + DatabaseHandler.site_parameters_cached.clear() + DatabaseHandler.model_parameters_cached.clear() def get_collections(self, db_name=None, model_collections_only=False): """ @@ -987,3 +931,34 @@ def get_collections(self, db_name=None, model_collections_only=False): if not collection.startswith("fs.") and collection != "metadata" ] return collections + + def _get_array_element_list(self, array_element_name, production_table): + """ + Return list of array elements (add design model if needed). + + Design model is added if found in the production table. + + Parameters + ---------- + array_element_name: str + Name of the array element. + production_table: dict + Production table. + + Returns + ------- + list + List of array elements + """ + if "-design" in array_element_name: + return [array_element_name] + try: + return [ + production_table["design_model"][array_element_name], + array_element_name, + ] + except KeyError: + return [ + f"{names.get_array_element_type_from_name(array_element_name)}-design", + array_element_name, + ] diff --git a/src/simtools/db/db_model_upload.py b/src/simtools/db/db_model_upload.py index 01e0c2bebb..8782a0b01d 100644 --- a/src/simtools/db/db_model_upload.py +++ b/src/simtools/db/db_model_upload.py @@ -119,10 +119,12 @@ def add_production_tables_to_db(args_dict, db): except KeyError as exc: logger.error(f"KeyError: {exc}") raise - if "design_model" in parameter_dict: + try: model_dict[collection]["design_model"][array_element] = parameter_dict[ "design_model" - ] + ][array_element] + except KeyError: + pass for collection, data in model_dict.items(): if not data["parameters"]: From 1fda8029d7d328d961816ce223eeeed9c8ffa26b Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 19 Jan 2025 17:46:49 +0100 Subject: [PATCH 054/124] new export_model_files api --- src/simtools/model/model_parameter.py | 6 +++--- src/simtools/model/site_model.py | 4 ++-- tests/unit_tests/model/test_site_model.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index 66392a7ca1..b9f32f4a04 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -481,7 +481,7 @@ def export_model_files(self): for par in self._added_parameter_files: pars_from_db.pop(par) - self.db.export_model_files(pars_from_db, self.config_file_directory) + self.db.export_model_files(parameters=pars_from_db, dest=self.config_file_directory) self._is_exported_model_files_up_to_date = True def export_config_file(self): @@ -559,7 +559,7 @@ def export_nsb_spectrum_to_telescope_altitude_correction_file(self, model_direct Model directory to export the file to. """ self.db.export_model_files( - { + parameters={ "nsb_spectrum_at_2200m": { "value": self._simulation_config_parameters["simtel"][ "correct_nsb_spectrum_to_telescope_altitude" @@ -567,5 +567,5 @@ def export_nsb_spectrum_to_telescope_altitude_correction_file(self, model_direct "file": True, } }, - model_directory, + dest=model_directory, ) diff --git a/src/simtools/model/site_model.py b/src/simtools/model/site_model.py index 94a22ee932..e346d5cba3 100644 --- a/src/simtools/model/site_model.py +++ b/src/simtools/model/site_model.py @@ -151,11 +151,11 @@ def export_atmospheric_transmission_file(self, model_directory: Path): Model directory to export the file to. """ self.db.export_model_files( - { + parameters={ "atmospheric_transmission_file": { "value": self.get_parameter_value("atmospheric_profile"), "file": True, } }, - model_directory, + dest=model_directory, ) diff --git a/tests/unit_tests/model/test_site_model.py b/tests/unit_tests/model/test_site_model.py index 3244fb1c58..9de21bddc6 100644 --- a/tests/unit_tests/model/test_site_model.py +++ b/tests/unit_tests/model/test_site_model.py @@ -95,11 +95,11 @@ def test_export_atmospheric_transmission_file(db_config, model_version, tmp_path _south.export_atmospheric_transmission_file(model_directory) _south.db.export_model_files.assert_called_once_with( - { + parameters={ "atmospheric_transmission_file": { "value": "test_atmospheric_profile", "file": True, } }, - model_directory, + dest=model_directory, ) From bc5073953195f1fa5999878f13e7e0210e9784b9 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 19 Jan 2025 20:07:51 +0100 Subject: [PATCH 055/124] MB to MByte --- src/simtools/corsika/corsika_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/simtools/corsika/corsika_config.py b/src/simtools/corsika/corsika_config.py index d6997c1ba9..ae5709ea03 100644 --- a/src/simtools/corsika/corsika_config.py +++ b/src/simtools/corsika/corsika_config.py @@ -316,7 +316,9 @@ def _corsika_configuration_debugging_parameters(self): def _input_config_io_buff(self, entry): """Return IO_BUFFER parameter CORSIKA format.""" - return f"{entry['value']}{entry['unit']}" + unit_map = {"Mbyte": "MB", "kbyte": "kB", "Gbyte": "GB"} + unit = unit_map.get(str(u.Unit(entry["unit"])), entry["unit"]) + return f"{entry['value']}{unit}" def _rotate_azimuth_by_180deg(self, az, correct_for_geomagnetic_field_alignment=True): """ From ef2e53595bac1c4105c845932d9d958004b8c608 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 19 Jan 2025 20:08:04 +0100 Subject: [PATCH 056/124] default is MB --- .../model_parameters/corsika_iact_io_buffer.schema.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml b/src/simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml index 9e6d5a12e9..25a2c15141 100644 --- a/src/simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml +++ b/src/simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml @@ -10,8 +10,8 @@ description: |- Maximum size of data blocks written by the CORSIKA IACT module. data: - type: int - unit: byte - default: 100000000 + unit: MB + default: 1000 activity: setting: - SetParameterFromExternal From bb348968990ae9082355de5eac5a517b10605d2d Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 19 Jan 2025 20:10:46 +0100 Subject: [PATCH 057/124] fix file uploading --- src/simtools/db/db_handler.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index 8d89e9f104..2e90d3ad1e 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -334,9 +334,9 @@ def export_model_files(self, parameters=None, file_names=None, dest=None, db_nam for info in parameters.values() if info and info.get("file") and info["value"] is not None ] - file_names = [] instance_ids = {} + self._logger.info(f"Exporting the following files: {file_names}") for file_name in file_names: if Path(dest).joinpath(file_name).exists(): instance_ids[file_name] = "file exits" @@ -346,11 +346,6 @@ def export_model_files(self, parameters=None, file_names=None, dest=None, db_nam instance_ids[file_name] = file_path_instance._id # pylint: disable=protected-access return instance_ids - @staticmethod - def _is_file(value): - """Verify if a parameter value is a file name.""" - return any(ext in str(value) for ext in DatabaseHandler.ALLOWED_FILE_EXTENSIONS) - def _get_query_from_parameter_version_table(self, parameter_version_table, array_element_name): """Return query based on parameter version table.""" return { From 67a5dd56553c404a007e3a947ac84748d91bedcc Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 19 Jan 2025 20:35:54 +0100 Subject: [PATCH 058/124] fixed docstring --- src/simtools/db/db_handler.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index 2e90d3ad1e..b0c3b10f60 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -356,15 +356,9 @@ def _get_query_from_parameter_version_table(self, parameter_version_table, array ], } - def read_mongo_db( - self, - query, - collection_name, - ): + def read_mongo_db(self, query, collection_name): """ - Build and execute query to Read the MongoDB for a specific array element. - - Also writes the files listed in the parameter values into the sim_telarray run location + Build and execute query to read the MongoDB for a collection. Parameters ---------- @@ -380,7 +374,7 @@ def read_mongo_db( Raises ------ ValueError - if query returned no results or if the collection is not found in the production table. + if query returned no results. """ db_name = self._get_db_name() collection = self.get_collection(db_name, collection_name) @@ -396,7 +390,6 @@ def read_mongo_db( parameters[par_now] = post parameters[par_now].pop("parameter", None) parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time - return parameters def get_site_parameters(self, site, model_version): From 4f8e87797b3c09e3e98f15703362320790b2c9fe Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 19 Jan 2025 21:52:09 +0100 Subject: [PATCH 059/124] simplificatons --- .../applications/db_get_parameter_from_db.py | 14 +- src/simtools/db/db_handler.py | 145 +++++------------- src/simtools/model/model_parameter.py | 11 +- 3 files changed, 52 insertions(+), 118 deletions(-) diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 4df2976726..3c1de26355 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -91,15 +91,16 @@ def main(): # noqa: D103 db = db_handler.DatabaseHandler(mongo_db_config=db_config) + # get parameter using 'parameter_version' if args_dict["parameter_version"] is not None: pars = db.get_model_parameter( parameter=args_dict["parameter"], parameter_version=args_dict["parameter_version"], site=args_dict["site"], array_element_name=args_dict["telescope"], - collection=(args_dict["db_collection"] if args_dict["db_collection"] else "telescopes"), + collection=args_dict.get("db_collection", "telescopes"), ) - + # get parameter using 'model_version' elif args_dict["telescope"]: pars = db.get_model_parameters( site=args_dict["site"], @@ -111,11 +112,12 @@ def main(): # noqa: D103 else "telescopes" ), ) - elif args_dict["db_collection"] == "configuration_corsika": - pars = db.get_corsika_configuration_parameters(model_version=args_dict["model_version"]) else: - pars = db.get_site_parameters( - site=args_dict["site"], model_version=args_dict["model_version"] + pars = db.get_model_parameters( + site=args_dict.get("site"), + model_version=args_dict["model_version"], + collection=args_dict["db_collection"], + array_element_name=None, ) param = args_dict["parameter"] if param not in pars: diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index b0c3b10f60..da6d3ec6c7 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -226,7 +226,7 @@ def get_model_parameters( Get model parameters from MongoDB. An array element can be e.g., a telescope or a calibration device. - Always queries parameters for design and for the specified array element (if necessary). + Queries parameters for design and for the specified array element (if necessary). Parameters ---------- @@ -244,13 +244,15 @@ def get_model_parameters( dict containing the parameters """ production_table = self.get_production_table_from_mongo_db(collection, model_version) - array_element_list = self._get_array_element_list(array_element_name, production_table) + array_element_list = self._get_array_element_list( + array_element_name, site, production_table, collection + ) pars = {} for array_element in array_element_list: cache_key, cache_dict = self._read_cache( DatabaseHandler.model_parameters_cached, - names.validate_site_name(site), + names.validate_site_name(site) if site else None, array_element, model_version, collection, @@ -265,15 +267,13 @@ def get_model_parameters( parameter_version_table = production_table["parameters"][array_element] except KeyError: continue - pars.update( - self.read_mongo_db( - query=self._get_query_from_parameter_version_table( - parameter_version_table, array_element - ), - collection_name=collection, - ) + DatabaseHandler.model_parameters_cached[cache_key] = self.read_mongo_db( + query=self._get_query_from_parameter_version_table( + parameter_version_table, array_element, site + ), + collection_name=collection, ) - DatabaseHandler.model_parameters_cached[cache_key] = pars + pars.update(DatabaseHandler.model_parameters_cached[cache_key]) return pars @@ -346,15 +346,21 @@ def export_model_files(self, parameters=None, file_names=None, dest=None, db_nam instance_ids[file_name] = file_path_instance._id # pylint: disable=protected-access return instance_ids - def _get_query_from_parameter_version_table(self, parameter_version_table, array_element_name): + def _get_query_from_parameter_version_table( + self, parameter_version_table, array_element_name, site + ): """Return query based on parameter version table.""" - return { - "instrument": array_element_name, + query_dict = { "$or": [ {"parameter": param, "parameter_version": version} for param, version in parameter_version_table.items() ], } + if array_element_name not in {"xSTx-design"} and not array_element_name.startswith("OBS-"): + query_dict["instrument"] = array_element_name + if site: + query_dict["site"] = site + return query_dict def read_mongo_db(self, query, collection_name): """ @@ -392,46 +398,6 @@ def read_mongo_db(self, query, collection_name): parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time return parameters - def get_site_parameters(self, site, model_version): - """ - Get site parameters from MongoDB. - - Parameters - ---------- - site: str - Site name. - model_version: str - Version of the model. - - Returns - ------- - dict containing the parameters - """ - site = names.validate_site_name(site) - production_table = self.get_production_table_from_mongo_db("sites", model_version) - cache_key, cache_dict = self._read_cache( - DatabaseHandler.site_parameters_cached, - self._cache_key(site, None, production_table.get("model_version")), - ) - if cache_dict: - return cache_dict - - try: - parameter_query = production_table["parameters"][f"OBS-{site}"] - except KeyError as exc: - raise ValueError(f"Site {site} not found in the production table") from exc - query = { - "site": site, - "$or": [ - {"parameter": param, "parameter_version": version} - for param, version in parameter_query.items() - ], - } - DatabaseHandler.site_parameters_cached[cache_key] = self.read_mongo_db( - query=query, collection_name="sites" - ) - return DatabaseHandler.site_parameters_cached[cache_key] - def get_production_table_from_mongo_db(self, collection_name, model_version): """ Get production table from MongoDB. @@ -520,7 +486,12 @@ def get_simulation_configuration_parameters( if simulation_software is not valid. """ if simulation_software == "corsika": - return self.get_corsika_configuration_parameters(model_version) + return self.get_model_parameters( + site, + array_element_name, + model_version, + collection="configuration_corsika", + ) if simulation_software == "simtel": return ( self.get_model_parameters( @@ -534,41 +505,6 @@ def get_simulation_configuration_parameters( ) raise ValueError(f"Unknown simulation software: {simulation_software}") - def get_corsika_configuration_parameters(self, model_version): - """ - Get CORSIKA configuration parameters from the DB. - - Parameters - ---------- - model_version : str - Version of the model. - - Returns - ------- - dict - Configuration parameters for CORSIKA - """ - _production_table = self.get_production_table_from_mongo_db( - "configuration_corsika", model_version - ) - cache_key, cache_dict = self._read_cache( - DatabaseHandler.corsika_configuration_parameters_cached, - None, - None, - _production_table.get("model_version"), - ) - if cache_dict: - return cache_dict - - DatabaseHandler.corsika_configuration_parameters_cached[cache_key] = self.read_mongo_db( - query=self._get_query_from_parameter_version_table( - _production_table["parameters"]["xSTx-design"], # design model for all telescopes - None, - ), - collection_name="configuration_corsika", - ) - return DatabaseHandler.corsika_configuration_parameters_cached[cache_key] - @staticmethod def _get_file_mongo_db(db_name, file_name): """ @@ -701,9 +637,7 @@ def add_production_table(self, db_name, production_table): collection = self.get_collection(db_name, "production_tables") self._logger.info(f"Adding production for {production_table.get('collection')} to to DB") collection.insert_one(production_table) - self._reset_production_table_cache( - production_table.get("collection"), production_table.get("model_version") - ) + DatabaseHandler.production_table_cached.clear() def add_new_parameter( self, @@ -869,21 +803,6 @@ def _read_cache( except KeyError: return cache_key, None - def _reset_production_table_cache(self, collection_name, model_version): - """ - Reset the cache for the production tables. - - Parameters - ---------- - collection_name: str - Collection name. - model_version: str - Model version. - """ - DatabaseHandler.production_table_cached.pop( - self._cache_key(model_version=model_version, collection=collection_name), None - ) - def _reset_parameter_cache(self): """Reset the cache for the parameters.""" DatabaseHandler.site_parameters_cached.clear() @@ -920,7 +839,7 @@ def get_collections(self, db_name=None, model_collections_only=False): ] return collections - def _get_array_element_list(self, array_element_name, production_table): + def _get_array_element_list(self, array_element_name, site, production_table, collection): """ Return list of array elements (add design model if needed). @@ -930,14 +849,22 @@ def _get_array_element_list(self, array_element_name, production_table): ---------- array_element_name: str Name of the array element. + site: str + Site name. production_table: dict Production table. + collection: str + collection of array element (e.g. telescopes, calibration_devices). Returns ------- list List of array elements """ + if collection == "configuration_corsika": + return ["xSTx-design"] # placeholder for any telescope design + if collection == "sites": + return [f"OBS-{site}"] if "-design" in array_element_name: return [array_element_name] try: diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index b9f32f4a04..7a4b42ecdf 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -309,9 +309,14 @@ def _load_parameters_from_db(self): ) if self.site is not None: - _site_pars = self.db.get_site_parameters(self.site, self.model_version) - self._parameters.update(_site_pars) - + self._parameters.update( + self.db.get_model_parameters( + self.site, + None, + self.model_version, + "sites", + ) + ) self._load_simulation_software_parameter() def set_extra_label(self, extra_label): From ed687b2f451460d7de0058513fe6afc0952bf3c1 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 20 Jan 2025 07:40:55 +0100 Subject: [PATCH 060/124] fix reading of corsika parameters --- src/simtools/db/db_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index da6d3ec6c7..c173b76dbb 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -356,7 +356,7 @@ def _get_query_from_parameter_version_table( for param, version in parameter_version_table.items() ], } - if array_element_name not in {"xSTx-design"} and not array_element_name.startswith("OBS-"): + if array_element_name: query_dict["instrument"] = array_element_name if site: query_dict["site"] = site @@ -487,8 +487,8 @@ def get_simulation_configuration_parameters( """ if simulation_software == "corsika": return self.get_model_parameters( - site, - array_element_name, + None, + None, model_version, collection="configuration_corsika", ) From 6752c98323de1e0dc80d5037ceea4892af335c8a Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 20 Jan 2025 11:10:51 +0100 Subject: [PATCH 061/124] improved logger message --- src/simtools/model/model_parameter.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index 7a4b42ecdf..36eae9da8a 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -292,10 +292,11 @@ def _load_simulation_software_parameter(self): simulation_software=simulation_software, ) ) - except ValueError: + except ValueError as exc: self._logger.warning( f"No {simulation_software} parameters found for " - f"{self.site}, {self.name} (model version {self.model_version})." + f"{self.site}, {self.name} (model version {self.model_version}). " + f" (Query {exc})" ) def _load_parameters_from_db(self): From aab80f8e2439e8a3a2c8744d46f92aa0faeb0aee Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 20 Jan 2025 11:11:02 +0100 Subject: [PATCH 062/124] debug --- .../config/simulate_light_emission_variable.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration_tests/config/simulate_light_emission_variable.yml b/tests/integration_tests/config/simulate_light_emission_variable.yml index 7b744927d4..9af989e31c 100644 --- a/tests/integration_tests/config/simulate_light_emission_variable.yml +++ b/tests/integration_tests/config/simulate_light_emission_variable.yml @@ -10,5 +10,6 @@ CTA_SIMPIPE: MODEL_VERSION: 6.0.0 LIGHT_SOURCE_TYPE: led OUTPUT_PATH: simtools-output + LOG_LEVEL: DEBUG INTEGRATION_TESTS: - OUTPUT_FILE: logfile.log From 0fc492a85c2a8aaa989990f98ca99aa74882c618 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 20 Jan 2025 11:38:59 +0100 Subject: [PATCH 063/124] logging --- src/simtools/db/db_handler.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index c173b76dbb..f58c6f0bfa 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -336,7 +336,6 @@ def export_model_files(self, parameters=None, file_names=None, dest=None, db_nam ] instance_ids = {} - self._logger.info(f"Exporting the following files: {file_names}") for file_name in file_names: if Path(dest).joinpath(file_name).exists(): instance_ids[file_name] = "file exits" @@ -356,7 +355,7 @@ def _get_query_from_parameter_version_table( for param, version in parameter_version_table.items() ], } - if array_element_name: + if array_element_name and array_element_name != "xSTx-design": query_dict["instrument"] = array_element_name if site: query_dict["site"] = site @@ -387,8 +386,7 @@ def read_mongo_db(self, query, collection_name): posts = list(collection.find(query).sort("parameter", ASCENDING)) if not posts: raise ValueError( - "The following query returned zero results! Check the input data and rerun.\n", - query, + f"The following query for {collection_name} returned zero results: {query} " ) parameters = {} for post in posts: @@ -862,7 +860,7 @@ def _get_array_element_list(self, array_element_name, site, production_table, co List of array elements """ if collection == "configuration_corsika": - return ["xSTx-design"] # placeholder for any telescope design + return ["xSTx-design"] # placeholder to ignore 'instrument' field in query. if collection == "sites": return [f"OBS-{site}"] if "-design" in array_element_name: From 3a6c7ce71f70ad4e16ddf20c19ef8474eea7cd9c Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 20 Jan 2025 13:28:38 +0100 Subject: [PATCH 064/124] remove ulr --- tests/conftest.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index cafb11c902..94781c303f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,7 +76,6 @@ def _mock_settings_env_vars(tmp_test_directory): "SIMTOOLS_DB_API_PORT": "42", "SIMTOOLS_DB_SERVER": "abc@def.de", "SIMTOOLS_DB_SIMULATION_MODEL": "sim_model", - "SIMTOOLS_DB_SIMULATION_MODEL_URL": _url, }, clear=True, ): @@ -149,7 +148,6 @@ def db_config(): "db_api_port", "db_server", "db_simulation_model", - "db_simulation_model_url", ) for _para in _db_para: if _para not in mongo_db_config: @@ -159,20 +157,6 @@ def db_config(): return mongo_db_config -@pytest.fixture -def simulation_model_url(db_config): - """Simulation model URL from .env file or default.""" - if ( - db_config["db_simulation_model_url"] is None - or len(db_config["db_simulation_model_url"]) == 0 - ): - db_config["db_simulation_model_url"] = ( - "https://gitlab.cta-observatory.org/cta-science/simulations/" - "simulation-model/model_parameters/-/raw/main/" - ) - return db_config["db_simulation_model_url"] - - @pytest.fixture def db(db_config): """Database object with configuration from .env file.""" From 84adc115f79d748704b53f6f5a7a598ae3bf04ed Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 20 Jan 2025 13:29:03 +0100 Subject: [PATCH 065/124] remove copy_array_element --- src/simtools/db/db_handler.py | 69 ----------------------------------- 1 file changed, 69 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index f58c6f0bfa..7686436187 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -10,7 +10,6 @@ from bson.objectid import ObjectId from packaging.version import Version from pymongo import ASCENDING, DESCENDING, MongoClient -from pymongo.errors import BulkWriteError from simtools.data_model import validate_data from simtools.io_operations import io_handler @@ -552,74 +551,6 @@ def _write_file_from_mongo_to_disk(db_name, path, file): with open(Path(path).joinpath(file.filename), "wb") as output_file: fs_output.download_to_stream_by_name(file.filename, output_file) - def copy_array_element( - self, - db_name, - element_to_copy, - version_to_copy, - new_array_element_name, - collection_name="telescopes", - db_to_copy_to=None, - collection_to_copy_to=None, - ): - """ - Copy a full array element configuration to a new array element name. - - Only a specific version is copied. - This function should be rarely used and is intended to simplify unit tests. - - Parameters - ---------- - db_name: str - the name of the DB to copy from - element_to_copy: str - The array element to copy - version_to_copy: str - The version of the configuration to copy - new_array_element_name: str - The name of the new array element - collection_name: str - The name of the collection to copy from. - db_to_copy_to: str - The name of the DB to copy to. - collection_to_copy_to: str - The name of the collection to copy to. - - Raises - ------ - BulkWriteError - """ - db_name = self._get_db_name(db_name) - if db_to_copy_to is None: - db_to_copy_to = db_name - - if collection_to_copy_to is None: - collection_to_copy_to = collection_name - - self._logger.info( - f"Copying version {version_to_copy} of {element_to_copy} " - f"to the new array element {new_array_element_name} in the {db_to_copy_to} DB" - ) - - collection = self.get_collection(db_name, collection_name) - db_entries = [] - - query = { - "instrument": element_to_copy, - "version": version_to_copy, - } - for post in collection.find(query): - post["instrument"] = new_array_element_name - post.pop("_id", None) - db_entries.append(post) - - self._logger.info(f"Creating new array element {new_array_element_name}") - collection = self.get_collection(db_to_copy_to, collection_to_copy_to) - try: - collection.insert_many(db_entries) - except BulkWriteError as exc: - raise BulkWriteError(str(exc.details)) from exc - def add_production_table(self, db_name, production_table): """ Add a production table for a given model version to the DB. From dbabe100972ad78d44f943438be71a1fbf3bf9cd Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 20 Jan 2025 14:48:51 +0100 Subject: [PATCH 066/124] unit tests --- src/simtools/db/db_handler.py | 76 +-- tests/unit_tests/db/test_db_handler.py | 789 +++++++++++++++++++------ 2 files changed, 635 insertions(+), 230 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index 7686436187..fed475909f 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -9,7 +9,7 @@ import jsonschema from bson.objectid import ObjectId from packaging.version import Version -from pymongo import ASCENDING, DESCENDING, MongoClient +from pymongo import MongoClient from simtools.data_model import validate_data from simtools.io_operations import io_handler @@ -212,7 +212,7 @@ def get_model_parameter( query["instrument"] = array_element_name if site is not None: query["site"] = site - return self.read_mongo_db(query=query, collection_name=collection) + return self._read_mongo_db(query=query, collection_name=collection) def get_model_parameters( self, @@ -266,7 +266,7 @@ def get_model_parameters( parameter_version_table = production_table["parameters"][array_element] except KeyError: continue - DatabaseHandler.model_parameters_cached[cache_key] = self.read_mongo_db( + DatabaseHandler.model_parameters_cached[cache_key] = self._read_mongo_db( query=self._get_query_from_parameter_version_table( parameter_version_table, array_element, site ), @@ -296,6 +296,37 @@ def get_collection(self, db_name, collection_name): db_name = self._get_db_name(db_name) return DatabaseHandler.db_client[db_name][collection_name] + def get_collections(self, db_name=None, model_collections_only=False): + """ + List of collections in the DB. + + Parameters + ---------- + db_name: str + Database name. + + Returns + ------- + list + List of collection names + model_collections_only: bool + If True, only return model collections (i.e. exclude fs.files, fs.chunks, metadata) + + """ + db_name = db_name or self._get_db_name() + if db_name not in self.list_of_collections: + self.list_of_collections[db_name] = DatabaseHandler.db_client[ + db_name + ].list_collection_names() + collections = self.list_of_collections[db_name] + if model_collections_only: + return [ + collection + for collection in collections + if not collection.startswith("fs.") and collection != "metadata" + ] + return collections + def export_model_files(self, parameters=None, file_names=None, dest=None, db_name=None): """ Export files from the DB to the model directory. @@ -360,7 +391,7 @@ def _get_query_from_parameter_version_table( query_dict["site"] = site return query_dict - def read_mongo_db(self, query, collection_name): + def _read_mongo_db(self, query, collection_name): """ Build and execute query to read the MongoDB for a collection. @@ -382,7 +413,7 @@ def read_mongo_db(self, query, collection_name): """ db_name = self._get_db_name() collection = self.get_collection(db_name, collection_name) - posts = list(collection.find(query).sort("parameter", ASCENDING)) + posts = list(collection.find(query)) if not posts: raise ValueError( f"The following query for {collection_name} returned zero results: {query} " @@ -393,7 +424,7 @@ def read_mongo_db(self, query, collection_name): parameters[par_now] = post parameters[par_now].pop("parameter", None) parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time - return parameters + return {k: parameters[k] for k in sorted(parameters)} def get_production_table_from_mongo_db(self, collection_name, model_version): """ @@ -415,7 +446,7 @@ def get_production_table_from_mongo_db(self, collection_name, model_version): query = {"model_version": model_version, "collection": collection_name} collection = self.get_collection(self._get_db_name(), "production_tables") - post = collection.find_one(query, sort=[("_id", DESCENDING)]) + post = collection.find_one(query) if not post: raise ValueError(f"The following query returned zero results: {query}") @@ -737,37 +768,6 @@ def _reset_parameter_cache(self): DatabaseHandler.site_parameters_cached.clear() DatabaseHandler.model_parameters_cached.clear() - def get_collections(self, db_name=None, model_collections_only=False): - """ - List of collections in the DB. - - Parameters - ---------- - db_name: str - Database name. - - Returns - ------- - list - List of collection names - model_collections_only: bool - If True, only return model collections (i.e. exclude fs.files, fs.chunks, metadata) - - """ - db_name = db_name or self._get_db_name() - if db_name not in self.list_of_collections: - self.list_of_collections[db_name] = DatabaseHandler.db_client[ - db_name - ].list_collection_names() - collections = self.list_of_collections[db_name] - if model_collections_only: - return [ - collection - for collection in collections - if not collection.startswith("fs.") and collection != "metadata" - ] - return collections - def _get_array_element_list(self, array_element_name, site, production_table, collection): """ Return list of array elements (add design model if needed). diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index d54e689bb3..d15ed16370 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -4,14 +4,31 @@ import logging import re import uuid +from unittest.mock import call import pytest +from bson.objectid import ObjectId from simtools.db import db_handler +from simtools.utils import names logger = logging.getLogger() +@pytest.fixture(autouse=True) +def reset_db_client(): + """Reset db_client before each test.""" + # If using the class-level db_client: + db_handler.DatabaseHandler.db_client = None + db_handler.production_table_cached = {} + db_handler.model_parameters_cached = {} + yield # allows the test to run + # After the test, reset any side-effects (if necessary): + db_handler.DatabaseHandler.db_client = None + db_handler.production_table_cached.clear() + db_handler.model_parameters_cached.clear() + + @pytest.fixture def random_id(): return uuid.uuid4().hex @@ -38,6 +55,7 @@ def fs_files(): return "fs.files" +# TODO remove? @pytest.fixture def _db_cleanup_file_sandbox(db_no_config_file, random_id, fs_files): yield @@ -47,6 +65,19 @@ def _db_cleanup_file_sandbox(db_no_config_file, random_id, fs_files): db_no_config_file.db_client[f"sandbox_{random_id}"][fs_files].drop() +def test_set_up_connection_no_config(): + """Test _set_up_connection with no configuration.""" + db = db_handler.DatabaseHandler(mongo_db_config=None) + db._set_up_connection() + assert db_handler.DatabaseHandler.db_client is None + + +def test_set_up_connection_with_config(db): + """Test _set_up_connection with valid configuration.""" + db._set_up_connection() + assert isinstance(db_handler.DatabaseHandler.db_client, db_handler.MongoClient) + + def test_valid_db_config(db, db_config): assert db.mongo_db_config == db._validate_mongo_db_config(db_config) assert db._validate_mongo_db_config(None) is None @@ -59,6 +90,28 @@ def test_valid_db_config(db, db_config): db._validate_mongo_db_config({"wrong_config": "wrong"}) +def test_open_mongo_db_direct_connection(mocker, db, db_config): + """Test _open_mongo_db with direct connection configuration.""" + db_config["db_server"] = "localhost" + mock_mongo_client = mocker.patch( + "simtools.db.db_handler.MongoClient", return_value="mock_client" + ) + db.mongo_db_config = db_config + client = db._open_mongo_db() + assert client == "mock_client" + mock_mongo_client.assert_called_once_with( + db_config["db_server"], + port=db_config["db_api_port"], + username=db_config["db_api_user"], + password=db_config["db_api_pw"], + authSource=db_config.get("db_api_authentication_database", "admin"), + directConnection=True, + ssl=False, + tlsallowinvalidhostnames=True, + tlsallowinvalidcertificates=True, + ) + + def test_find_latest_simulation_model_db(db, db_no_config_file, mocker): db_no_config_file._find_latest_simulation_model_db() @@ -92,91 +145,523 @@ def test_find_latest_simulation_model_db(db, db_no_config_file, mocker): assert db_copy.mongo_db_config["db_simulation_model"] == "CTAO-Simulation-Model-v0-3-19" -def test_reading_db_lst_without_simulation_repo(db, model_version): +def test_get_model_parameter(db, mocker): + """Test get_model_parameter method.""" + mock_read_mongo_db = mocker.patch.object( + db, "_read_mongo_db", return_value={"parameter": "value"} + ) - db_copy = copy.deepcopy(db) - db_copy.mongo_db_config["db_simulation_model_url"] = None - pars = db.get_model_parameters("North", "LSTN-01", model_version, collection="telescopes") - assert pars["parabolic_dish"]["value"] == 1 - assert pars["camera_pixels"]["value"] == 1855 + parameter = "camera_pixels" + parameter_version = "0.0.1" + site = "North" + array_element_name = "LSTN-01" + collection = "telescopes" + result = db.get_model_parameter( + parameter, parameter_version, site, array_element_name, collection + ) -def test_reading_db_lst(db, model_version): - logger.info("----Testing reading LST-North-----") - pars = db.get_model_parameters("North", "LSTN-01", model_version, collection="telescopes") - if db.mongo_db_config: - assert pars["parabolic_dish"]["value"] == 1 - assert pars["camera_pixels"]["value"] == 1855 - else: - assert pars["parabolic_dish"] == 1 - assert pars["camera_pixels"] == 1855 + query = { + "parameter_version": parameter_version, + "parameter": parameter, + "instrument": array_element_name, + "site": site, + } + mock_read_mongo_db.assert_called_once_with(query=query, collection_name=collection) + assert result == {"parameter": "value"} -def test_reading_db_mst_nc(db, model_version): - logger.info("----Testing reading MST-North-----") - pars = db.get_model_parameters("North", "MSTN-design", model_version, collection="telescopes") - if db.mongo_db_config: - assert pars["camera_pixels"]["value"] == 1855 - else: - assert pars["camera_pixels"] == 1855 +def test_get_model_parameter_no_site(db, mocker): + """Test get_model_parameter method without site.""" + mock_read_mongo_db = mocker.patch.object( + db, "_read_mongo_db", return_value={"parameter": "value"} + ) -def test_reading_db_mst_fc(db, model_version): - logger.info("----Testing reading MST-South-----") - pars = db.get_model_parameters("South", "MSTS-design", model_version, collection="telescopes") - if db.mongo_db_config: - assert pars["camera_pixels"]["value"] == 1764 - else: - assert pars["camera_pixels"] == 1764 + parameter = "mirror_list" + parameter_version = "1.0.0" + site = None + array_element_name = "LSTS-01" + collection = "telescopes" + result = db.get_model_parameter( + parameter, parameter_version, site, array_element_name, collection + ) -def test_reading_db_sst(db, model_version): - logger.info("----Testing reading SST-----") - pars = db.get_model_parameters("South", "SSTS-design", model_version, collection="telescopes") - if db.mongo_db_config: - assert pars["camera_pixels"]["value"] == 2048 - else: - assert pars["camera_pixels"] == 2048 + query = { + "parameter_version": parameter_version, + "parameter": parameter, + "instrument": array_element_name, + } + mock_read_mongo_db.assert_called_once_with(query=query, collection_name=collection) + assert result == {"parameter": "value"} -def test_get_sim_telarray_configuration_parameters(db, model_version): - _pars = db.get_model_parameters( - "North", "LSTN-01", model_version, collection="configuration_sim_telarray" +def test_get_model_parameter_no_array_element_name(db, mocker): + """Test get_model_parameter method without array element name.""" + mock_read_mongo_db = mocker.patch.object( + db, "_read_mongo_db", return_value={"parameter": "value"} ) - assert "min_photoelectrons" in _pars - _pars = db.get_model_parameters( - "North", "LSTN-design", model_version, collection="configuration_sim_telarray" + parameter = "corsika_iact_io_buffer" + parameter_version = "10.11.12" + site = "South" + array_element_name = None + collection = "configuration_corsika" + + result = db.get_model_parameter( + parameter, parameter_version, site, array_element_name, collection ) - assert "min_photoelectrons" in _pars + query = { + "parameter_version": parameter_version, + "parameter": parameter, + "site": site, + } + + mock_read_mongo_db.assert_called_once_with(query=query, collection_name=collection) + assert result == {"parameter": "value"} -@pytest.mark.usefixtures("_db_cleanup") -def test_copy_array_element_db(db, random_id, io_handler, model_version): - logger.info("----Testing copying a whole telescope-----") - db.copy_array_element( - db_name=None, - element_to_copy="LSTN-01", - version_to_copy=model_version, - new_array_element_name="LSTN-test", - collection_name="telescopes", - db_to_copy_to=f"sandbox_{random_id}", - collection_to_copy_to="telescopes", + +def test_get_model_parameters(db, mocker): + """Test get_model_parameters method.""" + mock_get_production_table = mocker.patch.object( + db, + "get_production_table_from_mongo_db", + return_value={"parameters": {"LSTN-01": {"param1": "v1"}}}, ) - pars = db.read_mongo_db( - db_name=f"sandbox_{random_id}", - array_element_name="LSTN-test", - model_version=model_version, - run_location=io_handler.get_output_directory(sub_dir="model"), - collection_name="telescopes", - write_files=False, + mock_get_array_element_list = mocker.patch.object( + db, "_get_array_element_list", return_value=["LSTN-design", "LSTN-01"] + ) + mock_read_cache = mocker.patch.object(db, "_read_cache", return_value=("cache_key", None)) + mock_read_mongo_db = mocker.patch.object( + db, "_read_mongo_db", return_value={"param1": {"value": "value1"}} + ) + + site = "North" + array_element_name = "LSTN-01" + model_version = "1.0.0" + collection = "telescopes" + + result = db.get_model_parameters(site, array_element_name, model_version, collection) + + mock_get_production_table.assert_called_once_with(collection, model_version) + mock_get_array_element_list.assert_called_once_with( + array_element_name, site, {"parameters": {"LSTN-01": {"param1": "v1"}}}, collection + ) + mock_read_cache.assert_has_calls( + [ + call( + db_handler.DatabaseHandler.model_parameters_cached, + names.validate_site_name(site), + "LSTN-design", + model_version, + collection, + ), + call( + db_handler.DatabaseHandler.model_parameters_cached, + names.validate_site_name(site), + "LSTN-01", + model_version, + collection, + ), + ] + ) + mock_read_mongo_db.assert_called_once_with( + query={ + "$or": [{"parameter": "param1", "parameter_version": "v1"}], + "instrument": "LSTN-01", + "site": site, + }, + collection_name=collection, + ) + assert result == {"param1": {"value": "value1"}} + + +def test_get_model_parameters_with_cache(db, mocker): + """Test get_model_parameters method with cache.""" + mock_get_production_table = mocker.patch.object( + db, + "get_production_table_from_mongo_db", + return_value={"parameters": {"LSTN-01": {"param1": "v1"}}}, + ) + mock_get_array_element_list = mocker.patch.object( + db, "_get_array_element_list", return_value=["LSTN-01"] + ) + mock_read_cache = mocker.patch.object( + db, "_read_cache", return_value=("cache_key", {"param1": {"value": "cached_value"}}) + ) + + site = "North" + array_element_name = "LSTN-01" + model_version = "1.0.0" + collection = "telescopes" + + result = db.get_model_parameters(site, array_element_name, model_version, collection) + + mock_get_production_table.assert_called_once_with(collection, model_version) + mock_get_array_element_list.assert_called_once_with( + array_element_name, site, {"parameters": {"LSTN-01": {"param1": "v1"}}}, collection + ) + mock_read_cache.assert_called_once_with( + db_handler.DatabaseHandler.model_parameters_cached, + names.validate_site_name(site), + "LSTN-01", + model_version, + collection, + ) + assert result == {"param1": {"value": "cached_value"}} + + +def test_get_model_parameters_no_parameters(db, mocker): + """Test get_model_parameters method with no parameters.""" + mock_get_production_table = mocker.patch.object( + db, "get_production_table_from_mongo_db", return_value={"parameters": {}} + ) + mock_get_array_element_list = mocker.patch.object( + db, "_get_array_element_list", return_value=["LSTN-01"] ) - assert pars["camera_pixels"]["value"] == 1855 + mock_read_cache = mocker.patch.object(db, "_read_cache", return_value=("cache_key", None)) + + site = "North" + array_element_name = "LSTN-01" + model_version = "1.0.0" + collection = "telescopes" + + result = db.get_model_parameters(site, array_element_name, model_version, collection) + + mock_get_production_table.assert_called_once_with(collection, model_version) + mock_get_array_element_list.assert_called_once_with( + array_element_name, site, {"parameters": {}}, collection + ) + mock_read_cache.assert_called_once_with( + db_handler.DatabaseHandler.model_parameters_cached, + names.validate_site_name(site), + "LSTN-01", + model_version, + collection, + ) + assert result == {} + + +def test_get_collection(db, mocker): + """Test get_collection method.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {"test_db": {"test_collection": "mock_collection"}} + ) + + db_name = "test_db" + collection_name = "test_collection" + + result = db.get_collection(db_name, collection_name) + + mock_get_db_name.assert_called_once_with(db_name) + assert result == "mock_collection" + + +def test_get_collections(db, db_config, fs_files): + collections = db.get_collections() + assert isinstance(collections, list) + assert "telescopes" in collections + + collections_from_name = db.get_collections(db_config["db_simulation_model"]) + assert isinstance(collections_from_name, list) + assert "telescopes" in collections_from_name + assert fs_files in collections_from_name + + collections_no_model = db.get_collections(db_config["db_simulation_model"], True) + assert isinstance(collections_no_model, list) + assert "telescopes" in collections_no_model + assert fs_files not in collections_no_model + assert "metadata" not in collections_no_model + + +def test_export_model_files_with_file_names(db, mocker): + """Test export_model_files method with file names.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_file_mongo_db = mocker.patch.object( + db, "_get_file_mongo_db", return_value=mocker.Mock(_id="file_id") + ) + mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") + + file_names = ["file1.dat", "file2.dat"] + dest = "/tmp" + + result = db.export_model_files(file_names=file_names, dest=dest) + + mock_get_db_name.assert_called() + mock_get_file_mongo_db.assert_has_calls( + [call("test_db", "file1.dat"), call("test_db", "file2.dat")] + ) + mock_write_file_from_mongo_to_disk.assert_has_calls( + [ + call("test_db", dest, mock_get_file_mongo_db.return_value), + call("test_db", dest, mock_get_file_mongo_db.return_value), + ] + ) + assert result == {"file1.dat": "file_id", "file2.dat": "file_id"} + + +def test_export_model_files_with_parameters(db, mocker): + """Test export_model_files method with parameters.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_file_mongo_db = mocker.patch.object( + db, "_get_file_mongo_db", return_value=mocker.Mock(_id="file_id") + ) + mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") + + parameters = { + "param1": {"file": True, "value": "file1.dat"}, + "param2": {"file": True, "value": "file2.dat"}, + } + dest = "/tmp" + + result = db.export_model_files(parameters=parameters, dest=dest) + + mock_get_db_name.assert_called() + mock_get_file_mongo_db.assert_has_calls( + [call("test_db", "file1.dat"), call("test_db", "file2.dat")] + ) + mock_write_file_from_mongo_to_disk.assert_has_calls( + [ + call("test_db", dest, mock_get_file_mongo_db.return_value), + call("test_db", dest, mock_get_file_mongo_db.return_value), + ] + ) + assert result == {"file1.dat": "file_id", "file2.dat": "file_id"} + + +def test_export_model_files_file_exists(db, mocker): + """Test export_model_files method when file already exists.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_file_mongo_db = mocker.patch.object(db, "_get_file_mongo_db") + mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") + mock_path_exists = mocker.patch("pathlib.Path.exists", return_value=True) + + file_names = ["file1.dat"] + dest = "/tmp" + + result = db.export_model_files(file_names=file_names, dest=dest) + + mock_get_db_name.assert_called() + mock_get_file_mongo_db.assert_not_called() + mock_write_file_from_mongo_to_disk.assert_not_called() + mock_path_exists.assert_called_once() + assert result == {"file1.dat": "file exits"} + + +def test_export_model_files_file_not_found(db, mocker): + """Test export_model_files method when file is not found in parameters.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_file_mongo_db = mocker.patch.object( + db, "_get_file_mongo_db", side_effect=FileNotFoundError + ) + mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") + + parameters = { + "param1": {"file": True, "value": "file1.dat"}, + } + dest = "/tmp" + + with pytest.raises(FileNotFoundError): + db.export_model_files(parameters=parameters, dest=dest) + + mock_get_db_name.assert_called() + mock_get_file_mongo_db.assert_called_once_with("test_db", "file1.dat") + mock_write_file_from_mongo_to_disk.assert_not_called() + + +def test_get_query_from_parameter_version_table(db): + """Test _get_query_from_parameter_version_table method.""" + or_list = [ + {"parameter": "param1", "parameter_version": "v1"}, + {"parameter": "param2", "parameter_version": "v2"}, + ] + parameter_version_table = { + "param1": "v1", + "param2": "v2", + } + array_element_name = "LSTN-01" + site = "North" + + result = db._get_query_from_parameter_version_table( + parameter_version_table, array_element_name, site + ) + + expected_query = { + "$or": or_list, + "instrument": array_element_name, + "site": site, + } + + assert result == expected_query + + # _get_query_from_parameter_version_table method without site. + array_element_name = "LSTN-01" + site = None + + result = db._get_query_from_parameter_version_table( + parameter_version_table, array_element_name, site + ) + + expected_query = { + "$or": or_list, + "instrument": array_element_name, + } + + assert result == expected_query + + # _get_query_from_parameter_version_table method without array element name. + array_element_name = None + site = "North" + + result = db._get_query_from_parameter_version_table( + parameter_version_table, array_element_name, site + ) + + expected_query = { + "$or": or_list, + "site": site, + } + + assert result == expected_query + + # _get_query_from_parameter_version_table method without site and array element name. + array_element_name = None + site = None + + result = db._get_query_from_parameter_version_table( + parameter_version_table, array_element_name, site + ) + + expected_query = { + "$or": or_list, + } + + assert result == expected_query + + # _get_query_from_parameter_version_table method with array element name 'xSTx-design'. + array_element_name = "xSTx-design" + site = "North" + + result = db._get_query_from_parameter_version_table( + parameter_version_table, array_element_name, site + ) + + expected_query = { + "$or": or_list, + "site": site, + } + + assert result == expected_query + + +def test_read_mongo_db(db, mocker): + """Test read_mongo_db method.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mock_find = mocker.patch.object( + db.get_collection.return_value, + "find", + return_value=[ + {"_id": ObjectId(), "parameter": "param1", "value": "value1"}, + {"_id": ObjectId(), "parameter": "param2", "value": "value2"}, + ], + ) + + query = {"parameter_version": "1.0.0"} + collection_name = "test_collection" + + result = db._read_mongo_db(query, collection_name) + + mock_get_db_name.assert_called_once_with() + mock_get_collection.assert_called_once_with("test_db", collection_name) + mock_find.assert_called_once_with(query) + assert result == { + "param1": { + "_id": mock_find.return_value[0]["_id"], + "value": "value1", + "entry_date": mock_find.return_value[0]["_id"].generation_time, + }, + "param2": { + "_id": mock_find.return_value[1]["_id"], + "value": "value2", + "entry_date": mock_find.return_value[1]["_id"].generation_time, + }, + } + + mock_find = mocker.patch.object(db.get_collection.return_value, "find", return_value=[]) + with pytest.raises( + ValueError, + match=r"The following query for test_collection returned zero results: {'parameter_version': '1.0.0'}", + ): + db._read_mongo_db(query, collection_name) + + +def test_get_production_table_from_mongo_db_with_cache(db, mocker): + """Test get_production_table_from_mongo_db method with cache.""" + collection_name = "telescopes" + model_version = "1.0.0" + param = {"param1": "value1"} + mock_cache_key = mocker.patch.object(db, "_cache_key", return_value="cache_key") + db_handler.DatabaseHandler.production_table_cached["cache_key"] = { + "collection": model_version, + "model_version": model_version, + "parameters": param, + "design_model": {}, + "entry_date": ObjectId().generation_time, + } + + result = db.get_production_table_from_mongo_db(collection_name, model_version) + + mock_cache_key.assert_called_once_with(None, None, model_version, collection_name) + assert result == db_handler.DatabaseHandler.production_table_cached["cache_key"] + + mock_cache_key = mocker.patch.object(db, "_cache_key", return_value="no_cache_key") + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_find_one = mocker.patch.object( + db.get_collection.return_value, + "find_one", + return_value={ + "_id": ObjectId(), + "collection": collection_name, + "model_version": model_version, + "parameters": param, + "design_model": {}, + }, + ) + + result = db.get_production_table_from_mongo_db(collection_name, model_version) + + mock_cache_key.assert_called_once_with(None, None, model_version, collection_name) + mock_get_collection.assert_called_once_with("test_db", "production_tables") + mock_find_one.assert_called_once_with( + {"model_version": model_version, "collection": collection_name} + ) + assert result == { + "collection": collection_name, + "model_version": model_version, + "parameters": param, + "design_model": {}, + "entry_date": mock_find_one.return_value["_id"].generation_time, + } + + # test with no results + mock_find_one = mocker.patch.object( + db.get_collection.return_value, "find_one", return_value=None + ) + with pytest.raises( + ValueError, + match=r"The following query returned zero results: {'model_version': '1.0.0', 'collection': 'telescopes'}", + ): + db.get_production_table_from_mongo_db(collection_name, model_version) @pytest.mark.usefixtures("_db_cleanup") -def test_adding_new_parameter_db(db, random_id, io_handler, model_version): +def ff_test_adding_new_parameter_db(db, random_id, io_handler, model_version): logger.info("----Testing adding a new parameter-----") test_model_version = "0.0.9876" tmp_par_dict = { @@ -249,7 +734,7 @@ def test_adding_new_parameter_db(db, random_id, io_handler, model_version): file_prefix="tests/resources", ) - pars = db.read_mongo_db( + pars = db._read_mongo_db( db_name=f"sandbox_{random_id}", array_element_name="LSTN-test", model_version=test_model_version, @@ -268,62 +753,8 @@ def test_adding_new_parameter_db(db, random_id, io_handler, model_version): assert db._cache_key("North", "LSTN-test", test_model_version) not in db.model_parameters_cached -def test_reading_db_sites(db, db_config, simulation_model_url, model_version): - logger.info("----Testing reading La Palma parameters-----") - db.mongo_db_config["db_simulation_model_url"] = None - pars = db.get_site_parameters("North", model_version) - if db.mongo_db_config: - _obs_level = pars["corsika_observation_level"].get("value") - assert _obs_level == pytest.approx(2156.0) - else: - assert pars["altitude"] == 2156 - - logger.info("----Testing reading Paranal parameters-----") - pars = db.get_site_parameters("South", model_version) - if db.mongo_db_config: - _obs_level = pars["corsika_observation_level"].get("value") - assert _obs_level == pytest.approx(2147.0) - else: - assert pars["altitude"] == 2147 - - db._reset_parameter_cache("South", None, model_version) - if db.mongo_db_config.get("db_simulation_model_url", None) is None: - db.mongo_db_config["db_simulation_model_url"] = simulation_model_url - pars = db.get_site_parameters("South", model_version) - assert pars["corsika_observation_level"]["value"] == 2147.0 - db.mongo_db_config["db_simulation_model_url"] = None # make sure that this is reset - - -def test_separating_get_and_write(db, io_handler, model_version): - logger.info("----Testing getting parameters and exporting model files-----") - pars = db.get_model_parameters("North", "LSTN-01", model_version, collection="telescopes") - - file_list = [] - for par_now in pars.values(): - if par_now["file"] and par_now["value"] is not None: - file_list.append(par_now["value"]) - db.export_model_files( - pars, - io_handler.get_output_directory(sub_dir="model"), - ) - logger.debug( - "Checking files were written to " f"{io_handler.get_output_directory(sub_dir='model')}" - ) - for file_now in file_list: - assert io_handler.get_output_file(file_now, sub_dir="model").exists() - - -def test_export_file_db(db, io_handler): - logger.info("----Testing exporting files from the DB-----") - output_dir = io_handler.get_output_directory(sub_dir="model") - file_name = "mirror_CTA-S-LST_v2020-04-07.dat" - file_to_export = output_dir / file_name - db.export_file_db(None, output_dir, file_name) - assert file_to_export.exists() - - @pytest.mark.usefixtures("_db_cleanup_file_sandbox") -def test_insert_files_db(db, io_handler, random_id, caplog): +def ff_test_insert_files_db(db, io_handler, random_id, caplog): logger.info("----Testing inserting files to the DB-----") logger.info( "Creating a temporary file in " f"{io_handler.get_output_directory(sub_dir='model')}" @@ -345,88 +776,62 @@ def test_insert_files_db(db, io_handler, random_id, caplog): ) -def test_get_all_versions(db, mocker, caplog): - - # not specifying any database names, collections, or parameters - all_versions = db.get_all_versions() - assert all(_v in all_versions for _v in ["5.0.0", "6.0.0"]) - assert any(key.endswith("None") for key in db.model_versions_cached) - - # not specifying a telescope model name and parameter - all_versions = db.get_all_versions( - array_element_name=None, - site="North", - parameter=None, - collection="telescopes", +def test_cache_key(db): + """Test _cache_key method.""" + # Test with all parameters + result = db._cache_key( + site="North", array_element_name="LSTN-01", model_version="1.0.0", collection="telescopes" ) - assert all(_v in all_versions for _v in ["5.0.0", "6.0.0"]) - assert any("telescopes" in key for key in db.model_versions_cached) + assert result == "1.0.0-telescopes-North-LSTN-01" - # using a specific parameter - all_versions = db.get_all_versions( - array_element_name="LSTN-01", - site="North", - parameter="camera_config_file", - collection="telescopes", - ) - assert all(_v in all_versions for _v in ["5.0.0", "6.0.0"]) - assert any( - key.endswith("telescopes-camera_config_file-LSTN-01") for key in db.model_versions_cached + # Test with missing site + result = db._cache_key( + site=None, array_element_name="LSTN-01", model_version="1.0.0", collection="telescopes" ) + assert result == "1.0.0-telescopes-LSTN-01" - all_versions = db.get_all_versions( - site="North", - parameter="corsika_observation_level", - collection="sites", + # Test with missing array_element_name + result = db._cache_key( + site="North", array_element_name=None, model_version="1.0.0", collection="telescopes" ) - assert all(_v in all_versions for _v in ["5.0.0", "6.0.0"]) - assert any( - key.endswith("sites-corsika_observation_level-North") for key in db.model_versions_cached - ) - - # no db_name defined - mocker.patch.object(db, "_get_db_name", return_value=None) - with caplog.at_level(logging.WARNING): - assert db.get_all_versions() == [] - assert "No database name defined to determine" in caplog.text - - -def test_cache_key(db, model_version_prod5): - - assert db._cache_key("North", "LSTN-01", model_version_prod5) == "North-LSTN-01-5.0.0" - assert db._cache_key("North", None, model_version_prod5) == "North-5.0.0" - assert db._cache_key(None, None, model_version_prod5) == "5.0.0" + assert result == "1.0.0-telescopes-North" + # Test with missing model_version + result = db._cache_key( + site="North", array_element_name="LSTN-01", model_version=None, collection="telescopes" + ) + assert result == "telescopes-North-LSTN-01" -def test_model_version(db): - - assert db.model_version(version="6.0.0") == "6.0.0" - - with pytest.raises(ValueError, match=r"Invalid model version test"): - db.model_version(version="test") - with pytest.raises(ValueError, match=r"Invalid model version 0.0.9876"): - db.model_version(version="0.0.9876") - - -def test_get_collections(db, db_config, fs_files): - - collections = db.get_collections() - assert isinstance(collections, list) - assert "telescopes" in collections + # Test with missing collection + result = db._cache_key( + site="North", array_element_name="LSTN-01", model_version="1.0.0", collection=None + ) + assert result == "1.0.0-North-LSTN-01" - collections_from_name = db.get_collections(db_config["db_simulation_model"]) - assert isinstance(collections_from_name, list) - assert "telescopes" in collections_from_name - assert fs_files in collections_from_name + # Test with only model_version + result = db._cache_key( + site=None, array_element_name=None, model_version="1.0.0", collection=None + ) + assert result == "1.0.0" - collections_no_model = db.get_collections(db_config["db_simulation_model"], True) - assert isinstance(collections_no_model, list) - assert "telescopes" in collections_no_model - assert fs_files not in collections_no_model - assert "metadata" not in collections_no_model + # Test with only collection + result = db._cache_key( + site=None, array_element_name=None, model_version=None, collection="telescopes" + ) + assert result == "telescopes" + # Test with only site + result = db._cache_key( + site="North", array_element_name=None, model_version=None, collection=None + ) + assert result == "North" -def test_model_version_empty(db, mocker): + # Test with only array_element_name + result = db._cache_key( + site=None, array_element_name="LSTN-01", model_version=None, collection=None + ) + assert result == "LSTN-01" - mocker.patch.object(db, "get_all_versions", return_value=[]) - assert db.model_version("6.0.0") is None + # Test with no parameters + result = db._cache_key(site=None, array_element_name=None, model_version=None, collection=None) + assert result == "" From 3460768b479c6469c4da4d4200a6bf1e06f0194d Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 20 Jan 2025 15:20:41 +0100 Subject: [PATCH 067/124] unit tests --- tests/unit_tests/db/test_db_handler.py | 582 ++++++++++++++++++++----- 1 file changed, 473 insertions(+), 109 deletions(-) diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index d15ed16370..9c2896b67c 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -2,8 +2,8 @@ import copy import logging -import re import uuid +from pathlib import Path from unittest.mock import call import pytest @@ -660,120 +660,321 @@ def test_get_production_table_from_mongo_db_with_cache(db, mocker): db.get_production_table_from_mongo_db(collection_name, model_version) -@pytest.mark.usefixtures("_db_cleanup") -def ff_test_adding_new_parameter_db(db, random_id, io_handler, model_version): - logger.info("----Testing adding a new parameter-----") - test_model_version = "0.0.9876" - tmp_par_dict = { - "parameter": None, - "instrument": "LSTN-test", - "site": "North", - "version": test_model_version, - "value": None, - "unit": None, - "type": None, - "applicable": True, - "file": False, +def test_get_array_elements_of_type(db, mocker): + """Test get_array_elements_of_type method.""" + mock_get_production_table = mocker.patch.object( + db, + "get_production_table_from_mongo_db", + return_value={ + "parameters": {"LSTN-01": "value1", "LSTN-02": "value2", "MSTS-01": "value3"} + }, + ) + + array_element_type = "LSTN" + model_version = "1.0.0" + collection = "telescopes" + + result = db.get_array_elements_of_type(array_element_type, model_version, collection) + + mock_get_production_table.assert_called_once_with(collection, model_version) + assert result == ["LSTN-01", "LSTN-02"] + + # Test with no matching array elements + mock_get_production_table.return_value = {"parameters": {"MSTS-01": "value3"}} + result = db.get_array_elements_of_type(array_element_type, model_version, collection) + assert result == [] + + # Test with array elements containing '-design' + mock_get_production_table.return_value = { + "parameters": {"LSTN-01": "value1", "LSTN-design": "value2"} } + result = db.get_array_elements_of_type(array_element_type, model_version, collection) + assert result == ["LSTN-01"] + + # Test with different array element type + array_element_type = "MSTS" + mock_get_production_table.return_value = { + "parameters": {"LSTN-01": "value1", "MSTS-01": "value3", "MSTS-02": "value4"} + } + result = db.get_array_elements_of_type(array_element_type, model_version, collection) + assert result == ["MSTS-01", "MSTS-02"] + + +def test_get_simulation_configuration_parameters(db, mocker): + return_value = {"parameter": "value"} + mock_get_model_parameters = mocker.patch.object( + db, "get_model_parameters", return_value=return_value + ) - par_dict_int = copy.deepcopy(tmp_par_dict) - par_dict_int["parameter"] = "num_gains" - par_dict_int["value"] = 3 - par_dict_int["type"] = "int64" - with pytest.raises( - ValueError, - match=re.escape("Value for column '0' out of range. ([3, 3], allowed_range: [1, 2])"), - ): - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_int, - collection_name="telescopes", - ) - par_dict_int["value"] = 2 - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_int, - collection_name="telescopes", - ) - - par_dict_list = copy.deepcopy(tmp_par_dict) - par_dict_list["parameter"] = "telescope_transmission" - par_dict_list["value"] = [0.969, 0.01, 0.0, 0.0, 0.0, 0.0] - par_dict_list["type"] = "float64" - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_list, - collection_name="telescopes", - ) - par_dict_quantity = copy.deepcopy(tmp_par_dict) - par_dict_quantity["parameter"] = "focal_length" - par_dict_quantity["value"] = 12.5 # test that value is converted to cm - par_dict_quantity["type"] = "float64" - par_dict_quantity["unit"] = "m" - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_quantity, - collection_name="telescopes", - ) - - par_dict_file = copy.deepcopy(tmp_par_dict) - par_dict_file["parameter"] = "mirror_list" - par_dict_file["value"] = "mirror_list_CTA-N-LST1_v2019-03-31_rotated_simtel.dat" - par_dict_file["type"] = "file" - par_dict_file["file"] = True - with pytest.raises(FileNotFoundError, match=r"^The location of the file to upload"): - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_file, - collection_name="telescopes", - ) - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_file, - collection_name="telescopes", - file_prefix="tests/resources", - ) - - pars = db._read_mongo_db( - db_name=f"sandbox_{random_id}", - array_element_name="LSTN-test", - model_version=test_model_version, - run_location=io_handler.get_output_directory(sub_dir="model"), - collection_name="telescopes", - write_files=False, - ) - assert pars["num_gains"]["value"] == 2 - assert pars["num_gains"]["type"] == "int64" - assert isinstance(pars["telescope_transmission"]["value"], list) - assert pars["focal_length"]["value"] == pytest.approx(1250.0) - assert pars["focal_length"]["unit"] == "cm" - assert pars["mirror_list"]["file"] is True - - # make sure that cache has been emptied after updating - assert db._cache_key("North", "LSTN-test", test_model_version) not in db.model_parameters_cached - - -@pytest.mark.usefixtures("_db_cleanup_file_sandbox") -def ff_test_insert_files_db(db, io_handler, random_id, caplog): - logger.info("----Testing inserting files to the DB-----") - logger.info( - "Creating a temporary file in " f"{io_handler.get_output_directory(sub_dir='model')}" - ) - file_name = io_handler.get_output_directory(sub_dir="model") / f"test_file_{random_id}.dat" - with open(file_name, "w") as f: - f.write("# This is a test file") - - file_id = db.insert_file_to_db(file_name, f"sandbox_{random_id}") assert ( - file_id == db._get_file_mongo_db(f"sandbox_{random_id}", f"test_file_{random_id}.dat")._id + db.get_simulation_configuration_parameters("corsika", "North", "LSTN-design", "6.0.0") + == return_value ) - logger.info("Now test inserting the same file again, this time expect a warning") - with caplog.at_level(logging.WARNING): - file_id = db.insert_file_to_db(file_name, f"sandbox_{random_id}") - assert "exists in the DB. Returning its ID" in caplog.text + assert mock_get_model_parameters.call_count == 1 + assert ( - file_id == db._get_file_mongo_db(f"sandbox_{random_id}", f"test_file_{random_id}.dat")._id + db.get_simulation_configuration_parameters("simtel", "North", "LSTN-design", "6.0.0") + == return_value + ) + assert mock_get_model_parameters.call_count == 2 + db.get_simulation_configuration_parameters("simtel", "North", None, "6.0.0") == {} + assert mock_get_model_parameters.call_count == 2 + db.get_simulation_configuration_parameters("simtel", None, "LSTN-design", "6.0.0") == {} + assert mock_get_model_parameters.call_count == 2 + assert db.get_simulation_configuration_parameters("simtel", None, None, "6.0.0") == {} + assert mock_get_model_parameters.call_count == 2 + + with pytest.raises(ValueError, match=r"Unknown simulation software: wrong"): + db.get_simulation_configuration_parameters("wrong", "North", "LSTN-design", "6.0.0") + + +def test_get_file_mongo_db_file(db, mocker): + """Test _get_file_mongo_db method when file exists.""" + mock_db_client = mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} + ) + mock_gridfs = mocker.patch("simtools.db.db_handler.gridfs.GridFS") + mock_file_system = mock_gridfs.return_value + mock_file_system.exists.return_value = True + mock_file_instance = mocker.Mock() + mock_file_system.find_one.return_value = mock_file_instance + + db_name = "test_db" + file_name = "test_file.dat" + + result = db._get_file_mongo_db(db_name, file_name) + + mock_gridfs.assert_called_once_with(mock_db_client[db_name]) + mock_file_system.exists.assert_called_once_with({"filename": file_name}) + mock_file_system.find_one.assert_called_once_with({"filename": file_name}) + assert result == mock_file_instance + + mock_file_system.exists.return_value = False + with pytest.raises( + FileNotFoundError, match=f"The file {file_name} does not exist in the database {db_name}" + ): + db._get_file_mongo_db(db_name, file_name) + + +def test_write_file_from_mongo_to_disk(db, mocker): + """Test _write_file_from_mongo_to_disk method.""" + mock_db_client = mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} + ) + mock_gridfs_bucket = mocker.patch("simtools.db.db_handler.gridfs.GridFSBucket") + mock_fs_output = mock_gridfs_bucket.return_value + mock_open = mocker.patch("builtins.open", mocker.mock_open()) + + db_name = "test_db" + path = "/tmp" + file = mocker.Mock() + file.filename = "test_file.dat" + + db._write_file_from_mongo_to_disk(db_name, path, file) + + mock_gridfs_bucket.assert_called_once_with(mock_db_client[db_name]) + mock_open.assert_called_once_with(Path(path).joinpath(file.filename), "wb") + mock_fs_output.download_to_stream_by_name.assert_called_once_with(file.filename, mock_open()) + + +def test_add_production_table(db, mocker): + """Test add_production_table method.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mock_insert_one = mocker.patch.object(db.get_collection.return_value, "insert_one") + + db_name = "test_db" + production_table = { + "collection": "telescopes", + "model_version": "1.0.0", + "parameters": {"param1": "value1"}, + } + + db.add_production_table(db_name, production_table) + + mock_get_db_name.assert_called_once_with(db_name) + mock_get_collection.assert_called_once_with("test_db", "production_tables") + mock_insert_one.assert_called_once_with(production_table) + + +def test_add_new_parameter(db, mocker): + """Test add_new_parameter method.""" + mock_validate_model_parameter = mocker.patch( + "simtools.db.db_handler.validate_data.DataValidator.validate_model_parameter", + return_value={"parameter": "param1", "value": "value1", "file": False}, + ) + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mock_insert_one = mocker.patch.object(db.get_collection.return_value, "insert_one") + mock_get_value_unit_type = mocker.patch( + "simtools.db.db_handler.value_conversion.get_value_unit_type", + return_value=("value1", "unit1", None), + ) + mock_reset_parameter_cache = mocker.patch.object(db, "_reset_parameter_cache") + + db_name = "test_db" + par_dict = {"parameter": "param1", "value": "value1", "file": False} + collection_name = "telescopes" + file_prefix = None + + db.add_new_parameter(db_name, par_dict, collection_name, file_prefix) + + mock_validate_model_parameter.assert_called_once_with(par_dict) + mock_get_db_name.assert_called_once_with(db_name) + mock_get_collection.assert_called_once_with("test_db", collection_name) + mock_get_value_unit_type.assert_called_once_with(value="value1", unit_str=None) + mock_insert_one.assert_called_once_with( + {"parameter": "param1", "value": "value1", "file": False, "unit": "unit1"} + ) + mock_reset_parameter_cache.assert_called_once() + + +def test_add_new_parameter_with_file(db, mocker): + """Test add_new_parameter method with file.""" + mock_validate_model_parameter = mocker.patch( + "simtools.db.db_handler.validate_data.DataValidator.validate_model_parameter", + return_value={"parameter": "param1", "value": "value1", "file": True}, + ) + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mock_insert_one = mocker.patch.object(db.get_collection.return_value, "insert_one") + mock_get_value_unit_type = mocker.patch( + "simtools.db.db_handler.value_conversion.get_value_unit_type", + return_value=("value1", "unit1", None), + ) + mock_insert_file_to_db = mocker.patch.object(db, "insert_file_to_db") + mock_reset_parameter_cache = mocker.patch.object(db, "_reset_parameter_cache") + + db_name = "test_db" + par_dict = {"parameter": "param1", "value": "value1", "file": True} + collection_name = "telescopes" + file_prefix = "/tmp" + + db.add_new_parameter(db_name, par_dict, collection_name, file_prefix) + + mock_validate_model_parameter.assert_called_once_with(par_dict) + mock_get_db_name.assert_called_once_with(db_name) + mock_get_collection.assert_called_once_with("test_db", collection_name) + mock_get_value_unit_type.assert_called_once_with(value="value1", unit_str=None) + mock_insert_one.assert_called_once_with( + {"parameter": "param1", "value": "value1", "file": True, "unit": "unit1"} ) + mock_insert_file_to_db.assert_called_once_with("/tmp/value1", "test_db") + mock_reset_parameter_cache.assert_called_once() + + +def test_add_new_parameter_with_file_no_prefix(db, mocker): + """Test add_new_parameter method with file but no file_prefix.""" + mock_validate_model_parameter = mocker.patch( + "simtools.db.db_handler.validate_data.DataValidator.validate_model_parameter", + return_value={"parameter": "param1", "value": "value1", "file": True}, + ) + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mock_get_value_unit_type = mocker.patch( + "simtools.db.db_handler.value_conversion.get_value_unit_type", + return_value=("value1", "unit1", None), + ) + mock_reset_parameter_cache = mocker.patch.object(db, "_reset_parameter_cache") + + db_name = "test_db" + par_dict = {"parameter": "param1", "value": "value1", "file": True} + collection_name = "telescopes" + file_prefix = None + + with pytest.raises( + FileNotFoundError, + match=r"The location of the file to upload, corresponding to the param1 parameter, must be provided.", + ): + db.add_new_parameter(db_name, par_dict, collection_name, file_prefix) + + mock_validate_model_parameter.assert_called_once_with(par_dict) + mock_get_db_name.assert_called_once_with(db_name) + mock_get_collection.assert_called_once_with("test_db", collection_name) + mock_get_value_unit_type.assert_called_once_with(value="value1", unit_str=None) + mock_reset_parameter_cache.assert_not_called() + + +def test_insert_file_to_db_file_exists(db, mocker): + """Test insert_file_to_db method when file already exists in the DB.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_db_client = mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} + ) + mock_gridfs = mocker.patch("simtools.db.db_handler.gridfs.GridFS") + mock_file_system = mock_gridfs.return_value + mock_file_system.exists.return_value = True + mock_file_instance = mocker.Mock() + mock_file_system.find_one.return_value = mock_file_instance + + file_name = "test_file.dat" + db_name = "test_db" + + result = db.insert_file_to_db(file_name, db_name) + + mock_get_db_name.assert_called_once_with(db_name) + mock_gridfs.assert_called_once_with(mock_db_client[db_name]) + mock_file_system.exists.assert_called_once_with({"filename": file_name}) + mock_file_system.find_one.assert_called_once_with({"filename": file_name}) + assert result == mock_file_instance._id + + +def test_insert_file_to_db_new_file(db, mocker): + """Test insert_file_to_db method when file does not exist in the DB.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_db_client = mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} + ) + mock_gridfs = mocker.patch("simtools.db.db_handler.gridfs.GridFS") + mock_file_system = mock_gridfs.return_value + mock_file_system.exists.return_value = False + mock_file_system.put.return_value = "new_file_id" + mock_open = mocker.patch("builtins.open", mocker.mock_open(read_data=b"file_content")) + + file_name = "test_file.dat" + db_name = "test_db" + + result = db.insert_file_to_db(file_name, db_name) + + mock_get_db_name.assert_called_once_with(db_name) + mock_gridfs.assert_called_once_with(mock_db_client[db_name]) + mock_file_system.exists.assert_called_once_with({"filename": file_name}) + mock_open.assert_called_once_with(file_name, "rb") + mock_file_system.put.assert_called_once_with( + mock_open(), content_type="ascii/dat", filename=file_name + ) + assert result == "new_file_id" + + +def test_insert_file_to_db_with_kwargs(db, mocker): + """Test insert_file_to_db method with additional kwargs.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_db_client = mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} + ) + mock_gridfs = mocker.patch("simtools.db.db_handler.gridfs.GridFS") + mock_file_system = mock_gridfs.return_value + mock_file_system.exists.return_value = False + mock_file_system.put.return_value = "new_file_id" + mock_open = mocker.patch("builtins.open", mocker.mock_open(read_data=b"file_content")) + + file_name = "test_file.dat" + db_name = "test_db" + kwargs = {"content_type": "application/octet-stream", "metadata": {"key": "value"}} + + result = db.insert_file_to_db(file_name, db_name, **kwargs) + + mock_get_db_name.assert_called_once_with(db_name) + mock_gridfs.assert_called_once_with(mock_db_client[db_name]) + mock_file_system.exists.assert_called_once_with({"filename": file_name}) + mock_open.assert_called_once_with(file_name, "rb") + mock_file_system.put.assert_called_once_with( + mock_open(), + content_type="application/octet-stream", + filename=file_name, + metadata={"key": "value"}, + ) + assert result == "new_file_id" def test_cache_key(db): @@ -835,3 +1036,166 @@ def test_cache_key(db): # Test with no parameters result = db._cache_key(site=None, array_element_name=None, model_version=None, collection=None) assert result == "" + + +def test_read_cache_with_cache_hit(db): + """Test _read_cache method when cache hit occurs.""" + cache_dict = {"1.0.0-telescopes-North-LSTN-01": {"param1": "value1"}} + site = "North" + array_element_name = "LSTN-01" + model_version = "1.0.0" + collection = "telescopes" + + cache_key, result = db._read_cache( + cache_dict, site, array_element_name, model_version, collection + ) + + assert cache_key == "1.0.0-telescopes-North-LSTN-01" + assert result == {"param1": "value1"} + + +def test_read_cache_with_cache_miss(db): + """Test _read_cache method when cache miss occurs.""" + cache_dict = {"1.0.0-telescopes-North-LSTN-01": {"param1": "value1"}} + site = "North" + array_element_name = "LSTN-02" + model_version = "1.0.0" + collection = "telescopes" + + cache_key, result = db._read_cache( + cache_dict, site, array_element_name, model_version, collection + ) + + assert cache_key == "1.0.0-telescopes-North-LSTN-02" + assert result is None + + +def test_read_cache_with_empty_cache(db): + """Test _read_cache method with empty cache.""" + cache_dict = {} + site = "North" + array_element_name = "LSTN-01" + model_version = "1.0.0" + collection = "telescopes" + + cache_key, result = db._read_cache( + cache_dict, site, array_element_name, model_version, collection + ) + + assert cache_key == "1.0.0-telescopes-North-LSTN-01" + assert result is None + + +def test_read_cache_with_partial_parameters(db): + """Test _read_cache method with partial parameters.""" + cache_dict = {"1.0.0-telescopes-North": {"param1": "value1"}} + site = "North" + array_element_name = None + model_version = "1.0.0" + collection = "telescopes" + + cache_key, result = db._read_cache( + cache_dict, site, array_element_name, model_version, collection + ) + + assert cache_key == "1.0.0-telescopes-North" + assert result == {"param1": "value1"} + + +def test_read_cache_with_no_parameters(db): + """Test _read_cache method with no parameters.""" + cache_dict = {"": {"param1": "value1"}} + site = None + array_element_name = None + model_version = None + collection = None + + cache_key, result = db._read_cache( + cache_dict, site, array_element_name, model_version, collection + ) + + assert cache_key == "" + assert result == {"param1": "value1"} + + +def test_reset_parameter_cache(db): + """Test _reset_parameter_cache method.""" + # Populate the cache dictionaries + db_handler.DatabaseHandler.site_parameters_cached = {"key1": "value1"} + db_handler.DatabaseHandler.model_parameters_cached = {"key2": "value2"} + + # Ensure the caches are populated + assert db_handler.DatabaseHandler.site_parameters_cached + assert db_handler.DatabaseHandler.model_parameters_cached + + # Call the method to reset the caches + db._reset_parameter_cache() + + # Check that the caches are cleared + assert not db_handler.DatabaseHandler.site_parameters_cached + assert not db_handler.DatabaseHandler.model_parameters_cached + + +def test_get_array_element_list_configuration_corsika(db): + """Test _get_array_element_list method for configuration_corsika collection.""" + array_element_name = "LSTN-01" + site = "North" + production_table = {} + collection = "configuration_corsika" + + result = db._get_array_element_list(array_element_name, site, production_table, collection) + + assert result == ["xSTx-design"] + + +def test_get_array_element_list_sites(db): + """Test _get_array_element_list method for sites collection.""" + array_element_name = "LSTN-01" + site = "North" + production_table = {} + collection = "sites" + + result = db._get_array_element_list(array_element_name, site, production_table, collection) + + assert result == ["OBS-North"] + + +def test_get_array_element_list_design_model(db): + """Test _get_array_element_list method when array element name contains '-design'.""" + array_element_name = "LSTN-design" + site = "North" + production_table = {} + collection = "telescopes" + + result = db._get_array_element_list(array_element_name, site, production_table, collection) + + assert result == ["LSTN-design"] + + +def test_get_array_element_list_with_design_model_in_production_table(db, mocker): + """Test _get_array_element_list method with design model in production table.""" + array_element_name = "LSTN-01" + site = "North" + production_table = {"design_model": {"LSTN-01": "LSTN-design"}} + collection = "telescopes" + + result = db._get_array_element_list(array_element_name, site, production_table, collection) + + assert result == ["LSTN-design", "LSTN-01"] + + +def test_get_array_element_list_without_design_model_in_production_table(db, mocker): + """Test _get_array_element_list method without design model in production table.""" + array_element_name = "LSTN-01" + site = "North" + production_table = {} + collection = "telescopes" + + mock_get_array_element_type_from_name = mocker.patch( + "simtools.utils.names.get_array_element_type_from_name", return_value="LSTN" + ) + + result = db._get_array_element_list(array_element_name, site, production_table, collection) + + mock_get_array_element_type_from_name.assert_called_once_with(array_element_name) + assert result == ["LSTN-design", "LSTN-01"] From aba93640af872d8476d6e54c6131b8977edf3301 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 20 Jan 2025 15:39:31 +0100 Subject: [PATCH 068/124] unit test fix --- tests/unit_tests/model/test_model_parameter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit_tests/model/test_model_parameter.py b/tests/unit_tests/model/test_model_parameter.py index b48239138e..269b0d3b69 100644 --- a/tests/unit_tests/model/test_model_parameter.py +++ b/tests/unit_tests/model/test_model_parameter.py @@ -161,11 +161,11 @@ def test_load_parameters_from_db(telescope_model_lst, mocker): telescope_copy = copy.deepcopy(telescope_model_lst) mock_db = mocker.patch.object(DatabaseHandler, "get_model_parameters") telescope_copy._load_parameters_from_db() - assert mock_db.call_count == 2 + assert mock_db.call_count == 4 telescope_copy.db = None telescope_copy._load_parameters_from_db() - assert mock_db.call_count == 2 + assert mock_db.call_count == 4 def test_extra_labels(telescope_model_lst): @@ -354,11 +354,11 @@ def test_export_nsb_spectrum_to_telescope_altitude_correction_file(telescope_mod telescope_copy.export_nsb_spectrum_to_telescope_altitude_correction_file(model_directory) mock_db_export.assert_called_once_with( - { + parameters={ "nsb_spectrum_at_2200m": { "value": "test_value", "file": True, } }, - model_directory, + dest=model_directory, ) From f22f88fdd7948ba777b68bf11fd93a381d22992f Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 20 Jan 2025 15:55:43 +0100 Subject: [PATCH 069/124] unit tests --- tests/unit_tests/layout/test_array_layout.py | 8 ++++---- tests/unit_tests/visualization/test_visualize.py | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/unit_tests/layout/test_array_layout.py b/tests/unit_tests/layout/test_array_layout.py index 2b25f896b5..0432ae23f6 100644 --- a/tests/unit_tests/layout/test_array_layout.py +++ b/tests/unit_tests/layout/test_array_layout.py @@ -283,9 +283,9 @@ def test_converting_center_coordinates_south(array_layout_south_four_lst_instanc def test_altitude_from_corsika_z( array_layout_north_four_lst_instance, array_layout_south_four_lst_instance ): - def test_one_site(instance, result1, result2): + def test_one_site(instance, telescope_name, result1, result2): instance.add_telescope( - telescope_name="LSTN-01", + telescope_name=telescope_name, crs_name="ground", xx=57.5 * u.m, yy=57.5 * u.m, @@ -302,8 +302,8 @@ def test_one_site(instance, result1, result2): instance._altitude_from_corsika_z(5.0, None, telescope_axis_height=16.0 * u.m) assert np.isnan(instance._altitude_from_corsika_z(None, None, None)) - test_one_site(array_layout_north_four_lst_instance, 2185.0, 45.0) - test_one_site(array_layout_south_four_lst_instance, 2176.0, 45.0) + test_one_site(array_layout_north_four_lst_instance, "LSTN-01", 2185.0, 45.0) + test_one_site(array_layout_south_four_lst_instance, "LSTS-01", 2176.0, 45.0) def test_try_set_altitude( diff --git a/tests/unit_tests/visualization/test_visualize.py b/tests/unit_tests/visualization/test_visualize.py index b670e6b49a..0e558c8684 100644 --- a/tests/unit_tests/visualization/test_visualize.py +++ b/tests/unit_tests/visualization/test_visualize.py @@ -26,10 +26,10 @@ def test_plot_1d(db, io_handler): title = "Test 1D plot" test_file_name = "ref_LST1_2022_04_01.dat" - db.export_file_db( + db.export_model_files( db_name=None, dest=io_handler.get_output_directory(sub_dir="model"), - file_name=test_file_name, + file_names=test_file_name, ) test_data_file = gen.find_file( test_file_name, From d3dadfc41fc8f4339db778f4923fd07efd29d9f9 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 20 Jan 2025 16:22:07 +0100 Subject: [PATCH 070/124] layout json --- src/simtools/layout/array_layout.py | 2 +- src/simtools/model/array_model.py | 13 ++++++++++--- tests/unit_tests/model/test_array_model.py | 2 +- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/simtools/layout/array_layout.py b/src/simtools/layout/array_layout.py index 7658a09b9e..fb174c3f8d 100644 --- a/src/simtools/layout/array_layout.py +++ b/src/simtools/layout/array_layout.py @@ -629,10 +629,10 @@ def export_one_telescope_as_json(self, crs_name): "instrument": table["telescope_name"][0], "site": self.site, "version": self.model_version, + "unique_id": None, "value": value_string, "unit": "m", "type": "float64", - "applicable": True, "file": False, } diff --git a/src/simtools/model/array_model.py b/src/simtools/model/array_model.py index 643d5d2648..29f52cf1f4 100644 --- a/src/simtools/model/array_model.py +++ b/src/simtools/model/array_model.py @@ -290,7 +290,13 @@ def _load_array_element_positions_from_file( } def _get_telescope_position_parameter( - self, telescope_name: str, site: str, x: u.Quantity, y: u.Quantity, z: u.Quantity + self, + telescope_name: str, + site: str, + x: u.Quantity, + y: u.Quantity, + z: u.Quantity, + parameter_version: str | None = None, ) -> dict: """ Return dictionary with telescope position parameters (following DB model database format). @@ -314,16 +320,17 @@ def _get_telescope_position_parameter( Dict with telescope position parameters. """ return { + "schema_version": "0.1.0", "parameter": "array_element_position_ground", "instrument": telescope_name, "site": site, - "version": self.model_version, + "parameter_version": parameter_version, + "unique_id": None, "value": general.convert_list_to_string( [x.to("m").value, y.to("m").value, z.to("m").value] ), "unit": "m", "type": "float64", - "applicable": True, "file": False, } diff --git a/tests/unit_tests/model/test_array_model.py b/tests/unit_tests/model/test_array_model.py index 8c6714ed86..77d469d743 100644 --- a/tests/unit_tests/model/test_array_model.py +++ b/tests/unit_tests/model/test_array_model.py @@ -107,7 +107,7 @@ def test_load_array_element_positions_from_file(array_model, io_handler, telesco def test_get_telescope_position_parameter(array_model, io_handler): am = array_model assert am._get_telescope_position_parameter( - "LSTN-01", "North", 10.0 * u.m, 200.0 * u.cm, 30.0 * u.m + "LSTN-01", "North", 10.0 * u.m, 200.0 * u.cm, 30.0 * u.m, "2.0.0" ) == { "schema_version": "0.1.0", "parameter": "array_element_position_ground", From 63efa2538e37e0c2b581bd4f63210ea788b83490 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 09:41:11 +0100 Subject: [PATCH 071/124] correct DB name --- .github/workflows/build-docker-images.yml | 3 --- database_scripts/upload_dump_to_local_db.sh | 2 +- docs/source/user-guide/databases.md | 9 ++++---- environment.yml | 2 +- tests/unit_tests/db/test_db_handler.py | 24 +++++++++++---------- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index f9b852980f..dce6b8cbde 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -89,8 +89,6 @@ jobs: - name: Test Docker image if: matrix.type == 'prod' run: > - # Clone simulation model repo for faster file access - git clone https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters.git # run docker docker run --rm -e "SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'" @@ -98,7 +96,6 @@ jobs: -e "SIMTOOLS_DB_API_USER=${{ env.SIMTOOLS_DB_API_USER }}" -e "SIMTOOLS_DB_API_PW=${{ env.SIMTOOLS_DB_API_PW }}" -e "SIMTOOLS_DB_SERVER=${{ env.SIMTOOLS_DB_SERVER }}" - -e "SIMTOOLS_DB_SIMULATION_MODEL_URL='./model_parameters'" -v "$(pwd):/workdir/external" ${{ env.TEST_TAG }} bash -c "cd /workdir/external && simtools-simulate-prod --config /workdir/external/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid.yml" shell: /usr/bin/bash -e {0} diff --git a/database_scripts/upload_dump_to_local_db.sh b/database_scripts/upload_dump_to_local_db.sh index c6941b9883..840b69ae4e 100755 --- a/database_scripts/upload_dump_to_local_db.sh +++ b/database_scripts/upload_dump_to_local_db.sh @@ -6,7 +6,7 @@ SIMTOOLS_NETWORK="simtools-mongo-network" CONTAINER_NAME="simtools-mongodb" -SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-v0-3-0' # Name of the database to be created +SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-v0-3-0' # Name of the database to be created # Check if podman is available, if not use docker if command -v podman &> /dev/null; then diff --git a/docs/source/user-guide/databases.md b/docs/source/user-guide/databases.md index d286779db1..308c57b9d8 100644 --- a/docs/source/user-guide/databases.md +++ b/docs/source/user-guide/databases.md @@ -16,7 +16,7 @@ This documentation is therefore incomplete. ### Model parameters DB -The name of the model parameter database needs to be indicated by `$SIMTOOLS_DB_SIMULATION_MODEL` environmental variable and defined e.g., in the `.env` file. Use `CTAO-Simulation-Model-LATEST` to use the latest version of the CTAO simulation model database (simtools will replace `LATEST` with the latest version number). +The name of the model parameter database needs to be indicated by `$SIMTOOLS_DB_SIMULATION_MODEL` environmental variable and defined e.g., in the `.env` file. Use `CTAO-Simulation-ModelParameters-LATEST` to use the latest version of the CTAO simulation model database (simtools will replace `LATEST` with the latest version number). Collections: @@ -43,8 +43,7 @@ SIMTOOLS_DB_SERVER='cta-simpipe-protodb.zeuthen.desy.de' # MongoDB server SIMTOOLS_DB_API_USER=YOUR_USERNAME # username for MongoDB: ask the responsible person SIMTOOLS_DB_API_PW=YOUR_PASSWORD # Password for MongoDB: ask the responsible person SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin' -SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-LATEST' -# SIMTOOLS_DB_SIMULATION_MODEL_URL='' +SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-LATEST' SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray' ``` @@ -129,7 +128,7 @@ SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin' SIMTOOLS_DB_SIMULATION_MODEL='STAGING-CTA-Simulation-Model-LATEST' ``` -`SIMTOOLS_DB_SIMULATION_MODEL` is set as an example here to `STAGING-CTAO-Simulation-Model-LATEST` and should be changed accordingly. +`SIMTOOLS_DB_SIMULATION_MODEL` is set as an example here to `STAGING-CTAO-Simulation-ModelParameters-LATEST` and should be changed accordingly. For using simtools inside a container: @@ -149,6 +148,6 @@ SIMTOOLS_DB_SERVER='simtools-mongodb' SIMTOOLS_DB_API_USER='api' # username for MongoDB SIMTOOLS_DB_API_PW='password' # Password for MongoDB SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin' -SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-LATEST' +SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-LATEST' SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray' ``` diff --git a/environment.yml b/environment.yml index 592f0b5f18..15c4b46e17 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - jsonschema - matplotlib-base - myst-parser - - numpy + - numpy~=2.1.0 - numpydoc - particle - pip diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index 9c2896b67c..d7b8794a8e 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -129,20 +129,22 @@ def test_find_latest_simulation_model_db(db, db_no_config_file, mocker): db_copy._find_latest_simulation_model_db() db_names = [ - "CTAO-Simulation-Model-v0-3-0", - "CTAO-Simulation-Model-v0-2-0", - "CTAO-Simulation-Model-v0-1-19", - "CTAO-Simulation-Model-v0-3-9", - "CTAO-Simulation-Model-v0-3-19", - "CTAO-Simulation-Model-v0-3-0", - "CTAO-Simulation-Model-v0-3-0-alpha-2", - "CTAO-Simulation-Model-v0-4-19-alpha-1", - "CTAO-Simulation-Model-v0-4-19-dev1", + "CTAO-Simulation-ModelParameters-v0-3-0", + "CTAO-Simulation-ModelParameters-v0-2-0", + "CTAO-Simulation-ModelParameters-v0-1-19", + "CTAO-Simulation-ModelParameters-v0-3-9", + "CTAO-Simulation-ModelParameters-v0-3-19", + "CTAO-Simulation-ModelParameters-v0-3-0", + "CTAO-Simulation-ModelParameters-v0-3-0-alpha-2", + "CTAO-Simulation-ModelParameters-v0-4-19-alpha-1", + "CTAO-Simulation-ModelParameters-v0-4-19-dev1", ] mocker.patch.object(db_copy.db_client, "list_database_names", return_value=db_names) - db_copy.mongo_db_config["db_simulation_model"] = "CTAO-Simulation-Model-LATEST" + db_copy.mongo_db_config["db_simulation_model"] = "CTAO-Simulation-ModelParameters-LATEST" db_copy._find_latest_simulation_model_db() - assert db_copy.mongo_db_config["db_simulation_model"] == "CTAO-Simulation-Model-v0-3-19" + assert ( + db_copy.mongo_db_config["db_simulation_model"] == "CTAO-Simulation-ModelParameters-v0-3-19" + ) def test_get_model_parameter(db, mocker): From c3cea59d329452ae4e27adee8fa07450df11d02c Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 09:46:30 +0100 Subject: [PATCH 072/124] improved comment --- database_scripts/upload_from_model_repository_to_db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_scripts/upload_from_model_repository_to_db.sh b/database_scripts/upload_from_model_repository_to_db.sh index 1bc9811952..84960f1131 100755 --- a/database_scripts/upload_from_model_repository_to_db.sh +++ b/database_scripts/upload_from_model_repository_to_db.sh @@ -44,7 +44,7 @@ if [[ $SIMTOOLS_DB_SERVER =~ $regex ]]; then fi fi -# upload files to DB +# upload model parameters to DB model_directory="./simulation-models/model_parameters/" simtools-db-add-simulation-model-from-repository-to-db \ --input_path "${model_directory}" \ From 29b3165f6e5745df2bcf5e48f038e879f74877f6 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 09:48:27 +0100 Subject: [PATCH 073/124] revert numpy version restriction --- environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment.yml b/environment.yml index 15c4b46e17..592f0b5f18 100644 --- a/environment.yml +++ b/environment.yml @@ -12,7 +12,7 @@ dependencies: - jsonschema - matplotlib-base - myst-parser - - numpy~=2.1.0 + - numpy - numpydoc - particle - pip From d10fc80067a8b9908c2fe93ad99b16327f9161d3 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 10:11:00 +0100 Subject: [PATCH 074/124] fix reading of list of files --- ...d_simulation_model_from_repository_to_db.py | 18 ++++++++++++++---- .../applications/db_get_file_from_db.py | 12 ++++++------ ...from_db_CTA-Simulation-Model-two-files.yml} | 5 +++-- 3 files changed, 23 insertions(+), 12 deletions(-) rename tests/integration_tests/config/{db_get_file_from_db_CTA-Simulation-Model-Derived-Values.yml => db_get_file_from_db_CTA-Simulation-Model-two-files.yml} (50%) diff --git a/src/simtools/applications/db_add_simulation_model_from_repository_to_db.py b/src/simtools/applications/db_add_simulation_model_from_repository_to_db.py index 434816035e..0da627be0d 100644 --- a/src/simtools/applications/db_add_simulation_model_from_repository_to_db.py +++ b/src/simtools/applications/db_add_simulation_model_from_repository_to_db.py @@ -1,9 +1,9 @@ #!/usr/bin/python3 r""" - Add parameters and production tables found in a model parameter repository to a new database. + Add parameters and production tables from a simulation model repository to a new database. Generates a new database with all required collections. - Follows the structure of the CTAO gitlab model parameters repository. + Follows the structure of the CTAO gitlab simulation model repository. This is an application for experts and should not be used by the general user. @@ -26,6 +26,16 @@ --input_path /path/to/repository \ --db_name new_db_name \ --type model_parameters + + Upload production tables to the DB: + + .. code-block:: console + + simtools-db-simulation-model-from-repository-to-db \ + --input_path /path/to/repository \ + --db_name new_db_name \ + --type production_tables + """ import logging @@ -55,13 +65,13 @@ def _parse(label=None, description=None): config = configurator.Configurator(label=label, description=description) config.parser.add_argument( "--input_path", - help="Path to model parameter repository.", + help="Path to simulation model repository.", type=Path, required=True, ) config.parser.add_argument( "--db_name", - help="Name of the new model parameter database to be created.", + help="Name of the new simulation model database to be created.", type=str.strip, required=True, ) diff --git a/src/simtools/applications/db_get_file_from_db.py b/src/simtools/applications/db_get_file_from_db.py index 17a22f9566..215f58f302 100644 --- a/src/simtools/applications/db_get_file_from_db.py +++ b/src/simtools/applications/db_get_file_from_db.py @@ -49,8 +49,9 @@ def _parse(): config.parser.add_argument( "--file_name", - help="The name of the file to be downloaded.", + help="The name of the file(s) to be downloaded (single file or space-separated list).", type=str, + nargs="+", required=True, ) return config.initialize(db_config=True, output=True) @@ -84,11 +85,10 @@ def main(): # noqa: D103 except FileNotFoundError: continue - if file_id.get(args_dict["file_name"]) is None: - logger.error( - f"The file {args_dict['file_name']} was not found in any of the available DBs." - ) - raise FileNotFoundError + for key, value in file_id.items(): + if value is None: + logger.error(f"The file {key} was not found in any of the available DBs.") + raise FileNotFoundError if __name__ == "__main__": diff --git a/tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-Derived-Values.yml b/tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-two-files.yml similarity index 50% rename from tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-Derived-Values.yml rename to tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-two-files.yml index ca02aa1848..fcfaa20ad4 100644 --- a/tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-Derived-Values.yml +++ b/tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-two-files.yml @@ -1,8 +1,9 @@ CTA_SIMPIPE: APPLICATION: simtools-db-get-file-from-db - TEST_NAME: CTA-Simulation-Model-Derived-Values + TEST_NAME: CTA-Simulation-Model-two-files CONFIGURATION: - FILE_NAME: ray-tracing-North-LST-1-d10.0-za20.0_validate_optics.ecsv + FILE_NAME: [mirror_CTA-S-LST_v2020-04-07.dat, ray-tracing-North-LST-1-d10.0-za20.0_validate_optics.ecsv] OUTPUT_PATH: simtools-output INTEGRATION_TESTS: + - OUTPUT_FILE: mirror_CTA-S-LST_v2020-04-07.dat - OUTPUT_FILE: ray-tracing-North-LST-1-d10.0-za20.0_validate_optics.ecsv From c4d4dc28bfb1589174e06683ed2ff1d7f0801339 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 10:25:25 +0100 Subject: [PATCH 075/124] consistent requirement for versions --- .../applications/db_get_parameter_from_db.py | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 3c1de26355..873425e4dc 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -101,24 +101,27 @@ def main(): # noqa: D103 collection=args_dict.get("db_collection", "telescopes"), ) # get parameter using 'model_version' - elif args_dict["telescope"]: - pars = db.get_model_parameters( - site=args_dict["site"], - array_element_name=args_dict["telescope"], - model_version=args_dict["model_version"], - collection=( - "configuration_sim_telarray" - if args_dict["db_collection"] == "configuration_sim_telarray" - else "telescopes" - ), - ) + elif args_dict["model_version"] is not None: + if args_dict["telescope"]: + pars = db.get_model_parameters( + site=args_dict["site"], + array_element_name=args_dict["telescope"], + model_version=args_dict["model_version"], + collection=( + "configuration_sim_telarray" + if args_dict["db_collection"] == "configuration_sim_telarray" + else "telescopes" + ), + ) + else: + pars = db.get_model_parameters( + site=args_dict.get("site"), + model_version=args_dict["model_version"], + collection=args_dict["db_collection"], + array_element_name=None, + ) else: - pars = db.get_model_parameters( - site=args_dict.get("site"), - model_version=args_dict["model_version"], - collection=args_dict["db_collection"], - array_element_name=None, - ) + raise ValueError("Either 'parameter_version' or 'model_version' must be provided.") param = args_dict["parameter"] if param not in pars: raise KeyError(f"The requested parameter, {args_dict['parameter']}, does not exist.") From 10605724995af0ad90e87699de5f9197fed60be5 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 10:38:37 +0100 Subject: [PATCH 076/124] correct handling of byte conversion --- src/simtools/corsika/corsika_config.py | 10 ++++++---- tests/unit_tests/corsika/test_corsika_config.py | 8 ++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/simtools/corsika/corsika_config.py b/src/simtools/corsika/corsika_config.py index ae5709ea03..0cad485967 100644 --- a/src/simtools/corsika/corsika_config.py +++ b/src/simtools/corsika/corsika_config.py @@ -315,10 +315,12 @@ def _corsika_configuration_debugging_parameters(self): } def _input_config_io_buff(self, entry): - """Return IO_BUFFER parameter CORSIKA format.""" - unit_map = {"Mbyte": "MB", "kbyte": "kB", "Gbyte": "GB"} - unit = unit_map.get(str(u.Unit(entry["unit"])), entry["unit"]) - return f"{entry['value']}{unit}" + """Return IO_BUFFER parameter CORSIKA format (Byte or MB required).""" + value = entry["value"] * u.Unit(entry["unit"]).to("Mbyte") + # check if value is integer-like + if value.is_integer(): + return f"{int(value)}MB" + return f"{int(entry['value'] * u.Unit(entry['unit']).to('byte'))}" def _rotate_azimuth_by_180deg(self, az, correct_for_geomagnetic_field_alignment=True): """ diff --git a/tests/unit_tests/corsika/test_corsika_config.py b/tests/unit_tests/corsika/test_corsika_config.py index c35f69fba5..f3a2ca5ef6 100644 --- a/tests/unit_tests/corsika/test_corsika_config.py +++ b/tests/unit_tests/corsika/test_corsika_config.py @@ -177,6 +177,14 @@ def test_input_config_io_buff(corsika_config_mock_array_model): corsika_config_mock_array_model._input_config_io_buff({"value": 800, "unit": "MB"}) == "800MB" ) + assert ( + corsika_config_mock_array_model._input_config_io_buff({"value": 8.5, "unit": "MB"}) + == "8500000" + ) + assert ( + corsika_config_mock_array_model._input_config_io_buff({"value": 800, "unit": "kB"}) + == "800000" + ) def test_corsika_configuration_debugging_parameters(corsika_config_mock_array_model): From 0790cb6f8bd587a69156c135ff7229a813e27646 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 11:39:49 +0100 Subject: [PATCH 077/124] docstrings --- src/simtools/db/db_handler.py | 54 ++++++++++++-------------- tests/unit_tests/db/test_db_handler.py | 21 +++++----- 2 files changed, 33 insertions(+), 42 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index fed475909f..a20a899321 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -67,10 +67,7 @@ class DatabaseHandler: db_client = None production_table_cached = {} - site_parameters_cached = {} model_parameters_cached = {} - model_versions_cached = {} - corsika_configuration_parameters_cached = {} def __init__(self, mongo_db_config=None): """Initialize the DatabaseHandler class.""" @@ -147,7 +144,7 @@ def _find_latest_simulation_model_db(self): db_simulation_model = self.mongo_db_config["db_simulation_model"] if not db_simulation_model.endswith("LATEST"): return - except TypeError: # if db_simulation_model is None + except TypeError: # db_simulation_model is None return prefix = db_simulation_model.replace("LATEST", "") @@ -184,7 +181,7 @@ def get_model_parameter( collection, ): """ - Get a single model parameter (using the parameter version). + Get a model parameter using the parameter version. Parameters ---------- @@ -222,9 +219,8 @@ def get_model_parameters( collection, ): """ - Get model parameters from MongoDB. + Get model parameters using the model version. - An array element can be e.g., a telescope or a calibration device. Queries parameters for design and for the specified array element (if necessary). Parameters @@ -232,17 +228,17 @@ def get_model_parameters( site: str Site name. array_element_name: str - Name of the array element model (e.g. LSTN-01, MSTS-design). + Name of the array element model (e.g. LSTN-01, MSTS-design, ILLN-01). model_version: str Version of the model. collection: str - collection of array element (e.g. telescopes, calibration_devices). + Collection of array element (e.g. telescopes, calibration_devices). Returns ------- dict containing the parameters """ - production_table = self.get_production_table_from_mongo_db(collection, model_version) + production_table = self._read_production_table_from_mongo_db(collection, model_version) array_element_list = self._get_array_element_list( array_element_name, site, production_table, collection ) @@ -264,7 +260,7 @@ def get_model_parameters( try: parameter_version_table = production_table["parameters"][array_element] - except KeyError: + except KeyError: # allow missing array elements (parameter dict is checked later) continue DatabaseHandler.model_parameters_cached[cache_key] = self._read_mongo_db( query=self._get_query_from_parameter_version_table( @@ -304,13 +300,13 @@ def get_collections(self, db_name=None, model_collections_only=False): ---------- db_name: str Database name. + model_collections_only: bool + If True, only return model collections (i.e. exclude fs.files, fs.chunks, metadata) Returns ------- list List of collection names - model_collections_only: bool - If True, only return model collections (i.e. exclude fs.files, fs.chunks, metadata) """ db_name = db_name or self._get_db_name() @@ -347,12 +343,6 @@ def export_model_files(self, parameters=None, file_names=None, dest=None, db_nam ------- file_id: dict of GridOut._id Dict of database IDs of files. - - Raises - ------ - FileNotFoundError - if a file in parameters.values is not found - """ db_name = self._get_db_name(db_name) @@ -393,14 +383,14 @@ def _get_query_from_parameter_version_table( def _read_mongo_db(self, query, collection_name): """ - Build and execute query to read the MongoDB for a collection. + Query MongoDB. Parameters ---------- query: dict - Dictionary describing the query to execute. + Query to execute. collection_name: str - The name of the collection to read from. + Collection name. Returns ------- @@ -426,9 +416,9 @@ def _read_mongo_db(self, query, collection_name): parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time return {k: parameters[k] for k in sorted(parameters)} - def get_production_table_from_mongo_db(self, collection_name, model_version): + def _read_production_table_from_mongo_db(self, collection_name, model_version): """ - Get production table from MongoDB. + Read production table from MongoDB. Parameters ---------- @@ -436,6 +426,11 @@ def get_production_table_from_mongo_db(self, collection_name, model_version): Name of the collection. model_version: str Version of the model. + + Raises + ------ + ValueError + if query returned no results. """ try: return DatabaseHandler.production_table_cached[ @@ -460,7 +455,7 @@ def get_production_table_from_mongo_db(self, collection_name, model_version): def get_array_elements_of_type(self, array_element_type, model_version, collection): """ - Get all array elements of a certain type (e.g. 'LSTN') from a collection in the DB. + Get array elements of a certain type (e.g. 'LSTN') for a DB collection. Parameters ---------- @@ -477,7 +472,7 @@ def get_array_elements_of_type(self, array_element_type, model_version, collecti list Sorted list of all array element names found in collection """ - production_table = self.get_production_table_from_mongo_db(collection, model_version) + production_table = self._read_production_table_from_mongo_db(collection, model_version) all_array_elements = production_table["parameters"] return sorted( [ @@ -584,7 +579,7 @@ def _write_file_from_mongo_to_disk(db_name, path, file): def add_production_table(self, db_name, production_table): """ - Add a production table for a given model version to the DB. + Add a production table to the DB. Parameters ---------- @@ -607,7 +602,7 @@ def add_new_parameter( file_prefix=None, ): """ - Add a parameter dictionary for a specific array element to the DB. + Add a new parameter dictionary to the DB. A new document will be added to the DB, with all fields taken from the input parameters. Parameter dictionaries are validated before submission using the corresponding schema. @@ -765,12 +760,11 @@ def _read_cache( def _reset_parameter_cache(self): """Reset the cache for the parameters.""" - DatabaseHandler.site_parameters_cached.clear() DatabaseHandler.model_parameters_cached.clear() def _get_array_element_list(self, array_element_name, site, production_table, collection): """ - Return list of array elements (add design model if needed). + Return list of array elements for DB queries (add design model if needed). Design model is added if found in the production table. diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index d7b8794a8e..8624a80322 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -230,7 +230,7 @@ def test_get_model_parameters(db, mocker): """Test get_model_parameters method.""" mock_get_production_table = mocker.patch.object( db, - "get_production_table_from_mongo_db", + "_read_production_table_from_mongo_db", return_value={"parameters": {"LSTN-01": {"param1": "v1"}}}, ) mock_get_array_element_list = mocker.patch.object( @@ -285,7 +285,7 @@ def test_get_model_parameters_with_cache(db, mocker): """Test get_model_parameters method with cache.""" mock_get_production_table = mocker.patch.object( db, - "get_production_table_from_mongo_db", + "_read_production_table_from_mongo_db", return_value={"parameters": {"LSTN-01": {"param1": "v1"}}}, ) mock_get_array_element_list = mocker.patch.object( @@ -319,7 +319,7 @@ def test_get_model_parameters_with_cache(db, mocker): def test_get_model_parameters_no_parameters(db, mocker): """Test get_model_parameters method with no parameters.""" mock_get_production_table = mocker.patch.object( - db, "get_production_table_from_mongo_db", return_value={"parameters": {}} + db, "_read_production_table_from_mongo_db", return_value={"parameters": {}} ) mock_get_array_element_list = mocker.patch.object( db, "_get_array_element_list", return_value=["LSTN-01"] @@ -602,8 +602,8 @@ def test_read_mongo_db(db, mocker): db._read_mongo_db(query, collection_name) -def test_get_production_table_from_mongo_db_with_cache(db, mocker): - """Test get_production_table_from_mongo_db method with cache.""" +def test__read_production_table_from_mongo_db_with_cache(db, mocker): + """Test _read_production_table_from_mongo_db method with cache.""" collection_name = "telescopes" model_version = "1.0.0" param = {"param1": "value1"} @@ -616,7 +616,7 @@ def test_get_production_table_from_mongo_db_with_cache(db, mocker): "entry_date": ObjectId().generation_time, } - result = db.get_production_table_from_mongo_db(collection_name, model_version) + result = db._read_production_table_from_mongo_db(collection_name, model_version) mock_cache_key.assert_called_once_with(None, None, model_version, collection_name) assert result == db_handler.DatabaseHandler.production_table_cached["cache_key"] @@ -636,7 +636,7 @@ def test_get_production_table_from_mongo_db_with_cache(db, mocker): }, ) - result = db.get_production_table_from_mongo_db(collection_name, model_version) + result = db._read_production_table_from_mongo_db(collection_name, model_version) mock_cache_key.assert_called_once_with(None, None, model_version, collection_name) mock_get_collection.assert_called_once_with("test_db", "production_tables") @@ -659,14 +659,14 @@ def test_get_production_table_from_mongo_db_with_cache(db, mocker): ValueError, match=r"The following query returned zero results: {'model_version': '1.0.0', 'collection': 'telescopes'}", ): - db.get_production_table_from_mongo_db(collection_name, model_version) + db._read_production_table_from_mongo_db(collection_name, model_version) def test_get_array_elements_of_type(db, mocker): """Test get_array_elements_of_type method.""" mock_get_production_table = mocker.patch.object( db, - "get_production_table_from_mongo_db", + "_read_production_table_from_mongo_db", return_value={ "parameters": {"LSTN-01": "value1", "LSTN-02": "value2", "MSTS-01": "value3"} }, @@ -1123,18 +1123,15 @@ def test_read_cache_with_no_parameters(db): def test_reset_parameter_cache(db): """Test _reset_parameter_cache method.""" # Populate the cache dictionaries - db_handler.DatabaseHandler.site_parameters_cached = {"key1": "value1"} db_handler.DatabaseHandler.model_parameters_cached = {"key2": "value2"} # Ensure the caches are populated - assert db_handler.DatabaseHandler.site_parameters_cached assert db_handler.DatabaseHandler.model_parameters_cached # Call the method to reset the caches db._reset_parameter_cache() # Check that the caches are cleared - assert not db_handler.DatabaseHandler.site_parameters_cached assert not db_handler.DatabaseHandler.model_parameters_cached From d6228416eada0ab83c407873bc4caef931debb80 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 11:45:54 +0100 Subject: [PATCH 078/124] docstrings --- src/simtools/db/db_model_upload.py | 2 +- src/simtools/model/model_parameter.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simtools/db/db_model_upload.py b/src/simtools/db/db_model_upload.py index 8782a0b01d..5d2b00560a 100644 --- a/src/simtools/db/db_model_upload.py +++ b/src/simtools/db/db_model_upload.py @@ -1,4 +1,4 @@ -"""Module for uploading a simulation model (parameters and production tables) to the database.""" +"""Upload a simulation model (parameters and production tables) to the database.""" import logging from pathlib import Path diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index 68d849f61c..d922d8389e 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -175,7 +175,7 @@ def get_parameter_value_with_unit(self, par_name): _value = self.get_parameter_value(par_name, _parameter) try: - if isinstance(_parameter.get("unit"), str): # TODO - check if still needed + if isinstance(_parameter.get("unit"), str): _unit = [item.strip() for item in _parameter.get("unit").split(",")] else: _unit = _parameter.get("unit") From 1fca7d64c85b95635c9d47110b367124eb17124b Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 12:00:15 +0100 Subject: [PATCH 079/124] code smells --- src/simtools/db/db_model_upload.py | 63 ++++++++++++++------------ tests/unit_tests/db/test_db_handler.py | 53 +++++++++------------- 2 files changed, 56 insertions(+), 60 deletions(-) diff --git a/src/simtools/db/db_model_upload.py b/src/simtools/db/db_model_upload.py index 5d2b00560a..4326012147 100644 --- a/src/simtools/db/db_model_upload.py +++ b/src/simtools/db/db_model_upload.py @@ -96,35 +96,7 @@ def add_production_tables_to_db(args_dict, db): logger.info(f"Reading production tables for model version {model.name}") model_dict = {} for file in sorted(model.rglob("*json")): - array_element = file.stem - collection = names.get_collection_name_from_array_element_name(array_element, False) - model_dict.setdefault( - collection, - { - "collection": collection, - "model_version": model.name, - "parameters": {}, - "design_model": {}, - }, - ) - parameter_dict = gen.collect_data_from_file(file_name=file) - logger.info(f"Reading production table for {array_element} (collection {collection})") - try: - if array_element in ("configuration_corsika", "configuration_sim_telarray"): - model_dict[collection]["parameters"] = parameter_dict["parameters"] - else: - model_dict[collection]["parameters"][array_element] = parameter_dict[ - "parameters" - ][array_element] - except KeyError as exc: - logger.error(f"KeyError: {exc}") - raise - try: - model_dict[collection]["design_model"][array_element] = parameter_dict[ - "design_model" - ][array_element] - except KeyError: - pass + _read_production_table(model_dict, file, model.name) for collection, data in model_dict.items(): if not data["parameters"]: @@ -135,3 +107,36 @@ def add_production_tables_to_db(args_dict, db): db_name=args_dict["db_name"], production_table=data, ) + + +def _read_production_table(model_dict, file, model_name): + """Read a single production table from file.""" + array_element = file.stem + collection = names.get_collection_name_from_array_element_name(array_element, False) + model_dict.setdefault( + collection, + { + "collection": collection, + "model_version": model_name, + "parameters": {}, + "design_model": {}, + }, + ) + parameter_dict = gen.collect_data_from_file(file_name=file) + logger.info(f"Reading production table for {array_element} (collection {collection})") + try: + if array_element in ("configuration_corsika", "configuration_sim_telarray"): + model_dict[collection]["parameters"] = parameter_dict["parameters"] + else: + model_dict[collection]["parameters"][array_element] = parameter_dict["parameters"][ + array_element + ] + except KeyError as exc: + logger.error(f"KeyError: {exc}") + raise + try: + model_dict[collection]["design_model"][array_element] = parameter_dict["design_model"][ + array_element + ] + except KeyError: + pass diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index 8624a80322..4c1db57655 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -55,7 +55,6 @@ def fs_files(): return "fs.files" -# TODO remove? @pytest.fixture def _db_cleanup_file_sandbox(db_no_config_file, random_id, fs_files): yield @@ -380,7 +379,7 @@ def test_get_collections(db, db_config, fs_files): assert "metadata" not in collections_no_model -def test_export_model_files_with_file_names(db, mocker): +def test_export_model_files_with_file_names(db, mocker, tmp_test_directory): """Test export_model_files method with file names.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_get_file_mongo_db = mocker.patch.object( @@ -389,9 +388,8 @@ def test_export_model_files_with_file_names(db, mocker): mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") file_names = ["file1.dat", "file2.dat"] - dest = "/tmp" - result = db.export_model_files(file_names=file_names, dest=dest) + result = db.export_model_files(file_names=file_names, dest=tmp_test_directory) mock_get_db_name.assert_called() mock_get_file_mongo_db.assert_has_calls( @@ -399,14 +397,14 @@ def test_export_model_files_with_file_names(db, mocker): ) mock_write_file_from_mongo_to_disk.assert_has_calls( [ - call("test_db", dest, mock_get_file_mongo_db.return_value), - call("test_db", dest, mock_get_file_mongo_db.return_value), + call("test_db", tmp_test_directory, mock_get_file_mongo_db.return_value), + call("test_db", tmp_test_directory, mock_get_file_mongo_db.return_value), ] ) assert result == {"file1.dat": "file_id", "file2.dat": "file_id"} -def test_export_model_files_with_parameters(db, mocker): +def test_export_model_files_with_parameters(db, mocker, tmp_test_directory): """Test export_model_files method with parameters.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_get_file_mongo_db = mocker.patch.object( @@ -418,9 +416,8 @@ def test_export_model_files_with_parameters(db, mocker): "param1": {"file": True, "value": "file1.dat"}, "param2": {"file": True, "value": "file2.dat"}, } - dest = "/tmp" - result = db.export_model_files(parameters=parameters, dest=dest) + result = db.export_model_files(parameters=parameters, dest=tmp_test_directory) mock_get_db_name.assert_called() mock_get_file_mongo_db.assert_has_calls( @@ -428,14 +425,14 @@ def test_export_model_files_with_parameters(db, mocker): ) mock_write_file_from_mongo_to_disk.assert_has_calls( [ - call("test_db", dest, mock_get_file_mongo_db.return_value), - call("test_db", dest, mock_get_file_mongo_db.return_value), + call("test_db", tmp_test_directory, mock_get_file_mongo_db.return_value), + call("test_db", tmp_test_directory, mock_get_file_mongo_db.return_value), ] ) assert result == {"file1.dat": "file_id", "file2.dat": "file_id"} -def test_export_model_files_file_exists(db, mocker): +def test_export_model_files_file_exists(db, mocker, tmp_test_directory): """Test export_model_files method when file already exists.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_get_file_mongo_db = mocker.patch.object(db, "_get_file_mongo_db") @@ -443,9 +440,8 @@ def test_export_model_files_file_exists(db, mocker): mock_path_exists = mocker.patch("pathlib.Path.exists", return_value=True) file_names = ["file1.dat"] - dest = "/tmp" - result = db.export_model_files(file_names=file_names, dest=dest) + result = db.export_model_files(file_names=file_names, dest=tmp_test_directory) mock_get_db_name.assert_called() mock_get_file_mongo_db.assert_not_called() @@ -454,7 +450,7 @@ def test_export_model_files_file_exists(db, mocker): assert result == {"file1.dat": "file exits"} -def test_export_model_files_file_not_found(db, mocker): +def test_export_model_files_file_not_found(db, mocker, tmp_test_directory): """Test export_model_files method when file is not found in parameters.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_get_file_mongo_db = mocker.patch.object( @@ -465,10 +461,9 @@ def test_export_model_files_file_not_found(db, mocker): parameters = { "param1": {"file": True, "value": "file1.dat"}, } - dest = "/tmp" with pytest.raises(FileNotFoundError): - db.export_model_files(parameters=parameters, dest=dest) + db.export_model_files(parameters=parameters, dest=tmp_test_directory) mock_get_db_name.assert_called() mock_get_file_mongo_db.assert_called_once_with("test_db", "file1.dat") @@ -594,7 +589,7 @@ def test_read_mongo_db(db, mocker): }, } - mock_find = mocker.patch.object(db.get_collection.return_value, "find", return_value=[]) + mocker.patch.object(db.get_collection.return_value, "find", return_value=[]) with pytest.raises( ValueError, match=r"The following query for test_collection returned zero results: {'parameter_version': '1.0.0'}", @@ -652,9 +647,7 @@ def test__read_production_table_from_mongo_db_with_cache(db, mocker): } # test with no results - mock_find_one = mocker.patch.object( - db.get_collection.return_value, "find_one", return_value=None - ) + mocker.patch.object(db.get_collection.return_value, "find_one", return_value=None) with pytest.raises( ValueError, match=r"The following query returned zero results: {'model_version': '1.0.0', 'collection': 'telescopes'}", @@ -719,9 +712,9 @@ def test_get_simulation_configuration_parameters(db, mocker): == return_value ) assert mock_get_model_parameters.call_count == 2 - db.get_simulation_configuration_parameters("simtel", "North", None, "6.0.0") == {} + assert db.get_simulation_configuration_parameters("simtel", "North", None, "6.0.0") == {} assert mock_get_model_parameters.call_count == 2 - db.get_simulation_configuration_parameters("simtel", None, "LSTN-design", "6.0.0") == {} + assert db.get_simulation_configuration_parameters("simtel", None, "LSTN-design", "6.0.0") == {} assert mock_get_model_parameters.call_count == 2 assert db.get_simulation_configuration_parameters("simtel", None, None, "6.0.0") == {} assert mock_get_model_parameters.call_count == 2 @@ -758,7 +751,7 @@ def test_get_file_mongo_db_file(db, mocker): db._get_file_mongo_db(db_name, file_name) -def test_write_file_from_mongo_to_disk(db, mocker): +def test_write_file_from_mongo_to_disk(db, mocker, tmp_test_directory): """Test _write_file_from_mongo_to_disk method.""" mock_db_client = mocker.patch.object( db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} @@ -768,14 +761,13 @@ def test_write_file_from_mongo_to_disk(db, mocker): mock_open = mocker.patch("builtins.open", mocker.mock_open()) db_name = "test_db" - path = "/tmp" file = mocker.Mock() file.filename = "test_file.dat" - db._write_file_from_mongo_to_disk(db_name, path, file) + db._write_file_from_mongo_to_disk(db_name, tmp_test_directory, file) mock_gridfs_bucket.assert_called_once_with(mock_db_client[db_name]) - mock_open.assert_called_once_with(Path(path).joinpath(file.filename), "wb") + mock_open.assert_called_once_with(Path(tmp_test_directory).joinpath(file.filename), "wb") mock_fs_output.download_to_stream_by_name.assert_called_once_with(file.filename, mock_open()) @@ -831,7 +823,7 @@ def test_add_new_parameter(db, mocker): mock_reset_parameter_cache.assert_called_once() -def test_add_new_parameter_with_file(db, mocker): +def test_add_new_parameter_with_file(db, mocker, tmp_test_directory): """Test add_new_parameter method with file.""" mock_validate_model_parameter = mocker.patch( "simtools.db.db_handler.validate_data.DataValidator.validate_model_parameter", @@ -850,9 +842,8 @@ def test_add_new_parameter_with_file(db, mocker): db_name = "test_db" par_dict = {"parameter": "param1", "value": "value1", "file": True} collection_name = "telescopes" - file_prefix = "/tmp" - db.add_new_parameter(db_name, par_dict, collection_name, file_prefix) + db.add_new_parameter(db_name, par_dict, collection_name, tmp_test_directory) mock_validate_model_parameter.assert_called_once_with(par_dict) mock_get_db_name.assert_called_once_with(db_name) @@ -861,7 +852,7 @@ def test_add_new_parameter_with_file(db, mocker): mock_insert_one.assert_called_once_with( {"parameter": "param1", "value": "value1", "file": True, "unit": "unit1"} ) - mock_insert_file_to_db.assert_called_once_with("/tmp/value1", "test_db") + mock_insert_file_to_db.assert_called_once_with(f"{tmp_test_directory!s}/value1", "test_db") mock_reset_parameter_cache.assert_called_once() From c6ed327a7e3bccb275c009b67954bbd25e6ba230 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 12:59:02 +0100 Subject: [PATCH 080/124] code smells --- tests/unit_tests/db/test_db_handler.py | 313 ++++++++++++------------- 1 file changed, 144 insertions(+), 169 deletions(-) diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index 4c1db57655..f3e69bc666 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -50,11 +50,41 @@ def _db_cleanup(db, random_id): db.db_client[f"sandbox_{random_id}"]["sites"].drop() +@pytest.fixture +def test_db(): + return "test_db" + + +@pytest.fixture +def test_file(): + return "test_file.dat" + + +@pytest.fixture +def test_file2(): + return "test_file2.dat" + + @pytest.fixture def fs_files(): return "fs.files" +@pytest.fixture +def value_unit_type(): + return "simtools.db.db_handler.value_conversion.get_value_unit_type" + + +@pytest.fixture +def mock_open(mocker): + return mocker.patch("builtins.open", mocker.mock_open(read_data=b"file_content")) + + +@pytest.fixture +def validate_model_parameter(): + return "simtools.db.db_handler.validate_data.DataValidator.validate_model_parameter" + + @pytest.fixture def _db_cleanup_file_sandbox(db_no_config_file, random_id, fs_files): yield @@ -346,19 +376,17 @@ def test_get_model_parameters_no_parameters(db, mocker): assert result == {} -def test_get_collection(db, mocker): +def test_get_collection(db, mocker, test_db): """Test get_collection method.""" - mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) mocker.patch.object( - db_handler.DatabaseHandler, "db_client", {"test_db": {"test_collection": "mock_collection"}} + db_handler.DatabaseHandler, "db_client", {test_db: {"test_collection": "mock_collection"}} ) - - db_name = "test_db" collection_name = "test_collection" - result = db.get_collection(db_name, collection_name) + result = db.get_collection(test_db, collection_name) - mock_get_db_name.assert_called_once_with(db_name) + mock_get_db_name.assert_called_once_with(test_db) assert result == "mock_collection" @@ -379,67 +407,67 @@ def test_get_collections(db, db_config, fs_files): assert "metadata" not in collections_no_model -def test_export_model_files_with_file_names(db, mocker, tmp_test_directory): +def test_export_model_files_with_file_names( + db, mocker, tmp_test_directory, test_db, test_file, test_file2 +): """Test export_model_files method with file names.""" - mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) mock_get_file_mongo_db = mocker.patch.object( db, "_get_file_mongo_db", return_value=mocker.Mock(_id="file_id") ) mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") - file_names = ["file1.dat", "file2.dat"] + file_names = [test_file, test_file2] result = db.export_model_files(file_names=file_names, dest=tmp_test_directory) mock_get_db_name.assert_called() - mock_get_file_mongo_db.assert_has_calls( - [call("test_db", "file1.dat"), call("test_db", "file2.dat")] - ) + mock_get_file_mongo_db.assert_has_calls([call(test_db, test_file), call(test_db, test_file2)]) mock_write_file_from_mongo_to_disk.assert_has_calls( [ - call("test_db", tmp_test_directory, mock_get_file_mongo_db.return_value), - call("test_db", tmp_test_directory, mock_get_file_mongo_db.return_value), + call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), + call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), ] ) - assert result == {"file1.dat": "file_id", "file2.dat": "file_id"} + assert result == {test_file: "file_id", test_file2: "file_id"} -def test_export_model_files_with_parameters(db, mocker, tmp_test_directory): +def test_export_model_files_with_parameters( + db, mocker, tmp_test_directory, test_db, test_file, test_file2 +): """Test export_model_files method with parameters.""" - mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) mock_get_file_mongo_db = mocker.patch.object( db, "_get_file_mongo_db", return_value=mocker.Mock(_id="file_id") ) mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") parameters = { - "param1": {"file": True, "value": "file1.dat"}, - "param2": {"file": True, "value": "file2.dat"}, + "param1": {"file": True, "value": test_file}, + "param2": {"file": True, "value": test_file2}, } result = db.export_model_files(parameters=parameters, dest=tmp_test_directory) mock_get_db_name.assert_called() - mock_get_file_mongo_db.assert_has_calls( - [call("test_db", "file1.dat"), call("test_db", "file2.dat")] - ) + mock_get_file_mongo_db.assert_has_calls([call(test_db, test_file), call(test_db, test_file2)]) mock_write_file_from_mongo_to_disk.assert_has_calls( [ - call("test_db", tmp_test_directory, mock_get_file_mongo_db.return_value), - call("test_db", tmp_test_directory, mock_get_file_mongo_db.return_value), + call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), + call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), ] ) - assert result == {"file1.dat": "file_id", "file2.dat": "file_id"} + assert result == {test_file: "file_id", test_file2: "file_id"} -def test_export_model_files_file_exists(db, mocker, tmp_test_directory): +def test_export_model_files_file_exists(db, mocker, tmp_test_directory, test_db, test_file): """Test export_model_files method when file already exists.""" - mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) mock_get_file_mongo_db = mocker.patch.object(db, "_get_file_mongo_db") mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") mock_path_exists = mocker.patch("pathlib.Path.exists", return_value=True) - file_names = ["file1.dat"] + file_names = [test_file] result = db.export_model_files(file_names=file_names, dest=tmp_test_directory) @@ -447,26 +475,26 @@ def test_export_model_files_file_exists(db, mocker, tmp_test_directory): mock_get_file_mongo_db.assert_not_called() mock_write_file_from_mongo_to_disk.assert_not_called() mock_path_exists.assert_called_once() - assert result == {"file1.dat": "file exits"} + assert result == {test_file: "file exits"} -def test_export_model_files_file_not_found(db, mocker, tmp_test_directory): +def test_export_model_files_file_not_found(db, mocker, tmp_test_directory, test_db, test_file): """Test export_model_files method when file is not found in parameters.""" - mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) mock_get_file_mongo_db = mocker.patch.object( db, "_get_file_mongo_db", side_effect=FileNotFoundError ) mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") parameters = { - "param1": {"file": True, "value": "file1.dat"}, + "param1": {"file": True, "value": test_file}, } with pytest.raises(FileNotFoundError): db.export_model_files(parameters=parameters, dest=tmp_test_directory) mock_get_db_name.assert_called() - mock_get_file_mongo_db.assert_called_once_with("test_db", "file1.dat") + mock_get_file_mongo_db.assert_called_once_with(test_db, test_file) mock_write_file_from_mongo_to_disk.assert_not_called() @@ -555,9 +583,9 @@ def test_get_query_from_parameter_version_table(db): assert result == expected_query -def test_read_mongo_db(db, mocker): +def test_read_mongo_db(db, mocker, test_db): """Test read_mongo_db method.""" - mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) mock_find = mocker.patch.object( db.get_collection.return_value, @@ -574,7 +602,7 @@ def test_read_mongo_db(db, mocker): result = db._read_mongo_db(query, collection_name) mock_get_db_name.assert_called_once_with() - mock_get_collection.assert_called_once_with("test_db", collection_name) + mock_get_collection.assert_called_once_with(test_db, collection_name) mock_find.assert_called_once_with(query) assert result == { "param1": { @@ -597,7 +625,7 @@ def test_read_mongo_db(db, mocker): db._read_mongo_db(query, collection_name) -def test__read_production_table_from_mongo_db_with_cache(db, mocker): +def test__read_production_table_from_mongo_db_with_cache(db, mocker, test_db): """Test _read_production_table_from_mongo_db method with cache.""" collection_name = "telescopes" model_version = "1.0.0" @@ -618,7 +646,7 @@ def test__read_production_table_from_mongo_db_with_cache(db, mocker): mock_cache_key = mocker.patch.object(db, "_cache_key", return_value="no_cache_key") mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) - mocker.patch.object(db, "_get_db_name", return_value="test_db") + mocker.patch.object(db, "_get_db_name", return_value=test_db) mock_find_one = mocker.patch.object( db.get_collection.return_value, "find_one", @@ -634,7 +662,7 @@ def test__read_production_table_from_mongo_db_with_cache(db, mocker): result = db._read_production_table_from_mongo_db(collection_name, model_version) mock_cache_key.assert_called_once_with(None, None, model_version, collection_name) - mock_get_collection.assert_called_once_with("test_db", "production_tables") + mock_get_collection.assert_called_once_with(test_db, "production_tables") mock_find_one.assert_called_once_with( {"model_version": model_version, "collection": collection_name} ) @@ -723,10 +751,10 @@ def test_get_simulation_configuration_parameters(db, mocker): db.get_simulation_configuration_parameters("wrong", "North", "LSTN-design", "6.0.0") -def test_get_file_mongo_db_file(db, mocker): +def test_get_file_mongo_db_file(db, mocker, test_db, test_file): """Test _get_file_mongo_db method when file exists.""" mock_db_client = mocker.patch.object( - db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} + db_handler.DatabaseHandler, "db_client", {test_db: mocker.Mock()} ) mock_gridfs = mocker.patch("simtools.db.db_handler.gridfs.GridFS") mock_file_system = mock_gridfs.return_value @@ -734,87 +762,82 @@ def test_get_file_mongo_db_file(db, mocker): mock_file_instance = mocker.Mock() mock_file_system.find_one.return_value = mock_file_instance - db_name = "test_db" - file_name = "test_file.dat" - - result = db._get_file_mongo_db(db_name, file_name) + result = db._get_file_mongo_db(test_db, test_file) - mock_gridfs.assert_called_once_with(mock_db_client[db_name]) - mock_file_system.exists.assert_called_once_with({"filename": file_name}) - mock_file_system.find_one.assert_called_once_with({"filename": file_name}) + mock_gridfs.assert_called_once_with(mock_db_client[test_db]) + mock_file_system.exists.assert_called_once_with({"filename": test_file}) + mock_file_system.find_one.assert_called_once_with({"filename": test_file}) assert result == mock_file_instance mock_file_system.exists.return_value = False with pytest.raises( - FileNotFoundError, match=f"The file {file_name} does not exist in the database {db_name}" + FileNotFoundError, match=f"The file {test_file} does not exist in the database {test_db}" ): - db._get_file_mongo_db(db_name, file_name) + db._get_file_mongo_db(test_db, test_file) -def test_write_file_from_mongo_to_disk(db, mocker, tmp_test_directory): +def test_write_file_from_mongo_to_disk( + db, mocker, tmp_test_directory, mock_open, test_db, test_file +): """Test _write_file_from_mongo_to_disk method.""" mock_db_client = mocker.patch.object( db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} ) mock_gridfs_bucket = mocker.patch("simtools.db.db_handler.gridfs.GridFSBucket") mock_fs_output = mock_gridfs_bucket.return_value - mock_open = mocker.patch("builtins.open", mocker.mock_open()) - db_name = "test_db" file = mocker.Mock() - file.filename = "test_file.dat" + file.filename = test_file - db._write_file_from_mongo_to_disk(db_name, tmp_test_directory, file) + db._write_file_from_mongo_to_disk(test_db, tmp_test_directory, file) - mock_gridfs_bucket.assert_called_once_with(mock_db_client[db_name]) + mock_gridfs_bucket.assert_called_once_with(mock_db_client[test_db]) mock_open.assert_called_once_with(Path(tmp_test_directory).joinpath(file.filename), "wb") mock_fs_output.download_to_stream_by_name.assert_called_once_with(file.filename, mock_open()) -def test_add_production_table(db, mocker): +def test_add_production_table(db, mocker, test_db): """Test add_production_table method.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) mock_insert_one = mocker.patch.object(db.get_collection.return_value, "insert_one") - db_name = "test_db" production_table = { "collection": "telescopes", "model_version": "1.0.0", "parameters": {"param1": "value1"}, } - db.add_production_table(db_name, production_table) + db.add_production_table(test_db, production_table) - mock_get_db_name.assert_called_once_with(db_name) + mock_get_db_name.assert_called_once_with(test_db) mock_get_collection.assert_called_once_with("test_db", "production_tables") mock_insert_one.assert_called_once_with(production_table) -def test_add_new_parameter(db, mocker): +def test_add_new_parameter(db, mocker, value_unit_type, validate_model_parameter, test_db): """Test add_new_parameter method.""" mock_validate_model_parameter = mocker.patch( - "simtools.db.db_handler.validate_data.DataValidator.validate_model_parameter", + validate_model_parameter, return_value={"parameter": "param1", "value": "value1", "file": False}, ) mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) mock_insert_one = mocker.patch.object(db.get_collection.return_value, "insert_one") mock_get_value_unit_type = mocker.patch( - "simtools.db.db_handler.value_conversion.get_value_unit_type", + value_unit_type, return_value=("value1", "unit1", None), ) mock_reset_parameter_cache = mocker.patch.object(db, "_reset_parameter_cache") - db_name = "test_db" par_dict = {"parameter": "param1", "value": "value1", "file": False} collection_name = "telescopes" file_prefix = None - db.add_new_parameter(db_name, par_dict, collection_name, file_prefix) + db.add_new_parameter(test_db, par_dict, collection_name, file_prefix) mock_validate_model_parameter.assert_called_once_with(par_dict) - mock_get_db_name.assert_called_once_with(db_name) + mock_get_db_name.assert_called_once_with(test_db) mock_get_collection.assert_called_once_with("test_db", collection_name) mock_get_value_unit_type.assert_called_once_with(value="value1", unit_str=None) mock_insert_one.assert_called_once_with( @@ -823,30 +846,31 @@ def test_add_new_parameter(db, mocker): mock_reset_parameter_cache.assert_called_once() -def test_add_new_parameter_with_file(db, mocker, tmp_test_directory): +def test_add_new_parameter_with_file( + db, mocker, tmp_test_directory, value_unit_type, validate_model_parameter, test_db +): """Test add_new_parameter method with file.""" mock_validate_model_parameter = mocker.patch( - "simtools.db.db_handler.validate_data.DataValidator.validate_model_parameter", + validate_model_parameter, return_value={"parameter": "param1", "value": "value1", "file": True}, ) mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) mock_insert_one = mocker.patch.object(db.get_collection.return_value, "insert_one") mock_get_value_unit_type = mocker.patch( - "simtools.db.db_handler.value_conversion.get_value_unit_type", + value_unit_type, return_value=("value1", "unit1", None), ) mock_insert_file_to_db = mocker.patch.object(db, "insert_file_to_db") mock_reset_parameter_cache = mocker.patch.object(db, "_reset_parameter_cache") - db_name = "test_db" par_dict = {"parameter": "param1", "value": "value1", "file": True} collection_name = "telescopes" - db.add_new_parameter(db_name, par_dict, collection_name, tmp_test_directory) + db.add_new_parameter(test_db, par_dict, collection_name, tmp_test_directory) mock_validate_model_parameter.assert_called_once_with(par_dict) - mock_get_db_name.assert_called_once_with(db_name) + mock_get_db_name.assert_called_once_with(test_db) mock_get_collection.assert_called_once_with("test_db", collection_name) mock_get_value_unit_type.assert_called_once_with(value="value1", unit_str=None) mock_insert_one.assert_called_once_with( @@ -856,21 +880,22 @@ def test_add_new_parameter_with_file(db, mocker, tmp_test_directory): mock_reset_parameter_cache.assert_called_once() -def test_add_new_parameter_with_file_no_prefix(db, mocker): +def test_add_new_parameter_with_file_no_prefix( + db, mocker, value_unit_type, validate_model_parameter, test_db +): """Test add_new_parameter method with file but no file_prefix.""" mock_validate_model_parameter = mocker.patch( - "simtools.db.db_handler.validate_data.DataValidator.validate_model_parameter", + validate_model_parameter, return_value={"parameter": "param1", "value": "value1", "file": True}, ) mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) mock_get_value_unit_type = mocker.patch( - "simtools.db.db_handler.value_conversion.get_value_unit_type", + value_unit_type, return_value=("value1", "unit1", None), ) mock_reset_parameter_cache = mocker.patch.object(db, "_reset_parameter_cache") - db_name = "test_db" par_dict = {"parameter": "param1", "value": "value1", "file": True} collection_name = "telescopes" file_prefix = None @@ -879,16 +904,16 @@ def test_add_new_parameter_with_file_no_prefix(db, mocker): FileNotFoundError, match=r"The location of the file to upload, corresponding to the param1 parameter, must be provided.", ): - db.add_new_parameter(db_name, par_dict, collection_name, file_prefix) + db.add_new_parameter(test_db, par_dict, collection_name, file_prefix) mock_validate_model_parameter.assert_called_once_with(par_dict) - mock_get_db_name.assert_called_once_with(db_name) + mock_get_db_name.assert_called_once_with(test_db) mock_get_collection.assert_called_once_with("test_db", collection_name) mock_get_value_unit_type.assert_called_once_with(value="value1", unit_str=None) mock_reset_parameter_cache.assert_not_called() -def test_insert_file_to_db_file_exists(db, mocker): +def test_insert_file_to_db_file_exists(db, mocker, test_db, test_file): """Test insert_file_to_db method when file already exists in the DB.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_db_client = mocker.patch.object( @@ -900,19 +925,16 @@ def test_insert_file_to_db_file_exists(db, mocker): mock_file_instance = mocker.Mock() mock_file_system.find_one.return_value = mock_file_instance - file_name = "test_file.dat" - db_name = "test_db" + result = db.insert_file_to_db(test_file, test_db) - result = db.insert_file_to_db(file_name, db_name) - - mock_get_db_name.assert_called_once_with(db_name) - mock_gridfs.assert_called_once_with(mock_db_client[db_name]) - mock_file_system.exists.assert_called_once_with({"filename": file_name}) - mock_file_system.find_one.assert_called_once_with({"filename": file_name}) + mock_get_db_name.assert_called_once_with(test_db) + mock_gridfs.assert_called_once_with(mock_db_client[test_db]) + mock_file_system.exists.assert_called_once_with({"filename": test_file}) + mock_file_system.find_one.assert_called_once_with({"filename": test_file}) assert result == mock_file_instance._id -def test_insert_file_to_db_new_file(db, mocker): +def test_insert_file_to_db_new_file(db, mocker, mock_open, test_db, test_file): """Test insert_file_to_db method when file does not exist in the DB.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_db_client = mocker.patch.object( @@ -922,24 +944,20 @@ def test_insert_file_to_db_new_file(db, mocker): mock_file_system = mock_gridfs.return_value mock_file_system.exists.return_value = False mock_file_system.put.return_value = "new_file_id" - mock_open = mocker.patch("builtins.open", mocker.mock_open(read_data=b"file_content")) - - file_name = "test_file.dat" - db_name = "test_db" - result = db.insert_file_to_db(file_name, db_name) + result = db.insert_file_to_db(test_file, test_db) - mock_get_db_name.assert_called_once_with(db_name) - mock_gridfs.assert_called_once_with(mock_db_client[db_name]) - mock_file_system.exists.assert_called_once_with({"filename": file_name}) - mock_open.assert_called_once_with(file_name, "rb") + mock_get_db_name.assert_called_once_with(test_db) + mock_gridfs.assert_called_once_with(mock_db_client[test_db]) + mock_file_system.exists.assert_called_once_with({"filename": test_file}) + mock_open.assert_called_once_with(test_file, "rb") mock_file_system.put.assert_called_once_with( - mock_open(), content_type="ascii/dat", filename=file_name + mock_open(), content_type="ascii/dat", filename=test_file ) assert result == "new_file_id" -def test_insert_file_to_db_with_kwargs(db, mocker): +def test_insert_file_to_db_with_kwargs(db, mocker, mock_open, test_db, test_file): """Test insert_file_to_db method with additional kwargs.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_db_client = mocker.patch.object( @@ -949,22 +967,19 @@ def test_insert_file_to_db_with_kwargs(db, mocker): mock_file_system = mock_gridfs.return_value mock_file_system.exists.return_value = False mock_file_system.put.return_value = "new_file_id" - mock_open = mocker.patch("builtins.open", mocker.mock_open(read_data=b"file_content")) - file_name = "test_file.dat" - db_name = "test_db" kwargs = {"content_type": "application/octet-stream", "metadata": {"key": "value"}} - result = db.insert_file_to_db(file_name, db_name, **kwargs) + result = db.insert_file_to_db(test_file, test_db, **kwargs) - mock_get_db_name.assert_called_once_with(db_name) - mock_gridfs.assert_called_once_with(mock_db_client[db_name]) - mock_file_system.exists.assert_called_once_with({"filename": file_name}) - mock_open.assert_called_once_with(file_name, "rb") + mock_get_db_name.assert_called_once_with(test_db) + mock_gridfs.assert_called_once_with(mock_db_client[test_db]) + mock_file_system.exists.assert_called_once_with({"filename": test_file}) + mock_open.assert_called_once_with(test_file, "rb") mock_file_system.put.assert_called_once_with( mock_open(), content_type="application/octet-stream", - filename=file_name, + filename=test_file, metadata={"key": "value"}, ) assert result == "new_file_id" @@ -1031,84 +1046,44 @@ def test_cache_key(db): assert result == "" -def test_read_cache_with_cache_hit(db): - """Test _read_cache method when cache hit occurs.""" - cache_dict = {"1.0.0-telescopes-North-LSTN-01": {"param1": "value1"}} - site = "North" - array_element_name = "LSTN-01" - model_version = "1.0.0" - collection = "telescopes" +def test_read_cache(db): + test_key = "1.0.0-telescopes-North-LSTN-01" + test_param1 = {"param1": "value1"} + cache_dict = {test_key: test_param1} - cache_key, result = db._read_cache( - cache_dict, site, array_element_name, model_version, collection - ) - - assert cache_key == "1.0.0-telescopes-North-LSTN-01" - assert result == {"param1": "value1"} - - -def test_read_cache_with_cache_miss(db): - """Test _read_cache method when cache miss occurs.""" - cache_dict = {"1.0.0-telescopes-North-LSTN-01": {"param1": "value1"}} + # Test _read_cache method when cache hit occurs. site = "North" - array_element_name = "LSTN-02" + array_element_name = "LSTN-01" model_version = "1.0.0" collection = "telescopes" cache_key, result = db._read_cache( cache_dict, site, array_element_name, model_version, collection ) + assert cache_key == test_key + assert result == test_param1 + # Test _read_cache method when cache miss occurs. + cache_key, result = db._read_cache(cache_dict, site, "LSTN-02", model_version, collection) assert cache_key == "1.0.0-telescopes-North-LSTN-02" assert result is None - -def test_read_cache_with_empty_cache(db): - """Test _read_cache method with empty cache.""" - cache_dict = {} - site = "North" - array_element_name = "LSTN-01" - model_version = "1.0.0" - collection = "telescopes" - - cache_key, result = db._read_cache( - cache_dict, site, array_element_name, model_version, collection - ) - + # Test _read_cache method with empty cache. + cache_key, result = db._read_cache({}, site, array_element_name, model_version, collection) assert cache_key == "1.0.0-telescopes-North-LSTN-01" assert result is None - -def test_read_cache_with_partial_parameters(db): - """Test _read_cache method with partial parameters.""" - cache_dict = {"1.0.0-telescopes-North": {"param1": "value1"}} - site = "North" - array_element_name = None - model_version = "1.0.0" - collection = "telescopes" - - cache_key, result = db._read_cache( - cache_dict, site, array_element_name, model_version, collection - ) - + # Test _read_cache method with partial parameters. + cache_dict = {"1.0.0-telescopes-North": test_param1} + cache_key, result = db._read_cache(cache_dict, site, None, model_version, collection) assert cache_key == "1.0.0-telescopes-North" - assert result == {"param1": "value1"} - - -def test_read_cache_with_no_parameters(db): - """Test _read_cache method with no parameters.""" - cache_dict = {"": {"param1": "value1"}} - site = None - array_element_name = None - model_version = None - collection = None - - cache_key, result = db._read_cache( - cache_dict, site, array_element_name, model_version, collection - ) + assert result == test_param1 + # Test _read_cache method with no parameters. + cache_dict = {"": test_param1} + cache_key, result = db._read_cache(cache_dict, None, None, None, None) assert cache_key == "" - assert result == {"param1": "value1"} + assert result == test_param1 def test_reset_parameter_cache(db): From c13135ce26c61c1f0af8c62227a7c43656e7ce0b Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 15:27:18 +0100 Subject: [PATCH 081/124] unit tests --- src/simtools/db/db_model_upload.py | 17 +-- tests/unit_tests/db/test_db_handler.py | 18 ++- tests/unit_tests/db/test_db_model_upload.py | 149 ++++++++++++++++++++ 3 files changed, 167 insertions(+), 17 deletions(-) create mode 100644 tests/unit_tests/db/test_db_model_upload.py diff --git a/src/simtools/db/db_model_upload.py b/src/simtools/db/db_model_upload.py index 4326012147..4b59e7ffba 100644 --- a/src/simtools/db/db_model_upload.py +++ b/src/simtools/db/db_model_upload.py @@ -63,16 +63,13 @@ def add_model_parameters_to_db(args_dict, db): logger.info(f"Reading model parameters for {element.name} into collection {collection}") files_to_insert = list(Path(element).rglob("*json")) for file in files_to_insert: - if collection == "files": - logger.info("Not yet implemented files") - else: - add_values_from_json_to_db( - file=file, - collection=collection, - db=db, - db_name=args_dict["db_name"], - file_prefix=input_path / "Files", - ) + add_values_from_json_to_db( + file=file, + collection=collection, + db=db, + db_name=args_dict["db_name"], + file_prefix=input_path / "Files", + ) def add_production_tables_to_db(args_dict, db): diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index f3e69bc666..a66e3d6879 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -85,6 +85,11 @@ def validate_model_parameter(): return "simtools.db.db_handler.validate_data.DataValidator.validate_model_parameter" +@pytest.fixture +def mock_gridfs(mocker): + return mocker.patch("simtools.db.db_handler.gridfs.GridFS") + + @pytest.fixture def _db_cleanup_file_sandbox(db_no_config_file, random_id, fs_files): yield @@ -751,12 +756,11 @@ def test_get_simulation_configuration_parameters(db, mocker): db.get_simulation_configuration_parameters("wrong", "North", "LSTN-design", "6.0.0") -def test_get_file_mongo_db_file(db, mocker, test_db, test_file): +def test_get_file_mongo_db_file(db, mocker, test_db, test_file, mock_gridfs): """Test _get_file_mongo_db method when file exists.""" mock_db_client = mocker.patch.object( db_handler.DatabaseHandler, "db_client", {test_db: mocker.Mock()} ) - mock_gridfs = mocker.patch("simtools.db.db_handler.gridfs.GridFS") mock_file_system = mock_gridfs.return_value mock_file_system.exists.return_value = True mock_file_instance = mocker.Mock() @@ -957,13 +961,12 @@ def test_insert_file_to_db_new_file(db, mocker, mock_open, test_db, test_file): assert result == "new_file_id" -def test_insert_file_to_db_with_kwargs(db, mocker, mock_open, test_db, test_file): +def test_insert_file_to_db_with_kwargs(db, mocker, mock_open, test_db, test_file, mock_gridfs): """Test insert_file_to_db method with additional kwargs.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_db_client = mocker.patch.object( db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} ) - mock_gridfs = mocker.patch("simtools.db.db_handler.gridfs.GridFS") mock_file_system = mock_gridfs.return_value mock_file_system.exists.return_value = False mock_file_system.put.return_value = "new_file_id" @@ -1070,13 +1073,14 @@ def test_read_cache(db): # Test _read_cache method with empty cache. cache_key, result = db._read_cache({}, site, array_element_name, model_version, collection) - assert cache_key == "1.0.0-telescopes-North-LSTN-01" + assert cache_key == test_key assert result is None # Test _read_cache method with partial parameters. - cache_dict = {"1.0.0-telescopes-North": test_param1} + test_key = "1.0.0-telescopes-North" + cache_dict = {test_key: test_param1} cache_key, result = db._read_cache(cache_dict, site, None, model_version, collection) - assert cache_key == "1.0.0-telescopes-North" + assert cache_key == test_key assert result == test_param1 # Test _read_cache method with no parameters. diff --git a/tests/unit_tests/db/test_db_model_upload.py b/tests/unit_tests/db/test_db_model_upload.py new file mode 100644 index 0000000000..c39bd13956 --- /dev/null +++ b/tests/unit_tests/db/test_db_model_upload.py @@ -0,0 +1,149 @@ +#!/usr/bin/python3 + +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest + +import simtools.db.db_model_upload as db_model_upload + + +@patch("simtools.db.db_model_upload.gen.collect_data_from_file") +def test_add_values_from_json_to_db(mock_collect_data_from_file): + mock_collect_data_from_file.return_value = { + "parameter": "test_param", + "parameter_version": "1.0", + } + mock_db = Mock() + file = "test_file.json" + collection = "test_collection" + db_name = "test_db" + file_prefix = "test_prefix" + + db_model_upload.add_values_from_json_to_db(file, collection, mock_db, db_name, file_prefix) + + mock_collect_data_from_file.assert_called_once_with(file_name=file) + mock_db.add_new_parameter.assert_called_once_with( + db_name=db_name, + par_dict={"parameter": "test_param", "parameter_version": "1.0"}, + collection_name=collection, + file_prefix=file_prefix, + ) + + +@patch("simtools.db.db_model_upload.gen.collect_data_from_file") +def test_read_production_table(mock_collect_data_from_file): + mock_collect_data_from_file.return_value = { + "parameters": {"LSTN-design": "param_value_1", "LSTN-01": "param_value_2"}, + "design_model": {"LSTN-design": "design_value_1", "LSTN-01": "design_value_2"}, + } + model_dict = {} + file = Mock() + file.stem = "LSTN-design" + model_name = "test_model" + + db_model_upload._read_production_table(model_dict, file, model_name) + + assert "LSTN-design" in model_dict["telescopes"]["parameters"] + assert model_dict["telescopes"]["parameters"]["LSTN-design"] == "param_value_1" + assert model_dict["telescopes"]["design_model"]["LSTN-design"] == "design_value_1" + + file.stem = "MSTN-design" + with pytest.raises(KeyError, match="MSTN-design"): + db_model_upload._read_production_table(model_dict, file, model_name) + + file.stem = "configuration_corsika" + mock_collect_data_from_file.return_value = {"parameters": "config_param_value"} + + db_model_upload._read_production_table(model_dict, file, model_name) + + assert model_dict["configuration_corsika"]["parameters"] == "config_param_value" + + +@patch("simtools.db.db_model_upload._read_production_table") +def test_add_production_tables_to_db(mock_read_production_table, tmp_test_directory, caplog): + mock_db = Mock() + args_dict = {"input_path": tmp_test_directory, "db_name": "test_db"} + input_path = Path(args_dict["input_path"]) + model_dir = input_path / "model_version_1" + model_dir.mkdir(parents=True, exist_ok=True) + (model_dir / "file1.json").touch() + (model_dir / "MSTS-02.json").touch() + + mock_read_production_table.side_effect = lambda model_dict, file, _: model_dict.update( + { + "telescopes": { + "parameters": {"MSTS-02": "param_value"}, + "design_model": {"MSTS-02": "MSTS-design"}, + } + } + ) + + with patch("simtools.db.db_model_upload.Path.iterdir", return_value=[model_dir]): + with patch("simtools.db.db_model_upload.Path.is_dir", return_value=True): + db_model_upload.add_production_tables_to_db(args_dict, mock_db) + + assert mock_read_production_table.call_count == 2 + mock_db.add_production_table.assert_called_once_with( + db_name="test_db", + production_table={ + "parameters": {"MSTS-02": "param_value"}, + "design_model": {"MSTS-02": "MSTS-design"}, + }, + ) + + mock_read_production_table.side_effect = lambda model_dict, file, _: model_dict.update( + {"telescopes": {"parameters": {}, "design_model": {}}} + ) + with caplog.at_level("INFO"): + db_model_upload.add_production_tables_to_db(args_dict, mock_db) + assert "No production table for telescopes in model version model_version_1" in caplog.text + + +@patch("simtools.db.db_model_upload.add_values_from_json_to_db") +def test_add_model_parameters_to_db(mock_add_values_from_json_to_db, tmp_test_directory): + mock_db = Mock() + args_dict = {"input_path": tmp_test_directory, "db_name": "test_db"} + input_path = Path(args_dict["input_path"]) + array_element_dir = input_path / "LSTS-01" + array_element_dir.mkdir(parents=True, exist_ok=True) + (array_element_dir / "num_gains-0.1.0.json").touch() + (array_element_dir / "mirror_list-0.2.1.json").touch() + + with patch("simtools.db.db_model_upload.Path.iterdir", return_value=[array_element_dir]): + with patch("simtools.db.db_model_upload.Path.is_dir", return_value=True): + db_model_upload.add_model_parameters_to_db(args_dict, mock_db) + + mock_add_values_from_json_to_db.assert_any_call( + file=array_element_dir / "num_gains-0.1.0.json", + collection="telescopes", + db=mock_db, + db_name="test_db", + file_prefix=input_path / "Files", + ) + mock_add_values_from_json_to_db.assert_any_call( + file=array_element_dir / "mirror_list-0.2.1.json", + collection="telescopes", + db=mock_db, + db_name="test_db", + file_prefix=input_path / "Files", + ) + assert mock_add_values_from_json_to_db.call_count == 2 + + +@patch("simtools.db.db_model_upload.add_values_from_json_to_db") +def test_add_model_parameters_to_db_skip_files_collection( + mock_add_values_from_json_to_db, tmp_test_directory +): + mock_db = Mock() + args_dict = {"input_path": tmp_test_directory, "db_name": "test_db"} + input_path = Path(args_dict["input_path"]) + files_dir = input_path / "Files" + files_dir.mkdir(parents=True, exist_ok=True) + (files_dir / "file1.json").touch() + + with patch("simtools.db.db_model_upload.Path.iterdir", return_value=[files_dir]): + with patch("simtools.db.db_model_upload.Path.is_dir", return_value=True): + db_model_upload.add_model_parameters_to_db(args_dict, mock_db) + + mock_add_values_from_json_to_db.assert_not_called() From 420ceee6bd5a57124494fa6139dab4c38212c053 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 17:20:19 +0100 Subject: [PATCH 082/124] code smells --- tests/unit_tests/db/test_db_handler.py | 6 ++-- tests/unit_tests/db/test_db_model_upload.py | 32 +++++++++++++++------ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index a66e3d6879..48a11d0994 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -917,13 +917,12 @@ def test_add_new_parameter_with_file_no_prefix( mock_reset_parameter_cache.assert_not_called() -def test_insert_file_to_db_file_exists(db, mocker, test_db, test_file): +def test_insert_file_to_db_file_exists(db, mocker, test_db, test_file, mock_gridfs): """Test insert_file_to_db method when file already exists in the DB.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_db_client = mocker.patch.object( db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} ) - mock_gridfs = mocker.patch("simtools.db.db_handler.gridfs.GridFS") mock_file_system = mock_gridfs.return_value mock_file_system.exists.return_value = True mock_file_instance = mocker.Mock() @@ -938,13 +937,12 @@ def test_insert_file_to_db_file_exists(db, mocker, test_db, test_file): assert result == mock_file_instance._id -def test_insert_file_to_db_new_file(db, mocker, mock_open, test_db, test_file): +def test_insert_file_to_db_new_file(db, mocker, mock_open, test_db, test_file, mock_gridfs): """Test insert_file_to_db method when file does not exist in the DB.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") mock_db_client = mocker.patch.object( db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} ) - mock_gridfs = mocker.patch("simtools.db.db_handler.gridfs.GridFS") mock_file_system = mock_gridfs.return_value mock_file_system.exists.return_value = False mock_file_system.put.return_value = "new_file_id" diff --git a/tests/unit_tests/db/test_db_model_upload.py b/tests/unit_tests/db/test_db_model_upload.py index c39bd13956..61abb97000 100644 --- a/tests/unit_tests/db/test_db_model_upload.py +++ b/tests/unit_tests/db/test_db_model_upload.py @@ -8,6 +8,16 @@ import simtools.db.db_model_upload as db_model_upload +@pytest.fixture +def iter_dir(): + return "simtools.db.db_model_upload.Path.iterdir" + + +@pytest.fixture +def is_dir(): + return "simtools.db.db_model_upload.Path.is_dir" + + @patch("simtools.db.db_model_upload.gen.collect_data_from_file") def test_add_values_from_json_to_db(mock_collect_data_from_file): mock_collect_data_from_file.return_value = { @@ -61,7 +71,9 @@ def test_read_production_table(mock_collect_data_from_file): @patch("simtools.db.db_model_upload._read_production_table") -def test_add_production_tables_to_db(mock_read_production_table, tmp_test_directory, caplog): +def test_add_production_tables_to_db( + mock_read_production_table, tmp_test_directory, caplog, iter_dir, is_dir +): mock_db = Mock() args_dict = {"input_path": tmp_test_directory, "db_name": "test_db"} input_path = Path(args_dict["input_path"]) @@ -79,8 +91,8 @@ def test_add_production_tables_to_db(mock_read_production_table, tmp_test_direct } ) - with patch("simtools.db.db_model_upload.Path.iterdir", return_value=[model_dir]): - with patch("simtools.db.db_model_upload.Path.is_dir", return_value=True): + with patch(iter_dir, return_value=[model_dir]): + with patch(is_dir, return_value=True): db_model_upload.add_production_tables_to_db(args_dict, mock_db) assert mock_read_production_table.call_count == 2 @@ -101,7 +113,9 @@ def test_add_production_tables_to_db(mock_read_production_table, tmp_test_direct @patch("simtools.db.db_model_upload.add_values_from_json_to_db") -def test_add_model_parameters_to_db(mock_add_values_from_json_to_db, tmp_test_directory): +def test_add_model_parameters_to_db( + mock_add_values_from_json_to_db, tmp_test_directory, iter_dir, is_dir +): mock_db = Mock() args_dict = {"input_path": tmp_test_directory, "db_name": "test_db"} input_path = Path(args_dict["input_path"]) @@ -110,8 +124,8 @@ def test_add_model_parameters_to_db(mock_add_values_from_json_to_db, tmp_test_di (array_element_dir / "num_gains-0.1.0.json").touch() (array_element_dir / "mirror_list-0.2.1.json").touch() - with patch("simtools.db.db_model_upload.Path.iterdir", return_value=[array_element_dir]): - with patch("simtools.db.db_model_upload.Path.is_dir", return_value=True): + with patch(iter_dir, return_value=[array_element_dir]): + with patch(is_dir, return_value=True): db_model_upload.add_model_parameters_to_db(args_dict, mock_db) mock_add_values_from_json_to_db.assert_any_call( @@ -133,7 +147,7 @@ def test_add_model_parameters_to_db(mock_add_values_from_json_to_db, tmp_test_di @patch("simtools.db.db_model_upload.add_values_from_json_to_db") def test_add_model_parameters_to_db_skip_files_collection( - mock_add_values_from_json_to_db, tmp_test_directory + mock_add_values_from_json_to_db, tmp_test_directory, iter_dir, is_dir ): mock_db = Mock() args_dict = {"input_path": tmp_test_directory, "db_name": "test_db"} @@ -142,8 +156,8 @@ def test_add_model_parameters_to_db_skip_files_collection( files_dir.mkdir(parents=True, exist_ok=True) (files_dir / "file1.json").touch() - with patch("simtools.db.db_model_upload.Path.iterdir", return_value=[files_dir]): - with patch("simtools.db.db_model_upload.Path.is_dir", return_value=True): + with patch(iter_dir, return_value=[files_dir]): + with patch(is_dir, return_value=True): db_model_upload.add_model_parameters_to_db(args_dict, mock_db) mock_add_values_from_json_to_db.assert_not_called() From cb1dbbd9df9cacf716b559552aaf825e7084d921 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 17:29:16 +0100 Subject: [PATCH 083/124] unit tests for names --- src/simtools/utils/names.py | 6 +++--- tests/unit_tests/utils/test_names.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/simtools/utils/names.py b/src/simtools/utils/names.py index b63594a8ce..43415e0290 100644 --- a/src/simtools/utils/names.py +++ b/src/simtools/utils/names.py @@ -313,14 +313,14 @@ def get_collection_name_from_array_element_name(name, array_elements_only=True): return array_elements()[get_array_element_type_from_name(name)]["collection"] except ValueError: pass + if name.startswith("OBS"): + return "sites" try: validate_site_name(name) return "sites" except ValueError as exc: if array_elements_only: - raise ValueError(f"Invalid array element name {name}: {exc}") from exc - if name.startswith("OBS"): - return "sites" + raise ValueError(f"Invalid array element name {name}") from exc if name in ( "configuration_sim_telarray", "configuration_corsika", diff --git a/tests/unit_tests/utils/test_names.py b/tests/unit_tests/utils/test_names.py index 2ed327e73a..c8de8bbc9b 100644 --- a/tests/unit_tests/utils/test_names.py +++ b/tests/unit_tests/utils/test_names.py @@ -153,6 +153,24 @@ def test_get_class_from_telescope_name(invalid_name): names.get_collection_name_from_array_element_name("Not_a_collection") +def test_get_collection_name_from_array_element_name(): + + assert "telescopes" == names.get_collection_name_from_array_element_name("LSTN-01") + assert "telescopes" == names.get_collection_name_from_array_element_name("MSTS-design") + assert "sites" == names.get_collection_name_from_array_element_name("OBS-North") + assert "sites" == names.get_collection_name_from_array_element_name("North") + assert "sites" == names.get_collection_name_from_array_element_name("OBS-North", False) + assert "configuration_sim_telarray" == names.get_collection_name_from_array_element_name( + "configuration_sim_telarray", False + ) + + with pytest.raises(ValueError, match=r"Invalid array element name configuration_sim_telarray"): + names.get_collection_name_from_array_element_name("configuration_sim_telarray", True) + + with pytest.raises(ValueError, match=r"Invalid array element name Not_a_collection"): + names.get_collection_name_from_array_element_name("Not_a_collection", False) + + def test_sanitize_name(caplog): assert names.sanitize_name("y_edges unit") == "y_edges_unit" assert names.sanitize_name("Y_EDGES UNIT") == "y_edges_unit" From 2a1137cb102dd1e164bfc11bea0378ad20465f3e Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 21 Jan 2025 17:46:40 +0100 Subject: [PATCH 084/124] unit tests --- .../configuration/test_commandline_parser.py | 15 ++++--- .../data_model/test_validate_data.py | 43 +++++++++++++++++++ 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/tests/unit_tests/configuration/test_commandline_parser.py b/tests/unit_tests/configuration/test_commandline_parser.py index 38e684a996..4f14afe607 100644 --- a/tests/unit_tests/configuration/test_commandline_parser.py +++ b/tests/unit_tests/configuration/test_commandline_parser.py @@ -179,25 +179,30 @@ def test_simulation_model(): assert all(action.dest != "site" for action in group._group_actions) assert all(action.dest != "telescope" for action in group._group_actions) - # Site model can exist without a telescope model + # Site model can exist without a telescope model, model and parameter version _parser_s = parser.CommandLineParser() - _parser_s.initialize_default_arguments(simulation_model=["site", "model_version"]) + _parser_s.initialize_default_arguments( + simulation_model=["site", "model_version", "parameter_version"] + ) job_groups = _parser_s._action_groups assert SIMULATION_MODEL_STRING in [str(group.title) for group in job_groups] for group in job_groups: if str(group.title) == SIMULATION_MODEL_STRING: assert any(action.dest == "model_version" for action in group._group_actions) + assert any(action.dest == "parameter_version" for action in group._group_actions) assert any(action.dest == "site" for action in group._group_actions) assert all(action.dest != "telescope" for action in group._group_actions) - # No telescope model without site model + # No telescope model without site model; parameter_version only _parser_t = parser.CommandLineParser() - _parser_t.initialize_default_arguments(simulation_model=["telescope", "site", "model_version"]) + _parser_t.initialize_default_arguments( + simulation_model=["telescope", "site", "parameter_version"] + ) job_groups = _parser_t._action_groups assert SIMULATION_MODEL_STRING in [str(group.title) for group in job_groups] for group in job_groups: if str(group.title) == SIMULATION_MODEL_STRING: - assert any(action.dest == "model_version" for action in group._group_actions) + assert any(action.dest == "parameter_version" for action in group._group_actions) assert any(action.dest == "site" for action in group._group_actions) assert any(action.dest == "telescope" for action in group._group_actions) diff --git a/tests/unit_tests/data_model/test_validate_data.py b/tests/unit_tests/data_model/test_validate_data.py index 83cff14c57..d5ae98fd3d 100644 --- a/tests/unit_tests/data_model/test_validate_data.py +++ b/tests/unit_tests/data_model/test_validate_data.py @@ -5,6 +5,7 @@ import shutil import sys from importlib.resources import files +from pathlib import Path import jsonschema import numpy as np @@ -661,6 +662,34 @@ def test_validate_data_dict(): data_validator_2.data_dict = {"name": "num_gains", "value": [2], "unit": ["null"]} data_validator_2._validate_data_dict() + data_validator_3 = validate_data.DataValidator( + schema_file=str(schema_dir) + "/random_focal_length.schema.yml" + ) + data_validator_3.data_dict = { + "name": "random_focal_length", + "value": [1.0, 2.0], + "unit": ["m", "m"], + } + result_3 = data_validator_3._validate_data_dict() + assert isinstance(result_3["value"], list) + result_3_str = data_validator_3._validate_data_dict(lists_as_strings=True) + assert isinstance(result_3_str["value"], str) + + +def test_convert_results_to_model_format(): + schema_dir = files("simtools").joinpath("schemas/model_parameters/") + data_validator_3 = validate_data.DataValidator( + schema_file=str(schema_dir) + "/random_focal_length.schema.yml" + ) + data_validator_3.data_dict = { + "name": "random_focal_length", + "value": [1.0, 2.0], + "unit": ["m", "m"], + } + data_validator_3._convert_results_to_model_format() + assert data_validator_3.data_dict["value"] == "1.0 2.0" + assert data_validator_3.data_dict["unit"] == "m m" + def test_prepare_model_parameter(): data_validator = validate_data.DataValidator() @@ -867,3 +896,17 @@ def test_validate_value_and_unit_for_dict(reference_columns): ) assert value == {"key": "value"} assert unit == "null" + + +def test_validate_model_parameter(mocker): + mocker.patch("simtools.data_model.validate_data.files", return_value=Path("tests/resources")) + mocker.patch( + "simtools.data_model.validate_data.DataValidator._read_validation_schema", + return_value=[{"name": "parameter", "type": "float", "unit": "km"}], + ) + + par_dict = {"parameter": "reference_point_altitude", "value": 1000.0, "unit": "km"} + + validated_data = validate_data.DataValidator.validate_model_parameter(par_dict) + assert validated_data["value"] == 1000.0 + assert validated_data["unit"] == "km" From 067895516dc98cf3354dda4a336525ba078b45f9 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 08:23:33 +0100 Subject: [PATCH 085/124] model validation --- .../applications/validate_file_using_schema.py | 10 ++++++++-- src/simtools/schemas/model_parameter.metaschema.yml | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/simtools/applications/validate_file_using_schema.py b/src/simtools/applications/validate_file_using_schema.py index b36252ef42..6deef1507b 100644 --- a/src/simtools/applications/validate_file_using_schema.py +++ b/src/simtools/applications/validate_file_using_schema.py @@ -37,6 +37,8 @@ from importlib.resources import files from pathlib import Path +import jsonschema + import simtools.utils.general as gen from simtools.configuration import configurator from simtools.data_model import metadata_collector, metadata_model, validate_data @@ -132,8 +134,12 @@ def validate_schema(args_dict, logger): except FileNotFoundError as exc: logger.error(f"Error reading schema file from {file_name}") raise exc - metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data)) - logger.info(f"Successful validation of schema file {file_name}") + try: + metadata_model.validate_schema(data, _get_schema_file_name(args_dict, data)) + except jsonschema.exceptions.ValidationError as exc: + logger.error(f"Failed validation of file {file_name}") + raise exc + logger.info(f"Successful validation of file {file_name}") def validate_data_files(args_dict, logger): diff --git a/src/simtools/schemas/model_parameter.metaschema.yml b/src/simtools/schemas/model_parameter.metaschema.yml index 5be0471dd9..22e2c2ede5 100644 --- a/src/simtools/schemas/model_parameter.metaschema.yml +++ b/src/simtools/schemas/model_parameter.metaschema.yml @@ -58,6 +58,8 @@ definitions: - type: boolean - type: number - type: string + - type: "null" + - type: array description: "Value of the parameter." parameter_version: anyOf: From 4e341545ad2459b9b976e584103fd203a8384e41 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 08:43:51 +0100 Subject: [PATCH 086/124] integration tests --- .../convert_geo_coordinates_of_array_elements.py | 6 ++++-- src/simtools/layout/array_layout.py | 11 +++++++++-- src/simtools/model/model_parameter.py | 2 +- .../config/plot_array_layout_from_list_south.yml | 1 + 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/src/simtools/applications/convert_geo_coordinates_of_array_elements.py b/src/simtools/applications/convert_geo_coordinates_of_array_elements.py index f103ba6140..5452a699e8 100644 --- a/src/simtools/applications/convert_geo_coordinates_of_array_elements.py +++ b/src/simtools/applications/convert_geo_coordinates_of_array_elements.py @@ -133,7 +133,7 @@ def _parse(label=None, description=None): output=True, require_command_line=True, db_config=True, - simulation_model=["model_version", "site"], + simulation_model=["model_version", "parameter_version", "site"], ) @@ -170,7 +170,9 @@ def main(): if args_dict["export"] is not None: product_data = ( - layout.export_one_telescope_as_json(crs_name=args_dict["export"]) + layout.export_one_telescope_as_json( + crs_name=args_dict["export"], parameter_version=args_dict["parameter_version"] + ) if args_dict.get("input", "").endswith(".json") else layout.export_telescope_list_table(crs_name=args_dict["export"]) ) diff --git a/src/simtools/layout/array_layout.py b/src/simtools/layout/array_layout.py index fb174c3f8d..9e407ab449 100644 --- a/src/simtools/layout/array_layout.py +++ b/src/simtools/layout/array_layout.py @@ -387,6 +387,8 @@ def _read_table_from_json_file(self, file_name): data = json.load(file) position = data["value"] + if isinstance(position, str): + position = gen.convert_string_to_list(position) self.site = data.get("site", None) table = QTable() @@ -579,7 +581,9 @@ def export_telescope_list_table(self, crs_name): return table - def export_one_telescope_as_json(self, crs_name): + def export_one_telescope_as_json( + self, crs_name, parameter_version=None, schema_version="0.2.0" + ): """ Return a list containing a single telescope in simtools-DB-style json. @@ -587,6 +591,8 @@ def export_one_telescope_as_json(self, crs_name): ---------- crs_name: str Name of coordinate system to be used for export. + schema_version: str + Version of the schema. Returns ------- @@ -625,10 +631,11 @@ def export_one_telescope_as_json(self, crs_name): ] ) return { + "schema_version": schema_version, "parameter": parameter_name, "instrument": table["telescope_name"][0], "site": self.site, - "version": self.model_version, + "parameter_version": parameter_version, "unique_id": None, "value": value_string, "unit": "m", diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index d922d8389e..32f69949fb 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -111,7 +111,7 @@ def _get_parameter_dict(self, par_name): try: return self._parameters[par_name] except (KeyError, ValueError) as e: - msg = f"Parameter {par_name} was not found in the model" + msg = f"Parameter {par_name} was not found in the model {self.name}, {self.site}." self._logger.error(msg) raise InvalidModelParameterError(msg) from e diff --git a/tests/integration_tests/config/plot_array_layout_from_list_south.yml b/tests/integration_tests/config/plot_array_layout_from_list_south.yml index b2e8ed20e6..09b876676c 100644 --- a/tests/integration_tests/config/plot_array_layout_from_list_south.yml +++ b/tests/integration_tests/config/plot_array_layout_from_list_south.yml @@ -6,6 +6,7 @@ CTA_SIMPIPE: MODEL_VERSION: 6.0.0 OUTPUT_PATH: simtools-output AXES_RANGE: 1200. + LOG_LEVEL: DEBUG INTEGRATION_TESTS: - OUTPUT_FILE: array_layout_list_South_ground_0deg.png - OUTPUT_FILE: array_layout_list_South_ground_0deg.pdf From 3c4cb1e51d2295c49c2e58f46d14587ce44f232c Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 09:27:32 +0100 Subject: [PATCH 087/124] integration tests --- src/simtools/applications/db_get_parameter_from_db.py | 11 +++++++---- src/simtools/model/model_parameter.py | 2 +- .../db_get_array_layouts_from_db_layout_list.yml | 2 +- tests/resources/model_parameters/num_gains.json | 5 +++-- 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 873425e4dc..174def9f3c 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -68,7 +68,6 @@ def _parse(): config.parser.add_argument( "--db_collection", help="DB collection to which to add the file", - default="telescopes", required=False, ) config.parser.add_argument( @@ -98,7 +97,9 @@ def main(): # noqa: D103 parameter_version=args_dict["parameter_version"], site=args_dict["site"], array_element_name=args_dict["telescope"], - collection=args_dict.get("db_collection", "telescopes"), + collection=( + args_dict.get("db_collection") if args_dict.get("db_collection") else "telescopes" + ), ) # get parameter using 'model_version' elif args_dict["model_version"] is not None: @@ -109,7 +110,7 @@ def main(): # noqa: D103 model_version=args_dict["model_version"], collection=( "configuration_sim_telarray" - if args_dict["db_collection"] == "configuration_sim_telarray" + if args_dict.get("db_collection") == "configuration_sim_telarray" else "telescopes" ), ) @@ -117,7 +118,9 @@ def main(): # noqa: D103 pars = db.get_model_parameters( site=args_dict.get("site"), model_version=args_dict["model_version"], - collection=args_dict["db_collection"], + collection=( + args_dict["db_collection"] if args_dict.get("db_collection") else "sites" + ), array_element_name=None, ) else: diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index 32f69949fb..adc5a9b370 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -510,7 +510,7 @@ def get_model_file_as_table(self, par_name): _par_entry[par_name] = self._parameters[par_name] except KeyError as exc: raise ValueError(f"Parameter {par_name} not found in the model.") from exc - self.db.export_model_files(_par_entry, self.config_file_directory) + self.db.export_model_files(parameters=_par_entry, dest=self.config_file_directory) if _par_entry[par_name]["value"].endswith("ecsv"): return Table.read( self.config_file_directory.joinpath(_par_entry[par_name]["value"]), diff --git a/tests/integration_tests/config/db_get_array_layouts_from_db_layout_list.yml b/tests/integration_tests/config/db_get_array_layouts_from_db_layout_list.yml index 1f0f79ddc8..f8630785bc 100644 --- a/tests/integration_tests/config/db_get_array_layouts_from_db_layout_list.yml +++ b/tests/integration_tests/config/db_get_array_layouts_from_db_layout_list.yml @@ -2,7 +2,7 @@ CTA_SIMPIPE: APPLICATION: simtools-db-get-array-layouts-from-db TEST_NAME: layout_list CONFIGURATION: - SITE: North + SITE: South MODEL_VERSION: "6.0.0" ARRAY_ELEMENT_LIST: ["LSTS", "MSTS", "SSTS"] COORDINATE_SYSTEM: ground diff --git a/tests/resources/model_parameters/num_gains.json b/tests/resources/model_parameters/num_gains.json index 5b101be397..2fd8cdbf57 100644 --- a/tests/resources/model_parameters/num_gains.json +++ b/tests/resources/model_parameters/num_gains.json @@ -1,11 +1,12 @@ { + "schema_version": "0.1.0", "parameter": "num_gains", "instrument": "LSTN-01", "site": "North", - "version": "6.0.0", + "parameter_version": "0.1.0", + "unique_id": null, "value": 2, "unit": null, "type": "int", - "applicable": true, "file": false } From 347a44a3d0c4fd52cfcdec51d7a5abc41cb1acfa Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 09:56:46 +0100 Subject: [PATCH 088/124] remove debug statements --- .../config/db_get_array_layouts_from_db_list_arrays.yml | 1 - .../config/plot_array_layout_from_list_south.yml | 1 - .../config/simulate_light_emission_variable.yml | 1 - 3 files changed, 3 deletions(-) diff --git a/tests/integration_tests/config/db_get_array_layouts_from_db_list_arrays.yml b/tests/integration_tests/config/db_get_array_layouts_from_db_list_arrays.yml index 67d5f64cbd..31c031e9f3 100644 --- a/tests/integration_tests/config/db_get_array_layouts_from_db_list_arrays.yml +++ b/tests/integration_tests/config/db_get_array_layouts_from_db_list_arrays.yml @@ -5,4 +5,3 @@ CTA_SIMPIPE: SITE: North MODEL_VERSION: "6.0.0" LIST_AVAILABLE_LAYOUTS: True - LOG_LEVEL: DEBUG diff --git a/tests/integration_tests/config/plot_array_layout_from_list_south.yml b/tests/integration_tests/config/plot_array_layout_from_list_south.yml index 09b876676c..b2e8ed20e6 100644 --- a/tests/integration_tests/config/plot_array_layout_from_list_south.yml +++ b/tests/integration_tests/config/plot_array_layout_from_list_south.yml @@ -6,7 +6,6 @@ CTA_SIMPIPE: MODEL_VERSION: 6.0.0 OUTPUT_PATH: simtools-output AXES_RANGE: 1200. - LOG_LEVEL: DEBUG INTEGRATION_TESTS: - OUTPUT_FILE: array_layout_list_South_ground_0deg.png - OUTPUT_FILE: array_layout_list_South_ground_0deg.pdf diff --git a/tests/integration_tests/config/simulate_light_emission_variable.yml b/tests/integration_tests/config/simulate_light_emission_variable.yml index 9af989e31c..7b744927d4 100644 --- a/tests/integration_tests/config/simulate_light_emission_variable.yml +++ b/tests/integration_tests/config/simulate_light_emission_variable.yml @@ -10,6 +10,5 @@ CTA_SIMPIPE: MODEL_VERSION: 6.0.0 LIGHT_SOURCE_TYPE: led OUTPUT_PATH: simtools-output - LOG_LEVEL: DEBUG INTEGRATION_TESTS: - OUTPUT_FILE: logfile.log From 8570ff861ef0ed53f7906ef02d7342bc9c5cc517 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 11:05:37 +0100 Subject: [PATCH 089/124] integration tests --- .../config/convert_model_parameter_from_simtel_num_gains.yml | 3 --- ...idate_file_using_schema_json_validate_model_parameter.yml | 2 +- ...validate_file_using_schema_json_validate_schema-0.1.0.yml | 2 +- ...validate_file_using_schema_json_validate_schema-0.2.0.yml | 2 +- .../{num_gains.json => num_gains-0.0.0.json} | 5 ++--- .../num_gains-0.2.0.json} | 0 6 files changed, 5 insertions(+), 9 deletions(-) rename tests/resources/model_parameters/{num_gains.json => num_gains-0.0.0.json} (63%) rename tests/resources/{num_gains-schema-0.2.0.json => model_parameters/num_gains-0.2.0.json} (100%) diff --git a/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml b/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml index 234afc4f78..166d7dec4a 100644 --- a/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml +++ b/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml @@ -10,6 +10,3 @@ CTA_SIMPIPE: OUTPUT_PATH: simtools-output OUTPUT_FILE: num_gains.json USE_PLAIN_OUTPUT_PATH: True - INTEGRATION_TESTS: - - REFERENCE_OUTPUT_FILE: | - ./tests/resources/model_parameters/num_gains.json diff --git a/tests/integration_tests/config/validate_file_using_schema_json_validate_model_parameter.yml b/tests/integration_tests/config/validate_file_using_schema_json_validate_model_parameter.yml index eed8237bd4..c6fc78e26e 100644 --- a/tests/integration_tests/config/validate_file_using_schema_json_validate_model_parameter.yml +++ b/tests/integration_tests/config/validate_file_using_schema_json_validate_model_parameter.yml @@ -3,5 +3,5 @@ CTA_SIMPIPE: TEST_NAME: json_validate_model_parameter CONFIGURATION: SCHEMA: tests/resources/num_gains.schema.yml - FILE_NAME: tests/resources/model_parameters/num_gains.json + FILE_NAME: tests/resources/model_parameters/num_gains-0.2.0.json DATA_TYPE: model_parameter diff --git a/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.1.0.yml b/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.1.0.yml index 8ec7f7dd57..bb7b660e4d 100644 --- a/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.1.0.yml +++ b/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.1.0.yml @@ -3,5 +3,5 @@ CTA_SIMPIPE: TEST_NAME: json_validate_schema-0.1.0 CONFIGURATION: SCHEMA: src/simtools/schemas/model_parameter.metaschema.yml - FILE_NAME: tests/resources/model_parameters/num_gains.json + FILE_NAME: tests/resources/model_parameters/num_gains-0.0.0.json DATA_TYPE: schema diff --git a/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.2.0.yml b/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.2.0.yml index 0280a891cb..4de540598a 100644 --- a/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.2.0.yml +++ b/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.2.0.yml @@ -3,5 +3,5 @@ CTA_SIMPIPE: TEST_NAME: json_validate_schema-0.2.0 CONFIGURATION: SCHEMA: src/simtools/schemas/model_parameter.metaschema.yml - FILE_NAME: tests/resources/num_gains-schema-0.2.0.json + FILE_NAME: tests/resources/model_parameters/num_gains-0.2.0.json DATA_TYPE: schema diff --git a/tests/resources/model_parameters/num_gains.json b/tests/resources/model_parameters/num_gains-0.0.0.json similarity index 63% rename from tests/resources/model_parameters/num_gains.json rename to tests/resources/model_parameters/num_gains-0.0.0.json index 2fd8cdbf57..5b101be397 100644 --- a/tests/resources/model_parameters/num_gains.json +++ b/tests/resources/model_parameters/num_gains-0.0.0.json @@ -1,12 +1,11 @@ { - "schema_version": "0.1.0", "parameter": "num_gains", "instrument": "LSTN-01", "site": "North", - "parameter_version": "0.1.0", - "unique_id": null, + "version": "6.0.0", "value": 2, "unit": null, "type": "int", + "applicable": true, "file": false } diff --git a/tests/resources/num_gains-schema-0.2.0.json b/tests/resources/model_parameters/num_gains-0.2.0.json similarity index 100% rename from tests/resources/num_gains-schema-0.2.0.json rename to tests/resources/model_parameters/num_gains-0.2.0.json From 61bf61bb95e42d0b7be2db5c5996b15fd90c2f47 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 11:13:01 +0100 Subject: [PATCH 090/124] unit tests --- tests/unit_tests/data_model/test_validate_data.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/unit_tests/data_model/test_validate_data.py b/tests/unit_tests/data_model/test_validate_data.py index 82da96abc6..2b3a90647b 100644 --- a/tests/unit_tests/data_model/test_validate_data.py +++ b/tests/unit_tests/data_model/test_validate_data.py @@ -110,7 +110,7 @@ def test_validate_and_transform(caplog, mocker): assert isinstance(_table, Table) assert "Validating tabled data from:" in caplog.text - data_validator.data_file_name = "tests/resources/model_parameters/num_gains.json" + data_validator.data_file_name = "tests/resources/model_parameters/num_gains-0.2.0.json" data_validator.schema_file_name = "tests/resources/num_gains.schema.yml" mock_prepare_model_parameter = mocker.patch( "simtools.data_model.validate_data.DataValidator._prepare_model_parameter" @@ -141,14 +141,14 @@ def test_validate_data_file(caplog): def test_validate_parameter_and_file_name(): data_validator = validate_data.DataValidator() - data_validator.data_file_name = "tests/resources/model_parameters/num_gains.json" + data_validator.data_file_name = "tests/resources/model_parameters/num_gains-0.2.0.json" data_validator.schema_file_name = "tests/resources/num_gains.schema.yml" data_validator.validate_and_transform() data_validator.data_dict["parameter"] = "incorrect_name" with pytest.raises( ValueError, - match="Parameter name in data dict incorrect_name and file name num_gains do not match.", + match="Parameter name in data dict incorrect_name and file name num_gains-0.2.0 do not match.", ): data_validator.validate_parameter_and_file_name() From fa9e1bc185ab853d18985cf24273ce9f410fbdfd Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 11:49:41 +0100 Subject: [PATCH 091/124] string fix --- src/simtools/data_model/validate_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simtools/data_model/validate_data.py b/src/simtools/data_model/validate_data.py index c27cae37ab..0c20dd6532 100644 --- a/src/simtools/data_model/validate_data.py +++ b/src/simtools/data_model/validate_data.py @@ -129,7 +129,7 @@ def validate_model_parameter(par_dict): Validated data dictionary """ data_validator = DataValidator( - schema_file=MODEL_PARAMETER_SCHEMA_PATH / "f{par_dict['parameter']}.schema.yml", + schema_file=MODEL_PARAMETER_SCHEMA_PATH / f"{par_dict['parameter']}.schema.yml", data_dict=par_dict, check_exact_data_type=False, ) From 1ce0c395d020d3ace605ea4a5befdbb75c75bb5d Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 12:09:35 +0100 Subject: [PATCH 092/124] correct handling of strings vs lists --- ...nvert_geo_coordinates_of_array_elements.py | 2 +- src/simtools/layout/array_layout.py | 41 ++++++++----------- 2 files changed, 19 insertions(+), 24 deletions(-) diff --git a/src/simtools/applications/convert_geo_coordinates_of_array_elements.py b/src/simtools/applications/convert_geo_coordinates_of_array_elements.py index 5452a699e8..cc246b7955 100644 --- a/src/simtools/applications/convert_geo_coordinates_of_array_elements.py +++ b/src/simtools/applications/convert_geo_coordinates_of_array_elements.py @@ -171,7 +171,7 @@ def main(): if args_dict["export"] is not None: product_data = ( layout.export_one_telescope_as_json( - crs_name=args_dict["export"], parameter_version=args_dict["parameter_version"] + crs_name=args_dict["export"], parameter_version=args_dict.get("parameter_version") ) if args_dict.get("input", "").endswith(".json") else layout.export_telescope_list_table(crs_name=args_dict["export"]) diff --git a/src/simtools/layout/array_layout.py b/src/simtools/layout/array_layout.py index 9e407ab449..9c039bdf7d 100644 --- a/src/simtools/layout/array_layout.py +++ b/src/simtools/layout/array_layout.py @@ -602,34 +602,29 @@ def export_one_telescope_as_json( table = self.export_telescope_list_table(crs_name) if len(table) != 1: raise ValueError("Only one telescope can be exported to json") - parameter_name = value_string = None + parameter_name = value = None if crs_name == "ground": parameter_name = "array_element_position_ground" - value_string = gen.convert_list_to_string( - [ - table["position_x"][0].value, - table["position_y"][0].value, - table["position_z"][0].value, - ] - ) + value = [ + table["position_x"][0].value, + table["position_y"][0].value, + table["position_z"][0].value, + ] elif crs_name == "utm": parameter_name = "array_element_position_utm" - value_string = gen.convert_list_to_string( - [ - table["utm_east"][0].value, - table["utm_north"][0].value, - table["altitude"][0].value, - ] - ) + value = [ + table["utm_east"][0].value, + table["utm_north"][0].value, + table["altitude"][0].value, + ] elif crs_name == "mercator": parameter_name = "array_element_position_mercator" - value_string = gen.convert_list_to_string( - [ - table["latitude"][0].value, - table["longitude"][0].value, - table["altitude"][0].value, - ] - ) + value = [ + table["latitude"][0].value, + table["longitude"][0].value, + table["altitude"][0].value, + ] + return { "schema_version": schema_version, "parameter": parameter_name, @@ -637,7 +632,7 @@ def export_one_telescope_as_json( "site": self.site, "parameter_version": parameter_version, "unique_id": None, - "value": value_string, + "value": value, "unit": "m", "type": "float64", "file": False, From cec3944b81e37aff24058e750a7e7543ff206710 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 12:25:59 +0100 Subject: [PATCH 093/124] towncrier doc --- docs/changes/1316.maintenance.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 docs/changes/1316.maintenance.md diff --git a/docs/changes/1316.maintenance.md b/docs/changes/1316.maintenance.md new file mode 100644 index 0000000000..13a992434c --- /dev/null +++ b/docs/changes/1316.maintenance.md @@ -0,0 +1 @@ +Major restructering of database routines plus introduction of new simulation model. From 50441bdcd3dedac98b5bc2efc1431ab289d5df67 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 14:51:11 +0100 Subject: [PATCH 094/124] Compare lists of floats in json/yaml files --- src/simtools/testing/validate_output.py | 35 +++++++++++--- .../testing/test_validate_output.py | 46 ++++++++++++++++++- 2 files changed, 74 insertions(+), 7 deletions(-) diff --git a/src/simtools/testing/validate_output.py b/src/simtools/testing/validate_output.py index c8dcfd5d44..803691f42c 100644 --- a/src/simtools/testing/validate_output.py +++ b/src/simtools/testing/validate_output.py @@ -133,7 +133,7 @@ def compare_files(file1, file2, tolerance=1.0e-5, test_columns=None): if _file1_suffix == ".ecsv": return compare_ecsv_files(file1, file2, tolerance, test_columns) if _file1_suffix in (".json", ".yaml", ".yml"): - return compare_json_or_yaml_files(file1, file2) + return compare_json_or_yaml_files(file1, file2, tolerance) _logger.warning(f"Unknown file type for files: {file1} and {file2}") return False @@ -168,11 +168,34 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2): if data1 == data2: return True - if "value" in data1 and isinstance(data1["value"], str): - value_list_1 = gen.convert_string_to_list(data1.pop("value")) - value_list_2 = gen.convert_string_to_list(data2.pop("value")) - return np.allclose(value_list_1, value_list_2, rtol=tolerance) - return data1 == data2 + if data1.keys() != data2.keys(): + return False + + is_equal = True + for key in data1.keys(): + if key == "value": + is_equal = is_equal and _compare_value_from_parameter_dict( + data1["value"], data2["value"], tolerance + ) + else: + is_equal = is_equal and (data1[key] == data2[key]) + + return is_equal + + +def _compare_value_from_parameter_dict(data1, data2, tolerance): + """Compare value fields given in different formats.""" + + def _as_list(value): + if isinstance(value, str): + return gen.convert_string_to_list(value) + if isinstance(value, list | np.ndarray): + return value + return [value] + + value_list_1 = _as_list(data1) + value_list_2 = _as_list(data2) + return np.allclose(value_list_1, value_list_2, rtol=tolerance) def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None): diff --git a/tests/unit_tests/testing/test_validate_output.py b/tests/unit_tests/testing/test_validate_output.py index 74bef55b38..7da869845a 100644 --- a/tests/unit_tests/testing/test_validate_output.py +++ b/tests/unit_tests/testing/test_validate_output.py @@ -100,6 +100,18 @@ def test_compare_json_files_float_strings(create_json_file, file_name): assert not validate_output.compare_json_or_yaml_files(file1, file3) +def test_compare_json_files_equal_dicts(create_json_file, file_name): + content1 = {"key": 1, "value": 5} + file1 = create_json_file(file_name(1, "json"), content1) + content2 = {"key": 1, "value": 5, "extra": "extra"} + file2 = create_json_file(file_name(2, "json"), content2) + assert not validate_output.compare_json_or_yaml_files(file1, file2) + + content3 = {"different_key": 1, "value": 5} + file3 = create_json_file(file_name(2, "json"), content3) + assert not validate_output.compare_json_or_yaml_files(file1, file3) + + def test_compare_json_files_equal_integers(create_json_file, file_name): content = {"key": 1, "value": 5} file1 = create_json_file(file_name(1, "json"), content) @@ -112,6 +124,36 @@ def test_compare_json_files_equal_integers(create_json_file, file_name): assert not validate_output.compare_json_or_yaml_files(file1, file3) +def test_compare_json_files_equal_floats(create_json_file, file_name): + content = {"key": 1, "value": 5.5} + file1 = create_json_file(file_name(1, "json"), content) + file2 = create_json_file(file_name(2, "json"), content) + + assert validate_output.compare_json_or_yaml_files(file1, file2) + + content3 = {"key": 1, "value": 5.75} + file3 = create_json_file(file_name(3, "json"), content3) + assert not validate_output.compare_json_or_yaml_files(file1, file3) + + assert validate_output.compare_json_or_yaml_files(file1, file3, tolerance=0.5) + + +def test_compare_json_files_list_of_floats(create_json_file, file_name): + content = {"key": 1, "value": [5.5, 10.5]} + file1 = create_json_file(file_name(1, "json"), content) + file2 = create_json_file(file_name(2, "json"), content) + + assert validate_output.compare_json_or_yaml_files(file1, file2) + + content3 = {"key": 1, "value": 5.75} + file3 = create_json_file(file_name(3, "json"), content3) + assert not validate_output.compare_json_or_yaml_files(file1, file3) + + content4 = {"key": 1, "value": [5.75, 10.75]} + file4 = create_json_file(file_name(3, "json"), content4) + assert validate_output.compare_json_or_yaml_files(file1, file4, tolerance=0.5) + + def test_compare_yaml_files_float_strings(create_yaml_file, file_name): content = {"key": 1, "value": "1.23 4.56 7.89"} file1 = create_yaml_file(file_name(1, "yaml"), content) @@ -119,10 +161,12 @@ def test_compare_yaml_files_float_strings(create_yaml_file, file_name): assert validate_output.compare_json_or_yaml_files(file1, file2) - content3 = {"key": 2, "value": "1.23 4.56 7.80"} + content3 = {"key": 1, "value": "1.23 4.56 7.80"} file3 = create_yaml_file(file_name(3, "yaml"), content3) assert not validate_output.compare_json_or_yaml_files(file1, file3) + assert validate_output.compare_json_or_yaml_files(file1, file3, tolerance=0.5) + def test_compare_yaml_files_equal_integers(create_yaml_file, file_name): content = {"key": 1, "value": 5} From ae017759f995d46fa737f9c47ae7e1109c1f09e5 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 16:46:53 +0100 Subject: [PATCH 095/124] code simplification --- src/simtools/testing/validate_output.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/src/simtools/testing/validate_output.py b/src/simtools/testing/validate_output.py index 803691f42c..8793288e35 100644 --- a/src/simtools/testing/validate_output.py +++ b/src/simtools/testing/validate_output.py @@ -167,20 +167,16 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2): if data1 == data2: return True - if data1.keys() != data2.keys(): return False - - is_equal = True - for key in data1.keys(): - if key == "value": - is_equal = is_equal and _compare_value_from_parameter_dict( - data1["value"], data2["value"], tolerance - ) - else: - is_equal = is_equal and (data1[key] == data2[key]) - - return is_equal + return all( + ( + _compare_value_from_parameter_dict(data1[k], data2[k], tolerance) + if k == "value" + else data1[k] == data2[k] + ) + for k in data1 + ) def _compare_value_from_parameter_dict(data1, data2, tolerance): From 5387876badaadb175d407d896a03e549e12a759d Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 22 Jan 2025 16:49:54 +0100 Subject: [PATCH 096/124] code simplification --- src/simtools/testing/validate_output.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/simtools/testing/validate_output.py b/src/simtools/testing/validate_output.py index 8793288e35..d0a578eb7e 100644 --- a/src/simtools/testing/validate_output.py +++ b/src/simtools/testing/validate_output.py @@ -189,9 +189,7 @@ def _as_list(value): return value return [value] - value_list_1 = _as_list(data1) - value_list_2 = _as_list(data2) - return np.allclose(value_list_1, value_list_2, rtol=tolerance) + return np.allclose(_as_list(data1), _as_list(data2), rtol=tolerance) def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None): From 9b3615aebef9aeb12f79783b4acd4868381b7516 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 23 Jan 2025 17:50:24 +0100 Subject: [PATCH 097/124] add comment on xSTx --- src/simtools/db/db_handler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index a20a899321..b991ae5506 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -375,6 +375,7 @@ def _get_query_from_parameter_version_table( for param, version in parameter_version_table.items() ], } + # 'xSTX-design' is a placeholder to ignore 'instrument' field in query. if array_element_name and array_element_name != "xSTx-design": query_dict["instrument"] = array_element_name if site: From 7fdd30ec12c333fac78924415f0ecdf177c02dbb Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 26 Jan 2025 20:54:15 +0100 Subject: [PATCH 098/124] Reference model corrections --- .../CTA-North-LSTN-01-5.0.0_test.cfg | 2 +- .../CTA-North-LSTN-02-5.0.0_test.cfg | 14 +++++++------- .../CTA-North-LSTN-03-5.0.0_test.cfg | 12 ++++++------ .../CTA-North-LSTN-04-5.0.0_test.cfg | 14 +++++++------- .../CTA-North-MSTN-01-5.0.0_test.cfg | 4 ++-- .../CTA-South-LSTS-01-5.0.0_test.cfg | 2 +- .../CTA-South-MSTS-01-5.0.0_test.cfg | 4 ++-- .../CTA-South-SSTS-01-5.0.0_test.cfg | 6 +++--- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg index 66e6180504..36203d221c 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg @@ -94,7 +94,7 @@ mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.238006 +nightsky_background = all: 0.23801 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg index 6a22e82a58..eca5a0d43c 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg @@ -19,7 +19,7 @@ atmospheric_transmission = atm_trans_2158_1_3_2_0_0_0.1_0.1.dat axes_offsets = 0.0 0.0 camera_body_diameter = 348.0 camera_body_shape = 2 -camera_config_file = camera_CTA-LST-1_analogsum21_v2020-04-14.dat +camera_config_file = camera_CTA-LST-234_analogsum21_v2020-04-14.dat camera_degraded_efficiency = 1.0 camera_degraded_map = none camera_depth = 0.0 @@ -89,12 +89,12 @@ mirror_align_random_vertical = 0.0039 28.0 0.0 0.0 mirror_class = 0 mirror_degraded_reflection = 1.0 mirror_focal_length = 0.0 -mirror_list = mirror_CTA-N-LST1_v2019-03-31.dat +mirror_list = mirror_CTA-N-LST2_v2020-04-07.dat mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.238006 +nightsky_background = all: 0.24499 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 @@ -104,12 +104,12 @@ photon_delay = 19.0 pixeltrg_time_step = 0.0 pm_average_gain = 40000.0 pm_collection_efficiency = 1.0 -pm_gain_index = 4.5 +pm_gain_index = 3.92 pm_photoelectron_spectrum = spe_LST_2020-05-09_AP2.0e-4.dat -pm_transit_time = 24.74 9.0 350.0 1066.0 -pm_voltage_variation = 0.041 +pm_transit_time = 20.89 9 350 1135 +pm_voltage_variation = 0.03 qe_variation = 0.03 -quantum_efficiency = qe_lst1_20200318_high+low.dat +quantum_efficiency = qe_lst2-4_20200318_high+low.dat random_focal_length = 0.0 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg index b284166cef..0d4fd36e3c 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg @@ -19,7 +19,7 @@ atmospheric_transmission = atm_trans_2158_1_3_2_0_0_0.1_0.1.dat axes_offsets = 0.0 0.0 camera_body_diameter = 348.0 camera_body_shape = 2 -camera_config_file = camera_CTA-LST-1_analogsum21_v2020-04-14.dat +camera_config_file = camera_CTA-LST-234_analogsum21_v2020-04-14.dat camera_degraded_efficiency = 1.0 camera_degraded_map = none camera_depth = 0.0 @@ -89,7 +89,7 @@ mirror_align_random_vertical = 0.0039 28.0 0.0 0.0 mirror_class = 0 mirror_degraded_reflection = 1.0 mirror_focal_length = 0.0 -mirror_list = mirror_CTA-N-LST1_v2019-03-31.dat +mirror_list = mirror_CTA-N-LST3_v2020-04-07.dat mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat @@ -104,12 +104,12 @@ photon_delay = 19.0 pixeltrg_time_step = 0.0 pm_average_gain = 40000.0 pm_collection_efficiency = 1.0 -pm_gain_index = 4.5 +pm_gain_index = 3.92 pm_photoelectron_spectrum = spe_LST_2020-05-09_AP2.0e-4.dat -pm_transit_time = 24.74 9.0 350.0 1066.0 -pm_voltage_variation = 0.041 +pm_transit_time = 20.89 9 350 1135 +pm_voltage_variation = 0.03 qe_variation = 0.03 -quantum_efficiency = qe_lst1_20200318_high+low.dat +quantum_efficiency = qe_lst2-4_20200318_high+low.dat random_focal_length = 0.0 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg index 95d4eef87c..942bc46ddd 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg @@ -19,7 +19,7 @@ atmospheric_transmission = atm_trans_2158_1_3_2_0_0_0.1_0.1.dat axes_offsets = 0.0 0.0 camera_body_diameter = 348.0 camera_body_shape = 2 -camera_config_file = camera_CTA-LST-1_analogsum21_v2020-04-14.dat +camera_config_file = camera_CTA-LST-234_analogsum21_v2020-04-14.dat camera_degraded_efficiency = 1.0 camera_degraded_map = none camera_depth = 0.0 @@ -89,12 +89,12 @@ mirror_align_random_vertical = 0.0039 28.0 0.0 0.0 mirror_class = 0 mirror_degraded_reflection = 1.0 mirror_focal_length = 0.0 -mirror_list = mirror_CTA-N-LST1_v2019-03-31.dat +mirror_list = mirror_CTA-N-LST4_v2020-04-07.dat mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.238006 +nightsky_background = all: 0.24499 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 @@ -104,12 +104,12 @@ photon_delay = 19.0 pixeltrg_time_step = 0.0 pm_average_gain = 40000.0 pm_collection_efficiency = 1.0 -pm_gain_index = 4.5 +pm_gain_index = 3.92 pm_photoelectron_spectrum = spe_LST_2020-05-09_AP2.0e-4.dat -pm_transit_time = 24.74 9.0 350.0 1066.0 -pm_voltage_variation = 0.041 +pm_transit_time = 20.89 9 350 1135 +pm_voltage_variation = 0.03 qe_variation = 0.03 -quantum_efficiency = qe_lst1_20200318_high+low.dat +quantum_efficiency = qe_lst2-4_20200318_high+low.dat random_focal_length = 0.0 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 diff --git a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg index a58c7641d4..2cec2b4039 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg @@ -107,7 +107,7 @@ mirror_offset = 0.0 mirror_reflection_random_angle = 0.006759 0.125 0.037 mirror_reflectivity = ref_AlSiO2HfO2.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.207706 +nightsky_background = all: 0.20771 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 @@ -126,7 +126,7 @@ quantum_efficiency = qe_R12992-100-05b.dat random_focal_length = 3.9 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 -telescope_transmission = 0.908661 0.0 0.0 0.0 0.0 0.0 +telescope_transmission = 0.908661 0.0154576 4 1.21166 0 0 teltrig_min_sigsum = 0.0 teltrig_min_time = 1.0 transit_time_calib_error = 0.0 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg index 9cab0f5f51..071a92c2b6 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg @@ -94,7 +94,7 @@ mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.244985 +nightsky_background = all: 0.24499 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg index e054a6db70..58ac86de31 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg @@ -107,7 +107,7 @@ mirror_offset = 0.0 mirror_reflection_random_angle = 0.006759 0.125 0.037 mirror_reflectivity = ref_AlSiO2HfO2.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.209051 +nightsky_background = all: 0.20905 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 1 @@ -126,7 +126,7 @@ quantum_efficiency = qe_R12992-100-05b.dat random_focal_length = 3.9 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 -telescope_transmission = 0.908661 0.0 0.0 0.0 0.0 0.0 +telescope_transmission = 0.908661 0.0154576 4 1.21166 0 0 teltrig_min_sigsum = 0.0 teltrig_min_time = 0.0 transit_time_calib_error = 0.0 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg index 5136f9c1b2..c04d11b7bd 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg @@ -56,7 +56,7 @@ dsum_pre_clipping = 0 dsum_prescale = 40 256 dsum_presum_max = 127 dsum_presum_shift = 1 -dsum_shaping_file = CTA-ULTRA6-dsum-shaping-FlashCam-2a-int.dat +dsum_shaping_file = none dsum_shaping_renormalize = 0 dsum_threshold = 0.0 dsum_zero_clip = 1 @@ -107,7 +107,7 @@ mirror_reflection_random_angle = 0.0255 0.0 0.0 mirror_reflectivity = ref_astri-2d_2018-01-17.dat mirror_secondary_reflectivity = ref_astri-2d_2018-01-17.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.0387262 +nightsky_background = all: 0.03873 nsb_autoscale_airmass = 0.83 0.2 nsb_gain_drop_scale = 0.0 nsb_offaxis = 1.0 0.04478 7.844 2.0282 0.0 @@ -146,7 +146,7 @@ transit_time_compensate_error = 0.2 transit_time_compensate_step = 1.0 transit_time_error = 0.1 transit_time_jitter = 0.01 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 2 iobuf_maximum = 1000000000 From cdf91369463439947624520fd56a65ad5418f7c5 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 26 Jan 2025 21:35:07 +0100 Subject: [PATCH 099/124] comments from OG --- .../upload_from_model_repository_to_db.sh | 2 +- .../convert_all_model_parameters_from_simtel.py | 4 ++-- .../applications/db_get_parameter_from_db.py | 6 ++---- src/simtools/db/db_handler.py | 12 +++++------- tests/unit_tests/db/test_db_handler.py | 9 ++++++--- 5 files changed, 16 insertions(+), 17 deletions(-) diff --git a/database_scripts/upload_from_model_repository_to_db.sh b/database_scripts/upload_from_model_repository_to_db.sh index 84960f1131..2a82112d26 100755 --- a/database_scripts/upload_from_model_repository_to_db.sh +++ b/database_scripts/upload_from_model_repository_to_db.sh @@ -5,7 +5,7 @@ # Cover 'source .env': the script ensure that this file exists: # shellcheck disable=SC1091 -DB_SIMULATION_MODEL_URL="https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/simulation-models" +DB_SIMULATION_MODEL_URL="https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/simulation-models.git" DB_SIMULATION_MODEL_BRANCH="v1.0.0-parameters" # Check that this script is not sourced but executed diff --git a/src/simtools/applications/convert_all_model_parameters_from_simtel.py b/src/simtools/applications/convert_all_model_parameters_from_simtel.py index c910d54d78..7c0624eec2 100644 --- a/src/simtools/applications/convert_all_model_parameters_from_simtel.py +++ b/src/simtools/applications/convert_all_model_parameters_from_simtel.py @@ -6,7 +6,7 @@ ready to be submitted to the model database. Prints out parameters which are not found in simtel configuration file and parameters which are not found in simtools schema files. - Note that all parameter are assigned the same parameter version. + Note that all parameters are assigned the same parameter version. Command line arguments ---------------------- @@ -32,7 +32,7 @@ --simtel_cfg_file all_telescope_config_la_palma.cfg\\ --simtel_telescope_name CT1\\ --telescope LSTN-01\\ - --parameter_version "0.0.1" + --parameter_version "1.0.0" The export of the model parameters from sim_telarray for 6.0.0 can be done e.g., as follows: diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 174def9f3c..0868de9815 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -40,7 +40,7 @@ simtools-db-get-parameter-from-db --parameter mirror_list \\ --site North --telescope LSTN-01 \\ - --parameter_version 5.0.0 + --parameter_version 1.0.0 """ @@ -97,9 +97,7 @@ def main(): # noqa: D103 parameter_version=args_dict["parameter_version"], site=args_dict["site"], array_element_name=args_dict["telescope"], - collection=( - args_dict.get("db_collection") if args_dict.get("db_collection") else "telescopes" - ), + collection=args_dict.get("db_collection", "telescopes"), ) # get parameter using 'model_version' elif args_dict["model_version"] is not None: diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index b991ae5506..fdb6488aaa 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -301,7 +301,7 @@ def get_collections(self, db_name=None, model_collections_only=False): db_name: str Database name. model_collections_only: bool - If True, only return model collections (i.e. exclude fs.files, fs.chunks, metadata) + If True, only return model collections (i.e. exclude fs.files, fs.chunks) Returns ------- @@ -316,11 +316,7 @@ def get_collections(self, db_name=None, model_collections_only=False): ].list_collection_names() collections = self.list_of_collections[db_name] if model_collections_only: - return [ - collection - for collection in collections - if not collection.startswith("fs.") and collection != "metadata" - ] + return [collection for collection in collections if not collection.startswith("fs.")] return collections def export_model_files(self, parameters=None, file_names=None, dest=None, db_name=None): @@ -358,7 +354,7 @@ def export_model_files(self, parameters=None, file_names=None, dest=None, db_nam instance_ids = {} for file_name in file_names: if Path(dest).joinpath(file_name).exists(): - instance_ids[file_name] = "file exits" + instance_ids[file_name] = "file exists" else: file_path_instance = self._get_file_mongo_db(self._get_db_name(), file_name) self._write_file_from_mongo_to_disk(self._get_db_name(), dest, file_path_instance) @@ -458,6 +454,8 @@ def get_array_elements_of_type(self, array_element_type, model_version, collecti """ Get array elements of a certain type (e.g. 'LSTN') for a DB collection. + Does not return 'design' models. + Parameters ---------- array_element_type: str diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index 48a11d0994..9f5c42f304 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -625,7 +625,8 @@ def test_read_mongo_db(db, mocker, test_db): mocker.patch.object(db.get_collection.return_value, "find", return_value=[]) with pytest.raises( ValueError, - match=r"The following query for test_collection returned zero results: {'parameter_version': '1.0.0'}", + match=r"The following query for test_collection returned zero results: " + r"{'parameter_version': '1.0.0'}", ): db._read_mongo_db(query, collection_name) @@ -683,7 +684,8 @@ def test__read_production_table_from_mongo_db_with_cache(db, mocker, test_db): mocker.patch.object(db.get_collection.return_value, "find_one", return_value=None) with pytest.raises( ValueError, - match=r"The following query returned zero results: {'model_version': '1.0.0', 'collection': 'telescopes'}", + match=r"The following query returned zero results: " + r"{'model_version': '1.0.0', 'collection': 'telescopes'}", ): db._read_production_table_from_mongo_db(collection_name, model_version) @@ -906,7 +908,8 @@ def test_add_new_parameter_with_file_no_prefix( with pytest.raises( FileNotFoundError, - match=r"The location of the file to upload, corresponding to the param1 parameter, must be provided.", + match=r"The location of the file to upload, corresponding to the param1 parameter, " + r"must be provided.", ): db.add_new_parameter(test_db, par_dict, collection_name, file_prefix) From 6629994ede3f84ce13e554a47a865176f78770fa Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Sun, 26 Jan 2025 21:38:13 +0100 Subject: [PATCH 100/124] unit test --- tests/unit_tests/db/test_db_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index 9f5c42f304..5435b3bd73 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -480,7 +480,7 @@ def test_export_model_files_file_exists(db, mocker, tmp_test_directory, test_db, mock_get_file_mongo_db.assert_not_called() mock_write_file_from_mongo_to_disk.assert_not_called() mock_path_exists.assert_called_once() - assert result == {test_file: "file exits"} + assert result == {test_file: "file exists"} def test_export_model_files_file_not_found(db, mocker, tmp_test_directory, test_db, test_file): From 62c46f59a7520236d48318fac0ab9c25237616d6 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 27 Jan 2025 10:38:07 +0100 Subject: [PATCH 101/124] fix call for collections --- src/simtools/applications/db_get_parameter_from_db.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 0868de9815..04d78e208c 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -97,7 +97,9 @@ def main(): # noqa: D103 parameter_version=args_dict["parameter_version"], site=args_dict["site"], array_element_name=args_dict["telescope"], - collection=args_dict.get("db_collection", "telescopes"), + collection=( + args_dict["db_collection"] if args_dict.get("db_collection") else "telescopes" + ), ) # get parameter using 'model_version' elif args_dict["model_version"] is not None: From 35b1ce3d209748147cef15955af04922912b5818 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 27 Jan 2025 11:51:40 +0100 Subject: [PATCH 102/124] code spell --- src/simtools/applications/simulate_prod.py | 2 +- src/simtools/corsika/corsika_histograms.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/simtools/applications/simulate_prod.py b/src/simtools/applications/simulate_prod.py index 665a8a1c01..c5e95498d0 100644 --- a/src/simtools/applications/simulate_prod.py +++ b/src/simtools/applications/simulate_prod.py @@ -31,7 +31,7 @@ Zenith angle in degrees. nshow (int, optional) Number of showers to simulate. - The Number of simulated events depends on the number of times a shower is re-used in the + The Number of simulated events depends on the number of times a shower is reused in the telescope simulation. The number provided here is before any reuse factors. start_run (int, required) Start run number such that the actual run number will be 'start_run' + 'run'. diff --git a/src/simtools/corsika/corsika_histograms.py b/src/simtools/corsika/corsika_histograms.py index fb3cb74a14..8e4f73faab 100644 --- a/src/simtools/corsika/corsika_histograms.py +++ b/src/simtools/corsika/corsika_histograms.py @@ -678,7 +678,7 @@ def individual_telescopes(self, new_individual_telescopes: bool): ---------- new_individual_telescopes: bool if False, the histograms are supposed to be filled for all telescopes. - if True, one histogram is set for each telescope sepparately. + if True, one histogram is set for each telescope separately. """ if new_individual_telescopes is None: self._individual_telescopes = False From 52c294c05a17840afbd4eabd89aec85224b3f3d1 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 27 Jan 2025 14:04:26 +0100 Subject: [PATCH 103/124] integration tests --- ..._get_parameter_from_db_telescope_model_version.yml | 2 +- ..._parameter_from_db_telescope_parameter_version.yml | 6 ++++++ tests/resources/model_parameters/mirror_list.json | 11 +++++++++++ tests/unit_tests/corsika/test_corsika_config.py | 6 +++--- 4 files changed, 21 insertions(+), 4 deletions(-) create mode 100644 tests/resources/model_parameters/mirror_list.json diff --git a/tests/integration_tests/config/db_get_parameter_from_db_telescope_model_version.yml b/tests/integration_tests/config/db_get_parameter_from_db_telescope_model_version.yml index 19f674dc1b..eab1256d72 100644 --- a/tests/integration_tests/config/db_get_parameter_from_db_telescope_model_version.yml +++ b/tests/integration_tests/config/db_get_parameter_from_db_telescope_model_version.yml @@ -1,6 +1,6 @@ CTA_SIMPIPE: APPLICATION: simtools-db-get-parameter-from-db - TEST_NAME: telescope_parameter_parameter_version + TEST_NAME: telescope_parameter_model_version CONFIGURATION: SITE: North TELESCOPE: LSTN-01 diff --git a/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml b/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml index ea57845b0f..7280170eb7 100644 --- a/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml +++ b/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml @@ -6,3 +6,9 @@ CTA_SIMPIPE: TELESCOPE: LSTN-01 PARAMETER: mirror_list PARAMETER_VERSION: 2.0.0 + OUTPUT_FILE: test.json + OUTPUT_PATH: simtools-output + LOG_LEVEL: DEBUG + INTEGRATION_TESTS: + - REFERENCE_OUTPUT_FILE: | + ./tests/resources/model_parameters/mirror_list.json diff --git a/tests/resources/model_parameters/mirror_list.json b/tests/resources/model_parameters/mirror_list.json new file mode 100644 index 0000000000..172113ce62 --- /dev/null +++ b/tests/resources/model_parameters/mirror_list.json @@ -0,0 +1,11 @@ +{ + "schema_version": "0.2.0", + "instrument": "LSTN-01", + "site": "North", + "parameter_version": "2.0.0", + "unique_id": null, + "value": "mirror_CTA-N-LST1_v2019-03-31.dat", + "unit": null, + "type": "string", + "file": true +} diff --git a/tests/unit_tests/corsika/test_corsika_config.py b/tests/unit_tests/corsika/test_corsika_config.py index f3a2ca5ef6..a818a03094 100644 --- a/tests/unit_tests/corsika/test_corsika_config.py +++ b/tests/unit_tests/corsika/test_corsika_config.py @@ -29,7 +29,7 @@ def corsika_configuration_parameters(gcm2): return { "corsika_iact_max_bunches": {"value": 1000000, "unit": None}, "corsika_cherenkov_photon_bunch_size": {"value": 5.0, "unit": None}, - "corsika_cherenkov_photon_wavelength_range": {"value": [240.0, 700.0], "unit": "nm"}, + "corsika_cherenkov_photon_wavelength_range": {"value": [240.0, 1000.0], "unit": "nm"}, "corsika_first_interaction_height": {"value": 0.0, "unit": "cm"}, "corsika_particle_kinetic_energy_cutoff": { "value": [0.3, 0.1, 0.020, 0.020], @@ -157,8 +157,8 @@ def test_corsika_configuration_cherenkov_parameters( def test_input_config_corsika_cherenkov_wavelength(corsika_config_mock_array_model): assert corsika_config_mock_array_model._input_config_corsika_cherenkov_wavelength( - {"value": [240.0, 700.0], "unit": "nm"} - ) == ["240.0", "700.0"] + {"value": [240.0, 1000.0], "unit": "nm"} + ) == ["240.0", "1000.0"] def test_corsika_configuration_iact_parameters( From 760c145b3045f626e35243c4a49a9448c4fdbfc5 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 27 Jan 2025 14:23:14 +0100 Subject: [PATCH 104/124] improved testing --- tests/unit_tests/db/test_db_handler.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index 5435b3bd73..4988497290 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -16,13 +16,19 @@ @pytest.fixture(autouse=True) -def reset_db_client(): +def reset_db_client(random_id): """Reset db_client before each test.""" # If using the class-level db_client: db_handler.DatabaseHandler.db_client = None db_handler.production_table_cached = {} db_handler.model_parameters_cached = {} yield # allows the test to run + logger.info(f"dropping sandbox_{random_id} collections") + if db_handler.DatabaseHandler.db_client is not None: + db_handler.DatabaseHandler.db_client[f"sandbox_{random_id}"]["telescopes"].drop() + db_handler.DatabaseHandler.db_client[f"sandbox_{random_id}"]["calibration_devices"].drop() + db_handler.DatabaseHandler.db_client[f"sandbox_{random_id}"]["sites"].drop() + # After the test, reset any side-effects (if necessary): db_handler.DatabaseHandler.db_client = None db_handler.production_table_cached.clear() @@ -40,16 +46,6 @@ def db_no_config_file(): return db_handler.DatabaseHandler(mongo_db_config=None) -@pytest.fixture -def _db_cleanup(db, random_id): - yield - # Cleanup - logger.info(f"dropping sandbox_{random_id} collections") - db.db_client[f"sandbox_{random_id}"]["telescopes"].drop() - db.db_client[f"sandbox_{random_id}"]["calibration_devices"].drop() - db.db_client[f"sandbox_{random_id}"]["sites"].drop() - - @pytest.fixture def test_db(): return "test_db" From 9e3e082df8fbb3acc18ada464fe4d89c37c1c03b Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 27 Jan 2025 14:45:30 +0100 Subject: [PATCH 105/124] remove SSTS from dsum and asum_clippling schemas --- src/simtools/schemas/model_parameters/asum_clipping.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_clipping.schema.yml | 1 - .../schemas/model_parameters/dsum_ignore_below.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_offset.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml | 1 - .../schemas/model_parameters/dsum_pre_clipping.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_prescale.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml | 1 - .../schemas/model_parameters/dsum_presum_shift.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_shaping.schema.yml | 1 - .../schemas/model_parameters/dsum_shaping_renormalize.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_threshold.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml | 1 - 13 files changed, 13 deletions(-) diff --git a/src/simtools/schemas/model_parameters/asum_clipping.schema.yml b/src/simtools/schemas/model_parameters/asum_clipping.schema.yml index 4fbcd3746e..c34294335d 100644 --- a/src/simtools/schemas/model_parameters/asum_clipping.schema.yml +++ b/src/simtools/schemas/model_parameters/asum_clipping.schema.yml @@ -24,7 +24,6 @@ instrument: - LSTS - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml b/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml index 1ff03a1be1..9ba72cb26b 100644 --- a/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml @@ -24,7 +24,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml b/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml index 45c769df28..25f252948e 100644 --- a/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml @@ -24,7 +24,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_offset.schema.yml b/src/simtools/schemas/model_parameters/dsum_offset.schema.yml index deb91b4f51..f3bdae9caa 100644 --- a/src/simtools/schemas/model_parameters/dsum_offset.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_offset.schema.yml @@ -23,7 +23,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml b/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml index 7942b5be1e..0025779348 100644 --- a/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml @@ -20,7 +20,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml b/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml index 3af1912a7e..dfed2f2332 100644 --- a/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml @@ -25,7 +25,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml b/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml index 29b278cc6b..6b6fccd25e 100644 --- a/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml @@ -30,7 +30,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml b/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml index f466f5e9b7..f994bca44a 100644 --- a/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml @@ -24,7 +24,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml b/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml index 697e92b478..60cdd5bb16 100644 --- a/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml @@ -31,7 +31,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml b/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml index 3300db113e..eff815f48f 100644 --- a/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml @@ -30,7 +30,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml b/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml index a62c001c82..2b2f80b949 100644 --- a/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml @@ -18,7 +18,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml b/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml index 6bbba24ab8..a6c70d3cec 100644 --- a/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml @@ -28,7 +28,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: diff --git a/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml b/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml index 81923cb713..83e87a71b4 100644 --- a/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml @@ -28,7 +28,6 @@ instrument: type: - MSTN - MSTS - - SSTS - SCTS activity: setting: From 34f76c2306b68c01b4054b0f48dfe155ee7543b3 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 27 Jan 2025 14:56:48 +0100 Subject: [PATCH 106/124] SST fixes --- .../CTA-South-SSTS-01-5.0.0_test.cfg | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg index c04d11b7bd..1718d78298 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg @@ -12,7 +12,7 @@ adjust_gain = 1.0 altitude = 2147.0 -asum_clipping = 0.0 +asum_clipping = 9999 asum_offset = 0.0 asum_shaping_file = none asum_threshold = 0.0 @@ -48,18 +48,18 @@ discriminator_var_gate_length = 0.0 discriminator_var_sigsum_over_threshold = 0.0 discriminator_var_threshold = 0.25 discriminator_var_time_over_threshold = 0.0 -dsum_clipping = 85 +dsum_clipping = 0 dsum_ignore_below = 0 dsum_offset = 0.0 -dsum_pedsub = 0 +dsum_pedsub = 1 dsum_pre_clipping = 0 -dsum_prescale = 40 256 -dsum_presum_max = 127 -dsum_presum_shift = 1 +dsum_prescale = all: 0 +dsum_presum_max = 0 +dsum_presum_shift = 0 dsum_shaping_file = none dsum_shaping_renormalize = 0 dsum_threshold = 0.0 -dsum_zero_clip = 1 +dsum_zero_clip = 0 effective_focal_length = 215.191 0.0 0.0 0.0 0.0 fadc_ac_coupled = 0 fadc_amplitude = 2.77 @@ -97,8 +97,8 @@ focal_surface_ref_radius = 1.0 focus_offset = 0.0 0.0 0.0 0.0 gain_variation = 0.05 mirror2_degraded_reflection = 1.0 -mirror_align_random_horizontal = 0.0 31.0 0.0135 0.012 -mirror_align_random_vertical = 0.0 31.0 0.0135 0.012 +mirror_align_random_horizontal = 0.0 0.0 0.0 0.0 +mirror_align_random_vertical = 0.0 0.0 0.0 0.0 mirror_class = 2 mirror_degraded_reflection = 1.0 mirror_list = none @@ -142,8 +142,8 @@ telescope_transmission = 0.92362 1.0 0.03668 1.7454 0.858 0.0 teltrig_min_sigsum = 0.0 teltrig_min_time = 0.5 transit_time_calib_error = 0.0 -transit_time_compensate_error = 0.2 -transit_time_compensate_step = 1.0 +transit_time_compensate_error = 0. +transit_time_compensate_step = 0. transit_time_error = 0.1 transit_time_jitter = 0.01 trigger_current_limit = 20.0 From 5a5ce317c56b9db3ef8d6066b12833f4636c43a0 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 27 Jan 2025 16:50:45 +0100 Subject: [PATCH 107/124] array window metaschema --- .../model_parameters/array_window.schema.yml | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 src/simtools/schemas/model_parameters/array_window.schema.yml diff --git a/src/simtools/schemas/model_parameters/array_window.schema.yml b/src/simtools/schemas/model_parameters/array_window.schema.yml new file mode 100644 index 0000000000..1318824a83 --- /dev/null +++ b/src/simtools/schemas/model_parameters/array_window.schema.yml @@ -0,0 +1,37 @@ +%YAML 1.2 +--- +title: Schema for array_window model parameter +version: 0.1.0 +meta_schema: simpipe-schema +meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml +meta_schema_version: 0.1.0 +name: array_window +description: |- + Length of a coincidence window of the default stereo trigger, + after correction of fixed (cable length, focal length, etc.) + and variable (view direction) delays. +short_description: |- + Length of a coincidence window of the default stereo trigger. +data: + - type: double + unit: ns + default: 100. +instrument: + class: Camera + type: + - LSTN + - LSTS + - MSTN + - MSTS + - SSTS + - SCTS +activity: + setting: + - SetParameterFromExternal + validation: + - ValidateParameterByExpert + - ValidateTriggerPerformance +source: + - Observation execution +simulation_software: + - name: sim_telarray From b3556350ee84ccb986c2d30b880dcba183961e78 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Mon, 27 Jan 2025 21:14:48 +0100 Subject: [PATCH 108/124] OG comments --- .../applications/db_get_parameter_from_db.py | 11 ++++----- tests/unit_tests/db/test_db_handler.py | 24 ++++++++----------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 04d78e208c..3b14c9df90 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -125,19 +125,18 @@ def main(): # noqa: D103 ) else: raise ValueError("Either 'parameter_version' or 'model_version' must be provided.") - param = args_dict["parameter"] - if param not in pars: + if args_dict["parameter"] not in pars: raise KeyError(f"The requested parameter, {args_dict['parameter']}, does not exist.") if args_dict["output_file"] is not None: _output_file = ( Path(io_handler.IOHandler().get_output_directory()) / args_dict["output_file"] ) - pars[param].pop("_id") - pars[param].pop("entry_date") + pars[args_dict["parameter"]].pop("_id") + pars[args_dict["parameter"]].pop("entry_date") with open(_output_file, "w", encoding="utf-8") as json_file: - json.dump(pars[param], json_file, indent=4) + json.dump(pars[args_dict["parameter"]], json_file, indent=4) else: - pprint(pars[param]) + pprint(pars[args_dict["parameter"]]) if __name__ == "__main__": diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index 4988497290..c2f417257b 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -20,8 +20,6 @@ def reset_db_client(random_id): """Reset db_client before each test.""" # If using the class-level db_client: db_handler.DatabaseHandler.db_client = None - db_handler.production_table_cached = {} - db_handler.model_parameters_cached = {} yield # allows the test to run logger.info(f"dropping sandbox_{random_id} collections") if db_handler.DatabaseHandler.db_client is not None: @@ -31,8 +29,6 @@ def reset_db_client(random_id): # After the test, reset any side-effects (if necessary): db_handler.DatabaseHandler.db_client = None - db_handler.production_table_cached.clear() - db_handler.model_parameters_cached.clear() @pytest.fixture @@ -57,8 +53,8 @@ def test_file(): @pytest.fixture -def test_file2(): - return "test_file2.dat" +def test_file_2(): + return "test_file_2.dat" @pytest.fixture @@ -409,7 +405,7 @@ def test_get_collections(db, db_config, fs_files): def test_export_model_files_with_file_names( - db, mocker, tmp_test_directory, test_db, test_file, test_file2 + db, mocker, tmp_test_directory, test_db, test_file, test_file_2 ): """Test export_model_files method with file names.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) @@ -418,23 +414,23 @@ def test_export_model_files_with_file_names( ) mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") - file_names = [test_file, test_file2] + file_names = [test_file, test_file_2] result = db.export_model_files(file_names=file_names, dest=tmp_test_directory) mock_get_db_name.assert_called() - mock_get_file_mongo_db.assert_has_calls([call(test_db, test_file), call(test_db, test_file2)]) + mock_get_file_mongo_db.assert_has_calls([call(test_db, test_file), call(test_db, test_file_2)]) mock_write_file_from_mongo_to_disk.assert_has_calls( [ call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), ] ) - assert result == {test_file: "file_id", test_file2: "file_id"} + assert result == {test_file: "file_id", test_file_2: "file_id"} def test_export_model_files_with_parameters( - db, mocker, tmp_test_directory, test_db, test_file, test_file2 + db, mocker, tmp_test_directory, test_db, test_file, test_file_2 ): """Test export_model_files method with parameters.""" mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) @@ -445,20 +441,20 @@ def test_export_model_files_with_parameters( parameters = { "param1": {"file": True, "value": test_file}, - "param2": {"file": True, "value": test_file2}, + "param2": {"file": True, "value": test_file_2}, } result = db.export_model_files(parameters=parameters, dest=tmp_test_directory) mock_get_db_name.assert_called() - mock_get_file_mongo_db.assert_has_calls([call(test_db, test_file), call(test_db, test_file2)]) + mock_get_file_mongo_db.assert_has_calls([call(test_db, test_file), call(test_db, test_file_2)]) mock_write_file_from_mongo_to_disk.assert_has_calls( [ call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), ] ) - assert result == {test_file: "file_id", test_file2: "file_id"} + assert result == {test_file: "file_id", test_file_2: "file_id"} def test_export_model_files_file_exists(db, mocker, tmp_test_directory, test_db, test_file): From 2eb8ee5411af5446784a42a4c72bcc802fedece5 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 28 Jan 2025 10:43:28 +0100 Subject: [PATCH 109/124] add parameter name --- tests/resources/model_parameters/mirror_list.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/resources/model_parameters/mirror_list.json b/tests/resources/model_parameters/mirror_list.json index 172113ce62..7371c99c62 100644 --- a/tests/resources/model_parameters/mirror_list.json +++ b/tests/resources/model_parameters/mirror_list.json @@ -1,5 +1,6 @@ { "schema_version": "0.2.0", + "parameter": "mirror_list", "instrument": "LSTN-01", "site": "North", "parameter_version": "2.0.0", From 82ac448546897bcb807f35a642e875f6a7481a18 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 28 Jan 2025 10:54:56 +0100 Subject: [PATCH 110/124] Allow null for instrument/site; check if files are found --- .../validate_file_using_schema.py | 24 ++++++++++++------- .../schemas/model_parameter.metaschema.yml | 9 +++++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/simtools/applications/validate_file_using_schema.py b/src/simtools/applications/validate_file_using_schema.py index a4ba71f4cb..f9dda5276c 100644 --- a/src/simtools/applications/validate_file_using_schema.py +++ b/src/simtools/applications/validate_file_using_schema.py @@ -116,6 +116,19 @@ def _get_schema_file_name(args_dict, data_dict=None): return schema_file +def _get_file_list(file_directory=None, file_name=None): + """Return list of json files in a directory.""" + file_list = [] + if file_directory is not None: + file_list = list(Path(file_directory).rglob("*.json")) + if not file_list: + raise FileNotFoundError(f"No files found in {file_directory}") + elif file_name is not None: + file_list = [file_name] + + return file_list + + def validate_schema(args_dict, logger): """ Validate a schema file (or several files) given in yaml or json format. @@ -124,11 +137,7 @@ def validate_schema(args_dict, logger): the metadata section of the data dictionary. """ - if args_dict.get("file_directory") is not None: - file_list = list(Path(args_dict["file_directory"]).rglob("*.json")) - else: - file_list = [args_dict["file_name"]] - for file_name in file_list: + for file_name in _get_file_list(args_dict.get("file_directory"), args_dict.get("file_name")): try: data = gen.collect_data_from_file(file_name=file_name) except FileNotFoundError as exc: @@ -144,10 +153,9 @@ def validate_schema(args_dict, logger): def validate_data_files(args_dict, logger): """Validate data files.""" - file_directory = args_dict.get("file_directory") - if file_directory is not None: + if args_dict.get("file_directory") is not None: tmp_args_dict = {} - for file_name in Path(file_directory).rglob("*.json"): + for file_name in _get_file_list(args_dict.get("file_directory")): tmp_args_dict["file_name"] = file_name parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem) schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml" diff --git a/src/simtools/schemas/model_parameter.metaschema.yml b/src/simtools/schemas/model_parameter.metaschema.yml index 22e2c2ede5..2e1fc5fab6 100644 --- a/src/simtools/schemas/model_parameter.metaschema.yml +++ b/src/simtools/schemas/model_parameter.metaschema.yml @@ -25,14 +25,19 @@ definitions: type: boolean description: "This parameter is a file." instrument: - type: string + type: + - string + - "null" description: "Associated instrument." site: - type: string + type: + - string + - "null" description: "Associated CTAO site." enum: - North - South + - null type: type: string description: "Data type" From f52a783c0d3cdefffb093c89ea6943b472d6b43f Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 28 Jan 2025 12:45:32 +0100 Subject: [PATCH 111/124] Corrections to reference models --- .../CTA-North-LSTN-01-5.0.0_test.cfg | 3 ++- .../CTA-North-LSTN-01-6.0.0_test.cfg | 1 + .../CTA-North-LSTN-02-5.0.0_test.cfg | 5 +++-- .../CTA-North-LSTN-02-6.0.0_test.cfg | 1 + .../CTA-North-LSTN-03-5.0.0_test.cfg | 5 +++-- .../CTA-North-LSTN-03-6.0.0_test.cfg | 1 + .../CTA-North-LSTN-04-5.0.0_test.cfg | 5 +++-- .../CTA-North-LSTN-04-6.0.0_test.cfg | 1 + .../CTA-North-MSTN-01-5.0.0_test.cfg | 5 +++-- .../CTA-North-MSTN-01-6.0.0_test.cfg | 1 + .../CTA-South-LSTS-01-5.0.0_test.cfg | 3 ++- .../CTA-South-LSTS-01-6.0.0_test.cfg | 1 + .../CTA-South-MSTS-01-5.0.0_test.cfg | 5 +++-- .../CTA-South-MSTS-01-6.0.0_test.cfg | 1 + .../CTA-South-SSTS-01-5.0.0_test.cfg | 9 +++++---- .../CTA-South-SSTS-01-6.0.0_test.cfg | 3 ++- 16 files changed, 33 insertions(+), 17 deletions(-) diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg index 36203d221c..59d3c22b22 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2158.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none @@ -94,7 +95,7 @@ mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.23801 +nightsky_background = all: 0.238006 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-6.0.0_test.cfg index 2f5d4ec29a..2866f46b8c 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-6.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2156.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg index eca5a0d43c..d5ab1b8e6a 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2158.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none @@ -94,7 +95,7 @@ mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.24499 +nightsky_background = all: 0.244985 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 @@ -106,7 +107,7 @@ pm_average_gain = 40000.0 pm_collection_efficiency = 1.0 pm_gain_index = 3.92 pm_photoelectron_spectrum = spe_LST_2020-05-09_AP2.0e-4.dat -pm_transit_time = 20.89 9 350 1135 +pm_transit_time = 20.89 9.0 350.0 1135.0 pm_voltage_variation = 0.03 qe_variation = 0.03 quantum_efficiency = qe_lst2-4_20200318_high+low.dat diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-6.0.0_test.cfg index 2be55beae9..bbcf6a03db 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-6.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2156.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg index 0d4fd36e3c..ae798dda40 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2158.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none @@ -94,7 +95,7 @@ mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.238006 +nightsky_background = all: 0.244985 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 @@ -106,7 +107,7 @@ pm_average_gain = 40000.0 pm_collection_efficiency = 1.0 pm_gain_index = 3.92 pm_photoelectron_spectrum = spe_LST_2020-05-09_AP2.0e-4.dat -pm_transit_time = 20.89 9 350 1135 +pm_transit_time = 20.89 9.0 350.0 1135.0 pm_voltage_variation = 0.03 qe_variation = 0.03 quantum_efficiency = qe_lst2-4_20200318_high+low.dat diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-6.0.0_test.cfg index a813caea12..32135153f0 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-6.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2156.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg index 942bc46ddd..af8dcf89c1 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2158.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none @@ -94,7 +95,7 @@ mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.24499 +nightsky_background = all: 0.244985 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 @@ -106,7 +107,7 @@ pm_average_gain = 40000.0 pm_collection_efficiency = 1.0 pm_gain_index = 3.92 pm_photoelectron_spectrum = spe_LST_2020-05-09_AP2.0e-4.dat -pm_transit_time = 20.89 9 350 1135 +pm_transit_time = 20.89 9.0 350.0 1135.0 pm_voltage_variation = 0.03 qe_variation = 0.03 quantum_efficiency = qe_lst2-4_20200318_high+low.dat diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-6.0.0_test.cfg index 7fdef1c87b..459b84c63b 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-6.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2156.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg index 2cec2b4039..984173b4f3 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2158.0 +array_window = 1000.0 asum_clipping = 0.0 asum_offset = 0.0 asum_shaping_file = none @@ -107,7 +108,7 @@ mirror_offset = 0.0 mirror_reflection_random_angle = 0.006759 0.125 0.037 mirror_reflectivity = ref_AlSiO2HfO2.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.20771 +nightsky_background = all: 0.207706 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 @@ -126,7 +127,7 @@ quantum_efficiency = qe_R12992-100-05b.dat random_focal_length = 3.9 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 -telescope_transmission = 0.908661 0.0154576 4 1.21166 0 0 +telescope_transmission = 0.908661 0.0154576 4.0 1.21166 teltrig_min_sigsum = 0.0 teltrig_min_time = 1.0 transit_time_calib_error = 0.0 diff --git a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg index 161c3b32ae..a87b837ec6 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2156.0 +array_window = 1000.0 asum_clipping = 0.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg index 071a92c2b6..11c3b8bc93 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2147.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none @@ -94,7 +95,7 @@ mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.24499 +nightsky_background = all: 0.244985 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-6.0.0_test.cfg index 51a89bc8ca..d17df29ae8 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-6.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2147.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg index 58ac86de31..06445e0161 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2147.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none @@ -107,7 +108,7 @@ mirror_offset = 0.0 mirror_reflection_random_angle = 0.006759 0.125 0.037 mirror_reflectivity = ref_AlSiO2HfO2.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.20905 +nightsky_background = all: 0.209051 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 1 @@ -126,7 +127,7 @@ quantum_efficiency = qe_R12992-100-05b.dat random_focal_length = 3.9 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 -telescope_transmission = 0.908661 0.0154576 4 1.21166 0 0 +telescope_transmission = 0.908661 0.0154576 4.0 1.21166 teltrig_min_sigsum = 0.0 teltrig_min_time = 0.0 transit_time_calib_error = 0.0 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg index c06b51487e..530a9496ae 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2147.0 +array_window = 1000.0 asum_clipping = 0.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg index bf9877ea7c..cbdb6ec992 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2147.0 +array_window = 1000.0 asum_clipping = 0.0 asum_offset = 0.0 asum_shaping_file = none @@ -107,7 +108,7 @@ mirror_reflection_random_angle = 0.0255 0.0 0.0 mirror_reflectivity = ref_astri-2d_2018-01-17.dat mirror_secondary_reflectivity = ref_astri-2d_2018-01-17.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.03873 +nightsky_background = all: 0.0387262 nsb_autoscale_airmass = 0.83 0.2 nsb_gain_drop_scale = 0.0 nsb_offaxis = 1.0 0.04478 7.844 2.0282 0.0 @@ -142,11 +143,11 @@ telescope_transmission = 0.92362 1.0 0.03668 1.7454 0.858 0.0 teltrig_min_sigsum = 0.0 teltrig_min_time = 0.5 transit_time_calib_error = 0.0 -transit_time_compensate_error = 0. -transit_time_compensate_step = 0. +transit_time_compensate_error = 0.0 +transit_time_compensate_step = 0.0 transit_time_error = 0.1 transit_time_jitter = 0.01 -trigger_current_limit = 20.0 +trigger_current_limit = 2000.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 2 iobuf_maximum = 1000000000 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg index b58061e91e..f202ad8f1d 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2147.0 +array_window = 1000.0 asum_clipping = 0.0 asum_offset = 0.0 asum_shaping_file = none @@ -146,7 +147,7 @@ transit_time_compensate_error = 0.0 transit_time_compensate_step = 0.0 transit_time_error = 0.3 transit_time_jitter = 0.1 -trigger_current_limit = 20.0 +trigger_current_limit = 2000.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 2 iobuf_maximum = 1000000000 From 193c1e33a6ec0f4eca3b71d928b79015fdd24f81 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 28 Jan 2025 13:23:05 +0100 Subject: [PATCH 112/124] minor source fix --- src/simtools/schemas/model_parameters/array_window.schema.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/simtools/schemas/model_parameters/array_window.schema.yml b/src/simtools/schemas/model_parameters/array_window.schema.yml index 1318824a83..c6918ecabf 100644 --- a/src/simtools/schemas/model_parameters/array_window.schema.yml +++ b/src/simtools/schemas/model_parameters/array_window.schema.yml @@ -32,6 +32,6 @@ activity: - ValidateParameterByExpert - ValidateTriggerPerformance source: - - Observation execution + - Initial instrument setup simulation_software: - name: sim_telarray From 9bf50b21c75fa8a3a60ba66f6d353e23683b01c2 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 28 Jan 2025 13:40:16 +0100 Subject: [PATCH 113/124] Simplifications of get simulation software from parameter name --- .../upload_from_model_repository_to_db.sh | 2 +- src/simtools/model/model_parameter.py | 11 ++--------- src/simtools/simtel/simtel_config_writer.py | 5 +---- src/simtools/utils/names.py | 12 +----------- tests/resources/model_parameters/mirror_list.json | 1 + tests/unit_tests/utils/test_names.py | 13 ------------- 6 files changed, 6 insertions(+), 38 deletions(-) diff --git a/database_scripts/upload_from_model_repository_to_db.sh b/database_scripts/upload_from_model_repository_to_db.sh index 2a82112d26..85d2bfd668 100755 --- a/database_scripts/upload_from_model_repository_to_db.sh +++ b/database_scripts/upload_from_model_repository_to_db.sh @@ -6,7 +6,7 @@ # shellcheck disable=SC1091 DB_SIMULATION_MODEL_URL="https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/simulation-models.git" -DB_SIMULATION_MODEL_BRANCH="v1.0.0-parameters" +DB_SIMULATION_MODEL_BRANCH="prod5-prod6-corrections" # Check that this script is not sourced but executed if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index adc5a9b370..248a984af2 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -344,7 +344,7 @@ def extra_label(self): """Return the extra label if defined, if not return ''.""" return self._extra_label if self._extra_label is not None else "" - def get_simtel_parameters(self, parameters=None, telescope_model=True, site_model=True): + def get_simtel_parameters(self, parameters=None): """ Get simtel parameters as name and value pairs. @@ -352,10 +352,6 @@ def get_simtel_parameters(self, parameters=None, telescope_model=True, site_mode ---------- parameters: dict Parameters (simtools) to be renamed (if necessary) - telescope_model: bool - If True, telescope model parameters are included. - site_model: bool - If True, site model parameters are included. Returns ------- @@ -369,10 +365,7 @@ def get_simtel_parameters(self, parameters=None, telescope_model=True, site_mode _simtel_parameter_value = {} for key in parameters: _par_name = names.get_simulation_software_name_from_parameter_name( - key, - simulation_software="sim_telarray", - search_telescope_parameters=telescope_model, - search_site_parameters=site_model, + key, simulation_software="sim_telarray" ) if _par_name is not None: _simtel_parameter_value[_par_name] = parameters[key].get("value") diff --git a/src/simtools/simtel/simtel_config_writer.py b/src/simtools/simtel/simtel_config_writer.py index 4c144192ff..1fee7d48d3 100644 --- a/src/simtools/simtel/simtel_config_writer.py +++ b/src/simtools/simtel/simtel_config_writer.py @@ -294,10 +294,7 @@ def _write_site_parameters(self, file, site_model, model_path, telescope_model): _site_parameters = site_model.get_simtel_parameters() for par, value in _site_parameters.items(): _simtel_name = names.get_simulation_software_name_from_parameter_name( - par, - simulation_software="sim_telarray", - search_telescope_parameters=False, - search_site_parameters=True, + par, simulation_software="sim_telarray" ) _simtel_name, value = self._convert_model_parameters_to_simtel_format( _simtel_name, value, model_path, telescope_model diff --git a/src/simtools/utils/names.py b/src/simtools/utils/names.py index 43415e0290..e4ee803caa 100644 --- a/src/simtools/utils/names.py +++ b/src/simtools/utils/names.py @@ -335,8 +335,6 @@ def get_collection_name_from_array_element_name(name, array_elements_only=True): def get_simulation_software_name_from_parameter_name( par_name, simulation_software="sim_telarray", - search_telescope_parameters=True, - search_site_parameters=True, ): """ Get the name used in the simulation software from the model parameter name. @@ -350,21 +348,13 @@ def get_simulation_software_name_from_parameter_name( Model parameter name. simulation_software: str Simulation software name. - search_telescope_parameters: bool - If True, telescope model parameters are included. - search_site_parameters: bool - If True, site model parameters are included. Returns ------- str Simtel parameter name. """ - _parameter_names = {} - if search_telescope_parameters: - _parameter_names.update(telescope_parameters()) - if search_site_parameters: - _parameter_names.update(site_parameters()) + _parameter_names = {**telescope_parameters(), **site_parameters()} try: _parameter = _parameter_names[par_name] diff --git a/tests/resources/model_parameters/mirror_list.json b/tests/resources/model_parameters/mirror_list.json index 172113ce62..7371c99c62 100644 --- a/tests/resources/model_parameters/mirror_list.json +++ b/tests/resources/model_parameters/mirror_list.json @@ -1,5 +1,6 @@ { "schema_version": "0.2.0", + "parameter": "mirror_list", "instrument": "LSTN-01", "site": "North", "parameter_version": "2.0.0", diff --git a/tests/unit_tests/utils/test_names.py b/tests/unit_tests/utils/test_names.py index c8de8bbc9b..dcf42c8a10 100644 --- a/tests/unit_tests/utils/test_names.py +++ b/tests/unit_tests/utils/test_names.py @@ -500,19 +500,6 @@ def test_get_simulation_software_name_from_parameter_name(): == "reference_point_longitude" ) - with pytest.raises(KeyError): - names.get_simulation_software_name_from_parameter_name( - "corsika_observation_level", - simulation_software="sim_telarray", - search_site_parameters=False, - ) - with pytest.raises(KeyError): - names.get_simulation_software_name_from_parameter_name( - "telescope_axis_height", - simulation_software="sim_telarray", - search_telescope_parameters=False, - ) - def test_get_parameter_name_from_simtel_name(): assert names.get_parameter_name_from_simtel_name("focal_length") == "focal_length" From 5ceedf5c104c7a74c6b6e06ed963024cf8df5245 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 28 Jan 2025 16:22:24 +0100 Subject: [PATCH 114/124] digital sum for MSTs only --- src/simtools/schemas/model_parameters/array_window.schema.yml | 2 +- src/simtools/schemas/model_parameters/asum_clipping.schema.yml | 3 --- src/simtools/schemas/model_parameters/dsum_clipping.schema.yml | 1 - .../schemas/model_parameters/dsum_ignore_below.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_offset.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml | 1 - .../schemas/model_parameters/dsum_pre_clipping.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_prescale.schema.yml | 1 - .../schemas/model_parameters/dsum_presum_max.schema.yml | 1 - .../schemas/model_parameters/dsum_presum_shift.schema.yml | 1 - src/simtools/schemas/model_parameters/dsum_shaping.schema.yml | 1 - .../model_parameters/dsum_shaping_renormalize.schema.yml | 1 - .../schemas/model_parameters/dsum_threshold.schema.yml | 1 - .../schemas/model_parameters/dsum_zero_clip.schema.yml | 1 - 14 files changed, 1 insertion(+), 16 deletions(-) diff --git a/src/simtools/schemas/model_parameters/array_window.schema.yml b/src/simtools/schemas/model_parameters/array_window.schema.yml index c6918ecabf..f6bb8a0544 100644 --- a/src/simtools/schemas/model_parameters/array_window.schema.yml +++ b/src/simtools/schemas/model_parameters/array_window.schema.yml @@ -15,7 +15,7 @@ short_description: |- data: - type: double unit: ns - default: 100. + default: 1000. instrument: class: Camera type: diff --git a/src/simtools/schemas/model_parameters/asum_clipping.schema.yml b/src/simtools/schemas/model_parameters/asum_clipping.schema.yml index c34294335d..1024977c14 100644 --- a/src/simtools/schemas/model_parameters/asum_clipping.schema.yml +++ b/src/simtools/schemas/model_parameters/asum_clipping.schema.yml @@ -22,9 +22,6 @@ instrument: type: - LSTN - LSTS - - MSTN - - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml b/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml index 9ba72cb26b..cb4dc9dbcf 100644 --- a/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml @@ -24,7 +24,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml b/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml index 25f252948e..03ae179b94 100644 --- a/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml @@ -24,7 +24,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_offset.schema.yml b/src/simtools/schemas/model_parameters/dsum_offset.schema.yml index f3bdae9caa..993dcc12d4 100644 --- a/src/simtools/schemas/model_parameters/dsum_offset.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_offset.schema.yml @@ -23,7 +23,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml b/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml index 0025779348..e507802ee6 100644 --- a/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml @@ -20,7 +20,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml b/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml index dfed2f2332..02fc297218 100644 --- a/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml @@ -25,7 +25,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml b/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml index 6b6fccd25e..b5a044b193 100644 --- a/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml @@ -30,7 +30,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml b/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml index f994bca44a..db7cc380d8 100644 --- a/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml @@ -24,7 +24,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml b/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml index 60cdd5bb16..d374f37930 100644 --- a/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml @@ -31,7 +31,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml b/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml index eff815f48f..1c32c62382 100644 --- a/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml @@ -30,7 +30,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml b/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml index 2b2f80b949..74047fd93e 100644 --- a/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml @@ -18,7 +18,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml b/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml index a6c70d3cec..af45603e50 100644 --- a/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml @@ -28,7 +28,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml b/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml index 83e87a71b4..9ae6c26af4 100644 --- a/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml @@ -28,7 +28,6 @@ instrument: type: - MSTN - MSTS - - SCTS activity: setting: - SetParameterFromExternal From df837756298b005151fe2e368952249cb90a15cc Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 28 Jan 2025 17:38:16 +0100 Subject: [PATCH 115/124] improve naming --- src/simtools/applications/validate_file_using_schema.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/simtools/applications/validate_file_using_schema.py b/src/simtools/applications/validate_file_using_schema.py index f9dda5276c..1ea17ca5ea 100644 --- a/src/simtools/applications/validate_file_using_schema.py +++ b/src/simtools/applications/validate_file_using_schema.py @@ -116,7 +116,7 @@ def _get_schema_file_name(args_dict, data_dict=None): return schema_file -def _get_file_list(file_directory=None, file_name=None): +def _get_json_file_list(file_directory=None, file_name=None): """Return list of json files in a directory.""" file_list = [] if file_directory is not None: @@ -137,7 +137,9 @@ def validate_schema(args_dict, logger): the metadata section of the data dictionary. """ - for file_name in _get_file_list(args_dict.get("file_directory"), args_dict.get("file_name")): + for file_name in _get_json_file_list( + args_dict.get("file_directory"), args_dict.get("file_name") + ): try: data = gen.collect_data_from_file(file_name=file_name) except FileNotFoundError as exc: @@ -155,7 +157,7 @@ def validate_data_files(args_dict, logger): """Validate data files.""" if args_dict.get("file_directory") is not None: tmp_args_dict = {} - for file_name in _get_file_list(args_dict.get("file_directory")): + for file_name in _get_json_file_list(args_dict.get("file_directory")): tmp_args_dict["file_name"] = file_name parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem) schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml" From aa58787a3f3cdc12510db3c8cfbe8309a2f52a39 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Tue, 28 Jan 2025 18:29:46 +0100 Subject: [PATCH 116/124] Fix integration test depending on json comparison --- src/simtools/db/db_handler.py | 1 - src/simtools/testing/validate_output.py | 10 +++++++++- ...oordinates_of_array_elements_ground_to_utm_json.yml | 3 ++- ...oordinates_of_array_elements_utm_to_ground_json.yml | 3 ++- ...parameter_from_db_array_element_position_ground.yml | 2 +- 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index fdb6488aaa..424de4c499 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -409,7 +409,6 @@ def _read_mongo_db(self, query, collection_name): for post in posts: par_now = post["parameter"] parameters[par_now] = post - parameters[par_now].pop("parameter", None) parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time return {k: parameters[k] for k in sorted(parameters)} diff --git a/src/simtools/testing/validate_output.py b/src/simtools/testing/validate_output.py index 8b8074ac79..b761526c8a 100644 --- a/src/simtools/testing/validate_output.py +++ b/src/simtools/testing/validate_output.py @@ -146,6 +146,7 @@ def compare_files(file1, file2, tolerance=1.0e-5, test_columns=None): """ _file1_suffix = Path(file1).suffix _file2_suffix = Path(file2).suffix + _logger.info("Comparing files: %s and %s", file1, file2) if _file1_suffix != _file2_suffix: raise ValueError(f"File suffixes do not match: {file1} and {file2}") if _file1_suffix == ".ecsv": @@ -185,9 +186,11 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2): if data1 == data2: return True + if data1.keys() != data2.keys(): + _logger.error(f"Keys do not match: {data1.keys()} and {data2.keys()}") return False - return all( + _comparison = all( ( _compare_value_from_parameter_dict(data1[k], data2[k], tolerance) if k == "value" @@ -195,6 +198,9 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2): ) for k in data1 ) + if not _comparison: + _logger.error(f"Values do not match: {data1} and {data2} (tolerance: {tolerance})") + return _comparison def _compare_value_from_parameter_dict(data1, data2, tolerance): @@ -207,6 +213,8 @@ def _as_list(value): return value return [value] + _logger.info(f"Comparing values: {data1} and {data2} (tolerance: {tolerance})") + return np.allclose(_as_list(data1), _as_list(data2), rtol=tolerance) diff --git a/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_ground_to_utm_json.yml b/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_ground_to_utm_json.yml index 7af3563556..df84bdec89 100644 --- a/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_ground_to_utm_json.yml +++ b/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_ground_to_utm_json.yml @@ -3,9 +3,10 @@ CTA_SIMPIPE: TEST_NAME: ground_to_utm_json CONFIGURATION: INPUT: tests/resources/model_parameters/array_element_position_ground.json - SITE: South + SITE: North EXPORT: utm MODEL_VERSION: "6.0.0" + PARAMETER_VERSION: "2.0.0" OUTPUT_PATH: simtools-output OUTPUT_FILE: test.json LOG_LEVEL: DEBUG diff --git a/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_utm_to_ground_json.yml b/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_utm_to_ground_json.yml index dace3a0460..e884abab10 100644 --- a/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_utm_to_ground_json.yml +++ b/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_utm_to_ground_json.yml @@ -3,9 +3,10 @@ CTA_SIMPIPE: TEST_NAME: utm_to_ground_json CONFIGURATION: INPUT: tests/resources/model_parameters/array_element_position_utm.json - SITE: South + SITE: North EXPORT: ground MODEL_VERSION: "6.0.0" + PARAMETER_VERSION: "2.0.0" OUTPUT_PATH: simtools-output OUTPUT_FILE: test.json LOG_LEVEL: DEBUG diff --git a/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml b/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml index 0af1b84807..40d615fa36 100644 --- a/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml +++ b/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml @@ -12,4 +12,4 @@ CTA_SIMPIPE: INTEGRATION_TESTS: - REFERENCE_OUTPUT_FILE: | ./tests/resources/model_parameters/array_element_position_ground.json - TOLERANCE: 1.e-2 + TOLERANCE: 1.e-5 From 102717b88511abae502c1056053e3847e91b7dee Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 29 Jan 2025 09:29:50 +0100 Subject: [PATCH 117/124] prod5 telescope transmission --- .../CTA-North-MSTN-01-5.0.0_test.cfg | 2 +- .../CTA-South-MSTS-01-5.0.0_test.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg index 984173b4f3..7dc9a39484 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg @@ -127,7 +127,7 @@ quantum_efficiency = qe_R12992-100-05b.dat random_focal_length = 3.9 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 -telescope_transmission = 0.908661 0.0154576 4.0 1.21166 +telescope_transmission = 0.908661 0.0 0.0 0.0 0.0 0.0 teltrig_min_sigsum = 0.0 teltrig_min_time = 1.0 transit_time_calib_error = 0.0 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg index 06445e0161..28d091a74a 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg @@ -127,7 +127,7 @@ quantum_efficiency = qe_R12992-100-05b.dat random_focal_length = 3.9 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 -telescope_transmission = 0.908661 0.0154576 4.0 1.21166 +telescope_transmission = 0.908661 0.0 0.0 0.0 0.0 0.0 teltrig_min_sigsum = 0.0 teltrig_min_time = 0.0 transit_time_calib_error = 0.0 From aefe6689fcca3c4538cdbaadfa4fff2b29780cae Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 29 Jan 2025 10:48:55 +0100 Subject: [PATCH 118/124] fix unit test --- tests/unit_tests/db/test_db_handler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index c2f417257b..af815a0dff 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -604,11 +604,13 @@ def test_read_mongo_db(db, mocker, test_db): assert result == { "param1": { "_id": mock_find.return_value[0]["_id"], + "parameter": "param1", "value": "value1", "entry_date": mock_find.return_value[0]["_id"].generation_time, }, "param2": { "_id": mock_find.return_value[1]["_id"], + "parameter": "param2", "value": "value2", "entry_date": mock_find.return_value[1]["_id"].generation_time, }, From 5f59a3a6929d877776064a7f6802425dfbb229e6 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 29 Jan 2025 14:27:06 +0100 Subject: [PATCH 119/124] set trigger current limit to 20 uA --- .../CTA-North-MSTN-01-5.0.0_test.cfg | 2 +- .../CTA-North-MSTN-01-6.0.0_test.cfg | 2 +- .../CTA-South-MSTS-01-5.0.0_test.cfg | 2 +- .../CTA-South-MSTS-01-6.0.0_test.cfg | 2 +- .../CTA-South-SSTS-01-5.0.0_test.cfg | 2 +- .../CTA-South-SSTS-01-6.0.0_test.cfg | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg index 7dc9a39484..5ab6e2c1ec 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg @@ -135,7 +135,7 @@ transit_time_compensate_error = 0.0 transit_time_compensate_step = 0.0 transit_time_error = 0.5 transit_time_jitter = 0.7 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 3 iobuf_maximum = 1000000000 diff --git a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg index a87b837ec6..ad82359215 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg @@ -135,7 +135,7 @@ transit_time_compensate_error = 0.0 transit_time_compensate_step = 0.0 transit_time_error = 0.5 transit_time_jitter = 0.7 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 3 iobuf_maximum = 1000000000 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg index 28d091a74a..6eaccbd233 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg @@ -135,7 +135,7 @@ transit_time_compensate_error = 0.2 transit_time_compensate_step = 1.0 transit_time_error = 0.0 transit_time_jitter = 0.7 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 3 iobuf_maximum = 1000000000 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg index 530a9496ae..b7751ba1df 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg @@ -135,7 +135,7 @@ transit_time_compensate_error = 0.2 transit_time_compensate_step = 1.0 transit_time_error = 0.0 transit_time_jitter = 0.7 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 99 iobuf_maximum = 1000000000 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg index cbdb6ec992..b379214c19 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg @@ -147,7 +147,7 @@ transit_time_compensate_error = 0.0 transit_time_compensate_step = 0.0 transit_time_error = 0.1 transit_time_jitter = 0.01 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 2 iobuf_maximum = 1000000000 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg index f202ad8f1d..215ba51226 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg @@ -147,7 +147,7 @@ transit_time_compensate_error = 0.0 transit_time_compensate_step = 0.0 transit_time_error = 0.3 transit_time_jitter = 0.1 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 2 iobuf_maximum = 1000000000 From eba8719a123b13dcd0d0201b02889d30fefd31e1 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 29 Jan 2025 17:09:59 +0100 Subject: [PATCH 120/124] Correct branch to main for simulation model upload --- database_scripts/upload_from_model_repository_to_db.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database_scripts/upload_from_model_repository_to_db.sh b/database_scripts/upload_from_model_repository_to_db.sh index 85d2bfd668..8a9a8213f2 100755 --- a/database_scripts/upload_from_model_repository_to_db.sh +++ b/database_scripts/upload_from_model_repository_to_db.sh @@ -6,7 +6,7 @@ # shellcheck disable=SC1091 DB_SIMULATION_MODEL_URL="https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/simulation-models.git" -DB_SIMULATION_MODEL_BRANCH="prod5-prod6-corrections" +DB_SIMULATION_MODEL_BRANCH="main" # Check that this script is not sourced but executed if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then From 3903f6ff879c2d99cd9061267b34cf131e7269c0 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Wed, 29 Jan 2025 20:59:57 +0100 Subject: [PATCH 121/124] Remove sandboxes --- .../applications/db_add_file_to_db.py | 3 +-- .../db_add_value_from_json_to_db.py | 3 +-- .../applications/db_get_file_from_db.py | 1 - src/simtools/db/db_handler.py | 1 + tests/unit_tests/db/test_db_handler.py | 23 +------------------ 5 files changed, 4 insertions(+), 27 deletions(-) diff --git a/src/simtools/applications/db_add_file_to_db.py b/src/simtools/applications/db_add_file_to_db.py index 3483bf6225..2efb77c21c 100644 --- a/src/simtools/applications/db_add_file_to_db.py +++ b/src/simtools/applications/db_add_file_to_db.py @@ -18,8 +18,7 @@ A directory with files to upload to the DB. \ All files in the directory with a predefined list of extensions will be uploaded. db (str) - The DB to insert the files to. \ - The choices are either the default CTA simulation DB or a sandbox for testing. + The DB to insert the files to. Example ------- diff --git a/src/simtools/applications/db_add_value_from_json_to_db.py b/src/simtools/applications/db_add_value_from_json_to_db.py index 93ee5317be..2046f1eec5 100644 --- a/src/simtools/applications/db_add_value_from_json_to_db.py +++ b/src/simtools/applications/db_add_value_from_json_to_db.py @@ -10,8 +10,7 @@ db_collection (str, required) The DB collection to which to add the file. db (str) - The DB to insert the files to. \ - The choices are either the default CTA simulation DB or a sandbox for testing. + The DB to insert the files to. Example ------- diff --git a/src/simtools/applications/db_get_file_from_db.py b/src/simtools/applications/db_get_file_from_db.py index 215f58f302..8510091d72 100644 --- a/src/simtools/applications/db_get_file_from_db.py +++ b/src/simtools/applications/db_get_file_from_db.py @@ -67,7 +67,6 @@ def main(): # noqa: D103 db = db_handler.DatabaseHandler(mongo_db_config=db_config) available_dbs = [ db_config["db_simulation_model"], - "sandbox", ] file_id = {} for db_name in available_dbs: diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index fdb6488aaa..f1d813708d 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -700,6 +700,7 @@ def insert_file_to_db(self, file_name, db_name=None, **kwargs): return file_system.find_one( # pylint: disable=protected-access {"filename": kwargs["filename"]} )._id + self._logger.debug(f"Writing file to DB: {file_name}") with open(file_name, "rb") as data_file: return file_system.put(data_file, **kwargs) diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index c2f417257b..d611f7b64f 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -2,7 +2,6 @@ import copy import logging -import uuid from pathlib import Path from unittest.mock import call @@ -16,26 +15,15 @@ @pytest.fixture(autouse=True) -def reset_db_client(random_id): +def reset_db_client(): """Reset db_client before each test.""" # If using the class-level db_client: db_handler.DatabaseHandler.db_client = None yield # allows the test to run - logger.info(f"dropping sandbox_{random_id} collections") - if db_handler.DatabaseHandler.db_client is not None: - db_handler.DatabaseHandler.db_client[f"sandbox_{random_id}"]["telescopes"].drop() - db_handler.DatabaseHandler.db_client[f"sandbox_{random_id}"]["calibration_devices"].drop() - db_handler.DatabaseHandler.db_client[f"sandbox_{random_id}"]["sites"].drop() - # After the test, reset any side-effects (if necessary): db_handler.DatabaseHandler.db_client = None -@pytest.fixture -def random_id(): - return uuid.uuid4().hex - - @pytest.fixture def db_no_config_file(): """Database object (without configuration).""" @@ -82,15 +70,6 @@ def mock_gridfs(mocker): return mocker.patch("simtools.db.db_handler.gridfs.GridFS") -@pytest.fixture -def _db_cleanup_file_sandbox(db_no_config_file, random_id, fs_files): - yield - # Cleanup - logger.info("Dropping the temporary files in the sandbox") - db_no_config_file.db_client[f"sandbox_{random_id}"]["fs.chunks"].drop() - db_no_config_file.db_client[f"sandbox_{random_id}"][fs_files].drop() - - def test_set_up_connection_no_config(): """Test _set_up_connection with no configuration.""" db = db_handler.DatabaseHandler(mongo_db_config=None) From 147670a1dd1579e14fa0323ca847f6de5ddbe80d Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 30 Jan 2025 12:27:08 +0100 Subject: [PATCH 122/124] Add fake_mirror_list parameter --- .../fake_mirror_list.schema.yml | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 src/simtools/schemas/model_parameters/fake_mirror_list.schema.yml diff --git a/src/simtools/schemas/model_parameters/fake_mirror_list.schema.yml b/src/simtools/schemas/model_parameters/fake_mirror_list.schema.yml new file mode 100644 index 0000000000..59735dd06d --- /dev/null +++ b/src/simtools/schemas/model_parameters/fake_mirror_list.schema.yml @@ -0,0 +1,33 @@ +%YAML 1.2 +--- +title: Schema for fake_mirror_list model parameter +version: 0.1.0 +meta_schema: simpipe-schema +meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml +meta_schema_version: 0.1.0 +developer_note: To be replaced by a data table +name: fake_mirror_list +short_description: Fake mirror list to be used for camera efficiency calculations ('testeff' program). +description: |- + Fake mirror list to be used for camera efficiency calculations ('testeff' program). + Allows to obtain realistic distributions of the photon incidence angles on the camera plan. + Not to be used for actual simulations. +data: + - type: file + unit: dimensionless + default: None +instrument: + class: Structure + type: + - SSTS + - SCTS +activity: + setting: + - SetParameterFromExternal + validation: + - ValidateParameterByExpert + - ValidateCameraEfficiency +source: + - Initial instrument setup +simulation_software: + - name: sim_telarray From 2a6362907bdc1fa4817e1e17c9565e2d1f5b0554 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 30 Jan 2025 12:38:28 +0100 Subject: [PATCH 123/124] new validation routine --- .../schemas/model_parameter_and_data_schema.metaschema.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml b/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml index fbcaca3a90..ef3ec52a9a 100644 --- a/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml +++ b/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml @@ -322,6 +322,7 @@ definitions: - ValidateArrayElementCoordinates - ValidateAtmosphericModel - ValidateCameraChargeResponse + - ValidateCameraEfficiency - ValidateCameraGainsAndEfficiency - ValidateCameraGeometry - ValidateCameraLinearity From 4e59984fc16cb43b8463ad72d3add21991b97062 Mon Sep 17 00:00:00 2001 From: Gernot Maier Date: Thu, 30 Jan 2025 13:45:58 +0100 Subject: [PATCH 124/124] array coordinate file used in test_array_model export files does not exist anymore --- tests/unit_tests/model/test_array_model.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unit_tests/model/test_array_model.py b/tests/unit_tests/model/test_array_model.py index 77d469d743..65806c4b1e 100644 --- a/tests/unit_tests/model/test_array_model.py +++ b/tests/unit_tests/model/test_array_model.py @@ -73,7 +73,6 @@ def test_exporting_config_files(db_config, io_handler, model_version): "CTA-North-LSTN-01-" + model_version + test_cfg, "CTA-North-MSTN-01-" + model_version + test_cfg, "CTA-test_layout-North-" + model_version + test_cfg, - "array_coordinates_LaPalma_alpha.dat", "NectarCAM_lightguide_efficiency_POP_131019.dat", "Pulse_template_nectarCam_17042020-noshift.dat", "array_triggers.dat",