diff --git a/.env_template b/.env_template index 99fe884107..2a8a55752c 100644 --- a/.env_template +++ b/.env_template @@ -6,7 +6,7 @@ SIMTOOLS_DB_SERVER='cta-simpipe-protodb.zeuthen.desy.de' # MongoDB server SIMTOOLS_DB_API_USER=YOUR_USERNAME # username for MongoDB: ask the responsible person SIMTOOLS_DB_API_PW=YOUR_PASSWORD # Password for MongoDB: ask the responsible person SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin' -SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-LATEST' +SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-LATEST' SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray' # The dashboards to monitor the MongoDB instance are in (accessible only from within DESY) diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml index a56aaaeba9..833f7ed6d3 100644 --- a/.github/workflows/CI-integrationtests.yml +++ b/.github/workflows/CI-integrationtests.yml @@ -7,7 +7,7 @@ env: SIMTOOLS_DB_API_USER: ${{ secrets.DB_API_USER }} SIMTOOLS_DB_API_PW: ${{ secrets.DB_API_PW }} SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }} - SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-Model-LATEST" + SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-ModelParameters-LATEST" SIMTOOLS_SIMTEL_PATH: "/workdir/sim_telarray/" on: diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml index dc01be539d..a73c9593e4 100644 --- a/.github/workflows/CI-unittests.yml +++ b/.github/workflows/CI-unittests.yml @@ -7,7 +7,7 @@ env: SIMTOOLS_DB_API_USER: ${{ secrets.DB_API_USER }} SIMTOOLS_DB_API_PW: ${{ secrets.DB_API_PW }} SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }} - SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-Model-LATEST" + SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-ModelParameters-LATEST" SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }} on: diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml index f9b852980f..dce6b8cbde 100644 --- a/.github/workflows/build-docker-images.yml +++ b/.github/workflows/build-docker-images.yml @@ -89,8 +89,6 @@ jobs: - name: Test Docker image if: matrix.type == 'prod' run: > - # Clone simulation model repo for faster file access - git clone https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters.git # run docker docker run --rm -e "SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'" @@ -98,7 +96,6 @@ jobs: -e "SIMTOOLS_DB_API_USER=${{ env.SIMTOOLS_DB_API_USER }}" -e "SIMTOOLS_DB_API_PW=${{ env.SIMTOOLS_DB_API_PW }}" -e "SIMTOOLS_DB_SERVER=${{ env.SIMTOOLS_DB_SERVER }}" - -e "SIMTOOLS_DB_SIMULATION_MODEL_URL='./model_parameters'" -v "$(pwd):/workdir/external" ${{ env.TEST_TAG }} bash -c "cd /workdir/external && simtools-simulate-prod --config /workdir/external/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid.yml" shell: /usr/bin/bash -e {0} diff --git a/database_scripts/upload_dump_to_local_db.sh b/database_scripts/upload_dump_to_local_db.sh index b13f372dce..840b69ae4e 100755 --- a/database_scripts/upload_dump_to_local_db.sh +++ b/database_scripts/upload_dump_to_local_db.sh @@ -6,8 +6,7 @@ SIMTOOLS_NETWORK="simtools-mongo-network" CONTAINER_NAME="simtools-mongodb" -SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-v0-3-0' # Name of the database to be created -SIMTOOLS_DB_SIMULATION_MODEL='Staging-CTA-Simulation-Model-Derived-Values' +SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-v0-3-0' # Name of the database to be created # Check if podman is available, if not use docker if command -v podman &> /dev/null; then diff --git a/database_scripts/upload_from_model_repository_to_db.sh b/database_scripts/upload_from_model_repository_to_db.sh index fe8f815466..8a9a8213f2 100755 --- a/database_scripts/upload_from_model_repository_to_db.sh +++ b/database_scripts/upload_from_model_repository_to_db.sh @@ -1,10 +1,11 @@ #!/bin/bash # Upload model parameter from repository to a local or remote mongoDB. # +# Execute this scripts from the ./database_scripts directory. # Cover 'source .env': the script ensure that this file exists: # shellcheck disable=SC1091 -DB_SIMULATION_MODEL_URL="https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters.git" +DB_SIMULATION_MODEL_URL="https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/simulation-models.git" DB_SIMULATION_MODEL_BRANCH="main" # Check that this script is not sourced but executed @@ -27,9 +28,9 @@ echo "Cloning model parameters from $DB_SIMULATION_MODEL_URL" rm -rf ./tmp_model_parameters git clone --depth=1 -b $DB_SIMULATION_MODEL_BRANCH $DB_SIMULATION_MODEL_URL ./tmp_model_parameters -CURRENTDIR=$(pwd) +CURRENT_DIR=$(pwd) cd ./tmp_model_parameters/ || exit -cp -f "$CURRENTDIR"/../.env .env +cp -f "$CURRENT_DIR"/../.env .env # ask for confirmation before uploading to remote DB source .env @@ -43,16 +44,20 @@ if [[ $SIMTOOLS_DB_SERVER =~ $regex ]]; then fi fi -# upload files to DB -model_directory="./model_versions/" -for dir in "${model_directory}"*/; do - simtools-db-add-model-parameters-from-repository-to-db \ - --model_version "$(basename "${dir}")" \ - --input_path "${dir}" \ +# upload model parameters to DB +model_directory="./simulation-models/model_parameters/" +simtools-db-add-simulation-model-from-repository-to-db \ + --input_path "${model_directory}" \ --db_name "$DB_SIMULATION_MODEL" \ --type "model_parameters" -done -cd "$CURRENTDIR" || exit +# upload production tables to DB +production_directory="./simulation-models/productions" +simtools-db-add-simulation-model-from-repository-to-db \ + --input_path "${production_directory}" \ + --db_name "$DB_SIMULATION_MODEL" \ + --type "production_tables" + +cd "$CURRENT_DIR" || exit rm -rf ./tmp_model_parameters diff --git a/docs/changes/1316.maintenance.md b/docs/changes/1316.maintenance.md new file mode 100644 index 0000000000..13a992434c --- /dev/null +++ b/docs/changes/1316.maintenance.md @@ -0,0 +1 @@ +Major restructering of database routines plus introduction of new simulation model. diff --git a/docs/source/api-reference/db_handler.md b/docs/source/api-reference/db_handler.md index 6fc06286eb..47b7a52255 100644 --- a/docs/source/api-reference/db_handler.md +++ b/docs/source/api-reference/db_handler.md @@ -13,11 +13,11 @@ Modules for database access. See the databases sections for details. :members: ``` -## db_array_elements +## db_model_upload -(db-array-elements)= +(db-module-upload)= ```{eval-rst} -.. automodule:: db.db_array_elements +.. automodule:: db.db_model_upload :members: ``` diff --git a/docs/source/user-guide/applications.md b/docs/source/user-guide/applications.md index acfa02c193..056ff10809 100644 --- a/docs/source/user-guide/applications.md +++ b/docs/source/user-guide/applications.md @@ -28,7 +28,7 @@ simtools-convert-all-model-parameters-from-simtel simtools-convert-model-parameter-from-simtel simtools-db-add-file-to-db -simtools-db-add-model-parameters-from-repository-to-db +simtools-db-add-simulation-model-from-repository-to-db simtools-db-add-value-from-json-to-db simtools-db-get-array-layouts-from-db simtools-db-get-file-from-db diff --git a/docs/source/user-guide/applications/simtools-db-add-model-parameters-from-repository-to-db.rst b/docs/source/user-guide/applications/simtools-db-add-model-parameters-from-repository-to-db.rst deleted file mode 100644 index 6fd7aa995d..0000000000 --- a/docs/source/user-guide/applications/simtools-db-add-model-parameters-from-repository-to-db.rst +++ /dev/null @@ -1,6 +0,0 @@ - -simtools-db-add-model-parameters-from-repository-to-db -====================================================== - -.. automodule:: db_add_model_parameters_from_repository_to_db - :members: diff --git a/docs/source/user-guide/applications/simtools-db-add-simulation-model-from-repository-to-db.rst b/docs/source/user-guide/applications/simtools-db-add-simulation-model-from-repository-to-db.rst new file mode 100644 index 0000000000..7049214841 --- /dev/null +++ b/docs/source/user-guide/applications/simtools-db-add-simulation-model-from-repository-to-db.rst @@ -0,0 +1,6 @@ + +simtools-db-add-simulation-model-from-repository-to-db +====================================================== + +.. automodule:: db_add_simulation_model_from_repository_to_db + :members: diff --git a/docs/source/user-guide/databases.md b/docs/source/user-guide/databases.md index 9f3d1b50c7..308c57b9d8 100644 --- a/docs/source/user-guide/databases.md +++ b/docs/source/user-guide/databases.md @@ -1,6 +1,6 @@ # Databases -The simtools package uses a prototype MongoDB database to store the simulation model parameters and derived data products. +The simtools package uses a prototype MongoDB database to store the simulation model parameters. Access to the DB is handled via a dedicated API module ([db_handler](#dbhandler)). Simulation model parameters are stored in databases (see the [Simulation Model](model_parameters.md#simulation-model) section) and synced with the [CTAO model parameter repository](https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters). @@ -16,7 +16,7 @@ This documentation is therefore incomplete. ### Model parameters DB -The name of the model parameter database needs to be indicated by `$SIMTOOLS_DB_SIMULATION_MODEL` environmental variable and defined e.g., in the `.env` file. Use `CTAO-Simulation-Model-LATEST` to use the latest version of the CTAO simulation model database (simtools will replace `LATEST` with the latest version number). +The name of the model parameter database needs to be indicated by `$SIMTOOLS_DB_SIMULATION_MODEL` environmental variable and defined e.g., in the `.env` file. Use `CTAO-Simulation-ModelParameters-LATEST` to use the latest version of the CTAO simulation model database (simtools will replace `LATEST` with the latest version number). Collections: @@ -27,15 +27,6 @@ Collections: * `metadata` containing tables describing the model versions * `fs.files` with all file type entries for the model parameters (e.g., the quantum-efficiency tables) -### Derived values DB - -Database with derived values DB (e.g., `Staging-CTA-Simulation-Model-Derived-Values` defined in `db_handler.DB_DERIVED_VALUES`). - -Collections are: - -* `derived_values` with the derived values for each telescope or site -* `fs.files` with file type derived results - ### Other databases All other currently available databases are not in use and kept for historical reasons. @@ -52,8 +43,7 @@ SIMTOOLS_DB_SERVER='cta-simpipe-protodb.zeuthen.desy.de' # MongoDB server SIMTOOLS_DB_API_USER=YOUR_USERNAME # username for MongoDB: ask the responsible person SIMTOOLS_DB_API_PW=YOUR_PASSWORD # Password for MongoDB: ask the responsible person SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin' -SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-LATEST' -# SIMTOOLS_DB_SIMULATION_MODEL_URL='' +SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-LATEST' SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray' ``` @@ -75,7 +65,7 @@ The following applications are important: * update or define a single model parameter from a json file (as defined in the model parameter repository): [db_add_value_from_json_to_db.py](db_add_value_from_json_to_db) * upload a model parameter file: [db_add_file_to_db.py](db_add_file_to_db) -* upload all model parameters and files from the model parameter repository: [db_add_model_parameters_from_repository_to_db.py](db_add_model_parameters_from_repository_to_db) +* upload all model parameters and files from the model parameter repository: [db_add_simulation_model_from_repository_to_db.py](db_add_simulation_model_from_repository_to_db) ## Configure and use a local copy of the model parameter database @@ -138,7 +128,7 @@ SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin' SIMTOOLS_DB_SIMULATION_MODEL='STAGING-CTA-Simulation-Model-LATEST' ``` -`SIMTOOLS_DB_SIMULATION_MODEL` is set as an example here to `STAGING-CTAO-Simulation-Model-LATEST` and should be changed accordingly. +`SIMTOOLS_DB_SIMULATION_MODEL` is set as an example here to `STAGING-CTAO-Simulation-ModelParameters-LATEST` and should be changed accordingly. For using simtools inside a container: @@ -158,6 +148,6 @@ SIMTOOLS_DB_SERVER='simtools-mongodb' SIMTOOLS_DB_API_USER='api' # username for MongoDB SIMTOOLS_DB_API_PW='password' # Password for MongoDB SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin' -SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-LATEST' +SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-LATEST' SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray' ``` diff --git a/docs/source/user-guide/model_parameters.md b/docs/source/user-guide/model_parameters.md index 3dd4d6cbd8..65425be19b 100644 --- a/docs/source/user-guide/model_parameters.md +++ b/docs/source/user-guide/model_parameters.md @@ -189,14 +189,15 @@ A typical model parameter file looks like: ```json { + "schema_version": "0.1.0", "parameter": "num_gains", "instrument": "LSTN-01", "site": "North", - "version": "6.0.0", + "parameter_version": "6.0.0", + "unique_id": null, "value": 2, "unit": null, "type": "int64", - "applicable": true, "file": false } ``` diff --git a/pyproject.toml b/pyproject.toml index 5addac82bb..920d7e44eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -73,7 +73,7 @@ scripts.simtools-convert-all-model-parameters-from-simtel = "simtools.applicatio scripts.simtools-convert-geo-coordinates-of-array-elements = "simtools.applications.convert_geo_coordinates_of_array_elements:main" scripts.simtools-convert-model-parameter-from-simtel = "simtools.applications.convert_model_parameter_from_simtel:main" scripts.simtools-db-add-file-to-db = "simtools.applications.db_add_file_to_db:main" -scripts.simtools-db-add-model-parameters-from-repository-to-db = "simtools.applications.db_add_model_parameters_from_repository_to_db:main" +scripts.simtools-db-add-simulation-model-from-repository-to-db = "simtools.applications.db_add_simulation_model_from_repository_to_db:main" scripts.simtools-db-add-value-from-json-to-db = "simtools.applications.db_add_value_from_json_to_db:main" scripts.simtools-db-get-array-layouts-from-db = "simtools.applications.db_get_array_layouts_from_db:main" scripts.simtools-db-get-file-from-db = "simtools.applications.db_get_file_from_db:main" diff --git a/src/simtools/applications/convert_all_model_parameters_from_simtel.py b/src/simtools/applications/convert_all_model_parameters_from_simtel.py index 29ce439947..7c0624eec2 100644 --- a/src/simtools/applications/convert_all_model_parameters_from_simtel.py +++ b/src/simtools/applications/convert_all_model_parameters_from_simtel.py @@ -6,6 +6,8 @@ ready to be submitted to the model database. Prints out parameters which are not found in simtel configuration file and parameters which are not found in simtools schema files. + Note that all parameters are assigned the same parameter version. + Command line arguments ---------------------- simtel_cfg_file (str) @@ -30,7 +32,7 @@ --simtel_cfg_file all_telescope_config_la_palma.cfg\\ --simtel_telescope_name CT1\\ --telescope LSTN-01\\ - --model_version "2024-03-06" + --parameter_version "1.0.0" The export of the model parameters from sim_telarray for 6.0.0 can be done e.g., as follows: @@ -103,7 +105,7 @@ def _parse(label=None, description=None): type=str, required=True, ) - return config.initialize(simulation_model=["telescope", "model_version"]) + return config.initialize(simulation_model=["telescope", "parameter_version"]) def get_list_of_parameters_and_schema_files(schema_directory): @@ -219,7 +221,6 @@ def read_and_export_parameters(args_dict, logger): """ Read and export parameters from simtel configuration file to json files. - Only applicable parameters are exported to json. Provide extensive logging information on the parameters found in the simtel configuration file. @@ -266,7 +267,7 @@ def read_and_export_parameters(args_dict, logger): parameter_name=_parameter, value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]), instrument=args_dict["telescope"], - model_version=args_dict["model_version"], + parameter_version=args_dict["parameter_version"], output_file=io_handler.get_output_file(f"{_parameter}.json"), ) diff --git a/src/simtools/applications/convert_geo_coordinates_of_array_elements.py b/src/simtools/applications/convert_geo_coordinates_of_array_elements.py index f103ba6140..cc246b7955 100644 --- a/src/simtools/applications/convert_geo_coordinates_of_array_elements.py +++ b/src/simtools/applications/convert_geo_coordinates_of_array_elements.py @@ -133,7 +133,7 @@ def _parse(label=None, description=None): output=True, require_command_line=True, db_config=True, - simulation_model=["model_version", "site"], + simulation_model=["model_version", "parameter_version", "site"], ) @@ -170,7 +170,9 @@ def main(): if args_dict["export"] is not None: product_data = ( - layout.export_one_telescope_as_json(crs_name=args_dict["export"]) + layout.export_one_telescope_as_json( + crs_name=args_dict["export"], parameter_version=args_dict.get("parameter_version") + ) if args_dict.get("input", "").endswith(".json") else layout.export_telescope_list_table(crs_name=args_dict["export"]) ) diff --git a/src/simtools/applications/convert_model_parameter_from_simtel.py b/src/simtools/applications/convert_model_parameter_from_simtel.py index 966b0260a5..ec6e064a03 100644 --- a/src/simtools/applications/convert_model_parameter_from_simtel.py +++ b/src/simtools/applications/convert_model_parameter_from_simtel.py @@ -78,7 +78,7 @@ def _parse(label=None, description=None): type=str, required=True, ) - return config.initialize(simulation_model=["telescope", "model_version"], output=True) + return config.initialize(simulation_model=["telescope", "parameter_version"], output=True) def main(): # noqa: D103 @@ -107,7 +107,7 @@ def main(): # noqa: D103 parameter_name=simtel_config_reader.parameter_name, value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]), instrument=args_dict["telescope"], - model_version=args_dict["model_version"], + parameter_version=args_dict["parameter_version"], output_file=args_dict["output_file"], output_path=args_dict.get("output_path"), use_plain_output_path=args_dict.get("use_plain_output_path"), diff --git a/src/simtools/applications/db_add_file_to_db.py b/src/simtools/applications/db_add_file_to_db.py index 3483bf6225..2efb77c21c 100644 --- a/src/simtools/applications/db_add_file_to_db.py +++ b/src/simtools/applications/db_add_file_to_db.py @@ -18,8 +18,7 @@ A directory with files to upload to the DB. \ All files in the directory with a predefined list of extensions will be uploaded. db (str) - The DB to insert the files to. \ - The choices are either the default CTA simulation DB or a sandbox for testing. + The DB to insert the files to. Example ------- diff --git a/src/simtools/applications/db_add_model_parameters_from_repository_to_db.py b/src/simtools/applications/db_add_model_parameters_from_repository_to_db.py deleted file mode 100644 index 715a6f0e19..0000000000 --- a/src/simtools/applications/db_add_model_parameters_from_repository_to_db.py +++ /dev/null @@ -1,176 +0,0 @@ -#!/usr/bin/python3 -r""" - Add parameters found in a model parameter repository to a new database. - - Generates a new database with all required collections. - Follows the structure of the CTAO gitlab model parameters repository. - file as input. - - This is an application for experts and should not be used by the general user. - - Command line arguments - - input_path (str, required) - Path of local copy of model parameter repository. - db_name (str, required) - Name of new DB to be created. - type (str, optional) - Type of data to be uploaded to the DB. Options are: model_parameters - - Examples - -------- - Upload model data repository to the DB: - - .. code-block:: console - - simtools-db-add_model-parameters-from-repository-to-db \ - --input_path /path/to/repository \ - --db_name new_db_name \ - --type model_parameters -""" - -import logging -from pathlib import Path - -import simtools.utils.general as gen -from simtools.configuration import configurator -from simtools.db import db_handler -from simtools.utils import names - - -def _parse(label=None, description=None): - """ - Parse command line configuration. - - Parameters - ---------- - label : str - Label describing application. - description : str - Description of application. - - Returns - ------- - CommandLineParser - Command line parser object. - """ - config = configurator.Configurator(label=label, description=description) - config.parser.add_argument( - "--input_path", - help="Path to model parameter repository.", - type=Path, - required=True, - ) - config.parser.add_argument( - "--db_name", - help="Name of the new model parameter database to be created.", - type=str.strip, - required=True, - ) - config.parser.add_argument( - "--type", - help="Type of data to be uploaded to the database.", - type=str, - required=False, - default="model_parameters", - choices=["model_parameters"], - ) - - args_dict, db_config = config.initialize( - output=True, require_command_line=True, db_config=True, simulation_model="version" - ) - db_config["db_simulation_model"] = args_dict["db_name"] # overwrite explicitly DB configuration - return args_dict, db_config - - -def add_values_from_json_to_db(file, collection, db, db_name, file_prefix, logger): - """ - Upload data from json files to db. - - Parameters - ---------- - file : list - Json file to be uploaded to the DB. - collection : str - The DB collection to which to add the file. - db : DatabaseHandler - Database handler object. - db_name : str - Name of the database to be created. - file_prefix : str - Path to location of all additional files to be uploaded. - logger : logging.Logger - Logger object. - """ - par_dict = gen.collect_data_from_file(file_name=file) - logger.info( - f"Adding the following parameter to the DB: {par_dict['parameter']} " - f"(collection {collection} in database {db_name})" - ) - db.add_new_parameter( - db_name=db_name, - par_dict=par_dict, - collection_name=collection, - file_prefix=file_prefix, - ) - - -def _add_model_parameters_to_db(args_dict, db, logger): - """ - Add model parameters to the DB. - - Parameters - ---------- - args_dict : dict - Command line arguments. - db : DatabaseHandler - Database handler object. - logger : logging.Logger - Logger object. - - """ - input_path = Path(args_dict["input_path"]) - array_elements = [d for d in input_path.iterdir() if d.is_dir()] - for element in array_elements: - try: - collection = names.get_collection_name_from_array_element_name(element.name) - except ValueError: - if element.name.startswith("OBS"): - collection = "sites" - elif element.name in {"configuration_sim_telarray", "configuration_corsika"}: - collection = element.name - elif element.name == "Files": - logger.info("Files are uploaded with the corresponding model parameters") - continue - logger.info(f"Reading model parameters for {element.name} into collection {collection}") - files_to_insert = list(Path(element).rglob("*json")) - for file in files_to_insert: - if collection == "files": - logger.info("Not yet implemented files") - else: - add_values_from_json_to_db( - file=file, - collection=collection, - db=db, - db_name=args_dict["db_name"], - file_prefix=input_path / "Files", - logger=logger, - ) - - -def main(): - """Application main.""" - label = Path(__file__).stem - args_dict, db_config = _parse( - label, description="Add or update a model parameter database to the DB" - ) - logger = logging.getLogger() - logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) - - db = db_handler.DatabaseHandler(mongo_db_config=db_config) - - _add_model_parameters_to_db(args_dict, db, logger) - - -if __name__ == "__main__": - main() diff --git a/src/simtools/applications/db_add_simulation_model_from_repository_to_db.py b/src/simtools/applications/db_add_simulation_model_from_repository_to_db.py new file mode 100644 index 0000000000..0da627be0d --- /dev/null +++ b/src/simtools/applications/db_add_simulation_model_from_repository_to_db.py @@ -0,0 +1,110 @@ +#!/usr/bin/python3 +r""" + Add parameters and production tables from a simulation model repository to a new database. + + Generates a new database with all required collections. + Follows the structure of the CTAO gitlab simulation model repository. + + This is an application for experts and should not be used by the general user. + + Command line arguments + + input_path (str, required) + Path of local copy of model parameter repository. + db_name (str, required) + Name of new DB to be created. + type (str, optional) + Type of data to be uploaded to the DB. Options are: model_parameters, production_tables. + + Examples + -------- + Upload model data repository to the DB: + + .. code-block:: console + + simtools-db-simulation-model-from-repository-to-db \ + --input_path /path/to/repository \ + --db_name new_db_name \ + --type model_parameters + + Upload production tables to the DB: + + .. code-block:: console + + simtools-db-simulation-model-from-repository-to-db \ + --input_path /path/to/repository \ + --db_name new_db_name \ + --type production_tables + +""" + +import logging +from pathlib import Path + +import simtools.utils.general as gen +from simtools.configuration import configurator +from simtools.db import db_handler, db_model_upload + + +def _parse(label=None, description=None): + """ + Parse command line configuration. + + Parameters + ---------- + label : str + Label describing application. + description : str + Description of application. + + Returns + ------- + CommandLineParser + Command line parser object. + """ + config = configurator.Configurator(label=label, description=description) + config.parser.add_argument( + "--input_path", + help="Path to simulation model repository.", + type=Path, + required=True, + ) + config.parser.add_argument( + "--db_name", + help="Name of the new simulation model database to be created.", + type=str.strip, + required=True, + ) + config.parser.add_argument( + "--type", + help="Type of data to be uploaded to the database.", + type=str, + required=False, + default="model_parameters", + choices=["model_parameters", "production_tables"], + ) + + args_dict, db_config = config.initialize(output=True, require_command_line=True, db_config=True) + db_config["db_simulation_model"] = args_dict["db_name"] # overwrite explicitly DB configuration + return args_dict, db_config + + +def main(): + """Application main.""" + label = Path(__file__).stem + args_dict, db_config = _parse( + label, description="Add or update a model parameter database to the DB" + ) + logger = logging.getLogger() + logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) + + db = db_handler.DatabaseHandler(mongo_db_config=db_config) + + if args_dict.get("type") == "model_parameters": + db_model_upload.add_model_parameters_to_db(args_dict, db) + elif args_dict.get("type") == "production_tables": + db_model_upload.add_production_tables_to_db(args_dict, db) + + +if __name__ == "__main__": + main() diff --git a/src/simtools/applications/db_add_value_from_json_to_db.py b/src/simtools/applications/db_add_value_from_json_to_db.py index 93ee5317be..2046f1eec5 100644 --- a/src/simtools/applications/db_add_value_from_json_to_db.py +++ b/src/simtools/applications/db_add_value_from_json_to_db.py @@ -10,8 +10,7 @@ db_collection (str, required) The DB collection to which to add the file. db (str) - The DB to insert the files to. \ - The choices are either the default CTA simulation DB or a sandbox for testing. + The DB to insert the files to. Example ------- diff --git a/src/simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py b/src/simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py index 1ea65ee87b..1dc4aa3670 100644 --- a/src/simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py +++ b/src/simtools/applications/db_development_tools/write_array_elements_positions_to_repository.py @@ -11,8 +11,8 @@ File containing a table of array element positions. repository_path : str Path of local copy of model parameter repository. - model_version : str - Model version. + parameter_version : str + Parameter version. site : str Observatory site. coordinate_system : str @@ -27,7 +27,7 @@ simtools-write-array-element-positions-to-repository \ --input /path/to/positions.txt \ --repository_path /path/to/repository \ - --model_version 1.0.0 \ + --parameter_version 0.1.0 \ --coordinate_system ground \ --site North @@ -82,7 +82,7 @@ def _parse(label=None, description=None): choices=["ground", "utm"], ) - return config.initialize(db_config=True, simulation_model="site") + return config.initialize(db_config=True, simulation_model=["site", "parameter_version"]) def write_utm_array_elements_to_repository(args_dict, logger): @@ -115,7 +115,7 @@ def write_utm_array_elements_to_repository(args_dict, logger): parameter_name="array_element_position_utm", instrument=instrument, value=f"{row['utm_east']} {row['utm_north']} {row['altitude']}", - model_version=args_dict["model_version"], + parameter_version=args_dict["parameter_version"], output_path=output_path, output_file="array_element_position_utm.json", ) @@ -137,7 +137,7 @@ def write_ground_array_elements_to_repository(args_dict, db_config, logger): """ array_model = ArrayModel( mongo_db_config=db_config, - model_version=args_dict["model_version"], + model_version=None, site=args_dict["site"], array_elements=args_dict["input"], ) diff --git a/src/simtools/applications/db_get_file_from_db.py b/src/simtools/applications/db_get_file_from_db.py index f1796d6924..8510091d72 100644 --- a/src/simtools/applications/db_get_file_from_db.py +++ b/src/simtools/applications/db_get_file_from_db.py @@ -49,8 +49,9 @@ def _parse(): config.parser.add_argument( "--file_name", - help="The name of the file to be downloaded.", + help="The name of the file(s) to be downloaded (single file or space-separated list).", type=str, + nargs="+", required=True, ) return config.initialize(db_config=True, output=True) @@ -66,15 +67,14 @@ def main(): # noqa: D103 db = db_handler.DatabaseHandler(mongo_db_config=db_config) available_dbs = [ db_config["db_simulation_model"], - db.DB_CTA_SIMULATION_MODEL_DESCRIPTIONS, - db.DB_DERIVED_VALUES, - "sandbox", ] - file_id = None + file_id = {} for db_name in available_dbs: try: - file_id = db.export_file_db( - db_name, _io_handler.get_output_directory(), args_dict["file_name"] + file_id = db.export_model_files( + db_name=db_name, + dest=_io_handler.get_output_directory(), + file_names=args_dict["file_name"], ) logger.info( f"Got file {args_dict['file_name']} from DB {db_name} " @@ -84,11 +84,10 @@ def main(): # noqa: D103 except FileNotFoundError: continue - if file_id is None: - logger.error( - f"The file {args_dict['file_name']} was not found in any of the available DBs." - ) - raise FileNotFoundError + for key, value in file_id.items(): + if value is None: + logger.error(f"The file {key} was not found in any of the available DBs.") + raise FileNotFoundError if __name__ == "__main__": diff --git a/src/simtools/applications/db_get_parameter_from_db.py b/src/simtools/applications/db_get_parameter_from_db.py index 812c487e37..3b14c9df90 100644 --- a/src/simtools/applications/db_get_parameter_from_db.py +++ b/src/simtools/applications/db_get_parameter_from_db.py @@ -26,7 +26,7 @@ Example ------- - Get the mirror_list parameter from the DB. + Get the mirror_list parameter used for a given model_version from the DB. .. code-block:: console @@ -34,18 +34,13 @@ --site North --telescope LSTN-01 \\ --model_version 5.0.0 - Expected final print-out message: + Get the mirror_list parameter using the parameter_version from the DB. .. code-block:: console - {'Applicable': True, - 'File': True, - 'Type': 'str', - 'Value': 'mirror_CTA-N-LST1_v2019-03-31.dat', - 'Version': '5.0.0', - '_id': ObjectId('608834f257df2db2531b8e78'), - 'entry_date': datetime.datetime(2021, 4, 27, 15, 59, 46, tzinfo=)} + simtools-db-get-parameter-from-db --parameter mirror_list \\ + --site North --telescope LSTN-01 \\ + --parameter_version 1.0.0 """ @@ -73,7 +68,6 @@ def _parse(): config.parser.add_argument( "--db_collection", help="DB collection to which to add the file", - default="telescopes", required=False, ) config.parser.add_argument( @@ -83,7 +77,9 @@ def _parse(): required=False, ) - return config.initialize(db_config=True, simulation_model=["telescope", "model_version"]) + return config.initialize( + db_config=True, simulation_model=["telescope", "parameter_version", "model_version"] + ) def main(): # noqa: D103 @@ -94,37 +90,53 @@ def main(): # noqa: D103 db = db_handler.DatabaseHandler(mongo_db_config=db_config) - if args_dict["db_collection"] == "configuration_sim_telarray": - pars = db.get_model_parameters( - args_dict["site"], - args_dict["telescope"], - args_dict["model_version"], - collection="configuration_sim_telarray", - ) - elif args_dict["db_collection"] == "configuration_corsika": - pars = db.get_corsika_configuration_parameters(args_dict["model_version"]) - elif args_dict["telescope"] is not None: - pars = db.get_model_parameters( - args_dict["site"], - args_dict["telescope"], - args_dict["model_version"], - collection="telescopes", + # get parameter using 'parameter_version' + if args_dict["parameter_version"] is not None: + pars = db.get_model_parameter( + parameter=args_dict["parameter"], + parameter_version=args_dict["parameter_version"], + site=args_dict["site"], + array_element_name=args_dict["telescope"], + collection=( + args_dict["db_collection"] if args_dict.get("db_collection") else "telescopes" + ), ) + # get parameter using 'model_version' + elif args_dict["model_version"] is not None: + if args_dict["telescope"]: + pars = db.get_model_parameters( + site=args_dict["site"], + array_element_name=args_dict["telescope"], + model_version=args_dict["model_version"], + collection=( + "configuration_sim_telarray" + if args_dict.get("db_collection") == "configuration_sim_telarray" + else "telescopes" + ), + ) + else: + pars = db.get_model_parameters( + site=args_dict.get("site"), + model_version=args_dict["model_version"], + collection=( + args_dict["db_collection"] if args_dict.get("db_collection") else "sites" + ), + array_element_name=None, + ) else: - pars = db.get_site_parameters(args_dict["site"], args_dict["model_version"]) + raise ValueError("Either 'parameter_version' or 'model_version' must be provided.") if args_dict["parameter"] not in pars: raise KeyError(f"The requested parameter, {args_dict['parameter']}, does not exist.") if args_dict["output_file"] is not None: - _io_handler = io_handler.IOHandler() + _output_file = ( + Path(io_handler.IOHandler().get_output_directory()) / args_dict["output_file"] + ) pars[args_dict["parameter"]].pop("_id") pars[args_dict["parameter"]].pop("entry_date") - _output_file = Path(_io_handler.get_output_directory()) / args_dict["output_file"] with open(_output_file, "w", encoding="utf-8") as json_file: json.dump(pars[args_dict["parameter"]], json_file, indent=4) else: - print() pprint(pars[args_dict["parameter"]]) - print() if __name__ == "__main__": diff --git a/src/simtools/applications/submit_model_parameter_from_external.py b/src/simtools/applications/submit_model_parameter_from_external.py index c0874998b8..0c89c8e636 100644 --- a/src/simtools/applications/submit_model_parameter_from_external.py +++ b/src/simtools/applications/submit_model_parameter_from_external.py @@ -16,8 +16,8 @@ instrument name. site (str) site location. - model_version (str) - Model version. + parameter_version (str) + Parameter version. input_meta (str, optional) input meta data file (yml format) @@ -33,7 +33,7 @@ --value 2 \\ --instrument LSTN-design \\ --site North \\ - --model_version 6.0.0 \\ + --parameter_version 0.1.0 \\ --input_meta num_gains.metadata.yml """ @@ -70,7 +70,9 @@ def _parse(label, description): ) config.parser.add_argument("--instrument", type=str, required=True, help="Instrument name") config.parser.add_argument("--site", type=str, required=True, help="Site location") - config.parser.add_argument("--model_version", type=str, required=True, help="Model version") + config.parser.add_argument( + "--parameter_version", type=str, required=True, help="Parameter version" + ) config.parser.add_argument( "--value", @@ -102,7 +104,7 @@ def main(): # noqa: D103 logger.setLevel(gen.get_log_level_from_user(args_dict["log_level"])) output_path = ( - Path(args_dict["output_path"]) / args_dict["model_version"] / args_dict["instrument"] + Path(args_dict["output_path"]) / args_dict["parameter_version"] / args_dict["instrument"] if args_dict.get("output_path") else None ) @@ -110,7 +112,7 @@ def main(): # noqa: D103 parameter_name=args_dict["parameter"], value=args_dict["value"], instrument=args_dict["instrument"], - model_version=args_dict["model_version"], + parameter_version=args_dict["parameter_version"], output_file=Path(args_dict["parameter"]).with_suffix(".json"), output_path=output_path, use_plain_output_path=args_dict.get("use_plain_output_path"), diff --git a/src/simtools/applications/validate_file_using_schema.py b/src/simtools/applications/validate_file_using_schema.py index a4ba71f4cb..1ea17ca5ea 100644 --- a/src/simtools/applications/validate_file_using_schema.py +++ b/src/simtools/applications/validate_file_using_schema.py @@ -116,6 +116,19 @@ def _get_schema_file_name(args_dict, data_dict=None): return schema_file +def _get_json_file_list(file_directory=None, file_name=None): + """Return list of json files in a directory.""" + file_list = [] + if file_directory is not None: + file_list = list(Path(file_directory).rglob("*.json")) + if not file_list: + raise FileNotFoundError(f"No files found in {file_directory}") + elif file_name is not None: + file_list = [file_name] + + return file_list + + def validate_schema(args_dict, logger): """ Validate a schema file (or several files) given in yaml or json format. @@ -124,11 +137,9 @@ def validate_schema(args_dict, logger): the metadata section of the data dictionary. """ - if args_dict.get("file_directory") is not None: - file_list = list(Path(args_dict["file_directory"]).rglob("*.json")) - else: - file_list = [args_dict["file_name"]] - for file_name in file_list: + for file_name in _get_json_file_list( + args_dict.get("file_directory"), args_dict.get("file_name") + ): try: data = gen.collect_data_from_file(file_name=file_name) except FileNotFoundError as exc: @@ -144,10 +155,9 @@ def validate_schema(args_dict, logger): def validate_data_files(args_dict, logger): """Validate data files.""" - file_directory = args_dict.get("file_directory") - if file_directory is not None: + if args_dict.get("file_directory") is not None: tmp_args_dict = {} - for file_name in Path(file_directory).rglob("*.json"): + for file_name in _get_json_file_list(args_dict.get("file_directory")): tmp_args_dict["file_name"] = file_name parameter_name = re.sub(r"-\d+\.\d+\.\d+", "", file_name.stem) schema_file = MODEL_PARAMETER_SCHEMA_PATH / f"{parameter_name}.schema.yml" diff --git a/src/simtools/configuration/commandline_parser.py b/src/simtools/configuration/commandline_parser.py index 449f5d09a2..4e5b26945d 100644 --- a/src/simtools/configuration/commandline_parser.py +++ b/src/simtools/configuration/commandline_parser.py @@ -239,7 +239,14 @@ def initialize_simulation_model_arguments(self, model_options): if "model_version" in model_options: _job_group.add_argument( "--model_version", - help="model version", + help="production model version", + type=str, + default=None, + ) + if "parameter_version" in model_options: + _job_group.add_argument( + "--parameter_version", + help="model parameter version", type=str, default=None, ) diff --git a/src/simtools/corsika/corsika_config.py b/src/simtools/corsika/corsika_config.py index 718b809a95..0cad485967 100644 --- a/src/simtools/corsika/corsika_config.py +++ b/src/simtools/corsika/corsika_config.py @@ -6,7 +6,6 @@ import numpy as np from astropy import units as u -import simtools.utils.general as gen from simtools.corsika.primary_particle import PrimaryParticle from simtools.io_operations import io_handler from simtools.model.model_parameter import ModelParameter @@ -111,8 +110,6 @@ def fill_corsika_configuration(self, args_dict, db_config=None): if args_dict is None: return {} - self._logger.debug("Setting CORSIKA parameters ") - self._is_file_updated = False self.azimuth_angle = int(args_dict["azimuth_angle"].to("deg").value) self.zenith_angle = args_dict["zenith_angle"].to("deg").value @@ -243,7 +240,7 @@ def _input_config_corsika_starting_grammage(self, entry): def _input_config_corsika_particle_kinetic_energy_cutoff(self, entry): """Return ECUTS parameter CORSIKA format.""" - e_cuts = gen.convert_string_to_list(entry["value"]) + e_cuts = entry["value"] return [ f"{e_cuts[0]*u.Unit(entry['unit']).to('GeV')} " f"{e_cuts[1]*u.Unit(entry['unit']).to('GeV')} " @@ -280,7 +277,7 @@ def _corsika_configuration_cherenkov_parameters(self, parameters_from_db): def _input_config_corsika_cherenkov_wavelength(self, entry): """Return CWAVLG parameter CORSIKA format.""" - wavelength_range = gen.convert_string_to_list(entry["value"]) + wavelength_range = entry["value"] return [ f"{wavelength_range[0]*u.Unit(entry['unit']).to('nm')}", f"{wavelength_range[1]*u.Unit(entry['unit']).to('nm')}", @@ -318,8 +315,12 @@ def _corsika_configuration_debugging_parameters(self): } def _input_config_io_buff(self, entry): - """Return IO_BUFFER parameter CORSIKA format.""" - return f"{entry['value']}{entry['unit']}" + """Return IO_BUFFER parameter CORSIKA format (Byte or MB required).""" + value = entry["value"] * u.Unit(entry["unit"]).to("Mbyte") + # check if value is integer-like + if value.is_integer(): + return f"{int(value)}MB" + return f"{int(entry['value'] * u.Unit(entry['unit']).to('byte'))}" def _rotate_azimuth_by_180deg(self, az, correct_for_geomagnetic_field_alignment=True): """ diff --git a/src/simtools/data_model/model_data_writer.py b/src/simtools/data_model/model_data_writer.py index f29352b01e..507e348119 100644 --- a/src/simtools/data_model/model_data_writer.py +++ b/src/simtools/data_model/model_data_writer.py @@ -122,7 +122,7 @@ def dump_model_parameter( parameter_name, value, instrument, - model_version, + parameter_version, output_file, output_path=None, use_plain_output_path=False, @@ -139,8 +139,8 @@ def dump_model_parameter( Value of the parameter. instrument: str Name of the instrument. - model_version: str - Version of the model. + parameter_version: str + Version of the parameter. output_file: str Name of output file. output_path: str or Path @@ -163,7 +163,7 @@ def dump_model_parameter( use_plain_output_path=use_plain_output_path, ) _json_dict = writer.get_validated_parameter_dict( - parameter_name, value, instrument, model_version + parameter_name, value, instrument, parameter_version ) writer.write_dict_to_model_parameter_json(output_file, _json_dict) if metadata_input_dict is not None: @@ -175,7 +175,9 @@ def dump_model_parameter( ) return _json_dict - def get_validated_parameter_dict(self, parameter_name, value, instrument, model_version): + def get_validated_parameter_dict( + self, parameter_name, value, instrument, parameter_version, schema_version="0.1.0" + ): """ Get validated parameter dictionary. @@ -187,8 +189,10 @@ def get_validated_parameter_dict(self, parameter_name, value, instrument, model_ Value of the parameter. instrument: str Name of the instrument. - model_version: str - Version of the model. + parameter_version: str + Version of the parameter. + schema_version: str + Version of the schema. Returns ------- @@ -203,22 +207,18 @@ def get_validated_parameter_dict(self, parameter_name, value, instrument, model_ except ValueError: # e.g. instrument is 'LSTN-01' site = names.get_site_from_array_element_name(instrument) - try: - applicable = self._get_parameter_applicability(instrument) - except ValueError: - applicable = True # Default to True (expect that this field goes in future) - value, unit = value_conversion.split_value_and_unit(value) data_dict = { + "schema_version": schema_version, "parameter": parameter_name, "instrument": instrument, "site": site, - "version": model_version, + "parameter_version": parameter_version, + "unique_id": None, "value": value, "unit": unit, "type": self._get_parameter_type(), - "applicable": applicable, "file": self._parameter_is_a_file(), } return self.validate_and_transform( @@ -273,36 +273,6 @@ def _parameter_is_a_file(self): pass return False - def _get_parameter_applicability(self, telescope_name): - """ - Check if a parameter is applicable for a given telescope using schema files. - - First check for exact telescope name (e.g., LSTN-01), if not listed in the schema - use telescope type (LSTN). - - Parameters - ---------- - telescope_name: str - Telescope name (e.g., LSTN-01) - - Returns - ------- - bool - True if parameter is applicable to telescope. - - """ - try: - if telescope_name in self.schema_dict["instrument"]["type"]: - return True - except KeyError as exc: - self._logger.error("Schema file does not contain 'instrument:type' key.") - raise exc - - return ( - names.get_array_element_type_from_name(telescope_name) - in self.schema_dict["instrument"]["type"] - ) - def _get_unit_from_schema(self): """ Return unit(s) from schema dict. diff --git a/src/simtools/data_model/validate_data.py b/src/simtools/data_model/validate_data.py index c27cae37ab..0c20dd6532 100644 --- a/src/simtools/data_model/validate_data.py +++ b/src/simtools/data_model/validate_data.py @@ -129,7 +129,7 @@ def validate_model_parameter(par_dict): Validated data dictionary """ data_validator = DataValidator( - schema_file=MODEL_PARAMETER_SCHEMA_PATH / "f{par_dict['parameter']}.schema.yml", + schema_file=MODEL_PARAMETER_SCHEMA_PATH / f"{par_dict['parameter']}.schema.yml", data_dict=par_dict, check_exact_data_type=False, ) diff --git a/src/simtools/db/db_array_elements.py b/src/simtools/db/db_array_elements.py deleted file mode 100644 index 4a0c2eff06..0000000000 --- a/src/simtools/db/db_array_elements.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Retrieval of array elements from the database.""" - -from functools import lru_cache - -from pymongo import ASCENDING - -from simtools.utils import names - - -@lru_cache -def get_array_elements(db_collection, model_version): - """ - Get all array element names and their design model for a given DB collection. - - Uses the 'design_model' parameter to determine the design model of the array element. - Assumes that a design model is defined for every array element. - - Parameters - ---------- - db_collection: - pymongo.collection.Collection - model_version: str - Model version. - - Returns - ------- - dict - Dict with array element names found and their design model - - Raises - ------ - ValueError - If query for collection name not implemented. - KeyError - If array element entry in the database is incomplete. - - """ - query = {"version": model_version} - results = db_collection.find(query, {"instrument": 1, "value": 1, "parameter": 1}).sort( - "instrument", ASCENDING - ) - - _all_available_array_elements = {} - for doc in results: - try: - if doc["parameter"] == "design_model": - _all_available_array_elements[doc["instrument"]] = doc["value"] - except KeyError as exc: - raise KeyError(f"Incomplete array element entry in the database: {doc}.") from exc - - if len(_all_available_array_elements) == 0: - raise ValueError(f"No array elements found in DB collection {db_collection}.") - - return _all_available_array_elements - - -def get_array_element_list_for_db_query(array_element_name, db, model_version, collection): - """ - Get array element name and design model for querying the database. - - Return a list of array element names to be used for querying the database for a given array - element. This is in most cases the array element itself and its design model. - In cases of no design model available, the design model of the array element is returned. - - Parameters - ---------- - array_element_name: str - Name of the array element model (e.g. MSTN-01). - db: DBHandler - Instance of the database handler - model_version: str - Model version. - collection: str - DB collection to get the array elements from (e.g., telescopes, calibration_devices) - - Returns - ------- - list - List of array element model names as used in the DB. - - """ - try: - _available_array_elements = get_array_elements( - db.get_collection(db_name=None, collection_name=collection), - db.model_version(model_version), - ) - except ValueError: - return [names.get_array_element_type_from_name(array_element_name) + "-design"] - try: - return [_available_array_elements[array_element_name], array_element_name] - except KeyError: - pass - - if array_element_name in _available_array_elements.values(): - return [array_element_name] - - raise ValueError(f"Array element {array_element_name} not found in DB.") - - -def get_array_elements_of_type(array_element_type, db, model_version, collection): - """ - Get all array elements of a certain type in the specified collection in the DB. - - Return e.g. for array_element_type='MSTS' all MSTS array elements found in the collection. - - Parameters - ---------- - array_element_type : str - Type of the array element (e.g. LSTN, MSTS) - model_version : str - Which version to get the array elements of - collection : str - Which collection to get the array elements from: - i.e. telescopes, calibration_devices - db_name : str - Database name - - Returns - ------- - list - Sorted list of all array element names found in collection - - """ - _available_array_elements = get_array_elements( - db.get_collection(db_name=None, collection_name=collection), - db.model_version(model_version), - ) - return sorted( - [entry for entry in _available_array_elements if entry.startswith(array_element_type)] - ) diff --git a/src/simtools/db/db_handler.py b/src/simtools/db/db_handler.py index a0f6dbde0d..b874efeac5 100644 --- a/src/simtools/db/db_handler.py +++ b/src/simtools/db/db_handler.py @@ -2,7 +2,6 @@ import logging import re -from importlib.resources import files from pathlib import Path from threading import Lock @@ -10,11 +9,9 @@ import jsonschema from bson.objectid import ObjectId from packaging.version import Version -from pymongo import ASCENDING, MongoClient -from pymongo.errors import BulkWriteError +from pymongo import MongoClient from simtools.data_model import validate_data -from simtools.db import db_array_elements from simtools.io_operations import io_handler from simtools.utils import names, value_conversion @@ -66,17 +63,11 @@ class DatabaseHandler: Dictionary with the MongoDB configuration (see jsonschema_db_dict for details). """ - DB_CTA_SIMULATION_MODEL_DESCRIPTIONS = "CTA-Simulation-Model-Descriptions" - # DB collection with updates field names - DB_DERIVED_VALUES = "Staging-CTA-Simulation-Model-Derived-Values" - ALLOWED_FILE_EXTENSIONS = [".dat", ".txt", ".lis", ".cfg", ".yml", ".yaml", ".ecsv"] db_client = None - site_parameters_cached = {} + production_table_cached = {} model_parameters_cached = {} - model_versions_cached = {} - corsika_configuration_parameters_cached = {} def __init__(self, mongo_db_config=None): """Initialize the DatabaseHandler class.""" @@ -150,12 +141,13 @@ def _find_latest_simulation_model_db(self): """ try: - if not self.mongo_db_config["db_simulation_model"].endswith("LATEST"): + db_simulation_model = self.mongo_db_config["db_simulation_model"] + if not db_simulation_model.endswith("LATEST"): return - except TypeError: + except TypeError: # db_simulation_model is None return - prefix = self.mongo_db_config["db_simulation_model"].replace("LATEST", "") + prefix = db_simulation_model.replace("LATEST", "") list_of_db_names = self.db_client.list_database_names() filtered_list_of_db_names = [s for s in list_of_db_names if s.startswith(prefix)] versioned_strings = [] @@ -180,66 +172,103 @@ def _find_latest_simulation_model_db(self): else: raise ValueError("Found LATEST in the DB name but no matching versions found in DB.") + def get_model_parameter( + self, + parameter, + parameter_version, + site, + array_element_name, + collection, + ): + """ + Get a model parameter using the parameter version. + + Parameters + ---------- + parameter: str + Name of the parameter. + parameter_version: str + Version of the parameter. + site: str + Site name. + array_element_name: str + Name of the array element model (e.g. MSTN, SSTS). + collection: str + Collection of array element (e.g. telescopes, calibration_devices). + + Returns + ------- + dict containing the parameter + + """ + query = { + "parameter_version": parameter_version, + "parameter": parameter, + } + if array_element_name is not None: + query["instrument"] = array_element_name + if site is not None: + query["site"] = site + return self._read_mongo_db(query=query, collection_name=collection) + def get_model_parameters( self, site, array_element_name, model_version, - collection="telescope", - only_applicable=False, + collection, ): """ - Get parameters from MongoDB or simulation model repository for an array element. + Get model parameters using the model version. - An array element can be e.g., a telescope or a calibration device. - Read parameters for design and for the specified array element (if necessary). This allows - to overwrite design parameters with specific parameters without having to copy - all model parameters when changing only a few. + Queries parameters for design and for the specified array element (if necessary). Parameters ---------- site: str Site name. array_element_name: str - Name of the array element model (e.g. LSTN-01, MSTS-design) + Name of the array element model (e.g. LSTN-01, MSTS-design, ILLN-01). model_version: str Version of the model. collection: str - collection of array element (e.g. telescopes, calibration_devices) - only_applicable: bool - If True, only applicable parameters will be read. + Collection of array element (e.g. telescopes, calibration_devices). Returns ------- dict containing the parameters - """ - _site, _array_element_name, _model_version = self._validate_model_input( - site, array_element_name, model_version - ) - array_element_list = db_array_elements.get_array_element_list_for_db_query( - _array_element_name, self, _model_version, collection + production_table = self._read_production_table_from_mongo_db(collection, model_version) + array_element_list = self._get_array_element_list( + array_element_name, site, production_table, collection ) + pars = {} for array_element in array_element_list: - _array_elements_cache_key = self._parameter_cache_key( - site, array_element, model_version, collection + cache_key, cache_dict = self._read_cache( + DatabaseHandler.model_parameters_cached, + names.validate_site_name(site) if site else None, + array_element, + model_version, + collection, ) + if cache_dict: + self._logger.debug(f"Found {array_element} in cache (key: {cache_key})") + pars.update(cache_dict) + continue + self._logger.debug(f"Did not find {array_element} in cache (key: {cache_key})") + try: - pars.update(DatabaseHandler.model_parameters_cached[_array_elements_cache_key]) - except KeyError: - pars.update( - self.read_mongo_db( - self.mongo_db_config.get("db_simulation_model", None), - array_element_name=array_element, - model_version=_model_version, - collection_name=collection, - run_location=None, - write_files=False, - only_applicable=only_applicable, - ) - ) - DatabaseHandler.model_parameters_cached[_array_elements_cache_key] = pars + parameter_version_table = production_table["parameters"][array_element] + except KeyError: # allow missing array elements (parameter dict is checked later) + continue + DatabaseHandler.model_parameters_cached[cache_key] = self._read_mongo_db( + query=self._get_query_from_parameter_version_table( + parameter_version_table, array_element, site + ), + collection_name=collection, + ) + pars.update(DatabaseHandler.model_parameters_cached[cache_key]) return pars @@ -263,99 +292,102 @@ def get_collection(self, db_name, collection_name): db_name = self._get_db_name(db_name) return DatabaseHandler.db_client[db_name][collection_name] - def export_file_db(self, db_name, dest, file_name): + def get_collections(self, db_name=None, model_collections_only=False): """ - Get file from the DB and write to disk. + List of collections in the DB. Parameters ---------- db_name: str - Name of the DB to search in. - dest: str or Path - Location where to write the file to. - file_name: str - Name of the file to get. + Database name. + model_collections_only: bool + If True, only return model collections (i.e. exclude fs.files, fs.chunks) Returns ------- - file_id: GridOut._id - the database ID the file was assigned when it was inserted to the DB. - - Raises - ------ - FileNotFoundError - If the desired file is not found. + list + List of collection names """ - db_name = self._get_db_name(db_name) - - self._logger.debug(f"Getting {file_name} from {db_name} and writing it to {dest}") - file_path_instance = self._get_file_mongo_db(db_name, file_name) - self._write_file_from_mongo_to_disk(db_name, dest, file_path_instance) - return file_path_instance._id # pylint: disable=protected-access; + db_name = db_name or self._get_db_name() + if db_name not in self.list_of_collections: + self.list_of_collections[db_name] = DatabaseHandler.db_client[ + db_name + ].list_collection_names() + collections = self.list_of_collections[db_name] + if model_collections_only: + return [collection for collection in collections if not collection.startswith("fs.")] + return collections - def export_model_files(self, parameters, dest): + def export_model_files(self, parameters=None, file_names=None, dest=None, db_name=None): """ - Export all the files in a model from the DB and write them to disk. + Export files from the DB to the model directory. + + The files to be exported can be specified by file_name or retrieved from the database + using the parameters dictionary. Parameters ---------- parameters: dict - Dict of model parameters. + Dict of model parameters + file_names: list, str + List (or string) of file names to export dest: str or Path Location where to write the files to. - Raises - ------ - FileNotFoundError - if a file in parameters.values is not found - + Returns + ------- + file_id: dict of GridOut._id + Dict of database IDs of files. """ - if self.mongo_db_config: - for info in parameters.values(): - if not info or not info.get("file") or info["value"] is None: - continue - if Path(dest).joinpath(info["value"]).exists(): - continue - file = self._get_file_mongo_db(self._get_db_name(), info["value"]) - self._write_file_from_mongo_to_disk(self._get_db_name(), dest, file) + db_name = self._get_db_name(db_name) - @staticmethod - def _is_file(value): - """Verify if a parameter value is a file name.""" - return any(ext in str(value) for ext in DatabaseHandler.ALLOWED_FILE_EXTENSIONS) + if file_names: + file_names = [file_names] if not isinstance(file_names, list) else file_names + elif parameters: + file_names = [ + info["value"] + for info in parameters.values() + if info and info.get("file") and info["value"] is not None + ] - def read_mongo_db( - self, - db_name, - array_element_name, - model_version, - run_location, - collection_name, - write_files=True, - only_applicable=False, + instance_ids = {} + for file_name in file_names: + if Path(dest).joinpath(file_name).exists(): + instance_ids[file_name] = "file exists" + else: + file_path_instance = self._get_file_mongo_db(self._get_db_name(), file_name) + self._write_file_from_mongo_to_disk(self._get_db_name(), dest, file_path_instance) + instance_ids[file_name] = file_path_instance._id # pylint: disable=protected-access + return instance_ids + + def _get_query_from_parameter_version_table( + self, parameter_version_table, array_element_name, site ): - """ - Build and execute query to Read the MongoDB for a specific array element. + """Return query based on parameter version table.""" + query_dict = { + "$or": [ + {"parameter": param, "parameter_version": version} + for param, version in parameter_version_table.items() + ], + } + # 'xSTX-design' is a placeholder to ignore 'instrument' field in query. + if array_element_name and array_element_name != "xSTx-design": + query_dict["instrument"] = array_element_name + if site: + query_dict["site"] = site + return query_dict - Also writes the files listed in the parameter values into the sim_telarray run location + def _read_mongo_db(self, query, collection_name): + """ + Query MongoDB. Parameters ---------- - db_name: str - the name of the DB - array_element_name: str - Name of the array element model (e.g. MSTN-design ...) - model_version: str - Version of the model. - run_location: Path or str - The sim_telarray run location to write the tabulated data files into. + query: dict + Query to execute. collection_name: str - The name of the collection to read from. - write_files: bool - If true, write the files to the run_location. - only_applicable: bool - If True, only applicable parameters will be read. + Collection name. Returns ------- @@ -364,154 +396,88 @@ def read_mongo_db( Raises ------ ValueError - if query returned zero results. - + if query returned no results. """ + db_name = self._get_db_name() collection = self.get_collection(db_name, collection_name) - _parameters = {} - - query = { - "instrument": array_element_name, - "version": self.model_version(model_version, db_name), - } - - if only_applicable: - query["applicable"] = True - if collection.count_documents(query) < 1: + posts = list(collection.find(query)) + if not posts: raise ValueError( - "The following query returned zero results! Check the input data and rerun.\n", - query, + f"The following query for {collection_name} returned zero results: {query} " ) - for post in collection.find(query).sort("parameter", ASCENDING): + parameters = {} + for post in posts: par_now = post["parameter"] - _parameters[par_now] = post - _parameters[par_now].pop("parameter", None) - _parameters[par_now].pop("instrument", None) - _parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time - if _parameters[par_now]["file"] and write_files: - file = self._get_file_mongo_db(db_name, _parameters[par_now]["value"]) - self._write_file_from_mongo_to_disk(db_name, run_location, file) + parameters[par_now] = post + parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time + return {k: parameters[k] for k in sorted(parameters)} - return _parameters - - def get_site_parameters( - self, - site, - model_version, - only_applicable=False, - ): + def _read_production_table_from_mongo_db(self, collection_name, model_version): """ - Get parameters from either MongoDB or simulation model repository for a specific site. + Read production table from MongoDB. Parameters ---------- - site: str - Site name. - model_version: str - Version of the model. - only_applicable: bool - If True, only applicable parameters will be read. - - Returns - ------- - dict containing the parameters - - """ - _site, _, _model_version = self._validate_model_input(site, None, model_version) - _db_name = self._get_db_name() - _site_cache_key = self._parameter_cache_key(site, None, model_version) - try: - return DatabaseHandler.site_parameters_cached[_site_cache_key] - except KeyError: - pass - - DatabaseHandler.site_parameters_cached[_site_cache_key] = ( - self._get_site_parameters_mongo_db( - _db_name, - _site, - _model_version, - only_applicable, - ) - ) - return DatabaseHandler.site_parameters_cached[_site_cache_key] - - def _get_site_parameters_mongo_db(self, db_name, site, model_version, only_applicable=False): - """ - Get parameters from MongoDB for a specific site. - - Parameters - ---------- - db_name: str - The name of the DB. - site: str - Site name. + collection_name: str + Name of the collection. model_version: str Version of the model. - only_applicable: bool - If True, only applicable parameters will be read. - - Returns - ------- - dict containing the parameters Raises ------ ValueError - if query returned zero results. - + if query returned no results. """ - collection = self.get_collection(db_name, "sites") - _parameters = {} + try: + return DatabaseHandler.production_table_cached[ + self._cache_key(None, None, model_version, collection_name) + ] + except KeyError: + pass - query = { - "site": site, - "version": model_version, + query = {"model_version": model_version, "collection": collection_name} + collection = self.get_collection(self._get_db_name(), "production_tables") + post = collection.find_one(query) + if not post: + raise ValueError(f"The following query returned zero results: {query}") + + return { + "collection": post["collection"], + "model_version": post["model_version"], + "parameters": post["parameters"], + "design_model": post.get("design_model", {}), + "entry_date": ObjectId(post["_id"]).generation_time, } - if only_applicable: - query["applicable"] = True - if collection.count_documents(query) < 1: - raise ValueError( - "The following query returned zero results! Check the input data and rerun.\n", - query, - ) - for post in collection.find(query).sort("parameter", ASCENDING): - par_now = post["parameter"] - _parameters[par_now] = post - _parameters[par_now].pop("parameter", None) - _parameters[par_now].pop("site", None) - _parameters[par_now]["entry_date"] = ObjectId(post["_id"]).generation_time - return _parameters - - def get_derived_values(self, site, array_element_name, model_version): + def get_array_elements_of_type(self, array_element_type, model_version, collection): """ - Get all derived values from the DB for a specific array element. + Get array elements of a certain type (e.g. 'LSTN') for a DB collection. + + Does not return 'design' models. Parameters ---------- - site: str - Site name. - array_element_name: str - Name of the array element model (e.g. MSTN, SSTS). + array_element_type: str + Type of the array element (e.g. LSTN, MSTS). model_version: str Version of the model. + collection: str + Which collection to get the array elements from: + i.e. telescopes, calibration_devices. Returns ------- - dict containing the parameters - - """ - _, _array_element_name, _model_version = self._validate_model_input( - site, array_element_name, model_version - ) - - return self.read_mongo_db( - DatabaseHandler.DB_DERIVED_VALUES, - _array_element_name, - _model_version, - run_location=None, - collection_name="derived_values", - write_files=False, + list + Sorted list of all array element names found in collection + """ + production_table = self._read_production_table_from_mongo_db(collection, model_version) + all_array_elements = production_table["parameters"] + return sorted( + [ + entry + for entry in all_array_elements + if entry.startswith(array_element_type) and "-design" not in entry + ] ) def get_simulation_configuration_parameters( @@ -541,63 +507,24 @@ def get_simulation_configuration_parameters( if simulation_software is not valid. """ if simulation_software == "corsika": - return self.get_corsika_configuration_parameters(model_version) + return self.get_model_parameters( + None, + None, + model_version, + collection="configuration_corsika", + ) if simulation_software == "simtel": - if site and array_element_name: - return self.get_model_parameters( - site, array_element_name, model_version, collection="configuration_sim_telarray" + return ( + self.get_model_parameters( + site, + array_element_name, + model_version, + collection="configuration_sim_telarray", ) - return {} - raise ValueError(f"Unknown simulation software: {simulation_software}") - - def get_corsika_configuration_parameters(self, model_version): - """ - Get CORSIKA configuration parameters from the DB. - - Parameters - ---------- - model_version : str - Version of the model. - - Returns - ------- - dict - Configuration parameters for CORSIKA - """ - _corsika_cache_key = self._parameter_cache_key(None, None, model_version) - try: - return DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key] - except KeyError: - pass - DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key] = ( - self.read_mongo_db( - db_name=self._get_db_name(), - array_element_name=None, - model_version=model_version, - run_location=None, - collection_name="configuration_corsika", - write_files=False, + if site and array_element_name + else {} ) - ) - return DatabaseHandler.corsika_configuration_parameters_cached[_corsika_cache_key] - - def _validate_model_input(self, site, array_element_name, model_version): - """ - Validate input for model parameter queries. - - site: str - Site name. - array_element_name: str - Name of the array element model (e.g. LSTN-01, MSTS-design) - model_version: str - Version of the model. - - """ - return ( - names.validate_site_name(site), - names.validate_array_element_name(array_element_name) if array_element_name else None, - self.model_version(model_version), - ) + raise ValueError(f"Unknown simulation software: {simulation_software}") @staticmethod def _get_file_mongo_db(db_name, file_name): @@ -648,76 +575,22 @@ def _write_file_from_mongo_to_disk(db_name, path, file): with open(Path(path).joinpath(file.filename), "wb") as output_file: fs_output.download_to_stream_by_name(file.filename, output_file) - def copy_array_element( - self, - db_name, - element_to_copy, - version_to_copy, - new_array_element_name, - collection_name="telescopes", - db_to_copy_to=None, - collection_to_copy_to=None, - ): + def add_production_table(self, db_name, production_table): """ - Copy a full array element configuration to a new array element name. - - Only a specific version is copied. - This function should be rarely used and is intended to simplify unit tests. + Add a production table to the DB. Parameters ---------- db_name: str - the name of the DB to copy from - element_to_copy: str - The array element to copy - version_to_copy: str - The version of the configuration to copy - new_array_element_name: str - The name of the new array element - collection_name: str - The name of the collection to copy from. - db_to_copy_to: str - The name of the DB to copy to. - collection_to_copy_to: str - The name of the collection to copy to. - - Raises - ------ - BulkWriteError - + the name of the DB. + production_table: dict + The production table to add to the DB. """ db_name = self._get_db_name(db_name) - if db_to_copy_to is None: - db_to_copy_to = db_name - - if collection_to_copy_to is None: - collection_to_copy_to = collection_name - - self._logger.info( - f"Copying version {version_to_copy} of {element_to_copy} " - f"to the new array element {new_array_element_name} in the {db_to_copy_to} DB" - ) - - collection = self.get_collection(db_name, collection_name) - db_entries = [] - - _version_to_copy = self.model_version(version_to_copy) - - query = { - "instrument": element_to_copy, - "version": _version_to_copy, - } - for post in collection.find(query): - post["instrument"] = new_array_element_name - post.pop("_id", None) - db_entries.append(post) - - self._logger.info(f"Creating new array element {new_array_element_name}") - collection = self.get_collection(db_to_copy_to, collection_to_copy_to) - try: - collection.insert_many(db_entries) - except BulkWriteError as exc: - raise BulkWriteError(str(exc.details)) from exc + collection = self.get_collection(db_name, "production_tables") + self._logger.info(f"Adding production for {production_table.get('collection')} to to DB") + collection.insert_one(production_table) + DatabaseHandler.production_table_cached.clear() def add_new_parameter( self, @@ -727,7 +600,7 @@ def add_new_parameter( file_prefix=None, ): """ - Add a parameter dictionary for a specific array element to the DB. + Add a new parameter dictionary to the DB. A new document will be added to the DB, with all fields taken from the input parameters. Parameter dictionaries are validated before submission using the corresponding schema. @@ -743,13 +616,7 @@ def add_new_parameter( file_prefix: str or Path where to find files to upload to the DB """ - data_validator = validate_data.DataValidator( - schema_file=files("simtools") - / f"schemas/model_parameters/{par_dict['parameter']}.schema.yml", - data_dict=par_dict, - check_exact_data_type=False, - ) - par_dict = data_validator.validate_and_transform(is_model_parameter=True) + par_dict = validate_data.DataValidator.validate_model_parameter(par_dict) db_name = self._get_db_name(db_name) collection = self.get_collection(db_name, collection_name) @@ -778,7 +645,7 @@ def add_new_parameter( self._logger.info(f"Will also add the file {file_to_insert_now} to the DB") self.insert_file_to_db(file_to_insert_now, db_name) - self._reset_parameter_cache(par_dict["site"], par_dict["instrument"], par_dict["version"]) + self._reset_parameter_cache() def _get_db_name(self, db_name=None): """ @@ -796,42 +663,6 @@ def _get_db_name(self, db_name=None): """ return self.mongo_db_config["db_simulation_model"] if db_name is None else db_name - def model_version(self, version, db_name=None): - """ - Return model version and check that it is valid. - - Queries the database for all available model versions and check if the - requested version is valid. - - Parameters - ---------- - version : str - Model version. - db_name : str - Database name. - - Returns - ------- - str - Model version. - - Raises - ------ - ValueError - if version not valid. - - """ - _all_versions = self.get_all_versions(db_name=db_name) - if version in _all_versions: - return version - if len(_all_versions) == 0: - return None - - raise ValueError( - f"Invalid model version {version} in DB {self._get_db_name(db_name)} " - f"(allowed are {_all_versions})" - ) - def insert_file_to_db(self, file_name, db_name=None, **kwargs): """ Insert a file to the DB. @@ -855,15 +686,11 @@ def insert_file_to_db(self, file_name, db_name=None, **kwargs): """ db_name = self._get_db_name(db_name) - db = DatabaseHandler.db_client[db_name] file_system = gridfs.GridFS(db) - if "content_type" not in kwargs: - kwargs["content_type"] = "ascii/dat" - - if "filename" not in kwargs: - kwargs["filename"] = Path(file_name).name + kwargs.setdefault("content_type", "ascii/dat") + kwargs.setdefault("filename", Path(file_name).name) if file_system.exists({"filename": kwargs["filename"]}): self._logger.warning( @@ -872,78 +699,11 @@ def insert_file_to_db(self, file_name, db_name=None, **kwargs): return file_system.find_one( # pylint: disable=protected-access {"filename": kwargs["filename"]} )._id + self._logger.debug(f"Writing file to DB: {file_name}") with open(file_name, "rb") as data_file: return file_system.put(data_file, **kwargs) - def get_all_versions( - self, - parameter=None, - array_element_name=None, - site=None, - db_name=None, - collection=None, - ): - """ - Get all version entries in the DB of collection and/or a specific parameter. - - Parameters - ---------- - parameter: str - Which parameter to get the versions of - array_element_name: str - Which array element to get the versions of (in case "collection_name" is not "sites") - site: str - Site name. - db_name: str - Database name. - collection_name: str - The name of the collection in which to update the parameter. - - Returns - ------- - all_versions: list - List of all versions found - - Raises - ------ - ValueError - If key to collection_name is not valid. - - """ - db_name = self._get_db_name() if db_name is None else db_name - if not db_name: - self._logger.warning("No database name defined to determine list of model versions") - return [] - _cache_key = f"model_versions_{db_name}-{collection}" - - query = {} - if parameter is not None: - query["parameter"] = parameter - _cache_key = f"{_cache_key}-{parameter}" - if collection in ["telescopes", "calibration_devices"] and array_element_name is not None: - query["instrument"] = names.validate_array_element_name(array_element_name) - _cache_key = f"{_cache_key}-{query['instrument']}" - elif collection == "sites" and site is not None: - query["site"] = names.validate_site_name(site) - _cache_key = f"{_cache_key}-{query['site']}" - - if _cache_key not in DatabaseHandler.model_versions_cached: - all_versions = set() - collections_to_query = ( - [collection] if collection else self.get_collections(db_name, True) - ) - for collection_name in collections_to_query: - db_collection = self.get_collection(db_name, collection_name) - sorted_posts = db_collection.find(query).sort("version", ASCENDING) - all_versions.update(post["version"] for post in sorted_posts) - DatabaseHandler.model_versions_cached[_cache_key] = list(all_versions) - - if len(DatabaseHandler.model_versions_cached[_cache_key]) == 0: - self._logger.warning(f"The query {query} did not return any results. No versions found") - - return DatabaseHandler.model_versions_cached[_cache_key] - - def _parameter_cache_key(self, site, array_element_name, model_version, collection=None): + def _cache_key(self, site=None, array_element_name=None, model_version=None, collection=None): """ Create a cache key for the parameter cache dictionaries. @@ -963,61 +723,79 @@ def _parameter_cache_key(self, site, array_element_name, model_version, collecti str Cache key. """ - parts = [] - if site: - parts.append(site) - if array_element_name: - parts.append(array_element_name) - parts.append(model_version) - if collection: - parts.append(collection) - return "-".join(parts) + return "-".join( + part for part in [model_version, collection, site, array_element_name] if part + ) - def _reset_parameter_cache(self, site, array_element_name, model_version): + def _read_cache( + self, cache_dict, site=None, array_element_name=None, model_version=None, collection=None + ): """ - Reset the cache for the parameters. + Read parameters from cache. Parameters ---------- + cache_dict: dict + Cache dictionary. site: str Site name. array_element_name: str Array element name. model_version: str Model version. + collection: str + DB collection name. + + Returns + ------- + str + Cache key. """ - self._logger.debug(f"Resetting cache for {site} {array_element_name} {model_version}") - _cache_key = self._parameter_cache_key(site, array_element_name, model_version) - DatabaseHandler.site_parameters_cached.pop(_cache_key, None) - DatabaseHandler.model_parameters_cached.pop(_cache_key, None) - db_array_elements.get_array_elements.cache_clear() + cache_key = self._cache_key(site, array_element_name, model_version, collection) + try: + return cache_key, cache_dict[cache_key] + except KeyError: + return cache_key, None - def get_collections(self, db_name=None, model_collections_only=False): + def _reset_parameter_cache(self): + """Reset the cache for the parameters.""" + DatabaseHandler.model_parameters_cached.clear() + + def _get_array_element_list(self, array_element_name, site, production_table, collection): """ - List of collections in the DB. + Return list of array elements for DB queries (add design model if needed). + + Design model is added if found in the production table. Parameters ---------- - db_name: str - Database name. + array_element_name: str + Name of the array element. + site: str + Site name. + production_table: dict + Production table. + collection: str + collection of array element (e.g. telescopes, calibration_devices). Returns ------- list - List of collection names - model_collections_only: bool - If True, only return model collections (i.e. exclude fs.files, fs.chunks, metadata) - - """ - db_name = self._get_db_name() if db_name is None else db_name - if db_name not in self.list_of_collections: - self.list_of_collections[db_name] = DatabaseHandler.db_client[ - db_name - ].list_collection_names() - if model_collections_only: + List of array elements + """ + if collection == "configuration_corsika": + return ["xSTx-design"] # placeholder to ignore 'instrument' field in query. + if collection == "sites": + return [f"OBS-{site}"] + if "-design" in array_element_name: + return [array_element_name] + try: + return [ + production_table["design_model"][array_element_name], + array_element_name, + ] + except KeyError: return [ - collection - for collection in self.list_of_collections[db_name] - if not collection.startswith("fs.") and collection != "metadata" + f"{names.get_array_element_type_from_name(array_element_name)}-design", + array_element_name, ] - return self.list_of_collections[db_name] diff --git a/src/simtools/db/db_model_upload.py b/src/simtools/db/db_model_upload.py new file mode 100644 index 0000000000..4b59e7ffba --- /dev/null +++ b/src/simtools/db/db_model_upload.py @@ -0,0 +1,139 @@ +"""Upload a simulation model (parameters and production tables) to the database.""" + +import logging +from pathlib import Path + +import simtools.utils.general as gen +from simtools.utils import names + +logger = logging.getLogger(__name__) + + +def add_values_from_json_to_db(file, collection, db, db_name, file_prefix): + """ + Upload new model parameter from json files to db. + + Parameters + ---------- + file : list + Json file to be uploaded to the DB. + collection : str + The DB collection to which to add the file. + db : DatabaseHandler + Database handler object. + db_name : str + Name of the database to be created. + file_prefix : str + Path to location of all additional files to be uploaded. + """ + par_dict = gen.collect_data_from_file(file_name=file) + logger.info( + f"Adding the following parameter to the DB: {par_dict['parameter']} " + f"version {par_dict['parameter_version']} " + f"(collection {collection} in database {db_name})" + ) + + db.add_new_parameter( + db_name=db_name, + par_dict=par_dict, + collection_name=collection, + file_prefix=file_prefix, + ) + + +def add_model_parameters_to_db(args_dict, db): + """ + Read model parameters from a directory and upload them to the database. + + Parameters + ---------- + args_dict : dict + Command line arguments. + db : DatabaseHandler + Database handler object. + """ + input_path = Path(args_dict["input_path"]) + logger.info(f"Reading model parameters from repository path {input_path}") + array_elements = [d for d in input_path.iterdir() if d.is_dir()] + for element in array_elements: + collection = names.get_collection_name_from_array_element_name(element.name, False) + if collection == "Files": + logger.info("Files (tables) are uploaded with the corresponding model parameters") + continue + logger.info(f"Reading model parameters for {element.name} into collection {collection}") + files_to_insert = list(Path(element).rglob("*json")) + for file in files_to_insert: + add_values_from_json_to_db( + file=file, + collection=collection, + db=db, + db_name=args_dict["db_name"], + file_prefix=input_path / "Files", + ) + + +def add_production_tables_to_db(args_dict, db): + """ + Read production tables from a directory and upload them to the database. + + One dictionary per collection is prepared for each model version, containing + tables of all array elements, sites, and configuration parameters. + + Parameters + ---------- + args_dict : dict + Command line arguments. + db : DatabaseHandler + Database handler object. + """ + input_path = Path(args_dict["input_path"]) + logger.info(f"Reading production tables from repository path {input_path}") + + for model in filter(Path.is_dir, input_path.iterdir()): + logger.info(f"Reading production tables for model version {model.name}") + model_dict = {} + for file in sorted(model.rglob("*json")): + _read_production_table(model_dict, file, model.name) + + for collection, data in model_dict.items(): + if not data["parameters"]: + logger.info(f"No production table for {collection} in model version {model.name}") + continue + logger.info(f"Adding production table for {collection} to the database") + db.add_production_table( + db_name=args_dict["db_name"], + production_table=data, + ) + + +def _read_production_table(model_dict, file, model_name): + """Read a single production table from file.""" + array_element = file.stem + collection = names.get_collection_name_from_array_element_name(array_element, False) + model_dict.setdefault( + collection, + { + "collection": collection, + "model_version": model_name, + "parameters": {}, + "design_model": {}, + }, + ) + parameter_dict = gen.collect_data_from_file(file_name=file) + logger.info(f"Reading production table for {array_element} (collection {collection})") + try: + if array_element in ("configuration_corsika", "configuration_sim_telarray"): + model_dict[collection]["parameters"] = parameter_dict["parameters"] + else: + model_dict[collection]["parameters"][array_element] = parameter_dict["parameters"][ + array_element + ] + except KeyError as exc: + logger.error(f"KeyError: {exc}") + raise + try: + model_dict[collection]["design_model"][array_element] = parameter_dict["design_model"][ + array_element + ] + except KeyError: + pass diff --git a/src/simtools/layout/array_layout.py b/src/simtools/layout/array_layout.py index 03f7dca5b2..9c039bdf7d 100644 --- a/src/simtools/layout/array_layout.py +++ b/src/simtools/layout/array_layout.py @@ -386,7 +386,9 @@ def _read_table_from_json_file(self, file_name): with Path(file_name).open("r", encoding="utf-8") as file: data = json.load(file) - position = gen.convert_string_to_list(data["value"]) + position = data["value"] + if isinstance(position, str): + position = gen.convert_string_to_list(position) self.site = data.get("site", None) table = QTable() @@ -579,7 +581,9 @@ def export_telescope_list_table(self, crs_name): return table - def export_one_telescope_as_json(self, crs_name): + def export_one_telescope_as_json( + self, crs_name, parameter_version=None, schema_version="0.2.0" + ): """ Return a list containing a single telescope in simtools-DB-style json. @@ -587,6 +591,8 @@ def export_one_telescope_as_json(self, crs_name): ---------- crs_name: str Name of coordinate system to be used for export. + schema_version: str + Version of the schema. Returns ------- @@ -596,43 +602,39 @@ def export_one_telescope_as_json(self, crs_name): table = self.export_telescope_list_table(crs_name) if len(table) != 1: raise ValueError("Only one telescope can be exported to json") - parameter_name = value_string = None + parameter_name = value = None if crs_name == "ground": parameter_name = "array_element_position_ground" - value_string = gen.convert_list_to_string( - [ - table["position_x"][0].value, - table["position_y"][0].value, - table["position_z"][0].value, - ] - ) + value = [ + table["position_x"][0].value, + table["position_y"][0].value, + table["position_z"][0].value, + ] elif crs_name == "utm": parameter_name = "array_element_position_utm" - value_string = gen.convert_list_to_string( - [ - table["utm_east"][0].value, - table["utm_north"][0].value, - table["altitude"][0].value, - ] - ) + value = [ + table["utm_east"][0].value, + table["utm_north"][0].value, + table["altitude"][0].value, + ] elif crs_name == "mercator": parameter_name = "array_element_position_mercator" - value_string = gen.convert_list_to_string( - [ - table["latitude"][0].value, - table["longitude"][0].value, - table["altitude"][0].value, - ] - ) + value = [ + table["latitude"][0].value, + table["longitude"][0].value, + table["altitude"][0].value, + ] + return { + "schema_version": schema_version, "parameter": parameter_name, "instrument": table["telescope_name"][0], "site": self.site, - "version": self.model_version, - "value": value_string, + "parameter_version": parameter_version, + "unique_id": None, + "value": value, "unit": "m", "type": "float64", - "applicable": True, "file": False, } diff --git a/src/simtools/model/array_model.py b/src/simtools/model/array_model.py index 672b4fa4a0..29f52cf1f4 100644 --- a/src/simtools/model/array_model.py +++ b/src/simtools/model/array_model.py @@ -7,7 +7,7 @@ from astropy.table import QTable from simtools.data_model import data_reader -from simtools.db import db_array_elements, db_handler +from simtools.db import db_handler from simtools.io_operations import io_handler from simtools.model.site_model import SiteModel from simtools.model.telescope_model import TelescopeModel @@ -290,7 +290,13 @@ def _load_array_element_positions_from_file( } def _get_telescope_position_parameter( - self, telescope_name: str, site: str, x: u.Quantity, y: u.Quantity, z: u.Quantity + self, + telescope_name: str, + site: str, + x: u.Quantity, + y: u.Quantity, + z: u.Quantity, + parameter_version: str | None = None, ) -> dict: """ Return dictionary with telescope position parameters (following DB model database format). @@ -314,16 +320,17 @@ def _get_telescope_position_parameter( Dict with telescope position parameters. """ return { + "schema_version": "0.1.0", "parameter": "array_element_position_ground", "instrument": telescope_name, "site": site, - "version": self.model_version, + "parameter_version": parameter_version, + "unique_id": None, "value": general.convert_list_to_string( [x.to("m").value, y.to("m").value, z.to("m").value] ), "unit": "m", "type": "float64", - "applicable": True, "file": False, } @@ -367,9 +374,8 @@ def _get_all_array_elements_of_type(self, array_element_type: str) -> dict: dict Dict with array elements. """ - all_elements = db_array_elements.get_array_elements_of_type( + all_elements = self.db.get_array_elements_of_type( array_element_type=array_element_type, - db=self.db, model_version=self.model_version, collection="telescopes", ) diff --git a/src/simtools/model/model_parameter.py b/src/simtools/model/model_parameter.py index 6106e3753c..248a984af2 100644 --- a/src/simtools/model/model_parameter.py +++ b/src/simtools/model/model_parameter.py @@ -67,7 +67,6 @@ def __init__( self._parameters = {} self._simulation_config_parameters = {"corsika": {}, "simtel": {}} - self._derived = None self.collection = collection self.label = label self.model_version = model_version @@ -111,12 +110,8 @@ def _get_parameter_dict(self, par_name): """ try: return self._parameters[par_name] - except KeyError: - pass - try: - return self.derived[par_name] except (KeyError, ValueError) as e: - msg = f"Parameter {par_name} was not found in the model" + msg = f"Parameter {par_name} was not found in the model {self.name}, {self.site}." self._logger.error(msg) raise InvalidModelParameterError(msg) from e @@ -180,7 +175,10 @@ def get_parameter_value_with_unit(self, par_name): _value = self.get_parameter_value(par_name, _parameter) try: - _unit = [item.strip() for item in _parameter.get("unit").split(",")] + if isinstance(_parameter.get("unit"), str): + _unit = [item.strip() for item in _parameter.get("unit").split(",")] + else: + _unit = _parameter.get("unit") # if there is only one value or the values share one unit if (isinstance(_value, (int | float))) or (len(_value) > len(_unit)): @@ -241,33 +239,6 @@ def get_parameter_file_flag(self, par_name): self._logger.debug(f"Parameter {par_name} does not have a file associated with it.") return False - @property - def derived(self): - """Load the derived values and export them if the class instance hasn't done it yet.""" - if self._derived is None: - self._load_derived_values() - self._export_derived_files() - return self._derived - - def _load_derived_values(self): - """Load derived values from the DB.""" - self._logger.debug("Reading derived values from DB") - self._derived = self.db.get_derived_values( - self.site, - self.name, - self.model_version, - ) - - def _export_derived_files(self): - """Write to disk a file from the derived values DB.""" - for par_now in self.derived.values(): - if par_now.get("File") or par_now.get("file"): - self.db.export_file_db( - db_name=self.db.DB_DERIVED_VALUES, - dest=self.config_file_directory, - file_name=(par_now.get("value") or par_now.get("Value")), - ) - def print_parameters(self): """Print parameters and their values for debugging purposes.""" for par in self._parameters: @@ -322,10 +293,11 @@ def _load_simulation_software_parameter(self): simulation_software=simulation_software, ) ) - except ValueError: + except ValueError as exc: self._logger.warning( f"No {simulation_software} parameters found for " - f"{self.site}, {self.name} (model version {self.model_version})." + f"{self.site}, {self.name} (model version {self.model_version}). " + f" (Query {exc})" ) def _load_parameters_from_db(self): @@ -335,15 +307,18 @@ def _load_parameters_from_db(self): if self.name is not None: self._parameters = self.db.get_model_parameters( - self.site, self.name, self.model_version, self.collection, only_applicable=True + self.site, self.name, self.model_version, self.collection ) if self.site is not None: - _site_pars = self.db.get_site_parameters( - self.site, self.model_version, only_applicable=True + self._parameters.update( + self.db.get_model_parameters( + self.site, + None, + self.model_version, + "sites", + ) ) - self._parameters.update(_site_pars) - self._load_simulation_software_parameter() def set_extra_label(self, extra_label): @@ -369,7 +344,7 @@ def extra_label(self): """Return the extra label if defined, if not return ''.""" return self._extra_label if self._extra_label is not None else "" - def get_simtel_parameters(self, parameters=None, telescope_model=True, site_model=True): + def get_simtel_parameters(self, parameters=None): """ Get simtel parameters as name and value pairs. @@ -377,10 +352,6 @@ def get_simtel_parameters(self, parameters=None, telescope_model=True, site_mode ---------- parameters: dict Parameters (simtools) to be renamed (if necessary) - telescope_model: bool - If True, telescope model parameters are included. - site_model: bool - If True, site model parameters are included. Returns ------- @@ -394,10 +365,7 @@ def get_simtel_parameters(self, parameters=None, telescope_model=True, site_mode _simtel_parameter_value = {} for key in parameters: _par_name = names.get_simulation_software_name_from_parameter_name( - key, - simulation_software="sim_telarray", - search_telescope_parameters=telescope_model, - search_site_parameters=site_model, + key, simulation_software="sim_telarray" ) if _par_name is not None: _simtel_parameter_value[_par_name] = parameters[key].get("value") @@ -513,7 +481,7 @@ def export_model_files(self): for par in self._added_parameter_files: pars_from_db.pop(par) - self.db.export_model_files(pars_from_db, self.config_file_directory) + self.db.export_model_files(parameters=pars_from_db, dest=self.config_file_directory) self._is_exported_model_files_up_to_date = True def get_model_file_as_table(self, par_name): @@ -535,7 +503,7 @@ def get_model_file_as_table(self, par_name): _par_entry[par_name] = self._parameters[par_name] except KeyError as exc: raise ValueError(f"Parameter {par_name} not found in the model.") from exc - self.db.export_model_files(_par_entry, self.config_file_directory) + self.db.export_model_files(parameters=_par_entry, dest=self.config_file_directory) if _par_entry[par_name]["value"].endswith("ecsv"): return Table.read( self.config_file_directory.joinpath(_par_entry[par_name]["value"]), @@ -620,7 +588,7 @@ def export_nsb_spectrum_to_telescope_altitude_correction_file(self, model_direct Model directory to export the file to. """ self.db.export_model_files( - { + parameters={ "nsb_spectrum_at_2200m": { "value": self._simulation_config_parameters["simtel"][ "correct_nsb_spectrum_to_telescope_altitude" @@ -628,5 +596,5 @@ def export_nsb_spectrum_to_telescope_altitude_correction_file(self, model_direct "file": True, } }, - model_directory, + dest=model_directory, ) diff --git a/src/simtools/model/site_model.py b/src/simtools/model/site_model.py index 94a22ee932..e346d5cba3 100644 --- a/src/simtools/model/site_model.py +++ b/src/simtools/model/site_model.py @@ -151,11 +151,11 @@ def export_atmospheric_transmission_file(self, model_directory: Path): Model directory to export the file to. """ self.db.export_model_files( - { + parameters={ "atmospheric_transmission_file": { "value": self.get_parameter_value("atmospheric_profile"), "file": True, } }, - model_directory, + dest=model_directory, ) diff --git a/src/simtools/schemas/model_parameter.metaschema.yml b/src/simtools/schemas/model_parameter.metaschema.yml index 22e2c2ede5..2e1fc5fab6 100644 --- a/src/simtools/schemas/model_parameter.metaschema.yml +++ b/src/simtools/schemas/model_parameter.metaschema.yml @@ -25,14 +25,19 @@ definitions: type: boolean description: "This parameter is a file." instrument: - type: string + type: + - string + - "null" description: "Associated instrument." site: - type: string + type: + - string + - "null" description: "Associated CTAO site." enum: - North - South + - null type: type: string description: "Data type" diff --git a/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml b/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml index fbcaca3a90..ef3ec52a9a 100644 --- a/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml +++ b/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml @@ -322,6 +322,7 @@ definitions: - ValidateArrayElementCoordinates - ValidateAtmosphericModel - ValidateCameraChargeResponse + - ValidateCameraEfficiency - ValidateCameraGainsAndEfficiency - ValidateCameraGeometry - ValidateCameraLinearity diff --git a/src/simtools/schemas/model_parameters/array_window.schema.yml b/src/simtools/schemas/model_parameters/array_window.schema.yml new file mode 100644 index 0000000000..f6bb8a0544 --- /dev/null +++ b/src/simtools/schemas/model_parameters/array_window.schema.yml @@ -0,0 +1,37 @@ +%YAML 1.2 +--- +title: Schema for array_window model parameter +version: 0.1.0 +meta_schema: simpipe-schema +meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml +meta_schema_version: 0.1.0 +name: array_window +description: |- + Length of a coincidence window of the default stereo trigger, + after correction of fixed (cable length, focal length, etc.) + and variable (view direction) delays. +short_description: |- + Length of a coincidence window of the default stereo trigger. +data: + - type: double + unit: ns + default: 1000. +instrument: + class: Camera + type: + - LSTN + - LSTS + - MSTN + - MSTS + - SSTS + - SCTS +activity: + setting: + - SetParameterFromExternal + validation: + - ValidateParameterByExpert + - ValidateTriggerPerformance +source: + - Initial instrument setup +simulation_software: + - name: sim_telarray diff --git a/src/simtools/schemas/model_parameters/asum_clipping.schema.yml b/src/simtools/schemas/model_parameters/asum_clipping.schema.yml index 4fbcd3746e..1024977c14 100644 --- a/src/simtools/schemas/model_parameters/asum_clipping.schema.yml +++ b/src/simtools/schemas/model_parameters/asum_clipping.schema.yml @@ -22,10 +22,6 @@ instrument: type: - LSTN - LSTS - - MSTN - - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml b/src/simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml index 9e6d5a12e9..25a2c15141 100644 --- a/src/simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml +++ b/src/simtools/schemas/model_parameters/corsika_iact_io_buffer.schema.yml @@ -10,8 +10,8 @@ description: |- Maximum size of data blocks written by the CORSIKA IACT module. data: - type: int - unit: byte - default: 100000000 + unit: MB + default: 1000 activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml b/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml index 1ff03a1be1..cb4dc9dbcf 100644 --- a/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_clipping.schema.yml @@ -24,8 +24,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml b/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml index 45c769df28..03ae179b94 100644 --- a/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_ignore_below.schema.yml @@ -24,8 +24,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_offset.schema.yml b/src/simtools/schemas/model_parameters/dsum_offset.schema.yml index deb91b4f51..993dcc12d4 100644 --- a/src/simtools/schemas/model_parameters/dsum_offset.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_offset.schema.yml @@ -23,8 +23,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml b/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml index 7942b5be1e..e507802ee6 100644 --- a/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_pedsub.schema.yml @@ -20,8 +20,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml b/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml index 3af1912a7e..02fc297218 100644 --- a/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_pre_clipping.schema.yml @@ -25,8 +25,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml b/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml index 29b278cc6b..b5a044b193 100644 --- a/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_prescale.schema.yml @@ -30,8 +30,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml b/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml index f466f5e9b7..db7cc380d8 100644 --- a/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_presum_max.schema.yml @@ -24,8 +24,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml b/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml index 697e92b478..d374f37930 100644 --- a/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_presum_shift.schema.yml @@ -31,8 +31,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml b/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml index 3300db113e..1c32c62382 100644 --- a/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_shaping.schema.yml @@ -30,8 +30,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml b/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml index a62c001c82..74047fd93e 100644 --- a/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_shaping_renormalize.schema.yml @@ -18,8 +18,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml b/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml index 6bbba24ab8..af45603e50 100644 --- a/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_threshold.schema.yml @@ -28,8 +28,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml b/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml index 81923cb713..9ae6c26af4 100644 --- a/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml +++ b/src/simtools/schemas/model_parameters/dsum_zero_clip.schema.yml @@ -28,8 +28,6 @@ instrument: type: - MSTN - MSTS - - SSTS - - SCTS activity: setting: - SetParameterFromExternal diff --git a/src/simtools/schemas/model_parameters/fake_mirror_list.schema.yml b/src/simtools/schemas/model_parameters/fake_mirror_list.schema.yml new file mode 100644 index 0000000000..59735dd06d --- /dev/null +++ b/src/simtools/schemas/model_parameters/fake_mirror_list.schema.yml @@ -0,0 +1,33 @@ +%YAML 1.2 +--- +title: Schema for fake_mirror_list model parameter +version: 0.1.0 +meta_schema: simpipe-schema +meta_schema_url: https://raw.githubusercontent.com/gammasim/simtools/main/src/simtools/schemas/model_parameter_and_data_schema.metaschema.yml +meta_schema_version: 0.1.0 +developer_note: To be replaced by a data table +name: fake_mirror_list +short_description: Fake mirror list to be used for camera efficiency calculations ('testeff' program). +description: |- + Fake mirror list to be used for camera efficiency calculations ('testeff' program). + Allows to obtain realistic distributions of the photon incidence angles on the camera plan. + Not to be used for actual simulations. +data: + - type: file + unit: dimensionless + default: None +instrument: + class: Structure + type: + - SSTS + - SCTS +activity: + setting: + - SetParameterFromExternal + validation: + - ValidateParameterByExpert + - ValidateCameraEfficiency +source: + - Initial instrument setup +simulation_software: + - name: sim_telarray diff --git a/src/simtools/schemas/production_tables.schema.yml b/src/simtools/schemas/production_tables.schema.yml new file mode 100644 index 0000000000..a12900e102 --- /dev/null +++ b/src/simtools/schemas/production_tables.schema.yml @@ -0,0 +1,41 @@ +--- +$schema: http://json-schema.org/draft-06/schema# +$ref: '#/definitions/ProductionModelTable' +title: SimPipe DB Production Model Metaschema +description: | + YAML representation of DB production model metaschema + (based on simulation model DB). +version: 0.1.0 +name: production_table.metaschema +type: object +additionalProperties: false + +definitions: + ProductionModelTable: + properties: + model_version: + type: string + description: Model version. + pattern: '^\d+\.\d+\.\d+$' + parameters: + type: object + description: Model parameters. + additionalProperties: + type: object + description: Model parameter. + additionalProperties: + type: string + description: Parameter version (semantical versioning). + pattern: '^\d+\.\d+\.\d+$' + propertyNames: + description: Allowed parameter name patterns. + pattern: '^([A-Za-z](ST|LL)[N,S,x]-\d{2,3}|[A-Za-z](ST|LL)[N,S,x]-design|OBS-(North|South)|Dummy-Telescope)$' + design_model: + type: object + description: Design models. + additionalProperties: + type: string + description: Design model of a telescope + required: + - model_version + - parameters diff --git a/src/simtools/simtel/simtel_config_writer.py b/src/simtools/simtel/simtel_config_writer.py index 4c144192ff..1fee7d48d3 100644 --- a/src/simtools/simtel/simtel_config_writer.py +++ b/src/simtools/simtel/simtel_config_writer.py @@ -294,10 +294,7 @@ def _write_site_parameters(self, file, site_model, model_path, telescope_model): _site_parameters = site_model.get_simtel_parameters() for par, value in _site_parameters.items(): _simtel_name = names.get_simulation_software_name_from_parameter_name( - par, - simulation_software="sim_telarray", - search_telescope_parameters=False, - search_site_parameters=True, + par, simulation_software="sim_telarray" ) _simtel_name, value = self._convert_model_parameters_to_simtel_format( _simtel_name, value, model_path, telescope_model diff --git a/src/simtools/testing/validate_output.py b/src/simtools/testing/validate_output.py index 63ab21428b..b761526c8a 100644 --- a/src/simtools/testing/validate_output.py +++ b/src/simtools/testing/validate_output.py @@ -146,12 +146,13 @@ def compare_files(file1, file2, tolerance=1.0e-5, test_columns=None): """ _file1_suffix = Path(file1).suffix _file2_suffix = Path(file2).suffix + _logger.info("Comparing files: %s and %s", file1, file2) if _file1_suffix != _file2_suffix: raise ValueError(f"File suffixes do not match: {file1} and {file2}") if _file1_suffix == ".ecsv": return compare_ecsv_files(file1, file2, tolerance, test_columns) if _file1_suffix in (".json", ".yaml", ".yml"): - return compare_json_or_yaml_files(file1, file2) + return compare_json_or_yaml_files(file1, file2, tolerance) _logger.warning(f"Unknown file type for files: {file1} and {file2}") return False @@ -186,11 +187,35 @@ def compare_json_or_yaml_files(file1, file2, tolerance=1.0e-2): if data1 == data2: return True - if "value" in data1 and isinstance(data1["value"], str): - value_list_1 = gen.convert_string_to_list(data1.pop("value")) - value_list_2 = gen.convert_string_to_list(data2.pop("value")) - return np.allclose(value_list_1, value_list_2, rtol=tolerance) - return data1 == data2 + if data1.keys() != data2.keys(): + _logger.error(f"Keys do not match: {data1.keys()} and {data2.keys()}") + return False + _comparison = all( + ( + _compare_value_from_parameter_dict(data1[k], data2[k], tolerance) + if k == "value" + else data1[k] == data2[k] + ) + for k in data1 + ) + if not _comparison: + _logger.error(f"Values do not match: {data1} and {data2} (tolerance: {tolerance})") + return _comparison + + +def _compare_value_from_parameter_dict(data1, data2, tolerance): + """Compare value fields given in different formats.""" + + def _as_list(value): + if isinstance(value, str): + return gen.convert_string_to_list(value) + if isinstance(value, list | np.ndarray): + return value + return [value] + + _logger.info(f"Comparing values: {data1} and {data2} (tolerance: {tolerance})") + + return np.allclose(_as_list(data1), _as_list(data2), rtol=tolerance) def compare_ecsv_files(file1, file2, tolerance=1.0e-5, test_columns=None): diff --git a/src/simtools/utils/names.py b/src/simtools/utils/names.py index 51caac9c2e..e4ee803caa 100644 --- a/src/simtools/utils/names.py +++ b/src/simtools/utils/names.py @@ -293,7 +293,7 @@ def get_site_from_array_element_name(name): return array_elements()[get_array_element_type_from_name(name)]["site"] -def get_collection_name_from_array_element_name(name): +def get_collection_name_from_array_element_name(name, array_elements_only=True): """ Get collection name (e.g., telescopes, calibration_devices, sites) of array element from name. @@ -301,6 +301,8 @@ def get_collection_name_from_array_element_name(name): ---------- name: str Array element name. + array_elements_only: bool + If True, only array elements are considered. Returns ------- @@ -311,18 +313,28 @@ def get_collection_name_from_array_element_name(name): return array_elements()[get_array_element_type_from_name(name)]["collection"] except ValueError: pass + if name.startswith("OBS"): + return "sites" try: validate_site_name(name) return "sites" except ValueError as exc: - raise ValueError(f"Invalid array element name {name}: {exc}") from exc + if array_elements_only: + raise ValueError(f"Invalid array element name {name}") from exc + if name in ( + "configuration_sim_telarray", + "configuration_corsika", + "Files", + "Dummy-Telescope", + ): + return name + + raise ValueError(f"Invalid array element name {name}") def get_simulation_software_name_from_parameter_name( par_name, simulation_software="sim_telarray", - search_telescope_parameters=True, - search_site_parameters=True, ): """ Get the name used in the simulation software from the model parameter name. @@ -336,21 +348,13 @@ def get_simulation_software_name_from_parameter_name( Model parameter name. simulation_software: str Simulation software name. - search_telescope_parameters: bool - If True, telescope model parameters are included. - search_site_parameters: bool - If True, site model parameters are included. Returns ------- str Simtel parameter name. """ - _parameter_names = {} - if search_telescope_parameters: - _parameter_names.update(telescope_parameters()) - if search_site_parameters: - _parameter_names.update(site_parameters()) + _parameter_names = {**telescope_parameters(), **site_parameters()} try: _parameter = _parameter_names[par_name] diff --git a/tests/conftest.py b/tests/conftest.py index cafb11c902..94781c303f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -76,7 +76,6 @@ def _mock_settings_env_vars(tmp_test_directory): "SIMTOOLS_DB_API_PORT": "42", "SIMTOOLS_DB_SERVER": "abc@def.de", "SIMTOOLS_DB_SIMULATION_MODEL": "sim_model", - "SIMTOOLS_DB_SIMULATION_MODEL_URL": _url, }, clear=True, ): @@ -149,7 +148,6 @@ def db_config(): "db_api_port", "db_server", "db_simulation_model", - "db_simulation_model_url", ) for _para in _db_para: if _para not in mongo_db_config: @@ -159,20 +157,6 @@ def db_config(): return mongo_db_config -@pytest.fixture -def simulation_model_url(db_config): - """Simulation model URL from .env file or default.""" - if ( - db_config["db_simulation_model_url"] is None - or len(db_config["db_simulation_model_url"]) == 0 - ): - db_config["db_simulation_model_url"] = ( - "https://gitlab.cta-observatory.org/cta-science/simulations/" - "simulation-model/model_parameters/-/raw/main/" - ) - return db_config["db_simulation_model_url"] - - @pytest.fixture def db(db_config): """Database object with configuration from .env file.""" diff --git a/tests/integration_tests/config/convert_all_model_parameters_from_simtel_num_gains.yml b/tests/integration_tests/config/convert_all_model_parameters_from_simtel_num_gains.yml index 46aa1c55a0..0095d610fa 100644 --- a/tests/integration_tests/config/convert_all_model_parameters_from_simtel_num_gains.yml +++ b/tests/integration_tests/config/convert_all_model_parameters_from_simtel_num_gains.yml @@ -5,7 +5,7 @@ CTA_SIMPIPE: SIMTEL_CFG_FILE: ./tests/resources/simtel_config_test_la_palma.cfg SIMTEL_TELESCOPE_NAME: CT1 TELESCOPE: LSTN-01 - MODEL_VERSION: "6.0.0" + PARAMETER_VERSION: "0.0.1" SCHEMA_DIRECTORY: ./tests/resources/ OUTPUT_PATH: simtools-output INTEGRATION_TESTS: diff --git a/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_ground_to_utm_json.yml b/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_ground_to_utm_json.yml index 7af3563556..df84bdec89 100644 --- a/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_ground_to_utm_json.yml +++ b/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_ground_to_utm_json.yml @@ -3,9 +3,10 @@ CTA_SIMPIPE: TEST_NAME: ground_to_utm_json CONFIGURATION: INPUT: tests/resources/model_parameters/array_element_position_ground.json - SITE: South + SITE: North EXPORT: utm MODEL_VERSION: "6.0.0" + PARAMETER_VERSION: "2.0.0" OUTPUT_PATH: simtools-output OUTPUT_FILE: test.json LOG_LEVEL: DEBUG diff --git a/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_utm_to_ground_json.yml b/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_utm_to_ground_json.yml index dace3a0460..e884abab10 100644 --- a/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_utm_to_ground_json.yml +++ b/tests/integration_tests/config/convert_geo_coordinates_of_array_elements_utm_to_ground_json.yml @@ -3,9 +3,10 @@ CTA_SIMPIPE: TEST_NAME: utm_to_ground_json CONFIGURATION: INPUT: tests/resources/model_parameters/array_element_position_utm.json - SITE: South + SITE: North EXPORT: ground MODEL_VERSION: "6.0.0" + PARAMETER_VERSION: "2.0.0" OUTPUT_PATH: simtools-output OUTPUT_FILE: test.json LOG_LEVEL: DEBUG diff --git a/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml b/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml index 5d69f70fa1..166d7dec4a 100644 --- a/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml +++ b/tests/integration_tests/config/convert_model_parameter_from_simtel_num_gains.yml @@ -5,11 +5,8 @@ CTA_SIMPIPE: SIMTEL_CFG_FILE: ./tests/resources/simtel_config_test_la_palma.cfg SIMTEL_TELESCOPE_NAME: CT1 TELESCOPE: LSTN-01 - MODEL_VERSION: "6.0.0" + PARAMETER_VERSION: "0.1.0" SCHEMA: ./tests/resources/num_gains.schema.yml OUTPUT_PATH: simtools-output OUTPUT_FILE: num_gains.json USE_PLAIN_OUTPUT_PATH: True - INTEGRATION_TESTS: - - REFERENCE_OUTPUT_FILE: | - ./tests/resources/model_parameters/num_gains.json diff --git a/tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-Derived-Values.yml b/tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-two-files.yml similarity index 50% rename from tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-Derived-Values.yml rename to tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-two-files.yml index ca02aa1848..fcfaa20ad4 100644 --- a/tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-Derived-Values.yml +++ b/tests/integration_tests/config/db_get_file_from_db_CTA-Simulation-Model-two-files.yml @@ -1,8 +1,9 @@ CTA_SIMPIPE: APPLICATION: simtools-db-get-file-from-db - TEST_NAME: CTA-Simulation-Model-Derived-Values + TEST_NAME: CTA-Simulation-Model-two-files CONFIGURATION: - FILE_NAME: ray-tracing-North-LST-1-d10.0-za20.0_validate_optics.ecsv + FILE_NAME: [mirror_CTA-S-LST_v2020-04-07.dat, ray-tracing-North-LST-1-d10.0-za20.0_validate_optics.ecsv] OUTPUT_PATH: simtools-output INTEGRATION_TESTS: + - OUTPUT_FILE: mirror_CTA-S-LST_v2020-04-07.dat - OUTPUT_FILE: ray-tracing-North-LST-1-d10.0-za20.0_validate_optics.ecsv diff --git a/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml b/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml index 1fa4eebdd6..40d615fa36 100644 --- a/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml +++ b/tests/integration_tests/config/db_get_parameter_from_db_array_element_position_ground.yml @@ -2,13 +2,14 @@ CTA_SIMPIPE: APPLICATION: simtools-db-get-parameter-from-db TEST_NAME: telescope_parameter_array_element_position_ground CONFIGURATION: - SITE: South - TELESCOPE: SSTS-09 + SITE: North + TELESCOPE: MSTN-09 PARAMETER: array_element_position_ground MODEL_VERSION: 6.0.0 OUTPUT_FILE: test.json OUTPUT_PATH: simtools-output + LOG_LEVEL: DEBUG INTEGRATION_TESTS: - REFERENCE_OUTPUT_FILE: | ./tests/resources/model_parameters/array_element_position_ground.json - TOLERANCE: 1.e-2 + TOLERANCE: 1.e-5 diff --git a/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter.yml b/tests/integration_tests/config/db_get_parameter_from_db_telescope_model_version.yml similarity index 78% rename from tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter.yml rename to tests/integration_tests/config/db_get_parameter_from_db_telescope_model_version.yml index 063a1999a6..eab1256d72 100644 --- a/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter.yml +++ b/tests/integration_tests/config/db_get_parameter_from_db_telescope_model_version.yml @@ -1,6 +1,6 @@ CTA_SIMPIPE: APPLICATION: simtools-db-get-parameter-from-db - TEST_NAME: telescope_parameter + TEST_NAME: telescope_parameter_model_version CONFIGURATION: SITE: North TELESCOPE: LSTN-01 diff --git a/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml b/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml new file mode 100644 index 0000000000..7280170eb7 --- /dev/null +++ b/tests/integration_tests/config/db_get_parameter_from_db_telescope_parameter_version.yml @@ -0,0 +1,14 @@ +CTA_SIMPIPE: + APPLICATION: simtools-db-get-parameter-from-db + TEST_NAME: telescope_parameter_parameter_version + CONFIGURATION: + SITE: North + TELESCOPE: LSTN-01 + PARAMETER: mirror_list + PARAMETER_VERSION: 2.0.0 + OUTPUT_FILE: test.json + OUTPUT_PATH: simtools-output + LOG_LEVEL: DEBUG + INTEGRATION_TESTS: + - REFERENCE_OUTPUT_FILE: | + ./tests/resources/model_parameters/mirror_list.json diff --git a/tests/integration_tests/config/submit_model_parameter_from_external_submit_focus_offset.yml b/tests/integration_tests/config/submit_model_parameter_from_external_submit_focus_offset.yml index 5c03c7679f..37d5ac75a6 100644 --- a/tests/integration_tests/config/submit_model_parameter_from_external_submit_focus_offset.yml +++ b/tests/integration_tests/config/submit_model_parameter_from_external_submit_focus_offset.yml @@ -6,7 +6,7 @@ CTA_SIMPIPE: VALUE: "6.55 cm, 0.0 deg, 0.0, 0.0" INSTRUMENT: LSTN-01 SITE: North - MODEL_VERSION: 6.0.0 + PARAMETER_VERSION: 0.1.0 INPUT_META: null OUTPUT_PATH: simtools-output USE_PLAIN_OUTPUT_PATH: true diff --git a/tests/integration_tests/config/submit_model_parameter_from_external_submit_mirror_list.yml b/tests/integration_tests/config/submit_model_parameter_from_external_submit_mirror_list.yml index 2090c964c1..fcd48ed4ab 100644 --- a/tests/integration_tests/config/submit_model_parameter_from_external_submit_mirror_list.yml +++ b/tests/integration_tests/config/submit_model_parameter_from_external_submit_mirror_list.yml @@ -6,7 +6,7 @@ CTA_SIMPIPE: VALUE: mirror_CTA-100_1.20-86-0.04.dat INSTRUMENT: MSTS-design SITE: South - MODEL_VERSION: 6.0.0 + PARAMETER_VERSION: 0.1.0 INPUT_META: null OUTPUT_PATH: simtools-output USE_PLAIN_OUTPUT_PATH: true diff --git a/tests/integration_tests/config/submit_model_parameter_from_external_submit_num_gains.yml b/tests/integration_tests/config/submit_model_parameter_from_external_submit_num_gains.yml index 82ca438853..b01258086e 100644 --- a/tests/integration_tests/config/submit_model_parameter_from_external_submit_num_gains.yml +++ b/tests/integration_tests/config/submit_model_parameter_from_external_submit_num_gains.yml @@ -6,7 +6,7 @@ CTA_SIMPIPE: VALUE: 2 INSTRUMENT: LSTN-design SITE: North - MODEL_VERSION: 6.0.0 + PARAMETER_VERSION: 0.1.0 INPUT_META: ./tests/resources/num_gains.meta.yml OUTPUT_PATH: simtools-output USE_PLAIN_OUTPUT_PATH: true diff --git a/tests/integration_tests/config/submit_model_parameter_from_external_submit_reference_point_altitude.yml b/tests/integration_tests/config/submit_model_parameter_from_external_submit_reference_point_altitude.yml index a734f65a7c..10584dd611 100644 --- a/tests/integration_tests/config/submit_model_parameter_from_external_submit_reference_point_altitude.yml +++ b/tests/integration_tests/config/submit_model_parameter_from_external_submit_reference_point_altitude.yml @@ -6,7 +6,7 @@ CTA_SIMPIPE: VALUE: "2.177 km" INSTRUMENT: North SITE: North - MODEL_VERSION: 6.0.0 + PARAMETER_VERSION: 0.1.0 INPUT_META: null OUTPUT_PATH: simtools-output USE_PLAIN_OUTPUT_PATH: true diff --git a/tests/integration_tests/config/validate_file_using_schema_json_validate_model_parameter.yml b/tests/integration_tests/config/validate_file_using_schema_json_validate_model_parameter.yml index eed8237bd4..c6fc78e26e 100644 --- a/tests/integration_tests/config/validate_file_using_schema_json_validate_model_parameter.yml +++ b/tests/integration_tests/config/validate_file_using_schema_json_validate_model_parameter.yml @@ -3,5 +3,5 @@ CTA_SIMPIPE: TEST_NAME: json_validate_model_parameter CONFIGURATION: SCHEMA: tests/resources/num_gains.schema.yml - FILE_NAME: tests/resources/model_parameters/num_gains.json + FILE_NAME: tests/resources/model_parameters/num_gains-0.2.0.json DATA_TYPE: model_parameter diff --git a/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.1.0.yml b/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.1.0.yml index 8ec7f7dd57..bb7b660e4d 100644 --- a/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.1.0.yml +++ b/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.1.0.yml @@ -3,5 +3,5 @@ CTA_SIMPIPE: TEST_NAME: json_validate_schema-0.1.0 CONFIGURATION: SCHEMA: src/simtools/schemas/model_parameter.metaschema.yml - FILE_NAME: tests/resources/model_parameters/num_gains.json + FILE_NAME: tests/resources/model_parameters/num_gains-0.0.0.json DATA_TYPE: schema diff --git a/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.2.0.yml b/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.2.0.yml index 0280a891cb..4de540598a 100644 --- a/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.2.0.yml +++ b/tests/integration_tests/config/validate_file_using_schema_json_validate_schema-0.2.0.yml @@ -3,5 +3,5 @@ CTA_SIMPIPE: TEST_NAME: json_validate_schema-0.2.0 CONFIGURATION: SCHEMA: src/simtools/schemas/model_parameter.metaschema.yml - FILE_NAME: tests/resources/num_gains-schema-0.2.0.json + FILE_NAME: tests/resources/model_parameters/num_gains-0.2.0.json DATA_TYPE: schema diff --git a/tests/resources/model_parameters/array_element_position_ground.json b/tests/resources/model_parameters/array_element_position_ground.json index 1d2ac2a396..63082c00e7 100644 --- a/tests/resources/model_parameters/array_element_position_ground.json +++ b/tests/resources/model_parameters/array_element_position_ground.json @@ -1,11 +1,16 @@ { + "schema_version": "0.2.0", "parameter": "array_element_position_ground", - "instrument": "SSTS-09", - "site": "South", - "version": "6.0.0", - "value": "-4.79 -499.24 39.75", + "instrument": "MSTN-09", + "site": "North", + "parameter_version": "2.0.0", + "unique_id": null, + "value": [ + 221.68, + -355.66, + 49.0 + ], "unit": "m", "type": "float64", - "applicable": true, "file": false } diff --git a/tests/resources/model_parameters/array_element_position_utm.json b/tests/resources/model_parameters/array_element_position_utm.json index 8cc265657c..159635e62c 100644 --- a/tests/resources/model_parameters/array_element_position_utm.json +++ b/tests/resources/model_parameters/array_element_position_utm.json @@ -1,11 +1,16 @@ { + "schema_version": "0.2.0", "parameter": "array_element_position_utm", - "instrument": "SSTS-09", - "site": "South", - "version": "6.0.0", - "value": "367321.0 7269466.0 2183.5", + "instrument": "MSTN-09", + "site": "North", + "parameter_version": "2.0.0", + "unique_id": null, + "value": [ + 217970.0, + 3185280.3, + 2196.0 + ], "unit": "m", "type": "float64", - "applicable": true, "file": false } diff --git a/tests/resources/model_parameters/mirror_list.json b/tests/resources/model_parameters/mirror_list.json new file mode 100644 index 0000000000..7371c99c62 --- /dev/null +++ b/tests/resources/model_parameters/mirror_list.json @@ -0,0 +1,12 @@ +{ + "schema_version": "0.2.0", + "parameter": "mirror_list", + "instrument": "LSTN-01", + "site": "North", + "parameter_version": "2.0.0", + "unique_id": null, + "value": "mirror_CTA-N-LST1_v2019-03-31.dat", + "unit": null, + "type": "string", + "file": true +} diff --git a/tests/resources/model_parameters/num_gains.json b/tests/resources/model_parameters/num_gains-0.0.0.json similarity index 100% rename from tests/resources/model_parameters/num_gains.json rename to tests/resources/model_parameters/num_gains-0.0.0.json diff --git a/tests/resources/num_gains-schema-0.2.0.json b/tests/resources/model_parameters/num_gains-0.2.0.json similarity index 100% rename from tests/resources/num_gains-schema-0.2.0.json rename to tests/resources/model_parameters/num_gains-0.2.0.json diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg index 66e6180504..59d3c22b22 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-5.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2158.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-6.0.0_test.cfg index 2f5d4ec29a..2866f46b8c 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-01-6.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2156.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg index 6a22e82a58..d5ab1b8e6a 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-5.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2158.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none @@ -19,7 +20,7 @@ atmospheric_transmission = atm_trans_2158_1_3_2_0_0_0.1_0.1.dat axes_offsets = 0.0 0.0 camera_body_diameter = 348.0 camera_body_shape = 2 -camera_config_file = camera_CTA-LST-1_analogsum21_v2020-04-14.dat +camera_config_file = camera_CTA-LST-234_analogsum21_v2020-04-14.dat camera_degraded_efficiency = 1.0 camera_degraded_map = none camera_depth = 0.0 @@ -89,12 +90,12 @@ mirror_align_random_vertical = 0.0039 28.0 0.0 0.0 mirror_class = 0 mirror_degraded_reflection = 1.0 mirror_focal_length = 0.0 -mirror_list = mirror_CTA-N-LST1_v2019-03-31.dat +mirror_list = mirror_CTA-N-LST2_v2020-04-07.dat mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.238006 +nightsky_background = all: 0.244985 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 @@ -104,12 +105,12 @@ photon_delay = 19.0 pixeltrg_time_step = 0.0 pm_average_gain = 40000.0 pm_collection_efficiency = 1.0 -pm_gain_index = 4.5 +pm_gain_index = 3.92 pm_photoelectron_spectrum = spe_LST_2020-05-09_AP2.0e-4.dat -pm_transit_time = 24.74 9.0 350.0 1066.0 -pm_voltage_variation = 0.041 +pm_transit_time = 20.89 9.0 350.0 1135.0 +pm_voltage_variation = 0.03 qe_variation = 0.03 -quantum_efficiency = qe_lst1_20200318_high+low.dat +quantum_efficiency = qe_lst2-4_20200318_high+low.dat random_focal_length = 0.0 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-6.0.0_test.cfg index 2be55beae9..bbcf6a03db 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-02-6.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2156.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg index b284166cef..ae798dda40 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-5.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2158.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none @@ -19,7 +20,7 @@ atmospheric_transmission = atm_trans_2158_1_3_2_0_0_0.1_0.1.dat axes_offsets = 0.0 0.0 camera_body_diameter = 348.0 camera_body_shape = 2 -camera_config_file = camera_CTA-LST-1_analogsum21_v2020-04-14.dat +camera_config_file = camera_CTA-LST-234_analogsum21_v2020-04-14.dat camera_degraded_efficiency = 1.0 camera_degraded_map = none camera_depth = 0.0 @@ -89,12 +90,12 @@ mirror_align_random_vertical = 0.0039 28.0 0.0 0.0 mirror_class = 0 mirror_degraded_reflection = 1.0 mirror_focal_length = 0.0 -mirror_list = mirror_CTA-N-LST1_v2019-03-31.dat +mirror_list = mirror_CTA-N-LST3_v2020-04-07.dat mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.238006 +nightsky_background = all: 0.244985 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 @@ -104,12 +105,12 @@ photon_delay = 19.0 pixeltrg_time_step = 0.0 pm_average_gain = 40000.0 pm_collection_efficiency = 1.0 -pm_gain_index = 4.5 +pm_gain_index = 3.92 pm_photoelectron_spectrum = spe_LST_2020-05-09_AP2.0e-4.dat -pm_transit_time = 24.74 9.0 350.0 1066.0 -pm_voltage_variation = 0.041 +pm_transit_time = 20.89 9.0 350.0 1135.0 +pm_voltage_variation = 0.03 qe_variation = 0.03 -quantum_efficiency = qe_lst1_20200318_high+low.dat +quantum_efficiency = qe_lst2-4_20200318_high+low.dat random_focal_length = 0.0 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-6.0.0_test.cfg index a813caea12..32135153f0 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-03-6.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2156.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg index 95d4eef87c..af8dcf89c1 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-5.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2158.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none @@ -19,7 +20,7 @@ atmospheric_transmission = atm_trans_2158_1_3_2_0_0_0.1_0.1.dat axes_offsets = 0.0 0.0 camera_body_diameter = 348.0 camera_body_shape = 2 -camera_config_file = camera_CTA-LST-1_analogsum21_v2020-04-14.dat +camera_config_file = camera_CTA-LST-234_analogsum21_v2020-04-14.dat camera_degraded_efficiency = 1.0 camera_degraded_map = none camera_depth = 0.0 @@ -89,12 +90,12 @@ mirror_align_random_vertical = 0.0039 28.0 0.0 0.0 mirror_class = 0 mirror_degraded_reflection = 1.0 mirror_focal_length = 0.0 -mirror_list = mirror_CTA-N-LST1_v2019-03-31.dat +mirror_list = mirror_CTA-N-LST4_v2020-04-07.dat mirror_offset = 93.25 mirror_reflection_random_angle = 0.0075 0.125 0.037 mirror_reflectivity = ref_LST_2020-04-23.dat multiplicity_offset = -0.5 -nightsky_background = all: 0.238006 +nightsky_background = all: 0.244985 nsb_autoscale_airmass = 0.84 0.29 nsb_offaxis = 0.0 0.0 0.0 0.0 0.0 num_gains = 2 @@ -104,12 +105,12 @@ photon_delay = 19.0 pixeltrg_time_step = 0.0 pm_average_gain = 40000.0 pm_collection_efficiency = 1.0 -pm_gain_index = 4.5 +pm_gain_index = 3.92 pm_photoelectron_spectrum = spe_LST_2020-05-09_AP2.0e-4.dat -pm_transit_time = 24.74 9.0 350.0 1066.0 -pm_voltage_variation = 0.041 +pm_transit_time = 20.89 9.0 350.0 1135.0 +pm_voltage_variation = 0.03 qe_variation = 0.03 -quantum_efficiency = qe_lst1_20200318_high+low.dat +quantum_efficiency = qe_lst2-4_20200318_high+low.dat random_focal_length = 0.0 0.0 telescope_random_angle = 0.0 telescope_random_error = 0.0 diff --git a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-6.0.0_test.cfg index 7fdef1c87b..459b84c63b 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-LSTN-04-6.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2156.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg index a58c7641d4..5ab6e2c1ec 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-5.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2158.0 +array_window = 1000.0 asum_clipping = 0.0 asum_offset = 0.0 asum_shaping_file = none @@ -134,7 +135,7 @@ transit_time_compensate_error = 0.0 transit_time_compensate_step = 0.0 transit_time_error = 0.5 transit_time_jitter = 0.7 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 3 iobuf_maximum = 1000000000 diff --git a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg index 161c3b32ae..ad82359215 100644 --- a/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-North-MSTN-01-6.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2156.0 +array_window = 1000.0 asum_clipping = 0.0 asum_offset = 0.0 asum_shaping_file = none @@ -134,7 +135,7 @@ transit_time_compensate_error = 0.0 transit_time_compensate_step = 0.0 transit_time_error = 0.5 transit_time_jitter = 0.7 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 3 iobuf_maximum = 1000000000 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg index 9cab0f5f51..11c3b8bc93 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-5.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2147.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-6.0.0_test.cfg index 51a89bc8ca..d17df29ae8 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-LSTS-01-6.0.0_test.cfg @@ -11,6 +11,7 @@ #endif altitude = 2147.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg index e054a6db70..6eaccbd233 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-5.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2147.0 +array_window = 1000.0 asum_clipping = 9999.0 asum_offset = 0.0 asum_shaping_file = none @@ -134,7 +135,7 @@ transit_time_compensate_error = 0.2 transit_time_compensate_step = 1.0 transit_time_error = 0.0 transit_time_jitter = 0.7 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 3 iobuf_maximum = 1000000000 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg index c06b51487e..b7751ba1df 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-MSTS-01-6.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2147.0 +array_window = 1000.0 asum_clipping = 0.0 asum_offset = 0.0 asum_shaping_file = none @@ -134,7 +135,7 @@ transit_time_compensate_error = 0.2 transit_time_compensate_step = 1.0 transit_time_error = 0.0 transit_time_jitter = 0.7 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 99 iobuf_maximum = 1000000000 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg index 5136f9c1b2..b379214c19 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-5.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2147.0 +array_window = 1000.0 asum_clipping = 0.0 asum_offset = 0.0 asum_shaping_file = none @@ -48,18 +49,18 @@ discriminator_var_gate_length = 0.0 discriminator_var_sigsum_over_threshold = 0.0 discriminator_var_threshold = 0.25 discriminator_var_time_over_threshold = 0.0 -dsum_clipping = 85 +dsum_clipping = 0 dsum_ignore_below = 0 dsum_offset = 0.0 -dsum_pedsub = 0 +dsum_pedsub = 1 dsum_pre_clipping = 0 -dsum_prescale = 40 256 -dsum_presum_max = 127 -dsum_presum_shift = 1 -dsum_shaping_file = CTA-ULTRA6-dsum-shaping-FlashCam-2a-int.dat +dsum_prescale = all: 0 +dsum_presum_max = 0 +dsum_presum_shift = 0 +dsum_shaping_file = none dsum_shaping_renormalize = 0 dsum_threshold = 0.0 -dsum_zero_clip = 1 +dsum_zero_clip = 0 effective_focal_length = 215.191 0.0 0.0 0.0 0.0 fadc_ac_coupled = 0 fadc_amplitude = 2.77 @@ -97,8 +98,8 @@ focal_surface_ref_radius = 1.0 focus_offset = 0.0 0.0 0.0 0.0 gain_variation = 0.05 mirror2_degraded_reflection = 1.0 -mirror_align_random_horizontal = 0.0 31.0 0.0135 0.012 -mirror_align_random_vertical = 0.0 31.0 0.0135 0.012 +mirror_align_random_horizontal = 0.0 0.0 0.0 0.0 +mirror_align_random_vertical = 0.0 0.0 0.0 0.0 mirror_class = 2 mirror_degraded_reflection = 1.0 mirror_list = none @@ -142,11 +143,11 @@ telescope_transmission = 0.92362 1.0 0.03668 1.7454 0.858 0.0 teltrig_min_sigsum = 0.0 teltrig_min_time = 0.5 transit_time_calib_error = 0.0 -transit_time_compensate_error = 0.2 -transit_time_compensate_step = 1.0 +transit_time_compensate_error = 0.0 +transit_time_compensate_step = 0.0 transit_time_error = 0.1 transit_time_jitter = 0.01 -trigger_current_limit = 2000.0 +trigger_current_limit = 20.0 trigger_delay_compensation = 0.0 0.0 0.0 0.0 trigger_pixels = 2 iobuf_maximum = 1000000000 diff --git a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg index b58061e91e..215ba51226 100644 --- a/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg +++ b/tests/resources/sim_telarray_configurations/CTA-South-SSTS-01-6.0.0_test.cfg @@ -12,6 +12,7 @@ adjust_gain = 1.0 altitude = 2147.0 +array_window = 1000.0 asum_clipping = 0.0 asum_offset = 0.0 asum_shaping_file = none diff --git a/tests/unit_tests/configuration/test_commandline_parser.py b/tests/unit_tests/configuration/test_commandline_parser.py index 38e684a996..4f14afe607 100644 --- a/tests/unit_tests/configuration/test_commandline_parser.py +++ b/tests/unit_tests/configuration/test_commandline_parser.py @@ -179,25 +179,30 @@ def test_simulation_model(): assert all(action.dest != "site" for action in group._group_actions) assert all(action.dest != "telescope" for action in group._group_actions) - # Site model can exist without a telescope model + # Site model can exist without a telescope model, model and parameter version _parser_s = parser.CommandLineParser() - _parser_s.initialize_default_arguments(simulation_model=["site", "model_version"]) + _parser_s.initialize_default_arguments( + simulation_model=["site", "model_version", "parameter_version"] + ) job_groups = _parser_s._action_groups assert SIMULATION_MODEL_STRING in [str(group.title) for group in job_groups] for group in job_groups: if str(group.title) == SIMULATION_MODEL_STRING: assert any(action.dest == "model_version" for action in group._group_actions) + assert any(action.dest == "parameter_version" for action in group._group_actions) assert any(action.dest == "site" for action in group._group_actions) assert all(action.dest != "telescope" for action in group._group_actions) - # No telescope model without site model + # No telescope model without site model; parameter_version only _parser_t = parser.CommandLineParser() - _parser_t.initialize_default_arguments(simulation_model=["telescope", "site", "model_version"]) + _parser_t.initialize_default_arguments( + simulation_model=["telescope", "site", "parameter_version"] + ) job_groups = _parser_t._action_groups assert SIMULATION_MODEL_STRING in [str(group.title) for group in job_groups] for group in job_groups: if str(group.title) == SIMULATION_MODEL_STRING: - assert any(action.dest == "model_version" for action in group._group_actions) + assert any(action.dest == "parameter_version" for action in group._group_actions) assert any(action.dest == "site" for action in group._group_actions) assert any(action.dest == "telescope" for action in group._group_actions) diff --git a/tests/unit_tests/corsika/test_corsika_config.py b/tests/unit_tests/corsika/test_corsika_config.py index b0aa400674..a818a03094 100644 --- a/tests/unit_tests/corsika/test_corsika_config.py +++ b/tests/unit_tests/corsika/test_corsika_config.py @@ -29,9 +29,12 @@ def corsika_configuration_parameters(gcm2): return { "corsika_iact_max_bunches": {"value": 1000000, "unit": None}, "corsika_cherenkov_photon_bunch_size": {"value": 5.0, "unit": None}, - "corsika_cherenkov_photon_wavelength_range": {"value": "240. 700.", "unit": "nm"}, + "corsika_cherenkov_photon_wavelength_range": {"value": [240.0, 1000.0], "unit": "nm"}, "corsika_first_interaction_height": {"value": 0.0, "unit": "cm"}, - "corsika_particle_kinetic_energy_cutoff": {"value": "0.3 0.1 0.020 0.020", "unit": "GeV"}, + "corsika_particle_kinetic_energy_cutoff": { + "value": [0.3, 0.1, 0.020, 0.020], + "unit": "GeV", + }, "corsika_longitudinal_shower_development": {"value": 20.0, "unit": gcm2}, "corsika_iact_split_auto": {"value": 15000000, "unit": None}, "corsika_starting_grammage": {"value": 0.0, "unit": gcm2}, @@ -125,10 +128,10 @@ def test_input_config_corsika_starting_grammage(corsika_config_mock_array_model, def test_input_config_corsika_particle_kinetic_energy_cutoff(corsika_config_mock_array_model): assert corsika_config_mock_array_model._input_config_corsika_particle_kinetic_energy_cutoff( - {"value": "0.3 0.1 0.020 0.020", "unit": "GeV"} + {"value": [0.3, 0.1, 0.020, 0.020], "unit": "GeV"} ) == ["0.3 0.1 0.02 0.02"] assert corsika_config_mock_array_model._input_config_corsika_particle_kinetic_energy_cutoff( - {"value": "0.3 0.1 0.020 0.020", "unit": "TeV"} + {"value": [0.3, 0.1, 0.020, 0.020], "unit": "TeV"} ) == ["300.0 100.0 20.0 20.0"] @@ -154,8 +157,8 @@ def test_corsika_configuration_cherenkov_parameters( def test_input_config_corsika_cherenkov_wavelength(corsika_config_mock_array_model): assert corsika_config_mock_array_model._input_config_corsika_cherenkov_wavelength( - {"value": "240. 700.", "unit": "nm"} - ) == ["240.0", "700.0"] + {"value": [240.0, 1000.0], "unit": "nm"} + ) == ["240.0", "1000.0"] def test_corsika_configuration_iact_parameters( @@ -174,6 +177,14 @@ def test_input_config_io_buff(corsika_config_mock_array_model): corsika_config_mock_array_model._input_config_io_buff({"value": 800, "unit": "MB"}) == "800MB" ) + assert ( + corsika_config_mock_array_model._input_config_io_buff({"value": 8.5, "unit": "MB"}) + == "8500000" + ) + assert ( + corsika_config_mock_array_model._input_config_io_buff({"value": 800, "unit": "kB"}) + == "800000" + ) def test_corsika_configuration_debugging_parameters(corsika_config_mock_array_model): diff --git a/tests/unit_tests/data_model/test_model_data_writer.py b/tests/unit_tests/data_model/test_model_data_writer.py index 55e1090b02..580d1622a9 100644 --- a/tests/unit_tests/data_model/test_model_data_writer.py +++ b/tests/unit_tests/data_model/test_model_data_writer.py @@ -208,14 +208,14 @@ def test_json_numpy_encoder(): def test_dump_model_parameter(tmp_test_directory): - model_version = "6.0.0" + parameter_version = "1.1.0" instrument = "LSTN-01" # single value, no unit num_gains_dict = writer.ModelDataWriter.dump_model_parameter( parameter_name="num_gains", value=2, instrument=instrument, - model_version=model_version, + parameter_version=parameter_version, output_file="num_gains.json", output_path=tmp_test_directory, use_plain_output_path=True, @@ -230,7 +230,7 @@ def test_dump_model_parameter(tmp_test_directory): parameter_name="array_element_position_utm", value=[217.6596 * u.km, 3184.9951 * u.km, 218500.0 * u.cm], instrument=instrument, - model_version=model_version, + parameter_version=parameter_version, output_file="array_element_position_utm.json", output_path=tmp_test_directory, use_plain_output_path=True, @@ -248,7 +248,7 @@ def test_dump_model_parameter(tmp_test_directory): parameter_name="focus_offset", value=[6.55 * u.cm, 0.0 * u.deg, 0.0, 0.0], instrument="LSTN-01", - model_version="6.0.0", + parameter_version=parameter_version, output_file="focus_offset.json", output_path=tmp_test_directory, use_plain_output_path=True, @@ -264,16 +264,17 @@ def test_get_validated_parameter_dict(): w1 = writer.ModelDataWriter() assert w1.get_validated_parameter_dict( - parameter_name="num_gains", value=2, instrument="MSTN-01", model_version="0.0.1" + parameter_name="num_gains", value=2, instrument="MSTN-01", parameter_version="0.0.1" ) == { + "schema_version": "0.1.0", "parameter": "num_gains", "instrument": "MSTN-01", "site": "North", - "version": "0.0.1", + "parameter_version": "0.0.1", + "unique_id": None, "value": 2, "unit": u.Unit(""), "type": "int", - "applicable": True, "file": False, } @@ -281,16 +282,17 @@ def test_get_validated_parameter_dict(): parameter_name="transit_time_error", value=5.0 * u.ns, instrument="LSTN-01", - model_version="0.0.1", + parameter_version="0.0.1", ) == { + "schema_version": "0.1.0", "parameter": "transit_time_error", "instrument": "LSTN-01", "site": "North", - "version": "0.0.1", + "parameter_version": "0.0.1", + "unique_id": None, "value": 5, "unit": u.Unit("ns"), "type": "double", - "applicable": True, "file": False, } @@ -298,40 +300,21 @@ def test_get_validated_parameter_dict(): parameter_name="reference_point_altitude", value=2.7 * u.km, instrument="North", - model_version="0.0.1", + parameter_version="0.0.1", ) == { + "schema_version": "0.1.0", "parameter": "reference_point_altitude", "instrument": "North", "site": "North", - "version": "0.0.1", + "parameter_version": "0.0.1", + "unique_id": None, "value": 2700.0, "unit": u.Unit("m"), "type": "double", - "applicable": True, "file": False, } -def test_get_parameter_applicability(num_gains_schema): - - w1 = writer.ModelDataWriter() - w1.schema_dict = num_gains_schema - - assert w1._get_parameter_applicability("LSTN-01") - - # illuminator does not have gains - assert not w1._get_parameter_applicability("ILLN-01") - - # change schema dict - w1.schema_dict["instrument"]["type"].append("LSTN-55") - assert w1._get_parameter_applicability("LSTN-55") - - # change schema dict - w1.schema_dict["instrument"].pop("type") - with pytest.raises(KeyError): - w1._get_parameter_applicability("LSTN-01") - - def test_prepare_data_dict_for_writing(): data_dict_1 = {} diff --git a/tests/unit_tests/data_model/test_validate_data.py b/tests/unit_tests/data_model/test_validate_data.py index 82da96abc6..2b3a90647b 100644 --- a/tests/unit_tests/data_model/test_validate_data.py +++ b/tests/unit_tests/data_model/test_validate_data.py @@ -110,7 +110,7 @@ def test_validate_and_transform(caplog, mocker): assert isinstance(_table, Table) assert "Validating tabled data from:" in caplog.text - data_validator.data_file_name = "tests/resources/model_parameters/num_gains.json" + data_validator.data_file_name = "tests/resources/model_parameters/num_gains-0.2.0.json" data_validator.schema_file_name = "tests/resources/num_gains.schema.yml" mock_prepare_model_parameter = mocker.patch( "simtools.data_model.validate_data.DataValidator._prepare_model_parameter" @@ -141,14 +141,14 @@ def test_validate_data_file(caplog): def test_validate_parameter_and_file_name(): data_validator = validate_data.DataValidator() - data_validator.data_file_name = "tests/resources/model_parameters/num_gains.json" + data_validator.data_file_name = "tests/resources/model_parameters/num_gains-0.2.0.json" data_validator.schema_file_name = "tests/resources/num_gains.schema.yml" data_validator.validate_and_transform() data_validator.data_dict["parameter"] = "incorrect_name" with pytest.raises( ValueError, - match="Parameter name in data dict incorrect_name and file name num_gains do not match.", + match="Parameter name in data dict incorrect_name and file name num_gains-0.2.0 do not match.", ): data_validator.validate_parameter_and_file_name() diff --git a/tests/unit_tests/db/test_db_array_elements.py b/tests/unit_tests/db/test_db_array_elements.py deleted file mode 100644 index 8d2a0f43e9..0000000000 --- a/tests/unit_tests/db/test_db_array_elements.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/python3 - -import time - -import pytest - -from simtools.db import db_array_elements - - -def test_get_array_elements(db, model_version): - - time_1 = time.time() - db_array_elements.get_array_elements( - db.get_collection(db_name=None, collection_name="telescopes"), - db.model_version(model_version), - ) - time_2 = time.time() - available_telescopes = db_array_elements.get_array_elements( - db.get_collection(db_name=None, collection_name="telescopes"), - db.model_version(model_version), - ) - time_3 = time.time() - - # check that the second call is much faster than the first one - assert (time_2 - time_1) > 0.1 * (time_3 - time_2) - - expected_telescope_names = { - "LSTN-01": "LSTN-design", - "LSTN-02": "LSTN-design", - "LSTN-03": "LSTN-design", - "LSTN-04": "LSTN-design", - "MSTN-15": "MSTN-design", - "MSTS-10": "MSTS-design", - "MSTS-301": "MSTN-design", - } - for _t in expected_telescope_names: - assert _t in available_telescopes - assert expected_telescope_names[_t] in available_telescopes[_t] - - available_calibration_devices = db_array_elements.get_array_elements( - db.get_collection(db_name=None, collection_name="calibration_devices"), - db.model_version(model_version), - ) - expected_calibration_devices = { - "ILLN-01": "ILLN-design", - "ILLS-02": "ILLS-design", - } - for _d in expected_calibration_devices: - assert _d in available_calibration_devices - assert expected_calibration_devices[_d] in available_calibration_devices[_d] - - with pytest.raises(ValueError, match=r"^No array elements found in DB collection"): - db_array_elements.get_array_elements( - db.get_collection(db_name=None, collection_name="wrong_collection"), - db.model_version(model_version), - ) - - -def test_get_array_element_list_for_db_query(db, model_version): - - assert db_array_elements.get_array_element_list_for_db_query( - "LSTN-01", db=db, model_version=model_version, collection="telescopes" - ) == ["LSTN-design", "LSTN-01"] - - assert db_array_elements.get_array_element_list_for_db_query( - "MSTS-10", db=db, model_version=model_version, collection="telescopes" - ) == ["MSTS-design", "MSTS-10"] - - assert db_array_elements.get_array_element_list_for_db_query( - "MSTS-301", db=db, model_version=model_version, collection="telescopes" - ) == ["MSTN-design", "MSTS-301"] - - assert db_array_elements.get_array_element_list_for_db_query( - "MSTS-design", db=db, model_version=model_version, collection="telescopes" - ) == ["MSTS-design"] - - with pytest.raises(ValueError, match=r"^Array element MSTS-301 not found in DB."): - db_array_elements.get_array_element_list_for_db_query( - "MSTS-301", db=db, model_version=model_version, collection="calibration_devices" - ) - - assert db_array_elements.get_array_element_list_for_db_query( - "LSTN-02", db=db, model_version=model_version, collection="configuration_sim_telarray" - ) == ["LSTN-design"] - - -def test_get_array_elements_of_type(db, model_version): - available_telescopes = db_array_elements.get_array_elements_of_type( - array_element_type="LSTN", db=db, model_version=model_version, collection="telescopes" - ) - assert available_telescopes == ["LSTN-01", "LSTN-02", "LSTN-03", "LSTN-04"] - - available_calibration_devices = db_array_elements.get_array_elements_of_type( - array_element_type="ILLS", - db=db, - model_version=model_version, - collection="calibration_devices", - ) - assert available_calibration_devices == ["ILLS-01", "ILLS-02", "ILLS-03", "ILLS-04"] diff --git a/tests/unit_tests/db/test_db_handler.py b/tests/unit_tests/db/test_db_handler.py index 73e5d5c36a..b28efdec92 100644 --- a/tests/unit_tests/db/test_db_handler.py +++ b/tests/unit_tests/db/test_db_handler.py @@ -2,19 +2,26 @@ import copy import logging -import re -import uuid +from pathlib import Path +from unittest.mock import call import pytest +from bson.objectid import ObjectId from simtools.db import db_handler +from simtools.utils import names logger = logging.getLogger() -@pytest.fixture -def random_id(): - return uuid.uuid4().hex +@pytest.fixture(autouse=True) +def reset_db_client(): + """Reset db_client before each test.""" + # If using the class-level db_client: + db_handler.DatabaseHandler.db_client = None + yield # allows the test to run + # After the test, reset any side-effects (if necessary): + db_handler.DatabaseHandler.db_client = None @pytest.fixture @@ -24,13 +31,18 @@ def db_no_config_file(): @pytest.fixture -def _db_cleanup(db, random_id): - yield - # Cleanup - logger.info(f"dropping sandbox_{random_id} collections") - db.db_client[f"sandbox_{random_id}"]["telescopes"].drop() - db.db_client[f"sandbox_{random_id}"]["calibration_devices"].drop() - db.db_client[f"sandbox_{random_id}"]["sites"].drop() +def test_db(): + return "test_db" + + +@pytest.fixture +def test_file(): + return "test_file.dat" + + +@pytest.fixture +def test_file_2(): + return "test_file_2.dat" @pytest.fixture @@ -39,12 +51,36 @@ def fs_files(): @pytest.fixture -def _db_cleanup_file_sandbox(db_no_config_file, random_id, fs_files): - yield - # Cleanup - logger.info("Dropping the temporary files in the sandbox") - db_no_config_file.db_client[f"sandbox_{random_id}"]["fs.chunks"].drop() - db_no_config_file.db_client[f"sandbox_{random_id}"][fs_files].drop() +def value_unit_type(): + return "simtools.db.db_handler.value_conversion.get_value_unit_type" + + +@pytest.fixture +def mock_open(mocker): + return mocker.patch("builtins.open", mocker.mock_open(read_data=b"file_content")) + + +@pytest.fixture +def validate_model_parameter(): + return "simtools.db.db_handler.validate_data.DataValidator.validate_model_parameter" + + +@pytest.fixture +def mock_gridfs(mocker): + return mocker.patch("simtools.db.db_handler.gridfs.GridFS") + + +def test_set_up_connection_no_config(): + """Test _set_up_connection with no configuration.""" + db = db_handler.DatabaseHandler(mongo_db_config=None) + db._set_up_connection() + assert db_handler.DatabaseHandler.db_client is None + + +def test_set_up_connection_with_config(db): + """Test _set_up_connection with valid configuration.""" + db._set_up_connection() + assert isinstance(db_handler.DatabaseHandler.db_client, db_handler.MongoClient) def test_valid_db_config(db, db_config): @@ -59,6 +95,28 @@ def test_valid_db_config(db, db_config): db._validate_mongo_db_config({"wrong_config": "wrong"}) +def test_open_mongo_db_direct_connection(mocker, db, db_config): + """Test _open_mongo_db with direct connection configuration.""" + db_config["db_server"] = "localhost" + mock_mongo_client = mocker.patch( + "simtools.db.db_handler.MongoClient", return_value="mock_client" + ) + db.mongo_db_config = db_config + client = db._open_mongo_db() + assert client == "mock_client" + mock_mongo_client.assert_called_once_with( + db_config["db_server"], + port=db_config["db_api_port"], + username=db_config["db_api_user"], + password=db_config["db_api_pw"], + authSource=db_config.get("db_api_authentication_database", "admin"), + directConnection=True, + ssl=False, + tlsallowinvalidhostnames=True, + tlsallowinvalidcertificates=True, + ) + + def test_find_latest_simulation_model_db(db, db_no_config_file, mocker): db_no_config_file._find_latest_simulation_model_db() @@ -76,360 +134,239 @@ def test_find_latest_simulation_model_db(db, db_no_config_file, mocker): db_copy._find_latest_simulation_model_db() db_names = [ - "CTAO-Simulation-Model-v0-3-0", - "CTAO-Simulation-Model-v0-2-0", - "CTAO-Simulation-Model-v0-1-19", - "CTAO-Simulation-Model-v0-3-9", - "CTAO-Simulation-Model-v0-3-19", - "CTAO-Simulation-Model-v0-3-0", - "CTAO-Simulation-Model-v0-3-0-alpha-2", - "CTAO-Simulation-Model-v0-4-19-alpha-1", - "CTAO-Simulation-Model-v0-4-19-dev1", + "CTAO-Simulation-ModelParameters-v0-3-0", + "CTAO-Simulation-ModelParameters-v0-2-0", + "CTAO-Simulation-ModelParameters-v0-1-19", + "CTAO-Simulation-ModelParameters-v0-3-9", + "CTAO-Simulation-ModelParameters-v0-3-19", + "CTAO-Simulation-ModelParameters-v0-3-0", + "CTAO-Simulation-ModelParameters-v0-3-0-alpha-2", + "CTAO-Simulation-ModelParameters-v0-4-19-alpha-1", + "CTAO-Simulation-ModelParameters-v0-4-19-dev1", ] mocker.patch.object(db_copy.db_client, "list_database_names", return_value=db_names) - db_copy.mongo_db_config["db_simulation_model"] = "CTAO-Simulation-Model-LATEST" + db_copy.mongo_db_config["db_simulation_model"] = "CTAO-Simulation-ModelParameters-LATEST" db_copy._find_latest_simulation_model_db() - assert db_copy.mongo_db_config["db_simulation_model"] == "CTAO-Simulation-Model-v0-3-19" + assert ( + db_copy.mongo_db_config["db_simulation_model"] == "CTAO-Simulation-ModelParameters-v0-3-19" + ) -def test_reading_db_lst_without_simulation_repo(db, model_version): +def test_get_model_parameter(db, mocker): + """Test get_model_parameter method.""" + mock_read_mongo_db = mocker.patch.object( + db, "_read_mongo_db", return_value={"parameter": "value"} + ) - db_copy = copy.deepcopy(db) - db_copy.mongo_db_config["db_simulation_model_url"] = None - pars = db.get_model_parameters("North", "LSTN-01", model_version, collection="telescopes") - assert pars["parabolic_dish"]["value"] == 1 - assert pars["camera_pixels"]["value"] == 1855 - - -def test_reading_db_lst(db, model_version): - logger.info("----Testing reading LST-North-----") - pars = db.get_model_parameters("North", "LSTN-01", model_version, collection="telescopes") - if db.mongo_db_config: - assert pars["parabolic_dish"]["value"] == 1 - assert pars["camera_pixels"]["value"] == 1855 - else: - assert pars["parabolic_dish"] == 1 - assert pars["camera_pixels"] == 1855 - - -def test_reading_db_mst_nc(db, model_version): - logger.info("----Testing reading MST-North-----") - pars = db.get_model_parameters("North", "MSTN-design", model_version, collection="telescopes") - if db.mongo_db_config: - assert pars["camera_pixels"]["value"] == 1855 - else: - assert pars["camera_pixels"] == 1855 - - -def test_reading_db_mst_fc(db, model_version): - logger.info("----Testing reading MST-South-----") - pars = db.get_model_parameters("South", "MSTS-design", model_version, collection="telescopes") - if db.mongo_db_config: - assert pars["camera_pixels"]["value"] == 1764 - else: - assert pars["camera_pixels"] == 1764 - - -def test_reading_db_sst(db, model_version): - logger.info("----Testing reading SST-----") - pars = db.get_model_parameters("South", "SSTS-design", model_version, collection="telescopes") - if db.mongo_db_config: - assert pars["camera_pixels"]["value"] == 2048 - else: - assert pars["camera_pixels"] == 2048 - - -@pytest.mark.xfail(reason="Test requires Derived-Values Database") -def test_get_derived_values(db, model_version_prod5): - logger.info("----Testing reading derived values-----") - try: - pars = db.get_derived_values("North", "LSTN-01", model_version_prod5) - assert ( - pars["ray_tracing"]["value"] - == "ray-tracing-North-LST-1-d10.0-za20.0_validate_optics.ecsv" - ) - except ValueError: - logger.error("Derived DB not updated for new telescope names. Expect failure") - raise AssertionError - - with pytest.raises(ValueError, match=r"^abc"): - pars = db.get_derived_values("North", None, model_version_prod5) - - -def test_get_sim_telarray_configuration_parameters(db, model_version): - - _pars = db.get_model_parameters( - "North", "LSTN-01", model_version, collection="configuration_sim_telarray" - ) - assert "min_photoelectrons" in _pars - - _pars = db.get_model_parameters( - "North", "LSTN-design", model_version, collection="configuration_sim_telarray" - ) - assert "min_photoelectrons" in _pars - - -@pytest.mark.usefixtures("_db_cleanup") -def test_copy_array_element_db(db, random_id, io_handler, model_version): - logger.info("----Testing copying a whole telescope-----") - db.copy_array_element( - db_name=None, - element_to_copy="LSTN-01", - version_to_copy=model_version, - new_array_element_name="LSTN-test", - collection_name="telescopes", - db_to_copy_to=f"sandbox_{random_id}", - collection_to_copy_to="telescopes", - ) - pars = db.read_mongo_db( - db_name=f"sandbox_{random_id}", - array_element_name="LSTN-test", - model_version=model_version, - run_location=io_handler.get_output_directory(sub_dir="model"), - collection_name="telescopes", - write_files=False, - ) - assert pars["camera_pixels"]["value"] == 1855 - - -@pytest.mark.usefixtures("_db_cleanup") -def test_adding_new_parameter_db(db, random_id, io_handler, model_version): - logger.info("----Testing adding a new parameter-----") - test_model_version = "0.0.9876" - tmp_par_dict = { - "parameter": None, - "instrument": "LSTN-test", - "site": "North", - "version": test_model_version, - "value": None, - "unit": None, - "type": None, - "applicable": True, - "file": False, + parameter = "camera_pixels" + parameter_version = "0.0.1" + site = "North" + array_element_name = "LSTN-01" + collection = "telescopes" + + result = db.get_model_parameter( + parameter, parameter_version, site, array_element_name, collection + ) + + query = { + "parameter_version": parameter_version, + "parameter": parameter, + "instrument": array_element_name, + "site": site, } - par_dict_int = copy.deepcopy(tmp_par_dict) - par_dict_int["parameter"] = "num_gains" - par_dict_int["value"] = 3 - par_dict_int["type"] = "int64" - with pytest.raises( - ValueError, - match=re.escape("Value for column '0' out of range. ([3, 3], allowed_range: [1, 2])"), - ): - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_int, - collection_name="telescopes", - ) - par_dict_int["value"] = 2 - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_int, - collection_name="telescopes", - ) - - par_dict_list = copy.deepcopy(tmp_par_dict) - par_dict_list["parameter"] = "telescope_transmission" - par_dict_list["value"] = [0.969, 0.01, 0.0, 0.0, 0.0, 0.0] - par_dict_list["type"] = "float64" - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_list, - collection_name="telescopes", - ) - par_dict_quantity = copy.deepcopy(tmp_par_dict) - par_dict_quantity["parameter"] = "focal_length" - par_dict_quantity["value"] = 12.5 # test that value is converted to cm - par_dict_quantity["type"] = "float64" - par_dict_quantity["unit"] = "m" - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_quantity, - collection_name="telescopes", - ) - - par_dict_file = copy.deepcopy(tmp_par_dict) - par_dict_file["parameter"] = "mirror_list" - par_dict_file["value"] = "mirror_list_CTA-N-LST1_v2019-03-31_rotated_simtel.dat" - par_dict_file["type"] = "file" - par_dict_file["file"] = True - with pytest.raises(FileNotFoundError, match=r"^The location of the file to upload"): - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_file, - collection_name="telescopes", - ) - db.add_new_parameter( - db_name=f"sandbox_{random_id}", - par_dict=par_dict_file, - collection_name="telescopes", - file_prefix="tests/resources", - ) - - pars = db.read_mongo_db( - db_name=f"sandbox_{random_id}", - array_element_name="LSTN-test", - model_version=test_model_version, - run_location=io_handler.get_output_directory(sub_dir="model"), - collection_name="telescopes", - write_files=False, - ) - assert pars["num_gains"]["value"] == 2 - assert pars["num_gains"]["type"] == "int64" - assert isinstance(pars["telescope_transmission"]["value"], list) - assert pars["focal_length"]["value"] == pytest.approx(1250.0) - assert pars["focal_length"]["unit"] == "cm" - assert pars["mirror_list"]["file"] is True - - # make sure that cache has been emptied after updating - assert ( - db._parameter_cache_key("North", "LSTN-test", test_model_version) - not in db.model_parameters_cached - ) - - -def test_reading_db_sites(db, db_config, simulation_model_url, model_version): - logger.info("----Testing reading La Palma parameters-----") - db.mongo_db_config["db_simulation_model_url"] = None - pars = db.get_site_parameters("North", model_version) - if db.mongo_db_config: - _obs_level = pars["corsika_observation_level"].get("value") - assert _obs_level == pytest.approx(2156.0) - else: - assert pars["altitude"] == 2156 - - logger.info("----Testing reading Paranal parameters-----") - pars = db.get_site_parameters("South", model_version) - if db.mongo_db_config: - _obs_level = pars["corsika_observation_level"].get("value") - assert _obs_level == pytest.approx(2147.0) - else: - assert pars["altitude"] == 2147 - - db._reset_parameter_cache("South", None, model_version) - if db.mongo_db_config.get("db_simulation_model_url", None) is None: - db.mongo_db_config["db_simulation_model_url"] = simulation_model_url - pars = db.get_site_parameters("South", model_version) - assert pars["corsika_observation_level"]["value"] == 2147.0 - db.mongo_db_config["db_simulation_model_url"] = None # make sure that this is reset - - -def test_separating_get_and_write(db, io_handler, model_version): - logger.info("----Testing getting parameters and exporting model files-----") - pars = db.get_model_parameters("North", "LSTN-01", model_version, collection="telescopes") - - file_list = [] - for par_now in pars.values(): - if par_now["file"] and par_now["value"] is not None: - file_list.append(par_now["value"]) - db.export_model_files( - pars, - io_handler.get_output_directory(sub_dir="model"), - ) - logger.debug( - "Checking files were written to " f"{io_handler.get_output_directory(sub_dir='model')}" - ) - for file_now in file_list: - assert io_handler.get_output_file(file_now, sub_dir="model").exists() - - -def test_export_file_db(db, io_handler): - logger.info("----Testing exporting files from the DB-----") - output_dir = io_handler.get_output_directory(sub_dir="model") - file_name = "mirror_CTA-S-LST_v2020-04-07.dat" - file_to_export = output_dir / file_name - db.export_file_db(None, output_dir, file_name) - assert file_to_export.exists() - - -@pytest.mark.usefixtures("_db_cleanup_file_sandbox") -def test_insert_files_db(db, io_handler, random_id, caplog): - logger.info("----Testing inserting files to the DB-----") - logger.info( - "Creating a temporary file in " f"{io_handler.get_output_directory(sub_dir='model')}" - ) - file_name = io_handler.get_output_directory(sub_dir="model") / f"test_file_{random_id}.dat" - with open(file_name, "w") as f: - f.write("# This is a test file") - - file_id = db.insert_file_to_db(file_name, f"sandbox_{random_id}") - assert ( - file_id == db._get_file_mongo_db(f"sandbox_{random_id}", f"test_file_{random_id}.dat")._id + mock_read_mongo_db.assert_called_once_with(query=query, collection_name=collection) + assert result == {"parameter": "value"} + + +def test_get_model_parameter_no_site(db, mocker): + """Test get_model_parameter method without site.""" + mock_read_mongo_db = mocker.patch.object( + db, "_read_mongo_db", return_value={"parameter": "value"} ) - logger.info("Now test inserting the same file again, this time expect a warning") - with caplog.at_level(logging.WARNING): - file_id = db.insert_file_to_db(file_name, f"sandbox_{random_id}") - assert "exists in the DB. Returning its ID" in caplog.text - assert ( - file_id == db._get_file_mongo_db(f"sandbox_{random_id}", f"test_file_{random_id}.dat")._id + + parameter = "mirror_list" + parameter_version = "1.0.0" + site = None + array_element_name = "LSTS-01" + collection = "telescopes" + + result = db.get_model_parameter( + parameter, parameter_version, site, array_element_name, collection ) + query = { + "parameter_version": parameter_version, + "parameter": parameter, + "instrument": array_element_name, + } + + mock_read_mongo_db.assert_called_once_with(query=query, collection_name=collection) + assert result == {"parameter": "value"} + -def test_get_all_versions(db, mocker, caplog): +def test_get_model_parameter_no_array_element_name(db, mocker): + """Test get_model_parameter method without array element name.""" + mock_read_mongo_db = mocker.patch.object( + db, "_read_mongo_db", return_value={"parameter": "value"} + ) - # not specifying any database names, collections, or parameters - all_versions = db.get_all_versions() - assert all(_v in all_versions for _v in ["5.0.0", "6.0.0"]) - assert any(key.endswith("None") for key in db.model_versions_cached) + parameter = "corsika_iact_io_buffer" + parameter_version = "10.11.12" + site = "South" + array_element_name = None + collection = "configuration_corsika" - # not specifying a telescope model name and parameter - all_versions = db.get_all_versions( - array_element_name=None, - site="North", - parameter=None, - collection="telescopes", + result = db.get_model_parameter( + parameter, parameter_version, site, array_element_name, collection ) - assert all(_v in all_versions for _v in ["5.0.0", "6.0.0"]) - assert any("telescopes" in key for key in db.model_versions_cached) - # using a specific parameter - all_versions = db.get_all_versions( - array_element_name="LSTN-01", - site="North", - parameter="camera_config_file", - collection="telescopes", + query = { + "parameter_version": parameter_version, + "parameter": parameter, + "site": site, + } + + mock_read_mongo_db.assert_called_once_with(query=query, collection_name=collection) + assert result == {"parameter": "value"} + + +def test_get_model_parameters(db, mocker): + """Test get_model_parameters method.""" + mock_get_production_table = mocker.patch.object( + db, + "_read_production_table_from_mongo_db", + return_value={"parameters": {"LSTN-01": {"param1": "v1"}}}, ) - assert all(_v in all_versions for _v in ["5.0.0", "6.0.0"]) - assert any( - key.endswith("telescopes-camera_config_file-LSTN-01") for key in db.model_versions_cached + mock_get_array_element_list = mocker.patch.object( + db, "_get_array_element_list", return_value=["LSTN-design", "LSTN-01"] ) + mock_read_cache = mocker.patch.object(db, "_read_cache", return_value=("cache_key", None)) + mock_read_mongo_db = mocker.patch.object( + db, "_read_mongo_db", return_value={"param1": {"value": "value1"}} + ) + + site = "North" + array_element_name = "LSTN-01" + model_version = "1.0.0" + collection = "telescopes" - all_versions = db.get_all_versions( - site="North", - parameter="corsika_observation_level", - collection="sites", + result = db.get_model_parameters(site, array_element_name, model_version, collection) + + mock_get_production_table.assert_called_once_with(collection, model_version) + mock_get_array_element_list.assert_called_once_with( + array_element_name, site, {"parameters": {"LSTN-01": {"param1": "v1"}}}, collection + ) + mock_read_cache.assert_has_calls( + [ + call( + db_handler.DatabaseHandler.model_parameters_cached, + names.validate_site_name(site), + "LSTN-design", + model_version, + collection, + ), + call( + db_handler.DatabaseHandler.model_parameters_cached, + names.validate_site_name(site), + "LSTN-01", + model_version, + collection, + ), + ] ) - assert all(_v in all_versions for _v in ["5.0.0", "6.0.0"]) - assert any( - key.endswith("sites-corsika_observation_level-North") for key in db.model_versions_cached + mock_read_mongo_db.assert_called_once_with( + query={ + "$or": [{"parameter": "param1", "parameter_version": "v1"}], + "instrument": "LSTN-01", + "site": site, + }, + collection_name=collection, ) + assert result == {"param1": {"value": "value1"}} - # no db_name defined - mocker.patch.object(db, "_get_db_name", return_value=None) - with caplog.at_level(logging.WARNING): - assert db.get_all_versions() == [] - assert "No database name defined to determine" in caplog.text +def test_get_model_parameters_with_cache(db, mocker): + """Test get_model_parameters method with cache.""" + mock_get_production_table = mocker.patch.object( + db, + "_read_production_table_from_mongo_db", + return_value={"parameters": {"LSTN-01": {"param1": "v1"}}}, + ) + mock_get_array_element_list = mocker.patch.object( + db, "_get_array_element_list", return_value=["LSTN-01"] + ) + mock_read_cache = mocker.patch.object( + db, "_read_cache", return_value=("cache_key", {"param1": {"value": "cached_value"}}) + ) -def test_parameter_cache_key(db, model_version_prod5): + site = "North" + array_element_name = "LSTN-01" + model_version = "1.0.0" + collection = "telescopes" - assert db._parameter_cache_key("North", "LSTN-01", model_version_prod5) == "North-LSTN-01-5.0.0" - assert db._parameter_cache_key("North", None, model_version_prod5) == "North-5.0.0" - assert db._parameter_cache_key(None, None, model_version_prod5) == "5.0.0" + result = db.get_model_parameters(site, array_element_name, model_version, collection) + mock_get_production_table.assert_called_once_with(collection, model_version) + mock_get_array_element_list.assert_called_once_with( + array_element_name, site, {"parameters": {"LSTN-01": {"param1": "v1"}}}, collection + ) + mock_read_cache.assert_called_once_with( + db_handler.DatabaseHandler.model_parameters_cached, + names.validate_site_name(site), + "LSTN-01", + model_version, + collection, + ) + assert result == {"param1": {"value": "cached_value"}} -def test_model_version(db): - assert db.model_version(version="6.0.0") == "6.0.0" +def test_get_model_parameters_no_parameters(db, mocker): + """Test get_model_parameters method with no parameters.""" + mock_get_production_table = mocker.patch.object( + db, "_read_production_table_from_mongo_db", return_value={"parameters": {}} + ) + mock_get_array_element_list = mocker.patch.object( + db, "_get_array_element_list", return_value=["LSTN-01"] + ) + mock_read_cache = mocker.patch.object(db, "_read_cache", return_value=("cache_key", None)) - with pytest.raises(ValueError, match=r"Invalid model version test"): - db.model_version(version="test") - with pytest.raises(ValueError, match=r"Invalid model version 0.0.9876"): - db.model_version(version="0.0.9876") + site = "North" + array_element_name = "LSTN-01" + model_version = "1.0.0" + collection = "telescopes" + result = db.get_model_parameters(site, array_element_name, model_version, collection) -def test_get_collections(db, db_config, fs_files): + mock_get_production_table.assert_called_once_with(collection, model_version) + mock_get_array_element_list.assert_called_once_with( + array_element_name, site, {"parameters": {}}, collection + ) + mock_read_cache.assert_called_once_with( + db_handler.DatabaseHandler.model_parameters_cached, + names.validate_site_name(site), + "LSTN-01", + model_version, + collection, + ) + assert result == {} + + +def test_get_collection(db, mocker, test_db): + """Test get_collection method.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) + mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {test_db: {"test_collection": "mock_collection"}} + ) + collection_name = "test_collection" + result = db.get_collection(test_db, collection_name) + + mock_get_db_name.assert_called_once_with(test_db) + assert result == "mock_collection" + + +def test_get_collections(db, db_config, fs_files): collections = db.get_collections() assert isinstance(collections, list) assert "telescopes" in collections @@ -446,7 +383,762 @@ def test_get_collections(db, db_config, fs_files): assert "metadata" not in collections_no_model -def test_model_version_empty(db, mocker): +def test_export_model_files_with_file_names( + db, mocker, tmp_test_directory, test_db, test_file, test_file_2 +): + """Test export_model_files method with file names.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) + mock_get_file_mongo_db = mocker.patch.object( + db, "_get_file_mongo_db", return_value=mocker.Mock(_id="file_id") + ) + mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") + + file_names = [test_file, test_file_2] + + result = db.export_model_files(file_names=file_names, dest=tmp_test_directory) + + mock_get_db_name.assert_called() + mock_get_file_mongo_db.assert_has_calls([call(test_db, test_file), call(test_db, test_file_2)]) + mock_write_file_from_mongo_to_disk.assert_has_calls( + [ + call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), + call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), + ] + ) + assert result == {test_file: "file_id", test_file_2: "file_id"} + + +def test_export_model_files_with_parameters( + db, mocker, tmp_test_directory, test_db, test_file, test_file_2 +): + """Test export_model_files method with parameters.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) + mock_get_file_mongo_db = mocker.patch.object( + db, "_get_file_mongo_db", return_value=mocker.Mock(_id="file_id") + ) + mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") + + parameters = { + "param1": {"file": True, "value": test_file}, + "param2": {"file": True, "value": test_file_2}, + } + + result = db.export_model_files(parameters=parameters, dest=tmp_test_directory) + + mock_get_db_name.assert_called() + mock_get_file_mongo_db.assert_has_calls([call(test_db, test_file), call(test_db, test_file_2)]) + mock_write_file_from_mongo_to_disk.assert_has_calls( + [ + call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), + call(test_db, tmp_test_directory, mock_get_file_mongo_db.return_value), + ] + ) + assert result == {test_file: "file_id", test_file_2: "file_id"} + + +def test_export_model_files_file_exists(db, mocker, tmp_test_directory, test_db, test_file): + """Test export_model_files method when file already exists.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) + mock_get_file_mongo_db = mocker.patch.object(db, "_get_file_mongo_db") + mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") + mock_path_exists = mocker.patch("pathlib.Path.exists", return_value=True) + + file_names = [test_file] + + result = db.export_model_files(file_names=file_names, dest=tmp_test_directory) + + mock_get_db_name.assert_called() + mock_get_file_mongo_db.assert_not_called() + mock_write_file_from_mongo_to_disk.assert_not_called() + mock_path_exists.assert_called_once() + assert result == {test_file: "file exists"} + + +def test_export_model_files_file_not_found(db, mocker, tmp_test_directory, test_db, test_file): + """Test export_model_files method when file is not found in parameters.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) + mock_get_file_mongo_db = mocker.patch.object( + db, "_get_file_mongo_db", side_effect=FileNotFoundError + ) + mock_write_file_from_mongo_to_disk = mocker.patch.object(db, "_write_file_from_mongo_to_disk") + + parameters = { + "param1": {"file": True, "value": test_file}, + } + + with pytest.raises(FileNotFoundError): + db.export_model_files(parameters=parameters, dest=tmp_test_directory) + + mock_get_db_name.assert_called() + mock_get_file_mongo_db.assert_called_once_with(test_db, test_file) + mock_write_file_from_mongo_to_disk.assert_not_called() + + +def test_get_query_from_parameter_version_table(db): + """Test _get_query_from_parameter_version_table method.""" + or_list = [ + {"parameter": "param1", "parameter_version": "v1"}, + {"parameter": "param2", "parameter_version": "v2"}, + ] + parameter_version_table = { + "param1": "v1", + "param2": "v2", + } + array_element_name = "LSTN-01" + site = "North" + + result = db._get_query_from_parameter_version_table( + parameter_version_table, array_element_name, site + ) + + expected_query = { + "$or": or_list, + "instrument": array_element_name, + "site": site, + } + + assert result == expected_query + + # _get_query_from_parameter_version_table method without site. + array_element_name = "LSTN-01" + site = None + + result = db._get_query_from_parameter_version_table( + parameter_version_table, array_element_name, site + ) + + expected_query = { + "$or": or_list, + "instrument": array_element_name, + } + + assert result == expected_query + + # _get_query_from_parameter_version_table method without array element name. + array_element_name = None + site = "North" + + result = db._get_query_from_parameter_version_table( + parameter_version_table, array_element_name, site + ) + + expected_query = { + "$or": or_list, + "site": site, + } + + assert result == expected_query + + # _get_query_from_parameter_version_table method without site and array element name. + array_element_name = None + site = None + + result = db._get_query_from_parameter_version_table( + parameter_version_table, array_element_name, site + ) + + expected_query = { + "$or": or_list, + } + + assert result == expected_query + + # _get_query_from_parameter_version_table method with array element name 'xSTx-design'. + array_element_name = "xSTx-design" + site = "North" + + result = db._get_query_from_parameter_version_table( + parameter_version_table, array_element_name, site + ) + + expected_query = { + "$or": or_list, + "site": site, + } + + assert result == expected_query + + +def test_read_mongo_db(db, mocker, test_db): + """Test read_mongo_db method.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value=test_db) + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mock_find = mocker.patch.object( + db.get_collection.return_value, + "find", + return_value=[ + {"_id": ObjectId(), "parameter": "param1", "value": "value1"}, + {"_id": ObjectId(), "parameter": "param2", "value": "value2"}, + ], + ) + + query = {"parameter_version": "1.0.0"} + collection_name = "test_collection" + + result = db._read_mongo_db(query, collection_name) + + mock_get_db_name.assert_called_once_with() + mock_get_collection.assert_called_once_with(test_db, collection_name) + mock_find.assert_called_once_with(query) + assert result == { + "param1": { + "_id": mock_find.return_value[0]["_id"], + "parameter": "param1", + "value": "value1", + "entry_date": mock_find.return_value[0]["_id"].generation_time, + }, + "param2": { + "_id": mock_find.return_value[1]["_id"], + "parameter": "param2", + "value": "value2", + "entry_date": mock_find.return_value[1]["_id"].generation_time, + }, + } + + mocker.patch.object(db.get_collection.return_value, "find", return_value=[]) + with pytest.raises( + ValueError, + match=r"The following query for test_collection returned zero results: " + r"{'parameter_version': '1.0.0'}", + ): + db._read_mongo_db(query, collection_name) + + +def test__read_production_table_from_mongo_db_with_cache(db, mocker, test_db): + """Test _read_production_table_from_mongo_db method with cache.""" + collection_name = "telescopes" + model_version = "1.0.0" + param = {"param1": "value1"} + mock_cache_key = mocker.patch.object(db, "_cache_key", return_value="cache_key") + db_handler.DatabaseHandler.production_table_cached["cache_key"] = { + "collection": model_version, + "model_version": model_version, + "parameters": param, + "design_model": {}, + "entry_date": ObjectId().generation_time, + } + + result = db._read_production_table_from_mongo_db(collection_name, model_version) + + mock_cache_key.assert_called_once_with(None, None, model_version, collection_name) + assert result == db_handler.DatabaseHandler.production_table_cached["cache_key"] + + mock_cache_key = mocker.patch.object(db, "_cache_key", return_value="no_cache_key") + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mocker.patch.object(db, "_get_db_name", return_value=test_db) + mock_find_one = mocker.patch.object( + db.get_collection.return_value, + "find_one", + return_value={ + "_id": ObjectId(), + "collection": collection_name, + "model_version": model_version, + "parameters": param, + "design_model": {}, + }, + ) + + result = db._read_production_table_from_mongo_db(collection_name, model_version) + + mock_cache_key.assert_called_once_with(None, None, model_version, collection_name) + mock_get_collection.assert_called_once_with(test_db, "production_tables") + mock_find_one.assert_called_once_with( + {"model_version": model_version, "collection": collection_name} + ) + assert result == { + "collection": collection_name, + "model_version": model_version, + "parameters": param, + "design_model": {}, + "entry_date": mock_find_one.return_value["_id"].generation_time, + } + + # test with no results + mocker.patch.object(db.get_collection.return_value, "find_one", return_value=None) + with pytest.raises( + ValueError, + match=r"The following query returned zero results: " + r"{'model_version': '1.0.0', 'collection': 'telescopes'}", + ): + db._read_production_table_from_mongo_db(collection_name, model_version) + + +def test_get_array_elements_of_type(db, mocker): + """Test get_array_elements_of_type method.""" + mock_get_production_table = mocker.patch.object( + db, + "_read_production_table_from_mongo_db", + return_value={ + "parameters": {"LSTN-01": "value1", "LSTN-02": "value2", "MSTS-01": "value3"} + }, + ) + + array_element_type = "LSTN" + model_version = "1.0.0" + collection = "telescopes" + + result = db.get_array_elements_of_type(array_element_type, model_version, collection) + + mock_get_production_table.assert_called_once_with(collection, model_version) + assert result == ["LSTN-01", "LSTN-02"] + + # Test with no matching array elements + mock_get_production_table.return_value = {"parameters": {"MSTS-01": "value3"}} + result = db.get_array_elements_of_type(array_element_type, model_version, collection) + assert result == [] + + # Test with array elements containing '-design' + mock_get_production_table.return_value = { + "parameters": {"LSTN-01": "value1", "LSTN-design": "value2"} + } + result = db.get_array_elements_of_type(array_element_type, model_version, collection) + assert result == ["LSTN-01"] + + # Test with different array element type + array_element_type = "MSTS" + mock_get_production_table.return_value = { + "parameters": {"LSTN-01": "value1", "MSTS-01": "value3", "MSTS-02": "value4"} + } + result = db.get_array_elements_of_type(array_element_type, model_version, collection) + assert result == ["MSTS-01", "MSTS-02"] + + +def test_get_simulation_configuration_parameters(db, mocker): + return_value = {"parameter": "value"} + mock_get_model_parameters = mocker.patch.object( + db, "get_model_parameters", return_value=return_value + ) + + assert ( + db.get_simulation_configuration_parameters("corsika", "North", "LSTN-design", "6.0.0") + == return_value + ) + assert mock_get_model_parameters.call_count == 1 + + assert ( + db.get_simulation_configuration_parameters("simtel", "North", "LSTN-design", "6.0.0") + == return_value + ) + assert mock_get_model_parameters.call_count == 2 + assert db.get_simulation_configuration_parameters("simtel", "North", None, "6.0.0") == {} + assert mock_get_model_parameters.call_count == 2 + assert db.get_simulation_configuration_parameters("simtel", None, "LSTN-design", "6.0.0") == {} + assert mock_get_model_parameters.call_count == 2 + assert db.get_simulation_configuration_parameters("simtel", None, None, "6.0.0") == {} + assert mock_get_model_parameters.call_count == 2 + + with pytest.raises(ValueError, match=r"Unknown simulation software: wrong"): + db.get_simulation_configuration_parameters("wrong", "North", "LSTN-design", "6.0.0") + + +def test_get_file_mongo_db_file(db, mocker, test_db, test_file, mock_gridfs): + """Test _get_file_mongo_db method when file exists.""" + mock_db_client = mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {test_db: mocker.Mock()} + ) + mock_file_system = mock_gridfs.return_value + mock_file_system.exists.return_value = True + mock_file_instance = mocker.Mock() + mock_file_system.find_one.return_value = mock_file_instance + + result = db._get_file_mongo_db(test_db, test_file) + + mock_gridfs.assert_called_once_with(mock_db_client[test_db]) + mock_file_system.exists.assert_called_once_with({"filename": test_file}) + mock_file_system.find_one.assert_called_once_with({"filename": test_file}) + assert result == mock_file_instance + + mock_file_system.exists.return_value = False + with pytest.raises( + FileNotFoundError, match=f"The file {test_file} does not exist in the database {test_db}" + ): + db._get_file_mongo_db(test_db, test_file) + + +def test_write_file_from_mongo_to_disk( + db, mocker, tmp_test_directory, mock_open, test_db, test_file +): + """Test _write_file_from_mongo_to_disk method.""" + mock_db_client = mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} + ) + mock_gridfs_bucket = mocker.patch("simtools.db.db_handler.gridfs.GridFSBucket") + mock_fs_output = mock_gridfs_bucket.return_value + + file = mocker.Mock() + file.filename = test_file + + db._write_file_from_mongo_to_disk(test_db, tmp_test_directory, file) + + mock_gridfs_bucket.assert_called_once_with(mock_db_client[test_db]) + mock_open.assert_called_once_with(Path(tmp_test_directory).joinpath(file.filename), "wb") + mock_fs_output.download_to_stream_by_name.assert_called_once_with(file.filename, mock_open()) + + +def test_add_production_table(db, mocker, test_db): + """Test add_production_table method.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mock_insert_one = mocker.patch.object(db.get_collection.return_value, "insert_one") + + production_table = { + "collection": "telescopes", + "model_version": "1.0.0", + "parameters": {"param1": "value1"}, + } + + db.add_production_table(test_db, production_table) + + mock_get_db_name.assert_called_once_with(test_db) + mock_get_collection.assert_called_once_with("test_db", "production_tables") + mock_insert_one.assert_called_once_with(production_table) + + +def test_add_new_parameter(db, mocker, value_unit_type, validate_model_parameter, test_db): + """Test add_new_parameter method.""" + mock_validate_model_parameter = mocker.patch( + validate_model_parameter, + return_value={"parameter": "param1", "value": "value1", "file": False}, + ) + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mock_insert_one = mocker.patch.object(db.get_collection.return_value, "insert_one") + mock_get_value_unit_type = mocker.patch( + value_unit_type, + return_value=("value1", "unit1", None), + ) + mock_reset_parameter_cache = mocker.patch.object(db, "_reset_parameter_cache") + + par_dict = {"parameter": "param1", "value": "value1", "file": False} + collection_name = "telescopes" + file_prefix = None + + db.add_new_parameter(test_db, par_dict, collection_name, file_prefix) + + mock_validate_model_parameter.assert_called_once_with(par_dict) + mock_get_db_name.assert_called_once_with(test_db) + mock_get_collection.assert_called_once_with("test_db", collection_name) + mock_get_value_unit_type.assert_called_once_with(value="value1", unit_str=None) + mock_insert_one.assert_called_once_with( + {"parameter": "param1", "value": "value1", "file": False, "unit": "unit1"} + ) + mock_reset_parameter_cache.assert_called_once() + + +def test_add_new_parameter_with_file( + db, mocker, tmp_test_directory, value_unit_type, validate_model_parameter, test_db +): + """Test add_new_parameter method with file.""" + mock_validate_model_parameter = mocker.patch( + validate_model_parameter, + return_value={"parameter": "param1", "value": "value1", "file": True}, + ) + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mock_insert_one = mocker.patch.object(db.get_collection.return_value, "insert_one") + mock_get_value_unit_type = mocker.patch( + value_unit_type, + return_value=("value1", "unit1", None), + ) + mock_insert_file_to_db = mocker.patch.object(db, "insert_file_to_db") + mock_reset_parameter_cache = mocker.patch.object(db, "_reset_parameter_cache") + + par_dict = {"parameter": "param1", "value": "value1", "file": True} + collection_name = "telescopes" + + db.add_new_parameter(test_db, par_dict, collection_name, tmp_test_directory) + + mock_validate_model_parameter.assert_called_once_with(par_dict) + mock_get_db_name.assert_called_once_with(test_db) + mock_get_collection.assert_called_once_with("test_db", collection_name) + mock_get_value_unit_type.assert_called_once_with(value="value1", unit_str=None) + mock_insert_one.assert_called_once_with( + {"parameter": "param1", "value": "value1", "file": True, "unit": "unit1"} + ) + mock_insert_file_to_db.assert_called_once_with(f"{tmp_test_directory!s}/value1", "test_db") + mock_reset_parameter_cache.assert_called_once() + + +def test_add_new_parameter_with_file_no_prefix( + db, mocker, value_unit_type, validate_model_parameter, test_db +): + """Test add_new_parameter method with file but no file_prefix.""" + mock_validate_model_parameter = mocker.patch( + validate_model_parameter, + return_value={"parameter": "param1", "value": "value1", "file": True}, + ) + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_get_collection = mocker.patch.object(db, "get_collection", return_value=mocker.Mock()) + mock_get_value_unit_type = mocker.patch( + value_unit_type, + return_value=("value1", "unit1", None), + ) + mock_reset_parameter_cache = mocker.patch.object(db, "_reset_parameter_cache") + + par_dict = {"parameter": "param1", "value": "value1", "file": True} + collection_name = "telescopes" + file_prefix = None + + with pytest.raises( + FileNotFoundError, + match=r"The location of the file to upload, corresponding to the param1 parameter, " + r"must be provided.", + ): + db.add_new_parameter(test_db, par_dict, collection_name, file_prefix) + + mock_validate_model_parameter.assert_called_once_with(par_dict) + mock_get_db_name.assert_called_once_with(test_db) + mock_get_collection.assert_called_once_with("test_db", collection_name) + mock_get_value_unit_type.assert_called_once_with(value="value1", unit_str=None) + mock_reset_parameter_cache.assert_not_called() + + +def test_insert_file_to_db_file_exists(db, mocker, test_db, test_file, mock_gridfs): + """Test insert_file_to_db method when file already exists in the DB.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_db_client = mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} + ) + mock_file_system = mock_gridfs.return_value + mock_file_system.exists.return_value = True + mock_file_instance = mocker.Mock() + mock_file_system.find_one.return_value = mock_file_instance + + result = db.insert_file_to_db(test_file, test_db) + + mock_get_db_name.assert_called_once_with(test_db) + mock_gridfs.assert_called_once_with(mock_db_client[test_db]) + mock_file_system.exists.assert_called_once_with({"filename": test_file}) + mock_file_system.find_one.assert_called_once_with({"filename": test_file}) + assert result == mock_file_instance._id + + +def test_insert_file_to_db_new_file(db, mocker, mock_open, test_db, test_file, mock_gridfs): + """Test insert_file_to_db method when file does not exist in the DB.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_db_client = mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} + ) + mock_file_system = mock_gridfs.return_value + mock_file_system.exists.return_value = False + mock_file_system.put.return_value = "new_file_id" + + result = db.insert_file_to_db(test_file, test_db) + + mock_get_db_name.assert_called_once_with(test_db) + mock_gridfs.assert_called_once_with(mock_db_client[test_db]) + mock_file_system.exists.assert_called_once_with({"filename": test_file}) + mock_open.assert_called_once_with(test_file, "rb") + mock_file_system.put.assert_called_once_with( + mock_open(), content_type="ascii/dat", filename=test_file + ) + assert result == "new_file_id" + + +def test_insert_file_to_db_with_kwargs(db, mocker, mock_open, test_db, test_file, mock_gridfs): + """Test insert_file_to_db method with additional kwargs.""" + mock_get_db_name = mocker.patch.object(db, "_get_db_name", return_value="test_db") + mock_db_client = mocker.patch.object( + db_handler.DatabaseHandler, "db_client", {"test_db": mocker.Mock()} + ) + mock_file_system = mock_gridfs.return_value + mock_file_system.exists.return_value = False + mock_file_system.put.return_value = "new_file_id" + + kwargs = {"content_type": "application/octet-stream", "metadata": {"key": "value"}} + + result = db.insert_file_to_db(test_file, test_db, **kwargs) + + mock_get_db_name.assert_called_once_with(test_db) + mock_gridfs.assert_called_once_with(mock_db_client[test_db]) + mock_file_system.exists.assert_called_once_with({"filename": test_file}) + mock_open.assert_called_once_with(test_file, "rb") + mock_file_system.put.assert_called_once_with( + mock_open(), + content_type="application/octet-stream", + filename=test_file, + metadata={"key": "value"}, + ) + assert result == "new_file_id" + + +def test_cache_key(db): + """Test _cache_key method.""" + # Test with all parameters + result = db._cache_key( + site="North", array_element_name="LSTN-01", model_version="1.0.0", collection="telescopes" + ) + assert result == "1.0.0-telescopes-North-LSTN-01" + + # Test with missing site + result = db._cache_key( + site=None, array_element_name="LSTN-01", model_version="1.0.0", collection="telescopes" + ) + assert result == "1.0.0-telescopes-LSTN-01" + + # Test with missing array_element_name + result = db._cache_key( + site="North", array_element_name=None, model_version="1.0.0", collection="telescopes" + ) + assert result == "1.0.0-telescopes-North" + + # Test with missing model_version + result = db._cache_key( + site="North", array_element_name="LSTN-01", model_version=None, collection="telescopes" + ) + assert result == "telescopes-North-LSTN-01" + + # Test with missing collection + result = db._cache_key( + site="North", array_element_name="LSTN-01", model_version="1.0.0", collection=None + ) + assert result == "1.0.0-North-LSTN-01" + + # Test with only model_version + result = db._cache_key( + site=None, array_element_name=None, model_version="1.0.0", collection=None + ) + assert result == "1.0.0" + + # Test with only collection + result = db._cache_key( + site=None, array_element_name=None, model_version=None, collection="telescopes" + ) + assert result == "telescopes" + + # Test with only site + result = db._cache_key( + site="North", array_element_name=None, model_version=None, collection=None + ) + assert result == "North" + + # Test with only array_element_name + result = db._cache_key( + site=None, array_element_name="LSTN-01", model_version=None, collection=None + ) + assert result == "LSTN-01" + + # Test with no parameters + result = db._cache_key(site=None, array_element_name=None, model_version=None, collection=None) + assert result == "" + + +def test_read_cache(db): + test_key = "1.0.0-telescopes-North-LSTN-01" + test_param1 = {"param1": "value1"} + cache_dict = {test_key: test_param1} + + # Test _read_cache method when cache hit occurs. + site = "North" + array_element_name = "LSTN-01" + model_version = "1.0.0" + collection = "telescopes" + + cache_key, result = db._read_cache( + cache_dict, site, array_element_name, model_version, collection + ) + assert cache_key == test_key + assert result == test_param1 + + # Test _read_cache method when cache miss occurs. + cache_key, result = db._read_cache(cache_dict, site, "LSTN-02", model_version, collection) + assert cache_key == "1.0.0-telescopes-North-LSTN-02" + assert result is None + + # Test _read_cache method with empty cache. + cache_key, result = db._read_cache({}, site, array_element_name, model_version, collection) + assert cache_key == test_key + assert result is None + + # Test _read_cache method with partial parameters. + test_key = "1.0.0-telescopes-North" + cache_dict = {test_key: test_param1} + cache_key, result = db._read_cache(cache_dict, site, None, model_version, collection) + assert cache_key == test_key + assert result == test_param1 + + # Test _read_cache method with no parameters. + cache_dict = {"": test_param1} + cache_key, result = db._read_cache(cache_dict, None, None, None, None) + assert cache_key == "" + assert result == test_param1 + + +def test_reset_parameter_cache(db): + """Test _reset_parameter_cache method.""" + # Populate the cache dictionaries + db_handler.DatabaseHandler.model_parameters_cached = {"key2": "value2"} + + # Ensure the caches are populated + assert db_handler.DatabaseHandler.model_parameters_cached + + # Call the method to reset the caches + db._reset_parameter_cache() + + # Check that the caches are cleared + assert not db_handler.DatabaseHandler.model_parameters_cached + + +def test_get_array_element_list_configuration_corsika(db): + """Test _get_array_element_list method for configuration_corsika collection.""" + array_element_name = "LSTN-01" + site = "North" + production_table = {} + collection = "configuration_corsika" + + result = db._get_array_element_list(array_element_name, site, production_table, collection) + + assert result == ["xSTx-design"] + + +def test_get_array_element_list_sites(db): + """Test _get_array_element_list method for sites collection.""" + array_element_name = "LSTN-01" + site = "North" + production_table = {} + collection = "sites" + + result = db._get_array_element_list(array_element_name, site, production_table, collection) + + assert result == ["OBS-North"] + + +def test_get_array_element_list_design_model(db): + """Test _get_array_element_list method when array element name contains '-design'.""" + array_element_name = "LSTN-design" + site = "North" + production_table = {} + collection = "telescopes" + + result = db._get_array_element_list(array_element_name, site, production_table, collection) + + assert result == ["LSTN-design"] + + +def test_get_array_element_list_with_design_model_in_production_table(db, mocker): + """Test _get_array_element_list method with design model in production table.""" + array_element_name = "LSTN-01" + site = "North" + production_table = {"design_model": {"LSTN-01": "LSTN-design"}} + collection = "telescopes" + + result = db._get_array_element_list(array_element_name, site, production_table, collection) + + assert result == ["LSTN-design", "LSTN-01"] + + +def test_get_array_element_list_without_design_model_in_production_table(db, mocker): + """Test _get_array_element_list method without design model in production table.""" + array_element_name = "LSTN-01" + site = "North" + production_table = {} + collection = "telescopes" + + mock_get_array_element_type_from_name = mocker.patch( + "simtools.utils.names.get_array_element_type_from_name", return_value="LSTN" + ) + + result = db._get_array_element_list(array_element_name, site, production_table, collection) - mocker.patch.object(db, "get_all_versions", return_value=[]) - assert db.model_version("6.0.0") is None + mock_get_array_element_type_from_name.assert_called_once_with(array_element_name) + assert result == ["LSTN-design", "LSTN-01"] diff --git a/tests/unit_tests/db/test_db_model_upload.py b/tests/unit_tests/db/test_db_model_upload.py new file mode 100644 index 0000000000..61abb97000 --- /dev/null +++ b/tests/unit_tests/db/test_db_model_upload.py @@ -0,0 +1,163 @@ +#!/usr/bin/python3 + +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest + +import simtools.db.db_model_upload as db_model_upload + + +@pytest.fixture +def iter_dir(): + return "simtools.db.db_model_upload.Path.iterdir" + + +@pytest.fixture +def is_dir(): + return "simtools.db.db_model_upload.Path.is_dir" + + +@patch("simtools.db.db_model_upload.gen.collect_data_from_file") +def test_add_values_from_json_to_db(mock_collect_data_from_file): + mock_collect_data_from_file.return_value = { + "parameter": "test_param", + "parameter_version": "1.0", + } + mock_db = Mock() + file = "test_file.json" + collection = "test_collection" + db_name = "test_db" + file_prefix = "test_prefix" + + db_model_upload.add_values_from_json_to_db(file, collection, mock_db, db_name, file_prefix) + + mock_collect_data_from_file.assert_called_once_with(file_name=file) + mock_db.add_new_parameter.assert_called_once_with( + db_name=db_name, + par_dict={"parameter": "test_param", "parameter_version": "1.0"}, + collection_name=collection, + file_prefix=file_prefix, + ) + + +@patch("simtools.db.db_model_upload.gen.collect_data_from_file") +def test_read_production_table(mock_collect_data_from_file): + mock_collect_data_from_file.return_value = { + "parameters": {"LSTN-design": "param_value_1", "LSTN-01": "param_value_2"}, + "design_model": {"LSTN-design": "design_value_1", "LSTN-01": "design_value_2"}, + } + model_dict = {} + file = Mock() + file.stem = "LSTN-design" + model_name = "test_model" + + db_model_upload._read_production_table(model_dict, file, model_name) + + assert "LSTN-design" in model_dict["telescopes"]["parameters"] + assert model_dict["telescopes"]["parameters"]["LSTN-design"] == "param_value_1" + assert model_dict["telescopes"]["design_model"]["LSTN-design"] == "design_value_1" + + file.stem = "MSTN-design" + with pytest.raises(KeyError, match="MSTN-design"): + db_model_upload._read_production_table(model_dict, file, model_name) + + file.stem = "configuration_corsika" + mock_collect_data_from_file.return_value = {"parameters": "config_param_value"} + + db_model_upload._read_production_table(model_dict, file, model_name) + + assert model_dict["configuration_corsika"]["parameters"] == "config_param_value" + + +@patch("simtools.db.db_model_upload._read_production_table") +def test_add_production_tables_to_db( + mock_read_production_table, tmp_test_directory, caplog, iter_dir, is_dir +): + mock_db = Mock() + args_dict = {"input_path": tmp_test_directory, "db_name": "test_db"} + input_path = Path(args_dict["input_path"]) + model_dir = input_path / "model_version_1" + model_dir.mkdir(parents=True, exist_ok=True) + (model_dir / "file1.json").touch() + (model_dir / "MSTS-02.json").touch() + + mock_read_production_table.side_effect = lambda model_dict, file, _: model_dict.update( + { + "telescopes": { + "parameters": {"MSTS-02": "param_value"}, + "design_model": {"MSTS-02": "MSTS-design"}, + } + } + ) + + with patch(iter_dir, return_value=[model_dir]): + with patch(is_dir, return_value=True): + db_model_upload.add_production_tables_to_db(args_dict, mock_db) + + assert mock_read_production_table.call_count == 2 + mock_db.add_production_table.assert_called_once_with( + db_name="test_db", + production_table={ + "parameters": {"MSTS-02": "param_value"}, + "design_model": {"MSTS-02": "MSTS-design"}, + }, + ) + + mock_read_production_table.side_effect = lambda model_dict, file, _: model_dict.update( + {"telescopes": {"parameters": {}, "design_model": {}}} + ) + with caplog.at_level("INFO"): + db_model_upload.add_production_tables_to_db(args_dict, mock_db) + assert "No production table for telescopes in model version model_version_1" in caplog.text + + +@patch("simtools.db.db_model_upload.add_values_from_json_to_db") +def test_add_model_parameters_to_db( + mock_add_values_from_json_to_db, tmp_test_directory, iter_dir, is_dir +): + mock_db = Mock() + args_dict = {"input_path": tmp_test_directory, "db_name": "test_db"} + input_path = Path(args_dict["input_path"]) + array_element_dir = input_path / "LSTS-01" + array_element_dir.mkdir(parents=True, exist_ok=True) + (array_element_dir / "num_gains-0.1.0.json").touch() + (array_element_dir / "mirror_list-0.2.1.json").touch() + + with patch(iter_dir, return_value=[array_element_dir]): + with patch(is_dir, return_value=True): + db_model_upload.add_model_parameters_to_db(args_dict, mock_db) + + mock_add_values_from_json_to_db.assert_any_call( + file=array_element_dir / "num_gains-0.1.0.json", + collection="telescopes", + db=mock_db, + db_name="test_db", + file_prefix=input_path / "Files", + ) + mock_add_values_from_json_to_db.assert_any_call( + file=array_element_dir / "mirror_list-0.2.1.json", + collection="telescopes", + db=mock_db, + db_name="test_db", + file_prefix=input_path / "Files", + ) + assert mock_add_values_from_json_to_db.call_count == 2 + + +@patch("simtools.db.db_model_upload.add_values_from_json_to_db") +def test_add_model_parameters_to_db_skip_files_collection( + mock_add_values_from_json_to_db, tmp_test_directory, iter_dir, is_dir +): + mock_db = Mock() + args_dict = {"input_path": tmp_test_directory, "db_name": "test_db"} + input_path = Path(args_dict["input_path"]) + files_dir = input_path / "Files" + files_dir.mkdir(parents=True, exist_ok=True) + (files_dir / "file1.json").touch() + + with patch(iter_dir, return_value=[files_dir]): + with patch(is_dir, return_value=True): + db_model_upload.add_model_parameters_to_db(args_dict, mock_db) + + mock_add_values_from_json_to_db.assert_not_called() diff --git a/tests/unit_tests/layout/test_array_layout.py b/tests/unit_tests/layout/test_array_layout.py index fd0f0b7d9f..0432ae23f6 100644 --- a/tests/unit_tests/layout/test_array_layout.py +++ b/tests/unit_tests/layout/test_array_layout.py @@ -283,9 +283,9 @@ def test_converting_center_coordinates_south(array_layout_south_four_lst_instanc def test_altitude_from_corsika_z( array_layout_north_four_lst_instance, array_layout_south_four_lst_instance ): - def test_one_site(instance, result1, result2): + def test_one_site(instance, telescope_name, result1, result2): instance.add_telescope( - telescope_name="LSTN-01", + telescope_name=telescope_name, crs_name="ground", xx=57.5 * u.m, yy=57.5 * u.m, @@ -302,8 +302,8 @@ def test_one_site(instance, result1, result2): instance._altitude_from_corsika_z(5.0, None, telescope_axis_height=16.0 * u.m) assert np.isnan(instance._altitude_from_corsika_z(None, None, None)) - test_one_site(array_layout_north_four_lst_instance, 2185.0, 45.0) - test_one_site(array_layout_south_four_lst_instance, 2176.0, 45.0) + test_one_site(array_layout_north_four_lst_instance, "LSTN-01", 2185.0, 45.0) + test_one_site(array_layout_south_four_lst_instance, "LSTS-01", 2176.0, 45.0) def test_try_set_altitude( @@ -455,7 +455,7 @@ def test_export_one_telescope_as_json(db_config, model_version, telescope_north_ ground_dict = layout.export_one_telescope_as_json(crs_name="ground") assert isinstance(ground_dict, dict) - assert ground_dict["instrument"] == "SSTS-09" + assert ground_dict["instrument"] == "MSTN-09" assert ground_dict["parameter"] == "array_element_position_ground" utm_dict = layout.export_one_telescope_as_json(crs_name="utm") diff --git a/tests/unit_tests/model/test_array_model.py b/tests/unit_tests/model/test_array_model.py index 1c884c7c24..65806c4b1e 100644 --- a/tests/unit_tests/model/test_array_model.py +++ b/tests/unit_tests/model/test_array_model.py @@ -73,7 +73,6 @@ def test_exporting_config_files(db_config, io_handler, model_version): "CTA-North-LSTN-01-" + model_version + test_cfg, "CTA-North-MSTN-01-" + model_version + test_cfg, "CTA-test_layout-North-" + model_version + test_cfg, - "array_coordinates_LaPalma_alpha.dat", "NectarCAM_lightguide_efficiency_POP_131019.dat", "Pulse_template_nectarCam_17042020-noshift.dat", "array_triggers.dat", @@ -94,7 +93,6 @@ def test_exporting_config_files(db_config, io_handler, model_version): ] for model_file in list_of_export_files: - logger.info("Checking file: %s", model_file) assert Path(am.get_config_directory()).joinpath(model_file).exists() @@ -108,16 +106,17 @@ def test_load_array_element_positions_from_file(array_model, io_handler, telesco def test_get_telescope_position_parameter(array_model, io_handler): am = array_model assert am._get_telescope_position_parameter( - "LSTN-01", "North", 10.0 * u.m, 200.0 * u.cm, 30.0 * u.m + "LSTN-01", "North", 10.0 * u.m, 200.0 * u.cm, 30.0 * u.m, "2.0.0" ) == { + "schema_version": "0.1.0", "parameter": "array_element_position_ground", "instrument": "LSTN-01", "site": "North", - "version": "6.0.0", + "parameter_version": "2.0.0", + "unique_id": None, "value": "10.0 2.0 30.0", "unit": "m", "type": "float64", - "applicable": True, "file": False, } diff --git a/tests/unit_tests/model/test_model_parameter.py b/tests/unit_tests/model/test_model_parameter.py index e1199a0307..cad287c129 100644 --- a/tests/unit_tests/model/test_model_parameter.py +++ b/tests/unit_tests/model/test_model_parameter.py @@ -162,11 +162,11 @@ def test_load_parameters_from_db(telescope_model_lst, mocker): telescope_copy = copy.deepcopy(telescope_model_lst) mock_db = mocker.patch.object(DatabaseHandler, "get_model_parameters") telescope_copy._load_parameters_from_db() - assert mock_db.call_count == 2 + assert mock_db.call_count == 4 telescope_copy.db = None telescope_copy._load_parameters_from_db() - assert mock_db.call_count == 2 + assert mock_db.call_count == 4 def test_extra_labels(telescope_model_lst): @@ -295,22 +295,6 @@ def test_updating_export_model_files(db_config, io_handler, model_version): assert False is tel._is_exported_model_files_up_to_date -@pytest.mark.xfail(reason="Test requires Derived-Values Database") -def test_export_derived_files(io_handler, db_config, model_version_prod5): - tel_model = TelescopeModel( - site="North", - telescope_name="LSTN-01", - model_version=model_version_prod5, - mongo_db_config=db_config, - label="test-telescope-model-lst", - ) - - _ = tel_model.derived - assert tel_model.config_file_directory.joinpath( - "ray-tracing-North-LST-1-d10.0-za20.0_validate_optics.ecsv" - ).exists() - - def test_export_parameter_file(telescope_model_lst, mocker): parameter = "array_coordinates_UTM" file_path = "tests/resources/telescope_positions-North-ground.ecsv" @@ -371,13 +355,13 @@ def test_export_nsb_spectrum_to_telescope_altitude_correction_file(telescope_mod telescope_copy.export_nsb_spectrum_to_telescope_altitude_correction_file(model_directory) mock_db_export.assert_called_once_with( - { + parameters={ "nsb_spectrum_at_2200m": { "value": "test_value", "file": True, } }, - model_directory, + dest=model_directory, ) diff --git a/tests/unit_tests/model/test_site_model.py b/tests/unit_tests/model/test_site_model.py index 3244fb1c58..9de21bddc6 100644 --- a/tests/unit_tests/model/test_site_model.py +++ b/tests/unit_tests/model/test_site_model.py @@ -95,11 +95,11 @@ def test_export_atmospheric_transmission_file(db_config, model_version, tmp_path _south.export_atmospheric_transmission_file(model_directory) _south.db.export_model_files.assert_called_once_with( - { + parameters={ "atmospheric_transmission_file": { "value": "test_atmospheric_profile", "file": True, } }, - model_directory, + dest=model_directory, ) diff --git a/tests/unit_tests/model/test_telescope_model.py b/tests/unit_tests/model/test_telescope_model.py index 26e77a1618..066f9177a1 100644 --- a/tests/unit_tests/model/test_telescope_model.py +++ b/tests/unit_tests/model/test_telescope_model.py @@ -55,7 +55,6 @@ def test_read_incidence_angle_distribution(telescope_model_sst): def test_calc_average_curve(telescope_model_sst_prod5): tel_model = telescope_model_sst_prod5 tel_model.export_config_file() - _ = tel_model.derived two_dim_file = tel_model.get_parameter_value("camera_filter") two_dim_dist = tel_model.read_two_dim_wavelength_angle(two_dim_file) @@ -72,7 +71,6 @@ def test_calc_average_curve(telescope_model_sst_prod5): def test_export_table_to_model_directory(telescope_model_sst_prod5): tel_model = telescope_model_sst_prod5 tel_model.export_config_file() - _ = tel_model.derived two_dim_file = tel_model.get_parameter_value("camera_filter") two_dim_dist = tel_model.read_two_dim_wavelength_angle(two_dim_file) diff --git a/tests/unit_tests/test_camera_efficiency.py b/tests/unit_tests/test_camera_efficiency.py index 9348d4efd9..a214eb5d63 100644 --- a/tests/unit_tests/test_camera_efficiency.py +++ b/tests/unit_tests/test_camera_efficiency.py @@ -144,7 +144,6 @@ def test_calc_reflectivity(camera_efficiency_lst, prepare_results_file): ) # Value for Prod5 LST-1 -@pytest.mark.xfail(reason="Missing ray_tracing for prod6 in Derived-DB") def test_calc_nsb_rate(camera_efficiency_lst, prepare_results_file): camera_efficiency_lst._read_results() camera_efficiency_lst.export_model_files() diff --git a/tests/unit_tests/testing/test_validate_output.py b/tests/unit_tests/testing/test_validate_output.py index f5eb056428..ede13d59c0 100644 --- a/tests/unit_tests/testing/test_validate_output.py +++ b/tests/unit_tests/testing/test_validate_output.py @@ -110,6 +110,18 @@ def test_compare_json_files_float_strings(create_json_file, file_name): assert not validate_output.compare_json_or_yaml_files(file1, file3) +def test_compare_json_files_equal_dicts(create_json_file, file_name): + content1 = {"key": 1, "value": 5} + file1 = create_json_file(file_name(1, "json"), content1) + content2 = {"key": 1, "value": 5, "extra": "extra"} + file2 = create_json_file(file_name(2, "json"), content2) + assert not validate_output.compare_json_or_yaml_files(file1, file2) + + content3 = {"different_key": 1, "value": 5} + file3 = create_json_file(file_name(2, "json"), content3) + assert not validate_output.compare_json_or_yaml_files(file1, file3) + + def test_compare_json_files_equal_integers(create_json_file, file_name): content = {"key": 1, "value": 5} file1 = create_json_file(file_name(1, "json"), content) @@ -122,6 +134,36 @@ def test_compare_json_files_equal_integers(create_json_file, file_name): assert not validate_output.compare_json_or_yaml_files(file1, file3) +def test_compare_json_files_equal_floats(create_json_file, file_name): + content = {"key": 1, "value": 5.5} + file1 = create_json_file(file_name(1, "json"), content) + file2 = create_json_file(file_name(2, "json"), content) + + assert validate_output.compare_json_or_yaml_files(file1, file2) + + content3 = {"key": 1, "value": 5.75} + file3 = create_json_file(file_name(3, "json"), content3) + assert not validate_output.compare_json_or_yaml_files(file1, file3) + + assert validate_output.compare_json_or_yaml_files(file1, file3, tolerance=0.5) + + +def test_compare_json_files_list_of_floats(create_json_file, file_name): + content = {"key": 1, "value": [5.5, 10.5]} + file1 = create_json_file(file_name(1, "json"), content) + file2 = create_json_file(file_name(2, "json"), content) + + assert validate_output.compare_json_or_yaml_files(file1, file2) + + content3 = {"key": 1, "value": 5.75} + file3 = create_json_file(file_name(3, "json"), content3) + assert not validate_output.compare_json_or_yaml_files(file1, file3) + + content4 = {"key": 1, "value": [5.75, 10.75]} + file4 = create_json_file(file_name(3, "json"), content4) + assert validate_output.compare_json_or_yaml_files(file1, file4, tolerance=0.5) + + def test_compare_yaml_files_float_strings(create_yaml_file, file_name): content = {"key": 1, "value": "1.23 4.56 7.89"} file1 = create_yaml_file(file_name(1, "yaml"), content) @@ -129,10 +171,12 @@ def test_compare_yaml_files_float_strings(create_yaml_file, file_name): assert validate_output.compare_json_or_yaml_files(file1, file2) - content3 = {"key": 2, "value": "1.23 4.56 7.80"} + content3 = {"key": 1, "value": "1.23 4.56 7.80"} file3 = create_yaml_file(file_name(3, "yaml"), content3) assert not validate_output.compare_json_or_yaml_files(file1, file3) + assert validate_output.compare_json_or_yaml_files(file1, file3, tolerance=0.5) + def test_compare_yaml_files_equal_integers(create_yaml_file, file_name): content = {"key": 1, "value": 5} diff --git a/tests/unit_tests/utils/test_names.py b/tests/unit_tests/utils/test_names.py index 2ed327e73a..dcf42c8a10 100644 --- a/tests/unit_tests/utils/test_names.py +++ b/tests/unit_tests/utils/test_names.py @@ -153,6 +153,24 @@ def test_get_class_from_telescope_name(invalid_name): names.get_collection_name_from_array_element_name("Not_a_collection") +def test_get_collection_name_from_array_element_name(): + + assert "telescopes" == names.get_collection_name_from_array_element_name("LSTN-01") + assert "telescopes" == names.get_collection_name_from_array_element_name("MSTS-design") + assert "sites" == names.get_collection_name_from_array_element_name("OBS-North") + assert "sites" == names.get_collection_name_from_array_element_name("North") + assert "sites" == names.get_collection_name_from_array_element_name("OBS-North", False) + assert "configuration_sim_telarray" == names.get_collection_name_from_array_element_name( + "configuration_sim_telarray", False + ) + + with pytest.raises(ValueError, match=r"Invalid array element name configuration_sim_telarray"): + names.get_collection_name_from_array_element_name("configuration_sim_telarray", True) + + with pytest.raises(ValueError, match=r"Invalid array element name Not_a_collection"): + names.get_collection_name_from_array_element_name("Not_a_collection", False) + + def test_sanitize_name(caplog): assert names.sanitize_name("y_edges unit") == "y_edges_unit" assert names.sanitize_name("Y_EDGES UNIT") == "y_edges_unit" @@ -482,19 +500,6 @@ def test_get_simulation_software_name_from_parameter_name(): == "reference_point_longitude" ) - with pytest.raises(KeyError): - names.get_simulation_software_name_from_parameter_name( - "corsika_observation_level", - simulation_software="sim_telarray", - search_site_parameters=False, - ) - with pytest.raises(KeyError): - names.get_simulation_software_name_from_parameter_name( - "telescope_axis_height", - simulation_software="sim_telarray", - search_telescope_parameters=False, - ) - def test_get_parameter_name_from_simtel_name(): assert names.get_parameter_name_from_simtel_name("focal_length") == "focal_length" diff --git a/tests/unit_tests/visualization/test_visualize.py b/tests/unit_tests/visualization/test_visualize.py index 0a3f38b647..482e9bba7d 100644 --- a/tests/unit_tests/visualization/test_visualize.py +++ b/tests/unit_tests/visualization/test_visualize.py @@ -27,10 +27,10 @@ def test_plot_1d(db, io_handler): title = "Test 1D plot" test_file_name = "ref_LST1_2022_04_01.dat" - db.export_file_db( + db.export_model_files( db_name=None, dest=io_handler.get_output_directory(sub_dir="model"), - file_name=test_file_name, + file_names=test_file_name, ) test_data_file = gen.find_file( test_file_name,