gammasim · GernotMaier · Jan 31, 2025 · Jan 13, 2025 · Jan 13, 2025 · Jan 13, 2025
diff --git a/.env_template b/.env_template
@@ -6,7 +6,7 @@ SIMTOOLS_DB_SERVER='cta-simpipe-protodb.zeuthen.desy.de' # MongoDB server
 SIMTOOLS_DB_API_USER=YOUR_USERNAME # username for MongoDB: ask the responsible person
 SIMTOOLS_DB_API_PW=YOUR_PASSWORD # Password for MongoDB: ask the responsible person
 SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin'
-SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-LATEST'
+SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-LATEST'
 SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'
 
 # The dashboards to monitor the MongoDB instance are in (accessible only from within DESY)

diff --git a/.github/workflows/CI-integrationtests.yml b/.github/workflows/CI-integrationtests.yml
@@ -7,7 +7,7 @@ env:
   SIMTOOLS_DB_API_USER: ${{ secrets.DB_API_USER }}
   SIMTOOLS_DB_API_PW: ${{ secrets.DB_API_PW }}
   SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }}
-  SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-Model-LATEST"
+  SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-ModelParameters-LATEST"
   SIMTOOLS_SIMTEL_PATH: "/workdir/sim_telarray/"
 
 on:

diff --git a/.github/workflows/CI-unittests.yml b/.github/workflows/CI-unittests.yml
@@ -7,7 +7,7 @@ env:
   SIMTOOLS_DB_API_USER: ${{ secrets.DB_API_USER }}
   SIMTOOLS_DB_API_PW: ${{ secrets.DB_API_PW }}
   SIMTOOLS_DB_API_PORT: ${{ secrets.DB_API_PORT }}
-  SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-Model-LATEST"
+  SIMTOOLS_DB_SIMULATION_MODEL: "CTAO-Simulation-ModelParameters-LATEST"
   SONAR_HOST_URL: ${{ secrets.SONAR_HOST_URL }}
 
 on:

diff --git a/.github/workflows/build-docker-images.yml b/.github/workflows/build-docker-images.yml
@@ -89,16 +89,13 @@ jobs:
       - name: Test Docker image
         if: matrix.type == 'prod'
         run: >
-          # Clone simulation model repo for faster file access
-          git clone https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters.git
           # run docker
           docker run --rm
           -e "SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'"
           -e "SIMTOOLS_DB_API_PORT=${{ env.SIMTOOLS_DB_API_PORT }}"
           -e "SIMTOOLS_DB_API_USER=${{ env.SIMTOOLS_DB_API_USER }}"
           -e "SIMTOOLS_DB_API_PW=${{ env.SIMTOOLS_DB_API_PW }}"
           -e "SIMTOOLS_DB_SERVER=${{ env.SIMTOOLS_DB_SERVER }}"
-          -e "SIMTOOLS_DB_SIMULATION_MODEL_URL='./model_parameters'"
           -v "$(pwd):/workdir/external" ${{ env.TEST_TAG }}
           bash -c "cd /workdir/external && simtools-simulate-prod --config /workdir/external/tests/integration_tests/config/simulate_prod_gamma_20_deg_pack_for_grid.yml"
         shell: /usr/bin/bash -e {0}

diff --git a/database_scripts/upload_dump_to_local_db.sh b/database_scripts/upload_dump_to_local_db.sh
@@ -6,8 +6,7 @@
 
 SIMTOOLS_NETWORK="simtools-mongo-network"
 CONTAINER_NAME="simtools-mongodb"
-SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-v0-3-0'  # Name of the database to be created
-SIMTOOLS_DB_SIMULATION_MODEL='Staging-CTA-Simulation-Model-Derived-Values'
+SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-v0-3-0'  # Name of the database to be created
 
 # Check if podman is available, if not use docker
 if command -v podman &> /dev/null; then

diff --git a/database_scripts/upload_from_model_repository_to_db.sh b/database_scripts/upload_from_model_repository_to_db.sh
@@ -1,10 +1,11 @@
 #!/bin/bash
 # Upload model parameter from repository to a local or remote mongoDB.
 #
+# Execute this scripts from the ./database_scripts directory.
 # Cover 'source .env': the script ensure that this file exists:
 # shellcheck disable=SC1091
 
-DB_SIMULATION_MODEL_URL="https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters.git"
+DB_SIMULATION_MODEL_URL="https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/simulation-models.git"
 DB_SIMULATION_MODEL_BRANCH="main"
 
 # Check that this script is not sourced but executed
@@ -27,9 +28,9 @@ echo "Cloning model parameters from $DB_SIMULATION_MODEL_URL"
 rm -rf ./tmp_model_parameters
 git clone --depth=1 -b $DB_SIMULATION_MODEL_BRANCH $DB_SIMULATION_MODEL_URL ./tmp_model_parameters
 
-CURRENTDIR=$(pwd)
+CURRENT_DIR=$(pwd)
 cd ./tmp_model_parameters/ || exit
-cp -f "$CURRENTDIR"/../.env .env
+cp -f "$CURRENT_DIR"/../.env .env
 
 # ask for confirmation before uploading to remote DB
 source .env
@@ -43,16 +44,20 @@ if [[ $SIMTOOLS_DB_SERVER =~ $regex ]]; then
   fi
 fi
 
-# upload files to DB
-model_directory="./model_versions/"
-for dir in "${model_directory}"*/; do
-  simtools-db-add-model-parameters-from-repository-to-db \
-  --model_version "$(basename "${dir}")" \
-  --input_path "${dir}" \
+# upload model parameters to DB
+model_directory="./simulation-models/model_parameters/"
+simtools-db-add-simulation-model-from-repository-to-db \
+  --input_path "${model_directory}" \
   --db_name "$DB_SIMULATION_MODEL" \
   --type "model_parameters"
-done
 
-cd "$CURRENTDIR" || exit
+# upload production tables to DB
+production_directory="./simulation-models/productions"
+simtools-db-add-simulation-model-from-repository-to-db \
+  --input_path "${production_directory}" \
+  --db_name "$DB_SIMULATION_MODEL" \
+  --type "production_tables"
+
+cd "$CURRENT_DIR" || exit
 
 rm -rf ./tmp_model_parameters
diff --git a/docs/changes/1316.maintenance.md b/docs/changes/1316.maintenance.md
@@ -0,0 +1 @@
+Major restructering of database routines plus introduction of new simulation model.
diff --git a/docs/source/api-reference/db_handler.md b/docs/source/api-reference/db_handler.md
@@ -13,11 +13,11 @@ Modules for database access. See the databases sections for details.
    :members:
 ```
 
-## db_array_elements
+## db_model_upload
 
-(db-array-elements)=
+(db-module-upload)=
 
 ```{eval-rst}
-.. automodule:: db.db_array_elements
+.. automodule:: db.db_model_upload
    :members:
 ```
diff --git a/docs/source/user-guide/applications.md b/docs/source/user-guide/applications.md
@@ -28,7 +28,7 @@ simtools-convert-all-model-parameters-from-simtel <applications/simtools-convert
 simtools-convert-geo-coordinates-of-array-elements <applications/simtools-convert-geo-coordinates-of-array-elements>
 simtools-convert-model-parameter-from-simtel <applications/simtools-convert-model-parameter-from-simtel>
 simtools-db-add-file-to-db <applications/simtools-db-add-file-to-db>
-simtools-db-add-model-parameters-from-repository-to-db <applications/simtools-db-add-model-parameters-from-repository-to-db>
+simtools-db-add-simulation-model-from-repository-to-db <applications/simtools-db-add-simulation-model-from-repository-to-db>
 simtools-db-add-value-from-json-to-db <applications/simtools-db-add-value-from-json-to-db>
 simtools-db-get-array-layouts-from-db <applications/simtools-db-get-array-layouts-from-db>
 simtools-db-get-file-from-db <applications/simtools-db-get-file-from-db>

diff --git a/...r-guide/applications/simtools-db-add-model-parameters-from-repository-to-db.rst b/...r-guide/applications/simtools-db-add-model-parameters-from-repository-to-db.rst
diff --git a/...r-guide/applications/simtools-db-add-simulation-model-from-repository-to-db.rst b/...r-guide/applications/simtools-db-add-simulation-model-from-repository-to-db.rst
@@ -0,0 +1,6 @@
+
+simtools-db-add-simulation-model-from-repository-to-db
+======================================================
+
+.. automodule:: db_add_simulation_model_from_repository_to_db
+   :members:
diff --git a/docs/source/user-guide/databases.md b/docs/source/user-guide/databases.md
@@ -1,6 +1,6 @@
 # Databases
 
-The simtools package uses a prototype MongoDB database to store the simulation model parameters and derived data products.
+The simtools package uses a prototype MongoDB database to store the simulation model parameters.
 Access to the DB is handled via a dedicated API module ([db_handler](#dbhandler)).
 
 Simulation model parameters are stored in databases (see the [Simulation Model](model_parameters.md#simulation-model) section) and synced with the [CTAO model parameter repository](https://gitlab.cta-observatory.org/cta-science/simulations/simulation-model/model_parameters).
@@ -16,7 +16,7 @@ This documentation is therefore incomplete.
 
 ### Model parameters DB
 
-The name of the model parameter database needs to be indicated by `$SIMTOOLS_DB_SIMULATION_MODEL` environmental variable and defined e.g., in the `.env` file. Use `CTAO-Simulation-Model-LATEST` to use the latest version of the CTAO simulation model database (simtools will replace `LATEST` with the latest version number).
+The name of the model parameter database needs to be indicated by `$SIMTOOLS_DB_SIMULATION_MODEL` environmental variable and defined e.g., in the `.env` file. Use `CTAO-Simulation-ModelParameters-LATEST` to use the latest version of the CTAO simulation model database (simtools will replace `LATEST` with the latest version number).
 
 Collections:
 
@@ -27,15 +27,6 @@ Collections:
 * `metadata` containing tables describing the model versions
 * `fs.files` with all file type entries for the model parameters (e.g., the quantum-efficiency tables)
 
-### Derived values DB
-
-Database with derived values DB (e.g., `Staging-CTA-Simulation-Model-Derived-Values` defined in `db_handler.DB_DERIVED_VALUES`).
-
-Collections are:
-
-* `derived_values` with the derived values for each telescope or site
-* `fs.files` with file type derived results
-
 ### Other databases
 
 All other currently available databases are not in use and kept for historical reasons.
@@ -52,8 +43,7 @@ SIMTOOLS_DB_SERVER='cta-simpipe-protodb.zeuthen.desy.de' # MongoDB server
 SIMTOOLS_DB_API_USER=YOUR_USERNAME # username for MongoDB: ask the responsible person
 SIMTOOLS_DB_API_PW=YOUR_PASSWORD # Password for MongoDB: ask the responsible person
 SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin'
-SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-LATEST'
-# SIMTOOLS_DB_SIMULATION_MODEL_URL=''
+SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-LATEST'
 SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'
 ```
 
@@ -75,7 +65,7 @@ The following applications are important:
 
 * update or define a single model parameter from a json file (as defined in the model parameter repository): [db_add_value_from_json_to_db.py](db_add_value_from_json_to_db)
 * upload a model parameter file: [db_add_file_to_db.py](db_add_file_to_db)
-* upload all model parameters and files from the model parameter repository: [db_add_model_parameters_from_repository_to_db.py](db_add_model_parameters_from_repository_to_db)
+* upload all model parameters and files from the model parameter repository: [db_add_simulation_model_from_repository_to_db.py](db_add_simulation_model_from_repository_to_db)
 
 ## Configure and use a local copy of the model parameter database
 
@@ -138,7 +128,7 @@ SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin'
 SIMTOOLS_DB_SIMULATION_MODEL='STAGING-CTA-Simulation-Model-LATEST'
 ```
 
-`SIMTOOLS_DB_SIMULATION_MODEL` is set as an example here to `STAGING-CTAO-Simulation-Model-LATEST` and should be changed accordingly.
+`SIMTOOLS_DB_SIMULATION_MODEL` is set as an example here to `STAGING-CTAO-Simulation-ModelParameters-LATEST` and should be changed accordingly.
 
 For using simtools inside a container:
 
@@ -158,6 +148,6 @@ SIMTOOLS_DB_SERVER='simtools-mongodb'
 SIMTOOLS_DB_API_USER='api' # username for MongoDB
 SIMTOOLS_DB_API_PW='password' # Password for MongoDB
 SIMTOOLS_DB_API_AUTHENTICATION_DATABASE='admin'
-SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-Model-LATEST'
+SIMTOOLS_DB_SIMULATION_MODEL='CTAO-Simulation-ModelParameters-LATEST'
 SIMTOOLS_SIMTEL_PATH='/workdir/sim_telarray'
 ```
diff --git a/docs/source/user-guide/model_parameters.md b/docs/source/user-guide/model_parameters.md
@@ -189,14 +189,15 @@ A typical model parameter file looks like:
 
 ```json
 {
+    "schema_version": "0.1.0",
     "parameter": "num_gains",
     "instrument": "LSTN-01",
     "site": "North",
-    "version": "6.0.0",
+    "parameter_version": "6.0.0",
+    "unique_id": null,
     "value": 2,
     "unit": null,
     "type": "int64",
-    "applicable": true,
     "file": false
 }
 ```

diff --git a/pyproject.toml b/pyproject.toml
@@ -73,7 +73,7 @@ scripts.simtools-convert-all-model-parameters-from-simtel = "simtools.applicatio
 scripts.simtools-convert-geo-coordinates-of-array-elements = "simtools.applications.convert_geo_coordinates_of_array_elements:main"
 scripts.simtools-convert-model-parameter-from-simtel = "simtools.applications.convert_model_parameter_from_simtel:main"
 scripts.simtools-db-add-file-to-db = "simtools.applications.db_add_file_to_db:main"
-scripts.simtools-db-add-model-parameters-from-repository-to-db = "simtools.applications.db_add_model_parameters_from_repository_to_db:main"
+scripts.simtools-db-add-simulation-model-from-repository-to-db = "simtools.applications.db_add_simulation_model_from_repository_to_db:main"
 scripts.simtools-db-add-value-from-json-to-db = "simtools.applications.db_add_value_from_json_to_db:main"
 scripts.simtools-db-get-array-layouts-from-db = "simtools.applications.db_get_array_layouts_from_db:main"
 scripts.simtools-db-get-file-from-db = "simtools.applications.db_get_file_from_db:main"

diff --git a/src/simtools/applications/convert_all_model_parameters_from_simtel.py b/src/simtools/applications/convert_all_model_parameters_from_simtel.py
@@ -6,6 +6,8 @@
     ready to be submitted to the model database. Prints out parameters which are not found
     in simtel configuration file and parameters which are not found in simtools schema files.
 
+    Note that all parameters are assigned the same parameter version.
+
     Command line arguments
     ----------------------
     simtel_cfg_file (str)
@@ -30,7 +32,7 @@
           --simtel_cfg_file all_telescope_config_la_palma.cfg\\
           --simtel_telescope_name CT1\\
           --telescope LSTN-01\\
-          --model_version "2024-03-06"
+          --parameter_version "1.0.0"
 
     The export of the model parameters from sim_telarray for 6.0.0 can be done e.g., as follows:
 
@@ -103,7 +105,7 @@ def _parse(label=None, description=None):
         type=str,
         required=True,
     )
-    return config.initialize(simulation_model=["telescope", "model_version"])
+    return config.initialize(simulation_model=["telescope", "parameter_version"])
 
 
 def get_list_of_parameters_and_schema_files(schema_directory):
@@ -219,7 +221,6 @@ def read_and_export_parameters(args_dict, logger):
     """
     Read and export parameters from simtel configuration file to json files.
 
-    Only applicable parameters are exported to json.
     Provide extensive logging information on the parameters found in the simtel
     configuration file.
 
@@ -266,7 +267,7 @@ def read_and_export_parameters(args_dict, logger):
             parameter_name=_parameter,
             value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]),
             instrument=args_dict["telescope"],
-            model_version=args_dict["model_version"],
+            parameter_version=args_dict["parameter_version"],
             output_file=io_handler.get_output_file(f"{_parameter}.json"),
         )
 

diff --git a/src/simtools/applications/convert_geo_coordinates_of_array_elements.py b/src/simtools/applications/convert_geo_coordinates_of_array_elements.py
@@ -133,7 +133,7 @@ def _parse(label=None, description=None):
         output=True,
         require_command_line=True,
         db_config=True,
-        simulation_model=["model_version", "site"],
+        simulation_model=["model_version", "parameter_version", "site"],
     )
 
 
@@ -170,7 +170,9 @@ def main():
 
     if args_dict["export"] is not None:
         product_data = (
-            layout.export_one_telescope_as_json(crs_name=args_dict["export"])
+            layout.export_one_telescope_as_json(
+                crs_name=args_dict["export"], parameter_version=args_dict.get("parameter_version")
+            )
             if args_dict.get("input", "").endswith(".json")
             else layout.export_telescope_list_table(crs_name=args_dict["export"])
         )

diff --git a/src/simtools/applications/convert_model_parameter_from_simtel.py b/src/simtools/applications/convert_model_parameter_from_simtel.py
@@ -78,7 +78,7 @@ def _parse(label=None, description=None):
         type=str,
         required=True,
     )
-    return config.initialize(simulation_model=["telescope", "model_version"], output=True)
+    return config.initialize(simulation_model=["telescope", "parameter_version"], output=True)
 
 
 def main():  # noqa: D103
@@ -107,7 +107,7 @@ def main():  # noqa: D103
         parameter_name=simtel_config_reader.parameter_name,
         value=simtel_config_reader.parameter_dict.get(args_dict["simtel_telescope_name"]),
         instrument=args_dict["telescope"],
-        model_version=args_dict["model_version"],
+        parameter_version=args_dict["parameter_version"],
         output_file=args_dict["output_file"],
         output_path=args_dict.get("output_path"),
         use_plain_output_path=args_dict.get("use_plain_output_path"),

diff --git a/src/simtools/applications/db_add_file_to_db.py b/src/simtools/applications/db_add_file_to_db.py
@@ -18,8 +18,7 @@
         A directory with files to upload to the DB. \
         All files in the directory with a predefined list of extensions will be uploaded.
     db (str)
-        The DB to insert the files to. \
-        The choices are either the default CTA simulation DB or a sandbox for testing.
+        The DB to insert the files to.
 
     Example
     -------
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Major restructering of database routines plus introduction of new simulation model.