Add integration tests to new simtools-prod image

.github/workflows/CI-integrationtests.yml

@@ -1,6 +1,5 @@
 ---
 name: CI-integrationtests
-# Integration tests for applications
 
 on:
   workflow_dispatch:
@@ -10,6 +9,29 @@
         required: false
         type: string
         default: 'main'
+  workflow_call:
+    inputs:
+      container_image:
+        description: 'Container image to use for tests'
+        required: false
+        type: string
+        default: 'ghcr.io/gammasim/simtools-dev:latest'
+      model_versions:
+        description: 'Model versions to test'
+        required: false
+        type: string
+        default: '["7.0.0","6.0.2","5.0.0","6.0.2,6.1.1"]'
+    secrets:
+      CLOUD_QGSJET3:
+        required: false
+      DB_SERVER:
+        required: false
+      DB_API_USER:
+        required: false
+      DB_API_PW:
+        required: false
+      DB_API_PORT:
+        required: false
   pull_request:
     types: [opened, synchronize]
     paths-ignore:
@@ -20,32 +42,33 @@
   release:
     types: [published]
 
+env:
+  CLOUD_URL: "https://syncandshare.desy.de/index.php/s/"
+
 jobs:
 
   test_building:
-    # Build the package and check if it is installable
-    # (tests among others that all components are there)
+    if: ${{ github.event_name != 'workflow_call' }}
     runs-on: ubuntu-latest
 
     defaults:
       run:
         shell: bash -leo pipefail {0}
 
     steps:
+      - uses: actions/checkout@v6
 
-      - name: checkout
-        uses: actions/checkout@v6
-
-      - name: Build the package
+      - name: Build package
         run: |
-          python -m pip install --upgrade pip
-          python -m pip install --upgrade build
+          python -m pip install --upgrade pip build
           python -m build
 
   integration_tests:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     container:
-      image: ghcr.io/gammasim/simtools-dev:latest
+      image: ${{ inputs.container_image || 'ghcr.io/gammasim/simtools-dev:latest' }}
       options: --user 0
 
     services:
@@ -63,60 +86,51 @@
     strategy:
       fail-fast: false
       matrix:
-        model_version: ['7.0.0', '6.0.2', '5.0.0', '6.0.2,6.1.1']
+        model_version: ${{ fromJSON(inputs.model_versions || '["7.0.0","6.0.2","5.0.0","6.0.2,6.1.1"]') }}
 
     defaults:
       run:
         shell: bash -leo pipefail {0}
 
     steps:
-      - name: checkout
-        uses: actions/checkout@v6
+      - uses: actions/checkout@v6
         with:
           fetch-depth: 0
 
-      - name: Set sim_telarray path
+      - name: Create runtime environment (.env)
         run: |
-          echo "PATH=\$PATH:/usr/bin:/usr/local/bin:$SIMTOOLS_SIM_TELARRAY_PATH" >> "$GITHUB_ENV"
+          set -a
+          source .env_template
+          set +a
+
+          DB_SERVER="mongodb"
+          DB_API_USER="api"
+          DB_API_PW="password"
+          DB_API_PORT="27017"
+          if [ "${{ github.event_name }}" = "schedule" ]; then
+            DB_SERVER="${{ secrets.DB_SERVER }}"
+            DB_API_USER="${{ secrets.DB_API_USER }}"
+            DB_API_PW="${{ secrets.DB_API_PW }}"
+            DB_API_PORT="${{ secrets.DB_API_PORT }}"
+          fi
 
-      - name: Create environment file (remote DB; schedule only)
-        if: ${{ github.event_name == 'schedule' }}
-        run: |
-          {
-            echo "SIMTOOLS_DB_SERVER=${{ secrets.DB_SERVER }}"
-            echo "SIMTOOLS_DB_API_USER=${{ secrets.DB_API_USER }}"
-            echo "SIMTOOLS_DB_API_PW=${{ secrets.DB_API_PW }}"
-            echo "SIMTOOLS_DB_API_PORT=${{ secrets.DB_API_PORT }}"
-            echo "SIMTOOLS_SIM_TELARRAY_PATH=$(grep '^SIMTOOLS_SIM_TELARRAY_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')"
-            echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')"
-            echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')"
-            echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')"
-          } > .env
-
-      - name: Extract SIMTOOLS_DB_SIMULATION_MODEL
-        run: |
-          SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL_VERSION=' .env_template | cut -d '=' -f2- | tr -d '"')
-          SIMTOOLS_DB_SIMULATION_MODEL=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL=' .env_template | cut -d '=' -f2- | tr -d '"')
           SIMTOOLS_DB_SIMULATION_MODEL="${SIMTOOLS_DB_SIMULATION_MODEL//\'/}"
-          echo "SIMTOOLS_DB_SIMULATION_MODEL=$SIMTOOLS_DB_SIMULATION_MODEL" >> "$GITHUB_ENV"
-          echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$SIMTOOLS_DB_SIMULATION_MODEL_VERSION" >> "$GITHUB_ENV"
-          echo "Simulation model set to: $SIMTOOLS_DB_SIMULATION_MODEL version $SIMTOOLS_DB_SIMULATION_MODEL_VERSION"
-
-      - name: Create environment file (local DB)
-        if: github.event_name != 'schedule'
-        run: |
           {
-            echo "SIMTOOLS_DB_SERVER=mongodb"
-            echo "SIMTOOLS_DB_API_USER=api"
-            echo "SIMTOOLS_DB_API_PW=password"
-            echo "SIMTOOLS_DB_API_PORT=27017"
-            echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }}"
-            echo "SIMTOOLS_DB_SIMULATION_MODEL=${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}"
-            echo "SIMTOOLS_SIM_TELARRAY_PATH=$(grep '^SIMTOOLS_SIM_TELARRAY_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')"
-            echo "SIMTOOLS_CORSIKA_PATH=$(grep '^SIMTOOLS_CORSIKA_PATH=' .env_template | cut -d '=' -f2- | tr -d '"')"
-            echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_HE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')"
-            echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$(grep '^SIMTOOLS_CORSIKA_LE_INTERACTION=' .env_template | cut -d '=' -f2- | tr -d '"')"
-          } > .env
+            echo "SIMTOOLS_DB_SERVER=$DB_SERVER"
+            echo "SIMTOOLS_DB_API_USER=$DB_API_USER"
+            echo "SIMTOOLS_DB_API_PW=$DB_API_PW"
+            echo "SIMTOOLS_DB_API_PORT=$DB_API_PORT"
+            echo "SIMTOOLS_DB_SIMULATION_MODEL=$SIMTOOLS_DB_SIMULATION_MODEL"
+            echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$SIMTOOLS_DB_SIMULATION_MODEL_VERSION"
+            echo "SIMTOOLS_SIM_TELARRAY_PATH=$SIMTOOLS_SIM_TELARRAY_PATH"
+            echo "SIMTOOLS_CORSIKA_PATH=$SIMTOOLS_CORSIKA_PATH"
+            echo "SIMTOOLS_CORSIKA_HE_INTERACTION=$SIMTOOLS_CORSIKA_HE_INTERACTION"
+            echo "SIMTOOLS_CORSIKA_LE_INTERACTION=$SIMTOOLS_CORSIKA_LE_INTERACTION"
+          } | tee .env >> "$GITHUB_ENV"
+
+      - name: Extend PATH (sim_telarray)
+        run: |
+          [ -n "$SIMTOOLS_SIM_TELARRAY_PATH" ] && echo "$SIMTOOLS_SIM_TELARRAY_PATH" >> "$GITHUB_PATH"
 
       - name: Determine simulation model branch
         env:
@@ -125,15 +139,13 @@
           EVENT_NAME: ${{ github.event_name }}
           INPUT_BRANCH: ${{ github.event.inputs.simulation_model_branch }}
         run: |
+          BRANCH="main"
           if [ "$EVENT_NAME" = "workflow_dispatch" ] && [ -n "$INPUT_BRANCH" ]; then
             BRANCH="$INPUT_BRANCH"
-          else
-            BRANCH="main"
           fi
+          CURRENT_REF="$REF_NAME"
           if [ "$EVENT_NAME" = "pull_request" ]; then
             CURRENT_REF="$HEAD_REF"
-          else
-            CURRENT_REF="$REF_NAME"
           fi
           # release candidates (e.g., v1.2.3-rc or v1.2.3-rc3) should be tested against released versions of the
           # simulation model database
@@ -143,34 +155,34 @@
           fi
           echo "SIMTOOLS_DB_SIMULATION_MODEL_BRANCH=$BRANCH" >> "$GITHUB_ENV"
 
-      - name: Print DB connection variables
+      - name: Install Python dependencies (prod container)
+        if: contains(inputs.container_image, 'simtools-prod')
+        run: |
+          pip install --no-cache-dir pytest pytest-cov pytest-requirements pytest-xdist pytest-retry
+
+      - name: Install Python dependencies (dev container)
+        if: ${{ !contains(inputs.container_image, 'simtools-prod') }}
         run: |
-          echo "Event name: ${{ github.event_name }}"
-          echo "DB Server: $SIMTOOLS_DB_SERVER"
-          echo "DB API User: $SIMTOOLS_DB_API_USER"
-          echo "DB API Port: $SIMTOOLS_DB_API_PORT"
-          echo "DB Simulation Model: $SIMTOOLS_DB_SIMULATION_MODEL"
-          echo "DB Simulation Model Version: $SIMTOOLS_DB_SIMULATION_MODEL_VERSION"
-          echo "DB Simulation Model Branch: $SIMTOOLS_DB_SIMULATION_MODEL_BRANCH"
+          pip install --no-cache-dir -e '.[tests,dev,doc]'
 
       - name: Upload data to MongoDB
         if: github.event_name != 'schedule'
         run: |
-          source /workdir/env/bin/activate
-          pip install --no-cache-dir -e '.[tests,dev,doc]'
           simtools-db-upload-model-repository \
             --db_simulation_model ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }} \
             --db_simulation_model_version ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} \
             --branch "$SIMTOOLS_DB_SIMULATION_MODEL_BRANCH"
 
+      - name: Download QGSJet (prod images without tables)
+        if: contains(inputs.container_image, 'simtools-prod')
+        run: |
+          wget -nv -O - "${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download" | \
+            bunzip2 > "${SIMTOOLS_CORSIKA_PATH}/qgsdat-III"
+
       - name: Integration tests
         shell: bash -l {0}
-        env:
-          SIMTOOLS_DB_SIMULATION_MODEL: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}
-          SIMTOOLS_DB_SIMULATION_MODEL_VERSION: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }}
-          SIMTOOLS_DB_SERVER: ${{ github.event_name == 'schedule' && secrets.DB_SERVER || 'mongodb' }}
         run: |
-          source /workdir/env/bin/activate
-          pip install --no-cache-dir -e '.[tests,dev,doc]'
+          cat .env
+          simtools-print-version
           pytest --model_version=${{ matrix.model_version }} --color=yes --durations=20 \
-           -n 4 --dist loadscope --retries 2 --retry-delay 5 --no-cov tests/integration_tests/
+            -n 4 --dist loadscope --retries 2 --retry-delay 5 --no-cov tests/integration_tests/

.github/workflows/build-simtools-prod.yml

@@ -22,6 +22,8 @@ jobs:
     permissions:
       contents: read
       packages: write
+    outputs:
+      no_opt_tag: ${{ steps.export-tags.outputs.no_opt_tag }}
     strategy:
       matrix:
         version:
@@ -68,18 +70,36 @@ jobs:
           images: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABEL }}
           flavor: latest=false
 
+      - name: Export no_opt tag for testing
+        id: export-tags
+        if: matrix.version.avx_flag == 'no_opt'
+        run: |
+          TAG=$(echo "${{ steps.meta.outputs.tags }}" | head -n1)
+          TAG=${TAG##*:}
+          echo "no_opt_tag=$TAG" >> "$GITHUB_OUTPUT"
+          echo "Exported no_opt tag: $TAG"
+
       - name: Build and push
         uses: docker/build-push-action@v6
         with:
           context: .
-          # Build multi-platform only for no_opt (non-optimized) variant, others are x86-specific
           platforms: ${{ matrix.version.avx_flag == 'no_opt' && 'linux/amd64,linux/arm64/v8' || 'linux/amd64' }}
           build-args: |
             CORSIKA_IMAGE_VERSION=${{ matrix.version.corsika }}
             SIMTEL_IMAGE_VERSION=${{ matrix.version.simtel_image }}
             AVX_FLAG=${{ matrix.version.avx_flag }}
             SIMTOOLS_BRANCH=${{ env.SIMTOOLS_BRANCH }}
-          push: ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' }}
+          push: ${{ github.event_name == 'release' || github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' || github.event_name == 'pull_request' }}
           file: ./docker/Dockerfile-simtools-prod
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
+
+  test-simtools-prod:
+    permissions:
+      contents: read
+    needs: build-simtools-prod
+    uses: ./.github/workflows/CI-integrationtests.yml
+    secrets: inherit
+    with:
+      container_image: ghcr.io/gammasim/simtools-prod:${{ needs.build-simtools-prod.outputs.no_opt_tag }}
+      model_versions: '["7.0.0","6.0.2"]'

docker/Dockerfile-simtools-prod

@@ -27,7 +27,7 @@ COPY --from=simtel_img /workdir/simulation_software/hessioxxx /workdir/simulatio
 COPY --from=simtel_img /workdir/simulation_software/stdtools /workdir/simulation_software/stdtools
 
 RUN microdnf update -y && microdnf install -y \
-    bc findutils gcc-c++ git gsl libgfortran procps wget zstd \
+    bc bzip2 findutils gcc-c++ git gsl libgfortran procps wget zstd \
     python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel && \
     microdnf clean all && \
     rm -rf /var/cache/dnf/* /tmp/* /var/tmp/* && \

docs/changes/1955.feature.md

@@ -0,0 +1 @@
+Add integration test run to updated production image building. Reuse existing integration tests and simplify building steps.