diff --git a/.github/workflows/build-corsika-simtel-legacy.yml b/.github/workflows/build-corsika-simtel-legacy.yml deleted file mode 100644 index 0e4f129180..0000000000 --- a/.github/workflows/build-corsika-simtel-legacy.yml +++ /dev/null @@ -1,160 +0,0 @@ ---- -name: build-corsika-simtel-legacy -# **legacy build** - will be deprecated in near future -# Build and push CORSIKA / sim_telarray base images for different CPU architectures. -# The generic (non-optimized) images are built as multi-platform images (x86_64 and arm64). -# -# The images are built and pushed when manually triggered (workflow_dispatch). - -on: - workflow_dispatch: - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - CLOUD_URL: "https://syncandshare.desy.de/index.php/s/" - -jobs: - download-corsika: - runs-on: ubuntu-latest - strategy: - matrix: - simtel_version: ['240205', '240927', '250903'] - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Download corsika-simtel-package - run: | - # CLOUD_SIMTEL_R_ secrets points to reduced tarballs without QGSJet and autoconf - wget --no-verbose ${{ env.CLOUD_URL }}/${{ secrets[format('CLOUD_SIMTEL_R_{0}', matrix.simtel_version)] }}/download -O corsika_simtelarray_${{ matrix.simtel_version }}.tar.gz - wget --no-verbose ${{ env.CLOUD_URL }}/${{ secrets.CLOUD_AUTOCONF }}/download -O autoconf.tar.gz - - name: Upload corsika-simtel-package - uses: actions/upload-artifact@v7 - with: - name: corsika-simtelarray-package-${{ matrix.simtel_version }} - path: | - corsika_simtelarray_${{ matrix.simtel_version }}.tar.gz - retention-days: 1 - - download-auxiliary-files: - runs-on: ubuntu-latest - - steps: - - - name: Download QGSJet and autoconf - run: | - wget --no-verbose ${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET2 }}/download -O qgsdat-II-04.bz2 - wget --no-verbose ${{ env.CLOUD_URL }}/${{ secrets.CLOUD_QGSJET3 }}/download -O qgsdat-III.bz2 - wget --no-verbose ${{ env.CLOUD_URL }}/${{ secrets.CLOUD_AUTOCONF }}/download -O autoconf.tar.gz - - name: Upload QGSJet files and autoconf - uses: actions/upload-artifact@v7 - with: - name: upload-auxiliary-files - path: | - qgsdat-II-04.bz2 - qgsdat-III.bz2 - autoconf.tar.gz - retention-days: 1 - - - build-base-images: - runs-on: ubuntu-latest - needs: [download-corsika, download-auxiliary-files] - permissions: - contents: read - packages: write - strategy: - matrix: - version: - # list of avx instructions: 'avx2', 'avx512f', 'sse4', 'avx' - - {simtel_version: '240205', corsika_version: '77500', bernlohr_version: '1.67', hadronic_model: 'qgs2', avx_flag: 'generic'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'generic'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'avx2'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'avx512f'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'sse4'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'generic'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'avx2'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'avx512f'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'sse4'} - production: - # list of productions build options: 'prod6-sc', 'prod6-baseline', 'prod5', 'prod5-sc' - # - {build_opt: 'prod6-sc', extra_def: ''} # currently not used - - {build_opt: 'prod6-baseline', extra_def: ''} - - env: - BASE_LABELS: "corsika-simtel-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_flag }}" - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Download corsika-simtel-package - uses: actions/download-artifact@v8 - with: - name: corsika-simtelarray-package-${{ matrix.version.simtel_version }} - path: ./downloaded-artifacts - - - name: Rename corsika-simtel-package files - run: | - mv ./downloaded-artifacts/corsika_simtelarray_${{ matrix.version.simtel_version }}.tar.gz corsika_simtelarray.tar.gz - - - name: Download auxiliary files - uses: actions/download-artifact@v8 - with: - name: upload-auxiliary-files - path: ./downloaded-artifacts - - - name: Rename auxiliary files - run: | - mv -v ./downloaded-artifacts/qgsdat-II-04.bz2 qgsdat-II-04.bz2 - mv -v ./downloaded-artifacts/qgsdat-III.bz2 qgsdat-III.bz2 - mv -v ./downloaded-artifacts/autoconf.tar.gz autoconf.tar.gz - ls -lh autoconf.tar.gz - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Log in to the Container registry - uses: docker/login-action@v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata for CORSIKA/sim_telarray base image (tags, labels) - id: meta-base - uses: docker/metadata-action@v6 - with: - tags: | - type=ref,event=pr - type=semver,pattern={{major}}.{{minor}}.{{patch}} - type=schedule,pattern={{date 'YYYYMMDD'}} - type=raw,value={{date 'YYYYMMDD-HHmmss'}} - images: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABELS }} - flavor: latest=true - - - name: Build and push CORSIKA/sim_telarray base image - uses: docker/build-push-action@v7 - with: - context: . - platforms: 'linux/amd64' - build-args: | - BUILD_OPT=${{ matrix.production.build_opt }} - EXTRA_DEF=${{ matrix.production.extra_def }} - HADRONIC_MODEL=${{ matrix.version.hadronic_model }} - AVX_FLAG=${{ matrix.version.avx_flag }} - CORSIKA_VERSION=${{ matrix.version.corsika_version }} - BERNLOHR_VERSION=${{ matrix.version.bernlohr_version }} - secrets: | - gitlab_token=${{ secrets.CORSIKA_OPT_PATCHES_GITLAB }} - push: ${{ github.event_name == 'workflow_dispatch' }} - file: ./docker/Dockerfile-corsika-simtel - tags: ${{ steps.meta-base.outputs.tags }} - labels: ${{ steps.meta-base.outputs.labels }} - - - name: Output image information - run: | - echo "Built base image: ${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABELS }}:latest" - echo "Image tags: ${{ steps.meta-base.outputs.tags }}" diff --git a/.github/workflows/build-sim_telarray.yml b/.github/workflows/build-sim_telarray.yml index 0f3759404a..96ef7b38c1 100644 --- a/.github/workflows/build-sim_telarray.yml +++ b/.github/workflows/build-sim_telarray.yml @@ -18,9 +18,32 @@ env: IMAGE_NAME: ${{ github.repository }} CLOUD_URL: "https://syncandshare.desy.de/index.php/s/" +permissions: + contents: read + jobs: + download-auxiliary-files: + runs-on: ubuntu-latest + + steps: + - name: download-auxiliary-files + run: | + curl -L --fail \ + --retry 5 --retry-delay 10 --retry-connrefused \ + --connect-timeout 30 \ + "${{ env.CLOUD_URL }}/${{ secrets.CLOUD_GSL }}/download" \ + -o gsl.tar.gz + - name: upload-auxiliary-files + uses: actions/upload-artifact@v7 + with: + name: upload-auxiliary-files + path: | + gsl.tar.gz + retention-days: 1 + build-sim_telarray: runs-on: ubuntu-latest + needs: [download-auxiliary-files] permissions: contents: read packages: write @@ -36,6 +59,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 + - name: Download auxiliary files + uses: actions/download-artifact@v8 + with: + name: upload-auxiliary-files + path: . + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v4 diff --git a/.github/workflows/build-simtools-legacy.yml b/.github/workflows/build-simtools-legacy.yml deleted file mode 100644 index 80f0cc5368..0000000000 --- a/.github/workflows/build-simtools-legacy.yml +++ /dev/null @@ -1,174 +0,0 @@ ---- -name: build-simtools-legacy -# **legacy build** - will be deprecated in near future -# Build and push simtools images using CORSIKA / sim_telarray base images. -# Integration tests are running on the generic (non-vector-optimized) images. - -on: - workflow_dispatch: - -env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - -jobs: - build-simtools-images: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - strategy: - matrix: - version: - # list of avx instructions: 'avx2', 'avx512f', 'sse4', 'avx' - - {simtel_version: '240205', corsika_version: '77500', bernlohr_version: '1.67', hadronic_model: 'qgs2', avx_flag: 'generic'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'generic'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'avx2'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'avx512f'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'sse4'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'generic'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'avx2'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'avx512f'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'sse4'} - production: - # list of productions build options: 'prod6-sc', 'prod6-baseline', 'prod5', 'prod5-sc' - # - {build_opt: 'prod6-sc', extra_def: ''} # currently not used - - {build_opt: 'prod6-baseline', extra_def: ''} - - env: - BASE_LABELS: "corsika-simtel-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_flag }}" - SIMTOOLS_LABELS: "simtools-sim-telarray-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_flag }}" - - steps: - - name: Checkout repository - uses: actions/checkout@v6 - - - name: Set build branch - run: | - if [[ "${{ github.event_name }}" == 'pull_request' ]]; then - echo "SIMTOOLS_BRANCH=${GITHUB_HEAD_REF}" >> "$GITHUB_ENV" - else - echo "SIMTOOLS_BRANCH=${GITHUB_REF#refs/heads/}" >> "$GITHUB_ENV" - fi - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v4 - - - name: Log in to the Container registry - uses: docker/login-action@v4 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Extract metadata for simtools image (tags, labels) - id: meta-simtools - uses: docker/metadata-action@v6 - with: - tags: | - type=ref,event=pr - type=semver,pattern={{major}}.{{minor}}.{{patch}} - type=schedule,pattern={{date 'YYYYMMDD'}} - type=raw,value={{date 'YYYYMMDD-HHmmss'}} - images: ${{ env.REGISTRY }}/gammasim/${{ env.SIMTOOLS_LABELS }} - flavor: latest=true - - - name: Build and push simtools image - uses: docker/build-push-action@v7 - with: - context: . - platforms: linux/amd64 - build-args: | - BASE_IMAGE=${{ env.REGISTRY }}/gammasim/${{ env.BASE_LABELS }}:latest - SIMTOOLS_BRANCH=${{ env.SIMTOOLS_BRANCH }} - push: ${{ github.event_name == 'release' || github.event.pull_request.merged == true || github.event_name == 'workflow_dispatch' }} - file: ./docker/Dockerfile-simtools-legacy - tags: ${{ steps.meta-simtools.outputs.tags }} - labels: ${{ steps.meta-simtools.outputs.labels }}-prod - - test-images: - runs-on: ubuntu-latest - needs: build-simtools-images - strategy: - matrix: - version: - # list of avx instructions: 'avx2', 'avx512f', 'sse4', 'avx' - - {simtel_version: '240205', corsika_version: '77500', bernlohr_version: '1.67', hadronic_model: 'qgs2', avx_flag: 'generic'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'generic'} - - {simtel_version: '240927', corsika_version: '77550', bernlohr_version: '1.68', hadronic_model: 'qgs2', avx_flag: 'avx2'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'generic'} - - {simtel_version: '250903', corsika_version: '78010', bernlohr_version: '1.70', hadronic_model: 'qgs3', avx_flag: 'avx2'} - production: - # list of productions build options: 'prod6-sc', 'prod6-baseline', 'prod5', 'prod5-sc' - # - {build_opt: 'prod6-sc', extra_def: ''} # currently not used - - {build_opt: 'prod6-baseline', extra_def: ''} - model_version: - - "6.0.2" - container: - image: ghcr.io/gammasim/simtools-sim-telarray-${{ matrix.version.simtel_version }}-corsika-${{ matrix.version.corsika_version }}-bernlohr-${{ matrix.version.bernlohr_version }}-${{ matrix.production.build_opt }}-${{ matrix.version.hadronic_model }}-${{ matrix.version.avx_flag }} - options: --user 0 - - services: - mongodb: - image: mongo:latest - env: - MONGO_INITDB_ROOT_USERNAME: api - MONGO_INITDB_ROOT_PASSWORD: password - options: >- - --health-cmd "mongosh --host localhost --port 27017 -u api -p password --authenticationDatabase admin --eval 'db.runCommand({ ping: 1 })'" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - defaults: - run: - shell: bash -leo pipefail {0} - - steps: - - name: checkout - uses: actions/checkout@v6 - with: - fetch-depth: 0 - - - name: Set sim_telarray path - run: | - echo "PATH=\$PATH:/usr/bin:/usr/local/bin:$SIMTOOLS_SIM_TELARRAY_PATH" >> "$GITHUB_ENV" - - - name: Extract SIMTOOLS_DB_SIMULATION_MODEL - run: | - SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL_VERSION=' .env_template | cut -d '=' -f2- | tr -d '"') - SIMTOOLS_DB_SIMULATION_MODEL=$(grep 'SIMTOOLS_DB_SIMULATION_MODEL=' .env_template | cut -d '=' -f2- | tr -d '"') - echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=$SIMTOOLS_DB_SIMULATION_MODEL_VERSION" >> "$GITHUB_ENV" - echo "SIMTOOLS_DB_SIMULATION_MODEL=$SIMTOOLS_DB_SIMULATION_MODEL" >> "$GITHUB_ENV" - echo "Simulation model set to: $SIMTOOLS_DB_SIMULATION_MODEL version $SIMTOOLS_DB_SIMULATION_MODEL_VERSION" - - - name: Create environment file (local DB) - run: | - { - echo "SIMTOOLS_DB_SERVER=mongodb" - echo "SIMTOOLS_DB_API_USER=api" - echo "SIMTOOLS_DB_API_PW=password" - echo "SIMTOOLS_DB_API_PORT=27017" - echo "SIMTOOLS_DB_SIMULATION_MODEL=${{ env.SIMTOOLS_DB_SIMULATION_MODEL }}" - echo "SIMTOOLS_DB_SIMULATION_MODEL_VERSION=${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }}" - echo "SIMTOOLS_SIM_TELARRAY_PATH=/workdir/sim_telarray/sim_telarray" - echo "SIMTOOLS_CORSIKA_PATH=/workdir/sim_telarray/corsika-run" - } > .env - - - name: Upload data to MongoDB - run: | - source /workdir/env/bin/activate - pip install --no-cache-dir -e '.[tests,dev,doc]' - simtools-db-upload-model-repository \ - --db_simulation_model ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }} \ - --db_simulation_model_version ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} - - - name: Integration tests - shell: bash -l {0} - env: - SIMTOOLS_DB_SIMULATION_MODEL: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL }} - SIMTOOLS_DB_SIMULATION_MODEL_VERSION: ${{ env.SIMTOOLS_DB_SIMULATION_MODEL_VERSION }} - SIMTOOLS_DB_SERVER: 'mongodb' - run: | - source /workdir/env/bin/activate - pip install --no-cache-dir pytest pytest-cov pytest-requirements pytest-xdist - pytest --model_version=${{ matrix.model_version }} --no-cov --color=yes -n auto tests/integration_tests/test_applications_from_config.py diff --git a/docker/Dockerfile-corsika-simtel b/docker/Dockerfile-corsika-simtel deleted file mode 100644 index a1c60e2bd0..0000000000 --- a/docker/Dockerfile-corsika-simtel +++ /dev/null @@ -1,210 +0,0 @@ -# syntax=docker/dockerfile:1.4 -# Dockerfile for base CORSIKA / sim_telarray build. -# -# Contains CORSIKA (possibly optimized; with and without curved atmosphere) -# and sim_telarray -# Requires CORSIKA and sim_telarray source code tarballs (plus tarball for -# CORSIKA optimized build) to be in the same directory as this Dockerfile. - -# Shared build arguments -ARG CORSIKA_VERSION="77550" -ARG CORSIKA_OPT_PATCH_VERSION="v1.1.0" -ARG BUILD_OPT="prod6-sc" -ARG EXTRA_DEF="-DMAXIMUM_TELESCOPES=128" -ARG HADRONIC_MODEL="qgs2" -ARG AVX_FLAG="avx2" -ARG BERNLOHR_VERSION="1.68" -ARG AUTOCONF_VERSION="2.71" - -# -# hadolint global ignore=DL3013,DL3033,DL3041,SC1091 -# - DL3013, DL3033: ignore warnings about installing non-specific versions with microdnf or yum) -# - SC1091: ignore warning about not being able to source the bashrc - -FROM almalinux:9.5 AS patches_stage -ARG CORSIKA_VERSION -ARG CORSIKA_OPT_PATCH_VERSION - -RUN yum update -y && yum install -y git && \ - yum clean all && \ - rm -rf /var/cache/yum/* /tmp/* /var/tmp/* - -RUN --mount=type=secret,id=gitlab_token \ - git config --global advice.detachedHead false && \ - git clone --depth 1 --branch "${CORSIKA_OPT_PATCH_VERSION}" "https://oauth2:$(cat /run/secrets/gitlab_token)@gitlab.cta-observatory.org/cta-computing/dpps/simpipe/corsika-opt-patches.git" && \ - cp -rL corsika-opt-patches/build_opt/corsikaOptPatch-$CORSIKA_VERSION/ ./corsika-patches/ && \ - rm -rf corsika-opt-patches - -FROM almalinux:9.5 AS build_image -ARG BUILD_OPT -ARG EXTRA_DEF -ARG HADRONIC_MODEL -ARG AVX_FLAG -ARG CORSIKA_VERSION -ARG BERNLOHR_VERSION -ARG CORSIKA_OPT_PATCH_VERSION -ARG AUTOCONF_VERSION - -ENV GCC_PATH="/usr/bin/" -ENV CORSIKA_DIR="corsika-${CORSIKA_VERSION}" - -RUN yum update -y && yum install -y \ - gcc-fortran gsl-devel libgfortran \ - automake csh patch perl which bzip2 \ - wget make tar m4 && \ - yum clean all && \ - rm -rf /var/cache/yum/* /tmp/* /var/tmp/* - -WORKDIR /tmp/ -ADD autoconf.tar.gz /tmp/ -# hadolint ignore=DL3003 -RUN cd "autoconf-$AUTOCONF_VERSION" && \ - ./configure --prefix=/usr && \ - make && make install && \ - cd / && rm -rf /tmp/autoconf-* - -WORKDIR /workdir/sim_telarray -COPY corsika_simtelarray.tar.gz . - -RUN tar -zxf corsika_simtelarray.tar.gz && \ - tar -zxf ${CORSIKA_DIR}.tar.gz && \ - tar -zxf bernlohr-$BERNLOHR_VERSION.tar.gz -C ${CORSIKA_DIR}/bernlohr && \ - rm -f examples-with-data.tar.gz sim_telarray_config.tar.gz \ - corsika_simtelarray.tar.gz ${CORSIKA_DIR}.tar.gz \ - bernlohr-$BERNLOHR_VERSION.tar.gz - -COPY --from=patches_stage /corsika-patches ./${CORSIKA_DIR}/ - -# Apply patches for non-optimized / optimized CORSIKA -WORKDIR /workdir/sim_telarray/${CORSIKA_DIR} -RUN if [ "$CORSIKA_VERSION" = "77550" ]; then \ - patch -b -p0 src/corsika.F < "../corsika-77550.patch"; \ - fi && \ - if [ "$AVX_FLAG" != "generic" ]; then \ - ./corsika_opt_patching.sh && \ - autoreconf; \ - fi - -# Build non-optimized / optimized CORSIKA -WORKDIR /workdir/sim_telarray -# hadolint ignore=DL3003 -RUN for variant in curved ""; do \ - (cd ./${CORSIKA_DIR} && \ - if [ "$AVX_FLAG" = "generic" ] || [ "$variant" = "curved" ]; then \ - ../corsika_build_script "$HADRONIC_MODEL" generic $variant; \ - else \ - ../corsika_build_script "$HADRONIC_MODEL" generic cerenkopt vlibm $variant; \ - fi) && \ - make -C "${CORSIKA_DIR}" clean && rm -f ./*/*.a lib/*/*.a && \ - AVX_EXTRA_FLAG=$([ "$AVX_FLAG" = "avx512f" ] && echo "-ffp-contract=off" || echo "") && \ - C_SPECIFIC_FLAGS="-std=c99" && \ - FORTRAN_SPECIFIC_FLAGS="-ffixed-line-length-132 -fno-automatic -frecord-marker=4 -std=legacy" && \ - if [ "$AVX_FLAG" = "generic" ] || [ "$variant" = "curved" ]; then \ - BASE_FLAGS="-g -O3"; \ - else \ - BASE_FLAGS="-g -DCERENKOPT -DVLIBM -O3 ${AVX_FLAG:+-m$AVX_FLAG} ${AVX_EXTRA_FLAG} -DVECTOR_SIZE=8 -DVECTOR_LENGTH=8"; \ - fi && \ - CFLAGS="${BASE_FLAGS} ${C_SPECIFIC_FLAGS}" && \ - CXXFLAGS="${BASE_FLAGS} ${C_SPECIFIC_FLAGS}" && \ - FFLAGS="${BASE_FLAGS} ${FORTRAN_SPECIFIC_FLAGS}" && \ - make -C "${CORSIKA_DIR}" \ - CC="${GCC_PATH}/gcc -static" \ - CXX="${GCC_PATH}/g++ -static" \ - CPP="${GCC_PATH}/gcc -static -E" \ - F77="${GCC_PATH}/gfortran -L/lib64/" \ - CFLAGS="${CFLAGS}" \ - CXXFLAGS="${CXXFLAGS}" \ - FFLAGS="${FFLAGS}" \ - install && \ - echo "=== Listing ${CORSIKA_DIR}/run/ after install (variant: $variant) ===" && \ - ls -la "${CORSIKA_DIR}/run/" && \ - mkdir -p corsika-run && \ - if [ "$variant" = "curved" ]; then \ - mv -v -f "${CORSIKA_DIR}/run/corsika${CORSIKA_VERSION}Linux_"* "corsika-run/corsika-$variant" && \ - cp -v -f "${CORSIKA_DIR}/src/corsikacompilefile.f" "corsika-run/corsikacompilefile_$variant.f" && \ - echo "=== After curved variant copy, corsika-run contains: ===" && \ - ls -la corsika-run/; \ - else \ - mv -v -f "${CORSIKA_DIR}/run/corsika${CORSIKA_VERSION}Linux_"* "corsika-run/corsika" && \ - cp -v -f "${CORSIKA_DIR}/src/corsikacompilefile.f" "corsika-run/corsikacompilefile.f" && \ - echo "=== After non-curved variant copy, corsika-run contains: ===" && \ - ls -la corsika-run/; \ - fi && \ - if [ "$variant" = "" ]; then \ - if [ -d "${CORSIKA_DIR}/run/" ]; then \ - cp -r -v "${CORSIKA_DIR}/run/"* corsika-run/; \ - else \ - echo "Skipping copy: source directory ${CORSIKA_DIR}/run/ does not exist."; \ - fi; \ - fi; \ - done - -# Build sim_telarray (with and without debug options) -RUN touch ${CORSIKA_DIR}.tar.gz && \ - # build with debug options; move binary to temporary location (to avoid being purged in next step) - export NO_CORSIKA=1 EXTRA_DEFINES="${EXTRA_DEF} -DDEBUG_TRACE_99" && ./build_all $BUILD_OPT $HADRONIC_MODEL gsl && \ - mv /workdir/sim_telarray/sim_telarray/sim_telarray \ - /workdir/sim_telarray/sim_telarray/sim_telarray_debug_trace && \ - # build without debug options; move debug option binary back to sim_telarray/bin - export NO_CORSIKA=1 EXTRA_DEFINES="${EXTRA_DEF}" && ./build_all $BUILD_OPT $HADRONIC_MODEL gsl && \ - mv /workdir/sim_telarray/sim_telarray/sim_telarray_debug_trace \ - /workdir/sim_telarray/sim_telarray/bin/sim_telarray_debug_trace - -# Cleanup. -# CORSIKA source code is removed to follow -# the CORSIKA non-distribution policy. -RUN rm -rf ./${CORSIKA_DIR}* && \ - find . -name "*.tar.gz" -delete && \ - find . -name "doc*" -type d -exec rm -rf "{}" + 2>/dev/null && \ - find . -name "*.o" -delete && \ - rm -rf /var/cache/yum/* /tmp/* /var/tmp/* && \ - rm -f build_all clean_all corsika_build_script init_batch_job_SGE && \ - rm -f new_export test_build_all && \ - rm -rf ./sim_telarray/cfg ./lib ./include && \ - rm -f ./example_scripts/Prod6/'*' - -# Use QGSJet tables downloaded externally (overwrite any existing copies) -WORKDIR /workdir/sim_telarray/corsika-run -COPY qgsdat-II-04.bz2 . -COPY qgsdat-III.bz2 . -RUN if [ "$HADRONIC_MODEL" = "qgs3" ]; then \ - if [ -f qgsdat-III.bz2 ]; then \ - bzip2 -dq qgsdat-III.bz2; \ - fi; \ - rm -f -v qgsdat-II-04.bz2; \ - fi && \ - if [ "$HADRONIC_MODEL" = "qgs2" ]; then \ - if [ -f qgsdat-II-04.bz2 ]; then \ - bzip2 -dq qgsdat-II-04.bz2; \ - fi; \ - rm -f -v qgsdat-III.bz2; \ - fi - -# Add a yml file in workdir with the build options -WORKDIR /workdir/sim_telarray -RUN echo "build_opt: $BUILD_OPT" > ./build_opts.yml && \ - echo "extra_def: $EXTRA_DEF" >> ./build_opts.yml && \ - echo "hadronic_model: $HADRONIC_MODEL" >> ./build_opts.yml && \ - echo "avx_flag: $AVX_FLAG" >> ./build_opts.yml && \ - echo "corsika_version: $CORSIKA_VERSION" >> ./build_opts.yml && \ - echo "corsika_opt_patch_version: $CORSIKA_OPT_PATCH_VERSION" >> ./build_opts.yml && \ - echo "bernlohr_version: $BERNLOHR_VERSION" >> ./build_opts.yml - -# Final stage - minimal base image with CORSIKA and sim_telarray -FROM almalinux:9.5-minimal -COPY --from=build_image /workdir/sim_telarray/ /workdir/sim_telarray/ -WORKDIR /workdir - -RUN microdnf update -y && microdnf install -y \ - bc \ - zstd \ - gsl-devel \ - gcc-fortran \ - procps \ - findutils \ - gcc-c++ \ - libgfortran && \ - microdnf clean all && \ - rm -rf /var/cache/dnf/* /tmp/* /var/tmp/* - -ENV SIMTEL_PATH="/workdir/sim_telarray" diff --git a/docker/Dockerfile-corsika7 b/docker/Dockerfile-corsika7 index 01ef58bda4..ab5ef99d00 100644 --- a/docker/Dockerfile-corsika7 +++ b/docker/Dockerfile-corsika7 @@ -14,6 +14,7 @@ ARG CORSIKA_VERSION="78010" ARG CORSIKA_OPT_PATCH_VERSION="v1.1.0" ARG CORSIKA_CONFIG_VERSION="v0.1.0" ARG AVX_FLAG="avx2" +ARG ALMALINUX_VERSION="9.7" # hadolint global ignore=DL3013,DL3033,DL3041,DL4006,SC1091,SC2086 # - DL3013, DL3033: ignore warnings about installing non-specific versions with microdnf or yum) @@ -24,7 +25,7 @@ ARG AVX_FLAG="avx2" # ----------------------------------------------------------------------------- # STAGE 1: source code preparation # ----------------------------------------------------------------------------- -FROM almalinux:9.5-minimal AS source_preparation +FROM almalinux:${ALMALINUX_VERSION}-minimal AS source_preparation ARG CORSIKA_REPO="gitlab.iap.kit.edu/AirShowerPhysics/corsika-legacy/corsika7.git" ARG CORSIKA_VERSION ARG CORSIKA_OPT_PATCH_REPO="gitlab.cta-observatory.org/cta-computing/dpps/simpipe/simulation_software/corsika-opt-patches.git" @@ -32,9 +33,7 @@ ARG CORSIKA_OPT_PATCH_VERSION ARG CORSIKA_CONFIG_REPO="gitlab.cta-observatory.org/cta-computing/dpps/simpipe/simulation_software/corsika7-config.git" ARG CORSIKA_CONFIG_VERSION -RUN microdnf update -y && microdnf install -y git && \ - microdnf clean all - +RUN microdnf update -y && microdnf install -y git && microdnf clean all WORKDIR /src # Clone CORSIKA configuration files from CTAO gitlab @@ -55,13 +54,13 @@ RUN git config --global advice.detachedHead false && \ cp -rL corsika-opt-patches/build_opt/corsikaOptPatch-$CORSIKA_VERSION/* corsika7/ && \ rm -rf corsika-opt-patches -# autoconf 2.71 (tar files need to present in current directory) +# autoconf 2.71 (tar file needs to present in current directory) ADD autoconf.tar.gz . # ----------------------------------------------------------------------------- # STAGE 2: build_image # ----------------------------------------------------------------------------- -FROM almalinux:9.5 AS build_image +FROM almalinux:${ALMALINUX_VERSION} AS build_image ARG AVX_FLAG ARG CORSIKA_VERSION ARG CORSIKA_OPT_PATCH_VERSION @@ -154,7 +153,7 @@ RUN CORSIKA_RUN_DIR="/workdir/corsika-run" && \ # ----------------------------------------------------------------------------- # STAGE 3: Runtime # ----------------------------------------------------------------------------- -FROM almalinux:9.5-minimal +FROM almalinux:${ALMALINUX_VERSION}-minimal WORKDIR /workdir/simulation_software COPY --from=build_image /workdir/corsika-run /workdir/simulation_software/corsika7 diff --git a/docker/Dockerfile-simtel_array b/docker/Dockerfile-simtel_array index 532732ad9f..63e97659b3 100644 --- a/docker/Dockerfile-simtel_array +++ b/docker/Dockerfile-simtel_array @@ -8,6 +8,8 @@ ARG EXTRA_DEFINES="-DCTA_PROD6 -DMAXIMUM_TELESCOPES=99" ARG SIMTEL_VERSION="master" ARG HESSIO_VERSION="master" ARG STDTOOLS_VERSION="master" +ARG ALMALINUX_VERSION="9.7" +ARG GSL_VERSION="2.8" # hadolint global ignore=DL3033,DL3041 # - DL3033, DL3041: ignore warnings about installing non-specific versions with microdnf or yum) @@ -15,7 +17,7 @@ ARG STDTOOLS_VERSION="master" # ----------------------------------------------------------------------------- # STAGE 1: source code preparation # ----------------------------------------------------------------------------- -FROM almalinux:9.5-minimal AS source_preparation +FROM almalinux:${ALMALINUX_VERSION}-minimal AS source_preparation ARG SIMTEL_REPO="gitlab.cta-observatory.org/cta-computing/dpps/simpipe/simulation_software/sim_telarray.git" ARG SIMTEL_VERSION ARG HESSIO_REPO="gitlab.cta-observatory.org/cta-computing/dpps/simpipe/simulation_software/hessio.git" @@ -43,26 +45,37 @@ RUN --mount=type=secret,id=stdtools_token \ git config --global advice.detachedHead false && \ git clone --depth 1 --branch "${STDTOOLS_VERSION}" "https://oauth2:$(cat /run/secrets/stdtools_token)@${STDTOOLS_REPO}" +# GSL (tar file needs to present in current directory) +ADD gsl.tar.gz . + # ----------------------------------------------------------------------------- # STAGE 2: build_image # ----------------------------------------------------------------------------- -FROM almalinux:9.5 AS build_image +FROM almalinux:${ALMALINUX_VERSION} AS build_image ARG EXTRA_DEFINES ARG SIMTEL_VERSION="master" ARG HESSIO_VERSION="master" ARG STDTOOLS_VERSION="master" ARG GSL_COMPILE="-DWITH_GSL -DWITH_GSL_RNG" +ARG GSL_VERSION ENV GCC_PATH="/usr/bin/" ENV HSEVENTIO="/workdir/simulation_software/hessioxxx/" ENV STDTOOLS_PATH="/workdir/simulation_software/stdtools/" RUN yum update -y && yum install -y \ - gcc gcc-c++ gsl-devel \ - automake csh patch perl \ + gcc gcc-c++ \ + automake csh diffutils patch perl \ make m4 which && \ yum clean all +WORKDIR /tmp +COPY --from=source_preparation /src/gsl-${GSL_VERSION} . +RUN ./configure --prefix=/usr/local && \ + make && \ + make install && \ + ldconfig + WORKDIR /workdir/simulation_software COPY --from=source_preparation /src . @@ -132,14 +145,20 @@ RUN { \ # ----------------------------------------------------------------------------- # STAGE 3: Runtime Image # ----------------------------------------------------------------------------- -FROM almalinux:9.5-minimal +FROM almalinux:${ALMALINUX_VERSION}-minimal WORKDIR /workdir/simulation_software COPY --from=build_image /workdir/simulation_software/hessioxxx /workdir/simulation_software/hessioxxx COPY --from=build_image /workdir/simulation_software/stdtools /workdir/simulation_software/stdtools COPY --from=build_image /workdir/simulation_software/sim_telarray /workdir/simulation_software/sim_telarray +COPY --from=build_image /usr/local/lib/libgsl.so* /usr/local/lib/ +COPY --from=build_image /usr/local/lib/libgslcblas.so* /usr/local/lib/ RUN microdnf update -y && microdnf install -y \ - bc findutils gsl procps zstd && \ - microdnf clean all + bc findutils procps zstd && \ + microdnf clean all && \ + echo /usr/local/lib > /etc/ld.so.conf.d/local.conf && \ + ldconfig + +ENV LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" WORKDIR /workdir/simulation_software/sim_telarray diff --git a/docker/Dockerfile-simtools-dev b/docker/Dockerfile-simtools-dev index 678c596a90..bb4a4a1ceb 100644 --- a/docker/Dockerfile-simtools-dev +++ b/docker/Dockerfile-simtools-dev @@ -15,24 +15,28 @@ ARG CORSIKA_IMAGE_VERSION="v78010" ARG SIMTEL_IMAGE_VERSION="latest" ARG AVX_FLAG="generic" ARG PYTHON_VERSION="3.12" +ARG ALMALINUX_VERSION="9.7" FROM ghcr.io/gammasim/corsika7:${CORSIKA_IMAGE_VERSION}-${AVX_FLAG} AS corsika_img FROM ghcr.io/gammasim/sim_telarray:${SIMTEL_IMAGE_VERSION} AS simtel_img -FROM almalinux:9.5-minimal +FROM almalinux:${ALMALINUX_VERSION}-minimal ARG PYTHON_VERSION -ARG SIMTOOLS_REPO="https://github.com/gammasim/simtools.git" COPY --from=corsika_img /workdir/simulation_software/corsika7 /workdir/simulation_software/corsika7 COPY --from=simtel_img /workdir/simulation_software/sim_telarray /workdir/simulation_software/sim_telarray COPY --from=simtel_img /workdir/simulation_software/hessioxxx /workdir/simulation_software/hessioxxx COPY --from=simtel_img /workdir/simulation_software/stdtools /workdir/simulation_software/stdtools +COPY --from=simtel_img /usr/local/lib/ /usr/local/lib/ RUN microdnf update -y && microdnf install -y \ - bc bzip2 findutils gcc-c++ git gsl libgfortran procps wget zstd \ - python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel && \ - microdnf clean all && \ - ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python + bc bzip2 findutils gcc-c++ git libgfortran procps wget zstd \ + python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel && \ + microdnf clean all && \ + rm -rf /var/cache/dnf/* /tmp/* /var/tmp/* && \ + ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python && \ + echo /usr/local/lib > /etc/ld.so.conf.d/local.conf && \ + ldconfig RUN { \ printf '%s\n' "if [ -n \"\${GITHUB_WORKSPACE:-}\" ]; then"; \ @@ -54,10 +58,12 @@ RUN wget --quiet https://raw.githubusercontent.com/gammasim/simtools/main/pyproj pip uninstall -y toml-to-requirements && \ pip install --no-cache-dir -r requirements.txt -ENV SIMTEL_PATH="/workdir/simulation_software/sim_telarray" +ENV SIMTOOLS_SIM_TELARRAY_PATH="/workdir/simulation_software/sim_telarray" +ENV SIMTOOLS_CORSIKA_PATH="/workdir/simulation_software/corsika7" ENV LD_LIBRARY_PATH="/workdir/simulation_software/hessioxxx/lib:\ /workdir/simulation_software/stdtools/lib:\ -/workdir/simulation_software/sim_telarray/LightEmission/lib" +/workdir/simulation_software/sim_telarray/LightEmission/lib:\ +/usr/local/lib" ENV PATH="/workdir/env/bin/:$PATH" SHELL ["/bin/bash", "-c"] diff --git a/docker/Dockerfile-simtools-legacy b/docker/Dockerfile-simtools-legacy deleted file mode 100644 index bbf6187a72..0000000000 --- a/docker/Dockerfile-simtools-legacy +++ /dev/null @@ -1,42 +0,0 @@ -# syntax=docker/dockerfile:1.4 -# Dockerfile for simtools build on top of CORSIKA/sim_telarray base image. -# -# This Dockerfile adds simtools to a base image that already contains -# CORSIKA and sim_telarray. -# -# hadolint global ignore=DL3007,DL3013,DL3041,SC1091 -# - DL3007, DL3041: ignore warnings about using latest -# - DL3013: ignore warnings about installing non-specific versions with microdnf) -# - SC1091: ignore warning about not being able to source the bashrc - -ARG BASE_IMAGE -ARG SIMTOOLS_BRANCH=main -ARG SIMTOOLS_REPO="https://github.com/gammasim/simtools.git" -ARG PYTHON_VERSION=3.12 - -FROM ${BASE_IMAGE} -ARG SIMTOOLS_BRANCH -ARG SIMTOOLS_REPO -ARG PYTHON_VERSION - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN microdnf update -y && microdnf install -y \ - git gcc gcc-c++ \ - python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel && \ - microdnf clean all && \ - rm -rf /var/cache/dnf/* /tmp/* /var/tmp/* && \ - ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python - -RUN SIMTOOLS_BRANCH=${SIMTOOLS_BRANCH//refs\/tags\//} \ - && git clone -b "$SIMTOOLS_BRANCH" "${SIMTOOLS_REPO}" --depth 1 \ - && python${PYTHON_VERSION} -m venv env \ - && . env/bin/activate \ - && pip install --no-cache-dir -U pip \ - && pip install --no-cache-dir "git+${SIMTOOLS_REPO}@${SIMTOOLS_BRANCH}" \ - && pip cache purge && rm -rf /root/.cache \ - && echo ". /workdir/env/bin/activate" >> ~/.bashrc - -ENV SIMTOOLS_SIMTEL_PATH="/workdir/sim_telarray/sim_telarray" -ENV SIMTOOLS_CORSIKA_PATH="/workdir/sim_telarray/corsika-run" -ENV PATH="/workdir/env/bin/:$PATH" diff --git a/docker/Dockerfile-simtools-prod b/docker/Dockerfile-simtools-prod index ff519550ca..b1cce40ada 100644 --- a/docker/Dockerfile-simtools-prod +++ b/docker/Dockerfile-simtools-prod @@ -12,11 +12,12 @@ ARG SIMTEL_IMAGE_VERSION="latest" ARG AVX_FLAG="generic" ARG PYTHON_VERSION="3.12" ARG SIMTOOLS_BRANCH=main +ARG ALMALINUX_VERSION="9.7" FROM ghcr.io/gammasim/corsika7:${CORSIKA_IMAGE_VERSION}-${AVX_FLAG} AS corsika_img FROM ghcr.io/gammasim/sim_telarray:${SIMTEL_IMAGE_VERSION} AS simtel_img -FROM almalinux:9.5-minimal +FROM almalinux:${ALMALINUX_VERSION}-minimal ARG PYTHON_VERSION ARG SIMTOOLS_BRANCH ARG SIMTOOLS_REPO="https://github.com/gammasim/simtools.git" @@ -25,14 +26,17 @@ COPY --from=corsika_img /workdir/simulation_software/corsika7 /workdir/simulati COPY --from=simtel_img /workdir/simulation_software/sim_telarray /workdir/simulation_software/sim_telarray COPY --from=simtel_img /workdir/simulation_software/hessioxxx /workdir/simulation_software/hessioxxx COPY --from=simtel_img /workdir/simulation_software/stdtools /workdir/simulation_software/stdtools +COPY --from=simtel_img /usr/local/lib/ /usr/local/lib/ RUN microdnf update -y && microdnf install -y \ - bc bzip2 findutils gcc-c++ git git-lfs gsl libgfortran procps wget zstd \ - python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel && \ - microdnf clean all && \ - rm -rf /var/cache/dnf/* /tmp/* /var/tmp/* && \ - ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python && \ - git lfs install --system + bc bzip2 findutils gcc-c++ git git-lfs libgfortran procps wget zstd \ + python${PYTHON_VERSION} python${PYTHON_VERSION}-pip python${PYTHON_VERSION}-devel && \ + microdnf clean all && \ + rm -rf /var/cache/dnf/* /tmp/* /var/tmp/* && \ + ln -sf /usr/bin/python${PYTHON_VERSION} /usr/bin/python && \ + git lfs install --system && \ + echo /usr/local/lib > /etc/ld.so.conf.d/local.conf && \ + ldconfig WORKDIR /workdir @@ -48,6 +52,8 @@ RUN SIMTOOLS_BRANCH="$(printf '%s' "$SIMTOOLS_BRANCH" | sed 's#refs/tags/##')" \ ENV SIMTOOLS_SIM_TELARRAY_PATH="/workdir/simulation_software/sim_telarray" ENV SIMTOOLS_CORSIKA_PATH="/workdir/simulation_software/corsika7" -ENV SIMTOOLS_CORSIKA_HE_INTERACTION="qgs3" -ENV SIMTOOLS_CORSIKA_LE_INTERACTION="urqmd" +ENV LD_LIBRARY_PATH="/workdir/simulation_software/hessioxxx/lib:\ +/workdir/simulation_software/stdtools/lib:\ +/workdir/simulation_software/sim_telarray/LightEmission/lib:\ +/usr/local/lib" ENV PATH="/workdir/env/bin/:$PATH" diff --git a/docs/changes/2153.maintenance.md b/docs/changes/2153.maintenance.md new file mode 100644 index 0000000000..838f6986db --- /dev/null +++ b/docs/changes/2153.maintenance.md @@ -0,0 +1 @@ +Update base images to alma linux 9.7. Install GSL from source for sim_telarray.