diff --git a/.github/workflows/cross-compile.yml b/.github/workflows/cross-compile.yml index a783209..ef3c12f 100644 --- a/.github/workflows/cross-compile.yml +++ b/.github/workflows/cross-compile.yml @@ -36,7 +36,8 @@ jobs: python -m pip install --upgrade pip pip install conan - - name: Cache Conan packages + - name: Cache Conan packages (Windows/macOS) + if: runner.os != 'Linux' uses: actions/cache@v4 with: path: ~/.conan2 @@ -44,6 +45,18 @@ jobs: restore-keys: | ${{ runner.os }}-conan- + # Keep the legacy cache separate from the normal Linux cache: Conan may cache + # build tools such as Boost's b2, and tools built on ubuntu-latest can require + # newer glibc symbols than the manylinux2014/CentOS 7 container provides. + - name: Cache legacy Linux Conan packages + if: runner.os == 'Linux' + uses: actions/cache@v4 + with: + path: ${{ runner.temp }}/dccc-conan2-legacy + key: ${{ runner.os }}-legacy-glibc217-conan-${{ hashFiles('localizer/src/conanfile.py', 'docker/Dockerfile.core.legacy', 'scripts/docker-build-core.sh', '.github/workflows/cross-compile.yml') }} + restore-keys: | + ${{ runner.os }}-legacy-glibc217-conan- + - name: Setup Ninja uses: seanmiddleditch/gha-setup-ninja@v4 @@ -55,8 +68,8 @@ jobs: echo "CC=gcc-12" >> $GITHUB_ENV echo "CXX=g++-12" >> $GITHUB_ENV - - name: Conan profile detect (Linux/macOS) - if: runner.os != 'Windows' + - name: Conan profile detect (macOS) + if: runner.os == 'macOS' working-directory: localizer/src shell: bash run: | @@ -69,8 +82,8 @@ jobs: run: | conan profile detect --force - - name: Conan install (Linux/macOS) - if: runner.os != 'Windows' + - name: Conan install (macOS) + if: runner.os == 'macOS' working-directory: localizer/src shell: bash run: | @@ -81,8 +94,8 @@ jobs: -o onetbb/*:tbbmalloc=False \ -o onetbb/*:tbbproxy=False - - name: Configure with CMake (Linux/macOS) - if: runner.os != 'Windows' + - name: Configure with CMake (macOS) + if: runner.os == 'macOS' working-directory: localizer/src shell: bash run: | @@ -93,18 +106,61 @@ jobs: -DCMAKE_BUILD_TYPE=Release \ -DCMAKE_INSTALL_PREFIX="./install" - - name: Build with CMake (Linux/macOS) - if: runner.os != 'Windows' + - name: Build with CMake (macOS) + if: runner.os == 'macOS' working-directory: localizer/src shell: bash run: cmake --build build --config Release - - name: Install with CMake (Linux/macOS) - if: runner.os != 'Windows' + - name: Install with CMake (macOS) + if: runner.os == 'macOS' working-directory: localizer/src shell: bash run: cmake --install build --config Release + - name: Build legacy Linux Docker image + if: runner.os == 'Linux' + shell: bash + run: | + docker build \ + --build-arg USER_UID="$(id -u)" \ + --build-arg USER_GID="$(id -g)" \ + -f docker/Dockerfile.core.legacy \ + -t dccc-core-dev:legacy-glibc217 . + + - name: Build and install with legacy Linux container + if: runner.os == 'Linux' + shell: bash + run: | + legacy_conan_home="$RUNNER_TEMP/dccc-conan2-legacy" + mkdir -p "$legacy_conan_home" + docker run --rm \ + -v "$GITHUB_WORKSPACE:/workspace" \ + -v "$legacy_conan_home:/home/dev/.conan2" \ + -e CONAN_CPPSTD=gnu17 \ + -e CONAN_BUILD_ARGS='--build=missing --build=b2/* --build=m4/* --build=autoconf/* --build=automake/* --build=libtool/* --build=pkgconf/*' \ + -e BUILD_TYPE=Release \ + -e INSTALL_PREFIX=./install \ + dccc-core-dev:legacy-glibc217 \ + /workspace/scripts/docker-build-core.sh + + - name: Verify Linux glibc compatibility floor + if: runner.os == 'Linux' + working-directory: localizer/src + shell: bash + run: | + set -euo pipefail + mapfile -t elf_files < <(find install/bin -maxdepth 1 -type f -exec sh -c 'file "$1" | grep -q ELF' sh {} \; -print) + max_glibc=$(objdump -T "${elf_files[@]}" \ + | sed -n 's/.*(GLIBC_\([0-9][0-9.]*\)).*/\1/p' \ + | sort -V \ + | tail -n 1) + echo "Maximum referenced GLIBC version: ${max_glibc:-none}" + if [[ -n "${max_glibc}" ]] && [[ "$(printf '%s\n%s\n' "2.17" "${max_glibc}" | sort -V | tail -n 1)" != "2.17" ]]; then + echo "Linux package references GLIBC_${max_glibc}; expected no symbols newer than GLIBC_2.17." >&2 + exit 1 + fi + - name: Run PR smoke tests on installed binary (Linux) if: github.event_name == 'pull_request' && runner.os == 'Linux' working-directory: localizer/src/tests diff --git a/docker-compose.core.yml b/docker-compose.core.yml index 0713f47..04e5116 100644 --- a/docker-compose.core.yml +++ b/docker-compose.core.yml @@ -15,6 +15,28 @@ services: tty: true stdin_open: true + + dccc-core-legacy: + build: + context: . + dockerfile: docker/Dockerfile.core.legacy + args: + USER_UID: "${UID:-1000}" + USER_GID: "${GID:-1000}" + image: dccc-core-dev:legacy-glibc217 + working_dir: /workspace/localizer/src + volumes: + - .:/workspace + - dccc-conan-cache-legacy:/home/dev/.conan2 + - dccc-cmake-build-legacy:/workspace/localizer/src/build + environment: + CONAN_CPPSTD: "gnu17" + CONAN_BUILD_ARGS: "--build=missing --build=b2/* --build=m4/* --build=autoconf/* --build=automake/* --build=libtool/* --build=pkgconf/*" + tty: true + stdin_open: true + volumes: dccc-conan-cache: dccc-cmake-build: + dccc-conan-cache-legacy: + dccc-cmake-build-legacy: diff --git a/docker/Dockerfile.core.legacy b/docker/Dockerfile.core.legacy new file mode 100644 index 0000000..3d1a2db --- /dev/null +++ b/docker/Dockerfile.core.legacy @@ -0,0 +1,55 @@ +FROM quay.io/pypa/manylinux2014_x86_64 + +ARG USERNAME=dev +ARG USER_UID=1000 +ARG USER_GID=1000 + +ENV TZ=Etc/UTC \ + PIP_DISABLE_PIP_VERSION_CHECK=1 \ + CONAN_HOME=/home/${USERNAME}/.conan2 \ + PATH=/opt/python/cp311-cp311/bin:/opt/rh/devtoolset-10/root/usr/bin:$PATH \ + LD_LIBRARY_PATH=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib \ + CC=gcc \ + CXX=g++ + +# Conan has to build several dependencies from source in the legacy glibc +# container. Install a small source-build baseline instead of discovering +# missing autotools/Perl pieces one CI failure at a time. OpenSSL's Configure +# uses Perl core modules such as IPC::Cmd and Time::Piece, and libcurl's +# autoreconf path needs GNU autotools utilities including m4. +RUN yum install -y \ + autoconf \ + automake \ + libtool \ + m4 \ + make \ + patch \ + perl-core \ + pkgconfig \ + sudo \ + which \ + zlib-devel \ + && yum clean all + +RUN perl -MIPC::Cmd -MTime::Piece -e 1 \ + && m4 --version \ + && autoreconf --version + +RUN python -m pip install --no-cache-dir --default-timeout=120 --retries=10 \ + "cmake>=3.27,<4" \ + "conan>=2.0,<3" \ + ninja \ + pandas \ + pytest + +RUN groupadd --gid ${USER_GID} ${USERNAME} \ + && useradd --uid ${USER_UID} --gid ${USER_GID} -m ${USERNAME} -s /bin/bash \ + && echo "${USERNAME} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/${USERNAME} \ + && chmod 0440 /etc/sudoers.d/${USERNAME} + +USER ${USERNAME} +WORKDIR /workspace/localizer/src + +RUN conan profile detect --force + +CMD ["/bin/bash"] diff --git a/localizer/src/README.md b/localizer/src/README.md index 20bff5f..6593a3a 100644 --- a/localizer/src/README.md +++ b/localizer/src/README.md @@ -161,7 +161,14 @@ Build and install the C++ core: docker compose -f docker-compose.core.yml run --rm dccc-core /workspace/scripts/docker-build-core.sh ``` -The installed executable will be written to `localizer/src/install/bin/DCCCcore`. Conan packages and the CMake build tree are kept in Docker volumes so later builds can reuse downloaded and compiled dependencies. The script defaults to `CONAN_CPPSTD=gnu17` on Linux to reuse more Conan Center binaries; use `CONAN_CPPSTD=17` if you need a strict non-GNU C++17 profile. +For Linux release builds that must run on older distributions without glibc 2.34, use the legacy compatibility container instead. It is based on the manylinux2014 / CentOS 7 toolchain so generated Linux binaries target glibc 2.17, which covers RHEL/CentOS 7-era images and avoids accidental `GLIBC_2.34` symbol requirements: + +```bash +docker compose -f docker-compose.core.yml build dccc-core-legacy +docker compose -f docker-compose.core.yml run --rm dccc-core-legacy /workspace/scripts/docker-build-core.sh +``` + +The installed executable will be written to `localizer/src/install/bin/DCCCcore`. Conan packages and the CMake build tree are kept in Docker volumes so later builds can reuse downloaded and compiled dependencies. The default development container and the legacy compatibility container use separate Conan and CMake volumes to avoid mixing dependency builds from different glibc baselines. The script defaults to `CONAN_CPPSTD=gnu17` on Linux to reuse more Conan Center binaries; use `CONAN_CPPSTD=17` if you need a strict non-GNU C++17 profile. The legacy container also sets `CONAN_BUILD_ARGS="--build=missing --build=b2/* --build=m4/* --build=autoconf/* --build=automake/* --build=libtool/* --build=pkgconf/*"` so native build tools such as Boost's `b2` and autotools packages used by libcurl are compiled inside the manylinux2014 environment instead of downloading ConanCenter binaries that may require newer glibc symbols such as `GLIBC_2.34`. The first run can still take a long time because Conan Center may not provide matching Linux binaries for heavy packages such as ITK, ONNX Runtime, Boost, HDF5, GDCM, or TBB. Let the first build finish once on a machine, then keep the Docker volumes for normal incremental development. diff --git a/scripts/docker-build-core.sh b/scripts/docker-build-core.sh index 6301095..d120a53 100755 --- a/scripts/docker-build-core.sh +++ b/scripts/docker-build-core.sh @@ -6,6 +6,12 @@ cd "$(dirname "${BASH_SOURCE[0]}")/../localizer/src" BUILD_TYPE="${BUILD_TYPE:-Release}" INSTALL_PREFIX="${INSTALL_PREFIX:-./install}" CONAN_CPPSTD="${CONAN_CPPSTD:-gnu17}" +CONAN_BUILD_ARGS="${CONAN_BUILD_ARGS:---build=missing}" +# Split the simple, space-delimited Conan build policy string into argv items. +# This lets the legacy Linux path force known build tools such as b2 to be +# rebuilt inside the manylinux2014 container instead of downloading binaries +# that may have been produced on newer glibc systems. +read -r -a conan_build_args <<< "${CONAN_BUILD_ARGS}" sudo mkdir -p build "${CONAN_HOME:-/home/dev/.conan2}" sudo chown -R "$(id -u):$(id -g)" build "${CONAN_HOME:-/home/dev/.conan2}" @@ -13,7 +19,7 @@ sudo chown -R "$(id -u):$(id -g)" build "${CONAN_HOME:-/home/dev/.conan2}" conan profile detect --force conan install . \ --output-folder=build \ - --build=missing \ + "${conan_build_args[@]}" \ -s build_type="${BUILD_TYPE}" \ -s compiler.cppstd="${CONAN_CPPSTD}" \ -c tools.cmake.cmaketoolchain:generator=Ninja \