diff --git a/.evergreen/build_all.sh b/.evergreen/build_all.sh index be0fce306..368b690da 100755 --- a/.evergreen/build_all.sh +++ b/.evergreen/build_all.sh @@ -15,6 +15,8 @@ set -eu : "${LIBMONGOCRYPT_COMPILE_FLAGS:=}" # Additional CMake flags that apply only to the libmongocrypt build. (Used by the C driver) : "${LIBMONGOCRYPT_EXTRA_CMAKE_FLAGS:=}" +# release_os_arch is set for release builds. +: "${release_os_arch:=}" # Control the build configuration that is generated. export CMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE:-RelWithDebInfo}" @@ -90,6 +92,27 @@ if [ "$CONFIGURE_ONLY" ]; then fi echo "Installing libmongocrypt" _cmake_with_env --build "$BINARY_DIR" --target install + +# If release_os_arch names a minimum glibc requirement (e.g. "linux-x86_64-glibc_2_17-nocrypto"), +# verify it matches the maximum glibc symbol used. +if [[ "$release_os_arch" == *glibc* ]]; then + expected_glibc=$(echo "$release_os_arch" | sed -r 's/.*glibc_([0-9]+)_([0-9]+).*/\1.\2/') + if [ -f "$CMAKE_INSTALL_PREFIX/lib64/libmongocrypt.so" ]; then + check_lib="$CMAKE_INSTALL_PREFIX/lib64/libmongocrypt.so" + elif [ -f "$CMAKE_INSTALL_PREFIX/lib/libmongocrypt.so" ]; then + check_lib="$CMAKE_INSTALL_PREFIX/lib/libmongocrypt.so" + else + echo "glibc version check failed: libmongocrypt.so not found under $CMAKE_INSTALL_PREFIX" + exit 1 + fi + actual_glibc=$(objdump -T "$check_lib" | grep 'GLIBC_' | sed -r -e 's/.*GLIBC_([0-9.]+).*/\1/' | sort -u | tail -1) + if [ "$actual_glibc" != "$expected_glibc" ]; then + echo "glibc version check failed: release_os_arch requires glibc $expected_glibc but library uses glibc $actual_glibc" + exit 1 + fi + echo "glibc version check passed: $actual_glibc" +fi + run_chdir "$BINARY_DIR" run_ctest # MONGOCRYPT-372, ensure macOS universal builds contain both x86_64 and arm64 architectures. diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 0f288eb26..5fa8770c0 100755 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -117,6 +117,7 @@ functions: export VS_VERSION=${vs_version|} export VS_TARGET_ARCH=${vs_target_arch|amd64} export CMAKE_GENERATOR=${CMAKE_GENERATOR|Ninja} + export release_os_arch=${release_os_arch} env ${compile_env|} \ bash "$EVG_DIR/env-run.sh" \ bash "$EVG_DIR/build_all.sh" @@ -628,7 +629,7 @@ tasks: commands: - func: "download and merge python releases" -- name: upload-java +- name: upload-java # TODO(MONGOCRYPT-894) remove in favor of `upload-release` tasks. depends_on: - variant: rhel-62-64-bit name: build-and-test-and-upload @@ -708,7 +709,7 @@ tasks: filenames: - libmongocrypt-java-${tag_upload_location!|*revision}.tar.gz -- name: upload-all +- name: upload-all # TODO(MONGOCRYPT-894) remove in favor of `upload-release` tasks. depends_on: - variant: ubuntu1604 name: build-and-test-and-upload @@ -947,7 +948,7 @@ tasks: filenames: - libmongocrypt-all-${tag_upload_location!|*revision}.tar.gz -- name: sign-all +- name: sign-all # TODO(MONGOCRYPT-894) remove in favor of `upload-release` tasks. patchable: false # Garasign credentials are marked as "Admin only" in Evergreen project. "Admin only" variables are not included in patch builds. To test a patch: temporarily unselect "Admin only". depends_on: upload-all commands: @@ -1044,7 +1045,7 @@ tasks: - func: "fetch source" - func: "setup packaging credentials" - func: "publish packages" -- name: windows-upload-release +- name: windows-upload-release # TODO(MONGOCRYPT-894) remove in favor of `upload-release` tasks. patchable: false # Garasign credentials are marked as "Admin only" in Evergreen and are not included in patch builds. depends_on: - variant: windows-test @@ -1162,6 +1163,98 @@ tasks: filenames: - libmongocrypt-windows-x86_64-${tag_upload_location!|*revision}.tar.gz +- name: upload-release + run_on: ubuntu2404-latest-small + patchable: false # Garasign credentials are marked as "Admin only" in Evergreen and are not included in patch builds. To test a patch: temporarily unselect "Admin only". + commands: + - func: "fetch source" + - command: s3.get # Download build. + params: + role_arn: ${upload_arn} + remote_file: 'libmongocrypt/${build_variant}/${branch_name}/${libmongocrypt_s3_suffix}/libmongocrypt.tar.gz' + bucket: ${upload_bucket} + extract_to: libmongocrypt_download + - command: shell.exec + params: + shell: bash + script: |- + set -o xtrace + set -o errexit + + # Move just the mongocrypt files needed into the final upload + mkdir libmongocrypt_upload + + if [[ "${release_os_arch}" == *"nocrypto"* ]]; then + # Publish libmongocrypt library without crypto dependency. + srcdir="libmongocrypt_download/nocrypto" + else + srcdir="libmongocrypt_download" + fi + + # Move headers + mkdir libmongocrypt_upload/include + mv $srcdir/include/mongocrypt libmongocrypt_upload/include + + # Move library + if [ -f "$srcdir/lib64/libmongocrypt.so" ]; then + mkdir libmongocrypt_upload/lib64 + mv $srcdir/lib64/libmongocrypt.so libmongocrypt_upload/lib64 + elif [ -f "$srcdir/lib/libmongocrypt.so" ]; then + mkdir libmongocrypt_upload/lib + mv $srcdir/lib/libmongocrypt.so libmongocrypt_upload/lib + elif [ -f "$srcdir/lib/libmongocrypt.dylib" ]; then + mkdir libmongocrypt_upload/lib + mv $srcdir/lib/libmongocrypt.dylib libmongocrypt_upload/lib + elif [ -f "$srcdir/bin/libmongocrypt.dll" ]; then + mkdir libmongocrypt_upload/bin + mv $srcdir/bin/libmongocrypt.dll libmongocrypt_upload/bin + fi + - command: archive.targz_pack + params: + target: libmongocrypt-${release_os_arch}-${tag_upload_location!|*revision}.tar.gz + source_dir: libmongocrypt_upload + include: [./**] + - command: s3.put # Upload tarball for GitHub Release. + params: + role_arn: ${upload_arn} + skip_existing: true + remote_file: 'libmongocrypt/${build_variant}/${branch_name}/${revision}/${version_id}/libmongocrypt-${release_os_arch}-${tag_upload_location!|*revision}.tar.gz' + display_name: libmongocrypt-${release_os_arch}-${tag_upload_location!|*revision}.tar.gz + bucket: ${upload_bucket} + permissions: ${upload_permissions} + visibility: ${upload_visibility} + local_file: 'libmongocrypt-${release_os_arch}-${tag_upload_location!|*revision}.tar.gz' + content_type: 'application/x-gzip' + - command: shell.exec + params: + shell: bash + script: |- + set -o errexit + # Copy file to sign into `libmongocrypt` directory to be used by Earthly. + cp libmongocrypt-${release_os_arch}-${tag_upload_location!|*revision}.tar.gz libmongocrypt + - func: "earthly" # Sign tarball. + vars: + args: --secret garasign_username=${garasign_username} --secret garasign_password=${garasign_password} +sign --file_to_sign=libmongocrypt-${release_os_arch}-${tag_upload_location!|*revision}.tar.gz --output_file=libmongocrypt_upload.asc + - command: s3.put # Upload signature for GitHub Release. + params: + role_arn: ${upload_arn} + skip_existing: true + remote_file: 'libmongocrypt/${build_variant}/${branch_name}/${revision}/${version_id}/libmongocrypt-${release_os_arch}-${tag_upload_location!|*revision}.asc' + display_name: libmongocrypt-${release_os_arch}-${tag_upload_location!|*revision}.asc + bucket: ${upload_bucket} + permissions: ${upload_permissions} + visibility: ${upload_visibility} + local_file: 'libmongocrypt/libmongocrypt_upload.asc' + content_type: 'application/pgp-signature' + - command: papertrail.trace + params: + key_id: ${papertrail_key_id} + secret_key: ${papertrail_secret_key} + product: ${papertrail_project} # libmongocrypt-dev or libmongocrypt-release + version: ${tag_upload_location!|*revision} # Use ${tag_upload_location} if non-empty, otherwise ${revision}. + filenames: + - libmongocrypt-${release_os_arch}-${tag_upload_location!|*revision}.tar.gz + - name: debian-package-build run_on: &deb-package-build-run_on - ubuntu2004 @@ -1419,6 +1512,7 @@ buildvariants: expansions: packager_distro: rhel72 packager_arch: s390x + release_os_arch: linux-s390x-glibc_2_7-nocrypto tasks: - build-and-test-and-upload - build-and-test-shared-bson @@ -1426,6 +1520,8 @@ buildvariants: - name: publish-packages distros: - rhel70-small + - name: upload-release + depends_on: build-and-test-and-upload - name: rhel83-zseries # rhel83-zseries has a new enough g++ to build the C++ tests. # rhel72-zseries-test does not build the C++ tests. @@ -1439,9 +1535,12 @@ buildvariants: expansions: vs_version: "15" vs_target_arch: amd64 + release_os_arch: windows-x86_64 tasks: - build-and-test-and-upload - build-and-test-shared-bson + - name: upload-release + depends_on: build-and-test-and-upload - name: windows-test-python display_name: "Windows Python" run_on: windows-64-vsMulti-small @@ -1600,6 +1699,7 @@ buildvariants: expansions: packager_distro: rhel70 packager_arch: x86_64 + release_os_arch: linux-x86_64-glibc_2_7-nocrypto tasks: - build-and-test-and-upload - build-and-test-shared-bson @@ -1607,12 +1707,15 @@ buildvariants: - name: publish-packages distros: - rhel70-small + - name: upload-release + depends_on: build-and-test-and-upload - name: rhel-71-ppc64el display_name: "RHEL 7.1 ppc64el" run_on: rhel71-power8-test expansions: packager_distro: rhel71 packager_arch: ppc64le + release_os_arch: linux-ppc64le-glibc_2_17-nocrypto tasks: - build-and-test-and-upload - build-and-test-shared-bson @@ -1620,6 +1723,8 @@ buildvariants: - name: publish-packages distros: - rhel70-small + - name: upload-release + depends_on: build-and-test-and-upload - name: rhel-80-64-bit display_name: "RHEL 8.0 64-bit" run_on: rhel80-test @@ -1641,6 +1746,7 @@ buildvariants: expansions: packager_distro: rhel82 packager_arch: aarch64 + release_os_arch: linux-arm64-glibc_2_17-nocrypto tasks: - build-and-test-and-upload - test-python @@ -1649,6 +1755,8 @@ buildvariants: - name: publish-packages distros: - rhel70-small + - name: upload-release + depends_on: build-and-test-and-upload - name: rhel-81-ppc64el display_name: "RHEL 8.1 ppc64el" run_on: rhel81-power8-small @@ -1886,10 +1994,13 @@ buildvariants: CMAKE=/opt/homebrew/bin/cmake # Disable Ninja to work around error "Bad CPU type in executable" CMAKE_GENERATOR: '' + release_os_arch: macos-universal tasks: - build-and-test-and-upload - name: test-python distros: macos-14-arm64 + - name: upload-release + depends_on: build-and-test-and-upload - name: windows-vs2017-32bit # Test Windows 32 bit builds for PHPC. PHPC builds libmongocrypt from source. See MONGOCRYPT-391. display_name: "Windows VS 2017 32-bit compile" @@ -1912,9 +2023,12 @@ buildvariants: display_name: "Alpine Linux 3.18 amd64 (via Earthly)" expansions: earthly_env: alpine + release_os_arch: linux-x86_64-musl_1_2-nocrypto tasks: - name: build-with-earthly run_on: ubuntu2204-small + - name: upload-release + depends_on: build-with-earthly - name: alpine-arm64-earthly display_name: "Alpine Linux 3.18 arm64 (via Earthly)" diff --git a/CHANGELOG.md b/CHANGELOG.md index 47c984a08..4f0c63980 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,11 +2,19 @@ ## Unreleased +### Added +- Signed binaries for macOS and Linux are now available on the GitHub release. + - Linux binaries including `nocrypto` in the name have no dependency on OpenSSL. Drivers using the `nocrypto` variant are expected to set crypto callbacks (e.g. call `mongocrypt_setopt_crypto_hooks`) to do operations requiring crypto to avoid an error. + - Drivers that package libmongocrypt binaries are encouraged to migrate release scripts to use these binaries. + - No reduction in platform support is expected. glibc dependencies were checked against existing builds on RHEL 6.2 and Ubuntu 16.04. ### Changed - Final release packages in the PPA are now available by specifying `release` in the repository configuration in place of the major/minor version (e.g., `1.17`). Details in `README.md`. +### Deprecated +- RHEL 6.2 builds are deprecated and may be removed in the future. The `linux-x86_64-glibc_2_7-nocrypto` release build may be used instead and has equivalent glibc requirements. + ### Removed - The configure-time CMake parameter `ENABLE_WINDOWS_STATIC_RUNTIME` has been diff --git a/doc/releasing.md b/doc/releasing.md index 143d76906..217d5e494 100644 --- a/doc/releasing.md +++ b/doc/releasing.md @@ -80,7 +80,12 @@ Do the following when releasing: - All `publish-packages` tasks. - If the `publish-packages` tasks fail with an error like `[curator] 2024/01/02 13:56:17 [p=emergency]: problem submitting repobuilder job: 404 (Not Found)`, this suggests the published path does not yet exist. Barque (the Linux package publishing service) has protection to avoid unintentional publishes. File a DEVPROD ticket ([example](https://jira.mongodb.org/browse/DEVPROD-15320)) and assign to the team called Release Infrastructure to request the path be created. Then re-run the failing `publish-packages` task. Ask in the slack channel `#ask-devprod-release-tools` for further help with `Barque` or `curator`. - Create the release from the GitHub releases page from the new tag. - - Attach the tarball and signature file from the Files tab of the `windows-upload-release` task. [Example](https://github.com/mongodb/libmongocrypt/releases/tag/1.13.0). + - Attach the tarball and signature files from the `upload-release` tasks. Use `etc/download-artifacts.py` to download all such files. Obtain the version ID from the Evergreen URL: + ```bash + # Example: the Evergreen URL: https://spruce.corp.mongodb.com/version/69cfada41e1f8400073c971e has VERSION_ID=69cfada41e1f8400073c971e + # Downloads to _build/artifacts + uv run etc/download-artifacts.py ${VERSION_ID:?} + ``` - Attach the Augmented SBOM file to the release as `cyclonedx.augmented.sbom.json`. Download the Augmented SBOM from a recent execution of the `sbom` task in an Evergreen patch or commit build. - Attach `etc/third_party_vulnerabilities.md` to the release. diff --git a/etc/download-artifacts.py b/etc/download-artifacts.py new file mode 100644 index 000000000..df9ab67da --- /dev/null +++ b/etc/download-artifacts.py @@ -0,0 +1,66 @@ +# Download libmongocrypt artifacts to upload to a GitHub release. +# +# /// script +# dependencies = [ +# "pyyaml", +# "requests", +# ] +# /// + +import argparse +import requests +import yaml +from pathlib import Path +from urllib.parse import urlsplit + + +def download_file(url: str): + url_parts = urlsplit(url) + path = Path(url_parts.path) + filename = path.name + destination_dir = Path("_build/artifacts") + destination_dir.mkdir(exist_ok=True, parents=True) + destination = destination_dir / filename + print(f"Downloading {url} to {destination}") + with requests.get(url, stream=True) as resp: + with destination.open("wb") as file: + for chunk in resp.iter_content(chunk_size=8192): + file.write(chunk) + + +def main(): + parser = argparse.ArgumentParser(description="Download release artifacts from an Evergreen version") + parser.add_argument("version_id", help="Evergreen version ID. (e.g. https://evergreen.mongodb.com/version/)") + args = parser.parse_args() + + version_id = args.version_id + + # Get Evergreen API credentials: + path: Path = Path().home() / ".evergreen.yml" + with path.open() as file: + evg_settings = yaml.load(file, Loader=yaml.FullLoader) + + api_server_host = "https://evergreen.mongodb.com/rest/v2" + api_key = evg_settings["api_key"] + api_user = evg_settings["user"] + headers = {'Api-User': api_user, 'Api-Key': api_key} + + resp: requests.Response = requests.get( + f"{api_server_host}/versions/{version_id}/builds", + params={"include_task_info": "true"}, + headers=headers) + builds = resp.json() + + for build in builds: + for task_info in build["task_cache"]: + if task_info["display_name"] == "upload-release": + task_id = task_info["id"] + resp = requests.get( + f"{api_server_host}/tasks/{task_id}", headers=headers) + task = resp.json() + for artifact in task["artifacts"]: + download_file(artifact["url"]) + + +if __name__ == "__main__": + main()