diff --git a/.github/actions/get-artifact-for-stage-tests/action.yml b/.github/actions/get-artifact-for-stage-tests/action.yml index 093612ae66..c638560fa8 100644 --- a/.github/actions/get-artifact-for-stage-tests/action.yml +++ b/.github/actions/get-artifact-for-stage-tests/action.yml @@ -78,5 +78,11 @@ runs: - name: Download artifact from JFrog if: ${{ inputs.get_from_jfrog == 'true' }} - run: jf rt dl --fail-no-op --flat --build python-client/${{ inputs.jfrog_build_version }} "${{ inputs.JFROG_REPO_NAME }}/**/${{ env.ARTIFACT_FILE_NAME_PATTERN }}" + run: | + bundle_version=$(jf rbs --format json --order-by created --project database versions python-client-release | jq -r '.release_bundles[0].release_bundle_version') + # Currently no way to download specific artifacts from a release bundle + # Using ARTIFACT_FILE_NAME_PATTERN with jf rt dl --bundle doesn't work + build_number="${bundle_version##*-}" + artifact_path=$(jf rt s --project database --fail-no-op --build python-client/$build_number "database-pypi-dev-local/**/$ARTIFACT_FILE_NAME_PATTERN" | jq -r '.[0].path') + jf rt dl --project database --fail-no-op --flat $artifact_path shell: bash diff --git a/.github/workflows/build-artifacts.yml b/.github/workflows/build-artifacts.yml index 3bddc6cecc..bcf3a0b730 100644 --- a/.github/workflows/build-artifacts.yml +++ b/.github/workflows/build-artifacts.yml @@ -89,6 +89,9 @@ on: required: false type: string default: '' + outputs: + build-version: + value: ${{ jobs.build-sdist.outputs.sdist-version }} secrets: DOCKER_HUB_BOT_USERNAME: required: true diff --git a/.github/workflows/build-python-client.bash b/.github/workflows/build-python-client.bash new file mode 100644 index 0000000000..8aed497e87 --- /dev/null +++ b/.github/workflows/build-python-client.bash @@ -0,0 +1,57 @@ +# On macOS and Windows, this is expected to run on bare metal +# On Linux, this is expected to run in a container from the manylinux image +os=$(uname -s) +running_on_windows=$([[ "$os" =~ CYGWIN* || "$os" =~ MINGW* ]]) +if [[ $running_on_windows ]]; then + nuget restore +fi + +if [[ $os =~ Darwin* ]]; then + brew install openssl@3 libyaml + + export MACOSX_DEPLOYMENT_TARGET + MACOSX_DEPLOYMENT_TARGET="$(sw_vers -productVersion | cut -d"." -f 1).0" + + if [[ $(uname -m) == "arm64" ]]; then + libraries=('libyaml' 'openssl') + for library in "${libraries[@]}"; do + LIBRARY_PATH="${LIBRARY_PATH}:$(brew --prefix "$library")/lib" + done + export LIBRARY_PATH="$LIBRARY_PATH" + else + export LDFLAGS='-headerpad_max_install_names' + fi +fi + +if [[ $os =~ Linux* ]]; then + yum install libyaml-devel -y +fi + +python3 -m pip install build -c requirements.txt +python3 -m build + +if [[ $os =~ Linux* ]]; then + unrepaired_wheel_path=$(find dist/ -type f -name '*.whl' | head -n 1) + if [[ "$DEBUG" != "" ]]; then + auditwheel show "$unrepaired_wheel_path" + WHEEL_DIR=wheel-contents + unzip "$unrepaired_wheel_path" -d $WHEEL_DIR + ldd $WHEEL_DIR/*.so | awk '{print $1}' | grep libssl.so.3 + ldd $WHEEL_DIR/*.so | awk '{print $1}' | grep libcrypto.so.3 + fi + + auditwheel repair -w wheelhouse/ "$unrepaired_wheel_path" + + if [[ "$DEBUG" != "" ]]; then + auditwheel show wheelhouse/* + fi + + # TODO: This should happen if any steps after creating the dir fail + rm -rf "$WHEEL_DIR" +elif [[ $os =~ Darwin* ]]; then + delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel} + delocate-listdeps {dest_dir}/*.whl | grep libcrypto.3.dylib + delocate-listdeps {dest_dir}/*.whl | grep libssl.3.dylib +elif [[ $running_on_windows ]]; then + delvewheel repair -vv --add-path ./aerospike-client-c/vs/x64/Release -w wheelhouse "$unrepaired_wheel_path" +fi diff --git a/.github/workflows/build-sdist.yml b/.github/workflows/build-sdist.yml index 85b9f074f5..308a66d805 100644 --- a/.github/workflows/build-sdist.yml +++ b/.github/workflows/build-sdist.yml @@ -9,6 +9,9 @@ on: sha_to_build: type: string required: true + outputs: + sdist-version: + value: ${{ jobs.build-sdist.outputs.sdist-version }} env: STATUS_CHECK_MESSAGE: "Build source distribution" @@ -16,6 +19,8 @@ env: jobs: build-sdist: + outputs: + sdist-version: ${{ steps.get-sdist-version.outputs.version }} name: Build source distribution runs-on: ubuntu-22.04 steps: @@ -48,6 +53,9 @@ jobs: - name: Build source distribution run: python3 -m build --sdist + - id: get-sdist-version + run: echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT + - name: Upload source distribution to GitHub uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml index 7b699850f5..eaf4a77805 100644 --- a/.github/workflows/build-wheels.yml +++ b/.github/workflows/build-wheels.yml @@ -263,6 +263,7 @@ jobs: - if: ${{ startsWith(inputs.platform-tag, 'manylinux') }} run: echo CIBW_MANYLINUX_AARCH64_IMAGE=${{ format(env.CUSTOM_IMAGE_NAME, 'aarch64', 'dadc371c3647efe5a1f3a7c99edddea1d8db542e3d2ad48bf63fa81364fb63b6') }} >> $GITHUB_ENV + # TODO: would need way to pull this outside of shared workflow - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0 if: ${{ startsWith(inputs.platform-tag, 'manylinux') }} with: diff --git a/.github/workflows/bump-stage-and-upload-to-jfrog.yml b/.github/workflows/bump-stage-and-upload-to-jfrog.yml index 922c94de93..f2a0cb1cbb 100644 --- a/.github/workflows/bump-stage-and-upload-to-jfrog.yml +++ b/.github/workflows/bump-stage-and-upload-to-jfrog.yml @@ -24,44 +24,12 @@ jobs: base_branch: ${{ vars.STAGE_BRANCH_NAME }} secrets: inherit - promote-dev-build-to-rc: - name: Bump (promote) the dev version to an RC version in the stage branch - needs: ff-stage-to-dev-tag - uses: ./.github/workflows/bump-version.yml - with: - change: 'promote-dev-build-to-rc' - ref: ${{ vars.STAGE_BRANCH_NAME }} - secrets: inherit - - rebuild-artifacts-with-rc-version: - needs: promote-dev-build-to-rc - uses: ./.github/workflows/build-artifacts.yml - with: - sha-to-build-and-test: ${{ needs.promote-dev-build-to-rc.outputs.bump_sha }} - secrets: inherit - - upload-rc-artifacts-to-jfrog: + promote-release-bundle-to-dev: needs: [ - rebuild-artifacts-with-rc-version, - # We need the new RC version to label the build in JFrog - promote-dev-build-to-rc + ff-stage-to-dev-tag, ] - name: Upload artifacts to JFrog - uses: ./.github/workflows/upload-to-jfrog.yml + uses: ./.github/workflows/promote-release-bundle.yml with: - version: ${{ needs.promote-dev-build-to-rc.outputs.new_version }} - secrets: inherit - - # See reason for deleting artifacts in dev-workflow-p2.yml - delete-artifacts: - needs: upload-rc-artifacts-to-jfrog - uses: ./.github/workflows/delete-artifacts.yml - - ff-dev-to-stage: - name: Fast forward dev branch to stage branch to include the bump to RC commit - needs: promote-dev-build-to-rc - uses: ./.github/workflows/fast-forward-merge.yml - with: - ref_to_merge: origin/${{ vars.STAGE_BRANCH_NAME }} - base_branch: ${{ vars.DEV_BRANCH_NAME }} - secrets: inherit + release-bundle-name: python-client-release + release-bundle-version: ${{ inputs.passed-dev-tag }} + new-environment: TEST diff --git a/.github/workflows/bump-version.yml b/.github/workflows/bump-version.yml index 7cd593e42e..618d5e2595 100644 --- a/.github/workflows/bump-version.yml +++ b/.github/workflows/bump-version.yml @@ -13,8 +13,18 @@ on: - bump-dev-num - promote-dev-build-to-rc - promote-rc-build-to-release + dry-run: + required: false + default: false + type: boolean + description: No tagging in repo workflow_call: inputs: + dry-run: + required: false + default: false + type: boolean + description: Dry run change: # Since workflow_call doesn't support 'options' input type, # we take in a string instead that must be a valid Python script name (excluding the .py part) @@ -37,8 +47,6 @@ on: outputs: new_version: value: ${{ jobs.get-new-version.outputs.new_version }} - bump_sha: - value: ${{ jobs.update-version-in-repo.outputs.bump_sha }} jobs: get-current-version: @@ -57,10 +65,14 @@ jobs: with: token: ${{ secrets.CLIENT_BOT_PAT }} ref: ${{ inputs.ref }} + # Get all tags + fetch-depth: 0 - - name: Get current version + - name: Get tag from previous commit id: get-current-version - run: echo current_version=$(cat VERSION) >> $GITHUB_OUTPUT + run: | + prev_commit=$(git rev-parse HEAD~) + echo current_version=$(git describe --tags --exact-match $prev_commit) >> $GITHUB_OUTPUT get-new-version: runs-on: ubuntu-22.04 @@ -93,4 +105,5 @@ jobs: with: new_version: ${{ needs.get-new-version.outputs.new_version }} ref: ${{ inputs.is_workflow_call && inputs.ref || github.ref }} + dry-run: ${{ inputs.dry-run }} secrets: inherit diff --git a/.github/workflows/dev-workflow-part2.yml b/.github/workflows/dev-workflow-part2.yml index 955d8f73bc..4651aa6ae9 100644 --- a/.github/workflows/dev-workflow-part2.yml +++ b/.github/workflows/dev-workflow-part2.yml @@ -6,33 +6,42 @@ on: branches: - 'dev*' workflow_dispatch: + inputs: + dry-run: + description: Don't create tag in repo + type: boolean + required: false + default: true jobs: - bump-dev-number: + tag-new-commit-with-dev-number: uses: ./.github/workflows/bump-version.yml with: change: 'bump-dev-num' + dry-run: ${{ inputs.dry-run }} secrets: inherit rebuild-artifacts-with-new-dev-num: - needs: bump-dev-number + needs: tag-new-commit-with-dev-number name: Rebuild artifacts with new dev number uses: ./.github/workflows/build-artifacts.yml with: # On pull_request_target, the bump version commit will be ignored # So we must pass it manually to the workflow - sha-to-build-and-test: ${{ needs.bump-dev-number.outputs.bump_sha }} + sha-to-build-and-test: ${{ github.sha }} + # override-version: ${{ needs.tag-new-commit-with-dev-number.outputs.new_version }} secrets: inherit upload-to-jfrog: name: Upload artifacts to JFrog needs: [ - bump-dev-number, + tag-new-commit-with-dev-number, rebuild-artifacts-with-new-dev-num ] uses: ./.github/workflows/upload-to-jfrog.yml with: version: ${{ needs.bump-dev-number.outputs.new_version }} + dry-run: ${{ inputs.dry-run }} secrets: inherit # We don't want the artifacts in JFrog to also exist in Github diff --git a/.github/workflows/promote-release-bundle.yml b/.github/workflows/promote-release-bundle.yml new file mode 100644 index 0000000000..ecb8989f7a --- /dev/null +++ b/.github/workflows/promote-release-bundle.yml @@ -0,0 +1,34 @@ +on: + workflow_call: + inputs: + release-bundle-name: + type: string + required: true + release-bundle-version: + type: string + required: true + new-environment: + type: string + required: true + jfrog-project: + type: string + required: true + jfrog-repo-name: + type: string + required: true + +jobs: + promote-release-bundle-to-dev: + runs-on: ubuntu-24.04 + env: + JF_PROJECT: ${{ inputs.jfrog-project }} + steps: + - name: Setup JFrog CLI + uses: jfrog/setup-jfrog-cli@5b06f730cc5a6f55d78b30753f8583454b08c0aa # v4.8.1 + env: + JF_URL: https://artifact.aerospike.io + with: + oidc-provider-name: ${{ vars.OIDC_PROVIDER_NAME }} + oidc-audience: ${{ vars.OIDC_AUDIENCE }} + + - run: jf rbp --signing-key=aerospike --include-repos ${{ inputs.jfrog-repo-name }} ${{ inputs.release-bundle-name }} ${{ inputs.release-bundle-version }} ${{ inputs.new-environment }} diff --git a/.github/workflows/run-dev-tests-and-promote-to-dev.yml b/.github/workflows/run-dev-tests-and-promote-to-dev.yml new file mode 100644 index 0000000000..757acaa13c --- /dev/null +++ b/.github/workflows/run-dev-tests-and-promote-to-dev.yml @@ -0,0 +1,64 @@ +name: Run dev tests and promote release bundle to DEV +run-name: Run dev tests and promote release bundle to DEV (dry-run=${{ inputs.dry-run }}, test-file=${{ inputs.test-file }}) + +permissions: + # We are building the artifacts in build-artifacts.yml + contents: read + # For shared reusable workflows + # TODO: look into more + id-token: write + statuses: write + # For downloading docker images from ghcr.io + packages: read + +on: + workflow_dispatch: + inputs: + # These inputs are in case we need to test against a server RC + registry-name: + type: string + required: true + description: Registry name + default: 'docker.io' + image-name: + type: string + required: true + description: Image name + default: 'aerospike/aerospike-server-enterprise' + server-tag: + type: string + required: true + default: 'latest' + description: 'Server docker image tag (e.g to test a client backport version)' + # For testing purposes + dry-run: + required: false + default: false + type: boolean + description: Don't create release bundle + run_tests: + required: false + default: false + type: boolean + description: Run dev tests + test-file: + required: false + type: string + description: For speeding up debugging + +jobs: + dev-workflow: + uses: aerospike/shared-workflows/.github/workflows/reusable_artifacts-cicd.yaml@8f53b63b33d86e63aeb37d951283dd974574885b + with: + gh-workflows-ref: 8f53b63b33d86e63aeb37d951283dd974574885b + jf-project: ${{ vars.JFROG_PROJECT_FOR_CLIENT_TEAM }} + jf-build-name: ${{ vars.JFROG_BUILD_NAME }} + # TODO: ${{ needs.run-dev-tests.outputs.build-version }}- needs to be applied to version + version: ${{ github.run_number }} + gh-artifact-directory: dist + build-script: | + make build + # Optional: + build-type: release # Freeform label, applied as build.type target-prop + internal: false # Set true to mark artifacts as internal-only + secrets: inherit diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml index 0157319624..e59a2f9963 100644 --- a/.github/workflows/update-version.yml +++ b/.github/workflows/update-version.yml @@ -9,6 +9,11 @@ on: type: string description: Version string to set in the repo required: true + dry-run: + required: false + default: false + type: boolean + description: No tagging in repo workflow_call: inputs: new_version: @@ -19,6 +24,11 @@ on: type: string description: commit/branch to change version required: false + dry-run: + required: false + default: false + type: boolean + description: No tagging in repo # A hack to tell if workflow is triggered by workflow_call or not # Calling workflows should not set this input # If workflow is triggered by workflow_dispatch, this should be set to the default boolean value: false @@ -51,19 +61,19 @@ jobs: token: ${{ secrets.CLIENT_BOT_PAT }} ref: ${{ inputs.ref }} - - name: Update version in repo - uses: ./.github/actions/update-version - with: - new_version: ${{ inputs.new_version }} + # - name: Update version in repo + # uses: ./.github/actions/update-version + # with: + # new_version: ${{ inputs.new_version }} - - name: Commit new version + - name: Tag new version uses: step-security/git-auto-commit-action@905c3cd6e9ed2b67b4d46ff401fdb6d745d0ff9d # v7.1.0 with: - commit_message: 'Auto-bump version to ${{ inputs.new_version }} [skip ci]' - commit_author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - tagging_message: ${{ inputs.new_version }} - branch: ${{ inputs.is_workflow_call && inputs.ref || github.ref }} + # commit_message: 'Auto-bump version to ${{ inputs.new_version }} [skip ci]' + # commit_author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> + tagging_message: ${{ inputs.dry-run == false && inputs.new_version || '' }} + # branch: ${{ inputs.new_version }} - - name: Output bump commit hash for next jobs to use - id: get-bump-commit-sha - run: echo "bump_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + # - name: Output bump commit hash for next jobs to use + # id: get-bump-commit-sha + # run: echo "bump_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 7b361dbe0c..19628ba4a6 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -8,6 +8,34 @@ on: description: 'new_test/' required: false default: "" + type: string + registry-name: + type: string + required: false + description: Registry name + default: 'docker.io' + image-name: + type: string + required: false + description: Image name + default: 'aerospike/aerospike-server-enterprise' + server-tag: + required: false + description: Server tag + default: latest + type: string + massif: + type: boolean + description: 'Use massif for testing memory usage' + required: false + default: false + workflow_call: + inputs: + test-file: + description: 'new_test/' + required: false + default: "" + type: string registry-name: type: string required: false @@ -22,17 +50,25 @@ on: required: false description: Server tag default: latest + type: string massif: type: boolean description: 'Use massif for testing memory usage' required: false default: false + # Workflow call specific inputs + # Will be false for workflow_dispatch trigger + download-wheel-from-github-artifacts: + type: boolean + required: false + default: true env: PYTHON_TAG: cp310 jobs: look-for-wheel-in-jfrog: + if: ${{ inputs.download-wheel-from-github-artifacts == false }} outputs: num_artifacts_found: ${{ steps.count_num_artifacts_found.outputs.num_artifacts }} # So we can pass the python tag to a reusable workflow @@ -101,7 +137,7 @@ jobs: build-manylinux-wheel: needs: look-for-wheel-in-jfrog - if: ${{ needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} + if: ${{ !cancelled() && needs.look-for-wheel-in-jfrog.result == 'success' && needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 }} uses: ./.github/workflows/build-wheels.yml with: python-tags: '["${{ needs.look-for-wheel-in-jfrog.outputs.python-tag }}"]' @@ -124,12 +160,12 @@ jobs: env: MASSIF_REPORT_FILE_NAME: massif.out needs: [ - look-for-wheel-in-jfrog, - build-manylinux-wheel + upload-built-wheel-to-jfrog ] - # Case 1: Found artifact in JFrog - # Case 2: Did not find artifact in JFrog, had to build it in GHA - if: ${{ !cancelled() && (needs.look-for-wheel-in-jfrog.result == 'success' && (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 1) || (needs.look-for-wheel-in-jfrog.outputs.num_artifacts_found == 0 && needs.build-manylinux-wheel.result == 'success')) }} + # Case 1: Did not find artifact in JFrog, had to build it in GHA + # Case 2: Found artifact in JFrog + # Case 3: Not using JFrog artifact (i.e using wheel from Github artifact) + if: ${{ !cancelled() && (needs.upload-built-wheel-to-jfrog.result == 'success' || needs.upload-built-wheel-to-jfrog.result == 'skipped') }} runs-on: ubuntu-22.04 steps: - name: Harden the runner (Audit all outbound calls) @@ -153,7 +189,7 @@ jobs: - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: - merge-multiple: true + name: ${{ env.PYTHON_TAG }}-manylinux_x86_64.build - name: Install client run: pip install ./*.whl @@ -179,7 +215,8 @@ jobs: - run: echo VALGRIND_ARGS="--leak-check=full" >> $GITHUB_ENV if: ${{ !inputs.massif }} - - run: PYTHONMALLOC=malloc valgrind --error-exitcode=1 ${{ env.VALGRIND_ARGS }} python3 -m pytest -v new_tests/${{ github.event.inputs.test-file }} + # TODO: removed --error-exitcode=1 for testing purposes + - run: PYTHONMALLOC=malloc valgrind ${{ env.VALGRIND_ARGS }} python3 -m pytest -v new_tests/${{ github.event.inputs.test-file }} working-directory: test # TODO: upload report as artifact @@ -195,5 +232,5 @@ jobs: valgrind ] # Workflow run must clean up after itself even if cancelled - if: ${{ always() }} + if: ${{ always() && inputs.download-wheel-from-github-artifacts == false }} uses: ./.github/workflows/delete-artifacts.yml