[CLIENT-3467] CI/CD: use orchestrator workflow instead of individual shared workflows

.github/actions/get-artifact-for-stage-tests/action.yml

@@ -78,5 +78,11 @@ runs:
 
   - name: Download artifact from JFrog
     if: ${{ inputs.get_from_jfrog == 'true' }}
-    run: jf rt dl --fail-no-op --flat --build python-client/${{ inputs.jfrog_build_version }} "${{ inputs.JFROG_REPO_NAME }}/**/${{ env.ARTIFACT_FILE_NAME_PATTERN }}"
+    run: |
+      bundle_version=$(jf rbs --format json --order-by created --project database versions python-client-release | jq -r '.release_bundles[0].release_bundle_version')
+      # Currently no way to download specific artifacts from a release bundle
+      # Using ARTIFACT_FILE_NAME_PATTERN with jf rt dl --bundle doesn't work
+      build_number="${bundle_version##*-}"
+      artifact_path=$(jf rt s --project database --fail-no-op --build python-client/$build_number "database-pypi-dev-local/**/$ARTIFACT_FILE_NAME_PATTERN" | jq -r '.[0].path')
+      jf rt dl --project database --fail-no-op --flat $artifact_path
     shell: bash

.github/workflows/build-artifacts.yml

@@ -89,6 +89,9 @@ on:
         required: false
         type: string
         default: ''
+    outputs:
+      build-version:
+        value: ${{ jobs.build-sdist.outputs.sdist-version }}
     secrets:
       DOCKER_HUB_BOT_USERNAME:
         required: true

.github/workflows/build-python-client.bash

@@ -0,0 +1,57 @@
+# On macOS and Windows, this is expected to run on bare metal
+# On Linux, this is expected to run in a container from the manylinux image
+os=$(uname -s)
+running_on_windows=$([[ "$os" =~ CYGWIN* || "$os" =~ MINGW* ]])
+if [[ $running_on_windows ]]; then
+    nuget restore
+fi
+
+if [[ $os =~ Darwin* ]]; then
+    brew install openssl@3 libyaml
+
+    export MACOSX_DEPLOYMENT_TARGET
+    MACOSX_DEPLOYMENT_TARGET="$(sw_vers -productVersion | cut -d"." -f 1).0"
+
+    if [[ $(uname -m) == "arm64" ]]; then
+        libraries=('libyaml' 'openssl')
+        for library in "${libraries[@]}"; do
+          LIBRARY_PATH="${LIBRARY_PATH}:$(brew --prefix "$library")/lib"
+        done
+        export LIBRARY_PATH="$LIBRARY_PATH"
+    else
+        export LDFLAGS='-headerpad_max_install_names'
+    fi
+fi
+
+if [[ $os =~ Linux* ]]; then
+    yum install libyaml-devel -y
+fi
+
+python3 -m pip install build -c requirements.txt
+python3 -m build
+
+if [[ $os =~ Linux* ]]; then
+    unrepaired_wheel_path=$(find dist/ -type f -name '*.whl' | head -n 1)
+    if [[ "$DEBUG" != "" ]]; then
+        auditwheel show "$unrepaired_wheel_path"
+        WHEEL_DIR=wheel-contents
+        unzip "$unrepaired_wheel_path" -d $WHEEL_DIR
+        ldd $WHEEL_DIR/*.so | awk '{print $1}' | grep libssl.so.3
+        ldd $WHEEL_DIR/*.so | awk '{print $1}' | grep libcrypto.so.3
+    fi
+
+    auditwheel repair -w wheelhouse/ "$unrepaired_wheel_path"
+
+    if [[ "$DEBUG" != "" ]]; then
+        auditwheel show wheelhouse/*
+    fi
+
+    # TODO: This should happen if any steps after creating the dir fail
+    rm -rf "$WHEEL_DIR"
+elif [[ $os =~ Darwin* ]]; then
+    delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}
+    delocate-listdeps {dest_dir}/*.whl | grep libcrypto.3.dylib
+    delocate-listdeps {dest_dir}/*.whl | grep libssl.3.dylib
+elif [[ $running_on_windows ]]; then
+    delvewheel repair -vv --add-path ./aerospike-client-c/vs/x64/Release -w wheelhouse "$unrepaired_wheel_path"
+fi

.github/workflows/build-sdist.yml

@@ -9,57 +9,65 @@
       sha_to_build:
         type: string
         required: true
+    outputs:
+      sdist-version:
+        value: ${{ jobs.build-sdist.outputs.sdist-version }}
 
 env:
   STATUS_CHECK_MESSAGE: "Build source distribution"
   COMMIT_SHA_TO_BUILD: ${{ inputs.is_workflow_call == true && inputs.sha_to_build || github.sha }}
 
 jobs:
   build-sdist:
+    outputs:
+      sdist-version: ${{ steps.get-sdist-version.outputs.version }}
     name: Build source distribution
     runs-on: ubuntu-22.04
     steps:
     - name: Harden the runner (Audit all outbound calls)
       uses: step-security/harden-runner@a90bcbc6539c36a85cdfeb73f7e2f433735f215b # v2.15.0
       with:
         egress-policy: audit
 
     - name: Show job status for commit
       # Commit status will already be shown by the calling workflow for push and pull request events, but not
       # for any other event like workflow_dispatch. so we have to do it manually
       # If workflow_call triggered this job, github.event_name will inherit the event of the calling workflow
       # The calling workflow can be triggered by push or pull request events, so there's that
       # https://github.com/actions/runner/issues/3146#issuecomment-2000017097
       if: ${{ github.event_name != 'push' && github.event_name != 'pull_request' }}
       uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
       with:
         sha: ${{ env.COMMIT_SHA_TO_BUILD }}
         context: ${{ env.STATUS_CHECK_MESSAGE }}
 
     - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
       with:
         submodules: recursive
         ref: ${{ env.COMMIT_SHA_TO_BUILD }}
         fetch-depth: 0
 
     - name: Install build dependencies (pip packages)
       run: python3 -m pip install -r requirements.txt
 
     - name: Build source distribution
       run: python3 -m build --sdist
 
+    - id: get-sdist-version
+      run: echo "version=$(cat VERSION)" >> $GITHUB_OUTPUT
+
     - name: Upload source distribution to GitHub
       uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
       with:
         path: ./dist/*.tar.gz
         name: sdist.build
 
     - name: Set final commit status
       uses: myrotvorets/set-commit-status-action@3730c0a348a2ace3c110851bed53331bc6406e9f # v2.0.1
       # Always run even if job failed or is cancelled
       # But we don't want to show anything if the calling workflow was triggered by these events
       if: ${{ always() && github.event_name != 'push' && github.event_name != 'pull_request' }}
       with:
         sha: ${{ env.COMMIT_SHA_TO_BUILD }}
         status: ${{ job.status }}
         context: ${{ env.STATUS_CHECK_MESSAGE }}

.github/workflows/build-wheels.yml

@@ -263,6 +263,7 @@ jobs:
     - if: ${{ startsWith(inputs.platform-tag, 'manylinux') }}
       run: echo CIBW_MANYLINUX_AARCH64_IMAGE=${{ format(env.CUSTOM_IMAGE_NAME, 'aarch64', 'dadc371c3647efe5a1f3a7c99edddea1d8db542e3d2ad48bf63fa81364fb63b6') }} >> $GITHUB_ENV
 
+    # TODO: would need way to pull this outside of shared workflow
     - uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # v3.6.0
       if: ${{ startsWith(inputs.platform-tag, 'manylinux') }}
       with:

.github/workflows/bump-stage-and-upload-to-jfrog.yml

@@ -24,44 +24,12 @@ jobs:
       base_branch: ${{ vars.STAGE_BRANCH_NAME }}
     secrets: inherit
 
-  promote-dev-build-to-rc:
-    name: Bump (promote) the dev version to an RC version in the stage branch
-    needs: ff-stage-to-dev-tag
-    uses: ./.github/workflows/bump-version.yml
-    with:
-      change: 'promote-dev-build-to-rc'
-      ref: ${{ vars.STAGE_BRANCH_NAME }}
-    secrets: inherit
-
-  rebuild-artifacts-with-rc-version:
-    needs: promote-dev-build-to-rc
-    uses: ./.github/workflows/build-artifacts.yml
-    with:
-      sha-to-build-and-test: ${{ needs.promote-dev-build-to-rc.outputs.bump_sha }}
-    secrets: inherit
-
-  upload-rc-artifacts-to-jfrog:
+  promote-release-bundle-to-dev:
     needs: [
-      rebuild-artifacts-with-rc-version,
-      # We need the new RC version to label the build in JFrog
-      promote-dev-build-to-rc
+      ff-stage-to-dev-tag,
     ]
-    name: Upload artifacts to JFrog
-    uses: ./.github/workflows/upload-to-jfrog.yml
+    uses: ./.github/workflows/promote-release-bundle.yml
     with:
-      version: ${{ needs.promote-dev-build-to-rc.outputs.new_version }}
-    secrets: inherit
-
-  # See reason for deleting artifacts in dev-workflow-p2.yml
-  delete-artifacts:
-    needs: upload-rc-artifacts-to-jfrog
-    uses: ./.github/workflows/delete-artifacts.yml
-
-  ff-dev-to-stage:
-    name: Fast forward dev branch to stage branch to include the bump to RC commit
-    needs: promote-dev-build-to-rc
-    uses: ./.github/workflows/fast-forward-merge.yml
-    with:
-      ref_to_merge: origin/${{ vars.STAGE_BRANCH_NAME }}
-      base_branch: ${{ vars.DEV_BRANCH_NAME }}
-    secrets: inherit
+      release-bundle-name: python-client-release
+      release-bundle-version: ${{ inputs.passed-dev-tag }}
+      new-environment: TEST

.github/workflows/bump-version.yml

@@ -13,8 +13,18 @@ on:
         - bump-dev-num
         - promote-dev-build-to-rc
         - promote-rc-build-to-release
+      dry-run:
+        required: false
+        default: false
+        type: boolean
+        description: No tagging in repo
   workflow_call:
     inputs:
+      dry-run:
+        required: false
+        default: false
+        type: boolean
+        description: Dry run
       change:
         # Since workflow_call doesn't support 'options' input type,
         # we take in a string instead that must be a valid Python script name (excluding the .py part)
@@ -37,8 +47,6 @@ on:
     outputs:
       new_version:
         value: ${{ jobs.get-new-version.outputs.new_version }}
-      bump_sha:
-        value: ${{ jobs.update-version-in-repo.outputs.bump_sha }}
 
 jobs:
   get-current-version:
@@ -57,10 +65,14 @@ jobs:
       with:
         token: ${{ secrets.CLIENT_BOT_PAT }}
         ref: ${{ inputs.ref }}
+        # Get all tags
+        fetch-depth: 0
 
-    - name: Get current version
+    - name: Get tag from previous commit
       id: get-current-version
-      run: echo current_version=$(cat VERSION) >> $GITHUB_OUTPUT
+      run: |
+        prev_commit=$(git rev-parse HEAD~)
+        echo current_version=$(git describe --tags --exact-match $prev_commit) >> $GITHUB_OUTPUT
 
   get-new-version:
     runs-on: ubuntu-22.04
@@ -93,4 +105,5 @@ jobs:
     with:
       new_version: ${{ needs.get-new-version.outputs.new_version }}
       ref: ${{ inputs.is_workflow_call && inputs.ref || github.ref }}
+      dry-run: ${{ inputs.dry-run }}
     secrets: inherit

.github/workflows/dev-workflow-part2.yml

@@ -6,33 +6,42 @@
     branches:
     - 'dev*'
   workflow_dispatch:
+    inputs:
+      dry-run:
+        description: Don't create tag in repo
+        type: boolean
+        required: false
+        default: true
 
 jobs:
-  bump-dev-number:
+  tag-new-commit-with-dev-number:
     uses: ./.github/workflows/bump-version.yml
     with:
       change: 'bump-dev-num'
+      dry-run: ${{ inputs.dry-run }}
     secrets: inherit
 
   rebuild-artifacts-with-new-dev-num:
-    needs: bump-dev-number
+    needs: tag-new-commit-with-dev-number
     name: Rebuild artifacts with new dev number
     uses: ./.github/workflows/build-artifacts.yml
     with:
       # On pull_request_target, the bump version commit will be ignored
       # So we must pass it manually to the workflow
-      sha-to-build-and-test: ${{ needs.bump-dev-number.outputs.bump_sha }}
+      sha-to-build-and-test: ${{ github.sha }}
+      # override-version: ${{ needs.tag-new-commit-with-dev-number.outputs.new_version }}
     secrets: inherit
 
   upload-to-jfrog:
     name: Upload artifacts to JFrog
     needs: [
-      bump-dev-number,
+      tag-new-commit-with-dev-number,
       rebuild-artifacts-with-new-dev-num
     ]
     uses: ./.github/workflows/upload-to-jfrog.yml
     with:
       version: ${{ needs.bump-dev-number.outputs.new_version }}
+      dry-run: ${{ inputs.dry-run }}
     secrets: inherit
 
   # We don't want the artifacts in JFrog to also exist in Github

.github/workflows/promote-release-bundle.yml

@@ -0,0 +1,34 @@
+on:
+  workflow_call:
+    inputs:
+      release-bundle-name:
+        type: string
+        required: true
+      release-bundle-version:
+        type: string
+        required: true
+      new-environment:
+        type: string
+        required: true
+      jfrog-project:
+        type: string
+        required: true
+      jfrog-repo-name:
+        type: string
+        required: true
+
+jobs:
+  promote-release-bundle-to-dev:
+    runs-on: ubuntu-24.04
+    env:
+      JF_PROJECT: ${{ inputs.jfrog-project }}
+    steps:
+    - name: Setup JFrog CLI
+      uses: jfrog/setup-jfrog-cli@5b06f730cc5a6f55d78b30753f8583454b08c0aa # v4.8.1
+      env:
+        JF_URL: https://artifact.aerospike.io
+      with:
+        oidc-provider-name: ${{ vars.OIDC_PROVIDER_NAME }}
+        oidc-audience: ${{ vars.OIDC_AUDIENCE }}
+
+    - run: jf rbp --signing-key=aerospike --include-repos ${{ inputs.jfrog-repo-name }} ${{ inputs.release-bundle-name }} ${{ inputs.release-bundle-version }} ${{ inputs.new-environment }}

.github/workflows/run-dev-tests-and-promote-to-dev.yml

@@ -0,0 +1,64 @@
+name: Run dev tests and promote release bundle to DEV
+run-name: Run dev tests and promote release bundle to DEV (dry-run=${{ inputs.dry-run }}, test-file=${{ inputs.test-file }})
+
+permissions:
+  # We are building the artifacts in build-artifacts.yml
+  contents: read
+  # For shared reusable workflows
+  # TODO: look into more
+  id-token: write
+  statuses: write
+  # For downloading docker images from ghcr.io
+  packages: read
+
+on:
+  workflow_dispatch:
+    inputs:
+      # These inputs are in case we need to test against a server RC
+      registry-name:
+        type: string
+        required: true
+        description: Registry name
+        default: 'docker.io'
+      image-name:
+        type: string
+        required: true
+        description: Image name
+        default: 'aerospike/aerospike-server-enterprise'
+      server-tag:
+        type: string
+        required: true
+        default: 'latest'
+        description: 'Server docker image tag (e.g to test a client backport version)'
+      # For testing purposes
+      dry-run:
+        required: false
+        default: false
+        type: boolean
+        description: Don't create release bundle
+      run_tests:
+        required: false
+        default: false
+        type: boolean
+        description: Run dev tests
+      test-file:
+        required: false
+        type: string
+        description: For speeding up debugging
+
+jobs:
+  dev-workflow:
+    uses: aerospike/shared-workflows/.github/workflows/reusable_artifacts-cicd.yaml@8f53b63b33d86e63aeb37d951283dd974574885b
+    with:
+      gh-workflows-ref: 8f53b63b33d86e63aeb37d951283dd974574885b
+      jf-project: ${{ vars.JFROG_PROJECT_FOR_CLIENT_TEAM }}
+      jf-build-name: ${{ vars.JFROG_BUILD_NAME }}
+      # TODO: ${{ needs.run-dev-tests.outputs.build-version }}- needs to be applied to version
+      version: ${{ github.run_number }}
+      gh-artifact-directory: dist
+      build-script: |
+        make build
+      # Optional:
+      build-type: release # Freeform label, applied as build.type target-prop
+      internal: false # Set true to mark artifacts as internal-only
+    secrets: inherit