Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions .github/workflows/build-cuvs-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,10 @@ jobs:
runs-on: "linux-${{ matrix.ARCH }}-cpu4"
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
fetch-depth: 0
persist-credentials: false
- name: Clean up condarc for release builds
run: |
GIT_DESCRIBE_TAG="$(git describe --tags --first-parent --abbrev=0)"
Expand All @@ -69,7 +70,7 @@ jobs:
echo "Most recent tag is an alpha. Build will use nightly channels."
fi
- name: Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
Expand All @@ -78,7 +79,7 @@ jobs:
run: |
docker context create builders
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3
with:
# Using the built-in config from NVIDIA's self-hosted runners means that 'docker build'
# will use NVIDIA's self-hosted DockerHub pull-through cache, which should mean faster builds,
Expand All @@ -95,7 +96,7 @@ jobs:
PYTHON_VER: ${{ inputs.PYTHON_VER }}
RAPIDS_VER: ${{ inputs.RAPIDS_VER }}
- name: Build cuVS Benchmarks GPU image
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: context
file: cuvs-bench/gpu/Dockerfile
Expand All @@ -109,7 +110,7 @@ jobs:
outputs: type=registry,oci-mediatypes=true
- name: Build cuVS Benchmarks CPU image
if: inputs.BUILD_CUVS_BENCH_CPU_IMAGE
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: context
file: cuvs-bench/cpu/Dockerfile
Expand Down
11 changes: 6 additions & 5 deletions .github/workflows/build-rapids-image.yml
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,10 @@ jobs:
runs-on: "linux-${{ matrix.ARCH }}-cpu4"
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
fetch-depth: 0
persist-credentials: false
- name: Clean up condarc for release builds
run: |
GIT_DESCRIBE_TAG="$(git describe --tags --first-parent --abbrev=0)"
Expand All @@ -70,7 +71,7 @@ jobs:
echo "Most recent tag is an alpha. Build will use nightly channels."
fi
- name: Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
Expand All @@ -79,7 +80,7 @@ jobs:
run: |
docker context create builders
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3 # zizmor: ignore[cache-poisoning]
with:
# Using the built-in config from NVIDIA's self-hosted runners means that 'docker build'
# will use NVIDIA's self-hosted DockerHub pull-through cache, which should mean faster builds,
Expand All @@ -100,7 +101,7 @@ jobs:
PYTHON_VER: ${{ inputs.PYTHON_VER }}
RAPIDS_VER: ${{ inputs.RAPIDS_VER }}
- name: Build base image
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: context
file: Dockerfile
Expand All @@ -113,7 +114,7 @@ jobs:
# ensure only OCI mediatypes are used: https://docs.docker.com/build/exporters/#oci-media-types
outputs: type=registry,oci-mediatypes=true
- name: Build notebooks image
uses: docker/build-push-action@v6
uses: docker/build-push-action@10e90e3645eae34f1e60eeb005ba3a3d33f178e8 # v6
with:
context: context
file: Dockerfile
Expand Down
43 changes: 27 additions & 16 deletions .github/workflows/build-test-publish-images.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,14 +42,16 @@ jobs:
- build-cuvs
- build-cuvs-multiarch-manifest
- test
secrets: inherit
uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main
secrets: inherit # zizmor: ignore[secrets-inherit]
uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main # zizmor: ignore[unpinned-uses]
checks:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false
- name: Run pre-commit
uses: pre-commit/action@v3.0.1
uses: pre-commit/action@2c7b3805fd2a0fd8c1884dcaebf91fc102a13ecd # v3.0.1
- name: Run hadolint
run: |
ci/lint-dockerfiles.sh
Expand All @@ -70,19 +72,22 @@ jobs:
ALPHA_TAG: ${{ steps.compute-rapids-ver.outputs.ALPHA_TAG }}
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
fetch-depth: 0
persist-credentials: false
- name: Compute matrix
id: compute-matrix
run: |
MATRIX=$(ci/compute-matrix.sh)
echo "MATRIX=${MATRIX}" | tee -a ${GITHUB_OUTPUT}
- name: Compute tag prefix
id: compute-tag-prefix
env:
BUILD_TYPE: ${{ inputs.build_type }}
run: |
TAG_PREFIX=""
if [ "${{ inputs.build_type }}" = "pull-request" ]; then
if [ "$BUILD_TYPE" = "pull-request" ]; then
pr_num="${GITHUB_REF_NAME##*/}"
BASE_TAG_PREFIX="docker-${pr_num}-"
NOTEBOOKS_TAG_PREFIX="docker-notebooks-${pr_num}-"
Expand All @@ -95,12 +100,14 @@ jobs:
echo "CUVS_BENCH_CPU_TAG_PREFIX=${CUVS_BENCH_CPU_TAG_PREFIX}" | tee -a ${GITHUB_OUTPUT}
- name: Compute image repo
id: compute-image-repo
env:
BUILD_TYPE: ${{ inputs.build_type }}
run: |
base_repo="base"
notebooks_repo="notebooks"
cuvs_bench_repo="cuvs-bench"
cuvs_bench_cpu_repo="cuvs-bench-cpu"
if [ "${{ inputs.build_type }}" = "pull-request" ]; then
if [ "$BUILD_TYPE" = "pull-request" ]; then
base_repo="staging"
notebooks_repo="staging"
cuvs_bench_repo="staging"
Expand All @@ -126,8 +133,10 @@ jobs:
echo "ALPHA_TAG=${ALPHA_TAG}" | tee -a ${GITHUB_OUTPUT}
- name: Compute test matrix
id: compute-test-matrix
env:
BUILD_TYPE: ${{ inputs.build_type }}
run: |
TEST_MATRIX=$(yq '.${{ inputs.build_type }}' matrix-test.yaml)
TEST_MATRIX=$(yq ".$BUILD_TYPE" matrix-test.yaml)
export TEST_MATRIX

echo "TEST_MATRIX=$(yq -n -o json 'env(TEST_MATRIX)' | jq -c '{include: .}')" | tee --append "${GITHUB_OUTPUT}"
Expand All @@ -136,7 +145,7 @@ jobs:
strategy:
matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
fail-fast: false
secrets: inherit
secrets: inherit # zizmor: ignore[secrets-inherit]
uses: ./.github/workflows/build-rapids-image.yml
# Referencing something from the 'matrix' context prevents GitHub auto-generating
# a hard-to-read name with all the matrix input values.
Expand Down Expand Up @@ -170,7 +179,7 @@ jobs:
strategy:
matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
fail-fast: false
secrets: inherit
secrets: inherit # zizmor: ignore[secrets-inherit]
uses: ./.github/workflows/build-cuvs-image.yml
# Referencing something from the 'matrix' context prevents GitHub auto-generating
# a hard-to-read name with all the matrix input values.
Expand Down Expand Up @@ -206,11 +215,12 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
fetch-depth: 0
persist-credentials: false
- name: Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
Expand Down Expand Up @@ -238,11 +248,12 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v6
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
fetch-depth: 0
persist-credentials: false
- name: Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
Expand All @@ -267,7 +278,7 @@ jobs:
strategy:
matrix: ${{ fromJSON(needs.compute-matrix.outputs.TEST_MATRIX) }}
fail-fast: false
secrets: inherit
secrets: inherit # zizmor: ignore[secrets-inherit]
uses: ./.github/workflows/validate.yml
# Referencing something from the 'matrix' context prevents GitHub auto-generating
# a hard-to-read name with all the matrix input values.
Expand Down Expand Up @@ -304,7 +315,7 @@ jobs:
strategy:
matrix: ${{ fromJSON(needs.compute-matrix.outputs.TEST_MATRIX) }}
fail-fast: false
secrets: inherit
secrets: inherit # zizmor: ignore[secrets-inherit]
uses: ./.github/workflows/test-notebooks.yml
# Referencing something from the 'matrix' context prevents GitHub auto-generating
# a hard-to-read name with all the matrix input values.
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ jobs:
needs:
- check-nightly-ci
- docker
uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main
permissions:
checks: write
pull-requests: write
uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@main # zizmor: ignore[unpinned-uses]
if: always()
with:
needs: ${{ toJSON(needs) }}
Expand All @@ -30,18 +33,24 @@ jobs:
steps:
- name: Get PR Info
id: get-pr-info
uses: nv-gha-runners/get-pr-info@main
uses: nv-gha-runners/get-pr-info@main # zizmor: ignore[unpinned-uses]
- name: Check if nightly CI is passing
uses: rapidsai/shared-actions/check_nightly_success/dispatch@main
uses: rapidsai/shared-actions/check_nightly_success/dispatch@main # zizmor: ignore[unpinned-uses]
with:
# default is 7 days, but this repo is downstream of all of RAPIDS so allow a bit longer window
max-days-without-success: 14
repo: ${{ github.repository }}
target-branch: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.ref }}
workflow-id: 'publish.yml'
docker:
permissions:
actions: read
contents: read
id-token: write
packages: read
pull-requests: read
uses: ./.github/workflows/build-test-publish-images.yml
with:
build_type: pull-request
run_tests: true
secrets: inherit
secrets: inherit # zizmor: ignore[secrets-inherit]
16 changes: 13 additions & 3 deletions .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,26 +21,36 @@ concurrency:

jobs:
docker:
permissions:
actions: read
contents: read
id-token: write
packages: read
pull-requests: read
uses: ./.github/workflows/build-test-publish-images.yml
with:
build_type: branch
run_tests: ${{ inputs.run_tests || false }}
secrets: inherit
secrets: inherit # zizmor: ignore[secrets-inherit]
readme:
runs-on: ubuntu-latest
needs: docker
if: startsWith(github.ref, 'refs/tags/v')
permissions:
contents: read
strategy:
matrix:
repo_name:
- rapidsai/base
- rapidsai/notebooks
steps:
- name: checkout code
uses: actions/checkout@v6
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false

- name: Update DockerHub README for ${{ matrix.repo_name }}
uses: peter-evans/dockerhub-description@v5
uses: peter-evans/dockerhub-description@1b9a80c056b620d92cedb9d9b5a223409c68ddfa # v5
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_PASSWORD }}
Expand Down
26 changes: 18 additions & 8 deletions .github/workflows/release-to-nvstaging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,15 @@ on:
jobs:
compute-matrix:
runs-on: ubuntu-latest
permissions:
contents: read
outputs:
matrix: ${{ steps.generate-matrix.outputs.matrix }}
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false

- name: Compute matrix
id: generate-matrix
Expand All @@ -29,37 +33,43 @@ jobs:
name: copy (${{ matrix.CUDA_VER }}, ${{ matrix.PYTHON_VER }})
needs: compute-matrix
runs-on: ubuntu-latest
permissions:
contents: read
strategy:
matrix: ${{fromJson(needs.compute-matrix.outputs.matrix)}}
steps:
- name: Checkout code
uses: actions/checkout@v6
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6
with:
persist-credentials: false

- name: Login to DockerHub
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}

- name: Login to NGC
uses: docker/login-action@v3
uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3
with:
registry: nvcr.io
username: ${{ secrets.NGC_DOCKER_USER }}
password: ${{ secrets.NGC_DOCKER_PASSWORD }}

- name: Release to NGC
env:
CUDA_VER: ${{ matrix.CUDA_VER }}
PYTHON_VER: ${{ matrix.PYTHON_VER }}
RAPIDS_VER: ${{ inputs.RAPIDS_VER }}
run: |
#!/bin/bash
set -e

CUDA_VER=${{ matrix.CUDA_VER }}
CUDA_MAJOR=${CUDA_VER%%.*}
PYTHON_VER=${{ matrix.PYTHON_VER }}

for type in base notebooks; do
source="rapidsai/$type:${{ inputs.RAPIDS_VER }}-cuda$CUDA_MAJOR-py$PYTHON_VER"
target="nvcr.io/nvstaging/rapids/$type:${{ inputs.RAPIDS_VER }}-cuda$CUDA_MAJOR-py$PYTHON_VER"
source="rapidsai/$type:${RAPIDS_VER}-cuda$CUDA_MAJOR-py$PYTHON_VER"
target="nvcr.io/nvstaging/rapids/$type:${RAPIDS_VER}-cuda$CUDA_MAJOR-py$PYTHON_VER"
echo "$source => $target"
docker run -v ~/.docker/config.json:/config.json quay.io/skopeo/stable:v1.20.0 copy --multi-arch all --dest-authfile=/config.json docker://$source docker://$target
done
Loading