Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
c803a00
Delegate CAGRA build heuristics to cuVS instead of hardcoding them
imotov Jul 20, 2026
98cea18
Fix checkstyle
imotov Jul 20, 2026
cd9eee9
Fix typo
imotov Jul 24, 2026
9c57e39
Remove redundant NN_DESCENT parameter
imotov Jul 24, 2026
d6b66df
Add test for the writer threads with custom strategy
imotov Jul 24, 2026
2b55668
Temporary switch to use NVIDIA/cuvs#2345
imotov Jul 24, 2026
692229e
Fix copyright
imotov Jul 24, 2026
b0e170f
Bump the build
imotov Jul 24, 2026
c64ac37
Test Build
imotov Jul 24, 2026
87e16ac
Copyright fix
imotov Jul 24, 2026
64e519e
Revert "Copyright fix"
imotov Jul 27, 2026
7a22b1e
Revert "Test Build"
imotov Jul 27, 2026
dfaaa90
Merge remote-tracking branch 'upstream/release/26.08' into issue-149/…
imotov Jul 27, 2026
e8167e2
Switch from fromHnswParams to fromDataset
imotov Jul 28, 2026
1b0fd04
Test panama binding theory
imotov Jul 28, 2026
3e7c48d
Test another panama approach
imotov Jul 28, 2026
8e6ebe3
Debug: log PR artifact contents
imotov Jul 28, 2026
d43cd9e
Debug: check PR libcuvs_c.so for new symbol
imotov Jul 28, 2026
ca62402
Remove conda libcuvs from LD_LIBRARY_PATH when using PR artifact
imotov Jul 28, 2026
6132d5f
Uninstall conda libcuvs when using PR artifact to avoid ldconfig shad…
imotov Jul 28, 2026
1ce03b4
Debug: one more try
imotov Jul 28, 2026
7b31187
Another build experiment
imotov Jul 28, 2026
1161c7d
Another build experiment
imotov Jul 28, 2026
888bb7c
Uninstall conda libcuvs when using PR artifact to avoid RPATH shadowing
imotov Jul 28, 2026
df35c3f
Use patchelf to tweak JVM RPATH instead of uninstalling conda libcuvs
imotov Jul 28, 2026
a6d6b94
Another experiment
imotov Jul 28, 2026
eb77f70
Move handling of PRs into ci/build.sh
imotov Jul 28, 2026
73c5828
Remove libcuvs first, download a new on after
imotov Jul 28, 2026
dce241d
Guard conda remove with set +u like conda activate
imotov Jul 28, 2026
1a19fdd
Fix conda removal
imotov Jul 28, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion RAPIDS_BRANCH
Original file line number Diff line number Diff line change
@@ -1 +1 @@
release/26.08
pull-request/2345
19 changes: 3 additions & 16 deletions build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -31,22 +31,7 @@ if hasArg --build-cuvs-java; then
pushd $CUVS_WORKDIR
fi

# libcuvs comes from the conda packages, so normally only the java bindings have
# to be built. For a pull request, the conda packages do not contain the PR's
# changes, so CI downloads the pre-built libcuvs artifact from the PR's own CI run.
CUVS_BUILD_TARGETS=("java")
if hasArg --use-pr-libcuvs && [[ "$BRANCH" == pull-request/* ]]; then
PR_NUM="${BRANCH#pull-request/}"
echo "Downloading libcuvs conda artifact from cuvs PR #${PR_NUM}..."
LIBCUVS_CONDA_DIR=$(rapids-get-pr-artifact NVIDIA/cuvs "$PR_NUM" cpp conda)
LIBCUVS_DIR=$(rapids-extract-conda-files "$LIBCUVS_CONDA_DIR")
# The downloaded library has to take precedence over the one provided by the
# conda packages, both here and while running the java tests.
LD_LIBRARY_PATH="$LIBCUVS_DIR/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
export LD_LIBRARY_PATH
echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH"
fi
./build.sh "${CUVS_BUILD_TARGETS[@]}"
./build.sh java
popd
fi

Expand All @@ -55,6 +40,8 @@ if ! hasArg --run-java-tests; then
MAVEN_VERIFY_ARGS=("-DskipTests")
fi

echo "LD_LIBRARY_PATH just before the test is: $LD_LIBRARY_PATH"

mvn clean verify "${MAVEN_VERIFY_ARGS[@]}" \
&& mvn jacoco:report \
&& mvn install:install-file -Dfile=./target/cuvs-lucene-$VERSION.jar -DgroupId=$GROUP_ID -DartifactId=cuvs-lucene -Dversion=$VERSION -Dpackaging=jar \
Expand Down
26 changes: 21 additions & 5 deletions ci/build_java.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,6 @@ fi
# Always build cuvs-java when running the pipeline
EXTRA_BUILD_ARGS+=("--build-cuvs-java")

# When RAPIDS_BRANCH points at a cuvs pull request, the libcuvs from the conda
# packages does not contain the PR's changes, so download the pre-built artifact
# from the PR's CI run instead. Only done in CI, local builds use the conda packages.
EXTRA_BUILD_ARGS+=("--use-pr-libcuvs")

# shellcheck disable=SC1091
. /opt/conda/etc/profile.d/conda.sh

Expand Down Expand Up @@ -66,6 +61,27 @@ else
exit 1
fi

# When RAPIDS_BRANCH points at a cuvs PR, the conda packages do not contain the
# PR's changes, so download the pre-built libcuvs artifact from the PR's own CI run.
BRANCH=$(cat "RAPIDS_BRANCH")
if [[ "$BRANCH" == pull-request/* ]]; then
rapids-logger "Remove libcuvs from conda environment"
# Uninstall the conda libcuvs so the JVM's RPATH (which points to the conda
# env's lib dir) cannot find the old libcuvs_c.so ahead of the PR artifact.
set +u
conda remove --yes --force-remove libcuvs
set -u
# Download PR artifact
PR_NUM="${BRANCH#pull-request/}"
rapids-logger "Downloading libcuvs conda artifact from cuvs PR #${PR_NUM}"
LIBCUVS_CONDA_DIR=$(rapids-get-pr-artifact NVIDIA/cuvs "$PR_NUM" cpp conda)
LIBCUVS_ARTIFACT_DIR=$(rapids-extract-conda-files "$LIBCUVS_CONDA_DIR")
# The PR artifact must take precedence over the conda-installed libcuvs both
# at runtime and for cmake's find_package (to pick up new C API headers for jextract).
export LD_LIBRARY_PATH="$LIBCUVS_ARTIFACT_DIR/lib:$LD_LIBRARY_PATH"
export cuvs_ROOT="$LIBCUVS_ARTIFACT_DIR"
fi

rapids-logger "Run Java build"

bash ./build.sh "${EXTRA_BUILD_ARGS[@]}"
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@ public class AcceleratedHNSWParams {

public static enum Strategy {
/*
* This strategy allows for automatic selection of the underlying CAGRA build algorithm.
* With this strategy we use NN_DESCENT for dataset less than 5M vectors, else we use IVF_PQ.
* Indexing parameters, especially for IVF_PQ, are heuristically identified automatically.
* This strategy delegates the derivation of the CAGRA build parameters (graph degrees, build
* algorithm and its parameters) to cuVS, based on HNSW-equivalent maxConn and beamWidth.
*
* This is the default and the recommended strategy.
*/
Expand Down Expand Up @@ -95,15 +94,14 @@ public static enum Strategy {
* @param writerThreads Number of cuVS writer threads to use.
* @param intermediateGraphDegree The intermediate graph degree while building the CAGRA index.
* @param graphdegree The graph degree to use while building the CAGRA index.
* @param indexType The type of index to build - CAGRA, BRUTEFORCE, or both.
* @param hnswLayers The number of HNSW layers to build in the HNSW index.
* @param maxConn The max connection parameter used when building HNSW index with the fallback mechanism.
* @param beamWidth The beam width parameter used when building HNSW index with the fallback mechanism.
* @param cagraGraphBuildAlgo The CAGRA graph build algorithm to use [NN_DESCENT, IVF_PQ].
* @param cuVSIvfPqParams An instance of CuVSIvfPqParams containing IVF_PQ specific parameters.
* @param numMergeWorkers The number of merge workers to use with the fallback mechanism.
* @param mergeExec The instance of {@link ExecutorService} to use with the fallback mechanism.
* @param strategy either HEURISTIC [Default] that automatically chooses build algorithm and its parameters based on data set size or CUSTOM that uses the parameters passed though this class.
* @param strategy either HEURISTIC [Default] that delegates the CAGRA build parameters to cuVS (derived from the HNSW-equivalent maxConn and beamWidth) or CUSTOM that uses the parameters passed through this class.
* @param cuvsDistanceType the cuvsDistanceType. The default option is L2Expanded.
* @param nnDescentNumIterations the number of Iterations to run if building with NN_DESCENT.
*/
Expand Down
11 changes: 5 additions & 6 deletions src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved.
* SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*/

Expand Down Expand Up @@ -74,7 +74,8 @@ public static GPUBuiltHnswGraph createSingleVectorHnswGraph(int size, int dimens

/**
* Creates a multi-layer HNSW graph with dynamic number of layers.
* M = cagraGraphDegree/2
* M = ceil(cagraGraphDegree / 2), where cagraGraphDegree is the CAGRA adjacency list's degree
* (its column count). Ceil is used to accommodate odd graph degrees.
* Each layer contains 1/M nodes from the previous layer
* Creates layers until the highest layer has ≤ M nodes
*/
Expand All @@ -85,13 +86,11 @@ public static GPUBuiltHnswGraph createMultiLayerHnswGraph(
CuVSMatrix adjacencyListMatrix,
List<?> vectors,
int hnswLayers,
int graphDegree,
CagraIndexParams params,
QuantizationType quantization)
throws Throwable {

// Calculate M as cagraGraphDegree/2
int M = graphDegree / 2;
int M = Math.ceilDiv((int) adjacencyListMatrix.columns(), 2);

// Store all layers data
List<int[]> layerNodes = new ArrayList<>();
Expand Down Expand Up @@ -320,7 +319,7 @@ public static void writeMeta(
meta.writeVLong(vectorIndexLength);
meta.writeVInt(field.getVectorDimension());
meta.writeInt(count);
meta.writeVInt(graphDegree / 2); // M = cagraGraphDegree/2
meta.writeVInt(Math.ceilDiv(graphDegree, 2)); // M = ceil(cagraGraphDegree / 2)

// write graph nodes on each level
if (graph == null) {
Expand Down
Loading
Loading