From c803a003d6809b4fc464690ef53a41772399b85b Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 20 Jul 2026 07:52:46 -1000 Subject: [PATCH 01/29] Delegate CAGRA build heuristics to cuVS instead of hardcoding them Replace the hand-ported heuristic and hard-coded 5M-row algorithm threshold in CagraIndexParamsFactory with cuVS's own APIs, so cuVS stays the single source of truth. Fixes #149 --- .../cuvs/lucene/AcceleratedHNSWParams.java | 11 +- .../cuvs/lucene/AcceleratedHNSWUtils.java | 9 +- .../cuvs/lucene/CagraIndexParamsFactory.java | 241 ++++-------------- .../cuvs/lucene/CuVS2510GPUVectorsWriter.java | 3 +- .../nvidia/cuvs/lucene/GPUSearchParams.java | 9 +- .../Lucene99AcceleratedHNSWVectorsWriter.java | 6 +- ...ratedHNSWBinaryQuantizedVectorsWriter.java | 6 +- ...ratedHNSWScalarQuantizedVectorsWriter.java | 6 +- .../TestAcceleratedHNSWOddGraphDegree.java | 111 ++++++++ .../lucene/TestCagraIndexParamsFactory.java | 97 +++++++ 10 files changed, 289 insertions(+), 210 deletions(-) create mode 100644 src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWOddGraphDegree.java create mode 100644 src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java diff --git a/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java b/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java index aac31415..a4764288 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java +++ b/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java @@ -7,6 +7,7 @@ import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.CagraIndexParams.CuvsDistanceType; +import com.nvidia.cuvs.CagraIndexParams.HnswHeuristicType; import com.nvidia.cuvs.CuVSIvfPqParams; import java.util.Objects; import java.util.concurrent.ExecutorService; @@ -17,9 +18,8 @@ public class AcceleratedHNSWParams { public static enum Strategy { /* - * This strategy allows for automatic selection of the underlying CAGRA build algorithm. - * With this strategy we use NN_DESCENT for dataset less than 5M vectors, else we use IVF_PQ. - * Indexing parameters, especially for IVF_PQ, are heuristically identified automatically. + * This strategy delegates the derivation of the CAGRA build parameters (graph degrees, build + * algorithm and its parameters) to cuVS, based on HNSW-equivalent maxConn and beamWidth. * * This is the default and the recommended strategy. */ @@ -64,6 +64,8 @@ public static enum Strategy { public static final Strategy DEFAULT_STRATEGY = Strategy.HEURISTIC; public static final CuvsDistanceType DEFAULT_CUVS_DISTANCE_TYPE = CuvsDistanceType.L2Expanded; public static final int DEFAULT_NN_DESCENT_NUM_ITERATIONS = 20; + public static final HnswHeuristicType DEFAULT_HNSW_HEURISTIC_TYPE = + HnswHeuristicType.SAME_GRAPH_FOOTPRINT; public static final Supplier DEFAULT_IVF_PQ_PARAMS = () -> { @@ -95,7 +97,6 @@ public static enum Strategy { * @param writerThreads Number of cuVS writer threads to use. * @param intermediateGraphDegree The intermediate graph degree while building the CAGRA index. * @param graphdegree The graph degree to use while building the CAGRA index. - * @param indexType The type of index to build - CAGRA, BRUTEFORCE, or both. * @param hnswLayers The number of HNSW layers to build in the HNSW index. * @param maxConn The max connection parameter used when building HNSW index with the fallback mechanism. * @param beamWidth The beam width parameter used when building HNSW index with the fallback mechanism. @@ -103,7 +104,7 @@ public static enum Strategy { * @param cuVSIvfPqParams An instance of CuVSIvfPqParams containing IVF_PQ specific parameters. * @param numMergeWorkers The number of merge workers to use with the fallback mechanism. * @param mergeExec The instance of {@link ExecutorService} to use with the fallback mechanism. - * @param strategy either HEURISTIC [Default] that automatically chooses build algorithm and its parameters based on data set size or CUSTOM that uses the parameters passed though this class. + * @param strategy either HEURISTIC [Default] that delegates the CAGRA build parameters to cuVS (derived from the HNSW-equivalent maxConn and beamWidth) or CUSTOM that uses the parameters passed though this class. * @param cuvsDistanceType the cuvsDistanceType. The default option is L2Expanded. * @param nnDescentNumIterations the number of Iterations to run if building with NN_DESCENT. */ diff --git a/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java b/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java index 2a8ab02f..d59a39d6 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java +++ b/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java @@ -74,7 +74,8 @@ public static GPUBuiltHnswGraph createSingleVectorHnswGraph(int size, int dimens /** * Creates a multi-layer HNSW graph with dynamic number of layers. - * M = cagraGraphDegree/2 + * M = ceil(cagraGraphDegree / 2), where cagraGraphDegree is the CAGRA adjacency list's degree + * (its column count). Ceil is used to accommodate odd graph degrees. * Each layer contains 1/M nodes from the previous layer * Creates layers until the highest layer has ≤ M nodes */ @@ -85,13 +86,11 @@ public static GPUBuiltHnswGraph createMultiLayerHnswGraph( CuVSMatrix adjacencyListMatrix, List vectors, int hnswLayers, - int graphDegree, CagraIndexParams params, QuantizationType quantization) throws Throwable { - // Calculate M as cagraGraphDegree/2 - int M = graphDegree / 2; + int M = Math.ceilDiv((int) adjacencyListMatrix.columns(), 2); // Store all layers data List layerNodes = new ArrayList<>(); @@ -320,7 +319,7 @@ public static void writeMeta( meta.writeVLong(vectorIndexLength); meta.writeVInt(field.getVectorDimension()); meta.writeInt(count); - meta.writeVInt(graphDegree / 2); // M = cagraGraphDegree/2 + meta.writeVInt(Math.ceilDiv(graphDegree, 2)); // M = ceil(cagraGraphDegree / 2) // write graph nodes on each level if (graph == null) { diff --git a/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java b/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java index 20fed04b..648362b5 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java +++ b/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java @@ -7,185 +7,49 @@ import com.nvidia.cuvs.CagraIndexParams; import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; -import com.nvidia.cuvs.CagraIndexParams.CodebookGen; -import com.nvidia.cuvs.CagraIndexParams.CudaDataType; -import com.nvidia.cuvs.CagraIndexParams.CuvsDistanceType; -import com.nvidia.cuvs.CuVSIvfPqIndexParams; -import com.nvidia.cuvs.CuVSIvfPqParams; -import com.nvidia.cuvs.CuVSIvfPqSearchParams; /** - * A centralized approach to producing instances of {@link CagraIndexParams} based on the chosen strategy + * A centralized place for producing {@link CagraIndexParams} from the cuvs-lucene input parameter + * classes based on the chosen strategy. + * + *

For the {@code HEURISTIC} strategy the build heuristics are delegated to cuVS. + * */ public class CagraIndexParamsFactory { - private static final int ALGO_SWITCH_THRESHOLD = 5_000_000; + private CagraIndexParamsFactory() {} /** - * Translation of the internal logic found here: - * https://github.com/rapidsai/cuvs/blob/main/cpp/include/cuvs/neighbors/ivf_pq.hpp#L3385-L3428 + * Creates an instance of {@link CagraIndexParams} for the GPU-native CAGRA index based on the + * chosen strategy in the {@link GPUSearchParams}. * - * Ideally we should hook into the internal API but this is currently replicated to avoid complications - * in other parts of code base. - */ - private static CuVSIvfPqParams getCuVSIvfPqParams(long rows, long dimension) { - - int pqDim; - int pqBits; - - if (dimension <= 32) { - pqDim = 16; - pqBits = 8; - } else { - pqBits = 4; - if (dimension <= 64) { - pqDim = 32; - } else if (dimension <= 128) { - pqDim = 64; - } else if (dimension <= 192) { - pqDim = 96; - } else { - pqDim = (int) roundUpSafe(dimension / 2, 128); - } - } - - int nLists = (int) Math.max(1, rows / 2000); - final int kmeansNIters = 10; - final double kMinPointsPerCluster = 32; - double minKmeansTrainsetPoints = kMinPointsPerCluster * nLists; - final double maxKmeansTrainsetFraction = 1.0; - double minKmeansTrainsetFraction = - Math.min(maxKmeansTrainsetFraction, minKmeansTrainsetPoints / rows); - double kmeansTrainsetFraction = - Math.clamp( - 1.0 / Math.sqrt(rows * 1e-5), minKmeansTrainsetFraction, maxKmeansTrainsetFraction); - final CodebookGen codebookKind = CodebookGen.PER_SUBSPACE; - int nProbes = (int) Math.round(Math.sqrt(nLists) / 20 + 4); - final int refinementRate = 1; - - CuVSIvfPqIndexParams cuVSIvfPqIndexParams = - new CuVSIvfPqIndexParams.Builder() - .withCodebookKind(codebookKind) - .withKmeansNIters(kmeansNIters) - .withKmeansTrainsetFraction(kmeansTrainsetFraction) - .withNLists(nLists) - .withPqBits(pqBits) - .withPqDim(pqDim) - .withAddDataOnBuild(true) - .withConservativeMemoryAllocation(true) - .build(); - - CuVSIvfPqSearchParams cuVSIvfPqSearchParams = - new CuVSIvfPqSearchParams.Builder() - .withLutDtype(CudaDataType.CUDA_R_16F) - .withInternalDistanceDtype(CudaDataType.CUDA_R_16F) - .withNProbes(nProbes) - .build(); - - CuVSIvfPqParams cuVSIvfPqParams = - new CuVSIvfPqParams.Builder() - .withCuVSIvfPqIndexParams(cuVSIvfPqIndexParams) - .withCuVSIvfPqSearchParams(cuVSIvfPqSearchParams) - .withRefinementRate(refinementRate) - .build(); - - return cuVSIvfPqParams; - } - - /* - * Rough translation from raft's internal utility found here: - * https://github.com/rapidsai/raft/blob/main/cpp/include/raft/util/integer_utils.hpp#L47-L56 - */ - private static long roundUpSafe(long numberToRound, long modulus) { - long remainder = numberToRound % modulus; - if (remainder == 0) { - return numberToRound; - } - long roundedUp = numberToRound - remainder + modulus; - return roundedUp; - } - - private static CagraIndexParams getNNDescentParams( - int graphDegree, - int intGraphDegree, - int writerThreads, - long nnDescentNumIterations, - CuvsDistanceType cuvsDistanceType) { - return new CagraIndexParams.Builder() - .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) - .withGraphDegree(graphDegree) - .withIntermediateGraphDegree(intGraphDegree) - .withNNDescentNumIterations(nnDescentNumIterations) - .withNumWriterThreads(writerThreads) - .withMetric(cuvsDistanceType) - .build(); - } - - private static CagraIndexParams getIVFPQParams( - int graphDegree, - int intGraphDegree, - int writerThreads, - long rows, - long dimension, - CuvsDistanceType cuvsDistanceType) { - return new CagraIndexParams.Builder() - .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.IVF_PQ) - .withCuVSIvfPqParams(getCuVSIvfPqParams(rows, dimension)) - .withNumWriterThreads(writerThreads) - .withIntermediateGraphDegree(intGraphDegree) - .withGraphDegree(graphDegree) - .withMetric(cuvsDistanceType) - .build(); - } - - /** - * Creates an instance of {@link CagraIndexParams} based on the chosen strategy in the {@link GPUSearchParams}. - * - * @param gPUSearchParams an instance of {@link GPUSearchParams} containing input params incoming via the build and search on the GPU API. - * @param rows number of vectors in the data set - * @param dimension the dimension of the vectors in the data set + * @param gpuSearchParams the input parameters for the build and search on the GPU API * @return an instance of {@link CagraIndexParams} */ - public static CagraIndexParams create( - GPUSearchParams gPUSearchParams, long rows, long dimension) { - if (gPUSearchParams.getStrategy().equals(GPUSearchParams.Strategy.HEURISTIC)) { - if (rows < ALGO_SWITCH_THRESHOLD) { - return getNNDescentParams( - gPUSearchParams.getGraphdegree(), - gPUSearchParams.getIntermediateGraphDegree(), - gPUSearchParams.getWriterThreads(), - gPUSearchParams.getnNDescentNumIterations(), - gPUSearchParams.getCuvsDistanceType()); - } else { - return getIVFPQParams( - gPUSearchParams.getGraphdegree(), - gPUSearchParams.getIntermediateGraphDegree(), - gPUSearchParams.getWriterThreads(), - rows, - dimension, - gPUSearchParams.getCuvsDistanceType()); - } + public static CagraIndexParams create(GPUSearchParams gpuSearchParams) { + CagraIndexParams.Builder builder = + new CagraIndexParams.Builder() + .withGraphDegree(gpuSearchParams.getGraphdegree()) + .withIntermediateGraphDegree(gpuSearchParams.getIntermediateGraphDegree()) + .withNumWriterThreads(gpuSearchParams.getWriterThreads()) + .withNNDescentNumIterations(gpuSearchParams.getnNDescentNumIterations()); + if (gpuSearchParams.getStrategy().equals(GPUSearchParams.Strategy.HEURISTIC)) { + builder + .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.AUTO_SELECT) + .withMetric(gpuSearchParams.getCuvsDistanceType()); } else { - return new CagraIndexParams.Builder() - .withNumWriterThreads(gPUSearchParams.getWriterThreads()) - .withIntermediateGraphDegree(gPUSearchParams.getIntermediateGraphDegree()) - .withGraphDegree(gPUSearchParams.getGraphdegree()) - .withCagraGraphBuildAlgo(gPUSearchParams.getCagraGraphBuildAlgo()) - .withCuVSIvfPqParams(gPUSearchParams.getCuVSIvfPqParams()) - .withNNDescentNumIterations(gPUSearchParams.getnNDescentNumIterations()) - .build(); + builder + .withCagraGraphBuildAlgo(gpuSearchParams.getCagraGraphBuildAlgo()) + .withCuVSIvfPqParams(gpuSearchParams.getCuVSIvfPqParams()); } + return builder.build(); } - /* - * Ideally there should be just one create method instead of two. - * We should do that when both the input parameter classes can be unified in the future. - */ - /** - * Creates an instance of {@link CagraIndexParams} based on the chosen strategy in the {@link AcceleratedHNSWParams}. + * Creates an instance of {@link CagraIndexParams} for the accelerated-HNSW index based on the + * chosen strategy in the {@link AcceleratedHNSWParams}. * - * @param acceleratedHNSWParams an instance of {@link AcceleratedHNSWParams} containing input params incoming via the build and search on the GPU API. + * @param acceleratedHNSWParams the input parameters for the build on the GPU API * @param rows number of vectors in the data set * @param dimension the dimension of the vectors in the data set * @return an instance of {@link CagraIndexParams} @@ -193,33 +57,36 @@ public static CagraIndexParams create( public static CagraIndexParams create( AcceleratedHNSWParams acceleratedHNSWParams, long rows, long dimension) { if (acceleratedHNSWParams.getStrategy().equals(AcceleratedHNSWParams.Strategy.HEURISTIC)) { - if (rows - < ALGO_SWITCH_THRESHOLD) { // TODO: maybe consider making this threshold configurable from - // outside later. - return getNNDescentParams( - acceleratedHNSWParams.getGraphdegree(), - acceleratedHNSWParams.getIntermediateGraphDegree(), - acceleratedHNSWParams.getWriterThreads(), - acceleratedHNSWParams.getNNDescentNumIterations(), - acceleratedHNSWParams.getCuvsDistanceType()); - } else { - return getIVFPQParams( - acceleratedHNSWParams.getGraphdegree(), - acceleratedHNSWParams.getIntermediateGraphDegree(), - acceleratedHNSWParams.getWriterThreads(), - rows, - dimension, - acceleratedHNSWParams.getCuvsDistanceType()); - } - } else { + // Delegate the derivation of the graph degrees, build algorithm and its parameters to cuVS, + // expressed in terms of the HNSW-equivalent maxConn/beamWidth. + CagraIndexParams derived = + CagraIndexParams.fromHnswParams( + rows, + dimension, + acceleratedHNSWParams.getMaxConn(), + acceleratedHNSWParams.getBeamWidth(), + AcceleratedHNSWParams.DEFAULT_HNSW_HEURISTIC_TYPE, + acceleratedHNSWParams.getCuvsDistanceType()); + // TODO: fromHnswParams has no writerThreads argument, so its result carries the cuVS default + // (not a heuristic value). We can rebuild the CagraIndexParams with the caller-supplied + // writerThreads for now but should fix this in cuVS in the future. return new CagraIndexParams.Builder() + .withGraphDegree(derived.getGraphDegree()) + .withIntermediateGraphDegree(derived.getIntermediateGraphDegree()) + .withCagraGraphBuildAlgo(derived.getCagraGraphBuildAlgo()) + .withCuVSIvfPqParams(derived.getCuVSIvfPqParams()) + .withNNDescentNumIterations(derived.getNNDescentNumIterations()) + .withMetric(derived.getCuvsDistanceType()) .withNumWriterThreads(acceleratedHNSWParams.getWriterThreads()) - .withIntermediateGraphDegree(acceleratedHNSWParams.getIntermediateGraphDegree()) - .withGraphDegree(acceleratedHNSWParams.getGraphdegree()) - .withCagraGraphBuildAlgo(acceleratedHNSWParams.getCagraGraphBuildAlgo()) - .withCuVSIvfPqParams(acceleratedHNSWParams.getCuVSIvfPqParams()) - .withNNDescentNumIterations(acceleratedHNSWParams.getNNDescentNumIterations()) .build(); } + return new CagraIndexParams.Builder() + .withNumWriterThreads(acceleratedHNSWParams.getWriterThreads()) + .withIntermediateGraphDegree(acceleratedHNSWParams.getIntermediateGraphDegree()) + .withGraphDegree(acceleratedHNSWParams.getGraphdegree()) + .withCagraGraphBuildAlgo(acceleratedHNSWParams.getCagraGraphBuildAlgo()) + .withCuVSIvfPqParams(acceleratedHNSWParams.getCuVSIvfPqParams()) + .withNNDescentNumIterations(acceleratedHNSWParams.getNNDescentNumIterations()) + .build(); } } diff --git a/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index e8fb304e..23911673 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -241,8 +241,7 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro * @throws Throwable */ private void writeCagraIndex(OutputStream os, CuVSMatrix dataset) throws Throwable { - CagraIndexParams params = - CagraIndexParamsFactory.create(gpuSearchParams, dataset.size(), dataset.columns()); + CagraIndexParams params = CagraIndexParamsFactory.create(gpuSearchParams); CagraIndex index = CagraIndex.newBuilder(getCuVSResourcesInstance()) .withDataset(dataset) diff --git a/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java b/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java index cc445268..46309a34 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java +++ b/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java @@ -16,15 +16,15 @@ public class GPUSearchParams { public static enum Strategy { /* - * This strategy allows for automatic selection of the underlining CAGRA build algorithm. - * With this strategy we use NN_DESCENT for data set less then 5M vectors else we use IVF_PQ. - * Indexing parameters, especially for IVF_PQ, are heuristically identified automatically. + * This strategy lets cuVS auto-select the CAGRA build algorithm (and its parameters) for the + * given dataset. * * This is the default and the recommended strategy. */ HEURISTIC, /* * This is an option when the end-user would want to use custom parameter values. + * * This strategy should only be used under expert guidance. */ CUSTOM @@ -77,8 +77,7 @@ public static enum Strategy { * @param cagraGraphBuildAlgo The CAGRA build algorithm to use. * @param indexType The type of index to build - CAGRA, BRUTEFORCE, or both. * @param cuVSIvfPqParams An instance of CuVSIvfPqParams containing IVF_PQ specific parameters. - * @param strategy either HEURISTIC [Default] that automatically chooses build algorithm and its parameters based on data set size or CUSTOM that uses the parameters passed though this class. - * @param heuristicType the heuristic type. The default option is SAME_GRAPH_FOOTPRINT. + * @param strategy either HEURISTIC [Default] that lets cuVS auto-select the build algorithm and its parameters or CUSTOM that uses the parameters passed though this class. * @param cuvsDistanceType the cuvsDistanceType. The default option is L2Expanded. * @param nnDescentNumIterations the number of Iterations to run if building with NN_DESCENT. */ diff --git a/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index 35209ce5..87d5e42a 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -170,6 +170,9 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro CuVSMatrix adjacencyListMatrix = cagraIndex.getGraph(); int size = (int) dataset.size(); int dimensions = fieldInfo.getVectorDimension(); + // The HNSW M is derived from the CAGRA graph degree, which is the adjacency list's column + // count (the degree cuVS actually built), used both below and when writing the meta. + int graphDegree = (int) adjacencyListMatrix.columns(); GPUBuiltHnswGraph hnswGraph = createMultiLayerHnswGraph( fieldInfo, @@ -178,7 +181,6 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro adjacencyListMatrix, vectors, acceleratedHNSWParams.getHnswLayers(), - acceleratedHNSWParams.getGraphdegree(), params, QuantizationType.NONE); long vectorIndexOffset = hnswVectorIndex.getFilePointer(); @@ -193,7 +195,7 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) thro size, hnswGraph, graphLevelNodeOffsets, - acceleratedHNSWParams.getGraphdegree()); + graphDegree); cagraIndex.close(); } catch (Throwable t) { Utils.handleThrowable(t); diff --git a/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java b/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java index d8a98cc2..80545468 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java +++ b/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java @@ -174,6 +174,9 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw CuVSMatrix adjacencyListMatrix = cagraIndex.getGraph(); int size = (int) dataset.size(); + // The CAGRA graph degree is the adjacency list's column count (the degree cuVS actually + // built); it is what the HNSW meta records as the basis for M. + int graphDegree = (int) adjacencyListMatrix.columns(); // Create multi-layer HNSW graph from CAGRA GPUBuiltHnswGraph hnswGraph = @@ -184,7 +187,6 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw adjacencyListMatrix, vectors, acceleratedHNSWParams.getHnswLayers(), - acceleratedHNSWParams.getGraphdegree(), params, QuantizationType.BINARY); @@ -203,7 +205,7 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throw size, hnswGraph, graphLevelNodeOffsets, - acceleratedHNSWParams.getGraphdegree()); + graphDegree); cagraIndex.close(); diff --git a/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java b/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java index 21b4be3f..436bedd2 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java +++ b/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java @@ -201,6 +201,9 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOE CuVSMatrix adjacencyListMatrix = cagraIndex.getGraph(); int size = (int) dataset.size(); + // The CAGRA graph degree is the adjacency list's column count (the degree cuVS actually + // built); it is what the HNSW meta records as the basis for M. + int graphDegree = (int) adjacencyListMatrix.columns(); GPUBuiltHnswGraph hnswGraph = createMultiLayerHnswGraph( fieldInfo, @@ -209,7 +212,6 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOE adjacencyListMatrix, unsignedVectors, acceleratedHNSWParams.getHnswLayers(), - acceleratedHNSWParams.getGraphdegree(), params, QuantizationType.SCALAR); @@ -230,7 +232,7 @@ private void writeFieldInternal(FieldInfo fieldInfo, List vectors) throws IOE size, hnswGraph, graphLevelNodeOffsets, - acceleratedHNSWParams.getGraphdegree()); + graphDegree); cagraIndex.close(); } catch (Throwable t) { diff --git a/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWOddGraphDegree.java b/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWOddGraphDegree.java new file mode 100644 index 00000000..7e1bce90 --- /dev/null +++ b/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWOddGraphDegree.java @@ -0,0 +1,111 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.TestUtils.generateDataset; +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; +import static org.apache.lucene.index.VectorSimilarityFunction.EUCLIDEAN; + +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Random; +import java.util.UUID; +import org.apache.commons.io.FileUtils; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.KnnFloatVectorField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.KnnFloatVectorQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +/** + * Regression test for the CAGRA-to-HNSW conversion when the CAGRA graph has an odd graph + * degree. + * + *

The HNSW max-connections parameter is derived from the CAGRA graph degree as {@code M = + * ceil(degree / 2)}, and Lucene's HNSW format allows up to {@code 2 * M} neighbors at level 0. With + * a plain integer {@code degree / 2}, an odd degree {@code d} yields {@code 2 * (d / 2) = d - 1 < d} + * — one fewer than the number of neighbors each node actually has — which makes the writer emit a + * graph the reader rejects with "too many neighbors: d". cuVS produces odd graph degrees when it + * clamps the degree for small datasets; here we force one deterministically via the CUSTOM strategy. + */ +@SuppressSysoutChecks(bugUrl = "") +public class TestAcceleratedHNSWOddGraphDegree extends LuceneTestCase { + + private static final String VECTOR_FIELD = "vector_field"; + private static final int ODD_GRAPH_DEGREE = 63; + + private Random random; + private Path indexDirPath; + + @Before + public void beforeTest() { + assumeTrue("cuVS not supported", isSupported()); + random = new Random(222); + indexDirPath = Paths.get(UUID.randomUUID().toString()); + } + + @Test + public void testOddGraphDegreeIndexesAndSearches() throws Exception { + // CUSTOM strategy passes the graph degree straight through to cuVS, so the built CAGRA graph + // (and thus its adjacency list's column count) has an odd degree. + AcceleratedHNSWParams params = + new AcceleratedHNSWParams.Builder() + .withStrategy(AcceleratedHNSWParams.Strategy.CUSTOM) + .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) + .withIntermediateGraphDegree(128) + .withGraphDegree(ODD_GRAPH_DEGREE) + .build(); + + Codec codec = new Lucene101AcceleratedHNSWCodec(params); + IndexWriterConfig config = new IndexWriterConfig().setCodec(codec).setUseCompoundFile(false); + + int numDocs = 1000; + int dimension = 32; + int topK = 10; + float[][] dataset = generateDataset(random, numDocs, dimension); + + try (Directory indexDirectory = FSDirectory.open(indexDirPath)) { + // Indexing (flush of the HNSW graph is where an odd degree would trip "too many neighbors"). + try (IndexWriter indexWriter = new IndexWriter(indexDirectory, config)) { + for (int i = 0; i < numDocs; i++) { + Document document = new Document(); + document.add(new KnnFloatVectorField(VECTOR_FIELD, dataset[i], EUCLIDEAN)); + indexWriter.addDocument(document); + } + indexWriter.commit(); + } + + // Searching (the Lucene HNSW reader validates neighbor counts against the stored M). + try (DirectoryReader reader = DirectoryReader.open(indexDirectory)) { + assertEquals(numDocs, reader.numDocs()); + IndexSearcher searcher = new IndexSearcher(reader); + float[] queryVector = generateDataset(random, 1, dimension)[0]; + TopDocs results = + searcher.search(new KnnFloatVectorQuery(VECTOR_FIELD, queryVector, topK), topK); + assertEquals(topK, results.scoreDocs.length); + } + } + } + + @After + public void afterTest() throws Exception { + var dir = indexDirPath.toFile(); + if (dir.exists() && dir.isDirectory()) { + FileUtils.deleteDirectory(dir); + } + } +} diff --git a/src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java b/src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java new file mode 100644 index 00000000..3f4f6fe8 --- /dev/null +++ b/src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java @@ -0,0 +1,97 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package com.nvidia.cuvs.lucene; + +import static com.nvidia.cuvs.lucene.ThreadLocalCuVSResourcesProvider.isSupported; + +import com.nvidia.cuvs.CagraIndexParams; +import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; +import com.nvidia.cuvs.CagraIndexParams.CuvsDistanceType; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.lucene.tests.util.LuceneTestCase.SuppressSysoutChecks; +import org.junit.Test; + +/** + * Verifies that {@link CagraIndexParamsFactory} keeps cuVS as the source of truth for the build + * heuristics: the GPU-native path defers the build-algorithm choice to cuVS via {@link + * CagraGraphBuildAlgo#AUTO_SELECT}, and the accelerated-HNSW path defers to cuVS' + * {@code fromHnswParams} heuristic. + */ +@SuppressSysoutChecks(bugUrl = "") +public class TestCagraIndexParamsFactory extends LuceneTestCase { + + /** + * The GPU-native HEURISTIC path hands the build-algorithm decision to cuVS (AUTO_SELECT) while + * keeping the caller-supplied CAGRA-native graph degrees and metric. Pure Java, no GPU needed. + */ + @Test + public void testGpuHeuristicUsesAutoSelect() { + GPUSearchParams params = + new GPUSearchParams.Builder() + .withStrategy(GPUSearchParams.Strategy.HEURISTIC) + .withGraphDegree(48) + .withIntermediateGraphDegree(96) + .withCuvsDistanceType(CuvsDistanceType.InnerProduct) + .withWriterThreads(4) + .build(); + + CagraIndexParams cagraParams = CagraIndexParamsFactory.create(params); + + assertEquals(CagraGraphBuildAlgo.AUTO_SELECT, cagraParams.getCagraGraphBuildAlgo()); + assertEquals(48, cagraParams.getGraphDegree()); + assertEquals(96, cagraParams.getIntermediateGraphDegree()); + assertEquals(4, cagraParams.getNumWriterThreads()); + assertEquals(CuvsDistanceType.InnerProduct, cagraParams.getCuvsDistanceType()); + } + + /** + * The GPU-native CUSTOM path passes the explicitly configured build parameters straight through. + * Pure Java, no GPU needed. + */ + @Test + public void testGpuCustomStrategyPassesThroughValues() { + GPUSearchParams params = + new GPUSearchParams.Builder() + .withStrategy(GPUSearchParams.Strategy.CUSTOM) + .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) + .withGraphDegree(32) + .withIntermediateGraphDegree(64) + .build(); + + CagraIndexParams cagraParams = CagraIndexParamsFactory.create(params); + + assertEquals(CagraGraphBuildAlgo.NN_DESCENT, cagraParams.getCagraGraphBuildAlgo()); + assertEquals(32, cagraParams.getGraphDegree()); + assertEquals(64, cagraParams.getIntermediateGraphDegree()); + assertNotNull(cagraParams.getCuVSIvfPqParams()); + } + + /** + * The accelerated-HNSW HEURISTIC path delegates to cuVS' native {@code fromHnswParams}, which + * derives the graph degrees from maxConn/beamWidth, and re-attaches the caller's writerThreads + * (which fromHnswParams itself cannot carry). Requires the native cuVS library. + */ + @Test + public void testHnswHeuristicDelegatesToCuVS() { + assumeTrue("cuVS not supported", isSupported()); + + AcceleratedHNSWParams params = + new AcceleratedHNSWParams.Builder() + .withStrategy(AcceleratedHNSWParams.Strategy.HEURISTIC) + .withMaxConn(16) + .withBeamWidth(100) + .withWriterThreads(7) + .build(); + + CagraIndexParams cagraParams = CagraIndexParamsFactory.create(params, 10_000, 128); + + // SAME_GRAPH_FOOTPRINT yields graph_degree = 2 * maxConn; cuVS owns the exact derivation, so we + // assert the footprint relationship it documents rather than a hardcoded value. + assertEquals(2L * params.getMaxConn(), cagraParams.getGraphDegree()); + // The caller-configured writerThreads must be honored on the HEURISTIC path. + assertEquals(7, cagraParams.getNumWriterThreads()); + } +} From 98cea184c5ef92eee9a176b68975b370d1047d5b Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 20 Jul 2026 09:31:43 -1000 Subject: [PATCH 02/29] Fix checkstyle --- src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java | 2 +- .../java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java | 2 +- src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java | 2 +- .../cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java | 2 +- .../LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java | 2 +- .../LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java | 2 +- .../nvidia/cuvs/lucene/TestAcceleratedHNSWOddGraphDegree.java | 2 +- .../com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java b/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java index d59a39d6..cb13f7af 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java +++ b/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWUtils.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java b/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java index 23911673..da5d0a60 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java +++ b/src/main/java/com/nvidia/cuvs/lucene/CuVS2510GPUVectorsWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java b/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java index 46309a34..fcb2cd38 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java +++ b/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ diff --git a/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java b/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java index 87d5e42a..b57dd6e2 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java +++ b/src/main/java/com/nvidia/cuvs/lucene/Lucene99AcceleratedHNSWVectorsWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java b/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java index 80545468..2cf5330a 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java +++ b/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWBinaryQuantizedVectorsWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java b/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java index 436bedd2..e25ef4af 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java +++ b/src/main/java/com/nvidia/cuvs/lucene/LuceneAcceleratedHNSWScalarQuantizedVectorsWriter.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWOddGraphDegree.java b/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWOddGraphDegree.java index 7e1bce90..ce56ca9a 100644 --- a/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWOddGraphDegree.java +++ b/src/test/java/com/nvidia/cuvs/lucene/TestAcceleratedHNSWOddGraphDegree.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; diff --git a/src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java b/src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java index 3f4f6fe8..2a093a89 100644 --- a/src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java +++ b/src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. + * SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ From cd9eee9eb20fc1083ef8953f14957a28c46ac49c Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 23 Jul 2026 14:08:49 -1000 Subject: [PATCH 03/29] Fix typo --- src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java | 2 +- src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java b/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java index a4764288..937064a1 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java +++ b/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java @@ -104,7 +104,7 @@ public static enum Strategy { * @param cuVSIvfPqParams An instance of CuVSIvfPqParams containing IVF_PQ specific parameters. * @param numMergeWorkers The number of merge workers to use with the fallback mechanism. * @param mergeExec The instance of {@link ExecutorService} to use with the fallback mechanism. - * @param strategy either HEURISTIC [Default] that delegates the CAGRA build parameters to cuVS (derived from the HNSW-equivalent maxConn and beamWidth) or CUSTOM that uses the parameters passed though this class. + * @param strategy either HEURISTIC [Default] that delegates the CAGRA build parameters to cuVS (derived from the HNSW-equivalent maxConn and beamWidth) or CUSTOM that uses the parameters passed through this class. * @param cuvsDistanceType the cuvsDistanceType. The default option is L2Expanded. * @param nnDescentNumIterations the number of Iterations to run if building with NN_DESCENT. */ diff --git a/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java b/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java index fcb2cd38..f035784d 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java +++ b/src/main/java/com/nvidia/cuvs/lucene/GPUSearchParams.java @@ -77,7 +77,7 @@ public static enum Strategy { * @param cagraGraphBuildAlgo The CAGRA build algorithm to use. * @param indexType The type of index to build - CAGRA, BRUTEFORCE, or both. * @param cuVSIvfPqParams An instance of CuVSIvfPqParams containing IVF_PQ specific parameters. - * @param strategy either HEURISTIC [Default] that lets cuVS auto-select the build algorithm and its parameters or CUSTOM that uses the parameters passed though this class. + * @param strategy either HEURISTIC [Default] that lets cuVS auto-select the build algorithm and its parameters or CUSTOM that uses the parameters passed through this class. * @param cuvsDistanceType the cuvsDistanceType. The default option is L2Expanded. * @param nnDescentNumIterations the number of Iterations to run if building with NN_DESCENT. */ From 9c57e39db2f2a38d5cd7a5e7d40cc99660b7b429 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 23 Jul 2026 14:10:42 -1000 Subject: [PATCH 04/29] Remove redundant NN_DESCENT parameter --- .../nvidia/cuvs/lucene/CagraIndexParamsFactory.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java b/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java index 648362b5..7d4c49b4 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java +++ b/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java @@ -31,16 +31,20 @@ public static CagraIndexParams create(GPUSearchParams gpuSearchParams) { new CagraIndexParams.Builder() .withGraphDegree(gpuSearchParams.getGraphdegree()) .withIntermediateGraphDegree(gpuSearchParams.getIntermediateGraphDegree()) - .withNumWriterThreads(gpuSearchParams.getWriterThreads()) - .withNNDescentNumIterations(gpuSearchParams.getnNDescentNumIterations()); + .withNumWriterThreads(gpuSearchParams.getWriterThreads()); if (gpuSearchParams.getStrategy().equals(GPUSearchParams.Strategy.HEURISTIC)) { + // AUTO_SELECT: cuVS picks the build algorithm and derives its parameters at build time, so + // the IVF-PQ params and nn-descent iterations are left to cuVS rather than forwarded here. builder .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.AUTO_SELECT) .withMetric(gpuSearchParams.getCuvsDistanceType()); } else { + // CUSTOM: forward the caller's algorithm and the parameters it consumes -- IVF-PQ params for + // IVF_PQ, nn-descent iterations for NN_DESCENT (each is ignored by the other algorithm). builder .withCagraGraphBuildAlgo(gpuSearchParams.getCagraGraphBuildAlgo()) - .withCuVSIvfPqParams(gpuSearchParams.getCuVSIvfPqParams()); + .withCuVSIvfPqParams(gpuSearchParams.getCuVSIvfPqParams()) + .withNNDescentNumIterations(gpuSearchParams.getnNDescentNumIterations()); } return builder.build(); } From d6b66df3fbef540285d4a68f50ddca3508495514 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Thu, 23 Jul 2026 14:42:28 -1000 Subject: [PATCH 05/29] Add test for the writer threads with custom strategy --- .../com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java b/src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java index 2a093a89..c949a0e3 100644 --- a/src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java +++ b/src/test/java/com/nvidia/cuvs/lucene/TestCagraIndexParamsFactory.java @@ -59,6 +59,7 @@ public void testGpuCustomStrategyPassesThroughValues() { .withCagraGraphBuildAlgo(CagraGraphBuildAlgo.NN_DESCENT) .withGraphDegree(32) .withIntermediateGraphDegree(64) + .withWriterThreads(12) .build(); CagraIndexParams cagraParams = CagraIndexParamsFactory.create(params); @@ -67,6 +68,8 @@ public void testGpuCustomStrategyPassesThroughValues() { assertEquals(32, cagraParams.getGraphDegree()); assertEquals(64, cagraParams.getIntermediateGraphDegree()); assertNotNull(cagraParams.getCuVSIvfPqParams()); + // The caller-configured writerThreads must be honored on the CUSTOM path. + assertEquals(12, cagraParams.getNumWriterThreads()); } /** From 2b5566823fbbe65841871ce2deb581e029be3032 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 24 Jul 2026 10:21:21 -1000 Subject: [PATCH 06/29] Temporary switch to use NVIDIA/cuvs#2345 Switches to build against NVIDIA/cuvs#2345 --- RAPIDS_BRANCH | 2 +- .../java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/RAPIDS_BRANCH b/RAPIDS_BRANCH index 8ce6b738..30897fb7 100644 --- a/RAPIDS_BRANCH +++ b/RAPIDS_BRANCH @@ -1 +1 @@ -release/26.08 +pull-request/2345 diff --git a/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index 07e64fb1..bb47cebd 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -165,4 +165,10 @@ public HnswIndex hnswIndexBuild(CuVSResources arg0, HnswIndexParams arg1, CuVSMa throws Throwable { return delegate.hnswIndexBuild(arg0, arg1, arg2); } + + @Override + public CagraIndexParams cagraIndexParamsFromDataset( + long arg0, long arg1, long arg2, CagraIndexParams.CuvsDistanceType arg3, long arg4) { + return delegate.cagraIndexParamsFromDataset(arg0, arg1, arg2, arg3, arg4); + } } From 692229eb074f4f590baa7d00c08a2def644b8e63 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 24 Jul 2026 10:26:51 -1000 Subject: [PATCH 07/29] Fix copyright --- src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java b/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java index bb47cebd..f5e904ec 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java +++ b/src/main/java/com/nvidia/cuvs/lucene/FilterCuVSProvider.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs.lucene; From b0e170f7507cfe6abaa6d48d73377d11856d2aac Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 24 Jul 2026 10:42:39 -1000 Subject: [PATCH 08/29] Bump the build From c64ac37b95b897b316fd6c3e34c2c337a2e0814e Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 24 Jul 2026 11:20:46 -1000 Subject: [PATCH 09/29] Test Build --- build.sh | 47 +++++++++++++++++++++++++++++++++++++++++++++-- ci/build_java.sh | 5 +++++ 2 files changed, 50 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 4ec8c087..7c40f207 100755 --- a/build.sh +++ b/build.sh @@ -15,21 +15,64 @@ function hasArg { (( NUMARGS != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") } +# Checks whether the checked out cuvs branch modifies the C/C++ sources of libcuvs +# with respect to the branch it is based on. Must be invoked from within the cuvs +# clone. +function cuvsModifiesCpp { + local base_ref merge_base + + # The branch a pull request is based on cannot be derived from the pull request + # ref itself, so fall back on the release branch matching the cuvs version and, + # if that one does not exist yet, on the default branch of the repository. + base_ref="${CUVS_BASE_REF:-}" + if [[ -z "$base_ref" ]]; then + base_ref="release/$(sed -E 's/^([0-9]+)\.([0-9]+).*$/\1.\2/' VERSION)" + if ! git rev-parse --verify --quiet "origin/$base_ref" > /dev/null; then + base_ref=$(git symbolic-ref --short refs/remotes/origin/HEAD) + base_ref="${base_ref#origin/}" + fi + fi + + if ! merge_base=$(git merge-base "origin/$base_ref" HEAD 2> /dev/null); then + echo "Could not determine the merge base with 'origin/$base_ref', assuming libcuvs was modified." + return 0 + fi + + echo "Comparing against 'origin/$base_ref' to detect changes in the cuvs C/C++ sources." + git diff --name-only "$merge_base" HEAD -- cpp/ \ + | grep -qE '(\.(c|cc|cpp|cxx|cu|cuh|h|hpp|hxx|cmake)|CMakeLists\.txt)$' +} + if hasArg --build-cuvs-java; then CUVS_WORKDIR="cuvs-workdir" CUVS_GIT_REPO="https://github.com/rapidsai/cuvs.git" + BRANCH=$(cat "RAPIDS_BRANCH") if [[ -d "$CUVS_WORKDIR" && -n "$(ls -A "$CUVS_WORKDIR")" ]]; then echo "Directory '$CUVS_WORKDIR' exists and is not empty." pushd $CUVS_WORKDIR git pull else - BRANCH=$(cat "RAPIDS_BRANCH") echo "Directory '$CUVS_WORKDIR' does not exist or is empty. Cloning the cuvs's '$BRANCH' branch." # Correct branch selection is crucial to avoid version mismatch issues when testing. git clone --branch "$BRANCH" $CUVS_GIT_REPO $CUVS_WORKDIR pushd $CUVS_WORKDIR fi - ./build.sh java + + # libcuvs comes from the conda packages, so normally only the java bindings have + # to be built. A pull request that changes the C/C++ sources is not part of those + # packages though, and the bindings would be built and tested against a stale + # library, so CI asks for libcuvs to be built from source in that case. + CUVS_BUILD_TARGETS=("java") + if hasArg --build-libcuvs-if-changed && [[ "$BRANCH" == pull-request/* ]] && cuvsModifiesCpp; then + echo "Branch '$BRANCH' modifies the cuvs C/C++ sources, building libcuvs from source as well." + CUVS_BUILD_TARGETS=("libcuvs" "${CUVS_BUILD_TARGETS[@]}") + # The libraries that were just built have to take precedence over the ones + # provided by the conda packages, both here and while running the java tests. + LD_LIBRARY_PATH="$PWD/cpp/build${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + export LD_LIBRARY_PATH + echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH" + fi + ./build.sh "${CUVS_BUILD_TARGETS[@]}" popd fi diff --git a/ci/build_java.sh b/ci/build_java.sh index 173ca55c..c929ca0e 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -16,6 +16,11 @@ fi # Always build cuvs-java when running the pipeline EXTRA_BUILD_ARGS+=("--build-cuvs-java") +# When RAPIDS_BRANCH points at a cuvs pull request that changes the C/C++ sources, +# the libcuvs from the conda packages does not contain those changes, so build it +# from source as well. Only done in CI, local builds keep using the conda packages. +EXTRA_BUILD_ARGS+=("--build-libcuvs-if-changed") + # shellcheck disable=SC1091 . /opt/conda/etc/profile.d/conda.sh From 87e16acae38af73e0d3f71bd5993c2d897aefcc7 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 24 Jul 2026 11:23:40 -1000 Subject: [PATCH 10/29] Copyright fix --- build.sh | 2 +- ci/build_java.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 7c40f207..c9f5e4e4 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -e -u -o pipefail diff --git a/ci/build_java.sh b/ci/build_java.sh index c929ca0e..aadbe809 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail From 64e519e144ffcd4664212065809cbb193530827c Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Jul 2026 13:37:03 -1000 Subject: [PATCH 11/29] Revert "Copyright fix" This reverts commit 87e16acae38af73e0d3f71bd5993c2d897aefcc7. --- build.sh | 2 +- ci/build_java.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index c9f5e4e4..7c40f207 100755 --- a/build.sh +++ b/build.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -e -u -o pipefail diff --git a/ci/build_java.sh b/ci/build_java.sh index aadbe809..c929ca0e 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -1,6 +1,6 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION. # SPDX-License-Identifier: Apache-2.0 set -euo pipefail From 7a22b1ebfe41a6a3ab8e844d3ac363a9c6d461e8 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Jul 2026 13:37:05 -1000 Subject: [PATCH 12/29] Revert "Test Build" This reverts commit c64ac37b95b897b316fd6c3e34c2c337a2e0814e. --- build.sh | 47 ++--------------------------------------------- ci/build_java.sh | 5 ----- 2 files changed, 2 insertions(+), 50 deletions(-) diff --git a/build.sh b/build.sh index 7c40f207..4ec8c087 100755 --- a/build.sh +++ b/build.sh @@ -15,64 +15,21 @@ function hasArg { (( NUMARGS != 0 )) && (echo " ${ARGS} " | grep -q " $1 ") } -# Checks whether the checked out cuvs branch modifies the C/C++ sources of libcuvs -# with respect to the branch it is based on. Must be invoked from within the cuvs -# clone. -function cuvsModifiesCpp { - local base_ref merge_base - - # The branch a pull request is based on cannot be derived from the pull request - # ref itself, so fall back on the release branch matching the cuvs version and, - # if that one does not exist yet, on the default branch of the repository. - base_ref="${CUVS_BASE_REF:-}" - if [[ -z "$base_ref" ]]; then - base_ref="release/$(sed -E 's/^([0-9]+)\.([0-9]+).*$/\1.\2/' VERSION)" - if ! git rev-parse --verify --quiet "origin/$base_ref" > /dev/null; then - base_ref=$(git symbolic-ref --short refs/remotes/origin/HEAD) - base_ref="${base_ref#origin/}" - fi - fi - - if ! merge_base=$(git merge-base "origin/$base_ref" HEAD 2> /dev/null); then - echo "Could not determine the merge base with 'origin/$base_ref', assuming libcuvs was modified." - return 0 - fi - - echo "Comparing against 'origin/$base_ref' to detect changes in the cuvs C/C++ sources." - git diff --name-only "$merge_base" HEAD -- cpp/ \ - | grep -qE '(\.(c|cc|cpp|cxx|cu|cuh|h|hpp|hxx|cmake)|CMakeLists\.txt)$' -} - if hasArg --build-cuvs-java; then CUVS_WORKDIR="cuvs-workdir" CUVS_GIT_REPO="https://github.com/rapidsai/cuvs.git" - BRANCH=$(cat "RAPIDS_BRANCH") if [[ -d "$CUVS_WORKDIR" && -n "$(ls -A "$CUVS_WORKDIR")" ]]; then echo "Directory '$CUVS_WORKDIR' exists and is not empty." pushd $CUVS_WORKDIR git pull else + BRANCH=$(cat "RAPIDS_BRANCH") echo "Directory '$CUVS_WORKDIR' does not exist or is empty. Cloning the cuvs's '$BRANCH' branch." # Correct branch selection is crucial to avoid version mismatch issues when testing. git clone --branch "$BRANCH" $CUVS_GIT_REPO $CUVS_WORKDIR pushd $CUVS_WORKDIR fi - - # libcuvs comes from the conda packages, so normally only the java bindings have - # to be built. A pull request that changes the C/C++ sources is not part of those - # packages though, and the bindings would be built and tested against a stale - # library, so CI asks for libcuvs to be built from source in that case. - CUVS_BUILD_TARGETS=("java") - if hasArg --build-libcuvs-if-changed && [[ "$BRANCH" == pull-request/* ]] && cuvsModifiesCpp; then - echo "Branch '$BRANCH' modifies the cuvs C/C++ sources, building libcuvs from source as well." - CUVS_BUILD_TARGETS=("libcuvs" "${CUVS_BUILD_TARGETS[@]}") - # The libraries that were just built have to take precedence over the ones - # provided by the conda packages, both here and while running the java tests. - LD_LIBRARY_PATH="$PWD/cpp/build${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - export LD_LIBRARY_PATH - echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH" - fi - ./build.sh "${CUVS_BUILD_TARGETS[@]}" + ./build.sh java popd fi diff --git a/ci/build_java.sh b/ci/build_java.sh index c929ca0e..173ca55c 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -16,11 +16,6 @@ fi # Always build cuvs-java when running the pipeline EXTRA_BUILD_ARGS+=("--build-cuvs-java") -# When RAPIDS_BRANCH points at a cuvs pull request that changes the C/C++ sources, -# the libcuvs from the conda packages does not contain those changes, so build it -# from source as well. Only done in CI, local builds keep using the conda packages. -EXTRA_BUILD_ARGS+=("--build-libcuvs-if-changed") - # shellcheck disable=SC1091 . /opt/conda/etc/profile.d/conda.sh From e8167e2a30d905db25766ae0a6d62009a9248844 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Jul 2026 15:49:28 -1000 Subject: [PATCH 13/29] Switch from fromHnswParams to fromDataset --- .../cuvs/lucene/AcceleratedHNSWParams.java | 3 --- .../cuvs/lucene/CagraIndexParamsFactory.java | 20 ++++++++++--------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java b/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java index 937064a1..8d511fef 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java +++ b/src/main/java/com/nvidia/cuvs/lucene/AcceleratedHNSWParams.java @@ -7,7 +7,6 @@ import com.nvidia.cuvs.CagraIndexParams.CagraGraphBuildAlgo; import com.nvidia.cuvs.CagraIndexParams.CuvsDistanceType; -import com.nvidia.cuvs.CagraIndexParams.HnswHeuristicType; import com.nvidia.cuvs.CuVSIvfPqParams; import java.util.Objects; import java.util.concurrent.ExecutorService; @@ -64,8 +63,6 @@ public static enum Strategy { public static final Strategy DEFAULT_STRATEGY = Strategy.HEURISTIC; public static final CuvsDistanceType DEFAULT_CUVS_DISTANCE_TYPE = CuvsDistanceType.L2Expanded; public static final int DEFAULT_NN_DESCENT_NUM_ITERATIONS = 20; - public static final HnswHeuristicType DEFAULT_HNSW_HEURISTIC_TYPE = - HnswHeuristicType.SAME_GRAPH_FOOTPRINT; public static final Supplier DEFAULT_IVF_PQ_PARAMS = () -> { diff --git a/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java b/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java index 7d4c49b4..376cc955 100644 --- a/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java +++ b/src/main/java/com/nvidia/cuvs/lucene/CagraIndexParamsFactory.java @@ -61,18 +61,20 @@ public static CagraIndexParams create(GPUSearchParams gpuSearchParams) { public static CagraIndexParams create( AcceleratedHNSWParams acceleratedHNSWParams, long rows, long dimension) { if (acceleratedHNSWParams.getStrategy().equals(AcceleratedHNSWParams.Strategy.HEURISTIC)) { - // Delegate the derivation of the graph degrees, build algorithm and its parameters to cuVS, - // expressed in terms of the HNSW-equivalent maxConn/beamWidth. + // Delegate the CAGRA build heuristic (algorithm choice and its parameters) to cuVS via the + // CAGRA-native fromDataset API. The codec's HNSW maxConn maps to a CAGRA graph degree of + // 2 * maxConn (so the resulting HNSW M = ceil(graphDegree / 2) == maxConn), and beamWidth to + // build_quality = beamWidth / 16. These reproduce what fromHnswParams(SAME_GRAPH_FOOTPRINT) + // produced before, so the built graph is unchanged. CagraIndexParams derived = - CagraIndexParams.fromHnswParams( + CagraIndexParams.fromDataset( rows, dimension, - acceleratedHNSWParams.getMaxConn(), - acceleratedHNSWParams.getBeamWidth(), - AcceleratedHNSWParams.DEFAULT_HNSW_HEURISTIC_TYPE, - acceleratedHNSWParams.getCuvsDistanceType()); - // TODO: fromHnswParams has no writerThreads argument, so its result carries the cuVS default - // (not a heuristic value). We can rebuild the CagraIndexParams with the caller-supplied + 2L * acceleratedHNSWParams.getMaxConn(), + acceleratedHNSWParams.getCuvsDistanceType(), + acceleratedHNSWParams.getBeamWidth() / 16); + // TODO: fromDataset has no writerThreads argument, so its result carries the cuVS default + // (not a heuristic value). We rebuild the CagraIndexParams with the caller-supplied // writerThreads for now but should fix this in cuVS in the future. return new CagraIndexParams.Builder() .withGraphDegree(derived.getGraphDegree()) From 1b0fd04ef22be80d84626002b7c577351a0709cc Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Jul 2026 16:32:13 -1000 Subject: [PATCH 14/29] Test panama binding theory --- build.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/build.sh b/build.sh index 39322568..d8451f4d 100755 --- a/build.sh +++ b/build.sh @@ -45,6 +45,20 @@ if hasArg --build-cuvs-java; then LD_LIBRARY_PATH="$LIBCUVS_DIR/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" export LD_LIBRARY_PATH echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH" + + # The conda-installed libcuvs headers may not include changes from the PR, so + # jextract would generate stale Panama bindings. Wrap generate-bindings.sh to + # overwrite the cmake-copied conda headers with the PR's source C headers before + # jextract runs. + mv java/panama-bindings/generate-bindings.sh java/panama-bindings/generate-bindings-orig.sh + cat > java/panama-bindings/generate-bindings.sh << 'EOF' +#!/bin/bash +SCRIPTDIR=$(cd "$(dirname "$0")"; pwd) +REPODIR=$(cd "$SCRIPTDIR/../.."; pwd) +cp -r "${REPODIR}/c/include/cuvs/" "${REPODIR}/java/internal/build/bindings/include/cuvs/" +exec "${SCRIPTDIR}/generate-bindings-orig.sh" "$@" +EOF + chmod +x java/panama-bindings/generate-bindings.sh fi ./build.sh "${CUVS_BUILD_TARGETS[@]}" popd From 3e7c48d04c39a5c1d315164a04c6e1047996e677 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Jul 2026 16:49:43 -1000 Subject: [PATCH 15/29] Test another panama approach --- build.sh | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/build.sh b/build.sh index d8451f4d..b39c545d 100755 --- a/build.sh +++ b/build.sh @@ -45,20 +45,10 @@ if hasArg --build-cuvs-java; then LD_LIBRARY_PATH="$LIBCUVS_DIR/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" export LD_LIBRARY_PATH echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH" - - # The conda-installed libcuvs headers may not include changes from the PR, so - # jextract would generate stale Panama bindings. Wrap generate-bindings.sh to - # overwrite the cmake-copied conda headers with the PR's source C headers before - # jextract runs. - mv java/panama-bindings/generate-bindings.sh java/panama-bindings/generate-bindings-orig.sh - cat > java/panama-bindings/generate-bindings.sh << 'EOF' -#!/bin/bash -SCRIPTDIR=$(cd "$(dirname "$0")"; pwd) -REPODIR=$(cd "$SCRIPTDIR/../.."; pwd) -cp -r "${REPODIR}/c/include/cuvs/" "${REPODIR}/java/internal/build/bindings/include/cuvs/" -exec "${SCRIPTDIR}/generate-bindings-orig.sh" "$@" -EOF - chmod +x java/panama-bindings/generate-bindings.sh + # Point cmake to the PR artifact so find_package(cuvs) picks up its headers + # instead of the conda-installed ones, ensuring jextract generates correct + # Panama bindings for any new C API in the PR. + export CMAKE_PREFIX_PATH="$LIBCUVS_DIR" fi ./build.sh "${CUVS_BUILD_TARGETS[@]}" popd From 8e6ebe3e70eccf9b92784810fd512b4fd51cb913 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Jul 2026 17:08:50 -1000 Subject: [PATCH 16/29] Debug: log PR artifact contents --- build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.sh b/build.sh index b39c545d..3814a348 100755 --- a/build.sh +++ b/build.sh @@ -45,6 +45,8 @@ if hasArg --build-cuvs-java; then LD_LIBRARY_PATH="$LIBCUVS_DIR/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" export LD_LIBRARY_PATH echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH" + echo "PR artifact contents:" + find "$LIBCUVS_DIR" -name "libcuvs*" # Point cmake to the PR artifact so find_package(cuvs) picks up its headers # instead of the conda-installed ones, ensuring jextract generates correct # Panama bindings for any new C API in the PR. From d43cd9e6f662e4bade68412681057750f1a6c3e1 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Jul 2026 18:40:10 -1000 Subject: [PATCH 17/29] Debug: check PR libcuvs_c.so for new symbol --- build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/build.sh b/build.sh index 3814a348..d9eb0748 100755 --- a/build.sh +++ b/build.sh @@ -47,6 +47,8 @@ if hasArg --build-cuvs-java; then echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH" echo "PR artifact contents:" find "$LIBCUVS_DIR" -name "libcuvs*" + echo "cuvsCagraIndexParamsFromDataset in PR libcuvs_c.so:" + nm -D "$LIBCUVS_DIR/lib/libcuvs_c.so" 2>/dev/null | grep cuvsCagraIndexParamsFromDataset || echo "NOT FOUND" # Point cmake to the PR artifact so find_package(cuvs) picks up its headers # instead of the conda-installed ones, ensuring jextract generates correct # Panama bindings for any new C API in the PR. From ca62402cbf0feb5b9ecaffc5ee2e00e4b677ba5a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Jul 2026 18:41:59 -1000 Subject: [PATCH 18/29] Remove conda libcuvs from LD_LIBRARY_PATH when using PR artifact --- build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.sh b/build.sh index d9eb0748..7d328db4 100755 --- a/build.sh +++ b/build.sh @@ -43,6 +43,9 @@ if hasArg --build-cuvs-java; then # The downloaded library has to take precedence over the one provided by the # conda packages, both here and while running the java tests. LD_LIBRARY_PATH="$LIBCUVS_DIR/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + # Remove the conda-cached libcuvs from LD_LIBRARY_PATH so it cannot + # shadow the PR artifact's libcuvs_c.so at runtime. + LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v "/pkgs/.*libcuvs" | paste -sd ':') export LD_LIBRARY_PATH echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH" echo "PR artifact contents:" From 6132d5fd5c5640e6e2f125fdbf8216bc4f48dea2 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Jul 2026 19:02:25 -1000 Subject: [PATCH 19/29] Uninstall conda libcuvs when using PR artifact to avoid ldconfig shadowing --- build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.sh b/build.sh index 7d328db4..71e44485 100755 --- a/build.sh +++ b/build.sh @@ -56,6 +56,9 @@ if hasArg --build-cuvs-java; then # instead of the conda-installed ones, ensuring jextract generates correct # Panama bindings for any new C API in the PR. export CMAKE_PREFIX_PATH="$LIBCUVS_DIR" + # Uninstall the conda-installed libcuvs so it cannot shadow the PR artifact + # at runtime via ldconfig (which takes precedence over LD_LIBRARY_PATH). + conda remove --yes --force-remove libcuvs || true fi ./build.sh "${CUVS_BUILD_TARGETS[@]}" popd From 1ce03b46ab53802133f2bcedb11adb3cfd81041a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 27 Jul 2026 19:21:55 -1000 Subject: [PATCH 20/29] Debug: one more try --- build.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/build.sh b/build.sh index 71e44485..2ce7e356 100755 --- a/build.sh +++ b/build.sh @@ -45,13 +45,9 @@ if hasArg --build-cuvs-java; then LD_LIBRARY_PATH="$LIBCUVS_DIR/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" # Remove the conda-cached libcuvs from LD_LIBRARY_PATH so it cannot # shadow the PR artifact's libcuvs_c.so at runtime. - LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v "/pkgs/.*libcuvs" | paste -sd ':') + # LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v "/pkgs/.*libcuvs" | paste -sd ':') export LD_LIBRARY_PATH echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH" - echo "PR artifact contents:" - find "$LIBCUVS_DIR" -name "libcuvs*" - echo "cuvsCagraIndexParamsFromDataset in PR libcuvs_c.so:" - nm -D "$LIBCUVS_DIR/lib/libcuvs_c.so" 2>/dev/null | grep cuvsCagraIndexParamsFromDataset || echo "NOT FOUND" # Point cmake to the PR artifact so find_package(cuvs) picks up its headers # instead of the conda-installed ones, ensuring jextract generates correct # Panama bindings for any new C API in the PR. From 7b31187e9e01c62a96f3528658d47751827ea725 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 28 Jul 2026 05:57:28 -1000 Subject: [PATCH 21/29] Another build experiment --- build.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 2ce7e356..9ca6e6e4 100755 --- a/build.sh +++ b/build.sh @@ -51,10 +51,11 @@ if hasArg --build-cuvs-java; then # Point cmake to the PR artifact so find_package(cuvs) picks up its headers # instead of the conda-installed ones, ensuring jextract generates correct # Panama bindings for any new C API in the PR. - export CMAKE_PREFIX_PATH="$LIBCUVS_DIR" + # export CMAKE_PREFIX_PATH="$LIBCUVS_DIR" + export cuvs_ROOT="$LIBCUVS_DIR" # Uninstall the conda-installed libcuvs so it cannot shadow the PR artifact # at runtime via ldconfig (which takes precedence over LD_LIBRARY_PATH). - conda remove --yes --force-remove libcuvs || true + # conda remove --yes --force-remove libcuvs || true fi ./build.sh "${CUVS_BUILD_TARGETS[@]}" popd From 1161c7de99f8fadb445c0f996c835a7817c90530 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 28 Jul 2026 06:19:01 -1000 Subject: [PATCH 22/29] Another build experiment --- build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 9ca6e6e4..782a56a6 100755 --- a/build.sh +++ b/build.sh @@ -53,9 +53,9 @@ if hasArg --build-cuvs-java; then # Panama bindings for any new C API in the PR. # export CMAKE_PREFIX_PATH="$LIBCUVS_DIR" export cuvs_ROOT="$LIBCUVS_DIR" - # Uninstall the conda-installed libcuvs so it cannot shadow the PR artifact - # at runtime via ldconfig (which takes precedence over LD_LIBRARY_PATH). - # conda remove --yes --force-remove libcuvs || true + # Preload the PR artifact's libcuvs_c.so so it takes precedence over the + # conda-installed one, which the JVM's RPATH would otherwise find first. + export LD_PRELOAD="$LIBCUVS_DIR/lib/libcuvs_c.so${LD_PRELOAD:+:$LD_PRELOAD}" fi ./build.sh "${CUVS_BUILD_TARGETS[@]}" popd From 888bb7c9169dd824ede5705216e68b8ab95c42e6 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 28 Jul 2026 06:30:33 -1000 Subject: [PATCH 23/29] Uninstall conda libcuvs when using PR artifact to avoid RPATH shadowing --- build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 782a56a6..5afae0cf 100755 --- a/build.sh +++ b/build.sh @@ -53,9 +53,9 @@ if hasArg --build-cuvs-java; then # Panama bindings for any new C API in the PR. # export CMAKE_PREFIX_PATH="$LIBCUVS_DIR" export cuvs_ROOT="$LIBCUVS_DIR" - # Preload the PR artifact's libcuvs_c.so so it takes precedence over the - # conda-installed one, which the JVM's RPATH would otherwise find first. - export LD_PRELOAD="$LIBCUVS_DIR/lib/libcuvs_c.so${LD_PRELOAD:+:$LD_PRELOAD}" + # Uninstall the conda-installed libcuvs so the JVM's RPATH cannot find it + # ahead of the PR artifact's libcuvs_c.so. + conda remove --yes --force-remove libcuvs fi ./build.sh "${CUVS_BUILD_TARGETS[@]}" popd From df35c3f321f3a42adaaf8875f6d55efb3c148afb Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 28 Jul 2026 06:31:59 -1000 Subject: [PATCH 24/29] Use patchelf to tweak JVM RPATH instead of uninstalling conda libcuvs --- build.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 5afae0cf..e1651b86 100755 --- a/build.sh +++ b/build.sh @@ -53,9 +53,12 @@ if hasArg --build-cuvs-java; then # Panama bindings for any new C API in the PR. # export CMAKE_PREFIX_PATH="$LIBCUVS_DIR" export cuvs_ROOT="$LIBCUVS_DIR" - # Uninstall the conda-installed libcuvs so the JVM's RPATH cannot find it - # ahead of the PR artifact's libcuvs_c.so. - conda remove --yes --force-remove libcuvs + # Prepend the PR artifact's lib dir to the JVM's RPATH so libcuvs_c.so is + # found there first, while conda-installed dependencies (librmm.so etc.) + # are still found via the existing RPATH entries. + JAVA_BIN=$(which java) + CURRENT_RPATH=$(patchelf --print-rpath "$JAVA_BIN") + patchelf --set-rpath "$LIBCUVS_DIR/lib:$CURRENT_RPATH" "$JAVA_BIN" fi ./build.sh "${CUVS_BUILD_TARGETS[@]}" popd From a6d6b948b373531e75a0f986ce0a2d8c87c9074e Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 28 Jul 2026 09:52:57 -1000 Subject: [PATCH 25/29] Another experiment --- build.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/build.sh b/build.sh index e1651b86..67a6db6e 100755 --- a/build.sh +++ b/build.sh @@ -53,12 +53,6 @@ if hasArg --build-cuvs-java; then # Panama bindings for any new C API in the PR. # export CMAKE_PREFIX_PATH="$LIBCUVS_DIR" export cuvs_ROOT="$LIBCUVS_DIR" - # Prepend the PR artifact's lib dir to the JVM's RPATH so libcuvs_c.so is - # found there first, while conda-installed dependencies (librmm.so etc.) - # are still found via the existing RPATH entries. - JAVA_BIN=$(which java) - CURRENT_RPATH=$(patchelf --print-rpath "$JAVA_BIN") - patchelf --set-rpath "$LIBCUVS_DIR/lib:$CURRENT_RPATH" "$JAVA_BIN" fi ./build.sh "${CUVS_BUILD_TARGETS[@]}" popd @@ -69,6 +63,8 @@ if ! hasArg --run-java-tests; then MAVEN_VERIFY_ARGS=("-DskipTests") fi +echo "LD_LIBRARY_PATH just before the test is: $LD_LIBRARY_PATH" + mvn clean verify "${MAVEN_VERIFY_ARGS[@]}" \ && mvn jacoco:report \ && mvn install:install-file -Dfile=./target/cuvs-lucene-$VERSION.jar -DgroupId=$GROUP_ID -DartifactId=cuvs-lucene -Dversion=$VERSION -Dpackaging=jar \ From eb77f70b10882517ed1d2958c5a3932966bafe23 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 28 Jul 2026 11:04:34 -1000 Subject: [PATCH 26/29] Move handling of PRs into ci/build.sh --- build.sh | 25 +------------------------ ci/build_java.sh | 22 +++++++++++++++++----- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/build.sh b/build.sh index 67a6db6e..4a857492 100755 --- a/build.sh +++ b/build.sh @@ -31,30 +31,7 @@ if hasArg --build-cuvs-java; then pushd $CUVS_WORKDIR fi - # libcuvs comes from the conda packages, so normally only the java bindings have - # to be built. For a pull request, the conda packages do not contain the PR's - # changes, so CI downloads the pre-built libcuvs artifact from the PR's own CI run. - CUVS_BUILD_TARGETS=("java") - if hasArg --use-pr-libcuvs && [[ "$BRANCH" == pull-request/* ]]; then - PR_NUM="${BRANCH#pull-request/}" - echo "Downloading libcuvs conda artifact from cuvs PR #${PR_NUM}..." - LIBCUVS_CONDA_DIR=$(rapids-get-pr-artifact NVIDIA/cuvs "$PR_NUM" cpp conda) - LIBCUVS_DIR=$(rapids-extract-conda-files "$LIBCUVS_CONDA_DIR") - # The downloaded library has to take precedence over the one provided by the - # conda packages, both here and while running the java tests. - LD_LIBRARY_PATH="$LIBCUVS_DIR/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - # Remove the conda-cached libcuvs from LD_LIBRARY_PATH so it cannot - # shadow the PR artifact's libcuvs_c.so at runtime. - # LD_LIBRARY_PATH=$(echo "$LD_LIBRARY_PATH" | tr ':' '\n' | grep -v "/pkgs/.*libcuvs" | paste -sd ':') - export LD_LIBRARY_PATH - echo "LD_LIBRARY_PATH is: $LD_LIBRARY_PATH" - # Point cmake to the PR artifact so find_package(cuvs) picks up its headers - # instead of the conda-installed ones, ensuring jextract generates correct - # Panama bindings for any new C API in the PR. - # export CMAKE_PREFIX_PATH="$LIBCUVS_DIR" - export cuvs_ROOT="$LIBCUVS_DIR" - fi - ./build.sh "${CUVS_BUILD_TARGETS[@]}" + ./build.sh java popd fi diff --git a/ci/build_java.sh b/ci/build_java.sh index f9e3c1af..b2122806 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -16,11 +16,6 @@ fi # Always build cuvs-java when running the pipeline EXTRA_BUILD_ARGS+=("--build-cuvs-java") -# When RAPIDS_BRANCH points at a cuvs pull request, the libcuvs from the conda -# packages does not contain the PR's changes, so download the pre-built artifact -# from the PR's CI run instead. Only done in CI, local builds use the conda packages. -EXTRA_BUILD_ARGS+=("--use-pr-libcuvs") - # shellcheck disable=SC1091 . /opt/conda/etc/profile.d/conda.sh @@ -66,6 +61,23 @@ else exit 1 fi +# When RAPIDS_BRANCH points at a cuvs PR, the conda packages do not contain the +# PR's changes, so download the pre-built libcuvs artifact from the PR's own CI run. +BRANCH=$(cat "RAPIDS_BRANCH") +if [[ "$BRANCH" == pull-request/* ]]; then + PR_NUM="${BRANCH#pull-request/}" + rapids-logger "Downloading libcuvs conda artifact from cuvs PR #${PR_NUM}" + LIBCUVS_CONDA_DIR=$(rapids-get-pr-artifact NVIDIA/cuvs "$PR_NUM" cpp conda) + LIBCUVS_ARTIFACT_DIR=$(rapids-extract-conda-files "$LIBCUVS_CONDA_DIR") + # The PR artifact must take precedence over the conda-installed libcuvs both + # at runtime and for cmake's find_package (to pick up new C API headers for jextract). + export LD_LIBRARY_PATH="$LIBCUVS_ARTIFACT_DIR/lib:$LD_LIBRARY_PATH" + export cuvs_ROOT="$LIBCUVS_ARTIFACT_DIR" + # Uninstall the conda libcuvs so the JVM's RPATH (which points to the conda + # env's lib dir) cannot find the old libcuvs_c.so ahead of the PR artifact. + conda remove --yes --force-remove libcuvs +fi + rapids-logger "Run Java build" bash ./build.sh "${EXTRA_BUILD_ARGS[@]}" From 73c58284f6144ccac1e1f7a069124146db634e9e Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 28 Jul 2026 11:37:26 -1000 Subject: [PATCH 27/29] Remove libcuvs first, download a new on after --- ci/build_java.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ci/build_java.sh b/ci/build_java.sh index b2122806..afadd546 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -65,6 +65,11 @@ fi # PR's changes, so download the pre-built libcuvs artifact from the PR's own CI run. BRANCH=$(cat "RAPIDS_BRANCH") if [[ "$BRANCH" == pull-request/* ]]; then + rapids-logger "Remove libcuvs from conda environment" + # Uninstall the conda libcuvs so the JVM's RPATH (which points to the conda + # env's lib dir) cannot find the old libcuvs_c.so ahead of the PR artifact. + conda remove --yes --force-remove libcuvs + # Download PR artifact PR_NUM="${BRANCH#pull-request/}" rapids-logger "Downloading libcuvs conda artifact from cuvs PR #${PR_NUM}" LIBCUVS_CONDA_DIR=$(rapids-get-pr-artifact NVIDIA/cuvs "$PR_NUM" cpp conda) @@ -73,9 +78,6 @@ if [[ "$BRANCH" == pull-request/* ]]; then # at runtime and for cmake's find_package (to pick up new C API headers for jextract). export LD_LIBRARY_PATH="$LIBCUVS_ARTIFACT_DIR/lib:$LD_LIBRARY_PATH" export cuvs_ROOT="$LIBCUVS_ARTIFACT_DIR" - # Uninstall the conda libcuvs so the JVM's RPATH (which points to the conda - # env's lib dir) cannot find the old libcuvs_c.so ahead of the PR artifact. - conda remove --yes --force-remove libcuvs fi rapids-logger "Run Java build" From dce241dbc9e1ac60659946e3bf76186d7838ca2a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 28 Jul 2026 11:48:45 -1000 Subject: [PATCH 28/29] Guard conda remove with set +u like conda activate --- ci/build_java.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/build_java.sh b/ci/build_java.sh index afadd546..1392a56f 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -68,7 +68,10 @@ if [[ "$BRANCH" == pull-request/* ]]; then rapids-logger "Remove libcuvs from conda environment" # Uninstall the conda libcuvs so the JVM's RPATH (which points to the conda # env's lib dir) cannot find the old libcuvs_c.so ahead of the PR artifact. + # Temporarily allow unbound variables for conda shell hooks. + set +u conda remove --yes --force-remove libcuvs + set -u # Download PR artifact PR_NUM="${BRANCH#pull-request/}" rapids-logger "Downloading libcuvs conda artifact from cuvs PR #${PR_NUM}" From 1a19fddd40aa97d20ca67e9fdd33ebc1781d26c6 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 28 Jul 2026 11:51:39 -1000 Subject: [PATCH 29/29] Fix conda removal --- ci/build_java.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/build_java.sh b/ci/build_java.sh index 1392a56f..4d7e24b0 100755 --- a/ci/build_java.sh +++ b/ci/build_java.sh @@ -68,7 +68,6 @@ if [[ "$BRANCH" == pull-request/* ]]; then rapids-logger "Remove libcuvs from conda environment" # Uninstall the conda libcuvs so the JVM's RPATH (which points to the conda # env's lib dir) cannot find the old libcuvs_c.so ahead of the PR artifact. - # Temporarily allow unbound variables for conda shell hooks. set +u conda remove --yes --force-remove libcuvs set -u