From e6e056ba3c45fb6ad1f516c32de2376574e963c2 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Tue, 25 Aug 2026 09:13:55 -0700 Subject: [PATCH 1/4] feat: add nemotron 500m rerank model with 512-passage batches nvidia/llama-3.2-nemoretriever-500m-rerank-v2 (NIM 2.x) accepts 512 passages per call, so a typical findAndRerank becomes a single reranking call instead of a fan-out of ten batch-10 calls, greatly reducing burst load on the reranking service. Non-default; the legacy rerankqa-1b-v2 model and the default /nvidia/v1/ranking route are unchanged. The model URL follows the GPU plane's per-model ingress route pattern; the route and NIM deployment land in the gpu-helm-charts repo and should be live before this model is put into use. --- .../resources/reranking-providers-config.yaml | 9 ++++++++- .../test-reranking-providers-config.yaml | 5 +++++ ...FindRerankingProvidersIntegrationTest.java | 20 +++++++++++++++---- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/main/resources/reranking-providers-config.yaml b/src/main/resources/reranking-providers-config.yaml index 67739fe0a2..6ba9e82c48 100644 --- a/src/main/resources/reranking-providers-config.yaml +++ b/src/main/resources/reranking-providers-config.yaml @@ -14,4 +14,11 @@ stargate: is-default: true url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking properties: - max-batch-size: 10 \ No newline at end of file + max-batch-size: 10 + # NIM 2.x model; accepts up to 512 passages per call, so a typical findAndRerank + # is a single reranking call instead of a fan-out of max-batch-size-10 batches. + - name: nvidia/llama-3.2-nemoretriever-500m-rerank-v2 + is-default: false + url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking/nvidia-llama-3-2-nemoretriever-500m-rerank-v2 + properties: + max-batch-size: 512 \ No newline at end of file diff --git a/src/main/resources/test-reranking-providers-config.yaml b/src/main/resources/test-reranking-providers-config.yaml index 7f1350470a..1b8629904c 100644 --- a/src/main/resources/test-reranking-providers-config.yaml +++ b/src/main/resources/test-reranking-providers-config.yaml @@ -15,6 +15,11 @@ stargate: url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking properties: max-batch-size: 10 + - name: nvidia/llama-3.2-nemoretriever-500m-rerank-v2 + is-default: false + url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking/nvidia-llama-3-2-nemoretriever-500m-rerank-v2 + properties: + max-batch-size: 512 - name: nvidia/a-random-deprecated-model api-model-support: status: DEPRECATED diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindRerankingProvidersIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindRerankingProvidersIntegrationTest.java index 0a0314644f..a20c8e0711 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindRerankingProvidersIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindRerankingProvidersIntegrationTest.java @@ -39,12 +39,18 @@ public final void defaultSupportModels() { .statusCode(200) .body("$", responseIsStatusOnly()) .body("status.rerankingProviders", notNullValue()) - .body("status.rerankingProviders.nvidia.models", hasSize(1)) + .body("status.rerankingProviders.nvidia.models", hasSize(2)) .body( "status.rerankingProviders.nvidia.models[0].name", - equalTo("nvidia/llama-3.2-nv-rerankqa-1b-v2")) + equalTo("nvidia/llama-3.2-nemoretriever-500m-rerank-v2")) .body( "status.rerankingProviders.nvidia.models[0].apiModelSupport.status", + equalTo(ApiModelSupport.SupportStatus.SUPPORTED.name())) + .body( + "status.rerankingProviders.nvidia.models[1].name", + equalTo("nvidia/llama-3.2-nv-rerankqa-1b-v2")) + .body( + "status.rerankingProviders.nvidia.models[1].apiModelSupport.status", equalTo(ApiModelSupport.SupportStatus.SUPPORTED.name())); } @@ -76,7 +82,7 @@ public final void returnModelsWithAllStatus(String filterModelStatus) { .statusCode(200) .body("$", responseIsStatusOnly()) .body("status.rerankingProviders", notNullValue()) - .body("status.rerankingProviders.nvidia.models", hasSize(3)) + .body("status.rerankingProviders.nvidia.models", hasSize(4)) .body( "status.rerankingProviders.nvidia.models[0].name", equalTo("nvidia/a-random-EOL-model")) @@ -91,9 +97,15 @@ public final void returnModelsWithAllStatus(String filterModelStatus) { equalTo(ApiModelSupport.SupportStatus.DEPRECATED.name())) .body( "status.rerankingProviders.nvidia.models[2].name", - equalTo("nvidia/llama-3.2-nv-rerankqa-1b-v2")) + equalTo("nvidia/llama-3.2-nemoretriever-500m-rerank-v2")) .body( "status.rerankingProviders.nvidia.models[2].apiModelSupport.status", + equalTo(ApiModelSupport.SupportStatus.SUPPORTED.name())) + .body( + "status.rerankingProviders.nvidia.models[3].name", + equalTo("nvidia/llama-3.2-nv-rerankqa-1b-v2")) + .body( + "status.rerankingProviders.nvidia.models[3].apiModelSupport.status", equalTo(ApiModelSupport.SupportStatus.SUPPORTED.name())); } From 0eef349e91fee7c9ad1e1ef36c88cfb6c7bc71a3 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Tue, 25 Aug 2026 10:14:46 -0700 Subject: [PATCH 2/4] perf: raise rerank max-batch-size to the NIM's 512-passage request limit The reranking NIM accepts up to 512 passages per request; it rejects 513+ with a 400 validation error. With max-batch-size 10, a findAndRerank with 100 candidates fans out into 10 concurrent reranking calls, multiplying request load on the reranking service by ~10x for no benefit. At 512 a typical findAndRerank is a single call. Verified against the dev reranking endpoint (rerankqa-1b-v2, NIM 1.3.1.0): 512-passage requests return 200 with all 512 rankings in ~0.36s warm; 600 passages returns 400 "List should have at most 512 items". Deployments that serve reranking config via the embedding gateway or a config override need the same value there for this to take effect. --- .../resources/reranking-providers-config.yaml | 10 +++------- .../test-reranking-providers-config.yaml | 5 ----- ...FindRerankingProvidersIntegrationTest.java | 20 ++++--------------- 3 files changed, 7 insertions(+), 28 deletions(-) diff --git a/src/main/resources/reranking-providers-config.yaml b/src/main/resources/reranking-providers-config.yaml index 6ba9e82c48..c4c9db61d0 100644 --- a/src/main/resources/reranking-providers-config.yaml +++ b/src/main/resources/reranking-providers-config.yaml @@ -14,11 +14,7 @@ stargate: is-default: true url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking properties: - max-batch-size: 10 - # NIM 2.x model; accepts up to 512 passages per call, so a typical findAndRerank - # is a single reranking call instead of a fan-out of max-batch-size-10 batches. - - name: nvidia/llama-3.2-nemoretriever-500m-rerank-v2 - is-default: false - url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking/nvidia-llama-3-2-nemoretriever-500m-rerank-v2 - properties: + # The reranking NIM accepts up to 512 passages per request (verified: it + # rejects 513+ with a 400). One call per findAndRerank instead of a fan-out + # of ten concurrent batch-10 calls. max-batch-size: 512 \ No newline at end of file diff --git a/src/main/resources/test-reranking-providers-config.yaml b/src/main/resources/test-reranking-providers-config.yaml index 1b8629904c..45c6887822 100644 --- a/src/main/resources/test-reranking-providers-config.yaml +++ b/src/main/resources/test-reranking-providers-config.yaml @@ -13,11 +13,6 @@ stargate: - name: nvidia/llama-3.2-nv-rerankqa-1b-v2 is-default: true url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking - properties: - max-batch-size: 10 - - name: nvidia/llama-3.2-nemoretriever-500m-rerank-v2 - is-default: false - url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking/nvidia-llama-3-2-nemoretriever-500m-rerank-v2 properties: max-batch-size: 512 - name: nvidia/a-random-deprecated-model diff --git a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindRerankingProvidersIntegrationTest.java b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindRerankingProvidersIntegrationTest.java index a20c8e0711..0a0314644f 100644 --- a/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindRerankingProvidersIntegrationTest.java +++ b/src/test/java/io/stargate/sgv2/jsonapi/api/v1/FindRerankingProvidersIntegrationTest.java @@ -39,18 +39,12 @@ public final void defaultSupportModels() { .statusCode(200) .body("$", responseIsStatusOnly()) .body("status.rerankingProviders", notNullValue()) - .body("status.rerankingProviders.nvidia.models", hasSize(2)) + .body("status.rerankingProviders.nvidia.models", hasSize(1)) .body( "status.rerankingProviders.nvidia.models[0].name", - equalTo("nvidia/llama-3.2-nemoretriever-500m-rerank-v2")) - .body( - "status.rerankingProviders.nvidia.models[0].apiModelSupport.status", - equalTo(ApiModelSupport.SupportStatus.SUPPORTED.name())) - .body( - "status.rerankingProviders.nvidia.models[1].name", equalTo("nvidia/llama-3.2-nv-rerankqa-1b-v2")) .body( - "status.rerankingProviders.nvidia.models[1].apiModelSupport.status", + "status.rerankingProviders.nvidia.models[0].apiModelSupport.status", equalTo(ApiModelSupport.SupportStatus.SUPPORTED.name())); } @@ -82,7 +76,7 @@ public final void returnModelsWithAllStatus(String filterModelStatus) { .statusCode(200) .body("$", responseIsStatusOnly()) .body("status.rerankingProviders", notNullValue()) - .body("status.rerankingProviders.nvidia.models", hasSize(4)) + .body("status.rerankingProviders.nvidia.models", hasSize(3)) .body( "status.rerankingProviders.nvidia.models[0].name", equalTo("nvidia/a-random-EOL-model")) @@ -97,15 +91,9 @@ public final void returnModelsWithAllStatus(String filterModelStatus) { equalTo(ApiModelSupport.SupportStatus.DEPRECATED.name())) .body( "status.rerankingProviders.nvidia.models[2].name", - equalTo("nvidia/llama-3.2-nemoretriever-500m-rerank-v2")) - .body( - "status.rerankingProviders.nvidia.models[2].apiModelSupport.status", - equalTo(ApiModelSupport.SupportStatus.SUPPORTED.name())) - .body( - "status.rerankingProviders.nvidia.models[3].name", equalTo("nvidia/llama-3.2-nv-rerankqa-1b-v2")) .body( - "status.rerankingProviders.nvidia.models[3].apiModelSupport.status", + "status.rerankingProviders.nvidia.models[2].apiModelSupport.status", equalTo(ApiModelSupport.SupportStatus.SUPPORTED.name())); } From 542dd47a7e37db16674464812d51301111656ec1 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Tue, 25 Aug 2026 10:20:06 -0700 Subject: [PATCH 3/4] Update reranking-providers-config.yaml --- src/main/resources/reranking-providers-config.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/resources/reranking-providers-config.yaml b/src/main/resources/reranking-providers-config.yaml index c4c9db61d0..7986ae91f1 100644 --- a/src/main/resources/reranking-providers-config.yaml +++ b/src/main/resources/reranking-providers-config.yaml @@ -14,7 +14,4 @@ stargate: is-default: true url: https://us-west-2.api-dev.ai.datastax.com/nvidia/v1/ranking properties: - # The reranking NIM accepts up to 512 passages per request (verified: it - # rejects 513+ with a 400). One call per findAndRerank instead of a fan-out - # of ten concurrent batch-10 calls. - max-batch-size: 512 \ No newline at end of file + max-batch-size: 512 From e53589e502d458383fe36c35f88ba41fadf04387 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Tue, 25 Aug 2026 10:20:20 -0700 Subject: [PATCH 4/4] Update reranking-providers-config.yaml