From 00bcc5b435e14971c03d8e5242958cf47ae1008d Mon Sep 17 00:00:00 2001 From: Karthik Tiruveedhi Date: Wed, 5 Aug 2026 13:59:10 -0700 Subject: [PATCH 1/2] Move off deprecated reranker and LLM endpoints The reranker this branch calls is already gone: the hosted llama-3_2-nv-rerankqa-1b-v2 endpoint returns HTTP 410 ("end of life on 2026-05-18"), so retrieval reranking fails today. The chat/summarization LLM nvidia-nemotron-nano-9b-v2 is in the August 2026 deprecation wave. Reranker moves to llama-nemotron-rerank-vl-1b-v2, self-hosted, mirroring the embedding-nim service this branch already runs. Self-hosting is not a preference here - the pinned langchain-nvidia-ai-endpoints==0.3.7 validates rerank model names against its own registry, which only knows four old-generation models, so the hosted successor is rejected client-side before a request is ever made. Upgrading the SDK is not viable either: 0.3.19 and langchain_core==0.3.21 resolve as ResolutionImpossible. LLM moves to nemotron-3-nano-30b-a3b in all three places. Unlike the reranker, ChatNVIDIA passes model names straight through, so this needed no plumbing change - verified against the pinned SDK rather than assumed. Also updates the hardcoded reranker defaults in milvus_db.py and aiq/utils.py, which still point at the dead endpoint. Config overrides them at runtime, so they are latent rather than active, but they are the copy a new call site inherits. Embedding and the Parakeet ASR NIM are deliberately untouched: both are self-hosted containers on this branch, so hosted-endpoint deprecation does not apply to them. Validated on an L4: the reranker NIM serves the model id the config asks for, and the pinned SDK reranks through it correctly - a GPU-related passage scores -5.41 for a query about GPUs, versus -7.50 and -7.91 for unrelated passages. --- config/config.yaml | 10 +++++----- docker/deploy/compose.yaml | 22 ++++++++++++++++++++++ src/vss_ctx_rag/aiq/utils.py | 2 +- src/vss_ctx_rag/tools/storage/milvus_db.py | 4 ++-- 4 files changed, 30 insertions(+), 8 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index a9468c0a..82cb8f2e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -17,7 +17,7 @@ summarization: enable: true method: "batch" llm: - model: nvidia/nvidia-nemotron-nano-9b-v2 + model: nvidia/nemotron-3-nano-30b-a3b base_url: https://integrate.api.nvidia.com/v1 max_tokens: 2048 temperature: 0.2 @@ -49,7 +49,7 @@ chat: top_k: 25 confidence_threshold: 0.0 llm: - model: nvidia/nvidia-nemotron-nano-9b-v2 + model: nvidia/nemotron-3-nano-30b-a3b base_url: https://integrate.api.nvidia.com/v1 max_tokens: 2048 temperature: 0.5 @@ -57,14 +57,14 @@ chat: model: "nvidia/llama-3.2-nv-embedqa-1b-v2" base_url: "http://embedding-nim:8000/v1/" reranker: - model: "nvidia/llama-3.2-nv-rerankqa-1b-v2" - base_url: https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking + model: "nvidia/llama-nemotron-rerank-vl-1b-v2" + base_url: "http://reranker-nim:8000/v1" notification: enable: false endpoint: "http://127.0.0.1:60000/via-alert-callback" llm: - model: nvidia/nvidia-nemotron-nano-9b-v2 + model: nvidia/nemotron-3-nano-30b-a3b base_url: https://integrate.api.nvidia.com/v1 max_tokens: 2048 temperature: 0.2 diff --git a/docker/deploy/compose.yaml b/docker/deploy/compose.yaml index 8a510967..ffb8444e 100644 --- a/docker/deploy/compose.yaml +++ b/docker/deploy/compose.yaml @@ -38,6 +38,28 @@ services: device_ids: ['${EMBEDDING_GPU_ID:-0}'] capabilities: [gpu] + # Reranking NIM. Self-hosted rather than hosted: the pinned + # langchain-nvidia-ai-endpoints validates rerank models against its own + # registry, which predates this model, and the SDK cannot be upgraded without + # breaking the langchain-core pin. + reranker-nim: + container_name: reranker-nim + image: "nvcr.io/nim/nvidia/llama-nemotron-rerank-vl-1b-v2:2.3.0" + volumes: + - ${MODEL_DIRECTORY:-.}:/opt/nim/.cache + ports: + - "8003:8000" + environment: + NGC_API_KEY: ${NVIDIA_API_KEY:?"NVIDIA_API_KEY is required"} + shm_size: 16gb + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ['${RERANKER_GPU_ID:-0}'] + capabilities: [gpu] + vss-ctx-rag-retriever: image: ctx_rag build: diff --git a/src/vss_ctx_rag/aiq/utils.py b/src/vss_ctx_rag/aiq/utils.py index 259cecc3..67501e41 100644 --- a/src/vss_ctx_rag/aiq/utils.py +++ b/src/vss_ctx_rag/aiq/utils.py @@ -141,7 +141,7 @@ class VssCtxRagToolConfig(FunctionBaseConfig, name=name): embedding_model_name: EmbedderRef - rerank_model_name: str = "nvidia/llama-3.2-nv-rerankqa-1b-v2" + rerank_model_name: str = "nvidia/llama-nemotron-rerank-vl-1b-v2" rerank_model_url: str = "https://integrate.api.nvidia.com/v1" rag_type: str = "vector-rag" # or "graph-rag" chat_batch_size: int = 1 diff --git a/src/vss_ctx_rag/tools/storage/milvus_db.py b/src/vss_ctx_rag/tools/storage/milvus_db.py index fa7aa189..536e6bdd 100644 --- a/src/vss_ctx_rag/tools/storage/milvus_db.py +++ b/src/vss_ctx_rag/tools/storage/milvus_db.py @@ -38,8 +38,8 @@ def __init__( port="19530", embedding_model_name="nvidia/llama-3.2-nv-embedqa-1b-v2", embedding_base_url="https://integrate.api.nvidia.com/v1", - reranker_model_name="nvidia/llama-3.2-nv-rerankqa-1b-v2", - reranker_base_url="https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking", + reranker_model_name="nvidia/llama-nemotron-rerank-vl-1b-v2", + reranker_base_url="http://reranker-nim:8000/v1", name="milvus_db", ) -> None: super().__init__(name) From 88f992cea8ce84d4f0069ddcd00a65d992f55133 Mon Sep 17 00:00:00 2001 From: Karthik Tiruveedhi Date: Wed, 5 Aug 2026 14:53:10 -0700 Subject: [PATCH 2/2] Also update the AIQ configs to the working reranker The first pass missed four files under src/vss_ctx_rag/aiq_config/, which still carried the dead llama-3_2-nv-rerankqa endpoint. These are not inert: the AIQ guides instruct users to run `aiq serve --config_file=` against exactly these workflow and function configs, so anyone following that path still hit the 410. Same change as the main config - the successor model, pointed at the self-hosted reranker service. --- .../aiq_config/function/config-ingestion-function.yml | 4 ++-- .../aiq_config/function/config-retrieval-function.yml | 4 ++-- .../aiq_config/workflow/config-ingestion-workflow.yml | 4 ++-- .../aiq_config/workflow/config-retrieval-workflow.yml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/vss_ctx_rag/aiq_config/function/config-ingestion-function.yml b/src/vss_ctx_rag/aiq_config/function/config-ingestion-function.yml index 8ce106d1..7d7ce152 100644 --- a/src/vss_ctx_rag/aiq_config/function/config-ingestion-function.yml +++ b/src/vss_ctx_rag/aiq_config/function/config-ingestion-function.yml @@ -46,8 +46,8 @@ functions: embedding_model_name: embedding_llm - rerank_model_name: "nvidia/llama-3.2-nv-rerankqa-1b-v2" - rerank_model_url: "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking" + rerank_model_name: "nvidia/llama-nemotron-rerank-vl-1b-v2" + rerank_model_url: "http://reranker-nim:8000/v1" rag_type: "vector-rag" # or "graph-rag" chat_batch_size: 1 summ_batch_size: 5 diff --git a/src/vss_ctx_rag/aiq_config/function/config-retrieval-function.yml b/src/vss_ctx_rag/aiq_config/function/config-retrieval-function.yml index a215e681..ad7d9294 100644 --- a/src/vss_ctx_rag/aiq_config/function/config-retrieval-function.yml +++ b/src/vss_ctx_rag/aiq_config/function/config-retrieval-function.yml @@ -46,8 +46,8 @@ functions: embedding_model_name: embedding_llm - rerank_model_name: "nvidia/llama-3.2-nv-rerankqa-1b-v2" - rerank_model_url: "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking" + rerank_model_name: "nvidia/llama-nemotron-rerank-vl-1b-v2" + rerank_model_url: "http://reranker-nim:8000/v1" rag_type: "vector-rag" # or "graph-rag" chat_batch_size: 1 summ_batch_size: 5 diff --git a/src/vss_ctx_rag/aiq_config/workflow/config-ingestion-workflow.yml b/src/vss_ctx_rag/aiq_config/workflow/config-ingestion-workflow.yml index c5631224..54d26848 100644 --- a/src/vss_ctx_rag/aiq_config/workflow/config-ingestion-workflow.yml +++ b/src/vss_ctx_rag/aiq_config/workflow/config-ingestion-workflow.yml @@ -45,8 +45,8 @@ workflow: embedding_model_name: embedding_llm - rerank_model_name: "nvidia/llama-3.2-nv-rerankqa-1b-v2" - rerank_model_url: "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking" + rerank_model_name: "nvidia/llama-nemotron-rerank-vl-1b-v2" + rerank_model_url: "http://reranker-nim:8000/v1" rag_type: "vector-rag" # or "graph-rag" chat_batch_size: 1 summ_batch_size: 5 diff --git a/src/vss_ctx_rag/aiq_config/workflow/config-retrieval-workflow.yml b/src/vss_ctx_rag/aiq_config/workflow/config-retrieval-workflow.yml index 893de080..59cdf8e3 100644 --- a/src/vss_ctx_rag/aiq_config/workflow/config-retrieval-workflow.yml +++ b/src/vss_ctx_rag/aiq_config/workflow/config-retrieval-workflow.yml @@ -45,8 +45,8 @@ workflow: embedding_model_name: embedding_llm - rerank_model_name: "nvidia/llama-3.2-nv-rerankqa-1b-v2" - rerank_model_url: "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking" + rerank_model_name: "nvidia/llama-nemotron-rerank-vl-1b-v2" + rerank_model_url: "http://reranker-nim:8000/v1" rag_type: "vector-rag" # or "graph-rag" chat_batch_size: 1 summ_batch_size: 5