Move off deprecated reranker and LLM endpoints

config/config.yaml

@@ -17,7 +17,7 @@ summarization:
   enable: true
   method: "batch"
   llm:
-    model: nvidia/nvidia-nemotron-nano-9b-v2
+    model: nvidia/nemotron-3-nano-30b-a3b
     base_url: https://integrate.api.nvidia.com/v1
     max_tokens: 2048
     temperature: 0.2
@@ -49,22 +49,22 @@ chat:
     top_k: 25
     confidence_threshold: 0.0
   llm:
-    model: nvidia/nvidia-nemotron-nano-9b-v2
+    model: nvidia/nemotron-3-nano-30b-a3b
     base_url: https://integrate.api.nvidia.com/v1
     max_tokens: 2048
     temperature: 0.5
   embedding:
     model: "nvidia/llama-3.2-nv-embedqa-1b-v2"
     base_url: "http://embedding-nim:8000/v1/"
   reranker:
-    model: "nvidia/llama-3.2-nv-rerankqa-1b-v2"
-    base_url: https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking
+    model: "nvidia/llama-nemotron-rerank-vl-1b-v2"
+    base_url: "http://reranker-nim:8000/v1"
 
 notification:
   enable: false
   endpoint: "http://127.0.0.1:60000/via-alert-callback"
   llm:
-    model: nvidia/nvidia-nemotron-nano-9b-v2
+    model: nvidia/nemotron-3-nano-30b-a3b
     base_url: https://integrate.api.nvidia.com/v1
     max_tokens: 2048
     temperature: 0.2

docker/deploy/compose.yaml

@@ -38,6 +38,28 @@ services:
               device_ids: ['${EMBEDDING_GPU_ID:-0}']
               capabilities: [gpu]
 
+  # Reranking NIM. Self-hosted rather than hosted: the pinned
+  # langchain-nvidia-ai-endpoints validates rerank models against its own
+  # registry, which predates this model, and the SDK cannot be upgraded without
+  # breaking the langchain-core pin.
+  reranker-nim:
+    container_name: reranker-nim
+    image: "nvcr.io/nim/nvidia/llama-nemotron-rerank-vl-1b-v2:2.3.0"
+    volumes:
+      - ${MODEL_DIRECTORY:-.}:/opt/nim/.cache
+    ports:
+      - "8003:8000"
+    environment:
+      NGC_API_KEY: ${NVIDIA_API_KEY:?"NVIDIA_API_KEY is required"}
+    shm_size: 16gb
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              device_ids: ['${RERANKER_GPU_ID:-0}']
+              capabilities: [gpu]
+
   vss-ctx-rag-retriever:
     image: ctx_rag
     build:

src/vss_ctx_rag/aiq/utils.py

@@ -141,7 +141,7 @@ class VssCtxRagToolConfig(FunctionBaseConfig, name=name):
 
         embedding_model_name: EmbedderRef
 
-        rerank_model_name: str = "nvidia/llama-3.2-nv-rerankqa-1b-v2"
+        rerank_model_name: str = "nvidia/llama-nemotron-rerank-vl-1b-v2"
         rerank_model_url: str = "https://integrate.api.nvidia.com/v1"
         rag_type: str = "vector-rag"  # or "graph-rag"
         chat_batch_size: int = 1

src/vss_ctx_rag/aiq_config/function/config-ingestion-function.yml

@@ -46,8 +46,8 @@ functions:
 
     embedding_model_name: embedding_llm
 
-    rerank_model_name: "nvidia/llama-3.2-nv-rerankqa-1b-v2"
-    rerank_model_url: "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking"
+    rerank_model_name: "nvidia/llama-nemotron-rerank-vl-1b-v2"
+    rerank_model_url: "http://reranker-nim:8000/v1"
     rag_type: "vector-rag" # or "graph-rag"
     chat_batch_size: 1
     summ_batch_size: 5

src/vss_ctx_rag/aiq_config/function/config-retrieval-function.yml

@@ -46,8 +46,8 @@ functions:
 
     embedding_model_name: embedding_llm
 
-    rerank_model_name: "nvidia/llama-3.2-nv-rerankqa-1b-v2"
-    rerank_model_url: "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking"
+    rerank_model_name: "nvidia/llama-nemotron-rerank-vl-1b-v2"
+    rerank_model_url: "http://reranker-nim:8000/v1"
     rag_type: "vector-rag" # or "graph-rag"
     chat_batch_size: 1
     summ_batch_size: 5

src/vss_ctx_rag/aiq_config/workflow/config-ingestion-workflow.yml

@@ -45,8 +45,8 @@ workflow:
 
   embedding_model_name: embedding_llm
 
-  rerank_model_name: "nvidia/llama-3.2-nv-rerankqa-1b-v2"
-  rerank_model_url: "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking"
+  rerank_model_name: "nvidia/llama-nemotron-rerank-vl-1b-v2"
+  rerank_model_url: "http://reranker-nim:8000/v1"
   rag_type: "vector-rag" # or "graph-rag"
   chat_batch_size: 1
   summ_batch_size: 5

src/vss_ctx_rag/aiq_config/workflow/config-retrieval-workflow.yml

@@ -45,8 +45,8 @@ workflow:
 
   embedding_model_name: embedding_llm
 
-  rerank_model_name: "nvidia/llama-3.2-nv-rerankqa-1b-v2"
-  rerank_model_url: "https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking"
+  rerank_model_name: "nvidia/llama-nemotron-rerank-vl-1b-v2"
+  rerank_model_url: "http://reranker-nim:8000/v1"
   rag_type: "vector-rag" # or "graph-rag"
   chat_batch_size: 1
   summ_batch_size: 5

src/vss_ctx_rag/tools/storage/milvus_db.py

@@ -38,8 +38,8 @@ def __init__(
         port="19530",
         embedding_model_name="nvidia/llama-3.2-nv-embedqa-1b-v2",
         embedding_base_url="https://integrate.api.nvidia.com/v1",
-        reranker_model_name="nvidia/llama-3.2-nv-rerankqa-1b-v2",
-        reranker_base_url="https://ai.api.nvidia.com/v1/retrieval/nvidia/llama-3_2-nv-rerankqa-1b-v2/reranking",
+        reranker_model_name="nvidia/llama-nemotron-rerank-vl-1b-v2",
+        reranker_base_url="http://reranker-nim:8000/v1",
         name="milvus_db",
     ) -> None:
         super().__init__(name)