RHIDP-14063: migrate inference providers from llama stack to lightspeed core

.github/workflows/sync-gitops.yml

@@ -21,6 +21,7 @@ on:
       - main
     paths:
       - "lightspeed-core-configs/rhdh-profile.py"
+      - "lightspeed-core-configs/lightspeed-stack.yaml"
       - "llama-stack-configs/config.yaml"
       - "images.yaml"
   workflow_dispatch:
@@ -58,6 +59,7 @@ jobs:
       - name: Copy generated manifests
         run: |
           cp generated/llama-stack-config.yaml ai-rolling-demo-gitops/charts/rhdh/templates/llama-stack-config.yaml
+          cp generated/lightspeed-stack-config.yaml ai-rolling-demo-gitops/charts/rhdh/templates/lightspeed-stack-config.yaml
           cp generated/rhdh-profile.py ai-rolling-demo-gitops/charts/rhdh/rhdh-profile.py
 
       - name: Export image versions
@@ -84,5 +86,5 @@ jobs:
 
             Automated sync triggered from [${{ github.repository }}@${{ github.sha }}](https://github.com/${{ github.repository }}/commit/${{ github.sha }}).
 
-            Syncs `llama-stack-config.yaml`, `rhdh-profile.py`, and `charts/rhdh/values.yaml` with the following image version:
+            Syncs `llama-stack-config.yaml`, `lightspeed-stack-config.yaml`, `rhdh-profile.py`, and `charts/rhdh/values.yaml` with the following image version:
             - `lightspeed-core`: `${{ env.LIGHTSPEED_CORE_IMAGE }}`

.gitignore

@@ -6,6 +6,7 @@ vector_db
 
 rag-content 
 values.env
+*.local.yaml
 
 .venv
 

Makefile

@@ -24,6 +24,12 @@ endif
 
 LOCAL_COMPOSE_FILES := -f compose/compose.yaml
 
+LIGHTSPEED_STACK_CONFIG := lightspeed-core-configs/lightspeed-stack.yaml
+ifneq ($(wildcard lightspeed-core-configs/lightspeed-stack.local.yaml),)
+LIGHTSPEED_STACK_CONFIG := lightspeed-core-configs/lightspeed-stack.local.yaml
+endif
+export LIGHTSPEED_STACK_CONFIG
+
 .PHONY: default
 default: help
 

compose/compose.yaml

@@ -19,7 +19,7 @@ services:
     ports:
       - "8080:8080"
     volumes:
-      - ../lightspeed-core-configs/lightspeed-stack.yaml:/app-root/lightspeed-stack.yaml:Z
+      - ../${LIGHTSPEED_STACK_CONFIG:-lightspeed-core-configs/lightspeed-stack.yaml}:/app-root/lightspeed-stack.yaml:Z
       - ../lightspeed-core-configs/rhdh-profile.py:/app-root/rhdh-profile.py:Z
       - ../llama-stack-configs/config.yaml:/app-root/config.yaml:Z
       - ../rag-content:/rag-content:Z

docs/CONTRIBUTING.md

@@ -20,6 +20,8 @@
 
 1. Copy `./env/default-values.env` to `./env/values.env` and fill in any provider-specific values (see [docs/PROVIDERS.md](./PROVIDERS.md)).
 
+   To configure inference providers without editing the git-tracked `lightspeed-stack.yaml`, copy `lightspeed-core-configs/lightspeed-stack.yaml` to `lightspeed-core-configs/lightspeed-stack.local.yaml` and make your edits there. `make local-up` mounts the `.local.yaml` file automatically when it's present, otherwise it falls back to `lightspeed-stack.yaml`. `lightspeed-stack.local.yaml` is gitignored, so it's safe to leave provider config there permanently.
+
 2. Pull the RAG content:
 
 ```sh
@@ -38,7 +40,7 @@ This starts Lightspeed Core using the mounted config/content below.
 
 Lightspeed Core uses mounted config/content in local compose:
 
-- `lightspeed-core-configs/lightspeed-stack.yaml` -> `/app-root/lightspeed-stack.yaml`
+- `lightspeed-core-configs/lightspeed-stack.yaml` (or `lightspeed-stack.local.yaml`, if present) -> `/app-root/lightspeed-stack.yaml`
 - `lightspeed-core-configs/rhdh-profile.py` -> `/app-root/rhdh-profile.py`
 - `llama-stack-configs/config.yaml` -> `/app-root/config.yaml`
 - `rag-content/` -> `/rag-content`
@@ -96,15 +98,14 @@ To enable validation, set the following in `env/values.env`:
 | `VALIDATION_PROVIDER` | Yes | Inference provider used by the validation shield, for example `vllm` or `openai` |
 | `VALIDATION_MODEL_NAME` | Yes | Model name served by the selected inference provider |
 
-You must also enable and configure the referenced inference provider. Examples:
+The referenced inference provider must also be present in `lightspeed-stack.yaml` (or `lightspeed-stack.local.yaml`) and configured via its env vars. See [docs/PROVIDERS.md](./PROVIDERS.md). Examples:
 
 ### Example: vLLM-backed validation
 
 ```env
 ENABLE_VALIDATION=true
 VALIDATION_PROVIDER=vllm
 VALIDATION_MODEL_NAME=<your-model-name>
-ENABLE_VLLM=true
 VLLM_URL=<your-vllm-endpoint>
 VLLM_API_KEY=<api-key>
 ```

docs/PROVIDERS.md

@@ -8,59 +8,159 @@ Each inference has its own environment variables. You can include all of these i
 > `VLLM_API_KEY=token` (recommended)
 >
 > `VLLM_API_KEY="token"` (can cause parsing issues)
+> 
+> You will notice the `api_key_env` field is not wrapped in curly-braces `{}`. This is due to Lightspeed Core wrapping them internally to curate a proper `{env.xyz}` to pass through to Llama Stack so the keys are not exposed internally.
 
 ## vLLM
 
-Required:
-
-```env
-ENABLE_VLLM=true
-VLLM_URL=<your-server-url>/v1
-VLLM_API_KEY=<your-api-key>
+To add the `vLLM` inference provider, include the following in [lightspeed-stack.yaml](../lightspeed-core-configs/lightspeed-stack.yaml):
+
+```yaml
+inference:
+  providers:
+    - type: vllm
+      id: <your-unique-id>
+      api_key_env: VLLM_API_KEY
+      extra:
+        base_url: ${env.VLLM_URL:=}
+        max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
+        network:
+          tls:
+            verify: ${env.VLLM_TLS_VERIFY:=true}
 ```
 
-Optional:
-
-```env
-VLLM_MAX_TOKENS=<defaults to 4096>
-VLLM_TLS_VERIFY=<defaults to true>
-```
+In order for `vLLM` to configure properly, you must include environment variables named `VLLM_API_KEY` and `VLLM_URL` to the Lightspeed Core container.
 
 ## Ollama
+> [!NOTE]
+> Lightspeed Core does not implement the "official" Ollama provider via Llama stack (remote::ollama), instead we can access it via the vLLM provider.
+>
 
-Required:
+To add the `ollama` inference provider, include the following in [lightspeed-stack.yaml](../lightspeed-core-configs/lightspeed-stack.yaml):
 
-```env
-ENABLE_OLLAMA=true
-OLLAMA_URL=<your-ollama-url>
+```yaml
+inference:
+  providers:
+    - type: vllm
+      id: <your-unique-id>
+      extra:
+        base_url: ${env.OLLAMA_URL:=http://localhost:11434/v1}
 ```
 
 `OLLAMA_URL` guidance:
 
-- If Lightspeed Core runs directly on your host, use `http://localhost:11434`.
-- If Lightspeed Core runs in a container, use `http://host.containers.internal:11434`.
+- If Lightspeed Core runs directly on your host, use `http://localhost:11434/v1`.
+- If Lightspeed Core runs in a container, use `http://host.containers.internal:11434/v1`.
 - On Linux, you may need to open firewall access to the Podman network or run with `--network host`.
 
 ## OpenAI
 
-Required:
+To add the `openai` inference provider, include the following in [lightspeed-stack.yaml](../lightspeed-core-configs/lightspeed-stack.yaml):
 
-```env
-ENABLE_OPENAI=true
-OPENAI_API_KEY=<your-api-key>
+```yaml
+inference:
+  providers:
+    - type: openai
+      id: <your-unique-id>
+      api_key_env: OPENAI_API_KEY
 ```
 
+You must pass the environment variable `OPENAI_API_KEY` to the Lightspeed Core container.
+
 Get your API key from [platform.openai.com](https://platform.openai.com/settings/organization/api-keys).
 
 ## Vertex AI (Gemini)
 
-Required:
+To add the `vertexai` inference provider, include the following in [lightspeed-stack.yaml](../lightspeed-core-configs/lightspeed-stack.yaml):
+
+```yaml
+inference:
+  providers:
+    - type: vertexai
+      id: <your-unique-id>
+      extra:
+        project: ${env.VERTEX_AI_PROJECT:=}
+        location: ${env.VERTEX_AI_LOCATION:=global}
+```
+
+Additionally, you need to ensure your Google Application Credentials are mounted to the Lightspeed Core container and the `GOOGLE_APPLICATION_CREDENTIALS` environment variable is the path to the mount location.
+
+Provider details: [Llama Stack (OGX) Vertex AI docs](https://ogx-ai.github.io/docs/providers/inference/remote_vertexai).
+
+## Restricting Models (`allowed_models`)
+
+Each of the providers above supports an `allowed_models` field to limit which models get registered with Llama Stack. This is most useful for `openai` and `vertexai`, since they auto-discover every model available to your account/project unless restricted.
+
+To use it, add `allowed_models` under the provider's `extra` block in [lightspeed-stack.yaml](../lightspeed-core-configs/lightspeed-stack.yaml).
 
-```env
-ENABLE_VERTEX_AI=true
-VERTEX_AI_PROJECT=
-VERTEX_AI_LOCATION=
-GOOGLE_APPLICATION_CREDENTIALS=
+Open AI example:
+```yaml
+inference:
+  providers:
+    - type: openai
+      id: <your-unique-id>
+      api_key_env: OPENAI_API_KEY
+      extra:
+        allowed_models:
+          - gpt-4o
+          - gpt-4o-mini
 ```
 
-Provider details: [Llama Stack Vertex AI docs](https://llamastack.github.io/v0.2.18/providers/inference/remote_vertexai.html).
+If `allowed_models` is omitted, all models the provider can see are registered.
+
+## Full Example
+
+```yaml
+name: lightspeed-core-stack
+service:
+  host: ${env.SERVICE_HOST:=127.0.0.1}
+  port: 8080
+  auth_enabled: false
+  workers: 1
+  color_log: true
+  access_log: true
+llama_stack:
+  use_as_library_client: true
+  config:
+    profile: /app-root/config.yaml
+inference:
+  providers:
+    - type: sentence_transformers
+    - type: openai
+      id: openai
+      api_key_env: OPENAI_API_KEY
+      extra:
+        allowed_models:
+          - gpt-4o
+          - gpt-4o-mini
+    - type: vllm
+      id: vllm-team
+      api_key_env: VLLM_API_KEY
+      extra:
+        base_url: ${env.VLLM_URL:=}
+        max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
+        network:
+          tls:
+            verify: ${env.VLLM_TLS_VERIFY:=true}
+    - type: vllm
+      id: ollama
+      extra:
+        base_url: ${env.OLLAMA_URL:=http://localhost:11434/v1}
+user_data_collection:
+  feedback_enabled: true
+  feedback_storage: '/tmp/data/feedback'
+authentication:
+  module: 'noop'
+conversation_cache:
+  type: 'sqlite'
+  sqlite:
+    db_path: '/tmp/cache.db'
+customization:
+  profile_path: '/app-root/rhdh-profile.py'
+mcp_servers:
+  - name: mcp-integration-tools
+    provider_id: 'model-context-protocol'
+    url: 'http://localhost:7007/api/mcp-actions/v1'
+    authorization_headers:
+      Authorization: 'client'
+```

env/default-values.env

@@ -2,20 +2,9 @@
 # You do not need to set them all if they will go unused.
 
 # Service Images
-LIGHTSPEED_CORE_IMAGE=quay.io/lightspeed-core/lightspeed-stack:dev-20260702-c62c782
+LIGHTSPEED_CORE_IMAGE=quay.io/lightspeed-core/lightspeed-stack:dev-20260710-2c3e52e
 RAG_CONTENT_IMAGE=quay.io/redhat-ai-dev/rag-content:release-1.10-lls-0.5.0-8c231a3b5177f12fff9db042dfa4091d8f2f26b3
 
-# Enable Inference Providers
-## Set any providers you want enabled to 'true'
-## E.g. ENABLE_VLLM=true
-## Leave all disabled providers EMPTY
-## E.g. ENABLE_OPENAI=
-ENABLE_VLLM=
-ENABLE_VERTEX_AI=
-ENABLE_OPENAI=
-ENABLE_OLLAMA=
-ENABLE_VALIDATION=
-
 # vLLM Inference Settings
 VLLM_URL=
 VLLM_API_KEY=
@@ -35,9 +24,9 @@ GOOGLE_APPLICATION_CREDENTIALS=
 OLLAMA_URL=
 
 # Question Validation Safety Shield Settings
-## Ensure VALIDATION_PROVIDER is one of your enabled Inference Providers
-## Only required for Llama Stack configs that use the Lightspeed Core provider
-## E.g. VALIDATION_PROVIDER=vllm if ENABLE_VLLM=true
+## Ensure VALIDATION_PROVIDER is one of the providers enabled in lightspeed-stack.yaml (or lightspeed-stack.local.yaml)
+## E.g. VALIDATION_PROVIDER=vllm
+ENABLE_VALIDATION=
 VALIDATION_PROVIDER=
 VALIDATION_MODEL_NAME=
 

images.yaml

@@ -15,7 +15,7 @@
 # limitations under the License.
 lightspeed-core:
   llama_version: 0.6.0
-  image: quay.io/lightspeed-core/lightspeed-stack:dev-20260702-c62c782
+  image: quay.io/lightspeed-core/lightspeed-stack:dev-20260710-2c3e52e
   latest_release: v0.6.0rc2 # using v prefix in catalog now it seems
 rag-content:
   llama_version: 0.5.0

lightspeed-core-configs/lightspeed-stack.yaml

@@ -23,7 +23,11 @@ service:
   access_log: true
 llama_stack:
   use_as_library_client: true
-  library_client_config_path: /app-root/config.yaml
+  config:
+    profile: /app-root/config.yaml
+inference:
+  providers:
+    - type: sentence_transformers
 user_data_collection:
   feedback_enabled: true
   feedback_storage: '/tmp/data/feedback'

llama-stack-configs/config.yaml

@@ -36,32 +36,6 @@ providers:
             backend: sql_default
       provider_id: meta-reference
       provider_type: inline::meta-reference
-  inference:
-    - provider_id: ${env.ENABLE_VLLM:+vllm}
-      provider_type: remote::vllm
-      config:
-        base_url: ${env.VLLM_URL:=}
-        api_token: ${env.VLLM_API_KEY:=}
-        max_tokens: ${env.VLLM_MAX_TOKENS:=4096}
-        network:
-          tls:
-            verify: ${env.VLLM_TLS_VERIFY:=true}
-    - provider_id: ${env.ENABLE_OLLAMA:+ollama}
-      provider_type: remote::ollama
-      config:
-        base_url: ${env.OLLAMA_URL:=http://localhost:11434/v1}
-    - provider_id: ${env.ENABLE_OPENAI:+openai}
-      provider_type: remote::openai
-      config:
-        api_key: ${env.OPENAI_API_KEY:=}
-    - provider_id: ${env.ENABLE_VERTEX_AI:+vertexai}
-      provider_type: remote::vertexai
-      config:
-        project: ${env.VERTEX_AI_PROJECT:=}
-        location: ${env.VERTEX_AI_LOCATION:=global}
-    - provider_id: sentence-transformers
-      provider_type: inline::sentence-transformers
-      config: {}
   tool_runtime:
     - provider_id: model-context-protocol
       provider_type: remote::model-context-protocol