diff --git a/.github/workflows/sync-gitops.yml b/.github/workflows/sync-gitops.yml index 9146dad..a0b10f2 100644 --- a/.github/workflows/sync-gitops.yml +++ b/.github/workflows/sync-gitops.yml @@ -21,6 +21,7 @@ on: - main paths: - "lightspeed-core-configs/rhdh-profile.py" + - "lightspeed-core-configs/lightspeed-stack.yaml" - "llama-stack-configs/config.yaml" - "images.yaml" workflow_dispatch: @@ -58,6 +59,7 @@ jobs: - name: Copy generated manifests run: | cp generated/llama-stack-config.yaml ai-rolling-demo-gitops/charts/rhdh/templates/llama-stack-config.yaml + cp generated/lightspeed-stack-config.yaml ai-rolling-demo-gitops/charts/rhdh/templates/lightspeed-stack-config.yaml cp generated/rhdh-profile.py ai-rolling-demo-gitops/charts/rhdh/rhdh-profile.py - name: Export image versions @@ -84,5 +86,5 @@ jobs: Automated sync triggered from [${{ github.repository }}@${{ github.sha }}](https://github.com/${{ github.repository }}/commit/${{ github.sha }}). - Syncs `llama-stack-config.yaml`, `rhdh-profile.py`, and `charts/rhdh/values.yaml` with the following image version: + Syncs `llama-stack-config.yaml`, `lightspeed-stack-config.yaml`, `rhdh-profile.py`, and `charts/rhdh/values.yaml` with the following image version: - `lightspeed-core`: `${{ env.LIGHTSPEED_CORE_IMAGE }}` diff --git a/.gitignore b/.gitignore index 50ca69d..363f308 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ vector_db rag-content values.env +*.local.yaml .venv diff --git a/Makefile b/Makefile index c06a39b..dfb49e3 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,12 @@ endif LOCAL_COMPOSE_FILES := -f compose/compose.yaml +LIGHTSPEED_STACK_CONFIG := lightspeed-core-configs/lightspeed-stack.yaml +ifneq ($(wildcard lightspeed-core-configs/lightspeed-stack.local.yaml),) +LIGHTSPEED_STACK_CONFIG := lightspeed-core-configs/lightspeed-stack.local.yaml +endif +export LIGHTSPEED_STACK_CONFIG + .PHONY: default default: help diff --git a/compose/compose.yaml b/compose/compose.yaml index 8573422..d9a4071 100644 --- a/compose/compose.yaml +++ b/compose/compose.yaml @@ -19,7 +19,7 @@ services: ports: - "8080:8080" volumes: - - ../lightspeed-core-configs/lightspeed-stack.yaml:/app-root/lightspeed-stack.yaml:Z + - ../${LIGHTSPEED_STACK_CONFIG:-lightspeed-core-configs/lightspeed-stack.yaml}:/app-root/lightspeed-stack.yaml:Z - ../lightspeed-core-configs/rhdh-profile.py:/app-root/rhdh-profile.py:Z - ../llama-stack-configs/config.yaml:/app-root/config.yaml:Z - ../rag-content:/rag-content:Z diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 9bd83dc..61c92ef 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -20,6 +20,8 @@ 1. Copy `./env/default-values.env` to `./env/values.env` and fill in any provider-specific values (see [docs/PROVIDERS.md](./PROVIDERS.md)). + To configure inference providers without editing the git-tracked `lightspeed-stack.yaml`, copy `lightspeed-core-configs/lightspeed-stack.yaml` to `lightspeed-core-configs/lightspeed-stack.local.yaml` and make your edits there. `make local-up` mounts the `.local.yaml` file automatically when it's present, otherwise it falls back to `lightspeed-stack.yaml`. `lightspeed-stack.local.yaml` is gitignored, so it's safe to leave provider config there permanently. + 2. Pull the RAG content: ```sh @@ -38,7 +40,7 @@ This starts Lightspeed Core using the mounted config/content below. Lightspeed Core uses mounted config/content in local compose: -- `lightspeed-core-configs/lightspeed-stack.yaml` -> `/app-root/lightspeed-stack.yaml` +- `lightspeed-core-configs/lightspeed-stack.yaml` (or `lightspeed-stack.local.yaml`, if present) -> `/app-root/lightspeed-stack.yaml` - `lightspeed-core-configs/rhdh-profile.py` -> `/app-root/rhdh-profile.py` - `llama-stack-configs/config.yaml` -> `/app-root/config.yaml` - `rag-content/` -> `/rag-content` @@ -96,7 +98,7 @@ To enable validation, set the following in `env/values.env`: | `VALIDATION_PROVIDER` | Yes | Inference provider used by the validation shield, for example `vllm` or `openai` | | `VALIDATION_MODEL_NAME` | Yes | Model name served by the selected inference provider | -You must also enable and configure the referenced inference provider. Examples: +The referenced inference provider must also be present in `lightspeed-stack.yaml` (or `lightspeed-stack.local.yaml`) and configured via its env vars. See [docs/PROVIDERS.md](./PROVIDERS.md). Examples: ### Example: vLLM-backed validation @@ -104,7 +106,6 @@ You must also enable and configure the referenced inference provider. Examples: ENABLE_VALIDATION=true VALIDATION_PROVIDER=vllm VALIDATION_MODEL_NAME= -ENABLE_VLLM=true VLLM_URL= VLLM_API_KEY= ``` diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index 5a5ffd1..9bd5d94 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -8,59 +8,159 @@ Each inference has its own environment variables. You can include all of these i > `VLLM_API_KEY=token` (recommended) > > `VLLM_API_KEY="token"` (can cause parsing issues) +> +> You will notice the `api_key_env` field is not wrapped in curly-braces `{}`. This is due to Lightspeed Core wrapping them internally to curate a proper `{env.xyz}` to pass through to Llama Stack so the keys are not exposed internally. ## vLLM -Required: - -```env -ENABLE_VLLM=true -VLLM_URL=/v1 -VLLM_API_KEY= +To add the `vLLM` inference provider, include the following in [lightspeed-stack.yaml](../lightspeed-core-configs/lightspeed-stack.yaml): + +```yaml +inference: + providers: + - type: vllm + id: + api_key_env: VLLM_API_KEY + extra: + base_url: ${env.VLLM_URL:=} + max_tokens: ${env.VLLM_MAX_TOKENS:=4096} + network: + tls: + verify: ${env.VLLM_TLS_VERIFY:=true} ``` -Optional: - -```env -VLLM_MAX_TOKENS= -VLLM_TLS_VERIFY= -``` +In order for `vLLM` to configure properly, you must include environment variables named `VLLM_API_KEY` and `VLLM_URL` to the Lightspeed Core container. ## Ollama +> [!NOTE] +> Lightspeed Core does not implement the "official" Ollama provider via Llama stack (remote::ollama), instead we can access it via the vLLM provider. +> -Required: +To add the `ollama` inference provider, include the following in [lightspeed-stack.yaml](../lightspeed-core-configs/lightspeed-stack.yaml): -```env -ENABLE_OLLAMA=true -OLLAMA_URL= +```yaml +inference: + providers: + - type: vllm + id: + extra: + base_url: ${env.OLLAMA_URL:=http://localhost:11434/v1} ``` `OLLAMA_URL` guidance: -- If Lightspeed Core runs directly on your host, use `http://localhost:11434`. -- If Lightspeed Core runs in a container, use `http://host.containers.internal:11434`. +- If Lightspeed Core runs directly on your host, use `http://localhost:11434/v1`. +- If Lightspeed Core runs in a container, use `http://host.containers.internal:11434/v1`. - On Linux, you may need to open firewall access to the Podman network or run with `--network host`. ## OpenAI -Required: +To add the `openai` inference provider, include the following in [lightspeed-stack.yaml](../lightspeed-core-configs/lightspeed-stack.yaml): -```env -ENABLE_OPENAI=true -OPENAI_API_KEY= +```yaml +inference: + providers: + - type: openai + id: + api_key_env: OPENAI_API_KEY ``` +You must pass the environment variable `OPENAI_API_KEY` to the Lightspeed Core container. + Get your API key from [platform.openai.com](https://platform.openai.com/settings/organization/api-keys). ## Vertex AI (Gemini) -Required: +To add the `vertexai` inference provider, include the following in [lightspeed-stack.yaml](../lightspeed-core-configs/lightspeed-stack.yaml): + +```yaml +inference: + providers: + - type: vertexai + id: + extra: + project: ${env.VERTEX_AI_PROJECT:=} + location: ${env.VERTEX_AI_LOCATION:=global} +``` + +Additionally, you need to ensure your Google Application Credentials are mounted to the Lightspeed Core container and the `GOOGLE_APPLICATION_CREDENTIALS` environment variable is the path to the mount location. + +Provider details: [Llama Stack (OGX) Vertex AI docs](https://ogx-ai.github.io/docs/providers/inference/remote_vertexai). + +## Restricting Models (`allowed_models`) + +Each of the providers above supports an `allowed_models` field to limit which models get registered with Llama Stack. This is most useful for `openai` and `vertexai`, since they auto-discover every model available to your account/project unless restricted. + +To use it, add `allowed_models` under the provider's `extra` block in [lightspeed-stack.yaml](../lightspeed-core-configs/lightspeed-stack.yaml). -```env -ENABLE_VERTEX_AI=true -VERTEX_AI_PROJECT= -VERTEX_AI_LOCATION= -GOOGLE_APPLICATION_CREDENTIALS= +Open AI example: +```yaml +inference: + providers: + - type: openai + id: + api_key_env: OPENAI_API_KEY + extra: + allowed_models: + - gpt-4o + - gpt-4o-mini ``` -Provider details: [Llama Stack Vertex AI docs](https://llamastack.github.io/v0.2.18/providers/inference/remote_vertexai.html). +If `allowed_models` is omitted, all models the provider can see are registered. + +## Full Example + +```yaml +name: lightspeed-core-stack +service: + host: ${env.SERVICE_HOST:=127.0.0.1} + port: 8080 + auth_enabled: false + workers: 1 + color_log: true + access_log: true +llama_stack: + use_as_library_client: true + config: + profile: /app-root/config.yaml +inference: + providers: + - type: sentence_transformers + - type: openai + id: openai + api_key_env: OPENAI_API_KEY + extra: + allowed_models: + - gpt-4o + - gpt-4o-mini + - type: vllm + id: vllm-team + api_key_env: VLLM_API_KEY + extra: + base_url: ${env.VLLM_URL:=} + max_tokens: ${env.VLLM_MAX_TOKENS:=4096} + network: + tls: + verify: ${env.VLLM_TLS_VERIFY:=true} + - type: vllm + id: ollama + extra: + base_url: ${env.OLLAMA_URL:=http://localhost:11434/v1} +user_data_collection: + feedback_enabled: true + feedback_storage: '/tmp/data/feedback' +authentication: + module: 'noop' +conversation_cache: + type: 'sqlite' + sqlite: + db_path: '/tmp/cache.db' +customization: + profile_path: '/app-root/rhdh-profile.py' +mcp_servers: + - name: mcp-integration-tools + provider_id: 'model-context-protocol' + url: 'http://localhost:7007/api/mcp-actions/v1' + authorization_headers: + Authorization: 'client' +``` \ No newline at end of file diff --git a/env/default-values.env b/env/default-values.env index 4655727..bd2a57c 100644 --- a/env/default-values.env +++ b/env/default-values.env @@ -2,20 +2,9 @@ # You do not need to set them all if they will go unused. # Service Images -LIGHTSPEED_CORE_IMAGE=quay.io/lightspeed-core/lightspeed-stack:dev-20260702-c62c782 +LIGHTSPEED_CORE_IMAGE=quay.io/lightspeed-core/lightspeed-stack:dev-20260710-2c3e52e RAG_CONTENT_IMAGE=quay.io/redhat-ai-dev/rag-content:release-1.10-lls-0.5.0-8c231a3b5177f12fff9db042dfa4091d8f2f26b3 -# Enable Inference Providers -## Set any providers you want enabled to 'true' -## E.g. ENABLE_VLLM=true -## Leave all disabled providers EMPTY -## E.g. ENABLE_OPENAI= -ENABLE_VLLM= -ENABLE_VERTEX_AI= -ENABLE_OPENAI= -ENABLE_OLLAMA= -ENABLE_VALIDATION= - # vLLM Inference Settings VLLM_URL= VLLM_API_KEY= @@ -35,9 +24,9 @@ GOOGLE_APPLICATION_CREDENTIALS= OLLAMA_URL= # Question Validation Safety Shield Settings -## Ensure VALIDATION_PROVIDER is one of your enabled Inference Providers -## Only required for Llama Stack configs that use the Lightspeed Core provider -## E.g. VALIDATION_PROVIDER=vllm if ENABLE_VLLM=true +## Ensure VALIDATION_PROVIDER is one of the providers enabled in lightspeed-stack.yaml (or lightspeed-stack.local.yaml) +## E.g. VALIDATION_PROVIDER=vllm +ENABLE_VALIDATION= VALIDATION_PROVIDER= VALIDATION_MODEL_NAME= diff --git a/images.yaml b/images.yaml index c10c194..76740a3 100644 --- a/images.yaml +++ b/images.yaml @@ -15,7 +15,7 @@ # limitations under the License. lightspeed-core: llama_version: 0.6.0 - image: quay.io/lightspeed-core/lightspeed-stack:dev-20260702-c62c782 + image: quay.io/lightspeed-core/lightspeed-stack:dev-20260710-2c3e52e latest_release: v0.6.0rc2 # using v prefix in catalog now it seems rag-content: llama_version: 0.5.0 diff --git a/lightspeed-core-configs/lightspeed-stack.yaml b/lightspeed-core-configs/lightspeed-stack.yaml index 9eeedbb..b813cca 100644 --- a/lightspeed-core-configs/lightspeed-stack.yaml +++ b/lightspeed-core-configs/lightspeed-stack.yaml @@ -23,7 +23,11 @@ service: access_log: true llama_stack: use_as_library_client: true - library_client_config_path: /app-root/config.yaml + config: + profile: /app-root/config.yaml +inference: + providers: + - type: sentence_transformers user_data_collection: feedback_enabled: true feedback_storage: '/tmp/data/feedback' diff --git a/llama-stack-configs/config.yaml b/llama-stack-configs/config.yaml index 7afd843..f24e926 100644 --- a/llama-stack-configs/config.yaml +++ b/llama-stack-configs/config.yaml @@ -36,32 +36,6 @@ providers: backend: sql_default provider_id: meta-reference provider_type: inline::meta-reference - inference: - - provider_id: ${env.ENABLE_VLLM:+vllm} - provider_type: remote::vllm - config: - base_url: ${env.VLLM_URL:=} - api_token: ${env.VLLM_API_KEY:=} - max_tokens: ${env.VLLM_MAX_TOKENS:=4096} - network: - tls: - verify: ${env.VLLM_TLS_VERIFY:=true} - - provider_id: ${env.ENABLE_OLLAMA:+ollama} - provider_type: remote::ollama - config: - base_url: ${env.OLLAMA_URL:=http://localhost:11434/v1} - - provider_id: ${env.ENABLE_OPENAI:+openai} - provider_type: remote::openai - config: - api_key: ${env.OPENAI_API_KEY:=} - - provider_id: ${env.ENABLE_VERTEX_AI:+vertexai} - provider_type: remote::vertexai - config: - project: ${env.VERTEX_AI_PROJECT:=} - location: ${env.VERTEX_AI_LOCATION:=global} - - provider_id: sentence-transformers - provider_type: inline::sentence-transformers - config: {} tool_runtime: - provider_id: model-context-protocol provider_type: remote::model-context-protocol diff --git a/scripts/generate-gitops-manifests.sh b/scripts/generate-gitops-manifests.sh index 2925a8d..6ad457a 100755 --- a/scripts/generate-gitops-manifests.sh +++ b/scripts/generate-gitops-manifests.sh @@ -50,18 +50,46 @@ data: config.yaml: | HEADER strip_license "${REPO_ROOT}/llama-stack-configs/config.yaml" \ - | awk '/api_key: \$\{env\.OPENAI_API_KEY:=\}/ { - print + | indent +} > "${OUTPUT_DIR}/llama-stack-config.yaml" + +echo "Generating lightspeed-stack ConfigMap..." +{ + cat << 'HEADER' +kind: ConfigMap +apiVersion: v1 +metadata: + name: lightspeed-stack-config + namespace: {{ .Release.Namespace }} +data: + lightspeed-stack.yaml: | +HEADER + strip_license "${REPO_ROOT}/lightspeed-core-configs/lightspeed-stack.yaml" \ + | awk '/^ - type: sentence_transformers$/ { + print " - type: vllm" + print " id: vllm" + print " api_key_env: VLLM_API_KEY" + print " extra:" + print " base_url: ${env.VLLM_URL:=}" + print " max_tokens: ${env.VLLM_MAX_TOKENS:=4096}" + print " network:" + print " tls:" + print " verify: ${env.VLLM_TLS_VERIFY:=true}" + print " - type: openai" + print " id: openai" + print " api_key_env: OPENAI_API_KEY" + print " extra:" print " allowed_models:" print " - gpt-4o-mini" print " - gpt-5.1" print " - gpt-4.1-mini" print " - gpt-4.1-nano" + print next } { print }' \ | indent -} > "${OUTPUT_DIR}/llama-stack-config.yaml" +} > "${OUTPUT_DIR}/lightspeed-stack-config.yaml" echo "Generating rhdh-profile.py..." cp "${REPO_ROOT}/lightspeed-core-configs/rhdh-profile.py" "${OUTPUT_DIR}/rhdh-profile.py"