From df06e85ac9f028d6e6d1b3428ed06ce50bcd8f4e Mon Sep 17 00:00:00 2001 From: Stephanie Date: Thu, 16 Jul 2026 12:07:55 -0400 Subject: [PATCH] enable google provider Signed-off-by: Stephanie --- compose/compose.yaml | 3 ++ docs/PROVIDERS.md | 30 +++++++++++++++++++ env/default-values.env | 11 ++++++- env/gcp-credentials-placeholder.json | 1 + lightspeed-core-configs/lightspeed-stack.yaml | 10 +++++++ 5 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 env/gcp-credentials-placeholder.json diff --git a/compose/compose.yaml b/compose/compose.yaml index d9a4071..5eb2bff 100644 --- a/compose/compose.yaml +++ b/compose/compose.yaml @@ -23,6 +23,9 @@ services: - ../lightspeed-core-configs/rhdh-profile.py:/app-root/rhdh-profile.py:Z - ../llama-stack-configs/config.yaml:/app-root/config.yaml:Z - ../rag-content:/rag-content:Z + # Only used when the vertexai provider is configured. Falls back to a + # harmless empty placeholder so the mount doesn't fail otherwise. + - ${GOOGLE_APPLICATION_CREDENTIALS_HOST_PATH:-../env/gcp-credentials-placeholder.json}:/app-root/gcp-credentials.json:Z,ro environment: - SERVICE_HOST=0.0.0.0 env_file: diff --git a/docs/PROVIDERS.md b/docs/PROVIDERS.md index 9bd5d94..0319f82 100644 --- a/docs/PROVIDERS.md +++ b/docs/PROVIDERS.md @@ -69,6 +69,26 @@ You must pass the environment variable `OPENAI_API_KEY` to the Lightspeed Core c Get your API key from [platform.openai.com](https://platform.openai.com/settings/organization/api-keys). +## Gemini + +There is no dedicated `gemini` provider type in Lightspeed Core's inference schema. Gemini exposes an OpenAI-compatible endpoint, so configure it as an `openai`-type provider with a custom `base_url` instead: + +```yaml +inference: + providers: + - type: openai + id: + api_key_env: GEMINI_API_KEY + extra: + base_url: https://generativelanguage.googleapis.com/v1beta/openai/ +``` + +You must pass the environment variable `GEMINI_API_KEY` to the Lightspeed Core container. + +Get your API key from [Google AI Studio](https://aistudio.google.com/apikey). + +This is the consumer Gemini API (`generativelanguage.googleapis.com`), authenticated with a simple API key. If you need enterprise features (VPC-SC, IAM, billing tied to a GCP project, etc.), use Vertex AI below instead. + ## Vertex AI (Gemini) To add the `vertexai` inference provider, include the following in [lightspeed-stack.yaml](../lightspeed-core-configs/lightspeed-stack.yaml): @@ -85,6 +105,16 @@ inference: Additionally, you need to ensure your Google Application Credentials are mounted to the Lightspeed Core container and the `GOOGLE_APPLICATION_CREDENTIALS` environment variable is the path to the mount location. +To set this up with the provided `compose/compose.yaml`, set `GOOGLE_APPLICATION_CREDENTIALS_HOST_PATH` to the path on your host machine of a GCP service account JSON key (or your `gcloud auth application-default login` credentials file). The compose file mounts that file into the container and points `GOOGLE_APPLICATION_CREDENTIALS` at the mounted path for you — do not set `GOOGLE_APPLICATION_CREDENTIALS` to a host path yourself, since that path won't exist inside the container. + +```env +VERTEX_AI_PROJECT= +VERTEX_AI_LOCATION= +GOOGLE_APPLICATION_CREDENTIALS_HOST_PATH= +``` + +The service account (or `gcloud auth application-default login` credentials) needs the `Vertex AI User` role, and the Vertex AI API must be enabled on `VERTEX_AI_PROJECT`. + Provider details: [Llama Stack (OGX) Vertex AI docs](https://ogx-ai.github.io/docs/providers/inference/remote_vertexai). ## Restricting Models (`allowed_models`) diff --git a/env/default-values.env b/env/default-values.env index bd2a57c..5d29372 100644 --- a/env/default-values.env +++ b/env/default-values.env @@ -15,10 +15,19 @@ VLLM_TLS_VERIFY= # OpenAI Inference Settings OPENAI_API_KEY= +# Gemini Inference Settings +GEMINI_API_KEY= + # Vertex AI Inference Settings VERTEX_AI_PROJECT= VERTEX_AI_LOCATION= -GOOGLE_APPLICATION_CREDENTIALS= +## Host path to your GCP service account JSON key. The compose file mounts +## this into the container for you; do not set GOOGLE_APPLICATION_CREDENTIALS +## to a host path yourself. +GOOGLE_APPLICATION_CREDENTIALS_HOST_PATH= +## In-container path where the credentials above are mounted. Leave as-is +## unless you also change the volume mount in compose/compose.yaml. +GOOGLE_APPLICATION_CREDENTIALS=/app-root/gcp-credentials.json # Ollama Inference Settings OLLAMA_URL= diff --git a/env/gcp-credentials-placeholder.json b/env/gcp-credentials-placeholder.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/env/gcp-credentials-placeholder.json @@ -0,0 +1 @@ +{} diff --git a/lightspeed-core-configs/lightspeed-stack.yaml b/lightspeed-core-configs/lightspeed-stack.yaml index b813cca..fbd3593 100644 --- a/lightspeed-core-configs/lightspeed-stack.yaml +++ b/lightspeed-core-configs/lightspeed-stack.yaml @@ -28,6 +28,16 @@ llama_stack: inference: providers: - type: sentence_transformers + - type: openai + id: gemini + api_key_env: GEMINI_API_KEY + extra: + base_url: https://generativelanguage.googleapis.com/v1beta/openai/ + - type: vertexai + id: vertexai + extra: + project: ${env.VERTEX_AI_PROJECT:=} + location: ${env.VERTEX_AI_LOCATION:=global} user_data_collection: feedback_enabled: true feedback_storage: '/tmp/data/feedback'