diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 342a1b2..8cb86ae 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -58,7 +58,6 @@ jobs: USE_TLS: ${{ matrix.USE_TLS }} USE_LOCAL_POSTGRES: ${{ matrix.USE_LOCAL_POSTGRES }} TIKA_VERSION: ${{ vars.TIKA_VERSION }} - TTM_VERSION: ${{ vars.TTM_VERSION }} SWIRL_VERSION: ${{ vars.SWIRL_VERSION }} steps: diff --git a/README.md b/README.md index e6eaa25..1b7192c 100755 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Please contact [hello@swirlaiconnect.com](mailto:hello@swirlaiconnect.com) for m - [Connecting to Microsoft IDP](doc/service-setup.md#connecting-to-microsoft-idp) - [Connecting to Google IDP](doc/service-setup.md#connecting-to-google-idp) - [Controlling SWIRL Service](doc/controlling-swirl-service.md) -- [Configuring MCP Proxy (optional)](doc/mcp-proxy-setup.md) +- [Configuring the SWIRL MCP Server (optional)](doc/mcp-setup.md) # SWIRL Documentation diff --git a/doc/mcp-proxy-setup.md b/doc/mcp-proxy-setup.md deleted file mode 100644 index cf86ef0..0000000 --- a/doc/mcp-proxy-setup.md +++ /dev/null @@ -1,18 +0,0 @@ -# Overview -The MCP server proxy acts as a TCP server that exposes a set of tools, resources, and prompts for clients to interact with. -It handles authentication, manages client sessions, and delegates requests (such as listing tools, resources, and prompts, or -executing commands) to internal logic. It serves as a gateway for clients to access and interact with the -MCP (Multi-Component Platform) backend services over TCP. - -## Setting up MCP Proxy -* MCP_ENABLED: Set to "true" to enable the MCP proxy service. -* MCP_VERSION: Specifies the version of the MCP proxy to use (e.g., "v1_0_5"). -* MCP_PORT: The port on which the MCP proxy will listen for incoming connections (defaults "9000"). -* SWIRL_MCP_USERNAME: Required authentication username for accessing the MCP proxy. -* SWIRL_MCP_PASSWORD: Required authentication password for accessing the MCP proxy. -* SWIRL_API_USERNAME: The username for the SWIRL API, used for MCP to SWIRL authentication. -* SWIRL_API_PASSWORD: The password for the SWIRL API, used for MCP to SWIRL authentication. -* MCP_SWIRL_BASE_PATH: The base URL path for the SWIRL API that MCP will interact with (e.g., "https://swirl.example.com/api"). -* MCP_TIMEOUT: Optional timeout setting for MCP operations (in seconds, defaults to "30"). - -The MCP proxy runs as a container named `swirl_mcp` and is controlled via the same systemd service as SWIRL. \ No newline at end of file diff --git a/doc/mcp-setup.md b/doc/mcp-setup.md new file mode 100644 index 0000000..5d99494 --- /dev/null +++ b/doc/mcp-setup.md @@ -0,0 +1,33 @@ +# Configuring the SWIRL MCP Server (optional) + +SWIRL 5 includes a built-in MCP (Model Context Protocol) server that exposes federated search, result paging, source discovery, document reading, and RAG answers with citations as tools for Claude, Microsoft Copilot, and any other MCP host. + +In this bundle the MCP server runs as its own container (`swirl_mcp`) from the same SWIRL image, using the streamable HTTP transport on port 8675. + +## Setup + +1. Set `SWIRL_MCP_TOKEN` in `.env` to a long random value (this is in the REQUIRED section). The setup job provisions a matching API token for the `admin` user, and the MCP container authenticates to SWIRL with it. +2. Start the stack with the `all` profile, or add `--profile mcp` to your usual profiles. +3. The MCP endpoint is available at `http://:8675/mcp`. Change the host port with `SWIRL_MCP_PORT` in `.env`. + +## Connecting a client + +Example Claude configuration (streamable HTTP): + +```json +{ + "mcpServers": { + "swirl": { + "type": "http", + "url": "http://:8675/mcp" + } + } +} +``` + +## Security notes + +- Treat `SWIRL_MCP_TOKEN` as a secret: all MCP requests act as the SWIRL user the token belongs to, with that user's source permissions. +- The MCP port has no additional client authentication; restrict it to trusted networks (do not expose 8675 to the public internet). + +Full tool reference: [MCP Server Guide](https://docs.swirlaiconnect.com/MCP-Guide-Enterprise). diff --git a/docker-compose.yml b/docker-compose.yml index 401cdeb..fae413b 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,29 +105,60 @@ services: interval: 10s timeout: 5s retries: 5 + qdrant: + profiles: ["all", "svc", "qdrant"] + image: qdrant/qdrant:${QDRANT_VERSION:-v1.18.2} + container_name: swirl_qdrant + restart: always + networks: ["swirl"] + volumes: + - qdrant_data:/qdrant/storage + + seaweedfs: + profiles: ["all", "svc", "seaweedfs"] + image: chrislusf/seaweedfs:${SEAWEEDFS_VERSION:-4.36} + container_name: swirl_seaweedfs + restart: always + command: ["server", "-dir=/data", "-s3", "-volume.max=0"] + networks: ["swirl"] + volumes: + - seaweed_data:/data + + ollama: + profiles: ["all", "svc", "ollama"] + image: ${OLLAMA_PATH:-swirlai/release-ollama-enterprise}:${OLLAMA_VERSION:-0.30.10} + container_name: swirl_ollama + restart: always + networks: ["swirl"] + healthcheck: + test: ["CMD", "ollama", "list"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 20s + mcp: profiles: [ "all", "mcp" ] - image: swirlai/release-mcp-proxy-enterprise:${MCP_VERSION} + image: ${SWIRL_PATH}:${SWIRL_VERSION} container_name: swirl_mcp + restart: always + env_file: + - .env networks: [ "swirl" ] - ports: - - "${MCP_PORT}:${MCP_PORT}" # expose the MCP port + command: ["python", "-m", "swirl_mcp", "--transport", "http", "--host", "0.0.0.0"] environment: - SWIRL_API_BASE: "${MCP_SWIRL_BASE_URL}" - SWIRL_API_BASE_PATH: "${MCP_SWIRL_BASE_PATH}" - SWIRL_API_PASSWORD: "${SWIRL_API_PASSWORD}" - SWIRL_API_USERNAME: "${SWIRL_API_USERNAME}" - SWIRL_MCP_SERVER_HOST: "0.0.0.0" - SWIRL_MCP_SERVER_PORT: "${MCP_PORT}" - SWIRL_MCP_SWIRL_API_TIMEOUT: "${MCP_TIMEOUT}" - SWIRL_MCP_PASSWORD: "${SWIRL_MCP_PASSWORD}" - SWIRL_MCP_USERNAME: "${SWIRL_MCP_USERNAME}" + SWIRL_MCP_BASE_URL: "http://swirl:${SWIRL_PORT:-8000}" + SWIRL_MCP_TOKEN: ${SWIRL_MCP_TOKEN:-} + ports: + - "${SWIRL_MCP_PORT:-8675}:8675" healthcheck: - test: [ "CMD-SHELL", "/app/scripts/check-mcp-health.sh || exit 1" ] + # The MCP endpoint answers non-protocol GETs with 4xx — any HTTP + # response at all means the server is up. + test: [ "CMD-SHELL", "curl -s -o /dev/null http://localhost:8675/mcp || exit 1" ] interval: 30s - timeout: 10s + timeout: 5s retries: 3 - start_period: 30s + start_period: 15s depends_on: swirl: condition: service_healthy @@ -138,12 +169,6 @@ services: container_name: swirl_tika networks: ["swirl"] - ttm: - profiles: ["all", "svc", "ttm"] - image: swirlai/release-topic-text-matcher-enterprise:${TTM_VERSION} - container_name: swirl_ttm - networks: ["swirl"] - swirl-job: profiles: ["all", "setup"] image: ${SWIRL_PATH}:${SWIRL_VERSION} @@ -182,7 +207,14 @@ services: condition: service_healthy redis-cache: condition: service_healthy - + qdrant: + condition: service_started + seaweedfs: + condition: service_started + tika: + condition: service_started + ollama: + condition: service_started swirl-init: condition: service_completed_successfully command: ["/bin/bash","/tmp/swirl-load.sh"] @@ -203,6 +235,12 @@ volumes: db_data: driver: local name: swirl_db_data + qdrant_data: + driver: local + name: swirl_qdrant_data + seaweed_data: + driver: local + name: swirl_seaweed_data networks: swirl: diff --git a/env.example b/env.example index a0b89bb..a24ecd4 100755 --- a/env.example +++ b/env.example @@ -19,12 +19,14 @@ SQL_USER="" CERTBOT_VERSION= NGINX_VERSION= POSTGRES_VERSION=16 -REDIS_VERSION=8 -SWIRL_VERSION=v4_4_1_1 +REDIS_VERSION=7.4 +SWIRL_VERSION=v5_0_0_0 SWIRL_PATH="swirlai/release-swirl-search-enterprise" -TIKA_VERSION=v4_4_1_1 -TTM_VERSION=v4_4_1_1 -MCP_VERSION=v1_0_6 +TIKA_VERSION=v5_0_0_0 +OLLAMA_PATH="swirlai/release-ollama-enterprise" +OLLAMA_VERSION=0.30.10 +QDRANT_VERSION=v1.18.2 +SEAWEEDFS_VERSION=4.36 ################################################################################ @@ -140,8 +142,24 @@ SWIRL_SVC="swirl" ## RAG SWIRL_RAG_CHAT_INTERACTION_APPROACH="ChatGAIGuided" SWIRL_RAG_DISTRIBUTION_STRATEGY="RoundRobin" -SWIRL_TEXT_SUMMARIZATION_URL="http://ttm:7029" TIKA_SERVER_ENDPOINT="http://tika:9998" +TIKA_CLIENT_ONLY="True" +# OCR for scanned documents: 'auto' runs OCR only on pages lacking text. +SWIRL_TIKA_OCR_STRATEGY="auto" + +## SEMANTIC CACHE SIDECARS (Qdrant vectors + SeaweedFS document storage + local Ollama models) +SWIRL_QDRANT_MODE="external" +SWIRL_QDRANT_URL="http://qdrant:6333" +SWIRL_STORAGE_MODE="external" +SWIRL_STORAGE_BACKEND="s3" +SWIRL_STORAGE_ENDPOINT="http://seaweedfs:8333" +SWIRL_STORAGE_ACCESS_KEY="swirl" +# CHANGE ME : Set to a unique value for this deployment. +SWIRL_STORAGE_SECRET_KEY="" +SWIRL_STORAGE_BUCKET="swirl-cache-bodies" +SWIRL_STORAGE_REGION="us-east-1" +# Local LLM sidecar with SWIRL's models baked in (llama3.1:8b + mxbai-embed-large). +SWIRL_OLLAMA_URL="http://ollama:11434" ## REDIS and CACHES CACHE_REDIS_URL="redis://redis-cache:6379/1" @@ -181,12 +199,15 @@ SEARCH_RESULTS_STORE_TIMEOUT="300" ################################################################################ # SECRETS -## MCP Endpoint Authentication -# CHANGE ME : To the user name that the MCP server will use to login to SWIRL, if using MCP -SWIRL_MCP_USERNAME="" -# CHANGE ME : To the password that the MCP server will use to login to SWIRL, if using MCP -SWIRL_MCP_PASSWORD="" -## User in SWIRL to which MCP requests will be authenticated. +## MCP SERVER (optional) +# Set to true to run SWIRL's built-in MCP server as the swirl_mcp container. +MCP_ENABLED=false +# REQUIRED when MCP_ENABLED=true : a long random value; the MCP container +# authenticates to SWIRL with it (a matching admin API token is provisioned). +SWIRL_MCP_TOKEN="" +# Host port for the MCP endpoint (http://:/mcp). +SWIRL_MCP_PORT="8675" +## Optional additional API user for MCP/agent use. SWIRL_API_USERNAME="" SWIRL_API_PASSWORD="" # CHANGE ME @@ -196,13 +217,14 @@ GOOGLE_APPLICATION_CREDENTIALS="" ################################################################################ # DEPRECATED +SWIRL_MCP_USERNAME="" +SWIRL_MCP_PASSWORD="" GOOGLE_CREDENTIALS="" LOGIN_REDIRECT_URL="" LOGOUT_REDIRECT_URL="" MICROSOFT_CLIENT_ID="" MICROSOFT_CLIENT_SECRET="" MICROSOFT_REDIRECT_URI="" -MCP_ENABLED=false MCP_PORT="9000" MCP_SWIRL_BASE_URL="0.0.0.0" MCP_SWIRL_BASE_PATH="/api/swirl" diff --git a/scripts/swirl-load-job.sh b/scripts/swirl-load-job.sh index 2fb20fd..0b6bea2 100755 --- a/scripts/swirl-load-job.sh +++ b/scripts/swirl-load-job.sh @@ -62,6 +62,15 @@ python swirl.py load_data python swirl.py reload_ai_prompts python swirl.py load_branding +# Point the preloaded Ollama AI Providers at the ollama sidecar +python manage.py reconcile_ollama_url + +# Provision the DRF token the built-in MCP sidecar authenticates with +if [ -n "$SWIRL_MCP_TOKEN" ]; then + echo "MCP Support: provisioning admin token for the MCP sidecar" + python manage.py ensure_token admin +fi + # Optionally SWIRL API User if environment variables are set if [ -n "$SWIRL_API_USERNAME" ] && [ -n "$SWIRL_API_PASSWORD" ]; then echo "MCP Support: Creating SWIRL API user: $SWIRL_API_USERNAME" diff --git a/scripts/swirl-service.sh b/scripts/swirl-service.sh index 4b5bbea..6a2bafc 100755 --- a/scripts/swirl-service.sh +++ b/scripts/swirl-service.sh @@ -104,8 +104,12 @@ if [ -z "${SWIRL_FQDN:-}" ]; then exit 1 fi -if [ -z "${SWIRL_VERSION:-}" ] || [ -z "${TIKA_VERSION:-}" ] || [ -z "${TTM_VERSION:-}" ]; then - error "SWIRL_VERSION, TIKA_VERSION, and TTM_VERSION must all be set in .env file." +if [ -z "${SWIRL_VERSION:-}" ] || [ -z "${TIKA_VERSION:-}" ]; then + error "SWIRL_VERSION and TIKA_VERSION must both be set in .env file." + exit 1 +fi +if [ "${MCP_ENABLED:-false}" = "true" ] && [ -z "${SWIRL_MCP_TOKEN:-}" ]; then + error "MCP_ENABLED is true but SWIRL_MCP_TOKEN is not set in .env file." exit 1 fi