From 73844ce72a93e62c62ece2ddfa3d4469e7129a81 Mon Sep 17 00:00:00 2001 From: janosmurai Date: Thu, 15 Oct 2026 12:15:22 +0200 Subject: [PATCH] feat(kc): add optional local Compose integration (SHD-1075) Signed-off-by: janosmurai --- .github/workflows/ci.yml | 6 + README.md | 4 + compose.kc.yaml | 255 +++++++++++++++++++++++++++ scripts/kc/README.md | 267 +++++++++++++++++++++++++++++ scripts/kc/entrypoint.sh | 34 ++++ scripts/kc/notifications.Caddyfile | 25 +++ scripts/kc/operate.py | 102 +++++++++++ scripts/kc/setup.py | 84 +++++++++ test/kc-compose/test_contract.py | 154 +++++++++++++++++ test/kc-compose/test_operator.py | 130 ++++++++++++++ 10 files changed, 1061 insertions(+) create mode 100644 compose.kc.yaml create mode 100644 scripts/kc/README.md create mode 100644 scripts/kc/entrypoint.sh create mode 100644 scripts/kc/notifications.Caddyfile create mode 100644 scripts/kc/operate.py create mode 100644 scripts/kc/setup.py create mode 100644 test/kc-compose/test_contract.py create mode 100644 test/kc-compose/test_operator.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 25e25d6..cf2d72e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,6 +14,12 @@ jobs: - name: Checkout code uses: actions/checkout@v4 + - name: Test kc Compose helpers + env: + TMPDIR: ${{ runner.temp }} + PYTHONDONTWRITEBYTECODE: '1' + run: python3 -W error::ResourceWarning -B -m unittest discover -s test/kc-compose -v + # Installs the version and components pinned in rust-toolchain.toml. - name: Install Rust toolchain run: | diff --git a/README.md b/README.md index 3568b1b..ae5bdfa 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,10 @@ docker run -p 8080:8080 \ ## Docker compose +For optional local KC storage, indexing and signed MCP retrieval, see +[the KC Compose operator guide](scripts/kc/README.md). Include `compose.kc.yaml` +explicitly; ordinary startup does not require KC images or credentials. + `compose.yaml` mirrors the k8s deployment and runs in one of two modes: ```sh diff --git a/compose.kc.yaml b/compose.kc.yaml new file mode 100644 index 0000000..7fb14cb --- /dev/null +++ b/compose.kc.yaml @@ -0,0 +1,255 @@ +# Explicit opt-in: -f compose.yaml -f compose.kc.yaml --profile server. +# Requires Compose >= 2.24.4 (!override) and operator-supplied local/private images. +# No private code, image builds, account registration or storage credentials in server. +services: + shaide_server: + build: !reset null + pull_policy: never + ports: !override + - "127.0.0.1:${SHAIDE_SERVER_PORT:-8080}:${SHAIDE_SERVER_PORT:-8080}" + environment: + ADMIN_PASSWORD: !reset null + JWT_SECRET: !reset null + shaide_ROOT: /data + KNOWLEDGE_CENTER_URL: http://knowledge-center:8080 + KNOWLEDGE_CENTER_ALLOW_LOCAL_HTTP: "true" + MCP_SERVERS: '[{"name":"knowledge-center","url":"http://kc-mcp:8080"}]' + MCP_COLLECTION_CONTEXT_TARGETS: '[{"server_id":"knowledge-center","url":"http://kc-mcp:8080","issuer":"shaide-local","kc_audience":"kc:knowledge-center","kid":"kc-context-v1","signing_key_file":"/run/secrets/private_key","gateway_token_file":"/run/secrets/gateway_token"}]' + MCP_COLLECTION_CONTEXT_ALLOW_LOCAL_HTTP: "true" + command: [sh, /opt/kc/entrypoint.sh, server] + volumes: !override + - kc-server-state:/data + - ./scripts/kc/entrypoint.sh:/opt/kc/entrypoint.sh:ro + secrets: + - source: kc_admin_password + target: admin_password + - source: kc_jwt_secret + target: jwt_secret + - source: kc_internal_token + target: internal_token + - source: kc_webhook_token + target: webhook_token + - source: kc_private_key + target: private_key + - source: kc_gateway_token + target: gateway_token + networks: [default, kc_api, kc_gateway, kc_notify_server] + cap_drop: [ALL] + cap_add: [DAC_OVERRIDE] + security_opt: [no-new-privileges:true] + init: true + stop_grace_period: 45s + + # Generic vectordb retains its URL/service identity and is not KC's vectors. + vectordb: + ports: !reset [] + control-panel: + user: "${KC_LOCAL_UID:?operator uid required}:${KC_LOCAL_GID:?operator gid required}" + ports: !reset [] + environment: + SESSION_SECRET: !reset null + command: [sh, /opt/kc/entrypoint.sh, control-panel] + volumes: + - ./scripts/kc/entrypoint.sh:/opt/kc/entrypoint.sh:ro + secrets: + - source: kc_session_secret + target: session_secret + webapp: + ports: !reset [] + # Legacy Docker gateway is independent of the server's signed KC gateway. + mcp-gateway: + ports: !reset [] + + knowledge-center: + image: ${SHAIDE_KC_IMAGE:?operator-supplied KC image required} + pull_policy: never + user: "${KC_LOCAL_UID:?operator uid required}:${KC_LOCAL_GID:?operator gid required}" + command: [sh, /opt/kc/entrypoint.sh, kc] + environment: + HOST: 0.0.0.0 + PORT: "8080" + S3_FQDN: kc-rustfs + S3_PORT: "9000" + VECTOR_DB_URL: http://kc-qdrant:6334 + SHAIDE_SERVER_URL: http://shaide_server:${SHAIDE_SERVER_PORT:-8080} + KC_SHAIDE_USERNAME: ${KC_SHAIDE_USERNAME:?dedicated role=user service account required} + RUSTFS_WEBHOOK_ARN: arn:rustfs:sqs:eu-central-1:shaide:webhook + KC_CONTEXT_ISSUER: shaide-local + KC_CONTEXT_TARGET: knowledge-center + KC_CONTEXT_AUDIENCE: kc:knowledge-center + KC_CONTEXT_MCP_AUDIENCE: kc-mcp:knowledge-center + KC_CONTEXT_KID: kc-context-v1 + KC_CONTEXT_PUBLIC_KEY_FILE: /run/secrets/public_key + KC_MCP_SCOPED_TOKEN_FILE: /run/secrets/scoped_token + KC_CONTEXT_ALLOW_LOCAL_HTTP: "true" + XBERG_CACHE_DIR: /tmp/xberg + volumes: + - ./scripts/kc/entrypoint.sh:/opt/kc/entrypoint.sh:ro + secrets: + - source: kc_internal_token + target: internal_token + - source: kc_embedding_password + target: embedding_password + - source: kc_storage_access_key + target: storage_access_key + - source: kc_storage_secret_key + target: storage_secret_key + - source: kc_public_key + target: public_key + - source: kc_scoped_token + target: scoped_token + networks: [kc_api, kc_storage, kc_vectors, kc_mcp] + depends_on: + kc-rustfs: + condition: service_started + kc-qdrant: + condition: service_started + healthcheck: + test: [CMD, shaide_knowledge_center, healthcheck] + interval: 15s + timeout: 5s + start_period: 15s + retries: 3 + read_only: true + tmpfs: [/tmp] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + mem_limit: 1g + cpus: 2 + init: true + stop_grace_period: 45s + + kc-mcp: + image: ${SHAIDE_KC_MCP_IMAGE:?operator-supplied KC-MCP image required} + pull_policy: never + user: "${KC_LOCAL_UID:?operator uid required}:${KC_LOCAL_GID:?operator gid required}" + environment: + HOST: 0.0.0.0 + PORT: "8080" + KNOWLEDGE_CENTER_URL: http://knowledge-center:8080 + KC_CONTEXT_ISSUER: shaide-local + KC_CONTEXT_TARGET: knowledge-center + KC_CONTEXT_AUDIENCE: kc-mcp:knowledge-center + KC_CONTEXT_KC_AUDIENCE: kc:knowledge-center + KC_CONTEXT_KID: kc-context-v1 + KC_CONTEXT_PUBLIC_KEY_FILE: /run/secrets/public_key + KC_MCP_GATEWAY_TOKEN_FILE: /run/secrets/gateway_token + KC_MCP_SCOPED_TOKEN_FILE: /run/secrets/scoped_token + KC_CONTEXT_ALLOW_LOCAL_HTTP: "true" + KC_MCP_ALLOWED_HOSTS: kc-mcp + secrets: + - source: kc_public_key + target: public_key + - source: kc_gateway_token + target: gateway_token + - source: kc_scoped_token + target: scoped_token + networks: [kc_gateway, kc_mcp] + depends_on: + knowledge-center: + condition: service_healthy + read_only: true + tmpfs: [/tmp] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + mem_limit: 256m + cpus: 1 + init: true + stop_grace_period: 35s + + kc-rustfs: + image: ${SHAIDE_RUSTFS_IMAGE:?operator-supplied RustFS image required} + pull_policy: never + # Root can read 0600 host-bound secrets and initialize only its named volume. + # Only DAC_OVERRIDE reads operator-owned 0600 secrets; no Docker socket. + user: "0:0" + entrypoint: [sh, /opt/kc/entrypoint.sh, rustfs] + environment: + RUSTFS_VOLUMES: /data + RUSTFS_ADDRESS: :9000 + RUSTFS_CONSOLE_ENABLE: "false" + RUSTFS_OBS_LOG_DIRECTORY: /logs + RUSTFS_NOTIFY_ENABLE: "true" + RUSTFS_NOTIFY_WEBHOOK_ENABLE_SHAIDE: "true" + RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SHAIDE: http://kc-notifications:8080/v1/object-storage/event + RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_SHAIDE: /data/notification-queue + RUSTFS_OUTBOUND_ALLOW_ORIGINS: http://kc-notifications:8080 + volumes: + - kc-object-state:/data + - ./scripts/kc/entrypoint.sh:/opt/kc/entrypoint.sh:ro + secrets: + - source: kc_storage_access_key + target: storage_access_key + - source: kc_storage_secret_key + target: storage_secret_key + - source: kc_webhook_token + target: webhook_token + networks: [kc_storage, kc_notify_storage] + read_only: true + tmpfs: [/tmp, /logs] + cap_drop: [ALL] + cap_add: [DAC_OVERRIDE] + security_opt: [no-new-privileges:true] + mem_limit: 1g + cpus: 2 + init: true + stop_grace_period: 45s + + kc-qdrant: + image: ${SHAIDE_KC_QDRANT_IMAGE:-qdrant/qdrant} + pull_policy: never + volumes: + - kc-vector-state:/qdrant/storage + networks: [kc_vectors] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + mem_limit: 512m + cpus: 1 + stop_grace_period: 45s + + # One fixed POST route bridges notification delivery, never S3 operations. + kc-notifications: + image: ${SHAIDE_KC_NOTIFICATION_IMAGE:-caddy:2} + pull_policy: never + user: "${KC_LOCAL_UID:?operator uid required}:${KC_LOCAL_GID:?operator gid required}" + command: [caddy, run, --config, /etc/caddy/Caddyfile, --adapter, caddyfile] + environment: + SHAIDE_SERVER_PORT: ${SHAIDE_SERVER_PORT:-8080} + volumes: + - ./scripts/kc/notifications.Caddyfile:/etc/caddy/Caddyfile:ro + networks: [kc_notify_storage, kc_notify_server] + read_only: true + tmpfs: [/tmp, /data, /config] + cap_drop: [ALL] + security_opt: [no-new-privileges:true] + # Caddy binary has this file capability even when listening on port 8080. + cap_add: [NET_BIND_SERVICE] + mem_limit: 128m + cpus: 1 + stop_grace_period: 35s + +networks: + kc_api: {internal: true} + kc_storage: {internal: true} + kc_vectors: {internal: true} + kc_gateway: {internal: true} + kc_mcp: {internal: true} + kc_notify_storage: {internal: true} + kc_notify_server: {internal: true} +volumes: + kc-server-state: + kc-object-state: + kc-vector-state: +secrets: + kc_admin_password: {file: '${KC_SECRETS_DIR:?protected credential directory required}/admin_password'} + kc_jwt_secret: {file: '${KC_SECRETS_DIR:?required}/jwt_secret'} + kc_session_secret: {file: '${KC_SECRETS_DIR:?required}/session_secret'} + kc_internal_token: {file: '${KC_SECRETS_DIR:?required}/internal_token'} + kc_webhook_token: {file: '${KC_SECRETS_DIR:?required}/webhook_token'} + kc_private_key: {file: '${KC_SECRETS_DIR:?required}/private.pem'} + kc_public_key: {file: '${KC_SECRETS_DIR:?required}/public.pem'} + kc_gateway_token: {file: '${KC_SECRETS_DIR:?required}/gateway_token'} + kc_scoped_token: {file: '${KC_SECRETS_DIR:?required}/scoped_token'} + kc_embedding_password: {file: '${KC_SECRETS_DIR:?required}/embedding_password'} + kc_storage_access_key: {file: '${KC_SECRETS_DIR:?required}/storage_access_key'} + kc_storage_secret_key: {file: '${KC_SECRETS_DIR:?required}/storage_secret_key'} diff --git a/scripts/kc/README.md b/scripts/kc/README.md new file mode 100644 index 0000000..63d117d --- /dev/null +++ b/scripts/kc/README.md @@ -0,0 +1,267 @@ +# Optional local Knowledge Center + +Include `compose.kc.yaml` explicitly to enable the product's public upload facade, +private Knowledge Center (KC), RustFS, dedicated Qdrant and signed KC-MCP retrieval. +The normal `compose.yaml`, profiles, Control Panel and generic `vectordb` behavior +are unchanged when this overlay is absent. No KC configuration is needed then. + +## Prerequisites + +- Docker Compose **2.24.4 or newer** (`!override` / `!reset` support), Python 3.10+ + and OpenSSL on the operator host. No sudo is needed by these helpers. +- Locally available, trusted server, private KC and private KC-MCP images that + implement the public upload/internal storage and signed collection-context + contracts. KC must contain `shaide_knowledge_center`, `sh`, and its extraction + runtime; KC-MCP must start `kc-mcp`. The server must contain `shaide` and `sh`. + Supply your own image references; this repository does not publish the private + images, promise a private registry tag, or build/pull them automatically. +- A locally available RustFS image with `/entrypoint.sh`, `rustfs`, `sh`, webhook + notifications and the `RUSTFS_NOTIFY_*_SHAIDE` settings below; Qdrant with gRPC on + 6334; and Caddy 2 for the notification relay. Pin trusted image digests in your + local configuration after validating compatibility. `pull_policy: never` makes + missing images a clear prerequisite failure. +- An authorized embedding provider with known dimensions, a dedicated **role=user** + shaide service account, and a registered embedding model. The setup helper only + creates files. Account/model registration is a separate explicit operator action. + Do not use the admin account for KC embedding or register real accounts for a test. +- A stable, private credential directory outside the source tree, and enough disk + for Docker's persistent volumes. The new server database is in its own named + volume; existing native/container databases are not imported or migrated. + +## Create protected files and start + +Choose a new destination whose parent already exists and is not a symlink. Replace +all angle-bracket placeholders with your actual local image references and path. +Image references and paths contain no passwords. The helper refuses existing files, +directories, symlinks and dangling symlinks, including symlink parents. It creates +its directory with mode 0700 and every file exclusively with mode 0600, regardless +of umask, and removes only its newly created files if setup fails. It never updates, +truncates or changes permissions on existing credentials. + +```sh +export SHAIDE_SERVER_IMAGE='' +python3 scripts/kc/setup.py '' \ + --kc-image '' \ + --kc-mcp-image '' \ + --rustfs-image '' +``` + +Set `KC_CONFIG` to the generated `config.env` path (a path, never a secret). The +shell function ensures every command uses the same project, overlay and settings: + +```sh +export KC_CONFIG='/config.env' +export SHAIDE_SERVER_PORT=18081 # choose an unused port; do not restart another stack +kc_compose() { + docker compose --env-file "$KC_CONFIG" -p shaide-kc-local \ + -f compose.yaml -f compose.kc.yaml --profile server "$@" +} +kc_compose config --quiet +kc_compose up -d --no-build --pull never +kc_compose ps +``` + +Use an unused **project name** for a new installation. Never reuse the name of a +running stack. The setup writes host UID/GID to `config.env` so nonroot KC, KC-MCP +and Control Panel can read their mounted mode-0600 files. The server and RustFS +run as root with all capabilities dropped except `DAC_OVERRIDE`, needed for these +operator-owned secret binds; they have no Docker socket or privileged mode. +Caddy retains only `NET_BIND_SERVICE`, which its executable file capability requires +even when listening on the internal port 8080. +The generated server admin password, JWT secret and Control Panel session secret +replace the base demo defaults only when the overlay is included. Container wrappers +read files into the component's required environment immediately before execution; +no secret is placed in Compose configuration, command arguments or helper output. +Do not enable shell tracing or publish process environments, secret files or logs. +Compose file-backed secrets are protected bind mounts, not an encrypted secret store. +The local Docker operator can inspect them and controls this trust boundary. + +`SHAIDE_KC_QDRANT_IMAGE` and `SHAIDE_KC_NOTIFICATION_IMAGE` optionally override the +locally present Qdrant and Caddy images. `KC_SHAIDE_USERNAME` defaults to +`kc-embedding-service`; if changed in `config.env`, pass the same `--username` to +`operate.py`. Keep the generated keys and tokens stable across restarts. Losing the +JWT secret invalidates sessions; changing storage credentials requires coordinated +RustFS/KC configuration, and changing signing keys/tokens requires coordinating +all participating services. The helper deliberately has no automatic rotation. + +The overlay removes host ports from Control Panel, webapp, generic vectordb and the +legacy optional Docker MCP gateway. Client ingress is **only** +`http://127.0.0.1:18081` (or your chosen server port); Control Panel is proxied by the +server at `/control-panel`. Add `--profile app` and `WEBAPP_URL=http://webapp:8787` to the function's +Compose command/environment to enable the existing `/app` webapp proxy. +The overlay runs `shaide` directly and replaces the base cloud credential mounts +with named database state; external HTTP providers work without host credential +mounts. Providers requiring cloud SDK/workload credentials need a private local +override with their documented runtime dependencies and read-only credential binds. +Never copy such credentials into this repository. + +## Explicit service account and model setup + +For a **new local instance**, after server login is available, create the dedicated +service account through the actual admin API. The helper loads the protected +`admin_password` and `embedding_password` files, logs in, calls `POST /v1/user`, and +prints neither the password nor bearer token. Choose a future RFC3339 expiry that +matches your policy. It does not reset an existing user on conflict. Skip creation +when the account already exists; verify its password/expiry instead. + +```sh +python3 scripts/kc/operate.py \ + --credentials '' --server http://127.0.0.1:18081 \ + create-service-account --expiry '' +``` + +For an existing account, configure `KC_SHAIDE_USERNAME` and replace the generated +embedding password **deliberately through your own secure credential workflow** +before start. The setup helper never touches existing credentials. No collection +memberships or administrator role are needed for the embedding service account. + +Create an operator-owned mode-0600 JSON file outside source with the exact registered +provider URL, model name, known vector size and provider/schema identifiers: + +```json +{ + "url": "", + "name": "", + "vector_size": 1024, + "platform": "", + "api_schema": "open_ai", + "daily_input_token_limit": 100000 +} +``` + +1024 is illustrative; use the provider's verified dimension, never infer it. +The supported platform/schema and provider authentication must match your server +image. The JSON is read from a protected regular file, not passed as command text. +Do not put credentials in endpoint URLs. Registration calls the real +`POST /v1/embedding_model` API and is not run at startup: + +```sh +python3 scripts/kc/operate.py \ + --credentials '' --server http://127.0.0.1:18081 \ + register-embedding --model-file '' +python3 scripts/kc/operate.py \ + --credentials '' --server http://127.0.0.1:18081 \ + check --model '' --vector-size '' +``` + +`check` verifies service login returns role=user, checks the catalog, and actually +embeds synthetic text through `POST /v1/embeddings`, including dimension checks. +It consumes a small amount of provider quota. It creates no account or model. +Account creation, model registration and this check were exercised with synthetic +disposable credentials and an embedding double through the real server; this does +not validate your provider or grant permission to register real users. No license +activation or private licensing workflow is automated here. + +## Traffic, trust and notifications + +| Path | Services and authorization | +| --- | --- | +| Upload | client → public server `/v1/object-storage/presigned-url`, then relative `/s3/...` PUT → authenticated KC internal storage facade → signed RustFS write | +| Notification | RustFS → fixed POST-only Caddy relay → server `/v1/object-storage/event`, authenticated by a separate webhook token | +| Index | server lifecycle orchestration → KC extraction → service-account login/embedding through server → private KC Qdrant | +| Retrieval | client → server `/v1/mcp/proxy/knowledge-center/mcp` → KC-MCP → KC → private KC Qdrant | + +Server and RustFS share **no network**. The relay shares a separate network with +each, accepts only the exact notification POST path, and cannot proxy bucket, +object or arbitrary-target requests. RustFS shares no network with KC-MCP or the +generic server Qdrant. KC has separate API, storage, vector and MCP networks; only +the existing server/frontend/generic services join the ordinary network. All added +networks are internal. No optional backend ports are published. + +Only KC and RustFS receive `storage_access_key` / `storage_secret_key`. KC creates +buckets, configures notifications, signs uploads, checks objects and deletes stored +objects/vectors through the frozen component APIs. The server's incoming +`S3_WEBHOOK_TOKEN` is notification authentication, not a storage credential. +RustFS notification target `SHAIDE` uses +`arn:rustfs:sqs:eu-central-1:shaide:webhook`, queue directory +`/data/notification-queue` and the fixed relay endpoint. KC provisions each bucket's +notification configuration when the server creates its collection; no root-key +bootstrap job performs storage operations on the server's behalf. + +The signed context target id is `knowledge-center`, issuer `shaide-local`, key id +`kc-context-v1`; server signs with the private RSA key, KC-MCP/KC verify the public +key. Audiences are respectively `kc-mcp:knowledge-center` and `kc:knowledge-center`. +The server/KC-MCP gateway token and KC-MCP/KC scoped token are separate from the +server/KC lifecycle token, webhook token, JWT secret and embedding password. Local +HTTP is allowed explicitly only for this private Compose topology. Clients supply +only their shaide bearer login, not downstream service/admin credentials. Do not +send fabricated membership headers or add unsigned retrieval callbacks. +The existing legacy `--profile mcp` Docker gateway is independent and is not needed +for KC; KC uses the server's existing signed MCP gateway implementation. + +## Liveness, readiness and recovery + +`knowledge-center`'s container healthcheck calls its built-in `healthcheck` +(`GET /v1/health`). This is **liveness only**: it checks neither storage, vectors, +embedding account nor provider. KC-MCP waits for that liveness check, not full +readiness. Storage/vector startup uses `service_started` to avoid a dependency +cycle with account provisioning/server notifications. The server's `/v1/health`, +relay `/health` and Qdrant's `/readyz` are useful individual checks; none proves +that the complete product can process or retrieve documents. KC-MCP has no +standalone readiness endpoint in this component version. `ps` showing running or +healthy is insufficient. + +After the embedding check, verify a disposable synthetic collection through the +public Control Panel/server: create a collection with the registered model and +memberships, request a five-minute relative upload URL, upload a small synthetic +PDF/DOCX through the public server, and wait for its file status to be `processed`. +Use the public signed MCP URL above to call `search_knowledge_center` and verify a +citation only for the caller's memberships. Verify file deletion through the public +API removes its object/vectors while preserving unrelated files. Do not expose +RustFS or KC to accomplish these steps. The operator helper does not auto-create +collections or upload fixtures. + +Inspect component status and sanitized logs for missing images/secrets, account +expiry/password, model dimensions/provider quota, RustFS notification reachability +and object/vector failures. A dependency outage must fail explicitly, never be +reported as processed or a successful empty retrieval. If an operation is marked +uncertain/quarantined, a timeout does not prove its remote writes stopped. Follow +the component's explicit reconciliation procedure before retry/delete/model changes; +do not clear persisted lifecycle locks by age or edit database rows to force success. + +## Preserve state on stop and restart + +Keep the same project name, configuration and volume names: + +```sh +kc_compose stop +kc_compose start +# Or remove only containers/networks, preserving all named volumes: +kc_compose down +kc_compose up -d --no-build --pull never +``` + +`kc-server-state` holds `/data/db/shaide-server.sqlite` and SQLite WAL state; +`kc-object-state` holds objects plus the notification queue; +`kc-vector-state` holds KC Qdrant data. KC has no metadata database. The generic +`vectordb` remains a separate service and URL; its preexisting persistence behavior +is not changed. Do not use `down --volumes`, prune volumes, delete credentials, or +switch project names to recover a failed startup. Before attaching preexisting +named volumes, inventory their exact names and compatibility and make a coordinated +backup. The overlay does not discover, rename, replace or restore another stack's +volumes, and does not import the old server database. + +Integration tests used both external disk bind overrides and the actual product +named volumes, and verified retained metadata/object/vector state after down/up. +The named-volume run also verifies the product permission configuration without +bind-mount ownership allowances. This proves restart persistence on the +**same disk**, not backup/restore, disaster recovery, migration or release-image +validation. Back up server DB/WAL, objects/notification queue, vectors and protected +credentials consistently while stopped before any destructive maintenance. Restore +and migration are separate operator procedures and have not been validated here. + +## Local verification + +```sh +# Use a temp/cache directory outside source with sufficient capacity. +TMPDIR='' python3 -m unittest discover -s test/kc-compose -v +# Ordinary startup still parses with no KC settings: +docker compose -f compose.yaml --profile server config --quiet +``` + +Tests create only disposable synthetic files and run Compose parsing; they do not +start or mutate a running stack. They cover exclusive creation, existing files and +symlinks, failure cleanup, secret-output absence, private network/port contracts, +signed trust configuration and persistent mount declarations. Runtime validation +uses a test-only override outside source with already verified native binaries and +local images, never a build or private binary/source in the public repository. diff --git a/scripts/kc/entrypoint.sh b/scripts/kc/entrypoint.sh new file mode 100644 index 0000000..955dbfc --- /dev/null +++ b/scripts/kc/entrypoint.sh @@ -0,0 +1,34 @@ +#!/bin/sh +# Components currently read these credentials from env, not *_FILE. Do not trace. +set -eu +read_secret() { + value=$(cat "/run/secrets/$1") + [ -n "$value" ] || { echo 'Missing or empty secret file' >&2; exit 1; } +} +case "$1" in + server) + read_secret admin_password; export ADMIN_PASSWORD="$value" + read_secret jwt_secret; export JWT_SECRET="$value" + read_secret internal_token; export KNOWLEDGE_CENTER_INTERNAL_TOKEN="$value" + read_secret webhook_token; export S3_WEBHOOK_TOKEN="$value" + unset value + exec shaide ;; + kc) + read_secret internal_token; export KNOWLEDGE_CENTER_INTERNAL_TOKEN="$value" + read_secret embedding_password; export KC_SHAIDE_PASSWORD="$value" + read_secret storage_access_key; export S3_USER="$value" + read_secret storage_secret_key; export S3_PASSWORD="$value" + unset value + exec shaide_knowledge_center ;; + rustfs) + read_secret storage_access_key; export RUSTFS_ACCESS_KEY="$value" + read_secret storage_secret_key; export RUSTFS_SECRET_KEY="$value" + read_secret webhook_token; export RUSTFS_NOTIFY_WEBHOOK_AUTH_TOKEN_SHAIDE="$value" + unset value + exec /entrypoint.sh rustfs ;; + control-panel) + read_secret session_secret; export SESSION_SECRET="$value" + unset value + exec node server.js ;; + *) echo 'Unknown component' >&2; exit 1 ;; +esac diff --git a/scripts/kc/notifications.Caddyfile b/scripts/kc/notifications.Caddyfile new file mode 100644 index 0000000..5bc6b2c --- /dev/null +++ b/scripts/kc/notifications.Caddyfile @@ -0,0 +1,25 @@ +{ + admin off + auto_https off +} +:8080 { + @event { + path /v1/object-storage/event + method POST + } + handle @event { + # Fixed destination; incoming bearer token is authenticated by shaide. + reverse_proxy shaide_server:{$SHAIDE_SERVER_PORT:8080} { + transport http { + dial_timeout 5s + response_header_timeout 120s + } + } + } + handle /health { + respond "ok" 200 + } + handle { + respond 404 + } +} diff --git a/scripts/kc/operate.py b/scripts/kc/operate.py new file mode 100644 index 0000000..b1b861c --- /dev/null +++ b/scripts/kc/operate.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Explicit local account/model setup and authenticated readiness. Never prints tokens.""" +import argparse +import json +import os +from pathlib import Path +import stat +import sys +import urllib.error +import urllib.parse +import urllib.request + + +def protected_read(path): + fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW) + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or stat.S_IMODE(info.st_mode) != 0o600 or info.st_uid != os.getuid(): + raise ValueError('input must be an operator-owned regular file with mode 0600') + with os.fdopen(fd, closefd=False) as stream: + return stream.read().strip() + finally: + os.close(fd) + + +class RejectRedirects(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + # Never forward account passwords, bearer tokens or provider input. + return None + + +def request(base, method, path, body=None, token=None): + headers = {'Content-Type': 'application/json'} + if token: + headers['Authorization'] = 'Bearer ' + token + raw = None if body is None else json.dumps(body).encode() + req = urllib.request.Request(base + path, data=raw, method=method, headers=headers) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), RejectRedirects()) + try: + with opener.open(req, timeout=30) as response: + return json.load(response) + except urllib.error.HTTPError as error: + # Server/provider messages can contain rejected URLs. Print only status. + error.close() + raise ValueError(f'API request failed with HTTP {error.code}') from None + + +def run(args): + url = urllib.parse.urlsplit(args.server) + if url.scheme != 'http' or url.hostname not in ('127.0.0.1', 'localhost', '::1') or url.username or url.password or url.query or url.fragment or url.path not in ('', '/'): + raise ValueError('server must be a loopback HTTP origin') + if args.action == 'check' and not 1 <= args.vector_size <= 65536: + raise ValueError('vector size must be within 1..65536') + base = args.server.rstrip('/') + dest = args.credentials + if args.action in ('create-service-account', 'register-embedding'): + login = request(base, 'POST', '/v1/login', {'username': 'admin', 'password': protected_read(dest / 'admin_password')}) + if login['role'] != 'admin': raise ValueError('administrator login required') + if args.action == 'create-service-account': + body = {'username': args.username, 'password': protected_read(dest / 'embedding_password'), 'expiry': args.expiry} + request(base, 'POST', '/v1/user', body, login['access_token']) + print('Created service account. Run check before using KC.') + else: + body = json.loads(protected_read(args.model_file)) + request(base, 'POST', '/v1/embedding_model', body, login['access_token']) + print('Registered embedding model. Verify its dimensions and provider readiness.') + else: + login = request(base, 'POST', '/v1/login', {'username': args.username, 'password': protected_read(dest / 'embedding_password')}) + if login['role'] != 'user': raise ValueError('KC requires a role=user service account') + catalog = request(base, 'GET', '/v1/embedding_models', token=login['access_token']) + if not any(m['name'] == args.model for m in catalog['models']): + raise ValueError('required embedding model is not registered') + response = request(base, 'POST', '/v1/embeddings', {'model': args.model, 'dimensions': args.vector_size, 'input': ['Synthetic local readiness check.']}, login['access_token']) + vectors = response.get('data', []) + if len(vectors) != 1 or len(vectors[0].get('embedding', [])) != args.vector_size: + raise ValueError('provider returned unexpected embedding dimensions') + print('Service login and actual embedding readiness passed. Storage/MCP still require an end-to-end smoke check.') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--credentials', required=True, type=Path) + parser.add_argument('--server', required=True) + parser.add_argument('--username', default='kc-embedding-service') + sub = parser.add_subparsers(dest='action', required=True) + account = sub.add_parser('create-service-account') + account.add_argument('--expiry', required=True, help='explicit future RFC3339 account expiry') + model = sub.add_parser('register-embedding') + model.add_argument('--model-file', required=True, type=Path) + check = sub.add_parser('check') + check.add_argument('--model', required=True) + check.add_argument('--vector-size', required=True, type=int) + args = parser.parse_args() + try: + run(args) + except (OSError, ValueError, KeyError, urllib.error.URLError): + print('Operator action failed. Check protected inputs, server, account, model and provider readiness.', file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': sys.exit(main()) diff --git a/scripts/kc/setup.py b/scripts/kc/setup.py new file mode 100644 index 0000000..5fd8366 --- /dev/null +++ b/scripts/kc/setup.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Create a NEW operator-owned credential directory. Never update existing state.""" +import argparse +import os +from pathlib import Path +import re +import secrets +import subprocess +import sys + +TOKENS = ('admin_password', 'jwt_secret', 'session_secret', 'internal_token', + 'webhook_token', 'gateway_token', 'scoped_token', 'embedding_password', + 'storage_access_key', 'storage_secret_key') + + +def write_exclusive(path, content): + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) + try: + os.fchmod(fd, 0o600) + with os.fdopen(fd, 'wb', closefd=False) as stream: + stream.write(content) + stream.flush() + os.fsync(stream.fileno()) + finally: + os.close(fd) + + +def generate(destination, kc_image, mcp_image, rustfs_image): + for image in (kc_image, mcp_image, rustfs_image): + if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._/:@-]*', image): + raise ValueError('image reference must contain only image-reference characters') + destination = Path(os.path.abspath(destination)) + for parent in destination.parents: + if parent.is_symlink(): + raise ValueError('symlink parents are not allowed') + # A single mkdir is the ownership boundary: EEXIST, including dangling + # symlinks, returns before chmod, writes or cleanup can touch user state. + destination.mkdir(mode=0o700) + try: + destination.chmod(0o700) + for name in TOKENS: + write_exclusive(destination / name, (secrets.token_urlsafe(36) + '\n').encode()) + # OpenSSL writes only to captured memory; we create files exclusively. + key = subprocess.run(['openssl', 'genpkey', '-algorithm', 'RSA', '-pkeyopt', + 'rsa_keygen_bits:2048'], check=True, capture_output=True).stdout + public = subprocess.run(['openssl', 'pkey', '-pubout'], input=key, + check=True, capture_output=True).stdout + write_exclusive(destination / 'private.pem', key) + write_exclusive(destination / 'public.pem', public) + # Compose single-quoted values preserve literal dollars in paths. + path = str(destination) + if any(c in path for c in "'\n\r"): + raise ValueError('destination cannot contain quotes or newlines') + env = (f"KC_SECRETS_DIR='{path}'\nKC_LOCAL_UID={os.getuid()}\nKC_LOCAL_GID={os.getgid()}\n" + f'SHAIDE_KC_IMAGE={kc_image}\nSHAIDE_KC_MCP_IMAGE={mcp_image}\n' + f'SHAIDE_RUSTFS_IMAGE={rustfs_image}\nKC_SHAIDE_USERNAME=kc-embedding-service\n') + write_exclusive(destination / 'config.env', env.encode()) + except BaseException: + # Only files in the newly created private directory are owned here. + for child in destination.iterdir(): + child.unlink() + destination.rmdir() + raise + return destination + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('destination', type=Path) + parser.add_argument('--kc-image', required=True) + parser.add_argument('--kc-mcp-image', required=True) + parser.add_argument('--rustfs-image', required=True) + args = parser.parse_args() + try: + generate(args.destination, args.kc_image, args.kc_mcp_image, args.rustfs_image) + except (OSError, ValueError, subprocess.SubprocessError): + print('Setup failed; use a new nonsymlink destination and check OpenSSL/image references.', file=sys.stderr) + return 1 + print('Created protected configuration. No accounts or models were registered.') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/kc-compose/test_contract.py b/test/kc-compose/test_contract.py new file mode 100644 index 0000000..a1bc912 --- /dev/null +++ b/test/kc-compose/test_contract.py @@ -0,0 +1,154 @@ +"""No Cargo, image builds or running-stack mutations; disposable setup/config checks.""" +import importlib.util +import json +import os +from pathlib import Path +import stat +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +SPEC = importlib.util.spec_from_file_location('setup', ROOT / 'scripts/kc/setup.py') +setup = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(setup) + +class SetupTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(dir=os.environ.get('TMPDIR')) + self.root = Path(self.tmp.name) + self.dest = self.root / 'credentials' + def tearDown(self): + self.tmp.cleanup() + def generate(self): + setup.generate(self.dest, 'local/kc:operator', 'local/mcp:operator', 'local/rustfs:operator') + def test_new_destination_modes_and_unique_tokens(self): + with patch.dict(os.environ, {}): + old = os.umask(0) + try: self.generate() + finally: os.umask(old) + self.assertEqual(stat.S_IMODE(self.dest.stat().st_mode), 0o700) + for p in self.dest.iterdir(): + self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o600) + values = [(self.dest / n).read_text() for n in setup.TOKENS] + self.assertEqual(len(values), len(set(values))) + self.assertTrue(all(len(v.strip()) >= 32 for v in values)) + def test_exact_cli_setup_keeps_secrets_off_output(self): + result = subprocess.run(['python3', str(ROOT / 'scripts/kc/setup.py'), str(self.dest), + '--kc-image', 'local/kc:operator', '--kc-mcp-image', 'local/mcp:operator', + '--rustfs-image', 'local/rustfs:operator'], capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + for name in setup.TOKENS: + self.assertNotIn((self.dest / name).read_text().strip(), result.stdout + result.stderr) + again = subprocess.run(['python3', str(ROOT / 'scripts/kc/setup.py'), str(self.dest), + '--kc-image', 'local/kc:operator', '--kc-mcp-image', 'local/mcp:operator', + '--rustfs-image', 'local/rustfs:operator'], capture_output=True, text=True) + self.assertNotEqual(again.returncode, 0) + + def test_existing_directory_untouched(self): + self.dest.mkdir(mode=0o755) + p = self.dest / 'admin_password'; p.write_text('keep'); p.chmod(0o644) + with self.assertRaises(FileExistsError): self.generate() + self.assertEqual(p.read_text(), 'keep') + self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o644) + def test_existing_file_untouched(self): + self.dest.write_text('keep'); self.dest.chmod(0o644) + with self.assertRaises(FileExistsError): self.generate() + self.assertEqual(self.dest.read_text(), 'keep') + self.assertEqual(stat.S_IMODE(self.dest.stat().st_mode), 0o644) + def test_symlinks_and_dangling_symlinks(self): + for target in [self.root / 'absent', self.root]: + with self.subTest(target=target): + self.dest.symlink_to(target) + with self.assertRaises((FileExistsError, ValueError)): self.generate() + self.assertTrue(self.dest.is_symlink()); self.dest.unlink() + def test_symlink_parent_rejected(self): + link = self.root / 'link'; link.symlink_to(self.root) + with self.assertRaises(ValueError): + setup.generate(link / 'credentials', 'kc', 'mcp', 'rustfs') + self.assertFalse(self.dest.exists()) + def test_key_generation_failure_cleans_owned_files(self): + with patch.object(setup.subprocess, 'run', side_effect=RuntimeError('injected failure')): + with self.assertRaises(RuntimeError): self.generate() + self.assertFalse(self.dest.exists()) + def test_partial_write_failure_cleanup(self): + real = setup.write_exclusive + def fail(path, content): + if path.name == 'jwt_secret': raise OSError('injected failure') + return real(path, content) + with patch.object(setup, 'write_exclusive', side_effect=fail): + with self.assertRaises(OSError): self.generate() + self.assertFalse(self.dest.exists()) + def test_exclusive_writer_preserves_existing_file_and_symlink(self): + p = self.root / 'file'; p.write_text('keep'); p.chmod(0o644) + for path in [p, self.root / 'link']: + if path != p: path.symlink_to(p) + with self.assertRaises(FileExistsError): setup.write_exclusive(path, b'replace') + self.assertEqual(p.read_text(), 'keep') + self.assertEqual(stat.S_IMODE(p.stat().st_mode), 0o644) + def test_invalid_image_does_not_create_destination(self): + with self.assertRaises(ValueError): + setup.generate(self.dest, 'kc\nINJECT=1', 'mcp', 'rustfs') + self.assertFalse(self.dest.exists()) + +class ComposeTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(dir=os.environ.get('TMPDIR')) + self.dest = Path(self.tmp.name) / 'credentials' + setup.generate(self.dest, 'local/kc:operator', 'local/mcp:operator', 'local/rustfs:operator') + def tearDown(self): self.tmp.cleanup() + def compose(self, overlay=True, env=True): + args = ['docker', 'compose'] + if env: args += ['--env-file', str(self.dest / 'config.env')] + args += ['-f', str(ROOT / 'compose.yaml')] + if overlay: args += ['-f', str(ROOT / 'compose.kc.yaml')] + args += ['--profile', 'server', 'config', '--format', 'json'] + clean = {k: v for k, v in os.environ.items() if not k.startswith(('KC_', 'SHAIDE_', 'MCP_', 'RUSTFS_'))} + return subprocess.run(args, capture_output=True, text=True, env=clean) + def model(self): + r = self.compose(); self.assertEqual(r.returncode, 0, r.stderr); return json.loads(r.stdout) + def test_no_kc_needs_no_kc_variables(self): + r = self.compose(False, False); self.assertEqual(r.returncode, 0, r.stderr) + m = json.loads(r.stdout); self.assertNotIn('knowledge-center', m['services']) + self.assertEqual(m['services']['vectordb']['ports'][0]['published'], '6333') + def test_missing_optional_settings_fail_only_overlay(self): + r = self.compose(True, False); self.assertNotEqual(r.returncode, 0) + self.assertIn('required', r.stderr) + def test_only_server_has_host_ports(self): + m = self.model() + for name, svc in m['services'].items(): + if name != 'shaide_server': self.assertFalse(svc.get('ports'), name) + self.assertEqual(m['services']['shaide_server']['ports'][0]['host_ip'], '127.0.0.1') + def test_storage_and_vectors_are_isolated_from_server_and_mcp(self): + m = self.model(); s = m['services'] + for backend in ['kc-rustfs', 'kc-qdrant']: + for facade in ['shaide_server', 'kc-mcp']: + self.assertTrue(set(s[backend]['networks']).isdisjoint(s[facade]['networks'])) + self.assertNotIn('default', s['knowledge-center']['networks']) + for name, net in m['networks'].items(): + if name != 'default': self.assertTrue(net['internal'], name) + def test_server_has_no_storage_credentials_or_mounts(self): + m = self.model(); s = m['services']['shaide_server'] + self.assertFalse(any(k.startswith(('S3_', 'RUSTFS_', 'AWS_')) and k != 'S3_WEBHOOK_TOKEN' for k in s['environment'])) + self.assertFalse(any('storage' in x['source'] for x in s['secrets'])) + def test_compose_output_contains_no_secret_values(self): + output = self.compose().stdout + for n in setup.TOKENS: + self.assertNotIn((self.dest / n).read_text().strip(), output) + def test_named_state_and_no_build_contract(self): + m = self.model(); s = m['services'] + for service in ['shaide_server', 'kc-rustfs', 'kc-qdrant']: + self.assertTrue(any(v['type'] == 'volume' for v in s[service]['volumes'])) + for name in ['knowledge-center', 'kc-mcp', 'kc-rustfs']: + self.assertNotIn('build', s[name]); self.assertEqual(s[name]['pull_policy'], 'never') + self.assertEqual(s['shaide_server']['environment']['VECTOR_DB_URL'], 'http://vectordb:6334') + def test_signed_context_and_notification_trust(self): + s = self.model()['services']; env = s['shaide_server']['environment'] + targets = json.loads(env['MCP_COLLECTION_CONTEXT_TARGETS']) + self.assertEqual(targets[0]['url'], 'http://kc-mcp:8080') + self.assertEqual(env['KNOWLEDGE_CENTER_URL'], 'http://knowledge-center:8080') + self.assertIn('kc-notifications:8080/v1/object-storage/event', s['kc-rustfs']['environment']['RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_SHAIDE']) + self.assertEqual(s['knowledge-center']['environment']['RUSTFS_WEBHOOK_ARN'], 'arn:rustfs:sqs:eu-central-1:shaide:webhook') + +if __name__ == '__main__': unittest.main() diff --git a/test/kc-compose/test_operator.py b/test/kc-compose/test_operator.py new file mode 100644 index 0000000..722780b --- /dev/null +++ b/test/kc-compose/test_operator.py @@ -0,0 +1,130 @@ +import importlib.util +from contextlib import contextmanager, redirect_stderr, redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import io +import json +import os +from pathlib import Path +import tempfile +import threading +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +ROOT = Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('operator_helper', ROOT / 'scripts/kc/operate.py') +op = importlib.util.module_from_spec(spec); spec.loader.exec_module(op) + +@contextmanager +def local_api(respond): + calls = [] + class Handler(BaseHTTPRequestHandler): + def log_message(self, *args): pass + def handle_request(self): + raw = self.rfile.read(int(self.headers.get('Content-Length', 0))) + calls.append((self.command, self.path, dict(self.headers), raw)) + status, headers, body = respond(self.command, self.path, raw) + payload = json.dumps(body).encode() + self.send_response(status) + for name, value in headers.items(): self.send_header(name, value) + self.send_header('Content-Length', str(len(payload))) + self.end_headers() + self.wfile.write(payload) + do_GET = do_POST = handle_request + server = ThreadingHTTPServer(('127.0.0.1', 0), Handler) + thread = threading.Thread(target=server.serve_forever, kwargs={'poll_interval': 0.01}) + thread.start() + try: + yield 'http://127.0.0.1:' + str(server.server_port), calls + finally: + server.shutdown(); server.server_close(); thread.join() + + +class OperatorTests(unittest.TestCase): + def check_args(self, server='http://127.0.0.1:55555', size=3): + return SimpleNamespace(server=server, credentials=Path('/unused'), action='check', + username='fixture', model='fixture', vector_size=size) + + def test_all_redirect_statuses_rejected_without_followup_or_secret_leak(self): + marker = 'synthetic-bearer-marker' + secret_body = {'password': 'synthetic-body-marker'} + with local_api(lambda *a: (200, {}, {'ok': True})) as (destination, received): + for status in (301, 302, 303, 307, 308): + for method in ('GET', 'POST'): + for cross_origin in (False, True): + with self.subTest(status=status, method=method, cross_origin=cross_origin): + received.clear() + def respond(command, path, raw): + return (status, {'Location': (destination if cross_origin else '') + '/destination?synthetic-url-marker'}, {}) if path == '/redirect' else (200, {}, {'ok': True}) + with local_api(respond) as (origin, sent): + with self.assertRaises(ValueError) as failure: + op.request(origin, method, '/redirect', secret_body if method == 'POST' else None, marker) + self.assertEqual(str(failure.exception), f'API request failed with HTTP {status}') + self.assertEqual(len(sent), 1) + self.assertEqual(received, []) + for value in (origin, destination, marker, secret_body['password'], 'synthetic-url-marker'): + self.assertNotIn(value, str(failure.exception)) + + def test_cli_redirect_failure_is_sanitized(self): + with local_api(lambda *a: (302, {'Location': 'http://127.0.0.1:1/synthetic-url-marker'}, {})) as (origin, calls): + stdout, stderr = io.StringIO(), io.StringIO() + argv = ['operate.py', '--credentials', '/unused', '--server', origin, + 'check', '--model', 'fixture', '--vector-size', '3'] + with patch.object(op, 'protected_read', return_value='synthetic-password-marker'), patch.object(op.sys, 'argv', argv), redirect_stdout(stdout), redirect_stderr(stderr): + self.assertEqual(op.main(), 1) + self.assertEqual(len(calls), 1) + self.assertEqual(stdout.getvalue(), '') + self.assertIn('Operator action failed.', stderr.getvalue()) + for value in (origin, 'synthetic-url-marker', 'synthetic-password-marker'): + self.assertNotIn(value, stderr.getvalue()) + + def test_invalid_dimensions_fail_before_credentials_or_network(self): + for size in (0, -1, 65537): + with self.subTest(size=size), patch.object(op, 'protected_read') as read, patch.object(op, 'request') as request: + with self.assertRaisesRegex(ValueError, '1..65536'): op.run(self.check_args(size=size)) + read.assert_not_called(); request.assert_not_called() + + def embedding_fixture(self, declared=3, actual=3): + def respond(method, path, raw): + if path == '/v1/login': return 200, {}, {'role': 'user', 'access_token': 'synthetic-token'} + if path == '/v1/embedding_models': return 200, {}, {'models': [{'name': 'fixture', 'vector_size': declared}]} + if path == '/v1/embeddings': + body = json.loads(raw) + # Mirrors the server's check_dimensions: omission is accepted. + if 'dimensions' in body and body['dimensions'] != declared: + return 400, {}, {'error': 'synthetic provider URL/token must not leak'} + return 200, {}, {'data': [{'embedding': [0.0] * actual}]} + self.fail('unexpected fixture request') + return local_api(respond) + + def test_declared_model_dimensions_mismatch_is_rejected_by_server(self): + with self.embedding_fixture(declared=4) as (origin, calls), patch.object(op, 'protected_read', return_value='synthetic'): + with self.assertRaisesRegex(ValueError, 'HTTP 400'): op.run(self.check_args(origin)) + self.assertEqual(json.loads(calls[-1][3])['dimensions'], 3) + + def test_provider_output_dimensions_remain_validated(self): + with self.embedding_fixture(actual=4) as (origin, calls), patch.object(op, 'protected_read', return_value='synthetic'): + with self.assertRaisesRegex(ValueError, 'unexpected embedding dimensions'): op.run(self.check_args(origin)) + self.assertEqual(json.loads(calls[-1][3])['dimensions'], 3) + + def test_supported_dimension_boundaries_are_sent(self): + for size in (1, 65536): + with self.subTest(size=size), self.embedding_fixture(declared=size, actual=size) as (origin, calls), patch.object(op, 'protected_read', return_value='synthetic'), redirect_stdout(io.StringIO()): + op.run(self.check_args(origin, size)) + self.assertEqual(json.loads(calls[-1][3])['dimensions'], size) + + def test_protected_inputs_reject_public_files_and_symlinks(self): + with tempfile.TemporaryDirectory(dir=os.environ.get('TMPDIR')) as directory: + p = Path(directory) / 'secret'; p.write_text('sensitive'); p.chmod(0o644) + with self.assertRaises(ValueError): op.protected_read(p) + p.chmod(0o600); self.assertEqual(op.protected_read(p), 'sensitive') + link = p.parent / 'link'; link.symlink_to(p) + with self.assertRaises(OSError): op.protected_read(link) + def test_admin_cannot_be_embedding_service(self): + args = SimpleNamespace(server='http://127.0.0.1:55555',credentials=Path('/unused'),action='check',username='fixture',model='fixture',vector_size=3) + with patch.object(op, 'protected_read', return_value='synthetic'), patch.object(op, 'request', return_value={'role': 'admin'}): + with self.assertRaises(ValueError): op.run(args) + def test_nonlocal_origins_rejected_before_secret_read(self): + for server in ['http://example.com', 'http://user:password@localhost', 'http://localhost?secret=bad', 'http://localhost/path']: + with self.subTest(server=server): + with self.assertRaises(ValueError): op.run(SimpleNamespace(server=server))