From 5e1bb6885fbc81305babe2086215aa58adab4cb5 Mon Sep 17 00:00:00 2001 From: Noah Cylich Date: Wed, 27 May 2026 11:51:31 -0700 Subject: [PATCH] Add v2 local HTTP server Introduce an OpenAI-compatible local server for prepared Cactus v2 bundles with chat completions, streaming responses, model discovery, tool-call mapping, WAV transcription, and warm model slot management. Wire the server into the CLI as cactus serve, add serving dependencies, and keep bundle loading on the direct Python FFI path instead of shelling out through cactus run. Add focused mocked server tests and live HTTP e2e tests that launch the server, exercise real generation, streaming, multi-turn chat, concurrency, and transcription paths. Signed-off-by: Noah Cylich --- python/cactus/cli/__init__.py | 20 ++ python/cactus/cli/serve.py | 71 ++++ python/cactus/server.py | 584 +++++++++++++++++++++++++++++++ python/pyproject.toml | 3 +- python/requirements.txt | 4 + python/tests/test_server.py | 213 +++++++++++ python/tests/test_server_live.py | 434 +++++++++++++++++++++++ 7 files changed, 1328 insertions(+), 1 deletion(-) create mode 100644 python/cactus/cli/serve.py create mode 100644 python/cactus/server.py create mode 100644 python/tests/test_server.py create mode 100644 python/tests/test_server_live.py diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 562d7ab58..1f45816aa 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -10,6 +10,7 @@ from .download import cmd_download from .compile import cmd_build from .run import cmd_run +from .serve import cmd_serve from .transcribe import cmd_transcribe from .test import cmd_test from .convert import cmd_convert @@ -68,6 +69,15 @@ def create_parser(): ----------------------------------------------------------------- + cactus serve [model] OpenAI-compatible local HTTP server + serves prepared v2 bundles only + + Optional flags: + --host bind address (default: 127.0.0.1) + --port port (default: 8080) + + ----------------------------------------------------------------- + cactus download fetch pre-converted CQ from Cactus-Compute Optional flags: @@ -199,6 +209,15 @@ def create_parser(): transcribe_parser.add_argument("--reconvert", action="store_true", help="Download original model and convert (instead of using pre-converted from Cactus-Compute)") + # ── serve ───────────────────────────────────────────────────────── + serve_parser = subparsers.add_parser("serve", help="Start OpenAI-compatible HTTP server") + serve_parser.add_argument("model", nargs="?", default=None, + help="Prepared v2 bundle path, local model dir name, or HF model ID") + serve_parser.add_argument("--host", default="127.0.0.1", + help="Bind address (default: 127.0.0.1)") + serve_parser.add_argument("--port", type=int, default=8080, + help="Port (default: 8080)") + # ── test ────────────────────────────────────────────────────────── test_parser = subparsers.add_parser("test", help="Run the test suite") test_parser.add_argument("--model", dest="model_id", default=DEFAULT_TEST_MODEL_ID, @@ -270,6 +289,7 @@ def create_parser(): "download": cmd_download, "build": cmd_build, "run": cmd_run, + "serve": cmd_serve, "transcribe": cmd_transcribe, "test": cmd_test, diff --git a/python/cactus/cli/serve.py b/python/cactus/cli/serve.py new file mode 100644 index 000000000..0a9d4fc3a --- /dev/null +++ b/python/cactus/cli/serve.py @@ -0,0 +1,71 @@ +from pathlib import Path + +from .common import BLUE, GREEN, RED, PROJECT_ROOT, is_repo_checkout, print_color +from .download import get_weights_dir + + +def _weights_root() -> Path: + if is_repo_checkout(): + return PROJECT_ROOT / "weights" + return Path.home() / ".cache" / "cactus" / "weights" + + +def _resolve_model_arg(model: str | None) -> tuple[Path | None, str | None]: + if not model: + return None, None + path = Path(model).expanduser() + if path.is_dir(): + return path, path.name + candidate = _weights_root() / model + if candidate.is_dir(): + return candidate, candidate.name + hf_candidate = get_weights_dir(model) + if hf_candidate.is_dir(): + return hf_candidate, hf_candidate.name + return None, model + + +def _is_valid_bundle(path: Path) -> bool: + return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() + + +def cmd_serve(args): + """Start the OpenAI-compatible HTTP server.""" + model_path, model_name = _resolve_model_arg(args.model) + if args.model and model_path is None: + print_color(RED, f"Error: model not found: {args.model}") + print("Prepare a v2 bundle first with `cactus run ` or `cactus convert `.") + return 1 + if model_path is not None and not _is_valid_bundle(model_path): + print_color(RED, f"Error: not a valid v2 Cactus bundle: {model_path}") + print("Expected config.txt and components/manifest.json.") + return 1 + + try: + import uvicorn + except ImportError: + print_color(RED, "Error: uvicorn not installed. Install the serve extra or run `pip install fastapi uvicorn python-multipart`.") + return 1 + + try: + from ..server import create_app + except ImportError: + print_color(RED, "Error: server dependencies not installed. Install the serve extra or run `pip install fastapi uvicorn python-multipart`.") + return 1 + + try: + application = create_app( + weights_root=_weights_root(), + model_path=model_path, + default_model=model_name, + ) + except RuntimeError as exc: + print_color(RED, f"Error: {exc}") + print("Prepare a v2 bundle first with `cactus run ` or `cactus convert `.") + return 1 + + models = sorted(application.state.registry.models) + print_color(GREEN, f"Available models: {', '.join(models)}") + print_color(BLUE, f"Starting server on {args.host}:{args.port}") + uvicorn.run(application, host=args.host, port=args.port, log_level="info") + return 0 diff --git a/python/cactus/server.py b/python/cactus/server.py new file mode 100644 index 000000000..7d5c9ad05 --- /dev/null +++ b/python/cactus/server.py @@ -0,0 +1,584 @@ +"""OpenAI-compatible local HTTP server for Cactus v2 bundles.""" +from __future__ import annotations + +import asyncio +import json +import logging +import time +import uuid +from contextlib import asynccontextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile +from fastapi.responses import PlainTextResponse, StreamingResponse +from pydantic import BaseModel + +from .bindings.cactus import ( + cactus_complete, + cactus_destroy, + cactus_get_last_error, + cactus_init, + cactus_reset, + cactus_transcribe, +) +from .cli.download import get_weights_dir +from .cli.common import DEFAULT_MODEL_ID, PROJECT_ROOT, is_repo_checkout + +LOGGER = logging.getLogger(__name__) + +LLM_MODEL_TYPES = {"gemma", "gemma3n", "gemma4", "lfm2", "qwen", "qwen3p5", "needle", "youtu"} +STT_MODEL_TYPES = {"whisper", "parakeet_tdt", "parakeet-tdt"} + + +@dataclass(frozen=True) +class ModelInfo: + id: str + path: Path + model_type: str + context_length: int + created: int + + +class ModelRegistry: + def __init__(self, weights_root: Path, extra_model: Path | None = None): + self.weights_root = weights_root + self.models: dict[str, ModelInfo] = {} + self._discover(weights_root) + if extra_model is not None: + info = self._info_for_dir(extra_model) + if info is None: + raise RuntimeError(f"Not a valid v2 Cactus bundle: {extra_model}") + self.models[info.id] = info + + @staticmethod + def _read_config_field(model_dir: Path, field: str) -> str: + config = model_dir / "config.txt" + if not config.exists(): + return "" + prefix = f"{field}=" + for line in config.read_text(encoding="utf-8").splitlines(): + if line.startswith(prefix): + return line.split("=", 1)[1].strip() + return "" + + @classmethod + def _info_for_dir(cls, path: Path) -> ModelInfo | None: + display_id = path.expanduser().name + resolved = path.expanduser().resolve() + if not (resolved / "config.txt").exists(): + return None + if not (resolved / "components" / "manifest.json").exists(): + return None + context_raw = cls._read_config_field(resolved, "context_length") + try: + context_length = int(context_raw or 0) + except ValueError: + context_length = 0 + stat = resolved.stat() + return ModelInfo( + id=display_id, + path=resolved, + model_type=cls._read_config_field(resolved, "model_type"), + context_length=context_length, + created=int(stat.st_mtime), + ) + + def _discover(self, root: Path) -> None: + if not root.exists(): + return + for entry in sorted(root.iterdir()): + if not entry.is_dir(): + continue + info = self._info_for_dir(entry) + if info is not None: + self.models[info.id] = info + + def require(self, model_id: str) -> ModelInfo: + info = self.models.get(model_id) + if info is None: + raise HTTPException(status_code=404, detail=f"Model '{model_id}' is not available") + return info + + def default_llm(self, preferred: str | None = None) -> ModelInfo: + if preferred: + info = self.models.get(preferred) + if info is None: + raise RuntimeError(f"Requested model '{preferred}' is not a valid v2 Cactus bundle") + if info.model_type not in LLM_MODEL_TYPES: + raise RuntimeError(f"Requested model '{preferred}' is not an LLM bundle") + return info + preferred_id = get_weights_dir(DEFAULT_MODEL_ID).name + if preferred_id in self.models and self.models[preferred_id].model_type in LLM_MODEL_TYPES: + return self.models[preferred_id] + for info in sorted(self.models.values(), key=lambda x: x.id): + if info.model_type in LLM_MODEL_TYPES: + return info + raise RuntimeError("No valid LLM bundles found. Prepare a transpiled bundle before running `cactus serve`.") + + def default_stt(self) -> ModelInfo | None: + for info in sorted(self.models.values(), key=lambda x: x.id): + if info.model_type in STT_MODEL_TYPES: + return info + return None + + def list_openai_models(self) -> list[dict[str, Any]]: + out = [] + for info in sorted(self.models.values(), key=lambda x: x.id): + out.append({ + "id": info.id, + "object": "model", + "created": info.created, + "owned_by": "cactus", + "context_window": info.context_length, + "model_type": info.model_type, + }) + return out + + +class _ModelSlot: + def __init__(self, info: ModelInfo, handle): + self.info = info + self.handle = handle + self.lock = asyncio.Lock() + self.last_used = time.monotonic() + self.active_requests = 0 + + def touch(self) -> None: + self.last_used = time.monotonic() + + +class ModelManager: + def __init__(self, registry: ModelRegistry, *, max_warm: int = 2): + self.registry = registry + self.max_warm = max_warm + self.slots: dict[str, _ModelSlot] = {} + self.swap_lock = asyncio.Lock() + + def _load(self, info: ModelInfo): + try: + return cactus_init(str(info.path)) + except RuntimeError as exc: + err = cactus_get_last_error() or str(exc) + raise HTTPException(status_code=500, detail=f"Failed to load model '{info.id}': {err}") from exc + + async def _get_slot(self, model_id: str) -> _ModelSlot: + info = self.registry.require(model_id) + async with self.swap_lock: + slot = self.slots.get(info.id) + if slot is not None: + slot.touch() + return slot + + if len(self.slots) >= self.max_warm: + idle = [s for s in self.slots.values() if s.active_requests == 0 and not s.lock.locked()] + if not idle: + raise HTTPException(status_code=503, detail="All warm model slots are busy") + victim = min(idle, key=lambda s: s.last_used) + self.slots.pop(victim.info.id, None) + cactus_destroy(victim.handle) + + handle = await asyncio.get_running_loop().run_in_executor(None, self._load, info) + slot = _ModelSlot(info, handle) + self.slots[info.id] = slot + return slot + + @asynccontextmanager + async def acquire(self, model_id: str): + slot = await self._get_slot(model_id) + async with self.swap_lock: + if slot.info.id not in self.slots: + raise HTTPException(status_code=503, detail="Model slot was evicted before use") + slot.active_requests += 1 + try: + yield slot + finally: + async with self.swap_lock: + slot.active_requests = max(0, slot.active_requests - 1) + slot.touch() + + async def preload(self, model_id: str) -> None: + async with self.acquire(model_id): + return + + def shutdown(self) -> None: + for slot in self.slots.values(): + cactus_destroy(slot.handle) + self.slots.clear() + + +class Permissive(BaseModel): + model_config = {"extra": "allow"} + + +class ChatMessage(Permissive): + role: str + content: str | list[Any] | None = None + tool_call_id: str | None = None + tool_calls: list[Any] | None = None + + +class ToolFunction(Permissive): + name: str + description: str | None = None + parameters: dict[str, Any] | None = None + + +class Tool(Permissive): + type: str = "function" + function: ToolFunction + + +class ToolChoiceFunction(Permissive): + name: str + + +class ToolChoiceObject(Permissive): + type: str = "function" + function: ToolChoiceFunction + + +class ChatRequest(Permissive): + model: str + messages: list[ChatMessage] + temperature: float | None = None + top_p: float | None = None + top_k: int | None = None + max_tokens: int | None = None + max_completion_tokens: int | None = None + stop: str | list[str] | None = None + stream: bool = False + tools: list[Tool] | None = None + tool_choice: str | ToolChoiceObject | None = None + + +def _flatten_message(msg: ChatMessage) -> dict[str, Any]: + out: dict[str, Any] = {"role": msg.role} + if isinstance(msg.content, list): + parts = [] + for part in msg.content: + if isinstance(part, dict) and part.get("type") == "text": + parts.append(str(part.get("text", ""))) + elif isinstance(part, str): + parts.append(part) + out["content"] = "\n".join(parts) + elif msg.content is not None: + out["content"] = msg.content + if msg.tool_call_id is not None: + out["tool_call_id"] = msg.tool_call_id + if msg.tool_calls is not None: + out["tool_calls"] = msg.tool_calls + return out + + +def _translate_tools(tools: list[Tool] | None, tool_choice) -> tuple[list[dict[str, Any]] | None, bool]: + if not tools or tool_choice == "none": + return None, False + if isinstance(tool_choice, ToolChoiceObject): + selected = [ + {"name": t.function.name, "description": t.function.description or "", "parameters": t.function.parameters or {}} + for t in tools + if t.function.name == tool_choice.function.name + ] + return selected or None, True + translated = [ + {"name": t.function.name, "description": t.function.description or "", "parameters": t.function.parameters or {}} + for t in tools + ] + return translated, tool_choice == "required" + + +def _make_tool_calls(function_calls: list[Any]) -> list[dict[str, Any]]: + out = [] + for call in function_calls: + if not isinstance(call, dict): + continue + args = call.get("arguments", {}) + out.append({ + "id": f"call_{uuid.uuid4().hex[:24]}", + "type": "function", + "function": { + "name": call.get("name", ""), + "arguments": json.dumps(args) if isinstance(args, dict) else str(args), + }, + }) + return out + + +def _chat_options(req: ChatRequest) -> dict[str, Any]: + options: dict[str, Any] = {} + for key in ("temperature", "top_p", "top_k"): + value = getattr(req, key) + if value is not None: + options[key] = value + max_tokens = req.max_tokens if req.max_tokens is not None else req.max_completion_tokens + if max_tokens is not None: + options["max_tokens"] = max_tokens + if req.stop: + options["stop_sequences"] = [req.stop] if isinstance(req.stop, str) else req.stop + return options + + +def _build_chat_response(result: dict[str, Any], model_id: str, request_id: str) -> dict[str, Any]: + function_calls = result.get("function_calls") or [] + tool_calls = _make_tool_calls(function_calls) + has_tool_calls = bool(tool_calls) + prefill = int(result.get("prefill_tokens") or 0) + decode = int(result.get("decode_tokens") or 0) + message: dict[str, Any] = {"role": "assistant", "content": None if has_tool_calls else result.get("response", "")} + if has_tool_calls: + message["tool_calls"] = tool_calls + return { + "id": request_id, + "object": "chat.completion", + "created": int(time.time()), + "model": model_id, + "system_fingerprint": None, + "choices": [{ + "index": 0, + "message": message, + "logprobs": None, + "finish_reason": "tool_calls" if has_tool_calls else "stop", + }], + "usage": { + "prompt_tokens": prefill, + "completion_tokens": decode, + "total_tokens": prefill + decode, + }, + } + + +def _event(data: dict[str, Any]) -> str: + return f"data: {json.dumps(data)}\n\n" + + +async def _stream_completion(manager: ModelManager, req: ChatRequest, request_id: str, messages, options, tools): + queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue() + loop = asyncio.get_running_loop() + + def on_token(token: str, token_id: int) -> None: + loop.call_soon_threadsafe(queue.put_nowait, ("token", token)) + + async def run_inference(): + error = None + result = None + try: + async with manager.acquire(req.model) as slot: + async with slot.lock: + cactus_reset(slot.handle) + result = await loop.run_in_executor( + None, + lambda: cactus_complete(slot.handle, messages, options, tools, on_token), + ) + except Exception as exc: + LOGGER.exception("Streaming completion failed") + error = exc + finally: + loop.call_soon_threadsafe(queue.put_nowait, ("done", (result, error))) + + task = asyncio.create_task(run_inference()) + created = int(time.time()) + yield _event({ + "id": request_id, + "object": "chat.completion.chunk", + "created": created, + "model": req.model, + "system_fingerprint": None, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}, "logprobs": None, "finish_reason": None}], + }) + + result = None + error = None + while True: + kind, value = await queue.get() + if kind == "token": + yield _event({ + "id": request_id, + "object": "chat.completion.chunk", + "created": created, + "model": req.model, + "system_fingerprint": None, + "choices": [{"index": 0, "delta": {"content": value}, "logprobs": None, "finish_reason": None}], + }) + elif kind == "done": + result, error = value + break + + await task + if error is not None: + yield f"event: error\ndata: {json.dumps({'error': str(error)})}\n\n" + yield "data: [DONE]\n\n" + return + + function_calls = result.get("function_calls") or [] + tool_calls = _make_tool_calls(function_calls) + finish_reason = "tool_calls" if tool_calls else "stop" + if tool_calls: + yield _event({ + "id": request_id, + "object": "chat.completion.chunk", + "created": created, + "model": req.model, + "system_fingerprint": None, + "choices": [{"index": 0, "delta": {"tool_calls": tool_calls}, "logprobs": None, "finish_reason": None}], + }) + final = { + "id": request_id, + "object": "chat.completion.chunk", + "created": created, + "model": req.model, + "system_fingerprint": None, + "choices": [{"index": 0, "delta": {}, "logprobs": None, "finish_reason": finish_reason}], + "usage": { + "prompt_tokens": int(result.get("prefill_tokens") or 0), + "completion_tokens": int(result.get("decode_tokens") or 0), + "total_tokens": int(result.get("total_tokens") or 0), + }, + } + yield _event(final) + yield "data: [DONE]\n\n" + + +def _requested_granularities(primary: list[str] | None, bracketed: list[str] | None) -> list[str]: + values = [] + for item in (primary or []) + (bracketed or []): + if item: + values.append(item) + return values + + +def create_app( + *, + weights_root: Path | None = None, + model_path: Path | None = None, + default_model: str | None = None, + max_warm: int = 2, + preload: bool = True, +) -> FastAPI: + root = Path(weights_root) if weights_root is not None else ( + PROJECT_ROOT / "weights" if is_repo_checkout() else Path.home() / ".cache" / "cactus" / "weights" + ) + registry = ModelRegistry(root, extra_model=model_path) + selected = registry.default_llm(default_model) + manager = ModelManager(registry, max_warm=max_warm) + + @asynccontextmanager + async def lifespan(app: FastAPI): + if preload: + await manager.preload(selected.id) + try: + yield + finally: + manager.shutdown() + + app = FastAPI(title="Cactus", version="0.1.0", lifespan=lifespan) + app.state.registry = registry + app.state.manager = manager + app.state.default_model = selected.id + app.state.default_stt_model = registry.default_stt().id if registry.default_stt() else None + + @app.get("/v1/models") + async def list_models(request: Request): + reg: ModelRegistry = request.app.state.registry + return {"object": "list", "data": reg.list_openai_models()} + + @app.post("/v1/chat/completions") + async def chat_completions(request: Request, req: ChatRequest): + reg: ModelRegistry = request.app.state.registry + info = reg.require(req.model) + if info.model_type not in LLM_MODEL_TYPES: + raise HTTPException(status_code=400, detail=f"Model '{req.model}' is not an LLM model") + messages = [_flatten_message(m) for m in req.messages] + tools, force_tools = _translate_tools(req.tools, req.tool_choice) + options = _chat_options(req) + if force_tools: + options["force_tools"] = True + request_id = f"chatcmpl-{uuid.uuid4().hex[:29]}" + mgr: ModelManager = request.app.state.manager + if req.stream: + return StreamingResponse( + _stream_completion(mgr, req, request_id, messages, options or None, tools), + media_type="text/event-stream", + ) + async with mgr.acquire(req.model) as slot: + async with slot.lock: + cactus_reset(slot.handle) + result = await asyncio.get_running_loop().run_in_executor( + None, + lambda: cactus_complete(slot.handle, messages, options or None, tools, None), + ) + if not result.get("success", False): + raise HTTPException(status_code=500, detail=result.get("error") or "completion failed") + return _build_chat_response(result, req.model, request_id) + + @app.post("/v1/audio/transcriptions") + async def create_transcription( + request: Request, + file: UploadFile = File(...), + model: str = Form(""), + language: str | None = Form(None), + prompt: str | None = Form(None), + response_format: str = Form("json"), + temperature: float | None = Form(None), + timestamp_granularities: list[str] | None = Form(None), + timestamp_granularities_array: list[str] | None = Form(None, alias="timestamp_granularities[]"), + ): + if response_format not in {"json", "text", "verbose_json"}: + raise HTTPException(status_code=400, detail=f"Unsupported transcription response_format: {response_format}") + granularities = _requested_granularities(timestamp_granularities, timestamp_granularities_array) + if "word" in granularities: + raise HTTPException(status_code=400, detail="Word timestamp granularity is not supported") + if any(g != "segment" for g in granularities): + raise HTTPException(status_code=400, detail=f"Unsupported timestamp granularity: {', '.join(granularities)}") + suffix = Path(file.filename or "").suffix.lower() + if suffix != ".wav": + raise HTTPException(status_code=400, detail="Only .wav transcription uploads are supported for now") + model_id = model or request.app.state.default_stt_model + if not model_id: + raise HTTPException(status_code=400, detail="No STT model available") + reg: ModelRegistry = request.app.state.registry + info = reg.require(model_id) + if info.model_type not in STT_MODEL_TYPES: + raise HTTPException(status_code=400, detail=f"Model '{model_id}' is not a speech-to-text model") + import tempfile + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + tmp.write(await file.read()) + tmp_path = Path(tmp.name) + try: + options: dict[str, Any] = {} + if temperature is not None: + options["temperature"] = temperature + if language: + options["language"] = language + mgr: ModelManager = request.app.state.manager + async with mgr.acquire(model_id) as slot: + async with slot.lock: + cactus_reset(slot.handle) + result = await asyncio.get_running_loop().run_in_executor( + None, + lambda: cactus_transcribe(slot.handle, str(tmp_path), prompt, options or None, None), + ) + finally: + tmp_path.unlink(missing_ok=True) + if not result.get("success", False): + raise HTTPException(status_code=500, detail=result.get("error") or "transcription failed") + text = result.get("response", "") + if response_format == "text": + return PlainTextResponse(text) + if response_format == "verbose_json": + segments = result.get("segments") or [] + return { + "task": "transcribe", + "language": language or "", + "duration": segments[-1]["end"] if segments else 0.0, + "text": text, + "segments": [ + {"id": i, "start": seg["start"], "end": seg["end"], "text": seg["text"]} + for i, seg in enumerate(segments) + ], + } + return {"text": text} + + return app diff --git a/python/pyproject.toml b/python/pyproject.toml index 0425c1b2a..0c351a568 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -39,7 +39,8 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0"] +dev = ["pytest>=8.0", "httpx>=0.25.0"] +serve = ["fastapi>=0.100.0", "uvicorn>=0.20.0", "python-multipart>=0.0.5"] [project.urls] Homepage = "https://cactuscompute.com" diff --git a/python/requirements.txt b/python/requirements.txt index 1ba381ab7..4e509d70b 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -5,3 +5,7 @@ numpy==2.4.6 pillow==11.3.0 huggingface_hub==1.15.0 scipy==1.17.1 +fastapi>=0.100.0 +uvicorn>=0.20.0 +python-multipart>=0.0.5 +httpx>=0.25.0 diff --git a/python/tests/test_server.py b/python/tests/test_server.py new file mode 100644 index 000000000..5f570e070 --- /dev/null +++ b/python/tests/test_server.py @@ -0,0 +1,213 @@ +from pathlib import Path +from types import SimpleNamespace +import asyncio + +import pytest +from fastapi import HTTPException +from fastapi.testclient import TestClient + +import cactus.server as server + + +def _bundle(root: Path, name: str, model_type: str, *, manifest: bool = True) -> Path: + path = root / name + path.mkdir(parents=True) + (path / "config.txt").write_text( + f"model_type={model_type}\ncontext_length=1024\n", + encoding="utf-8", + ) + if manifest: + (path / "components").mkdir() + (path / "components" / "manifest.json").write_text('{"components":[]}', encoding="utf-8") + return path + + +def _app(tmp_path: Path, monkeypatch, *, preload: bool = False): + _bundle(tmp_path, "llm", "gemma4") + monkeypatch.setattr(server, "cactus_init", lambda *args, **kwargs: SimpleNamespace(args=args, kwargs=kwargs)) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: None) + return server.create_app(weights_root=tmp_path, default_model="llm", preload=preload) + + +def test_models_lists_only_v2_bundles(tmp_path: Path, monkeypatch) -> None: + _bundle(tmp_path, "valid", "gemma4") + _bundle(tmp_path, "old_weights", "lfm2", manifest=False) + monkeypatch.setattr(server, "cactus_init", lambda *args, **kwargs: object()) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: None) + + app = server.create_app(weights_root=tmp_path, default_model="valid", preload=False) + with TestClient(app) as client: + data = client.get("/v1/models").json()["data"] + + assert [m["id"] for m in data] == ["valid"] + assert data[0]["context_window"] == 1024 + + +def test_chat_completion_shapes_openai_response(tmp_path: Path, monkeypatch) -> None: + app = _app(tmp_path, monkeypatch) + + def fake_complete(handle, messages, options, tools, callback): + assert messages == [{"role": "user", "content": "hello"}] + assert options["max_tokens"] == 3 + return { + "success": True, + "response": "hi", + "function_calls": [], + "prefill_tokens": 2, + "decode_tokens": 1, + } + + monkeypatch.setattr(server, "cactus_complete", fake_complete) + monkeypatch.setattr(server, "cactus_reset", lambda handle: None) + + with TestClient(app) as client: + res = client.post("/v1/chat/completions", json={ + "model": "llm", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 3, + }) + + assert res.status_code == 200 + body = res.json() + assert body["object"] == "chat.completion" + assert body["choices"][0]["message"]["content"] == "hi" + assert body["usage"] == {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3} + + +def test_chat_tool_calls_are_translated(tmp_path: Path, monkeypatch) -> None: + app = _app(tmp_path, monkeypatch) + monkeypatch.setattr(server, "cactus_reset", lambda handle: None) + monkeypatch.setattr(server, "cactus_complete", lambda *args, **kwargs: { + "success": True, + "response": "", + "function_calls": [{"name": "lookup", "arguments": {"q": "x"}}], + "prefill_tokens": 1, + "decode_tokens": 1, + }) + + with TestClient(app) as client: + res = client.post("/v1/chat/completions", json={ + "model": "llm", + "messages": [{"role": "user", "content": "call tool"}], + "tools": [{"type": "function", "function": {"name": "lookup", "parameters": {}}}], + "tool_choice": "required", + }) + + choice = res.json()["choices"][0] + assert choice["finish_reason"] == "tool_calls" + assert choice["message"]["content"] is None + assert choice["message"]["tool_calls"][0]["function"]["name"] == "lookup" + assert choice["message"]["tool_calls"][0]["function"]["arguments"] == '{"q": "x"}' + + +def test_streaming_completion_error_terminates(tmp_path: Path, monkeypatch) -> None: + app = _app(tmp_path, monkeypatch) + monkeypatch.setattr(server, "cactus_reset", lambda handle: None) + + def fail_complete(*args, **kwargs): + raise RuntimeError("boom") + + monkeypatch.setattr(server, "cactus_complete", fail_complete) + + with TestClient(app) as client: + with client.stream("POST", "/v1/chat/completions", json={ + "model": "llm", + "messages": [{"role": "user", "content": "hello"}], + "stream": True, + }) as res: + body = "".join(res.iter_text()) + + assert res.status_code == 200 + assert "event: error" in body + assert "boom" in body + assert "data: [DONE]" in body + + +def test_transcription_rejects_non_wav(tmp_path: Path, monkeypatch) -> None: + _bundle(tmp_path, "llm", "gemma4") + _bundle(tmp_path, "stt", "parakeet_tdt") + monkeypatch.setattr(server, "cactus_init", lambda *args, **kwargs: object()) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: None) + app = server.create_app(weights_root=tmp_path, default_model="llm", preload=False) + + with TestClient(app) as client: + res = client.post( + "/v1/audio/transcriptions", + files={"file": ("audio.mp3", b"not wav", "audio/mpeg")}, + data={"model": "stt"}, + ) + + assert res.status_code == 400 + assert "Only .wav" in res.json()["detail"] + + +def test_transcription_verbose_json_with_segments(tmp_path: Path, monkeypatch) -> None: + _bundle(tmp_path, "llm", "gemma4") + _bundle(tmp_path, "stt", "parakeet_tdt") + monkeypatch.setattr(server, "cactus_init", lambda *args, **kwargs: object()) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: None) + monkeypatch.setattr(server, "cactus_reset", lambda handle: None) + monkeypatch.setattr(server, "cactus_transcribe", lambda *args, **kwargs: { + "success": True, + "response": "hello", + "segments": [{"start": 0.0, "end": 1.0, "text": "hello"}], + }) + app = server.create_app(weights_root=tmp_path, default_model="llm", preload=False) + + with TestClient(app) as client: + res = client.post( + "/v1/audio/transcriptions", + files={"file": ("audio.wav", b"wav", "audio/wav")}, + data={ + "model": "stt", + "response_format": "verbose_json", + "timestamp_granularities[]": "segment", + }, + ) + + assert res.status_code == 200 + assert res.json()["segments"] == [{"id": 0, "start": 0.0, "end": 1.0, "text": "hello"}] + + +def test_transcription_rejects_word_timestamps(tmp_path: Path, monkeypatch) -> None: + _bundle(tmp_path, "llm", "gemma4") + _bundle(tmp_path, "stt", "parakeet_tdt") + monkeypatch.setattr(server, "cactus_init", lambda *args, **kwargs: object()) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: None) + app = server.create_app(weights_root=tmp_path, default_model="llm", preload=False) + + with TestClient(app) as client: + res = client.post( + "/v1/audio/transcriptions", + files={"file": ("audio.wav", b"wav", "audio/wav")}, + data={"model": "stt", "timestamp_granularities[]": "word"}, + ) + + assert res.status_code == 400 + assert "Word timestamp" in res.json()["detail"] + + +def test_model_manager_does_not_evict_active_slots(tmp_path: Path, monkeypatch) -> None: + for name in ("a", "b", "c"): + _bundle(tmp_path, name, "gemma4") + destroyed = [] + monkeypatch.setattr(server, "cactus_init", lambda path, **kwargs: Path(path).name) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: destroyed.append(handle)) + + registry = server.ModelRegistry(tmp_path) + manager = server.ModelManager(registry, max_warm=2) + + async def run(): + async with manager.acquire("a"): + async with manager.acquire("b"): + with pytest.raises(HTTPException) as exc: + async with manager.acquire("c"): + pass + assert exc.value.status_code == 503 + assert destroyed == [] + + async with manager.acquire("c"): + pass + + asyncio.run(run()) + assert len(destroyed) == 1 diff --git a/python/tests/test_server_live.py b/python/tests/test_server_live.py new file mode 100644 index 000000000..724444625 --- /dev/null +++ b/python/tests/test_server_live.py @@ -0,0 +1,434 @@ +from __future__ import annotations + +import json +import os +import socket +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import httpx +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +WEIGHTS = PROJECT_ROOT / "weights" +DEFAULT_LLM_BUNDLE = Path("weights/gemma-4-e2b-it") +ASSETS = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" +LLM_TYPES = {"gemma", "gemma3n", "gemma4", "lfm2", "qwen", "qwen3p5", "needle", "youtu"} +STT_TYPES = {"whisper", "parakeet_tdt", "parakeet-tdt"} + + +def _read_model_type(bundle: Path) -> str: + for line in (bundle / "config.txt").read_text(encoding="utf-8").splitlines(): + if line.startswith("model_type="): + return line.split("=", 1)[1].strip() + return "" + + +def _valid_bundle(path: Path) -> bool: + return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() + + +def _require_bundle(relative: Path, types: set[str]) -> Path: + candidate = PROJECT_ROOT / relative + if not candidate.exists(): + pytest.fail( + f"Live-test model not found: {relative}\n" + f"Prepare it with `cactus convert google/gemma-4-E2B-it` from {PROJECT_ROOT}." + ) + if not _valid_bundle(candidate): + pytest.fail( + f"Live-test model is not a prepared v2 bundle: {relative}\n" + "Expected config.txt and components/manifest.json.\n" + f"Prepare it with `cactus convert google/gemma-4-E2B-it` from {PROJECT_ROOT}." + ) + model_type = _read_model_type(candidate) + if model_type not in types: + pytest.fail( + f"Live-test model has unsupported type {model_type!r}: {relative}\n" + f"Expected one of: {sorted(types)}." + ) + return candidate + + +def _find_bundle(preferred: list[str], types: set[str]) -> Path: + for name in preferred: + candidate = WEIGHTS / name + if candidate.exists() and _valid_bundle(candidate) and _read_model_type(candidate) in types: + return candidate + for candidate in sorted(WEIGHTS.iterdir()): + if candidate.is_dir() and _valid_bundle(candidate) and _read_model_type(candidate) in types: + return candidate + pytest.fail(f"No valid live-test bundle found under {WEIGHTS} for model types: {sorted(types)}") + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _start_server(bundle: Path, port: int) -> subprocess.Popen: + env = os.environ.copy() + python_path = str(PROJECT_ROOT / "python") + env["PYTHONPATH"] = python_path + os.pathsep + env.get("PYTHONPATH", "") + return subprocess.Popen( + [ + sys.executable, + "-m", + "cactus", + "serve", + str(bundle), + "--host", + "127.0.0.1", + "--port", + str(port), + ], + cwd=PROJECT_ROOT, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + + +def _wait_ready(proc: subprocess.Popen, base_url: str) -> None: + deadline = time.time() + 120 + last_error = None + while time.time() < deadline: + if proc.poll() is not None: + output = proc.stdout.read() if proc.stdout else "" + pytest.fail(f"server exited before becoming ready:\n{output}") + try: + with httpx.Client(timeout=2) as client: + res = client.get(f"{base_url}/v1/models") + if res.status_code == 200: + return + except Exception as exc: + last_error = exc + time.sleep(0.5) + pytest.fail(f"server did not become ready: {last_error}") + + +@pytest.fixture(scope="module") +def live_server(): + _require_bundle(DEFAULT_LLM_BUNDLE, LLM_TYPES) + port = _free_port() + proc = _start_server(DEFAULT_LLM_BUNDLE, port) + base_url = f"http://127.0.0.1:{port}" + try: + _wait_ready(proc, base_url) + yield base_url, DEFAULT_LLM_BUNDLE.name + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +def test_live_models(live_server) -> None: + base_url, _ = live_server + res = httpx.get(f"{base_url}/v1/models", timeout=10) + assert res.status_code == 200 + models = res.json()["data"] + assert models + for model in models: + bundle = WEIGHTS / model["id"] + assert _valid_bundle(bundle) + + +def test_live_chat_rejects_unknown_model(live_server) -> None: + base_url, _ = live_server + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": "definitely-not-a-live-model", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 4, + }, + timeout=30, + ) + assert res.status_code == 404 + + +def test_live_chat_completion(live_server) -> None: + base_url, model = live_server + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": "Reply with one word."}], "max_tokens": 4}, + timeout=120, + ) + assert res.status_code == 200 + body = res.json() + assert body["choices"][0]["message"]["content"] + assert body["usage"]["completion_tokens"] > 0 + + +def test_live_chat_text_content_parts(live_server) -> None: + base_url, model = live_server + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "Reply with exactly "}, + {"type": "text", "text": "OK"}, + ], + }], + "max_tokens": 8, + "temperature": 0.0, + }, + timeout=180, + ) + assert res.status_code == 200 + assert res.json()["choices"][0]["message"]["content"] + + +def test_live_long_gemma_generation(live_server) -> None: + base_url, model = live_server + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": [{ + "role": "user", + "content": ( + "Write a numbered list of ten short items about why local HTTP " + "inference servers need end-to-end tests. Keep each item concise." + ), + }], + "max_tokens": 96, + "temperature": 0.0, + }, + timeout=180, + ) + assert res.status_code == 200 + body = res.json() + text = body["choices"][0]["message"]["content"] + assert isinstance(text, str) + assert len(text.split()) >= 20 + assert body["usage"]["completion_tokens"] >= 16 + assert body["choices"][0]["finish_reason"] == "stop" + + +def test_live_multi_turn_conversation(live_server) -> None: + base_url, model = live_server + messages = [ + {"role": "system", "content": "You answer directly and remember the user's chosen keyword."}, + {"role": "user", "content": "My keyword is cactus. Reply with only: remembered."}, + {"role": "assistant", "content": "remembered."}, + {"role": "user", "content": "What keyword did I give you? Reply with only the keyword."}, + ] + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": messages, + "max_tokens": 12, + "temperature": 0.0, + }, + timeout=180, + ) + assert res.status_code == 200 + body = res.json() + text = body["choices"][0]["message"]["content"].lower() + assert "cactus" in text + assert body["usage"]["prompt_tokens"] > body["usage"]["completion_tokens"] + + +def test_live_tool_call_completion(live_server) -> None: + base_url, model = live_server + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": [{"role": "user", "content": "Use the weather tool for Paris."}], + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + }], + "tool_choice": "required", + "max_tokens": 64, + "temperature": 0.0, + }, + timeout=180, + ) + assert res.status_code == 200 + body = res.json() + choice = body["choices"][0] + assert choice["finish_reason"] in {"tool_calls", "stop"} + if choice["finish_reason"] == "tool_calls": + calls = choice["message"]["tool_calls"] + assert calls + assert calls[0]["type"] == "function" + assert calls[0]["function"]["name"] + + +def test_live_concurrent_chat_requests(live_server) -> None: + base_url, model = live_server + + def request(i: int) -> int: + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": [{"role": "user", "content": f"Reply with the number {i}."}], + "max_tokens": 8, + "temperature": 0.0, + }, + timeout=180, + ) + assert res.status_code == 200 + assert res.json()["choices"][0]["message"]["content"] + return res.status_code + + with ThreadPoolExecutor(max_workers=2) as pool: + assert list(pool.map(request, [1, 2])) == [200, 200] + + +def test_live_chat_stream(live_server) -> None: + base_url, model = live_server + with httpx.stream( + "POST", + f"{base_url}/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": "Say hi."}], "max_tokens": 4, "stream": True}, + timeout=120, + ) as res: + assert res.status_code == 200 + text = "".join(res.iter_text()) + assert "chat.completion.chunk" in text + assert "data: [DONE]" in text + + +def test_live_long_streaming_generation(live_server) -> None: + base_url, model = live_server + with httpx.stream( + "POST", + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": [{"role": "user", "content": "Write one short paragraph about robust server testing."}], + "max_tokens": 64, + "stream": True, + }, + timeout=180, + ) as res: + assert res.status_code == 200 + text = "".join(res.iter_text()) + assert "chat.completion.chunk" in text + assert text.count('"delta"') >= 3 + assert "data: [DONE]" in text + + +def test_live_transcription_wav(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + audio = ASSETS / "test.wav" + assert audio.exists() + with audio.open("rb") as f: + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={"model": stt.name, "response_format": "json"}, + files={"file": ("test.wav", f, "audio/wav")}, + timeout=180, + ) + assert res.status_code == 200 + assert res.json()["text"] + + +def test_live_transcription_text_response(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + audio = ASSETS / "test.wav" + with audio.open("rb") as f: + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={"model": stt.name, "response_format": "text"}, + files={"file": ("test.wav", f, "audio/wav")}, + timeout=180, + ) + assert res.status_code == 200 + assert res.text.strip() + assert res.headers["content-type"].startswith("text/plain") + + +def test_live_transcription_verbose_json_segments(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + audio = ASSETS / "test.wav" + with audio.open("rb") as f: + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={ + "model": stt.name, + "response_format": "verbose_json", + "timestamp_granularities[]": "segment", + }, + files={"file": ("test.wav", f, "audio/wav")}, + timeout=180, + ) + assert res.status_code == 200 + body = res.json() + assert body["text"] + assert "segments" in body + + +def test_live_transcription_rejects_unsupported_format(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + audio = ASSETS / "test.wav" + with audio.open("rb") as f: + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={"model": stt.name, "response_format": "srt"}, + files={"file": ("test.wav", f, "audio/wav")}, + timeout=30, + ) + assert res.status_code == 400 + assert "Unsupported transcription response_format" in json.dumps(res.json()) + + +def test_live_transcription_rejects_word_timestamps(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + audio = ASSETS / "test.wav" + with audio.open("rb") as f: + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={ + "model": stt.name, + "response_format": "verbose_json", + "timestamp_granularities[]": "word", + }, + files={"file": ("test.wav", f, "audio/wav")}, + timeout=30, + ) + assert res.status_code == 400 + assert "Word timestamp" in json.dumps(res.json()) + + +def test_live_transcription_rejects_non_wav(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={"model": stt.name}, + files={"file": ("test.mp3", b"nope", "audio/mpeg")}, + timeout=30, + ) + assert res.status_code == 400 + assert "Only .wav" in json.dumps(res.json())