diff --git a/python/cactus/cli/__init__.py b/python/cactus/cli/__init__.py index 562d7ab58..1f45816aa 100644 --- a/python/cactus/cli/__init__.py +++ b/python/cactus/cli/__init__.py @@ -10,6 +10,7 @@ from .download import cmd_download from .compile import cmd_build from .run import cmd_run +from .serve import cmd_serve from .transcribe import cmd_transcribe from .test import cmd_test from .convert import cmd_convert @@ -68,6 +69,15 @@ def create_parser(): ----------------------------------------------------------------- + cactus serve [model] OpenAI-compatible local HTTP server + serves prepared v2 bundles only + + Optional flags: + --host bind address (default: 127.0.0.1) + --port port (default: 8080) + + ----------------------------------------------------------------- + cactus download fetch pre-converted CQ from Cactus-Compute Optional flags: @@ -199,6 +209,15 @@ def create_parser(): transcribe_parser.add_argument("--reconvert", action="store_true", help="Download original model and convert (instead of using pre-converted from Cactus-Compute)") + # ── serve ───────────────────────────────────────────────────────── + serve_parser = subparsers.add_parser("serve", help="Start OpenAI-compatible HTTP server") + serve_parser.add_argument("model", nargs="?", default=None, + help="Prepared v2 bundle path, local model dir name, or HF model ID") + serve_parser.add_argument("--host", default="127.0.0.1", + help="Bind address (default: 127.0.0.1)") + serve_parser.add_argument("--port", type=int, default=8080, + help="Port (default: 8080)") + # ── test ────────────────────────────────────────────────────────── test_parser = subparsers.add_parser("test", help="Run the test suite") test_parser.add_argument("--model", dest="model_id", default=DEFAULT_TEST_MODEL_ID, @@ -270,6 +289,7 @@ def create_parser(): "download": cmd_download, "build": cmd_build, "run": cmd_run, + "serve": cmd_serve, "transcribe": cmd_transcribe, "test": cmd_test, diff --git a/python/cactus/cli/serve.py b/python/cactus/cli/serve.py new file mode 100644 index 000000000..0a9d4fc3a --- /dev/null +++ b/python/cactus/cli/serve.py @@ -0,0 +1,71 @@ +from pathlib import Path + +from .common import BLUE, GREEN, RED, PROJECT_ROOT, is_repo_checkout, print_color +from .download import get_weights_dir + + +def _weights_root() -> Path: + if is_repo_checkout(): + return PROJECT_ROOT / "weights" + return Path.home() / ".cache" / "cactus" / "weights" + + +def _resolve_model_arg(model: str | None) -> tuple[Path | None, str | None]: + if not model: + return None, None + path = Path(model).expanduser() + if path.is_dir(): + return path, path.name + candidate = _weights_root() / model + if candidate.is_dir(): + return candidate, candidate.name + hf_candidate = get_weights_dir(model) + if hf_candidate.is_dir(): + return hf_candidate, hf_candidate.name + return None, model + + +def _is_valid_bundle(path: Path) -> bool: + return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() + + +def cmd_serve(args): + """Start the OpenAI-compatible HTTP server.""" + model_path, model_name = _resolve_model_arg(args.model) + if args.model and model_path is None: + print_color(RED, f"Error: model not found: {args.model}") + print("Prepare a v2 bundle first with `cactus run ` or `cactus convert `.") + return 1 + if model_path is not None and not _is_valid_bundle(model_path): + print_color(RED, f"Error: not a valid v2 Cactus bundle: {model_path}") + print("Expected config.txt and components/manifest.json.") + return 1 + + try: + import uvicorn + except ImportError: + print_color(RED, "Error: uvicorn not installed. Install the serve extra or run `pip install fastapi uvicorn python-multipart`.") + return 1 + + try: + from ..server import create_app + except ImportError: + print_color(RED, "Error: server dependencies not installed. Install the serve extra or run `pip install fastapi uvicorn python-multipart`.") + return 1 + + try: + application = create_app( + weights_root=_weights_root(), + model_path=model_path, + default_model=model_name, + ) + except RuntimeError as exc: + print_color(RED, f"Error: {exc}") + print("Prepare a v2 bundle first with `cactus run ` or `cactus convert `.") + return 1 + + models = sorted(application.state.registry.models) + print_color(GREEN, f"Available models: {', '.join(models)}") + print_color(BLUE, f"Starting server on {args.host}:{args.port}") + uvicorn.run(application, host=args.host, port=args.port, log_level="info") + return 0 diff --git a/python/cactus/server.py b/python/cactus/server.py new file mode 100644 index 000000000..7d5c9ad05 --- /dev/null +++ b/python/cactus/server.py @@ -0,0 +1,584 @@ +"""OpenAI-compatible local HTTP server for Cactus v2 bundles.""" +from __future__ import annotations + +import asyncio +import json +import logging +import time +import uuid +from contextlib import asynccontextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from fastapi import FastAPI, File, Form, HTTPException, Request, UploadFile +from fastapi.responses import PlainTextResponse, StreamingResponse +from pydantic import BaseModel + +from .bindings.cactus import ( + cactus_complete, + cactus_destroy, + cactus_get_last_error, + cactus_init, + cactus_reset, + cactus_transcribe, +) +from .cli.download import get_weights_dir +from .cli.common import DEFAULT_MODEL_ID, PROJECT_ROOT, is_repo_checkout + +LOGGER = logging.getLogger(__name__) + +LLM_MODEL_TYPES = {"gemma", "gemma3n", "gemma4", "lfm2", "qwen", "qwen3p5", "needle", "youtu"} +STT_MODEL_TYPES = {"whisper", "parakeet_tdt", "parakeet-tdt"} + + +@dataclass(frozen=True) +class ModelInfo: + id: str + path: Path + model_type: str + context_length: int + created: int + + +class ModelRegistry: + def __init__(self, weights_root: Path, extra_model: Path | None = None): + self.weights_root = weights_root + self.models: dict[str, ModelInfo] = {} + self._discover(weights_root) + if extra_model is not None: + info = self._info_for_dir(extra_model) + if info is None: + raise RuntimeError(f"Not a valid v2 Cactus bundle: {extra_model}") + self.models[info.id] = info + + @staticmethod + def _read_config_field(model_dir: Path, field: str) -> str: + config = model_dir / "config.txt" + if not config.exists(): + return "" + prefix = f"{field}=" + for line in config.read_text(encoding="utf-8").splitlines(): + if line.startswith(prefix): + return line.split("=", 1)[1].strip() + return "" + + @classmethod + def _info_for_dir(cls, path: Path) -> ModelInfo | None: + display_id = path.expanduser().name + resolved = path.expanduser().resolve() + if not (resolved / "config.txt").exists(): + return None + if not (resolved / "components" / "manifest.json").exists(): + return None + context_raw = cls._read_config_field(resolved, "context_length") + try: + context_length = int(context_raw or 0) + except ValueError: + context_length = 0 + stat = resolved.stat() + return ModelInfo( + id=display_id, + path=resolved, + model_type=cls._read_config_field(resolved, "model_type"), + context_length=context_length, + created=int(stat.st_mtime), + ) + + def _discover(self, root: Path) -> None: + if not root.exists(): + return + for entry in sorted(root.iterdir()): + if not entry.is_dir(): + continue + info = self._info_for_dir(entry) + if info is not None: + self.models[info.id] = info + + def require(self, model_id: str) -> ModelInfo: + info = self.models.get(model_id) + if info is None: + raise HTTPException(status_code=404, detail=f"Model '{model_id}' is not available") + return info + + def default_llm(self, preferred: str | None = None) -> ModelInfo: + if preferred: + info = self.models.get(preferred) + if info is None: + raise RuntimeError(f"Requested model '{preferred}' is not a valid v2 Cactus bundle") + if info.model_type not in LLM_MODEL_TYPES: + raise RuntimeError(f"Requested model '{preferred}' is not an LLM bundle") + return info + preferred_id = get_weights_dir(DEFAULT_MODEL_ID).name + if preferred_id in self.models and self.models[preferred_id].model_type in LLM_MODEL_TYPES: + return self.models[preferred_id] + for info in sorted(self.models.values(), key=lambda x: x.id): + if info.model_type in LLM_MODEL_TYPES: + return info + raise RuntimeError("No valid LLM bundles found. Prepare a transpiled bundle before running `cactus serve`.") + + def default_stt(self) -> ModelInfo | None: + for info in sorted(self.models.values(), key=lambda x: x.id): + if info.model_type in STT_MODEL_TYPES: + return info + return None + + def list_openai_models(self) -> list[dict[str, Any]]: + out = [] + for info in sorted(self.models.values(), key=lambda x: x.id): + out.append({ + "id": info.id, + "object": "model", + "created": info.created, + "owned_by": "cactus", + "context_window": info.context_length, + "model_type": info.model_type, + }) + return out + + +class _ModelSlot: + def __init__(self, info: ModelInfo, handle): + self.info = info + self.handle = handle + self.lock = asyncio.Lock() + self.last_used = time.monotonic() + self.active_requests = 0 + + def touch(self) -> None: + self.last_used = time.monotonic() + + +class ModelManager: + def __init__(self, registry: ModelRegistry, *, max_warm: int = 2): + self.registry = registry + self.max_warm = max_warm + self.slots: dict[str, _ModelSlot] = {} + self.swap_lock = asyncio.Lock() + + def _load(self, info: ModelInfo): + try: + return cactus_init(str(info.path)) + except RuntimeError as exc: + err = cactus_get_last_error() or str(exc) + raise HTTPException(status_code=500, detail=f"Failed to load model '{info.id}': {err}") from exc + + async def _get_slot(self, model_id: str) -> _ModelSlot: + info = self.registry.require(model_id) + async with self.swap_lock: + slot = self.slots.get(info.id) + if slot is not None: + slot.touch() + return slot + + if len(self.slots) >= self.max_warm: + idle = [s for s in self.slots.values() if s.active_requests == 0 and not s.lock.locked()] + if not idle: + raise HTTPException(status_code=503, detail="All warm model slots are busy") + victim = min(idle, key=lambda s: s.last_used) + self.slots.pop(victim.info.id, None) + cactus_destroy(victim.handle) + + handle = await asyncio.get_running_loop().run_in_executor(None, self._load, info) + slot = _ModelSlot(info, handle) + self.slots[info.id] = slot + return slot + + @asynccontextmanager + async def acquire(self, model_id: str): + slot = await self._get_slot(model_id) + async with self.swap_lock: + if slot.info.id not in self.slots: + raise HTTPException(status_code=503, detail="Model slot was evicted before use") + slot.active_requests += 1 + try: + yield slot + finally: + async with self.swap_lock: + slot.active_requests = max(0, slot.active_requests - 1) + slot.touch() + + async def preload(self, model_id: str) -> None: + async with self.acquire(model_id): + return + + def shutdown(self) -> None: + for slot in self.slots.values(): + cactus_destroy(slot.handle) + self.slots.clear() + + +class Permissive(BaseModel): + model_config = {"extra": "allow"} + + +class ChatMessage(Permissive): + role: str + content: str | list[Any] | None = None + tool_call_id: str | None = None + tool_calls: list[Any] | None = None + + +class ToolFunction(Permissive): + name: str + description: str | None = None + parameters: dict[str, Any] | None = None + + +class Tool(Permissive): + type: str = "function" + function: ToolFunction + + +class ToolChoiceFunction(Permissive): + name: str + + +class ToolChoiceObject(Permissive): + type: str = "function" + function: ToolChoiceFunction + + +class ChatRequest(Permissive): + model: str + messages: list[ChatMessage] + temperature: float | None = None + top_p: float | None = None + top_k: int | None = None + max_tokens: int | None = None + max_completion_tokens: int | None = None + stop: str | list[str] | None = None + stream: bool = False + tools: list[Tool] | None = None + tool_choice: str | ToolChoiceObject | None = None + + +def _flatten_message(msg: ChatMessage) -> dict[str, Any]: + out: dict[str, Any] = {"role": msg.role} + if isinstance(msg.content, list): + parts = [] + for part in msg.content: + if isinstance(part, dict) and part.get("type") == "text": + parts.append(str(part.get("text", ""))) + elif isinstance(part, str): + parts.append(part) + out["content"] = "\n".join(parts) + elif msg.content is not None: + out["content"] = msg.content + if msg.tool_call_id is not None: + out["tool_call_id"] = msg.tool_call_id + if msg.tool_calls is not None: + out["tool_calls"] = msg.tool_calls + return out + + +def _translate_tools(tools: list[Tool] | None, tool_choice) -> tuple[list[dict[str, Any]] | None, bool]: + if not tools or tool_choice == "none": + return None, False + if isinstance(tool_choice, ToolChoiceObject): + selected = [ + {"name": t.function.name, "description": t.function.description or "", "parameters": t.function.parameters or {}} + for t in tools + if t.function.name == tool_choice.function.name + ] + return selected or None, True + translated = [ + {"name": t.function.name, "description": t.function.description or "", "parameters": t.function.parameters or {}} + for t in tools + ] + return translated, tool_choice == "required" + + +def _make_tool_calls(function_calls: list[Any]) -> list[dict[str, Any]]: + out = [] + for call in function_calls: + if not isinstance(call, dict): + continue + args = call.get("arguments", {}) + out.append({ + "id": f"call_{uuid.uuid4().hex[:24]}", + "type": "function", + "function": { + "name": call.get("name", ""), + "arguments": json.dumps(args) if isinstance(args, dict) else str(args), + }, + }) + return out + + +def _chat_options(req: ChatRequest) -> dict[str, Any]: + options: dict[str, Any] = {} + for key in ("temperature", "top_p", "top_k"): + value = getattr(req, key) + if value is not None: + options[key] = value + max_tokens = req.max_tokens if req.max_tokens is not None else req.max_completion_tokens + if max_tokens is not None: + options["max_tokens"] = max_tokens + if req.stop: + options["stop_sequences"] = [req.stop] if isinstance(req.stop, str) else req.stop + return options + + +def _build_chat_response(result: dict[str, Any], model_id: str, request_id: str) -> dict[str, Any]: + function_calls = result.get("function_calls") or [] + tool_calls = _make_tool_calls(function_calls) + has_tool_calls = bool(tool_calls) + prefill = int(result.get("prefill_tokens") or 0) + decode = int(result.get("decode_tokens") or 0) + message: dict[str, Any] = {"role": "assistant", "content": None if has_tool_calls else result.get("response", "")} + if has_tool_calls: + message["tool_calls"] = tool_calls + return { + "id": request_id, + "object": "chat.completion", + "created": int(time.time()), + "model": model_id, + "system_fingerprint": None, + "choices": [{ + "index": 0, + "message": message, + "logprobs": None, + "finish_reason": "tool_calls" if has_tool_calls else "stop", + }], + "usage": { + "prompt_tokens": prefill, + "completion_tokens": decode, + "total_tokens": prefill + decode, + }, + } + + +def _event(data: dict[str, Any]) -> str: + return f"data: {json.dumps(data)}\n\n" + + +async def _stream_completion(manager: ModelManager, req: ChatRequest, request_id: str, messages, options, tools): + queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue() + loop = asyncio.get_running_loop() + + def on_token(token: str, token_id: int) -> None: + loop.call_soon_threadsafe(queue.put_nowait, ("token", token)) + + async def run_inference(): + error = None + result = None + try: + async with manager.acquire(req.model) as slot: + async with slot.lock: + cactus_reset(slot.handle) + result = await loop.run_in_executor( + None, + lambda: cactus_complete(slot.handle, messages, options, tools, on_token), + ) + except Exception as exc: + LOGGER.exception("Streaming completion failed") + error = exc + finally: + loop.call_soon_threadsafe(queue.put_nowait, ("done", (result, error))) + + task = asyncio.create_task(run_inference()) + created = int(time.time()) + yield _event({ + "id": request_id, + "object": "chat.completion.chunk", + "created": created, + "model": req.model, + "system_fingerprint": None, + "choices": [{"index": 0, "delta": {"role": "assistant", "content": ""}, "logprobs": None, "finish_reason": None}], + }) + + result = None + error = None + while True: + kind, value = await queue.get() + if kind == "token": + yield _event({ + "id": request_id, + "object": "chat.completion.chunk", + "created": created, + "model": req.model, + "system_fingerprint": None, + "choices": [{"index": 0, "delta": {"content": value}, "logprobs": None, "finish_reason": None}], + }) + elif kind == "done": + result, error = value + break + + await task + if error is not None: + yield f"event: error\ndata: {json.dumps({'error': str(error)})}\n\n" + yield "data: [DONE]\n\n" + return + + function_calls = result.get("function_calls") or [] + tool_calls = _make_tool_calls(function_calls) + finish_reason = "tool_calls" if tool_calls else "stop" + if tool_calls: + yield _event({ + "id": request_id, + "object": "chat.completion.chunk", + "created": created, + "model": req.model, + "system_fingerprint": None, + "choices": [{"index": 0, "delta": {"tool_calls": tool_calls}, "logprobs": None, "finish_reason": None}], + }) + final = { + "id": request_id, + "object": "chat.completion.chunk", + "created": created, + "model": req.model, + "system_fingerprint": None, + "choices": [{"index": 0, "delta": {}, "logprobs": None, "finish_reason": finish_reason}], + "usage": { + "prompt_tokens": int(result.get("prefill_tokens") or 0), + "completion_tokens": int(result.get("decode_tokens") or 0), + "total_tokens": int(result.get("total_tokens") or 0), + }, + } + yield _event(final) + yield "data: [DONE]\n\n" + + +def _requested_granularities(primary: list[str] | None, bracketed: list[str] | None) -> list[str]: + values = [] + for item in (primary or []) + (bracketed or []): + if item: + values.append(item) + return values + + +def create_app( + *, + weights_root: Path | None = None, + model_path: Path | None = None, + default_model: str | None = None, + max_warm: int = 2, + preload: bool = True, +) -> FastAPI: + root = Path(weights_root) if weights_root is not None else ( + PROJECT_ROOT / "weights" if is_repo_checkout() else Path.home() / ".cache" / "cactus" / "weights" + ) + registry = ModelRegistry(root, extra_model=model_path) + selected = registry.default_llm(default_model) + manager = ModelManager(registry, max_warm=max_warm) + + @asynccontextmanager + async def lifespan(app: FastAPI): + if preload: + await manager.preload(selected.id) + try: + yield + finally: + manager.shutdown() + + app = FastAPI(title="Cactus", version="0.1.0", lifespan=lifespan) + app.state.registry = registry + app.state.manager = manager + app.state.default_model = selected.id + app.state.default_stt_model = registry.default_stt().id if registry.default_stt() else None + + @app.get("/v1/models") + async def list_models(request: Request): + reg: ModelRegistry = request.app.state.registry + return {"object": "list", "data": reg.list_openai_models()} + + @app.post("/v1/chat/completions") + async def chat_completions(request: Request, req: ChatRequest): + reg: ModelRegistry = request.app.state.registry + info = reg.require(req.model) + if info.model_type not in LLM_MODEL_TYPES: + raise HTTPException(status_code=400, detail=f"Model '{req.model}' is not an LLM model") + messages = [_flatten_message(m) for m in req.messages] + tools, force_tools = _translate_tools(req.tools, req.tool_choice) + options = _chat_options(req) + if force_tools: + options["force_tools"] = True + request_id = f"chatcmpl-{uuid.uuid4().hex[:29]}" + mgr: ModelManager = request.app.state.manager + if req.stream: + return StreamingResponse( + _stream_completion(mgr, req, request_id, messages, options or None, tools), + media_type="text/event-stream", + ) + async with mgr.acquire(req.model) as slot: + async with slot.lock: + cactus_reset(slot.handle) + result = await asyncio.get_running_loop().run_in_executor( + None, + lambda: cactus_complete(slot.handle, messages, options or None, tools, None), + ) + if not result.get("success", False): + raise HTTPException(status_code=500, detail=result.get("error") or "completion failed") + return _build_chat_response(result, req.model, request_id) + + @app.post("/v1/audio/transcriptions") + async def create_transcription( + request: Request, + file: UploadFile = File(...), + model: str = Form(""), + language: str | None = Form(None), + prompt: str | None = Form(None), + response_format: str = Form("json"), + temperature: float | None = Form(None), + timestamp_granularities: list[str] | None = Form(None), + timestamp_granularities_array: list[str] | None = Form(None, alias="timestamp_granularities[]"), + ): + if response_format not in {"json", "text", "verbose_json"}: + raise HTTPException(status_code=400, detail=f"Unsupported transcription response_format: {response_format}") + granularities = _requested_granularities(timestamp_granularities, timestamp_granularities_array) + if "word" in granularities: + raise HTTPException(status_code=400, detail="Word timestamp granularity is not supported") + if any(g != "segment" for g in granularities): + raise HTTPException(status_code=400, detail=f"Unsupported timestamp granularity: {', '.join(granularities)}") + suffix = Path(file.filename or "").suffix.lower() + if suffix != ".wav": + raise HTTPException(status_code=400, detail="Only .wav transcription uploads are supported for now") + model_id = model or request.app.state.default_stt_model + if not model_id: + raise HTTPException(status_code=400, detail="No STT model available") + reg: ModelRegistry = request.app.state.registry + info = reg.require(model_id) + if info.model_type not in STT_MODEL_TYPES: + raise HTTPException(status_code=400, detail=f"Model '{model_id}' is not a speech-to-text model") + import tempfile + + with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: + tmp.write(await file.read()) + tmp_path = Path(tmp.name) + try: + options: dict[str, Any] = {} + if temperature is not None: + options["temperature"] = temperature + if language: + options["language"] = language + mgr: ModelManager = request.app.state.manager + async with mgr.acquire(model_id) as slot: + async with slot.lock: + cactus_reset(slot.handle) + result = await asyncio.get_running_loop().run_in_executor( + None, + lambda: cactus_transcribe(slot.handle, str(tmp_path), prompt, options or None, None), + ) + finally: + tmp_path.unlink(missing_ok=True) + if not result.get("success", False): + raise HTTPException(status_code=500, detail=result.get("error") or "transcription failed") + text = result.get("response", "") + if response_format == "text": + return PlainTextResponse(text) + if response_format == "verbose_json": + segments = result.get("segments") or [] + return { + "task": "transcribe", + "language": language or "", + "duration": segments[-1]["end"] if segments else 0.0, + "text": text, + "segments": [ + {"id": i, "start": seg["start"], "end": seg["end"], "text": seg["text"]} + for i, seg in enumerate(segments) + ], + } + return {"text": text} + + return app diff --git a/python/pyproject.toml b/python/pyproject.toml index 0425c1b2a..0c351a568 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -39,7 +39,8 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0"] +dev = ["pytest>=8.0", "httpx>=0.25.0"] +serve = ["fastapi>=0.100.0", "uvicorn>=0.20.0", "python-multipart>=0.0.5"] [project.urls] Homepage = "https://cactuscompute.com" diff --git a/python/requirements.txt b/python/requirements.txt index 1ba381ab7..4e509d70b 100644 --- a/python/requirements.txt +++ b/python/requirements.txt @@ -5,3 +5,7 @@ numpy==2.4.6 pillow==11.3.0 huggingface_hub==1.15.0 scipy==1.17.1 +fastapi>=0.100.0 +uvicorn>=0.20.0 +python-multipart>=0.0.5 +httpx>=0.25.0 diff --git a/python/tests/test_server.py b/python/tests/test_server.py new file mode 100644 index 000000000..5f570e070 --- /dev/null +++ b/python/tests/test_server.py @@ -0,0 +1,213 @@ +from pathlib import Path +from types import SimpleNamespace +import asyncio + +import pytest +from fastapi import HTTPException +from fastapi.testclient import TestClient + +import cactus.server as server + + +def _bundle(root: Path, name: str, model_type: str, *, manifest: bool = True) -> Path: + path = root / name + path.mkdir(parents=True) + (path / "config.txt").write_text( + f"model_type={model_type}\ncontext_length=1024\n", + encoding="utf-8", + ) + if manifest: + (path / "components").mkdir() + (path / "components" / "manifest.json").write_text('{"components":[]}', encoding="utf-8") + return path + + +def _app(tmp_path: Path, monkeypatch, *, preload: bool = False): + _bundle(tmp_path, "llm", "gemma4") + monkeypatch.setattr(server, "cactus_init", lambda *args, **kwargs: SimpleNamespace(args=args, kwargs=kwargs)) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: None) + return server.create_app(weights_root=tmp_path, default_model="llm", preload=preload) + + +def test_models_lists_only_v2_bundles(tmp_path: Path, monkeypatch) -> None: + _bundle(tmp_path, "valid", "gemma4") + _bundle(tmp_path, "old_weights", "lfm2", manifest=False) + monkeypatch.setattr(server, "cactus_init", lambda *args, **kwargs: object()) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: None) + + app = server.create_app(weights_root=tmp_path, default_model="valid", preload=False) + with TestClient(app) as client: + data = client.get("/v1/models").json()["data"] + + assert [m["id"] for m in data] == ["valid"] + assert data[0]["context_window"] == 1024 + + +def test_chat_completion_shapes_openai_response(tmp_path: Path, monkeypatch) -> None: + app = _app(tmp_path, monkeypatch) + + def fake_complete(handle, messages, options, tools, callback): + assert messages == [{"role": "user", "content": "hello"}] + assert options["max_tokens"] == 3 + return { + "success": True, + "response": "hi", + "function_calls": [], + "prefill_tokens": 2, + "decode_tokens": 1, + } + + monkeypatch.setattr(server, "cactus_complete", fake_complete) + monkeypatch.setattr(server, "cactus_reset", lambda handle: None) + + with TestClient(app) as client: + res = client.post("/v1/chat/completions", json={ + "model": "llm", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 3, + }) + + assert res.status_code == 200 + body = res.json() + assert body["object"] == "chat.completion" + assert body["choices"][0]["message"]["content"] == "hi" + assert body["usage"] == {"prompt_tokens": 2, "completion_tokens": 1, "total_tokens": 3} + + +def test_chat_tool_calls_are_translated(tmp_path: Path, monkeypatch) -> None: + app = _app(tmp_path, monkeypatch) + monkeypatch.setattr(server, "cactus_reset", lambda handle: None) + monkeypatch.setattr(server, "cactus_complete", lambda *args, **kwargs: { + "success": True, + "response": "", + "function_calls": [{"name": "lookup", "arguments": {"q": "x"}}], + "prefill_tokens": 1, + "decode_tokens": 1, + }) + + with TestClient(app) as client: + res = client.post("/v1/chat/completions", json={ + "model": "llm", + "messages": [{"role": "user", "content": "call tool"}], + "tools": [{"type": "function", "function": {"name": "lookup", "parameters": {}}}], + "tool_choice": "required", + }) + + choice = res.json()["choices"][0] + assert choice["finish_reason"] == "tool_calls" + assert choice["message"]["content"] is None + assert choice["message"]["tool_calls"][0]["function"]["name"] == "lookup" + assert choice["message"]["tool_calls"][0]["function"]["arguments"] == '{"q": "x"}' + + +def test_streaming_completion_error_terminates(tmp_path: Path, monkeypatch) -> None: + app = _app(tmp_path, monkeypatch) + monkeypatch.setattr(server, "cactus_reset", lambda handle: None) + + def fail_complete(*args, **kwargs): + raise RuntimeError("boom") + + monkeypatch.setattr(server, "cactus_complete", fail_complete) + + with TestClient(app) as client: + with client.stream("POST", "/v1/chat/completions", json={ + "model": "llm", + "messages": [{"role": "user", "content": "hello"}], + "stream": True, + }) as res: + body = "".join(res.iter_text()) + + assert res.status_code == 200 + assert "event: error" in body + assert "boom" in body + assert "data: [DONE]" in body + + +def test_transcription_rejects_non_wav(tmp_path: Path, monkeypatch) -> None: + _bundle(tmp_path, "llm", "gemma4") + _bundle(tmp_path, "stt", "parakeet_tdt") + monkeypatch.setattr(server, "cactus_init", lambda *args, **kwargs: object()) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: None) + app = server.create_app(weights_root=tmp_path, default_model="llm", preload=False) + + with TestClient(app) as client: + res = client.post( + "/v1/audio/transcriptions", + files={"file": ("audio.mp3", b"not wav", "audio/mpeg")}, + data={"model": "stt"}, + ) + + assert res.status_code == 400 + assert "Only .wav" in res.json()["detail"] + + +def test_transcription_verbose_json_with_segments(tmp_path: Path, monkeypatch) -> None: + _bundle(tmp_path, "llm", "gemma4") + _bundle(tmp_path, "stt", "parakeet_tdt") + monkeypatch.setattr(server, "cactus_init", lambda *args, **kwargs: object()) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: None) + monkeypatch.setattr(server, "cactus_reset", lambda handle: None) + monkeypatch.setattr(server, "cactus_transcribe", lambda *args, **kwargs: { + "success": True, + "response": "hello", + "segments": [{"start": 0.0, "end": 1.0, "text": "hello"}], + }) + app = server.create_app(weights_root=tmp_path, default_model="llm", preload=False) + + with TestClient(app) as client: + res = client.post( + "/v1/audio/transcriptions", + files={"file": ("audio.wav", b"wav", "audio/wav")}, + data={ + "model": "stt", + "response_format": "verbose_json", + "timestamp_granularities[]": "segment", + }, + ) + + assert res.status_code == 200 + assert res.json()["segments"] == [{"id": 0, "start": 0.0, "end": 1.0, "text": "hello"}] + + +def test_transcription_rejects_word_timestamps(tmp_path: Path, monkeypatch) -> None: + _bundle(tmp_path, "llm", "gemma4") + _bundle(tmp_path, "stt", "parakeet_tdt") + monkeypatch.setattr(server, "cactus_init", lambda *args, **kwargs: object()) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: None) + app = server.create_app(weights_root=tmp_path, default_model="llm", preload=False) + + with TestClient(app) as client: + res = client.post( + "/v1/audio/transcriptions", + files={"file": ("audio.wav", b"wav", "audio/wav")}, + data={"model": "stt", "timestamp_granularities[]": "word"}, + ) + + assert res.status_code == 400 + assert "Word timestamp" in res.json()["detail"] + + +def test_model_manager_does_not_evict_active_slots(tmp_path: Path, monkeypatch) -> None: + for name in ("a", "b", "c"): + _bundle(tmp_path, name, "gemma4") + destroyed = [] + monkeypatch.setattr(server, "cactus_init", lambda path, **kwargs: Path(path).name) + monkeypatch.setattr(server, "cactus_destroy", lambda handle: destroyed.append(handle)) + + registry = server.ModelRegistry(tmp_path) + manager = server.ModelManager(registry, max_warm=2) + + async def run(): + async with manager.acquire("a"): + async with manager.acquire("b"): + with pytest.raises(HTTPException) as exc: + async with manager.acquire("c"): + pass + assert exc.value.status_code == 503 + assert destroyed == [] + + async with manager.acquire("c"): + pass + + asyncio.run(run()) + assert len(destroyed) == 1 diff --git a/python/tests/test_server_live.py b/python/tests/test_server_live.py new file mode 100644 index 000000000..724444625 --- /dev/null +++ b/python/tests/test_server_live.py @@ -0,0 +1,434 @@ +from __future__ import annotations + +import json +import os +import socket +import subprocess +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import httpx +import pytest + +PROJECT_ROOT = Path(__file__).resolve().parents[2] +WEIGHTS = PROJECT_ROOT / "weights" +DEFAULT_LLM_BUNDLE = Path("weights/gemma-4-e2b-it") +ASSETS = PROJECT_ROOT / "cactus-engine" / "tests" / "assets" +LLM_TYPES = {"gemma", "gemma3n", "gemma4", "lfm2", "qwen", "qwen3p5", "needle", "youtu"} +STT_TYPES = {"whisper", "parakeet_tdt", "parakeet-tdt"} + + +def _read_model_type(bundle: Path) -> str: + for line in (bundle / "config.txt").read_text(encoding="utf-8").splitlines(): + if line.startswith("model_type="): + return line.split("=", 1)[1].strip() + return "" + + +def _valid_bundle(path: Path) -> bool: + return (path / "config.txt").exists() and (path / "components" / "manifest.json").exists() + + +def _require_bundle(relative: Path, types: set[str]) -> Path: + candidate = PROJECT_ROOT / relative + if not candidate.exists(): + pytest.fail( + f"Live-test model not found: {relative}\n" + f"Prepare it with `cactus convert google/gemma-4-E2B-it` from {PROJECT_ROOT}." + ) + if not _valid_bundle(candidate): + pytest.fail( + f"Live-test model is not a prepared v2 bundle: {relative}\n" + "Expected config.txt and components/manifest.json.\n" + f"Prepare it with `cactus convert google/gemma-4-E2B-it` from {PROJECT_ROOT}." + ) + model_type = _read_model_type(candidate) + if model_type not in types: + pytest.fail( + f"Live-test model has unsupported type {model_type!r}: {relative}\n" + f"Expected one of: {sorted(types)}." + ) + return candidate + + +def _find_bundle(preferred: list[str], types: set[str]) -> Path: + for name in preferred: + candidate = WEIGHTS / name + if candidate.exists() and _valid_bundle(candidate) and _read_model_type(candidate) in types: + return candidate + for candidate in sorted(WEIGHTS.iterdir()): + if candidate.is_dir() and _valid_bundle(candidate) and _read_model_type(candidate) in types: + return candidate + pytest.fail(f"No valid live-test bundle found under {WEIGHTS} for model types: {sorted(types)}") + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _start_server(bundle: Path, port: int) -> subprocess.Popen: + env = os.environ.copy() + python_path = str(PROJECT_ROOT / "python") + env["PYTHONPATH"] = python_path + os.pathsep + env.get("PYTHONPATH", "") + return subprocess.Popen( + [ + sys.executable, + "-m", + "cactus", + "serve", + str(bundle), + "--host", + "127.0.0.1", + "--port", + str(port), + ], + cwd=PROJECT_ROOT, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ) + + +def _wait_ready(proc: subprocess.Popen, base_url: str) -> None: + deadline = time.time() + 120 + last_error = None + while time.time() < deadline: + if proc.poll() is not None: + output = proc.stdout.read() if proc.stdout else "" + pytest.fail(f"server exited before becoming ready:\n{output}") + try: + with httpx.Client(timeout=2) as client: + res = client.get(f"{base_url}/v1/models") + if res.status_code == 200: + return + except Exception as exc: + last_error = exc + time.sleep(0.5) + pytest.fail(f"server did not become ready: {last_error}") + + +@pytest.fixture(scope="module") +def live_server(): + _require_bundle(DEFAULT_LLM_BUNDLE, LLM_TYPES) + port = _free_port() + proc = _start_server(DEFAULT_LLM_BUNDLE, port) + base_url = f"http://127.0.0.1:{port}" + try: + _wait_ready(proc, base_url) + yield base_url, DEFAULT_LLM_BUNDLE.name + finally: + proc.terminate() + try: + proc.wait(timeout=10) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + + +def test_live_models(live_server) -> None: + base_url, _ = live_server + res = httpx.get(f"{base_url}/v1/models", timeout=10) + assert res.status_code == 200 + models = res.json()["data"] + assert models + for model in models: + bundle = WEIGHTS / model["id"] + assert _valid_bundle(bundle) + + +def test_live_chat_rejects_unknown_model(live_server) -> None: + base_url, _ = live_server + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": "definitely-not-a-live-model", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 4, + }, + timeout=30, + ) + assert res.status_code == 404 + + +def test_live_chat_completion(live_server) -> None: + base_url, model = live_server + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": "Reply with one word."}], "max_tokens": 4}, + timeout=120, + ) + assert res.status_code == 200 + body = res.json() + assert body["choices"][0]["message"]["content"] + assert body["usage"]["completion_tokens"] > 0 + + +def test_live_chat_text_content_parts(live_server) -> None: + base_url, model = live_server + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "Reply with exactly "}, + {"type": "text", "text": "OK"}, + ], + }], + "max_tokens": 8, + "temperature": 0.0, + }, + timeout=180, + ) + assert res.status_code == 200 + assert res.json()["choices"][0]["message"]["content"] + + +def test_live_long_gemma_generation(live_server) -> None: + base_url, model = live_server + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": [{ + "role": "user", + "content": ( + "Write a numbered list of ten short items about why local HTTP " + "inference servers need end-to-end tests. Keep each item concise." + ), + }], + "max_tokens": 96, + "temperature": 0.0, + }, + timeout=180, + ) + assert res.status_code == 200 + body = res.json() + text = body["choices"][0]["message"]["content"] + assert isinstance(text, str) + assert len(text.split()) >= 20 + assert body["usage"]["completion_tokens"] >= 16 + assert body["choices"][0]["finish_reason"] == "stop" + + +def test_live_multi_turn_conversation(live_server) -> None: + base_url, model = live_server + messages = [ + {"role": "system", "content": "You answer directly and remember the user's chosen keyword."}, + {"role": "user", "content": "My keyword is cactus. Reply with only: remembered."}, + {"role": "assistant", "content": "remembered."}, + {"role": "user", "content": "What keyword did I give you? Reply with only the keyword."}, + ] + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": messages, + "max_tokens": 12, + "temperature": 0.0, + }, + timeout=180, + ) + assert res.status_code == 200 + body = res.json() + text = body["choices"][0]["message"]["content"].lower() + assert "cactus" in text + assert body["usage"]["prompt_tokens"] > body["usage"]["completion_tokens"] + + +def test_live_tool_call_completion(live_server) -> None: + base_url, model = live_server + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": [{"role": "user", "content": "Use the weather tool for Paris."}], + "tools": [{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + }], + "tool_choice": "required", + "max_tokens": 64, + "temperature": 0.0, + }, + timeout=180, + ) + assert res.status_code == 200 + body = res.json() + choice = body["choices"][0] + assert choice["finish_reason"] in {"tool_calls", "stop"} + if choice["finish_reason"] == "tool_calls": + calls = choice["message"]["tool_calls"] + assert calls + assert calls[0]["type"] == "function" + assert calls[0]["function"]["name"] + + +def test_live_concurrent_chat_requests(live_server) -> None: + base_url, model = live_server + + def request(i: int) -> int: + res = httpx.post( + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": [{"role": "user", "content": f"Reply with the number {i}."}], + "max_tokens": 8, + "temperature": 0.0, + }, + timeout=180, + ) + assert res.status_code == 200 + assert res.json()["choices"][0]["message"]["content"] + return res.status_code + + with ThreadPoolExecutor(max_workers=2) as pool: + assert list(pool.map(request, [1, 2])) == [200, 200] + + +def test_live_chat_stream(live_server) -> None: + base_url, model = live_server + with httpx.stream( + "POST", + f"{base_url}/v1/chat/completions", + json={"model": model, "messages": [{"role": "user", "content": "Say hi."}], "max_tokens": 4, "stream": True}, + timeout=120, + ) as res: + assert res.status_code == 200 + text = "".join(res.iter_text()) + assert "chat.completion.chunk" in text + assert "data: [DONE]" in text + + +def test_live_long_streaming_generation(live_server) -> None: + base_url, model = live_server + with httpx.stream( + "POST", + f"{base_url}/v1/chat/completions", + json={ + "model": model, + "messages": [{"role": "user", "content": "Write one short paragraph about robust server testing."}], + "max_tokens": 64, + "stream": True, + }, + timeout=180, + ) as res: + assert res.status_code == 200 + text = "".join(res.iter_text()) + assert "chat.completion.chunk" in text + assert text.count('"delta"') >= 3 + assert "data: [DONE]" in text + + +def test_live_transcription_wav(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + audio = ASSETS / "test.wav" + assert audio.exists() + with audio.open("rb") as f: + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={"model": stt.name, "response_format": "json"}, + files={"file": ("test.wav", f, "audio/wav")}, + timeout=180, + ) + assert res.status_code == 200 + assert res.json()["text"] + + +def test_live_transcription_text_response(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + audio = ASSETS / "test.wav" + with audio.open("rb") as f: + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={"model": stt.name, "response_format": "text"}, + files={"file": ("test.wav", f, "audio/wav")}, + timeout=180, + ) + assert res.status_code == 200 + assert res.text.strip() + assert res.headers["content-type"].startswith("text/plain") + + +def test_live_transcription_verbose_json_segments(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + audio = ASSETS / "test.wav" + with audio.open("rb") as f: + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={ + "model": stt.name, + "response_format": "verbose_json", + "timestamp_granularities[]": "segment", + }, + files={"file": ("test.wav", f, "audio/wav")}, + timeout=180, + ) + assert res.status_code == 200 + body = res.json() + assert body["text"] + assert "segments" in body + + +def test_live_transcription_rejects_unsupported_format(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + audio = ASSETS / "test.wav" + with audio.open("rb") as f: + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={"model": stt.name, "response_format": "srt"}, + files={"file": ("test.wav", f, "audio/wav")}, + timeout=30, + ) + assert res.status_code == 400 + assert "Unsupported transcription response_format" in json.dumps(res.json()) + + +def test_live_transcription_rejects_word_timestamps(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + audio = ASSETS / "test.wav" + with audio.open("rb") as f: + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={ + "model": stt.name, + "response_format": "verbose_json", + "timestamp_granularities[]": "word", + }, + files={"file": ("test.wav", f, "audio/wav")}, + timeout=30, + ) + assert res.status_code == 400 + assert "Word timestamp" in json.dumps(res.json()) + + +def test_live_transcription_rejects_non_wav(live_server) -> None: + base_url, _ = live_server + stt = _find_bundle(["parakeet-tdt-0.6b-v3-transpiled"], STT_TYPES) + res = httpx.post( + f"{base_url}/v1/audio/transcriptions", + data={"model": stt.name}, + files={"file": ("test.mp3", b"nope", "audio/mpeg")}, + timeout=30, + ) + assert res.status_code == 400 + assert "Only .wav" in json.dumps(res.json())