diff --git a/docs/python-client.md b/docs/python-client.md index fec151503..56b78e4aa 100644 --- a/docs/python-client.md +++ b/docs/python-client.md @@ -37,6 +37,18 @@ The context manager (`with`) is the recommended form; it guarantees shutdown. Th Useful managed-mode arguments: `binary=`, `host=`/`port=` (forces TCP and binds there), `socket=` (an explicit Unix socket path), `api_key=`, `ctx_size=`, `n_predict=`, `alias=`, `warmup=`, `extra_args=[...]` (forwarded verbatim to `mlxcel serve`), and `startup_timeout=`. +The last group (`ctx_size=`, `n_predict=`, `alias=`, `warmup=`, `extra_args=`) travels through `**server_kwargs`: any extra keyword argument configures the spawned `mlxcel serve` process rather than the client. + +```python +with mlxcel.LLM( + "mlx-community/Qwen3-4B-4bit", + ctx_size=8192, + warmup=False, + extra_args=["--parallel", "1"], +) as llm: + print(llm.generate("hello", max_tokens=32)) +``` + ## Connect mode Pass `base_url=` or `socket=` (but not a model) to talk to a server you started yourself. No subprocess is launched or managed. diff --git a/python/README.md b/python/README.md index ad1ec9a6a..12b86d694 100644 --- a/python/README.md +++ b/python/README.md @@ -51,6 +51,20 @@ Async usage mirrors the sync API via `mlxcel.AsyncLLM` (`await llm.generate(...) On POSIX, managed mode defaults to a Unix domain socket for low-overhead local IPC. Keep socket paths short (`sun_path` is about 104 bytes on macOS, 108 on Linux); the default lives under `/tmp`. Pass `socket=` to override. Windows uses TCP. +## Server options (`**server_kwargs`) + +In managed mode, extra keyword arguments to `LLM` / `AsyncLLM` are forwarded to the spawned `mlxcel serve` process: `ctx_size`, `n_predict`, `alias`, and `warmup` map to the matching CLI flags, `extra_args=[...]` appends raw CLI arguments verbatim, and `shutdown_grace` sets the termination grace period on close. + +```python +with mlxcel.LLM( + "mlx-community/Qwen3-4B-4bit", + ctx_size=8192, + warmup=False, + extra_args=["--parallel", "1"], +) as llm: + print(llm.generate("hello", max_tokens=32)) +``` + ## Sampling parameters `generate`, `stream`, `chat`, and `chat_stream` accept OpenAI sampling fields directly: `max_tokens`, `temperature`, `top_p`, `stop`, `seed`, `presence_penalty`, `frequency_penalty`, `logit_bias`, `response_format`. Server-specific knobs (`top_k`, `min_p`, `repetition_penalty`, DRY settings) are forwarded in the request body; you can also pass an explicit `extra_body={...}`. diff --git a/python/src/mlxcel/_async_client.py b/python/src/mlxcel/_async_client.py index 9eb7b4910..824d6f47e 100644 --- a/python/src/mlxcel/_async_client.py +++ b/python/src/mlxcel/_async_client.py @@ -55,6 +55,47 @@ def __init__( transport: Optional[httpx.AsyncBaseTransport] = None, **server_kwargs: Any, ) -> None: + """Create an async client in managed or connect mode. + + The arguments mirror :class:`mlxcel.LLM`. Passing ``model`` selects managed + mode: a local ``mlxcel serve`` process is spawned and supervised, and the + spawn plus readiness wait runs *synchronously* inside ``__init__`` (it is a + one-time blocking setup). Passing ``base_url``, ``socket`` without a model, or + ``transport`` selects connect mode: no subprocess. Combining a model with + ``base_url`` or ``transport`` raises :class:`~mlxcel.errors.MlxcelError`. + Unlike the sync client, the model id resolves lazily on the first request. + + Args: + model: Model to serve in managed mode (passed to ``mlxcel serve -m``): a + HuggingFace repo id or a local checkpoint path. + base_url: Base URL of a running server for connect mode, e.g. + ``"http://localhost:8080/v1"`` (a missing ``/v1`` suffix is added). + socket: Unix domain socket path. With ``model`` it is the bind path for the + spawned server (default: a short unique path under ``/tmp``); without + one it is the connect target. + api_key: API key sent as the bearer token on every request. In managed mode + it is also handed to the spawned server via the ``LLAMA_API_KEY`` + environment variable (never argv). + binary: Path to (or name of) the ``mlxcel`` executable. Falls back to the + ``MLXCEL_BIN`` environment variable, then ``mlxcel`` on ``PATH``. + Managed mode only. + host: TCP host for the spawned server. Setting it (or ``port``) forces TCP + instead of the default Unix socket. Managed mode only. + port: TCP port for the spawned server; defaults to a free ephemeral port + when TCP is in use. Managed mode only. + timeout: httpx timeout for requests. Defaults to + :data:`~mlxcel._common.DEFAULT_TIMEOUT` (generous 600-second read so a + slow first token does not abort a long generation). + startup_timeout: Seconds to wait for the spawned server's ``/health`` to + report ready before raising + :class:`~mlxcel.errors.MlxcelTimeoutError`. Managed mode only. + transport: Injected async httpx transport (e.g. ``httpx.MockTransport`` in + tests); implies connect mode. + **server_kwargs: Extra options for the spawned ``mlxcel serve`` process: + ``ctx_size``, ``n_predict``, ``alias``, ``warmup`` (map to the + matching CLI flags), ``extra_args`` (a list of raw CLI arguments + appended verbatim), and ``shutdown_grace``. Managed mode only. + """ # Set _closed before any call that can raise so __del__ never sees a # missing attribute even when __init__ fails early (e.g. bad arg combo). self._closed = False diff --git a/python/src/mlxcel/_client.py b/python/src/mlxcel/_client.py index e7ca831ab..1c1136b68 100644 --- a/python/src/mlxcel/_client.py +++ b/python/src/mlxcel/_client.py @@ -67,6 +67,46 @@ def __init__( transport: Optional[httpx.BaseTransport] = None, **server_kwargs: Any, ) -> None: + """Create a client in managed or connect mode. + + Passing ``model`` selects managed mode: a local ``mlxcel serve`` process is + spawned and supervised, and the constructor blocks until it reports ready. + Passing ``base_url``, ``socket`` without a model, or ``transport`` selects + connect mode: no subprocess, the client talks to an already-running server. + Combining a model with ``base_url`` or ``transport`` raises + :class:`~mlxcel.errors.MlxcelError`. + + Args: + model: Model to serve in managed mode (passed to ``mlxcel serve -m``): a + HuggingFace repo id or a local checkpoint path. + base_url: Base URL of a running server for connect mode, e.g. + ``"http://localhost:8080/v1"`` (a missing ``/v1`` suffix is added). + socket: Unix domain socket path. With ``model`` it is the bind path for the + spawned server (default: a short unique path under ``/tmp``); without + one it is the connect target. + api_key: API key sent as the bearer token on every request. In managed mode + it is also handed to the spawned server via the ``LLAMA_API_KEY`` + environment variable (never argv). + binary: Path to (or name of) the ``mlxcel`` executable. Falls back to the + ``MLXCEL_BIN`` environment variable, then ``mlxcel`` on ``PATH``. + Managed mode only. + host: TCP host for the spawned server. Setting it (or ``port``) forces TCP + instead of the default Unix socket. Managed mode only. + port: TCP port for the spawned server; defaults to a free ephemeral port + when TCP is in use. Managed mode only. + timeout: httpx timeout for requests. Defaults to + :data:`~mlxcel._common.DEFAULT_TIMEOUT` (generous 600-second read so a + slow first token does not abort a long generation). + startup_timeout: Seconds to wait for the spawned server's ``/health`` to + report ready before raising + :class:`~mlxcel.errors.MlxcelTimeoutError`. Managed mode only. + transport: Injected httpx transport (e.g. ``httpx.MockTransport`` in + tests); implies connect mode. + **server_kwargs: Extra options for the spawned ``mlxcel serve`` process: + ``ctx_size``, ``n_predict``, ``alias``, ``warmup`` (map to the + matching CLI flags), ``extra_args`` (a list of raw CLI arguments + appended verbatim), and ``shutdown_grace``. Managed mode only. + """ managed = is_managed(model, base_url, socket, transport) self._server: Optional[ManagedServer] = None