diff --git a/docs/basics/infrastructure.mdx b/docs/basics/infrastructure.mdx index d52f0919..71169ce4 100644 --- a/docs/basics/infrastructure.mdx +++ b/docs/basics/infrastructure.mdx @@ -16,6 +16,12 @@ Summarize the external services Neural touches (REST, WebSocket, FIX), their lat Latency reference: REST polling at 1s intervals, FIX round-trips ~5–10 ms, WebSocket delivers pushes \<100 ms once enabled. +## Deployment runtime model + +- The OSS SDK includes a built-in Docker deployment provider. +- Additional runtimes can be loaded as provider plugins through `neural.deployment`. +- This enables private or environment-specific deployment backends without forking SDK APIs. + ## Quick smoke tests ```bash @@ -57,4 +63,4 @@ REST polling (baseline) ─┬─> Strategy / Aggregator ──> TradingClient - Verify install & credentials: `getting-started` - Review execution options: `trading/overview` - Plan deployment workflows: `workflows/promotion-checklist` - +- Build custom runtime integrations: `workflows/deployment-providers` diff --git a/docs/mint.json b/docs/mint.json index 7546a7fe..db846e0e 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -106,7 +106,8 @@ "pages": [ "workflows/build-first-bot", "workflows/promotion-checklist", - "workflows/data-pipeline" + "workflows/data-pipeline", + "workflows/deployment-providers" ] }, { @@ -120,4 +121,4 @@ "github": "https://github.com/IntelIP/Neural", "twitter": "https://twitter.com/neural_sdk" } -} \ No newline at end of file +} diff --git a/docs/workflows/deployment-providers.mdx b/docs/workflows/deployment-providers.mdx new file mode 100644 index 00000000..3acfe2b1 --- /dev/null +++ b/docs/workflows/deployment-providers.mdx @@ -0,0 +1,62 @@ +--- +title: 'Deployment Provider Plugins' +description: 'Extend Neural deployment with external provider plugins (for example, private Daytona runtimes).' +--- + +Neural ships with a built-in Docker deployment provider and a registry for loading external providers. +This lets you keep proprietary runtime logic outside the OSS SDK while using the same deployment interface. + +## Built-in provider + +```python +from neural.deployment import create_provider + +docker = create_provider("docker") +``` + +The built-in `docker` provider maps to `DockerDeploymentProvider`. + +## Register providers programmatically + +```python +from neural.deployment import create_provider, register_provider +from neural.deployment.base import DeploymentProvider + +class MyProvider(DeploymentProvider): + ... + +register_provider("my_provider", MyProvider) +provider = create_provider("my_provider") +``` + +If a provider name already exists, pass `replace=True` to overwrite it. + +## Register providers via entry points + +External packages can register providers with Python entry points: + +```toml +[project.entry-points."neural.deployment.providers"] +daytona = "neural_daytona_runtime.provider:create_provider" +``` + +Neural auto-discovers this group and exposes the provider through `create_provider("daytona")`. + +## Error behavior + +- Unknown provider names raise `ProviderNotFoundError` with available providers listed. +- Plugins that fail to load raise `ProviderNotFoundError` with the plugin load error details. +- Factories that return non-`DeploymentProvider` objects raise `ConfigurationError`. + +## Plugin authoring checklist + +1. Implement `DeploymentProvider` methods: `deploy`, `status`, `logs`, `stop`, `list_deployments`, `cleanup`. +2. Keep secret loading in your provider package (not in SDK source). +3. Map runtime errors into Neural deployment exceptions with clear operator-facing messages. +4. Add unit tests for discovery and lifecycle methods. + +## Next + +- Production workflow progression: `workflows/promotion-checklist` +- Deployment and runtime dependencies: `basics/infrastructure` + diff --git a/neural/auth/polymarket_us_env.py b/neural/auth/polymarket_us_env.py index 35aaad31..bacd9ac3 100644 --- a/neural/auth/polymarket_us_env.py +++ b/neural/auth/polymarket_us_env.py @@ -1,3 +1,5 @@ +"""Helpers for resolving Polymarket US auth credentials from env/files.""" + from __future__ import annotations import base64 @@ -14,6 +16,7 @@ def _read_text_file(path: str | Path, label: str) -> str: + """Read a UTF-8 secret file and return stripped text content.""" try: return Path(path).read_text(encoding="utf-8").strip() except FileNotFoundError: @@ -23,6 +26,7 @@ def _read_text_file(path: str | Path, label: str) -> str: def _read_bytes_file(path: str | Path, label: str) -> bytes: + """Read a binary secret file.""" try: return Path(path).read_bytes() except FileNotFoundError: @@ -32,10 +36,12 @@ def _read_bytes_file(path: str | Path, label: str) -> bytes: def get_polymarket_us_base_url() -> str: + """Return the normalized base API URL for Polymarket US.""" return os.getenv("POLYMARKET_US_API_URL", POLYMARKET_US_API_URL).rstrip("/") def get_polymarket_us_api_key() -> str: + """Resolve API key from env var first, then fallback secret file path.""" value = os.getenv("POLYMARKET_US_API_KEY") if value: return value @@ -44,6 +50,7 @@ def get_polymarket_us_api_key() -> str: def get_polymarket_us_passphrase() -> str: + """Resolve API passphrase from env var first, then fallback secret file path.""" value = os.getenv("POLYMARKET_US_API_PASSPHRASE") if value: return value @@ -52,6 +59,7 @@ def get_polymarket_us_passphrase() -> str: def get_polymarket_us_api_secret() -> bytes: + """Resolve API secret bytes from env vars or fallback secret file path.""" b64_value = os.getenv("POLYMARKET_US_API_SECRET_BASE64") if b64_value: return base64.b64decode(b64_value) @@ -73,6 +81,7 @@ def get_polymarket_us_api_secret() -> bytes: def get_polymarket_us_credentials() -> dict[str, object]: + """Build full credential payload for signer initialization.""" return { "api_key": get_polymarket_us_api_key(), "api_secret": get_polymarket_us_api_secret(), diff --git a/neural/auth/signers/polymarket_us.py b/neural/auth/signers/polymarket_us.py index ee284fff..3d223f9c 100644 --- a/neural/auth/signers/polymarket_us.py +++ b/neural/auth/signers/polymarket_us.py @@ -61,12 +61,16 @@ def headers(self, method: str, path: str, body: str = "") -> dict[str, str]: } @classmethod - def from_env(cls, values: dict[str, Any], now_ms: TimestampFn | None = None) -> PolymarketUSSigner: + def from_env( + cls, values: dict[str, Any], now_ms: TimestampFn | None = None + ) -> PolymarketUSSigner: api_key = values.get("api_key") api_secret = values.get("api_secret") passphrase = values.get("passphrase") if api_key is None or api_secret is None or passphrase is None: - raise ValueError("Missing required Polymarket signer config: api_key, api_secret, passphrase") + raise ValueError( + "Missing required Polymarket signer config: api_key, api_secret, passphrase" + ) if isinstance(api_secret, str): secret_bytes = api_secret.encode("utf-8") diff --git a/neural/deployment/__init__.py b/neural/deployment/__init__.py index 9aa66787..f258fc14 100644 --- a/neural/deployment/__init__.py +++ b/neural/deployment/__init__.py @@ -31,6 +31,8 @@ ``` """ +from typing import Any + # Core abstractions from neural.deployment.base import DeploymentContext, DeploymentProvider @@ -45,15 +47,6 @@ MonitoringConfig, ) -# Docker provider -from neural.deployment.docker import ( - DockerDeploymentProvider, - render_compose_file, - render_dockerfile, - render_dockerignore, - write_compose_file, -) - # Exceptions from neural.deployment.exceptions import ( ConfigurationError, @@ -67,6 +60,57 @@ ProviderNotFoundError, ResourceLimitExceededError, ) +from neural.deployment.registry import create_provider, list_providers, register_provider + +_DOCKER_AVAILABLE = True +_DOCKER_IMPORT_ERROR: Exception | None = None + +try: + # Docker provider + from neural.deployment.docker import ( + DockerDeploymentProvider, + render_compose_file, + render_dockerfile, + render_dockerignore, + write_compose_file, + ) +except Exception as exc: # pragma: no cover - depends on optional dependency presence + _DOCKER_AVAILABLE = False + _DOCKER_IMPORT_ERROR = exc + + class DockerDeploymentProvider: # type: ignore[no-redef] + """Placeholder that raises when Docker deployment extras are missing.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise ProviderNotFoundError( + "Docker deployment provider is unavailable. " + "Install optional dependencies with: pip install 'neural-sdk[deployment]'" + ) from _DOCKER_IMPORT_ERROR + + def render_compose_file(*args: Any, **kwargs: Any) -> str: + raise ProviderNotFoundError( + "Docker compose rendering is unavailable. " + "Install optional dependencies with: pip install 'neural-sdk[deployment]'" + ) from _DOCKER_IMPORT_ERROR + + def render_dockerfile(*args: Any, **kwargs: Any) -> str: + raise ProviderNotFoundError( + "Dockerfile rendering is unavailable. " + "Install optional dependencies with: pip install 'neural-sdk[deployment]'" + ) from _DOCKER_IMPORT_ERROR + + def render_dockerignore() -> str: + raise ProviderNotFoundError( + "Docker ignore rendering is unavailable. " + "Install optional dependencies with: pip install 'neural-sdk[deployment]'" + ) from _DOCKER_IMPORT_ERROR + + def write_compose_file(*args: Any, **kwargs: Any) -> Any: + raise ProviderNotFoundError( + "Docker compose writing is unavailable. " + "Install optional dependencies with: pip install 'neural-sdk[deployment]'" + ) from _DOCKER_IMPORT_ERROR + __all__ = [ # Core abstractions @@ -83,6 +127,9 @@ "DeploymentInfo", # Providers "DockerDeploymentProvider", + "register_provider", + "create_provider", + "list_providers", # Docker utilities "render_dockerfile", "render_dockerignore", @@ -101,6 +148,10 @@ "MonitoringError", ] +# Register built-in providers so callers can use create_provider("docker", ...). +if _DOCKER_AVAILABLE: + register_provider("docker", DockerDeploymentProvider, replace=True) + # Convenience function for deploying with context manager def deploy( diff --git a/neural/deployment/registry.py b/neural/deployment/registry.py new file mode 100644 index 00000000..41e46d25 --- /dev/null +++ b/neural/deployment/registry.py @@ -0,0 +1,135 @@ +""" +Provider registry and plugin discovery for deployment backends. + +External packages can register deployment providers via setuptools entry points +under the group ``neural.deployment.providers``. +""" + +from __future__ import annotations + +from collections.abc import Callable +from importlib import metadata +from typing import Any + +from neural.deployment.base import DeploymentProvider +from neural.deployment.exceptions import ConfigurationError, ProviderNotFoundError + +PROVIDER_ENTRYPOINT_GROUP = "neural.deployment.providers" + +ProviderFactory = Callable[..., DeploymentProvider] + +_provider_factories: dict[str, ProviderFactory] = {} +_provider_load_errors: dict[str, str] = {} +_plugins_discovered = False + + +def _normalize_provider_name(name: str) -> str: + return name.strip().lower() + + +def register_provider(name: str, factory: ProviderFactory, *, replace: bool = False) -> None: + """Register a deployment provider factory. + + Args: + name: Provider name (e.g., "docker", "daytona") + factory: Callable that returns a DeploymentProvider instance + replace: Whether to overwrite an existing provider with the same name + + Raises: + ConfigurationError: If the provider name/factory is invalid or name is duplicated + """ + normalized_name = _normalize_provider_name(name) + if not normalized_name: + raise ConfigurationError("Provider name must be a non-empty string.") + if not callable(factory): + raise ConfigurationError(f"Provider '{normalized_name}' factory must be callable.") + if normalized_name in _provider_factories and not replace: + raise ConfigurationError( + f"Provider '{normalized_name}' is already registered. " + "Use replace=True to override it." + ) + + _provider_factories[normalized_name] = factory + # Clear stale loader error if the provider was successfully registered later. + _provider_load_errors.pop(normalized_name, None) + + +def discover_providers(*, force: bool = False) -> None: + """Discover provider plugins from Python entry points. + + Discovery is cached after the first successful pass unless ``force=True``. + """ + global _plugins_discovered + if _plugins_discovered and not force: + return + + entry_points = metadata.entry_points() + if hasattr(entry_points, "select"): + providers = entry_points.select(group=PROVIDER_ENTRYPOINT_GROUP) + else: # pragma: no cover - compatibility branch for older runtimes + providers = entry_points.get(PROVIDER_ENTRYPOINT_GROUP, []) + + for entry_point in providers: + provider_name = _normalize_provider_name(entry_point.name) + if provider_name in _provider_factories: + # Keep built-in providers deterministic; external plugins can use a different name. + continue + try: + loaded_factory = entry_point.load() + if not callable(loaded_factory): + raise TypeError("entry point did not resolve to a callable provider factory") + register_provider(provider_name, loaded_factory) + except Exception as exc: + _provider_load_errors[provider_name] = ( + f"Provider plugin '{provider_name}' failed to load from '{entry_point.value}': {exc}" + ) + + _plugins_discovered = True + + +def list_providers() -> list[str]: + """Return all successfully registered provider names.""" + discover_providers() + return sorted(_provider_factories.keys()) + + +def create_provider(name: str, **kwargs: Any) -> DeploymentProvider: + """Create a provider instance by name. + + Args: + name: Registered provider name + **kwargs: Arguments forwarded to the provider factory + + Raises: + ProviderNotFoundError: If provider is missing or failed to load + ConfigurationError: If the factory returns the wrong object type + """ + discover_providers() + normalized_name = _normalize_provider_name(name) + + load_error = _provider_load_errors.get(normalized_name) + if load_error: + raise ProviderNotFoundError(load_error) + + factory = _provider_factories.get(normalized_name) + if not factory: + available = ", ".join(sorted(_provider_factories.keys())) or "(none)" + raise ProviderNotFoundError( + f"Provider '{normalized_name}' is not registered. Available providers: {available}." + ) + + provider = factory(**kwargs) + if not isinstance(provider, DeploymentProvider): + raise ConfigurationError( + f"Provider '{normalized_name}' factory returned {type(provider).__name__}, " + "expected a DeploymentProvider instance." + ) + return provider + + +def _reset_registry_for_tests() -> None: + """Reset global state for unit tests.""" + global _plugins_discovered + _provider_factories.clear() + _provider_load_errors.clear() + _plugins_discovered = False diff --git a/tests/deployment/test_registry.py b/tests/deployment/test_registry.py new file mode 100644 index 00000000..ae80b067 --- /dev/null +++ b/tests/deployment/test_registry.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + +import pytest + +from neural.deployment.base import DeploymentProvider +from neural.deployment.config import DeploymentInfo, DeploymentResult, DeploymentStatus +from neural.deployment.exceptions import ConfigurationError, ProviderNotFoundError +from neural.deployment.registry import ( + _reset_registry_for_tests, + create_provider, + discover_providers, + list_providers, + register_provider, +) + + +@dataclass +class _DummyProvider(DeploymentProvider): + marker: str = "ok" + + async def deploy(self, config) -> DeploymentResult: # pragma: no cover - not used in tests + return DeploymentResult(deployment_id="x", status="running") + + async def stop(self, deployment_id: str) -> bool: # pragma: no cover - not used in tests + return True + + async def status(self, deployment_id: str) -> DeploymentStatus: # pragma: no cover + return DeploymentStatus(deployment_id=deployment_id, status="running") + + async def logs(self, deployment_id: str, tail: int = 100) -> list[str]: # pragma: no cover + return [] + + async def list_deployments(self) -> list[DeploymentInfo]: # pragma: no cover + return [] + + async def cleanup(self) -> None: # pragma: no cover + return None + + +@pytest.fixture(autouse=True) +def _reset_registry_state() -> None: + _reset_registry_for_tests() + yield + _reset_registry_for_tests() + + +def test_register_list_and_create_provider() -> None: + register_provider("dummy", _DummyProvider) + + assert list_providers() == ["dummy"] + + provider = create_provider("dummy", marker="custom") + assert isinstance(provider, _DummyProvider) + assert provider.marker == "custom" + + +def test_register_duplicate_provider_without_replace_fails() -> None: + register_provider("dummy", _DummyProvider) + with pytest.raises(ConfigurationError, match="already registered"): + register_provider("dummy", _DummyProvider) + + +def test_create_provider_unknown_name_shows_available() -> None: + register_provider("dummy", _DummyProvider) + + with pytest.raises(ProviderNotFoundError, match="Available providers: dummy"): + create_provider("missing") + + +def test_discover_providers_loads_entry_points_and_tracks_broken_plugins(monkeypatch) -> None: + class _FakeEntryPoint: + def __init__(self, name: str, value: str, loader: Any): + self.name = name + self.value = value + self._loader = loader + + def load(self): + return self._loader() + + class _FakeEntryPoints: + def __init__(self, items: list[_FakeEntryPoint]): + self._items = items + + def select(self, **kwargs): + if kwargs.get("group") == "neural.deployment.providers": + return self._items + return [] + + def _good_loader(): + return _DummyProvider + + def _bad_loader(): + raise RuntimeError("boom") + + monkeypatch.setattr( + "neural.deployment.registry.metadata.entry_points", + lambda: _FakeEntryPoints( + [ + _FakeEntryPoint("good_plugin", "pkg.good:factory", _good_loader), + _FakeEntryPoint("bad_plugin", "pkg.bad:factory", _bad_loader), + ] + ), + ) + + discover_providers(force=True) + + assert "good_plugin" in list_providers() + provider = create_provider("good_plugin", marker="plugin") + assert isinstance(provider, _DummyProvider) + assert provider.marker == "plugin" + + with pytest.raises(ProviderNotFoundError, match="failed to load"): + create_provider("bad_plugin") + + +def test_factory_must_return_provider_instance() -> None: + register_provider("bad", lambda **_: object()) + + with pytest.raises(ConfigurationError, match="expected a DeploymentProvider instance"): + create_provider("bad") diff --git a/tests/exchanges/test_polymarket_streaming.py b/tests/exchanges/test_polymarket_streaming.py index 6227664c..35017483 100644 --- a/tests/exchanges/test_polymarket_streaming.py +++ b/tests/exchanges/test_polymarket_streaming.py @@ -44,7 +44,9 @@ async def __call__( extra_headers: dict[str, str], open_timeout: int, ) -> FakeConnection: - self.calls.append({"url": url, "extra_headers": extra_headers, "open_timeout": open_timeout}) + self.calls.append( + {"url": url, "extra_headers": extra_headers, "open_timeout": open_timeout} + ) if not self._conns: raise RuntimeError("No available websocket connections") return self._conns.pop(0) @@ -122,7 +124,9 @@ def test_sequence_rules_dedupe_and_gap() -> None: signer=_new_signer(), ) - first = client._apply_sequence_rules({"channel": "markets", "market_id": "MKT-1", "sequence": 1}) + first = client._apply_sequence_rules( + {"channel": "markets", "market_id": "MKT-1", "sequence": 1} + ) duplicate = client._apply_sequence_rules( {"channel": "markets", "market_id": "MKT-1", "sequence": 1} )