From 3ca3302c91a2ff2ccf2813ccd4fd5308a3e27a61 Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Fri, 4 Sep 2026 11:47:36 +0800 Subject: [PATCH 01/12] [fix][video-gateway] recover stale discovery watches safely Detect silent etcd watch stalls and compare the Runtime worker view with etcd discovery truth. Coordinate HA replica restarts with an etcd lease lock, preserve drain behavior, and expose bounded consistency metrics. The watchdog is opt-in and validated for etcd-backed Video Gateway deployments, so ordinary Router, DeepSeek, GLM, and other LLM paths keep their existing behavior. --- dingo/video_gateway/__main__.py | 42 ++++- dingo/video_gateway/config.py | 76 +++++++- dingo/video_gateway/dispatcher.py | 163 ++++++++++++++++-- dingo/video_gateway/etcd_http.py | 6 +- dingo/video_gateway/task_store.py | 99 +++++++++++ dingo/video_gateway/telemetry.py | 26 +++ tests/video_gateway/test_config.py | 72 ++++++++ .../video_gateway/test_discovery_watchdog.py | 154 +++++++++++++++++ tests/video_gateway/test_etcd_http.py | 53 ++++++ tests/video_gateway/test_etcd_task_store.py | 55 ++++++ tests/video_gateway/test_main.py | 33 ++++ tests/video_gateway/test_telemetry.py | 7 + 12 files changed, 760 insertions(+), 26 deletions(-) create mode 100644 tests/video_gateway/test_discovery_watchdog.py diff --git a/dingo/video_gateway/__main__.py b/dingo/video_gateway/__main__.py index 907fac4d96c2..ba0245f7d5ad 100644 --- a/dingo/video_gateway/__main__.py +++ b/dingo/video_gateway/__main__.py @@ -47,9 +47,15 @@ async def _wait_for_shutdown(runtime, stopped: asyncio.Event) -> bool: signal. A cancelled Runtime cannot rebuild its discovery clients, so the Video Gateway must let Kubernetes create a fresh process. """ + async def wait_runtime_shutdown() -> None: + # PyO3's future_into_py returns an asyncio Future, while test doubles + # and pure-Python runtimes may return a coroutine. Await it inside a + # coroutine so create_task supports both forms consistently. + await runtime.wait_shutdown() + signal_wait = asyncio.create_task(stopped.wait(), name="video-gateway-signal") runtime_wait = asyncio.create_task( - runtime.wait_shutdown(), name="video-gateway-runtime-shutdown" + wait_runtime_shutdown(), name="video-gateway-runtime-shutdown" ) waits = {signal_wait, runtime_wait} try: @@ -93,6 +99,9 @@ async def run(args: argparse.Namespace) -> None: EtcdHttpClient( config.task_store.endpoints, timeout_s=config.task_store.request_timeout_s, + watch_response_timeout_s=( + config.task_store.watch_response_timeout_s + ), telemetry=telemetry, ), prefix=config.task_store.prefix, @@ -128,10 +137,33 @@ def request_stop() -> None: for sig in (signal.SIGINT, signal.SIGTERM): loop.add_signal_handler(sig, request_stop) - if await _wait_for_shutdown(runtime, stopped): - reason = "Dynamo discovery Runtime terminated" - logger.critical("%s; restarting Video Gateway", reason) - raise RuntimeError(reason) + process_shutdown = asyncio.create_task( + _wait_for_shutdown(runtime, stopped), + name="video-gateway-process-shutdown", + ) + dispatcher_fatal = asyncio.create_task( + dispatcher.wait_fatal(), name="video-gateway-dispatcher-fatal" + ) + try: + done, _pending = await asyncio.wait( + {process_shutdown, dispatcher_fatal}, + return_when=asyncio.FIRST_COMPLETED, + ) + if dispatcher_fatal in done: + reason = dispatcher_fatal.result() + logger.critical("%s", reason) + raise RuntimeError(reason) + if await process_shutdown: + reason = "Dynamo discovery Runtime terminated" + logger.critical("%s; restarting Video Gateway", reason) + raise RuntimeError(reason) + finally: + for waiter in (process_shutdown, dispatcher_fatal): + if not waiter.done(): + waiter.cancel() + await asyncio.gather( + process_shutdown, dispatcher_fatal, return_exceptions=True + ) finally: try: if runner is not None: diff --git a/dingo/video_gateway/config.py b/dingo/video_gateway/config.py index ac50367b1d3d..011fff6ff71a 100644 --- a/dingo/video_gateway/config.py +++ b/dingo/video_gateway/config.py @@ -79,11 +79,21 @@ def canonical_backend_target(value: str) -> str: return "dyn://" + ".".join(parts) +@dataclass(frozen=True, slots=True) +class DiscoveryWatchdogConfig: + enabled: bool = False + interval_s: float = 2.0 + mismatch_grace_s: float = 6.0 + + @dataclass(frozen=True, slots=True) class RuntimeConfig: discovery_backend: str = "etcd" request_plane: str = "tcp" event_plane: str | None = "zmq" + discovery_watchdog: DiscoveryWatchdogConfig = field( + default_factory=DiscoveryWatchdogConfig + ) @dataclass(frozen=True, slots=True) @@ -133,6 +143,7 @@ class TaskStoreConfig: endpoints: tuple[str, ...] = () prefix: str = "/dingo/video-gateway/v1" request_timeout_s: float = 5.0 + watch_response_timeout_s: float | None = None @dataclass(frozen=True, slots=True) @@ -214,7 +225,35 @@ def pool_for_model(self, model: str) -> PoolConfig | None: def _runtime_config(raw: Any) -> RuntimeConfig: data = _mapping(raw or {}, "runtime") - _only(data, {"discovery_backend", "request_plane", "event_plane"}, "runtime") + _only( + data, + { + "discovery_backend", + "request_plane", + "event_plane", + "discovery_watchdog", + }, + "runtime", + ) + watchdog_data = _mapping( + data.get("discovery_watchdog") or {}, "runtime.discovery_watchdog" + ) + _only( + watchdog_data, + {"enabled", "interval_s", "mismatch_grace_s"}, + "runtime.discovery_watchdog", + ) + watchdog_enabled = _boolean(watchdog_data, "enabled", False) + watchdog_interval_s = float(watchdog_data.get("interval_s", 2.0)) + watchdog_mismatch_grace_s = float( + watchdog_data.get("mismatch_grace_s", 6.0) + ) + if watchdog_interval_s <= 0: + raise ValueError("runtime.discovery_watchdog.interval_s must be positive") + if watchdog_mismatch_grace_s < watchdog_interval_s: + raise ValueError( + "runtime.discovery_watchdog.mismatch_grace_s must be at least interval_s" + ) event_plane = data.get("event_plane", "zmq") if event_plane is not None and not isinstance(event_plane, str): raise ValueError("runtime.event_plane must be a string or null") @@ -222,6 +261,11 @@ def _runtime_config(raw: Any) -> RuntimeConfig: discovery_backend=str(data.get("discovery_backend", "etcd")), request_plane=str(data.get("request_plane", "tcp")), event_plane=event_plane, + discovery_watchdog=DiscoveryWatchdogConfig( + enabled=watchdog_enabled, + interval_s=watchdog_interval_s, + mismatch_grace_s=watchdog_mismatch_grace_s, + ), ) if config.discovery_backend not in {"etcd", "kubernetes", "file", "mem"}: raise ValueError("runtime.discovery_backend is not supported") @@ -331,7 +375,14 @@ def _task_store_config(raw: Any) -> TaskStoreConfig: data = _mapping(raw or {}, "task_store") _only( data, - {"kind", "url", "endpoints", "prefix", "request_timeout_s"}, + { + "kind", + "url", + "endpoints", + "prefix", + "request_timeout_s", + "watch_response_timeout_s", + }, "task_store", ) kind = str(data.get("kind", "etcd_http")) @@ -374,12 +425,19 @@ def _task_store_config(raw: Any) -> TaskStoreConfig: timeout = float(data.get("request_timeout_s", 5.0)) if timeout <= 0: raise ValueError("task_store.request_timeout_s must be positive") + watch_timeout_raw = data.get("watch_response_timeout_s") + watch_timeout = ( + None if watch_timeout_raw is None else float(watch_timeout_raw) + ) + if watch_timeout is not None and watch_timeout <= 0: + raise ValueError("task_store.watch_response_timeout_s must be positive") return TaskStoreConfig( kind=kind, url=url.strip().rstrip("/") if isinstance(url, str) else None, endpoints=endpoints, prefix=prefix.rstrip("/"), request_timeout_s=timeout, + watch_response_timeout_s=watch_timeout, ) @@ -644,13 +702,23 @@ def parse_config(raw: Any) -> GatewayConfig: f"http.default_model {http.default_model!r} is not a configured served model" ) + runtime = _runtime_config(data.get("runtime")) + task_store = _task_store_config(data.get("task_store")) + if runtime.discovery_watchdog.enabled and ( + runtime.discovery_backend != "etcd" or task_store.kind != "etcd_http" + ): + raise ValueError( + "runtime.discovery_watchdog requires etcd Runtime discovery and " + "an etcd_http task_store" + ) + return GatewayConfig( schema_version=schema_version, deployment_id=deployment_id, - runtime=_runtime_config(data.get("runtime")), + runtime=runtime, http=http, media=media, - task_store=_task_store_config(data.get("task_store")), + task_store=task_store, artifact_store=_artifact_store_config(data.get("artifact_store")), lifecycle=_lifecycle_config(data.get("lifecycle")), pools=pools, diff --git a/dingo/video_gateway/dispatcher.py b/dingo/video_gateway/dispatcher.py index 21f7ba6b1d24..6fb87fe11e76 100644 --- a/dingo/video_gateway/dispatcher.py +++ b/dingo/video_gateway/dispatcher.py @@ -51,6 +51,9 @@ _DETACHED_WAIT_RETRY_INITIAL_S = 0.2 _DETACHED_WAIT_RETRY_MAX_S = 5.0 _DETACHED_WORKER_STALE_S = 20.0 +_DISCOVERY_MISMATCH_MIN_CHECKS = 3 +_DISCOVERY_RECOVERY_LOCK_TTL_S = 15 +_DISCOVERY_RESTART_DRAIN_S = 5.0 class _DetachedWorkerCancelled(RuntimeError): @@ -184,8 +187,12 @@ def __init__( self._artifact_released_bytes = 0 self._gateway_lease_id: int | None = None self._gateway_owner_healthy = not store.gateway_owner_supported + self._fatal_restart_pending = False self._fatal_error: str | None = None + self._fatal_event = asyncio.Event() self._orphan_recovery_lock = asyncio.Lock() + self._discovery_mismatch_started: dict[str, float] = {} + self._discovery_mismatch_checks: dict[str, int] = {} @property def ready(self) -> bool: @@ -224,6 +231,28 @@ def begin_drain(self) -> bool: pool.wakeup.set() return True + async def wait_fatal(self) -> str: + await self._fatal_event.wait() + return self._fatal_error or "video Gateway requested restart" + + async def _request_fatal_restart( + self, reason: str, *, drain_s: float = 0.0 + ) -> None: + if self._fatal_restart_pending or self._fatal_error is not None: + return + self._fatal_restart_pending = True + self.begin_drain() + self._ready = False + if drain_s > 0: + try: + await asyncio.wait_for(self._stop.wait(), timeout=drain_s) + except asyncio.TimeoutError: + pass + if self._stop.is_set(): + return + self._fatal_error = reason + self._fatal_event.set() + async def start(self) -> None: await self.store.health() await self.artifacts.health() @@ -261,6 +290,17 @@ async def start(self) -> None: self._pool_loop(pool), name=f"video-dispatch-{pool.config.pool_id}" ) ) + if self.config.runtime.discovery_watchdog.enabled: + if not self.store.discovery_truth_supported: + raise RuntimeError( + "Dynamo discovery watchdog requires an etcd discovery truth source" + ) + self._loops.append( + asyncio.create_task( + self._discovery_watchdog_loop(), + name="video-discovery-watchdog", + ) + ) self._loops.append( asyncio.create_task(self._sweeper_loop(), name="video-task-sweeper") ) @@ -331,22 +371,7 @@ async def _gateway_owner_loop(self) -> None: self.telemetry.increment( "dingo_video_gateway_owner_lease_lost_total" ) - self._fatal_error = "Gateway owner lease was lost" - self._ready = False - self._stop.set() - self._wake_task_waiters() - for pool in self.pools.values(): - pool.wakeup.set() - for running in list(self.running_calls.values()): - if running.detached: - running.execution.cancel() - continue - try: - running.context.stop_generating() - except Exception: - logger.exception( - "failed to stop task after Gateway owner lease loss" - ) + await self._request_fatal_restart("Gateway owner lease was lost") return def has_workers(self, pool_id: str) -> bool: @@ -571,6 +596,112 @@ async def _refresh_instances(self, pool: PoolRuntime) -> None: pool.instance_ids = [] pool.discovery_healthy = False + def _record_discovery_match( + self, + pool: PoolRuntime, + runtime_ids: set[int], + truth_ids: set[int], + ) -> bool: + missing = truth_ids - runtime_ids + stale = runtime_ids - truth_ids + labels = {"pool": pool.config.pool_id} + self.telemetry.set_gauge( + "dingo_video_discovery_consistent", 0 if missing or stale else 1, + labels=labels, + ) + self.telemetry.set_gauge( + "dingo_video_discovery_missing_instances", len(missing), labels=labels + ) + self.telemetry.set_gauge( + "dingo_video_discovery_stale_instances", len(stale), labels=labels + ) + if not missing and not stale: + self._discovery_mismatch_started.pop(pool.config.pool_id, None) + self._discovery_mismatch_checks.pop(pool.config.pool_id, None) + self.telemetry.set_gauge( + "dingo_video_discovery_last_consistent_timestamp_seconds", + time.time(), + labels=labels, + ) + return True + if missing: + self.telemetry.increment( + "dingo_video_discovery_mismatch_checks_total", + labels={**labels, "direction": "missing_in_runtime"}, + ) + if stale: + self.telemetry.increment( + "dingo_video_discovery_mismatch_checks_total", + labels={**labels, "direction": "stale_in_runtime"}, + ) + self._discovery_mismatch_started.setdefault( + pool.config.pool_id, time.monotonic() + ) + self._discovery_mismatch_checks[pool.config.pool_id] = ( + self._discovery_mismatch_checks.get(pool.config.pool_id, 0) + 1 + ) + return False + + async def _discovery_watchdog_loop(self) -> None: + watchdog = self.config.runtime.discovery_watchdog + targets = [pool.config.backend_target for pool in self.pools.values()] + while not self._stop.is_set(): + started = time.monotonic() + try: + truth = await self.store.discovery_instance_snapshot(targets) + mature_mismatches: list[str] = [] + for pool in self.pools.values(): + runtime_ids = set(pool.client.instance_ids()) + truth_ids = truth[pool.config.backend_target] + if self._record_discovery_match(pool, runtime_ids, truth_ids): + continue + mismatch_started = self._discovery_mismatch_started[ + pool.config.pool_id + ] + mismatch_checks = self._discovery_mismatch_checks[ + pool.config.pool_id + ] + if ( + mismatch_checks >= _DISCOVERY_MISMATCH_MIN_CHECKS + and time.monotonic() - mismatch_started + >= watchdog.mismatch_grace_s + ): + mature_mismatches.append(pool.config.pool_id) + if mature_mismatches: + acquired = await self.store.try_acquire_discovery_recovery( + self.generation, + ttl_s=_DISCOVERY_RECOVERY_LOCK_TTL_S, + ) + if acquired: + pools = ",".join(sorted(mature_mismatches)) + self.telemetry.increment( + "dingo_video_discovery_watchdog_restarts_total" + ) + reason = ( + "Dynamo discovery view remained inconsistent for pools=" + f"{pools}; restarting this Video Gateway replica" + ) + logger.error(reason) + await self._request_fatal_restart( + reason, drain_s=_DISCOVERY_RESTART_DRAIN_S + ) + return + except asyncio.CancelledError: + raise + except Exception: + self.telemetry.increment( + "dingo_video_discovery_watchdog_errors_total" + ) + logger.exception("Dynamo discovery watchdog check failed") + remaining = watchdog.interval_s - (time.monotonic() - started) + if remaining <= 0: + await asyncio.sleep(0) + continue + try: + await asyncio.wait_for(self._stop.wait(), timeout=remaining) + except asyncio.TimeoutError: + pass + async def _resync_lease_cache(self, pool: PoolRuntime) -> None: pool.lease_watch_healthy = False leases, revision = await self.store.lease_snapshot(pool.config.pool_id) diff --git a/dingo/video_gateway/etcd_http.py b/dingo/video_gateway/etcd_http.py index abc13031249b..abcfcf7af3cc 100644 --- a/dingo/video_gateway/etcd_http.py +++ b/dingo/video_gateway/etcd_http.py @@ -93,6 +93,7 @@ def __init__( url: str | Sequence[str], *, timeout_s: float = 5.0, + watch_response_timeout_s: float | None = None, telemetry: GatewayTelemetry | None = None, ) -> None: raw_urls = (url,) if isinstance(url, str) else tuple(url) @@ -110,6 +111,9 @@ def __init__( # request code uses the endpoint set below. self.url = self.urls[0] self.timeout = aiohttp.ClientTimeout(total=timeout_s) + if watch_response_timeout_s is not None and watch_response_timeout_s <= 0: + raise ValueError("watch response timeout must be positive") + self.watch_response_timeout_s = watch_response_timeout_s self.telemetry = telemetry self._session: aiohttp.ClientSession | None = None self._endpoint_index = 0 @@ -220,7 +224,7 @@ async def _stream_post( timeout = aiohttp.ClientTimeout( total=None, sock_connect=self.timeout.total, - sock_read=None, + sock_read=self.watch_response_timeout_s, ) endpoints = await self._endpoint_order() for position, (index, endpoint) in enumerate(endpoints): diff --git a/dingo/video_gateway/task_store.py b/dingo/video_gateway/task_store.py index c2c4fd93f426..79e9dd25755d 100644 --- a/dingo/video_gateway/task_store.py +++ b/dingo/video_gateway/task_store.py @@ -102,6 +102,26 @@ def task_watch_supported(self) -> bool: def gateway_owner_supported(self) -> bool: return False + @property + def discovery_truth_supported(self) -> bool: + return False + + async def discovery_instance_snapshot( + self, backend_targets: Iterable[str] + ) -> dict[str, set[int]]: + del backend_targets + raise NotImplementedError( + "this TaskStore cannot read Dynamo discovery instance truth" + ) + + async def try_acquire_discovery_recovery( + self, gateway_id: str, *, ttl_s: int + ) -> bool: + del gateway_id, ttl_s + raise NotImplementedError( + "this TaskStore does not support discovery recovery fencing" + ) + async def lease_snapshot( self, pool_id: str ) -> tuple[dict[str, WorkerLease], int]: @@ -693,6 +713,10 @@ def task_watch_supported(self) -> bool: def gateway_owner_supported(self) -> bool: return True + @property + def discovery_truth_supported(self) -> bool: + return True + def _task_key(self, task_id: str) -> str: return f"{self.root}/tasks/{task_id}" @@ -726,6 +750,12 @@ def _lease_heartbeat_key(self, pool_id: str, worker_key_value: str) -> str: def _gateway_key(self, gateway_id: str) -> str: return f"{self.root}/gateways/{gateway_id}" + def _gateway_prefix(self) -> str: + return f"{self.root}/gateways/" + + def _discovery_recovery_lock_key(self) -> str: + return self._meta_key("discovery-recovery-lock") + def _owner_task_key(self, gateway_id: str, task_id: str) -> str: return f"{self.root}/gateway-owners/{gateway_id}/tasks/{task_id}" @@ -1593,6 +1623,75 @@ async def keepalive_gateway(self, lease_id: int) -> None: async def unregister_gateway(self, lease_id: int) -> None: await self.client.lease_revoke(lease_id) + async def discovery_instance_snapshot( + self, backend_targets: Iterable[str] + ) -> dict[str, set[int]]: + snapshots: dict[str, set[int]] = {} + for target in dict.fromkeys(backend_targets): + endpoint_path = target.removeprefix("dyn://") + parts = endpoint_path.split(".") + if len(parts) != 3 or any(not part for part in parts): + raise ValueError(f"invalid Dynamo backend target: {target!r}") + namespace, component, endpoint = parts + prefix = f"v1/instances/{namespace}/{component}/{endpoint}/" + values, _revision = await self.client.range_all(prefix, prefix=True) + instances: set[int] = set() + for value in values: + try: + payload = json.loads(value.value) + if not isinstance(payload, Mapping): + raise TypeError("discovery value must be an object") + instance_id = int(payload["instance_id"]) + except (KeyError, TypeError, ValueError, json.JSONDecodeError) as exc: + raise RuntimeError( + f"invalid Dynamo discovery value at {value.key!r}" + ) from exc + if ( + payload.get("namespace") != namespace + or payload.get("component") != component + or payload.get("endpoint") != endpoint + ): + raise RuntimeError( + f"Dynamo discovery value does not match key {value.key!r}" + ) + instances.add(instance_id) + snapshots[target] = instances + return snapshots + + async def try_acquire_discovery_recovery( + self, gateway_id: str, *, ttl_s: int + ) -> bool: + if ttl_s <= 0: + raise ValueError("discovery recovery lock ttl must be positive") + gateways, _revision = await self.client.range_all( + self._gateway_prefix(), prefix=True, keys_only=True + ) + # A single live Gateway must stay available and surface the mismatch; + # only an HA deployment can repair replicas one at a time. + if len(gateways) < 2: + return False + native_lease = await self.client.lease_grant(ttl_s) + lock_key = self._discovery_recovery_lock_key() + succeeded, _revision = await self.client.txn( + [self.client.compare_version(lock_key, 0)], + [ + self.client.put( + lock_key, + self._encode( + { + "gateway_id": gateway_id, + "acquired_at_ms": now_ms(), + "lease_id": native_lease.lease_id, + } + ), + lease_id=native_lease.lease_id, + ) + ], + ) + if not succeeded: + await self.client.lease_revoke(native_lease.lease_id) + return succeeded + async def iter_orphaned_active_tasks(self) -> AsyncIterator[StoredTask]: prefix = self._owner_task_prefix() cursor = prefix.encode() diff --git a/dingo/video_gateway/telemetry.py b/dingo/video_gateway/telemetry.py index 2a8b6090b8d8..f8834c232a8e 100644 --- a/dingo/video_gateway/telemetry.py +++ b/dingo/video_gateway/telemetry.py @@ -103,6 +103,9 @@ class GatewayTelemetry: def __init__(self) -> None: self._lock = threading.Lock() self._counters: Counter[tuple[str, tuple[tuple[str, str], ...]]] = Counter() + self._gauges: dict[ + tuple[str, tuple[tuple[str, str], ...]], float + ] = {} self._histograms: dict[ tuple[str, tuple[tuple[str, str], ...]], _Histogram ] = {} @@ -119,6 +122,19 @@ def increment( with self._lock: self._counters[(name, _labels(labels))] += amount + def set_gauge( + self, + name: str, + value: float | int, + *, + labels: Mapping[str, str] | None = None, + ) -> None: + numeric = float(value) + if not math.isfinite(numeric): + raise ValueError("gauge values must be finite") + with self._lock: + self._gauges[(name, _labels(labels))] = numeric + def observe( self, name: str, @@ -244,6 +260,7 @@ def record_transition( def render_prometheus(self) -> list[str]: with self._lock: counters = list(self._counters.items()) + gauges = list(self._gauges.items()) histograms = [ (key, _Histogram( value.buckets, @@ -264,6 +281,15 @@ def render_prometheus(self) -> list[str]: f"{name}{_format_labels(labels)} {_format_number(value)}" ) + gauge_names = sorted({name for (name, _), _value in gauges}) + for name in gauge_names: + lines.append(f"# TYPE {name} gauge") + for (metric_name, labels), value in sorted(gauges): + if metric_name == name: + lines.append( + f"{name}{_format_labels(labels)} {_format_number(value)}" + ) + histogram_names = sorted({name for (name, _), _value in histograms}) for name in histogram_names: lines.append(f"# TYPE {name} histogram") diff --git a/tests/video_gateway/test_config.py b/tests/video_gateway/test_config.py index 4cd79cdcc990..fbf238a4859b 100644 --- a/tests/video_gateway/test_config.py +++ b/tests/video_gateway/test_config.py @@ -148,6 +148,51 @@ def test_unknown_discovery_backend_fails_config_validation(tmp_path): parse_config(raw) +def test_video_discovery_watchdog_is_opt_in_and_configurable(tmp_path): + raw = _raw(tmp_path) + assert parse_config(raw).runtime.discovery_watchdog.enabled is False + + raw["runtime"] = { + "discovery_backend": "etcd", + "discovery_watchdog": { + "enabled": True, + "interval_s": 1.5, + "mismatch_grace_s": 4.5, + }, + } + raw["task_store"] = { + "kind": "etcd_http", + "endpoints": ["http://etcd-0:2379", "http://etcd-1:2379"], + } + + watchdog = parse_config(raw).runtime.discovery_watchdog + assert watchdog.enabled is True + assert watchdog.interval_s == 1.5 + assert watchdog.mismatch_grace_s == 4.5 + + +def test_video_discovery_watchdog_rejects_non_etcd_state(tmp_path): + raw = _raw(tmp_path) + raw["runtime"] = {"discovery_watchdog": {"enabled": True}} + + with pytest.raises(ValueError, match="requires etcd"): + parse_config(raw) + + +def test_video_discovery_watchdog_rejects_too_short_grace(tmp_path): + raw = _raw(tmp_path) + raw["runtime"] = { + "discovery_watchdog": { + "enabled": True, + "interval_s": 2, + "mismatch_grace_s": 1, + } + } + + with pytest.raises(ValueError, match="at least interval_s"): + parse_config(raw) + + def test_vllm_omni_http_compatibility_options(tmp_path): raw = _raw(tmp_path) raw["http"] = { @@ -222,6 +267,33 @@ def test_lifecycle_and_artifact_watermarks_are_configurable(tmp_path): assert config.artifact_store.soft_min_free_bytes == 4096 +def test_task_store_watch_response_timeout_is_opt_in(tmp_path): + raw = _raw(tmp_path) + assert parse_config(raw).task_store.watch_response_timeout_s is None + + raw["task_store"] = { + "kind": "etcd_http", + "url": "http://etcd:2379", + "watch_response_timeout_s": 6, + } + config = parse_config(raw) + + assert config.task_store.watch_response_timeout_s == 6 + + +@pytest.mark.parametrize("value", [0, -1]) +def test_task_store_watch_response_timeout_must_be_positive(tmp_path, value): + raw = _raw(tmp_path) + raw["task_store"] = { + "kind": "etcd_http", + "url": "http://etcd:2379", + "watch_response_timeout_s": value, + } + + with pytest.raises(ValueError, match="watch_response_timeout_s"): + parse_config(raw) + + def test_artifact_soft_watermark_cannot_be_below_hard_watermark(tmp_path): raw = _raw(tmp_path) raw["artifact_store"].update( diff --git a/tests/video_gateway/test_discovery_watchdog.py b/tests/video_gateway/test_discovery_watchdog.py new file mode 100644 index 000000000000..87064745ea43 --- /dev/null +++ b/tests/video_gateway/test_discovery_watchdog.py @@ -0,0 +1,154 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +from dataclasses import replace + +import dingo.video_gateway.dispatcher as dispatcher_module +from dingo.video_gateway.artifact_store import FileArtifactStore +from dingo.video_gateway.config import DiscoveryWatchdogConfig +from dingo.video_gateway.dispatcher import VideoDispatcher +from dingo.video_gateway.task_store import MemoryTaskStore + + +class FakeEndpointClient: + def __init__(self, instance_ids: set[int]) -> None: + self.ids = instance_ids + + def instance_ids(self) -> list[int]: + return sorted(self.ids) + + async def direct(self, payload, instance_id, context): # pragma: no cover + del payload, instance_id, context + raise AssertionError("watchdog tests must not dispatch requests") + + +class DiscoveryMemoryTaskStore(MemoryTaskStore): + def __init__(self, truth: dict[str, set[int]], *, acquire: bool = True) -> None: + super().__init__() + self.truth = truth + self.acquire = acquire + self.lock_calls: list[tuple[str, int]] = [] + + @property + def discovery_truth_supported(self) -> bool: + return True + + async def discovery_instance_snapshot(self, backend_targets): + return { + target: set(self.truth.get(target, set())) + for target in backend_targets + } + + async def try_acquire_discovery_recovery(self, gateway_id, *, ttl_s): + self.lock_calls.append((gateway_id, ttl_s)) + return self.acquire + + +def _dispatcher(make_gateway_config, *, acquire: bool): + config = make_gateway_config() + config = replace( + config, + runtime=replace( + config.runtime, + discovery_watchdog=DiscoveryWatchdogConfig( + enabled=True, + interval_s=0.01, + mismatch_grace_s=0.02, + ), + ), + ) + pool = config.pools[0] + store = DiscoveryMemoryTaskStore( + {pool.backend_target: {8}}, acquire=acquire + ) + dispatcher = VideoDispatcher( + config, + store, + FileArtifactStore(config.artifact_store.root), + {pool.pool_id: FakeEndpointClient({7})}, + {pool.pool_id: object()}, + generation="watchdog-gateway", + ) + return dispatcher, store + + +async def test_watchdog_requests_fenced_restart_after_grace( + make_gateway_config, monkeypatch +): + monkeypatch.setattr(dispatcher_module, "_DISCOVERY_RESTART_DRAIN_S", 0) + dispatcher, store = _dispatcher(make_gateway_config, acquire=True) + watchdog = asyncio.create_task(dispatcher._discovery_watchdog_loop()) + try: + reason = await asyncio.wait_for(dispatcher.wait_fatal(), timeout=1) + assert "Dynamo discovery view remained inconsistent" in reason + assert dispatcher.draining is True + assert dispatcher.live is False + assert store.lock_calls == [("watchdog-gateway", 15)] + metrics = "\n".join(dispatcher.telemetry.render_prometheus()) + assert 'dingo_video_discovery_consistent{pool="fl-pool"} 0' in metrics + assert "dingo_video_discovery_watchdog_restarts_total 1" in metrics + finally: + dispatcher._stop.set() + watchdog.cancel() + await asyncio.gather(watchdog, return_exceptions=True) + + +async def test_watchdog_stays_live_during_drain_before_fatal_restart( + make_gateway_config, monkeypatch +): + monkeypatch.setattr(dispatcher_module, "_DISCOVERY_RESTART_DRAIN_S", 0.05) + dispatcher, _store = _dispatcher(make_gateway_config, acquire=True) + watchdog = asyncio.create_task(dispatcher._discovery_watchdog_loop()) + try: + for _ in range(100): + if dispatcher.draining: + break + await asyncio.sleep(0.001) + assert dispatcher.draining is True + assert dispatcher.ready is False + assert dispatcher.live is True + assert not dispatcher._fatal_event.is_set() + + await asyncio.wait_for(dispatcher.wait_fatal(), timeout=1) + assert dispatcher.live is False + finally: + dispatcher._stop.set() + watchdog.cancel() + await asyncio.gather(watchdog, return_exceptions=True) + + +async def test_watchdog_keeps_serving_when_recovery_lock_is_unavailable( + make_gateway_config, +): + dispatcher, store = _dispatcher(make_gateway_config, acquire=False) + watchdog = asyncio.create_task(dispatcher._discovery_watchdog_loop()) + try: + await asyncio.sleep(0.06) + assert not dispatcher._fatal_event.is_set() + assert dispatcher.draining is False + assert len(store.lock_calls) >= 1 + finally: + dispatcher._stop.set() + watchdog.cancel() + await asyncio.gather(watchdog, return_exceptions=True) + + +async def test_watchdog_clears_transient_mismatch_before_grace(make_gateway_config): + dispatcher, store = _dispatcher(make_gateway_config, acquire=True) + pool = dispatcher.config.pools[0] + watchdog = asyncio.create_task(dispatcher._discovery_watchdog_loop()) + try: + await asyncio.sleep(0.012) + store.truth[pool.backend_target] = {7} + await asyncio.sleep(0.04) + assert not dispatcher._fatal_event.is_set() + assert store.lock_calls == [] + metrics = "\n".join(dispatcher.telemetry.render_prometheus()) + assert 'dingo_video_discovery_consistent{pool="fl-pool"} 1' in metrics + finally: + dispatcher._stop.set() + watchdog.cancel() + await asyncio.gather(watchdog, return_exceptions=True) diff --git a/tests/video_gateway/test_etcd_http.py b/tests/video_gateway/test_etcd_http.py index fe1615659136..96d111ae18b5 100644 --- a/tests/video_gateway/test_etcd_http.py +++ b/tests/video_gateway/test_etcd_http.py @@ -3,9 +3,13 @@ from __future__ import annotations +import asyncio import base64 +import json import pytest +from aiohttp import web +from aiohttp.test_utils import TestServer from dingo.video_gateway.errors import StoreUnavailable from dingo.video_gateway.etcd_http import ( @@ -349,3 +353,52 @@ async def test_watch_prefix_reports_server_cancel(): with pytest.raises(StoreUnavailable, match="permission denied"): async for _response in client.watch_prefix("/workers/"): pass + + +async def test_watch_response_timeout_marks_silent_endpoint_failed(): + hold_open = asyncio.Event() + + async def stalled_watch(request): + await request.json() + response = web.StreamResponse( + status=200, headers={"Content-Type": "application/json"} + ) + await response.prepare(request) + await response.write( + json.dumps( + { + "result": { + "header": {"revision": "20"}, + "watch_id": "7", + "created": True, + } + } + ).encode() + + b"\n" + ) + await hold_open.wait() + return response + + app = web.Application() + app.router.add_post("/v3/watch", stalled_watch) + server = TestServer(app) + await server.start_server() + client = EtcdHttpClient( + [str(server.make_url("")).rstrip("/"), "http://next-etcd"], + timeout_s=1, + watch_response_timeout_s=0.05, + ) + watch = client.watch_prefix("/workers/") + try: + created = await asyncio.wait_for(anext(watch), timeout=1) + assert created.created is True + + with pytest.raises(StoreUnavailable, match="stream failed"): + await asyncio.wait_for(anext(watch), timeout=1) + + assert (await client._endpoint_order())[0][1] == "http://next-etcd" + finally: + await watch.aclose() + await client.close() + hold_open.set() + await server.close() diff --git a/tests/video_gateway/test_etcd_task_store.py b/tests/video_gateway/test_etcd_task_store.py index c75f87578d7e..446e8306db17 100644 --- a/tests/video_gateway/test_etcd_task_store.py +++ b/tests/video_gateway/test_etcd_task_store.py @@ -755,3 +755,58 @@ async def test_two_gateway_owners_cannot_reserve_the_same_task(): left.owner_generation, right.owner_generation, } + + +async def test_discovery_snapshot_uses_full_backend_target_not_k8s_namespace(): + client = FakeEtcd() + store = EtcdTaskStore( + client, prefix="/isolated/video", deployment_id="discovery" + ) + client.revision = 11 + key = "v1/instances/arbitrary-scope/backend/generate/000000000000002a" + client.values[key] = EtcdValue( + key, + json.dumps( + { + "type": "Endpoint", + "namespace": "arbitrary-scope", + "component": "backend", + "endpoint": "generate", + "instance_id": 42, + } + ).encode(), + 11, + 11, + 1, + ) + + snapshot = await store.discovery_instance_snapshot( + ["dyn://arbitrary-scope.backend.generate", "other.backend.generate"] + ) + + assert snapshot == { + "dyn://arbitrary-scope.backend.generate": {42}, + "other.backend.generate": set(), + } + + +async def test_discovery_recovery_lock_requires_ha_and_has_single_winner(): + client = FakeEtcd() + store = EtcdTaskStore( + client, prefix="/isolated/video", deployment_id="discovery-lock" + ) + await store.register_gateway("gateway-a", ttl_s=15) + assert not await store.try_acquire_discovery_recovery( + "gateway-a", ttl_s=15 + ) + + await store.register_gateway("gateway-b", ttl_s=15) + results = await asyncio.gather( + store.try_acquire_discovery_recovery("gateway-a", ttl_s=15), + store.try_acquire_discovery_recovery("gateway-b", ttl_s=15), + ) + + assert sorted(results) == [False, True] + lock = await client.get(store._discovery_recovery_lock_key()) + assert lock is not None + assert json.loads(lock.value)["gateway_id"] in {"gateway-a", "gateway-b"} diff --git a/tests/video_gateway/test_main.py b/tests/video_gateway/test_main.py index 78d46bbd577b..ff79dff17ffa 100644 --- a/tests/video_gateway/test_main.py +++ b/tests/video_gateway/test_main.py @@ -21,6 +21,16 @@ async def wait_shutdown(self) -> None: raise +class FutureRuntime: + """Match the asyncio Future returned by PyO3 future_into_py.""" + + def __init__(self) -> None: + self.stopped = asyncio.get_running_loop().create_future() + + def wait_shutdown(self) -> asyncio.Future[None]: + return self.stopped + + async def test_wait_for_shutdown_detects_runtime_termination(): runtime = FakeRuntime() stopped = asyncio.Event() @@ -51,3 +61,26 @@ async def test_wait_for_shutdown_prefers_normal_signal_when_both_are_set(): stopped.set() assert await _wait_for_shutdown(runtime, stopped) is False + + +async def test_wait_for_shutdown_accepts_pyo3_future(): + runtime = FutureRuntime() + stopped = asyncio.Event() + waiting = asyncio.create_task(_wait_for_shutdown(runtime, stopped)) + await asyncio.sleep(0) + + runtime.stopped.set_result(None) + + assert await asyncio.wait_for(waiting, timeout=1) is True + + +async def test_wait_for_shutdown_cancels_pyo3_future_after_signal(): + runtime = FutureRuntime() + stopped = asyncio.Event() + waiting = asyncio.create_task(_wait_for_shutdown(runtime, stopped)) + await asyncio.sleep(0) + + stopped.set() + + assert await asyncio.wait_for(waiting, timeout=1) is False + assert runtime.stopped.cancelled() diff --git a/tests/video_gateway/test_telemetry.py b/tests/video_gateway/test_telemetry.py index 1ae840d5cc5e..db07f4638673 100644 --- a/tests/video_gateway/test_telemetry.py +++ b/tests/video_gateway/test_telemetry.py @@ -20,6 +20,9 @@ def test_telemetry_renders_bounded_task_and_etcd_metrics(caplog): failed.error = terminal_error("worker_lease_lost", "lease lost") telemetry.record_submission(task.pool_id, "created", "async") + telemetry.set_gauge( + "dingo_video_discovery_consistent", 1, labels={"pool": task.pool_id} + ) telemetry.record_stage_duration(task.pool_id, "queue", 0.125) telemetry.record_etcd_request("kv/range", 0.01, succeeded=False) with caplog.at_level(logging.INFO, logger="dingo.video_gateway.audit"): @@ -43,6 +46,10 @@ def test_telemetry_renders_bounded_task_and_etcd_metrics(caplog): assert 'stage="queue"' in text assert 'dingo_video_etcd_request_errors_total{operation="kv/range"} 1' in text assert 'error_code="worker_lease_lost"' in text + assert ( + 'dingo_video_discovery_consistent{pool="pool-a"} 1' + in text + ) event = json.loads(caplog.records[-1].message) assert event["log_type"] == "video_task_lifecycle" From 6954663932c4d19bb44bb218424e4dbf36fdbb9f Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Fri, 4 Sep 2026 11:47:51 +0800 Subject: [PATCH 02/12] [fix][omni] restart detached workers after Runtime termination Add a process-level Runtime termination guard for detached vLLM-Omni video workers. Unexpected permanent Runtime shutdown cancels the component cleanly and exits for supervisor restart, while normal signal-driven shutdown retains graceful drain behavior. The guard is enabled only when detached_video_task_root is configured; ordinary Omni and non-video worker startup paths are unchanged. --- dingo/common/utils/runtime_termination.py | 65 +++++++++++++ dingo/vllm/omni/main.py | 14 ++- .../video_gateway/test_runtime_termination.py | 92 +++++++++++++++++++ 3 files changed, 170 insertions(+), 1 deletion(-) create mode 100644 dingo/common/utils/runtime_termination.py create mode 100644 tests/video_gateway/test_runtime_termination.py diff --git a/dingo/common/utils/runtime_termination.py b/dingo/common/utils/runtime_termination.py new file mode 100644 index 000000000000..5595202c2995 --- /dev/null +++ b/dingo/common/utils/runtime_termination.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Process-level guard for a permanently terminated Dynamo Runtime.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable +from typing import Any, TypeVar + + +_T = TypeVar("_T") + + +async def run_with_runtime_termination_guard( + operation: Awaitable[_T], + runtime: Any, + shutdown_event: asyncio.Event, + *, + component: str, +) -> _T: + """Run ``operation`` until it completes or the Runtime terminates. + + A Dynamo Runtime whose cancellation token has fired cannot rebuild its + discovery clients in place. An unexpected ``wait_shutdown`` completion + therefore cancels the component coroutine so its normal cleanup runs, then + raises and lets the process supervisor create a fresh Runtime. + + ``shutdown_event`` distinguishes this condition from an intentional signal + shutdown. Signal handling sets that event before calling + ``runtime.shutdown()``, so the component retains its ordinary graceful + drain behavior in that path. + """ + + operation_task = asyncio.ensure_future(operation) + runtime_task = asyncio.ensure_future(runtime.wait_shutdown()) + try: + done, _pending = await asyncio.wait( + {operation_task, runtime_task}, return_when=asyncio.FIRST_COMPLETED + ) + if runtime_task in done and not shutdown_event.is_set(): + runtime_error: BaseException | None = None + if not runtime_task.cancelled(): + runtime_error = runtime_task.exception() + if not operation_task.done(): + operation_task.cancel() + await asyncio.gather(operation_task, return_exceptions=True) + error = RuntimeError( + f"{component} Dynamo Runtime terminated unexpectedly; " + "the process must restart" + ) + if runtime_error is not None: + raise error from runtime_error + raise error + return await operation_task + except BaseException: + if not operation_task.done(): + operation_task.cancel() + await asyncio.gather(operation_task, return_exceptions=True) + raise + finally: + if not runtime_task.done(): + runtime_task.cancel() + await asyncio.gather(runtime_task, return_exceptions=True) diff --git a/dingo/vllm/omni/main.py b/dingo/vllm/omni/main.py index 8ec2e279ba11..edb7f72e8ca6 100644 --- a/dingo/vllm/omni/main.py +++ b/dingo/vllm/omni/main.py @@ -15,6 +15,9 @@ from dingo.common.utils.graceful_shutdown import install_signal_handlers from dingo.common.utils.output_modalities import get_output_modalities from dingo.common.utils.runtime import create_runtime +from dingo.common.utils.runtime_termination import ( + run_with_runtime_termination_guard, +) from dynamo.llm import ModelInput, ModelType, WorkerType, fetch_model, register_model from dynamo.runtime import DistributedRuntime from dynamo.runtime.logging import configure_dynamo_logging @@ -167,7 +170,16 @@ async def worker(): await init_omni_realtime(runtime, config, shutdown_endpoints, shutdown_event) logger.debug("init_omni_realtime completed, exiting...") else: - await init_omni(runtime, config, shutdown_event) + omni = init_omni(runtime, config, shutdown_event) + if config.detached_video_task_root is not None: + await run_with_runtime_termination_guard( + omni, + runtime, + shutdown_event, + component="detached vLLM-Omni Video Worker", + ) + else: + await omni logger.debug("Omni worker completed, exiting...") diff --git a/tests/video_gateway/test_runtime_termination.py b/tests/video_gateway/test_runtime_termination.py new file mode 100644 index 000000000000..ae7158f89f00 --- /dev/null +++ b/tests/video_gateway/test_runtime_termination.py @@ -0,0 +1,92 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import asyncio +import importlib.util +from pathlib import Path + +import pytest + + +_SOURCE = ( + Path(__file__).parents[2] + / "dingo" + / "common" + / "utils" + / "runtime_termination.py" +) +_SPEC = importlib.util.spec_from_file_location("runtime_termination", _SOURCE) +assert _SPEC is not None and _SPEC.loader is not None +_MODULE = importlib.util.module_from_spec(_SPEC) +_SPEC.loader.exec_module(_MODULE) +run_with_runtime_termination_guard = _MODULE.run_with_runtime_termination_guard + + +class _Runtime: + def __init__(self) -> None: + loop = asyncio.get_running_loop() + self.terminated: asyncio.Future[None] = loop.create_future() + + def wait_shutdown(self) -> asyncio.Future[None]: + return self.terminated + + +async def test_unexpected_runtime_termination_cancels_component() -> None: + runtime = _Runtime() + shutdown_event = asyncio.Event() + component_stopped = asyncio.Event() + + async def component() -> None: + try: + await asyncio.Event().wait() + finally: + component_stopped.set() + + guarded = asyncio.create_task( + run_with_runtime_termination_guard( + component(), runtime, shutdown_event, component="detached Video Worker" + ) + ) + await asyncio.sleep(0) + runtime.terminated.set_result(None) + + with pytest.raises(RuntimeError, match="process must restart"): + await guarded + assert component_stopped.is_set() + + +async def test_signal_shutdown_keeps_graceful_component_drain() -> None: + runtime = _Runtime() + shutdown_event = asyncio.Event() + component_release = asyncio.Event() + + async def component() -> str: + await component_release.wait() + return "drained" + + guarded = asyncio.create_task( + run_with_runtime_termination_guard( + component(), runtime, shutdown_event, component="detached Video Worker" + ) + ) + await asyncio.sleep(0) + shutdown_event.set() + runtime.terminated.set_result(None) + await asyncio.sleep(0) + assert not guarded.done() + component_release.set() + assert await guarded == "drained" + + +async def test_component_completion_cancels_only_runtime_waiter() -> None: + runtime = _Runtime() + result = await run_with_runtime_termination_guard( + asyncio.sleep(0, result=7), + runtime, + asyncio.Event(), + component="detached Video Worker", + ) + assert result == 7 + assert runtime.terminated.cancelled() From ce0a80268ef9a59e14ee6339f59dc3b78d2a2271 Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Tue, 8 Sep 2026 18:52:03 +0800 Subject: [PATCH 03/12] [fix][video-gateway] harden admission and retry lost workers --- dingo/video_gateway/adapters/minimax_h3.py | 4 +- dingo/video_gateway/dispatcher.py | 276 +++++++++++++++++--- dingo/video_gateway/errors.py | 29 ++ dingo/video_gateway/task_store.py | 233 ++++++++++++++++- tests/video_gateway/test_worker_liveness.py | 190 ++++++++++++++ 5 files changed, 694 insertions(+), 38 deletions(-) create mode 100644 tests/video_gateway/test_worker_liveness.py diff --git a/dingo/video_gateway/adapters/minimax_h3.py b/dingo/video_gateway/adapters/minimax_h3.py index 37323e678ef8..13e4af44da7a 100644 --- a/dingo/video_gateway/adapters/minimax_h3.py +++ b/dingo/video_gateway/adapters/minimax_h3.py @@ -24,7 +24,7 @@ resolve_output_shape, ) from dingo.video_gateway.config import PoolConfig -from dingo.video_gateway.errors import GatewayError +from dingo.video_gateway.errors import GatewayError, worker_execution_error _IMAGE_TYPE_ALIASES = { "image/jpeg": "image/jpeg", @@ -584,7 +584,7 @@ def finish(self) -> WorkerVideoResult: # while the Gateway validates and publishes the decoded artifact. self._terminal = None if terminal.get("status") == "failed": - raise RuntimeError(str(terminal.get("error") or "Worker generation failed")) + raise worker_execution_error(terminal.get("error")) data = terminal.get("data") if ( terminal.get("status") != "completed" diff --git a/dingo/video_gateway/dispatcher.py b/dingo/video_gateway/dispatcher.py index 6fb87fe11e76..25b34e3b8a21 100644 --- a/dingo/video_gateway/dispatcher.py +++ b/dingo/video_gateway/dispatcher.py @@ -8,6 +8,7 @@ import asyncio import hashlib import logging +import os import secrets import time import uuid @@ -23,7 +24,7 @@ EndpointClient, create_context, ) -from dingo.video_gateway.errors import ResultTooLarge, StoreConflict +from dingo.video_gateway.errors import ResultTooLarge, StoreConflict, WorkerUnavailable, worker_execution_error from dingo.video_gateway.memory_budget import ( MemoryBudgetSnapshot, WeightedMemoryBudget, @@ -51,6 +52,8 @@ _DETACHED_WAIT_RETRY_INITIAL_S = 0.2 _DETACHED_WAIT_RETRY_MAX_S = 5.0 _DETACHED_WORKER_STALE_S = 20.0 +_WORKER_LIVENESS_CHECK_INTERVAL_S = 5.0 +_WORKER_LIVENESS_CHECK_CONCURRENCY = 4 _DISCOVERY_MISMATCH_MIN_CHECKS = 3 _DISCOVERY_RECOVERY_LOCK_TTL_S = 15 _DISCOVERY_RESTART_DRAIN_S = 5.0 @@ -60,6 +63,9 @@ class _DetachedWorkerCancelled(RuntimeError): pass +_RetryableWorkerFailure = WorkerUnavailable + + class _CancellationConfirmationTimedOut(RuntimeError): pass @@ -145,6 +151,14 @@ def __init__( self.artifacts = artifacts self.context_factory = context_factory self.generation = generation or uuid.uuid4().hex + self._worker_retry_once = os.getenv("DINGO_VIDEO_WORKER_RETRY_ONCE", "0") == "1" + self._retry_budget_limit = int(os.getenv("DINGO_VIDEO_RETRY_BUDGET", "32")) + self._retry_wait_timeout_s = float(os.getenv("DINGO_VIDEO_RETRY_WAIT_TIMEOUT_S", "600")) + self._failed_instance_backoff_s = float(os.getenv("DINGO_VIDEO_RETRY_FAILED_INSTANCE_BACKOFF_S", "30")) + if not 1 <= self._retry_budget_limit <= 1024 or not 0 < self._retry_wait_timeout_s <= 86400: + raise ValueError("invalid retry budget or wait timeout") + if not 0 <= self._failed_instance_backoff_s <= 86400: + raise ValueError("invalid failed instance backoff") self.telemetry = telemetry or GatewayTelemetry() self.pools: dict[str, PoolRuntime] = { pool.pool_id: PoolRuntime( @@ -191,6 +205,9 @@ def __init__( self._fatal_error: str | None = None self._fatal_event = asyncio.Event() self._orphan_recovery_lock = asyncio.Lock() + self._worker_liveness_checks = asyncio.Semaphore( + _WORKER_LIVENESS_CHECK_CONCURRENCY + ) self._discovery_mismatch_started: dict[str, float] = {} self._discovery_mismatch_checks: dict[str, int] = {} @@ -817,28 +834,50 @@ async def _release_reusable_leases(self, pool: PoolRuntime) -> None: await self.store.release_lease(pool.config.pool_id, lease.worker_key) async def _dispatch_once(self, pool: PoolRuntime) -> bool: - queued = await self.store.list_queued(pool.config.pool_id, limit=1) + ledger = hasattr(self.store, "retry_budget_used") + budget_limit = getattr(self, "_retry_budget_limit", 32) + queued = await self.store.list_queued(pool.config.pool_id, limit=min(10000, pool.config.scheduling.queue_limit + budget_limit)) + used = 0 + if ledger: + waiting = await self.store.retry_queue_depth(pool.config.pool_id) + used = await self.store.retry_budget_used(pool.config.pool_id) + self.telemetry.set_gauge("dingo_video_retry_waiting_tasks", waiting, labels={"pool": pool.config.pool_id}) + self.telemetry.set_gauge("dingo_video_retry_credits_used", used, labels={"pool": pool.config.pool_id}) + self.telemetry.set_gauge("dingo_video_normal_queue_depth", max(0, await self.store.queue_depth(pool.config.pool_id) - waiting), labels={"pool": pool.config.pool_id}) if not queued or not pool.instance_ids: await self._clear_budget_waiter(pool) return False - task_id = queued[0].task.id - if pool.budget_waiter_id not in {None, task_id}: - await self.memory_budget.cancel_waiter(pool.budget_waiter_id) - pool.budget_waiter_id = None if self.store.lease_watch_supported and not pool.lease_watch_healthy: await self._clear_budget_waiter(pool) return False leased = { lease.worker_key for lease in await self.pool_leases(pool.config.pool_id) } - available = [ - instance_id - for instance_id in pool.instance_ids - if worker_key(pool.config.backend_target, instance_id) not in leased - ] - if not available: + selected = None + available = [] + # Retry first, but do not let a retry that excludes the only available + # instance block unrelated runnable work behind it. + for candidate in sorted(queued, key=lambda item: item.task.attempt == 0): + if candidate.task.id in self.running_calls or candidate.task.expires_at_ms <= now_ms(): + continue + if (ledger and getattr(self, "_worker_retry_once", False) + and candidate.task.attempt == 0 and used >= budget_limit): + continue + available = [instance for instance in pool.instance_ids + if worker_key(pool.config.backend_target, instance) not in leased + and not (candidate.task.attempt > 0 + and worker_key(pool.config.backend_target, instance) == candidate.task.worker_key)] + if available: + selected = candidate + break + if selected is None: await self._clear_budget_waiter(pool) return False + queued = [selected] + task_id = selected.task.id + if pool.budget_waiter_id not in {None, task_id}: + await self.memory_budget.cancel_waiter(pool.budget_waiter_id) + pool.budget_waiter_id = None weight_bytes = ( queued[0].task.estimated_payload_bytes or self.config.media.max_task_memory_bytes @@ -868,8 +907,11 @@ async def _dispatch_once(self, pool: PoolRuntime) -> bool: ) deadline = now_ms() + int(pool.config.scheduling.execution_timeout_s * 1000) try: + retry_options = ({"reserve_retry": getattr(self, "_worker_retry_once", False) + and pool.config.execution_mode == "detached", "retry_limit": budget_limit} + if ledger else {}) reserved = await self.store.reserve( - queued[0], lease, deadline_at_ms=deadline + queued[0], lease, deadline_at_ms=deadline, **retry_options ) except Exception: await self.memory_budget.release(task_id) @@ -932,15 +974,21 @@ async def _run_with_lease_monitor( operation: Any, heartbeat: asyncio.Task, cancellation: Any | None = None, + liveness: Any | None = None, ) -> Any: operation_task = asyncio.create_task(operation) cancellation_task = ( asyncio.create_task(cancellation) if cancellation is not None else None ) + liveness_task = ( + asyncio.create_task(liveness) if liveness is not None else None + ) try: monitored = {operation_task, heartbeat} if cancellation_task is not None: monitored.add(cancellation_task) + if liveness_task is not None: + monitored.add(liveness_task) done, _pending = await asyncio.wait( monitored, return_when=asyncio.FIRST_COMPLETED, @@ -952,20 +1000,99 @@ async def _run_with_lease_monitor( if cancellation_task is not None and cancellation_task in done: await cancellation_task raise RuntimeError("cancellation monitor ended unexpectedly") - if heartbeat.cancelled(): - raise _WorkerLeaseLost("Worker execution lease monitor stopped") - error = heartbeat.exception() - if isinstance(error, _WorkerLeaseLost): - raise error - raise _WorkerLeaseLost("Worker execution lease monitor failed") from error + # Losing execution ownership takes precedence over a simultaneous + # suspicion of Worker loss. An etcd outage is not a Worker crash. + if heartbeat in done: + self._raise_if_heartbeat_stopped(heartbeat) + if liveness_task is not None and liveness_task in done: + await liveness_task + raise RuntimeError("Worker liveness monitor ended unexpectedly") + raise RuntimeError("execution monitor ended unexpectedly") finally: - if not operation_task.done(): - operation_task.cancel() - await asyncio.gather(operation_task, return_exceptions=True) - if cancellation_task is not None: - if not cancellation_task.done(): - cancellation_task.cancel() - await asyncio.gather(cancellation_task, return_exceptions=True) + children = [operation_task] + children.extend( + monitor for monitor in (cancellation_task, liveness_task) + if monitor is not None + ) + for child in children: + if not child.done(): + child.cancel() + await asyncio.gather(*children, return_exceptions=True) + + def _can_check_missing_worker(self, pool: PoolRuntime, task: Any) -> bool: + return ( + self.store.discovery_truth_supported + and self._gateway_owner_healthy + and self._task_watch_healthy + and pool.discovery_healthy + and pool.lease_watch_healthy + and task.worker_instance_id not in pool.instance_ids + and (task.deadline_at_ms or 0) > now_ms() + ) + + async def _confirm_worker_loss(self, pool: PoolRuntime, task: Any) -> bool: + """Read remote evidence only for a locally missing execution instance. + + Unknown/corrupt/unavailable evidence is not positive proof of failure. + The normal registered-Worker path never reads etcd or artifact files. + """ + if not self._can_check_missing_worker(pool, task): + return False + async with self._worker_liveness_checks: + # Waiting for the shared concurrency limit may have made the + # suspicion obsolete, or the control plane may have become sick. + if not self._can_check_missing_worker(pool, task): + return False + try: + truth = await self.store.discovery_instance_snapshot( + [pool.config.backend_target] + ) + if task.worker_instance_id in truth[pool.config.backend_target]: + return False + current = await self._current_owned_execution(task) + if current is None: + raise _TaskOwnershipLost("task ownership moved during Worker liveness check") + if (current.task.status != TaskStatus.IN_PROGRESS + or current.task.cancel_requested_at_ms is not None): + return False + status = await self.artifacts.read_detached_status( + task.deployment_id, task.pool_id, task.id, task.attempt, + task.execution_token, + ) + if status is None or status.get("state") not in {"accepted", "running"}: + return False + updated_at_ms = status.get("updated_at_ms") + if (not isinstance(updated_at_ms, int) + or isinstance(updated_at_ms, bool) + or now_ms() - updated_at_ms <= int(_DETACHED_WORKER_STALE_S * 1000)): + return False + return self._can_check_missing_worker(pool, task) + except _TaskOwnershipLost: + raise + except asyncio.CancelledError: + raise + except Exception: + self.telemetry.increment( + "dingo_video_worker_liveness_checks_total", + labels={"pool": pool.config.pool_id, "outcome": "inconclusive"}, + ) + logger.debug("Worker liveness evidence unavailable for %s", task.id, exc_info=True) + return False + + async def _monitor_worker_liveness( + self, pool: PoolRuntime, task: Any, running: RunningCall + ) -> None: + """Independent of the possibly stuck result/terminal-wait stream.""" + while True: + await asyncio.sleep(_WORKER_LIVENESS_CHECK_INTERVAL_S) + if running.worker_accepted and await self._confirm_worker_loss(pool, task): + self.telemetry.increment( + "dingo_video_worker_liveness_checks_total", + labels={"pool": pool.config.pool_id, "outcome": "confirmed_lost"}, + ) + raise _RetryableWorkerFailure( + "detached Worker disappeared and its heartbeat is stale" + ) async def _monitor_cancellation( self, @@ -1189,6 +1316,7 @@ async def _consume_worker_stream() -> None: self._monitor_cancellation( pool, stored.task, context, running_call ), + self._monitor_worker_liveness(pool, stored.task, running_call), ) worker_stream_finished = True else: @@ -1367,6 +1495,7 @@ async def _consume_worker_stream() -> None: "gateway_shutdown", "Gateway stopped during generation", quarantine=True, + expected_execution=task, ) raise except asyncio.TimeoutError: @@ -1382,6 +1511,7 @@ async def _consume_worker_stream() -> None: "execution_timeout", "video generation timed out", quarantine=True, + expected_execution=task, ) except _WorkerLeaseLost as exc: if await self._current_owned_execution(task) is None: @@ -1405,6 +1535,7 @@ async def _consume_worker_stream() -> None: "worker_lease_lost", str(exc), quarantine=worker_accepted, + expected_execution=task, ) except _DetachedWorkerCancelled: latest = await self.store.get_task(task.id) @@ -1436,12 +1567,14 @@ async def _consume_worker_stream() -> None: "result_too_large", str(exc), quarantine=False, + expected_execution=task, ) except Exception as exc: if final_path is not None: await asyncio.to_thread(final_path.unlink, True) final_path = None - if await self._current_owned_execution(task) is None: + current = await self._current_owned_execution(task) + if current is None: logger.info( "ignored stale task failure after execution ownership moved: %s", task.id, @@ -1464,7 +1597,25 @@ async def _consume_worker_stream() -> None: context.stop_generating() except Exception: logger.exception("failed to stop Worker after task error") - logger.exception("video task failed: %s", task.id) + if ( + isinstance(exc, StoreConflict) + and running_call is None + and task.status == TaskStatus.DISPATCHING + and current.task.status == TaskStatus.DISPATCHING + and current.task.cancel_requested_at_ms is not None + ): + # A cancellation can advance the reserved task revision before + # dispatch. Keep the existing cancellation/lease cleanup below; + # this confirmed pre-dispatch race is not a Worker failure. + logger.info( + "task %s cancelled during dispatch preparation; " + "handling expected task revision conflict", + task.id, + ) + elif isinstance(exc, _RetryableWorkerFailure): + logger.exception("video Worker attempt failed: %s", task.id) + else: + logger.exception("video task failed: %s", task.id) await self._finish_failed( pool, task.id, @@ -1475,6 +1626,8 @@ async def _consume_worker_stream() -> None: and running_call.worker_accepted and not worker_stream_finished ), + retryable=isinstance(exc, _RetryableWorkerFailure), + expected_execution=task, ) finally: payload = None @@ -1597,7 +1750,7 @@ async def _consume_status(value: dict[str, Any]) -> bool: return True if state == "failed": running_call.worker_accepted = False - raise RuntimeError("detached Worker reported execution failure") + raise worker_execution_error(worker_status.get("error")) if state == "cancelled": running_call.worker_accepted = False raise _DetachedWorkerCancelled("detached Worker cancelled task") @@ -1609,11 +1762,11 @@ async def _consume_status(value: dict[str, Any]) -> bool: if ( state in {"accepted", "running"} and isinstance(updated_at_ms, int) - and task.worker_instance_id not in pool.instance_ids and now_ms() - updated_at_ms > int(_DETACHED_WORKER_STALE_S * 1000) + and await self._confirm_worker_loss(pool, task) ): - raise RuntimeError( + raise _RetryableWorkerFailure( "detached Worker disappeared and its heartbeat is stale" ) return False @@ -1912,6 +2065,45 @@ async def _finish_cancelled( revision=cancelled.revision, ) + async def _try_worker_retry(self, pool: PoolRuntime, stored: StoredTask, *, quarantine: bool) -> bool: + task = stored.task + if (not getattr(self, "_worker_retry_once", False) + or pool.config.execution_mode != "detached" + or task.attempt != 1 + or task.status not in {TaskStatus.DISPATCHING, TaskStatus.IN_PROGRESS} + or task.cancel_requested_at_ms is not None + or task.owner_generation != self.generation + or not hasattr(self.store, "retry_budget_used")): + return False + retry = None + for _ in range(16): + try: + retry = await self.store.requeue_failed_attempt( + stored, queue_limit=pool.config.scheduling.queue_limit, + retry_wait_timeout_s=getattr(self, "_retry_wait_timeout_s", 600), + quarantine_until_ms=(max(task.deadline_at_ms or now_ms(), now_ms()) + + int(pool.config.scheduling.abort_grace_s * 1000)) if quarantine + else now_ms() + int(getattr(self, "_failed_instance_backoff_s", 30) * 1000), + ) + break + except StoreConflict: + current = await self.store.get_task(task.id) + if (current is None or current.task.status not in {TaskStatus.DISPATCHING, TaskStatus.IN_PROGRESS} + or not self._same_execution_owner(current.task, task) + or current.task.cancel_requested_at_ms is not None): + return False + stored = current + await asyncio.sleep(0.01) + if retry is None: + return False + self.telemetry.increment("dingo_video_worker_retries_total", labels={"pool": task.pool_id}) + self.telemetry.record_transition("worker_retry_queued", task, retry.task, + gateway_generation=self.generation, revision=retry.revision) + logger.warning("queued one Worker retry for task %s after attempt %s; excluding instance %s", + task.id, task.attempt, task.worker_instance_id) + pool.wakeup.set() + return True + async def _finish_failed( self, pool: PoolRuntime, @@ -1920,10 +2112,27 @@ async def _finish_failed( message: str, *, quarantine: bool, + retryable: bool = False, + expected_execution: Any | None = None, ) -> None: latest = await self.store.get_task(task_id) if latest is None or latest.task.status in TERMINAL_STATUSES: return + if expected_execution is not None and not self._same_execution_owner(latest.task, expected_execution): + return + if retryable and latest.task.cancel_requested_at_ms is None: + previous = latest + try: + if await self._try_worker_retry(pool, latest, quarantine=quarantine): + return + except Exception: + # A transaction reply may be ambiguous. Re-read ownership + # before falling back; never fail an already requeued attempt. + logger.exception("Worker retry decision failed for task %s", task_id) + latest = await self.store.get_task(task_id) + if (latest is None or latest.task.status in TERMINAL_STATUSES + or not self._same_execution_owner(latest.task, previous.task)): + return if latest.task.cancel_requested_at_ms is not None: try: await self._finish_cancelled(pool, latest, quarantine=quarantine) @@ -2310,7 +2519,8 @@ async def _sweep_once(self) -> None: "expires_at_ms": current + int(self.config.lifecycle.failed_ttl_s * 1000), "error": terminal_error( - "queue_timeout", "video task expired while queued" + "retry_wait_timeout" if task.attempt > 0 else "queue_timeout", + "video retry wait expired" if task.attempt > 0 else "video task expired while queued" ), }, ) @@ -2320,7 +2530,7 @@ async def _sweep_once(self) -> None: failed.task, gateway_generation=self.generation, revision=failed.revision, - reason="queue_timeout", + reason="retry_wait_timeout" if task.attempt > 0 else "queue_timeout", ) elif task.status in { TaskStatus.COMPLETED, diff --git a/dingo/video_gateway/errors.py b/dingo/video_gateway/errors.py index 4fe4e0046fbe..bf7a2ba3a21a 100644 --- a/dingo/video_gateway/errors.py +++ b/dingo/video_gateway/errors.py @@ -44,3 +44,32 @@ class StoreUnavailable(RuntimeError): class ResultTooLarge(RuntimeError): """Raised when a Worker result exceeds the configured artifact policy.""" + + +class WorkerExecutionFailed(RuntimeError): + """A Worker reported failure; not automatically retryable.""" + + +class WorkerUnavailable(WorkerExecutionFailed): + """Positive evidence of a lost Worker or unavailable execution engine.""" + + +def worker_execution_error(error: object) -> WorkerExecutionFailed: + """Conservative compatibility classifier for this deployed Worker protocol. + + Worker errors currently have no infrastructure-specific structured code. + Match only complete, known engine-unavailable messages, never substrings, + arbitrary RuntimeError, or the generic worker_failed code by itself. + Unknown, parameter, media and model errors are deliberately not retried. + """ + message = error + if isinstance(error, dict): + if error.get("code") != "worker_failed": + return WorkerExecutionFailed(str(error.get("message") or "Worker generation failed")) + message = error.get("message") + if isinstance(message, str) and message in { + "Executor shut down", + "Stage-0 has no live replica", + }: + return WorkerUnavailable(message) + return WorkerExecutionFailed(str(message or "Worker generation failed")) diff --git a/dingo/video_gateway/task_store.py b/dingo/video_gateway/task_store.py index 79e9dd25755d..466aa15ede4c 100644 --- a/dingo/video_gateway/task_store.py +++ b/dingo/video_gateway/task_store.py @@ -9,6 +9,7 @@ import copy import hashlib import json +import random from abc import ABC, abstractmethod from collections.abc import AsyncIterator, Iterable, Mapping from dataclasses import dataclass @@ -700,6 +701,9 @@ def __init__( self.client = client self.root = f"{prefix.rstrip('/')}/deployments/{deployment_id}" self.execution_lease_ttl_s = execution_lease_ttl_s + # All pools share the sequence counter; serialize only admission here. + # Other Gateways still synchronize through the existing etcd transaction. + self._create_task_lock = asyncio.Lock() @property def lease_watch_supported(self) -> bool: @@ -738,6 +742,37 @@ def _ordered_queue_prefix(self, pool_id: str) -> str: def _counter_key(self, pool_id: str) -> str: return f"{self.root}/pools/{pool_id}/counters/queued" + def _retry_counter_key(self, pool_id: str, kind: str) -> str: + return f"{self.root}/pools/{pool_id}/counters/retry-{kind}" + + def _retry_credit_key(self, task: VideoTask) -> str: + return f"{self.root}/pools/{task.pool_id}/retry-credits/{task.id}" + + @staticmethod + def _is_retry_waiting(task: VideoTask) -> bool: + return task.status == TaskStatus.QUEUED and task.attempt == 1 and task.worker_key is not None + + async def _retry_counter(self, pool_id: str, kind: str) -> tuple[int, EtcdValue | None]: + value = await self.client.get(self._retry_counter_key(pool_id, kind)) + count = int(value.value) if value is not None else 0 + if count < 0: + raise RuntimeError(f"negative retry {kind} counter") + return count, value + + async def retry_queue_depth(self, pool_id: str) -> int: + return (await self._retry_counter(pool_id, "waiting"))[0] + + async def retry_budget_used(self, pool_id: str) -> int: + return (await self._retry_counter(pool_id, "credits"))[0] + + async def _require_retry_counter(self, stored: StoredTask, count: int, kind: str) -> None: + if count > 0: + return + current = await self.get_task(stored.task.id) + if current is None or current.revision != stored.revision: + raise StoreConflict("task changed while reading retry accounting") + raise RuntimeError(f"retry {kind} counter is inconsistent") + def _lease_key(self, pool_id: str, worker_key_value: str) -> str: return f"{self.root}/pools/{pool_id}/worker-leases/{worker_key_value}" @@ -998,6 +1033,36 @@ async def create_task( principal_hash: str, idempotency_hash: str | None, queue_limit: int, + ) -> tuple[StoredTask, bool]: + loop = asyncio.get_running_loop() + deadline = loop.time() + 5.0 + try: + await asyncio.wait_for(self._create_task_lock.acquire(), timeout=5.0) + except asyncio.TimeoutError as exc: + raise GatewayError( + 503, "store_busy", + "task admission is busy; retry with the same Idempotency-Key", + error_type="service_unavailable_error", headers={"Retry-After": "1"}, + ) from exc + try: + return await self._create_task_serialized( + task, + principal_hash=principal_hash, + idempotency_hash=idempotency_hash, + queue_limit=queue_limit, + deadline=deadline, + ) + finally: + self._create_task_lock.release() + + async def _create_task_serialized( + self, + task: VideoTask, + *, + principal_hash: str, + idempotency_hash: str | None, + queue_limit: int, + deadline: float, ) -> tuple[StoredTask, bool]: task = _apply_patch( task, @@ -1014,7 +1079,8 @@ async def create_task( if idempotency_hash is not None else None ) - for _ in range(8): + attempt = 0 + while asyncio.get_running_loop().time() < deadline: if idem_key is not None: existing = await self.client.get(idem_key) if existing is not None: @@ -1031,14 +1097,16 @@ async def create_task( return stored, False count, counter = await self._counter(task.pool_id) + retry_count, retry_counter = await self._retry_counter(task.pool_id, "waiting") sequence, sequence_value = await self._sequence() - if count >= queue_limit: + if count - retry_count >= queue_limit: raise GatewayError(429, "queue_full", "video queue is full") assigned = _apply_patch(task, {"created_seq": sequence + 1}) compare = [ self.client.compare_version(task_key, 0), self.client.compare_version(queue_key, 0), *self._counter_compare(counter_key, counter), + *self._counter_compare(self._retry_counter_key(task.pool_id, "waiting"), retry_counter), *self._counter_compare(self._sequence_key(), sequence_value), ] success = [ @@ -1066,7 +1134,102 @@ async def create_task( succeeded, revision = await self.client.txn(compare, success) if succeeded: return StoredTask(assigned, revision), True - raise StoreConflict("unable to create task after repeated etcd CAS conflicts") + # Do not cancel an in-flight txn: its commit result may be ambiguous. + # The existing client bounds each RPC; this deadline bounds retries. + remaining = deadline - asyncio.get_running_loop().time() + if remaining <= 0: + break + delay = random.uniform(0.001, min(0.05, 0.002 * (2 ** min(attempt, 5)))) + await asyncio.sleep(min(delay, remaining)) + attempt += 1 + raise GatewayError( + 503, "store_busy", + "task admission contention timed out; retry with the same Idempotency-Key", + error_type="service_unavailable_error", headers={"Retry-After": "1"}, + ) + + async def requeue_failed_attempt( + self, stored: StoredTask, *, queue_limit: int, + quarantine_until_ms: int | None = None, + retry_wait_timeout_s: float = 600, + ) -> StoredTask | None: + """Atomically admit one retry without reopening a terminal task. + + Preserve the original FIFO sequence and idempotency identity. The old + worker key is retained only to exclude that instance on the retry. + This dedicated transition intentionally does not broaden the generic + state machine's allowed transitions. + """ + task = stored.task + if (task.status not in {TaskStatus.DISPATCHING, TaskStatus.IN_PROGRESS} + or task.attempt != 1 or task.cancel_requested_at_ms is not None + or not task.owner_generation or not task.worker_key + or not task.execution_token): + return None + count, counter = await self._counter(task.pool_id) + credit = await self.client.get(self._retry_credit_key(task)) + if credit is None: + return None + waiting, waiting_value = await self._retry_counter(task.pool_id, "waiting") + lease_key = self._lease_key(task.pool_id, task.worker_key) + lease_value = await self.client.get(lease_key) + if lease_value is None: + return None + lease = WorkerLease.from_dict(json.loads(lease_value.value)) + if (lease.task_id != task.id or lease.owner_generation != task.owner_generation + or lease.execution_token != task.execution_token): + return None + updated = _clone_task(task) + updated.status = TaskStatus.QUEUED + updated.queued_at_ms = now_ms() + updated.owner_generation = None + updated.execution_token = None + updated.worker_lease_id = None + updated.assigned_at_ms = None + updated.started_at_ms = None + updated.deadline_at_ms = now_ms() + int(retry_wait_timeout_s * 1000) + updated.expires_at_ms = min(task.expires_at_ms, updated.deadline_at_ms) + updated.completed_at_ms = None + updated.queue_wait_s = None + updated.inference_time_s = None + updated.finalize_time_s = None + updated.stage_durations = None + updated.error = None + counter_key = self._counter_key(task.pool_id) + queue_key = self._queue_key(task.pool_id, task.id) + compare = [ + self.client.compare_mod(self._task_key(task.id), stored.revision), + self.client.compare_mod(lease_key, lease_value.mod_revision), + self.client.compare_mod(self._retry_credit_key(task), credit.mod_revision), + self.client.compare_version(queue_key, 0), + self.client.compare_version(self._gateway_key(task.owner_generation), 0, result="GREATER"), + *self._counter_compare(counter_key, counter), + *self._counter_compare(self._retry_counter_key(task.pool_id, "waiting"), waiting_value), + ] + success = [ + self.client.put(self._task_key(task.id), self._encode(updated.to_dict())), + self.client.put(queue_key, str(updated.queued_at_ms)), + self.client.put(self._ordered_queue_key(updated), task.id), + self.client.put(counter_key, str(count + 1)), + self.client.put(self._retry_counter_key(task.pool_id, "waiting"), str(waiting + 1)), + self.client.delete(self._expiry_index_key(task)), + self.client.put(self._expiry_index_key(updated), task.id), + self.client.delete(self._owner_task_key(task.owner_generation, task.id)), + self.client.delete(self._lease_heartbeat_key(task.pool_id, task.worker_key)), + *[self.client.delete(key) for key in self._status_task_index_keys(task, task.status)], + *[self.client.put(key, task.id) for key in self._status_task_index_keys(updated, updated.status)], + ] + if quarantine_until_ms is None: + success.append(self.client.delete(lease_key)) + else: + lease.state = "quarantined" + lease.reuse_after_ms = quarantine_until_ms + lease.heartbeat_at_ms = now_ms() + success.append(self.client.put(lease_key, self._encode(lease.to_dict()))) + succeeded, revision = await self.client.txn(compare, success) + if not succeeded: + raise StoreConflict("retry admission lost an etcd CAS race") + return StoredTask(updated, revision) if succeeded else None async def get_task(self, task_id: str) -> StoredTask | None: value = await self.client.get(self._task_key(task_id)) @@ -1255,6 +1418,8 @@ async def reserve( lease: WorkerLease, *, deadline_at_ms: int, + reserve_retry: bool = False, + retry_limit: int = 32, ) -> StoredTask | None: task_key = self._task_key(stored.task.id) queue_key = self._queue_key(stored.task.pool_id, stored.task.id) @@ -1264,6 +1429,38 @@ async def reserve( count, counter = await self._counter(stored.task.pool_id) if count <= 0: return None + retry_waiting = self._is_retry_waiting(stored.task) + if retry_waiting and (stored.task.expires_at_ms <= now_ms() + or stored.task.worker_key == lease.worker_key): + return None + accounting_compare, accounting_success = [], [] + credit_key = self._retry_credit_key(stored.task) + if retry_waiting or (reserve_retry and stored.task.attempt == 0): + used, used_value = await self._retry_counter(stored.task.pool_id, "credits") + used_key = self._retry_counter_key(stored.task.pool_id, "credits") + accounting_compare.extend(self._counter_compare(used_key, used_value)) + if retry_waiting: + waiting, waiting_value = await self._retry_counter(stored.task.pool_id, "waiting") + await self._require_retry_counter(stored, waiting, "waiting") + await self._require_retry_counter(stored, used, "credits") + credit = await self.client.get(credit_key) + if credit is None: + return None + accounting_compare.extend([ + self.client.compare_mod(credit_key, credit.mod_revision), + *self._counter_compare(self._retry_counter_key(stored.task.pool_id, "waiting"), waiting_value), + ]) + accounting_success.extend([ + self.client.delete(credit_key), + self.client.put(used_key, str(used - 1)), + self.client.put(self._retry_counter_key(stored.task.pool_id, "waiting"), str(waiting - 1)), + ]) + else: + if used >= retry_limit: + return None + accounting_compare.append(self.client.compare_version(credit_key, 0)) + accounting_success.extend([self.client.put(credit_key, stored.task.id), + self.client.put(used_key, str(used + 1))]) native_lease = await self.client.lease_grant(self.execution_lease_ttl_s) lease = copy.deepcopy(lease) lease.etcd_lease_id = native_lease.lease_id @@ -1279,9 +1476,11 @@ async def reserve( "execution_token": lease.execution_token, "assigned_at_ms": now_ms(), "deadline_at_ms": deadline_at_ms, + "expires_at_ms": max(stored.task.expires_at_ms, deadline_at_ms) if retry_waiting else stored.task.expires_at_ms, }, ) compare = [ + *accounting_compare, self.client.compare_mod(task_key, stored.revision), self.client.compare_version(queue_key, 0, result="GREATER"), self.client.compare_version(lease_key, 0), @@ -1292,6 +1491,7 @@ async def reserve( *self._counter_compare(counter_key, counter), ] success = [ + *accounting_success, self.client.put(task_key, self._encode(updated.to_dict())), self.client.put( lease_key, @@ -1324,6 +1524,9 @@ async def reserve( for key in self._status_task_index_keys(updated, updated.status) ], ] + if updated.expires_at_ms != stored.task.expires_at_ms: + success.extend([self.client.delete(self._expiry_index_key(stored.task)), + self.client.put(self._expiry_index_key(updated), updated.id)]) try: succeeded, revision = await self.client.txn(compare, success) except Exception: @@ -1404,6 +1607,13 @@ async def transition( counter_key = self._counter_key(stored.task.pool_id) count, counter = await self._counter(stored.task.pool_id) if count <= 0: + # Task and counter are separate reads. A concurrent reservation + # or cancellation may have dequeued this task between them. + # Let callers retry that race, but keep detecting a genuinely + # inconsistent counter when the queued task is unchanged. + current = await self.get_task(task_id) + if current is None or current.revision != stored.revision: + raise StoreConflict("task changed while reading queue counter") raise RuntimeError("queue counter is inconsistent") compare.extend( [ @@ -1419,6 +1629,23 @@ async def transition( ] ) + if self._is_retry_waiting(stored.task) and updated.status != TaskStatus.QUEUED: + waiting, waiting_value = await self._retry_counter(stored.task.pool_id, "waiting") + await self._require_retry_counter(stored, waiting, "waiting") + key = self._retry_counter_key(stored.task.pool_id, "waiting") + compare.extend(self._counter_compare(key, waiting_value)) + success.append(self.client.put(key, str(waiting - 1))) + if updated.status in TERMINAL_STATUSES and stored.task.status not in TERMINAL_STATUSES: + credit_key = self._retry_credit_key(stored.task) + credit = await self.client.get(credit_key) + if credit is not None: + used, used_value = await self._retry_counter(stored.task.pool_id, "credits") + await self._require_retry_counter(stored, used, "credits") + key = self._retry_counter_key(stored.task.pool_id, "credits") + compare.extend([self.client.compare_mod(credit_key, credit.mod_revision), + *self._counter_compare(key, used_value)]) + success.extend([self.client.delete(credit_key), self.client.put(key, str(used - 1))]) + if release_lease and stored.task.worker_key is not None: lease_key = self._lease_key(stored.task.pool_id, stored.task.worker_key) heartbeat_key = self._lease_heartbeat_key( diff --git a/tests/video_gateway/test_worker_liveness.py b/tests/video_gateway/test_worker_liveness.py new file mode 100644 index 000000000000..406aacf6afbe --- /dev/null +++ b/tests/video_gateway/test_worker_liveness.py @@ -0,0 +1,190 @@ +import asyncio +from types import SimpleNamespace as NS +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from dingo.video_gateway import dispatcher as m +from dingo.video_gateway.models import TaskStatus, now_ms +from dingo.common.video_task_protocol import WAIT_TERMINAL_CAPABILITY + + +def fixture(): + d = object.__new__(m.VideoDispatcher) + d._gateway_owner_healthy = d._task_watch_healthy = True + d._worker_liveness_checks = asyncio.Semaphore(4) + d.telemetry = MagicMock() + task = NS(id='liveness-test', deployment_id='isolated', pool_id='pool', + attempt=1, execution_token='a' * 32, worker_instance_id=101, + deadline_at_ms=now_ms() + 60000) + target = 'dyn://isolated.backend.generate' + pool = NS(config=NS(pool_id='pool', backend_target=target), instance_ids=[], + discovery_healthy=True, lease_watch_healthy=True) + d.store = NS(discovery_truth_supported=True, + discovery_instance_snapshot=AsyncMock(return_value={target: set()})) + d._current_owned_execution = AsyncMock(return_value=NS(task=NS( + status=TaskStatus.IN_PROGRESS, cancel_requested_at_ms=None))) + status = dict(schema_version=1, deployment_id=task.deployment_id, + pool_id=task.pool_id, task_id=task.id, attempt=task.attempt, + execution_token=task.execution_token, state='running', + updated_at_ms=now_ms() - 60000, + capabilities=[WAIT_TERMINAL_CAPABILITY]) + d.artifacts = NS(read_detached_status=AsyncMock(return_value=status)) + return d, pool, task, status + + +async def forever(): + await asyncio.Event().wait() + + +async def test_five_second_default_and_four_check_limit(): + assert m._WORKER_LIVENESS_CHECK_INTERVAL_S == 5.0 + assert m._WORKER_LIVENESS_CHECK_CONCURRENCY == 4 + + +async def test_100_registered_executions_do_not_read_remote_state(): + d, pool, task, _ = fixture() + pool.instance_ids = [101] + assert not any(await asyncio.gather(*(d._confirm_worker_loss(pool, task) for _ in range(100)))) + d.store.discovery_instance_snapshot.assert_not_awaited() + d.artifacts.read_detached_status.assert_not_awaited() + + +@pytest.mark.parametrize('flag', ['owner', 'task_watch', 'discovery', 'lease_watch']) +async def test_unhealthy_control_plane_never_means_worker_loss(flag): + d, pool, task, _ = fixture() + if flag == 'owner': d._gateway_owner_healthy = False + if flag == 'task_watch': d._task_watch_healthy = False + if flag == 'discovery': pool.discovery_healthy = False + if flag == 'lease_watch': pool.lease_watch_healthy = False + assert not await d._confirm_worker_loss(pool, task) + d.store.discovery_instance_snapshot.assert_not_awaited() + + +async def test_local_missing_but_authoritatively_present_is_not_lost(): + d, pool, task, _ = fixture() + d.store.discovery_instance_snapshot.return_value = {pool.config.backend_target: {101}} + assert not await d._confirm_worker_loss(pool, task) + d.artifacts.read_detached_status.assert_not_awaited() + + +@pytest.mark.parametrize('where', ['discovery', 'artifact']) +async def test_failed_evidence_reads_are_inconclusive(where): + d, pool, task, _ = fixture() + call = d.store.discovery_instance_snapshot if where == 'discovery' else d.artifacts.read_detached_status + call.side_effect = RuntimeError('unavailable/corrupt test evidence') + assert not await d._confirm_worker_loss(pool, task) + + +@pytest.mark.parametrize('state', ['completed', 'failed', 'cancelled', 'not_found']) +async def test_reported_terminal_or_unknown_status_is_not_worker_loss(state): + d, pool, task, status = fixture(); status['state'] = state + assert not await d._confirm_worker_loss(pool, task) + + +@pytest.mark.parametrize('heartbeat', [None, True, 'invalid', 'fresh', 'future']) +async def test_invalid_or_recent_heartbeat_does_not_trigger_retry(heartbeat): + d, pool, task, status = fixture() + status['updated_at_ms'] = now_ms() + (60000 if heartbeat == 'future' else 0) if heartbeat in ('fresh', 'future') else heartbeat + assert not await d._confirm_worker_loss(pool, task) + + +async def test_cancel_and_owner_change_prevent_retry(): + d, pool, task, _ = fixture() + d._current_owned_execution.return_value.task.cancel_requested_at_ms = now_ms() + assert not await d._confirm_worker_loss(pool, task) + d._current_owned_execution.return_value = None + with pytest.raises(m._TaskOwnershipLost): await d._confirm_worker_loss(pool, task) + + +async def test_expired_execution_deadline_is_not_reclassified(): + d, pool, task, _ = fixture(); task.deadline_at_ms = now_ms() - 1 + assert not await d._confirm_worker_loss(pool, task) + d.store.discovery_instance_snapshot.assert_not_awaited() + + +async def test_max_four_remote_confirmations_and_recheck_after_waiting(): + d, pool, task, _ = fixture(); entered = 0; maximum = 0; active = 0 + gate = asyncio.Event() + async def snapshot(_): + nonlocal entered, maximum, active + entered += 1; active += 1; maximum = max(maximum, active) + try: await gate.wait(); return {pool.config.backend_target: {101}} + finally: active -= 1 + d.store.discovery_instance_snapshot.side_effect = snapshot + tasks = [asyncio.create_task(d._confirm_worker_loss(pool, task)) for _ in range(100)] + try: + for _ in range(10): await asyncio.sleep(0) + assert entered == maximum == 4 + pool.instance_ids = [101]; gate.set() + assert not any(await asyncio.gather(*tasks)) + assert entered == 4 + finally: + gate.set() + for t in tasks: t.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + + +async def test_hung_attached_wait_is_interrupted_and_children_are_cleaned(monkeypatch): + d, pool, task, status = fixture() + monkeypatch.setattr(m, '_WORKER_LIVENESS_CHECK_INTERVAL_S', .01) + attached = asyncio.Event(); stream_closed = asyncio.Event() + async def stream(): + try: + yield {**status, 'state': 'watching'} + attached.set() + await forever() + finally: stream_closed.set() + async def direct(*_): return stream() + pool.client = NS(direct=direct); pool.instance_ids = [101] + d._shrink_to_result_memory_budget = AsyncMock() + running = NS(worker_accepted=True) + heartbeat = asyncio.create_task(forever()) + operation = asyncio.create_task(d._run_with_lease_monitor( + d._consume_detached_worker(pool, NS(task=task), None, None, None, + running, initial_worker_status=status), + heartbeat, forever(), d._monitor_worker_liveness(pool, task, running))) + try: + await asyncio.wait_for(attached.wait(), 1) + pool.instance_ids = [] + with pytest.raises(m._RetryableWorkerFailure): await asyncio.wait_for(operation, 1) + assert stream_closed.is_set() + assert d.store.discovery_instance_snapshot.await_count >= 1 + finally: + heartbeat.cancel(); operation.cancel() + await asyncio.gather(heartbeat, operation, return_exceptions=True) + + +async def test_registered_long_wait_and_cancel_cleanup(monkeypatch): + d, pool, task, _ = fixture(); pool.instance_ids = [101] + monkeypatch.setattr(m, '_WORKER_LIVENESS_CHECK_INTERVAL_S', .01) + heartbeat = asyncio.create_task(forever()) + operation = asyncio.create_task(d._run_with_lease_monitor( + forever(), heartbeat, forever(), d._monitor_worker_liveness(pool, task, NS(worker_accepted=True)))) + try: + await asyncio.sleep(.055) + assert not operation.done() + d.store.discovery_instance_snapshot.assert_not_awaited() + d.artifacts.read_detached_status.assert_not_awaited() + finally: + operation.cancel(); heartbeat.cancel() + await asyncio.gather(operation, heartbeat, return_exceptions=True) + + +async def test_completed_result_wins_simultaneous_failure(): + d, _, _, _ = fixture() + async def completed(): return 'result' + async def lost(): raise m._RetryableWorkerFailure('lost') + heartbeat = asyncio.create_task(forever()) + try: assert await d._run_with_lease_monitor(completed(), heartbeat, None, lost()) == 'result' + finally: + heartbeat.cancel(); await asyncio.gather(heartbeat, return_exceptions=True) + + +async def test_lease_loss_wins_simultaneous_liveness_failure(): + d, _, _, _ = fixture() + async def lease_lost(): raise m._WorkerLeaseLost('owner lost') + async def worker_lost(): raise m._RetryableWorkerFailure('worker lost') + heartbeat = asyncio.create_task(lease_lost()) + with pytest.raises(m._WorkerLeaseLost): + await d._run_with_lease_monitor(forever(), heartbeat, None, worker_lost()) From 0f2b19671d526974f7c7dd31d4fc580f0cb821f6 Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Thu, 17 Sep 2026 03:15:54 +0800 Subject: [PATCH 04/12] [fix][container] align Omni 0.29 runtime and media dependencies Pin vLLM 0.29.0 and Omni 0.29.0rc1, retain compiled framework constraints, include and validate ffprobe, and honor the selected native build parallelism. --- container/context.yaml | 6 +- container/deps/vllm/install_vllm_omni.sh | 7 +- container/deps/vllm/validate_media_probe.py | 75 ++++++++++++++++++++ container/templates/vllm_runtime.Dockerfile | 9 ++- container/templates/wheel_builder.Dockerfile | 8 +++ pyproject.toml | 2 +- 6 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 container/deps/vllm/validate_media_probe.py diff --git a/container/context.yaml b/container/context.yaml index 55f55f16b70d..d5e99dcc58cc 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -54,9 +54,9 @@ vllm: base_image: nvcr.io/nvidia/cuda-dl-base runtime_image: vllm/vllm-openai base_image_tag: 25.11-cuda13.0-devel-ubuntu24.04 - runtime_image_tag: v0.27.1-ubuntu2404 + runtime_image_tag: v0.29.0-ubuntu2404 # Keep using the existing CUDA 13.0.2 compliance corpus for this Dingo CI - # image update. Refreshing the corpus for vLLM 0.27.1's CUDA 13.0.3 base is + # image update. Refreshing the corpus for the vLLM 0.29 runtime base is # intentionally deferred because this workflow does not publish SBOM or # provenance artifacts. Per-arch stem; the licenses stage appends # -${TARGETARCH}.cdx.json. @@ -74,7 +74,7 @@ vllm: runtime_image_tag: v0.24.0 # baseline_sbom: not yet captured for cpu — runtime build runs without subtraction flashinf_ref: v0.6.16.post3 - vllm_omni_ref: "v0.27.0rc1" + vllm_omni_ref: "v0.29.0rc1" nixl_ref: v1.3.1 max_jobs: "10" enable_media_ffmpeg: "false" diff --git a/container/deps/vllm/install_vllm_omni.sh b/container/deps/vllm/install_vllm_omni.sh index 929e9a4821cd..7306278a6bbb 100755 --- a/container/deps/vllm/install_vllm_omni.sh +++ b/container/deps/vllm/install_vllm_omni.sh @@ -17,7 +17,7 @@ cleanup() { trap cleanup EXIT -python3 - "${VLLM_OMNI_PROTECTED_PACKAGES_FILE}" <<'PY' > "${PROTECTED_CONSTRAINTS}" +python3 - "${VLLM_OMNI_PROTECTED_PACKAGES_FILE}" "${VLLM_OMNI_VERSION}" <<'PY' > "${PROTECTED_CONSTRAINTS}" import importlib.metadata as md from pathlib import Path import sys @@ -26,6 +26,11 @@ for raw_line in Path(sys.argv[1]).read_text().splitlines(): name = raw_line.strip() if not name or name.startswith("#"): continue + # Omni 0.29.0rc1 requires transformers>=5.13,<5.15, while the pinned + # upstream vLLM 0.29 image ships 5.16.1. Let Omni resolve this pure-Python + # API stack and its paired tokenizer, retaining the compiled core pins. + if sys.argv[2] == "0.29.0rc1" and name in {"transformers", "tokenizers"}: + continue try: dist = md.distribution(name) except Exception: diff --git a/container/deps/vllm/validate_media_probe.py b/container/deps/vllm/validate_media_probe.py new file mode 100644 index 000000000000..abf1272884e6 --- /dev/null +++ b/container/deps/vllm/validate_media_probe.py @@ -0,0 +1,75 @@ +"""CPU-only FFmpeg/ffprobe smoke after replacing the upstream media packages.""" + +import json +import shutil +import subprocess +import tempfile +from pathlib import Path + +import av + + +def main(): + for executable in ("ffmpeg", "ffprobe"): + path = shutil.which(executable) + if path != "/usr/local/bin/" + executable: + raise RuntimeError(f"Expected in-tree {executable}, found {path!r}") + subprocess.run([path, "-version"], check=True, capture_output=True, timeout=15) + + with tempfile.TemporaryDirectory(prefix="dingo-media-probe-") as directory: + path = Path(directory) / "sample.mp4" + with av.open(str(path), "w", format="mp4") as output: + video = output.add_stream("libx264", rate=8) + video.width, video.height, video.pix_fmt = 32, 32, "yuv420p" + audio = output.add_stream("aac", rate=48000) + audio.layout = "stereo" + for index in range(4): + frame = av.VideoFrame(32, 32, "yuv420p") + frame.pts = index + for plane in frame.planes: + plane.update(bytes(plane.buffer_size)) + for packet in video.encode(frame): + output.mux(packet) + for packet in video.encode(None): + output.mux(packet) + frame = av.AudioFrame(format="fltp", layout="stereo", samples=24000) + frame.sample_rate, frame.pts = 48000, 0 + for plane in frame.planes: + plane.update(bytes(plane.buffer_size)) + for packet in audio.encode(frame): + output.mux(packet) + for packet in audio.encode(None): + output.mux(packet) + result = subprocess.run( + [ + "ffprobe", + "-v", + "error", + "-count_frames", + "-show_streams", + "-show_format", + "-of", + "json", + str(path), + ], + capture_output=True, + text=True, + check=True, + timeout=30, + ) + document = json.loads(result.stdout) + streams = {stream["codec_type"]: stream for stream in document["streams"]} + assert streams["video"]["codec_name"] == "h264" + assert (streams["video"]["width"], streams["video"]["height"]) == (32, 32) + assert int(streams["video"]["nb_read_frames"]) == 4 + assert streams["audio"]["codec_name"] == "aac" + assert int(streams["audio"]["sample_rate"]) == 48000 + assert int(streams["audio"]["channels"]) == 2 + assert float(document["format"]["duration"]) > 0 + print( + "DINGO_MEDIA_PROBE=PASS (H.264 video + stereo AAC, ffprobe JSON and frame count)" + ) + + +if __name__ == "__main__": + main() diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 1964e7be306d..8b2a9de6d984 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -266,7 +266,7 @@ RUN set -eux; \ RUN --mount=type=bind,source=./container/deps/vllm/validate_torch_compile_smoke.py,target=/tmp/validate_torch_compile_smoke.py,readonly \ python3 /tmp/validate_torch_compile_smoke.py -# Copy the LGPL ffmpeg from wheel_builder: versioned shared libs (libav*.so*, +# Copy the LGPL ffmpeg and ffprobe from wheel_builder: versioned shared libs (libav*.so*, # libsw*.so*) + libvpx + the LGPL CLI binary that imageio/diffusers target via # IMAGEIO_FFMPEG_EXE. Ungated by enable_media_ffmpeg because the base GPL ffmpeg # was just purged, so the LGPL CLI must always be present for the omni @@ -278,9 +278,16 @@ RUN --mount=type=bind,from=wheel_builder,source=/usr/local/,target=/tmp/usr/loca cp -nL /tmp/usr/local/lib/lib*vpx*.so* /usr/local/lib/ 2>/dev/null || true && \ cp -nL /tmp/usr/local/lib/pkgconfig/libav*.pc /tmp/usr/local/lib/pkgconfig/libsw*.pc /usr/local/lib/pkgconfig/ && \ cp -nL /tmp/usr/local/bin/ffmpeg /usr/local/bin/ffmpeg && \ + cp -nL /tmp/usr/local/bin/ffprobe /usr/local/bin/ffprobe && \ cp -r /tmp/usr/local/src/ffmpeg /usr/local/src/ && \ ldconfig ENV IMAGEIO_FFMPEG_EXE=/usr/local/bin/ffmpeg + +# H3 reference-media preprocessing invokes ffprobe directly. The upstream +# package purge removes /usr/bin/ffprobe too; copying only ffmpeg leaves a +# runtime failure even though the Python imports and output encoding work. +RUN --mount=type=bind,source=./container/deps/vllm/validate_media_probe.py,target=/tmp/validate_media_probe.py,readonly \ + python3 /tmp/validate_media_probe.py {% endif %} # Replace the upstream vllm/vllm-openai image's imageio-ffmpeg (which ships a diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index 03215e58ac34..dca66bb89227 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -535,6 +535,11 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token FROM wheel_builder_base AS runtime_wheel_builder +# Re-declare after FROM so --builder-image does not retain its baked-in +# compilation parallelism when the current build has a smaller resource budget. +ARG CARGO_BUILD_JOBS +ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS:-16} + {% if target not in ("dev", "local-dev") %} # Copy source code (order matters for layer caching) COPY .cargo/ /opt/dynamo/.cargo/ @@ -759,6 +764,9 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token # so code-only commits never invalidate the published dependency image. FROM reusable_builder_base AS wheel_builder +ARG CARGO_BUILD_JOBS +ENV CARGO_BUILD_JOBS=${CARGO_BUILD_JOBS:-16} + ARG TARGETARCH ARG DEVICE ARG USE_SCCACHE diff --git a/pyproject.toml b/pyproject.toml index ce0f9d2cfc03..ba51e060ffee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ video-gateway = [ vllm = [ "uvloop", "nixl[cu13]==1.3.1", - "vllm[flashinfer,runai,otel]==0.27.1", + "vllm[flashinfer,runai,otel]==0.29.0", # vllm-omni is not part of ai-dingo[vllm]: container builds inherit the # framework stack from vllm/vllm-openai, and pip/uv dependency resolution # for omni can override the vLLM torch stack. From 1429696682da6321fa1a4820193c08131e39485e Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Thu, 17 Sep 2026 03:16:44 +0800 Subject: [PATCH 05/12] [feat][video-gateway] support continuous multi-slot execution Add fenced slot capacity, durable result handoff, independent recoverable finalization, bounded Worker prefetch, binary artifacts and native reusable frame conversion. Keep blocking filesystem work off the event loop and harden shared-counter CAS transitions. Document capacity metrics and upgrade boundaries. Validated with 451 Gateway tests, detached manager tests, 40/80/120 virtual Worker stages and real Omni continuous-execution fault cases. --- dingo/common/video_encoding.py | 76 ++ dingo/common/video_result_file.py | 189 ++++ dingo/common/video_task_protocol.py | 9 + dingo/video_gateway/CONTINUOUS_EXECUTION.md | 76 ++ dingo/video_gateway/METRICS.md | 60 ++ dingo/video_gateway/__main__.py | 6 +- dingo/video_gateway/adapters/base.py | 11 +- dingo/video_gateway/adapters/minimax_h3.py | 274 ++++-- dingo/video_gateway/api.py | 113 ++- dingo/video_gateway/artifact_store.py | 466 +++++++--- dingo/video_gateway/config.py | 69 +- dingo/video_gateway/dispatcher.py | 859 +++++++++++++++--- dingo/video_gateway/errors.py | 8 +- dingo/video_gateway/file_io.py | 72 ++ dingo/video_gateway/finalization.py | 247 +++++ dingo/video_gateway/form_parser.py | 17 +- dingo/video_gateway/result_handoff.py | 82 ++ dingo/video_gateway/service.py | 21 +- dingo/video_gateway/task_store.py | 431 +++++++-- dingo/vllm/omni/args.py | 54 +- dingo/vllm/omni/detached_tasks.py | 367 ++++++-- dingo/vllm/omni/main.py | 32 +- dingo/vllm/omni/output_formatter.py | 83 +- dingo/vllm/omni/stage_router.py | 9 +- dingo/vllm/tests/omni/test_detached_tasks.py | 101 +- dingo/vllm/tests/omni/test_omni_args.py | 43 +- .../vllm/tests/omni/test_omni_base_handler.py | 23 +- .../vllm/tests/omni/test_output_formatter.py | 49 +- .../test_artifact_metadata_probe.py | 91 ++ tests/video_gateway/test_binary_result.py | 842 +++++++++++++++++ tests/video_gateway/test_config.py | 59 +- .../test_continuous_execution_timing.py | 99 ++ .../test_detached_terminal_reuse.py | 108 +++ tests/video_gateway/test_dispatch_idle.py | 129 +++ .../test_early_release_dispatcher.py | 161 ++++ tests/video_gateway/test_execution_slots.py | 137 +++ tests/video_gateway/test_file_io.py | 395 ++++++++ tests/video_gateway/test_finalization.py | 233 +++++ .../video_gateway/test_minimax_h3_adapter.py | 135 ++- .../video_gateway/test_real_etcd_contract.py | 123 ++- tests/video_gateway/test_result_handoff.py | 188 ++++ tests/video_gateway/test_slot_dispatcher.py | 192 ++++ tests/video_gateway/test_slot_metrics.py | 156 ++++ .../video_gateway/test_terminal_cas_retry.py | 288 ++++++ tests/video_gateway/test_video_encoding.py | 165 ++++ tests/video_gateway/test_worker_prefetch.py | 174 ++++ 46 files changed, 6816 insertions(+), 706 deletions(-) create mode 100644 dingo/common/video_encoding.py create mode 100644 dingo/common/video_result_file.py create mode 100644 dingo/video_gateway/CONTINUOUS_EXECUTION.md create mode 100644 dingo/video_gateway/METRICS.md create mode 100644 dingo/video_gateway/file_io.py create mode 100644 dingo/video_gateway/finalization.py create mode 100644 dingo/video_gateway/result_handoff.py create mode 100644 tests/video_gateway/test_artifact_metadata_probe.py create mode 100644 tests/video_gateway/test_binary_result.py create mode 100644 tests/video_gateway/test_continuous_execution_timing.py create mode 100644 tests/video_gateway/test_detached_terminal_reuse.py create mode 100644 tests/video_gateway/test_dispatch_idle.py create mode 100644 tests/video_gateway/test_early_release_dispatcher.py create mode 100644 tests/video_gateway/test_execution_slots.py create mode 100644 tests/video_gateway/test_file_io.py create mode 100644 tests/video_gateway/test_finalization.py create mode 100644 tests/video_gateway/test_result_handoff.py create mode 100644 tests/video_gateway/test_slot_dispatcher.py create mode 100644 tests/video_gateway/test_slot_metrics.py create mode 100644 tests/video_gateway/test_terminal_cas_retry.py create mode 100644 tests/video_gateway/test_video_encoding.py create mode 100644 tests/video_gateway/test_worker_prefetch.py diff --git a/dingo/common/video_encoding.py b/dingo/common/video_encoding.py new file mode 100644 index 000000000000..1da30c26820c --- /dev/null +++ b/dingo/common/video_encoding.py @@ -0,0 +1,76 @@ +"""Optional bounded Omni frame conversion; no GPU dependencies at import time.""" + +import inspect +import os +import threading + + +def frame_conversion_workers() -> int: + """Deployment-only control; eight workers matches the native Omni service.""" + try: + value = int(os.environ.get("DINGO_VIDEO_FRAME_CONVERSION_WORKERS", "8")) + except ValueError as exc: + raise ValueError( + "DINGO_VIDEO_FRAME_CONVERSION_WORKERS must be an integer from 1 to 16" + ) from exc + if not 1 <= value <= 16: + raise ValueError( + "DINGO_VIDEO_FRAME_CONVERSION_WORKERS must be an integer from 1 to 16" + ) + return value + + +class VideoEncoder: + """One reusable native converter per formatter, shared across requests. + + Shutdown fences new work and waits for active encodes before closing the + converter. This also covers encode threads finishing after task cancellation. + """ + + def __init__(self, workers: int): + if type(workers) is not int or not 1 <= workers <= 16: + raise ValueError("frame conversion workers must be an integer from 1 to 16") + self.workers = workers + self._condition = threading.Condition() + self._active = 0 + self._closed = False + self._initialized = False + self._converter = None + + def encode(self, api, video, **kwargs) -> bytes: + with self._condition: + if self._closed: + raise RuntimeError("video encoder is closed") + if not self._initialized: + factory = getattr(api, "_PlanarFrameConverter", None) + try: + supported = ( + "frame_converter" + in inspect.signature(api._encode_video_bytes).parameters + ) + except (TypeError, ValueError): + supported = False + if self.workers > 1 and factory is not None and supported: + self._converter = factory(max_workers=self.workers) + self._initialized = True + converter = self._converter + self._active += 1 + try: + if converter is not None: + return api._encode_video_bytes( + video, frame_converter=converter, **kwargs + ) + return api._encode_video_bytes(video, **kwargs) + finally: + with self._condition: + self._active -= 1 + self._condition.notify_all() + + def close(self) -> None: + with self._condition: + self._closed = True + while self._active: + self._condition.wait() + converter, self._converter = self._converter, None + if converter is not None: + converter.shutdown() diff --git a/dingo/common/video_result_file.py b/dingo/common/video_result_file.py new file mode 100644 index 000000000000..5159055e3abd --- /dev/null +++ b/dingo/common/video_result_file.py @@ -0,0 +1,189 @@ +"""Private, task-scoped binary result handoff; no public request path accepts it.""" + +from __future__ import annotations + +import asyncio +import contextvars +import hashlib +import json +import math +import os +import re +import time +import uuid +from pathlib import Path +from typing import Any + +BINARY_RESULT_WRITER: contextvars.ContextVar[BinaryResultWriter | None] = ( + contextvars.ContextVar("dingo_binary_result_writer", default=None) +) +INLINE_RESULT_FORMAT = "binary_mp4_inline_v1" + + +def normalize_inline_result(value: Any) -> dict[str, Any]: + """Bound the small private status payload; never admit Base64 or paths.""" + if not isinstance(value, dict) or value.get("status") not in { + "completed", + "failed", + "cancelled", + }: + raise ValueError("inline Worker response is not terminal") + result = {"status": value["status"]} + if value["status"] == "completed": + data = value.get("data") + if ( + not isinstance(data, list) + or len(data) != 1 + or not isinstance(data[0], dict) + or set(data[0]) != {"output_format", "artifact"} + or data[0]["output_format"] != "mp4" + ): + raise ValueError( + "inline Worker response must contain one binary MP4 descriptor" + ) + result["data"] = [ + { + "output_format": "mp4", + "artifact": validate_descriptor(data[0]["artifact"]), + } + ] + elif "error" in value: + error = value["error"] + if isinstance(error, str) and len(error) <= 1024: + result["error"] = error + elif isinstance(error, dict) and all( + isinstance(error.get(k), str) and len(error[k]) <= limit + for k, limit in [("code", 128), ("message", 1024)] + ): + result["error"] = {k: error[k] for k in ["code", "message"]} + else: + raise ValueError("invalid inline Worker error") + if value.get("inference_time_s") is not None: + duration = value["inference_time_s"] + if ( + type(duration) not in {int, float} + or not math.isfinite(duration) + or duration < 0 + ): + raise ValueError("invalid inline inference duration") + result["inference_time_s"] = duration + if value.get("stage_durations") is not None: + stages = value["stage_durations"] + if ( + not isinstance(stages, dict) + or len(stages) > 32 + or any( + not isinstance(k, str) + or len(k) > 128 + or type(v) not in {int, float} + or not math.isfinite(v) + or v < 0 + for k, v in stages.items() + ) + ): + raise ValueError("invalid inline stage durations") + result["stage_durations"] = dict(stages) + if len(json.dumps(result, ensure_ascii=False).encode()) > 8192: + raise ValueError("inline Worker result exceeds metadata limit") + return result + + +def validate_descriptor(value: Any) -> dict[str, Any]: + if not isinstance(value, dict) or set(value) != { + "schema_version", + "filename", + "bytes", + "sha256", + }: + raise ValueError("invalid binary result descriptor") + if ( + type(value["schema_version"]) is not int + or value["schema_version"] != 1 + or not isinstance(value["filename"], str) + or re.fullmatch(r"worker-video-[0-9a-f]{32}\.mp4", value["filename"]) is None + or type(value["bytes"]) is not int + or value["bytes"] <= 0 + or not isinstance(value["sha256"], str) + or re.fullmatch(r"[0-9a-f]{64}", value["sha256"]) is None + ): + raise ValueError("invalid binary result descriptor") + return dict(value) + + +class BinaryResultWriter: + def __init__(self, root: Path, max_bytes: int = 128 * 1024 * 1024): + self.root = root + self.max_bytes = max_bytes + self.used = False + self.stage_durations: dict[str, float] = {} + self._submitted_at: float | None = None + + def _write(self, data: bytes) -> dict[str, Any]: + started = time.perf_counter() + self.stage_durations["artifact_queue_s"] = max( + 0.0, started - (self._submitted_at or started) + ) + name = f"worker-video-{uuid.uuid4().hex}.mp4" + temporary = self.root / (name + ".part") + final = self.root / name + renamed = False + try: + mark = time.perf_counter() + digest = hashlib.sha256(data).hexdigest() + self.stage_durations["artifact_hash_s"] = time.perf_counter() - mark + mark = time.perf_counter() + with temporary.open("xb") as stream: + self.stage_durations["artifact_open_s"] = time.perf_counter() - mark + mark = time.perf_counter() + stream.write(data) + stream.flush() + self.stage_durations["artifact_write_s"] = time.perf_counter() - mark + mark = time.perf_counter() + os.fsync(stream.fileno()) + self.stage_durations["artifact_file_fsync_s"] = ( + time.perf_counter() - mark + ) + mark = time.perf_counter() + self.stage_durations["artifact_close_s"] = time.perf_counter() - mark + mark = time.perf_counter() + os.replace(temporary, final) + self.stage_durations["artifact_rename_s"] = time.perf_counter() - mark + renamed = True + mark = time.perf_counter() + descriptor = os.open(self.root, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(descriptor) + finally: + os.close(descriptor) + self.stage_durations["artifact_dir_fsync_s"] = time.perf_counter() - mark + return dict(schema_version=1, filename=name, bytes=len(data), sha256=digest) + finally: + if not renamed: + temporary.unlink(missing_ok=True) + self._finished_at = time.perf_counter() + self.stage_durations["artifact_work_s"] = self._finished_at - started + + async def write(self, data: bytes) -> dict[str, Any]: + if self.used: + raise RuntimeError("multiple binary video results are not supported") + if not isinstance(data, bytes) or not 0 < len(data) <= self.max_bytes: + raise ValueError("binary video result exceeds configured size or is empty") + self.used = True + self._submitted_at = time.perf_counter() + work = asyncio.create_task(asyncio.to_thread(self._write, data)) + cancelled = False + while True: + try: + result = await asyncio.shield(work) + break + except asyncio.CancelledError: + cancelled = True + if work.done(): + work.result() + raise + if cancelled: + raise asyncio.CancelledError + self.stage_durations["artifact_resume_s"] = max( + 0.0, time.perf_counter() - self._finished_at + ) + return result diff --git a/dingo/common/video_task_protocol.py b/dingo/common/video_task_protocol.py index 959df8903ca1..3e8c9c0590d0 100644 --- a/dingo/common/video_task_protocol.py +++ b/dingo/common/video_task_protocol.py @@ -14,6 +14,8 @@ ENVELOPE_KEY = "_dingo_video_task" SCHEMA_VERSION = 1 WAIT_TERMINAL_CAPABILITY = "wait_terminal_v1" +EXECUTION_CAPACITY_CAPABILITY = "execution_capacity_v1" +PREFETCH_CAPABILITY = "execution_prefetch_v1" IDENTIFIER = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") TASK_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$") TOKEN = re.compile(r"^[0-9a-f]{32,64}$") @@ -73,12 +75,16 @@ def from_envelope(cls, value: Mapping[str, Any]) -> "DetachedTaskIdentity": "attempt", "execution_token", "payload", + "deadline_at_ms", } unknown = set(value) - allowed if unknown: raise ValueError(f"unknown detached task fields: {sorted(unknown)}") if value.get("schema_version") != SCHEMA_VERSION: raise ValueError("unsupported detached task schema_version") + deadline = value.get("deadline_at_ms") + if deadline is not None and (type(deadline) is not int or deadline <= 0): + raise ValueError("invalid detached deadline_at_ms") identity = cls( deployment_id=str(value.get("deployment_id", "")), pool_id=str(value.get("pool_id", "")), @@ -116,6 +122,7 @@ def detached_envelope( attempt: int, execution_token: str, payload: dict[str, Any] | None = None, + deadline_at_ms: int | None = None, ) -> dict[str, Any]: value: dict[str, Any] = { "schema_version": SCHEMA_VERSION, @@ -128,5 +135,7 @@ def detached_envelope( } if payload is not None: value["payload"] = payload + if deadline_at_ms is not None: + value["deadline_at_ms"] = deadline_at_ms DetachedTaskIdentity.from_envelope(value) return {ENVELOPE_KEY: value} diff --git a/dingo/video_gateway/CONTINUOUS_EXECUTION.md b/dingo/video_gateway/CONTINUOUS_EXECUTION.md new file mode 100644 index 000000000000..d58deb39472b --- /dev/null +++ b/dingo/video_gateway/CONTINUOUS_EXECUTION.md @@ -0,0 +1,76 @@ +# Detached Worker capacity and continuous execution + +These are opt-in deployment settings, not public HTTP request parameters. +They do not claim that a model gains throughput from GPU concurrency. + +## Enable + +Upgrade all Gateways to a version that understands durable result handoff before +enabling it. Upgrade Workers before requesting multiple execution/prefetch slots. +Gateway and Worker must share the same artifact filesystem and task root. + +For one executing request and one prefetched request per physical Worker: + +```yaml +# Within a pool using execution_mode: detached +scheduling: + worker_capacity: 1 + worker_prefetch_capacity: 1 + early_release_slot: true + finalization_concurrency: 4 + finalization_pending_limit: 64 + finalization_timeout_s: 60 + finalization_max_retries: 2 + finalization_retry_delay_s: 0.1 +``` + +Start the Omni Worker with `--max-num-seqs 1` and +`--detached-video-prefetch-capacity 1`, plus its existing detached task-root +arguments. Set `DINGO_VIDEO_BINARY_RESULTS=1` and `DINGO_VIDEO_INLINE_RESULT=1`. +The first enables durable MP4 descriptors; the second avoids an extra response +JSONL file. Neither enables an arbitrary-path public upload/download API. + +For MiniMax-H3 with N > 1, set matching `worker_capacity` / `--max-num-seqs`, and +enable `--step-execution`. The current integration rejects Cache-DiT with step +execution. The extra prefetch slot does not increase engine concurrency: N=2/P=1 +means at most two executions and one waiting task, not three model executions. +Verify memory capacity and model behavior separately before increasing N. + +Gateway negotiates capacity per physical registration and uses separate etcd +slot leases. Failed/legacy capability probes never infer N slots from config: +fallback is at most one. Two Gateways share reservations; their capacity gauges +must not be summed. See [metrics](METRICS.md). + +## Result boundary and failures + +Worker writes and synchronizes the MP4 before publishing a terminal descriptor. +Gateway then atomically persists the current attempt's descriptor, enters +`finalizing`, releases that task's slot and model-retry credit, and wakes dispatch. +Independent result processing subsequently publishes `completed`. + +A transient finalization error retries only result processing, with a persisted +retry count and deadline. Permanent validation failure or timeout fails that +task; it does not run the model again or release a reused slot. Cancellation +targets the task/attempt/token. A replacement Gateway can claim a durable +handoff without a live Worker. Before the handoff, Worker loss still follows +the configured one-retry policy; parameter/OOM/ordinary execution errors are +not automatically retried. + +Finalizer concurrency and pending limits are per Gateway, not cluster totals. +When its pending threshold is reached, that Gateway pauses new dispatch; work +already admitted can still finish. Memory admission remains independently +bounded. Size queue/retry/memory budgets for N+P admissions and pending results. +These controls cannot make permanently blocked filesystem calls cancellable. + +## Disable or roll back + +Drain accepted work and durable `finalizing` handoffs before downgrading to code +that does not understand this protocol. Disabling prefetch/early release does +not intentionally change the task configuration fingerprint, but that is not +permission to mix incompatible Gateway binaries. Keep all Gateways' pool +configuration aligned and check both shared ledgers and local finalizer gauges. + +The default is N=1/P=0 with early release off. The native frame converter is a +separate optimization: `DINGO_VIDEO_FRAME_CONVERSION_WORKERS` defaults to 8, +accepts integers 1–16, and 1 selects the legacy conversion path. It affects CPU +frame conversion only, not model execution slots or generated media parameters. diff --git a/dingo/video_gateway/METRICS.md b/dingo/video_gateway/METRICS.md new file mode 100644 index 000000000000..f2760515f123 --- /dev/null +++ b/dingo/video_gateway/METRICS.md @@ -0,0 +1,60 @@ +# Multi-slot capacity and statistics + +All metrics below use the `dingo_video_` prefix and a bounded `pool` label. + +| Metric suffix | Meaning | +| --- | --- | +| workers | Discovered physical Worker registrations, not Pods or slots | +| worker_busy | Distinct registered Workers with a non-quarantined lease | +| worker_execution_capacity | Negotiated execution concurrency, clamped to Gateway configuration | +| worker_prefetch_capacity | Negotiated additional prefetch admission capacity, not compute capacity | +| worker_admission_capacity | Execution plus prefetch capacity | +| worker_slots_busy | Non-quarantined leases mapped to current admission slots | +| worker_slots_quarantined | Quarantined leases mapped to current admission slots | +| worker_slots_free | Unleased admission slots; zero if discovery/lease-watch view is unhealthy | +| worker_unmapped_leases | Leases outside current slot map, e.g. disappeared registrations or unknown/reduced capacity | +| worker_capacity_view_healthy | Whether local discovery and the shared lease view are healthy | +| worker_execution_capacity_configured | Per-Worker execution ceiling, not pool total | +| worker_prefetch_capacity_configured | Per-Worker prefetch ceiling | +| early_release_slot_enabled | Local Gateway feature flag | +| finalization_pending_local | This Gateway's tracked finalizers, including waiting/running work | +| finalization_concurrency_configured | Local finalizer concurrency limit | +| finalization_pending_limit_configured | Local finalizer pending limit | + +`worker_busy` previously counted leases: for a multi-slot Worker this overstated +the number of physical busy Workers. Consumers needing lease occupancy should +switch to `worker_slots_busy`. Single-slot registered-Worker behavior is unchanged. +Disappeared registrations are excluded from physical busy count; their leases +remain visible in `worker_unmapped_leases` until recovered. + +Example: one Worker with N=2/P=1 has execution capacity 2 and admission capacity 3. +With two active leases, physical busy=1, slot busy=2 and slot free=1. This does NOT +prove that two model computations are currently running: a lease covers Worker +queuing, execution and output publication. Slot IDs are interchangeable; the +highest slot is not a permanently designated prefetch task. Similarly, task +`in_progress` and its `execution_started` lifecycle boundary are Gateway execution +states, not GPU kernel start measurements. Exact engine-running/queued counts +require Worker-side telemetry; do not infer them from admission leases. + +Early result handoff releases the lease before Gateway finalization. Historical +task Worker assignments must not count as current occupancy. Shared task counts +by status still include finalizing tasks; finalization_pending_local is a local +scheduler/backpressure view, not a replacement for that shared task count. + +Capacity snapshots reuse discovery capability caches and lease snapshots; scraping +does not call every Worker. Unknown multi-slot capacity contributes zero until +probed. Legacy fallback contributes at most one slot. A fully busy model remains +available for queuing; a model with no negotiated slots is unavailable. + +In HA, each Gateway exposes a view of the SAME shared pool and task counts. +Do not sum these gauges across Gateway replicas: deduplicate by deployment/pool +(e.g. max for a dashboard, checking view health and agreement). Local finalizer +and process-memory metrics are per replica. Free slots are observational, not an +admission guarantee: quarantine, memory, retries and finalization backpressure +may still block dispatch; etcd reservation CAS remains authoritative. + +Compare configured gauges across Gateways to detect configuration skew. Prefetch +and early-release flags intentionally retain the existing task configuration +fingerprint; changing it would fail queued tasks during recovery. This is not +permission to mix incompatible binaries: upgrade all Gateways before enabling +handoff, and drain durable handoffs before downgrading to older code. diff --git a/dingo/video_gateway/__main__.py b/dingo/video_gateway/__main__.py index ba0245f7d5ad..6c09bee41311 100644 --- a/dingo/video_gateway/__main__.py +++ b/dingo/video_gateway/__main__.py @@ -20,6 +20,7 @@ from dingo.video_gateway.dingo_adapter import create_pool_clients from dingo.video_gateway.dispatcher import VideoDispatcher from dingo.video_gateway.etcd_http import EtcdHttpClient +from dingo.video_gateway.file_io import run_file_io from dingo.video_gateway.service import VideoGatewayService from dingo.video_gateway.task_store import EtcdTaskStore, MemoryTaskStore from dingo.video_gateway.telemetry import GatewayTelemetry @@ -47,6 +48,7 @@ async def _wait_for_shutdown(runtime, stopped: asyncio.Event) -> bool: signal. A cancelled Runtime cannot rebuild its discovery clients, so the Video Gateway must let Kubernetes create a fresh process. """ + async def wait_runtime_shutdown() -> None: # PyO3's future_into_py returns an asyncio Future, while test doubles # and pure-Python runtimes may return a coroutine. Await it inside a @@ -75,7 +77,7 @@ async def run(args: argparse.Namespace) -> None: from dynamo.runtime.logging import configure_dynamo_logging configure_dynamo_logging() - config = load_config(args.config) + config = await run_file_io(load_config, args.config) if config.task_store.kind == "memory" and not args.allow_memory_store: raise RuntimeError("memory Task Store requires --allow-memory-store") adapters = {pool.pool_id: create_adapter(pool) for pool in config.pools} @@ -89,7 +91,7 @@ async def run(args: argparse.Namespace) -> None: runner: web.AppRunner | None = None try: clients = await create_pool_clients(runtime, config) - artifacts = FileArtifactStore(config.artifact_store.root) + artifacts = await run_file_io(FileArtifactStore, config.artifact_store.root) telemetry = GatewayTelemetry() if config.task_store.kind == "memory": store = MemoryTaskStore() diff --git a/dingo/video_gateway/adapters/base.py b/dingo/video_gateway/adapters/base.py index 45a3503dd000..c63b0850ae28 100644 --- a/dingo/video_gateway/adapters/base.py +++ b/dingo/video_gateway/adapters/base.py @@ -39,6 +39,7 @@ class WorkerVideoResult: output_format: str inference_time_s: float | None = None stage_durations: Mapping[str, float] | None = None + artifact: Mapping[str, Any] | None = None class WorkerStreamConsumer(Protocol): @@ -83,6 +84,12 @@ def validate_artifact( self, path: Path, normalized: Mapping[str, Any] ) -> dict[str, Any]: ... - def prepare_artifact( + def prepare_artifact(self, path: Path, normalized: Mapping[str, Any]) -> None: ... + + def artifact_requires_processing( + self, path: Path, normalized: Mapping[str, Any] + ) -> bool: ... + + def inspect_artifact_for_publication( self, path: Path, normalized: Mapping[str, Any] - ) -> None: ... + ) -> tuple[bool, dict[str, Any] | None]: ... diff --git a/dingo/video_gateway/adapters/minimax_h3.py b/dingo/video_gateway/adapters/minimax_h3.py index 13e4af44da7a..d83de3cf9e8b 100644 --- a/dingo/video_gateway/adapters/minimax_h3.py +++ b/dingo/video_gateway/adapters/minimax_h3.py @@ -13,10 +13,12 @@ import os import secrets from collections.abc import Mapping, Sequence +from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path from typing import Any +from dingo.common.video_result_file import validate_descriptor from dingo.video_gateway.adapters.base import UploadedArtifact, WorkerVideoResult from dingo.video_gateway.adapters.h3_shape import ( align_frame_count, @@ -43,6 +45,57 @@ "audio/vnd.wave": "audio/wav", } _GENERIC_BINARY_TYPES = {"", "application/octet-stream", "binary/octet-stream"} + + +@contextmanager +def _open_artifact_metadata(path: Path): + """Avoid pixel reconstruction while probing indexed H.264/AAC results. + + This is metadata validation, not a full bitstream/decode integrity scan. + Incomplete/unsupported metadata falls back to the original probe. Restore + normal decode settings before callers can use the frame-count fallback. + """ + import av + + container = None + try: + try: + container = av.open(str(path), options={"skip_frame": "all"}) + videos = list(container.streams.video) + audios = list(container.streams.audio) + complete = ( + len(videos) == 1 + and len(container.streams) == len(videos) + len(audios) + and videos[0].codec_context.name == "h264" + and bool(videos[0].codec_context.extradata) + and videos[0].width > 0 + and videos[0].height > 0 + and bool(videos[0].average_rate) + and int(videos[0].frames or 0) > 0 + and all( + a.codec_context.name == "aac" + and a.codec_context.extradata + and a.duration + and a.time_base + for a in audios + ) + ) + except av.error.FFmpegError: + complete = False + if not complete: + if container is not None: + container.close() + container = av.open(str(path)) + else: + for stream in container.streams: + stream.codec_context.options.pop("skip_frame", None) + stream.codec_context.skip_frame = "DEFAULT" + yield container + finally: + if container is not None: + container.close() + + _HEIF_BRANDS = {b"heic", b"heix", b"hevc", b"hevx", b"heim", b"heis", b"mif1", b"msf1"} _ALLOWED_FIELDS = { "model", @@ -258,8 +311,7 @@ def _iso_bmff_brands(header: bytes) -> set[bytes]: return set() brands = {header[8:12]} brands.update( - header[offset : offset + 4] - for offset in range(16, len(header) - 3, 4) + header[offset : offset + 4] for offset in range(16, len(header) - 3, 4) ) return brands @@ -397,14 +449,10 @@ def _probe_image( 400, "invalid_media", "image could not be decoded", upload.field_name ) from exc _validate_image_dimensions(width, height, upload) - return _MediaProbe( - "image", width=width, height=height, content_type=content_type - ) + return _MediaProbe("image", width=width, height=height, content_type=content_type) -def _probe_av( - upload: UploadedArtifact, kind: str, content_type: str -) -> _MediaProbe: +def _probe_av(upload: UploadedArtifact, kind: str, content_type: str) -> _MediaProbe: try: import av except ImportError as exc: @@ -447,9 +495,7 @@ def _probe_av( streams = list(container.streams.audio) if not streams: raise ValueError("missing audio stream") - _validate_reference_audio_codec( - streams[0].codec_context.name, upload - ) + _validate_reference_audio_codec(streams[0].codec_context.name, upload) if next(container.decode(streams[0]), None) is None: raise ValueError("audio stream has no decodable frames") return _MediaProbe( @@ -465,9 +511,7 @@ def _probe_av( ) from exc -def _probe_upload( - upload: UploadedArtifact, limits: _MiniMaxH3Limits -) -> _MediaProbe: +def _probe_upload(upload: UploadedArtifact, limits: _MiniMaxH3Limits) -> _MediaProbe: with upload.path.open("rb") as stream: header = stream.read(64) content_type = _canonical_content_type(upload, header) @@ -599,7 +643,13 @@ def finish(self) -> WorkerVideoResult: raise TypeError("Worker video data has an invalid shape") output_format = str(first.get("output_format") or "") b64_json = first.get("b64_json") - if output_format != "mp4" or not isinstance(b64_json, str) or not b64_json: + artifact = first.get("artifact") + if artifact is not None: + artifact = validate_descriptor(artifact) + if b64_json is not None or output_format != "mp4": + raise RuntimeError("ambiguous binary Worker result") + b64_json = "" + elif output_format != "mp4" or not isinstance(b64_json, str) or not b64_json: raise RuntimeError("Worker must return output_format=mp4 with b64_json") inference_time_raw = terminal.get("inference_time_s") inference_time = ( @@ -629,6 +679,7 @@ def finish(self) -> WorkerVideoResult: stage_durations[name] = duration return WorkerVideoResult( b64_json=b64_json, + artifact=artifact, output_format=output_format, inference_time_s=inference_time, stage_durations=stage_durations, @@ -705,9 +756,7 @@ def __init__(self, pool: PoolConfig) -> None: @property def max_encoded_reference_bytes(self) -> int: - return int( - self.options.get("max_encoded_reference_bytes", 384 * 1024 * 1024) - ) + return int(self.options.get("max_encoded_reference_bytes", 384 * 1024 * 1024)) def capabilities(self, *, max_result_bytes: int) -> dict[str, Any]: supports_reference_media = self.workflow == "ref2va" @@ -1271,14 +1320,58 @@ def consume_worker_stream(self, chunks: Sequence[Any]) -> WorkerVideoResult: consumer.consume(chunk) return consumer.finish() - def prepare_artifact( + def inspect_artifact_for_publication( self, path: Path, normalized: Mapping[str, Any] - ) -> None: + ) -> tuple[bool, dict[str, Any] | None]: + """Probe once; return reusable validation only for unchanged output. + + The caller may reuse this media result only when publishing the same + checked inode. Copies and transformed outputs must be validated anew. + """ + if bool(normalized.get("generate_sound", True)): + return False, self.validate_artifact(path, normalized) + try: + import av # noqa: F401 - fail early with the optional-extra diagnostic + except ImportError as exc: + raise RuntimeError( + "MiniMax-H3 media inspection requires the video-gateway optional extra" + ) from exc + with path.open("rb") as stream: + header = stream.read(32) + with _open_artifact_metadata(path) as container: + if not ( + len(container.streams) == 1 + and len(container.streams.video) == 1 + and len(header) >= 12 + and header[4:8] == b"ftyp" + ): + return True, None + return False, self._validate_open_artifact(container, normalized) + + def artifact_requires_processing( + self, path: Path, normalized: Mapping[str, Any] + ) -> bool: + if bool(normalized.get("generate_sound", True)): + return False + import av + + with av.open(str(path)) as source: + with path.open("rb") as stream: + header = stream.read(12) + return not ( + len(source.streams) == 1 + and len(source.streams.video) == 1 + and len(header) == 12 + and header[4:8] == b"ftyp" + ) + + def prepare_artifact(self, path: Path, normalized: Mapping[str, Any]) -> None: """Match vLLM-Omni's generate_sound=false response semantics. MiniMax-H3 always computes the audio branch in the currently supported Worker path. Remuxing only the H.264 packets avoids a second video encode - and therefore does not change image quality. + and therefore does not change image quality. An already video-only MP4 + needs no rewrite; the normal artifact validator still runs afterwards. """ if bool(normalized.get("generate_sound", True)): @@ -1290,12 +1383,21 @@ def prepare_artifact( "MiniMax-H3 audio stripping requires the video-gateway optional extra" ) from exc remuxed = path.with_name(f"{path.name}.video-only.mp4") + remux_started = False try: with av.open(str(path)) as source: video_streams = list(source.streams.video) if not video_streams: raise RuntimeError("Worker MP4 is missing a video stream") + # Preserve the old stripping behavior for additional streams + # and non-MP4 inputs. Do not bypass the caller's media validator. + if len(source.streams) == 1: + with path.open("rb") as stream: + header = stream.read(12) + if len(header) == 12 and header[4:8] == b"ftyp": + return source_video = video_streams[0] + remux_started = True with av.open(str(remuxed), mode="w", format="mp4") as output: output_video = output.add_stream_from_template(source_video) for packet in source.demux(source_video): @@ -1307,7 +1409,7 @@ def prepare_artifact( os.fsync(stream.fileno()) os.replace(remuxed, path) finally: - if remuxed.exists(): + if remux_started and remuxed.exists(): remuxed.unlink() def validate_artifact( @@ -1320,71 +1422,75 @@ def validate_artifact( if not bool(self.options.get("validate_media", True)): return {"container": "mp4"} try: - import av + import av # noqa: F401 - fail early with the optional-extra diagnostic except ImportError as exc: raise RuntimeError( "MiniMax-H3 media validation requires the video-gateway optional extra" ) from exc - with av.open(str(path)) as container: - video_streams = list(container.streams.video) - audio_streams = list(container.streams.audio) - if not video_streams or video_streams[0].codec_context.name != "h264": - raise RuntimeError("MP4 must contain an H.264 video stream") - generate_sound = bool(normalized.get("generate_sound", True)) - if generate_sound: - if not audio_streams or audio_streams[0].codec_context.name != "aac": - raise RuntimeError("MP4 must contain an AAC audio stream") - elif audio_streams: - raise RuntimeError("MP4 must not contain audio when generate_sound=false") - video = video_streams[0] - if ( - video.width != normalized["width"] - or video.height != normalized["height"] - ): - raise RuntimeError( - f"MP4 dimensions {video.width}x{video.height} do not match request" - ) - average_rate = float(video.average_rate) if video.average_rate else 0.0 - if abs(average_rate - 24.0) > 0.05: - raise RuntimeError( - f"MP4 frame rate {average_rate} does not match 24 fps" - ) - frame_count = int(video.frames or 0) - if frame_count <= 0: - frame_count = sum(1 for _frame in container.decode(video=0)) - expected_frames = align_frame_count(int(normalized["num_frames"])) - if frame_count != expected_frames: + with _open_artifact_metadata(path) as container: + return self._validate_open_artifact(container, normalized) + + def _validate_open_artifact( + self, container, normalized: Mapping[str, Any] + ) -> dict[str, Any]: + if not bool(self.options.get("validate_media", True)): + return {"container": "mp4"} + import av + + video_streams = list(container.streams.video) + audio_streams = list(container.streams.audio) + if not video_streams or video_streams[0].codec_context.name != "h264": + raise RuntimeError("MP4 must contain an H.264 video stream") + generate_sound = bool(normalized.get("generate_sound", True)) + if generate_sound: + if not audio_streams or audio_streams[0].codec_context.name != "aac": + raise RuntimeError("MP4 must contain an AAC audio stream") + elif audio_streams: + raise RuntimeError("MP4 must not contain audio when generate_sound=false") + video = video_streams[0] + if video.width != normalized["width"] or video.height != normalized["height"]: + raise RuntimeError( + f"MP4 dimensions {video.width}x{video.height} do not match request" + ) + average_rate = float(video.average_rate) if video.average_rate else 0.0 + if abs(average_rate - 24.0) > 0.05: + raise RuntimeError(f"MP4 frame rate {average_rate} does not match 24 fps") + frame_count = int(video.frames or 0) + if frame_count <= 0: + frame_count = sum(1 for _frame in container.decode(video=0)) + expected_frames = align_frame_count(int(normalized["num_frames"])) + if frame_count != expected_frames: + raise RuntimeError( + f"MP4 frame count {frame_count} does not match expected " + f"MiniMax-H3 aligned count {expected_frames}" + ) + video_duration = frame_count / average_rate + container_duration = ( + float(container.duration / av.time_base) + if container.duration is not None + else 0.0 + ) + audio_duration: float | None = None + if generate_sound: + audio = audio_streams[0] + audio_duration = _stream_duration(container, audio) + if audio_duration <= 0: + raise RuntimeError("MP4 AAC stream has no measurable duration") + if abs(video_duration - audio_duration) > 0.1: raise RuntimeError( - f"MP4 frame count {frame_count} does not match expected " - f"MiniMax-H3 aligned count {expected_frames}" + "MP4 audio/video duration difference exceeds 100 milliseconds" ) - video_duration = frame_count / average_rate - container_duration = ( - float(container.duration / av.time_base) - if container.duration is not None - else 0.0 - ) - audio_duration: float | None = None - if generate_sound: - audio = audio_streams[0] - audio_duration = _stream_duration(container, audio) - if audio_duration <= 0: - raise RuntimeError("MP4 AAC stream has no measurable duration") - if abs(video_duration - audio_duration) > 0.1: - raise RuntimeError( - "MP4 audio/video duration difference exceeds 100 milliseconds" - ) - return { - "container": "mp4", - "video_codec": "h264", - "audio_codec": "aac" if generate_sound else None, - "width": video.width, - "height": video.height, - "fps": average_rate, - "frames": frame_count, - "duration_s": max( - video_duration, audio_duration or 0.0, container_duration - ), - "video_duration_s": video_duration, - "audio_duration_s": audio_duration, - } + return { + "container": "mp4", + "video_codec": "h264", + "audio_codec": "aac" if generate_sound else None, + "width": video.width, + "height": video.height, + "fps": average_rate, + "frames": frame_count, + "duration_s": max( + video_duration, audio_duration or 0.0, container_duration + ), + "video_duration_s": video_duration, + "audio_duration_s": audio_duration, + } diff --git a/dingo/video_gateway/api.py b/dingo/video_gateway/api.py index 6e9f212ff585..5cd758dbaa0b 100644 --- a/dingo/video_gateway/api.py +++ b/dingo/video_gateway/api.py @@ -5,7 +5,6 @@ from __future__ import annotations -import asyncio import ipaddress import logging import os @@ -16,6 +15,7 @@ from aiohttp import web from dingo.video_gateway.errors import GatewayError, StoreUnavailable +from dingo.video_gateway.file_io import opened_file, run_file_io from dingo.video_gateway.form_parser import parse_multipart from dingo.video_gateway.models import TERMINAL_STATUSES, TaskStatus from dingo.video_gateway.service import VideoGatewayService @@ -166,7 +166,9 @@ async def _submit(request: web.Request, *, delivery_mode: str): service = _service(request) if service.dispatcher.draining: raise _gateway_draining_error() - anticipated_input = request.content_length or service.config.media.max_total_file_bytes + anticipated_input = ( + request.content_length or service.config.media.max_total_file_bytes + ) await service.ensure_submission_capacity( anticipated_input + service.config.media.max_result_bytes ) @@ -328,26 +330,27 @@ async def _content_response( if task.status in {TaskStatus.FAILED, TaskStatus.CANCELLED}: raise GatewayError(422, "video_generation_failed", task.status.value) raise GatewayError(409, "video_not_ready", "video result is not ready") - try: - path = service.artifacts.result_path(task.result_path) - except (FileNotFoundError, RuntimeError) as exc: - raise GatewayError( - 410, "video_expired", "video result artifact is unavailable" - ) from exc if not task.result_sha256: raise GatewayError(410, "video_expired", "video result checksum is unavailable") flags = os.O_RDONLY | getattr(os, "O_CLOEXEC", 0) | getattr(os, "O_NOFOLLOW", 0) - try: - descriptor = os.open(path, flags) - except OSError as exc: - raise GatewayError( - 410, "video_expired", "video result artifact is unavailable" - ) from exc - stream = os.fdopen(descriptor, "rb", closefd=True) - try: - metadata = os.fstat(descriptor) + def _open(): + try: + path = service.artifacts.result_path(task.result_path) + descriptor = os.open(path, flags) + except (OSError, RuntimeError) as exc: + raise GatewayError( + 410, "video_expired", "video result artifact is unavailable" + ) from exc + try: + return os.fdopen(descriptor, "rb", closefd=True) + except BaseException: + os.close(descriptor) + raise + + async with opened_file(_open) as stream: + metadata = await run_file_io(os.fstat, stream.fileno()) if not stat.S_ISREG(metadata.st_mode): raise GatewayError( 410, "video_expired", "video result artifact is unavailable" @@ -373,12 +376,10 @@ async def _content_response( if media.get("frames") is not None: common_headers["X-Video-Frames"] = str(media["frames"]) if media.get("fps") is not None: - common_headers["X-Video-FPS"] = f'{float(media["fps"]):.9g}' + common_headers["X-Video-FPS"] = f"{float(media['fps']):.9g}" duration = media.get("duration_s", media.get("video_duration_s")) if duration is not None: - common_headers["X-Video-Duration-Seconds"] = ( - f"{float(duration):.9g}" - ) + common_headers["X-Video-Duration-Seconds"] = f"{float(duration):.9g}" if _etag_matches(request.headers.get("If-None-Match"), etag): return web.Response(status=304, headers=common_headers) @@ -399,20 +400,16 @@ async def _content_response( response = web.StreamResponse(status=206 if partial else 200, headers=headers) await response.prepare(request) if request.method != "HEAD" and length: - await asyncio.to_thread(stream.seek, start) + await run_file_io(stream.seek, start) remaining = length while remaining: - chunk = await asyncio.to_thread( - stream.read, min(1024 * 1024, remaining) - ) + chunk = await run_file_io(stream.read, min(1024 * 1024, remaining)) if not chunk: raise ConnectionError("video artifact ended during download") await response.write(chunk) remaining -= len(chunk) await response.write_eof() return response - finally: - stream.close() async def get_video_content(request: web.Request) -> web.StreamResponse: @@ -494,6 +491,7 @@ async def metrics(request: web.Request) -> web.Response: "# TYPE dingo_video_worker_busy gauge", "# TYPE dingo_video_queue_depth gauge", "# TYPE dingo_video_tasks gauge", + "# TYPE dingo_video_finalization_pending_local gauge", "# TYPE dingo_video_media_memory_budget_bytes gauge", "# TYPE dingo_video_media_memory_used_bytes gauge", "# TYPE dingo_video_media_memory_peak_bytes gauge", @@ -525,8 +523,7 @@ async def metrics(request: web.Request) -> web.Response: f"{media.legacy_input_encoded_bytes}", "dingo_video_media_legacy_output_encoded_bytes_total " f"{media.legacy_output_encoded_bytes}", - "dingo_video_media_payload_build_seconds_total " - f"{media.payload_build_seconds}", + f"dingo_video_media_payload_build_seconds_total {media.payload_build_seconds}", f"dingo_video_media_payload_build_total {media.payload_build_count}", f"dingo_video_media_finalize_seconds_total {media.finalize_seconds}", f"dingo_video_media_finalize_total {media.finalize_count}", @@ -536,25 +533,63 @@ async def metrics(request: web.Request) -> web.Response: f"dingo_video_artifact_total_bytes {capacity.total_bytes}", f"dingo_video_artifact_free_bytes {capacity.free_bytes}", f"dingo_video_artifact_sweep_due_tasks {artifact.sweep_due_tasks}", - "dingo_video_artifact_expired_tasks_total " - f"{artifact.expired_tasks_total}", + f"dingo_video_artifact_expired_tasks_total {artifact.expired_tasks_total}", "dingo_video_artifact_orphan_candidates_total " f"{artifact.orphan_candidates_total}", - "dingo_video_artifact_orphan_trashed_total " - f"{artifact.orphan_trashed_total}", + f"dingo_video_artifact_orphan_trashed_total {artifact.orphan_trashed_total}", "dingo_video_artifact_cleanup_failures_total " f"{artifact.cleanup_failures_total}", - "dingo_video_artifact_released_bytes_total " - f"{artifact.released_bytes_total}", + f"dingo_video_artifact_released_bytes_total {artifact.released_bytes_total}", ] for pool in service.config.pools: - workers = len(service.dispatcher.pool_instances(pool.pool_id)) queue = await service.store.queue_depth(pool.pool_id) + # Dispatch may skip all queue/counter reads when no Worker is free. + # Refresh these gauges at scrape time instead of exposing stale values + # merely because the pool is saturated or has no registered Workers. + if hasattr(service.store, "retry_budget_used"): + waiting = await service.store.retry_queue_depth(pool.pool_id) + credits = await service.store.retry_budget_used(pool.pool_id) + service.telemetry.set_gauge( + "dingo_video_retry_waiting_tasks", + waiting, + labels={"pool": pool.pool_id}, + ) + service.telemetry.set_gauge( + "dingo_video_retry_credits_used", + credits, + labels={"pool": pool.pool_id}, + ) + service.telemetry.set_gauge( + "dingo_video_normal_queue_depth", + max(0, queue - waiting), + labels={"pool": pool.pool_id}, + ) leases = await service.dispatcher.pool_leases(pool.pool_id) - busy = sum(lease.state != "quarantined" for lease in leases) + worker_capacity = service.dispatcher.pool_capacity_snapshot( + pool.pool_id, leases + ) + scheduling = service.dispatcher.pools[pool.pool_id].config.scheduling + worker_capacity.update( + { + "worker_execution_capacity_configured": scheduling.worker_capacity, + "worker_prefetch_capacity_configured": scheduling.worker_prefetch_capacity, + "early_release_slot_enabled": int(scheduling.early_release_slot), + "finalization_concurrency_configured": scheduling.finalization_concurrency, + "finalization_pending_limit_configured": scheduling.finalization_pending_limit, + } + ) counts = await service.store.task_counts(pool.pool_id) - lines.append(f'dingo_video_workers{{pool="{pool.pool_id}"}} {workers}') - lines.append(f'dingo_video_worker_busy{{pool="{pool.pool_id}"}} {busy}') + for name, value in worker_capacity.items(): + if pool is service.config.pools[0] and name not in { + "workers", + "worker_busy", + }: + lines.append(f"# TYPE dingo_video_{name} gauge") + lines.append(f'dingo_video_{name}{{pool="{pool.pool_id}"}} {value}') + lines.append( + f'dingo_video_finalization_pending_local{{pool="{pool.pool_id}"}} ' + f"{service.dispatcher.pool_finalization_pending(pool.pool_id)}" + ) lines.append(f'dingo_video_queue_depth{{pool="{pool.pool_id}"}} {queue}') lines.extend( f'dingo_video_tasks{{pool="{pool.pool_id}",status="{status.value}"}} ' diff --git a/dingo/video_gateway/artifact_store.py b/dingo/video_gateway/artifact_store.py index 3ef71831d513..356b96e1bcae 100644 --- a/dingo/video_gateway/artifact_store.py +++ b/dingo/video_gateway/artifact_store.py @@ -8,11 +8,13 @@ import asyncio import base64 import binascii +import errno import hashlib import json import os import re import shutil +import stat import time import uuid from collections.abc import Callable, Mapping @@ -20,8 +22,10 @@ from pathlib import Path from typing import Any +from dingo.common.video_result_file import validate_descriptor from dingo.common.video_task_protocol import detached_attempt_root from dingo.video_gateway.errors import ResultTooLarge +from dingo.video_gateway.file_io import run_cancellable_file_io, run_file_io @dataclass(frozen=True, slots=True) @@ -57,7 +61,9 @@ def _contained(self, path: Path) -> Path: return resolved def _lexically_contained(self, path: Path) -> Path: - absolute = path.absolute() + # Lexical normalization performs no filesystem I/O. Keep the separate + # realpath/symlink checks at the operation boundary in a worker thread. + absolute = Path(os.path.abspath(path)) if absolute != self.root and self.root not in absolute.parents: raise RuntimeError(f"artifact path escaped configured root: {absolute}") return absolute @@ -84,14 +90,17 @@ def _probe() -> None: os.fsync(stream.fileno()) probe.unlink() - await asyncio.to_thread(_probe) + await run_file_io(_probe) async def create_upload(self) -> Path: path = self.upload_root / uuid.uuid4().hex - await asyncio.to_thread(path.mkdir, 0o700, True, False) - inputs = path / "inputs" - await asyncio.to_thread(inputs.mkdir, 0o700, False, False) - return self._contained(path) + + def _create() -> Path: + path.mkdir(0o700, True, False) + (path / "inputs").mkdir(0o700, False, False) + return self._contained(path) + + return await run_file_io(_create) @staticmethod def _tree_size(path: Path) -> int: @@ -114,33 +123,30 @@ def _tree_size(path: Path) -> int: return total async def discard(self, path: Path) -> int: - target = self._lexically_contained(path) - if target.is_symlink(): - try: - await asyncio.to_thread(target.unlink) - except FileNotFoundError: - # Concurrent idempotent DELETE/sweeper cleanup won the race. - pass - return 0 - elif target.exists(): - resolved = self._contained(target) - size = await asyncio.to_thread(self._tree_size, resolved) - - def _remove_tree() -> bool: + def _discard() -> int: + target = self._lexically_contained(path) + if target == self.root: + raise RuntimeError("refusing to discard artifact root") + # Validate parents before unlinking a leaf symlink; never follow + # an escaped parent directory into another deployment's files. + self._contained(target.parent) + if target.is_symlink(): + target.unlink(missing_ok=True) + return 0 + if target.exists(): + resolved = self._contained(target) + size = self._tree_size(resolved) try: shutil.rmtree(resolved) except FileNotFoundError: - # rmtree may observe a concurrently removed child even - # when the root existed at the containment check above. - return False - return True + return 0 + return size + return 0 - removed = await asyncio.to_thread(_remove_tree) - return size if removed else 0 - return 0 + return await run_file_io(_discard) async def capacity(self) -> ArtifactCapacity: - usage = await asyncio.to_thread(shutil.disk_usage, self.root) + usage = await run_file_io(shutil.disk_usage, self.root) return ArtifactCapacity(usage.total, usage.used, usage.free) async def cleanup_orphan_uploads(self, *, minimum_age_s: float = 3600.0) -> int: @@ -166,13 +172,19 @@ def _cleanup() -> int: removed += 1 return removed - return await asyncio.to_thread(_cleanup) + return await run_file_io(_cleanup) def task_root(self, deployment_id: str, pool_id: str, task_id: str) -> Path: + """Synchronous checked path API; async callers use resolve_task_root.""" return self._contained( self.root / deployment_id / "v1" / "pools" / pool_id / "tasks" / task_id ) + async def resolve_task_root( + self, deployment_id: str, pool_id: str, task_id: str + ) -> Path: + return await run_file_io(self.task_root, deployment_id, pool_id, task_id) + async def commit_upload( self, upload_root: Path, @@ -182,12 +194,18 @@ async def commit_upload( *, artifact_manifest: Mapping[str, Any] | None = None, ) -> Path: - source = self._contained(upload_root) - target = self.task_root(deployment_id, pool_id, task_id) - await asyncio.to_thread(target.parent.mkdir, 0o750, True, True) - if target.exists(): - raise FileExistsError(f"task artifact directory already exists: {task_id}") - await asyncio.to_thread(os.replace, source, target) + def _commit() -> Path: + source = self._contained(upload_root) + target = self.task_root(deployment_id, pool_id, task_id) + target.parent.mkdir(0o750, True, True) + if target.exists(): + raise FileExistsError( + f"task artifact directory already exists: {task_id}" + ) + os.replace(source, target) + return target + + target = await run_file_io(_commit) if artifact_manifest is not None: try: await self.write_json(target / "_artifact.json", artifact_manifest) @@ -204,19 +222,14 @@ async def orphan_task_candidates( minimum_age_s: float, ) -> list[TaskArtifactCandidate]: cutoff = time.time() - minimum_age_s - root_device = (await asyncio.to_thread(self.root.stat)).st_dev + root_device = (await run_file_io(self.root.stat)).st_dev def _scan() -> list[TaskArtifactCandidate]: candidates: list[TaskArtifactCandidate] = [] for pool_id in pool_ids: try: tasks_root = self._symlink_free( - self.root - / deployment_id - / "v1" - / "pools" - / pool_id - / "tasks" + self.root / deployment_id / "v1" / "pools" / pool_id / "tasks" ) except (FileNotFoundError, RuntimeError): continue @@ -228,7 +241,9 @@ def _scan() -> list[TaskArtifactCandidate]: metadata = entry.stat(follow_symlinks=False) except FileNotFoundError: continue - if entry.is_symlink() or not entry.is_dir(follow_symlinks=False): + if entry.is_symlink() or not entry.is_dir( + follow_symlinks=False + ): continue if metadata.st_dev != root_device or metadata.st_mtime > cutoff: continue @@ -263,7 +278,7 @@ def _scan() -> list[TaskArtifactCandidate]: ) return candidates - return await asyncio.to_thread(_scan) + return await run_file_io(_scan) async def trash_orphan( self, candidate: TaskArtifactCandidate, *, dry_run: bool = False @@ -273,30 +288,32 @@ async def trash_orphan( raise RuntimeError("orphan candidate is not a task directory") if dry_run: return source - if not source.exists() and not source.is_symlink(): - return None - source_parent = self._symlink_free(source.parent) target = self.trash_root / f"{uuid.uuid4().hex}-{candidate.task_id}" - def _move() -> None: + def _move() -> Path | None: + if not source.exists() and not source.is_symlink(): + return None + source_parent = self._symlink_free(source.parent) directory_flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) nofollow = getattr(os, "O_NOFOLLOW", 0) source_fd = os.open(source_parent, directory_flags | nofollow) - trash_fd = os.open(self.trash_root, directory_flags | nofollow) try: - os.rename( - source.name, - target.name, - src_dir_fd=source_fd, - dst_dir_fd=trash_fd, - ) + trash_fd = os.open(self.trash_root, directory_flags | nofollow) + try: + os.rename( + source.name, + target.name, + src_dir_fd=source_fd, + dst_dir_fd=trash_fd, + ) + finally: + os.close(trash_fd) finally: os.close(source_fd) - os.close(trash_fd) os.utime(target, None, follow_symlinks=False) + return self._contained(target) - await asyncio.to_thread(_move) - return self._contained(target) + return await run_file_io(_move) async def cleanup_trash(self, *, minimum_age_s: float) -> tuple[int, int]: cutoff = time.time() - minimum_age_s @@ -324,33 +341,34 @@ def _cleanup() -> tuple[int, int]: removed += 1 return removed, released - return await asyncio.to_thread(_cleanup) + return await run_file_io(_cleanup) async def write_json(self, path: Path, value: Any) -> None: - target = self._contained(path) - await asyncio.to_thread(target.parent.mkdir, 0o750, True, True) - payload = json.dumps( - value, ensure_ascii=False, sort_keys=True, separators=(",", ":") - ).encode("utf-8") - temporary = target.with_name(target.name + ".part-" + uuid.uuid4().hex) - def _write() -> None: - with temporary.open("xb") as stream: - stream.write(payload) - stream.flush() - os.fsync(stream.fileno()) - os.replace(temporary, target) + target = self._contained(path) + target.parent.mkdir(0o750, True, True) + payload = json.dumps( + value, ensure_ascii=False, sort_keys=True, separators=(",", ":") + ).encode("utf-8") + temporary = target.with_name(target.name + ".part-" + uuid.uuid4().hex) + try: + with temporary.open("xb") as stream: + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + os.replace(temporary, target) + finally: + temporary.unlink(missing_ok=True) - await asyncio.to_thread(_write) + await run_file_io(_write) async def read_json(self, path: str | Path) -> Any: - target = self._contained(Path(path)) - def _read() -> Any: + target = self._contained(Path(path)) with target.open("r", encoding="utf-8") as stream: return json.load(stream) - return await asyncio.to_thread(_read) + return await run_file_io(_read) def detached_attempt_root( self, @@ -378,12 +396,11 @@ async def read_detached_status( attempt: int, execution_token: str, ) -> dict[str, Any] | None: - attempt_root = self.detached_attempt_root( - deployment_id, pool_id, task_id, attempt, execution_token - ) - path = attempt_root / "worker-status.json" - def _read() -> dict[str, Any] | None: + attempt_root = self.detached_attempt_root( + deployment_id, pool_id, task_id, attempt, execution_token + ) + path = attempt_root / "worker-status.json" try: if path.is_symlink(): raise RuntimeError("detached Worker status is a symlink") @@ -411,7 +428,7 @@ def _read() -> dict[str, Any] | None: raise RuntimeError("unsupported detached Worker status schema") return value - return await asyncio.to_thread(_read) + return await run_file_io(_read) async def request_detached_cancel( self, @@ -421,17 +438,16 @@ async def request_detached_cancel( attempt: int, execution_token: str, ) -> None: - attempt_root = self.detached_attempt_root( - deployment_id, pool_id, task_id, attempt, execution_token - ) - path = attempt_root / "cancel.requested" - def _write() -> None: + attempt_root = self.detached_attempt_root( + deployment_id, pool_id, task_id, attempt, execution_token + ) + path = attempt_root / "cancel.requested" attempt_root.mkdir(mode=0o750, parents=True, exist_ok=True) descriptor = os.open(path, os.O_WRONLY | os.O_CREAT, 0o600) os.close(descriptor) - await asyncio.to_thread(_write) + await run_file_io(_write) async def consume_detached_response( self, @@ -445,12 +461,11 @@ async def consume_detached_response( expected_sha256: str, max_response_bytes: int, ) -> int: - attempt_root = self.detached_attempt_root( - deployment_id, pool_id, task_id, attempt, execution_token - ) - path = attempt_root / "worker-response.jsonl" - def _consume() -> int: + attempt_root = self.detached_attempt_root( + deployment_id, pool_id, task_id, attempt, execution_token + ) + path = attempt_root / "worker-response.jsonl" if path.is_symlink() or not path.is_file(): raise RuntimeError("detached Worker response is not a regular file") digest = hashlib.sha256() @@ -486,7 +501,148 @@ def _consume() -> int: raise RuntimeError("detached Worker response checksum mismatch") return consumed - return await asyncio.to_thread(_consume) + return await run_file_io(_consume) + + async def finalize_worker_mp4( + self, + deployment_id, + pool_id, + task_id, + attempt, + execution_token, + descriptor, + normalized, + validator, + processor, + requires_processing, + *, + max_result_bytes=128 * 1024 * 1024, + inspector=None, + ): + descriptor = validate_descriptor(dict(descriptor)) + if descriptor["bytes"] > max_result_bytes: + raise ResultTooLarge("Worker binary result exceeds configured maximum") + final = None + + def finalize(cancelled): + nonlocal final + + def check(): + if cancelled.is_set(): + raise asyncio.CancelledError + + check() + attempt_root = self.detached_attempt_root( + deployment_id, pool_id, task_id, attempt, execution_token + ) + source = attempt_root / descriptor["filename"] + try: + verified = source.lstat() + except FileNotFoundError as exc: + raise RuntimeError( + "Worker binary result is not a regular file" + ) from exc + if not stat.S_ISREG(verified.st_mode): + raise RuntimeError("Worker binary result is not a regular file") + size = verified.st_size + if size != descriptor["bytes"]: + raise RuntimeError("Worker binary result size mismatch") + # Trusted Worker/shared-store handoff: retain metadata/media checks, + # but do not reread the whole unmodified MP4 to repeat its digest. + # This deliberately does not detect same-size content corruption. + result_sha256 = descriptor["sha256"] + root = self.task_root(deployment_id, pool_id, task_id) + result_dir = root / "result" + result_dir.mkdir(mode=0o750, parents=True, exist_ok=True) + final = result_dir / f"video-a{attempt}-{uuid.uuid4().hex}.mp4" + check() + inspected_media = None + if inspector is None: + processing = requires_processing(source, normalized) + else: + processing, inspected_media = inspector(source, normalized) + current = source.lstat() + if ( + current.st_dev, + current.st_ino, + current.st_size, + current.st_mtime_ns, + ) != ( + verified.st_dev, + verified.st_ino, + verified.st_size, + verified.st_mtime_ns, + ): + raise RuntimeError("Worker binary result changed during validation") + copied = processing + if processing: + shutil.copyfile(source, final) + processor(final, normalized) + else: + try: + os.link(source, final, follow_symlinks=False) + linked = final.lstat() + if (linked.st_dev, linked.st_ino) != ( + verified.st_dev, + verified.st_ino, + ): + raise RuntimeError( + "Worker binary result changed before publication" + ) + except OSError as exc: + if exc.errno not in {errno.EXDEV, errno.EOPNOTSUPP, errno.ENOSYS}: + raise + shutil.copyfile(source, final) + copied = True + check() + media = ( + inspected_media + if not copied and inspected_media is not None + else validator(final, normalized) + ) + if processing: + digest = hashlib.sha256() + size = 0 + with final.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + check() + size += len(chunk) + if size > max_result_bytes: + raise ResultTooLarge( + "processed Worker result exceeds maximum" + ) + digest.update(chunk) + result_sha256 = digest.hexdigest() + published = final.lstat() + if not stat.S_ISREG(published.st_mode) or published.st_size != size: + raise RuntimeError("published Worker result size/type changed") + # DingoFS updates mtime when creating a hardlink. Compare against + # the post-link snapshot, not the pre-link Worker-file timestamp. + if not copied and ( + published.st_dev, + published.st_ino, + published.st_mtime_ns, + ) != (linked.st_dev, linked.st_ino, linked.st_mtime_ns): + raise RuntimeError( + "Worker binary result changed during media validation" + ) + if copied: + with final.open("rb") as stream: + os.fsync(stream.fileno()) + check() + directory = os.open(result_dir, os.O_RDONLY | os.O_DIRECTORY) + try: + os.fsync(directory) + finally: + os.close(directory) + return final, size, result_sha256, media + + try: + return await run_cancellable_file_io(finalize) + except BaseException: + if final is not None: + await run_file_io(final.unlink, missing_ok=True) + raise async def finalize_b64_mp4( self, @@ -499,70 +655,86 @@ async def finalize_b64_mp4( max_result_bytes: int = 128 * 1024 * 1024, publication_scope: str | None = None, ) -> tuple[Path, int, str, dict[str, Any]]: - root = self._contained(task_root) - result_dir = root / "result" - temporary_dir = root / "tmp" - await asyncio.to_thread(result_dir.mkdir, 0o750, True, True) - await asyncio.to_thread(temporary_dir.mkdir, 0o750, True, True) estimated = (len(b64_json) // 4) * 3 if estimated > max_result_bytes: - raise ResultTooLarge( - "Worker base64 result exceeds configured maximum" - ) + raise ResultTooLarge("Worker base64 result exceeds configured maximum") if len(b64_json) % 4: raise RuntimeError("Worker base64 result has invalid padding length") - temporary = temporary_dir / f"result-{uuid.uuid4().hex}.part" scope = publication_scope or "legacy" if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", scope) is None: raise ValueError("result publication scope is invalid") - # The task CAS publishes one unique candidate. A stale Gateway can - # safely unlink its own losing candidate without touching the winner. - final = result_dir / f"video-{scope}-{uuid.uuid4().hex}.mp4" - - def _decode() -> tuple[int, str]: - digest = hashlib.sha256() - written = 0 - chunk_chars = 4 * 1024 * 1024 - with temporary.open("xb") as output: - for offset in range(0, len(b64_json), chunk_chars): - encoded = b64_json[offset : offset + chunk_chars] - try: - decoded = base64.b64decode(encoded, validate=True) - except (binascii.Error, ValueError) as exc: - raise RuntimeError( - "Worker returned invalid base64 video data" - ) from exc - written += len(decoded) - if written > max_result_bytes: - raise ResultTooLarge( - "Worker result exceeds configured maximum" - ) - digest.update(decoded) - output.write(decoded) - output.flush() - os.fsync(output.fileno()) - return written, digest.hexdigest() - - def _digest() -> tuple[int, str]: - digest = hashlib.sha256() - size = 0 - with temporary.open("rb") as stream: - for chunk in iter(lambda: stream.read(4 * 1024 * 1024), b""): - size += len(chunk) - digest.update(chunk) - return size, digest.hexdigest() + final: Path | None = None + + def _finalize(cancelled) -> tuple[Path, int, str, dict[str, Any]]: + nonlocal final + + def check_cancelled() -> None: + if cancelled.is_set(): + raise asyncio.CancelledError + + check_cancelled() + root = self._contained(task_root) + result_dir = root / "result" + temporary_dir = root / "tmp" + result_dir.mkdir(mode=0o750, parents=True, exist_ok=True) + temporary_dir.mkdir(mode=0o750, parents=True, exist_ok=True) + temporary = temporary_dir / f"result-{uuid.uuid4().hex}.part" + # Only the later etcd CAS publishes this unique candidate. Neither + # cancellation nor a stale owner may remove another owner's file. + final = result_dir / f"video-{scope}-{uuid.uuid4().hex}.mp4" + try: + check_cancelled() + written = 0 + chunk_chars = 4 * 1024 * 1024 + with temporary.open("xb") as output: + for offset in range(0, len(b64_json), chunk_chars): + check_cancelled() + encoded = b64_json[offset : offset + chunk_chars] + try: + decoded = base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError) as exc: + raise RuntimeError( + "Worker returned invalid base64 video data" + ) from exc + written += len(decoded) + if written > max_result_bytes: + raise ResultTooLarge( + "Worker result exceeds configured maximum" + ) + output.write(decoded) + output.flush() + os.fsync(output.fileno()) + check_cancelled() + if processor is not None: + processor(temporary, normalized) + check_cancelled() + media = validator(temporary, normalized) + check_cancelled() + digest = hashlib.sha256() + size = 0 + with temporary.open("rb") as stream: + for chunk in iter(lambda: stream.read(4 * 1024 * 1024), b""): + check_cancelled() + size += len(chunk) + digest.update(chunk) + check_cancelled() + os.replace(temporary, final) + return final, size, digest.hexdigest(), media + finally: + temporary.unlink(missing_ok=True) + # One bounded off-loop pipeline, rather than a separate thread/loop + # round trip for each dependent file operation. The media memory budget + # remains held by the caller until this operation has fully drained. try: - await asyncio.to_thread(_decode) - if processor is not None: - await asyncio.to_thread(processor, temporary, normalized) - media = await asyncio.to_thread(validator, temporary, normalized) - size, sha256 = await asyncio.to_thread(_digest) - await asyncio.to_thread(os.replace, temporary, final) - return final, size, sha256, media - finally: - if temporary.exists(): - await asyncio.to_thread(temporary.unlink) + return await run_cancellable_file_io(_finalize) + except BaseException: + # Cancellation can arrive after rename but before the thread result + # is delivered. The thread is drained here; this unpublished file + # belongs only to this invocation and is now safe to remove. + if final is not None: + await run_file_io(final.unlink, missing_ok=True) + raise def result_path(self, path: str | Path) -> Path: original = self._lexically_contained(Path(path)) diff --git a/dingo/video_gateway/config.py b/dingo/video_gateway/config.py index 011fff6ff71a..35f24eb13393 100644 --- a/dingo/video_gateway/config.py +++ b/dingo/video_gateway/config.py @@ -130,10 +130,7 @@ def result_task_memory_bytes(self) -> int: def max_task_memory_bytes(self) -> int: # Mixed-reference construction briefly holds both the individual data # URLs and their final JSON envelope. - return ( - 2 * self.max_encoded_reference_bytes - + self.result_task_memory_bytes - ) + return 2 * self.max_encoded_reference_bytes + self.result_task_memory_bytes @dataclass(frozen=True, slots=True) @@ -173,6 +170,13 @@ class LifecycleConfig: @dataclass(frozen=True, slots=True) class SchedulingConfig: worker_capacity: int = 1 + worker_prefetch_capacity: int = 0 + early_release_slot: bool = False + finalization_timeout_s: float = 60.0 + finalization_max_retries: int = 2 + finalization_retry_delay_s: float = 0.1 + finalization_concurrency: int = 4 + finalization_pending_limit: int = 64 queue_limit: int = 32 accept_without_workers: bool = False execution_timeout_s: float = 1800.0 @@ -245,9 +249,7 @@ def _runtime_config(raw: Any) -> RuntimeConfig: ) watchdog_enabled = _boolean(watchdog_data, "enabled", False) watchdog_interval_s = float(watchdog_data.get("interval_s", 2.0)) - watchdog_mismatch_grace_s = float( - watchdog_data.get("mismatch_grace_s", 6.0) - ) + watchdog_mismatch_grace_s = float(watchdog_data.get("mismatch_grace_s", 6.0)) if watchdog_interval_s <= 0: raise ValueError("runtime.discovery_watchdog.interval_s must be positive") if watchdog_mismatch_grace_s < watchdog_interval_s: @@ -401,8 +403,7 @@ def _task_store_config(raw: Any) -> TaskStoreConfig: not isinstance(endpoints_raw, list) or not endpoints_raw or any( - not isinstance(item, str) or not item.strip() - for item in endpoints_raw + not isinstance(item, str) or not item.strip() for item in endpoints_raw ) ): raise ValueError( @@ -426,9 +427,7 @@ def _task_store_config(raw: Any) -> TaskStoreConfig: if timeout <= 0: raise ValueError("task_store.request_timeout_s must be positive") watch_timeout_raw = data.get("watch_response_timeout_s") - watch_timeout = ( - None if watch_timeout_raw is None else float(watch_timeout_raw) - ) + watch_timeout = None if watch_timeout_raw is None else float(watch_timeout_raw) if watch_timeout is not None and watch_timeout <= 0: raise ValueError("task_store.watch_response_timeout_s must be positive") return TaskStoreConfig( @@ -488,8 +487,7 @@ def _lifecycle_config(raw: Any) -> LifecycleConfig: allowed = duration_fields | {"sweeper_batch_size", "orphan_cleanup_dry_run"} _only(data, allowed, "lifecycle") durations = { - name: float(data.get(name, getattr(defaults, name))) - for name in duration_fields + name: float(data.get(name, getattr(defaults, name))) for name in duration_fields } if any(value <= 0 for value in durations.values()): raise ValueError("lifecycle TTL, grace and interval values must be positive") @@ -511,6 +509,13 @@ def _scheduling_config(raw: Any, pool_name: str) -> SchedulingConfig: data, { "worker_capacity", + "worker_prefetch_capacity", + "early_release_slot", + "finalization_timeout_s", + "finalization_max_retries", + "finalization_retry_delay_s", + "finalization_concurrency", + "finalization_pending_limit", "queue_limit", "accept_without_workers", "execution_timeout_s", @@ -520,8 +525,24 @@ def _scheduling_config(raw: Any, pool_name: str) -> SchedulingConfig: }, f"pools[{pool_name}].scheduling", ) + for key in ( + "worker_capacity", + "worker_prefetch_capacity", + "finalization_max_retries", + "finalization_concurrency", + "finalization_pending_limit", + ): + if key in data and type(data[key]) is not int: + raise ValueError(f"scheduling.{key} must be an integer") config = SchedulingConfig( worker_capacity=int(data.get("worker_capacity", 1)), + worker_prefetch_capacity=int(data.get("worker_prefetch_capacity", 0)), + early_release_slot=_boolean(data, "early_release_slot", False), + finalization_timeout_s=float(data.get("finalization_timeout_s", 60)), + finalization_max_retries=int(data.get("finalization_max_retries", 2)), + finalization_retry_delay_s=float(data.get("finalization_retry_delay_s", 0.1)), + finalization_concurrency=int(data.get("finalization_concurrency", 4)), + finalization_pending_limit=int(data.get("finalization_pending_limit", 64)), queue_limit=int(data.get("queue_limit", 32)), accept_without_workers=_boolean(data, "accept_without_workers", False), execution_timeout_s=float(data.get("execution_timeout_s", 1800.0)), @@ -529,8 +550,20 @@ def _scheduling_config(raw: Any, pool_name: str) -> SchedulingConfig: discovery_interval_s=float(data.get("discovery_interval_s", 1.0)), dispatch_interval_s=float(data.get("dispatch_interval_s", 0.25)), ) - if config.worker_capacity != 1: - raise ValueError("version 1 requires scheduling.worker_capacity=1") + if config.worker_capacity < 1: + raise ValueError("scheduling.worker_capacity must be positive") + if config.worker_prefetch_capacity not in {0, 1}: + raise ValueError("scheduling.worker_prefetch_capacity must be 0 or 1") + if config.worker_prefetch_capacity and not config.early_release_slot: + raise ValueError("Worker prefetch requires early_release_slot") + if ( + not 0 <= config.finalization_max_retries <= 16 + or config.finalization_concurrency < 1 + or config.finalization_pending_limit < config.finalization_concurrency + or not 0 < config.finalization_timeout_s <= 86400 + or not 0 < config.finalization_retry_delay_s <= 60 + ): + raise ValueError("invalid scheduling finalization limits") if config.queue_limit < 1: raise ValueError("scheduling.queue_limit must be positive") if ( @@ -620,6 +653,10 @@ def _pool_config(raw: Any, index: int) -> PoolConfig: }, ) scheduling = _scheduling_config(data.get("scheduling"), pool_id) + if scheduling.early_release_slot and execution_mode != "detached": + raise ValueError("early slot release requires detached execution_mode") + if scheduling.worker_capacity > 1 and execution_mode != "detached": + raise ValueError("multiple Worker slots require detached execution_mode") revision_payload = { "pool_id": pool_id, "served_models": served_models, diff --git a/dingo/video_gateway/dispatcher.py b/dingo/video_gateway/dispatcher.py index 25b34e3b8a21..4cb070c41f19 100644 --- a/dingo/video_gateway/dispatcher.py +++ b/dingo/video_gateway/dispatcher.py @@ -16,6 +16,14 @@ from dataclasses import dataclass, field from typing import Any +from dingo.common.video_result_file import INLINE_RESULT_FORMAT, normalize_inline_result +from dingo.common.video_task_protocol import ( + ENVELOPE_KEY, + EXECUTION_CAPACITY_CAPABILITY, + PREFETCH_CAPABILITY, + WAIT_TERMINAL_CAPABILITY, + detached_envelope, +) from dingo.video_gateway.adapters.base import VideoBackendAdapter from dingo.video_gateway.artifact_store import FileArtifactStore from dingo.video_gateway.config import GatewayConfig, PoolConfig @@ -24,7 +32,15 @@ EndpointClient, create_context, ) -from dingo.video_gateway.errors import ResultTooLarge, StoreConflict, WorkerUnavailable, worker_execution_error +from dingo.video_gateway.errors import ( + HandoffReservationLost, + ResultTooLarge, + StoreConflict, + WorkerUnavailable, + worker_execution_error, +) +from dingo.video_gateway.file_io import run_file_io +from dingo.video_gateway.finalization import ResultFinalizer from dingo.video_gateway.memory_budget import ( MemoryBudgetSnapshot, WeightedMemoryBudget, @@ -33,16 +49,19 @@ ACTIVE_STATUSES, TERMINAL_STATUSES, StoredTask, + TaskError, TaskStatus, WorkerLease, now_ms, ) -from dingo.video_gateway.task_store import TaskStore, terminal_error, worker_key -from dingo.video_gateway.telemetry import GatewayTelemetry -from dingo.common.video_task_protocol import ( - WAIT_TERMINAL_CAPABILITY, - detached_envelope, +from dingo.video_gateway.result_handoff import HANDOFF_KEY, make_handoff, read_handoff +from dingo.video_gateway.task_store import ( + TaskStore, + retry_excludes_worker, + terminal_error, + worker_key, ) +from dingo.video_gateway.telemetry import GatewayTelemetry logger = logging.getLogger(__name__) _GATEWAY_OWNER_TTL_S = 15 @@ -131,6 +150,9 @@ class PoolRuntime: lease_cache: dict[str, WorkerLease] = field(default_factory=dict) lease_revision: int = 0 lease_watch_healthy: bool = True + # Fixed per physical registration; failed probes are retried with backoff. + capacity_cache: dict[int, tuple[int, float]] = field(default_factory=dict) + prefetch_instances: set[int] = field(default_factory=set) class VideoDispatcher: @@ -153,9 +175,16 @@ def __init__( self.generation = generation or uuid.uuid4().hex self._worker_retry_once = os.getenv("DINGO_VIDEO_WORKER_RETRY_ONCE", "0") == "1" self._retry_budget_limit = int(os.getenv("DINGO_VIDEO_RETRY_BUDGET", "32")) - self._retry_wait_timeout_s = float(os.getenv("DINGO_VIDEO_RETRY_WAIT_TIMEOUT_S", "600")) - self._failed_instance_backoff_s = float(os.getenv("DINGO_VIDEO_RETRY_FAILED_INSTANCE_BACKOFF_S", "30")) - if not 1 <= self._retry_budget_limit <= 1024 or not 0 < self._retry_wait_timeout_s <= 86400: + self._retry_wait_timeout_s = float( + os.getenv("DINGO_VIDEO_RETRY_WAIT_TIMEOUT_S", "600") + ) + self._failed_instance_backoff_s = float( + os.getenv("DINGO_VIDEO_RETRY_FAILED_INSTANCE_BACKOFF_S", "30") + ) + if ( + not 1 <= self._retry_budget_limit <= 1024 + or not 0 < self._retry_wait_timeout_s <= 86400 + ): raise ValueError("invalid retry budget or wait timeout") if not 0 <= self._failed_instance_backoff_s <= 86400: raise ValueError("invalid failed instance backoff") @@ -172,6 +201,14 @@ def __init__( for pool in config.pools } self.running_calls: dict[str, RunningCall] = {} + self._finalizing: dict[str, str] = {} + self._finalization_slots = { + pool.pool_id: asyncio.Semaphore(pool.scheduling.finalization_concurrency) + for pool in config.pools + } + self._finalizer = ResultFinalizer( + store, artifacts, config, self.telemetry, self.generation + ) self.memory_budget = WeightedMemoryBudget( config.media.inflight_memory_budget_bytes ) @@ -327,6 +364,12 @@ async def start(self) -> None: self._orphan_recovery_loop(), name="video-orphan-recovery" ) ) + self._loops.append( + asyncio.create_task( + self._owned_finalization_recovery_loop(), + name="video-finalization-recovery", + ) + ) self._ready = True async def stop(self) -> None: @@ -393,11 +436,66 @@ async def _gateway_owner_loop(self) -> None: def has_workers(self, pool_id: str) -> bool: pool = self.pools[pool_id] - return bool(pool.instance_ids) + # Registration alone is not capacity; saturation still allows queuing. + return bool(self._worker_slots(pool)) def pool_instances(self, pool_id: str) -> list[int]: return list(self.pools[pool_id].instance_ids) + def pool_capacity_snapshot( + self, pool_id: str, leases: list[WorkerLease] + ) -> dict[str, int]: + """Local discovery + shared leases, without per-Worker scrape RPCs. + + Admission leases are NOT engine-running tasks. With prefetch enabled, + a lease can cover execution, Worker queuing or output writing. Slot + numbers are fungible and cannot identify the prefetch occupant. + """ + pool = self.pools[pool_id] + slots = self._worker_slots(pool) + keys = { + worker_key(pool.config.backend_target, instance, slot) + for instance, slot in slots + } + registered = {str(i) for i in pool.instance_ids} + mapped = { + lease.worker_key: lease for lease in leases if lease.worker_key in keys + } + busy = sum(lease.state != "quarantined" for lease in mapped.values()) + quarantined = len(mapped) - busy + physical_busy = { + str(lease.worker_instance_id) + for lease in leases + if str(lease.worker_instance_id) in registered + and lease.backend_target == pool.config.backend_target + and lease.state != "quarantined" + } + prefetch = sum( + instance in pool.prefetch_instances + for instance in {instance for instance, _ in slots} + ) + healthy = pool.discovery_healthy and ( + not self.store.lease_watch_supported or pool.lease_watch_healthy + ) + return { + "workers": len(registered), + "worker_busy": len(physical_busy), + "worker_execution_capacity": len(slots) - prefetch, + "worker_prefetch_capacity": prefetch, + "worker_admission_capacity": len(slots), + "worker_slots_busy": busy, + "worker_slots_quarantined": quarantined, + "worker_slots_free": len(keys - mapped.keys()) if healthy else 0, + "worker_unmapped_leases": len( + {lease.worker_key for lease in leases} - keys + ), + "worker_capacity_view_healthy": int(healthy), + } + + def pool_finalization_pending(self, pool_id: str) -> int: + """This Gateway's pending/running finalizers, not a shared pool total.""" + return sum(p == pool_id for p in self._finalizing.values()) + async def pool_leases(self, pool_id: str) -> list[WorkerLease]: pool = self.pools[pool_id] if self.store.lease_watch_supported: @@ -612,6 +710,140 @@ async def _refresh_instances(self, pool: PoolRuntime) -> None: ) pool.instance_ids = [] pool.discovery_healthy = False + pool.capacity_cache = { + i: v for i, v in pool.capacity_cache.items() if i in pool.instance_ids + } + if hasattr(pool, "prefetch_instances"): + pool.prefetch_instances.intersection_update(pool.instance_ids) + if pool.config.scheduling.worker_capacity > 1 or getattr( + pool.config.scheduling, "worker_prefetch_capacity", 0 + ): + semaphore = asyncio.Semaphore(8) + + async def probe(instance: int) -> None: + if pool.capacity_cache.get(instance, (0, 0))[1] > time.monotonic(): + return + async with semaphore: + pool.capacity_cache[instance] = await self._worker_capacity( + pool, instance + ) + + await asyncio.gather(*(probe(i) for i in pool.instance_ids)) + + async def _worker_capacity( + self, pool: PoolRuntime, instance: int + ) -> tuple[int, float]: + pool.prefetch_instances.discard(instance) + context = self.context_factory( + f"capacity-{uuid.uuid4().hex}", {"pool_id": pool.config.pool_id} + ) + + async def query() -> int: + stream = await pool.client.direct( + {ENVELOPE_KEY: {"schema_version": 1, "op": "capabilities"}}, + instance, + context, + ) + response = None + async for item in stream: + if hasattr(item, "is_error") and item.is_error(): + raise RuntimeError("Worker capability query failed") + value = item.data() if hasattr(item, "data") else item + if response is not None or not isinstance(value, dict): + raise ValueError("invalid Worker capabilities response") + response = value + if ( + response is None + or response.get("schema_version") != 1 + or EXECUTION_CAPACITY_CAPABILITY not in response.get("capabilities", []) + ): + raise ValueError("Worker does not advertise execution capacity") + capacity = response.get("execution_capacity") + if ( + isinstance(capacity, bool) + or not isinstance(capacity, int) + or capacity < 1 + ): + raise ValueError("invalid Worker execution capacity") + if response.get("accepting") is not True: + return 0 + effective = min(capacity, pool.config.scheduling.worker_capacity) + prefetch = getattr(pool.config.scheduling, "worker_prefetch_capacity", 0) + if ( + prefetch + and capacity == pool.config.scheduling.worker_capacity + and PREFETCH_CAPABILITY in response.get("capabilities", []) + ): + advertised = response.get("prefetch_capacity") + admission = response.get("admission_capacity") + if ( + type(advertised) is not int + or advertised not in {0, 1} + or type(admission) is not int + or admission != capacity + advertised + ): + raise ValueError("invalid Worker prefetch capacity") + if advertised: + pool.prefetch_instances.add(instance) + effective += min(prefetch, advertised) + return effective + + try: + capacity = await asyncio.wait_for(query(), timeout=2.0) + return capacity, float("inf") if capacity else time.monotonic() + 5.0 + except Exception: + # Legacy Workers are still usable as single-slot instances. Never + # infer N slots from the configured ceiling or a failed RPC. + logger.warning("Worker %s capacity unavailable; using one slot", instance) + return 1, time.monotonic() + 30.0 + finally: + context.stop_generating() + + @staticmethod + def _worker_slots(pool: PoolRuntime) -> list[tuple[int, int]]: + ceiling = pool.config.scheduling.worker_capacity + getattr( + pool.config.scheduling, "worker_prefetch_capacity", 0 + ) + return [ + (instance, slot) + for instance in pool.instance_ids + for slot in range( + 1 + if ceiling == 1 + else min(ceiling, pool.capacity_cache.get(instance, (0, 0))[0]) + ) + ] + + async def _detached_submit_ack(self, pool, task, submit, context, validate): + """Busy means not executed: retry admission, not the task attempt.""" + while True: + if task.deadline_at_ms is not None and now_ms() >= task.deadline_at_ms: + raise asyncio.TimeoutError("Worker admission deadline exceeded") + stream = await pool.client.direct( + submit, int(task.worker_instance_id), context + ) + response = None + async for item in stream: + if hasattr(item, "is_error") and item.is_error(): + comments = item.comments() if hasattr(item, "comments") else [] + raise RuntimeError( + "; ".join(comments) or "detached Worker submit failed" + ) + value = item.data() if hasattr(item, "data") else item + if response is not None or not isinstance(value, dict): + raise RuntimeError("invalid detached Worker acknowledgement") + response = validate(value) + if response is None: + raise RuntimeError("detached Worker returned no acknowledgement") + if response.get("state") != "busy": + return response + if response.get("accepted") is not False: + raise RuntimeError("busy Worker incorrectly acknowledged execution") + self.telemetry.increment( + "dingo_video_worker_admission_busy_total", + labels={"pool": pool.config.pool_id}, + ) + await asyncio.sleep(0.25) def _record_discovery_match( self, @@ -623,7 +855,8 @@ def _record_discovery_match( stale = runtime_ids - truth_ids labels = {"pool": pool.config.pool_id} self.telemetry.set_gauge( - "dingo_video_discovery_consistent", 0 if missing or stale else 1, + "dingo_video_discovery_consistent", + 0 if missing or stale else 1, labels=labels, ) self.telemetry.set_gauge( @@ -706,9 +939,7 @@ async def _discovery_watchdog_loop(self) -> None: except asyncio.CancelledError: raise except Exception: - self.telemetry.increment( - "dingo_video_discovery_watchdog_errors_total" - ) + self.telemetry.increment("dingo_video_discovery_watchdog_errors_total") logger.exception("Dynamo discovery watchdog check failed") remaining = watchdog.interval_s - (time.monotonic() - started) if remaining <= 0: @@ -834,16 +1065,66 @@ async def _release_reusable_leases(self, pool: PoolRuntime) -> None: await self.store.release_lease(pool.config.pool_id, lease.worker_key) async def _dispatch_once(self, pool: PoolRuntime) -> bool: + if ( + sum( + p == pool.config.pool_id + for p in getattr(self, "_finalizing", {}).values() + ) + >= pool.config.scheduling.finalization_pending_limit + ): + await self._clear_budget_waiter(pool) + return False + # A healthy lease view can prove that no instance is usable without + # reading queue indexes, task bodies, or shared retry counters. Keep + # the later checks and reserve CAS: availability can change while the + # awaited queue reads are in flight. + skip_reason = None + if not pool.instance_ids: + skip_reason = "no_workers" + elif self.store.lease_watch_supported and not pool.lease_watch_healthy: + skip_reason = "lease_watch_unhealthy" + else: + occupied = { + lease.worker_key + for lease in await self.pool_leases(pool.config.pool_id) + } + if all( + worker_key(pool.config.backend_target, instance, slot) in occupied + for instance, slot in self._worker_slots(pool) + ): + skip_reason = "no_free_worker" + if skip_reason is not None: + await self._clear_budget_waiter(pool) + self.telemetry.increment( + "dingo_video_dispatch_skips_total", + labels={"pool": pool.config.pool_id, "reason": skip_reason}, + ) + return False ledger = hasattr(self.store, "retry_budget_used") budget_limit = getattr(self, "_retry_budget_limit", 32) - queued = await self.store.list_queued(pool.config.pool_id, limit=min(10000, pool.config.scheduling.queue_limit + budget_limit)) + queued = await self.store.list_queued( + pool.config.pool_id, + limit=min(10000, pool.config.scheduling.queue_limit + budget_limit), + ) used = 0 if ledger: waiting = await self.store.retry_queue_depth(pool.config.pool_id) used = await self.store.retry_budget_used(pool.config.pool_id) - self.telemetry.set_gauge("dingo_video_retry_waiting_tasks", waiting, labels={"pool": pool.config.pool_id}) - self.telemetry.set_gauge("dingo_video_retry_credits_used", used, labels={"pool": pool.config.pool_id}) - self.telemetry.set_gauge("dingo_video_normal_queue_depth", max(0, await self.store.queue_depth(pool.config.pool_id) - waiting), labels={"pool": pool.config.pool_id}) + self.telemetry.set_gauge( + "dingo_video_retry_waiting_tasks", + waiting, + labels={"pool": pool.config.pool_id}, + ) + self.telemetry.set_gauge( + "dingo_video_retry_credits_used", + used, + labels={"pool": pool.config.pool_id}, + ) + self.telemetry.set_gauge( + "dingo_video_normal_queue_depth", + max(0, await self.store.queue_depth(pool.config.pool_id) - waiting), + labels={"pool": pool.config.pool_id}, + ) if not queued or not pool.instance_ids: await self._clear_budget_waiter(pool) return False @@ -858,15 +1139,26 @@ async def _dispatch_once(self, pool: PoolRuntime) -> bool: # Retry first, but do not let a retry that excludes the only available # instance block unrelated runnable work behind it. for candidate in sorted(queued, key=lambda item: item.task.attempt == 0): - if candidate.task.id in self.running_calls or candidate.task.expires_at_ms <= now_ms(): + if ( + candidate.task.id in self.running_calls + or candidate.task.expires_at_ms <= now_ms() + ): continue - if (ledger and getattr(self, "_worker_retry_once", False) - and candidate.task.attempt == 0 and used >= budget_limit): + if ( + ledger + and getattr(self, "_worker_retry_once", False) + and candidate.task.attempt == 0 + and used >= budget_limit + ): continue - available = [instance for instance in pool.instance_ids - if worker_key(pool.config.backend_target, instance) not in leased - and not (candidate.task.attempt > 0 - and worker_key(pool.config.backend_target, instance) == candidate.task.worker_key)] + available = [ + (instance, slot) + for instance, slot in self._worker_slots(pool) + if worker_key(pool.config.backend_target, instance, slot) not in leased + and not retry_excludes_worker( + candidate.task, pool.config.backend_target, instance + ) + ] if available: selected = candidate break @@ -890,9 +1182,9 @@ async def _dispatch_once(self, pool: PoolRuntime) -> bool: await self.memory_budget.release(task_id) return False index = pool.cursor % len(available) - instance_id = available[index] + instance_id, slot_id = available[index] pool.cursor = (index + 1) % len(available) - key = worker_key(pool.config.backend_target, instance_id) + key = worker_key(pool.config.backend_target, instance_id, slot_id) lease = WorkerLease( pool_id=pool.config.pool_id, worker_key=key, @@ -907,9 +1199,15 @@ async def _dispatch_once(self, pool: PoolRuntime) -> bool: ) deadline = now_ms() + int(pool.config.scheduling.execution_timeout_s * 1000) try: - retry_options = ({"reserve_retry": getattr(self, "_worker_retry_once", False) - and pool.config.execution_mode == "detached", "retry_limit": budget_limit} - if ledger else {}) + retry_options = ( + { + "reserve_retry": getattr(self, "_worker_retry_once", False) + and pool.config.execution_mode == "detached", + "retry_limit": budget_limit, + } + if ledger + else {} + ) reserved = await self.store.reserve( queued[0], lease, deadline_at_ms=deadline, **retry_options ) @@ -980,9 +1278,7 @@ async def _run_with_lease_monitor( cancellation_task = ( asyncio.create_task(cancellation) if cancellation is not None else None ) - liveness_task = ( - asyncio.create_task(liveness) if liveness is not None else None - ) + liveness_task = asyncio.create_task(liveness) if liveness is not None else None try: monitored = {operation_task, heartbeat} if cancellation_task is not None: @@ -1011,7 +1307,8 @@ async def _run_with_lease_monitor( finally: children = [operation_task] children.extend( - monitor for monitor in (cancellation_task, liveness_task) + monitor + for monitor in (cancellation_task, liveness_task) if monitor is not None ) for child in children: @@ -1051,20 +1348,29 @@ async def _confirm_worker_loss(self, pool: PoolRuntime, task: Any) -> bool: return False current = await self._current_owned_execution(task) if current is None: - raise _TaskOwnershipLost("task ownership moved during Worker liveness check") - if (current.task.status != TaskStatus.IN_PROGRESS - or current.task.cancel_requested_at_ms is not None): + raise _TaskOwnershipLost( + "task ownership moved during Worker liveness check" + ) + if ( + current.task.status != TaskStatus.IN_PROGRESS + or current.task.cancel_requested_at_ms is not None + ): return False status = await self.artifacts.read_detached_status( - task.deployment_id, task.pool_id, task.id, task.attempt, + task.deployment_id, + task.pool_id, + task.id, + task.attempt, task.execution_token, ) if status is None or status.get("state") not in {"accepted", "running"}: return False updated_at_ms = status.get("updated_at_ms") - if (not isinstance(updated_at_ms, int) - or isinstance(updated_at_ms, bool) - or now_ms() - updated_at_ms <= int(_DETACHED_WORKER_STALE_S * 1000)): + if ( + not isinstance(updated_at_ms, int) + or isinstance(updated_at_ms, bool) + or now_ms() - updated_at_ms <= int(_DETACHED_WORKER_STALE_S * 1000) + ): return False return self._can_check_missing_worker(pool, task) except _TaskOwnershipLost: @@ -1076,7 +1382,11 @@ async def _confirm_worker_loss(self, pool: PoolRuntime, task: Any) -> bool: "dingo_video_worker_liveness_checks_total", labels={"pool": pool.config.pool_id, "outcome": "inconclusive"}, ) - logger.debug("Worker liveness evidence unavailable for %s", task.id, exc_info=True) + logger.debug( + "Worker liveness evidence unavailable for %s", + task.id, + exc_info=True, + ) return False async def _monitor_worker_liveness( @@ -1131,9 +1441,7 @@ async def _monitor_cancellation( if remaining_s <= 0: raise _CancellationConfirmationTimedOut(expected.id) try: - await asyncio.wait_for( - running.task_changed.wait(), timeout=remaining_s - ) + await asyncio.wait_for(running.task_changed.wait(), timeout=remaining_s) except asyncio.TimeoutError as exc: raise _CancellationConfirmationTimedOut(expected.id) from exc @@ -1197,18 +1505,16 @@ async def _run_reserved(self, pool: PoolRuntime, stored: StoredTask) -> None: if initial_worker_status is None: manifest = await self.artifacts.read_json(task.input_manifest_path) payload_build_started = time.monotonic() - payload = await asyncio.to_thread( + payload = await run_file_io( pool.adapter.build_worker_payload, normalized, manifest, - self.artifacts.task_root( + await self.artifacts.resolve_task_root( task.deployment_id, task.pool_id, task.id ), ) self._payload_build_count += 1 - self._payload_build_seconds += ( - time.monotonic() - payload_build_started - ) + self._payload_build_seconds += time.monotonic() - payload_build_started self._raise_if_heartbeat_stopped(heartbeat) if stored.task.status == TaskStatus.DISPATCHING: before = stored @@ -1221,9 +1527,7 @@ async def _run_reserved(self, pool: PoolRuntime, stored: StoredTask) -> None: "started_at_ms": stored.task.started_at_ms or now_ms(), "queue_wait_s": stored.task.queue_wait_s if stored.task.queue_wait_s is not None - else max( - 0.0, (now_ms() - task.queued_at_ms) / 1000.0 - ), + else max(0.0, (now_ms() - task.queued_at_ms) / 1000.0), }, ) self.telemetry.record_transition( @@ -1350,19 +1654,36 @@ async def _consume_worker_stream() -> None: result = response_consumer.finish() response_consumer = None encoded_result = result.b64_json + binary_artifact = result.artifact + if binary_artifact is not None and not detached: + raise RuntimeError( + "binary artifact references require detached execution" + ) inference_time_s = result.inference_time_s stage_durations = dict(result.stage_durations or {}) result = None self._legacy_output_encoded_bytes += len(encoded_result) if len(encoded_result) > self.config.media.max_result_encoded_bytes: self._result_oversize_count += 1 - raise ResultTooLarge( - "Worker base64 result exceeds configured maximum" - ) + raise ResultTooLarge("Worker base64 result exceeds configured maximum") if inference_time_s is not None: self.telemetry.record_stage_duration( task.pool_id, "execution", inference_time_s ) + if ( + detached + and binary_artifact is not None + and pool.config.scheduling.early_release_slot + and latest.task.status == TaskStatus.IN_PROGRESS + ): + handed_off = await self._commit_result_handoff( + pool, task, binary_artifact, inference_time_s, stage_durations + ) + if handed_off is not None: + self.running_calls.pop(task.id, None) + pool.wakeup.set() + await self._run_result_finalizer(pool, handed_off) + return if latest.task.status == TaskStatus.IN_PROGRESS: finalizing = await self.store.transition( task.id, @@ -1391,9 +1712,34 @@ async def _consume_worker_stream() -> None: (stored.task.execution_token or "legacy").encode() ).hexdigest()[:16] try: - final_path, size, sha256, media = ( - await self.artifacts.finalize_b64_mp4( - self.artifacts.task_root( + if binary_artifact is not None: + ( + final_path, + size, + sha256, + media, + ) = await self.artifacts.finalize_worker_mp4( + task.deployment_id, + task.pool_id, + task.id, + task.attempt, + task.execution_token, + binary_artifact, + normalized, + pool.adapter.validate_artifact, + pool.adapter.prepare_artifact, + pool.adapter.artifact_requires_processing, + inspector=pool.adapter.inspect_artifact_for_publication, + max_result_bytes=self.config.media.max_result_bytes, + ) + else: + ( + final_path, + size, + sha256, + media, + ) = await self.artifacts.finalize_b64_mp4( + await self.artifacts.resolve_task_root( task.deployment_id, task.pool_id, task.id ), encoded_result, @@ -1403,7 +1749,6 @@ async def _consume_worker_stream() -> None: max_result_bytes=self.config.media.max_result_bytes, publication_scope=f"a{stored.task.attempt}-{token_digest}", ) - ) finally: # Decoding/validation has either published a file candidate or # failed. No later state transition needs the Base64 string. @@ -1418,12 +1763,12 @@ async def _consume_worker_stream() -> None: if latest is None: raise RuntimeError("task disappeared during finalization") if latest.task.status in TERMINAL_STATUSES: - await asyncio.to_thread(final_path.unlink, True) + await run_file_io(final_path.unlink, True) final_path = None return self._require_execution_owner(latest.task, task) if latest.task.cancel_requested_at_ms is not None: - await asyncio.to_thread(final_path.unlink, True) + await run_file_io(final_path.unlink, True) final_path = None await self._finish_cancelled(pool, latest, quarantine=False) return @@ -1451,7 +1796,7 @@ async def _consume_worker_stream() -> None: except StoreConflict: # Another owner may have published a different immutable # candidate. Delete only this Gateway's candidate. - await asyncio.to_thread(final_path.unlink, True) + await run_file_io(final_path.unlink, True) final_path = None current = await self.store.get_task(task.id) if current is not None and current.task.status in TERMINAL_STATUSES: @@ -1471,7 +1816,7 @@ async def _consume_worker_stream() -> None: ) except _TaskOwnershipLost: if final_path is not None: - await asyncio.to_thread(final_path.unlink, True) + await run_file_io(final_path.unlink, True) final_path = None logger.info( "Gateway relinquished task %s after execution ownership moved", @@ -1516,9 +1861,7 @@ async def _consume_worker_stream() -> None: except _WorkerLeaseLost as exc: if await self._current_owned_execution(task) is None: return - worker_accepted = ( - running_call is not None and running_call.worker_accepted - ) + worker_accepted = running_call is not None and running_call.worker_accepted if detached and task.execution_token is not None and worker_accepted: try: await self._request_detached_cancel(task) @@ -1556,7 +1899,7 @@ async def _consume_worker_stream() -> None: await self._finish_cancelled(pool, latest, quarantine=True) except ResultTooLarge as exc: if final_path is not None: - await asyncio.to_thread(final_path.unlink, True) + await run_file_io(final_path.unlink, True) final_path = None if await self._current_owned_execution(task) is None: return @@ -1571,7 +1914,7 @@ async def _consume_worker_stream() -> None: ) except Exception as exc: if final_path is not None: - await asyncio.to_thread(final_path.unlink, True) + await run_file_io(final_path.unlink, True) final_path = None current = await self._current_owned_execution(task) if current is None: @@ -1635,6 +1978,7 @@ async def _consume_worker_stream() -> None: result = None encoded_result = None released_budget = await self.memory_budget.release(task.id) + self._finalizing.pop(task.id, None) self.running_calls.pop(task.id, None) if heartbeat is not None: heartbeat.cancel() @@ -1656,6 +2000,171 @@ async def _request_detached_cancel(self, task: Any) -> None: task.execution_token, ) + async def _commit_result_handoff( + self, pool, expected, artifact, inference_time_s, stage_durations + ): + """Retry ambiguous storage responses by reading the durable task first.""" + reservation_lost = False + while not self._stop.is_set(): + try: + latest = await self.store.get_task(expected.id) + if latest is None or latest.task.status in TERMINAL_STATUSES: + return None + self._require_execution_owner(latest.task, expected) + if ( + latest.task.status == TaskStatus.FINALIZING + and read_handoff(latest.task) is not None + ): + return latest + if reservation_lost: + # A definite fencing failure is not transient CAS contention. + # End only our task, never cancel/release/quarantine the new + # occupant of the historical Worker slot. + await self.store.transition( + expected.id, + expected={TaskStatus.IN_PROGRESS}, + expected_revision=latest.revision, + release_lease=False, + patch={ + "status": TaskStatus.FAILED, + "error": TaskError( + "result_handoff_lost_reservation", + "execution reservation changed before result handoff", + ), + "completed_at_ms": now_ms(), + "expires_at_ms": now_ms() + + int(self.config.lifecycle.failed_ttl_s * 1000), + }, + ) + return None + if latest.task.cancel_requested_at_ms is not None: + await self._finish_cancelled(pool, latest, quarantine=False) + return None + reference = make_handoff( + latest.task, + artifact, + timeout_s=pool.config.scheduling.finalization_timeout_s, + ) + result = await self.store.transition( + expected.id, + expected={TaskStatus.IN_PROGRESS}, + expected_revision=latest.revision, + release_lease=True, + release_execution=True, + patch={ + "status": TaskStatus.FINALIZING, + "worker_lease_id": None, + "normalized_request": { + **latest.task.normalized_request, + HANDOFF_KEY: reference, + }, + "inference_time_s": inference_time_s, + "stage_durations": stage_durations, + }, + ) + self.telemetry.record_transition( + "result_handoff", + latest.task, + result.task, + gateway_generation=self.generation, + revision=result.revision, + ) + return result + except HandoffReservationLost: + reservation_lost = True + except (_TaskOwnershipLost, ValueError): + raise + except asyncio.CancelledError: + raise + except Exception: + # Includes a lost transaction response. Never discard the source + # or call the Worker again while the outcome is uncertain. + logger.warning( + "result handoff awaiting storage for task %s", + expected.id, + exc_info=True, + ) + await asyncio.sleep(0.2) + raise asyncio.CancelledError + + async def _run_result_finalizer(self, pool, stored): + self._finalizing[stored.task.id] = pool.config.pool_id + try: + async with self._finalization_slots[pool.config.pool_id]: + await self._finalizer.run(stored, pool) + except asyncio.CancelledError: + raise + except Exception: + # Keep the durable handoff for our live-owner recovery loop or HA + # takeover. Do not route storage failures through model retry/cancel. + logger.exception("postprocessing suspended for task %s", stored.task.id) + + async def _resume_result_finalizer(self, pool, stored): + try: + await self._run_result_finalizer(pool, stored) + finally: + await self.memory_budget.release(stored.task.id) + self._finalizing.pop(stored.task.id, None) + pool.wakeup.set() + + async def _schedule_result_finalizer(self, pool, stored): + if stored.task.id in self._finalizing or stored.task.id in self.running_calls: + return + if ( + sum(p == pool.config.pool_id for p in self._finalizing.values()) + >= pool.config.scheduling.finalization_pending_limit + ): + return + # Reserve before the first await: owner recovery and the live-owner + # scanner can otherwise start two finalizers sharing one allocation. + self._finalizing[stored.task.id] = pool.config.pool_id + scheduled = False + try: + if not await self.memory_budget.try_acquire( + stored.task.id, self.config.media.result_task_memory_bytes + ): + return + execution = asyncio.create_task( + self._resume_result_finalizer(pool, stored), + name=f"video-finalizing-{stored.task.id}", + ) + scheduled = True + finally: + if not scheduled: + await self.memory_budget.release(stored.task.id) + self._finalizing.pop(stored.task.id, None) + self._executions.add(execution) + execution.add_done_callback(self._execution_done) + + async def _owned_finalization_recovery_loop(self): + while not self._stop.is_set(): + try: + after = None + while not self._stop.is_set(): + page = await self.store.list_tasks( + status=TaskStatus.FINALIZING, after=after, limit=256 + ) + if not page: + break + for stored in page: + if ( + stored.task.owner_generation != self.generation + or read_handoff(stored.task) is None + ): + continue + pool = self.pools.get(stored.task.pool_id) + if pool is not None: + await self._schedule_result_finalizer(pool, stored) + after = page[-1].task.id + except asyncio.CancelledError: + raise + except Exception: + logger.exception("owned finalization recovery failed") + try: + await asyncio.wait_for(self._stop.wait(), timeout=1.0) + except asyncio.TimeoutError: + pass + async def _consume_detached_worker( self, pool: PoolRuntime, @@ -1719,6 +2228,38 @@ async def _consume_status(value: dict[str, Any]) -> bool: worker_status = _validate_identity(value) state = worker_status.get("state") if state == "completed": + queue_wait = worker_status.get("worker_queue_wait_s") + if ( + isinstance(queue_wait, (int, float)) + and not isinstance(queue_wait, bool) + and 0 <= queue_wait <= 86400 + ): + self.telemetry.record_stage_duration( + task.pool_id, "worker_queue", queue_wait + ) + if "result_format" in worker_status or "inline_result" in worker_status: + if worker_status.get( + "result_format" + ) != INLINE_RESULT_FORMAT or any( + key in worker_status + for key in ( + "response_path", + "response_bytes", + "response_sha256", + ) + ): + raise _DetachedWaitProtocolError( + "unsupported or ambiguous inline Worker result" + ) + response_consumer.consume( + normalize_inline_result(worker_status.get("inline_result")) + ) + self.telemetry.increment( + "dingo_video_detached_inline_results_total", + labels={"pool": task.pool_id}, + ) + running_call.worker_accepted = False + return True response_sha256 = worker_status.get("response_sha256") response_bytes = worker_status.get("response_bytes") if ( @@ -1730,9 +2271,7 @@ async def _consume_status(value: dict[str, Any]) -> bool: or response_bytes > self.config.media.max_result_encoded_bytes + 1024 * 1024 ): - raise RuntimeError( - "detached Worker completed metadata is invalid" - ) + raise RuntimeError("detached Worker completed metadata is invalid") consumed = await self.artifacts.consume_detached_response( task.deployment_id, task.pool_id, @@ -1762,8 +2301,7 @@ async def _consume_status(value: dict[str, Any]) -> bool: if ( state in {"accepted", "running"} and isinstance(updated_at_ms, int) - and now_ms() - updated_at_ms - > int(_DETACHED_WORKER_STALE_S * 1000) + and now_ms() - updated_at_ms > int(_DETACHED_WORKER_STALE_S * 1000) and await self._confirm_worker_loss(pool, task) ): raise _RetryableWorkerFailure( @@ -1789,12 +2327,9 @@ async def _wait_event(attached: asyncio.Event) -> dict[str, Any]: count = 0 async for item in stream: if hasattr(item, "is_error") and item.is_error(): - comments = ( - item.comments() if hasattr(item, "comments") else [] - ) + comments = item.comments() if hasattr(item, "comments") else [] raise _DetachedWaitUnavailable( - "; ".join(comments) - or "detached Worker wait request failed" + "; ".join(comments) or "detached Worker wait request failed" ) value = _validate_identity( item.data() if hasattr(item, "data") else item @@ -1858,9 +2393,7 @@ async def _wait_once() -> dict[str, Any]: "dingo_video_detached_wait_connections_total", labels={"pool": task.pool_id, "outcome": "attached"}, ) - remaining_s = ( - (task.deadline_at_ms or 0) - now_ms() - ) / 1000.0 + remaining_s = ((task.deadline_at_ms or 0) - now_ms()) / 1000.0 if remaining_s <= 0: raise asyncio.TimeoutError return await asyncio.wait_for(wait_task, timeout=remaining_s) @@ -1887,29 +2420,20 @@ async def _wait_once() -> dict[str, Any]: attempt=task.attempt, execution_token=task.execution_token, payload=payload, + deadline_at_ms=( + task.deadline_at_ms + if task.worker_instance_id in pool.prefetch_instances + else None + ), ) - stream = await pool.client.direct( - submit, int(task.worker_instance_id), context + acknowledgement = await asyncio.wait_for( + self._detached_submit_ack( + pool, task, submit, context, _validate_identity + ), + timeout=max( + 0.0, ((task.deadline_at_ms or now_ms()) - now_ms()) / 1000.0 + ), ) - acknowledgements: list[dict[str, Any]] = [] - async for item in stream: - if hasattr(item, "is_error") and item.is_error(): - comments = item.comments() if hasattr(item, "comments") else [] - raise RuntimeError( - "; ".join(comments) or "detached Worker submit failed" - ) - value = item.data() if hasattr(item, "data") else item - if not isinstance(value, dict): - raise RuntimeError("detached Worker acknowledgement is invalid") - acknowledgements.append(value) - if len(acknowledgements) > 1: - raise RuntimeError( - "detached Worker returned multiple acknowledgements" - ) - if len(acknowledgements) != 1: - raise RuntimeError("detached Worker returned no acknowledgement") - acknowledgement = acknowledgements[0] - acknowledgement = _validate_identity(acknowledgement) if acknowledgement.get("state") not in { "accepted", "running", @@ -1922,9 +2446,7 @@ async def _wait_once() -> dict[str, Any]: # The direct response stream is exhausted and the Worker has # durably accepted this detached execution. Drop the envelope and # its Base64 references before entering the long status-poll loop. - acknowledgements.clear() del acknowledgement - del stream del submit else: running_call.worker_accepted = worker_status.get("state") in { @@ -1966,9 +2488,7 @@ async def _wait_once() -> dict[str, Any]: labels={"pool": task.pool_id, "outcome": "unavailable"}, ) next_wait_retry = time.monotonic() + retry_delay_s - retry_delay_s = min( - retry_delay_s * 2.0, _DETACHED_WAIT_RETRY_MAX_S - ) + retry_delay_s = min(retry_delay_s * 2.0, _DETACHED_WAIT_RETRY_MAX_S) worker_status = await _status() self.telemetry.increment( @@ -1984,9 +2504,7 @@ async def _wait_once() -> dict[str, Any]: raise asyncio.TimeoutError sleep_s = min(_DETACHED_STATUS_FALLBACK_S, remaining_ms / 1000.0) if supports_wait: - sleep_s = min( - sleep_s, max(0.0, next_wait_retry - time.monotonic()) - ) + sleep_s = min(sleep_s, max(0.0, next_wait_retry - time.monotonic())) if sleep_s > 0: await asyncio.sleep(sleep_s) @@ -2065,42 +2583,66 @@ async def _finish_cancelled( revision=cancelled.revision, ) - async def _try_worker_retry(self, pool: PoolRuntime, stored: StoredTask, *, quarantine: bool) -> bool: + async def _try_worker_retry( + self, pool: PoolRuntime, stored: StoredTask, *, quarantine: bool + ) -> bool: task = stored.task - if (not getattr(self, "_worker_retry_once", False) - or pool.config.execution_mode != "detached" - or task.attempt != 1 - or task.status not in {TaskStatus.DISPATCHING, TaskStatus.IN_PROGRESS} - or task.cancel_requested_at_ms is not None - or task.owner_generation != self.generation - or not hasattr(self.store, "retry_budget_used")): + if ( + not getattr(self, "_worker_retry_once", False) + or pool.config.execution_mode != "detached" + or task.attempt != 1 + or task.status not in {TaskStatus.DISPATCHING, TaskStatus.IN_PROGRESS} + or task.cancel_requested_at_ms is not None + or task.owner_generation != self.generation + or not hasattr(self.store, "retry_budget_used") + ): return False retry = None for _ in range(16): try: retry = await self.store.requeue_failed_attempt( - stored, queue_limit=pool.config.scheduling.queue_limit, + stored, + queue_limit=pool.config.scheduling.queue_limit, retry_wait_timeout_s=getattr(self, "_retry_wait_timeout_s", 600), - quarantine_until_ms=(max(task.deadline_at_ms or now_ms(), now_ms()) - + int(pool.config.scheduling.abort_grace_s * 1000)) if quarantine - else now_ms() + int(getattr(self, "_failed_instance_backoff_s", 30) * 1000), + quarantine_until_ms=( + max(task.deadline_at_ms or now_ms(), now_ms()) + + int(pool.config.scheduling.abort_grace_s * 1000) + ) + if quarantine + else now_ms() + + int(getattr(self, "_failed_instance_backoff_s", 30) * 1000), ) break except StoreConflict: current = await self.store.get_task(task.id) - if (current is None or current.task.status not in {TaskStatus.DISPATCHING, TaskStatus.IN_PROGRESS} - or not self._same_execution_owner(current.task, task) - or current.task.cancel_requested_at_ms is not None): + if ( + current is None + or current.task.status + not in {TaskStatus.DISPATCHING, TaskStatus.IN_PROGRESS} + or not self._same_execution_owner(current.task, task) + or current.task.cancel_requested_at_ms is not None + ): return False stored = current await asyncio.sleep(0.01) if retry is None: return False - self.telemetry.increment("dingo_video_worker_retries_total", labels={"pool": task.pool_id}) - self.telemetry.record_transition("worker_retry_queued", task, retry.task, - gateway_generation=self.generation, revision=retry.revision) - logger.warning("queued one Worker retry for task %s after attempt %s; excluding instance %s", - task.id, task.attempt, task.worker_instance_id) + self.telemetry.increment( + "dingo_video_worker_retries_total", labels={"pool": task.pool_id} + ) + self.telemetry.record_transition( + "worker_retry_queued", + task, + retry.task, + gateway_generation=self.generation, + revision=retry.revision, + ) + logger.warning( + "queued one Worker retry for task %s after attempt %s; excluding instance %s", + task.id, + task.attempt, + task.worker_instance_id, + ) pool.wakeup.set() return True @@ -2118,7 +2660,9 @@ async def _finish_failed( latest = await self.store.get_task(task_id) if latest is None or latest.task.status in TERMINAL_STATUSES: return - if expected_execution is not None and not self._same_execution_owner(latest.task, expected_execution): + if expected_execution is not None and not self._same_execution_owner( + latest.task, expected_execution + ): return if retryable and latest.task.cancel_requested_at_ms is None: previous = latest @@ -2130,8 +2674,11 @@ async def _finish_failed( # before falling back; never fail an already requeued attempt. logger.exception("Worker retry decision failed for task %s", task_id) latest = await self.store.get_task(task_id) - if (latest is None or latest.task.status in TERMINAL_STATUSES - or not self._same_execution_owner(latest.task, previous.task)): + if ( + latest is None + or latest.task.status in TERMINAL_STATUSES + or not self._same_execution_owner(latest.task, previous.task) + ): return if latest.task.cancel_requested_at_ms is not None: try: @@ -2207,9 +2754,7 @@ async def _recover(self) -> None: "status": TaskStatus.FAILED, "completed_at_ms": now_ms(), "expires_at_ms": now_ms() - + int( - self.config.lifecycle.failed_ttl_s * 1000 - ), + + int(self.config.lifecycle.failed_ttl_s * 1000), "error": terminal_error( "configuration_changed", "task pool configuration changed before dispatch", @@ -2249,6 +2794,18 @@ async def _recover(self) -> None: async def _recover_orphaned_active(self, stored: StoredTask) -> None: task = stored.task pool = self.pools.get(task.pool_id) + if task.status == TaskStatus.FINALIZING and read_handoff(task) is not None: + if pool is None: + return + if task.owner_generation == self.generation: + await self._schedule_result_finalizer(pool, stored) + elif self.store.gateway_owner_supported: + claimed = await self.store.claim_finalizing( + stored, new_owner_generation=self.generation + ) + if claimed is not None: + await self._schedule_result_finalizer(pool, claimed) + return if ( self.store.gateway_owner_supported and pool is not None @@ -2258,8 +2815,7 @@ async def _recover_orphaned_active(self, stored: StoredTask) -> None: and task.worker_instance_id is not None ): weight_bytes = ( - task.estimated_payload_bytes - or self.config.media.max_task_memory_bytes + task.estimated_payload_bytes or self.config.media.max_task_memory_bytes ) if not await self.memory_budget.try_acquire(task.id, weight_bytes): return @@ -2345,11 +2901,7 @@ async def _recover_orphaned_active(self, stored: StoredTask) -> None: release_lease=True, quarantine_until_ms=max(task.deadline_at_ms or now_ms(), now_ms()) + int( - ( - pool.config.scheduling.abort_grace_s - if pool is not None - else 30.0 - ) + (pool.config.scheduling.abort_grace_s if pool is not None else 30.0) * 1000 ), ) @@ -2432,7 +2984,7 @@ async def expire_terminal(self, stored: StoredTask) -> StoredTask: return stored if task.artifact_deleted_at_ms is not None: return stored - task_root = self.artifacts.task_root( + task_root = await self.artifacts.resolve_task_root( task.deployment_id, task.pool_id, task.id ) self._artifact_released_bytes += await self.artifacts.discard(task_root) @@ -2475,7 +3027,10 @@ async def _cleanup_orphan_tasks(self) -> None: candidate, dry_run=self.config.lifecycle.orphan_cleanup_dry_run, ) - if moved is not None and not self.config.lifecycle.orphan_cleanup_dry_run: + if ( + moved is not None + and not self.config.lifecycle.orphan_cleanup_dry_run + ): self._orphan_trashed_total += 1 except Exception: self._artifact_cleanup_failures += 1 @@ -2519,8 +3074,12 @@ async def _sweep_once(self) -> None: "expires_at_ms": current + int(self.config.lifecycle.failed_ttl_s * 1000), "error": terminal_error( - "retry_wait_timeout" if task.attempt > 0 else "queue_timeout", - "video retry wait expired" if task.attempt > 0 else "video task expired while queued" + "retry_wait_timeout" + if task.attempt > 0 + else "queue_timeout", + "video retry wait expired" + if task.attempt > 0 + else "video task expired while queued", ), }, ) @@ -2530,7 +3089,9 @@ async def _sweep_once(self) -> None: failed.task, gateway_generation=self.generation, revision=failed.revision, - reason="retry_wait_timeout" if task.attempt > 0 else "queue_timeout", + reason="retry_wait_timeout" + if task.attempt > 0 + else "queue_timeout", ) elif task.status in { TaskStatus.COMPLETED, diff --git a/dingo/video_gateway/errors.py b/dingo/video_gateway/errors.py index bf7a2ba3a21a..118d3cf62550 100644 --- a/dingo/video_gateway/errors.py +++ b/dingo/video_gateway/errors.py @@ -42,6 +42,10 @@ class StoreUnavailable(RuntimeError): """Raised when the configured Task Store cannot be reached safely.""" +class HandoffReservationLost(StoreConflict): + """Result handoff definitively no longer owns its execution reservation.""" + + class ResultTooLarge(RuntimeError): """Raised when a Worker result exceeds the configured artifact policy.""" @@ -65,7 +69,9 @@ def worker_execution_error(error: object) -> WorkerExecutionFailed: message = error if isinstance(error, dict): if error.get("code") != "worker_failed": - return WorkerExecutionFailed(str(error.get("message") or "Worker generation failed")) + return WorkerExecutionFailed( + str(error.get("message") or "Worker generation failed") + ) message = error.get("message") if isinstance(message, str) and message in { "Executor shut down", diff --git a/dingo/video_gateway/file_io.py b/dingo/video_gateway/file_io.py new file mode 100644 index 000000000000..57f45ac87c40 --- /dev/null +++ b/dingo/video_gateway/file_io.py @@ -0,0 +1,72 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Run filesystem work off-loop without abandoning in-flight file operations.""" + +from __future__ import annotations + +import asyncio +from contextlib import asynccontextmanager +from threading import Event +from typing import Any, Callable + + +async def run_file_io(function: Callable[..., Any], /, *args, **kwargs) -> Any: + """Drain a started operation before propagating cancellation to its owner. + + Cancelling an asyncio future cannot stop an OS filesystem call. Waiting for + that call to finish keeps subsequent close/unlink and budget release from + racing with a still-running read or write. It does not block the event loop. + A permanently hung filesystem still requires storage/node recovery. + """ + work = asyncio.create_task(asyncio.to_thread(function, *args, **kwargs)) + return await _drain_on_cancel(work) + + +async def run_cancellable_file_io( + function: Callable[..., Any], /, *args, **kwargs +) -> Any: + """Run a multi-step operation with a cooperative cancellation Event. + + The function receives the Event as its first argument and checks it between + safe stages. Cancellation still drains the active syscall/thread before + returning, including when the caller is cancelled repeatedly. + """ + cancelled = Event() + work = asyncio.create_task(asyncio.to_thread(function, cancelled, *args, **kwargs)) + return await _drain_on_cancel(work, on_cancel=cancelled.set) + + +async def _drain_on_cancel( + work: asyncio.Task, on_cancel: Callable[[], None] | None = None +) -> Any: + try: + return await asyncio.shield(work) + except asyncio.CancelledError: + if on_cancel is not None: + on_cancel() + while not work.done(): + try: + await asyncio.shield(work) + except asyncio.CancelledError: + continue + except BaseException: + break + if not work.cancelled(): + work.exception() # retrieve any worker error; preserve cancellation + raise + + +@asynccontextmanager +async def opened_file(opener: Callable[..., Any], /, *args, **kwargs): + """Close even when cancellation arrives while the file is being opened.""" + opened = [] + + def acquire(): + stream = opener(*args, **kwargs) + opened.append(stream) + return stream + + try: + yield await run_file_io(acquire) + finally: + if opened: + await run_file_io(opened[0].close) diff --git a/dingo/video_gateway/finalization.py b/dingo/video_gateway/finalization.py new file mode 100644 index 000000000000..9cd116636471 --- /dev/null +++ b/dingo/video_gateway/finalization.py @@ -0,0 +1,247 @@ +"""Recoverable postprocessing: no Worker calls, slot leases or model retries.""" + +from __future__ import annotations + +import asyncio +import errno +import time + +from dingo.video_gateway.artifact_store import ResultTooLarge +from dingo.video_gateway.errors import StoreConflict +from dingo.video_gateway.file_io import run_file_io +from dingo.video_gateway.models import TaskError, TaskStatus, now_ms +from dingo.video_gateway.result_handoff import HANDOFF_KEY, read_handoff + + +def same_owner(current, expected): + return (current.owner_generation, current.attempt, current.execution_token) == ( + expected.owner_generation, + expected.attempt, + expected.execution_token, + ) + + +def transient_result_error(exc): + return isinstance(exc, (TimeoutError, ConnectionError)) or ( + isinstance(exc, OSError) + and exc.errno + in { + errno.EAGAIN, + errno.EINTR, + errno.ETIMEDOUT, + errno.EIO, + errno.ESTALE, + errno.EBUSY, + errno.ENOSPC, + } + ) + + +class ResultFinalizer: + def __init__(self, store, artifacts, config, telemetry, generation): + self.store, self.artifacts, self.config = store, artifacts, config + self.telemetry, self.generation = telemetry, generation + + async def _terminal(self, stored, status, error): + ttl = ( + self.config.lifecycle.cancelled_ttl_s + if status == TaskStatus.CANCELLED + else self.config.lifecycle.failed_ttl_s + ) + result = await self.store.transition( + stored.task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=stored.revision, + patch={ + "status": status, + "error": TaskError(*error), + "completed_at_ms": now_ms(), + "expires_at_ms": now_ms() + int(ttl * 1000), + }, + release_lease=False, + ) + self.telemetry.record_transition( + status.value, + stored.task, + result.task, + gateway_generation=self.generation, + revision=result.revision, + ) + + async def run(self, stored, pool): + expected = stored.task + settings = pool.config.scheduling + while True: + current = await self.store.get_task(expected.id) + if ( + current is None + or current.task.status != TaskStatus.FINALIZING + or not same_owner(current.task, expected) + ): + return + task = current.task + reference = read_handoff(task) + if reference is None: + raise ValueError("finalizer requires a durable handoff") + if task.cancel_requested_at_ms is not None: + try: + await self._terminal( + current, + TaskStatus.CANCELLED, + ("cancelled", "video task was cancelled"), + ) + return + except StoreConflict: + continue + remaining = (reference["deadline_at_ms"] - now_ms()) / 1000 + if remaining <= 0: + try: + await self._terminal( + current, + TaskStatus.FAILED, + ( + "finalization_timeout", + "result finalization deadline exceeded", + ), + ) + return + except StoreConflict: + continue + final_path = None + started = time.monotonic() + try: + try: + final_path, size, digest, media = await asyncio.wait_for( + self.artifacts.finalize_worker_mp4( + task.deployment_id, + task.pool_id, + task.id, + task.attempt, + task.execution_token, + reference["artifact"], + task.normalized_request, + pool.adapter.validate_artifact, + pool.adapter.prepare_artifact, + pool.adapter.artifact_requires_processing, + inspector=pool.adapter.inspect_artifact_for_publication, + max_result_bytes=self.config.media.max_result_bytes, + ), + timeout=remaining, + ) + except asyncio.CancelledError: + raise + except Exception as exc: + latest = await self.store.get_task(task.id) + if ( + latest is None + or not same_owner(latest.task, expected) + or latest.task.status != TaskStatus.FINALIZING + ): + return + if latest.task.cancel_requested_at_ms is not None: + continue + ref = read_handoff(latest.task) + failures = int(ref.get("failures", 0)) + if ( + transient_result_error(exc) + and failures < settings.finalization_max_retries + and now_ms() < ref["deadline_at_ms"] + ): + ref["failures"] = failures + 1 + try: + await self.store.transition( + task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=latest.revision, + patch={ + "normalized_request": { + **latest.task.normalized_request, + HANDOFF_KEY: ref, + } + }, + ) + except StoreConflict: + continue + self.telemetry.increment( + "dingo_video_finalization_retries_total", + labels={"pool": task.pool_id}, + ) + await asyncio.sleep( + min( + settings.finalization_retry_delay_s * 2**failures, + max(0, (ref["deadline_at_ms"] - now_ms()) / 1000), + ) + ) + continue + if now_ms() >= ref["deadline_at_ms"]: + code = "finalization_timeout" + else: + code = ( + "result_too_large" + if isinstance(exc, ResultTooLarge) + else "finalization_failed" + ) + try: + await self._terminal( + latest, + TaskStatus.FAILED, + (code, "result validation or finalization failed"), + ) + return + except StoreConflict: + continue + duration = time.monotonic() - started + latest = await self.store.get_task(task.id) + if ( + latest is None + or latest.task.status != TaskStatus.FINALIZING + or not same_owner(latest.task, expected) + ): + return + if latest.task.cancel_requested_at_ms is not None: + continue + # A synchronous filesystem operation can finish after wait_for's + # logical deadline. Never publish a result past that deadline. + if now_ms() >= reference["deadline_at_ms"]: + continue + try: + completed = await self.store.transition( + task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=latest.revision, + patch={ + "status": TaskStatus.COMPLETED, + "completed_at_ms": now_ms(), + "expires_at_ms": now_ms() + + int(self.config.lifecycle.completed_ttl_s * 1000), + "result_path": str(final_path), + "result_bytes": size, + "result_sha256": digest, + "normalized_request": { + **latest.task.normalized_request, + "_result_media": media, + }, + "finalize_time_s": duration, + }, + release_lease=False, + ) + except StoreConflict: + continue + except Exception: + # Ambiguous etcd response: do not destroy a possibly committed + # candidate. Unique files not referenced after recovery are GC'd. + final_path = None + raise + final_path = None + self.telemetry.record_transition( + "completed", + latest.task, + completed.task, + gateway_generation=self.generation, + revision=completed.revision, + ) + self.telemetry.record_stage_duration(task.pool_id, "finalize", duration) + return + finally: + if final_path is not None: + await run_file_io(final_path.unlink, missing_ok=True) diff --git a/dingo/video_gateway/form_parser.py b/dingo/video_gateway/form_parser.py index 8d46eae4565d..d12e31969e92 100644 --- a/dingo/video_gateway/form_parser.py +++ b/dingo/video_gateway/form_parser.py @@ -5,7 +5,6 @@ from __future__ import annotations -import asyncio import hashlib import os from dataclasses import dataclass @@ -17,6 +16,7 @@ from dingo.video_gateway.artifact_store import FileArtifactStore from dingo.video_gateway.config import MediaConfig from dingo.video_gateway.errors import GatewayError +from dingo.video_gateway.file_io import opened_file, run_file_io @dataclass(slots=True) @@ -95,8 +95,7 @@ async def parse_multipart( temporary = destination.with_suffix(".part") digest = hashlib.sha256() size = 0 - output = temporary.open("xb") - try: + async with opened_file(temporary.open, "xb") as output: while True: chunk = await part.read_chunk(size=1024 * 1024) if not chunk: @@ -118,12 +117,10 @@ async def parse_multipart( "reference files exceed the aggregate upload limit", ) digest.update(chunk) - await asyncio.to_thread(output.write, chunk) - await asyncio.to_thread(output.flush) - await asyncio.to_thread(os.fsync, output.fileno()) - finally: - output.close() - await asyncio.to_thread(os.replace, temporary, destination) + await run_file_io(output.write, chunk) + await run_file_io(output.flush) + await run_file_io(os.fsync, output.fileno()) + await run_file_io(os.replace, temporary, destination) filename = Path(part.filename).name or f"upload-{ordinal}" uploads.append( UploadedArtifact( @@ -144,6 +141,6 @@ async def parse_multipart( upload_root=upload_root, total_bytes=total_file_bytes, ) - except Exception: + except BaseException: await artifacts.discard(upload_root) raise diff --git a/dingo/video_gateway/result_handoff.py b/dingo/video_gateway/result_handoff.py new file mode 100644 index 000000000000..d1a1aff35d03 --- /dev/null +++ b/dingo/video_gateway/result_handoff.py @@ -0,0 +1,82 @@ +"""Small, durable result references; never embed media bytes in task records.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from typing import Any + +from dingo.common.video_result_file import validate_descriptor +from dingo.video_gateway.models import TaskStatus, VideoTask, now_ms + +HANDOFF_KEY = "_result_handoff_v1" + + +def make_handoff( + task: VideoTask, descriptor: Mapping[str, Any], *, timeout_s: float +) -> dict[str, Any]: + if task.attempt < 1 or not task.execution_token or not task.owner_generation: + raise ValueError("result handoff requires a fenced execution") + if ( + isinstance(timeout_s, bool) + or not math.isfinite(timeout_s) + or not 0 < timeout_s <= 86400 + ): + raise ValueError("finalization timeout must be finite and within (0, 86400]") + timestamp = now_ms() + return { + "schema_version": 1, + "task_id": task.id, + "attempt": task.attempt, + "execution_token": task.execution_token, + "artifact": validate_descriptor(dict(descriptor)), + "created_at_ms": timestamp, + "deadline_at_ms": timestamp + int(timeout_s * 1000), + "failures": 0, + } + + +def read_handoff(task: VideoTask) -> dict[str, Any] | None: + raw = task.normalized_request.get(HANDOFF_KEY) + if raw is None: + return None + if not isinstance(raw, dict) or raw.get("schema_version") != 1: + raise ValueError("invalid result handoff schema") + if (raw.get("task_id"), raw.get("attempt"), raw.get("execution_token")) != ( + task.id, + task.attempt, + task.execution_token, + ): + raise ValueError("result handoff execution identity mismatch") + if ( + not task.execution_token + or type(raw.get("deadline_at_ms")) is not int + or type(raw.get("created_at_ms")) is not int + or raw["deadline_at_ms"] < raw["created_at_ms"] + or type(raw.get("failures")) is not int + or raw["failures"] < 0 + ): + raise ValueError("incomplete result handoff") + validate_descriptor(dict(raw["artifact"])) + return dict(raw) + + +def validate_handoff_transition(before: VideoTask, after: VideoTask) -> None: + if before.status != TaskStatus.IN_PROGRESS or after.status != TaskStatus.FINALIZING: + raise ValueError("execution handoff requires in_progress -> finalizing") + if before.cancel_requested_at_ms is not None: + raise ValueError("cannot hand off a cancelled execution") + if ( + before.owner_generation, + before.attempt, + before.execution_token, + before.worker_key, + ) != ( + after.owner_generation, + after.attempt, + after.execution_token, + after.worker_key, + ): + raise ValueError("handoff cannot change execution identity") + if read_handoff(after) is None: + raise ValueError("durable result reference missing") diff --git a/dingo/video_gateway/service.py b/dingo/video_gateway/service.py index ecce6b8a9908..ecb4b772b91c 100644 --- a/dingo/video_gateway/service.py +++ b/dingo/video_gateway/service.py @@ -5,7 +5,6 @@ from __future__ import annotations -import asyncio import hashlib import json from collections.abc import Mapping, Sequence @@ -17,6 +16,7 @@ from dingo.video_gateway.config import GatewayConfig, PoolConfig from dingo.video_gateway.dispatcher import VideoDispatcher from dingo.video_gateway.errors import GatewayError, StoreConflict +from dingo.video_gateway.file_io import run_file_io from dingo.video_gateway.ids import new_video_id from dingo.video_gateway.models import StoredTask, TaskStatus, VideoTask, now_ms from dingo.video_gateway.task_store import TaskStore @@ -60,12 +60,9 @@ async def ensure_submission_capacity(self, required_bytes: int) -> None: watermarks = self.config.artifact_store capacity = await self.artifacts.capacity() remaining = capacity.free_bytes - required_bytes - if ( - remaining < watermarks.hard_min_free_bytes - or ( - watermarks.soft_min_free_bytes > 0 - and remaining < watermarks.soft_min_free_bytes - ) + if remaining < watermarks.hard_min_free_bytes or ( + watermarks.soft_min_free_bytes > 0 + and remaining < watermarks.soft_min_free_bytes ): await self.dispatcher.sweep_now() capacity = await self.artifacts.capacity() @@ -126,7 +123,7 @@ async def submit( normalized_fields = {key: list(values) for key, values in fields.items()} normalized_fields["model"] = [model] try: - normalized = await asyncio.to_thread( + normalized = await run_file_io( adapter.normalize_request, normalized_fields, uploads, model ) except Exception: @@ -228,17 +225,15 @@ async def submit( ) raise GatewayError(429, "queue_full", "video queue is full") except Exception: - if upload_root.exists(): - await self.artifacts.discard(upload_root) + # discard is idempotent and performs its existence check off-loop. + await self.artifacts.discard(upload_root) raise task_root: Path | None = None try: task_id = new_video_id() created_at = now_ms() - expires_at_ms = created_at + int( - self.config.lifecycle.queue_ttl_s * 1000 - ) + expires_at_ms = created_at + int(self.config.lifecycle.queue_ttl_s * 1000) task_root = await self.artifacts.commit_upload( upload_root, self.config.deployment_id, diff --git a/dingo/video_gateway/task_store.py b/dingo/video_gateway/task_store.py index 466aa15ede4c..f5885d63d7bf 100644 --- a/dingo/video_gateway/task_store.py +++ b/dingo/video_gateway/task_store.py @@ -15,7 +15,11 @@ from dataclasses import dataclass from typing import Any -from dingo.video_gateway.errors import GatewayError, StoreConflict +from dingo.video_gateway.errors import ( + GatewayError, + HandoffReservationLost, + StoreConflict, +) from dingo.video_gateway.etcd_http import EtcdHttpClient, EtcdValue from dingo.video_gateway.models import ( ACTIVE_STATUSES, @@ -28,6 +32,7 @@ WorkerLease, now_ms, ) +from dingo.video_gateway.result_handoff import read_handoff, validate_handoff_transition _INDEX_SCHEMA_VERSION = 3 _SEQUENCE_WIDTH = 20 @@ -49,10 +54,31 @@ class TaskWatchEvent: created: bool = False -def worker_key(backend_target: str, instance_id: int | str) -> str: - return hashlib.sha256( +def worker_key(backend_target: str, instance_id: int | str, slot_id: int = 0) -> str: + """Opaque lease identity; slot zero preserves every existing persisted key.""" + if isinstance(slot_id, bool) or not isinstance(slot_id, int) or slot_id < 0: + raise ValueError("slot_id must be a non-negative integer") + physical = hashlib.sha256( backend_target.encode() + b"\0" + str(instance_id).encode() ).hexdigest() + if slot_id == 0: + return physical + return hashlib.sha256(f"{physical}\0slot\0{slot_id}".encode()).hexdigest() + + +def retry_excludes_worker( + task: VideoTask, backend_target: str, instance_id: int | str +) -> bool: + """A retry must leave the entire failed instance, not merely its busy slot.""" + if task.attempt == 0: + return False + if task.worker_instance_id is None: + # Missing physical identity is not permission to reuse an unknown + # failed process. Well-formed historical records include this field. + return True + return worker_key(task.backend_target, task.worker_instance_id) == worker_key( + backend_target, instance_id + ) def _clone_task(task: VideoTask) -> VideoTask: @@ -80,6 +106,12 @@ def _apply_patch(task: VideoTask, patch: Mapping[str, Any]) -> VideoTask: class TaskStore(ABC): + async def claim_finalizing( + self, stored: StoredTask, *, new_owner_generation: str + ) -> StoredTask | None: + """Take over durable postprocessing without acquiring any Worker slot.""" + raise NotImplementedError("this TaskStore cannot take over finalization") + @abstractmethod async def health(self) -> None: ... @@ -123,9 +155,7 @@ async def try_acquire_discovery_recovery( "this TaskStore does not support discovery recovery fencing" ) - async def lease_snapshot( - self, pool_id: str - ) -> tuple[dict[str, WorkerLease], int]: + async def lease_snapshot(self, pool_id: str) -> tuple[dict[str, WorkerLease], int]: leases = await self.list_leases(pool_id) return {lease.worker_key: lease for lease in leases}, 0 @@ -231,6 +261,7 @@ async def transition( patch: Mapping[str, Any], expected_revision: int | None = None, release_lease: bool = False, + release_execution: bool = False, quarantine_until_ms: int | None = None, ) -> StoredTask: ... @@ -536,6 +567,7 @@ async def transition( patch: Mapping[str, Any], expected_revision: int | None = None, release_lease: bool = False, + release_execution: bool = False, quarantine_until_ms: int | None = None, ) -> StoredTask: expected_set = set(expected) @@ -551,6 +583,23 @@ async def transition( + ", ".join(sorted(status.value for status in expected_set)) ) updated = _apply_patch(current[0], patch) + if release_execution: + validate_handoff_transition(current[0], updated) + lease = self._leases.get((current[0].pool_id, current[0].worker_key)) + if ( + not release_lease + or quarantine_until_ms is not None + or lease is None + or (lease.task_id, lease.owner_generation, lease.execution_token) + != ( + task_id, + current[0].owner_generation, + current[0].execution_token, + ) + ): + raise HandoffReservationLost( + "result handoff lost its Worker reservation" + ) if ( current[0].status == TaskStatus.QUEUED and updated.status != TaskStatus.QUEUED @@ -750,9 +799,15 @@ def _retry_credit_key(self, task: VideoTask) -> str: @staticmethod def _is_retry_waiting(task: VideoTask) -> bool: - return task.status == TaskStatus.QUEUED and task.attempt == 1 and task.worker_key is not None + return ( + task.status == TaskStatus.QUEUED + and task.attempt == 1 + and task.worker_key is not None + ) - async def _retry_counter(self, pool_id: str, kind: str) -> tuple[int, EtcdValue | None]: + async def _retry_counter( + self, pool_id: str, kind: str + ) -> tuple[int, EtcdValue | None]: value = await self.client.get(self._retry_counter_key(pool_id, kind)) count = int(value.value) if value is not None else 0 if count < 0: @@ -765,7 +820,9 @@ async def retry_queue_depth(self, pool_id: str) -> int: async def retry_budget_used(self, pool_id: str) -> int: return (await self._retry_counter(pool_id, "credits"))[0] - async def _require_retry_counter(self, stored: StoredTask, count: int, kind: str) -> None: + async def _require_retry_counter( + self, stored: StoredTask, count: int, kind: str + ) -> None: if count > 0: return current = await self.get_task(stored.task.id) @@ -907,9 +964,7 @@ async def prepare(self) -> None: tasks = [self._task(value) for value in values] tasks.sort(key=lambda item: (item.task.created_at_ms, item.task.id)) sequence, sequence_value = await self._sequence() - next_sequence = max( - [sequence, *(item.task.created_seq for item in tasks)] - ) + next_sequence = max([sequence, *(item.task.created_seq for item in tasks)]) conflicted = False for stored in tasks: task = stored.task @@ -944,11 +999,7 @@ async def prepare(self) -> None: ) ) succeeded, _revision = await self.client.txn( - [ - self.client.compare_mod( - self._task_key(task.id), stored.revision - ) - ], + [self.client.compare_mod(self._task_key(task.id), stored.revision)], success, ) if not succeeded: @@ -962,9 +1013,7 @@ async def prepare(self) -> None: compare = self._counter_compare( self._sequence_key(), current_sequence_value ) - success = [ - self.client.put(self._sequence_key(), str(target_sequence)) - ] + success = [self.client.put(self._sequence_key(), str(target_sequence))] if ( current_sequence == target_sequence and current_sequence_value is not None @@ -987,9 +1036,10 @@ async def prepare(self) -> None: if succeeded: return marker = await self.client.get(marker_key) - if marker is not None and marker.value == str( - _INDEX_SCHEMA_VERSION - ).encode(): + if ( + marker is not None + and marker.value == str(_INDEX_SCHEMA_VERSION).encode() + ): return raise StoreConflict("unable to prepare task indexes after repeated races") @@ -1040,9 +1090,11 @@ async def create_task( await asyncio.wait_for(self._create_task_lock.acquire(), timeout=5.0) except asyncio.TimeoutError as exc: raise GatewayError( - 503, "store_busy", + 503, + "store_busy", "task admission is busy; retry with the same Idempotency-Key", - error_type="service_unavailable_error", headers={"Retry-After": "1"}, + error_type="service_unavailable_error", + headers={"Retry-After": "1"}, ) from exc try: return await self._create_task_serialized( @@ -1097,7 +1149,9 @@ async def _create_task_serialized( return stored, False count, counter = await self._counter(task.pool_id) - retry_count, retry_counter = await self._retry_counter(task.pool_id, "waiting") + retry_count, retry_counter = await self._retry_counter( + task.pool_id, "waiting" + ) sequence, sequence_value = await self._sequence() if count - retry_count >= queue_limit: raise GatewayError(429, "queue_full", "video queue is full") @@ -1106,7 +1160,9 @@ async def _create_task_serialized( self.client.compare_version(task_key, 0), self.client.compare_version(queue_key, 0), *self._counter_compare(counter_key, counter), - *self._counter_compare(self._retry_counter_key(task.pool_id, "waiting"), retry_counter), + *self._counter_compare( + self._retry_counter_key(task.pool_id, "waiting"), retry_counter + ), *self._counter_compare(self._sequence_key(), sequence_value), ] success = [ @@ -1143,13 +1199,18 @@ async def _create_task_serialized( await asyncio.sleep(min(delay, remaining)) attempt += 1 raise GatewayError( - 503, "store_busy", + 503, + "store_busy", "task admission contention timed out; retry with the same Idempotency-Key", - error_type="service_unavailable_error", headers={"Retry-After": "1"}, + error_type="service_unavailable_error", + headers={"Retry-After": "1"}, ) async def requeue_failed_attempt( - self, stored: StoredTask, *, queue_limit: int, + self, + stored: StoredTask, + *, + queue_limit: int, quarantine_until_ms: int | None = None, retry_wait_timeout_s: float = 600, ) -> StoredTask | None: @@ -1161,10 +1222,14 @@ async def requeue_failed_attempt( state machine's allowed transitions. """ task = stored.task - if (task.status not in {TaskStatus.DISPATCHING, TaskStatus.IN_PROGRESS} - or task.attempt != 1 or task.cancel_requested_at_ms is not None - or not task.owner_generation or not task.worker_key - or not task.execution_token): + if ( + task.status not in {TaskStatus.DISPATCHING, TaskStatus.IN_PROGRESS} + or task.attempt != 1 + or task.cancel_requested_at_ms is not None + or not task.owner_generation + or not task.worker_key + or not task.execution_token + ): return None count, counter = await self._counter(task.pool_id) credit = await self.client.get(self._retry_credit_key(task)) @@ -1176,8 +1241,11 @@ async def requeue_failed_attempt( if lease_value is None: return None lease = WorkerLease.from_dict(json.loads(lease_value.value)) - if (lease.task_id != task.id or lease.owner_generation != task.owner_generation - or lease.execution_token != task.execution_token): + if ( + lease.task_id != task.id + or lease.owner_generation != task.owner_generation + or lease.execution_token != task.execution_token + ): return None updated = _clone_task(task) updated.status = TaskStatus.QUEUED @@ -1202,22 +1270,36 @@ async def requeue_failed_attempt( self.client.compare_mod(lease_key, lease_value.mod_revision), self.client.compare_mod(self._retry_credit_key(task), credit.mod_revision), self.client.compare_version(queue_key, 0), - self.client.compare_version(self._gateway_key(task.owner_generation), 0, result="GREATER"), + self.client.compare_version( + self._gateway_key(task.owner_generation), 0, result="GREATER" + ), *self._counter_compare(counter_key, counter), - *self._counter_compare(self._retry_counter_key(task.pool_id, "waiting"), waiting_value), + *self._counter_compare( + self._retry_counter_key(task.pool_id, "waiting"), waiting_value + ), ] success = [ self.client.put(self._task_key(task.id), self._encode(updated.to_dict())), self.client.put(queue_key, str(updated.queued_at_ms)), self.client.put(self._ordered_queue_key(updated), task.id), self.client.put(counter_key, str(count + 1)), - self.client.put(self._retry_counter_key(task.pool_id, "waiting"), str(waiting + 1)), + self.client.put( + self._retry_counter_key(task.pool_id, "waiting"), str(waiting + 1) + ), self.client.delete(self._expiry_index_key(task)), self.client.put(self._expiry_index_key(updated), task.id), self.client.delete(self._owner_task_key(task.owner_generation, task.id)), - self.client.delete(self._lease_heartbeat_key(task.pool_id, task.worker_key)), - *[self.client.delete(key) for key in self._status_task_index_keys(task, task.status)], - *[self.client.put(key, task.id) for key in self._status_task_index_keys(updated, updated.status)], + self.client.delete( + self._lease_heartbeat_key(task.pool_id, task.worker_key) + ), + *[ + self.client.delete(key) + for key in self._status_task_index_keys(task, task.status) + ], + *[ + self.client.put(key, task.id) + for key in self._status_task_index_keys(updated, updated.status) + ], ] if quarantine_until_ms is None: success.append(self.client.delete(lease_key)) @@ -1430,8 +1512,12 @@ async def reserve( if count <= 0: return None retry_waiting = self._is_retry_waiting(stored.task) - if retry_waiting and (stored.task.expires_at_ms <= now_ms() - or stored.task.worker_key == lease.worker_key): + if retry_waiting and ( + stored.task.expires_at_ms <= now_ms() + or retry_excludes_worker( + stored.task, lease.backend_target, lease.worker_instance_id + ) + ): return None accounting_compare, accounting_success = [], [] credit_key = self._retry_credit_key(stored.task) @@ -1440,27 +1526,43 @@ async def reserve( used_key = self._retry_counter_key(stored.task.pool_id, "credits") accounting_compare.extend(self._counter_compare(used_key, used_value)) if retry_waiting: - waiting, waiting_value = await self._retry_counter(stored.task.pool_id, "waiting") + waiting, waiting_value = await self._retry_counter( + stored.task.pool_id, "waiting" + ) await self._require_retry_counter(stored, waiting, "waiting") await self._require_retry_counter(stored, used, "credits") credit = await self.client.get(credit_key) if credit is None: return None - accounting_compare.extend([ - self.client.compare_mod(credit_key, credit.mod_revision), - *self._counter_compare(self._retry_counter_key(stored.task.pool_id, "waiting"), waiting_value), - ]) - accounting_success.extend([ - self.client.delete(credit_key), - self.client.put(used_key, str(used - 1)), - self.client.put(self._retry_counter_key(stored.task.pool_id, "waiting"), str(waiting - 1)), - ]) + accounting_compare.extend( + [ + self.client.compare_mod(credit_key, credit.mod_revision), + *self._counter_compare( + self._retry_counter_key(stored.task.pool_id, "waiting"), + waiting_value, + ), + ] + ) + accounting_success.extend( + [ + self.client.delete(credit_key), + self.client.put(used_key, str(used - 1)), + self.client.put( + self._retry_counter_key(stored.task.pool_id, "waiting"), + str(waiting - 1), + ), + ] + ) else: if used >= retry_limit: return None accounting_compare.append(self.client.compare_version(credit_key, 0)) - accounting_success.extend([self.client.put(credit_key, stored.task.id), - self.client.put(used_key, str(used + 1))]) + accounting_success.extend( + [ + self.client.put(credit_key, stored.task.id), + self.client.put(used_key, str(used + 1)), + ] + ) native_lease = await self.client.lease_grant(self.execution_lease_ttl_s) lease = copy.deepcopy(lease) lease.etcd_lease_id = native_lease.lease_id @@ -1476,7 +1578,9 @@ async def reserve( "execution_token": lease.execution_token, "assigned_at_ms": now_ms(), "deadline_at_ms": deadline_at_ms, - "expires_at_ms": max(stored.task.expires_at_ms, deadline_at_ms) if retry_waiting else stored.task.expires_at_ms, + "expires_at_ms": max(stored.task.expires_at_ms, deadline_at_ms) + if retry_waiting + else stored.task.expires_at_ms, }, ) compare = [ @@ -1515,9 +1619,7 @@ async def reserve( ), *[ self.client.delete(key) - for key in self._status_task_index_keys( - stored.task, stored.task.status - ) + for key in self._status_task_index_keys(stored.task, stored.task.status) ], *[ self.client.put(key, updated.id) @@ -1525,8 +1627,12 @@ async def reserve( ], ] if updated.expires_at_ms != stored.task.expires_at_ms: - success.extend([self.client.delete(self._expiry_index_key(stored.task)), - self.client.put(self._expiry_index_key(updated), updated.id)]) + success.extend( + [ + self.client.delete(self._expiry_index_key(stored.task)), + self.client.put(self._expiry_index_key(updated), updated.id), + ] + ) try: succeeded, revision = await self.client.txn(compare, success) except Exception: @@ -1550,9 +1656,81 @@ async def transition( patch: Mapping[str, Any], expected_revision: int | None = None, release_lease: bool = False, + release_execution: bool = False, quarantine_until_ms: int | None = None, ) -> StoredTask: + # A transition also compares shared queue/retry counters and the Worker + # lease. Losing one of those comparisons does not mean this task changed. + # Pin its original revision: never retry across cancellation or takeover. stored = await self.get_task(task_id) + if stored is None: + raise KeyError(task_id) + revision = stored.revision if expected_revision is None else expected_revision + expected = tuple(expected) + for attempt in range(32): + try: + return await self._transition_once( + task_id, + expected=expected, + patch=patch, + expected_revision=revision, + release_lease=release_lease, + release_execution=release_execution, + quarantine_until_ms=quarantine_until_ms, + task_hint=stored, + ) + except StoreConflict: + current = await self.get_task(task_id) + if ( + current is None + or current.revision != revision + or current.task.status not in expected + or attempt == 31 + ): + raise + await asyncio.sleep( + random.uniform(0.001, min(0.05, 0.002 * 2 ** min(attempt, 5))) + ) + raise AssertionError("unreachable") + + async def _transition_once( + self, + task_id: str, + *, + expected: Iterable[TaskStatus], + patch: Mapping[str, Any], + expected_revision: int | None = None, + release_lease: bool = False, + release_execution: bool = False, + quarantine_until_ms: int | None = None, + task_hint: StoredTask | None = None, + ) -> StoredTask: + ledger_snapshot: dict[str, EtcdValue | None] | None = None + # The hint only identifies keys. Re-read the task and its dependent + # ledger/lease records in one MVCC snapshot, then validate its pinned + # revision before using any of them. Never authorize a write from a + # caller's possibly stale task object. + if ( + task_hint is not None + and task_hint.task.id == task_id + and expected_revision is not None + and task_hint.task.status in ACTIVE_STATUSES + and patch.get("status") in TERMINAL_STATUSES + ): + hint = task_hint.task + keys = [ + self._task_key(task_id), + self._retry_credit_key(hint), + self._retry_counter_key(hint.pool_id, "credits"), + ] + if release_lease and hint.worker_key is not None: + keys.append(self._lease_key(hint.pool_id, hint.worker_key)) + values, _snapshot_revision = await self.client.get_many(keys) + ledger_snapshot = dict(zip(keys, values, strict=True)) + task_value = values[0] + stored = None if task_value is None else self._task(task_value) + else: + stored = await self.get_task(task_id) if stored is None: raise KeyError(task_id) expected_set = set(expected) @@ -1564,6 +1742,10 @@ async def transition( if expected_revision is not None and stored.revision != expected_revision: raise StoreConflict("task revision changed") updated = _apply_patch(stored.task, patch) + if release_execution: + validate_handoff_transition(stored.task, updated) + if not release_lease or quarantine_until_ms is not None: + raise ValueError("handoff must release, never quarantine execution") task_key = self._task_key(task_id) compare: list[dict] = [self.client.compare_mod(task_key, stored.revision)] success: list[dict] = [ @@ -1573,9 +1755,7 @@ async def transition( if stored.task.status != updated.status: success.extend( self.client.delete(key) - for key in self._status_task_index_keys( - stored.task, stored.task.status - ) + for key in self._status_task_index_keys(stored.task, stored.task.status) ) success.extend( self.client.put(key, updated.id) @@ -1630,28 +1810,71 @@ async def transition( ) if self._is_retry_waiting(stored.task) and updated.status != TaskStatus.QUEUED: - waiting, waiting_value = await self._retry_counter(stored.task.pool_id, "waiting") + waiting, waiting_value = await self._retry_counter( + stored.task.pool_id, "waiting" + ) await self._require_retry_counter(stored, waiting, "waiting") key = self._retry_counter_key(stored.task.pool_id, "waiting") compare.extend(self._counter_compare(key, waiting_value)) success.append(self.client.put(key, str(waiting - 1))) - if updated.status in TERMINAL_STATUSES and stored.task.status not in TERMINAL_STATUSES: + if ( + updated.status in TERMINAL_STATUSES or release_execution + ) and stored.task.status not in TERMINAL_STATUSES: credit_key = self._retry_credit_key(stored.task) - credit = await self.client.get(credit_key) + credit = ( + ledger_snapshot[credit_key] + if ledger_snapshot is not None + else await self.client.get(credit_key) + ) if credit is not None: - used, used_value = await self._retry_counter(stored.task.pool_id, "credits") - await self._require_retry_counter(stored, used, "credits") key = self._retry_counter_key(stored.task.pool_id, "credits") - compare.extend([self.client.compare_mod(credit_key, credit.mod_revision), - *self._counter_compare(key, used_value)]) - success.extend([self.client.delete(credit_key), self.client.put(key, str(used - 1))]) + if ledger_snapshot is None: + used, used_value = await self._retry_counter( + stored.task.pool_id, "credits" + ) + else: + used_value = ledger_snapshot[key] + used = int(used_value.value) if used_value is not None else 0 + if used < 0: + raise RuntimeError("negative retry credits counter") + await self._require_retry_counter(stored, used, "credits") + compare.extend( + [ + self.client.compare_mod(credit_key, credit.mod_revision), + *self._counter_compare(key, used_value), + ] + ) + success.extend( + [ + self.client.delete(credit_key), + self.client.put(key, str(used - 1)), + ] + ) if release_lease and stored.task.worker_key is not None: lease_key = self._lease_key(stored.task.pool_id, stored.task.worker_key) heartbeat_key = self._lease_heartbeat_key( stored.task.pool_id, stored.task.worker_key ) - lease_value = await self.client.get(lease_key) + lease_value = ( + ledger_snapshot[lease_key] + if ledger_snapshot is not None + else await self.client.get(lease_key) + ) + if release_execution: + if lease_value is None: + raise HandoffReservationLost( + "result handoff lost its Worker reservation" + ) + held = WorkerLease.from_dict(json.loads(lease_value.value)) + if (held.task_id, held.owner_generation, held.execution_token) != ( + task_id, + stored.task.owner_generation, + stored.task.execution_token, + ): + raise HandoffReservationLost( + "result handoff lost its Worker reservation" + ) if lease_value is not None: lease = WorkerLease.from_dict(json.loads(lease_value.value)) if lease.task_id == task_id: @@ -1693,9 +1916,7 @@ async def transition( compare.append(self.client.compare_version(lease_key, 0)) success.extend( [ - self.client.put( - lease_key, self._encode(quarantine.to_dict()) - ), + self.client.put(lease_key, self._encode(quarantine.to_dict())), self.client.delete(heartbeat_key), ] ) @@ -1746,9 +1967,7 @@ async def list_leases(self, pool_id: str) -> list[WorkerLease]: values = await self.client.range(self._lease_prefix(pool_id), prefix=True) return [WorkerLease.from_dict(json.loads(value.value)) for value in values] - async def lease_snapshot( - self, pool_id: str - ) -> tuple[dict[str, WorkerLease], int]: + async def lease_snapshot(self, pool_id: str) -> tuple[dict[str, WorkerLease], int]: values, revision = await self.client.range_all( self._lease_prefix(pool_id), prefix=True, @@ -1982,6 +2201,10 @@ async def claim_orphaned_active( self, stored: StoredTask, *, new_owner_generation: str ) -> StoredTask | None: task = stored.task + if task.status == TaskStatus.FINALIZING and read_handoff(task) is not None: + return await self.claim_finalizing( + stored, new_owner_generation=new_owner_generation + ) if ( task.status not in ACTIVE_STATUSES or not task.owner_generation @@ -2035,9 +2258,7 @@ async def claim_orphaned_active( ) compare = [ self.client.compare_mod(task_key, stored.revision), - self.client.compare_version( - self._gateway_key(task.owner_generation), 0 - ), + self.client.compare_version(self._gateway_key(task.owner_generation), 0), self.client.compare_version( self._gateway_key(new_owner_generation), 0, result="GREATER" ), @@ -2060,9 +2281,7 @@ async def claim_orphaned_active( ), lease_id=native_lease.lease_id, ), - self.client.delete( - self._owner_task_key(task.owner_generation, task.id) - ), + self.client.delete(self._owner_task_key(task.owner_generation, task.id)), self.client.put( self._owner_task_key(new_owner_generation, task.id), task.id ), @@ -2083,6 +2302,34 @@ async def claim_orphaned_active( return None return StoredTask(updated, revision) + async def claim_finalizing( + self, stored: StoredTask, *, new_owner_generation: str + ) -> StoredTask | None: + task = stored.task + if task.status != TaskStatus.FINALIZING or read_handoff(task) is None: + return None + if not task.owner_generation or task.owner_generation == new_owner_generation: + return None + updated = _apply_patch( + task, {"owner_generation": new_owner_generation, "worker_lease_id": None} + ) + comparisons = [ + self.client.compare_mod(self._task_key(task.id), stored.revision), + self.client.compare_version(self._gateway_key(task.owner_generation), 0), + self.client.compare_version( + self._gateway_key(new_owner_generation), 0, result="GREATER" + ), + ] + success = [ + self.client.put(self._task_key(task.id), self._encode(updated.to_dict())), + self.client.delete(self._owner_task_key(task.owner_generation, task.id)), + self.client.put( + self._owner_task_key(new_owner_generation, task.id), task.id + ), + ] + succeeded, revision = await self.client.txn(comparisons, success) + return StoredTask(updated, revision) if succeeded else None + async def release_lease(self, pool_id: str, worker_key_value: str) -> None: key = self._lease_key(pool_id, worker_key_value) heartbeat_key = self._lease_heartbeat_key(pool_id, worker_key_value) @@ -2160,9 +2407,7 @@ async def delete_expired(self, stored: StoredTask) -> bool: if stored.task.owner_generation: success.append( self.client.delete( - self._owner_task_key( - stored.task.owner_generation, stored.task.id - ) + self._owner_task_key(stored.task.owner_generation, stored.task.id) ) ) if stored.task.principal_hash and stored.task.idempotency_hash: @@ -2194,9 +2439,7 @@ async def reconcile_pool(self, pool_id: str) -> None: await self.prepare() for _ in range(8): index_values, snapshot_revision = await self.client.range_all( - self._task_index_prefix( - pool_id=pool_id, status=TaskStatus.QUEUED - ), + self._task_index_prefix(pool_id=pool_id, status=TaskStatus.QUEUED), prefix=True, ) task_ids = [value.value.decode() for value in index_values] @@ -2365,9 +2608,7 @@ async def reconcile_pool(self, pool_id: str) -> None: ordered_key = self._ordered_queue_key(task.task) succeeded, _ = await self.client.txn( [ - self.client.compare_mod( - self._task_key(task_id), task.revision - ), + self.client.compare_mod(self._task_key(task_id), task.revision), self.client.compare_version(ordered_key, 0), ], [self.client.put(ordered_key, task_id)], diff --git a/dingo/vllm/omni/args.py b/dingo/vllm/omni/args.py index 1cab48fe7bdf..38411609ebc6 100644 --- a/dingo/vllm/omni/args.py +++ b/dingo/vllm/omni/args.py @@ -80,6 +80,9 @@ class OmniDiffusionKwargs: enable_diffusion_pipeline_profiler: bool = False diffusion_attention_backend: Optional[str] = None diffusion_quantization_config: Optional[dict] = None + # Opt-in diffusion scheduler controls. None preserves the upstream default. + step_execution: bool = False + max_num_seqs: Optional[int] = None @dataclasses.dataclass @@ -193,8 +196,31 @@ def add_arguments(self, parser) -> None: "accepted video tasks during graceful shutdown." ), ) + add_argument( + g, + flag_name="--detached-video-prefetch-capacity", + env_var="DYN_OMNI_DETACHED_VIDEO_PREFETCH_CAPACITY", + arg_type=int, + default=0, + help="Additional detached requests waiting locally; 0 or 1, separate from engine execution capacity.", + ) # OmniDiffusionKwargs fields + add_negatable_bool_argument( + g, + flag_name="--step-execution", + env_var="DYN_OMNI_STEP_EXECUTION", + default=False, + help="Enable Omni diffusion step execution (required for concurrent H3 tasks).", + ) + add_argument( + g, + flag_name="--max-num-seqs", + env_var="DYN_OMNI_MAX_NUM_SEQS", + default=None, + arg_type=int, + help="Maximum concurrent sequences in the Omni diffusion engine; unset keeps its default.", + ) add_negatable_bool_argument( g, flag_name="--enable-layerwise-offload", @@ -515,6 +541,7 @@ class OmniConfig(DynamoRuntimeConfig): request_adapter_media_max_bytes: int = 2 * 1024 * 1024 * 1024 detached_video_task_root: Optional[str] = None detached_video_drain_timeout: float = 1800.0 + detached_video_prefetch_capacity: int = 0 # Nested structs — each group of fields has a clear destination diffusion: OmniDiffusionKwargs = dataclasses.field( @@ -559,6 +586,20 @@ def from_cli_args(cls, args: argparse.Namespace) -> "OmniConfig": def validate(self) -> None: DynamoRuntimeConfig.validate(self) + capacity = self.diffusion.max_num_seqs + if capacity is not None and ( + isinstance(capacity, bool) or not isinstance(capacity, int) or capacity < 1 + ): + raise ValueError("--max-num-seqs must be a positive integer") + if self.diffusion.step_execution and self.diffusion.cache_backend: + raise ValueError("--step-execution cannot be combined with --cache-backend") + if ( + self.request_adapter == "minimax_h3" + and capacity is not None + and capacity > 1 + and not self.diffusion.step_execution + ): + raise ValueError("concurrent MiniMax-H3 requires --step-execution") if self.default_video_fps <= 0: raise ValueError("--default-video-fps must be > 0") if self.request_adapter is None and self.request_adapter_workflow is not None: @@ -580,8 +621,7 @@ def validate(self) -> None: raise ValueError("--detached-video-task-root must not be empty") if self.request_adapter != "minimax_h3": raise ValueError( - "--detached-video-task-root requires " - "--request-adapter minimax_h3" + "--detached-video-task-root requires --request-adapter minimax_h3" ) if self.stage_id is not None or self.omni_router or self.realtime: raise ValueError( @@ -589,6 +629,12 @@ def validate(self) -> None: ) if self.detached_video_drain_timeout <= 0: raise ValueError("--detached-video-drain-timeout must be > 0") + if type( + self.detached_video_prefetch_capacity + ) is not int or self.detached_video_prefetch_capacity not in {0, 1}: + raise ValueError("--detached-video-prefetch-capacity must be 0 or 1") + if self.detached_video_prefetch_capacity and not self.detached_video_task_root: + raise ValueError("detached prefetch requires --detached-video-task-root") if self.parallel.ulysses_degree <= 0: raise ValueError("--ulysses-degree must be > 0") if self.parallel.ring_degree <= 0: @@ -670,6 +716,10 @@ def parse_omni_args() -> OmniConfig: ) engine_args = OmniEngineArgs.from_cli_args(vllm_args) + # The Omni parser owns this flag, but disaggregated modes also read the + # native engine_args object. Keep the two views consistent. + if config.diffusion.max_num_seqs is not None: + engine_args.max_num_seqs = config.diffusion.max_num_seqs if getattr(engine_args, "served_model_name", None) is not None: served = engine_args.served_model_name diff --git a/dingo/vllm/omni/detached_tasks.py b/dingo/vllm/omni/detached_tasks.py index 758c7f3521fb..973c3fd5fe22 100644 --- a/dingo/vllm/omni/detached_tasks.py +++ b/dingo/vllm/omni/detached_tasks.py @@ -11,6 +11,7 @@ from __future__ import annotations import asyncio +import copy import hashlib import json import logging @@ -22,13 +23,23 @@ from pathlib import Path from typing import Any +from dingo.common.video_result_file import ( + BINARY_RESULT_WRITER, + INLINE_RESULT_FORMAT, + BinaryResultWriter, + normalize_inline_result, +) from dingo.common.video_task_protocol import ( ENVELOPE_KEY, + EXECUTION_CAPACITY_CAPABILITY, + PREFETCH_CAPABILITY, SCHEMA_VERSION, WAIT_TERMINAL_CAPABILITY, DetachedTaskIdentity, detached_attempt_root, - detached_envelope, +) +from dingo.common.video_task_protocol import ( + detached_envelope as detached_envelope, # retained compatibility re-export ) logger = logging.getLogger(__name__) @@ -76,6 +87,10 @@ class _RunningTask: identity: DetachedTaskIdentity context: _DetachedContext execution: asyncio.Task[None] + # Published only after the terminal write succeeds. Existing waiters hold + # this execution-scoped record; completed tasks are not retained globally. + persisted_terminal: dict[str, Any] | None = None + started: bool = False class DetachedOmniTaskManager: @@ -89,9 +104,21 @@ def __init__( drain_timeout_s: float = 1800.0, cancel_poll_interval_s: float = 0.25, cancel_grace_s: float = 5.0, + binary_results: bool | None = None, + inline_results: bool | None = None, + execution_capacity: int = 1, + prefetch_capacity: int = 0, ) -> None: + if ( + isinstance(execution_capacity, bool) + or not isinstance(execution_capacity, int) + or execution_capacity < 1 + ): + raise ValueError("detached execution capacity must be a positive integer") if drain_timeout_s <= 0: raise ValueError("detached drain timeout must be positive") + if type(prefetch_capacity) is not int or prefetch_capacity not in {0, 1}: + raise ValueError("detached prefetch capacity must be 0 or 1") if cancel_poll_interval_s <= 0: raise ValueError("detached cancel poll interval must be positive") if cancel_grace_s <= 0: @@ -105,21 +132,70 @@ def __init__( self._running: dict[tuple[str, str, str, int, str], _RunningTask] = {} self._lock = asyncio.Lock() self._accepting = True + self.execution_capacity = execution_capacity + self.prefetch_capacity = prefetch_capacity + self._execution_slots = asyncio.Semaphore(execution_capacity) + self._direct_running = 0 + self.binary_results = ( + os.getenv("DINGO_VIDEO_BINARY_RESULTS") == "1" + if binary_results is None + else binary_results + ) + self.inline_results = ( + os.getenv("DINGO_VIDEO_INLINE_RESULT") == "1" + if inline_results is None + else inline_results + ) + if self.inline_results and not self.binary_results: + raise ValueError("inline results require binary results") async def generate( self, request: dict[str, Any], context: Any ) -> AsyncGenerator[dict[str, Any], None]: envelope = request.get(ENVELOPE_KEY) if envelope is None: - async for chunk in self.handler.generate(request, context): - yield chunk + async with self._lock: + if not self._accepting: + raise RuntimeError("detached Worker is draining") + if len(self._running) + self._direct_running >= self.execution_capacity: + raise RuntimeError("Worker execution capacity exhausted") + self._direct_running += 1 + try: + async with self._execution_slots: + async for chunk in self.handler.generate(request, context): + yield chunk + finally: + self._direct_running -= 1 return if set(request) != {ENVELOPE_KEY} or not isinstance(envelope, Mapping): raise ValueError("detached task request must contain only its envelope") - identity = DetachedTaskIdentity.from_envelope(envelope) op = envelope.get("op") + if op == "capabilities": + if ( + set(envelope) != {"schema_version", "op"} + or envelope.get("schema_version") != SCHEMA_VERSION + ): + raise ValueError("invalid detached capabilities request") + yield { + "schema_version": SCHEMA_VERSION, + "capabilities": [ + WAIT_TERMINAL_CAPABILITY, + EXECUTION_CAPACITY_CAPABILITY, + PREFETCH_CAPABILITY, + ], + "execution_capacity": self.execution_capacity, + "prefetch_capacity": self.prefetch_capacity, + "admission_capacity": self.execution_capacity + self.prefetch_capacity, + "accepting": self._accepting, + } + return + identity = DetachedTaskIdentity.from_envelope(envelope) if op == "submit": - yield await self._submit(identity, envelope.get("payload")) + yield await self._submit( + identity, + envelope.get("payload"), + deadline_at_ms=envelope.get("deadline_at_ms"), + ) elif op == "wait": async for status in self._wait_terminal(identity): yield status @@ -148,7 +224,7 @@ def _response_path(self, identity: DetachedTaskIdentity) -> Path: def _cancel_path(self, identity: DetachedTaskIdentity) -> Path: return self._attempt_root(identity) / "cancel.requested" - def _validate_task_manifest(self, identity: DetachedTaskIdentity) -> None: + def _validate_task_manifest(self, identity: DetachedTaskIdentity) -> Path: attempt_root = self._attempt_root(identity) task_root = attempt_root.parent.parent manifest_path = task_root / "_artifact.json" @@ -171,22 +247,32 @@ def _validate_task_manifest(self, identity: DetachedTaskIdentity) -> None: manifest.get(key) != value for key, value in expected.items() ): raise RuntimeError("detached task manifest identity mismatch") + return attempt_root @staticmethod def _atomic_json(path: Path, value: Mapping[str, Any]) -> None: - path.parent.mkdir(mode=0o750, parents=True, exist_ok=True) temporary = path.with_name(path.name + f".part-{uuid.uuid4().hex}") payload = json.dumps( value, ensure_ascii=False, sort_keys=True, separators=(",", ":") ).encode("utf-8") + renamed = False try: - with temporary.open("xb") as stream: + try: + stream = temporary.open("xb") + except FileNotFoundError: + # Existing attempt directories are the common path. Retain + # creation behavior without a redundant mkdir on every update. + path.parent.mkdir(mode=0o750, parents=True, exist_ok=True) + stream = temporary.open("xb") + with stream: stream.write(payload) stream.flush() os.fsync(stream.fileno()) os.replace(temporary, path) + renamed = True finally: - temporary.unlink(missing_ok=True) + if not renamed: + temporary.unlink(missing_ok=True) @staticmethod def _read_status(path: Path) -> dict[str, Any] | None: @@ -215,22 +301,41 @@ def _base_status( } async def _submit( - self, identity: DetachedTaskIdentity, payload: Any + self, + identity: DetachedTaskIdentity, + payload: Any, + *, + deadline_at_ms: int | None = None, ) -> dict[str, Any]: if not self._accepting: raise RuntimeError("detached Worker is draining") if not isinstance(payload, dict): raise ValueError("detached submit payload must be an object") - await asyncio.to_thread(self._validate_task_manifest, identity) - attempt_root = self._attempt_root(identity) - status_path = self._status_path(identity) + attempt_root = await asyncio.to_thread(self._validate_task_manifest, identity) + status_path = attempt_root / "worker-status.json" async with self._lock: running = self._running.get(identity.key) if running is not None: - return {**self._base_status(identity, "running"), "accepted": False} + state = ( + "accepted" + if self.prefetch_capacity and not running.started + else "running" + ) + return {**self._base_status(identity, state), "accepted": False} existing = await asyncio.to_thread(self._read_status, status_path) if existing is not None and existing.get("state") in _TERMINAL: return {**existing, "accepted": False} + if ( + len(self._running) + self._direct_running + >= self.execution_capacity + self.prefetch_capacity + ): + # No execution lock/status is written for rejected work. A + # caller can requeue it without mistaking it for an execution. + return { + **self._base_status(identity, "busy"), + "accepted": False, + "execution_capacity": self.execution_capacity, + } await asyncio.to_thread(attempt_root.mkdir, 0o750, True, True) lock_path = attempt_root / "execution.lock" @@ -253,14 +358,23 @@ def _claim() -> bool: current = existing or self._base_status(identity, "running") return {**current, "accepted": False} initial = self._base_status(identity, "accepted") + queued_at_ms = int(time.time() * 1000) + if self.prefetch_capacity: + initial["queued_at_ms"] = queued_at_ms await asyncio.to_thread(self._atomic_json, status_path, initial) request_id = ( - f"{identity.task_id}-{identity.attempt}-" - f"{identity.execution_token[:12]}" + f"{identity.task_id}-{identity.attempt}-{identity.execution_token[:12]}" ) detached_context = _DetachedContext(request_id) execution = asyncio.create_task( - self._execute(identity, payload, detached_context), + self._execute( + identity, + payload, + detached_context, + deadline_at_ms=deadline_at_ms, + queued_at_ms=queued_at_ms, + status_path=status_path, + ), name=f"omni-detached-{identity.task_id}-{identity.attempt}", ) self._running[identity.key] = _RunningTask( @@ -291,7 +405,7 @@ def _execution_done( async def _watch_cancel( self, identity: DetachedTaskIdentity, context: _DetachedContext ) -> None: - path = self._cancel_path(identity) + path = await asyncio.to_thread(self._cancel_path, identity) while not context.is_stopped(): if await asyncio.to_thread(path.exists): context.stop_generating() @@ -330,24 +444,112 @@ async def _heartbeat_status( }, ) + async def _record_terminal( + self, identity: DetachedTaskIdentity, path: Path, status: dict[str, Any] + ) -> None: + """Notify local waiters only after the terminal write completes.""" + await asyncio.to_thread(self._atomic_json, path, status) + running = self._running.get(identity.key) + if running is not None: + running.persisted_terminal = copy.deepcopy(status) + async def _execute( + self, + identity, + payload, + context, + *, + deadline_at_ms=None, + queued_at_ms=None, + status_path=None, + ): + if not self.prefetch_capacity: + return await self._execute_started(identity, payload, context) + # Submission already resolved this path before publishing accepted. + # Do not insert a filesystem await before acquiring the FIFO permit: + # two concurrent metadata reads can complete in the opposite order. + if status_path is None: + status_path = await asyncio.to_thread(self._status_path, identity) + acquire = asyncio.create_task(self._execution_slots.acquire()) + stopped = context.async_killed_or_stopped() + cancel_watch = asyncio.create_task(self._watch_cancel(identity, context)) + try: + timeout = ( + None + if deadline_at_ms is None + else max(0, (deadline_at_ms - int(time.time() * 1000)) / 1000) + ) + done, _ = await asyncio.wait( + {acquire, stopped}, timeout=timeout, return_when=asyncio.FIRST_COMPLETED + ) + if stopped in done or context.is_stopped(): + await self._record_terminal( + identity, status_path, self._base_status(identity, "cancelled") + ) + return + if acquire not in done or ( + deadline_at_ms is not None and int(time.time() * 1000) >= deadline_at_ms + ): + await self._record_terminal( + identity, + status_path, + { + **self._base_status(identity, "failed"), + "error": { + "code": "worker_queue_timeout", + "message": "Worker prefetch wait expired", + }, + }, + ) + return + await self._execute_started( + identity, payload, context, queued_at_ms=queued_at_ms + ) + except asyncio.CancelledError: + context.stop_generating() + running = self._running.get(identity.key) + if running is None or running.persisted_terminal is None: + await self._record_terminal( + identity, status_path, self._base_status(identity, "cancelled") + ) + raise + finally: + if not acquire.done(): + acquire.cancel() + cancel_watch.cancel() + stopped.cancel() + await asyncio.gather(acquire, cancel_watch, stopped, return_exceptions=True) + if ( + not acquire.cancelled() + and acquire.exception() is None + and acquire.result() + ): + self._execution_slots.release() + await context.close() + + async def _execute_started( self, identity: DetachedTaskIdentity, payload: dict[str, Any], context: _DetachedContext, + *, + queued_at_ms: int | None = None, ) -> None: - attempt_root = self._attempt_root(identity) - status_path = self._status_path(identity) - response_path = self._response_path(identity) + running = self._running.get(identity.key) + if running is not None: + running.started = True + # Validate the directory once for this operation, off the event loop; + # deriving sibling filenames does not require rechecking each parent. + attempt_root = await asyncio.to_thread(self._attempt_root, identity) + status_path = attempt_root / "worker-status.json" + response_path = attempt_root / "worker-response.jsonl" temporary = response_path.with_name( response_path.name + f".part-{uuid.uuid4().hex}" ) execution = asyncio.current_task() assert execution is not None cancel_watch = asyncio.create_task(self._watch_cancel(identity, context)) - cancel_enforcer = asyncio.create_task( - self._enforce_cancel(context, execution) - ) + cancel_enforcer = asyncio.create_task(self._enforce_cancel(context, execution)) started_at_ms = int(time.time() * 1000) status_stop = asyncio.Event() status_heartbeat = asyncio.create_task( @@ -359,29 +561,57 @@ async def _execute( status_path, {**self._base_status(identity, "running"), "started_at_ms": started_at_ms}, ) + result_token = BINARY_RESULT_WRITER.set( + BinaryResultWriter(attempt_root) if self.binary_results else None + ) try: - digest = hashlib.sha256() - written = 0 - with temporary.open("xb") as stream: + if self.inline_results: + terminal = None async for chunk in self.handler.generate(payload, context): if not isinstance(chunk, dict): raise RuntimeError( "Omni detached response chunk is not an object" ) - encoded = await asyncio.to_thread( - lambda value: json.dumps( - value, - ensure_ascii=False, - separators=(",", ":"), - ).encode("utf-8") - + b"\n", - chunk, - ) - await asyncio.to_thread(stream.write, encoded) - digest.update(encoded) - written += len(encoded) - await asyncio.to_thread(stream.flush) - await asyncio.to_thread(os.fsync, stream.fileno()) + if chunk.get("status") in _TERMINAL: + if terminal is not None: + raise RuntimeError( + "Worker returned multiple terminal responses" + ) + terminal = normalize_inline_result(chunk) + if terminal is None and not context.is_stopped(): + raise RuntimeError("Worker stream ended without terminal response") + response_fields = { + "result_format": INLINE_RESULT_FORMAT, + "inline_result": terminal, + } + else: + digest = hashlib.sha256() + written = 0 + with temporary.open("xb") as stream: + async for chunk in self.handler.generate(payload, context): + if not isinstance(chunk, dict): + raise RuntimeError( + "Omni detached response chunk is not an object" + ) + encoded = await asyncio.to_thread( + lambda value: ( + json.dumps( + value, ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + + b"\n" + ), + chunk, + ) + await asyncio.to_thread(stream.write, encoded) + digest.update(encoded) + written += len(encoded) + await asyncio.to_thread(stream.flush) + await asyncio.to_thread(os.fsync, stream.fileno()) + response_fields = { + "response_path": str(response_path), + "response_bytes": written, + "response_sha256": digest.hexdigest(), + } if context.is_stopped(): temporary.unlink(missing_ok=True) status_stop.set() @@ -392,17 +622,20 @@ async def _execute( {**self._base_status(identity, "cancelled")}, ) return - await asyncio.to_thread(os.replace, temporary, response_path) + if not self.inline_results: + await asyncio.to_thread(os.replace, temporary, response_path) status_stop.set() await asyncio.gather(status_heartbeat, return_exceptions=True) completed = { **self._base_status(identity, "completed"), - "response_path": str(response_path), - "response_bytes": written, - "response_sha256": digest.hexdigest(), + **response_fields, "inference_time_s": max(0.0, time.monotonic() - started), } - await asyncio.to_thread(self._atomic_json, status_path, completed) + if queued_at_ms is not None: + completed["worker_queue_wait_s"] = max( + 0.0, (started_at_ms - queued_at_ms) / 1000 + ) + await self._record_terminal(identity, status_path, completed) except asyncio.CancelledError: context.stop_generating() temporary.unlink(missing_ok=True) @@ -426,8 +659,9 @@ async def _execute( "message": str(exc)[:1024] or "detached Omni task failed", }, } - await asyncio.to_thread(self._atomic_json, status_path, failed) + await self._record_terminal(identity, status_path, failed) finally: + BINARY_RESULT_WRITER.reset(result_token) cancel_watch.cancel() cancel_enforcer.cancel() status_stop.set() @@ -441,9 +675,10 @@ async def _execute( await context.close() async def _status(self, identity: DetachedTaskIdentity) -> dict[str, Any]: - value = await asyncio.to_thread( - self._read_status, self._status_path(identity) - ) + def read(): + return self._read_status(self._status_path(identity)) + + value = await asyncio.to_thread(read) if value is None: return {**self._base_status(identity, "not_found")} return value @@ -459,21 +694,20 @@ async def _wait_terminal( detached inference task, so the execution is always shielded. """ - status_path = self._status_path(identity) - status = await asyncio.to_thread(self._read_status, status_path) - if status is None: - yield self._base_status(identity, "not_found") - return - if status.get("state") in _TERMINAL: - yield status - return + def read(): + path = self._status_path(identity) + return path, self._read_status(path) running = self._running.get(identity.key) if running is None: - # A different or restarted process may own the durable status. - # The Gateway will use its shared-filesystem fallback rather than - # treating the missing local waiter as an inference failure. - yield status + # Reconnects and restarted processes still recover from disk. A + # local execution is already validated against the full identity. + _, status = await asyncio.to_thread(read) + yield ( + status + if status is not None + else self._base_status(identity, "not_found") + ) return yield self._base_status(identity, "watching") @@ -487,7 +721,11 @@ async def _wait_terminal( # exception across the private protocol. pass - terminal = await asyncio.to_thread(self._read_status, status_path) + terminal = copy.deepcopy(running.persisted_terminal) + if terminal is None: + # Cancellation, legacy subclasses and exceptional recorder paths + # still read disk. Never invent a successful completion. + _, terminal = await asyncio.to_thread(read) if terminal is None or terminal.get("state") not in _TERMINAL: raise RuntimeError( "detached execution ended without a durable terminal status" @@ -495,9 +733,8 @@ async def _wait_terminal( yield terminal async def _cancel(self, identity: DetachedTaskIdentity) -> dict[str, Any]: - path = self._cancel_path(identity) - def _write_cancel() -> None: + path = self._cancel_path(identity) path.parent.mkdir(mode=0o750, parents=True, exist_ok=True) descriptor = os.open(path, os.O_WRONLY | os.O_CREAT, 0o600) os.close(descriptor) diff --git a/dingo/vllm/omni/main.py b/dingo/vllm/omni/main.py index edb7f72e8ca6..e17097e53987 100644 --- a/dingo/vllm/omni/main.py +++ b/dingo/vllm/omni/main.py @@ -8,8 +8,11 @@ import os import uvloop - from dynamo import prometheus_names +from dynamo.llm import ModelInput, ModelType, WorkerType, fetch_model, register_model +from dynamo.runtime import DistributedRuntime +from dynamo.runtime.logging import configure_dynamo_logging + from dingo.common.config_dump import dump_config from dingo.common.storage import get_fs from dingo.common.utils.graceful_shutdown import install_signal_handlers @@ -18,14 +21,8 @@ from dingo.common.utils.runtime_termination import ( run_with_runtime_termination_guard, ) -from dynamo.llm import ModelInput, ModelType, WorkerType, fetch_model, register_model -from dynamo.runtime import DistributedRuntime -from dynamo.runtime.logging import configure_dynamo_logging from dingo.vllm.health_check import VllmOmniHealthCheckPayload from dingo.vllm.main import setup_metrics_collection -from dingo.vllm.omni.realtime_utils import init_omni_realtime -from dingo.vllm.omni.stage_router import init_omni_stage_router -from dingo.vllm.omni.stage_worker import init_omni_stage from .args import OmniConfig, parse_omni_args @@ -71,6 +68,8 @@ async def init_omni( handler, config.detached_video_task_root, drain_timeout_s=config.detached_video_drain_timeout, + execution_capacity=config.diffusion.max_num_seqs or 1, + prefetch_capacity=config.detached_video_prefetch_capacity, ) serve_handler = detached_manager.generate logger.info( @@ -135,9 +134,14 @@ async def init_omni( raise finally: logger.debug("Cleaning up Omni worker") - if detached_manager is not None: - await detached_manager.shutdown() - handler.cleanup() + try: + if detached_manager is not None: + await detached_manager.shutdown() + finally: + try: + await asyncio.to_thread(handler.output_formatter.close) + finally: + handler.cleanup() async def worker(): @@ -161,12 +165,20 @@ async def worker(): install_signal_handlers(loop, runtime, shutdown_endpoints, shutdown_event) if config.stage_id is not None: + # Optional execution modes have their own upstream dependencies. Do not + # import them when serving a full diffusion pipeline (or parsing --help). + from dingo.vllm.omni.stage_worker import init_omni_stage + await init_omni_stage(runtime, config, shutdown_endpoints, shutdown_event) logger.debug("init_omni_stage completed (stage %d)", config.stage_id) elif config.omni_router: + from dingo.vllm.omni.stage_router import init_omni_stage_router + await init_omni_stage_router(runtime, config, shutdown_endpoints) logger.debug("init_omni_stage_router completed") elif config.realtime: + from dingo.vllm.omni.realtime_utils import init_omni_realtime + await init_omni_realtime(runtime, config, shutdown_endpoints, shutdown_event) logger.debug("init_omni_realtime completed, exiting...") else: diff --git a/dingo/vllm/omni/output_formatter.py b/dingo/vllm/omni/output_formatter.py index 7e9d58937925..02abb8896963 100644 --- a/dingo/vllm/omni/output_formatter.py +++ b/dingo/vllm/omni/output_formatter.py @@ -10,6 +10,7 @@ import asyncio import base64 +import json import logging import time import uuid @@ -19,6 +20,7 @@ import numpy as np import soundfile as sf import torch + from dingo.common.protocols.audio_protocol import AudioData, NvAudioSpeechResponse from dingo.common.protocols.image_protocol import ImageData, NvImagesResponse from dingo.common.protocols.video_protocol import NvVideosResponse, VideoData @@ -26,6 +28,8 @@ from dingo.common.utils.engine_response import normalize_finish_reason from dingo.common.utils.output_modalities import RequestType from dingo.common.utils.video_utils import normalize_video_frames +from dingo.common.video_encoding import VideoEncoder, frame_conversion_workers +from dingo.common.video_result_file import BINARY_RESULT_WRITER logger = logging.getLogger(__name__) @@ -76,8 +80,8 @@ def format( class DiffusionFormatter: """Formats diffusion output (images/video frames) for the frontend. - Handles both image and video — routes by request_type since vllm-omni - reports final_output_type="image" for all diffusion outputs. + Handles both image and video. Older vllm-omni reports "image" for video; + 0.29 also reports the explicit final_output_type="video". """ def __init__( @@ -91,6 +95,10 @@ def __init__( self._media_fs = media_fs self._media_http_url = media_http_url self._default_fps = default_fps + self._video_encoder = VideoEncoder(frame_conversion_workers()) + + def close(self) -> None: + self._video_encoder.close() async def format( self, stage_output: Any, request_id: str, *, request_type: Any, **ctx: Any @@ -141,13 +149,13 @@ def _extract_audio(stage_output: Any) -> tuple[Any, Optional[int]]: return None, None metadata = mm_output.get("metadata") - audio_metadata = ( - metadata.get("audio") if isinstance(metadata, dict) else None - ) + audio_metadata = metadata.get("audio") if isinstance(metadata, dict) else None for candidate in ( mm_output.get("audio_sample_rate"), mm_output.get("sr"), - audio_metadata.get("sample_rate") if isinstance(audio_metadata, dict) else None, + audio_metadata.get("sample_rate") + if isinstance(audio_metadata, dict) + else None, ): if candidate is None: continue @@ -186,6 +194,8 @@ async def _encode_video( ) try: start_time = time.time() + output_started = time.perf_counter() + stages: Dict[str, float] = {} frame_list = normalize_video_frames(images) encode_kwargs: Dict[str, Any] = { @@ -204,10 +214,7 @@ async def _encode_video( # AAC audio into an MP4 byte string. It handles the conversion from # normalized frame tensors to the byte format expected by the # frontend, including audio sample rate negotiation for AAC encoding. - from vllm_omni.entrypoints.openai.video_api_utils import ( - _encode_video_bytes, - ) - + from vllm_omni.entrypoints.openai import video_api_utils from vllm_omni.entrypoints.openai.serving_video import ( OmniOpenAIServingVideo, ) @@ -219,13 +226,48 @@ async def _encode_video( f"video encoding requires a single normalized video tensor" ) - video_bytes = await asyncio.to_thread( - _encode_video_bytes, - normalized[0], - fps=fps, - **encode_kwargs, + stages["output_normalize_s"] = time.perf_counter() - output_started + submitted = time.perf_counter() + + def encode(): + began = time.perf_counter() + stages["encode_queue_s"] = began - submitted + try: + return self._video_encoder.encode( + video_api_utils, normalized[0], fps=fps, **encode_kwargs + ) + finally: + stages["encode_work_s"] = time.perf_counter() - began + + video_bytes = await asyncio.to_thread(encode) + stages["encode_resume_s"] = max( + 0.0, + time.perf_counter() + - submitted + - stages["encode_queue_s"] + - stages["encode_work_s"], ) + binary_writer = BINARY_RESULT_WRITER.get() + if binary_writer is not None: + artifact = await binary_writer.write(video_bytes) + stages.update(binary_writer.stage_durations) + stages["output_total_s"] = time.perf_counter() - output_started + logger.info( + "video_output_timing request_id=%s stages=%s", + request_id, + json.dumps(stages, sort_keys=True), + ) + return { + "id": request_id, + "object": "video", + "model": self._model_name, + "status": "completed", + "progress": 100, + "data": [{"output_format": "mp4", "artifact": artifact}], + "inference_time_s": time.time() - start_time, + "stage_durations": stages, + } if response_format == "b64_json": video_data = VideoData( output_format=output_format, @@ -541,14 +583,19 @@ def __init__( media_http_url: Optional[str] = None, default_fps: int = 16, ) -> None: + diffusion = DiffusionFormatter( + model_name, media_fs, media_http_url, default_fps + ) self._formatters: Dict[str, Any] = { "text": TextFormatter(model_name), - "image": DiffusionFormatter( - model_name, media_fs, media_http_url, default_fps - ), + "image": diffusion, + "video": diffusion, "audio": AudioFormatter(model_name, media_fs, media_http_url), } + def close(self) -> None: + self._formatters["video"].close() + async def format( self, stage_output: Any, diff --git a/dingo/vllm/omni/stage_router.py b/dingo/vllm/omni/stage_router.py index 5f3c7daf5bae..b37cfebd11f5 100644 --- a/dingo/vllm/omni/stage_router.py +++ b/dingo/vllm/omni/stage_router.py @@ -3,22 +3,23 @@ """Stage router for disaggregated omni pipelines.""" +import asyncio import json import logging import uuid from typing import Any, AsyncGenerator, Dict, List +from dynamo import prometheus_names +from dynamo.llm import ModelInput, WorkerType, register_model +from dynamo.runtime import DistributedRuntime from vllm_omni.entrypoints.utils import load_and_resolve_stage_configs -from dynamo import prometheus_names from dingo.common.storage import get_fs from dingo.common.utils.output_modalities import ( RequestType, get_output_modalities, parse_request_type, ) -from dynamo.llm import ModelInput, WorkerType, register_model -from dynamo.runtime import DistributedRuntime from dingo.vllm.main import setup_metrics_collection from dingo.vllm.omni.args import OmniConfig from dingo.vllm.omni.output_formatter import OutputFormatter @@ -235,3 +236,5 @@ async def init_omni_stage_router( except Exception as e: logger.error("OmniStageRouter endpoint failed: %s", e) raise + finally: + await asyncio.to_thread(router._formatter.close) diff --git a/dingo/vllm/tests/omni/test_detached_tasks.py b/dingo/vllm/tests/omni/test_detached_tasks.py index ef6821bfaefa..34332e4aae09 100644 --- a/dingo/vllm/tests/omni/test_detached_tasks.py +++ b/dingo/vllm/tests/omni/test_detached_tasks.py @@ -61,12 +61,17 @@ def is_stopped(self) -> bool: return False -def _request(*, op: str = "submit", payload: dict[str, Any] | None = None): +def _request( + *, + op: str = "submit", + payload: dict[str, Any] | None = None, + task_id: str = "01TASK", +): return detached_envelope( op=op, deployment_id="deployment-a", pool_id="fl-pool", - task_id="01TASK", + task_id=task_id, attempt=1, execution_token="a" * 32, payload=payload, @@ -86,8 +91,8 @@ async def _terminal_status(manager: DetachedOmniTaskManager) -> dict[str, Any]: raise AssertionError("detached task did not become terminal") -def _write_manifest(root: Path) -> None: - task_root = root / "deployment-a" / "v1" / "pools" / "fl-pool" / "tasks" / "01TASK" +def _write_manifest(root: Path, task_id: str = "01TASK") -> None: + task_root = root / "deployment-a" / "v1" / "pools" / "fl-pool" / "tasks" / task_id task_root.mkdir(parents=True) (task_root / "_artifact.json").write_text( json.dumps( @@ -95,12 +100,98 @@ def _write_manifest(root: Path) -> None: "schema_version": 1, "deployment_id": "deployment-a", "pool_id": "fl-pool", - "task_id": "01TASK", + "task_id": task_id, } ) ) +async def test_capabilities_are_explicit_and_do_not_create_task_files(tmp_path): + from dingo.common.video_task_protocol import ( + ENVELOPE_KEY, + EXECUTION_CAPACITY_CAPABILITY, + ) + + manager = DetachedOmniTaskManager(_Handler(), tmp_path, execution_capacity=2) + value = ( + await _one(manager, {ENVELOPE_KEY: {"schema_version": 1, "op": "capabilities"}}) + )[0] + assert value["execution_capacity"] == 2 and value["accepting"] + assert EXECUTION_CAPACITY_CAPABILITY in value["capabilities"] + assert not list(tmp_path.iterdir()) + with pytest.raises(ValueError, match="capabilities"): + await _one( + manager, + {ENVELOPE_KEY: {"schema_version": 1, "op": "capabilities", "payload": {}}}, + ) + await manager.shutdown() + + +@pytest.mark.parametrize("capacity", [0, -1, True, 1.5]) +async def test_invalid_execution_capacity(tmp_path, capacity): + with pytest.raises(ValueError, match="capacity"): + DetachedOmniTaskManager(_Handler(), tmp_path, execution_capacity=capacity) + + +async def test_two_slots_reject_overflow_without_persisting_execution(tmp_path): + handler = _Handler() + manager = DetachedOmniTaskManager( + handler, tmp_path, execution_capacity=2, drain_timeout_s=1 + ) + ids = [f"task-{i}" for i in range(10)] + for task_id in ids: + _write_manifest(tmp_path, task_id) + try: + rows = await asyncio.gather( + *[_one(manager, _request(task_id=i, payload={})) for i in ids] + ) + accepted = [i for i, row in zip(ids, rows) if row[0]["accepted"]] + rejected = [i for i, row in zip(ids, rows) if row[0]["state"] == "busy"] + assert len(accepted) == 2 and len(rejected) == 8 + assert len(manager._running) == 2 + duplicate = (await _one(manager, _request(task_id=accepted[0], payload={})))[0] + assert duplicate["state"] == "running" and not duplicate["accepted"] + for task_id in rejected: + attempt = detached_attempt_root( + tmp_path, "deployment-a", "fl-pool", task_id, 1, "a" * 32 + ) + assert not attempt.exists() + handler.release.set() + await asyncio.gather( + *(item.execution for item in list(manager._running.values())) + ) + await asyncio.sleep(0) + assert not manager._running + assert (await _one(manager, _request(task_id=rejected[0], payload={})))[0][ + "accepted" + ] + finally: + handler.release.set() + await manager.shutdown() + + +async def test_direct_requests_share_detached_execution_budget(tmp_path): + handler = _Handler() + manager = DetachedOmniTaskManager( + handler, tmp_path, execution_capacity=1, drain_timeout_s=1 + ) + _write_manifest(tmp_path) + direct = asyncio.create_task(_one(manager, {"prompt": "direct"})) + try: + await asyncio.wait_for(handler.started.wait(), 1) + assert (await _one(manager, _request(payload={})))[0]["state"] == "busy" + with pytest.raises(RuntimeError, match="capacity"): + await _one(manager, {"prompt": "another direct"}) + handler.release.set() + await direct + assert manager._direct_running == 0 + assert (await _one(manager, _request(payload={})))[0]["accepted"] + finally: + handler.release.set() + await direct + await manager.shutdown() + + async def test_submit_ack_is_independent_and_duplicate_is_idempotent(tmp_path: Path): handler = _Handler() manager = DetachedOmniTaskManager(handler, tmp_path, drain_timeout_s=1) diff --git a/dingo/vllm/tests/omni/test_omni_args.py b/dingo/vllm/tests/omni/test_omni_args.py index 7743159dd821..9c4247330cbe 100644 --- a/dingo/vllm/tests/omni/test_omni_args.py +++ b/dingo/vllm/tests/omni/test_omni_args.py @@ -31,6 +31,37 @@ _PARALLEL_FIELDS = {f.name for f in dataclasses.fields(OmniParallelKwargs)} +@pytest.mark.parametrize("capacity", [0, -1, True, 1.5]) +def test_rejects_invalid_diffusion_capacity(capacity): + config = _make_omni_config(max_num_seqs=capacity) + with pytest.raises(ValueError, match="max-num-seqs"): + config.validate() + + +def test_concurrent_h3_requires_step_execution(): + config = _make_omni_config( + request_adapter="minimax_h3", request_adapter_workflow="fl2va", max_num_seqs=2 + ) + with pytest.raises(ValueError, match="requires --step-execution"): + config.validate() + + +def test_concurrent_h3_step_execution_valid(): + config = _make_omni_config( + request_adapter="minimax_h3", + request_adapter_workflow="fl2va", + max_num_seqs=2, + step_execution=True, + ) + config.validate() + + +def test_step_execution_rejects_cache_backend(): + config = _make_omni_config(step_execution=True, cache_backend="cache_dit") + with pytest.raises(ValueError, match="cannot be combined"): + config.validate() + + def _make_omni_config(**overrides) -> OmniConfig: """Build a minimal OmniConfig with valid defaults, applying overrides. @@ -274,18 +305,18 @@ def test_diffusion_kwargs_expose_runtime_wrapper_fields(): "diffusion_attention_backend", "diffusion_quantization_config", } - assert expected.issubset( - _DIFFUSION_FIELDS - ), f"Missing diffusion kwargs: {expected - _DIFFUSION_FIELDS}" + assert expected.issubset(_DIFFUSION_FIELDS), ( + f"Missing diffusion kwargs: {expected - _DIFFUSION_FIELDS}" + ) def test_parallel_kwargs_expose_runtime_wrapper_fields(): """Fields previously injected by launch_worker._build_h3_tuned_omni_kwargs are now first-class OmniParallelKwargs members.""" expected = {"text_encoder_tp_size", "vae_parallel_mode"} - assert expected.issubset( - _PARALLEL_FIELDS - ), f"Missing parallel kwargs: {expected - _PARALLEL_FIELDS}" + assert expected.issubset(_PARALLEL_FIELDS), ( + f"Missing parallel kwargs: {expected - _PARALLEL_FIELDS}" + ) def test_diffusion_kwargs_defaults_match_vllm_omni(): diff --git a/dingo/vllm/tests/omni/test_omni_base_handler.py b/dingo/vllm/tests/omni/test_omni_base_handler.py index caa675e0c86e..8f977ed5cc05 100644 --- a/dingo/vllm/tests/omni/test_omni_base_handler.py +++ b/dingo/vllm/tests/omni/test_omni_base_handler.py @@ -25,12 +25,18 @@ pytest.mark.pre_merge, ] -# These fields are not exposed in OmniParallelKwargs, because they are derived from other fields. +# Fields deliberately not exposed in OmniParallelKwargs. Derived fields and +# upstream-only tuning controls retain the native DiffusionParallelConfig values. _SKIP_FIELDS = { "sequence_parallel_size", "enable_expert_parallel", "ulysses_mode", "mask_sp_padding", + # 0.29 sequence-parallel transport tuning, independent of request slots. + # Dingo does not currently expose these controls; retain upstream defaults + # instead of changing the existing TP/SP topology during slot migration. + "ulysses_a2a_permute", + "allgather_degree", } @@ -61,6 +67,21 @@ def _build_kwargs(config): return handler._build_omni_kwargs(config) +def test_step_execution_and_capacity_forwarded(): + config = _make_config() + config.diffusion = dataclasses.replace( + config.diffusion, step_execution=True, max_num_seqs=2 + ) + kwargs = _build_kwargs(config) + assert kwargs["step_execution"] is True + assert kwargs["max_num_seqs"] == 2 + + +def test_unset_capacity_preserves_upstream_default(): + kwargs = _build_kwargs(_make_config()) + assert "max_num_seqs" not in kwargs + + class TestDiffusionParallelConfigCoverage: def test_all_diffusion_parallel_config_fields_covered(self): """Every DiffusionParallelConfig field must be in OmniParallelKwargs, engine_args, or _SKIP_FIELDS. diff --git a/dingo/vllm/tests/omni/test_output_formatter.py b/dingo/vllm/tests/omni/test_output_formatter.py index 758db1bd2bd3..45318c7bebe8 100644 --- a/dingo/vllm/tests/omni/test_output_formatter.py +++ b/dingo/vllm/tests/omni/test_output_formatter.py @@ -464,6 +464,44 @@ async def test_routes_image(self): ) assert chunk["choices"][0]["delta"]["content"][0]["type"] == "image_url" + @pytest.mark.asyncio + @pytest.mark.parametrize("output_type", ["image", "video"]) + async def test_routes_legacy_and_029_video(self, output_type): + from types import SimpleNamespace + from unittest.mock import AsyncMock + + from dingo.common.utils.output_modalities import RequestType + from dingo.vllm.omni.output_formatter import OutputFormatter + + f = OutputFormatter(model_name="test-model") + stage = SimpleNamespace( + final_output_type=output_type, images=[object()], multimodal_output=None + ) + expected = {"status": "completed", "data": [{"output_format": "mp4"}]} + with patch.object( + f._formatters[output_type], + "_encode_video", + new_callable=AsyncMock, + return_value=expected, + ) as encode: + result = await f.format( + stage, + "req-video", + request_type=RequestType.VIDEO_GENERATION, + fps=24, + output_format="mp4", + ) + assert result == expected + encode.assert_awaited_once_with( + stage.images, + "req-video", + fps=24, + response_format=None, + output_format="mp4", + audio=None, + audio_sample_rate=None, + ) + @pytest.mark.asyncio async def test_routes_audio(self): import numpy as np @@ -587,14 +625,17 @@ def _patches(self): "dingo.vllm.omni.output_formatter.normalize_video_frames", return_value=[MagicMock()], ), - _patch("dingo.vllm.omni.output_formatter.export_to_video"), + _patch( + "vllm_omni.entrypoints.openai.video_api_utils._encode_video_bytes", + return_value=b"encoded-mp4-bytes", + ), _patch( "dingo.vllm.omni.output_formatter.upload_to_fs", return_value="http://x/v.mp4", ), _patch( - "dingo.vllm.omni.output_formatter.asyncio.to_thread", - side_effect=lambda fn, *a, **kw: fn(*a, **kw), + "vllm_omni.entrypoints.openai.serving_video.OmniOpenAIServingVideo._normalize_video_outputs", + return_value=[object()], ), ) @@ -648,7 +689,7 @@ async def test_video_b64_response_format(self): assert result["data"][0]["output_format"] == "mp4" assert result["data"][0].get("url") is None assert result["data"][0]["b64_json"] is not None - base64.b64decode(result["data"][0]["b64_json"]) # must be valid base64 + assert base64.b64decode(result["data"][0]["b64_json"]) == b"encoded-mp4-bytes" mock_upload.assert_not_called() @pytest.mark.asyncio diff --git a/tests/video_gateway/test_artifact_metadata_probe.py b/tests/video_gateway/test_artifact_metadata_probe.py new file mode 100644 index 000000000000..e9fde8000e4e --- /dev/null +++ b/tests/video_gateway/test_artifact_metadata_probe.py @@ -0,0 +1,91 @@ +"""Fast media probing keeps validation and the full-decode fallback usable.""" + +from types import SimpleNamespace + +import pytest + +from dingo.video_gateway.adapters import create_adapter +from dingo.video_gateway.adapters.minimax_h3 import _open_artifact_metadata +from tests.video_gateway.test_minimax_h3_adapter import _write_h264_aac_mp4 + + +@pytest.mark.parametrize("sound", [False, True]) +def test_fast_probe_matches_original_and_restores_decoding( + tmp_path, make_gateway_config, sound +): + import av + + adapter = create_adapter(make_gateway_config().pools[0]) + adapter.options["validate_media"] = True + path = tmp_path / "sample.mp4" + _write_h264_aac_mp4(path, frames=124, width=256, height=256) + normalized = dict(width=256, height=256, num_frames=124, generate_sound=sound) + adapter.prepare_artifact(path, normalized) + with av.open(str(path)) as original: + expected = adapter._validate_open_artifact(original, normalized) + assert adapter.validate_artifact(path, normalized) == expected + with _open_artifact_metadata(path) as optimized: + assert adapter._validate_open_artifact(optimized, normalized) == expected + assert optimized.streams.video[0].codec_context.skip_frame == "DEFAULT" + # The probe option must not leak into subsequent decoder initialization. + assert sum(1 for _ in optimized.decode(video=0)) == 124 + + +@pytest.mark.parametrize("missing", ["frames", "extradata", "rate", "width"]) +def test_incomplete_probe_reopens_with_original_defaults( + tmp_path, make_gateway_config, monkeypatch, missing +): + import av + + adapter = create_adapter(make_gateway_config().pools[0]) + adapter.options["validate_media"] = True + path = tmp_path / "sample.mp4" + _write_h264_aac_mp4(path, frames=124, width=256, height=256) + normalized = dict(width=256, height=256, num_frames=124, generate_sound=True) + expected = adapter.validate_artifact(path, normalized) + original_open = av.open + calls = [] + + def opened(*args, **kwargs): + calls.append(kwargs) + container = original_open(*args, **kwargs) + if not kwargs.get("options"): + return container + video = container.streams.video[0] + proxy = SimpleNamespace( + width=video.width, + height=video.height, + frames=video.frames, + average_rate=video.average_rate, + codec_context=SimpleNamespace( + name=video.codec_context.name, extradata=video.codec_context.extradata + ), + ) + if missing == "extradata": + proxy.codec_context.extradata = b"" + else: + setattr(proxy, {"rate": "average_rate"}.get(missing, missing), 0) + + class Streams: + video = [proxy] + audio = container.streams.audio + + def __len__(self): + return len(container.streams) + + return SimpleNamespace(streams=Streams(), close=container.close) + + monkeypatch.setattr(av, "open", opened) + assert adapter.validate_artifact(path, normalized) == expected + assert calls == [{"options": {"skip_frame": "all"}}, {}] + + +def test_truncated_mp4_not_accepted_by_fast_probe(tmp_path, make_gateway_config): + adapter = create_adapter(make_gateway_config().pools[0]) + adapter.options["validate_media"] = True + path = tmp_path / "broken.mp4" + path.write_bytes(b"\x00\x00\x00\x18ftypisom\x00\x00\x00\x00isommp42") + with pytest.raises(Exception): + adapter.validate_artifact( + path, dict(width=256, height=256, num_frames=124, generate_sound=False) + ) diff --git a/tests/video_gateway/test_binary_result.py b/tests/video_gateway/test_binary_result.py new file mode 100644 index 000000000000..b589c1978695 --- /dev/null +++ b/tests/video_gateway/test_binary_result.py @@ -0,0 +1,842 @@ +import asyncio +import errno +import hashlib +import json +import os +import threading + +import pytest + +from dingo.common.video_result_file import ( + BINARY_RESULT_WRITER, + BinaryResultWriter, + validate_descriptor, +) +from dingo.video_gateway.artifact_store import FileArtifactStore +from dingo.video_gateway.errors import ResultTooLarge +from dingo.video_gateway.models import TaskStatus +from dingo.vllm.omni.detached_tasks import DetachedOmniTaskManager +from tests.video_gateway.test_dispatcher import ( + _MINIMAL_MP4, + _DetachedClient, + _pool, + _stack, + _submit, +) + +REAL_THREAD = asyncio.to_thread + + +def test_atomic_status_existing_directory_needs_no_mkdir_or_unlink( + tmp_path, monkeypatch +): + from pathlib import Path + + path = tmp_path / "worker-status.json" + + def unexpected(*args, **kwargs): + raise AssertionError("redundant metadata operation") + + with monkeypatch.context() as patch: + patch.setattr(Path, "mkdir", unexpected) + patch.setattr(Path, "unlink", unexpected) + DetachedOmniTaskManager._atomic_json(path, {"state": "accepted"}) + DetachedOmniTaskManager._atomic_json(path, {"state": "completed"}) + assert json.loads(path.read_text()) == {"state": "completed"} + + +def test_atomic_status_still_creates_missing_parent(tmp_path): + path = tmp_path / "new" / "attempt" / "worker-status.json" + DetachedOmniTaskManager._atomic_json(path, {"state": "accepted"}) + assert json.loads(path.read_text()) == {"state": "accepted"} + assert not list(tmp_path.rglob("*.part-*")) + + +@pytest.mark.parametrize("operation", ["fsync", "replace"]) +def test_atomic_status_failure_keeps_old_status_and_cleans_temporary( + tmp_path, monkeypatch, operation +): + path = tmp_path / "worker-status.json" + DetachedOmniTaskManager._atomic_json(path, {"state": "running"}) + + def fail(*args, **kwargs): + raise OSError("injected " + operation) + + monkeypatch.setattr(os, operation, fail) + with pytest.raises(OSError, match="injected"): + DetachedOmniTaskManager._atomic_json(path, {"state": "completed"}) + assert json.loads(path.read_text()) == {"state": "running"} + assert not list(tmp_path.glob("*.part-*")) + + +async def test_binary_success_keeps_both_fsyncs_without_missing_temp_unlink( + tmp_path, monkeypatch +): + from pathlib import Path + + original = os.fsync + calls = [] + + def fsync(fd): + calls.append(fd) + return original(fd) + + def unexpected(*args, **kwargs): + raise AssertionError("unlink after successful rename") + + with monkeypatch.context() as patch: + patch.setattr(Path, "unlink", unexpected) + patch.setattr(os, "fsync", fsync) + desc = await BinaryResultWriter(tmp_path).write(_MINIMAL_MP4) + assert len(calls) == 2 + assert (tmp_path / desc["filename"]).read_bytes() == _MINIMAL_MP4 + + +async def test_binary_output_timings_are_bounded_and_descriptor_unchanged(tmp_path): + import math + + from dingo.common.video_result_file import normalize_inline_result + + writer = BinaryResultWriter(tmp_path) + desc = await writer.write(_MINIMAL_MP4) + assert set(desc) == {"schema_version", "filename", "bytes", "sha256"} + stages = writer.stage_durations + assert set(stages) == { + "artifact_queue_s", + "artifact_hash_s", + "artifact_open_s", + "artifact_write_s", + "artifact_file_fsync_s", + "artifact_rename_s", + "artifact_dir_fsync_s", + "artifact_close_s", + "artifact_work_s", + "artifact_resume_s", + } + assert all(math.isfinite(v) and v >= 0 for v in stages.values()) + assert ( + sum( + stages[k] + for k in [ + "artifact_hash_s", + "artifact_open_s", + "artifact_write_s", + "artifact_file_fsync_s", + "artifact_rename_s", + "artifact_dir_fsync_s", + "artifact_close_s", + ] + ) + <= stages["artifact_work_s"] + ) + result = normalize_inline_result( + { + "status": "completed", + "stage_durations": stages, + "data": [{"output_format": "mp4", "artifact": desc}], + } + ) + assert result["stage_durations"] == stages + + +@pytest.mark.parametrize("operation", ["fsync", "replace"]) +async def test_binary_failure_cleans_unpublished_temp(tmp_path, monkeypatch, operation): + def fail(*args, **kwargs): + raise OSError("injected " + operation) + + monkeypatch.setattr(os, operation, fail) + with pytest.raises(OSError, match="injected"): + await BinaryResultWriter(tmp_path).write(_MINIMAL_MP4) + assert not list(tmp_path.iterdir()) + + +async def test_detached_path_checks_run_off_loop_for_all_operations( + tmp_path, monkeypatch +): + from dingo.common.video_task_protocol import DetachedTaskIdentity + + monkeypatch.setattr(asyncio, "to_thread", REAL_THREAD) + started = asyncio.Event() + release = asyncio.Event() + + class Handler: + async def generate(self, request, context): + started.set() + await release.wait() + yield { + "status": "completed", + "data": [ + { + "output_format": "mp4", + "artifact": await BINARY_RESULT_WRITER.get().write( + _MINIMAL_MP4 + ), + } + ], + } + + manager = DetachedOmniTaskManager( + Handler(), tmp_path, binary_results=True, inline_results=True + ) + identity = DetachedTaskIdentity("deployment", "pool", "task", 1, "a" * 32) + task_root = manager._attempt_root(identity).parent.parent + task_root.mkdir(parents=True) + (task_root / "_artifact.json").write_text( + json.dumps( + dict( + schema_version=1, + deployment_id="deployment", + pool_id="pool", + task_id="task", + ) + ) + ) + loop_thread = threading.get_ident() + original = manager._attempt_root + checks = [] + + def checked(value): + assert threading.get_ident() != loop_thread + checks.append(value.key) + return original(value) + + monkeypatch.setattr(manager, "_attempt_root", checked) + try: + assert (await manager._submit(identity, {}))["accepted"] + await asyncio.wait_for(started.wait(), 2) + assert (await manager._status(identity))["state"] == "running" + waiter = manager._wait_terminal(identity) + assert (await anext(waiter))["state"] == "watching" + release.set() + assert (await anext(waiter))["state"] == "completed" + await waiter.aclose() + assert (await manager._cancel(identity))["state"] == "cancel_requested" + assert checks + finally: + release.set() + await manager.shutdown() + + +async def test_detached_revalidates_symlinks_on_later_status_operation(tmp_path): + from dingo.common.video_task_protocol import DetachedTaskIdentity + + manager = DetachedOmniTaskManager(object(), tmp_path / "artifacts") + identity = DetachedTaskIdentity("deployment", "pool", "task", 1, "a" * 32) + assert (await manager._status(identity))["state"] == "not_found" + outside = tmp_path / "outside" + outside.mkdir() + (manager.root / "deployment").symlink_to(outside, target_is_directory=True) + with pytest.raises(RuntimeError, match="symlink"): + await manager._status(identity) + assert not list(outside.iterdir()) + + +async def setup_result(tmp_path): + store = FileArtifactStore(tmp_path / "artifacts") + root = store.detached_attempt_root("deployment", "pool", "task", 1, "a" * 32) + root.mkdir(parents=True) + desc = await BinaryResultWriter(root).write(_MINIMAL_MP4) + return store, root, desc + + +async def publish(store, desc, **kwargs): + return await store.finalize_worker_mp4( + "deployment", + "pool", + "task", + 1, + "a" * 32, + desc, + {}, + kwargs.pop("validator", lambda p, n: {"container": "mp4"}), + kwargs.pop("processor", lambda p, n: None), + kwargs.pop("requires_processing", lambda p, n: False), + **kwargs, + ) + + +async def test_binary_candidates_are_independent_and_share_no_duplicate_data(tmp_path): + store, root, desc = await setup_result(tmp_path) + left, size, sha, _ = await publish(store, desc) + right, *_ = await publish(store, desc) + assert left != right + assert size == len(_MINIMAL_MP4) and sha == hashlib.sha256(_MINIMAL_MP4).hexdigest() + assert left.stat().st_ino == (root / desc["filename"]).stat().st_ino + left.unlink() + assert right.read_bytes() == _MINIMAL_MP4 + assert (root / desc["filename"]).read_bytes() == _MINIMAL_MP4 + + +@pytest.mark.parametrize("fault", ["size", "missing", "symlink", "other_attempt"]) +async def test_binary_rejects_bad_or_cross_attempt_results(tmp_path, fault): + store, root, desc = await setup_result(tmp_path) + source = root / desc["filename"] + if fault == "size": + desc["bytes"] += 1 + if fault == "missing": + source.unlink() + if fault == "symlink": + source.unlink() + source.symlink_to(tmp_path / "outside") + if fault == "other_attempt": + other = root.parent / "2-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + other.mkdir() + source.rename(other / source.name) + with pytest.raises(RuntimeError): + await publish(store, desc) + + +async def test_binary_trusts_worker_digest_without_full_read(tmp_path, monkeypatch): + from pathlib import Path + + store, root, desc = await setup_result(tmp_path) + desc["sha256"] = ( + "0" * 64 + ) # intentional policy: no independent content digest comparison + + def unexpected_read(*args, **kwargs): + raise AssertionError("unexpected whole-file read") + + def unexpected_hash(*args, **kwargs): + raise AssertionError("unexpected Gateway hash") + + with monkeypatch.context() as check: + check.setattr(Path, "open", unexpected_read) + check.setattr(hashlib, "sha256", unexpected_hash) + final, size, sha, _ = await publish(store, desc) + assert size == len(_MINIMAL_MP4) and sha == desc["sha256"] + assert final.read_bytes() == _MINIMAL_MP4 + + +async def test_binary_rechecks_candidate_size_after_media_validation(tmp_path): + store, root, desc = await setup_result(tmp_path) + + def mutate(path, normalized): + path.write_bytes(b"short") + return {} + + with pytest.raises(RuntimeError, match="size/type changed"): + await publish(store, desc, validator=mutate) + assert not list(store.root.rglob("video-a*.mp4")) + + +@pytest.mark.parametrize("merged", [False, True]) +async def test_binary_allows_dingofs_hardlink_mtime_update( + tmp_path, monkeypatch, merged +): + store, root, desc = await setup_result(tmp_path) + original = os.link + before = (root / desc["filename"]).stat().st_mtime_ns + + def dingofs_link(src, dst, **kwargs): + original(src, dst, **kwargs) + value = os.stat(dst) + os.utime(dst, ns=(value.st_atime_ns, value.st_mtime_ns + 1_000_000_000)) + + monkeypatch.setattr(os, "link", dingofs_link) + final, size, sha, _ = await publish( + store, + desc, + inspector=(lambda p, n: (False, {"container": "mp4"})) if merged else None, + ) + assert final.stat().st_mtime_ns != before + assert final.read_bytes() == _MINIMAL_MP4 and size == len(_MINIMAL_MP4) + assert sha == desc["sha256"] + + +@pytest.mark.parametrize( + "name", ["../outside.mp4", "/tmp/out.mp4", "worker-video-x.mp4"] +) +def test_descriptor_rejects_paths(name): + with pytest.raises(ValueError): + validate_descriptor( + dict(schema_version=1, filename=name, bytes=1, sha256="a" * 64) + ) + + +async def test_binary_processing_never_changes_worker_source(tmp_path): + store, root, desc = await setup_result(tmp_path) + processed = b"changed-video" + path, size, sha, _ = await publish( + store, + desc, + requires_processing=lambda p, n: True, + processor=lambda p, n: p.write_bytes(processed), + ) + assert path.read_bytes() == processed and size == len(processed) + assert sha == hashlib.sha256(processed).hexdigest() + assert (root / desc["filename"]).read_bytes() == _MINIMAL_MP4 + + +async def test_binary_validation_failure_cleans_only_candidate(tmp_path): + store, root, desc = await setup_result(tmp_path) + good, *_ = await publish(store, desc) + + def invalid(p, n): + raise RuntimeError("invalid media") + + with pytest.raises(RuntimeError, match="invalid media"): + await publish(store, desc, validator=invalid) + assert list(good.parent.iterdir()) == [good] + assert (root / desc["filename"]).exists() + + +async def test_binary_limit_and_duplicate_output(tmp_path): + store, root, desc = await setup_result(tmp_path) + with pytest.raises(ResultTooLarge): + await publish(store, desc, max_result_bytes=1) + writer = BinaryResultWriter(root, max_bytes=4) + with pytest.raises(ValueError): + await writer.write(b"12345") + await writer.write(b"1234") + with pytest.raises(RuntimeError): + await writer.write(b"1") + + +async def test_binary_hardlink_unsupported_falls_back_to_copy(tmp_path, monkeypatch): + store, root, desc = await setup_result(tmp_path) + + def unsupported(*args, **kwargs): + raise OSError(errno.EOPNOTSUPP, "unsupported") + + monkeypatch.setattr(os, "link", unsupported) + final, *_ = await publish(store, desc) + assert final.read_bytes() == _MINIMAL_MP4 + assert final.stat().st_ino != (root / desc["filename"]).stat().st_ino + + +async def test_binary_cancel_drains_writer_before_return(tmp_path, monkeypatch): + monkeypatch.setattr(asyncio, "to_thread", REAL_THREAD) + writer = BinaryResultWriter(tmp_path) + started = threading.Event() + release = threading.Event() + original = writer._write + + def slow(data): + started.set() + release.wait(3) + return original(data) + + monkeypatch.setattr(writer, "_write", slow) + task = asyncio.create_task(writer.write(_MINIMAL_MP4)) + try: + assert await REAL_THREAD(started.wait, 1) + task.cancel() + await asyncio.sleep(0.01) + task.cancel() + await asyncio.sleep(0.01) + assert not task.done() + finally: + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert not list(tmp_path.glob("*.part")) + + +@pytest.mark.parametrize("inline", [False, True]) +@pytest.mark.parametrize("wait_supported", [False, True]) +async def test_dispatcher_finishes_binary_worker_and_keeps_small_manifest( + make_gateway_config, inline, wait_supported +): + class Handler: + async def generate(self, request, context): + desc = await BINARY_RESULT_WRITER.get().write(_MINIMAL_MP4) + yield { + "status": "completed", + "data": [{"output_format": "mp4", "artifact": desc}], + } + + pool = _pool("fl-pool", "public-fl", "dyn://scope.backend.generate") + pool["execution_mode"] = "detached" + config = make_gateway_config(pools=[pool]) + + class Manager(DetachedOmniTaskManager): + def _base_status(self, identity, state): + status = super()._base_status(identity, state) + if not wait_supported: + status.pop("capabilities", None) + return status + + manager = Manager( + Handler(), + config.artifact_store.root, + binary_results=True, + inline_results=inline, + ) + store, artifacts, dispatcher, service = _stack( + config, {"fl-pool": _DetachedClient(manager)} + ) + await dispatcher.start() + try: + submitted = await _submit(service, "public-fl") + terminal = await dispatcher.wait_terminal(submitted.stored.task.id, 3) + assert terminal.task.status == TaskStatus.COMPLETED, terminal.task.error + assert terminal.task.result_bytes == len(_MINIMAL_MP4) + responses = list(artifacts.root.rglob("worker-response.jsonl")) + if inline: + assert not responses + status = json.loads( + next(artifacts.root.rglob("worker-status.json")).read_text() + ) + assert status["result_format"] == "binary_mp4_inline_v1" + assert "artifact" in status["inline_result"]["data"][0] + assert ( + not {"response_path", "response_bytes", "response_sha256"} + & status.keys() + ) + else: + assert len(responses) == 1 and responses[0].stat().st_size < 1024 + assert "b64_json" not in responses[0].read_text() + assert not await store.list_leases("fl-pool") + finally: + await dispatcher.stop() + await manager.shutdown() + assert BINARY_RESULT_WRITER.get() is None + + +@pytest.mark.parametrize("sound", [True, False]) +@pytest.mark.parametrize("merged", [False, True]) +async def test_binary_real_media_audio_policy( + tmp_path, make_gateway_config, sound, merged +): + from dingo.video_gateway.adapters import create_adapter + from tests.video_gateway.test_minimax_h3_adapter import _write_h264_aac_mp4 + + sample = tmp_path / "sample.mp4" + _write_h264_aac_mp4(sample, frames=124, width=256, height=256) + store = FileArtifactStore(tmp_path / "artifacts") + root = store.detached_attempt_root("deployment", "pool", "task", 1, "a" * 32) + root.mkdir(parents=True) + desc = await BinaryResultWriter(root).write(sample.read_bytes()) + config = make_gateway_config() + adapter = create_adapter(config.pools[0]) + adapter.options["validate_media"] = True + normalized = { + "width": 256, + "height": 256, + "fps": 24, + "num_frames": 124, + "seconds": 5, + "generate_sound": sound, + } + final, _, _, media = await store.finalize_worker_mp4( + "deployment", + "pool", + "task", + 1, + "a" * 32, + desc, + normalized, + adapter.validate_artifact, + adapter.prepare_artifact, + adapter.artifact_requires_processing, + inspector=adapter.inspect_artifact_for_publication if merged else None, + ) + assert media["audio_codec"] == ("aac" if sound else None) + assert ( + hashlib.sha256((root / desc["filename"]).read_bytes()).hexdigest() + == desc["sha256"] + ) + assert (final.stat().st_ino == (root / desc["filename"]).stat().st_ino) == sound + + +@pytest.mark.parametrize("fallback", [False, True]) +@pytest.mark.parametrize("sound", [False, True]) +async def test_binary_merged_probe_opens_once_unless_copied( + tmp_path, make_gateway_config, monkeypatch, fallback, sound +): + import av + + from dingo.video_gateway.adapters import create_adapter + from tests.video_gateway.test_minimax_h3_adapter import _write_h264_aac_mp4 + + adapter = create_adapter(make_gateway_config().pools[0]) + adapter.options["validate_media"] = True + normalized = { + "width": 256, + "height": 256, + "num_frames": 124, + "generate_sound": sound, + } + sample = tmp_path / "sample.mp4" + _write_h264_aac_mp4(sample, frames=124, width=256, height=256) + adapter.prepare_artifact(sample, normalized) + expected = adapter.validate_artifact(sample, normalized) + store = FileArtifactStore(tmp_path / "artifacts") + root = store.detached_attempt_root("deployment", "pool", "task", 1, "a" * 32) + root.mkdir(parents=True) + desc = await BinaryResultWriter(root).write(sample.read_bytes()) + opened = [] + original = av.open + + def counted(path, *args, **kwargs): + opened.append(str(path)) + return original(path, *args, **kwargs) + + monkeypatch.setattr(av, "open", counted) + if fallback: + + def unsupported(*args, **kwargs): + raise OSError(errno.EXDEV, "cross-device") + + monkeypatch.setattr(os, "link", unsupported) + final, _, _, media = await store.finalize_worker_mp4( + "deployment", + "pool", + "task", + 1, + "a" * 32, + desc, + normalized, + adapter.validate_artifact, + adapter.prepare_artifact, + adapter.artifact_requires_processing, + inspector=adapter.inspect_artifact_for_publication, + ) + assert media == expected + assert opened == [str(root / desc["filename"])] + ([str(final)] if fallback else []) + + +@pytest.mark.parametrize("fault", ["dimensions", "frames", "audio_required"]) +async def test_binary_merged_probe_keeps_media_rejections( + tmp_path, make_gateway_config, fault +): + from dingo.video_gateway.adapters import create_adapter + from tests.video_gateway.test_minimax_h3_adapter import _write_h264_aac_mp4 + + adapter = create_adapter(make_gateway_config().pools[0]) + adapter.options["validate_media"] = True + normalized = { + "width": 256, + "height": 256, + "num_frames": 124, + "generate_sound": False, + } + sample = tmp_path / "sample.mp4" + _write_h264_aac_mp4(sample, frames=124, width=256, height=256) + adapter.prepare_artifact(sample, normalized) + store = FileArtifactStore(tmp_path / "artifacts") + root = store.detached_attempt_root("deployment", "pool", "task", 1, "a" * 32) + root.mkdir(parents=True) + desc = await BinaryResultWriter(root).write(sample.read_bytes()) + if fault == "dimensions": + normalized["width"] = 512 + if fault == "frames": + normalized["num_frames"] = 200 + if fault == "audio_required": + normalized["generate_sound"] = True + with pytest.raises( + RuntimeError, + match={ + "dimensions": "dimensions", + "frames": "frame count", + "audio_required": "AAC", + }[fault], + ): + await store.finalize_worker_mp4( + "deployment", + "pool", + "task", + 1, + "a" * 32, + desc, + normalized, + adapter.validate_artifact, + adapter.prepare_artifact, + adapter.artifact_requires_processing, + inspector=adapter.inspect_artifact_for_publication, + ) + assert not list(store.root.rglob("video-a*.mp4")) + assert (root / desc["filename"]).exists() + + +async def test_binary_inspected_source_change_is_rejected(tmp_path): + store, root, desc = await setup_result(tmp_path) + + def inspect(path, normalized): + path.write_bytes(b"changed") + return False, {"container": "mp4"} + + with pytest.raises(RuntimeError, match="changed during validation"): + await publish(store, desc, inspector=inspect) + assert not list(store.root.rglob("video-a*.mp4")) + + +@pytest.mark.parametrize("disk_failure", [False, True]) +@pytest.mark.parametrize("inline", [False, True]) +async def test_binary_worker_restart_is_idempotent_and_write_failure_is_not_completed( + tmp_path, monkeypatch, disk_failure, inline +): + from dingo.common.video_task_protocol import detached_envelope + + calls = 0 + + class Handler: + async def generate(self, request, context): + nonlocal calls + calls += 1 + desc = await BINARY_RESULT_WRITER.get().write(_MINIMAL_MP4) + yield { + "status": "completed", + "data": [{"output_format": "mp4", "artifact": desc}], + } + + if disk_failure: + + def fail(*args): + raise OSError("controlled disk failure") + + monkeypatch.setattr(BinaryResultWriter, "_write", fail) + store = FileArtifactStore(tmp_path / "artifacts") + task_root = store.task_root("deployment", "pool", "task") + task_root.mkdir(parents=True) + (task_root / "_artifact.json").write_text( + json.dumps( + dict( + schema_version=1, + deployment_id="deployment", + pool_id="pool", + task_id="task", + ) + ) + ) + identity = dict( + deployment_id="deployment", + pool_id="pool", + task_id="task", + attempt=1, + execution_token="a" * 32, + ) + manager = DetachedOmniTaskManager( + Handler(), store.root, binary_results=True, inline_results=inline + ) + + async def operation(m, op): + return [ + x + async for x in m.generate( + detached_envelope( + op=op, **identity, payload={} if op == "submit" else None + ), + None, + ) + ] + + await operation(manager, "submit") + terminal = (await operation(manager, "wait"))[-1] + assert terminal["state"] == ("failed" if disk_failure else "completed") + await manager.shutdown() + replacement = DetachedOmniTaskManager( + Handler(), store.root, binary_results=True, inline_results=inline + ) + try: + reply = (await operation(replacement, "submit"))[0] + assert reply["accepted"] is False and reply["state"] == terminal["state"] + assert calls == 1 + if not disk_failure: + attempt_root = store.detached_attempt_root(**identity) + response = ( + reply["inline_result"] + if inline + else json.loads((attempt_root / "worker-response.jsonl").read_text()) + ) + desc = response["data"][0]["artifact"] + final, *_ = await publish(store, desc) + assert final.read_bytes() == _MINIMAL_MP4 + finally: + await replacement.shutdown() + + +@pytest.mark.parametrize( + "error", + [ + "Executor shut down", + {"code": "invalid_request", "message": "bad parameter"}, + {"code": "worker_failed", "message": "Executor shut down"}, + ], +) +def test_inline_errors_preserve_retry_classification(error): + from dingo.common.video_result_file import normalize_inline_result + from dingo.video_gateway.errors import worker_execution_error + + new = normalize_inline_result({"status": "failed", "error": error})["error"] + assert type(worker_execution_error(new)) is type(worker_execution_error(error)) + assert str(worker_execution_error(new)) == str(worker_execution_error(error)) + + +@pytest.mark.parametrize( + "data", + [ + None, + [], + [{"output_format": "mp4", "b64_json": "AAAA"}], + [{"output_format": "mp4", "artifact": {"filename": "../other"}}], + ], +) +def test_inline_descriptor_rejects_missing_binary_or_legacy_payload(data): + from dingo.common.video_result_file import normalize_inline_result + + with pytest.raises(ValueError): + normalize_inline_result({"status": "completed", "data": data}) + + +async def test_inline_cancel_after_binary_write_never_publishes_completed(tmp_path): + from dingo.common.video_task_protocol import detached_envelope + + store = FileArtifactStore(tmp_path / "artifacts") + root = store.task_root("deployment", "pool", "task") + root.mkdir(parents=True) + (root / "_artifact.json").write_text( + json.dumps( + dict( + schema_version=1, + deployment_id="deployment", + pool_id="pool", + task_id="task", + ) + ) + ) + ready = asyncio.Event() + + class Handler: + async def generate(self, request, context): + descriptor = await BINARY_RESULT_WRITER.get().write(_MINIMAL_MP4) + ready.set() + yield { + "status": "completed", + "data": [{"output_format": "mp4", "artifact": descriptor}], + } + await context.wait_stopped() + + manager = DetachedOmniTaskManager( + Handler(), + store.root, + binary_results=True, + inline_results=True, + cancel_poll_interval_s=0.01, + ) + + def request(op): + return detached_envelope( + op=op, + deployment_id="deployment", + pool_id="pool", + task_id="task", + attempt=1, + execution_token="a" * 32, + payload={} if op == "submit" else None, + ) + + async def call(op): + return [x async for x in manager.generate(request(op), None)] + + await call("submit") + try: + await asyncio.wait_for(ready.wait(), 1) + assert (await call("status"))[0]["state"] == "running" + await call("cancel") + terminal = (await asyncio.wait_for(call("wait"), 1))[-1] + assert terminal["state"] == "cancelled" and "inline_result" not in terminal + assert not list(store.root.rglob("worker-response.jsonl")) + finally: + await manager.shutdown() diff --git a/tests/video_gateway/test_config.py b/tests/video_gateway/test_config.py index fbf238a4859b..113025bda947 100644 --- a/tests/video_gateway/test_config.py +++ b/tests/video_gateway/test_config.py @@ -61,6 +61,15 @@ def test_config_maps_arbitrary_full_targets_without_namespace_assumptions(tmp_pa assert config.pools[0].execution_mode == "stream" +@pytest.mark.parametrize("capacity", [True, False, 1.5, "2", None]) +def test_worker_capacity_requires_integer_without_coercion(tmp_path, capacity): + raw = _raw(tmp_path) + raw["pools"][0]["execution_mode"] = "detached" + raw["pools"][0]["scheduling"] = {"worker_capacity": capacity} + with pytest.raises(ValueError, match="worker_capacity must be an integer"): + parse_config(raw) + + def test_legacy_compatibility_version_is_derived_without_revision_drift(tmp_path): raw = _raw(tmp_path) without_legacy = parse_config(deepcopy(raw)) @@ -98,6 +107,53 @@ def test_detached_execution_requires_explicit_pool_setting(tmp_path): parse_config(raw) +@pytest.mark.parametrize( + "settings", + [ + {"worker_prefetch_capacity": 1}, + {"worker_prefetch_capacity": 2, "early_release_slot": True}, + {"worker_prefetch_capacity": True, "early_release_slot": True}, + {"worker_prefetch_capacity": 1.5, "early_release_slot": True}, + {"finalization_timeout_s": float("nan")}, + {"finalization_timeout_s": float("inf")}, + {"finalization_timeout_s": 0}, + {"finalization_max_retries": -1}, + {"finalization_max_retries": 17}, + {"finalization_concurrency": 0}, + {"finalization_concurrency": 4, "finalization_pending_limit": 3}, + ], +) +def test_invalid_continuous_execution_settings_fail_fast(tmp_path, settings): + raw = _raw(tmp_path) + raw["pools"][0]["execution_mode"] = "detached" + raw["pools"][0]["scheduling"] = settings + with pytest.raises(ValueError): + parse_config(raw) + + +def test_early_release_requires_detached_mode(tmp_path): + raw = _raw(tmp_path) + raw["pools"][0]["scheduling"] = {"early_release_slot": True} + with pytest.raises(ValueError, match="detached"): + parse_config(raw) + + +def test_continuous_execution_defaults_off_and_does_not_change_adapter_revision( + tmp_path, +): + raw = _raw(tmp_path) + raw["pools"][0]["execution_mode"] = "detached" + original = parse_config(raw).pools[0] + assert original.scheduling.early_release_slot is False + assert original.scheduling.worker_prefetch_capacity == 0 + raw["pools"][0]["scheduling"] = { + "early_release_slot": True, + "worker_prefetch_capacity": 1, + } + updated = parse_config(raw).pools[0] + assert updated.configuration_revision == original.configuration_revision + + @pytest.mark.parametrize( "target", ["two.parts", "four.parts.are.invalid", "dyn://scope.bad/name.endpoint", ""], @@ -320,8 +376,7 @@ def test_media_limits_have_safe_defaults_and_derived_budget_weight(tmp_path): assert config.media.max_single_file_bytes == 50 * 1024 * 1024 assert config.media.max_result_encoded_bytes >= config.media.max_result_bytes assert ( - config.media.inflight_memory_budget_bytes - >= config.media.max_task_memory_bytes + config.media.inflight_memory_budget_bytes >= config.media.max_task_memory_bytes ) diff --git a/tests/video_gateway/test_continuous_execution_timing.py b/tests/video_gateway/test_continuous_execution_timing.py new file mode 100644 index 000000000000..e57e227bd0bc --- /dev/null +++ b/tests/video_gateway/test_continuous_execution_timing.py @@ -0,0 +1,99 @@ +"""Opt-in local continuity experiment, not a GPU/DingoFS throughput benchmark.""" + +import asyncio +import json +import os +import time + +import pytest + +from dingo.common.video_result_file import BINARY_RESULT_WRITER +from dingo.video_gateway.models import TaskStatus +from dingo.vllm.omni.detached_tasks import DetachedOmniTaskManager +from tests.video_gateway.test_dispatcher import ( + _MINIMAL_MP4, + _DetachedClient, + _pool, + _stack, + _submit, +) + + +@pytest.mark.skipif( + os.environ.get("DINGO_CONTINUOUS_TIMING") != "1", + reason="opt-in wall-clock continuity experiment (~2 minutes)", +) +@pytest.mark.parametrize("compute_s", [3, 5]) +@pytest.mark.parametrize("enabled", [False, True]) +async def test_worker_continuity_with_half_second_postprocessing( + make_gateway_config, monkeypatch, compute_s, enabled, record_property +): + starts, finishes = [], [] + + class Handler: + async def generate(self, request, context): + starts.append(time.monotonic()) + await asyncio.sleep(compute_s) + descriptor = await BINARY_RESULT_WRITER.get().write(_MINIMAL_MP4) + finishes.append(time.monotonic()) + yield { + "status": "completed", + "data": [{"output_format": "mp4", "artifact": descriptor}], + } + + pool = _pool("fl-pool", "public-fl", "dyn://scope.backend.generate") + pool["execution_mode"] = "detached" + pool["scheduling"].update( + early_release_slot=enabled, + worker_prefetch_capacity=int(enabled), + execution_timeout_s=60, + ) + config = make_gateway_config(pools=[pool]) + manager = DetachedOmniTaskManager( + Handler(), + config.artifact_store.root, + binary_results=True, + inline_results=True, + prefetch_capacity=int(enabled), + ) + store, artifacts, dispatcher, service = _stack( + config, {"fl-pool": _DetachedClient(manager)} + ) + original = artifacts.finalize_worker_mp4 + + async def slow(*args, **kwargs): + await asyncio.sleep(0.5) + return await original(*args, **kwargs) + + monkeypatch.setattr(artifacts, "finalize_worker_mp4", slow) + await dispatcher.start() + try: + started = time.monotonic() + submitted = [await _submit(service, "public-fl") for _ in range(6)] + for item in submitted: + terminal = await dispatcher.wait_terminal(item.stored.task.id, 60) + assert terminal.task.status == TaskStatus.COMPLETED + elapsed = time.monotonic() - started + gaps = [starts[i] - finishes[i - 1] for i in range(1, len(starts))] + measured = dict( + compute_s=compute_s, + early_release=enabled, + prefetch=int(enabled), + tasks=len(starts), + elapsed_s=elapsed, + requests_per_s=len(starts) / elapsed, + mean_execution_gap_s=sum(gaps) / len(gaps), + max_execution_gap_s=max(gaps), + execution_gaps_s=gaps, + ) + print("CONTINUOUS_TIMING " + json.dumps(measured), flush=True) + record_property("continuous_timing", json.dumps(measured)) + assert len(starts) == 6 and len(finishes) == 6 + assert not await store.list_leases("fl-pool") + if enabled: + assert max(gaps) < 0.25, measured + else: + assert min(gaps) >= 0.5, measured + finally: + await dispatcher.stop() + await manager.shutdown() diff --git a/tests/video_gateway/test_detached_terminal_reuse.py b/tests/video_gateway/test_detached_terminal_reuse.py new file mode 100644 index 000000000000..244e4c87218d --- /dev/null +++ b/tests/video_gateway/test_detached_terminal_reuse.py @@ -0,0 +1,108 @@ +"""Execution-local notification must never get ahead of the status write.""" + +import asyncio +import json +import threading + +import pytest + +from dingo.common.video_result_file import BINARY_RESULT_WRITER +from dingo.common.video_task_protocol import DetachedTaskIdentity +from dingo.vllm.omni.detached_tasks import DetachedOmniTaskManager +from tests.video_gateway.test_dispatcher import _MINIMAL_MP4 + +REAL_THREAD = asyncio.to_thread + + +@pytest.mark.parametrize("write_fails", [False, True]) +async def test_wait_reuses_only_successfully_written_terminal( + tmp_path, monkeypatch, write_fails +): + monkeypatch.setattr(asyncio, "to_thread", REAL_THREAD) + started, release = asyncio.Event(), asyncio.Event() + writing, finish_write = threading.Event(), threading.Event() + + class Handler: + async def generate(self, request, context): + started.set() + await release.wait() + descriptor = await BINARY_RESULT_WRITER.get().write(_MINIMAL_MP4) + yield { + "status": "completed", + "data": [{"output_format": "mp4", "artifact": descriptor}], + } + + manager = DetachedOmniTaskManager( + Handler(), tmp_path, binary_results=True, inline_results=True + ) + identity = DetachedTaskIdentity("deployment", "pool", "task", 1, "a" * 32) + task_root = manager._attempt_root(identity).parent.parent + task_root.mkdir(parents=True) + (task_root / "_artifact.json").write_text( + json.dumps( + dict( + schema_version=1, + deployment_id="deployment", + pool_id="pool", + task_id="task", + ) + ) + ) + original_write, original_read = manager._atomic_json, manager._read_status + reads = [] + + def write(path, value): + if value["state"] == "completed": + writing.set() + assert finish_write.wait(5), "test did not release status writer" + if write_fails: + raise OSError("injected terminal write failure") + original_write(path, value) + + def read(path): + reads.append(path) + return original_read(path) + + monkeypatch.setattr(manager, "_atomic_json", write) + try: + assert (await manager._submit(identity, {}))["accepted"] + await asyncio.wait_for(started.wait(), 2) + running = manager._running[identity.key] + monkeypatch.setattr(manager, "_read_status", read) + first, second = ( + manager._wait_terminal(identity), + manager._wait_terminal(identity), + ) + assert (await anext(first))["state"] == "watching" + assert (await anext(second))["state"] == "watching" + result = asyncio.create_task(anext(first)) + release.set() + assert await REAL_THREAD(writing.wait, 3) + assert running.persisted_terminal is None + assert not result.done() + finish_write.set() + terminal = await asyncio.wait_for(result, 3) + expected = "failed" if write_fails else "completed" + assert terminal["state"] == expected + assert not reads, "local wait must not reread status or recheck paths" + terminal["state"] = "tampered" + assert (await anext(second))["state"] == expected + await first.aclose() + await second.aclose() + await manager.shutdown() + await asyncio.sleep(0) + assert not manager._running, "no global cache of completed tasks" + assert (await anext(manager._wait_terminal(identity)))["state"] == expected + assert len(reads) == 1, "late reconnect must use persisted status" + replacement = DetachedOmniTaskManager(object(), tmp_path) + assert (await anext(replacement._wait_terminal(identity)))["state"] == expected + finally: + finish_write.set() + release.set() + await manager.shutdown() + + +async def test_wait_other_execution_token_cannot_use_local_execution(tmp_path): + manager = DetachedOmniTaskManager(object(), tmp_path) + identity = DetachedTaskIdentity("deployment", "pool", "task", 1, "b" * 32) + assert (await anext(manager._wait_terminal(identity)))["state"] == "not_found" diff --git a/tests/video_gateway/test_dispatch_idle.py b/tests/video_gateway/test_dispatch_idle.py new file mode 100644 index 000000000000..5a9569d5c787 --- /dev/null +++ b/tests/video_gateway/test_dispatch_idle.py @@ -0,0 +1,129 @@ +"""A saturated pool must not fetch queued task bodies just to reject dispatch.""" + +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from dingo.video_gateway.adapters import create_adapter +from dingo.video_gateway.artifact_store import FileArtifactStore +from dingo.video_gateway.dispatcher import VideoDispatcher +from dingo.video_gateway.task_store import StoredTask, worker_key +from tests.video_gateway.test_dispatcher import ( + FakeClient, + FakeContext, + WatchMemoryTaskStore, +) +from tests.video_gateway.test_task_store import _task + + +def stack(make_gateway_config): + config = make_gateway_config() + store = WatchMemoryTaskStore() + client = FakeClient() + dispatcher = VideoDispatcher( + config, + store, + FileArtifactStore(config.artifact_store.root), + {"fl-pool": client}, + {"fl-pool": create_adapter(config.pools[0])}, + context_factory=FakeContext, + ) + pool = dispatcher.pools["fl-pool"] + pool.instance_ids = [7] + pool.lease_watch_healthy = True + for name, value in [ + ("list_queued", []), + ("retry_queue_depth", 0), + ("retry_budget_used", 0), + ("queue_depth", 0), + ("reserve", None), + ]: + setattr(store, name, AsyncMock(return_value=value)) + return dispatcher, pool, store, client + + +@pytest.mark.parametrize( + "reason", ["no_workers", "lease_watch_unhealthy", "no_free_worker"] +) +async def test_unusable_pool_skips_all_queue_and_counter_reads( + make_gateway_config, reason +): + dispatcher, pool, store, client = stack(make_gateway_config) + if reason == "no_workers": + pool.instance_ids = [] + elif reason == "lease_watch_unhealthy": + pool.lease_watch_healthy = False + else: + key = worker_key(pool.config.backend_target, 7) + pool.lease_cache[key] = SimpleNamespace(worker_key=key) + capacity = (await dispatcher.memory_budget.snapshot()).capacity_bytes + assert await dispatcher.memory_budget.try_acquire("holder", capacity) + assert not await dispatcher.memory_budget.try_acquire("stalled", 1) + pool.budget_waiter_id = "stalled" + for _ in range(3): + assert not await dispatcher._dispatch_once(pool) + for name in [ + "list_queued", + "retry_queue_depth", + "retry_budget_used", + "queue_depth", + "reserve", + ]: + getattr(store, name).assert_not_awaited() + assert pool.budget_waiter_id is None + assert (await dispatcher.memory_budget.snapshot()).waiting_tasks == 0 + assert not client.calls + + +@pytest.mark.parametrize("race", ["leased", "watch_lost", "none"]) +async def test_free_worker_still_rechecks_and_uses_reserve_cas( + make_gateway_config, race +): + dispatcher, pool, store, client = stack(make_gateway_config) + task = _task("task", pool_id=pool.config.pool_id) + task.backend_target = pool.config.backend_target + stored = StoredTask(task, 1) + + async def read_queue(*_args, **_kwargs): + if race == "leased": + key = worker_key(pool.config.backend_target, 7) + pool.lease_cache[key] = SimpleNamespace(worker_key=key) + elif race == "watch_lost": + pool.lease_watch_healthy = False + return [stored] + + store.list_queued.side_effect = read_queue + outcome = await dispatcher._dispatch_once(pool) + if race == "none": + # None is a lost reservation CAS, not authority to call the Worker. + store.reserve.assert_awaited_once() + assert outcome is True + assert (await dispatcher.memory_budget.snapshot()).used_bytes == 0 + else: + store.reserve.assert_not_awaited() + assert outcome is False + assert not client.calls + + +async def test_retry_gauges_refresh_without_dispatch(make_gateway_config, monkeypatch): + from dingo.video_gateway.api import _SERVICE_KEY + from tests.video_gateway.test_api import _client + + client = await _client(make_gateway_config, FakeClient(available=False)) + try: + service = client.server.app[_SERVICE_KEY] + monkeypatch.setattr( + service.store, "retry_queue_depth", AsyncMock(return_value=2), raising=False + ) + monkeypatch.setattr( + service.store, "retry_budget_used", AsyncMock(return_value=3), raising=False + ) + monkeypatch.setattr(service.store, "queue_depth", AsyncMock(return_value=5)) + response = await client.get("/metrics") + body = await response.text() + assert 'dingo_video_retry_waiting_tasks{pool="fl-pool"} 2' in body + assert 'dingo_video_retry_credits_used{pool="fl-pool"} 3' in body + assert 'dingo_video_normal_queue_depth{pool="fl-pool"} 3' in body + finally: + await client.close() diff --git a/tests/video_gateway/test_early_release_dispatcher.py b/tests/video_gateway/test_early_release_dispatcher.py new file mode 100644 index 000000000000..ac240bc17569 --- /dev/null +++ b/tests/video_gateway/test_early_release_dispatcher.py @@ -0,0 +1,161 @@ +import asyncio + +import pytest + +from dingo.common.video_result_file import BINARY_RESULT_WRITER +from dingo.video_gateway.models import TaskStatus +from dingo.video_gateway.result_handoff import read_handoff +from dingo.vllm.omni.detached_tasks import DetachedOmniTaskManager +from tests.video_gateway.test_dispatcher import ( + _MINIMAL_MP4, + _DetachedClient, + _pool, + _stack, + _submit, +) + + +@pytest.mark.parametrize("prefetch", [0, 1]) +async def test_next_task_starts_while_previous_gateway_finalization_is_blocked( + make_gateway_config, monkeypatch, prefetch +): + calls = [] + second_started = asyncio.Event() + first_finalizing = asyncio.Event() + finish = asyncio.Event() + + class Handler: + async def generate(self, request, context): + calls.append(context) + if len(calls) == 2: + second_started.set() + desc = await BINARY_RESULT_WRITER.get().write(_MINIMAL_MP4) + yield { + "status": "completed", + "data": [{"output_format": "mp4", "artifact": desc}], + } + + pool = _pool("fl-pool", "public-fl", "dyn://scope.backend.generate") + pool["execution_mode"] = "detached" + pool["scheduling"]["early_release_slot"] = True + pool["scheduling"]["worker_prefetch_capacity"] = prefetch + config = make_gateway_config(pools=[pool]) + manager = DetachedOmniTaskManager( + Handler(), + config.artifact_store.root, + binary_results=True, + inline_results=True, + prefetch_capacity=prefetch, + ) + store, artifacts, dispatcher, service = _stack( + config, {"fl-pool": _DetachedClient(manager)} + ) + original = artifacts.finalize_worker_mp4 + count = 0 + + async def slow(*args, **kwargs): + nonlocal count + count += 1 + if count == 1: + first_finalizing.set() + await finish.wait() + return await original(*args, **kwargs) + + monkeypatch.setattr(artifacts, "finalize_worker_mp4", slow) + await dispatcher.start() + try: + first = await _submit(service, "public-fl") + await asyncio.wait_for(first_finalizing.wait(), 3) + current = await store.get_task(first.stored.task.id) + assert current.task.status == TaskStatus.FINALIZING and read_handoff( + current.task + ) + assert not await store.list_leases("fl-pool") + second = await _submit(service, "public-fl") + await asyncio.wait_for(second_started.wait(), 2) + assert not finish.is_set() + finish.set() + for submitted in [first, second]: + assert ( + await dispatcher.wait_terminal(submitted.stored.task.id, 3) + ).task.status == TaskStatus.COMPLETED + assert len(calls) == 2 + finally: + finish.set() + await dispatcher.stop() + await manager.shutdown() + + +@pytest.mark.parametrize( + "fault", ["lost_handoff_reply", "completion_store_unavailable", "lost_reservation"] +) +async def test_handoff_storage_faults_do_not_rerun_model( + make_gateway_config, monkeypatch, fault +): + from dingo.video_gateway.errors import HandoffReservationLost + + calls = 0 + + class Handler: + async def generate(self, request, context): + nonlocal calls + calls += 1 + desc = await BINARY_RESULT_WRITER.get().write(_MINIMAL_MP4) + yield { + "status": "completed", + "data": [{"output_format": "mp4", "artifact": desc}], + } + + pool = _pool("fl-pool", "public-fl", "dyn://scope.backend.generate") + pool["execution_mode"] = "detached" + pool["scheduling"]["early_release_slot"] = True + config = make_gateway_config(pools=[pool]) + manager = DetachedOmniTaskManager( + Handler(), config.artifact_store.root, binary_results=True, inline_results=True + ) + store, artifacts, dispatcher, service = _stack( + config, {"fl-pool": _DetachedClient(manager)} + ) + original = store.transition + injected = False + + async def transition(*args, **kwargs): + nonlocal injected + handoff = kwargs.get("release_execution", False) + if not injected and handoff and fault == "lost_reservation": + injected = True + raise HandoffReservationLost("injected reservation replacement") + if ( + not injected + and fault == "completion_store_unavailable" + and kwargs["patch"].get("status") == TaskStatus.COMPLETED + ): + injected = True + raise ConnectionError("injected etcd outage before publication") + result = await original(*args, **kwargs) + if not injected and handoff and fault == "lost_handoff_reply": + injected = True + raise ConnectionError("injected lost handoff reply after commit") + return result + + monkeypatch.setattr(store, "transition", transition) + await dispatcher.start() + try: + submitted = await _submit(service, "public-fl") + terminal = await dispatcher.wait_terminal(submitted.stored.task.id, 4) + assert injected and calls == 1 and terminal.task.attempt == 1 + if fault == "lost_reservation": + assert terminal.task.status == TaskStatus.FAILED + assert terminal.task.error.code == "result_handoff_lost_reservation" + else: + assert terminal.task.status == TaskStatus.COMPLETED + assert not await store.list_leases("fl-pool") + for _ in range(100): + if (await dispatcher.memory_budget.snapshot()).used_bytes == 0: + break + await asyncio.sleep(0.01) + assert (await dispatcher.memory_budget.snapshot()).used_bytes == 0 + assert not dispatcher._finalizing + finally: + await dispatcher.stop() + await manager.shutdown() diff --git a/tests/video_gateway/test_execution_slots.py b/tests/video_gateway/test_execution_slots.py new file mode 100644 index 000000000000..4b09c976e7e3 --- /dev/null +++ b/tests/video_gateway/test_execution_slots.py @@ -0,0 +1,137 @@ +"""Slot lease identity and physical-instance retry fencing.""" + +import asyncio +import hashlib +from unittest.mock import AsyncMock + +import pytest + +from dingo.video_gateway.models import ( + StoredTask, + TaskStatus, + VideoTask, + WorkerLease, + now_ms, +) +from dingo.video_gateway.task_store import ( + EtcdTaskStore, + MemoryTaskStore, + retry_excludes_worker, + worker_key, +) + + +def task(name): + return VideoTask( + schema_version=1, + id=name, + deployment_id="slots", + pool_id="pool", + model="model", + backend_model="model", + backend_target="dyn://slot.backend.generate", + configuration_revision="r1", + delivery_mode="async", + status=TaskStatus.QUEUED, + request_digest=name, + request_path="/test/request", + input_manifest_path="/test/manifest", + created_at_ms=now_ms(), + queued_at_ms=now_ms(), + expires_at_ms=now_ms() + 60000, + ) + + +def lease(value, slot): + return WorkerLease( + pool_id=value.pool_id, + worker_key=worker_key(value.backend_target, 7, slot), + worker_instance_id=7, + backend_target=value.backend_target, + task_id=value.id, + owner_generation="gateway-a", + state="reserved", + heartbeat_at_ms=now_ms(), + ) + + +def test_slot_zero_preserves_legacy_key_and_other_slots_are_distinct(): + target = "dyn://slot.backend.generate" + legacy = hashlib.sha256(target.encode() + b"\0" + b"7").hexdigest() + assert worker_key(target, 7) == worker_key(target, "7", 0) == legacy + assert len({worker_key(target, 7, slot) for slot in range(100)}) == 100 + + +@pytest.mark.parametrize("slot", [-1, True, 1.5, "1"]) +def test_invalid_slot_id_rejected(slot): + with pytest.raises(ValueError, match="slot_id"): + worker_key("target", 7, slot) + + +@pytest.mark.parametrize("failed_slot", [0, 1, 9]) +def test_retry_excludes_physical_instance_across_all_slots(failed_slot): + value = task("retry") + assert not retry_excludes_worker(value, value.backend_target, 7) + value.attempt = 1 + value.worker_instance_id = 7 + value.worker_key = worker_key(value.backend_target, 7, failed_slot) + assert retry_excludes_worker(value, value.backend_target, "7") + assert not retry_excludes_worker(value, value.backend_target, 8) + value.worker_instance_id = None + assert retry_excludes_worker(value, value.backend_target, 8) + + +@pytest.mark.asyncio +async def test_two_slots_reserve_independently_and_same_slot_does_not_oversell(): + store = MemoryTaskStore() + values = [task(str(i)) for i in range(3)] + stored = [ + ( + await store.create_task( + v, principal_hash="p", idempotency_hash=None, queue_limit=4 + ) + )[0] + for v in values + ] + results = await asyncio.gather( + *[ + store.reserve( + stored[i], lease(values[i], i % 2), deadline_at_ms=now_ms() + 30000 + ) + for i in range(3) + ] + ) + assert sum(r is not None for r in results) == 2 + leases = await store.list_leases("pool") + assert len(leases) == 2 and {l.worker_instance_id for l in leases} == {7} + assert len({l.worker_key for l in leases}) == 2 + victim = next(r for r in results if r is not None) + await store.transition( + victim.task.id, + expected={TaskStatus.DISPATCHING}, + patch={"status": TaskStatus.CANCELLED}, + release_lease=True, + ) + remaining = await store.list_leases("pool") + assert len(remaining) == 1 and remaining[0].task_id != victim.task.id + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failed_slot,new_slot", [(0, 1), (1, 0), (1, 2)]) +async def test_etcd_reserve_rejects_retry_to_another_slot_of_failed_worker( + failed_slot, new_slot +): + value = task("retry") + value.attempt = 1 + value.worker_instance_id = 7 + value.worker_key = worker_key(value.backend_target, 7, failed_slot) + store = EtcdTaskStore(object(), prefix="/unit/slots", deployment_id="slots") + store._counter = AsyncMock(return_value=(1, None)) + store._retry_counter = AsyncMock( + side_effect=AssertionError("must reject before reserving credits") + ) + result = await store.reserve( + StoredTask(value, 1), lease(value, new_slot), deadline_at_ms=now_ms() + 30000 + ) + assert result is None + store._retry_counter.assert_not_awaited() diff --git a/tests/video_gateway/test_file_io.py b/tests/video_gateway/test_file_io.py new file mode 100644 index 000000000000..2e7ded5d61d7 --- /dev/null +++ b/tests/video_gateway/test_file_io.py @@ -0,0 +1,395 @@ +"""Real-thread regressions: slow metadata must not block the event loop.""" + +import asyncio +import base64 +import hashlib +import threading +import time + +import pytest + +from dingo.video_gateway.artifact_store import FileArtifactStore, TaskArtifactCandidate +from dingo.video_gateway.file_io import ( + opened_file, + run_cancellable_file_io, + run_file_io, +) + +REAL_TO_THREAD = asyncio.to_thread + + +async def test_finalize_uses_one_thread_dispatch(tmp_path, monkeypatch): + calls = [] + + async def counted(function, /, *args, **kwargs): + calls.append(function.__name__) + return await REAL_TO_THREAD(function, *args, **kwargs) + + monkeypatch.setattr(asyncio, "to_thread", counted) + store = FileArtifactStore(tmp_path / "artifacts") + main_thread = threading.get_ident() + stages = [] + + def processor(path, _normalized): + stages.append(threading.get_ident()) + assert threading.get_ident() != main_thread + path.write_bytes(b"processed-video") + + def validator(path, _normalized): + stages.append(threading.get_ident()) + assert path.read_bytes() == b"processed-video" + return {"validated": True} + + final, size, digest, media = await store.finalize_b64_mp4( + store.task_root("deployment", "pool", "task"), + base64.b64encode(b"source-video").decode(), + {}, + validator, + processor, + ) + assert calls == ["_finalize"] + assert len(set(stages)) == 1 and stages[0] != main_thread + assert final.read_bytes() == b"processed-video" + assert size == len(b"processed-video") + assert digest == hashlib.sha256(b"processed-video").hexdigest() + assert media == {"validated": True} + + +async def test_cancellable_file_io_receives_signal_and_drains(tmp_path, monkeypatch): + monkeypatch.setattr(asyncio, "to_thread", REAL_TO_THREAD) + entered, release = threading.Event(), threading.Event() + events = [] + + def work(cancelled): + entered.set() + assert release.wait(2) + assert cancelled.is_set() + events.append("drained") + raise asyncio.CancelledError + + task = asyncio.create_task(run_cancellable_file_io(work)) + try: + while not entered.is_set(): + await asyncio.sleep(0.001) + task.cancel() + await asyncio.sleep(0.01) + task.cancel() + await asyncio.sleep(0.01) + assert not task.done() + finally: + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert events == ["drained"] + + +@pytest.mark.parametrize("boundary", ["processor", "validator", "renamed"]) +async def test_cancelled_pipeline_cleans_only_own_candidate( + tmp_path, monkeypatch, boundary +): + import dingo.video_gateway.artifact_store as module + + monkeypatch.setattr(asyncio, "to_thread", REAL_TO_THREAD) + entered, release = threading.Event(), threading.Event() + store = FileArtifactStore(tmp_path / "artifacts") + task_root = store.task_root("deployment", "pool", "task") + result_dir = task_root / "result" + result_dir.mkdir(parents=True) + winner = result_dir / "video-other-owner.mp4" + winner.write_bytes(b"winner") + stages = [] + original_replace = module.os.replace + + def pause(): + entered.set() + assert release.wait(2) + + def processor(_path, _normalized): + stages.append("processor") + if boundary == "processor": + pause() + + def validator(_path, _normalized): + stages.append("validator") + if boundary == "validator": + pause() + return {} + + def replace(source, target): + original_replace(source, target) + if boundary == "renamed": + pause() + + monkeypatch.setattr(module.os, "replace", replace) + task = asyncio.create_task( + store.finalize_b64_mp4( + task_root, + base64.b64encode(b"video").decode(), + {}, + validator, + processor, + publication_scope="cancelled-owner", + ) + ) + try: + while not entered.is_set(): + await asyncio.sleep(0.001) + task.cancel() + await asyncio.sleep(0.01) + task.cancel() + await asyncio.sleep(0.01) + assert not task.done(), "released file ownership before the thread drained" + assert winner.read_bytes() == b"winner" + finally: + release.set() + with pytest.raises(asyncio.CancelledError): + await task + if boundary == "processor": + assert stages == ["processor"], "continued expensive work after cancellation" + assert list(result_dir.iterdir()) == [winner] + assert not list((task_root / "tmp").iterdir()) + assert winner.read_bytes() == b"winner" + + +@pytest.mark.parametrize("boundary", ["processor", "validator"]) +async def test_failed_pipeline_cleans_partial_result(tmp_path, monkeypatch, boundary): + monkeypatch.setattr(asyncio, "to_thread", REAL_TO_THREAD) + store = FileArtifactStore(tmp_path / "artifacts") + task_root = store.task_root("deployment", "pool", "task") + + def processor(_path, _normalized): + if boundary == "processor": + raise RuntimeError("processor failed") + + def validator(_path, _normalized): + raise RuntimeError("validator failed") + + with pytest.raises(RuntimeError, match=boundary + " failed"): + await store.finalize_b64_mp4( + task_root, base64.b64encode(b"video").decode(), {}, validator, processor + ) + assert not list((task_root / "result").iterdir()) + assert not list((task_root / "tmp").iterdir()) + + +@pytest.mark.parametrize( + "operation", + [ + "create", + "task_root", + "commit", + "write", + "read", + "status", + "cancel", + "consume", + "finalize", + "discard", + "trash", + "open_result", + ], +) +async def test_slow_containment_runs_off_loop(tmp_path, monkeypatch, operation): + monkeypatch.setattr(asyncio, "to_thread", REAL_TO_THREAD) + store = FileArtifactStore(tmp_path / "artifacts") + upload = await store.create_upload() + task = store.task_root("deployment", "pool", "task") + task.mkdir(parents=True) + (task / "x.json").write_text("{}") + result = task / "result.mp4" + result.write_bytes(b"video") + token = "a" * 32 + attempt = store.detached_attempt_root("deployment", "pool", "task", 1, token) + attempt.mkdir(parents=True) + payload = b'{"status":"completed"}\n' + (attempt / "worker-response.jsonl").write_bytes(payload) + main_thread = threading.get_ident() + original = store._contained + calls = [] + + def slow(path): + assert threading.get_ident() != main_thread, ( + "filesystem metadata ran on event loop" + ) + calls.append(path) + time.sleep(0.03) + return original(path) + + monkeypatch.setattr(store, "_contained", slow) + ticks = [] + + async def heartbeat(): + while True: + await asyncio.sleep(0.002) + ticks.append(time.monotonic()) + + beat = asyncio.create_task(heartbeat()) + try: + if operation == "create": + await store.create_upload() + elif operation == "task_root": + await store.resolve_task_root("deployment", "pool", "task") + elif operation == "commit": + await store.commit_upload( + upload, "deployment", "pool", "new-task", artifact_manifest={} + ) + elif operation == "write": + await store.write_json(task / "new.json", {}) + elif operation == "read": + await store.read_json(task / "x.json") + elif operation == "status": + assert ( + await store.read_detached_status("deployment", "pool", "task", 1, token) + is None + ) + elif operation == "cancel": + await store.request_detached_cancel("deployment", "pool", "task", 1, token) + elif operation == "consume": + + class Consumer: + def consume(self, value): + assert value["status"] == "completed" + + await store.consume_detached_response( + "deployment", + "pool", + "task", + 1, + token, + Consumer(), + expected_sha256=hashlib.sha256(payload).hexdigest(), + max_response_bytes=1024, + ) + elif operation == "finalize": + await store.finalize_b64_mp4( + task, base64.b64encode(b"video").decode(), {}, lambda *_: {} + ) + elif operation == "discard": + await store.discard(task) + elif operation == "trash": + await store.trash_orphan(TaskArtifactCandidate("task", task, 10, True)) + elif operation == "open_result": + async with opened_file( + lambda: store.result_path(result).open("rb") + ) as stream: + assert await run_file_io(stream.read) == b"video" + assert calls + assert len(ticks) >= 2, "event loop heartbeat stopped during metadata access" + finally: + beat.cancel() + await asyncio.gather(beat, return_exceptions=True) + + +async def test_cancel_waits_for_actual_io_completion(tmp_path, monkeypatch): + monkeypatch.setattr(asyncio, "to_thread", REAL_TO_THREAD) + entered, release = threading.Event(), threading.Event() + completed = [] + + def write(): + entered.set() + assert release.wait(2) + (tmp_path / "data").write_bytes(b"ok") + completed.append(True) + + task = asyncio.create_task(run_file_io(write)) + try: + while not entered.is_set(): + await asyncio.sleep(0.001) + task.cancel() + await asyncio.sleep(0.02) + task.cancel() # repeated cancellation must not abandon the worker + await asyncio.sleep(0.02) + assert not task.done() + assert not completed + finally: + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert completed + + +async def test_cancel_during_open_closes_returned_handle(tmp_path, monkeypatch): + monkeypatch.setattr(asyncio, "to_thread", REAL_TO_THREAD) + entered, release = threading.Event(), threading.Event() + streams = [] + + def open_slow(): + stream = (tmp_path / "data").open("wb") + streams.append(stream) + entered.set() + assert release.wait(2) + return stream + + async def owner(): + async with opened_file(open_slow): + pytest.fail("cancelled open must not enter the body") + + task = asyncio.create_task(owner()) + try: + while not entered.is_set(): + await asyncio.sleep(0.001) + task.cancel() + await asyncio.sleep(0.02) + assert not streams[0].closed + finally: + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert streams[0].closed + + +async def test_discard_rejects_traversal_and_escaped_parent(tmp_path): + store = FileArtifactStore(tmp_path / "artifacts") + outside = tmp_path / "outside" + outside.mkdir() + victim = outside / "victim" + victim.write_text("keep") + link = outside / "link" + link.symlink_to(victim) + (store.root / "escape").symlink_to(outside, target_is_directory=True) + for path in [ + store.root / ".." / "outside" / "link", + store.root / "escape" / "link", + store.root, + ]: + with pytest.raises(RuntimeError): + await store.discard(path) + assert victim.read_text() == "keep" + assert link.is_symlink() + + +async def test_live_responds_while_submission_path_lookup_is_blocked( + make_gateway_config, monkeypatch +): + from dingo.video_gateway.api import _SERVICE_KEY + from tests.video_gateway.test_api import _client, _form + + monkeypatch.setattr(asyncio, "to_thread", REAL_TO_THREAD) + client = await _client(make_gateway_config) + artifacts = client.server.app[_SERVICE_KEY].artifacts + original = artifacts._contained + entered, release = threading.Event(), threading.Event() + main_thread = threading.get_ident() + + def blocked(path): + assert threading.get_ident() != main_thread + entered.set() + assert release.wait(2) + return original(path) + + monkeypatch.setattr(artifacts, "_contained", blocked) + submission = asyncio.create_task(client.post("/v1/videos", data=_form())) + try: + + async def started(): + while not entered.is_set(): + await asyncio.sleep(0.001) + + await asyncio.wait_for(started(), 0.5) + response = await asyncio.wait_for(client.get("/live"), 0.5) + assert response.status == 200 + assert not submission.done() + finally: + release.set() + await asyncio.gather(submission, return_exceptions=True) + await client.close() diff --git a/tests/video_gateway/test_finalization.py b/tests/video_gateway/test_finalization.py new file mode 100644 index 000000000000..46b647f8918a --- /dev/null +++ b/tests/video_gateway/test_finalization.py @@ -0,0 +1,233 @@ +import asyncio +import dataclasses +import errno +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from dingo.video_gateway.finalization import ResultFinalizer +from dingo.video_gateway.models import TaskStatus, now_ms +from dingo.video_gateway.result_handoff import HANDOFF_KEY +from tests.video_gateway.test_result_handoff import patch_for, setup +from tests.video_gateway.test_task_store import _lease, _task + + +async def ready_task(make_gateway_config, tmp_path, operation): + store, _, stored, _ = await setup("memory") + ready = await store.transition( + stored.task.id, + expected={TaskStatus.IN_PROGRESS}, + expected_revision=stored.revision, + patch=patch_for(stored), + release_lease=True, + release_execution=True, + ) + config = make_gateway_config() + scheduling = dataclasses.replace( + config.pools[0].scheduling, finalization_retry_delay_s=0.001 + ) + pool = SimpleNamespace( + config=SimpleNamespace(scheduling=scheduling), + adapter=SimpleNamespace( + validate_artifact=None, + prepare_artifact=None, + artifact_requires_processing=None, + inspect_artifact_for_publication=None, + ), + ) + artifacts = SimpleNamespace(finalize_worker_mp4=operation) + finalizer = ResultFinalizer(store, artifacts, config, Mock(), "generation") + return store, ready, pool, finalizer + + +async def test_slow_finalization_does_not_hold_or_release_new_worker_slot( + make_gateway_config, tmp_path +): + entered, finish = asyncio.Event(), asyncio.Event() + path = tmp_path / "candidate.mp4" + + async def operation(*args, **kwargs): + entered.set() + await finish.wait() + path.write_bytes(b"mp4") + return path, 3, "a" * 64, {} + + store, ready, pool, f = await ready_task(make_gateway_config, tmp_path, operation) + pending = asyncio.create_task(f.run(ready, pool)) + try: + await asyncio.wait_for(entered.wait(), 1) + task = _task("next") + nxt, _ = await store.create_task( + task, principal_hash="p", idempotency_hash=None, queue_limit=8 + ) + lease = _lease(task) + lease.execution_token = "d" * 32 + assert ( + await store.reserve(nxt, lease, deadline_at_ms=now_ms() + 60000) is not None + ) + finish.set() + await asyncio.wait_for(pending, 1) + assert (await store.get_task("first")).task.status == TaskStatus.COMPLETED + assert (await store.list_leases(task.pool_id))[0].task_id == "next" + finally: + finish.set() + await asyncio.gather(pending, return_exceptions=True) + + +async def test_transient_error_retries_only_postprocessing( + make_gateway_config, tmp_path +): + count = 0 + path = tmp_path / "result.mp4" + + async def operation(*args, **kwargs): + nonlocal count + count += 1 + if count < 3: + raise OSError(errno.EIO, "transient storage error") + path.write_bytes(b"mp4") + return path, 3, "a" * 64, {} + + store, ready, pool, f = await ready_task(make_gateway_config, tmp_path, operation) + await f.run(ready, pool) + task = (await store.get_task("first")).task + assert task.status == TaskStatus.COMPLETED and task.attempt == 1 + assert count == 3 and task.normalized_request[HANDOFF_KEY]["failures"] == 2 + assert not await store.list_leases(task.pool_id) + + +async def test_permanent_result_failure_is_not_worker_failure( + make_gateway_config, tmp_path +): + async def operation(*args, **kwargs): + raise ValueError("invalid video") + + store, ready, pool, f = await ready_task(make_gateway_config, tmp_path, operation) + await f.run(ready, pool) + task = (await store.get_task("first")).task + assert task.status == TaskStatus.FAILED and task.error.code == "finalization_failed" + assert task.attempt == 1 and not await store.list_leases(task.pool_id) + + +async def test_cancel_during_finalize_discards_only_own_candidate( + make_gateway_config, tmp_path +): + entered, finish = asyncio.Event(), asyncio.Event() + path = tmp_path / "candidate.mp4" + + async def operation(*args, **kwargs): + entered.set() + await finish.wait() + path.write_bytes(b"mp4") + return path, 3, "a" * 64, {} + + store, ready, pool, f = await ready_task(make_gateway_config, tmp_path, operation) + pending = asyncio.create_task(f.run(ready, pool)) + try: + await asyncio.wait_for(entered.wait(), 1) + await store.request_cancel("first") + finish.set() + await asyncio.wait_for(pending, 1) + assert (await store.get_task("first")).task.status == TaskStatus.CANCELLED + assert not path.exists() + finally: + finish.set() + await asyncio.gather(pending, return_exceptions=True) + + +async def test_lost_commit_response_does_not_delete_published_result( + make_gateway_config, tmp_path +): + path = tmp_path / "candidate.mp4" + + async def operation(*args, **kwargs): + path.write_bytes(b"mp4") + return path, 3, "a" * 64, {} + + store, ready, pool, f = await ready_task(make_gateway_config, tmp_path, operation) + original = store.transition + + async def ambiguous(*args, **kwargs): + result = await original(*args, **kwargs) + if kwargs["patch"].get("status") == TaskStatus.COMPLETED: + raise ConnectionError("lost etcd reply after commit") + return result + + store.transition = ambiguous + with pytest.raises(ConnectionError): + await f.run(ready, pool) + assert path.read_bytes() == b"mp4" + await f.run(ready, pool) + assert (await store.get_task("first")).task.status == TaskStatus.COMPLETED + + +async def test_transient_errors_exhaust_bounded_retry_budget( + make_gateway_config, tmp_path +): + calls = 0 + + async def operation(*args, **kwargs): + nonlocal calls + calls += 1 + raise OSError(errno.EIO, "persistent outage") + + store, ready, pool, f = await ready_task(make_gateway_config, tmp_path, operation) + await f.run(ready, pool) + assert ( + calls == 3 and (await store.get_task("first")).task.status == TaskStatus.FAILED + ) + + +async def test_processing_deadline_is_not_a_worker_failure( + make_gateway_config, tmp_path +): + calls = 0 + + async def operation(*args, **kwargs): + nonlocal calls + calls += 1 + await asyncio.sleep(10) + + store, ready, pool, f = await ready_task(make_gateway_config, tmp_path, operation) + ref = dict(ready.task.normalized_request[HANDOFF_KEY]) + ref["deadline_at_ms"] = now_ms() + 30 + ready = await store.transition( + "first", + expected={TaskStatus.FINALIZING}, + patch={ + "normalized_request": {**ready.task.normalized_request, HANDOFF_KEY: ref} + }, + ) + await asyncio.wait_for(f.run(ready, pool), 1) + task = (await store.get_task("first")).task + assert calls == 1 and task.status == TaskStatus.FAILED + assert task.error.code == "finalization_timeout" and task.attempt == 1 + + +async def test_resuming_finalizer_uses_original_durable_result( + make_gateway_config, tmp_path +): + path = tmp_path / "result.mp4" + entered = asyncio.Event() + + async def operation(*args, **kwargs): + entered.set() + await asyncio.sleep(10) + + store, ready, pool, f = await ready_task(make_gateway_config, tmp_path, operation) + pending = asyncio.create_task(f.run(ready, pool)) + await asyncio.wait_for(entered.wait(), 1) + pending.cancel() + await asyncio.gather(pending, return_exceptions=True) + assert (await store.get_task("first")).task.status == TaskStatus.FINALIZING + + async def recovered(*args, **kwargs): + assert args[5] == ready.task.normalized_request[HANDOFF_KEY]["artifact"] + path.write_bytes(b"mp4") + return path, 3, "a" * 64, {} + + f.artifacts.finalize_worker_mp4 = recovered + await f.run(ready, pool) + assert (await store.get_task("first")).task.status == TaskStatus.COMPLETED + assert not await store.list_leases(ready.task.pool_id) diff --git a/tests/video_gateway/test_minimax_h3_adapter.py b/tests/video_gateway/test_minimax_h3_adapter.py index 82f2f2b72fcf..a1eb3a27502b 100644 --- a/tests/video_gateway/test_minimax_h3_adapter.py +++ b/tests/video_gateway/test_minimax_h3_adapter.py @@ -427,9 +427,7 @@ def test_media_type_is_normalized_from_mime_and_signature( ("audio/mpeg", b"ID3\x04\0\0\0\0\0\0"), ], ) -def test_media_not_supported_by_current_worker_is_rejected( - tmp_path, declared, payload -): +def test_media_not_supported_by_current_worker_is_rejected(tmp_path, declared, payload): path = tmp_path / "reference.bin" path.write_bytes(payload) upload = UploadedArtifact( @@ -512,9 +510,7 @@ def test_reference_wav_rejects_non_pcm_codec(tmp_path, codec): ("width", "height"), [(255, 512), (512, 5761), (256, 1024), (1024, 256)], ) -def test_reference_image_shape_is_rejected_before_worker( - tmp_path, width, height -): +def test_reference_image_shape_is_rejected_before_worker(tmp_path, width, height): path = tmp_path / "reference.png" path.write_bytes(b"unused") upload = UploadedArtifact( @@ -663,19 +659,13 @@ async def load_image(self, value): envelope = json.dumps( { "type": "ref2va_mixed_v1", - "videos": [ - _data_url( - "video/quicktime", b"\0\0\0\x18ftypqt \0\0\0\0qt " - ) - ], + "videos": [_data_url("video/quicktime", b"\0\0\0\x18ftypqt \0\0\0\0qt ")], "audios": [_data_url("audio/wav", b"RIFF\0\0\0\0WAVEpayload")], } ) async with adapter.request_scope("request-1") as scope: - reference = await adapter._decode_mixed_envelope( - envelope, Loader(), scope - ) + reference = await adapter._decode_mixed_envelope(envelope, Loader(), scope) assert [Path(item.path).suffix for item in reference.videos] == [".mov"] assert [Path(item.path).suffix for item in reference.audios] == [".wav"] inputs = adapter.build_engine_inputs( @@ -797,3 +787,120 @@ def test_generate_sound_false_remuxes_without_reencoding_video( assert len(container.streams.audio) == 0 assert media["video_codec"] == "h264" assert media["audio_codec"] is None + + +def test_video_only_mp4_is_not_rewritten(tmp_path, make_gateway_config, monkeypatch): + import av + + adapter = _adapter(make_gateway_config) + adapter.options["validate_media"] = True + result = tmp_path / "result.mp4" + _write_h264_aac_mp4(result, frames=124, width=256, height=256) + normalized = { + "width": 256, + "height": 256, + "num_frames": 120, + "generate_sound": False, + } + # First strip real AAC audio, then exercise the already-video-only path. + adapter.prepare_artifact(result, normalized) + before = result.read_bytes() + before_stat = result.stat() + original_open = av.open + + def read_only_open(*args, **kwargs): + assert kwargs.get("mode", "r") == "r", "unexpected MP4 rewrite" + return original_open(*args, **kwargs) + + def no_write(*_args, **_kwargs): + pytest.fail("already-video-only MP4 must not be replaced or fsynced") + + with monkeypatch.context() as patch: + patch.setattr(av, "open", read_only_open) + patch.setattr(h3_module.os, "replace", no_write) + patch.setattr(h3_module.os, "fsync", no_write) + adapter.prepare_artifact(result, normalized) + media = adapter.validate_artifact(result, normalized) + assert result.read_bytes() == before + assert result.stat().st_ino == before_stat.st_ino + assert result.stat().st_mtime_ns == before_stat.st_mtime_ns + assert media["frames"] == 124 and media["audio_codec"] is None + assert not result.with_name("result.mp4.video-only.mp4").exists() + + +@pytest.mark.parametrize("normalized", [{"generate_sound": True}, {}]) +def test_preserve_audio_does_not_enter_remux_path( + tmp_path, make_gateway_config, monkeypatch, normalized +): + import av + + adapter = _adapter(make_gateway_config) + result = tmp_path / "result.mp4" + _write_h264_aac_mp4(result, frames=124, width=256, height=256) + before = result.read_bytes() + + def unexpected_open(*_args, **_kwargs): + pytest.fail("audio-preserving requests must not enter remux processing") + + monkeypatch.setattr(av, "open", unexpected_open) + adapter.prepare_artifact(result, normalized) + assert result.read_bytes() == before + + +async def test_video_only_fast_path_still_validates_before_publication( + tmp_path, make_gateway_config +): + from dingo.video_gateway.artifact_store import FileArtifactStore + + adapter = _adapter(make_gateway_config) + adapter.options["validate_media"] = True + result = tmp_path / "result.mp4" + _write_h264_aac_mp4(result, frames=124, width=256, height=256) + adapter.prepare_artifact(result, {"generate_sound": False}) + store = FileArtifactStore(tmp_path / "artifacts") + task_root = store.task_root("deployment", "pool", "task") + with pytest.raises(RuntimeError, match="dimensions"): + await store.finalize_b64_mp4( + task_root, + base64.b64encode(result.read_bytes()).decode(), + {"width": 128, "height": 256, "num_frames": 120, "generate_sound": False}, + adapter.validate_artifact, + adapter.prepare_artifact, + ) + assert not list((task_root / "result").iterdir()) + assert not list((task_root / "tmp").iterdir()) + + +@pytest.mark.parametrize( + "stream_count,header", [(2, b"\0\0\0\x18ftypisom"), (1, b"not-an-mp4!!")] +) +def test_fast_path_preserves_other_stream_and_container_handling( + tmp_path, make_gateway_config, monkeypatch, stream_count, header +): + from contextlib import nullcontext + + import av + + class Streams: + video = [object()] + + def __len__(self): + return stream_count + + result = tmp_path / "result.mp4" + result.write_bytes(header) + writes = [] + + def fake_open(path, **kwargs): + if kwargs.get("mode") == "w": + writes.append(path) + raise RuntimeError("remux attempted") + return nullcontext(SimpleNamespace(streams=Streams())) + + monkeypatch.setattr(av, "open", fake_open) + with pytest.raises(RuntimeError, match="remux attempted"): + _adapter(make_gateway_config).prepare_artifact( + result, {"generate_sound": False} + ) + assert len(writes) == 1 + assert result.read_bytes() == header diff --git a/tests/video_gateway/test_real_etcd_contract.py b/tests/video_gateway/test_real_etcd_contract.py index 045123939973..54a5d26406e7 100644 --- a/tests/video_gateway/test_real_etcd_contract.py +++ b/tests/video_gateway/test_real_etcd_contract.py @@ -30,6 +30,102 @@ _ETCD_URL = os.environ.get("DINGO_VIDEO_TEST_ETCD_URL") +@pytest.mark.skipif(not _ETCD_URL, reason="requires a real etcd v3 endpoint") +async def test_result_handoff_and_owner_takeover_never_touch_reused_slot(): + from dingo.video_gateway.models import now_ms + from tests.video_gateway.test_result_handoff import patch_for + + client = EtcdHttpClient(str(_ETCD_URL), timeout_s=5.0) + prefix = f"/dingo/continuous-execution-contract/{uuid.uuid4().hex}" + store = EtcdTaskStore(client, prefix=prefix, deployment_id="contract") + owners = [] + try: + for generation in ["generation", "gateway-b", "gateway-c"]: + owners.append(await store.register_gateway(generation, ttl_s=30)) + first = _task("first") + stored, _ = await store.create_task( + first, principal_hash="p", idempotency_hash=None, queue_limit=8 + ) + lease = _lease(first) + lease.execution_token = "c" * 32 + stored = await store.reserve( + stored, lease, deadline_at_ms=now_ms() + 60000, reserve_retry=True + ) + stored = await store.transition( + first.id, + expected={TaskStatus.DISPATCHING}, + patch={"status": TaskStatus.IN_PROGRESS}, + ) + ready = await store.transition( + first.id, + expected={TaskStatus.IN_PROGRESS}, + expected_revision=stored.revision, + patch=patch_for(stored), + release_lease=True, + release_execution=True, + ) + assert not await store.list_leases(first.pool_id) + assert await client.get(store._retry_credit_key(ready.task)) is None + assert ( + int( + ( + await client.get(store._retry_counter_key(first.pool_id, "credits")) + ).value + ) + == 0 + ) + second = _task("second") + queued, _ = await store.create_task( + second, principal_hash="p", idempotency_hash=None, queue_limit=8 + ) + next_lease = _lease(second) + next_lease.owner_generation = "gateway-b" + next_lease.execution_token = "d" * 32 + reserved = await store.reserve( + queued, next_lease, deadline_at_ms=now_ms() + 60000, reserve_retry=True + ) + assert reserved is not None + key = store._lease_key(second.pool_id, reserved.task.worker_key) + occupant = await client.get(key) + assert ( + await store.claim_finalizing(ready, new_owner_generation="gateway-b") + is None + ) + await store.unregister_gateway(owners.pop(0)) + claims = await asyncio.gather( + *[ + store.claim_orphaned_active(ready, new_owner_generation=owner) + for owner in ["gateway-b", "gateway-c"] + ] + ) + winners = [claim for claim in claims if claim is not None] + assert len(winners) == 1 + claimed = winners[0] + assert claimed.task.worker_lease_id is None + assert await client.get(key) == occupant + await store.transition( + first.id, + expected={TaskStatus.FINALIZING}, + expected_revision=claimed.revision, + patch={"status": TaskStatus.COMPLETED}, + release_lease=False, + ) + assert await client.get(key) == occupant + assert ( + int( + ( + await client.get(store._retry_counter_key(first.pool_id, "credits")) + ).value + ) + == 1 + ) + finally: + for lease_id in owners: + await store.unregister_gateway(lease_id) + await _delete_prefix(client, prefix) + await client.close() + + async def _delete_prefix(client: EtcdHttpClient, prefix: str) -> None: values, _revision = await client.range_all(prefix, prefix=True) for offset in range(0, len(values), 100): @@ -106,10 +202,7 @@ async def test_real_etcd_range_all_reads_every_page_from_one_snapshot(): for offset in range(0, len(items), 100): succeeded, _revision = await client.txn( [], - [ - client.put(key, value) - for key, value in items[offset : offset + 100] - ], + [client.put(key, value) for key, value in items[offset : offset + 100]], ) assert succeeded is True @@ -186,9 +279,7 @@ async def test_real_etcd_count_descending_and_batch_get_contract(): ) assert succeeded is True assert await client.count_prefix(prefix) == len(keys) - descending = await client.range( - prefix, prefix=True, limit=2, descending=True - ) + descending = await client.range(prefix, prefix=True, limit=2, descending=True) assert [value.key for value in descending] == list(reversed(keys[-2:])) values, snapshot_revision = await client.get_many( [keys[1], prefix + "missing", keys[3]], revision=revision @@ -339,9 +430,7 @@ async def test_two_gateways_preserve_task_owned_by_healthy_peer( context_factory=FakeContext, generation="gateway-b", ) - service_a = VideoGatewayService( - config, store_a, artifacts, dispatcher_a, adapters - ) + service_a = VideoGatewayService(config, store_a, artifacts, dispatcher_a, adapters) started_a = False started_b = False try: @@ -385,9 +474,7 @@ async def test_detached_task_survives_owner_gateway_shutdown_and_is_claimed( make_gateway_config, ): prefix = f"/dingo/video-gateway-detached-ha-tests/{uuid.uuid4().hex}" - pool_raw = _pool( - "fl-pool", "public-fl", "dyn://scope-a.backend.generate" - ) + pool_raw = _pool("fl-pool", "public-fl", "dyn://scope-a.backend.generate") pool_raw["execution_mode"] = "detached" config = make_gateway_config(pools=[pool_raw]) cleanup_client = EtcdHttpClient(str(_ETCD_URL), timeout_s=5.0) @@ -427,16 +514,14 @@ async def test_detached_task_survives_owner_gateway_shutdown_and_is_claimed( context_factory=FakeContext, generation="detached-gateway-b", ) - service_a = VideoGatewayService( - config, store_a, artifacts, dispatcher_a, adapters - ) + service_a = VideoGatewayService(config, store_a, artifacts, dispatcher_a, adapters) started_a = False started_b = False try: await dispatcher_a.start() started_a = True - await dispatcher_b.start() - started_b = True + # Submitting through A does not force A to win a shared queue. First + # establish the intended owner, then start its standby before failure. submission = await _submit(service_a, "public-fl") await asyncio.wait_for(handler.started.wait(), timeout=2) active = await store_b.get_task(submission.stored.task.id) @@ -444,6 +529,8 @@ async def test_detached_task_survives_owner_gateway_shutdown_and_is_claimed( assert active.task.status == TaskStatus.IN_PROGRESS assert active.task.owner_generation == "detached-gateway-a" + await dispatcher_b.start() + started_b = True await dispatcher_a.stop() started_a = False handler.release.set() diff --git a/tests/video_gateway/test_result_handoff.py b/tests/video_gateway/test_result_handoff.py new file mode 100644 index 000000000000..640e41624eb9 --- /dev/null +++ b/tests/video_gateway/test_result_handoff.py @@ -0,0 +1,188 @@ +import dataclasses +import json + +import pytest + +from dingo.video_gateway.errors import StoreConflict +from dingo.video_gateway.models import TaskStatus, now_ms +from dingo.video_gateway.result_handoff import HANDOFF_KEY, make_handoff, read_handoff +from dingo.video_gateway.task_store import EtcdTaskStore, MemoryTaskStore +from tests.video_gateway.test_etcd_task_store import FakeEtcd +from tests.video_gateway.test_task_store import _lease, _task + +ARTIFACT = dict( + schema_version=1, + filename="worker-video-" + "a" * 32 + ".mp4", + bytes=100, + sha256="b" * 64, +) + + +async def setup(kind): + client = FakeEtcd() if kind == "etcd" else None + store = ( + EtcdTaskStore(client, prefix="/handoff-test", deployment_id="deployment") + if client + else MemoryTaskStore() + ) + original = _task("first") + stored, _ = await store.create_task( + original, principal_hash="p", idempotency_hash=None, queue_limit=8 + ) + lease = _lease(original) + lease.execution_token = "c" * 32 + if client: + owner = await store.register_gateway("generation", ttl_s=15) + else: + owner = None + options = {"reserve_retry": True} if client else {} + stored = await store.reserve( + stored, lease, deadline_at_ms=now_ms() + 60000, **options + ) + stored = await store.transition( + stored.task.id, + expected={TaskStatus.DISPATCHING}, + patch={"status": TaskStatus.IN_PROGRESS}, + ) + return store, client, stored, owner + + +def patch_for(stored): + return { + "status": TaskStatus.FINALIZING, + "worker_lease_id": None, + "normalized_request": { + **stored.task.normalized_request, + HANDOFF_KEY: make_handoff(stored.task, ARTIFACT, timeout_s=30), + }, + } + + +@pytest.mark.parametrize("kind", ["memory", "etcd"]) +async def test_atomic_handoff_releases_slot_and_preserves_result(kind): + store, client, stored, owner = await setup(kind) + ready = await store.transition( + stored.task.id, + expected={TaskStatus.IN_PROGRESS}, + expected_revision=stored.revision, + patch=patch_for(stored), + release_lease=True, + release_execution=True, + ) + assert ready.task.status == TaskStatus.FINALIZING + assert read_handoff(ready.task)["artifact"] == ARTIFACT + assert not await store.list_leases(stored.task.pool_id) + if client: + assert await client.get(store._retry_credit_key(stored.task)) is None + assert ( + int( + ( + await client.get( + store._retry_counter_key(stored.task.pool_id, "credits") + ) + ).value + ) + == 0 + ) + # A new task can use exactly the same slot before old postprocessing ends. + next_task = _task("second") + next_stored, _ = await store.create_task( + next_task, principal_hash="p", idempotency_hash=None, queue_limit=8 + ) + next_lease = _lease(next_task) + next_lease.execution_token = "d" * 32 + reserved = await store.reserve( + next_stored, next_lease, deadline_at_ms=now_ms() + 60000 + ) + assert reserved is not None + await store.transition( + ready.task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=ready.revision, + patch={"status": TaskStatus.COMPLETED}, + release_lease=True, + ) + assert (await store.list_leases(stored.task.pool_id))[0].task_id == "second" + + +@pytest.mark.parametrize("kind", ["memory", "etcd"]) +async def test_cancel_race_cannot_release_lease_or_publish_handoff(kind): + store, client, stored, owner = await setup(kind) + await store.request_cancel(stored.task.id) + with pytest.raises(StoreConflict): + await store.transition( + stored.task.id, + expected={TaskStatus.IN_PROGRESS}, + expected_revision=stored.revision, + patch=patch_for(stored), + release_lease=True, + release_execution=True, + ) + current = await store.get_task(stored.task.id) + assert read_handoff(current.task) is None + assert len(await store.list_leases(stored.task.pool_id)) == 1 + + +async def test_handoff_recovery_does_not_touch_reused_worker_slot(): + store, client, stored, owner = await setup("etcd") + ready = await store.transition( + stored.task.id, + expected={TaskStatus.IN_PROGRESS}, + expected_revision=stored.revision, + patch=patch_for(stored), + release_lease=True, + release_execution=True, + ) + second = _task("second") + nxt, _ = await store.create_task( + second, principal_hash="p", idempotency_hash=None, queue_limit=8 + ) + nxt_lease = _lease(second) + nxt_lease.execution_token = "d" * 32 + await store.register_gateway("gateway-b", ttl_s=15) + nxt_lease.owner_generation = "gateway-b" + reserved = await store.reserve(nxt, nxt_lease, deadline_at_ms=now_ms() + 60000) + slot_key = store._lease_key(second.pool_id, reserved.task.worker_key) + slot_before = await client.get(slot_key) + assert await store.claim_finalizing(ready, new_owner_generation="gateway-b") is None + await store.unregister_gateway(owner) + claimed = await store.claim_orphaned_active(ready, new_owner_generation="gateway-b") + assert claimed.task.owner_generation == "gateway-b" + assert claimed.task.worker_lease_id is None + assert await client.get(slot_key) == slot_before + with pytest.raises(StoreConflict): + await store.transition( + ready.task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=ready.revision, + patch={"status": TaskStatus.FAILED}, + release_lease=True, + ) + assert await client.get(slot_key) == slot_before + + +async def test_conflicting_token_cannot_handoff_someone_elses_reservation(): + store, client, stored, owner = await setup("etcd") + key = store._lease_key(stored.task.pool_id, stored.task.worker_key) + value = await client.get(key) + data = json.loads(value.value) + data["execution_token"] = "e" * 32 + client.values[key] = dataclasses.replace(value, value=json.dumps(data).encode()) + with pytest.raises(StoreConflict): + await store.transition( + stored.task.id, + expected={TaskStatus.IN_PROGRESS}, + expected_revision=stored.revision, + patch=patch_for(stored), + release_lease=True, + release_execution=True, + ) + assert read_handoff((await store.get_task(stored.task.id)).task) is None + assert await client.get(store._retry_credit_key(stored.task)) is not None + + +@pytest.mark.parametrize("timeout", [float("nan"), float("inf"), 0, -1, True, 86401]) +async def test_handoff_rejects_invalid_deadline(timeout): + _, _, stored, _ = await setup("memory") + with pytest.raises(ValueError): + make_handoff(stored.task, ARTIFACT, timeout_s=timeout) diff --git a/tests/video_gateway/test_slot_dispatcher.py b/tests/video_gateway/test_slot_dispatcher.py new file mode 100644 index 000000000000..8a2506c86fff --- /dev/null +++ b/tests/video_gateway/test_slot_dispatcher.py @@ -0,0 +1,192 @@ +import asyncio +import dataclasses +import time +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest + +from dingo.video_gateway.dispatcher import VideoDispatcher +from dingo.video_gateway.models import now_ms +from dingo.video_gateway.task_store import StoredTask, worker_key +from tests.video_gateway.test_dispatch_idle import stack +from tests.video_gateway.test_task_store import _task + + +def probe_stack(response): + calls = [] + + async def direct(payload, instance, context): + calls.append(payload) + + async def stream(): + yield response + + return stream() + + dispatcher = VideoDispatcher.__new__(VideoDispatcher) + dispatcher.context_factory = lambda *a: MagicMock() + dispatcher.telemetry = MagicMock() + pool = SimpleNamespace( + config=SimpleNamespace( + pool_id="p", scheduling=SimpleNamespace(worker_capacity=2) + ), + client=SimpleNamespace(direct=direct, instance_ids=lambda: [7]), + instance_ids=[], + capacity_cache={}, + prefetch_instances=set(), + ) + return dispatcher, pool, calls + + +@pytest.mark.asyncio +async def test_capacity_cached_per_physical_registration_and_clamped(): + d, p, calls = probe_stack( + dict( + schema_version=1, + capabilities=["execution_capacity_v1"], + execution_capacity=8, + accepting=True, + ) + ) + await d._refresh_instances(p) + await d._refresh_instances(p) + assert len(calls) == 1 + assert d._worker_slots(p) == [(7, 0), (7, 1)] + p.client.instance_ids = lambda: [] + await d._refresh_instances(p) + assert not p.capacity_cache + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "response", + [ + {}, + { + "schema_version": 1, + "capabilities": ["execution_capacity_v1"], + "execution_capacity": True, + "accepting": True, + }, + ], +) +async def test_unsupported_or_malformed_capacity_never_infers_multiple_slots(response): + d, p, calls = probe_stack(response) + await d._refresh_instances(p) + await d._refresh_instances(p) + assert len(calls) == 1 and d._worker_slots(p) == [(7, 0)] + assert p.capacity_cache[7][1] > time.monotonic() + + +@pytest.mark.asyncio +async def test_single_slot_does_not_query_capabilities(): + d, p, calls = probe_stack({}) + p.config.scheduling.worker_capacity = 1 + await d._refresh_instances(p) + assert not calls and d._worker_slots(p) == [(7, 0)] + + +@pytest.mark.asyncio +async def test_prefetch_capacity_is_distinct_from_execution_capacity(): + d, p, calls = probe_stack( + dict( + schema_version=1, + capabilities=["execution_capacity_v1", "execution_prefetch_v1"], + execution_capacity=1, + prefetch_capacity=1, + admission_capacity=2, + accepting=True, + ) + ) + p.config.scheduling.worker_capacity = 1 + p.config.scheduling.worker_prefetch_capacity = 1 + p.prefetch_instances = set() + await d._refresh_instances(p) + assert d._worker_slots(p) == [(7, 0), (7, 1)] and p.prefetch_instances == {7} + + +@pytest.mark.asyncio +async def test_prefetch_is_disabled_for_legacy_workers_and_mismatched_engine_n(): + for caps, n in [ + (["execution_capacity_v1"], 1), + (["execution_capacity_v1", "execution_prefetch_v1"], 2), + ]: + d, p, calls = probe_stack( + dict( + schema_version=1, + capabilities=caps, + execution_capacity=n, + prefetch_capacity=1, + admission_capacity=n + 1, + accepting=True, + ) + ) + p.config.scheduling.worker_capacity = 1 + p.config.scheduling.worker_prefetch_capacity = 1 + p.prefetch_instances = set() + await d._refresh_instances(p) + assert d._worker_slots(p) == [(7, 0)] and not p.prefetch_instances + + +async def test_failed_reprobe_clears_prefetch_protocol_capability(): + d, p, _ = probe_stack({}) + p.prefetch_instances.add(7) + p.config.scheduling.worker_prefetch_capacity = 1 + assert (await d._worker_capacity(p, 7))[0] == 1 + assert not p.prefetch_instances + + +@pytest.mark.asyncio +async def test_dispatch_uses_free_second_slot(make_gateway_config): + d, p, store, client = stack(make_gateway_config) + p.config = dataclasses.replace( + p.config, scheduling=dataclasses.replace(p.config.scheduling, worker_capacity=2) + ) + p.capacity_cache[7] = (2, float("inf")) + p.lease_cache["first"] = SimpleNamespace( + worker_key=worker_key(p.config.backend_target, 7) + ) + task = _task("queued", pool_id=p.config.pool_id) + task.backend_target = p.config.backend_target + store.list_queued.return_value = [StoredTask(task, 1)] + assert await d._dispatch_once(p) + reserved_lease = store.reserve.call_args.args[1] + assert reserved_lease.worker_instance_id == 7 + assert reserved_lease.worker_key == worker_key(task.backend_target, 7, 1) + assert (await d.memory_budget.snapshot()).used_bytes == 0 + + +@pytest.mark.asyncio +async def test_busy_retries_submission_without_changing_attempt(): + d, p, calls = probe_stack(None) + responses = iter( + [{"state": "busy", "accepted": False}, {"state": "accepted", "accepted": True}] + ) + + async def direct(payload, instance, context): + calls.append(payload) + + async def stream(): + yield next(responses) + + return stream() + + p.client.direct = direct + task = SimpleNamespace( + worker_instance_id=7, deadline_at_ms=now_ms() + 3000, attempt=1 + ) + ack = await d._detached_submit_ack( + p, task, {"submit": "same-token"}, MagicMock(), lambda v: v + ) + assert ack["accepted"] and task.attempt == 1 + assert calls == [{"submit": "same-token"}] * 2 + + +@pytest.mark.asyncio +async def test_busy_stops_at_deadline(): + d, p, calls = probe_stack({"state": "busy", "accepted": False}) + task = SimpleNamespace(worker_instance_id=7, deadline_at_ms=now_ms() - 1) + with pytest.raises(asyncio.TimeoutError): + await d._detached_submit_ack(p, task, {}, MagicMock(), lambda v: v) + assert not calls diff --git a/tests/video_gateway/test_slot_metrics.py b/tests/video_gateway/test_slot_metrics.py new file mode 100644 index 000000000000..9c4c3305bf89 --- /dev/null +++ b/tests/video_gateway/test_slot_metrics.py @@ -0,0 +1,156 @@ +"""Physical Workers, admission leases and engine concurrency are distinct.""" + +import dataclasses +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from dingo.video_gateway.task_store import worker_key +from tests.video_gateway.test_dispatch_idle import stack + + +def setup_pool(make_gateway_config, *, n=2, p=1): + dispatcher, pool, store, _ = stack(make_gateway_config) + pool.config = dataclasses.replace( + pool.config, + scheduling=dataclasses.replace( + pool.config.scheduling, worker_capacity=n, worker_prefetch_capacity=p + ), + ) + pool.capacity_cache = {7: (n + p, float("inf"))} + pool.prefetch_instances = {7} if p else set() + pool.discovery_healthy = True + return dispatcher, pool, store + + +def lease(pool, slot=0, *, instance=7, state="active"): + return SimpleNamespace( + worker_key=worker_key(pool.config.backend_target, instance, slot), + worker_instance_id=instance, + backend_target=pool.config.backend_target, + state=state, + ) + + +def test_two_slots_are_one_busy_worker(make_gateway_config): + d, p, _ = setup_pool(make_gateway_config) + s = d.pool_capacity_snapshot(p.config.pool_id, [lease(p, 0), lease(p, 1)]) + assert s == dict( + workers=1, + worker_busy=1, + worker_execution_capacity=2, + worker_prefetch_capacity=1, + worker_admission_capacity=3, + worker_slots_busy=2, + worker_slots_quarantined=0, + worker_slots_free=1, + worker_unmapped_leases=0, + worker_capacity_view_healthy=1, + ) + + +def test_quarantine_and_disappeared_worker_do_not_inflate_capacity(make_gateway_config): + d, p, _ = setup_pool(make_gateway_config) + leases = [lease(p, 0), lease(p, 1, state="quarantined"), lease(p, instance=8)] + s = d.pool_capacity_snapshot(p.config.pool_id, leases) + assert s["worker_busy"] == 1 + assert s["worker_slots_busy"] == s["worker_slots_quarantined"] == 1 + assert s["worker_slots_free"] == s["worker_unmapped_leases"] == 1 + p.instance_ids = [] + s = d.pool_capacity_snapshot(p.config.pool_id, leases) + assert ( + s["worker_busy"] + == s["worker_admission_capacity"] + == s["worker_slots_free"] + == 0 + ) + assert s["worker_unmapped_leases"] == 3 + assert not d.has_workers(p.config.pool_id) + + +@pytest.mark.parametrize("n,prefetch", [(1, 0), (2, 0), (2, 1)]) +def test_saturation_does_not_disable_model(make_gateway_config, n, prefetch): + d, p, _ = setup_pool(make_gateway_config, n=n, p=prefetch) + s = d.pool_capacity_snapshot( + p.config.pool_id, [lease(p, i) for i in range(n + prefetch)] + ) + assert s["worker_slots_free"] == 0 + assert d.has_workers(p.config.pool_id) + + +@pytest.mark.parametrize("fault", ["discovery", "watch"]) +def test_unhealthy_view_never_reports_free_slots(make_gateway_config, fault): + d, p, _ = setup_pool(make_gateway_config) + if fault == "discovery": + p.discovery_healthy = False + else: + p.lease_watch_healthy = False + s = d.pool_capacity_snapshot(p.config.pool_id, []) + assert s["worker_capacity_view_healthy"] == s["worker_slots_free"] == 0 + + +def test_unknown_and_not_accepting_capacity_then_legacy_fallback(make_gateway_config): + d, p, _ = setup_pool(make_gateway_config, p=0) + for cache in ({}, {7: (0, 0)}): + p.capacity_cache = cache + assert not d.has_workers(p.config.pool_id) + assert ( + d.pool_capacity_snapshot(p.config.pool_id, [])["worker_execution_capacity"] + == 0 + ) + p.capacity_cache = {7: (1, 0)} + assert d.has_workers(p.config.pool_id) + assert ( + d.pool_capacity_snapshot(p.config.pool_id, [])["worker_execution_capacity"] == 1 + ) + + +def test_handoff_release_counts_leases_not_historical_task_assignment( + make_gateway_config, +): + d, p, _ = setup_pool(make_gateway_config) + d._finalizing = {"old-task": p.config.pool_id, "elsewhere": "another-pool"} + s = d.pool_capacity_snapshot(p.config.pool_id, []) + assert s["worker_busy"] == s["worker_slots_busy"] == 0 + assert s["worker_slots_free"] == 3 + assert d.pool_finalization_pending(p.config.pool_id) == 1 + + +async def test_metrics_endpoint_uses_slot_snapshot(make_gateway_config, monkeypatch): + from dingo.video_gateway.api import _SERVICE_KEY + from tests.video_gateway.test_api import _client + + client = await _client(make_gateway_config) + try: + service = client.server.app[_SERVICE_KEY] + d = service.dispatcher + p = d.pools["fl-pool"] + p.config = dataclasses.replace( + p.config, + scheduling=dataclasses.replace( + p.config.scheduling, worker_capacity=2, worker_prefetch_capacity=1 + ), + ) + # Freeze discovery so the legacy fake's next refresh cannot change the snapshot. + monkeypatch.setattr(d, "_refresh_instances", AsyncMock()) + p.instance_ids = [7] + p.capacity_cache = {7: (3, float("inf"))} + p.prefetch_instances = {7} + monkeypatch.setattr( + d, "pool_leases", AsyncMock(return_value=[lease(p), lease(p, 1)]) + ) + response = await client.get("/metrics") + assert response.status == 200 + body = await response.text() + for metric, value in ( + ("workers", 1), + ("worker_busy", 1), + ("worker_slots_busy", 2), + ("worker_execution_capacity", 2), + ("worker_prefetch_capacity", 1), + ("worker_admission_capacity", 3), + ): + assert f'dingo_video_{metric}{{pool="fl-pool"}} {value}\n' in body + finally: + await client.close() diff --git a/tests/video_gateway/test_terminal_cas_retry.py b/tests/video_gateway/test_terminal_cas_retry.py new file mode 100644 index 000000000000..23e40bb38f74 --- /dev/null +++ b/tests/video_gateway/test_terminal_cas_retry.py @@ -0,0 +1,288 @@ +"""Terminal writes must survive unrelated pool-ledger CAS contention.""" + +import asyncio +import json + +import pytest + +from dingo.video_gateway.errors import StoreConflict +from dingo.video_gateway.models import TaskStatus, now_ms +from dingo.video_gateway.task_store import EtcdTaskStore +from tests.video_gateway.test_etcd_task_store import FakeEtcd, _decode +from tests.video_gateway.test_task_store import _lease, _task + + +async def prepared(): + client = FakeEtcd() + store = EtcdTaskStore(client, prefix="/isolated/terminal-cas", deployment_id="cas") + task = _task("video-cas") + stored, _ = await store.create_task( + task, principal_hash="p", idempotency_hash=None, queue_limit=8 + ) + lease = _lease(task) + await store.register_gateway(lease.owner_generation, ttl_s=15) + stored = await store.reserve( + stored, + lease, + deadline_at_ms=now_ms() + 60000, + reserve_retry=True, + retry_limit=32, + ) + for status in [TaskStatus.IN_PROGRESS, TaskStatus.FINALIZING]: + stored = await store.transition( + task.id, + expected={stored.task.status}, + expected_revision=stored.revision, + patch={"status": status}, + ) + return client, store, stored + + +@pytest.mark.parametrize( + "terminal", [TaskStatus.COMPLETED, TaskStatus.FAILED, TaskStatus.CANCELLED] +) +async def test_terminal_write_retries_shared_credit_cas(monkeypatch, terminal): + client, store, stored = await prepared() + original = client.txn + credit_key = store._retry_counter_key(stored.task.pool_id, "credits") + conflicts = 0 + + async def racing_txn(compare, success, failure=()): + nonlocal conflicts + if conflicts < 2 and any( + _decode(c["key"]).decode() == credit_key for c in compare + ): + # Another task reserves and then releases a retry credit between + # our counter read and transaction: value unchanged, revision new. + value = int((await client.get(credit_key)).value) + await original([], [client.put(credit_key, str(value + 1))]) + await original([], [client.put(credit_key, str(value))]) + conflicts += 1 + return await original(compare, success, failure) + + monkeypatch.setattr(client, "txn", racing_txn) + result = await store.transition( + stored.task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=stored.revision, + patch={"status": terminal}, + release_lease=True, + ) + assert conflicts == 2 + assert result.task.status == terminal + assert await store.list_leases(stored.task.pool_id) == [] + assert int((await client.get(credit_key)).value) == 0 + + +async def test_terminal_retry_never_overwrites_changed_task(monkeypatch): + client, store, stored = await prepared() + original = client.txn + changed = False + + async def racing_txn(compare, success, failure=()): + nonlocal changed + if not changed: + changed = True + task = stored.task.to_dict() + task["cancel_requested_at_ms"] = now_ms() + await original( + [], [client.put(store._task_key(stored.task.id), store._encode(task))] + ) + return await original(compare, success, failure) + + monkeypatch.setattr(client, "txn", racing_txn) + with pytest.raises(StoreConflict): + await store.transition( + stored.task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=stored.revision, + patch={"status": TaskStatus.COMPLETED}, + release_lease=True, + ) + current = await store.get_task(stored.task.id) + assert current.task.cancel_requested_at_ms is not None + assert current.task.status == TaskStatus.FINALIZING + + +@pytest.mark.parametrize("release", [False, True]) +async def test_terminal_ledger_uses_one_snapshot(monkeypatch, release): + client, store, stored = await prepared() + gets, batches = [], [] + original_get, original_many = client.get, client.get_many + + async def get(key): + gets.append(key) + return await original_get(key) + + async def many(keys, **kwargs): + batches.append(list(keys)) + return await original_many(keys, **kwargs) + + monkeypatch.setattr(client, "get", get) + monkeypatch.setattr(client, "get_many", many) + result = await store.transition( + stored.task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=stored.revision, + patch={"status": TaskStatus.COMPLETED}, + release_lease=release, + ) + assert result.task.status == TaskStatus.COMPLETED + assert gets == [store._task_key(stored.task.id)] + assert len(batches) == 1 + assert len(batches[0]) == (4 if release else 3) + assert bool(await store.list_leases(stored.task.pool_id)) is not release + + +async def test_terminal_snapshot_rejects_changed_hint(monkeypatch): + client, store, stored = await prepared() + original_many = client.get_many + + async def many(keys, **kwargs): + task = stored.task.to_dict() + task["cancel_requested_at_ms"] = now_ms() + await client.txn( + [], [client.put(store._task_key(task["id"]), store._encode(task))] + ) + return await original_many(keys, **kwargs) + + monkeypatch.setattr(client, "get_many", many) + with pytest.raises(StoreConflict): + await store.transition( + stored.task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=stored.revision, + patch={"status": TaskStatus.COMPLETED}, + release_lease=True, + ) + current = await store.get_task(stored.task.id) + assert current.task.status == TaskStatus.FINALIZING + assert current.task.cancel_requested_at_ms is not None + assert len(await store.list_leases(stored.task.pool_id)) == 1 + + +@pytest.mark.parametrize("count", ["0", "-1", None]) +async def test_terminal_snapshot_preserves_counter_invariants(count): + client, store, stored = await prepared() + key = store._retry_counter_key(stored.task.pool_id, "credits") + await client.txn( + [], [client.delete(key) if count is None else client.put(key, count)] + ) + with pytest.raises(RuntimeError): + await store.transition( + stored.task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=stored.revision, + patch={"status": TaskStatus.COMPLETED}, + release_lease=True, + ) + assert (await store.get_task(stored.task.id)).revision == stored.revision + assert len(await store.list_leases(stored.task.pool_id)) == 1 + + +async def test_terminal_snapshot_no_credit_and_foreign_lease(): + client, store, stored = await prepared() + lease_key = store._lease_key(stored.task.pool_id, stored.task.worker_key) + lease = json.loads((await client.get(lease_key)).value) + lease["task_id"] = "other-task" + await client.txn( + [], + [ + client.delete(store._retry_credit_key(stored.task)), + client.put(store._retry_counter_key(stored.task.pool_id, "credits"), "0"), + client.put(lease_key, store._encode(lease)), + ], + ) + result = await store.transition( + stored.task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=stored.revision, + patch={"status": TaskStatus.COMPLETED}, + release_lease=True, + ) + assert result.task.status == TaskStatus.COMPLETED + assert json.loads((await client.get(lease_key)).value)["task_id"] == "other-task" + + +async def test_terminal_snapshot_missing_task(monkeypatch): + client, store, stored = await prepared() + original_many = client.get_many + + async def many(keys, **kwargs): + await client.txn([], [client.delete(store._task_key(stored.task.id))]) + return await original_many(keys, **kwargs) + + monkeypatch.setattr(client, "get_many", many) + with pytest.raises(KeyError): + await store.transition( + stored.task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=stored.revision, + patch={"status": TaskStatus.COMPLETED}, + release_lease=True, + ) + assert len(await store.list_leases(stored.task.pool_id)) == 1 + + +async def test_concurrent_terminal_snapshots_conserve_credits(monkeypatch): + client = FakeEtcd() + store = EtcdTaskStore( + client, prefix="/isolated/batch-concurrency", deployment_id="cas" + ) + await store.register_gateway("generation", ttl_s=15) + tasks = [] + for index in range(8): + task = _task(f"video-batch-{index}") + stored, _ = await store.create_task( + task, principal_hash="p", idempotency_hash=None, queue_limit=16 + ) + stored = await store.reserve( + stored, + _lease(task, instance_id=index), + deadline_at_ms=now_ms() + 60000, + reserve_retry=True, + retry_limit=16, + ) + for status in [TaskStatus.IN_PROGRESS, TaskStatus.FINALIZING]: + stored = await store.transition( + task.id, + expected={stored.task.status}, + expected_revision=stored.revision, + patch={"status": status}, + ) + tasks.append(stored) + original_many = client.get_many + barrier = asyncio.Event() + first_reads = 0 + + async def many(keys, **kwargs): + nonlocal first_reads + snapshot = await original_many(keys, **kwargs) + if first_reads < len(tasks): + first_reads += 1 + if first_reads == len(tasks): + barrier.set() + await barrier.wait() # all eight deliberately see the same counter + return snapshot + + monkeypatch.setattr(client, "get_many", many) + results = await asyncio.wait_for( + asyncio.gather( + *[ + store.transition( + t.task.id, + expected={TaskStatus.FINALIZING}, + expected_revision=t.revision, + patch={"status": TaskStatus.COMPLETED}, + release_lease=True, + ) + for t in tasks + ] + ), + timeout=5, + ) + assert all(t.task.status == TaskStatus.COMPLETED for t in results) + assert await store.retry_budget_used(tasks[0].task.pool_id) == 0 + assert await store.list_leases(tasks[0].task.pool_id) == [] + for task in tasks: + assert await client.get(store._retry_credit_key(task.task)) is None diff --git a/tests/video_gateway/test_video_encoding.py b/tests/video_gateway/test_video_encoding.py new file mode 100644 index 000000000000..db1844964d01 --- /dev/null +++ b/tests/video_gateway/test_video_encoding.py @@ -0,0 +1,165 @@ +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from dingo.common.video_encoding import VideoEncoder, frame_conversion_workers + + +@pytest.mark.parametrize("value", ["0", "17", "-1", "1.5", "true", ""]) +def test_invalid_deployment_thread_count(monkeypatch, value): + monkeypatch.setenv("DINGO_VIDEO_FRAME_CONVERSION_WORKERS", value) + with pytest.raises(ValueError): + frame_conversion_workers() + + +def test_default_and_valid_deployment_thread_counts(monkeypatch): + monkeypatch.delenv("DINGO_VIDEO_FRAME_CONVERSION_WORKERS", raising=False) + assert frame_conversion_workers() == 8 + monkeypatch.setenv("DINGO_VIDEO_FRAME_CONVERSION_WORKERS", "2") + assert frame_conversion_workers() == 2 + + +@pytest.mark.parametrize("workers", [True, 0, 17, 2.5]) +def test_no_implicit_thread_count_coercion(workers): + with pytest.raises(ValueError): + VideoEncoder(workers) + + +@pytest.mark.parametrize("enabled", [False, True]) +def test_default_and_old_encoder_preserve_call(enabled): + def encoder(video, fps, audio): + assert video == "frames" and fps == 24 and audio == "sound" + return b"mp4" + + factory = Mock(side_effect=AssertionError("old encoder must not create converter")) + api = SimpleNamespace(_encode_video_bytes=encoder, _PlanarFrameConverter=factory) + shared = VideoEncoder(2 if enabled else 1) + assert shared.encode(api, "frames", fps=24, audio="sound") == b"mp4" + shared.close() + factory.assert_not_called() + + +def test_missing_converter_keeps_legacy_encoder(): + encoder = Mock(return_value=b"mp4") + api = SimpleNamespace(_encode_video_bytes=encoder) + shared = VideoEncoder(2) + assert shared.encode(api, "frames", fps=24) == b"mp4" + shared.close() + encoder.assert_called_once_with("frames", fps=24) + + +@pytest.mark.parametrize("failure", [False, True]) +def test_bounded_converter_is_closed_on_success_and_error(failure): + converter = SimpleNamespace(shutdown=Mock()) + factory = Mock(return_value=converter) + calls = [] + + def encoder(video, fps, audio, video_codec_options, frame_converter=None): + calls.append(video) + assert frame_converter is converter and fps == 24 and audio == "sound" + assert video_codec_options == {"preset": "ultrafast", "threads": "0"} + if failure: + raise RuntimeError("encoder failed") + return b"mp4" + + api = SimpleNamespace(_encode_video_bytes=encoder, _PlanarFrameConverter=factory) + shared = VideoEncoder(2) + + def run(): + return shared.encode( + api, + "frames", + fps=24, + audio="sound", + video_codec_options={"preset": "ultrafast", "threads": "0"}, + ) + + if failure: + with pytest.raises(RuntimeError, match="encoder failed"): + run() + else: + assert run() == b"mp4" + factory.assert_called_once_with(max_workers=2) + converter.shutdown.assert_not_called() + shared.close() + converter.shutdown.assert_called_once_with() + assert calls == ["frames"] + + +def test_native_converter_is_reused_until_shutdown(): + converter = SimpleNamespace(shutdown=Mock()) + factory = Mock(return_value=converter) + + def encode(video, *, frame_converter=None): + assert frame_converter is converter + return video + + api = SimpleNamespace(_encode_video_bytes=encode, _PlanarFrameConverter=factory) + encoder = VideoEncoder(8) + assert encoder.encode(api, b"first") == b"first" + assert encoder.encode(api, b"second") == b"second" + factory.assert_called_once_with(max_workers=8) + converter.shutdown.assert_not_called() + encoder.close() + encoder.close() + converter.shutdown.assert_called_once_with() + with pytest.raises(RuntimeError, match="closed"): + encoder.encode(api, b"third") + + +def test_native_encoder_failure_does_not_discard_pool(): + converter = SimpleNamespace(shutdown=Mock()) + factory = Mock(return_value=converter) + + def encode(video, *, frame_converter=None): + if video == b"bad": + raise ValueError("bad frame") + return video + + api = SimpleNamespace(_encode_video_bytes=encode, _PlanarFrameConverter=factory) + encoder = VideoEncoder(8) + with pytest.raises(ValueError, match="bad frame"): + encoder.encode(api, b"bad") + assert encoder.encode(api, b"good") == b"good" + factory.assert_called_once() + converter.shutdown.assert_not_called() + encoder.close() + converter.shutdown.assert_called_once() + + +def test_native_shutdown_waits_for_inflight_encode(): + import threading + from concurrent.futures import ThreadPoolExecutor + + started = threading.Event() + release = threading.Event() + closed = threading.Event() + converter = SimpleNamespace(shutdown=Mock()) + + def encode(video, *, frame_converter=None): + started.set() + assert release.wait(5) + return video + + api = SimpleNamespace( + _encode_video_bytes=encode, _PlanarFrameConverter=Mock(return_value=converter) + ) + encoder = VideoEncoder(8) + + def close(): + encoder.close() + closed.set() + + with ThreadPoolExecutor(max_workers=2) as pool: + result = pool.submit(encoder.encode, api, b"video") + try: + assert started.wait(2) + closing = pool.submit(close) + assert not closed.wait(0.05) + converter.shutdown.assert_not_called() + finally: + release.set() + assert result.result(timeout=2) == b"video" + closing.result(timeout=2) + converter.shutdown.assert_called_once() diff --git a/tests/video_gateway/test_worker_prefetch.py b/tests/video_gateway/test_worker_prefetch.py new file mode 100644 index 000000000000..5b1cf5ecf0fc --- /dev/null +++ b/tests/video_gateway/test_worker_prefetch.py @@ -0,0 +1,174 @@ +import asyncio +import json +import time + +import pytest + +from dingo.common.video_task_protocol import DetachedTaskIdentity +from dingo.vllm.omni.detached_tasks import DetachedOmniTaskManager + + +def identity(manager, name): + value = DetachedTaskIdentity("deployment", "pool", name, 1, "a" * 32) + root = manager._attempt_root(value).parent.parent + root.mkdir(parents=True, exist_ok=True) + (root / "_artifact.json").write_text( + json.dumps( + dict( + schema_version=1, + deployment_id="deployment", + pool_id="pool", + task_id=name, + ) + ) + ) + return value + + +class Handler: + def __init__(self): + self.running = 0 + self.peak = 0 + self.started = [] + self.release = asyncio.Event() + + async def generate(self, payload, context): + self.running += 1 + self.peak = max(self.peak, self.running) + self.started.append(payload["name"]) + try: + await self.release.wait() + yield {"phase": "done"} + finally: + self.running -= 1 + + +async def until(predicate): + async def wait(): + while not predicate(): + await asyncio.sleep(0.001) + + await asyncio.wait_for(wait(), 2) + + +@pytest.mark.parametrize("capacity", [1, 2]) +async def test_prefetch_admits_one_extra_but_never_executes_above_n(tmp_path, capacity): + handler = Handler() + manager = DetachedOmniTaskManager( + handler, + tmp_path, + execution_capacity=capacity, + prefetch_capacity=1, + cancel_poll_interval_s=0.01, + ) + values = [identity(manager, str(i)) for i in range(capacity + 2)] + try: + for i in range(capacity): + assert (await manager._submit(values[i], {"name": str(i)}))["accepted"] + await until(lambda: len(handler.started) == capacity) + assert (await manager._submit(values[capacity], {"name": str(capacity)}))[ + "accepted" + ] + duplicate = await manager._submit(values[capacity], {"name": str(capacity)}) + assert not duplicate["accepted"] and duplicate["state"] == "accepted" + assert (await manager._submit(values[-1], {"name": "rejected"}))[ + "state" + ] == "busy" + await asyncio.sleep(0.02) + assert handler.running == capacity and len(handler.started) == capacity + handler.release.set() + await until(lambda: not manager._running) + assert handler.peak == capacity + assert handler.started == [str(i) for i in range(capacity + 1)] + assert (await manager._status(values[capacity]))["worker_queue_wait_s"] > 0 + finally: + handler.release.set() + await manager.shutdown() + + +@pytest.mark.parametrize("cancel_kind", ["rpc", "file"]) +async def test_queued_cancel_never_enters_model_and_frees_admission( + tmp_path, cancel_kind +): + handler = Handler() + manager = DetachedOmniTaskManager( + handler, tmp_path, prefetch_capacity=1, cancel_poll_interval_s=0.01 + ) + a, b, c = [identity(manager, name) for name in ["a", "b", "c"]] + try: + await manager._submit(a, {"name": "a"}) + await until(lambda: handler.running == 1) + await manager._submit(b, {"name": "b"}) + if cancel_kind == "rpc": + await manager._cancel(b) + else: + manager._cancel_path(b).touch() + await until(lambda: b.key not in manager._running) + assert (await manager._status(b))["state"] == "cancelled" + assert handler.started == ["a"] + assert (await manager._submit(c, {"name": "c"}))["accepted"] + handler.release.set() + await until(lambda: not manager._running) + assert handler.started == ["a", "c"] + finally: + handler.release.set() + await manager.shutdown() + + +async def test_prefetch_deadline_expires_without_model_execution(tmp_path): + handler = Handler() + manager = DetachedOmniTaskManager(handler, tmp_path, prefetch_capacity=1) + a, b = [identity(manager, name) for name in ["a", "b"]] + try: + await manager._submit(a, {"name": "a"}) + await until(lambda: handler.running == 1) + await manager._submit( + b, {"name": "b"}, deadline_at_ms=int(time.time() * 1000) + 20 + ) + await until(lambda: b.key not in manager._running) + status = await manager._status(b) + assert ( + status["state"] == "failed" + and status["error"]["code"] == "worker_queue_timeout" + ) + assert handler.started == ["a"] + finally: + handler.release.set() + await manager.shutdown() + + +async def test_direct_admission_cannot_bypass_prefetched_execution_budget(tmp_path): + handler = Handler() + manager = DetachedOmniTaskManager(handler, tmp_path, prefetch_capacity=1) + a, b = [identity(manager, name) for name in ["a", "b"]] + try: + await manager._submit(a, {"name": "a"}) + await until(lambda: handler.running == 1) + await manager._submit(b, {"name": "b"}) + with pytest.raises(RuntimeError, match="capacity"): + await anext(manager.generate({"name": "direct"}, None)) + assert handler.started == ["a"] + finally: + handler.release.set() + await manager.shutdown() + + +@pytest.mark.parametrize("capacity", [-1, 2, True, "1"]) +def test_prefetch_capacity_rejects_invalid_values(tmp_path, capacity): + with pytest.raises(ValueError, match="prefetch"): + DetachedOmniTaskManager(object(), tmp_path, prefetch_capacity=capacity) + + +async def test_shutdown_cancels_waiting_task_without_entering_model(tmp_path): + handler = Handler() + manager = DetachedOmniTaskManager( + handler, tmp_path, prefetch_capacity=1, drain_timeout_s=0.02 + ) + a, b = [identity(manager, name) for name in ["a", "b"]] + await manager._submit(a, {"name": "a"}) + await until(lambda: handler.running == 1) + await manager._submit(b, {"name": "b"}) + await asyncio.wait_for(manager.shutdown(), 2) + assert handler.started == ["a"] + assert (await manager._status(b))["state"] == "cancelled" + assert manager._execution_slots._value == 1 From 3e22a8a775868dd4584bd8f6ee95d8f1be4b9009 Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Thu, 17 Sep 2026 10:44:57 +0800 Subject: [PATCH 06/12] [test][video-gateway] allow explicit remote contract wait budgets --- .../video_gateway/test_real_etcd_contract.py | 21 ++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/tests/video_gateway/test_real_etcd_contract.py b/tests/video_gateway/test_real_etcd_contract.py index 54a5d26406e7..85e5f70e58db 100644 --- a/tests/video_gateway/test_real_etcd_contract.py +++ b/tests/video_gateway/test_real_etcd_contract.py @@ -30,6 +30,14 @@ _ETCD_URL = os.environ.get("DINGO_VIDEO_TEST_ETCD_URL") +def _eventual_timeout(default: float) -> float: + """Remote contract transports need an explicit non-SLO wait budget.""" + value = float(os.environ.get("DINGO_VIDEO_TEST_EVENTUAL_TIMEOUT_S", default)) + if not 0 < value <= 120: + raise ValueError("DINGO_VIDEO_TEST_EVENTUAL_TIMEOUT_S must be in (0, 120]") + return value + + @pytest.mark.skipif(not _ETCD_URL, reason="requires a real etcd v3 endpoint") async def test_result_handoff_and_owner_takeover_never_touch_reused_slot(): from dingo.video_gateway.models import now_ms @@ -437,10 +445,9 @@ async def test_two_gateways_preserve_task_owned_by_healthy_peer( await dispatcher_a.start() started_a = True submission = await _submit(service_a, "public-fl") - for _ in range(100): - if client_a.calls: - break - await asyncio.sleep(0.01) + async with asyncio.timeout(_eventual_timeout(1)): + while not client_a.calls: + await asyncio.sleep(0.01) assert len(client_a.calls) == 1 await dispatcher_b.start() @@ -452,7 +459,7 @@ async def test_two_gateways_preserve_task_owned_by_healthy_peer( assert client_b.calls == [] client_a.release.set() - terminal = await dispatcher_b.wait_terminal(active.task.id, 2) + terminal = await dispatcher_b.wait_terminal(active.task.id, _eventual_timeout(2)) assert terminal.task.status == TaskStatus.COMPLETED assert len(client_a.calls) + len(client_b.calls) == 1 finally: @@ -523,7 +530,7 @@ async def test_detached_task_survives_owner_gateway_shutdown_and_is_claimed( # Submitting through A does not force A to win a shared queue. First # establish the intended owner, then start its standby before failure. submission = await _submit(service_a, "public-fl") - await asyncio.wait_for(handler.started.wait(), timeout=2) + await asyncio.wait_for(handler.started.wait(), timeout=_eventual_timeout(2)) active = await store_b.get_task(submission.stored.task.id) assert active is not None assert active.task.status == TaskStatus.IN_PROGRESS @@ -534,7 +541,7 @@ async def test_detached_task_survives_owner_gateway_shutdown_and_is_claimed( await dispatcher_a.stop() started_a = False handler.release.set() - terminal = await dispatcher_b.wait_terminal(active.task.id, 12) + terminal = await dispatcher_b.wait_terminal(active.task.id, _eventual_timeout(12)) assert terminal.task.status == TaskStatus.COMPLETED assert terminal.task.owner_generation == "detached-gateway-b" assert handler.calls == 1 From 6373a67c7ca34377bd7142bbdd25e1137c7279eb Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Mon, 21 Sep 2026 00:36:23 +0800 Subject: [PATCH 07/12] [fix][container] preserve upstream vLLM media tools --- container/deps/vllm/validate_media_probe.py | 109 ++++++++++++++------ container/templates/vllm_runtime.Dockerfile | 72 ++----------- 2 files changed, 85 insertions(+), 96 deletions(-) diff --git a/container/deps/vllm/validate_media_probe.py b/container/deps/vllm/validate_media_probe.py index abf1272884e6..a7a03eeb12cc 100644 --- a/container/deps/vllm/validate_media_probe.py +++ b/container/deps/vllm/validate_media_probe.py @@ -1,4 +1,4 @@ -"""CPU-only FFmpeg/ffprobe smoke after replacing the upstream media packages.""" +"""CPU-only smoke for the upstream FFmpeg path required by Ref2VA.""" import json import shutil @@ -6,40 +6,85 @@ import tempfile from pathlib import Path -import av - def main(): for executable in ("ffmpeg", "ffprobe"): path = shutil.which(executable) - if path != "/usr/local/bin/" + executable: - raise RuntimeError(f"Expected in-tree {executable}, found {path!r}") + if not path: + raise RuntimeError(f"Missing upstream {executable}") + if Path(path).resolve().is_relative_to(Path("/usr/local")): + raise RuntimeError( + f"Expected upstream {executable}, found in-tree replacement {path!r}" + ) subprocess.run([path, "-version"], check=True, capture_output=True, timeout=15) + encoders = subprocess.run( + ["ffmpeg", "-hide_banner", "-encoders"], + check=True, + capture_output=True, + text=True, + timeout=15, + ).stdout + if "libx264rgb" not in encoders: + raise RuntimeError("Upstream FFmpeg is missing the libx264rgb encoder") + with tempfile.TemporaryDirectory(prefix="dingo-media-probe-") as directory: - path = Path(directory) / "sample.mp4" - with av.open(str(path), "w", format="mp4") as output: - video = output.add_stream("libx264", rate=8) - video.width, video.height, video.pix_fmt = 32, 32, "yuv420p" - audio = output.add_stream("aac", rate=48000) - audio.layout = "stereo" - for index in range(4): - frame = av.VideoFrame(32, 32, "yuv420p") - frame.pts = index - for plane in frame.planes: - plane.update(bytes(plane.buffer_size)) - for packet in video.encode(frame): - output.mux(packet) - for packet in video.encode(None): - output.mux(packet) - frame = av.AudioFrame(format="fltp", layout="stereo", samples=24000) - frame.sample_rate, frame.pts = 48000, 0 - for plane in frame.planes: - plane.update(bytes(plane.buffer_size)) - for packet in audio.encode(frame): - output.mux(packet) - for packet in audio.encode(None): - output.mux(packet) + directory_path = Path(directory) + source = directory_path / "source.rgb" + encoded = directory_path / "prepared.mp4" + decoded = directory_path / "decoded.rgb" + frame = bytes((index % 251 for index in range(32 * 32 * 3))) + source.write_bytes(frame * 4) + subprocess.run( + [ + "ffmpeg", + "-y", + "-loglevel", + "error", + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + "-s", + "32x32", + "-r", + "8", + "-i", + str(source), + "-frames:v", + "4", + "-c:v", + "libx264rgb", + "-crf", + "0", + "-preset", + "veryfast", + "-pix_fmt", + "rgb24", + str(encoded), + ], + check=True, + timeout=30, + ) + subprocess.run( + [ + "ffmpeg", + "-y", + "-loglevel", + "error", + "-i", + str(encoded), + "-f", + "rawvideo", + "-pix_fmt", + "rgb24", + str(decoded), + ], + check=True, + timeout=30, + ) + if decoded.read_bytes() != source.read_bytes(): + raise RuntimeError("libx264rgb lossless round trip changed RGB pixels") result = subprocess.run( [ "ffprobe", @@ -50,7 +95,7 @@ def main(): "-show_format", "-of", "json", - str(path), + str(encoded), ], capture_output=True, text=True, @@ -60,14 +105,12 @@ def main(): document = json.loads(result.stdout) streams = {stream["codec_type"]: stream for stream in document["streams"]} assert streams["video"]["codec_name"] == "h264" + assert streams["video"]["pix_fmt"] == "gbrp" assert (streams["video"]["width"], streams["video"]["height"]) == (32, 32) assert int(streams["video"]["nb_read_frames"]) == 4 - assert streams["audio"]["codec_name"] == "aac" - assert int(streams["audio"]["sample_rate"]) == 48000 - assert int(streams["audio"]["channels"]) == 2 assert float(document["format"]["duration"]) > 0 print( - "DINGO_MEDIA_PROBE=PASS (H.264 video + stereo AAC, ffprobe JSON and frame count)" + "DINGO_MEDIA_PROBE=PASS (upstream libx264rgb + rawvideo + ffprobe)" ) diff --git a/container/templates/vllm_runtime.Dockerfile b/container/templates/vllm_runtime.Dockerfile index 8b2a9de6d984..ddf5dc1c7db9 100644 --- a/container/templates/vllm_runtime.Dockerfile +++ b/container/templates/vllm_runtime.Dockerfile @@ -231,76 +231,22 @@ RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ {% endif %} {% if device == "cuda" %} -# The upstream vllm/vllm-openai base image ships a GPL/GPL-3.0 ffmpeg built -# against libx264/libx265/libmp3lame. Purge ONLY the explicitly-named ffmpeg + -# codec packages and replace them with the LGPL-only in-tree ffmpeg built in -# wheel_builder (--disable-gpl --disable-nonfree; H.264 via NVENC, VP9 via -# libvpx). PyAV, torchaudio, torchvision, soundfile and Pillow all bundle their -# own libraries and do not link the system ffmpeg/codecs, so removing them is -# safe. dpkg-query keeps the match robust across base-image/arch version -# suffixes (e.g. libavcodec58 vs 60). -# -# This grep is the COMPLETE, auditable set of what leaves the image: there is -# deliberately NO apt-get autoremove, so the removal can never cascade into -# unrelated auto-installed packages. That matters because the base image marks -# both the gcc/g++/make toolchain (torch.inductor/Triton JIT shell out to it at -# runtime) and the CUDA math libs (libcublas/libcusolver/libcusparse — the torch -# wheels here ship no bundled cublas and load the system copies) as -# auto-installed. A bare `autoremove --purge` sweeps all of those as "orphaned", -# which broke runtime JIT (missing C compiler) in the 1.3.0 rc image. Any -# LGPL/BSD media libs left orphaned (libva, libvdpau, ...) are license-clean -# dead weight, not a compliance issue. -RUN set -eux; \ - purge=$(dpkg-query -W -f='${Package}\n' 2>/dev/null \ - | grep -E '^(ffmpeg|libav[a-z]|libsw[a-z]|libpostproc|libx264|libx265|libmp3lame|libaom|libdav1d|libvpx|libtheora|libvorbis|libopus|libsoxr|libcaca|libcdio|libzvbi|libgme|libvidstab|libdc1394|libraw1394|libiec61883|libtwolame|libshine|libsrt[0-9]|libudfread|libsvtav1|libbs2b|librubberband|libchromaprint|libcodec2|libgsm|libass[0-9]|libbluray|libxvidcore|libflite)' \ - || true); \ - if [ -n "$purge" ]; then \ - DEBIAN_FRONTEND=noninteractive apt-get purge -y $purge; \ - fi; \ - rm -rf /var/lib/apt/lists/* - -# Regression guard for the codec purge above: torch.inductor/Triton JIT shell -# out to a host C/C++ compiler at runtime, so a missing toolchain only surfaces -# on the first compile in production. Reproduce that compile path at build time -# (CPU-only) so a missing compiler aborts the build instead of shipping. +# Preserve the upstream vllm/vllm-openai FFmpeg stack. vLLM-Omni's Ref2VA +# preprocessing requires its software libx264rgb encoder and rawvideo support; +# replacing it with the reduced in-tree FFmpeg silently breaks video-reference +# requests. The runtime probe below exercises the same lossless RGB path. + +# TorchInductor/Triton JIT shells out to a host C/C++ compiler at runtime. +# Reproduce that compile path at build time so a missing compiler aborts the +# build instead of surfacing on the first production request. RUN --mount=type=bind,source=./container/deps/vllm/validate_torch_compile_smoke.py,target=/tmp/validate_torch_compile_smoke.py,readonly \ python3 /tmp/validate_torch_compile_smoke.py -# Copy the LGPL ffmpeg and ffprobe from wheel_builder: versioned shared libs (libav*.so*, -# libsw*.so*) + libvpx + the LGPL CLI binary that imageio/diffusers target via -# IMAGEIO_FFMPEG_EXE. Ungated by enable_media_ffmpeg because the base GPL ffmpeg -# was just purged, so the LGPL CLI must always be present for the omni -# video-export path to have something to encode with. -RUN --mount=type=bind,from=wheel_builder,source=/usr/local/,target=/tmp/usr/local/ \ - mkdir -p /usr/local/lib/pkgconfig && \ - cp -rnL /tmp/usr/local/include/libav* /tmp/usr/local/include/libsw* /usr/local/include/ && \ - cp -nL /tmp/usr/local/lib/libav*.so* /tmp/usr/local/lib/libsw*.so* /usr/local/lib/ && \ - cp -nL /tmp/usr/local/lib/lib*vpx*.so* /usr/local/lib/ 2>/dev/null || true && \ - cp -nL /tmp/usr/local/lib/pkgconfig/libav*.pc /tmp/usr/local/lib/pkgconfig/libsw*.pc /usr/local/lib/pkgconfig/ && \ - cp -nL /tmp/usr/local/bin/ffmpeg /usr/local/bin/ffmpeg && \ - cp -nL /tmp/usr/local/bin/ffprobe /usr/local/bin/ffprobe && \ - cp -r /tmp/usr/local/src/ffmpeg /usr/local/src/ && \ - ldconfig -ENV IMAGEIO_FFMPEG_EXE=/usr/local/bin/ffmpeg - -# H3 reference-media preprocessing invokes ffprobe directly. The upstream -# package purge removes /usr/bin/ffprobe too; copying only ffmpeg leaves a -# runtime failure even though the Python imports and output encoding work. +# Guard the upstream media tools and the exact Ref2VA codec path. RUN --mount=type=bind,source=./container/deps/vllm/validate_media_probe.py,target=/tmp/validate_media_probe.py,readonly \ python3 /tmp/validate_media_probe.py {% endif %} -# Replace the upstream vllm/vllm-openai image's imageio-ffmpeg (which ships a -# GPL-encumbered prebuilt ffmpeg binary in /imageio_ffmpeg/binaries/) -# with a source install that leaves no binary on disk. On cuda, IMAGEIO_FFMPEG_EXE -# (set above) points imageio at the LGPL CLI copied from wheel_builder. The -# --no-binary directive lives in the requirements file itself. -RUN --mount=type=bind,source=./container/deps/requirements.vllm.txt,target=/tmp/requirements.vllm.txt \ - --mount=type=cache,target=/root/.cache/uv,sharing=locked \ - export UV_CACHE_DIR=/root/.cache/uv && \ - uv pip install {{ pip_target }} --reinstall-package imageio-ffmpeg --no-deps \ - --requirement /tmp/requirements.vllm.txt - # Remove the vLLM source tree shipped in the base image to avoid pytest # collection conflicts (duplicate conftest plugin registration) and stale # tool scripts referencing files not present in Dynamo's build context. From de8e0b3304bb50687073dc85b2e1a0f206b05ebe Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Mon, 21 Sep 2026 00:36:50 +0800 Subject: [PATCH 08/12] [fix][video-gateway] harden recovery and task observability --- dingo/video_gateway/CONTINUOUS_EXECUTION.md | 27 ++ .../GATEWAY_CONFIGURATION_CONSISTENCY.md | 161 ++++++++ .../RECOVERY_AND_UPLOAD_CLEANUP.md | 59 +++ dingo/video_gateway/api.py | 73 +++- dingo/video_gateway/artifact_store.py | 50 ++- dingo/video_gateway/dispatcher.py | 343 ++++++++++++------ dingo/video_gateway/etcd_http.py | 19 +- dingo/video_gateway/form_parser.py | 5 +- dingo/video_gateway/models.py | 5 +- dingo/video_gateway/task_store.py | 37 +- dingo/video_gateway/telemetry.py | 22 ++ dingo/vllm/omni/detached_tasks.py | 99 ++++- tests/video_gateway/test_artifact_store.py | 95 ++++- tests/video_gateway/test_dispatcher.py | 32 ++ .../test_early_release_dispatcher.py | 106 +++++- tests/video_gateway/test_etcd_http.py | 68 +++- tests/video_gateway/test_handoff_recovery.py | 164 +++++++++ tests/video_gateway/test_result_handoff.py | 18 +- tests/video_gateway/test_task_store.py | 20 + tests/video_gateway/test_telemetry.py | 7 + tests/video_gateway/test_upload_lifecycle.py | 194 ++++++++++ tests/video_gateway/test_worker_prefetch.py | 84 +++++ 22 files changed, 1535 insertions(+), 153 deletions(-) create mode 100644 dingo/video_gateway/GATEWAY_CONFIGURATION_CONSISTENCY.md create mode 100644 dingo/video_gateway/RECOVERY_AND_UPLOAD_CLEANUP.md create mode 100644 tests/video_gateway/test_handoff_recovery.py create mode 100644 tests/video_gateway/test_upload_lifecycle.py diff --git a/dingo/video_gateway/CONTINUOUS_EXECUTION.md b/dingo/video_gateway/CONTINUOUS_EXECUTION.md index d58deb39472b..736d26b120ec 100644 --- a/dingo/video_gateway/CONTINUOUS_EXECUTION.md +++ b/dingo/video_gateway/CONTINUOUS_EXECUTION.md @@ -62,6 +62,33 @@ already admitted can still finish. Memory admission remains independently bounded. Size queue/retry/memory budgets for N+P admissions and pending results. These controls cannot make permanently blocked filesystem calls cancellable. +## Cancellation boundary + +Video task cancellation is a logical result and scheduling operation. After a +DELETE request, Gateway signals the detached Worker and waits up to +`abort_grace_s` for its durable terminal status. A confirmed `cancelled` status +releases the task's lease immediately. If the Worker does not confirm, Gateway +quarantines the old slot through the task deadline plus the abort grace. A +restarted Worker registers with a new instance ID and is not blocked by the old +instance's quarantine. + +Engine compute interruption is backend-dependent. In particular, a vLLM-Omni +orchestrator can acknowledge an abort and unwind the outer request while an +already-running diffusion forward continues to its next interruptible boundary. +DingoRouter still prevents the cancelled attempt from publishing a result. With +engine concurrency one, a following task remains serialized inside vLLM-Omni +until that forward returns, even though its Gateway lifecycle has entered +`in_progress`. + +This behavior is an accepted integration boundary, not a task-state correctness +failure. `execution_started` means that the detached Worker admitted the task; +it is not proof that a GPU kernel started. `worker_queue_wait_s` measures the +detached Worker's admission queue and does not include an opaque queue inside +the engine. Immediate reclamation of in-flight diffusion compute requires the +engine to implement interruption within its execution loop. Do not use a +successful DELETE response or a `cancelled` task status as proof that GPU use +has already fallen to zero. + ## Disable or roll back Drain accepted work and durable `finalizing` handoffs before downgrading to code diff --git a/dingo/video_gateway/GATEWAY_CONFIGURATION_CONSISTENCY.md b/dingo/video_gateway/GATEWAY_CONFIGURATION_CONSISTENCY.md new file mode 100644 index 000000000000..a69d484d71d6 --- /dev/null +++ b/dingo/video_gateway/GATEWAY_CONFIGURATION_CONSISTENCY.md @@ -0,0 +1,161 @@ +# Video Gateway 多副本配置一致性约束 + +本文说明同一负载均衡入口后、共同服务同一组任务和 Worker pool 的多个 +Video Gateway Pod，哪些配置可以长期不同，哪些只允许在滚动升级期间短暂不同， +以及哪些配置必须始终一致或保持协议兼容。 + +本文中的“相同”不是指 YAML 文本逐字相同。例如，每个 Gateway 可以配置不同的 +etcd endpoint 顺序；只要它们最终连接的是同一个 etcd 集群和同一个逻辑 key 空间， +就满足一致性要求。 + +## 基本原则 + +通常应由同一个 Deployment、ConfigMap 和镜像版本生成所有 Gateway Pod，以减少人为 +配置漂移。但是，多副本安全性不依赖所有本机性能参数完全相同。 + +判断一个参数属于哪一类时，使用以下规则： + +1. 只限制当前 Gateway 自身资源消耗的参数，可以长期不同。 +2. 改变请求准入、等待时间、重试或清理策略，但不改变共享数据含义的参数，可以在 + 滚动升级期间短暂不同。 +3. 改变 etcd key 空间、任务状态含义、Worker slot 身份、fencing、artifact 定位或 + Worker wire protocol 的参数，必须始终一致或向前、向后兼容。 + +CAS、lease、attempt、owner generation 和 execution token fencing 是防止重复终态、 +错误释放 slot 和旧结果覆盖新结果的安全边界。调度策略短暂不同不应绕过这些边界。 + +## A. 可以在不同 Gateway Pod 之间长期不一致 + +这些参数仅控制单个 Gateway 的资源占用、检查频率或非共享连接行为。不同 Pod 可以 +根据自身 CPU、内存和运维需求独立设置。 + +| 配置 | 允许差异 | 影响 | +|---|---|---| +| `scheduling.finalization_concurrency` | 可以长期不同 | 每个 Gateway 同时执行多少个本地后处理任务 | +| `scheduling.finalization_pending_limit` | 可以长期不同 | 每个 Gateway 接受多少个本地待后处理任务 | +| `scheduling.finalization_retry_delay_s` | 可以长期不同 | 当前 Gateway 后处理暂态失败后的本地退避速度 | +| `scheduling.discovery_interval_s` | 可以长期不同 | 当前 Gateway 刷新 Worker 视图的频率 | +| `scheduling.dispatch_interval_s` | 可以长期不同 | 当前 Gateway 尝试派发任务的频率 | +| `runtime.discovery_watchdog.interval_s` | 可以长期不同 | 当前 Pod 的 watchdog 检查频率 | +| `runtime.discovery_watchdog.mismatch_grace_s` | 可以长期不同 | 当前 Pod 对 discovery 短暂不一致的容忍时间 | +| `task_store.request_timeout_s` | 可以长期不同 | 当前 Gateway 单次 etcd 请求的客户端超时 | +| `task_store.watch_response_timeout_s` | 可以长期不同 | 当前 Gateway watch 长连接的客户端超时 | +| etcd endpoint 顺序或首选 endpoint | 可以长期不同 | 必须仍指向同一个 etcd 集群 | +| 日志级别、日志格式、指标抓取配置 | 可以长期不同 | 只影响当前 Pod 的可观测性 | +| Pod CPU、内存、线程池等资源配置 | 可以长期不同 | 只影响当前 Pod 的吞吐与延迟 | + +`http.host` 和 `http.port` 也可以因 Pod 网络布局而不同，但 Kubernetes Service 的 +`targetPort` 和健康检查必须能够正确访问每个 Pod。 + +## B. 允许在滚动升级期间短暂不一致 + +这些参数不改变共享记录和 fencing 的含义，但会改变外部行为或任务策略。正常情况下 +仍建议所有副本最终收敛到相同值。滚动升级期间允许新旧版本短暂共存。 + +| 配置 | 短暂不一致时可能出现的现象 | +|---|---| +| `DINGO_VIDEO_WORKER_RETRY_ONCE` | Worker 故障后，有的任务重试，有的任务直接失败 | +| `DINGO_VIDEO_RETRY_BUDGET` | 不同 Gateway 对共享重试额度是否已满采用不同阈值 | +| `DINGO_VIDEO_RETRY_WAIT_TIMEOUT_S` | 重试任务的最长等待时间不同 | +| `DINGO_VIDEO_RETRY_FAILED_INSTANCE_BACKOFF_S` | 不同 Gateway 对刚失败 Worker 的避让时长不同 | +| `scheduling.queue_limit` | 相同负载下，请求打到一个 Gateway 被接收，打到另一个返回 429 | +| `scheduling.accept_without_workers` | 无已注册 Worker 时，不同 Gateway 的准入决定不同 | +| `scheduling.execution_timeout_s` | 新旧 Gateway 对执行超时的判断时间不同 | +| `scheduling.abort_grace_s` | 取消或超时后等待 Worker 确认的时长不同 | +| `scheduling.finalization_timeout_s` | 后处理被认定为超时的时间不同 | +| `scheduling.finalization_max_retries` | 后处理失败后的尝试次数不同 | +| `scheduling.worker_capacity` | 新旧 Gateway 使用 Worker 并发容量的上限不同 | +| `scheduling.worker_prefetch_capacity` | 新旧 Gateway 是否使用 Worker 预取额度不同 | +| `scheduling.early_release_slot` | 新旧 Gateway 释放执行 slot 的时点不同 | +| `http.max_body_bytes` 及 `media.*` 限制 | 同一个输入可能被一个 Gateway 接收、被另一个拒绝 | +| `http.sync_timeout_s` | 同步请求等待时长不同；异步任务状态不受影响 | +| `http.async_submit_status_code` | 提交成功时可能短暂混用 HTTP 200 和 202 | +| `http.default_model`、`pools[*].served_models` | 新增或移除模型期间，部分 Pod 可能暂时不识别该模型 | +| `lifecycle.*` 保留及扫描参数 | 任务和垃圾制品的回收时间可能短暂不同 | +| `artifact_store.hard_min_free_bytes`、`soft_min_free_bytes` | 不同 Gateway 可能做出不同磁盘准入决定 | + +这一类差异的接受边界如下： + +- 允许行为和性能短暂不一致，但不能绕过共享 lease、CAS 和 fencing。 +- 旧 Gateway 创建的任务被新 Gateway 接管时，以接管者当前策略为准。当前系统不承诺 + “任务创建时的策略永久固定”。如果未来需要该语义，应把策略快照写入任务记录。 +- 滚动升级完成后必须检查所有 Ready Pod 已使用目标配置，不应长期保留无意的漂移。 +- 调整 `worker_capacity` 或 prefetch 时，Gateway 仍必须以 Worker 实际声明容量为上限； + 不允许通过配置绕过容量握手。 +- 改动 media 上限时，不得超过代码中的协议 hard cap。 + +这些差异本身不会使同一任务产生两个有效成功终态，不会破坏 etcd，也不会允许旧 +attempt 覆盖新 attempt。Worker 失联且实际仍在计算时，重试可能造成短暂的重复物理 +计算，但只有通过当前 fencing 校验的执行结果可以成为有效结果。 + +## C. 必须始终一致或保持协议兼容 + +以下配置定义了多个 Gateway 共同操作的数据和协议。服务同一个逻辑部署的 Pod 不应 +在这些项目上使用互不兼容的值。 + +| 配置或协议 | 要求 | 不一致的后果 | +|---|---|---| +| `schema_version` | 必须为当前代码支持的同一配置 schema | Pod 可能无法启动或错误解释配置 | +| `deployment_id` | 同一逻辑部署必须相同 | 形成相互不可见的任务和 artifact 空间 | +| `task_store.kind` | 必须使用兼容实现 | Memory store 不能替代生产 etcd 的共享语义 | +| `task_store.prefix` | 必须相同 | Gateway 会看到不同的任务、队列、lease 和 retry ledger | +| etcd 集群身份 | 必须指向同一集群 | 即使 prefix 相同也会形成 split-brain | +| etcd task、index、lease、retry ledger 的 key/value schema | 新旧代码必须双向兼容 | 可能无法接管任务或正确维护计数器 | +| 任务状态机和 CAS 前置条件 | 新旧代码必须兼容 | 可能错误拒绝合法转换；不得放宽安全条件 | +| attempt、owner generation、execution token 语义 | 必须兼容 | 这是阻止旧执行写回和错释放 slot 的 fencing 边界 | +| Worker slot key 和 slot 编号规则 | 必须兼容 | 可能对同一执行资源形成不同身份认知 | +| `artifact_store.kind` 及逻辑存储 | 必须指向同一份共享制品存储 | 接管任务的 Gateway 可能找不到输入或结果 | +| `artifact_store.root` 对应的逻辑路径布局 | 所有 Pod 和 Worker 必须能解析到同一制品 | 可能出现结果存在但查询、下载或清理失败 | +| `runtime.request_plane`、`event_plane` | 必须与 Worker 和新旧 Gateway 协议兼容 | 请求无法发送、结果事件无法接收 | +| `runtime.discovery_backend` | 同一 pool 必须发现同一组 Worker 身份 | Gateway 的 Worker 视图可能分裂 | +| `pool_id` 的含义 | 同一个 ID 必须始终代表同一个逻辑池 | 队列、slot 和重试账本可能被不同业务混用 | +| `backend_model`、`backend_target` | 同一 pool 内必须保持语义一致 | 任务可能发送到错误模型或错误 endpoint | +| `execution_mode` | 新旧 Gateway 和 Worker 必须兼容 | 生命周期、等待和取消协议可能不匹配 | +| adapter 名称、workflow、协议版本和 wire-format options | 新旧两端必须兼容 | 输入或 Worker 返回值可能被错误解释 | + +如果需要对这一类配置做不兼容变更，应采用新的 `deployment_id`、`pool_id` 或独立 +task-store prefix 做蓝绿部署，而不是让不兼容版本同时操作同一个共享空间。确认旧任务 +已排空后，再切换流量并清理旧空间。 + +## `configuration_revision` 的作用与限制 + +当前每个任务记录 `configuration_revision`，用于阻止配置语义不同的 Gateway 盲目恢复 +排队任务。当前 revision 覆盖： + +- `pool_id`、served models； +- backend model 和 target； +- execution mode； +- adapter workflow、协议版本及 options； +- `worker_capacity`； +- `execution_timeout_s`。 + +它是一道恢复保护，不是完整的集群配置协调器。当前重试环境变量、prefetch、 +`early_release_slot` 和多数本机参数不在 revision 中。因此： + +- 不应把 revision 相同理解为所有 Gateway 参数逐项相同； +- 不应仅依赖 revision 发现运维配置漂移； +- 共享协议的不兼容升级仍应使用蓝绿部署； +- 普通滚动升级结束后仍应核对 Pod 镜像、ConfigMap hash 和关键环境变量。 + +## 运维检查清单 + +滚动升级前： + +1. 判断改动属于 A、B、C 哪一类。 +2. C 类不兼容改动改用蓝绿部署，不进行原地混跑。 +3. 确认新旧版本使用相同 etcd 集群、prefix、deployment ID 和共享 artifact store。 +4. 确认 Worker wire protocol 与新旧 Gateway 均兼容。 + +滚动升级期间： + +1. 允许 B 类行为短暂不同，并关注 429、失败、重试和超时指标。 +2. 不单独修改某个 Pod 来绕过 Worker 声明容量或 fencing。 +3. 若出现任务恢复失败，首先比较镜像、配置 revision、pool 定义和 etcd prefix。 + +滚动升级后： + +1. 确认所有 Ready Gateway 使用目标镜像和 ConfigMap hash。 +2. 确认 B 类参数已经收敛；刻意保留的差异应有运维记录。 +3. 确认 queue、retry credits、Worker leases 和 finalization backlog 能正常回落。 +4. 至少执行一次提交、查询、下载、删除以及 Worker 故障重试的冒烟验证。 + diff --git a/dingo/video_gateway/RECOVERY_AND_UPLOAD_CLEANUP.md b/dingo/video_gateway/RECOVERY_AND_UPLOAD_CLEANUP.md new file mode 100644 index 000000000000..e503db95bee0 --- /dev/null +++ b/dingo/video_gateway/RECOVERY_AND_UPLOAD_CLEANUP.md @@ -0,0 +1,59 @@ +# Result handoff 与上传临时目录回收 + +## Result handoff 异常处理（M5） + +Worker 完成后，Gateway 用 etcd 事务写入结果引用、将任务置为 `finalizing`，并释放执行 +slot。遇到超时或断连，必须先重新读取任务，不能根据客户端异常推断事务没有提交。 + +- 已提交匹配的 handoff：接着完成后处理，不再执行模型。 +- CAS 竞争、存储暂时不可用：重新读取、指数退避，加少量抖动，上限 5 秒；停止进程时 + 可以中断等待。不会因为重试次数达到某个数就把未确认的结果当作失败。 +- 构造或写入 handoff 的内部错误：核对状态后，只允许将同一 revision 的 `in_progress` + 任务置为失败；取消请求同样按这个窄条件写入终态。 +- 写失败/取消状态后响应丢失：仍然读回核对。若终态写入本身持续发生程序错误，当前 + Gateway 停止准入并请求进程重启，由现有 owner 接管机制恢复，避免落入通用 Worker + 失败/重试路径。无法解析的持久化记录可能仍需人工修复，不承诺任意数据损坏自动恢复。 +- 释放 slot 必须同时匹配 task、owner generation、execution token；不匹配时只结束旧 + 任务，不碰当前 slot 占用者。终态 fenced release 不允许修改执行身份。 +- 日志、指标异常不改变已提交结果。正常路径不增加额外存储读取。 + +## 上传临时目录回收（M6） + +上传活跃性属于一次 HTTP 请求，不属于整个 Gateway 进程。 + +每个新 `_uploads/` 目录有一个空文件 `.upload-heartbeat`。从开始接收 multipart +到输入校验、任务提交结束，当前请求每 `min(30秒, upload_grace_s / 3)` 更新该文件的 +mtime。没有 JSON owner、额外 etcd key、每上传租约或 Gateway owner 查询。 + +清理器只删除目录和心跳都超过 `lifecycle.upload_grace_s` 的临时上传。默认 1 小时按 +**最后一次成功心跳**计算。后台扫描的实际回收时间还取决于 `sweeper_interval_s` +（默认 30 秒）及文件系统响应时间。 + +- 正常提交：目录原子移动到任务目录，心跳标记移除。 +- 正常失败、客户端取消：停止并等待请求与心跳协程，然后主动删除临时目录。 +- 删除暂时失败：心跳已经停止，即使 Gateway 不重启，之后仍能超时回收残留。 +- Gateway 崩溃：心跳自然停止，其他 Gateway 可按同一规则回收。 +- 心跳 I/O 失败或心跳已超龄：中止上传；不会默默停止续期后继续接收请求。 +- 心跳通过打开的文件描述符更新，目录提交重命名不会触发误报。取消时会等待已开始的 + 文件操作结束后再关闭描述符和删除目录，包括重复取消的情况。 + +共享文件系统必须能在 Gateway 间及时反映文件 mtime，节点时钟应保持同步。超过宽限 +期的进程暂停/存储中断按上传失效处理；此机制不承诺无限期暂停后还能恢复同一上传。 +不使用跨 Pod `flock`：当前 DingoFS 实测没有提供需要的锁互斥效果。 + +### 从旧 owner 版本迁移 + +新旧清理器会保守跳过不识别的标记。旧 `.upload-owner.json` 或无心跳的历史目录不会 +自动删除；确认旧版本全部下线、旧上传均已结束后，再人工清理这些历史目录。 +新请求使用 `.upload-heartbeat`，不再有“必须等 Gateway 重启才能回收”的行为。 + +## 针对性回归 + +- `tests/video_gateway/test_handoff_recovery.py`：不确定提交、失败/取消回包丢失、CAS、 + 中断存储等待、损坏终态写入出口及身份约束。 +- `tests/video_gateway/test_early_release_dispatcher.py`：完整派发链路、模型调用次数、 + slot 复用与恢复失败时停止准入。 +- `tests/video_gateway/test_upload_lifecycle.py`：真实慢 multipart、取消、心跳失败、 + 清理失败残留、重复取消时文件操作收尾。 +- `tests/video_gateway/test_artifact_store.py`：跨实例清理、停止续期后的回收、超龄心跳 + 不再复活和目录提交重命名。 diff --git a/dingo/video_gateway/api.py b/dingo/video_gateway/api.py index 5cd758dbaa0b..72eba6362926 100644 --- a/dingo/video_gateway/api.py +++ b/dingo/video_gateway/api.py @@ -5,17 +5,20 @@ from __future__ import annotations +import asyncio import ipaddress import logging import os import resource import stat +import time import aiohttp from aiohttp import web +from dingo.video_gateway.artifact_store import UPLOAD_HEARTBEAT_NAME from dingo.video_gateway.errors import GatewayError, StoreUnavailable -from dingo.video_gateway.file_io import opened_file, run_file_io +from dingo.video_gateway.file_io import _drain_on_cancel, opened_file, run_file_io from dingo.video_gateway.form_parser import parse_multipart from dingo.video_gateway.models import TERMINAL_STATUSES, TaskStatus from dingo.video_gateway.service import VideoGatewayService @@ -34,6 +37,20 @@ "transfer-encoding", "upgrade", } +_UPLOAD_HEARTBEAT_MAX_INTERVAL_S = 30.0 + + +async def _keep_upload_active(marker_fd: int, grace_s: float) -> None: + def refresh() -> None: + # A delayed request must not revive an already expired upload while + # another Gateway is deleting it. The fd survives the commit rename. + if time.time() - os.fstat(marker_fd).st_mtime >= grace_s: + raise TimeoutError("upload heartbeat expired") + os.utime(marker_fd, None) + + while True: + await asyncio.sleep(min(_UPLOAD_HEARTBEAT_MAX_INTERVAL_S, grace_s / 3)) + await run_file_io(refresh) def _service(request: web.Request) -> VideoGatewayService: @@ -172,8 +189,15 @@ async def _submit(request: web.Request, *, delivery_mode: str): await service.ensure_submission_capacity( anticipated_input + service.config.media.max_result_bytes ) - parsed = await parse_multipart(request, service.artifacts, service.config.media) - try: + upload_root = await service.artifacts.create_upload() + + async def submit(): + parsed = await parse_multipart( + request, + service.artifacts, + service.config.media, + upload_root=upload_root, + ) return await service.submit( fields=parsed.fields, uploads=parsed.uploads, @@ -181,9 +205,46 @@ async def _submit(request: web.Request, *, delivery_mode: str): delivery_mode=delivery_mode, idempotency_key=request.headers.get("Idempotency-Key"), ) - except Exception: - await service.artifacts.discard(parsed.upload_root) - raise + + try: + async with opened_file( + (upload_root / UPLOAD_HEARTBEAT_NAME).open, "rb" + ) as marker: + operation = asyncio.create_task(submit(), name="video-upload-submit") + heartbeat = asyncio.create_task( + _keep_upload_active( + marker.fileno(), service.config.lifecycle.upload_grace_s + ), + name="video-upload-heartbeat", + ) + try: + done, _ = await asyncio.wait( + {operation, heartbeat}, return_when=asyncio.FIRST_COMPLETED + ) + if operation in done: + return await operation + # Fail closed if heartbeat I/O fails; do not let an unprotected + # upload continue. Drain its file work before closing/unlinking. + try: + await heartbeat + except Exception as exc: + raise GatewayError( + 503, + "upload_heartbeat_failed", + "upload storage heartbeat could not be maintained", + error_type="service_unavailable_error", + ) from exc + raise RuntimeError("upload heartbeat stopped unexpectedly") + finally: + operation.cancel() + heartbeat.cancel() + await _drain_on_cancel( + asyncio.gather(operation, heartbeat, return_exceptions=True) + ) + finally: + # commit_upload moved the directory on success. Cleanup also runs on + # disconnect, malformed multipart, and cancellation during validation. + await service.artifacts.discard(upload_root) async def create_video(request: web.Request) -> web.Response: diff --git a/dingo/video_gateway/artifact_store.py b/dingo/video_gateway/artifact_store.py index 356b96e1bcae..37163f89485f 100644 --- a/dingo/video_gateway/artifact_store.py +++ b/dingo/video_gateway/artifact_store.py @@ -27,6 +27,8 @@ from dingo.video_gateway.errors import ResultTooLarge from dingo.video_gateway.file_io import run_cancellable_file_io, run_file_io +UPLOAD_HEARTBEAT_NAME = ".upload-heartbeat" + @dataclass(frozen=True, slots=True) class ArtifactCapacity: @@ -97,8 +99,13 @@ async def create_upload(self) -> Path: def _create() -> Path: path.mkdir(0o700, True, False) - (path / "inputs").mkdir(0o700, False, False) - return self._contained(path) + try: + (path / "inputs").mkdir(0o700, False, False) + (path / UPLOAD_HEARTBEAT_NAME).touch(exist_ok=False) + return self._contained(path) + except Exception: + shutil.rmtree(path, ignore_errors=True) + raise return await run_file_io(_create) @@ -149,8 +156,16 @@ async def capacity(self) -> ArtifactCapacity: usage = await run_file_io(shutil.disk_usage, self.root) return ArtifactCapacity(usage.total, usage.used, usage.free) - async def cleanup_orphan_uploads(self, *, minimum_age_s: float = 3600.0) -> int: - """Remove only stale staging directories that were never task-owned.""" + async def cleanup_orphan_uploads( + self, + *, + minimum_age_s: float = 3600.0, + ) -> int: + """Remove staging directories whose request heartbeat has expired. + + Missing/legacy markers are kept for explicit migration cleanup. Neither + a live Gateway nor ordinary writes to inputs/ renew this heartbeat. + """ cutoff = time.time() - minimum_age_s @@ -166,7 +181,26 @@ def _cleanup() -> int: if candidate.is_symlink(): candidate.unlink(missing_ok=True) elif candidate.is_dir(): - shutil.rmtree(self._contained(candidate)) + marker_path = candidate / UPLOAD_HEARTBEAT_NAME + try: + marker = marker_path.lstat() + if ( + not stat.S_ISREG(marker.st_mode) + or marker.st_mtime > cutoff + ): + continue + except OSError: + # Old Gateway versions do not renew request heartbeats. + # Do not delete their staging directories during rollout. + continue + try: + # Recheck after scanning: another process may have + # refreshed or committed this upload in the meantime. + if marker_path.lstat() != marker: + continue + shutil.rmtree(self._contained(candidate)) + except FileNotFoundError: + continue else: candidate.unlink(missing_ok=True) removed += 1 @@ -203,6 +237,12 @@ def _commit() -> Path: f"task artifact directory already exists: {task_id}" ) os.replace(source, target) + try: + (target / UPLOAD_HEARTBEAT_NAME).unlink(missing_ok=True) + except OSError: + # The marker is harmless outside _uploads. A completed atomic + # move must not fail solely because marker cleanup was denied. + pass return target target = await run_file_io(_commit) diff --git a/dingo/video_gateway/dispatcher.py b/dingo/video_gateway/dispatcher.py index 4cb070c41f19..4bb2f66ef820 100644 --- a/dingo/video_gateway/dispatcher.py +++ b/dingo/video_gateway/dispatcher.py @@ -8,7 +8,9 @@ import asyncio import hashlib import logging +import math import os +import random import secrets import time import uuid @@ -36,9 +38,11 @@ HandoffReservationLost, ResultTooLarge, StoreConflict, + StoreUnavailable, WorkerUnavailable, worker_execution_error, ) +from dingo.video_gateway.etcd_http import EtcdWatchIdleTimeout from dingo.video_gateway.file_io import run_file_io from dingo.video_gateway.finalization import ResultFinalizer from dingo.video_gateway.memory_budget import ( @@ -70,6 +74,8 @@ _DETACHED_WAIT_ATTACH_TIMEOUT_S = 1.0 _DETACHED_WAIT_RETRY_INITIAL_S = 0.2 _DETACHED_WAIT_RETRY_MAX_S = 5.0 +_HANDOFF_RETRY_INITIAL_S = 0.2 +_HANDOFF_RETRY_MAX_S = 5.0 _DETACHED_WORKER_STALE_S = 20.0 _WORKER_LIVENESS_CHECK_INTERVAL_S = 5.0 _WORKER_LIVENESS_CHECK_CONCURRENCY = 4 @@ -647,9 +653,10 @@ def _wake_task_waiters(self, task_id: str | None = None) -> None: if not waiter.done(): waiter.set_result(None) - async def _resync_task_watch(self) -> None: - self._task_watch_healthy = False - self._task_watch_ready.clear() + async def _resync_task_watch(self, *, preserve_health: bool = False) -> None: + if not preserve_health: + self._task_watch_healthy = False + self._task_watch_ready.clear() self._task_watch_revision = await self.store.task_watch_revision() if self._task_watch_revision < 0: raise RuntimeError("task watch snapshot omitted its store revision") @@ -680,19 +687,30 @@ async def _task_watch_loop(self) -> None: raise RuntimeError("task watch ended unexpectedly") except asyncio.CancelledError: raise - except Exception: - self._task_watch_healthy = False - self._task_watch_ready.clear() + except Exception as exc: + idle_refresh = isinstance(exc, EtcdWatchIdleTimeout) + if not idle_refresh: + self._task_watch_healthy = False + self._task_watch_ready.clear() self.telemetry.increment( - "dingo_video_etcd_watch_rebuilds_total", + ( + "dingo_video_etcd_watch_idle_refreshes_total" + if idle_refresh + else "dingo_video_etcd_watch_rebuilds_total" + ), labels={"watch": "tasks", "pool": "_all"}, ) - logger.exception("task watch failed and will be rebuilt") + if idle_refresh: + logger.debug("task watch idle timeout; refreshing from snapshot") + else: + logger.exception("task watch failed and will be rebuilt") try: - await self._resync_task_watch() + await self._resync_task_watch(preserve_health=idle_refresh) except asyncio.CancelledError: raise except Exception: + self._task_watch_healthy = False + self._task_watch_ready.clear() logger.exception("task watch snapshot rebuild failed") try: await asyncio.wait_for(self._stop.wait(), timeout=backoff_s) @@ -950,8 +968,11 @@ async def _discovery_watchdog_loop(self) -> None: except asyncio.TimeoutError: pass - async def _resync_lease_cache(self, pool: PoolRuntime) -> None: - pool.lease_watch_healthy = False + async def _resync_lease_cache( + self, pool: PoolRuntime, *, preserve_health: bool = False + ) -> None: + if not preserve_health: + pool.lease_watch_healthy = False leases, revision = await self.store.lease_snapshot(pool.config.pool_id) if revision <= 0: raise RuntimeError("Worker lease snapshot omitted its etcd revision") @@ -980,21 +1001,36 @@ async def _lease_watch_loop(self, pool: PoolRuntime) -> None: raise RuntimeError("Worker lease watch ended unexpectedly") except asyncio.CancelledError: raise - except Exception: - pool.lease_watch_healthy = False + except Exception as exc: + idle_refresh = isinstance(exc, EtcdWatchIdleTimeout) + if not idle_refresh: + pool.lease_watch_healthy = False self.telemetry.increment( - "dingo_video_etcd_watch_rebuilds_total", + ( + "dingo_video_etcd_watch_idle_refreshes_total" + if idle_refresh + else "dingo_video_etcd_watch_rebuilds_total" + ), labels={"watch": "worker_leases", "pool": pool.config.pool_id}, ) - logger.exception( - "Worker lease watch failed and will be rebuilt: %s", - pool.config.pool_id, - ) + if idle_refresh: + logger.debug( + "Worker lease watch idle timeout; refreshing from snapshot: %s", + pool.config.pool_id, + ) + else: + logger.exception( + "Worker lease watch failed and will be rebuilt: %s", + pool.config.pool_id, + ) try: - await self._resync_lease_cache(pool) + await self._resync_lease_cache( + pool, preserve_health=idle_refresh + ) except asyncio.CancelledError: raise except Exception: + pool.lease_watch_healthy = False logger.exception( "Worker lease snapshot rebuild failed: %s", pool.config.pool_id, @@ -1601,6 +1637,7 @@ async def _consume_worker_stream() -> None: ) worker_stream_finished = True + worker_queue_wait_s: float | None = None if detached: detached_consumer = self._consume_detached_worker( pool, @@ -1614,7 +1651,7 @@ async def _consume_worker_stream() -> None: # Ownership has moved into the detached consumer coroutine. # Do not keep a second reference in this long-lived frame. payload = None - await self._run_with_lease_monitor( + worker_queue_wait_s = await self._run_with_lease_monitor( detached_consumer, heartbeat, self._monitor_cancellation( @@ -1661,6 +1698,8 @@ async def _consume_worker_stream() -> None: ) inference_time_s = result.inference_time_s stage_durations = dict(result.stage_durations or {}) + if worker_queue_wait_s is not None: + stage_durations["worker_queue_wait"] = worker_queue_wait_s result = None self._legacy_output_encoded_bytes += len(encoded_result) if len(encoded_result) > self.config.media.max_result_encoded_bytes: @@ -1676,9 +1715,19 @@ async def _consume_worker_stream() -> None: and pool.config.scheduling.early_release_slot and latest.task.status == TaskStatus.IN_PROGRESS ): - handed_off = await self._commit_result_handoff( - pool, task, binary_artifact, inference_time_s, stage_durations - ) + try: + handed_off = await self._commit_result_handoff( + pool, task, binary_artifact, inference_time_s, stage_durations + ) + except _TaskOwnershipLost: + raise + except Exception: + # A failed reconciliation must never fall through to the + # generic Worker failure/cancel/retry path. Preserve the + # result and let owner-lease recovery reconcile durable state. + logger.exception("result handoff recovery suspended: %s", task.id) + await self._request_fatal_restart("result handoff recovery failed") + return if handed_off is not None: self.running_calls.pop(task.id, None) pool.wakeup.set() @@ -2003,88 +2052,161 @@ async def _request_detached_cancel(self, task: Any) -> None: async def _commit_result_handoff( self, pool, expected, artifact, inference_time_s, stage_durations ): - """Retry ambiguous storage responses by reading the durable task first.""" - reservation_lost = False - while not self._stop.is_set(): + """Reconcile every uncertain write before publishing or failing a result.""" + failure_code = None + settlement_error = None + reference = None + retries = 0 + delay = _HANDOFF_RETRY_INITIAL_S + started = time.monotonic() + + def observe(event, before=None, after=None): try: - latest = await self.store.get_task(expected.id) - if latest is None or latest.task.status in TERMINAL_STATUSES: - return None - self._require_execution_owner(latest.task, expected) - if ( - latest.task.status == TaskStatus.FINALIZING - and read_handoff(latest.task) is not None - ): - return latest - if reservation_lost: - # A definite fencing failure is not transient CAS contention. - # End only our task, never cancel/release/quarantine the new - # occupant of the historical Worker slot. - await self.store.transition( - expected.id, - expected={TaskStatus.IN_PROGRESS}, - expected_revision=latest.revision, - release_lease=False, - patch={ - "status": TaskStatus.FAILED, - "error": TaskError( - "result_handoff_lost_reservation", - "execution reservation changed before result handoff", - ), - "completed_at_ms": now_ms(), - "expires_at_ms": now_ms() - + int(self.config.lifecycle.failed_ttl_s * 1000), - }, + if before is not None and after is not None: + self.telemetry.record_transition( + event, + before.task, + after.task, + gateway_generation=self.generation, + revision=after.revision, ) - return None - if latest.task.cancel_requested_at_ms is not None: - await self._finish_cancelled(pool, latest, quarantine=False) - return None - reference = make_handoff( - latest.task, - artifact, - timeout_s=pool.config.scheduling.finalization_timeout_s, + self.telemetry.increment( + "dingo_video_result_handoff_total", + labels={"pool": expected.pool_id, "outcome": event}, ) - result = await self.store.transition( + self.telemetry.record_stage_duration( + expected.pool_id, "result_handoff", time.monotonic() - started + ) + except Exception: + logger.exception("handoff telemetry failed: %s", expected.id) + + async def backoff(reason): + nonlocal retries, delay + retries += 1 + if retries == 1 or retries & (retries - 1) == 0: + logger.warning( + "handoff recovery task_id=%s attempt=%s reason=%s retries=%s elapsed_s=%.3f", expected.id, - expected={TaskStatus.IN_PROGRESS}, - expected_revision=latest.revision, - release_lease=True, - release_execution=True, - patch={ - "status": TaskStatus.FINALIZING, - "worker_lease_id": None, - "normalized_request": { - **latest.task.normalized_request, - HANDOFF_KEY: reference, - }, - "inference_time_s": inference_time_s, - "stage_durations": stage_durations, - }, + expected.attempt, + reason, + retries, + time.monotonic() - started, ) - self.telemetry.record_transition( - "result_handoff", - latest.task, - result.task, - gateway_generation=self.generation, - revision=result.revision, + try: + self.telemetry.increment( + "dingo_video_result_handoff_retries_total", + labels={"pool": expected.pool_id, "reason": reason}, ) - return result - except HandoffReservationLost: - reservation_lost = True - except (_TaskOwnershipLost, ValueError): - raise - except asyncio.CancelledError: - raise except Exception: - # Includes a lost transaction response. Never discard the source - # or call the Worker again while the outcome is uncertain. - logger.warning( - "result handoff awaiting storage for task %s", + logger.exception("handoff retry telemetry failed: %s", expected.id) + try: + await asyncio.wait_for( + self._stop.wait(), timeout=random.uniform(0.8 * delay, delay) + ) + except TimeoutError: + pass + delay = min(delay * 2, _HANDOFF_RETRY_MAX_S) + + while not self._stop.is_set(): + try: + latest = await self.store.get_task(expected.id) + except (StoreUnavailable, TimeoutError, ConnectionError): + await backoff("store_unavailable") + continue + if latest is None or latest.task.status in TERMINAL_STATUSES: + observe("already_terminal") + return None + self._require_execution_owner(latest.task, expected) + if latest.task.status == TaskStatus.FINALIZING: + if read_handoff(latest.task) is None: + raise ValueError("finalizing task is missing its durable handoff") + observe("confirmed_after_read") + return latest + if latest.task.status != TaskStatus.IN_PROGRESS: + raise ValueError("handoff requires the current in_progress execution") + if settlement_error is not None: + # One read-back confirmed the attempted failure/cancellation did + # not finish. Do not loop on a broken terminal writer indefinitely. + raise RuntimeError( + "handoff terminal reconciliation failed" + ) from settlement_error + + lost = failure_code == "result_handoff_lost_reservation" + cancelled = latest.task.cancel_requested_at_ms is not None + terminal = failure_code is not None or cancelled + if terminal: + status = TaskStatus.CANCELLED if cancelled else TaskStatus.FAILED + code = "cancelled" if cancelled else failure_code + message = { + "cancelled": "video task was cancelled", + "result_handoff_lost_reservation": "execution reservation changed before result handoff", + "result_handoff_failed": "result handoff failed before durable publication", + }[code] + ttl = ( + self.config.lifecycle.cancelled_ttl_s + if cancelled + else self.config.lifecycle.failed_ttl_s + ) + patch = { + "status": status, + "error": TaskError(code, message), + "completed_at_ms": now_ms(), + "expires_at_ms": now_ms() + int(ttl * 1000), + } + else: + if reference is None: + try: + reference = make_handoff( + latest.task, + artifact, + timeout_s=pool.config.scheduling.finalization_timeout_s, + ) + except Exception: + failure_code = "result_handoff_failed" + logger.exception("handoff construction failed: %s", expected.id) + continue + patch = { + "status": TaskStatus.FINALIZING, + "worker_lease_id": None, + "normalized_request": { + **latest.task.normalized_request, + HANDOFF_KEY: reference, + }, + "inference_time_s": inference_time_s, + "stage_durations": stage_durations, + } + try: + result = await self.store.transition( expected.id, - exc_info=True, + expected={TaskStatus.IN_PROGRESS}, + expected_revision=latest.revision, + patch=patch, + release_lease=not lost, + release_execution=not lost, ) - await asyncio.sleep(0.2) + except HandoffReservationLost: + failure_code = "result_handoff_lost_reservation" + continue + except StoreConflict: + await backoff("cas_conflict") + continue + except (StoreUnavailable, TimeoutError, ConnectionError): + await backoff("store_unavailable") + continue + except Exception as exc: + # Even a local exception may follow a committed txn. Read back + # before failing, and reconcile a failed terminal write once too. + if terminal: + settlement_error = exc + failure_code = failure_code or "result_handoff_failed" + logger.exception("handoff write needs reconciliation: %s", expected.id) + continue + observe( + result.task.status.value if terminal else "result_handoff", + latest, + result, + ) + return None if terminal else result raise asyncio.CancelledError async def _run_result_finalizer(self, pool, stored): @@ -2175,7 +2297,7 @@ async def _consume_detached_worker( running_call: RunningCall, *, initial_worker_status: dict[str, Any] | None = None, - ) -> None: + ) -> float | None: task = stored.task if ( task.execution_token is None @@ -2224,18 +2346,27 @@ def _supports_wait(value: dict[str, Any] | None) -> bool: and WAIT_TERMINAL_CAPABILITY in capabilities ) + worker_queue_wait_s: float | None = None + async def _consume_status(value: dict[str, Any]) -> bool: + nonlocal worker_queue_wait_s worker_status = _validate_identity(value) state = worker_status.get("state") if state == "completed": queue_wait = worker_status.get("worker_queue_wait_s") - if ( - isinstance(queue_wait, (int, float)) - and not isinstance(queue_wait, bool) - and 0 <= queue_wait <= 86400 - ): + if queue_wait is not None: + if ( + not isinstance(queue_wait, (int, float)) + or isinstance(queue_wait, bool) + or not math.isfinite(float(queue_wait)) + or not 0 <= float(queue_wait) <= 86400 + ): + raise _DetachedWaitProtocolError( + "detached Worker returned invalid worker_queue_wait_s" + ) + worker_queue_wait_s = float(queue_wait) self.telemetry.record_stage_duration( - task.pool_id, "worker_queue", queue_wait + task.pool_id, "worker_queue", worker_queue_wait_s ) if "result_format" in worker_status or "inline_result" in worker_status: if worker_status.get( @@ -2460,7 +2591,7 @@ async def _wait_once() -> dict[str, Any]: assert worker_status is not None supports_wait = _supports_wait(worker_status) if await _consume_status(worker_status): - return + return worker_queue_wait_s retry_delay_s = _DETACHED_WAIT_RETRY_INITIAL_S next_wait_retry = time.monotonic() @@ -2476,7 +2607,7 @@ async def _wait_once() -> dict[str, Any]: worker_status = await _wait_once() retry_delay_s = _DETACHED_WAIT_RETRY_INITIAL_S if await _consume_status(worker_status): - return + return worker_queue_wait_s raise _DetachedWaitUnavailable( "detached Worker could not attach a local terminal waiter" ) @@ -2498,7 +2629,7 @@ async def _wait_once() -> dict[str, Any]: if worker_status is not None: supports_wait = supports_wait or _supports_wait(worker_status) if await _consume_status(worker_status): - return + return worker_queue_wait_s remaining_ms = (task.deadline_at_ms or 0) - now_ms() if remaining_ms <= 0: raise asyncio.TimeoutError diff --git a/dingo/video_gateway/etcd_http.py b/dingo/video_gateway/etcd_http.py index abcfcf7af3cc..f24483b2af4f 100644 --- a/dingo/video_gateway/etcd_http.py +++ b/dingo/video_gateway/etcd_http.py @@ -87,6 +87,10 @@ def __init__(self, compact_revision: int, reason: str = "") -> None: ) +class EtcdWatchIdleTimeout(StoreUnavailable): + """A healthy watch stream produced no events or progress before its deadline.""" + + class EtcdHttpClient: def __init__( self, @@ -271,7 +275,20 @@ async def _stream_post( await self._mark_endpoint(index, succeeded=False) if yielded or position + 1 == len(endpoints): raise - except (aiohttp.ClientError, TimeoutError) as exc: + except TimeoutError as exc: + if yielded: + # An established watch can legitimately be idle. Refresh it + # from a new snapshot without penalizing the etcd endpoint. + raise EtcdWatchIdleTimeout( + f"etcd endpoint {endpoint} {path} watch was idle for " + f"{self.watch_response_timeout_s} seconds" + ) from exc + await self._mark_endpoint(index, succeeded=False) + if position + 1 == len(endpoints): + raise StoreUnavailable( + f"etcd endpoint {endpoint} {path} stream failed: {exc}" + ) from exc + except aiohttp.ClientError as exc: await self._mark_endpoint(index, succeeded=False) if yielded or position + 1 == len(endpoints): raise StoreUnavailable( diff --git a/dingo/video_gateway/form_parser.py b/dingo/video_gateway/form_parser.py index d12e31969e92..af10d2c3e8e3 100644 --- a/dingo/video_gateway/form_parser.py +++ b/dingo/video_gateway/form_parser.py @@ -58,6 +58,8 @@ async def parse_multipart( request: web.Request, artifacts: FileArtifactStore, limits: MediaConfig, + *, + upload_root: Path | None = None, ) -> ParsedMultipart: if request.content_type != "multipart/form-data": raise GatewayError( @@ -65,7 +67,8 @@ async def parse_multipart( "unsupported_media_type", "video requests require multipart/form-data", ) - upload_root = await artifacts.create_upload() + if upload_root is None: + upload_root = await artifacts.create_upload() fields: dict[str, list[str]] = {} uploads: list[UploadedArtifact] = [] total_file_bytes = 0 diff --git a/dingo/video_gateway/models.py b/dingo/video_gateway/models.py index cd9db84590d1..2aa90c6dd53f 100644 --- a/dingo/video_gateway/models.py +++ b/dingo/video_gateway/models.py @@ -198,16 +198,20 @@ def public_dict(self) -> dict[str, Any]: result["video_duration_s"] = media["video_duration_s"] if media.get("audio_duration_s") is not None: result["audio_duration_s"] = media["audio_duration_s"] + durations = dict(self.stage_durations or {}) + worker_queue_wait_s = durations.get("worker_queue_wait") if any( value is not None for value in ( self.queue_wait_s, + worker_queue_wait_s, self.inference_time_s, self.finalize_time_s, ) ): result["metrics"] = { "queue_wait_s": self.queue_wait_s, + "worker_queue_wait_s": worker_queue_wait_s, "inference_time_s": self.inference_time_s, "finalize_time_s": self.finalize_time_s, } @@ -218,7 +222,6 @@ def public_dict(self) -> dict[str, Any]: result["inference_time_s"] = max( 0.0, (self.completed_at_ms - self.created_at_ms) / 1000.0 ) - durations = dict(self.stage_durations or {}) if self.queue_wait_s is not None: durations.setdefault("queue_wait", self.queue_wait_s) if self.finalize_time_s is not None: diff --git a/dingo/video_gateway/task_store.py b/dingo/video_gateway/task_store.py index f5885d63d7bf..b6399521b6d1 100644 --- a/dingo/video_gateway/task_store.py +++ b/dingo/video_gateway/task_store.py @@ -85,6 +85,29 @@ def _clone_task(task: VideoTask) -> VideoTask: return VideoTask.from_dict(task.to_dict()) +def _validate_fenced_release(before: VideoTask, after: VideoTask) -> None: + if after.status == TaskStatus.FINALIZING: + validate_handoff_transition(before, after) + return + if before.status != TaskStatus.IN_PROGRESS or after.status not in { + TaskStatus.FAILED, + TaskStatus.CANCELLED, + }: + raise ValueError("fenced terminal release requires an in_progress execution") + for name in ( + "id", + "pool_id", + "backend_target", + "worker_key", + "worker_instance_id", + "owner_generation", + "attempt", + "execution_token", + ): + if getattr(before, name) != getattr(after, name): + raise ValueError("fenced terminal release cannot change execution identity") + + def _apply_patch(task: VideoTask, patch: Mapping[str, Any]) -> VideoTask: result = _clone_task(task) for key, value in patch.items(): @@ -584,7 +607,7 @@ async def transition( ) updated = _apply_patch(current[0], patch) if release_execution: - validate_handoff_transition(current[0], updated) + _validate_fenced_release(current[0], updated) lease = self._leases.get((current[0].pool_id, current[0].worker_key)) if ( not release_lease @@ -1679,6 +1702,12 @@ async def transition( quarantine_until_ms=quarantine_until_ms, task_hint=stored, ) + except HandoffReservationLost: + # This is a definitive fencing result, not shared-ledger CAS + # contention. Retrying cannot restore ownership of a reused + # Worker reservation and only delays the caller's safe + # task-only failure path. + raise except StoreConflict: current = await self.get_task(task_id) if ( @@ -1743,9 +1772,11 @@ async def _transition_once( raise StoreConflict("task revision changed") updated = _apply_patch(stored.task, patch) if release_execution: - validate_handoff_transition(stored.task, updated) + _validate_fenced_release(stored.task, updated) if not release_lease or quarantine_until_ms is not None: - raise ValueError("handoff must release, never quarantine execution") + raise ValueError( + "fenced execution transition must release, never quarantine" + ) task_key = self._task_key(task_id) compare: list[dict] = [self.client.compare_mod(task_key, stored.revision)] success: list[dict] = [ diff --git a/dingo/video_gateway/telemetry.py b/dingo/video_gateway/telemetry.py index f8834c232a8e..82684acf39a4 100644 --- a/dingo/video_gateway/telemetry.py +++ b/dingo/video_gateway/telemetry.py @@ -232,6 +232,28 @@ def audit_task( "error_code": task.error.code if task.error is not None else None, } payload.update({key: value for key, value in optional.items() if value is not None}) + if event in { + "finalization_started", + "completed", + "failed", + "cancelled", + }: + stages = dict(task.stage_durations or {}) + timing = { + "gateway_queue_wait_s": task.queue_wait_s, + "worker_queue_wait_s": stages.get("worker_queue_wait"), + "worker_inference_time_s": task.inference_time_s, + "gateway_finalize_time_s": task.finalize_time_s, + "end_to_end_s": ( + max(0.0, (task.completed_at_ms - task.created_at_ms) / 1000.0) + if task.completed_at_ms is not None + else None + ), + "stage_durations": stages or None, + } + payload.update( + {key: value for key, value in timing.items() if value is not None} + ) if extra: payload.update(extra) audit_logger.info(json.dumps(payload, sort_keys=True, separators=(",", ":"))) diff --git a/dingo/vllm/omni/detached_tasks.py b/dingo/vllm/omni/detached_tasks.py index 973c3fd5fe22..8feed97a09f7 100644 --- a/dingo/vllm/omni/detached_tasks.py +++ b/dingo/vllm/omni/detached_tasks.py @@ -300,6 +300,43 @@ def _base_status( "capabilities": [WAIT_TERMINAL_CAPABILITY], } + @staticmethod + def _log_task_timing( + event: str, + identity: DetachedTaskIdentity, + status: Mapping[str, Any], + ) -> None: + inline = status.get("inline_result") + inline = inline if isinstance(inline, Mapping) else {} + stages = inline.get("stage_durations") + error = status.get("error") + payload = { + "log_type": "video_worker_task_timing", + "event": event, + "timestamp_ms": int(time.time() * 1000), + "task_id": identity.task_id, + "deployment_id": identity.deployment_id, + "pool_id": identity.pool_id, + "attempt": identity.attempt, + "state": status.get("state"), + "queued_at_ms": status.get("queued_at_ms"), + "started_at_ms": status.get("started_at_ms"), + "completed_at_ms": status.get("completed_at_ms"), + "worker_queue_wait_s": status.get("worker_queue_wait_s"), + "worker_execution_s": status.get("inference_time_s"), + "inference_time_s": inline.get("inference_time_s"), + "stage_durations": stages if isinstance(stages, Mapping) else None, + "error_code": error.get("code") if isinstance(error, Mapping) else None, + } + print( + json.dumps( + {key: value for key, value in payload.items() if value is not None}, + sort_keys=True, + separators=(",", ":"), + ), + flush=True, + ) + async def _submit( self, identity: DetachedTaskIdentity, @@ -362,6 +399,7 @@ def _claim() -> bool: if self.prefetch_capacity: initial["queued_at_ms"] = queued_at_ms await asyncio.to_thread(self._atomic_json, status_path, initial) + self._log_task_timing("accepted", identity, initial) request_id = ( f"{identity.task_id}-{identity.attempt}-{identity.execution_token[:12]}" ) @@ -452,6 +490,7 @@ async def _record_terminal( running = self._running.get(identity.key) if running is not None: running.persisted_terminal = copy.deepcopy(status) + self._log_task_timing("terminal", identity, status) async def _execute( self, @@ -559,7 +598,25 @@ async def _execute_started( await asyncio.to_thread( self._atomic_json, status_path, - {**self._base_status(identity, "running"), "started_at_ms": started_at_ms}, + { + **self._base_status(identity, "running"), + "queued_at_ms": queued_at_ms, + "started_at_ms": started_at_ms, + }, + ) + self._log_task_timing( + "execution_started", + identity, + { + "state": "running", + "queued_at_ms": queued_at_ms, + "started_at_ms": started_at_ms, + "worker_queue_wait_s": ( + max(0.0, (started_at_ms - queued_at_ms) / 1000) + if queued_at_ms is not None + else None + ), + }, ) result_token = BINARY_RESULT_WRITER.set( BinaryResultWriter(attempt_root) if self.binary_results else None @@ -616,10 +673,15 @@ async def _execute_started( temporary.unlink(missing_ok=True) status_stop.set() await asyncio.gather(status_heartbeat, return_exceptions=True) - await asyncio.to_thread( - self._atomic_json, + await self._record_terminal( + identity, status_path, - {**self._base_status(identity, "cancelled")}, + { + **self._base_status(identity, "cancelled"), + "queued_at_ms": queued_at_ms, + "started_at_ms": started_at_ms, + "completed_at_ms": int(time.time() * 1000), + }, ) return if not self.inline_results: @@ -630,6 +692,9 @@ async def _execute_started( **self._base_status(identity, "completed"), **response_fields, "inference_time_s": max(0.0, time.monotonic() - started), + "queued_at_ms": queued_at_ms, + "started_at_ms": started_at_ms, + "completed_at_ms": int(time.time() * 1000), } if queued_at_ms is not None: completed["worker_queue_wait_s"] = max( @@ -641,10 +706,15 @@ async def _execute_started( temporary.unlink(missing_ok=True) status_stop.set() await asyncio.gather(status_heartbeat, return_exceptions=True) - await asyncio.to_thread( - self._atomic_json, + await self._record_terminal( + identity, status_path, - {**self._base_status(identity, "cancelled")}, + { + **self._base_status(identity, "cancelled"), + "queued_at_ms": queued_at_ms, + "started_at_ms": started_at_ms, + "completed_at_ms": int(time.time() * 1000), + }, ) raise except Exception as exc: @@ -654,6 +724,9 @@ async def _execute_started( logger.exception("detached Omni task failed: %s", identity.task_id) failed = { **self._base_status(identity, "failed"), + "queued_at_ms": queued_at_ms, + "started_at_ms": started_at_ms, + "completed_at_ms": int(time.time() * 1000), "error": { "code": "worker_failed", "message": str(exc)[:1024] or "detached Omni task failed", @@ -714,7 +787,17 @@ def read(): try: await asyncio.shield(running.execution) except asyncio.CancelledError: - raise + current = asyncio.current_task() + if ( + (current is not None and current.cancelling()) + or not running.execution.cancelled() + ): + # The wait request itself was cancelled. Its lifetime never + # owns the detached execution, which remains shielded. + raise + # The detached execution was cancelled after persisting its + # durable terminal state. Return that state below instead of + # presenting a normal task cancellation as a broken wait stream. except Exception: # _execute records its terminal failure before it returns. Read # that durable status below instead of leaking an internal task diff --git a/tests/video_gateway/test_artifact_store.py b/tests/video_gateway/test_artifact_store.py index 280f277148bc..3f7e3979ad86 100644 --- a/tests/video_gateway/test_artifact_store.py +++ b/tests/video_gateway/test_artifact_store.py @@ -3,14 +3,16 @@ from __future__ import annotations +import asyncio import base64 import hashlib -import os import json +import os import pytest -from dingo.video_gateway.artifact_store import FileArtifactStore +from dingo.video_gateway.api import _keep_upload_active +from dingo.video_gateway.artifact_store import UPLOAD_HEARTBEAT_NAME, FileArtifactStore from dingo.video_gateway.errors import ResultTooLarge @@ -43,9 +45,7 @@ async def test_detached_status_and_response_are_identity_and_checksum_checked(tm } (root / "worker-status.json").write_text(json.dumps(status)) - loaded = await store.read_detached_status( - "deployment", "pool", "task", 1, token - ) + loaded = await store.read_detached_status("deployment", "pool", "task", 1, token) consumer = _Consumer() consumed = await store.consume_detached_response( "deployment", @@ -77,7 +77,9 @@ async def test_detached_status_and_response_are_identity_and_checksum_checked(tm async def test_finalize_decodes_validated_mp4_atomically(tmp_path): store = FileArtifactStore(tmp_path / "artifacts") upload = await store.create_upload() + assert (upload / UPLOAD_HEARTBEAT_NAME).is_file() task_root = await store.commit_upload(upload, "deployment", "pool", "video-id") + assert not (task_root / UPLOAD_HEARTBEAT_NAME).exists() payload = b"\x00\x00\x00\x18ftypisomvalidated" final, size, sha256, media = await store.finalize_b64_mp4( @@ -181,12 +183,95 @@ async def test_orphan_cleanup_only_removes_stale_staging_directories(tmp_path): stale = await store.create_upload() current = await store.create_upload() os.utime(stale, (0, 0)) + os.utime(stale / UPLOAD_HEARTBEAT_NAME, (0, 0)) removed = await store.cleanup_orphan_uploads(minimum_age_s=60) assert removed == 1 assert not stale.exists() assert current.is_dir() + await store.discard(current) + + +async def test_orphan_cleanup_never_removes_an_active_upload(tmp_path): + store = FileArtifactStore(tmp_path / "artifacts") + active = await store.create_upload() + os.utime(active, (0, 0)) + + # A different Gateway sees the same heartbeat marker on shared storage. + other_gateway = FileArtifactStore(store.root) + removed = await other_gateway.cleanup_orphan_uploads(minimum_age_s=60) + + assert removed == 0 + assert active.is_dir() + await store.discard(active) + + +async def test_orphan_cleanup_removes_stale_upload_while_gateway_stays_alive(tmp_path): + store = FileArtifactStore(tmp_path / "artifacts") + abandoned = await store.create_upload() + os.utime(abandoned, (0, 0)) + os.utime(abandoned / UPLOAD_HEARTBEAT_NAME, (0, 0)) + + removed = await store.cleanup_orphan_uploads(minimum_age_s=60) + + assert removed == 1 + assert not abandoned.exists() + + +async def test_orphan_cleanup_keeps_legacy_upload_without_owner(tmp_path): + store = FileArtifactStore(tmp_path / "artifacts") + legacy = store.upload_root / "legacy-stale" + (legacy / "inputs").mkdir(parents=True) + os.utime(legacy, (0, 0)) + + removed = await store.cleanup_orphan_uploads(minimum_age_s=60) + + assert removed == 0 + assert legacy.is_dir() + + +async def test_heartbeat_stops_with_request_and_does_not_resurrect_expired_upload( + tmp_path, +): + store = FileArtifactStore(tmp_path / "artifacts") + active = await store.create_upload() + marker = active / UPLOAD_HEARTBEAT_NAME + os.utime(active, (0, 0)) + with marker.open("rb") as stream: + before = marker.stat().st_mtime_ns + heartbeat = asyncio.create_task(_keep_upload_active(stream.fileno(), 0.3)) + try: + for _ in range(100): + if marker.stat().st_mtime_ns > before: + break + await asyncio.sleep(0.01) + assert marker.stat().st_mtime_ns > before + assert await store.cleanup_orphan_uploads(minimum_age_s=0.3) == 0 + finally: + heartbeat.cancel() + await asyncio.gather(heartbeat, return_exceptions=True) + stopped = marker.stat().st_mtime_ns + await asyncio.sleep(0.12) + assert marker.stat().st_mtime_ns == stopped + os.utime(marker, (0, 0)) + with pytest.raises(TimeoutError, match="expired"): + await _keep_upload_active(stream.fileno(), 0.03) + assert await store.cleanup_orphan_uploads(minimum_age_s=0.3) == 1 + + +async def test_heartbeat_fd_survives_commit_rename(tmp_path): + store = FileArtifactStore(tmp_path / "artifacts") + upload = await store.create_upload() + with (upload / UPLOAD_HEARTBEAT_NAME).open("rb") as stream: + heartbeat = asyncio.create_task(_keep_upload_active(stream.fileno(), 0.3)) + try: + target = await store.commit_upload(upload, "deployment", "pool", "task") + await asyncio.sleep(0.15) + assert target.is_dir() and not heartbeat.done() + finally: + heartbeat.cancel() + await asyncio.gather(heartbeat, return_exceptions=True) async def test_task_orphan_is_manifested_trashed_and_deleted_in_two_steps(tmp_path): diff --git a/tests/video_gateway/test_dispatcher.py b/tests/video_gateway/test_dispatcher.py index f219e0fdbaa9..e8932ea1e261 100644 --- a/tests/video_gateway/test_dispatcher.py +++ b/tests/video_gateway/test_dispatcher.py @@ -301,6 +301,38 @@ async def _submit(service, model): ) +async def test_idle_watch_snapshot_refresh_preserves_ready_state( + make_gateway_config, +): + config = make_gateway_config() + client = FakeClient() + store = WatchMemoryTaskStore() + artifacts = FileArtifactStore(config.artifact_store.root) + adapters = {pool.pool_id: create_adapter(pool) for pool in config.pools} + dispatcher = VideoDispatcher( + config, + store, + artifacts, + {"fl-pool": client}, + adapters, + context_factory=FakeContext, + generation="idle-watch-refresh-test", + ) + pool = dispatcher.pools["fl-pool"] + dispatcher._task_watch_healthy = True + dispatcher._task_watch_ready.set() + pool.lease_watch_healthy = True + + await dispatcher._resync_task_watch(preserve_health=True) + await dispatcher._resync_lease_cache(pool, preserve_health=True) + + assert dispatcher._task_watch_healthy is True + assert dispatcher._task_watch_ready.is_set() + assert pool.lease_watch_healthy is True + assert dispatcher._task_watch_revision >= 0 + assert pool.lease_revision > 0 + + async def test_two_pools_with_same_numeric_instance_never_cross_route( make_gateway_config, ): diff --git a/tests/video_gateway/test_early_release_dispatcher.py b/tests/video_gateway/test_early_release_dispatcher.py index ac240bc17569..f81efa8ecf09 100644 --- a/tests/video_gateway/test_early_release_dispatcher.py +++ b/tests/video_gateway/test_early_release_dispatcher.py @@ -75,10 +75,24 @@ async def slow(*args, **kwargs): await asyncio.wait_for(second_started.wait(), 2) assert not finish.is_set() finish.set() - for submitted in [first, second]: - assert ( - await dispatcher.wait_terminal(submitted.stored.task.id, 3) - ).task.status == TaskStatus.COMPLETED + terminals = [ + await dispatcher.wait_terminal(submitted.stored.task.id, 3) + for submitted in [first, second] + ] + assert all(item.task.status == TaskStatus.COMPLETED for item in terminals) + if prefetch: + assert all( + item.task.stage_durations is not None + and item.task.stage_durations["worker_queue_wait"] >= 0 + and item.task.public_dict()["metrics"]["worker_queue_wait_s"] + == item.task.stage_durations["worker_queue_wait"] + for item in terminals + ) + else: + assert all( + "worker_queue_wait" not in (item.task.stage_durations or {}) + for item in terminals + ) assert len(calls) == 2 finally: finish.set() @@ -87,12 +101,21 @@ async def slow(*args, **kwargs): @pytest.mark.parametrize( - "fault", ["lost_handoff_reply", "completion_store_unavailable", "lost_reservation"] + "fault", + [ + "lost_handoff_reply", + "completion_store_unavailable", + "lost_reservation", + "store_unavailable_then_recovers", + "deterministic_handoff_error", + "deterministic_handoff_error_after_slot_reuse", + "broken_terminal_writer", + ], ) async def test_handoff_storage_faults_do_not_rerun_model( make_gateway_config, monkeypatch, fault ): - from dingo.video_gateway.errors import HandoffReservationLost + from dingo.video_gateway.errors import HandoffReservationLost, StoreUnavailable calls = 0 @@ -118,13 +141,42 @@ async def generate(self, request, context): ) original = store.transition injected = False + handoff_attempts = 0 async def transition(*args, **kwargs): - nonlocal injected - handoff = kwargs.get("release_execution", False) + nonlocal handoff_attempts, injected + handoff = ( + kwargs.get("release_execution", False) + and kwargs["patch"].get("status") == TaskStatus.FINALIZING + ) + if handoff: + handoff_attempts += 1 if not injected and handoff and fault == "lost_reservation": injected = True raise HandoffReservationLost("injected reservation replacement") + if handoff and fault in { + "deterministic_handoff_error", + "broken_terminal_writer", + }: + injected = True + raise RuntimeError("injected deterministic handoff bug") + if ( + fault == "broken_terminal_writer" + and kwargs["patch"].get("status") == TaskStatus.FAILED + ): + raise RuntimeError("terminal writer is broken too") + if handoff and fault == "deterministic_handoff_error_after_slot_reuse": + injected = True + key = next(iter(store._leases)) + store._leases[key].execution_token = "e" * 32 + raise RuntimeError("injected deterministic handoff bug after slot reuse") + if ( + handoff + and fault == "store_unavailable_then_recovers" + and handoff_attempts <= 3 + ): + injected = True + raise StoreUnavailable("injected etcd outage before publication") if ( not injected and fault == "completion_store_unavailable" @@ -142,20 +194,52 @@ async def transition(*args, **kwargs): await dispatcher.start() try: submitted = await _submit(service, "public-fl") - terminal = await dispatcher.wait_terminal(submitted.stored.task.id, 4) + if fault == "broken_terminal_writer": + await asyncio.wait_for(dispatcher._fatal_event.wait(), 3) + assert injected and calls == handoff_attempts == 1 + assert not dispatcher.ready and dispatcher.draining + current = await store.get_task(submitted.stored.task.id) + assert current.task.status == TaskStatus.IN_PROGRESS + for _ in range(100): + if (await dispatcher.memory_budget.snapshot()).used_bytes == 0: + break + await asyncio.sleep(0.01) + assert (await dispatcher.memory_budget.snapshot()).used_bytes == 0 + return + terminal = await dispatcher.wait_terminal(submitted.stored.task.id, 6) assert injected and calls == 1 and terminal.task.attempt == 1 - if fault == "lost_reservation": + if fault in { + "lost_reservation", + "deterministic_handoff_error", + "deterministic_handoff_error_after_slot_reuse", + }: assert terminal.task.status == TaskStatus.FAILED - assert terminal.task.error.code == "result_handoff_lost_reservation" + assert terminal.task.error.code == ( + "result_handoff_failed" + if fault == "deterministic_handoff_error" + else "result_handoff_lost_reservation" + ) else: assert terminal.task.status == TaskStatus.COMPLETED assert not await store.list_leases("fl-pool") + if fault in { + "deterministic_handoff_error", + "deterministic_handoff_error_after_slot_reuse", + }: + assert handoff_attempts == 1 + elif fault == "store_unavailable_then_recovers": + assert handoff_attempts == 4 for _ in range(100): if (await dispatcher.memory_budget.snapshot()).used_bytes == 0: break await asyncio.sleep(0.01) assert (await dispatcher.memory_budget.snapshot()).used_bytes == 0 assert not dispatcher._finalizing + if fault == "deterministic_handoff_error": + assert not await store.list_leases("fl-pool") + elif fault == "deterministic_handoff_error_after_slot_reuse": + leases = await store.list_leases("fl-pool") + assert len(leases) == 1 and leases[0].execution_token == "e" * 32 finally: await dispatcher.stop() await manager.shutdown() diff --git a/tests/video_gateway/test_etcd_http.py b/tests/video_gateway/test_etcd_http.py index 96d111ae18b5..4c89a21894af 100644 --- a/tests/video_gateway/test_etcd_http.py +++ b/tests/video_gateway/test_etcd_http.py @@ -15,6 +15,7 @@ from dingo.video_gateway.etcd_http import ( EtcdHttpClient, EtcdWatchCompacted, + EtcdWatchIdleTimeout, ) @@ -355,7 +356,7 @@ async def test_watch_prefix_reports_server_cancel(): pass -async def test_watch_response_timeout_marks_silent_endpoint_failed(): +async def test_watch_response_timeout_refreshes_silent_endpoint_without_penalty(): hold_open = asyncio.Event() async def stalled_watch(request): @@ -393,12 +394,73 @@ async def stalled_watch(request): created = await asyncio.wait_for(anext(watch), timeout=1) assert created.created is True - with pytest.raises(StoreUnavailable, match="stream failed"): + with pytest.raises(EtcdWatchIdleTimeout, match="watch was idle"): await asyncio.wait_for(anext(watch), timeout=1) - assert (await client._endpoint_order())[0][1] == "http://next-etcd" + assert (await client._endpoint_order())[0][1] == str( + server.make_url("") + ).rstrip("/") finally: await watch.aclose() await client.close() hold_open.set() await server.close() + + +async def test_watch_without_response_timeout_remains_pending_while_range_works(): + hold_open = asyncio.Event() + + async def stalled_watch(request): + await request.json() + response = web.StreamResponse( + status=200, headers={"Content-Type": "application/json"} + ) + await response.prepare(request) + await response.write( + json.dumps( + { + "result": { + "header": {"revision": "20"}, + "watch_id": "7", + "created": True, + } + } + ).encode() + + b"\n" + ) + await hold_open.wait() + return response + + async def range_request(request): + await request.json() + return web.json_response( + {"header": {"revision": "21"}, "kvs": [], "more": False} + ) + + app = web.Application() + app.router.add_post("/v3/watch", stalled_watch) + app.router.add_post("/v3/kv/range", range_request) + server = TestServer(app) + await server.start_server() + client = EtcdHttpClient( + str(server.make_url("")).rstrip("/"), + timeout_s=1, + watch_response_timeout_s=None, + ) + watch = client.watch_prefix("/workers/") + pending = None + try: + created = await asyncio.wait_for(anext(watch), timeout=1) + assert created.created is True + pending = asyncio.create_task(anext(watch)) + await asyncio.sleep(0.05) + assert not pending.done() + assert (await client.range_page("/probe")).revision == 21 + finally: + if pending is not None: + pending.cancel() + await asyncio.gather(pending, return_exceptions=True) + await watch.aclose() + await client.close() + hold_open.set() + await server.close() diff --git a/tests/video_gateway/test_handoff_recovery.py b/tests/video_gateway/test_handoff_recovery.py new file mode 100644 index 000000000000..ca2f9d89ac0b --- /dev/null +++ b/tests/video_gateway/test_handoff_recovery.py @@ -0,0 +1,164 @@ +import asyncio +from types import SimpleNamespace + +import pytest + +from dingo.video_gateway.dispatcher import VideoDispatcher +from dingo.video_gateway.errors import StoreConflict, StoreUnavailable +from dingo.video_gateway.models import TaskStatus +from dingo.video_gateway.result_handoff import read_handoff +from dingo.video_gateway.telemetry import GatewayTelemetry +from tests.video_gateway.test_result_handoff import ARTIFACT, setup + + +def dispatcher_for(store, config, monkeypatch): + import dingo.video_gateway.dispatcher as module + + monkeypatch.setattr(module, "_HANDOFF_RETRY_INITIAL_S", 0.001) + monkeypatch.setattr(module, "_HANDOFF_RETRY_MAX_S", 0.002) + obj = object.__new__(VideoDispatcher) + obj.store = store + obj.config = config + obj.generation = "generation" + obj.telemetry = GatewayTelemetry() + obj._stop = asyncio.Event() + return obj, SimpleNamespace(config=config.pools[0]) + + +@pytest.mark.parametrize("kind", ["memory", "etcd"]) +@pytest.mark.parametrize( + "fault", + [ + "reply_lost", + "local_error_after_commit", + "write_unavailable", + "read_unavailable", + "cas", + "failure_reply_lost", + "cancellation_reply_lost", + "telemetry", + ], +) +async def test_reconcile_handoff_and_terminal_outcomes( + kind, fault, make_gateway_config, monkeypatch +): + store, client, active, _ = await setup(kind) + dispatcher, pool = dispatcher_for(store, make_gateway_config(), monkeypatch) + original = store.transition + original_get = store.get_task + writes, reads = 0, 0 + + async def read(task_id): + nonlocal reads + reads += 1 + if fault == "read_unavailable" and reads <= 3: + raise StoreUnavailable("read timeout") + return await original_get(task_id) + + async def transition(*args, **kwargs): + nonlocal writes + writes += 1 + if fault in {"failure_reply_lost", "cancellation_reply_lost"} and writes == 1: + if fault == "cancellation_reply_lost": + # Use the original writer to avoid recursively invoking this wrapper. + await original( + active.task.id, + expected={TaskStatus.IN_PROGRESS}, + patch={"cancel_requested_at_ms": 1}, + ) + raise TypeError("deterministic error before handoff") + if fault == "write_unavailable" and writes <= 3: + raise StoreUnavailable("write timeout") + if fault == "cas" and writes <= 3: + raise StoreConflict("contended") + result = await original(*args, **kwargs) + if (fault == "reply_lost" and writes == 1) or ( + fault.endswith("reply_lost") and writes == 2 + ): + raise StoreUnavailable("response lost after commit") + if fault == "local_error_after_commit" and writes == 1: + raise TypeError("decode failed after commit") + return result + + monkeypatch.setattr(store, "transition", transition) + monkeypatch.setattr(store, "get_task", read) + if fault == "telemetry": + + def fail(*args, **kwargs): + raise RuntimeError("telemetry unavailable") + + monkeypatch.setattr(dispatcher.telemetry, "record_transition", fail) + monkeypatch.setattr(dispatcher.telemetry, "increment", fail) + result = await dispatcher._commit_result_handoff(pool, active.task, ARTIFACT, 1, {}) + stored = await original_get(active.task.id) + if fault in {"failure_reply_lost", "cancellation_reply_lost"}: + assert result is None + assert stored.task.status == ( + TaskStatus.FAILED if fault == "failure_reply_lost" else TaskStatus.CANCELLED + ) + else: + assert result.task.status == TaskStatus.FINALIZING + assert read_handoff(stored.task)["artifact"] == ARTIFACT + assert not await store.list_leases(active.task.pool_id) + if client: + assert await store.retry_budget_used(active.task.pool_id) == 0 + assert ( + await client.get( + store._lease_heartbeat_key(active.task.pool_id, active.task.worker_key) + ) + is None + ) + + +async def test_broken_terminal_writer_has_recovery_exit( + make_gateway_config, monkeypatch +): + store, _, active, _ = await setup("memory") + dispatcher, pool = dispatcher_for(store, make_gateway_config(), monkeypatch) + calls = 0 + + async def broken(*args, **kwargs): + nonlocal calls + calls += 1 + raise TypeError("permanent serializer defect") + + monkeypatch.setattr(store, "transition", broken) + with pytest.raises(RuntimeError, match="reconciliation failed"): + await dispatcher._commit_result_handoff(pool, active.task, ARTIFACT, 1, {}) + assert calls == 2 # one handoff, one guarded terminal write; never spin + assert (await store.get_task(active.task.id)).task.status == TaskStatus.IN_PROGRESS + + +async def test_storage_outage_retry_is_interruptible(make_gateway_config, monkeypatch): + store, _, active, _ = await setup("memory") + dispatcher, pool = dispatcher_for(store, make_gateway_config(), monkeypatch) + entered = asyncio.Event() + + async def unavailable(*args): + entered.set() + raise StoreUnavailable("outage") + + monkeypatch.setattr(store, "get_task", unavailable) + running = asyncio.create_task( + dispatcher._commit_result_handoff(pool, active.task, ARTIFACT, 1, {}) + ) + await entered.wait() + dispatcher._stop.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(running, 1) + + +@pytest.mark.parametrize("kind", ["memory", "etcd"]) +async def test_fenced_terminal_cannot_change_attempt(kind): + store, _, active, _ = await setup(kind) + with pytest.raises(ValueError, match="identity"): + await store.transition( + active.task.id, + expected={TaskStatus.IN_PROGRESS}, + expected_revision=active.revision, + patch={"status": TaskStatus.FAILED, "attempt": 99}, + release_lease=True, + release_execution=True, + ) + assert (await store.get_task(active.task.id)).task.status == TaskStatus.IN_PROGRESS + assert len(await store.list_leases(active.task.pool_id)) == 1 diff --git a/tests/video_gateway/test_result_handoff.py b/tests/video_gateway/test_result_handoff.py index 640e41624eb9..0b9552481f3c 100644 --- a/tests/video_gateway/test_result_handoff.py +++ b/tests/video_gateway/test_result_handoff.py @@ -3,7 +3,7 @@ import pytest -from dingo.video_gateway.errors import StoreConflict +from dingo.video_gateway.errors import HandoffReservationLost, StoreConflict from dingo.video_gateway.models import TaskStatus, now_ms from dingo.video_gateway.result_handoff import HANDOFF_KEY, make_handoff, read_handoff from dingo.video_gateway.task_store import EtcdTaskStore, MemoryTaskStore @@ -161,14 +161,25 @@ async def test_handoff_recovery_does_not_touch_reused_worker_slot(): assert await client.get(slot_key) == slot_before -async def test_conflicting_token_cannot_handoff_someone_elses_reservation(): +async def test_conflicting_token_fails_handoff_without_retrying_fencing( + monkeypatch, +): store, client, stored, owner = await setup("etcd") key = store._lease_key(stored.task.pool_id, stored.task.worker_key) value = await client.get(key) data = json.loads(value.value) data["execution_token"] = "e" * 32 client.values[key] = dataclasses.replace(value, value=json.dumps(data).encode()) - with pytest.raises(StoreConflict): + original = store._transition_once + calls = 0 + + async def counted(*args, **kwargs): + nonlocal calls + calls += 1 + return await original(*args, **kwargs) + + monkeypatch.setattr(store, "_transition_once", counted) + with pytest.raises(HandoffReservationLost): await store.transition( stored.task.id, expected={TaskStatus.IN_PROGRESS}, @@ -177,6 +188,7 @@ async def test_conflicting_token_cannot_handoff_someone_elses_reservation(): release_lease=True, release_execution=True, ) + assert calls == 1 assert read_handoff((await store.get_task(stored.task.id)).task) is None assert await client.get(store._retry_credit_key(stored.task)) is not None diff --git a/tests/video_gateway/test_task_store.py b/tests/video_gateway/test_task_store.py index 60dab7358620..cde4925385fd 100644 --- a/tests/video_gateway/test_task_store.py +++ b/tests/video_gateway/test_task_store.py @@ -86,6 +86,26 @@ def test_public_task_reports_final_seed_and_validated_media_metadata(): assert public["audio_duration_s"] == 5.175 +def test_public_task_reports_persisted_worker_queue_wait(): + task = _task("video-worker-queue") + task.status = TaskStatus.COMPLETED + task.completed_at_ms = task.created_at_ms + 2_000 + task.queue_wait_s = 0.25 + task.inference_time_s = 1.5 + task.finalize_time_s = 0.05 + task.stage_durations = { + "worker_queue_wait": 0.75, + "output_total_s": 0.2, + } + + public = task.public_dict() + + assert public["metrics"]["worker_queue_wait_s"] == 0.75 + assert public["stage_durations"]["worker_queue_wait"] == 0.75 + assert public["stage_durations"]["queue_wait"] == 0.25 + assert public["stage_durations"]["finalize"] == 0.05 + + async def test_idempotency_returns_original_task_and_detects_conflict(): store = MemoryTaskStore() first, created = await store.create_task( diff --git a/tests/video_gateway/test_telemetry.py b/tests/video_gateway/test_telemetry.py index db07f4638673..6934760b14bb 100644 --- a/tests/video_gateway/test_telemetry.py +++ b/tests/video_gateway/test_telemetry.py @@ -18,6 +18,9 @@ def test_telemetry_renders_bounded_task_and_etcd_metrics(caplog): failed = _task("video-observed") failed.status = TaskStatus.FAILED failed.error = terminal_error("worker_lease_lost", "lease lost") + failed.queue_wait_s = 0.1 + failed.inference_time_s = 1.2 + failed.stage_durations = {"worker_queue_wait": 0.3} telemetry.record_submission(task.pool_id, "created", "async") telemetry.set_gauge( @@ -56,4 +59,8 @@ def test_telemetry_renders_bounded_task_and_etcd_metrics(caplog): assert event["event"] == "failed" assert event["task_id"] == task.id assert event["gateway_generation"] == "gateway-a" + assert event["gateway_queue_wait_s"] == 0.1 + assert event["worker_queue_wait_s"] == 0.3 + assert event["worker_inference_time_s"] == 1.2 + assert event["stage_durations"] == {"worker_queue_wait": 0.3} assert "prompt" not in event diff --git a/tests/video_gateway/test_upload_lifecycle.py b/tests/video_gateway/test_upload_lifecycle.py new file mode 100644 index 000000000000..08a0bd450c15 --- /dev/null +++ b/tests/video_gateway/test_upload_lifecycle.py @@ -0,0 +1,194 @@ +import asyncio +import dataclasses +import os +import threading +from types import SimpleNamespace + +import pytest +from aiohttp.test_utils import TestClient, TestServer + +from dingo.video_gateway import api +from dingo.video_gateway.app import create_app +from dingo.video_gateway.artifact_store import UPLOAD_HEARTBEAT_NAME, FileArtifactStore +from dingo.video_gateway.errors import GatewayError +from dingo.video_gateway.file_io import run_file_io +from tests.video_gateway.test_dispatcher import FakeClient, _stack + +_REAL_TO_THREAD = asyncio.to_thread + + +def stack(make_gateway_config): + config = make_gateway_config() + config = dataclasses.replace( + config, lifecycle=dataclasses.replace(config.lifecycle, upload_grace_s=0.3) + ) + return _stack(config, {"fl-pool": FakeClient()}) + + +def pending_heartbeats(): + return [t for t in asyncio.all_tasks() if t.get_name() == "video-upload-heartbeat"] + + +async def test_real_slow_multipart_survives_other_gateway_cleanup(make_gateway_config): + _, artifacts, _, service = stack(make_gateway_config) + cleaner = FileArtifactStore(artifacts.root) + client = TestClient(TestServer(create_app(service))) + await client.start_server() + + async def body(): + yield b'--slow\r\nContent-Disposition: form-data; name="prompt"\r\n\r\n' + for _ in range(12): + await asyncio.sleep(0.05) + for upload in artifacts.upload_root.iterdir(): + if upload.is_dir(): + os.utime(upload, (0, 0)) + assert await cleaner.cleanup_orphan_uploads(minimum_age_s=0.3) == 0 + yield b"a slow request " + yield b"\r\n" + for key, value in [ + ("model", "public-fl"), + ("seconds", "5"), + ("size", "1344x768"), + ]: + yield ( + f'--slow\r\nContent-Disposition: form-data; name="{key}"\r\n\r\n' + f"{value}\r\n" + ).encode() + yield b"--slow--\r\n" + + try: + response = await client.post( + "/v1/videos", + data=body(), + headers={"Content-Type": "multipart/form-data; boundary=slow"}, + ) + assert response.status == 202, await response.text() + assert not list(artifacts.upload_root.iterdir()) + assert not pending_heartbeats() + finally: + await client.close() + + +@pytest.mark.parametrize("phase", ["parsing", "validation"]) +async def test_cancel_submission_stops_heartbeat_and_discards_staging( + make_gateway_config, monkeypatch, phase +): + _, artifacts, _, service = stack(make_gateway_config) + entered = asyncio.Event() + + async def parse(*args, upload_root, **kwargs): + if phase == "parsing": + entered.set() + await asyncio.Event().wait() + return SimpleNamespace(fields={}, uploads=[], upload_root=upload_root) + + async def submit(**kwargs): + entered.set() + await asyncio.Event().wait() + + monkeypatch.setattr(api, "parse_multipart", parse) + monkeypatch.setattr(service, "submit", submit) + request = SimpleNamespace( + app={api._SERVICE_KEY: service}, content_length=10, headers={} + ) + operation = asyncio.create_task(api._submit(request, delivery_mode="async")) + await asyncio.wait_for(entered.wait(), 1) + operation.cancel() + with pytest.raises(asyncio.CancelledError): + await operation + assert not pending_heartbeats() + assert not list(artifacts.upload_root.iterdir()) + + +async def test_heartbeat_error_aborts_upload_instead_of_leaving_unprotected_request( + make_gateway_config, monkeypatch +): + _, artifacts, _, service = stack(make_gateway_config) + unwound = asyncio.Event() + + async def parse(*args, **kwargs): + try: + await asyncio.Event().wait() + finally: + unwound.set() + + async def failed_heartbeat(*args): + await asyncio.sleep(0) + raise OSError("injected storage failure") + + monkeypatch.setattr(api, "parse_multipart", parse) + monkeypatch.setattr(api, "_keep_upload_active", failed_heartbeat) + request = SimpleNamespace( + app={api._SERVICE_KEY: service}, content_length=10, headers={} + ) + with pytest.raises(GatewayError) as raised: + await api._submit(request, delivery_mode="async") + assert raised.value.status == 503 and raised.value.code == "upload_heartbeat_failed" + assert unwound.is_set() and not pending_heartbeats() + assert not list(artifacts.upload_root.iterdir()) + + +async def test_cleanup_failure_does_not_keep_request_heartbeat_alive( + make_gateway_config, monkeypatch +): + _, artifacts, _, service = stack(make_gateway_config) + real_discard = artifacts.discard + + async def parse(*args, **kwargs): + raise ValueError("bad multipart") + + async def failed_discard(*args, **kwargs): + raise OSError("unlink temporarily unavailable") + + monkeypatch.setattr(api, "parse_multipart", parse) + monkeypatch.setattr(artifacts, "discard", failed_discard) + request = SimpleNamespace( + app={api._SERVICE_KEY: service}, content_length=10, headers={} + ) + with pytest.raises(OSError): + await api._submit(request, delivery_mode="async") + assert not pending_heartbeats() + [residual] = list(artifacts.upload_root.iterdir()) + os.utime(residual, (0, 0)) + os.utime(residual / UPLOAD_HEARTBEAT_NAME, (0, 0)) + monkeypatch.setattr(artifacts, "discard", real_discard) + assert ( + await FileArtifactStore(artifacts.root).cleanup_orphan_uploads(minimum_age_s=60) + == 1 + ) + + +async def test_repeated_cancel_drains_file_work_before_unlink( + make_gateway_config, monkeypatch +): + monkeypatch.setattr(asyncio, "to_thread", _REAL_TO_THREAD) + _, artifacts, _, service = stack(make_gateway_config) + entered = asyncio.Event() + release = threading.Event() + loop = asyncio.get_running_loop() + + def file_work(): + loop.call_soon_threadsafe(entered.set) + assert release.wait(3) + + async def parse(*args, **kwargs): + await run_file_io(file_work) + + monkeypatch.setattr(api, "parse_multipart", parse) + request = SimpleNamespace( + app={api._SERVICE_KEY: service}, content_length=10, headers={} + ) + operation = asyncio.create_task(api._submit(request, delivery_mode="async")) + try: + await asyncio.wait_for(entered.wait(), 1) + operation.cancel() + await asyncio.sleep(0.02) + operation.cancel() + await asyncio.sleep(0.02) + assert not operation.done() + assert len(list(artifacts.upload_root.iterdir())) == 1 + finally: + release.set() + await asyncio.gather(operation, return_exceptions=True) + assert not pending_heartbeats() + assert not list(artifacts.upload_root.iterdir()) diff --git a/tests/video_gateway/test_worker_prefetch.py b/tests/video_gateway/test_worker_prefetch.py index 5b1cf5ecf0fc..51cb34aa04e5 100644 --- a/tests/video_gateway/test_worker_prefetch.py +++ b/tests/video_gateway/test_worker_prefetch.py @@ -86,6 +86,90 @@ async def test_prefetch_admits_one_extra_but_never_executes_above_n(tmp_path, ca await manager.shutdown() +async def test_prefetch_emits_structured_task_timing_logs(tmp_path, capsys): + handler = Handler() + manager = DetachedOmniTaskManager( + handler, + tmp_path, + prefetch_capacity=1, + cancel_poll_interval_s=0.01, + ) + value = identity(manager, "logged") + try: + assert (await manager._submit(value, {"name": "logged"}))["accepted"] + await until(lambda: handler.running == 1) + handler.release.set() + await until(lambda: value.key not in manager._running) + events = [ + json.loads(line) + for line in capsys.readouterr().out.splitlines() + if line.startswith("{") + ] + assert [event["event"] for event in events] == [ + "accepted", + "execution_started", + "terminal", + ] + assert all(event["task_id"] == "logged" for event in events) + assert events[-1]["state"] == "completed" + assert events[-1]["worker_queue_wait_s"] >= 0 + assert "execution_token" not in events[-1] + finally: + handler.release.set() + await manager.shutdown() + + +async def test_wait_returns_durable_terminal_when_execution_is_cancelled(tmp_path): + handler = Handler() + manager = DetachedOmniTaskManager( + handler, tmp_path, binary_results=False, inline_results=False + ) + value = identity(manager, "execution-cancelled") + try: + assert (await manager._submit(value, {"name": value.task_id}))["accepted"] + await until(lambda: handler.running == 1) + waiter = manager._wait_terminal(value) + assert (await anext(waiter))["state"] == "watching" + terminal = asyncio.create_task(anext(waiter)) + + manager._running[value.key].execution.cancel() + + assert (await asyncio.wait_for(terminal, 2))["state"] == "cancelled" + assert (await manager._status(value))["state"] == "cancelled" + await waiter.aclose() + finally: + handler.release.set() + await manager.shutdown() + + +async def test_cancelling_waiter_keeps_detached_execution_running(tmp_path): + handler = Handler() + manager = DetachedOmniTaskManager( + handler, tmp_path, binary_results=False, inline_results=False + ) + value = identity(manager, "waiter-cancelled") + try: + assert (await manager._submit(value, {"name": value.task_id}))["accepted"] + await until(lambda: handler.running == 1) + execution = manager._running[value.key].execution + waiter = manager._wait_terminal(value) + assert (await anext(waiter))["state"] == "watching" + terminal = asyncio.create_task(anext(waiter)) + + terminal.cancel() + with pytest.raises(asyncio.CancelledError): + await terminal + assert not execution.done() + + handler.release.set() + await until(lambda: value.key not in manager._running) + assert (await manager._status(value))["state"] == "completed" + await waiter.aclose() + finally: + handler.release.set() + await manager.shutdown() + + @pytest.mark.parametrize("cancel_kind", ["rpc", "file"]) async def test_queued_cancel_never_enters_model_and_frees_admission( tmp_path, cancel_kind From e2b8e169fd2de02c8887561d50f3d5b3cc0757e4 Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Mon, 21 Sep 2026 00:37:06 +0800 Subject: [PATCH 09/12] [fix][omni] isolate detached engine health monitoring --- dingo/vllm/omni/engine_monitor.py | 109 ++++++++++++++++++ dingo/vllm/omni/main.py | 8 ++ .../tests/omni/test_omni_engine_monitor.py | 61 ++++++++++ 3 files changed, 178 insertions(+) create mode 100644 dingo/vllm/omni/engine_monitor.py create mode 100644 dingo/vllm/tests/omni/test_omni_engine_monitor.py diff --git a/dingo/vllm/omni/engine_monitor.py b/dingo/vllm/omni/engine_monitor.py new file mode 100644 index 000000000000..efc302169e53 --- /dev/null +++ b/dingo/vllm/omni/engine_monitor.py @@ -0,0 +1,109 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Health monitor isolated to the vLLM-Omni worker process.""" + +from __future__ import annotations + +import asyncio +import logging +import math +import os +import signal +import traceback + +from vllm.v1.engine.exceptions import EngineDeadError +from vllm_omni.entrypoints.async_omni import AsyncOmni + +from dingo.common.engine_monitor import EngineHealthMonitorConfig +from dynamo.runtime import DistributedRuntime + +logger = logging.getLogger(__name__) + + +class OmniEngineMonitor: + """Exit an Omni Worker when AsyncOmni reports a dead stage or rank.""" + + def __init__( + self, + runtime: DistributedRuntime, + engine_client: AsyncOmni, + shutdown_event: asyncio.Event | None = None, + ) -> None: + if not isinstance(runtime, DistributedRuntime): + raise ValueError( + f"{self.__class__.__name__} requires a DistributedRuntime" + ) + if not isinstance(engine_client, AsyncOmni): + raise ValueError(f"{self.__class__.__name__} requires an AsyncOmni") + self.runtime = runtime + self.engine_client = engine_client + self.shutdown_event = shutdown_event + self.health_config = EngineHealthMonitorConfig.from_env() + self._monitor_task = asyncio.create_task(self._check_engine_health()) + logger.info("OmniEngineMonitor initialized and health check task started") + + def __del__(self) -> None: + monitor_task = getattr(self, "_monitor_task", None) + if monitor_task is not None: + monitor_task.cancel() + + def _shutdown_engine(self) -> None: + def timeout_handler(signum, frame): + del signum, frame + raise TimeoutError("Omni engine shutdown timed out") + + previous_handler = None + if self.health_config.shutdown_timeout > 0: + previous_handler = signal.getsignal(signal.SIGALRM) + signal.signal(signal.SIGALRM, timeout_handler) + signal.alarm(math.ceil(self.health_config.shutdown_timeout)) + try: + self.engine_client.shutdown() + except Exception as exc: + logger.warning("vLLM-Omni engine shutdown failed: %s", exc) + finally: + if self.health_config.shutdown_timeout > 0: + signal.alarm(0) + signal.signal(signal.SIGALRM, previous_handler) + + async def _check_engine_health(self) -> None: + while True: + try: + if self.shutdown_event and self.shutdown_event.is_set(): + logger.info( + "OmniEngineMonitor: shutdown event detected; stopping monitor" + ) + return + await self._run_health_check() + if self.shutdown_event: + try: + await asyncio.wait_for( + self.shutdown_event.wait(), + timeout=self.health_config.interval, + ) + return + except asyncio.TimeoutError: + pass + else: + await asyncio.sleep(self.health_config.interval) + except (EngineDeadError, asyncio.TimeoutError) as exc: + logger.error("Traceback: %s", traceback.format_exc()) + logger.error("vLLM-Omni health check failed: %s", exc) + logger.warning("Initiating Dynamo Runtime shutdown") + self._shutdown_engine() + self.runtime.shutdown() + os._exit(1) + except asyncio.CancelledError: + logger.debug("OmniEngineMonitor health check task cancelled") + return + + async def _run_health_check(self) -> None: + health_check = self.engine_client.check_health() + if self.health_config.check_timeout > 0: + await asyncio.wait_for( + health_check, + timeout=self.health_config.check_timeout, + ) + return + await health_check diff --git a/dingo/vllm/omni/main.py b/dingo/vllm/omni/main.py index e17097e53987..d14a14edc964 100644 --- a/dingo/vllm/omni/main.py +++ b/dingo/vllm/omni/main.py @@ -23,6 +23,7 @@ ) from dingo.vllm.health_check import VllmOmniHealthCheckPayload from dingo.vllm.main import setup_metrics_collection +from dingo.vllm.omni.engine_monitor import OmniEngineMonitor from .args import OmniConfig, parse_omni_args @@ -60,6 +61,13 @@ async def init_omni( media_output_fs=media_fs, media_output_http_url=config.media_output_http_url, ) + # Monitor AsyncOmni independently from the standard AsyncLLM worker path so + # a dead stage/rank cannot leave a registered endpoint with no live engine. + _engine_monitor = OmniEngineMonitor( + runtime, + handler.engine_client, + shutdown_event, + ) serve_handler = handler.generate if config.detached_video_task_root is not None: from dingo.vllm.omni.detached_tasks import DetachedOmniTaskManager diff --git a/dingo/vllm/tests/omni/test_omni_engine_monitor.py b/dingo/vllm/tests/omni/test_omni_engine_monitor.py new file mode 100644 index 000000000000..69ca862782e7 --- /dev/null +++ b/dingo/vllm/tests/omni/test_omni_engine_monitor.py @@ -0,0 +1,61 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import asyncio +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from vllm.v1.engine.exceptions import EngineDeadError + +from dingo.common.engine_monitor import EngineHealthMonitorConfig +from dingo.vllm.omni.engine_monitor import OmniEngineMonitor + +pytestmark = [pytest.mark.unit, pytest.mark.vllm, pytest.mark.pre_merge] + + +def _monitor(engine): + monitor = object.__new__(OmniEngineMonitor) + monitor.runtime = MagicMock() + monitor.engine_client = engine + monitor.shutdown_event = None + monitor.health_config = EngineHealthMonitorConfig( + interval=0.01, + check_timeout=0.01, + shutdown_timeout=0.01, + ) + monitor._monitor_task = asyncio.get_event_loop().create_future() + return monitor + + +@pytest.mark.asyncio +async def test_health_check_timeout_is_fatal(): + async def blocked_health_check(): + await asyncio.sleep(1) + + engine = AsyncMock() + engine.check_health = blocked_health_check + monitor = _monitor(engine) + + with pytest.raises(asyncio.TimeoutError): + await monitor._run_health_check() + + +@pytest.mark.asyncio +async def test_engine_death_shuts_down_runtime_and_worker(): + engine = AsyncMock() + engine.check_health.side_effect = EngineDeadError("engine is dead") + monitor = _monitor(engine) + + with ( + patch.object(monitor, "_shutdown_engine") as shutdown_engine, + patch( + "dingo.vllm.omni.engine_monitor.os._exit", + side_effect=SystemExit(1), + ) as exit_process, + pytest.raises(SystemExit, match="1"), + ): + await monitor._check_engine_health() + + shutdown_engine.assert_called_once_with() + monitor.runtime.shutdown.assert_called_once_with() + exit_process.assert_called_once_with(1) From f48e0f2d115f5b23348ac25c634edea38154d566 Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Wed, 23 Sep 2026 17:36:38 +0800 Subject: [PATCH 10/12] [fix][video-gateway] tolerate delayed artifact visibility --- dingo/video_gateway/dispatcher.py | 97 ++++++++++++++++++- .../test_dispatch_artifact_visibility.py | 94 ++++++++++++++++++ 2 files changed, 189 insertions(+), 2 deletions(-) create mode 100644 tests/video_gateway/test_dispatch_artifact_visibility.py diff --git a/dingo/video_gateway/dispatcher.py b/dingo/video_gateway/dispatcher.py index 4bb2f66ef820..ddf16a3559c6 100644 --- a/dingo/video_gateway/dispatcher.py +++ b/dingo/video_gateway/dispatcher.py @@ -82,6 +82,14 @@ _DISCOVERY_MISMATCH_MIN_CHECKS = 3 _DISCOVERY_RECOVERY_LOCK_TTL_S = 15 _DISCOVERY_RESTART_DRAIN_S = 5.0 +_ARTIFACT_VISIBILITY_RETRY_DELAYS_S = ( + 0.1, + 0.25, + 0.5, + 1.0, + 2.0, + 4.0, +) + (5.0,) * 10 class _DetachedWorkerCancelled(RuntimeError): @@ -111,6 +119,14 @@ class _DetachedWaitProtocolError(RuntimeError): pass +class _ArtifactUnavailable(RuntimeError): + pass + + +class _DispatchCancelledDuringArtifactWait(RuntimeError): + pass + + @dataclass(slots=True) class RunningCall: context: Any @@ -1503,6 +1519,63 @@ async def _current_owned_execution(self, expected: Any) -> StoredTask | None: return None return current + async def _read_dispatch_artifact_json( + self, expected: Any, path: str, artifact: str + ) -> Any: + """Tolerate bounded shared-filesystem visibility lag before dispatch.""" + + started = time.monotonic() + misses = 0 + while True: + try: + value = await self.artifacts.read_json(path) + except FileNotFoundError as exc: + current = await self.store.get_task(expected.id) + if ( + current is None + or current.task.status in TERMINAL_STATUSES + or not self._same_execution_owner(current.task, expected) + ): + raise _TaskOwnershipLost( + "video task ownership changed while waiting for input artifacts" + ) from exc + if current.task.cancel_requested_at_ms is not None: + raise _DispatchCancelledDuringArtifactWait(expected.id) from exc + if misses >= len(_ARTIFACT_VISIBILITY_RETRY_DELAYS_S): + waited = time.monotonic() - started + self.telemetry.increment( + "dingo_video_artifact_visibility_failures_total", + labels={"pool": expected.pool_id, "artifact": artifact}, + ) + raise _ArtifactUnavailable( + f"{artifact} remained unavailable for {waited:.3f}s" + ) from exc + delay = _ARTIFACT_VISIBILITY_RETRY_DELAYS_S[misses] + misses += 1 + self.telemetry.increment( + "dingo_video_artifact_visibility_retries_total", + labels={"pool": expected.pool_id, "artifact": artifact}, + ) + await asyncio.sleep(delay) + continue + if misses: + waited = time.monotonic() - started + self.telemetry.increment( + "dingo_video_artifact_visibility_recoveries_total", + labels={"pool": expected.pool_id, "artifact": artifact}, + ) + self.telemetry.record_stage_duration( + expected.pool_id, "artifact_visibility", waited + ) + logger.warning( + "task %s %s became visible after %s retries and %.3fs", + expected.id, + artifact, + misses, + waited, + ) + return value + async def _run_reserved(self, pool: PoolRuntime, stored: StoredTask) -> None: context: Any | None = None heartbeat: asyncio.Task | None = None @@ -1525,7 +1598,9 @@ async def _run_reserved(self, pool: PoolRuntime, stored: StoredTask) -> None: ) await asyncio.sleep(0) self._raise_if_heartbeat_stopped(heartbeat) - normalized = await self.artifacts.read_json(task.request_path) + normalized = await self._read_dispatch_artifact_json( + task, task.request_path, "request" + ) if detached: if task.execution_token is None or task.attempt < 1: raise RuntimeError( @@ -1539,7 +1614,9 @@ async def _run_reserved(self, pool: PoolRuntime, stored: StoredTask) -> None: task.execution_token, ) if initial_worker_status is None: - manifest = await self.artifacts.read_json(task.input_manifest_path) + manifest = await self._read_dispatch_artifact_json( + task, task.input_manifest_path, "input_manifest" + ) payload_build_started = time.monotonic() payload = await run_file_io( pool.adapter.build_worker_payload, @@ -1946,6 +2023,22 @@ async def _consume_worker_stream() -> None: latest = await self._current_owned_execution(task) if latest is not None and latest.task.cancel_requested_at_ms is not None: await self._finish_cancelled(pool, latest, quarantine=True) + except _DispatchCancelledDuringArtifactWait: + latest = await self._current_owned_execution(task) + if latest is not None: + await self._finish_cancelled(pool, latest, quarantine=False) + except _ArtifactUnavailable as exc: + if await self._current_owned_execution(task) is None: + return + logger.error("video task input artifacts unavailable: %s", task.id) + await self._finish_failed( + pool, + task.id, + "artifact_unavailable", + str(exc), + quarantine=False, + expected_execution=task, + ) except ResultTooLarge as exc: if final_path is not None: await run_file_io(final_path.unlink, True) diff --git a/tests/video_gateway/test_dispatch_artifact_visibility.py b/tests/video_gateway/test_dispatch_artifact_visibility.py new file mode 100644 index 000000000000..2f13772242a7 --- /dev/null +++ b/tests/video_gateway/test_dispatch_artifact_visibility.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest + +from dingo.video_gateway import dispatcher as dispatcher_module +from dingo.video_gateway.dispatcher import VideoDispatcher +from dingo.video_gateway.models import TaskStatus + + +def _task(*, cancelled=False): + return SimpleNamespace( + id="video-test", + pool_id="pool", + status=TaskStatus.DISPATCHING, + owner_generation="gateway-a", + attempt=1, + execution_token="token", + cancel_requested_at_ms=1 if cancelled else None, + ) + + +def _dispatcher(expected, reads, current=None): + value = object.__new__(VideoDispatcher) + value.artifacts = SimpleNamespace(read_json=AsyncMock(side_effect=reads)) + value.store = SimpleNamespace( + get_task=AsyncMock( + return_value=SimpleNamespace(task=current or expected, revision=1) + ) + ) + value.telemetry = SimpleNamespace( + increment=Mock(), record_stage_duration=Mock() + ) + return value + + +@pytest.mark.asyncio +async def test_dispatch_artifact_visibility_recovers(monkeypatch): + monkeypatch.setattr( + dispatcher_module, "_ARTIFACT_VISIBILITY_RETRY_DELAYS_S", (0, 0) + ) + task = _task() + value = _dispatcher( + task, + [FileNotFoundError(), FileNotFoundError(), {"prompt": "ready"}], + ) + + result = await value._read_dispatch_artifact_json( + task, "/artifacts/request.json", "request" + ) + + assert result == {"prompt": "ready"} + assert value.artifacts.read_json.await_count == 3 + assert value.telemetry.increment.call_count == 3 + value.telemetry.record_stage_duration.assert_called_once() + + +@pytest.mark.asyncio +async def test_dispatch_artifact_visibility_exhaustion_is_not_worker_failure( + monkeypatch, +): + monkeypatch.setattr( + dispatcher_module, "_ARTIFACT_VISIBILITY_RETRY_DELAYS_S", (0,) + ) + task = _task() + value = _dispatcher(task, FileNotFoundError()) + + with pytest.raises(dispatcher_module._ArtifactUnavailable): + await value._read_dispatch_artifact_json( + task, "/artifacts/request.json", "request" + ) + + assert value.artifacts.read_json.await_count == 2 + value.telemetry.record_stage_duration.assert_not_called() + + +@pytest.mark.asyncio +async def test_dispatch_artifact_visibility_stops_for_cancellation(monkeypatch): + monkeypatch.setattr( + dispatcher_module, "_ARTIFACT_VISIBILITY_RETRY_DELAYS_S", (0,) + ) + task = _task() + current = _task(cancelled=True) + value = _dispatcher(task, FileNotFoundError(), current=current) + + with pytest.raises(dispatcher_module._DispatchCancelledDuringArtifactWait): + await value._read_dispatch_artifact_json( + task, "/artifacts/request.json", "request" + ) + + assert value.artifacts.read_json.await_count == 1 From 95d1a99e58cd162e979df7253c85c23c88f4fc28 Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Wed, 23 Sep 2026 17:36:52 +0800 Subject: [PATCH 11/12] [fix][video-gateway] avoid optional result media reads --- dingo/video_gateway/adapters/minimax_h3.py | 167 ++++++++++++++---- .../test_media_timing_diagnostics.py | 139 +++++++++++++++ .../video_gateway/test_minimax_h3_adapter.py | 17 ++ 3 files changed, 293 insertions(+), 30 deletions(-) create mode 100644 tests/video_gateway/test_media_timing_diagnostics.py diff --git a/dingo/video_gateway/adapters/minimax_h3.py b/dingo/video_gateway/adapters/minimax_h3.py index d83de3cf9e8b..054f7fccffa5 100644 --- a/dingo/video_gateway/adapters/minimax_h3.py +++ b/dingo/video_gateway/adapters/minimax_h3.py @@ -9,9 +9,11 @@ import hashlib import importlib.util import json +import logging import math import os import secrets +import time from collections.abc import Mapping, Sequence from contextlib import contextmanager from dataclasses import dataclass @@ -45,10 +47,21 @@ "audio/vnd.wave": "audio/wav", } _GENERIC_BINARY_TYPES = {"", "application/octet-stream", "binary/octet-stream"} +_MEDIA_TIMING_ENABLED = os.environ.get("DINGO_VIDEO_MEDIA_TIMING") == "1" +_LOGGER = logging.getLogger(__name__) + + +def _record_media_timing( + timings: dict[str, Any] | None, stage: str, started: float +) -> None: + if timings is not None: + timings[f"{stage}_s"] = round(time.perf_counter() - started, 6) @contextmanager -def _open_artifact_metadata(path: Path): +def _open_artifact_metadata( + path: Path, timings: dict[str, Any] | None = None +): """Avoid pixel reconstruction while probing indexed H.264/AAC results. This is metadata validation, not a full bitstream/decode integrity scan. @@ -60,7 +73,12 @@ def _open_artifact_metadata(path: Path): container = None try: try: - container = av.open(str(path), options={"skip_frame": "all"}) + started = time.perf_counter() + try: + container = av.open(str(path), options={"skip_frame": "all"}) + finally: + _record_media_timing(timings, "indexed_open", started) + started = time.perf_counter() videos = list(container.streams.video) audios = list(container.streams.audio) complete = ( @@ -80,20 +98,33 @@ def _open_artifact_metadata(path: Path): for a in audios ) ) + _record_media_timing(timings, "indexed_metadata", started) except av.error.FFmpegError: complete = False + if timings is not None: + timings["indexed_fast_path"] = complete if not complete: if container is not None: + started = time.perf_counter() container.close() - container = av.open(str(path)) + _record_media_timing(timings, "indexed_close", started) + started = time.perf_counter() + try: + container = av.open(str(path)) + finally: + _record_media_timing(timings, "fallback_open", started) else: + started = time.perf_counter() for stream in container.streams: stream.codec_context.options.pop("skip_frame", None) stream.codec_context.skip_frame = "DEFAULT" + _record_media_timing(timings, "decode_flags_reset", started) yield container finally: if container is not None: + started = time.perf_counter() container.close() + _record_media_timing(timings, "container_close", started) _HEIF_BRANDS = {b"heic", b"heix", b"hevc", b"hevx", b"heim", b"heis", b"mif1", b"msf1"} @@ -717,7 +748,10 @@ def __init__(self, pool: PoolConfig) -> None: raise ValueError( "unknown MiniMax-H3 adapter options: " + ", ".join(unknown_options) ) - if not isinstance(self.options.get("validate_media", True), bool): + # Media-content reads can exhibit large shared-filesystem tails. Keep + # them opt-in; descriptor/path/stat/size publication checks remain in + # the artifact store when this is disabled. + if not isinstance(self.options.setdefault("validate_media", False), bool): raise TypeError("MiniMax-H3 validate_media must be a boolean") raw_limits = self.options.get("limits", {}) if not isinstance(raw_limits, Mapping): @@ -1328,25 +1362,75 @@ def inspect_artifact_for_publication( The caller may reuse this media result only when publishing the same checked inode. Copies and transformed outputs must be validated anew. """ - if bool(normalized.get("generate_sound", True)): - return False, self.validate_artifact(path, normalized) + timings: dict[str, Any] | None = {} if _MEDIA_TIMING_ENABLED else None + started_total = time.perf_counter() + generate_sound = bool(normalized.get("generate_sound", True)) + needs_processing: bool | None = None + outcome = "completed" try: - import av # noqa: F401 - fail early with the optional-extra diagnostic - except ImportError as exc: - raise RuntimeError( - "MiniMax-H3 media inspection requires the video-gateway optional extra" - ) from exc - with path.open("rb") as stream: - header = stream.read(32) - with _open_artifact_metadata(path) as container: - if not ( - len(container.streams) == 1 - and len(container.streams.video) == 1 - and len(header) >= 12 - and header[4:8] == b"ftyp" - ): - return True, None - return False, self._validate_open_artifact(container, normalized) + if generate_sound: + needs_processing = False + if not bool(self.options.get("validate_media", False)): + return False, {"container": "mp4"} + return False, self.validate_artifact( + path, normalized, diagnostic_timings=timings + ) + started = time.perf_counter() + try: + import av # noqa: F401 - fail early with the optional-extra diagnostic + except ImportError as exc: + raise RuntimeError( + "MiniMax-H3 media inspection requires the video-gateway optional extra" + ) from exc + _record_media_timing(timings, "av_import", started) + started = time.perf_counter() + with path.open("rb") as stream: + header = stream.read(32) + _record_media_timing(timings, "header_read", started) + with _open_artifact_metadata(path, timings) as container: + started = time.perf_counter() + needs_processing = not ( + len(container.streams) == 1 + and len(container.streams.video) == 1 + and len(header) >= 12 + and header[4:8] == b"ftyp" + ) + _record_media_timing(timings, "stream_topology", started) + if needs_processing: + return True, None + return False, self._validate_open_artifact( + container, normalized, diagnostic_timings=timings + ) + except BaseException as exc: + outcome = type(exc).__name__ + raise + finally: + if timings is not None: + try: + timings.update( + task_id=next( + ( + part + for part in path.parts + if part.startswith("video-") and len(part) == 32 + ), + None, + ), + generate_sound=generate_sound, + validate_media=bool( + self.options.get("validate_media", False) + ), + needs_processing=needs_processing, + outcome=outcome, + total_s=round(time.perf_counter() - started_total, 6), + ) + _LOGGER.info( + "video_media_inspection_timing %s", + json.dumps(timings, sort_keys=True, separators=(",", ":")), + ) + except Exception: + # Even a broken diagnostic must not change result publication. + pass def artifact_requires_processing( self, path: Path, normalized: Mapping[str, Any] @@ -1413,32 +1497,47 @@ def prepare_artifact(self, path: Path, normalized: Mapping[str, Any]) -> None: remuxed.unlink() def validate_artifact( - self, path: Path, normalized: Mapping[str, Any] + self, + path: Path, + normalized: Mapping[str, Any], + diagnostic_timings: dict[str, Any] | None = None, ) -> dict[str, Any]: + if not bool(self.options.get("validate_media", False)): + return {"container": "mp4"} + started = time.perf_counter() with path.open("rb") as stream: header = stream.read(32) + _record_media_timing(diagnostic_timings, "header_read", started) if len(header) < 12 or header[4:8] != b"ftyp": raise RuntimeError("Worker result is not an ISO BMFF/MP4 file") - if not bool(self.options.get("validate_media", True)): - return {"container": "mp4"} + started = time.perf_counter() try: import av # noqa: F401 - fail early with the optional-extra diagnostic except ImportError as exc: raise RuntimeError( "MiniMax-H3 media validation requires the video-gateway optional extra" ) from exc - with _open_artifact_metadata(path) as container: - return self._validate_open_artifact(container, normalized) + _record_media_timing(diagnostic_timings, "av_import", started) + with _open_artifact_metadata(path, diagnostic_timings) as container: + return self._validate_open_artifact( + container, normalized, diagnostic_timings=diagnostic_timings + ) def _validate_open_artifact( - self, container, normalized: Mapping[str, Any] + self, + container, + normalized: Mapping[str, Any], + diagnostic_timings: dict[str, Any] | None = None, ) -> dict[str, Any]: - if not bool(self.options.get("validate_media", True)): + if not bool(self.options.get("validate_media", False)): return {"container": "mp4"} import av + started_validation = time.perf_counter() + started = time.perf_counter() video_streams = list(container.streams.video) audio_streams = list(container.streams.audio) + _record_media_timing(diagnostic_timings, "validation_streams", started) if not video_streams or video_streams[0].codec_context.name != "h264": raise RuntimeError("MP4 must contain an H.264 video stream") generate_sound = bool(normalized.get("generate_sound", True)) @@ -1457,7 +1556,9 @@ def _validate_open_artifact( raise RuntimeError(f"MP4 frame rate {average_rate} does not match 24 fps") frame_count = int(video.frames or 0) if frame_count <= 0: + started = time.perf_counter() frame_count = sum(1 for _frame in container.decode(video=0)) + _record_media_timing(diagnostic_timings, "frame_count_decode", started) expected_frames = align_frame_count(int(normalized["num_frames"])) if frame_count != expected_frames: raise RuntimeError( @@ -1473,14 +1574,16 @@ def _validate_open_artifact( audio_duration: float | None = None if generate_sound: audio = audio_streams[0] + started = time.perf_counter() audio_duration = _stream_duration(container, audio) + _record_media_timing(diagnostic_timings, "audio_duration", started) if audio_duration <= 0: raise RuntimeError("MP4 AAC stream has no measurable duration") if abs(video_duration - audio_duration) > 0.1: raise RuntimeError( "MP4 audio/video duration difference exceeds 100 milliseconds" ) - return { + result = { "container": "mp4", "video_codec": "h264", "audio_codec": "aac" if generate_sound else None, @@ -1494,3 +1597,7 @@ def _validate_open_artifact( "video_duration_s": video_duration, "audio_duration_s": audio_duration, } + _record_media_timing( + diagnostic_timings, "validation_total", started_validation + ) + return result diff --git a/tests/video_gateway/test_media_timing_diagnostics.py b/tests/video_gateway/test_media_timing_diagnostics.py new file mode 100644 index 000000000000..a51ac90ffcc0 --- /dev/null +++ b/tests/video_gateway/test_media_timing_diagnostics.py @@ -0,0 +1,139 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. +# SPDX-License-Identifier: Apache-2.0 + +"""Diagnostic media timing must never change artifact inspection results.""" + +import json +import sys +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from dingo.video_gateway.adapters import minimax_h3 + + +class _FakeAdapter: + options = {"validate_media": True} + + def validate_artifact(self, path, normalized, diagnostic_timings=None): + return {"container": "mp4"} + + def _validate_open_artifact(self, container, normalized, diagnostic_timings=None): + return {"container": "mp4"} + + +class _FakeContainer: + def __init__(self, *, with_audio): + video = SimpleNamespace( + width=256, + height=256, + average_rate=24, + frames=124, + codec_context=SimpleNamespace( + name="h264", extradata=b"x", options={}, skip_frame="ALL" + ), + ) + audio = SimpleNamespace( + duration=124, + time_base=1 / 24, + codec_context=SimpleNamespace( + name="aac", extradata=b"x", options={}, skip_frame="ALL" + ), + ) + items = [video, audio] if with_audio else [video] + self.streams = _FakeStreams(items) + + def close(self): + pass + + +class _FakeStreams: + def __init__(self, items): + self.items = items + self.video = items[:1] + self.audio = items[1:] + + def __len__(self): + return len(self.items) + + def __iter__(self): + return iter(self.items) + + +class MediaTimingDiagnosticTests(unittest.TestCase): + def test_sound_request_with_validation_disabled_never_reads_media(self): + adapter = _FakeAdapter() + adapter.options = {"validate_media": False} + path = Path("/tmp/video-01M36AFXE5GZQPEVRFQANDH7R9/result.mp4") + with ( + patch.object(minimax_h3, "_MEDIA_TIMING_ENABLED", True), + patch.object(adapter, "validate_artifact") as validate, + patch.object(minimax_h3._LOGGER, "info") as logged, + ): + result = minimax_h3.MiniMaxH3VideoAdapter.inspect_artifact_for_publication( + adapter, path, {"generate_sound": True} + ) + self.assertEqual(result, (False, {"container": "mp4"})) + validate.assert_not_called() + payload = json.loads(logged.call_args.args[1]) + self.assertFalse(payload["validate_media"]) + self.assertNotIn("header_read_s", payload) + + def test_direct_validation_disabled_never_opens_media(self): + adapter = _FakeAdapter() + adapter.options = {"validate_media": False} + path = Path("/tmp/video-01M36AFXE5GZQPEVRFQANDH7R9/result.mp4") + with patch.object(Path, "open", side_effect=AssertionError("media read")): + result = minimax_h3.MiniMaxH3VideoAdapter.validate_artifact( + adapter, path, {"generate_sound": True} + ) + self.assertEqual(result, {"container": "mp4"}) + + def test_sound_request_preserves_result_when_logging_fails(self): + adapter = _FakeAdapter() + path = Path("/tmp/video-01M36AFXE5GZQPEVRFQANDH7R9/result.mp4") + with ( + patch.object(minimax_h3, "_MEDIA_TIMING_ENABLED", True), + patch.object(minimax_h3.json, "dumps", side_effect=RuntimeError("log failed")), + ): + result = minimax_h3.MiniMaxH3VideoAdapter.inspect_artifact_for_publication( + adapter, path, {"generate_sound": True} + ) + self.assertEqual(result, (False, {"container": "mp4"})) + + def test_silent_request_logs_fast_open_and_processing_decision(self): + adapter = _FakeAdapter() + fake_av = SimpleNamespace( + open=lambda *_args, **_kwargs: _FakeContainer(with_audio=True), + error=SimpleNamespace(FFmpegError=RuntimeError), + ) + with tempfile.TemporaryDirectory() as directory: + task_root = Path(directory) / "video-01M36AFXE5GZQPEVRFQANDH7R9" + task_root.mkdir() + path = task_root / "result.mp4" + path.write_bytes(b"\x00\x00\x00\x18ftypisom" + b"\x00" * 20) + with ( + patch.dict(sys.modules, {"av": fake_av}), + patch.object(minimax_h3, "_MEDIA_TIMING_ENABLED", True), + patch.object(minimax_h3._LOGGER, "info") as logged, + ): + result = ( + minimax_h3.MiniMaxH3VideoAdapter.inspect_artifact_for_publication( + adapter, path, {"generate_sound": False} + ) + ) + self.assertEqual(result, (True, None)) + payload = json.loads(logged.call_args.args[1]) + self.assertEqual(payload["task_id"], task_root.name) + self.assertFalse(payload["generate_sound"]) + self.assertTrue(payload["needs_processing"]) + self.assertTrue(payload["indexed_fast_path"]) + self.assertIn("indexed_open_s", payload) + self.assertIn("indexed_metadata_s", payload) + self.assertIn("stream_topology_s", payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/video_gateway/test_minimax_h3_adapter.py b/tests/video_gateway/test_minimax_h3_adapter.py index a1eb3a27502b..6b4923c48f00 100644 --- a/tests/video_gateway/test_minimax_h3_adapter.py +++ b/tests/video_gateway/test_minimax_h3_adapter.py @@ -635,6 +635,23 @@ def test_pool_limits_can_only_tighten_adapter_capability(make_gateway_config): MiniMaxH3VideoAdapter(make_gateway_config(pools=[pool]).pools[0]) +def test_media_validation_is_opt_in(make_gateway_config): + pool = { + "pool_id": "default-media-policy", + "served_models": ["public-fl"], + "backend_model": "worker-fl", + "backend_target": "dyn://scope.backend.generate", + "adapter": { + "name": "minimax_h3", + "workflow": "fl2va", + }, + } + + adapter = MiniMaxH3VideoAdapter(make_gateway_config(pools=[pool]).pools[0]) + + assert adapter.options["validate_media"] is False + + def test_ref2va_capabilities_only_advertise_verified_media(make_gateway_config): capabilities = _adapter(make_gateway_config, "ref2va").capabilities( max_result_bytes=128 * 1024 * 1024 From 4e8c3242f8b8236311b21a443f2323d9f5512454 Mon Sep 17 00:00:00 2001 From: dingjianshan Date: Wed, 23 Sep 2026 17:37:04 +0800 Subject: [PATCH 12/12] [docs][video-gateway] document API and lease invariants --- .../video_gateway/LEASE_TIMING_INVARIANTS.md | 45 ++ docs/video-gateway/README.md | 94 ++++ docs/video-gateway/video-gateway-api.md | 418 ++++++++++++++++++ .../video-gateway-protocol-compatibility.md | 187 ++++++++ 4 files changed, 744 insertions(+) create mode 100644 dingo/video_gateway/LEASE_TIMING_INVARIANTS.md create mode 100644 docs/video-gateway/README.md create mode 100644 docs/video-gateway/video-gateway-api.md create mode 100644 docs/video-gateway/video-gateway-protocol-compatibility.md diff --git a/dingo/video_gateway/LEASE_TIMING_INVARIANTS.md b/dingo/video_gateway/LEASE_TIMING_INVARIANTS.md new file mode 100644 index 000000000000..e1f4940d5ed9 --- /dev/null +++ b/dingo/video_gateway/LEASE_TIMING_INVARIANTS.md @@ -0,0 +1,45 @@ +# Worker 租约时间常量耦合说明 + +本文记录 Worker 执行租约相关的时间常量及其联动约束。这些值当前固定且相互匹配， +不计划调整；本文的目的不是推动修改，而是防止未来对其中某一个值做孤立改动。 + +## 常量清单 + +| 常量 | 当前值 | 位置 | 作用 | +|---|---|---|---| +| `_GATEWAY_OWNER_TTL_S` | 15s | `dispatcher.py` | Gateway owner 租约 TTL；keepalive 每 TTL/3=5s；连续失败累计满 TTL 触发 fatal restart | +| `_WORKER_LEASE_HEARTBEAT_INTERVAL_S` | 5s | `dispatcher.py` | Worker 执行租约的心跳间隔 | +| 心跳失败阈值 | 2 次 | `dispatcher.py` `_heartbeat` | 连续失败 2 次判 `_WorkerLeaseLost`，停止本地执行 | +| `owner_expires_at_ms` 记账 | `now + 15_000` | `dispatcher.py` reserve/recovery 两处；`task_store.py` 两个 `heartbeat_lease` | lease 记录上的逻辑过期时间（纯记账字段） | +| `execution_lease_ttl_s` | 15（下限 5） | `task_store.py` `EtcdTaskStore.__init__` | 原生 etcd 租约 TTL；heartbeat key 挂在它上面。当前未暴露为 Gateway 配置项，只能改代码 | +| `_DETACHED_WORKER_STALE_S` | 20s | `dispatcher.py` | Worker 状态文件陈旧判定（Worker 每 5s 刷新，容许丢 4 拍） | +| `scheduling.abort_grace_s` | 15s | `config.py` | 取消确认宽限；quarantine 复用时点 = 任务 deadline + 该宽限 | + +## 必须保持的不变量 + +1. **止损先于复用**：心跳间隔（5s）× 失败阈值（2）= 10s，必须小于原生租约 TTL（15s）。 + 当前余量 5s。含义：本地执行必须在 slot 可能被他人复用之前停止。TTL 降到 10s 及以下 + 会出现"租约已过期被复用、本 Gateway 仍在心跳重试"的窗口，同一 slot 可能双重占用。 +2. **keepalive 节奏装进 owner TTL**：Gateway owner 每 TTL/3（5s）keepalive，单次 etcd + 请求超时（默认 5s）与该节奏叠加后仍须在 TTL 内留有余量。 +3. **HA 接管时延下限 = TTL**：`claim_orphaned_active` 要求 heartbeat key 已随原生租约 + 自然消失才允许接管。TTL 调大则故障接管和 owner fatal restart 都变慢，恢复 SLA 变差； + 调小则先破坏不变量 1。 +4. **相邻计时器一并复核**：Worker 状态文件心跳（5s/陈旧 20s）与 `abort_grace_s`（15s） + 不由 TTL 推导，但调整 TTL 或心跳参数时应一并复核。 + +## 修改 `execution_lease_ttl_s` 时的联动清单 + +- 调小（如 5s）：先破坏不变量 1（止损 10s > 租约 5s），产生 slot 双卖窗口；同时 + `dispatcher.py` 与 `task_store.py` 中硬编码的 `owner_expires_at_ms = now + 15_000` + 记账与真实租约脱节。注意 `claim_orphaned_active` 使用的是该参数本身，而两个 + `heartbeat_lease` 实现和 dispatcher 的 reserve/recovery 路径是硬编码 15s，不会随 + 参数自动变化。 +- 调大（如 60s）：不变量 1 更安全，但 HA 接管与 owner fatal restart 时延同步变大。 +- 任一方向都必须同步复核：心跳间隔与失败阈值、owner keepalive、四处 + `owner_expires_at_ms` 写入点、Worker 状态文件陈旧阈值、取消宽限。 + +## 当前决策 + +2026-09-21 确认：上述值相互匹配且已经过故障注入验证，保持现状不做参数化推导。 +如确需调整，按上节清单整体联动修改并重新做 Worker 失联、HA 接管故障注入验证。 diff --git a/docs/video-gateway/README.md b/docs/video-gateway/README.md new file mode 100644 index 000000000000..4cf1adfb595c --- /dev/null +++ b/docs/video-gateway/README.md @@ -0,0 +1,94 @@ +# Video Gateway + +Dingo Video Gateway 是面向视频生成业务的持久化任务入口。它位于客户端与 +vLLM-Omni Worker 之间，为耗时较长、结果文件较大的视频任务提供统一的提交、排队、 +状态查询、取消、故障恢复和制品下载接口。 + +它不是模型推理引擎。模型执行仍由后端 Worker 完成；Gateway 负责把一次 HTTP 请求转换为 +可持久化、可恢复、可观测的任务，并协调多个 Gateway 和多个 Worker 安全地处理同一套队列。 + +## 核心能力 + +- 同时提供异步任务接口和同步 MP4 返回接口，两种模式共用同一套任务生命周期。 +- 使用 etcd 保存任务、队列、Worker lease、重试额度和 Gateway owner 信息，支持双 Gateway + 共同服务及故障接管。 +- 使用共享制品存储保存上传素材和生成结果，使任一 Gateway 都能查询和下载已完成任务。 +- 支持请求幂等，客户端可在连接中断或响应丢失后安全重试提交。 +- 区分 Gateway 排队、Worker 预取等待、Worker 执行和 Gateway 后处理阶段。 +- Worker 明确失联时可按策略重试；参数、媒体或模型错误不会被无差别重试。 +- 支持取消、结果过期清理、HTTP Range、ETag 和 Prometheus 指标。 +- 当前 MiniMax-H3 适配器支持 FL2VA 和 Ref2VA 两个独立的 Worker pool。 + +## 组件关系 + +```mermaid +flowchart LR + C[客户端或上层应用] --> LB[LoadBalancer] + LB --> G1[Gateway 1] + LB --> G2[Gateway 2] + G1 <--> E[(etcd)] + G2 <--> E + G1 --> W1[FL2VA Worker pool] + G1 --> W2[Ref2VA Worker pool] + G2 --> W1 + G2 --> W2 + G1 <--> A[(共享制品存储)] + G2 <--> A + W1 <--> A + W2 <--> A +``` + +etcd 保存协调状态，不保存视频内容。上传文件、中间任务目录和最终 MP4 位于共享制品存储。 +多 Gateway 部署要求所有 Gateway 和 Worker 看到同一份制品目录。 + +## 任务生命周期 + +```text +queued -> in_progress -> completed + -> failed + -> cancelled +``` + +公开 API 将内部的 `dispatching` 和 `finalizing` 都显示为 `in_progress`。因此 +`in_progress` 可能表示正在下发、位于 Worker 预取队列、正在推理，或者 Gateway 正在发布 +最终制品。具体阶段耗时可从单任务响应的 `metrics` 和 `stage_durations` 查看。 + +任务完成后结果不会永久保留。生命周期清理会把到期任务转为 `expired` 并释放制品空间； +对终态任务执行 DELETE 也会触发结果清理。 + +## 如何选择接口 + +- 业务系统通常应使用 `POST /v1/videos`：提交后保存任务 ID，通过状态接口轮询，完成后下载。 + 该模式能容忍客户端断线、长时间推理和 Gateway 滚动重启。 +- 命令行验证或调用方必须直接接收文件时，可使用 `POST /v1/videos/sync`。同步请求超时后 + 任务不会消失，仍可使用响应头中的任务 ID 继续查询。 +- 每次业务请求都建议提供稳定的 `Idempotency-Key`。重试同一请求时复用原 key，创建新任务 + 时使用新 key。 + +完整的请求字段、媒体限制、curl 示例、响应结构和错误处理见 +[接口使用说明](video-gateway-api.md)。 + +## 文档导航 + +| 文档 | 内容 | 面向对象 | +|---|---|---| +| [接口使用说明](video-gateway-api.md) | 模型查询、任务提交、状态、下载、取消、FL2VA/Ref2VA 输入和错误码 | 接口调用方、联调人员 | +| [协议与兼容性](video-gateway-protocol-compatibility.md) | 与 OpenAI Videos API、vLLM-Omni Videos API 的相同点和差异 | 接口调用方、适配开发人员 | +| [`CONTINUOUS_EXECUTION.md`](../../dingo/video_gateway/CONTINUOUS_EXECUTION.md) | Worker slot 提前释放、预取和连续执行的状态约束 | Gateway 开发与评审人员 | +| [`GATEWAY_CONFIGURATION_CONSISTENCY.md`](../../dingo/video_gateway/GATEWAY_CONFIGURATION_CONSISTENCY.md) | 双 Gateway 配置哪些必须一致、哪些可在滚动升级期间短暂不同 | 部署和运维人员 | + +Kubernetes 部署 recipe 和回归测试维护在独立的 +`dingo-video-gateway-regression` 项目中，接口文档不复制具体集群地址、PVC、节点或镜像配置。 + +## 当前边界 + +- Gateway 当前不实现最终用户认证、租户鉴权和业务配额；这些能力由上层应用或入口网关负责。 +- `/ready` 表示 Gateway、任务存储和制品存储可用，不代表每个模型都有已注册 Worker。 + 模型可用性应查询 `/v1/models` 中的 `available` 字段。 +- `available=true` 表示至少发现一个 Worker，不表示存在空闲执行名额；满载时任务会排队， + 队列达到上限时返回 `429 queue_full`。 +- 当前不提供逐去噪步的稳定进度百分比；`progress` 仅在完成时为 100，其他状态为 0。 +- `POST /v1/videos/stream` 尚未实现。 + +实现入口位于 `dingo/video_gateway/`，主要模块包括 `api.py`、`service.py`、`dispatcher.py`、 +`task_store.py`、`artifact_store.py` 和 `adapters/minimax_h3.py`。 diff --git a/docs/video-gateway/video-gateway-api.md b/docs/video-gateway/video-gateway-api.md new file mode 100644 index 000000000000..1c810edd4d4e --- /dev/null +++ b/docs/video-gateway/video-gateway-api.md @@ -0,0 +1,418 @@ +# Video Gateway 接口使用说明 + +本文说明 Dingo Video Gateway 当前提供的视频生成接口。示例使用 MiniMax-H3 +双模型部署中的模型名： + +- `minimax-h3`：FL2VA，可执行纯文本生成视频，也可使用一张或两张关键帧图片。 +- `minimax-h3-ref2va`：Ref2VA，使用图片、视频和可选音频作为参考素材。 + +实际模型名、可用状态和媒体限制应以目标环境的 `GET /v1/models` 返回值为准。 + +## 快速开始 + +先设置 Gateway 地址并查询模型： + +```bash +export GATEWAY_URL=http://:8000 +curl -sS "$GATEWAY_URL/v1/models" | jq +``` + +提交一个异步 FL2VA 文生视频任务： + +```bash +curl -sS -X POST "$GATEWAY_URL/v1/videos" \ + -H 'Idempotency-Key: example-fl2va-001' \ + -F model=minimax-h3 \ + -F 'prompt=A cinematic sunrise over snow mountains' \ + -F seconds=5 \ + -F size=1344x768 \ + -F fps=24 \ + -F num_inference_steps=50 \ + -F seed=1101 +``` + +成功后返回任务 JSON，响应头 `Location` 指向任务查询地址。保存返回的 `id`： + +```bash +export TASK_ID= +curl -sS "$GATEWAY_URL/v1/videos/$TASK_ID" | jq +``` + +任务完成后下载 MP4： + +```bash +curl -fL "$GATEWAY_URL/v1/videos/$TASK_ID/content" \ + -o "$TASK_ID.mp4" +``` + +## 接口列表 + +| 方法 | 路径 | 用途 | +|---|---|---| +| `GET` | `/v1/models` | 查询模型、可用状态和媒体能力 | +| `POST` | `/v1/videos` | 异步提交任务，返回任务 JSON | +| `POST` | `/v1/videos/sync` | 同步提交任务，成功时直接返回 MP4 | +| `GET` | `/v1/videos/{task_id}` | 查询单个任务状态和阶段耗时 | +| `GET` | `/v1/videos` | 分页查询任务列表 | +| `GET`/`HEAD` | `/v1/videos/{task_id}/content` | 下载或检查生成结果 | +| `DELETE` | `/v1/videos/{task_id}` | 取消活跃任务，或删除终态任务制品 | +| `GET` | `/live` | 进程及 Gateway owner lease 存活检查 | +| `GET` | `/ready`、`/health` | 服务就绪检查 | +| `GET` | `/metrics` | Prometheus 文本指标 | + +`POST /v1/videos/stream` 当前未实现，固定返回 `404 unsupported_endpoint`。 +`/ready` 表示 Gateway 自身、任务存储和制品存储可用，不要求每个模型都有 Worker。 +模型是否至少发现一个已注册 Worker，应查看 `/v1/models` 中对应项的 `available`； +`available=true` 也不表示 Worker 当前空闲，满载时任务仍可能进入队列。 + +## 提交请求 + +视频提交接口只接受 `multipart/form-data`。普通参数是文本 part，参考文件使用 +`input_reference` 或可重复的 `input_references` 文件 part。 + +### 通用参数 + +| 参数 | 必填 | 默认值 | 约束 | +|---|---:|---|---| +| `model` | 未配置默认模型且有多个视频模型时必填 | 配置中的默认模型；仅有一个视频模型时可自动选用 | 必须属于 Gateway 已配置的 Worker pool；`/v1/models` 可能还列出不能用于视频提交的上游模型 | +| `prompt` | 是 | 无 | UTF-8，不超过 32 KiB | +| `seconds` | 三选一 | 无 | 与 `num_frames`、`extra_params.duration` 三选一；范围 4–15 秒 | +| `num_frames` | 三选一 | 无 | 范围 96–360；与另外两种时长写法互斥 | +| `size` | 二选一 | 无 | `WIDTHxHEIGHT`，也可改用 `width` 与 `height` | +| `width`、`height` | 二选一 | 无 | 必须同时出现，不能与 `size` 同时使用 | +| `aspect_ratio` | 否 | 根据尺寸推导 | `21:9`、`16:9`、`4:3`、`1:1`、`3:4`、`9:16`，必须与尺寸一致 | +| `fps` | 否 | `24` | MiniMax-H3 当前只接受 24 | +| `num_inference_steps` | 否 | `50` | 2–200；别名为 `steps`，两者不能同时提供 | +| `guidance_scale` | 否 | 模型默认值 | 0–20 | +| `seed` | 否 | 随机生成 | 无符号 32 位整数；最终 seed 会在任务状态中返回 | +| `negative_prompt` | 否 | 无 | UTF-8，不超过 32 KiB | +| `generate_sound` | 否 | `true` | 接受 `true/false/1/0` | +| `output_format` | 否 | `mp4` | 当前只支持 `mp4` | +| `num_outputs_per_prompt` | 否 | `1` | 当前只支持 1 | +| `user` | 否 | 无 | 记录在规范化请求中的调用方标签；不用于认证 | + +输出尺寸必须同时满足： + +- 宽和高各为 256–2048，且都是 32 的倍数； +- 总像素不超过 `768 × 1344`； +- 宽高比与支持的命名比例之一相差不超过 5%。 + +`task`、`flow_shift`、`audio_flow_shift` 和 `extra_params` 主要用于兼容底层 +vLLM-Omni 调用。普通调用建议省略，由 Gateway 按目标 pool 填充。 +`quality`、`image_reference`、`video_reference` 和 `audio_reference` 等字段当前不支持； +参考文件统一使用下文的 `input_reference(s)`。 + +默认配置还限制单次请求的参考文件合计不超过 256 MiB、multipart part 不超过 32 个； +部署可以把这些限制调低。`GET /v1/models` 中的 `video_capabilities` 会返回模型适配器的 +格式、单文件大小、参考数量和结果大小上限。 + +### 幂等提交 + +建议每次业务请求携带稳定且唯一的 `Idempotency-Key`： + +```text +Idempotency-Key: <1 至 256 字节的业务请求键> +``` + +在原任务及其幂等记录仍保留、且请求所用 pool 配置与适配器兼容版本未变化时， +相同 key 和相同请求会返回原任务，不会重复生成。请求摘要包含这些版本信息： +滚动升级或配置变更期间， +即使 HTTP 请求内容相同，也可能返回 `409 idempotency_conflict`。原任务及其记录被清理后， +再次使用相同 key 可能创建新任务。网络超时后仍应首先保持 key 和请求内容不变进行重试； +若遇到 409，应查询已知任务 ID 或核对部署配置，不要盲目改用新 key 提交。 + +### FL2VA 请求 + +FL2VA 支持三种输入： + +| 输入 | 文件字段 | 行为 | +|---|---|---| +| 无参考图 | 不上传文件 | 文本生成视频，内部任务为 `t2va` | +| 一张图片 | `input_reference` 或一个 `input_references` | 默认作为首帧；可用 `frame_indices=[-1]` 指定为尾帧 | +| 两张图片 | 两个 `input_references` | 分别作为首帧和尾帧，`frame_indices` 为 `[0,-1]` | + +支持 JPEG、PNG、WebP。每张图片不超过 30 MiB，宽高各为 256–5760，宽高比范围 +0.4–2.5。两张关键帧尺寸必须相同；输出尺寸的宽高比必须与关键帧基本一致。 + +一张首帧图片示例： + +```bash +curl -sS -X POST "$GATEWAY_URL/v1/videos" \ + -H 'Idempotency-Key: example-fl2va-image-001' \ + -F model=minimax-h3 \ + -F 'prompt=The camera slowly moves forward' \ + -F seconds=5 \ + -F size=1344x768 \ + -F num_inference_steps=50 \ + -F 'input_reference=@/path/to/first.png;type=image/png' +``` + +首尾两张图片示例： + +```bash +curl -sS -X POST "$GATEWAY_URL/v1/videos" \ + -H 'Idempotency-Key: example-fl2va-two-images-001' \ + -F model=minimax-h3 \ + -F 'prompt=Transition naturally between the two scenes' \ + -F seconds=5 \ + -F size=1344x768 \ + -F num_inference_steps=50 \ + -F 'frame_indices=[0,-1]' \ + -F 'input_references=@/path/to/first.png;type=image/png' \ + -F 'input_references=@/path/to/last.png;type=image/png' +``` + +### Ref2VA 请求 + +Ref2VA 至少需要一张图片或一个视频；音频不能单独使用。文件按 multipart 中出现的顺序 +传给模型。可组合： + +- 最多 9 张图片； +- 最多 3 个视频； +- 最多 3 个音频； +- 所有参考文件合计最多 12 个； +- 参考视频总时长不超过 15 秒，参考音频总时长不超过 15 秒。 + +媒体限制如下： + +| 类型 | 格式/编码 | 单文件大小 | 其他限制 | +|---|---|---:|---| +| 图片 | JPEG、PNG、WebP | 30 MiB | 宽高 256–5760，宽高比 0.4–2.5 | +| 视频 | MP4/MOV，H.264 或 H.265；如含音轨须为 AAC | 50 MiB | 2–15 秒，23.976–60 FPS，宽高 256–5760 | +| 音频 | WAV、PCM 编码 | 15 MiB | 2–15 秒 | + +图片、视频和音频混合示例： + +```bash +curl -sS -X POST "$GATEWAY_URL/v1/videos" \ + -H 'Idempotency-Key: example-ref2va-001' \ + -F model=minimax-h3-ref2va \ + -F 'prompt=Create a coherent five second scene using these references' \ + -F seconds=5 \ + -F size=1344x768 \ + -F num_inference_steps=50 \ + -F 'input_references=@/path/to/reference.png;type=image/png' \ + -F 'input_references=@/path/to/reference.mp4;type=video/mp4' \ + -F 'input_references=@/path/to/reference.wav;type=audio/wav' +``` + +Gateway 始终检查上传参考文件的签名、声明的 MIME 类型以及媒体是否可以解码。对于 Worker +生成的最终结果，adapter `validate_media` 默认是 `false`：Gateway 仍检查可信 Worker +结果描述符、任务/attempt fencing、安全路径、普通文件、文件大小及上限，但有声结果在发布 +阶段不打开或读取 MP4 内容，避免共享存储首次读取的长尾。部署显式设置 +`validate_media: true` 后，Gateway 才会读取 MP4 签名，并通过 PyAV 检查 H.264/AAC、帧率、 +帧数、分辨率及音视频时长；该严格校验会增加完成阶段耗时。`generate_sound=false` 仍需打开 +结果并按需去除音轨，不属于上述有声快速路径。仅修改上传文件扩展名或伪造 +`Content-Type` 不会绕过输入校验。 + +## 任务与结果 + +### 异步任务 + +`POST /v1/videos` 在任务被持久化并进入队列后返回。默认 HTTP 状态为 `202`；部署可为 +兼容客户端配置成 `200`，调用方应以响应体中的任务状态为准。 + +典型响应： + +```json +{ + "id": "video-...", + "object": "video", + "model": "minimax-h3", + "status": "queued", + "progress": 0, + "created_at": 1789950000, + "expires_at": 1790036400, + "size": "1344x768", + "seconds": 5.0, + "num_frames": 120, + "fps": 24, + "seed": 1101, + "seed_generated": false +} +``` + +公开状态及含义： + +| 状态 | 含义 | +|---|---| +| `queued` | 在 Gateway 队列中等待调度 | +| `in_progress` | 正在下发、在 Worker 预取队列等待、执行或由 Gateway 后处理 | +| `completed` | 结果可下载 | +| `failed` | 任务失败；查看 `error` | +| `cancelled` | 任务已取消 | +| `expired` | 结果已过期或已通过 DELETE 清理 | + +当前 `progress` 只表示终态：完成为 100，其余为 0，不代表去噪步数进度。 + +完成后的状态会增加结果和耗时字段，例如： + +```json +{ + "id": "video-...", + "object": "video", + "model": "minimax-h3", + "status": "completed", + "progress": 100, + "media_type": "video/mp4", + "file_name": "video-....mp4", + "bytes": 20971520, + "sha256": "...", + "requested_seconds": 5.0, + "seconds": 5.175, + "duration_s": 5.175, + "metrics": { + "queue_wait_s": 0.12, + "worker_queue_wait_s": 0.35, + "inference_time_s": 5.0, + "finalize_time_s": 0.28 + }, + "inference_time_s": 5.86, + "stage_durations": { + "queue_wait": 0.12, + "worker_queue_wait": 0.35, + "finalize": 0.28 + } +} +``` + +耗时字段含义： + +- 顶层 `inference_time_s`：从 Gateway 创建任务到任务完成的端到端时间。 +- `metrics.queue_wait_s`：任务在 Gateway 队列等待的时间。 +- `metrics.worker_queue_wait_s`：任务已交给 Worker 后，在 Worker 预取队列等待执行的时间。 +- `metrics.inference_time_s`：Worker/vLLM-Omni 报告的推理时间；其具体起止点由 Worker + 版本定义，可能包含 Worker 内部排队，因此分析时应结合 `worker_queue_wait_s`。 +- `metrics.finalize_time_s`：Gateway 校验、处理并发布结果制品的耗时；计时在读取最新任务及 + 向 etcd 写入 `completed` 终态之前结束，**不包含最终任务状态写回耗时**。 +- `stage_durations`：以上持久化阶段以及 Worker 返回的细分阶段；新增阶段名属于兼容性扩展， + 调用方不应要求固定键集合。 + +### 同步任务 + +`POST /v1/videos/sync` 使用相同的校验、队列、Worker 和制品流程。成功时直接返回 +`video/mp4`，常用响应头包括： + +- `X-Video-Id`：任务 ID；同步请求也会创建可查询的持久化任务。 +- `X-Video-Seed`：实际 seed。 +- `X-Video-Frames`、`X-Video-FPS`、`X-Video-Duration-Seconds`：实际输出信息。 +- `ETag`：基于结果 SHA-256 的实体标签。 + +示例： + +```bash +curl -fL -X POST "$GATEWAY_URL/v1/videos/sync" \ + -H 'Idempotency-Key: example-sync-001' \ + -F model=minimax-h3 \ + -F 'prompt=A short cinematic shot' \ + -F seconds=5 \ + -F size=1344x768 \ + -F num_inference_steps=50 \ + -D /tmp/video-response.headers \ + -o /tmp/result.mp4 +``` + +同步等待超时时返回 `504 gateway_timeout`，响应头 `X-Video-Id` 中的任务仍可继续通过 +异步查询接口轮询和下载。生成失败返回 `422 video_generation_failed`，同样携带 +`X-Video-Id`。 + +### 列表与分页 + +```bash +curl -sS "$GATEWAY_URL/v1/videos?model=minimax-h3&status=completed&limit=20&order=desc" | jq +``` + +查询参数： + +- `model`：按模型对应的 pool 过滤； +- `status`：按任务状态过滤； +- `limit`：1–100，默认 20； +- `order`：`asc` 或 `desc`，默认 `desc`； +- `after`：使用上一页的 `last_id` 作为游标。 + +响应包含 `data`、`has_more`、`first_id` 和 `last_id`。 + +### 下载结果 + +完整下载： + +```bash +curl -fL "$GATEWAY_URL/v1/videos/$TASK_ID/content" -o "$TASK_ID.mp4" +``` + +只检查元数据： + +```bash +curl -sSI "$GATEWAY_URL/v1/videos/$TASK_ID/content" +``` + +接口支持单段 HTTP Range 和 `If-None-Match`： + +```bash +curl -sS -H 'Range: bytes=0-1048575' \ + "$GATEWAY_URL/v1/videos/$TASK_ID/content" -o first-megabyte.bin +``` + +成功响应可能包含 `Content-Length`、`Content-Range`、`Accept-Ranges: bytes`、`ETag`、 +`X-Video-Id`、`X-Video-Seed`、`X-Video-Frames`、`X-Video-FPS` 和 +`X-Video-Duration-Seconds`。 + +结果未完成时返回 `409 video_not_ready`；任务失败或取消返回 +`422 video_generation_failed`；任务仍保留过期记录、或已完成任务的制品不可用时返回 +`410 video_expired`。过期记录最终清理后，任务查询和内容下载均返回 `404 video_not_found`。 +不支持多段 Range，无效范围返回 `416 range_not_satisfiable`。 + +### 取消与删除 + +```bash +curl -sS -X DELETE "$GATEWAY_URL/v1/videos/$TASK_ID" | jq +``` + +- 排队任务可立即取消，返回 `200` 和 `video.deleted`。 +- Worker 正在执行时，Gateway 记录取消请求并返回 `202`；继续轮询任务直到进入终态。 +- 对 `completed`、`failed`、`cancelled` 或 `expired` 任务执行 DELETE 会清理结果并将任务 + 标记为 `expired`，返回 `200`。 +- 任务记录仍存在时，重复 DELETE 不会重新创建任务；过期记录最终清理后，再对该 ID + 执行 DELETE 返回 `404 video_not_found`，并非始终返回 200。 + +如果底层推理引擎不能中途停止 GPU kernel，API 已接受取消并不代表 GPU 计算在同一时刻停止。 + +## 错误处理与接入 + +### 错误格式与重试建议 + +所有可预期 JSON 错误使用统一格式： + +```json +{ + "error": { + "message": "video queue is full", + "type": "invalid_request_error", + "param": null, + "code": "queue_full" + } +} +``` + +常见 HTTP 状态： + +| 状态 | 常见 code | 调用方处理 | +|---:|---|---| +| 400 | `missing_required_field`、`invalid_size`、`invalid_duration` | 修正参数，不自动重试 | +| 404 | `model_not_found`、`video_not_found` | 检查模型名或任务 ID | +| 409 | `idempotency_conflict`、`video_not_ready` | 使用原请求或继续轮询 | +| 413 | `payload_too_large`、`file_too_large` | 缩小参考素材 | +| 415 | `unsupported_media_type`、`unsupported_media_format` | 使用 multipart 和受支持格式 | +| 422 | `video_generation_failed` | 查看任务 `error`，通常不应原样盲重试 | +| 429 | `queue_full` | 保持相同 `Idempotency-Key`，退避后重试 | +| 503 | `no_worker_available`、`not_ready`、`service_unavailable` | 退避并切换/重试 Gateway | +| 507 | `insufficient_artifact_storage` | 服务端需要释放或扩展制品存储 | + +对于连接中断、429、502、503、504 等不确定结果，先使用相同 `Idempotency-Key` 重试提交， +不要生成新 key，否则可能创建重复任务。 + +### 认证与兼容性 + +Video Gateway 当前不实现最终用户认证和授权，应由上层应用、API Gateway 或入口 LB +完成身份校验、租户隔离和限流。调用方应忽略未知的响应字段，以便 Gateway 在滚动升级中 +增加指标和媒体元数据而不破坏客户端。 diff --git a/docs/video-gateway/video-gateway-protocol-compatibility.md b/docs/video-gateway/video-gateway-protocol-compatibility.md new file mode 100644 index 000000000000..993f35ea57cb --- /dev/null +++ b/docs/video-gateway/video-gateway-protocol-compatibility.md @@ -0,0 +1,187 @@ +# Dingo Video Gateway API 协议与兼容性 + +本文描述当前 DingoRouter Video Gateway 对外提供的视频 API，以及它与 OpenAI Videos API、vLLM-Omni Videos API 的关系。以 2026-09-23 的 DingoRouter 实现为准；实际模型名、可用状态和媒体限制应查询目标部署的 `GET /v1/models`。 + +## 一句话说明 + +我们的接口是 **OpenAI 风格的视频任务 API，加上 vLLM-Omni 常用的视频生成参数和 MiniMax-H3 专用适配**。它复用 `/v1/videos` 的提交、查询、下载等资源路径，但不是完整的 OpenAI API，也不是对 vLLM-Omni HTTP 服务的透明代理。 + +```text +客户端 --multipart /v1/videos--> Dingo Video Gateway + --持久化任务、排队、租约、重试--> FL2VA / Ref2VA Worker + <--共享制品目录中的 MP4 与任务结果-- +客户端 <--任务状态 / MP4 下载-- Gateway +``` + +Gateway 将公开请求规范化为 Worker 请求。Worker 执行仍依赖 vLLM-Omni；当前部署通过 Dynamo 内部请求平面和共享制品目录交换任务与结果。双 Gateway 的任务协调使用 etcd。 + +例如，Gateway 会把 `seconds`/`size` 等公开字段换算为 Worker 使用的帧数和尺寸，再构造包含 `model`、`prompt`、`size`、`output_format: mp4`、`nvext.fps`、`nvext.num_frames`、`nvext.num_inference_steps` 和 `nvext.seed` 的内部请求。参考文件经校验后才编码为底层 `input_reference`；这层转换不属于公开 API。 + +## 对外接口 + +| 方法 | 路径 | 当前行为 | 来源与差异 | +|---|---|---|---| +| `GET` | `/v1/models` | 列出模型、`available` 和 `video_capabilities` | OpenAI 风格列表；后两个字段是 Gateway 扩展 | +| `POST` | `/v1/videos` | multipart 异步提交，返回持久化任务 JSON 和 `Location` | 与 OpenAI、vLLM-Omni 的基本路径一致；当前部署配置返回 HTTP 200，代码默认 202 | +| `GET` | `/v1/videos/{id}` | 查询状态、错误、结果信息和阶段耗时 | 同类资源查询；Gateway 增加持久化指标和清理状态 | +| `GET` | `/v1/videos` | 分页查询任务 | 同类资源列表；Gateway 支持按模型、状态等过滤 | +| `GET`/`HEAD` | `/v1/videos/{id}/content` | 下载 MP4；支持 Range、ETag | 基本下载路径相同；HEAD/Range 是 Gateway 提供的能力 | +| `DELETE` | `/v1/videos/{id}` | 活跃任务请求取消；终态任务清理制品 | Gateway 同时承担取消与制品清理；取消不保证底层 GPU 计算立即停止 | +| `POST` | `/v1/videos/sync` | 使用相同任务流程，完成后直接返回 MP4 字节 | 与 vLLM-Omni 的同步扩展同路径；Gateway 同步任务仍可按 ID 查询 | +| `POST` | `/v1/videos/stream` | 返回 `404 unsupported_endpoint` | 当前未实现 | + +`/live`、`/ready`、`/health` 和 `/metrics` 是部署与运维接口。当前 Gateway 未配置 `http.upstream_url` 时，其他未列出的路径返回 404；不能把它当作通用 OpenAI Chat/Responses 代理。 + +## 逐项对照 OpenAI Videos API + +这里的“完全一致”严格指**所列出的那一部分协议**一致，不代表整个接口的请求、响应、错误和认证都可以原样替换。按 2026-09-23 的[官方 Videos API 参考](https://developers.openai.com/api/reference/resources/videos)与本仓库 `api.py`、`models.py`、MiniMax-H3 adapter 对照：**目前没有一个完整的 Videos 端点可以声明与 OpenAI 全契约完全一致**。 + +### 完全一致的部分 + +| 范围 | 一致之处 | 边界 | +|---|---|---| +| 基础资源路由 | `POST /v1/videos`、`GET /v1/videos`、`GET /v1/videos/{id}`、`GET /v1/videos/{id}/content`、`DELETE /v1/videos/{id}` 的方法与路径相同 | 路径相同不等于入参、返回体或生命周期完全相同 | +| 异步基本流程 | 提交取得 ID，随后按 ID 查询；完成后通过 `/content` 下载 MP4 | 具体提交格式、参数范围、状态和错误码见下文 | +| 任务对象的共同字段 | `id` 为字符串，`object` 为 `"video"`；`model`、`status`、`created_at`、`size` 使用同名字段；`created_at`/已完成时的 `completed_at` 是 Unix 秒时间戳 | `model` 的可用值不同；`status` 的取值集合不同；`size` 的可用值不同 | +| 列表外壳 | 返回 `object: "list"`、`data`、`first_id`、`last_id`、`has_more`；共同支持 `after`、`limit`、`order=asc/desc` | 条目仍是上述有差异的任务对象；`limit` 边界不同 | +| 删除终态任务的成功返回体 | 对已完成/失败任务，成功时返回 `{ "id": "...", "object": "video.deleted", "deleted": true }` | 对活跃任务的 DELETE 是我们的取消扩展；内部清理与保留语义不能据此视为完全一致 | + +### 相同端点上的差异与我们的扩展 + +| 端点/项目 | OpenAI | 当前 Gateway | 分类 | +|---|---|---|---| +| `POST /v1/videos` 的编码 | 官方示例可用 multipart；官方参数参考定义 `prompt` 等字段 | **只接受** `multipart/form-data`；提交后返回任务 JSON 与 `Location` | 共同路径，编码能力不等同 | +| `POST /v1/videos` 的 HTTP 状态 | 返回新建视频任务 | 状态码可配置；当前部署为 **200**，代码默认 **202**；幂等命中返回 200 | 差异 | +| `prompt` | 必填文本，官方长度上限 32000 字符 | 必填文本；按 UTF-8 **字节数**受部署/adapter 限制 | 字段用途相同，校验边界不同 | +| `model` | 可选，默认 Sora 模型；接受官方 Sora 型号 | 只能选已配置 Worker pool 的模型，如 `minimax-h3`、`minimax-h3-ref2va`；是否可省略取决于默认模型/池数 | 字段名相同，取值与默认规则不同；**不支持 Sora 模型** | +| `seconds` | 可省略，默认 `4`；允许字符串 `"4"`、`"8"`、`"12"` | 必须在 `seconds`、`num_frames`、`extra_params.duration` 中恰选一种；对应 4–15 秒，支持如 5 秒 | 字段名相同，必填规则和范围不同 | +| `size` | 可省略，默认 `720x1280`；只允许官方列出的四种尺寸 | 必须提供 `size` 或 `width`+`height`，按 MiniMax-H3 尺寸约束校验 | 字段名相同，必填规则和值域不同 | +| `input_reference` | 官方参数是含 `file_id` **或** `image_url` 的引用对象 | 同名字段只接收一个 **multipart 上传文件**，由 Gateway 保存并校验 | **同名但格式不兼容**；不支持直接传官方 `file_id`/`image_url` 引用对象 | +| `GET /v1/videos/{id}` | 返回官方视频对象 | 返回任务对象，并增加阶段耗时、结果媒体信息、取消/过期信息；不保证官方所有字段 | 共同路径、扩展响应 | +| `GET /v1/videos` | `after`、`limit`（官方允许 0–100）、`order` | 还支持 `model`、`status` 过滤；`limit` 只允许 **1–100** | 扩展筛选，边界差异 | +| `GET /v1/videos/{id}/content` | 默认 MP4；`variant=video/thumbnail/spritesheet` 可选 | 只提供 MP4；额外支持 `HEAD`、单区间 `Range`、`ETag`、`If-None-Match` 和 `X-Video-*` 响应头 | MP4 下载同路径；预览变体不支持，HTTP 下载能力是扩展 | +| `DELETE /v1/videos/{id}` | 删除已完成/失败视频及其资产 | 终态清理返回同形状 JSON；排队或执行中也可请求取消，部分情况返回 202 `video.cancel`；取消不保证 GPU 立即停算 | 终态成功体相同，活跃任务行为是扩展 | +| `GET /v1/models` | 标准模型列表端点 | 列表项额外有 `available`、`video_capabilities`；可能同时列出上游模型，但只有已配置 Worker pool 的模型可提交视频任务 | 扩展，不是 Sora 模型目录 | + +特别注意：当前 `/content?variant=thumbnail` 或 `spritesheet` **不会生成对应预览图**；现有 handler 不解析 `variant`，仍按 MP4 下载处理。客户端不能仅以 HTTP 200 判断拿到了所请求的变体。 + +任务对象本身也要逐字段看，不能只看到 `object: "video"` 就按 OpenAI 类型反序列化： + +| 字段 | 对照结论 | +|---|---| +| `id`、`object`、`created_at`、已完成时的 `completed_at` | 字段名和基本类型/时间单位一致；ID 的具体前缀没有兼容保证 | +| `model`、`size` | 字段名及字符串形态一致，可用模型/尺寸集合不同 | +| `status` | 官方的 `queued`、`in_progress`、`completed`、`failed` 均有；我们另有 `cancelled`、`expired` | +| `seconds` | 官方视频对象是**字符串**；我们是**数字**，完成后可能改为测得的实际时长，并另给 `requested_seconds` | +| `progress` | 官方是近似 0–100 的完成百分比；我们仅返回未完成 0、完成 100 | +| `expires_at` | 同名 Unix 秒时间戳；我们按本地任务保留策略设置，不能假定与 OpenAI 保留期限一致 | +| `error` | 官方视频错误对象可含 `headers`、`misalignment`；我们失败时仅返回本地 `code`、`message`、`retryable`，且正常状态通常不返回 `error` 键 | +| `prompt`、`remixed_from_video_id` | 官方视频对象定义了这些字段；当前任务 JSON 不提供 | +| `metrics`、`stage_durations`、`inference_time_s`、`num_frames`、`fps`、`seed`、`bytes`、`sha256` 等 | 我们的附加字段，不是 OpenAI Videos 对象的必备/标准字段 | + +### 我们新增、但不是 OpenAI Videos API 的能力 + +| 能力 | 当前接口/字段 | +|---|---| +| 同步返回 MP4 | `POST /v1/videos/sync`；沿用持久化任务流程，超时后还可凭 `X-Video-Id` 查询 | +| 幂等提交 | 请求头 `Idempotency-Key`；任务与幂等记录保留、pool 配置及适配器兼容版本不变时，相同请求复用任务；同 key 的请求摘要不一致返回 409；记录清理后同 key 可能创建新任务 | +| 多参考与 MiniMax-H3 生成控制 | 重复的文件 part `input_references`，以及 `width`、`height`、`num_frames`、`fps`、`num_inference_steps`、`seed`、`generate_sound`、`frame_indices` 等；按模型/pool 校验，不是任意透传 | +| 更多任务状态和观测字段 | `cancelled`、`expired`；`metrics`、`stage_durations`、`num_frames`、`fps`、`seed`、`bytes`、`sha256`、`duration_s` 等；`inference_time_s` 也不是 OpenAI Videos 标准字段 | +| 下载与运维 | `HEAD`、Range/ETag/条件下载；`/live`、`/ready`、`/health`、`/metrics` | + +`user` 虽可作为我们请求中的标签，但不是上述 OpenAI **Videos Create** 参考列出的参数，不负责认证或租户隔离。`response_format`、`output_format` 等字段源于 vLLM-Omni/内部适配，也不属于 OpenAI Videos Create 的公开参数。 + +### OpenAI 有、当前 Gateway 不支持或不等价的部分 + +| 官方能力 | 当前情况 | +|---|---| +| `POST /v1/videos/{id}/remix` | 无本地实现 | +| `POST /v1/videos/edits` | 无本地实现 | +| `POST /v1/videos/extensions` | 无本地实现 | +| `POST /v1/videos/characters` | 无本地实现 | +| `GET /v1/videos/characters/{id}` | 无本地实现 | +| Sora 型号、官方尺寸/时长默认值与取值集合 | 不支持；只能按本部署模型和约束提交 | +| 官方 `input_reference.file_id` / `image_url` | 不支持；需上传媒体文件 part | +| 下载 `variant=thumbnail` / `spritesheet` | 不生成；只能获得 MP4 | +| 官方视频对象的 `prompt`、`remixed_from_video_id`，及官方错误对象的 `headers`/`misalignment` 等完整契约 | 当前任务查询不提供这些官方字段/结构；失败任务的 `error` 使用 Gateway 自身格式 | +| OpenAI 平台 Bearer API key、项目权限与隔离 | Gateway 自身未实现；必须由上层 LB/API 网关或应用负责鉴权和租户边界 | + +此表按**当前未配置 `http.upstream_url` 的部署**说明。若将来配置通用 upstream，未注册路由可能由上游处理；这并不意味着 Gateway 自己实现了这些 OpenAI 能力，也不能把上述“不支持”理解为对任意上游配置的断言。OpenAI 的 Chat、Responses、Files 等其他 API 也不属于本视频 Gateway 的本地实现范围。 + +## Dingo 协议细节与 vLLM-Omni 适配 + +### 请求格式与 MiniMax-H3 适配 + +提交接口只接受 `multipart/form-data`。普通字段是文本 part，媒体输入是文件 part。业务调用方通常只需提供 `model`、`prompt`、时长、尺寸、推理步数和参考文件；`task`、`flow_shift` 等 Worker 参数由 Gateway 根据 pool 填充。 + +| 字段 | 与 OpenAI / vLLM-Omni 的关系 | Gateway 当前行为 | +|---|---|---| +| `model`、`prompt`、`seconds`、`size` | 与 OpenAI Videos Create 同名，vLLM-Omni 也支持 | 模型必须属于已配置 pool；时长和尺寸按 MiniMax-H3 限制校验；必填/默认值/取值并不与 OpenAI 相同 | +| `user` | 不是当前 OpenAI Videos Create 参考列出的字段 | 仅作标签，不用于认证 | +| `width` + `height`、`num_frames` | vLLM-Omni 扩展 | 可替代 `size`、`seconds`；时长必须在 `seconds`、`num_frames`、`extra_params.duration` 中恰选一种 | +| `fps`、`num_inference_steps`、`seed`、`guidance_scale`、`negative_prompt` | vLLM-Omni 常见生成参数 | MiniMax-H3 当前只接受 24 FPS；步数允许 2–200；seed 可省略并由 Gateway 生成 | +| `generate_sound` | vLLM-Omni 扩展 | Gateway 默认 `true`；请求 `false` 时，输出不得带音轨，必要时 Gateway 去音轨 | +| `input_reference` | 与 OpenAI 同名，但官方要求 `file_id`/`image_url` 引用对象 | 当前只接收一个 multipart 上传文件；FL2VA 可作为首帧或尾帧；不能直接使用官方引用对象 | +| 重复的 `input_references` | Gateway 的多参考文件扩展 | FL2VA 支持两张首尾帧；Ref2VA 支持图片、视频及可选音频组合，文件顺序保留 | +| `frame_indices`、`extra_params` | 为 MiniMax-H3 与 vLLM-Omni 参数映射保留 | 只允许实现明确支持的键；不是任意扩展 JSON 的透传 | +| `response_format`、`output_format` | vLLM-Omni 输出参数 | 仅接受内部 `b64_json` 和 `mp4`；对外结果通过下载接口或同步响应返回 MP4，不在任务 JSON 中返回 Base64 视频 | + +当前对外模型名通常是： + +- `minimax-h3`：FL2VA，文本生成视频，或使用一张/两张关键帧图片。 +- `minimax-h3-ref2va`：Ref2VA，至少一张图片或一个视频，可附带音频等参考素材。 + +Ref2VA 的多媒体组合会被适配器转换成底层 `input_reference` 可理解的编码；Gateway 不要求客户端自行构造该内部封装。具体格式、大小、时长和示例见[接口使用说明](video-gateway-api.md)。 + +目前不接受 `quality`、`image_reference`、`video_reference`、`audio_reference` 等直传字段，也未开放 vLLM-Omni 最新文档中的 `control_reference`、MiniMax-H3 latent-mask 编辑等能力。不能因为底层 vLLM-Omni 支持某字段，就假定 Gateway 已对外开放。 + +#### 最小调用示例 + +```bash +RESPONSE=$(curl -sS -X POST "$GATEWAY_URL/v1/videos" \ + -H 'Idempotency-Key: order-123' \ + -F model=minimax-h3 \ + -F 'prompt=A sunrise over snow mountains' \ + -F seconds=5 \ + -F size=1344x768 \ + -F num_inference_steps=50) +TASK_ID=$(printf '%s' "$RESPONSE" | jq -r '.id') + +curl -sS "$GATEWAY_URL/v1/videos/$TASK_ID" +curl -fL "$GATEWAY_URL/v1/videos/$TASK_ID/content" -o "$TASK_ID.mp4" +``` + +`Idempotency-Key` 是 Gateway 扩展：在原任务及幂等记录仍保留、pool 配置及适配器兼容 +版本不变时，同一 key、同一请求重复提交会返回原任务；请求摘要不一致则返回 409。 +摘要包含这些版本信息，因此滚动升级期间即使 HTTP 请求相同也可能冲突；记录清理后 +同 key 可能创建新任务。 +发生网络超时时，仍应先复用原 key 重试，遇到 409 时查询已知任务 ID 或核对配置。 + +### 响应和任务生命周期 + +异步提交先返回任务 ID 与 `object: "video"`。公开状态为 `queued`、`in_progress`、`completed`、`failed`、`cancelled`、`expired`。内部 `dispatching` 和 `finalizing` 都映射为 `in_progress`；`in_progress` 也可能表示 Worker 内预取等待。当前 `progress` 只有完成时 100、其他状态 0，不提供逐去噪步进度。 + +完成后，`GET /v1/videos/{id}` 提供结果字节数、SHA256、实际视频时长以及 `metrics` / `stage_durations`。其中 `queue_wait_s`、`worker_queue_wait_s`、`inference_time_s`、`finalize_time_s` 用于区分 Gateway 排队、Worker 预取、推理与 Gateway 结果处理。已受理任务失败时查看任务的 `error.code`，例如 `worker_failed` 或 `finalization_timeout`；没有可用 Worker 时，提交请求可直接收到 HTTP 503 `no_worker_available`。 + +这里也有类型和语义差异：OpenAI 视频对象的 `seconds` 是字符串，当前 Gateway 返回数值型时长；OpenAI 文档中的 `progress` 表示近似完成百分比，而当前 Gateway 仅使用 0/100。客户端不应依赖官方 SDK 对视频对象的全部类型约束与可选字段都完全一致。 + +任务与结果会过期，也可通过 DELETE 主动清理。调用方应保存任务 ID，完成后及时下载；不能把制品目录当成永久对象存储。 + +### 与 vLLM-Omni 的边界 + +| 主题 | vLLM-Omni Videos API | 当前 Dingo Video Gateway | +|---|---|---| +| 服务定位 | 单模型推理服务的视频 HTTP 接口 | 多 Gateway、多 Worker pool 的持久化调度与制品服务；不是到 Worker HTTP 接口的透明代理 | +| 同步生成 | 提供 `/v1/videos/sync` | 同路径，但同步任务也持久化，超时后可按 ID 查询 | +| 参考素材 | 提供单参考与其他模型扩展字段 | 接收单文件 `input_reference` 或多文件 `input_references`，按 FL2VA/Ref2VA 限制校验，再转换为 Worker 输入 | +| 模型参数 | 可依模型支持更多视频生成和编辑字段 | 只开放当前 MiniMax-H3 适配器明确支持的字段；不直接透传任意 Worker 参数 | +| 任务与结果 | 由单个推理服务管理 | etcd 协调跨 Gateway 任务、Worker 租约与重试；共享制品存储支持跨 Gateway 查询和下载 | +| 认证 | 由具体部署决定 | Gateway 自身不做最终用户认证与租户鉴权，需由上层入口处理 | + +因此，兼容性的准确表述是：**可复用 OpenAI 风格的视频资源路径和 vLLM-Omni 的一部分请求字段；不能保证任意 OpenAI SDK 调用或任意 vLLM-Omni 模型扩展只替换 `base_url` 就能工作。** 调用方应以本文和[接口使用说明](video-gateway-api.md)为契约。 + +## 依据与版本说明 + +- [OpenAI Videos API 参考](https://developers.openai.com/api/reference/resources/videos)：用于对照公开资源路径和任务对象；该页面在本文写作时将相关 Videos 接口标为 Deprecated，不能据此推断本项目接口也已废弃。 +- [vLLM-Omni Videos API](https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/videos_api/)：用于对照最新公开协议；`latest` 文档可能超前于本部署所用的 vLLM-Omni 0.29 系列镜像。 +- 本仓库 `dingo/video_gateway/api.py`、`adapters/minimax_h3.py`、`models.py` 和 `docs/video-gateway/video-gateway-api.md`：决定当前 Gateway 的实际行为。