From 079e8c58ce398bbf9ad4b3a84549abffd7ad21ff Mon Sep 17 00:00:00 2001 From: Barada Sahu Date: Sun, 5 Jul 2026 11:58:08 +0530 Subject: [PATCH 1/8] feat(cua): expose /v1/cua pure-Holo3 instruction loop on Modal (+H100 fallback) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds an autonomous /v1/cua endpoint to the Modal server so every step is a 100%-student Holo3 action with planner-layer modelio + per-token logprobs — the on-policy substrate SA-OPSD / GRPO training needs. Modal's run_holo3 was suite-only; the pure-Holo3 loop lived only in baseten_server.run_pure_cua and was never reachable from the Modal deployment. - _run_holo3_executor: new `_mode == "pure_cua"` branch (sibling of the _micro_plan fork) that reuses the built brain + env, drives a bare GymRunner(task=instruction, start_url), and — critically — opens a LIVE AugurAdapter onto runner._augur so publish_modelio_context publishes layer="planner" records with logprobs. On Baseten nothing sets runner._augur, so modelio silently no-ops; this path fixes that. - build_api_app: new POST /v1/cua route (contract in docs/client/pure-cua.md) that translates instruction/start_url/... into a _mode="pure_cua" suite and dispatches through a shared _submit_detached_run helper extracted from /v1/predict (one lifecycle, two entrypoints). - Honest terminal_status/halt_reason on the pure_cua envelope so _do_action maps the wire status truthfully. - H100 GPU fallback on run_holo3 (gpu=["A100-80GB","H100"]) — the A100-80GB pool starves; bundled here as it was a stranded working-tree change. Verified live: a /v1/cua run emits 4 layer="planner" modelio records, each carrying populated per-token logprobs (172 tokens/step with top-K alternatives). Requires MANTIS_CAPTURE_LOGPROBS=1 in the deploy env. Tests: /v1/cua request->suite translation + lifecycle; pure_cua branch selection + the runner._augur (logprobs-capture) guarantee. 9 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- deploy/modal/modal_cua_server.py | 423 ++++++++++++++++++++++++---- tests/test_holo3_pure_cua_branch.py | 255 +++++++++++++++++ tests/test_modal_cua_route.py | 215 ++++++++++++++ 3 files changed, 839 insertions(+), 54 deletions(-) create mode 100644 tests/test_holo3_pure_cua_branch.py create mode 100644 tests/test_modal_cua_route.py diff --git a/deploy/modal/modal_cua_server.py b/deploy/modal/modal_cua_server.py index 1b1db693..d263e02c 100644 --- a/deploy/modal/modal_cua_server.py +++ b/deploy/modal/modal_cua_server.py @@ -1068,6 +1068,214 @@ def _run_holo3_executor( except Exception as exc: # noqa: BLE001 print(f" WARNING: external_pause init failed: {exc}") + # ── Pure-CUA mode: autonomous Holo3 loop (100% student actions) ── + # Sibling of the ``_micro_plan`` fork below, selected by the /v1/cua + # route (``_mode == "pure_cua"``). Mirrors + # ``baseten_server.runtime.run_pure_cua``: a bare GymRunner ↔ + # XdotoolGymEnv loop with NO decomposer / grounding / extractor on the + # default path — whatever the brain emits, xdotool executes. + # + # The whole point of this path is planner-layer modelio + per-token + # logprobs on every step (for RL / SFT curation). GymRunner.run + # captures that via ``publish_modelio_context(_planner_augur, + # layer="planner", …)`` where ``_planner_augur`` falls back to + # ``getattr(self, "_augur", None)`` when no modelio context is active + # (the pure-cua case — nothing wraps a ``model`` context around this + # loop). Nothing sets ``runner._augur`` on this path by default, so on + # Baseten it silently no-ops (zero logprobs). We fix that here: mint an + # ``augur_run_id`` and open a LIVE AugurAdapter onto ``runner._augur`` + # so ``publish_modelio_context`` (which requires ``adapter.active``) + # fires and ``Holo3Brain._record_modelio_if_active`` writes + # ``layer="planner"`` records. ``MANTIS_CAPTURE_LOGPROBS`` is on in the + # deploy env, so the SDK stamps per-token logprobs onto those records. + if task_suite.get("_mode") == "pure_cua": + from mantis_agent.gym.runner import GymRunner, RoutingPolicy + from mantis_agent.observability.augur import AugurAdapter + + instruction = str(task_suite.get("_instruction") or "").strip() + if not instruction: + raise ValueError("pure_cua requires a non-empty _instruction") + + pure_max_steps = int(task_suite.get("_max_steps", max_steps) or max_steps) + print(f"\n === PURE-CUA MODE (holo3, max_steps={pure_max_steps}) ===") + print(f" Instruction: {instruction[:120]}") + + # #300: attach PageDiscovery when the env exposes the CDP DOM shim + # so the SoM click path can fire (no-op on legacy callers). Mirrors + # the Baseten pure-cua wiring. + pure_page_discovery = None + if hasattr(env, "cdp_evaluate"): + from mantis_agent.gym.page_discovery import PageDiscovery + pure_page_discovery = PageDiscovery(env=env) + + # #300 follow-up: per-request RoutingPolicy override. ``None`` + # defers to ``MANTIS_ROUTE_SOM_CLICKS``. + pure_routing_policy = RoutingPolicy.from_env() + _route_override = task_suite.get("_route_som_clicks") + if _route_override is not None: + pure_routing_policy = RoutingPolicy( + plan_executor_enabled=pure_routing_policy.plan_executor_enabled, + som_enabled=pure_routing_policy.som_enabled, + som_for_unstructured_clicks=bool(_route_override), + ) + + # #931 P2: opt-in screenshot grounding for click precision. Pure + # CUA is brain-only by default (grounding=None); reuse the + # ClaudeGrounding built above only when the caller opted in AND a + # key is present — otherwise brain-only (never hard-fail). + pure_grounding = None + if bool(task_suite.get("_ground_clicks")) and os.environ.get("ANTHROPIC_API_KEY"): + pure_grounding = grounding + print(" [pure_cua] ground_clicks=true — screenshot grounding enabled") + elif bool(task_suite.get("_ground_clicks")): + print( + " [pure_cua] ground_clicks requested but no ANTHROPIC_API_KEY " + "— running brain-only" + ) + + pure_runner = GymRunner( + brain=brain, + env=env, + max_steps=pure_max_steps, + frames_per_inference=frames_per_inference, + grounding=pure_grounding, # None = pure brain; set when ground_clicks opts in + page_discovery=pure_page_discovery, + routing_policy=pure_routing_policy, + on_step=viewer_event_bus.emit if viewer_event_bus else None, + ) + + # #649-shaped session id: workflow_id-prefixed for searchability, + # uuid-suffixed for uniqueness (same recipe as the micro branch). + import uuid as _uuid_for_cua + _pure_workflow_id = str(task_suite.get("_workflow_id", "") or "") + pure_augur_run_id = ( + f"{_pure_workflow_id}-{_uuid_for_cua.uuid4().hex[:8]}" + if _pure_workflow_id else _uuid_for_cua.uuid4().hex[:12] + ) + try: + if api_run_id and api_tenant_id: + _write_augur_metadata(api_tenant_id, api_run_id, pure_augur_run_id) + except Exception as _augur_exc: # noqa: BLE001 + print(f" WARNING: augur metadata write failed: {_augur_exc}") + + # THE logprobs-capture guarantee. The micro path opens this adapter + # inside ``run_executor`` and stamps it on ``runner._augur``; the + # pure-cua path doesn't go through run_executor, so we open it here + # and assign it ourselves. ``AugurAdapter.__init__`` drives + # ``DebugSession.__enter__`` so ``.active`` is True on return — the + # precondition ``publish_modelio_context`` checks before publishing. + _pure_model_name = str(getattr(brain, "model_name", "") or "") + pure_runner._augur = AugurAdapter( + run_id=pure_augur_run_id, + tenant_id=str(api_tenant_id or ""), + session_name=session_name, + extra_tags={ + "mode": "pure_cua", + "workflow_id": _pure_workflow_id, + "model": _pure_model_name, + }, + brain_model_name=_pure_model_name, + ) + + # #931 P3: opt-in decompose-then-cua. Seed an ordered sub-goal + # roadmap up front (Claude) and drive the brain with the augmented + # task. Falls back to the raw instruction on no-key / failure — + # never hard-fails the run. + pure_task = instruction + if bool(task_suite.get("_decompose")) and os.environ.get("ANTHROPIC_API_KEY"): + try: + from mantis_agent.plan_decomposer import PlanDecomposer + _dplan = PlanDecomposer( + api_key=os.environ["ANTHROPIC_API_KEY"] + ).decompose_text(instruction) + _subgoals = [ + s.intent for s in getattr(_dplan, "steps", []) + if getattr(s, "intent", "") + ] + if _subgoals: + _lines = "\n".join( + f" {i + 1}. {s}" for i, s in enumerate(_subgoals) + ) + pure_task = ( + f"{instruction}\n\nComplete the task as the following " + "ordered sub-goals. Do them in order and confirm each " + "visibly succeeded before moving to the next:\n" + f"{_lines}" + ) + print(f" [pure_cua] decompose=true — seeded {len(_subgoals)} sub-goals") + except Exception as _de: # noqa: BLE001 — fall back to raw instruction + print(f" [pure_cua] decompose failed ({_de}) — raw instruction") + elif bool(task_suite.get("_decompose")): + print( + " [pure_cua] decompose requested but no ANTHROPIC_API_KEY " + "— raw instruction" + ) + + pure_gym_result = None + try: + pure_gym_result = pure_runner.run( + task=pure_task, task_id="cua", start_url=base_url, + ) + finally: + # Flush the Augur bundle (mirrors ``run_executor._finalize``). + # Best-effort — telemetry teardown never masks a run error. + try: + _pure_term = ( + "succeeded" + if (pure_gym_result is not None and pure_gym_result.success) + else "halted" + ) + pure_runner._augur.close(_pure_term) + except Exception as _close_exc: # noqa: BLE001 + print(f" WARNING: augur close failed: {_close_exc}") + + pure_elapsed = time.time() - t0 + pure_trajectory = list(getattr(pure_gym_result, "trajectory", []) or []) + pure_success = bool(pure_gym_result.success) + pure_termination = str( + getattr(pure_gym_result, "termination_reason", "") or "" + ) + # Honest terminal fields (cua-issues 2026-06-29): the /v1/cua + # envelope MUST carry ``terminal_status`` / ``halt_reason`` so + # ``_do_action`` maps the wire status truthfully — success → + # ``completed`` → ``succeeded``; anything else → ``halted`` with + # the termination_reason. Without these the poll path would stamp + # a blanket success on every run. + pure_terminal_status = "completed" if pure_success else "halted" + pure_halt_reason = "" if pure_success else pure_termination + + result = { + "run_id": run_id, + "mode": "pure_cua", + "provider": "modal", + "session_name": session_name, + "model": _pure_model_name or "holo3", + "instruction": instruction, + "start_url": base_url, + "success": pure_success, + "termination_reason": pure_termination, + "terminal_status": pure_terminal_status, + "halt_reason": pure_halt_reason, + "steps": int(pure_gym_result.total_steps), + "duration_s": round(pure_elapsed), + "elapsed_seconds": pure_elapsed, + "trajectory_len": len(pure_trajectory), + "augur_run_id": pure_augur_run_id, + } + print( + f"\n Pure-CUA complete: success={pure_success} " + f"steps={result['steps']} reason={pure_termination or ''}" + ) + + env.close() + llama_proc.terminate() + if viewer_ctx: + try: + viewer_ctx.__exit__(None, None, None) + except Exception: + pass + return result + # ── Micro-intent mode: run MicroPlanRunner ── micro_plan_data = task_suite.get("_micro_plan") if micro_plan_data: @@ -2800,6 +3008,7 @@ def build_api_app(executor_resolver=None, function_call_lookup=None, from mantis_agent.server.run_dispatch import ( DispatchError, acquire_profile_lock, + prepare_cua_payload, prepare_predict_payload, read_profile_lock, release_profile_lock as _release_phase1_profile_lock, @@ -2858,54 +3067,29 @@ def models() -> dict: ], } - @fastapi_app.post("/v1/predict") - async def predict( - request: Request, - tenant: TenantConfig = Depends(require_run_scope), + def _submit_detached_run( + tenant: TenantConfig, + *, + task_file_contents: str, + model: str, + profile_id: str, + workflow_id: str, + state_key: str, + max_steps: int, + live_viewer: bool = False, + mode: str = "detached", ) -> dict: - try: - raw = await request.json() - except Exception: - raise HTTPException(400, "request body must be JSON") - try: - payload = prepare_predict_payload(raw, tenant) - except DispatchError as exc: - raise HTTPException(exc.status_code, exc.detail) from exc - - # Action mode: poll/cancel an existing run. - if payload.get("action"): - return _do_action(payload, tenant) - - # Run mode: build suite, lock the profile, spawn the executor. - try: - task_file_contents = _build_suite_from_payload(payload) - except (ValueError, FileNotFoundError) as exc: - raise HTTPException(400, str(exc)) from exc - - # #508 parity: forward a top-level `extraction_schema` into the suite as - # `_extraction_schema` so the executor's extractor honors it (baseten did - # this; the Modal CUA path silently dropped it → `no_schema_configured`). - _payload_schema = payload.get("extraction_schema") - if isinstance(_payload_schema, dict): - try: - _suite_obj = json.loads(task_file_contents) - _suite_obj["_extraction_schema"] = _payload_schema - task_file_contents = json.dumps(_suite_obj) - except (ValueError, TypeError): - pass # non-JSON suite (rare) → leave as-is - - # DX-3 (#785 follow-up): dry-run preview. Returns the resolved - # task_suite + summary + cost estimate without acquiring the - # Chrome lock or spawning the executor. Lets devs iterate on - # plan_text → MicroPlan without burning GPU credit. - if bool(payload.get("dry_run")): - from mantis_agent.server.dry_run import build_dry_run_response - return build_dry_run_response( - task_file_contents, payload, tenant.tenant_id - ) - - profile_id = payload["profile_id"] - workflow_id = payload["workflow_id"] + """Shared submit path for ``/v1/predict`` and ``/v1/cua``. + + Mints a run_id, acquires the Phase-1 profile lock + Phase-2 + snapshot lock, resolves + spawns the executor, persists the + initial ``queued`` status + suite snapshot, and returns the + detached envelope. Both routes go through this one helper so the + status/result lifecycle (poll → terminal write in ``_do_action``) + is byte-identical regardless of which entrypoint submitted the + run — the /v1/cua route just builds a different + ``task_file_contents`` (a ``_mode="pure_cua"`` suite). + """ run_id = _mint_run_id() # 409 if another run is using this profile (Chrome lock). @@ -2937,13 +3121,12 @@ async def predict( "Use a different profile_id or wait for the existing run to finish.", ) from exc - model = payload.get("cua_model") or payload.get("model") or "holo3" executor_fn = resolve_executor(model) # Forward only what the executors accept. `task_file_contents` # is positional; everything else is **kwargs. spawn_kwargs: dict = { - "max_steps": int(payload.get("max_steps", 30)), + "max_steps": int(max_steps), # Per-tenant, per-profile Chrome user-data-dir (#341). # Until this was added, Modal executors fell back to # XdotoolGymEnv's default ``/data/chrome-profile`` and @@ -2959,7 +3142,7 @@ async def predict( # side status.json. The executor needs the API's run_id + # tenant_id to know which status file to update (the # executor's internal run_id differs). - if payload.get("live_viewer"): + if live_viewer: spawn_kwargs["viewer"] = True spawn_kwargs["api_run_id"] = run_id spawn_kwargs["api_tenant_id"] = tenant.tenant_id @@ -2982,9 +3165,9 @@ async def predict( "tenant_id": tenant.tenant_id, "profile_id": profile_id, "workflow_id": workflow_id, - "state_key": payload["state_key"], + "state_key": state_key, "model": model, - "max_steps": int(payload.get("max_steps", 30)), + "max_steps": int(max_steps), } _write_status(tenant.tenant_id, run_id, status) # #347: snapshot the suite so a later action=resume can re-spawn @@ -2994,7 +3177,7 @@ async def predict( run_dir = _run_dir(tenant.tenant_id, run_id) return { "status": "queued", - "mode": "detached", + "mode": mode, "run_id": run_id, "created_at": now, "updated_at": now, @@ -3009,6 +3192,138 @@ async def predict( "events_path": str(run_dir / "events.log"), } + @fastapi_app.post("/v1/predict") + async def predict( + request: Request, + tenant: TenantConfig = Depends(require_run_scope), + ) -> dict: + try: + raw = await request.json() + except Exception: + raise HTTPException(400, "request body must be JSON") + try: + payload = prepare_predict_payload(raw, tenant) + except DispatchError as exc: + raise HTTPException(exc.status_code, exc.detail) from exc + + # Action mode: poll/cancel an existing run. + if payload.get("action"): + return _do_action(payload, tenant) + + # Run mode: build suite, lock the profile, spawn the executor. + try: + task_file_contents = _build_suite_from_payload(payload) + except (ValueError, FileNotFoundError) as exc: + raise HTTPException(400, str(exc)) from exc + + # #508 parity: forward a top-level `extraction_schema` into the suite as + # `_extraction_schema` so the executor's extractor honors it (baseten did + # this; the Modal CUA path silently dropped it → `no_schema_configured`). + _payload_schema = payload.get("extraction_schema") + if isinstance(_payload_schema, dict): + try: + _suite_obj = json.loads(task_file_contents) + _suite_obj["_extraction_schema"] = _payload_schema + task_file_contents = json.dumps(_suite_obj) + except (ValueError, TypeError): + pass # non-JSON suite (rare) → leave as-is + + # DX-3 (#785 follow-up): dry-run preview. Returns the resolved + # task_suite + summary + cost estimate without acquiring the + # Chrome lock or spawning the executor. Lets devs iterate on + # plan_text → MicroPlan without burning GPU credit. + if bool(payload.get("dry_run")): + from mantis_agent.server.dry_run import build_dry_run_response + return build_dry_run_response( + task_file_contents, payload, tenant.tenant_id + ) + + model = payload.get("cua_model") or payload.get("model") or "holo3" + return _submit_detached_run( + tenant, + task_file_contents=task_file_contents, + model=model, + profile_id=payload["profile_id"], + workflow_id=payload["workflow_id"], + state_key=payload["state_key"], + max_steps=int(payload.get("max_steps", 30)), + live_viewer=bool(payload.get("live_viewer")), + ) + + @fastapi_app.post("/v1/cua") + async def cua( + request: Request, + tenant: TenantConfig = Depends(require_run_scope), + ) -> dict: + """Pure-CUA loop — 100%-student Holo3 (``brain ↔ XdotoolGymEnv``). + + Same auth / caps / allowlist / lock / status-lifecycle plumbing + as ``/v1/predict``: we validate through the shared + :func:`prepare_cua_payload` (identity resolution + URL allowlist), + translate the request into a ``_mode="pure_cua"`` ``task_suite``, + force ``cua_model="holo3"``, and dispatch through the exact same + submit/spawn/status-write machinery via + :func:`_submit_detached_run`. The executor's + ``_run_holo3_executor`` picks the pure-cua branch off ``_mode``. + + Runs are always detached on Modal (spawn + poll), same as + ``/v1/predict`` — poll terminal status/result via + ``GET /v1/runs/{run_id}/status|result``. + """ + try: + raw = await request.json() + except Exception as exc: + raise HTTPException(400, "request body must be JSON") from exc + + try: + payload = prepare_cua_payload(raw, tenant) + except DispatchError as exc: + raise HTTPException(exc.status_code, exc.detail) from exc + + instruction = str(payload.get("instruction") or "").strip() + if not instruction: + raise HTTPException(400, "instruction is required") + + # Translate the validated request into the pure-cua suite the + # holo3 executor forks on. Mirrors the ``task_suite`` shape + # ``baseten_server.runtime._run_pure_cua`` builds, plus the + # ``_mode`` / ``_instruction`` selectors the Modal branch reads. + task_suite = { + "session_name": "pure_cua", + "base_url": str(payload.get("start_url") or ""), + "_mode": "pure_cua", + "_instruction": instruction, + "_ground_clicks": bool(payload.get("ground_clicks", False)), + "_decompose": bool(payload.get("decompose", False)), + "_max_steps": int(payload.get("max_steps", 30)), + "_settle_time": float(payload.get("settle_time", 4.0)), + # #300 follow-up: per-request SoM routing override (None → + # deployment env). Forwarded so the executor can flip it. + "_route_som_clicks": payload.get("route_som_clicks"), + # Identity: forwarded so the executor mints a searchable, + # workflow-prefixed Augur run_id (the logprobs session). + "_workflow_id": payload.get("workflow_id", ""), + # Proxy controls (allowlist-gated upstream). + "_proxy_city": payload.get("proxy_city") or "", + "_proxy_state": payload.get("proxy_state") or "", + "_proxy_disabled": bool(payload.get("proxy_disabled", False)), + # Human-readable plan family for the Augur Runs list. + "_plan_name": "pure_cua", + } + task_file_contents = json.dumps(task_suite) + + return _submit_detached_run( + tenant, + task_file_contents=task_file_contents, + model="holo3", # pure-cua is a Holo3-only path + profile_id=payload["profile_id"], + workflow_id=payload["workflow_id"], + state_key=payload["state_key"], + max_steps=int(payload.get("max_steps", 30)), + live_viewer=bool(payload.get("live_viewer")), + mode="pure_cua", + ) + def _do_action(payload: dict, tenant: TenantConfig) -> dict: """Handle ``action=status|result|cancel|resume`` against an existing run.""" run_id = str(payload.get("run_id") or "") @@ -4690,7 +5005,7 @@ def computer_plane(): @app.function( - gpu="A100-80GB", + gpu=["A100-80GB", "H100"], # H100 fallback: A100-80GB pool starves (2026-07-04) image=planner_base_image.run_commands( # #stealth-parity: install fonts/locale alongside the existing # Chrome + Xvfb deps. ``run_holo3`` is the most-used executor diff --git a/tests/test_holo3_pure_cua_branch.py b/tests/test_holo3_pure_cua_branch.py new file mode 100644 index 00000000..bf14c090 --- /dev/null +++ b/tests/test_holo3_pure_cua_branch.py @@ -0,0 +1,255 @@ +"""``_run_holo3_executor`` pure-CUA branch selection + logprobs guarantee. + +The Modal holo3 executor forks on ``task_suite["_mode"]``: + +* ``_mode == "pure_cua"`` → a bare ``GymRunner ↔ XdotoolGymEnv`` loop + (100%-student Holo3 actions), with a LIVE Augur adapter stamped onto + ``runner._augur`` so ``publish_modelio_context(layer="planner")`` in + ``GymRunner.run`` fires and per-token logprobs are captured. Without a + live ``runner._augur`` the planner modelio silently no-ops — the exact + gap this branch fixes. +* ``_micro_plan`` present → the orchestrated ``MicroPlanRunner`` path. + +These tests mock away the GPU / llama-server / Chrome setup so we can +assert branch selection + the ``runner._augur`` wiring without any real +infra. +""" + +from __future__ import annotations + +import importlib +import json +import sys +import types +from pathlib import Path + +import pytest + +pytest.importorskip("modal") + +_DEPLOY_MODAL = Path(__file__).resolve().parent.parent / "deploy" / "modal" +if str(_DEPLOY_MODAL) not in sys.path: + sys.path.insert(0, str(_DEPLOY_MODAL)) + + +# ── fakes ──────────────────────────────────────────────────────────── + + +class _FakeGymResult: + def __init__(self) -> None: + self.success = True + self.termination_reason = "done" + self.total_steps = 5 + self.trajectory = [object(), object(), object(), object(), object()] + + +class _FakeAugurAdapter: + """Stand-in for the real AugurAdapter — records construction + close.""" + + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + self.active = True + self.closed_with: str | None = None + + def close(self, status: str | None = None) -> None: + self.closed_with = status + + +class _ReachedMicro(Exception): + """Sentinel raised by the fake MicroPlanRunner ctor so a test can + prove the executor entered the micro branch (not the pure branch).""" + + +@pytest.fixture +def holo3_env(monkeypatch, tmp_path): + """Mock the GGUF download / llama-server / env / brain / runner so + ``_run_holo3_executor`` reaches its mode fork with zero real infra. + + Returns ``(mcs, created)`` where ``created`` is a bucket recording + the GymRunner + AugurAdapter instances the executor built. + """ + import modal_cua_server as mcs # type: ignore[import-not-found] + importlib.reload(mcs) + + # 1) Skip the GGUF download: point HOLO3_MODEL_DIR at a tmp dir whose + # ``.download_complete`` marker already exists. + model_dir = tmp_path / "holo3_gguf" + model_dir.mkdir() + (model_dir / ".download_complete").write_text("done") + monkeypatch.setattr(mcs, "HOLO3_MODEL_DIR", str(model_dir)) + + # 2) Stub the llama-server subprocess + readiness wait. + class _FakeProc: + def terminate(self) -> None: + return None + + monkeypatch.setattr(mcs.subprocess, "Popen", lambda *a, **k: _FakeProc()) + monkeypatch.setattr(mcs, "wait_for_openai_server", lambda *a, **k: None) + monkeypatch.setattr(mcs, "_kill_stale_inference_server", lambda *a, **k: None) + + # 3) Fake brain (built via ``from mantis_agent.brain_holo3 import Holo3Brain``). + class _FakeBrain: + model_name = "Holo3-35B-A3B" + + def __init__(self, *a, **k) -> None: + pass + + def load(self) -> None: + return None + + import mantis_agent.brain_holo3 as _bh + monkeypatch.setattr(_bh, "Holo3Brain", _FakeBrain) + + # 4) Fake env (no ``cdp_evaluate`` → PageDiscovery stays off) + viewer. + class _FakeEnv: + screen_size = (1280, 720) + + def close(self) -> None: + return None + + import mantis_agent.task_loop as _tl + monkeypatch.setattr(_tl, "setup_env", lambda **k: (_FakeEnv(), None, {})) + monkeypatch.setattr(_tl, "setup_viewer", lambda *a, **k: (None, None, "")) + + # 5) Fake the Claude helpers the setup path constructs unconditionally. + import mantis_agent.grounding as _gr + monkeypatch.setattr(_gr, "ClaudeGrounding", lambda *a, **k: object()) + import mantis_agent.extraction as _ex + monkeypatch.setattr(_ex, "ClaudeExtractor", lambda *a, **k: object()) + + # 6) Record the GymRunner + AugurAdapter the pure branch builds. + created: dict[str, object] = {} + + class _FakeGymRunner: + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + self._augur = None + created["runner"] = self + + def run(self, *, task, task_id, start_url): + self.run_call = {"task": task, "task_id": task_id, "start_url": start_url} + return _FakeGymResult() + + import mantis_agent.gym.runner as _rn + monkeypatch.setattr(_rn, "GymRunner", _FakeGymRunner) + + def _mk_augur(**kwargs): + adapter = _FakeAugurAdapter(**kwargs) + created["augur"] = adapter + return adapter + + import mantis_agent.observability.augur as _ag + monkeypatch.setattr(_ag, "AugurAdapter", _mk_augur) + + return mcs, created + + +# ── pure_cua branch ───────────────────────────────────────────────── + + +def test_pure_cua_branch_runs_gymrunner_with_live_augur(holo3_env) -> None: + mcs, created = holo3_env + suite = { + "session_name": "pure_cua", + "base_url": "https://example.com/start", + "_mode": "pure_cua", + "_instruction": "click the first result", + "_claude_fallback_disabled": True, + "_workflow_id": "tenantA__wf1", + } + + result = mcs._run_holo3_executor(json.dumps(suite), max_steps=25) + + # Selected the pure branch → built our fake GymRunner (not MicroPlanRunner). + runner = created["runner"] + assert runner.run_call["task"] == "click the first result" + assert runner.run_call["task_id"] == "cua" + assert runner.run_call["start_url"] == "https://example.com/start" + # grounding=None by default (brain-only); page_discovery off (no cdp). + assert runner.kwargs["grounding"] is None + assert runner.kwargs["page_discovery"] is None + + # THE logprobs guarantee: runner._augur is a LIVE (non-None) adapter, + # so publish_modelio_context(layer="planner") fires in GymRunner.run. + assert runner._augur is not None + assert runner._augur is created["augur"] + assert runner._augur.active is True + # Session id is workflow-prefixed (searchable) + uuid-suffixed (unique). + assert runner._augur.kwargs["run_id"].startswith("tenantA__wf1-") + assert runner._augur.kwargs["extra_tags"]["mode"] == "pure_cua" + # Adapter closed at teardown (mirrors run_executor._finalize). + assert runner._augur.closed_with == "succeeded" + + # Result envelope is lifecycle-compatible: honest terminal fields. + assert result["mode"] == "pure_cua" + assert result["success"] is True + assert result["steps"] == 5 + assert result["terminal_status"] == "completed" + assert result["halt_reason"] == "" + + +def test_pure_cua_halt_maps_to_halted_terminal_status(holo3_env, monkeypatch) -> None: + """A non-success gym result must surface an honest ``halted`` terminal + status (+ the termination_reason), never a blanket success.""" + mcs, created = holo3_env + + class _Halted(_FakeGymResult): + def __init__(self) -> None: + super().__init__() + self.success = False + self.termination_reason = "max_steps" + self.total_steps = 25 + + import mantis_agent.gym.runner as _rn + + class _HaltRunner: + def __init__(self, **kwargs) -> None: + self._augur = None + created["runner"] = self + + def run(self, *, task, task_id, start_url): + return _Halted() + + monkeypatch.setattr(_rn, "GymRunner", _HaltRunner) + + suite = { + "base_url": "https://example.com", + "_mode": "pure_cua", + "_instruction": "do a long thing", + "_claude_fallback_disabled": True, + } + result = mcs._run_holo3_executor(json.dumps(suite)) + + assert result["success"] is False + assert result["terminal_status"] == "halted" + assert result["halt_reason"] == "max_steps" + # Adapter still closed — with the halted status. + assert created["augur"].closed_with == "halted" + + +# ── negative: _micro_plan must NOT take the pure branch ────────────── + + +def test_micro_plan_does_not_take_pure_branch(holo3_env, monkeypatch) -> None: + mcs, created = holo3_env + + # A fake MicroPlanRunner ctor that flags the micro branch was reached. + def _boom(*a, **k): + raise _ReachedMicro() + + import mantis_agent.gym.micro_runner as _mr + monkeypatch.setattr(_mr, "MicroPlanRunner", _boom) + + suite = { + "session_name": "boattrader", + "base_url": "https://example.com", + # No ``_mode`` — but a ``_micro_plan`` is present. + "_micro_plan": [{"intent": "click the title", "type": "click"}], + "_claude_fallback_disabled": True, + } + + with pytest.raises(_ReachedMicro): + mcs._run_holo3_executor(json.dumps(suite)) + + # The pure GymRunner branch was NOT taken. + assert "runner" not in created diff --git a/tests/test_modal_cua_route.py b/tests/test_modal_cua_route.py new file mode 100644 index 00000000..eb3f495a --- /dev/null +++ b/tests/test_modal_cua_route.py @@ -0,0 +1,215 @@ +"""Tests for the Modal ``/v1/cua`` route — pure-Holo3 autonomous loop. + +Drives the FastAPI app through ``TestClient`` with a mocked executor +spawner (same harness as ``test_modal_endpoint.py``), so we exercise the +request → task_suite translation + shared submit lifecycle without going +through Modal's runtime or a real GPU. + +The route's job is narrow: validate through ``prepare_cua_payload``, +translate the ``PureCUARequest`` body into a ``_mode="pure_cua"`` suite, +force ``cua_model="holo3"``, and dispatch through the SAME +submit/spawn/status-write machinery ``/v1/predict`` uses. +""" + +from __future__ import annotations + +import importlib +import json +import sys +from pathlib import Path +from typing import Any + +import pytest + +# ``deploy/modal/modal_cua_server.py`` imports the ``modal`` SDK at module +# scope — a deploy-time dep, not a test dep. Skip cleanly when absent. +pytest.importorskip("modal") + +from fastapi.testclient import TestClient + +from mantis_agent import tenant_auth as ta_mod + +_DEPLOY_MODAL = Path(__file__).resolve().parent.parent / "deploy" / "modal" +if str(_DEPLOY_MODAL) not in sys.path: + sys.path.insert(0, str(_DEPLOY_MODAL)) + + +class _StubFunctionCall: + def __init__(self, result: Any = None) -> None: + self.object_id = "fc-stub-cua" + self._result = result + + def get(self, timeout: float = 0.1) -> Any: + return self._result + + def cancel(self) -> None: + return None + + +class _StubExecutor: + def __init__(self, call: _StubFunctionCall) -> None: + self.call = call + self.spawn_kwargs: dict[str, Any] = {} + + def spawn(self, *, task_file_contents: str, **kwargs) -> _StubFunctionCall: + self.spawn_kwargs = {"task_file_contents": task_file_contents, **kwargs} + return self.call + + +@pytest.fixture +def app_with_stub(monkeypatch, tmp_path): + monkeypatch.setenv("MANTIS_API_TOKEN", "test-token") + monkeypatch.setenv("MANTIS_DATA_DIR", str(tmp_path / "mantis-data")) + monkeypatch.delenv("MANTIS_TENANT_KEYS_PATH", raising=False) + ta_mod.reset_key_store() + + import modal_cua_server as mcs # type: ignore[import-not-found] + importlib.reload(mcs) + + stub_call = _StubFunctionCall(result={"mode": "pure_cua", "success": True}) + stub_executor = _StubExecutor(stub_call) + + app = mcs.build_api_app( + executor_resolver=lambda model: stub_executor, + function_call_lookup=lambda call_id: stub_call, + ) + return TestClient(app), stub_executor, stub_call + + +def _auth_headers() -> dict[str, str]: + return {"X-Mantis-Token": "test-token", "Content-Type": "application/json"} + + +# ── translation + dispatch ────────────────────────────────────────── + + +def test_cua_returns_queued_pure_cua_envelope(app_with_stub) -> None: + client, _executor, _call = app_with_stub + r = client.post( + "/v1/cua", + headers=_auth_headers(), + data=json.dumps({ + "instruction": "Find the first music event and click it", + "start_url": "https://lu.ma/discover", + "profile_id": "alice", + }), + ) + assert r.status_code == 200, r.text + body = r.json() + assert body["status"] == "queued" + assert body["mode"] == "pure_cua" + assert body["run_id"] + # Runs through the shared detached envelope → identity round-trips. + assert body["payload"]["profile_id"].endswith("__alice") + + +def test_cua_translates_request_into_pure_cua_suite(app_with_stub) -> None: + """The core contract: the request body becomes a ``_mode="pure_cua"`` + task_suite, dispatched to the holo3 executor with ``cua_model=holo3``. + """ + client, executor, _call = app_with_stub + r = client.post( + "/v1/cua", + headers=_auth_headers(), + data=json.dumps({ + "instruction": "Search for boats under $50k", + "start_url": "https://example.com/boats", + "max_steps": 17, + "settle_time": 6.0, + "ground_clicks": True, + "decompose": True, + "proxy_city": "austin", + "proxy_state": "tx", + "proxy_disabled": False, + "route_som_clicks": True, + "profile_id": "bob", + }), + ) + assert r.status_code == 200, r.text + + # Forced holo3 model on the spawn. + assert executor.spawn_kwargs["cua_model"] == "holo3" + # max_steps threaded to the spawn kwargs (shared submit path). + assert executor.spawn_kwargs["max_steps"] == 17 + + suite = json.loads(executor.spawn_kwargs["task_file_contents"]) + assert suite["_mode"] == "pure_cua" + assert suite["_instruction"] == "Search for boats under $50k" + assert suite["base_url"] == "https://example.com/boats" + assert suite["session_name"] == "pure_cua" + # Forwarded knobs. + assert suite["_max_steps"] == 17 + assert suite["_settle_time"] == 6.0 + assert suite["_ground_clicks"] is True + assert suite["_decompose"] is True + assert suite["_route_som_clicks"] is True + # Proxy fields forwarded verbatim. + assert suite["_proxy_city"] == "austin" + assert suite["_proxy_state"] == "tx" + assert suite["_proxy_disabled"] is False + + +def test_cua_defaults_match_pure_cua_request(app_with_stub) -> None: + """Unset optional knobs fall through to PureCUARequest defaults.""" + client, executor, _call = app_with_stub + r = client.post( + "/v1/cua", + headers=_auth_headers(), + data=json.dumps({"instruction": "do a thing", "profile_id": "carol"}), + ) + assert r.status_code == 200, r.text + suite = json.loads(executor.spawn_kwargs["task_file_contents"]) + assert suite["base_url"] == "" # start_url default + assert suite["_max_steps"] == 30 # max_steps default + assert suite["_ground_clicks"] is False # off by default + assert suite["_decompose"] is False # off by default + assert suite["_proxy_disabled"] is False + + +# ── auth + validation parity with /v1/predict ─────────────────────── + + +def test_cua_rejects_missing_token(app_with_stub) -> None: + client, _executor, _call = app_with_stub + r = client.post( + "/v1/cua", + headers={"Content-Type": "application/json"}, + data=json.dumps({"instruction": "x"}), + ) + assert r.status_code == 401 + + +def test_cua_rejects_missing_instruction(app_with_stub) -> None: + client, _executor, _call = app_with_stub + r = client.post( + "/v1/cua", + headers=_auth_headers(), + data=json.dumps({"start_url": "https://example.com"}), + ) + # PureCUARequest requires a non-empty instruction (min_length=1). + assert r.status_code == 400, r.text + + +def test_cua_status_lifecycle_is_shared_with_predict(app_with_stub) -> None: + """A /v1/cua run polls terminal status through the same _do_action + machinery as /v1/predict (result carries an honest terminal_status).""" + client, _executor, call = app_with_stub + call._result = { + "mode": "pure_cua", + "success": True, + "terminal_status": "completed", + "steps": 12, + } + submit = client.post( + "/v1/cua", + headers=_auth_headers(), + data=json.dumps({"instruction": "go", "profile_id": "dave"}), + ) + run_id = submit.json()["run_id"] + poll = client.post( + "/v1/predict", + headers=_auth_headers(), + data=json.dumps({"action": "status", "run_id": run_id}), + ) + assert poll.status_code == 200, poll.text + assert poll.json()["status"] == "succeeded" From 5dbeda076cd397f8fa1dae8c937f6b4eed1e0dc2 Mon Sep 17 00:00:00 2001 From: Barada Sahu Date: Sun, 5 Jul 2026 12:38:15 +0530 Subject: [PATCH 2/8] =?UTF-8?q?fix(cua):=20Holo3=20native-text=20parser=20?= =?UTF-8?q?drops=20normalized-float=20clicks=20=E2=86=92=20WAIT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /v1/cua pure-CUA loop drives Holo3's native "Action: click({...})" text format (Strategy 2 in _parse_response). That path took a SEPARATE coord conversion from the tool_calls path that the S09 fix repaired: ``_safe_int(0.324) == 0`` truncated the normalized fraction, ``_has_valid_xy`` then read it as the (0,0) origin and rejected the click, and the harness substituted WAIT(1s) — silently dropping a sensible, on-policy action the student had chosen (and often correctly predicted the outcome of). Only the tool_calls Strategy 1 had been routed through the fraction-aware ``_coords_to_screen``. Fix: route the native-text (Strategy 2) and JSON (Strategy 3) click / double_click branches through ``_coords_to_screen`` (same helper the tool_calls path uses), and validate x/y as FLOAT in ``_has_valid_xy`` so a fraction isn't truncated to the origin before the no-target gate. The #574 (0,0)-origin reject is preserved (now on the float value). Belongs with the /v1/cua endpoint (#960): without this, pure-CUA runs emit WAITs instead of the clicks the policy chose. Logprobs capture is unaffected/verified. Tests: normalized fraction now scales on the native + JSON paths (0.324→415, 0.5→640); explicit (0,0) and resized ints still honor the #574 contract. 68 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/mantis_agent/brain_holo3.py | 41 +++++++++++------ tests/test_holo3_normalized_coords.py | 66 +++++++++++++++++++++++++++ 2 files changed, 92 insertions(+), 15 deletions(-) diff --git a/src/mantis_agent/brain_holo3.py b/src/mantis_agent/brain_holo3.py index 71d050bb..8aac004d 100644 --- a/src/mantis_agent/brain_holo3.py +++ b/src/mantis_agent/brain_holo3.py @@ -759,15 +759,27 @@ def _has_valid_xy(args: dict) -> bool: explicit ``(0, 0)`` both indicate the model didn't pin a target — better to fall through to the WAIT(1s) fallback than dispatch to the page origin. + + Validate as FLOAT, not ``_safe_int``: Holo3's pure-CUA output + emits NORMALIZED fractions like ``click(x=0.324, y=0.338)``. + ``_safe_int`` truncated 0.324 → 0, so a perfectly good click + read as the (0,0) origin and got rejected — the harness then + substituted WAIT(1s) and silently dropped an on-policy action + (cua-issues 2026-06-29 S09, native-text path). ``_coords_to_screen`` + (used below) scales the fraction; here we only gate on it being + a real, non-origin target. """ if "x" not in args or "y" not in args: return False - x = _safe_int(args["x"], default=-1) - y = _safe_int(args["y"], default=-1) - if x < 0 or y < 0: - # Malformed value that didn't parse to a non-negative int. + try: + fx = float(str(args["x"]).strip()) + fy = float(str(args["y"]).strip()) + except (ValueError, TypeError): + # Malformed value that didn't parse to a number. + return False + if fx < 0 or fy < 0: return False - return (x, y) != (0, 0) + return (fx, fy) != (0.0, 0.0) # Map to Action if func_name in ("click",): @@ -780,17 +792,17 @@ def _has_valid_xy(args: dict) -> bool: # level → clean no-op step. if not _has_valid_xy(args): return None - x = _safe_int(args["x"]) - y = _safe_int(args["y"]) - sx, sy = _model_coords_to_screen(x, y, screen_size[0], screen_size[1]) + # #S09: normalized-fraction-aware conversion (same helper the + # tool_calls path uses via _convert_coords). Handles both + # ``0.324`` fractions and resized-space integers — do NOT + # pre-truncate with _safe_int (that reintroduces the origin bug). + sx, sy = self._coords_to_screen(args["x"], args["y"], screen_size) return Action(ActionType.CLICK, {"x": sx, "y": sy, "button": args.get("button", "left")}) if func_name in ("double_click", "doubleclick"): if not _has_valid_xy(args): return None - x = _safe_int(args["x"]) - y = _safe_int(args["y"]) - sx, sy = _model_coords_to_screen(x, y, screen_size[0], screen_size[1]) + sx, sy = self._coords_to_screen(args["x"], args["y"], screen_size) return Action(ActionType.DOUBLE_CLICK, {"x": sx, "y": sy}) if func_name in ("type_text", "type", "typewrite"): @@ -848,14 +860,13 @@ def _parse_json_action(self, text: str, screen_size: tuple[int, int]) -> Action return self._parse_holo3_action(code, screen_size) if action_type == "click": - x, y = int(obj.get("x", 0)), int(obj.get("y", 0)) - sx, sy = _model_coords_to_screen(x, y, screen_size[0], screen_size[1]) + # #S09: normalized-fraction-aware (int() truncated 0.324 → 0). + sx, sy = self._coords_to_screen(obj.get("x", 0), obj.get("y", 0), screen_size) button = obj.get("button", "left") return Action(ActionType.CLICK, {"x": sx, "y": sy, "button": button}) if action_type in ("double_click", "doubleclick", "doubleClick"): - x, y = int(obj.get("x", 0)), int(obj.get("y", 0)) - sx, sy = _model_coords_to_screen(x, y, screen_size[0], screen_size[1]) + sx, sy = self._coords_to_screen(obj.get("x", 0), obj.get("y", 0), screen_size) return Action(ActionType.DOUBLE_CLICK, {"x": sx, "y": sy}) if action_type in ("type", "type_text", "typewrite", "write"): diff --git a/tests/test_holo3_normalized_coords.py b/tests/test_holo3_normalized_coords.py index ac431bbe..300cdb87 100644 --- a/tests/test_holo3_normalized_coords.py +++ b/tests/test_holo3_normalized_coords.py @@ -10,12 +10,25 @@ from __future__ import annotations +from mantis_agent.actions import ActionType from mantis_agent.brain_holo3 import Holo3Brain SCREEN = (1280, 720) +def _parse_native(text: str): + return Holo3Brain._parse_holo3_action( + Holo3Brain.__new__(Holo3Brain), text, SCREEN, + ) + + +def _parse_json(text: str): + return Holo3Brain._parse_json_action( + Holo3Brain.__new__(Holo3Brain), text, SCREEN, + ) + + def test_normalized_fraction_scales_to_pixels(): # 0.324 * 1280 = 414.7 → 415 ; 0.338 * 720 = 243.4 → 243 x, y = Holo3Brain._coords_to_screen("0.324", "0.338", SCREEN) @@ -47,3 +60,56 @@ def test_integer_one_is_not_a_fraction(): # bare integer 1 (no dot) → resized-space, NOT full-width. x, y = Holo3Brain._coords_to_screen("1", "1", SCREEN) assert x < 1280 and y < 720 + + +# ── /v1/cua parse path (native "Action: name(...)" text) ──────────── +# The pure-CUA loop drives Holo3's native text format (Strategy 2 in +# _parse_response), which took a SEPARATE coord path from the tool_calls +# Strategy 1 that S09 fixed: ``_safe_int(0.324) == 0`` truncated the +# fraction, ``_has_valid_xy`` then read it as the (0,0) origin and +# rejected the click → the harness substituted WAIT(1s), silently +# dropping a sensible on-policy action. These lock the native path onto +# the same ``_coords_to_screen`` scaling. + + +def test_native_click_normalized_json_scales_not_dropped(): + action = _parse_native("Action: click({'x': 0.324, 'y': 0.338})") + assert action is not None + assert action.action_type == ActionType.CLICK + # 0.324*1280→415, 0.338*720→243 — NOT (0,0), NOT a WAIT. + assert (action.params["x"], action.params["y"]) == (415, 243) + + +def test_native_click_normalized_kv_scales(): + action = _parse_native("Action: click(x=0.5, y=0.5)") + assert action is not None + assert action.action_type == ActionType.CLICK + assert (action.params["x"], action.params["y"]) == (640, 360) + + +def test_native_double_click_normalized_scales(): + action = _parse_native("Action: double_click({'x': 0.5, 'y': 0.5})") + assert action is not None + assert action.action_type == ActionType.DOUBLE_CLICK + assert (action.params["x"], action.params["y"]) == (640, 360) + + +def test_native_click_explicit_zero_still_rejected(): + # #574 must survive: the true (0,0) origin is still a no-target reject. + assert _parse_native("Action: click({'x': 0, 'y': 0})") is None + + +def test_native_click_resized_integer_still_scales(): + # Regression guard for the #574 valid-int contract on the native path. + action = _parse_native("Action: click({'x': 500, 'y': 300})") + assert action is not None + assert action.action_type == ActionType.CLICK + assert action.params["x"] != 0 or action.params["y"] != 0 + + +def test_json_action_click_normalized_scales(): + # Strategy 3 (_parse_json_action) had the identical int() truncation. + action = _parse_json('{"action": "click", "x": 0.324, "y": 0.338}') + assert action is not None + assert action.action_type == ActionType.CLICK + assert (action.params["x"], action.params["y"]) == (415, 243) From 4bb91eaf15fd2c4426bf1a2e87a2f463e8686180 Mon Sep 17 00:00:00 2001 From: Barada Sahu Date: Sun, 5 Jul 2026 13:26:12 +0530 Subject: [PATCH 3/8] fix(cua): pure-CUA clicks via xdotool screen-space, not SoM/CDP (chrome-offset) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The deploy runs with MANTIS_ROUTE_SOM_CLICKS=enabled, which flips som_for_unstructured_clicks on globally. That routed pure-CUA brain-clicks through the SoM/CDP path (try_som_click → cdp_click_at_point), which translates screen→viewport by the browser chrome offset (outerHeight - innerHeight ≈ 95px). But the pure-CUA brain grounds on FULL-WINDOW screenshots, so every click landed ~95px too high — the model re-clicked a target it never actually hit and looped out (observed live: 6 iterations all clicking the same "Go to feed" button, 0 progress). Fix: the pure_cua branch forces som_for_unstructured_clicks=False, overriding the global flag, so clicks dispatch via env.step → xdotool in screen-space — the same coordinate space as the full-window screenshot (like the rest of the computer plane). Explicit _route_som_clicks:true still opts back into SoM routing per-request. Tests: pure-CUA forces xdotool even when MANTIS_ROUTE_SOM_CLICKS=enabled; explicit opt-in re-enables SoM. 5 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- deploy/modal/modal_cua_server.py | 28 ++++++++++++------- tests/test_holo3_pure_cua_branch.py | 42 +++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 9 deletions(-) diff --git a/deploy/modal/modal_cua_server.py b/deploy/modal/modal_cua_server.py index d263e02c..66279893 100644 --- a/deploy/modal/modal_cua_server.py +++ b/deploy/modal/modal_cua_server.py @@ -1108,16 +1108,26 @@ def _run_holo3_executor( from mantis_agent.gym.page_discovery import PageDiscovery pure_page_discovery = PageDiscovery(env=env) - # #300 follow-up: per-request RoutingPolicy override. ``None`` - # defers to ``MANTIS_ROUTE_SOM_CLICKS``. - pure_routing_policy = RoutingPolicy.from_env() + # Pure-CUA clicks dispatch in SCREEN space (xdotool), NOT via the + # SoM/CDP path — even when the global ``MANTIS_ROUTE_SOM_CLICKS`` is + # on. The brain grounds on FULL-WINDOW screenshots, but + # ``cdp_click_at_point`` translates screen→viewport by the chrome + # offset (~95px, ``outerHeight - innerHeight``). On a full-window + # pure-CUA frame that translation lands every click ~95px high, so + # the brain re-clicks a target it never actually hits and loops out. + # env.step → xdotool clicks the physical display at the same + # coordinates the brain saw (like the rest of the computer plane), + # so screenshot and click share one coordinate space. Callers can + # still opt back into SoM routing per-request via ``_route_som_clicks``. + _base_routing = RoutingPolicy.from_env() _route_override = task_suite.get("_route_som_clicks") - if _route_override is not None: - pure_routing_policy = RoutingPolicy( - plan_executor_enabled=pure_routing_policy.plan_executor_enabled, - som_enabled=pure_routing_policy.som_enabled, - som_for_unstructured_clicks=bool(_route_override), - ) + pure_routing_policy = RoutingPolicy( + plan_executor_enabled=_base_routing.plan_executor_enabled, + som_enabled=_base_routing.som_enabled, + som_for_unstructured_clicks=( + bool(_route_override) if _route_override is not None else False + ), + ) # #931 P2: opt-in screenshot grounding for click precision. Pure # CUA is brain-only by default (grounding=None); reuse the diff --git a/tests/test_holo3_pure_cua_branch.py b/tests/test_holo3_pure_cua_branch.py index bf14c090..911561a8 100644 --- a/tests/test_holo3_pure_cua_branch.py +++ b/tests/test_holo3_pure_cua_branch.py @@ -168,6 +168,9 @@ def test_pure_cua_branch_runs_gymrunner_with_live_augur(holo3_env) -> None: # grounding=None by default (brain-only); page_discovery off (no cdp). assert runner.kwargs["grounding"] is None assert runner.kwargs["page_discovery"] is None + # Clicks dispatch in screen-space (xdotool), NOT the SoM/CDP path — the + # full-window-screenshot ↔ viewport chrome-offset (~95px) click bug. + assert runner.kwargs["routing_policy"].som_for_unstructured_clicks is False # THE logprobs guarantee: runner._augur is a LIVE (non-None) adapter, # so publish_modelio_context(layer="planner") fires in GymRunner.run. @@ -227,6 +230,45 @@ def run(self, *, task, task_id, start_url): assert created["augur"].closed_with == "halted" +# ── click routing: xdotool screen-space, not SoM/CDP viewport ──────── + + +def test_pure_cua_forces_xdotool_even_when_global_som_on(holo3_env, monkeypatch) -> None: + """Deploy sets ``MANTIS_ROUTE_SOM_CLICKS=enabled``; pure-CUA must STILL + route clicks through xdotool (screen-space) — the SoM/CDP path applies a + ~95px chrome offset that misaligns with the full-window screenshot the + brain grounded on, so clicks land high and the loop re-clicks forever.""" + mcs, created = holo3_env + monkeypatch.setenv("MANTIS_ROUTE_SOM_CLICKS", "enabled") + + suite = { + "base_url": "https://example.com", + "_mode": "pure_cua", + "_instruction": "click the first result", + "_claude_fallback_disabled": True, + } + mcs._run_holo3_executor(json.dumps(suite), max_steps=6) + + assert created["runner"].kwargs["routing_policy"].som_for_unstructured_clicks is False + + +def test_pure_cua_route_som_clicks_opt_in_re_enables_som(holo3_env, monkeypatch) -> None: + """Explicit ``_route_som_clicks: true`` still opts back into SoM routing.""" + mcs, created = holo3_env + monkeypatch.setenv("MANTIS_ROUTE_SOM_CLICKS", "disabled") + + suite = { + "base_url": "https://example.com", + "_mode": "pure_cua", + "_instruction": "click the first result", + "_route_som_clicks": True, + "_claude_fallback_disabled": True, + } + mcs._run_holo3_executor(json.dumps(suite), max_steps=6) + + assert created["runner"].kwargs["routing_policy"].som_for_unstructured_clicks is True + + # ── negative: _micro_plan must NOT take the pure branch ────────────── From c5ee9b812494f8eb1a7289aae385d86f4d294d42 Mon Sep 17 00:00:00 2001 From: Barada Sahu Date: Sun, 5 Jul 2026 17:20:22 +0530 Subject: [PATCH 4/8] =?UTF-8?q?fix(cua):=20forward=20=5Fbrowser=5Fextra=5F?= =?UTF-8?q?headers=20on=20/v1/cua=20=E2=86=92=20unblock=20Daytona=20rollou?= =?UTF-8?q?ts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The /v1/cua route validated the request (PureCUARequest, extra:allow) but never copied ``_browser_extra_headers`` into the ``_mode="pure_cua"`` task_suite it builds, so the executor's ``setup_env(extra_http_headers=…)`` always got None. The browser therefore never sent Daytona's ``X-Daytona-Skip-Preview-Warning`` / ``X-Daytona-Preview-Token`` headers, and every Daytona-hosted rollout stranded on the preview interstitial at frame one (confirmed live: URL bar showed /feed/ but the frame was the warning page). Modal-hosted envs were unaffected (no header needed), which masked it. Fix: forward the headers into the suite (accept ``_browser_extra_headers`` or a bare ``browser_extra_headers``). The env already applies them persistently via ``Network.setExtraHTTPHeaders`` (xdotool_env), which honors the ``x-daytona-*`` headers on every request. Tests: the route-translation test now asserts the headers land in the suite. 6 passed. Co-Authored-By: Claude Opus 4.8 (1M context) --- deploy/modal/modal_cua_server.py | 11 +++++++++++ tests/test_modal_cua_route.py | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/deploy/modal/modal_cua_server.py b/deploy/modal/modal_cua_server.py index 66279893..e08020e9 100644 --- a/deploy/modal/modal_cua_server.py +++ b/deploy/modal/modal_cua_server.py @@ -3310,6 +3310,17 @@ async def cua( # #300 follow-up: per-request SoM routing override (None → # deployment env). Forwarded so the executor can flip it. "_route_som_clicks": payload.get("route_som_clicks"), + # Persistent browser request headers (CDP Network.setExtraHTTPHeaders). + # WITHOUT forwarding this the browser never sends Daytona's + # ``X-Daytona-Skip-Preview-Warning`` / ``X-Daytona-Preview-Token``, + # so every Daytona rollout strands on the preview interstitial at + # frame one. Accept either the documented ``_browser_extra_headers`` + # or a bare ``browser_extra_headers`` (``extra: allow`` keeps both). + "_browser_extra_headers": ( + payload.get("_browser_extra_headers") + or payload.get("browser_extra_headers") + or None + ), # Identity: forwarded so the executor mints a searchable, # workflow-prefixed Augur run_id (the logprobs session). "_workflow_id": payload.get("workflow_id", ""), diff --git a/tests/test_modal_cua_route.py b/tests/test_modal_cua_route.py index eb3f495a..4440c710 100644 --- a/tests/test_modal_cua_route.py +++ b/tests/test_modal_cua_route.py @@ -123,6 +123,10 @@ def test_cua_translates_request_into_pure_cua_suite(app_with_stub) -> None: "proxy_disabled": False, "route_som_clicks": True, "profile_id": "bob", + "_browser_extra_headers": { + "X-Daytona-Skip-Preview-Warning": "true", + "X-Daytona-Preview-Token": "pt123", + }, }), ) assert r.status_code == 200, r.text @@ -147,6 +151,12 @@ def test_cua_translates_request_into_pure_cua_suite(app_with_stub) -> None: assert suite["_proxy_city"] == "austin" assert suite["_proxy_state"] == "tx" assert suite["_proxy_disabled"] is False + # Browser headers forwarded (Daytona skip-warning + preview-token) — without + # this every Daytona rollout strands on the preview interstitial. + assert suite["_browser_extra_headers"] == { + "X-Daytona-Skip-Preview-Warning": "true", + "X-Daytona-Preview-Token": "pt123", + } def test_cua_defaults_match_pure_cua_request(app_with_stub) -> None: From bef7845fe2c6a789c50f878f3eee71d7382ee74b Mon Sep 17 00:00:00 2001 From: Barada Sahu Date: Sun, 5 Jul 2026 19:12:00 +0530 Subject: [PATCH 5/8] fix(cua): kill false "no visible change" on navigations + Daytona auto-inject + imgdim probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three fixes that unblock the graded E1 baseline (the 0/12 was dominated by the annotation defect, not model incapacity): 1. runner.py — URL-diff guard on the perceptual-diff verifier. A click that NAVIGATED is a real change even when pre/post frames hash identical (the new page renders after the post-action capture, or looks similar). Firing "no observed effect" there contradicts the "page navigated to " line _build_feedback already added, poisoning context until loop-recovery thrashes the step budget — the single largest rollout-quality depressor. Snapshot the pre-action URL and suppress the warning when it changed. Also carries the [cua-offset] grounding-delta WARNING (aim observability). 2. modal_cua_server.py — /v1/cua auto-injects X-Daytona-Skip-Preview-Warning for any daytonaproxy start_url, so callers only pass the preview token and never strand on the interstitial by forgetting the skip header. 3. brain_holo3.py — [cua-imgdim] WARNING logs the model's actual frame size vs the assumed screen_size, to root-cause the Holo3 y-over-emit (pairs with the raw coord in modelio + mapped/grounded in [cua-offset]). Tests: navigation suppresses the false warning + keeps "navigated to"; the /v1/cua auto-inject fires for Daytona URLs, not others. Co-Authored-By: Claude Opus 4.8 (1M context) --- deploy/modal/modal_cua_server.py | 26 ++++++++--- src/mantis_agent/brain_holo3.py | 20 +++++++++ src/mantis_agent/gym/runner.py | 33 +++++++++++++- tests/test_gym_runner_perceptual_diff.py | 56 ++++++++++++++++++++++++ tests/test_modal_cua_route.py | 39 +++++++++++++++++ 5 files changed, 166 insertions(+), 8 deletions(-) diff --git a/deploy/modal/modal_cua_server.py b/deploy/modal/modal_cua_server.py index e08020e9..67afe6de 100644 --- a/deploy/modal/modal_cua_server.py +++ b/deploy/modal/modal_cua_server.py @@ -3294,6 +3294,23 @@ async def cua( if not instruction: raise HTTPException(400, "instruction is required") + # Persistent browser headers. AUTO-INJECT the Daytona skip-warning + # header when start_url is a Daytona preview URL, so callers never + # strand on the interstitial by forgetting it — the org-wide removal + # (Tier 3 / custom proxy) is the dashboard-side path; this is the + # per-request safety net. The preview TOKEN stays caller-supplied + # (it's per-sandbox and can't be inferred). + _start_url = str(payload.get("start_url") or "") + _bxh = ( + payload.get("_browser_extra_headers") + or payload.get("browser_extra_headers") + or {} + ) + if not isinstance(_bxh, dict): + _bxh = {} + if "daytonaproxy" in _start_url: + _bxh.setdefault("X-Daytona-Skip-Preview-Warning", "true") + # Translate the validated request into the pure-cua suite the # holo3 executor forks on. Mirrors the ``task_suite`` shape # ``baseten_server.runtime._run_pure_cua`` builds, plus the @@ -3314,13 +3331,8 @@ async def cua( # WITHOUT forwarding this the browser never sends Daytona's # ``X-Daytona-Skip-Preview-Warning`` / ``X-Daytona-Preview-Token``, # so every Daytona rollout strands on the preview interstitial at - # frame one. Accept either the documented ``_browser_extra_headers`` - # or a bare ``browser_extra_headers`` (``extra: allow`` keeps both). - "_browser_extra_headers": ( - payload.get("_browser_extra_headers") - or payload.get("browser_extra_headers") - or None - ), + # frame one; the Daytona skip-warning header is auto-added above. + "_browser_extra_headers": _bxh or None, # Identity: forwarded so the executor mints a searchable, # workflow-prefixed Augur run_id (the logprobs session). "_workflow_id": payload.get("workflow_id", ""), diff --git a/src/mantis_agent/brain_holo3.py b/src/mantis_agent/brain_holo3.py index 8aac004d..84055149 100644 --- a/src/mantis_agent/brain_holo3.py +++ b/src/mantis_agent/brain_holo3.py @@ -473,6 +473,26 @@ def _build_messages( content: list[dict] = [] n_frames = len(frames) + # Aim diagnostic (WARNING so it survives Modal log suppression): the + # image the model localizes on vs the screen_size the coord-mapping + # assumes. If the CURRENT frame is TALLER than screen_size the model + # over-emits y and _model_coords_to_screen (which divides by the + # smart-resize of screen_size) can dispatch off-screen. Pairs with the + # raw coord in modelio ``response.text`` and the mapped/grounded coord + # in the runner's ``[cua-offset]`` line to fully explain any y-miss. + if n_frames and hasattr(frames[-1], "size"): + _cw, _ch = frames[-1].size + if (_cw, _ch) != (screen_size[0], screen_size[1]): + logger.warning( + "[cua-imgdim] current_frame=%sx%s screen_size=%sx%s " + "(MISMATCH → coord space skewed)", + _cw, _ch, screen_size[0], screen_size[1], + ) + else: + logger.warning( + "[cua-imgdim] current_frame=%sx%s screen_size=%sx%s (match)", + _cw, _ch, screen_size[0], screen_size[1], + ) for i, frame in enumerate(frames): label = "CURRENT" if i == n_frames - 1 else f"t-{n_frames - 1 - i}" content.append({"type": "text", "text": f"[Frame {label}]"}) diff --git a/src/mantis_agent/gym/runner.py b/src/mantis_agent/gym/runner.py index 20d724ba..2d3668f8 100644 --- a/src/mantis_agent/gym/runner.py +++ b/src/mantis_agent/gym/runner.py @@ -1630,6 +1630,22 @@ def _save_frame(img: Any, step: int) -> None: initial_x=orig_x, initial_y=orig_y, ) + # Offset measurement (WARNING so it survives Modal + # log suppression): the model's raw click coord vs + # the grounded (actual-element) coord IS the exact + # per-click offset. Uniform ~chrome-height vertical + # → a systematic mapping shift; large/variable 2D → + # Holo3 spatial-grounding imprecision. + try: + _dx = int(gr.x) - int(orig_x) + _dy = int(gr.y) - int(orig_y) + except (TypeError, ValueError): + _dx = _dy = None + logger.warning( + "[cua-offset] model=(%s,%s) grounded=(%s,%s) " + "delta=(%s,%s) conf=%.2f", + orig_x, orig_y, gr.x, gr.y, _dx, _dy, gr.confidence, + ) if gr.confidence > 0.3: print(f" [grounding] ({orig_x},{orig_y}) → ({gr.x},{gr.y}) conf={gr.confidence:.1f}") action = Action( @@ -1683,6 +1699,10 @@ def _save_frame(img: Any, step: int) -> None: # screenshot the brain reasoned on, so it's the right # baseline for "did this action change the page?". pre_action_frame = frame_history[-1] if frame_history else None + # Snapshot the pre-action URL before ``last_url`` is reassigned + # to the post-action URL below — the URL-diff guard in the + # perceptual-diff block needs the BEFORE value. + _pre_action_url = last_url for pre_action in pre_actions: logger.warning( @@ -1903,12 +1923,23 @@ def _save_frame(img: Any, step: int) -> None: and isinstance(focused_input, dict) and focused_input.get("focused_via_click") ) + # URL-diff guard: a click that NAVIGATED is a real change even + # when the perceptual-diff sees near-identical frames (the new + # page renders after the post-action capture, or simply looks + # similar). Firing "no observed effect" here contradicts the + # "page navigated to " line ``_build_feedback`` already + # added and traps otherwise-recoverable trajectories in + # Return-loops — the single largest rollout-quality depressor. + # Uses the URL the env already surfaces in ``info`` (post-action + # verification, not target derivation). + _post_url = str(gym_result.info.get("url", "") or "") + _navigated_this_step = bool(_post_url and _post_url != _pre_action_url) if clicked_into_field: feedback = ( f"{feedback}; input field focused — type your text now " f"(do not click it again)" ) - elif effect_check.effect_observed is False: + elif effect_check.effect_observed is False and not _navigated_this_step: feedback = ( f"{feedback}; WARNING: high-risk action had no observed effect " f"({effect_check.reason})" diff --git a/tests/test_gym_runner_perceptual_diff.py b/tests/test_gym_runner_perceptual_diff.py index 38f2d526..b0f62222 100644 --- a/tests/test_gym_runner_perceptual_diff.py +++ b/tests/test_gym_runner_perceptual_diff.py @@ -273,3 +273,59 @@ def test_trajectory_step_round_trips_action_effect_observed() -> None: assert payload["action_effect_observed"] is False rehydrated = _trajectory_step_from_dict(payload) assert rehydrated.action_effect_observed is False + + +# ── URL-diff guard: navigation must not read as "no observed effect" ──── + + +class _StaticFrameNavEnv(GymEnvironment): + """Identical screenshot every step (perceptual-diff sees no pixel change) + BUT the URL changes — a real navigation to a similar-looking page. The + URL-diff guard must suppress the false "no observed effect" warning that + would otherwise trap the brain in a Return-loop.""" + + def __init__(self) -> None: + self._frame = _noisy_img(seed=42) + self._step = 0 + + @property + def screen_size(self) -> tuple[int, int]: + return (200, 200) + + def reset(self, task: str, **kwargs: Any) -> GymObservation: + self._step = 0 + return GymObservation(screenshot=self._frame) + + def step(self, action: Action) -> GymResult: + self._step += 1 + return GymResult( + GymObservation(screenshot=self._frame), 0.0, False, + info={"url": f"https://example.com/page{self._step}"}, + ) + + def close(self) -> None: + pass + + +def test_navigation_suppresses_false_no_effect_warning( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A high-risk click whose post-frame is pixel-identical BUT whose URL + changed is a real navigation — the perceptual-diff must NOT warn "no + observed effect", and the feedback should carry the navigation instead.""" + monkeypatch.delenv("MANTIS_PERCEPTUAL_VERIFY", raising=False) + + runner = GymRunner( + brain=_ScriptedBrain([_high_risk_click()]), + env=_StaticFrameNavEnv(), + max_steps=3, + ) + result = runner.run("task") + click_step = next( + t for t in result.trajectory + if t.action.action_type == ActionType.CLICK + ) + # URL-diff guard suppressed the contradictory warning... + assert "no observed effect" not in click_step.feedback + # ...and the brain still gets the positive navigation signal. + assert "navigated to" in click_step.feedback diff --git a/tests/test_modal_cua_route.py b/tests/test_modal_cua_route.py index 4440c710..33cfcd55 100644 --- a/tests/test_modal_cua_route.py +++ b/tests/test_modal_cua_route.py @@ -223,3 +223,42 @@ def test_cua_status_lifecycle_is_shared_with_predict(app_with_stub) -> None: ) assert poll.status_code == 200, poll.text assert poll.json()["status"] == "succeeded" + + +def test_cua_autoinjects_daytona_skip_header(app_with_stub) -> None: + """A daytonaproxy start_url auto-adds X-Daytona-Skip-Preview-Warning even + when the caller only passed the preview token — no more stranding on the + interstitial by forgetting the skip header.""" + client, executor, _call = app_with_stub + r = client.post( + "/v1/cua", + headers=_auth_headers(), + data=json.dumps({ + "instruction": "read the feed", + "start_url": "https://8080-abc.daytonaproxy01.net/feed/", + "_browser_extra_headers": {"X-Daytona-Preview-Token": "pt9"}, + "profile_id": "bob", + }), + ) + assert r.status_code == 200, r.text + suite = json.loads(executor.spawn_kwargs["task_file_contents"]) + hdrs = suite["_browser_extra_headers"] + assert hdrs["X-Daytona-Skip-Preview-Warning"] == "true" # auto-injected + assert hdrs["X-Daytona-Preview-Token"] == "pt9" # caller-supplied kept + + +def test_cua_no_autoinject_for_non_daytona(app_with_stub) -> None: + """Non-Daytona start_url must NOT get the skip header.""" + client, executor, _call = app_with_stub + r = client.post( + "/v1/cua", + headers=_auth_headers(), + data=json.dumps({ + "instruction": "read", + "start_url": "https://example.com/", + "profile_id": "bob", + }), + ) + assert r.status_code == 200, r.text + suite = json.loads(executor.spawn_kwargs["task_file_contents"]) + assert suite["_browser_extra_headers"] is None From 5b9d1b96385c5409a83da15119c76f3c8f82c730 Mon Sep 17 00:00:00 2001 From: Barada Sahu Date: Sun, 5 Jul 2026 19:29:13 +0530 Subject: [PATCH 6/8] =?UTF-8?q?fix(cua):=20populate=20info["url"]=20in=20X?= =?UTF-8?q?dotoolGymEnv.step=20=E2=80=94=20kill=20false=20"no=20visible=20?= =?UTF-8?q?change"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the "clicked (no visible change)" trap (the dominant depressor of E1 rollout quality): the env's step() built ``info = {}`` and set ``type_verified`` / ``focused_input`` but NEVER ``info["url"]``. Both the "page navigated to " annotation and the ``clicked (no visible change)`` gate in ``_build_feedback`` read ``gym_result.info.get("url")`` — so with no URL in info, a navigating click never produced a "navigated to" line, fell through to "clicked (no visible change)", and the brain thrashed the step budget re-clicking a target that had in fact navigated. Fix: step() now populates ``info["url"] = current_url`` (best-effort CDP read, post-action verification — not target derivation). "page navigated to" fires on a real nav → parts non-empty → the false "no visible change" is suppressed by the existing ``not parts`` gate. Complements the earlier perceptual-diff URL-diff guard (bef7845), which targeted the separate "no observed effect" string. Known minor edge (noted for follow-up): step 1 can emit one "navigated to " since last_url starts empty — harmless, single step. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/mantis_agent/gym/xdotool_env.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/mantis_agent/gym/xdotool_env.py b/src/mantis_agent/gym/xdotool_env.py index 1ba5fcfc..e04014bc 100644 --- a/src/mantis_agent/gym/xdotool_env.py +++ b/src/mantis_agent/gym/xdotool_env.py @@ -1745,6 +1745,20 @@ def step(self, action: Action) -> GymResult: "focused_via_click": True, "value": str(probe.get("text") or ""), } + + # Post-action URL so the runner's navigation-aware feedback works. + # ``_build_feedback`` (and the perceptual-diff URL-diff guard) read + # ``info["url"]`` to emit "page navigated to " and to SUPPRESS the + # false "clicked (no visible change)" on a real navigation — the single + # largest rollout-quality depressor. Without this the info dict never + # carries a URL, so every navigating click reads as a no-op and the + # brain thrashes the step budget in Return-loops. Best-effort CDP read + # (post-action verification, not target derivation). + try: + if hasattr(self, "current_url"): + info["url"] = self.current_url or "" + except Exception: # noqa: BLE001 — feedback URL is best-effort + pass return GymResult(observation=obs, reward=0.0, done=False, info=info) def close(self) -> None: From d21a3f99c1e6842b728d2088d50d72aa377f4801 Mon Sep 17 00:00:00 2001 From: Barada Sahu Date: Sun, 5 Jul 2026 22:03:35 +0530 Subject: [PATCH 7/8] =?UTF-8?q?fix(cua):=20cdp=5Finput=5Finsert=20?= =?UTF-8?q?=E2=80=94=20focus-hardened=20