diff --git a/deploy/modal/modal_cua_server.py b/deploy/modal/modal_cua_server.py index 1b1db693..67afe6de 100644 --- a/deploy/modal/modal_cua_server.py +++ b/deploy/modal/modal_cua_server.py @@ -1068,6 +1068,224 @@ def _run_holo3_executor( except Exception as exc: # noqa: BLE001 print(f" WARNING: external_pause init failed: {exc}") + # ── Pure-CUA mode: autonomous Holo3 loop (100% student actions) ── + # Sibling of the ``_micro_plan`` fork below, selected by the /v1/cua + # route (``_mode == "pure_cua"``). Mirrors + # ``baseten_server.runtime.run_pure_cua``: a bare GymRunner ↔ + # XdotoolGymEnv loop with NO decomposer / grounding / extractor on the + # default path — whatever the brain emits, xdotool executes. + # + # The whole point of this path is planner-layer modelio + per-token + # logprobs on every step (for RL / SFT curation). GymRunner.run + # captures that via ``publish_modelio_context(_planner_augur, + # layer="planner", …)`` where ``_planner_augur`` falls back to + # ``getattr(self, "_augur", None)`` when no modelio context is active + # (the pure-cua case — nothing wraps a ``model`` context around this + # loop). Nothing sets ``runner._augur`` on this path by default, so on + # Baseten it silently no-ops (zero logprobs). We fix that here: mint an + # ``augur_run_id`` and open a LIVE AugurAdapter onto ``runner._augur`` + # so ``publish_modelio_context`` (which requires ``adapter.active``) + # fires and ``Holo3Brain._record_modelio_if_active`` writes + # ``layer="planner"`` records. ``MANTIS_CAPTURE_LOGPROBS`` is on in the + # deploy env, so the SDK stamps per-token logprobs onto those records. + if task_suite.get("_mode") == "pure_cua": + from mantis_agent.gym.runner import GymRunner, RoutingPolicy + from mantis_agent.observability.augur import AugurAdapter + + instruction = str(task_suite.get("_instruction") or "").strip() + if not instruction: + raise ValueError("pure_cua requires a non-empty _instruction") + + pure_max_steps = int(task_suite.get("_max_steps", max_steps) or max_steps) + print(f"\n === PURE-CUA MODE (holo3, max_steps={pure_max_steps}) ===") + print(f" Instruction: {instruction[:120]}") + + # #300: attach PageDiscovery when the env exposes the CDP DOM shim + # so the SoM click path can fire (no-op on legacy callers). Mirrors + # the Baseten pure-cua wiring. + pure_page_discovery = None + if hasattr(env, "cdp_evaluate"): + from mantis_agent.gym.page_discovery import PageDiscovery + pure_page_discovery = PageDiscovery(env=env) + + # Pure-CUA clicks dispatch in SCREEN space (xdotool), NOT via the + # SoM/CDP path — even when the global ``MANTIS_ROUTE_SOM_CLICKS`` is + # on. The brain grounds on FULL-WINDOW screenshots, but + # ``cdp_click_at_point`` translates screen→viewport by the chrome + # offset (~95px, ``outerHeight - innerHeight``). On a full-window + # pure-CUA frame that translation lands every click ~95px high, so + # the brain re-clicks a target it never actually hits and loops out. + # env.step → xdotool clicks the physical display at the same + # coordinates the brain saw (like the rest of the computer plane), + # so screenshot and click share one coordinate space. Callers can + # still opt back into SoM routing per-request via ``_route_som_clicks``. + _base_routing = RoutingPolicy.from_env() + _route_override = task_suite.get("_route_som_clicks") + pure_routing_policy = RoutingPolicy( + plan_executor_enabled=_base_routing.plan_executor_enabled, + som_enabled=_base_routing.som_enabled, + som_for_unstructured_clicks=( + bool(_route_override) if _route_override is not None else False + ), + ) + + # #931 P2: opt-in screenshot grounding for click precision. Pure + # CUA is brain-only by default (grounding=None); reuse the + # ClaudeGrounding built above only when the caller opted in AND a + # key is present — otherwise brain-only (never hard-fail). + pure_grounding = None + if bool(task_suite.get("_ground_clicks")) and os.environ.get("ANTHROPIC_API_KEY"): + pure_grounding = grounding + print(" [pure_cua] ground_clicks=true — screenshot grounding enabled") + elif bool(task_suite.get("_ground_clicks")): + print( + " [pure_cua] ground_clicks requested but no ANTHROPIC_API_KEY " + "— running brain-only" + ) + + pure_runner = GymRunner( + brain=brain, + env=env, + max_steps=pure_max_steps, + frames_per_inference=frames_per_inference, + grounding=pure_grounding, # None = pure brain; set when ground_clicks opts in + page_discovery=pure_page_discovery, + routing_policy=pure_routing_policy, + on_step=viewer_event_bus.emit if viewer_event_bus else None, + ) + + # #649-shaped session id: workflow_id-prefixed for searchability, + # uuid-suffixed for uniqueness (same recipe as the micro branch). + import uuid as _uuid_for_cua + _pure_workflow_id = str(task_suite.get("_workflow_id", "") or "") + pure_augur_run_id = ( + f"{_pure_workflow_id}-{_uuid_for_cua.uuid4().hex[:8]}" + if _pure_workflow_id else _uuid_for_cua.uuid4().hex[:12] + ) + try: + if api_run_id and api_tenant_id: + _write_augur_metadata(api_tenant_id, api_run_id, pure_augur_run_id) + except Exception as _augur_exc: # noqa: BLE001 + print(f" WARNING: augur metadata write failed: {_augur_exc}") + + # THE logprobs-capture guarantee. The micro path opens this adapter + # inside ``run_executor`` and stamps it on ``runner._augur``; the + # pure-cua path doesn't go through run_executor, so we open it here + # and assign it ourselves. ``AugurAdapter.__init__`` drives + # ``DebugSession.__enter__`` so ``.active`` is True on return — the + # precondition ``publish_modelio_context`` checks before publishing. + _pure_model_name = str(getattr(brain, "model_name", "") or "") + pure_runner._augur = AugurAdapter( + run_id=pure_augur_run_id, + tenant_id=str(api_tenant_id or ""), + session_name=session_name, + extra_tags={ + "mode": "pure_cua", + "workflow_id": _pure_workflow_id, + "model": _pure_model_name, + }, + brain_model_name=_pure_model_name, + ) + + # #931 P3: opt-in decompose-then-cua. Seed an ordered sub-goal + # roadmap up front (Claude) and drive the brain with the augmented + # task. Falls back to the raw instruction on no-key / failure — + # never hard-fails the run. + pure_task = instruction + if bool(task_suite.get("_decompose")) and os.environ.get("ANTHROPIC_API_KEY"): + try: + from mantis_agent.plan_decomposer import PlanDecomposer + _dplan = PlanDecomposer( + api_key=os.environ["ANTHROPIC_API_KEY"] + ).decompose_text(instruction) + _subgoals = [ + s.intent for s in getattr(_dplan, "steps", []) + if getattr(s, "intent", "") + ] + if _subgoals: + _lines = "\n".join( + f" {i + 1}. {s}" for i, s in enumerate(_subgoals) + ) + pure_task = ( + f"{instruction}\n\nComplete the task as the following " + "ordered sub-goals. Do them in order and confirm each " + "visibly succeeded before moving to the next:\n" + f"{_lines}" + ) + print(f" [pure_cua] decompose=true — seeded {len(_subgoals)} sub-goals") + except Exception as _de: # noqa: BLE001 — fall back to raw instruction + print(f" [pure_cua] decompose failed ({_de}) — raw instruction") + elif bool(task_suite.get("_decompose")): + print( + " [pure_cua] decompose requested but no ANTHROPIC_API_KEY " + "— raw instruction" + ) + + pure_gym_result = None + try: + pure_gym_result = pure_runner.run( + task=pure_task, task_id="cua", start_url=base_url, + ) + finally: + # Flush the Augur bundle (mirrors ``run_executor._finalize``). + # Best-effort — telemetry teardown never masks a run error. + try: + _pure_term = ( + "succeeded" + if (pure_gym_result is not None and pure_gym_result.success) + else "halted" + ) + pure_runner._augur.close(_pure_term) + except Exception as _close_exc: # noqa: BLE001 + print(f" WARNING: augur close failed: {_close_exc}") + + pure_elapsed = time.time() - t0 + pure_trajectory = list(getattr(pure_gym_result, "trajectory", []) or []) + pure_success = bool(pure_gym_result.success) + pure_termination = str( + getattr(pure_gym_result, "termination_reason", "") or "" + ) + # Honest terminal fields (cua-issues 2026-06-29): the /v1/cua + # envelope MUST carry ``terminal_status`` / ``halt_reason`` so + # ``_do_action`` maps the wire status truthfully — success → + # ``completed`` → ``succeeded``; anything else → ``halted`` with + # the termination_reason. Without these the poll path would stamp + # a blanket success on every run. + pure_terminal_status = "completed" if pure_success else "halted" + pure_halt_reason = "" if pure_success else pure_termination + + result = { + "run_id": run_id, + "mode": "pure_cua", + "provider": "modal", + "session_name": session_name, + "model": _pure_model_name or "holo3", + "instruction": instruction, + "start_url": base_url, + "success": pure_success, + "termination_reason": pure_termination, + "terminal_status": pure_terminal_status, + "halt_reason": pure_halt_reason, + "steps": int(pure_gym_result.total_steps), + "duration_s": round(pure_elapsed), + "elapsed_seconds": pure_elapsed, + "trajectory_len": len(pure_trajectory), + "augur_run_id": pure_augur_run_id, + } + print( + f"\n Pure-CUA complete: success={pure_success} " + f"steps={result['steps']} reason={pure_termination or ''}" + ) + + env.close() + llama_proc.terminate() + if viewer_ctx: + try: + viewer_ctx.__exit__(None, None, None) + except Exception: + pass + return result + # ── Micro-intent mode: run MicroPlanRunner ── micro_plan_data = task_suite.get("_micro_plan") if micro_plan_data: @@ -2800,6 +3018,7 @@ def build_api_app(executor_resolver=None, function_call_lookup=None, from mantis_agent.server.run_dispatch import ( DispatchError, acquire_profile_lock, + prepare_cua_payload, prepare_predict_payload, read_profile_lock, release_profile_lock as _release_phase1_profile_lock, @@ -2858,54 +3077,29 @@ def models() -> dict: ], } - @fastapi_app.post("/v1/predict") - async def predict( - request: Request, - tenant: TenantConfig = Depends(require_run_scope), + def _submit_detached_run( + tenant: TenantConfig, + *, + task_file_contents: str, + model: str, + profile_id: str, + workflow_id: str, + state_key: str, + max_steps: int, + live_viewer: bool = False, + mode: str = "detached", ) -> dict: - try: - raw = await request.json() - except Exception: - raise HTTPException(400, "request body must be JSON") - try: - payload = prepare_predict_payload(raw, tenant) - except DispatchError as exc: - raise HTTPException(exc.status_code, exc.detail) from exc - - # Action mode: poll/cancel an existing run. - if payload.get("action"): - return _do_action(payload, tenant) - - # Run mode: build suite, lock the profile, spawn the executor. - try: - task_file_contents = _build_suite_from_payload(payload) - except (ValueError, FileNotFoundError) as exc: - raise HTTPException(400, str(exc)) from exc - - # #508 parity: forward a top-level `extraction_schema` into the suite as - # `_extraction_schema` so the executor's extractor honors it (baseten did - # this; the Modal CUA path silently dropped it → `no_schema_configured`). - _payload_schema = payload.get("extraction_schema") - if isinstance(_payload_schema, dict): - try: - _suite_obj = json.loads(task_file_contents) - _suite_obj["_extraction_schema"] = _payload_schema - task_file_contents = json.dumps(_suite_obj) - except (ValueError, TypeError): - pass # non-JSON suite (rare) → leave as-is - - # DX-3 (#785 follow-up): dry-run preview. Returns the resolved - # task_suite + summary + cost estimate without acquiring the - # Chrome lock or spawning the executor. Lets devs iterate on - # plan_text → MicroPlan without burning GPU credit. - if bool(payload.get("dry_run")): - from mantis_agent.server.dry_run import build_dry_run_response - return build_dry_run_response( - task_file_contents, payload, tenant.tenant_id - ) - - profile_id = payload["profile_id"] - workflow_id = payload["workflow_id"] + """Shared submit path for ``/v1/predict`` and ``/v1/cua``. + + Mints a run_id, acquires the Phase-1 profile lock + Phase-2 + snapshot lock, resolves + spawns the executor, persists the + initial ``queued`` status + suite snapshot, and returns the + detached envelope. Both routes go through this one helper so the + status/result lifecycle (poll → terminal write in ``_do_action``) + is byte-identical regardless of which entrypoint submitted the + run — the /v1/cua route just builds a different + ``task_file_contents`` (a ``_mode="pure_cua"`` suite). + """ run_id = _mint_run_id() # 409 if another run is using this profile (Chrome lock). @@ -2937,13 +3131,12 @@ async def predict( "Use a different profile_id or wait for the existing run to finish.", ) from exc - model = payload.get("cua_model") or payload.get("model") or "holo3" executor_fn = resolve_executor(model) # Forward only what the executors accept. `task_file_contents` # is positional; everything else is **kwargs. spawn_kwargs: dict = { - "max_steps": int(payload.get("max_steps", 30)), + "max_steps": int(max_steps), # Per-tenant, per-profile Chrome user-data-dir (#341). # Until this was added, Modal executors fell back to # XdotoolGymEnv's default ``/data/chrome-profile`` and @@ -2959,7 +3152,7 @@ async def predict( # side status.json. The executor needs the API's run_id + # tenant_id to know which status file to update (the # executor's internal run_id differs). - if payload.get("live_viewer"): + if live_viewer: spawn_kwargs["viewer"] = True spawn_kwargs["api_run_id"] = run_id spawn_kwargs["api_tenant_id"] = tenant.tenant_id @@ -2982,9 +3175,9 @@ async def predict( "tenant_id": tenant.tenant_id, "profile_id": profile_id, "workflow_id": workflow_id, - "state_key": payload["state_key"], + "state_key": state_key, "model": model, - "max_steps": int(payload.get("max_steps", 30)), + "max_steps": int(max_steps), } _write_status(tenant.tenant_id, run_id, status) # #347: snapshot the suite so a later action=resume can re-spawn @@ -2994,7 +3187,7 @@ async def predict( run_dir = _run_dir(tenant.tenant_id, run_id) return { "status": "queued", - "mode": "detached", + "mode": mode, "run_id": run_id, "created_at": now, "updated_at": now, @@ -3009,6 +3202,161 @@ async def predict( "events_path": str(run_dir / "events.log"), } + @fastapi_app.post("/v1/predict") + async def predict( + request: Request, + tenant: TenantConfig = Depends(require_run_scope), + ) -> dict: + try: + raw = await request.json() + except Exception: + raise HTTPException(400, "request body must be JSON") + try: + payload = prepare_predict_payload(raw, tenant) + except DispatchError as exc: + raise HTTPException(exc.status_code, exc.detail) from exc + + # Action mode: poll/cancel an existing run. + if payload.get("action"): + return _do_action(payload, tenant) + + # Run mode: build suite, lock the profile, spawn the executor. + try: + task_file_contents = _build_suite_from_payload(payload) + except (ValueError, FileNotFoundError) as exc: + raise HTTPException(400, str(exc)) from exc + + # #508 parity: forward a top-level `extraction_schema` into the suite as + # `_extraction_schema` so the executor's extractor honors it (baseten did + # this; the Modal CUA path silently dropped it → `no_schema_configured`). + _payload_schema = payload.get("extraction_schema") + if isinstance(_payload_schema, dict): + try: + _suite_obj = json.loads(task_file_contents) + _suite_obj["_extraction_schema"] = _payload_schema + task_file_contents = json.dumps(_suite_obj) + except (ValueError, TypeError): + pass # non-JSON suite (rare) → leave as-is + + # DX-3 (#785 follow-up): dry-run preview. Returns the resolved + # task_suite + summary + cost estimate without acquiring the + # Chrome lock or spawning the executor. Lets devs iterate on + # plan_text → MicroPlan without burning GPU credit. + if bool(payload.get("dry_run")): + from mantis_agent.server.dry_run import build_dry_run_response + return build_dry_run_response( + task_file_contents, payload, tenant.tenant_id + ) + + model = payload.get("cua_model") or payload.get("model") or "holo3" + return _submit_detached_run( + tenant, + task_file_contents=task_file_contents, + model=model, + profile_id=payload["profile_id"], + workflow_id=payload["workflow_id"], + state_key=payload["state_key"], + max_steps=int(payload.get("max_steps", 30)), + live_viewer=bool(payload.get("live_viewer")), + ) + + @fastapi_app.post("/v1/cua") + async def cua( + request: Request, + tenant: TenantConfig = Depends(require_run_scope), + ) -> dict: + """Pure-CUA loop — 100%-student Holo3 (``brain ↔ XdotoolGymEnv``). + + Same auth / caps / allowlist / lock / status-lifecycle plumbing + as ``/v1/predict``: we validate through the shared + :func:`prepare_cua_payload` (identity resolution + URL allowlist), + translate the request into a ``_mode="pure_cua"`` ``task_suite``, + force ``cua_model="holo3"``, and dispatch through the exact same + submit/spawn/status-write machinery via + :func:`_submit_detached_run`. The executor's + ``_run_holo3_executor`` picks the pure-cua branch off ``_mode``. + + Runs are always detached on Modal (spawn + poll), same as + ``/v1/predict`` — poll terminal status/result via + ``GET /v1/runs/{run_id}/status|result``. + """ + try: + raw = await request.json() + except Exception as exc: + raise HTTPException(400, "request body must be JSON") from exc + + try: + payload = prepare_cua_payload(raw, tenant) + except DispatchError as exc: + raise HTTPException(exc.status_code, exc.detail) from exc + + instruction = str(payload.get("instruction") or "").strip() + if not instruction: + raise HTTPException(400, "instruction is required") + + # Persistent browser headers. AUTO-INJECT the Daytona skip-warning + # header when start_url is a Daytona preview URL, so callers never + # strand on the interstitial by forgetting it — the org-wide removal + # (Tier 3 / custom proxy) is the dashboard-side path; this is the + # per-request safety net. The preview TOKEN stays caller-supplied + # (it's per-sandbox and can't be inferred). + _start_url = str(payload.get("start_url") or "") + _bxh = ( + payload.get("_browser_extra_headers") + or payload.get("browser_extra_headers") + or {} + ) + if not isinstance(_bxh, dict): + _bxh = {} + if "daytonaproxy" in _start_url: + _bxh.setdefault("X-Daytona-Skip-Preview-Warning", "true") + + # Translate the validated request into the pure-cua suite the + # holo3 executor forks on. Mirrors the ``task_suite`` shape + # ``baseten_server.runtime._run_pure_cua`` builds, plus the + # ``_mode`` / ``_instruction`` selectors the Modal branch reads. + task_suite = { + "session_name": "pure_cua", + "base_url": str(payload.get("start_url") or ""), + "_mode": "pure_cua", + "_instruction": instruction, + "_ground_clicks": bool(payload.get("ground_clicks", False)), + "_decompose": bool(payload.get("decompose", False)), + "_max_steps": int(payload.get("max_steps", 30)), + "_settle_time": float(payload.get("settle_time", 4.0)), + # #300 follow-up: per-request SoM routing override (None → + # deployment env). Forwarded so the executor can flip it. + "_route_som_clicks": payload.get("route_som_clicks"), + # Persistent browser request headers (CDP Network.setExtraHTTPHeaders). + # WITHOUT forwarding this the browser never sends Daytona's + # ``X-Daytona-Skip-Preview-Warning`` / ``X-Daytona-Preview-Token``, + # so every Daytona rollout strands on the preview interstitial at + # frame one; the Daytona skip-warning header is auto-added above. + "_browser_extra_headers": _bxh or None, + # Identity: forwarded so the executor mints a searchable, + # workflow-prefixed Augur run_id (the logprobs session). + "_workflow_id": payload.get("workflow_id", ""), + # Proxy controls (allowlist-gated upstream). + "_proxy_city": payload.get("proxy_city") or "", + "_proxy_state": payload.get("proxy_state") or "", + "_proxy_disabled": bool(payload.get("proxy_disabled", False)), + # Human-readable plan family for the Augur Runs list. + "_plan_name": "pure_cua", + } + task_file_contents = json.dumps(task_suite) + + return _submit_detached_run( + tenant, + task_file_contents=task_file_contents, + model="holo3", # pure-cua is a Holo3-only path + profile_id=payload["profile_id"], + workflow_id=payload["workflow_id"], + state_key=payload["state_key"], + max_steps=int(payload.get("max_steps", 30)), + live_viewer=bool(payload.get("live_viewer")), + mode="pure_cua", + ) + def _do_action(payload: dict, tenant: TenantConfig) -> dict: """Handle ``action=status|result|cancel|resume`` against an existing run.""" run_id = str(payload.get("run_id") or "") @@ -4690,7 +5038,7 @@ def computer_plane(): @app.function( - gpu="A100-80GB", + gpu=["A100-80GB", "H100"], # H100 fallback: A100-80GB pool starves (2026-07-04) image=planner_base_image.run_commands( # #stealth-parity: install fonts/locale alongside the existing # Chrome + Xvfb deps. ``run_holo3`` is the most-used executor diff --git a/src/mantis_agent/brain_holo3.py b/src/mantis_agent/brain_holo3.py index 71d050bb..84055149 100644 --- a/src/mantis_agent/brain_holo3.py +++ b/src/mantis_agent/brain_holo3.py @@ -473,6 +473,26 @@ def _build_messages( content: list[dict] = [] n_frames = len(frames) + # Aim diagnostic (WARNING so it survives Modal log suppression): the + # image the model localizes on vs the screen_size the coord-mapping + # assumes. If the CURRENT frame is TALLER than screen_size the model + # over-emits y and _model_coords_to_screen (which divides by the + # smart-resize of screen_size) can dispatch off-screen. Pairs with the + # raw coord in modelio ``response.text`` and the mapped/grounded coord + # in the runner's ``[cua-offset]`` line to fully explain any y-miss. + if n_frames and hasattr(frames[-1], "size"): + _cw, _ch = frames[-1].size + if (_cw, _ch) != (screen_size[0], screen_size[1]): + logger.warning( + "[cua-imgdim] current_frame=%sx%s screen_size=%sx%s " + "(MISMATCH → coord space skewed)", + _cw, _ch, screen_size[0], screen_size[1], + ) + else: + logger.warning( + "[cua-imgdim] current_frame=%sx%s screen_size=%sx%s (match)", + _cw, _ch, screen_size[0], screen_size[1], + ) for i, frame in enumerate(frames): label = "CURRENT" if i == n_frames - 1 else f"t-{n_frames - 1 - i}" content.append({"type": "text", "text": f"[Frame {label}]"}) @@ -759,15 +779,27 @@ def _has_valid_xy(args: dict) -> bool: explicit ``(0, 0)`` both indicate the model didn't pin a target — better to fall through to the WAIT(1s) fallback than dispatch to the page origin. + + Validate as FLOAT, not ``_safe_int``: Holo3's pure-CUA output + emits NORMALIZED fractions like ``click(x=0.324, y=0.338)``. + ``_safe_int`` truncated 0.324 → 0, so a perfectly good click + read as the (0,0) origin and got rejected — the harness then + substituted WAIT(1s) and silently dropped an on-policy action + (cua-issues 2026-06-29 S09, native-text path). ``_coords_to_screen`` + (used below) scales the fraction; here we only gate on it being + a real, non-origin target. """ if "x" not in args or "y" not in args: return False - x = _safe_int(args["x"], default=-1) - y = _safe_int(args["y"], default=-1) - if x < 0 or y < 0: - # Malformed value that didn't parse to a non-negative int. + try: + fx = float(str(args["x"]).strip()) + fy = float(str(args["y"]).strip()) + except (ValueError, TypeError): + # Malformed value that didn't parse to a number. + return False + if fx < 0 or fy < 0: return False - return (x, y) != (0, 0) + return (fx, fy) != (0.0, 0.0) # Map to Action if func_name in ("click",): @@ -780,17 +812,17 @@ def _has_valid_xy(args: dict) -> bool: # level → clean no-op step. if not _has_valid_xy(args): return None - x = _safe_int(args["x"]) - y = _safe_int(args["y"]) - sx, sy = _model_coords_to_screen(x, y, screen_size[0], screen_size[1]) + # #S09: normalized-fraction-aware conversion (same helper the + # tool_calls path uses via _convert_coords). Handles both + # ``0.324`` fractions and resized-space integers — do NOT + # pre-truncate with _safe_int (that reintroduces the origin bug). + sx, sy = self._coords_to_screen(args["x"], args["y"], screen_size) return Action(ActionType.CLICK, {"x": sx, "y": sy, "button": args.get("button", "left")}) if func_name in ("double_click", "doubleclick"): if not _has_valid_xy(args): return None - x = _safe_int(args["x"]) - y = _safe_int(args["y"]) - sx, sy = _model_coords_to_screen(x, y, screen_size[0], screen_size[1]) + sx, sy = self._coords_to_screen(args["x"], args["y"], screen_size) return Action(ActionType.DOUBLE_CLICK, {"x": sx, "y": sy}) if func_name in ("type_text", "type", "typewrite"): @@ -848,14 +880,13 @@ def _parse_json_action(self, text: str, screen_size: tuple[int, int]) -> Action return self._parse_holo3_action(code, screen_size) if action_type == "click": - x, y = int(obj.get("x", 0)), int(obj.get("y", 0)) - sx, sy = _model_coords_to_screen(x, y, screen_size[0], screen_size[1]) + # #S09: normalized-fraction-aware (int() truncated 0.324 → 0). + sx, sy = self._coords_to_screen(obj.get("x", 0), obj.get("y", 0), screen_size) button = obj.get("button", "left") return Action(ActionType.CLICK, {"x": sx, "y": sy, "button": button}) if action_type in ("double_click", "doubleclick", "doubleClick"): - x, y = int(obj.get("x", 0)), int(obj.get("y", 0)) - sx, sy = _model_coords_to_screen(x, y, screen_size[0], screen_size[1]) + sx, sy = self._coords_to_screen(obj.get("x", 0), obj.get("y", 0), screen_size) return Action(ActionType.DOUBLE_CLICK, {"x": sx, "y": sy}) if action_type in ("type", "type_text", "typewrite", "write"): diff --git a/src/mantis_agent/gym/runner.py b/src/mantis_agent/gym/runner.py index 20d724ba..2d3668f8 100644 --- a/src/mantis_agent/gym/runner.py +++ b/src/mantis_agent/gym/runner.py @@ -1630,6 +1630,22 @@ def _save_frame(img: Any, step: int) -> None: initial_x=orig_x, initial_y=orig_y, ) + # Offset measurement (WARNING so it survives Modal + # log suppression): the model's raw click coord vs + # the grounded (actual-element) coord IS the exact + # per-click offset. Uniform ~chrome-height vertical + # → a systematic mapping shift; large/variable 2D → + # Holo3 spatial-grounding imprecision. + try: + _dx = int(gr.x) - int(orig_x) + _dy = int(gr.y) - int(orig_y) + except (TypeError, ValueError): + _dx = _dy = None + logger.warning( + "[cua-offset] model=(%s,%s) grounded=(%s,%s) " + "delta=(%s,%s) conf=%.2f", + orig_x, orig_y, gr.x, gr.y, _dx, _dy, gr.confidence, + ) if gr.confidence > 0.3: print(f" [grounding] ({orig_x},{orig_y}) → ({gr.x},{gr.y}) conf={gr.confidence:.1f}") action = Action( @@ -1683,6 +1699,10 @@ def _save_frame(img: Any, step: int) -> None: # screenshot the brain reasoned on, so it's the right # baseline for "did this action change the page?". pre_action_frame = frame_history[-1] if frame_history else None + # Snapshot the pre-action URL before ``last_url`` is reassigned + # to the post-action URL below — the URL-diff guard in the + # perceptual-diff block needs the BEFORE value. + _pre_action_url = last_url for pre_action in pre_actions: logger.warning( @@ -1903,12 +1923,23 @@ def _save_frame(img: Any, step: int) -> None: and isinstance(focused_input, dict) and focused_input.get("focused_via_click") ) + # URL-diff guard: a click that NAVIGATED is a real change even + # when the perceptual-diff sees near-identical frames (the new + # page renders after the post-action capture, or simply looks + # similar). Firing "no observed effect" here contradicts the + # "page navigated to " line ``_build_feedback`` already + # added and traps otherwise-recoverable trajectories in + # Return-loops — the single largest rollout-quality depressor. + # Uses the URL the env already surfaces in ``info`` (post-action + # verification, not target derivation). + _post_url = str(gym_result.info.get("url", "") or "") + _navigated_this_step = bool(_post_url and _post_url != _pre_action_url) if clicked_into_field: feedback = ( f"{feedback}; input field focused — type your text now " f"(do not click it again)" ) - elif effect_check.effect_observed is False: + elif effect_check.effect_observed is False and not _navigated_this_step: feedback = ( f"{feedback}; WARNING: high-risk action had no observed effect " f"({effect_check.reason})" diff --git a/src/mantis_agent/gym/xdotool_env.py b/src/mantis_agent/gym/xdotool_env.py index 1ba5fcfc..050c3498 100644 --- a/src/mantis_agent/gym/xdotool_env.py +++ b/src/mantis_agent/gym/xdotool_env.py @@ -745,6 +745,81 @@ def _cdp_insert_text(self, text: str) -> bool: ok, _ = self._cdp_call("Input.insertText", {"text": text}) return ok + def cdp_input_insert(self, text: str) -> bool: + """Focus-hardened fill for ``