diff --git a/src/mantis_agent/gym/runner.py b/src/mantis_agent/gym/runner.py index 4a8a4815..20d724ba 100644 --- a/src/mantis_agent/gym/runner.py +++ b/src/mantis_agent/gym/runner.py @@ -1739,6 +1739,18 @@ def _save_frame(img: Any, step: int) -> None: ) gym_result = self.env.step(action) + # type-stall fix: a SoM-dispatched click swaps the action to a + # no-op WAIT, so env.step's CLICK-branch focus probe never runs. + # If the SoM click focused a text field (cdp_click_at_point did + # el.focus()), surface focused_input here so the brain is told + # to type next instead of stalling on "no visible change". + if pending_executor_backend == "som": + som_diag = getattr(self.env, "_last_som_diag", None) + if isinstance(som_diag, dict) and som_diag.get("focused"): + gym_result.info["focused_input"] = { + "focused_via_click": True, + "value": "", + } action_history.append(action) for post_action in post_actions: logger.warning( diff --git a/src/mantis_agent/gym/xdotool_env.py b/src/mantis_agent/gym/xdotool_env.py index fc526879..74c26623 100644 --- a/src/mantis_agent/gym/xdotool_env.py +++ b/src/mantis_agent/gym/xdotool_env.py @@ -1124,9 +1124,22 @@ def cdp_click_at_point(self, x: int, y: int) -> bool: "(e.className && typeof e.className === 'string' ? '.' + e.className.split(' ').slice(0,2).join('.') : '')) : null;" "const text = e => e ? (e.innerText || e.textContent || '').trim().slice(0, 40) : '';" "let ok = false;" - "if (elv) { try { elv.click(); ok = true; } catch (e) { ok = false; } }" + "let focused = false;" + # type-stall fix: a real mouse click focuses an input on mousedown, + # but ``elv.click()`` only fires the click event — it does NOT focus. + # So SoM clicks on a search/login/composer field left it UNFOCUSED; + # the brain then had no field to type into and stalled (it never + # reached a type()). Focus focusable targets first so the next + # TYPE lands. Mirrors native click ordering (focus → click). + "if (elv) { try {" + " const tagU = (elv.tagName || '').toUpperCase();" + " if ((tagU === 'INPUT' || tagU === 'TEXTAREA' || tagU === 'SELECT' || elv.isContentEditable) && typeof elv.focus === 'function') {" + " try { elv.focus(); focused = (document.activeElement === elv); } catch (e) {}" + " }" + " elv.click(); ok = true;" + "} catch (e) { ok = false; } }" "return {" - "ok: ok," + "ok: ok, focused: focused," "outerHeight: oh, innerHeight: ih, chromeH: chromeH," "vx: vx, vy: vy," "elv_tag: tag(elv), elv_text: text(elv)," @@ -1157,6 +1170,9 @@ def cdp_click_at_point(self, x: int, y: int) -> bool: "els_tag": str(result.get("els_tag") or ""), "els_text": str(result.get("els_text") or ""), "ok": bool(result.get("ok")), + # type-stall fix: did the SoM click focus a text field? Read by + # the runner to surface focused_input → "type now" feedback. + "focused": bool(result.get("focused")), } # One-line WARNING log so the SoM click's coordinate translation # is visible in Modal's INFO-suppressed capture without an diff --git a/tests/test_cua_input_focus_feedback.py b/tests/test_cua_input_focus_feedback.py index f0010ffa..0759eaa8 100644 --- a/tests/test_cua_input_focus_feedback.py +++ b/tests/test_cua_input_focus_feedback.py @@ -72,3 +72,33 @@ def _boom(self): monkeypatch.setattr(XdotoolGymEnv, "_active_field_probe", _boom) result = click_env.step(_click()) # must not raise assert "focused_input" not in result.info + + +def test_som_click_focuses_input_and_reports_focused(monkeypatch: pytest.MonkeyPatch) -> None: + """cdp_click_at_point must el.focus() a focusable target and surface + ``focused`` on _last_som_diag — the type-stall fix for the SoM path, where + el.click() alone never focused the input.""" + env = XdotoolGymEnv.__new__(XdotoolGymEnv) + env.som_diagnostic = False + captured: dict = {} + + def _fake_eval(self, js): # noqa: ANN001 + captured["js"] = js + return { + "ok": True, "focused": True, + "elv_tag": "INPUT", "elv_text": "", + "els_tag": "DIV", "els_text": "", + "outerHeight": 800, "innerHeight": 720, "chromeH": 80, + "vx": 270, "vy": 42, + } + + monkeypatch.setattr(XdotoolGymEnv, "cdp_evaluate", _fake_eval) + + ok = env.cdp_click_at_point(270, 122) + + assert ok is True + # the JS payload focuses focusable elements before clicking + assert "focus" in captured["js"] + assert ".focus()" in captured["js"] + # the focus result is surfaced for the runner to read + assert env._last_som_diag["focused"] is True