From 7a1f067cc597445e213280d20b6acccae6a53103 Mon Sep 17 00:00:00 2001 From: Barada Sahu Date: Tue, 30 Jun 2026 16:28:26 +0530 Subject: [PATCH] =?UTF-8?q?fix(cua):=20SoM=20click=20must=20focus=20the=20?= =?UTF-8?q?input=20=E2=80=94=20the=20core=20of=20the=20type-stall?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live verification of #951 showed typing STILL failed: the brain SoM-clicks the search box but the run halts with the brain only emitting wait()/Return. Root cause is one layer deeper than #951: the SoM click path (cdp_click_at_point, which the runner uses for unstructured clicks) dispatches `el.click()` via Runtime.evaluate — and programmatic el.click() fires a click event but does NOT focus a text input the way a real mouse click does (focus happens on mousedown). So search/login/composer fields were left UNFOCUSED, the brain had no field to type into, and it stalled. #951's env CLICK-branch probe never even ran because SoM swaps the action to a no-op WAIT. Fix: - cdp_click_at_point: focus focusable targets (INPUT/TEXTAREA/SELECT/ contenteditable) before el.click(), mirroring native focus→click ordering; report `focused` on the result + _last_som_diag. - runner: after a SoM-dispatched click that focused a field, surface focused_input → the "input field focused — type your text now" feedback (#951) now fires on the SoM path too, so the brain types instead of waiting. Tests: test_cua_input_focus_feedback.py — SoM payload focuses + reports focused. 215 adjacent tests green; ruff clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/mantis_agent/gym/runner.py | 12 +++++++++++ src/mantis_agent/gym/xdotool_env.py | 20 +++++++++++++++-- tests/test_cua_input_focus_feedback.py | 30 ++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/src/mantis_agent/gym/runner.py b/src/mantis_agent/gym/runner.py index 4a8a4815..20d724ba 100644 --- a/src/mantis_agent/gym/runner.py +++ b/src/mantis_agent/gym/runner.py @@ -1739,6 +1739,18 @@ def _save_frame(img: Any, step: int) -> None: ) gym_result = self.env.step(action) + # type-stall fix: a SoM-dispatched click swaps the action to a + # no-op WAIT, so env.step's CLICK-branch focus probe never runs. + # If the SoM click focused a text field (cdp_click_at_point did + # el.focus()), surface focused_input here so the brain is told + # to type next instead of stalling on "no visible change". + if pending_executor_backend == "som": + som_diag = getattr(self.env, "_last_som_diag", None) + if isinstance(som_diag, dict) and som_diag.get("focused"): + gym_result.info["focused_input"] = { + "focused_via_click": True, + "value": "", + } action_history.append(action) for post_action in post_actions: logger.warning( diff --git a/src/mantis_agent/gym/xdotool_env.py b/src/mantis_agent/gym/xdotool_env.py index fc526879..74c26623 100644 --- a/src/mantis_agent/gym/xdotool_env.py +++ b/src/mantis_agent/gym/xdotool_env.py @@ -1124,9 +1124,22 @@ def cdp_click_at_point(self, x: int, y: int) -> bool: "(e.className && typeof e.className === 'string' ? '.' + e.className.split(' ').slice(0,2).join('.') : '')) : null;" "const text = e => e ? (e.innerText || e.textContent || '').trim().slice(0, 40) : '';" "let ok = false;" - "if (elv) { try { elv.click(); ok = true; } catch (e) { ok = false; } }" + "let focused = false;" + # type-stall fix: a real mouse click focuses an input on mousedown, + # but ``elv.click()`` only fires the click event — it does NOT focus. + # So SoM clicks on a search/login/composer field left it UNFOCUSED; + # the brain then had no field to type into and stalled (it never + # reached a type()). Focus focusable targets first so the next + # TYPE lands. Mirrors native click ordering (focus → click). + "if (elv) { try {" + " const tagU = (elv.tagName || '').toUpperCase();" + " if ((tagU === 'INPUT' || tagU === 'TEXTAREA' || tagU === 'SELECT' || elv.isContentEditable) && typeof elv.focus === 'function') {" + " try { elv.focus(); focused = (document.activeElement === elv); } catch (e) {}" + " }" + " elv.click(); ok = true;" + "} catch (e) { ok = false; } }" "return {" - "ok: ok," + "ok: ok, focused: focused," "outerHeight: oh, innerHeight: ih, chromeH: chromeH," "vx: vx, vy: vy," "elv_tag: tag(elv), elv_text: text(elv)," @@ -1157,6 +1170,9 @@ def cdp_click_at_point(self, x: int, y: int) -> bool: "els_tag": str(result.get("els_tag") or ""), "els_text": str(result.get("els_text") or ""), "ok": bool(result.get("ok")), + # type-stall fix: did the SoM click focus a text field? Read by + # the runner to surface focused_input → "type now" feedback. + "focused": bool(result.get("focused")), } # One-line WARNING log so the SoM click's coordinate translation # is visible in Modal's INFO-suppressed capture without an diff --git a/tests/test_cua_input_focus_feedback.py b/tests/test_cua_input_focus_feedback.py index f0010ffa..0759eaa8 100644 --- a/tests/test_cua_input_focus_feedback.py +++ b/tests/test_cua_input_focus_feedback.py @@ -72,3 +72,33 @@ def _boom(self): monkeypatch.setattr(XdotoolGymEnv, "_active_field_probe", _boom) result = click_env.step(_click()) # must not raise assert "focused_input" not in result.info + + +def test_som_click_focuses_input_and_reports_focused(monkeypatch: pytest.MonkeyPatch) -> None: + """cdp_click_at_point must el.focus() a focusable target and surface + ``focused`` on _last_som_diag — the type-stall fix for the SoM path, where + el.click() alone never focused the input.""" + env = XdotoolGymEnv.__new__(XdotoolGymEnv) + env.som_diagnostic = False + captured: dict = {} + + def _fake_eval(self, js): # noqa: ANN001 + captured["js"] = js + return { + "ok": True, "focused": True, + "elv_tag": "INPUT", "elv_text": "", + "els_tag": "DIV", "els_text": "", + "outerHeight": 800, "innerHeight": 720, "chromeH": 80, + "vx": 270, "vy": 42, + } + + monkeypatch.setattr(XdotoolGymEnv, "cdp_evaluate", _fake_eval) + + ok = env.cdp_click_at_point(270, 122) + + assert ok is True + # the JS payload focuses focusable elements before clicking + assert "focus" in captured["js"] + assert ".focus()" in captured["js"] + # the focus result is surfaced for the runner to read + assert env._last_som_diag["focused"] is True