Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions src/mantis_agent/gym/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -1739,6 +1739,18 @@ def _save_frame(img: Any, step: int) -> None:
)

gym_result = self.env.step(action)
# type-stall fix: a SoM-dispatched click swaps the action to a
# no-op WAIT, so env.step's CLICK-branch focus probe never runs.
# If the SoM click focused a text field (cdp_click_at_point did
# el.focus()), surface focused_input here so the brain is told
# to type next instead of stalling on "no visible change".
if pending_executor_backend == "som":
som_diag = getattr(self.env, "_last_som_diag", None)
if isinstance(som_diag, dict) and som_diag.get("focused"):
gym_result.info["focused_input"] = {
"focused_via_click": True,
"value": "",
}
action_history.append(action)
for post_action in post_actions:
logger.warning(
Expand Down
20 changes: 18 additions & 2 deletions src/mantis_agent/gym/xdotool_env.py
Original file line number Diff line number Diff line change
Expand Up @@ -1124,9 +1124,22 @@ def cdp_click_at_point(self, x: int, y: int) -> bool:
"(e.className && typeof e.className === 'string' ? '.' + e.className.split(' ').slice(0,2).join('.') : '')) : null;"
"const text = e => e ? (e.innerText || e.textContent || '').trim().slice(0, 40) : '';"
"let ok = false;"
"if (elv) { try { elv.click(); ok = true; } catch (e) { ok = false; } }"
"let focused = false;"
# type-stall fix: a real mouse click focuses an input on mousedown,
# but ``elv.click()`` only fires the click event — it does NOT focus.
# So SoM clicks on a search/login/composer field left it UNFOCUSED;
# the brain then had no field to type into and stalled (it never
# reached a type()). Focus focusable targets first so the next
# TYPE lands. Mirrors native click ordering (focus → click).
"if (elv) { try {"
" const tagU = (elv.tagName || '').toUpperCase();"
" if ((tagU === 'INPUT' || tagU === 'TEXTAREA' || tagU === 'SELECT' || elv.isContentEditable) && typeof elv.focus === 'function') {"
" try { elv.focus(); focused = (document.activeElement === elv); } catch (e) {}"
" }"
" elv.click(); ok = true;"
"} catch (e) { ok = false; } }"
"return {"
"ok: ok,"
"ok: ok, focused: focused,"
"outerHeight: oh, innerHeight: ih, chromeH: chromeH,"
"vx: vx, vy: vy,"
"elv_tag: tag(elv), elv_text: text(elv),"
Expand Down Expand Up @@ -1157,6 +1170,9 @@ def cdp_click_at_point(self, x: int, y: int) -> bool:
"els_tag": str(result.get("els_tag") or ""),
"els_text": str(result.get("els_text") or ""),
"ok": bool(result.get("ok")),
# type-stall fix: did the SoM click focus a text field? Read by
# the runner to surface focused_input → "type now" feedback.
"focused": bool(result.get("focused")),
}
# One-line WARNING log so the SoM click's coordinate translation
# is visible in Modal's INFO-suppressed capture without an
Expand Down
30 changes: 30 additions & 0 deletions tests/test_cua_input_focus_feedback.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,3 +72,33 @@ def _boom(self):
monkeypatch.setattr(XdotoolGymEnv, "_active_field_probe", _boom)
result = click_env.step(_click()) # must not raise
assert "focused_input" not in result.info


def test_som_click_focuses_input_and_reports_focused(monkeypatch: pytest.MonkeyPatch) -> None:
"""cdp_click_at_point must el.focus() a focusable target and surface
``focused`` on _last_som_diag — the type-stall fix for the SoM path, where
el.click() alone never focused the input."""
env = XdotoolGymEnv.__new__(XdotoolGymEnv)
env.som_diagnostic = False
captured: dict = {}

def _fake_eval(self, js): # noqa: ANN001
captured["js"] = js
return {
"ok": True, "focused": True,
"elv_tag": "INPUT", "elv_text": "",
"els_tag": "DIV", "els_text": "",
"outerHeight": 800, "innerHeight": 720, "chromeH": 80,
"vx": 270, "vy": 42,
}

monkeypatch.setattr(XdotoolGymEnv, "cdp_evaluate", _fake_eval)

ok = env.cdp_click_at_point(270, 122)

assert ok is True
# the JS payload focuses focusable elements before clicking
assert "focus" in captured["js"]
assert ".focus()" in captured["js"]
# the focus result is surfaced for the runner to read
assert env._last_som_diag["focused"] is True
Loading