From aae4e3652321baf2544589faa30ca2210625cfce Mon Sep 17 00:00:00 2001 From: Subash Shibu Date: Mon, 17 Nov 2025 14:43:41 -0800 Subject: [PATCH 1/3] feat: add hover tool --- package.json | 4 +- pnpm-lock.yaml | 10 +- src/server/server-factory.ts | 20 ++++ src/tools/hover.ts | 208 +++++++++++++++++++++++++++++++++++ 4 files changed, 235 insertions(+), 7 deletions(-) create mode 100644 src/tools/hover.ts diff --git a/package.json b/package.json index 186c214..3d34f8b 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gbox.ai/mcp-server", - "version": "0.1.15", + "version": "0.1.17", "description": "MCP server exposing Gbox Android and Linux control tools via Model Context Protocol", "homepage": "https://gbox.ai", "author": "Gbox Team", @@ -44,7 +44,7 @@ "axios": "^1.12.2", "dotenv": "^17.2.3", "express": "^5.1.0", - "gbox-sdk": "^0.37.0", + "gbox-sdk": "^0.42.0", "jose": "^6.1.0", "zod": "^3.25.76" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9dab849..78b2102 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: ^5.1.0 version: 5.1.0 gbox-sdk: - specifier: ^0.37.0 - version: 0.37.0(typescript@5.9.3) + specifier: ^0.42.0 + version: 0.42.0(typescript@5.9.3) jose: specifier: ^6.1.0 version: 6.1.0 @@ -950,8 +950,8 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} - gbox-sdk@0.37.0: - resolution: {integrity: sha512-v6aQjTxTLvkvGWqsqp/yIDsOJoQDi58+weF29L0ZMWMV9HFXHgXeA5nnch1qAlrMY9Mrr7lfAKuzSH6aU6eo+A==} + gbox-sdk@0.42.0: + resolution: {integrity: sha512-qXm/XjAfsbWplHO4juThD0SoEqN3Mp9jbPMlKzXm1BfoXHtOjSFSC0pN+1aPMkveQZiiywleKuk/qhKnvOmlQA==} engines: {node: '>=16.0.0', npm: please-use-yarn, yarn: '>=1.22.0'} get-intrinsic@1.3.0: @@ -2505,7 +2505,7 @@ snapshots: function-bind@1.1.2: {} - gbox-sdk@0.37.0(typescript@5.9.3): + gbox-sdk@0.42.0(typescript@5.9.3): dependencies: smol-toml: 1.4.2 typedoc: 0.28.14(typescript@5.9.3) diff --git a/src/server/server-factory.ts b/src/server/server-factory.ts index fb69f65..1027daf 100644 --- a/src/server/server-factory.ts +++ b/src/server/server-factory.ts @@ -133,6 +133,12 @@ import { CLOSE_TAB_TOOL, closeTabParamsSchema, } from "../tools/close-tab.js"; +import { + handleHover, + HOVER_DESCRIPTION, + HOVER_TOOL, + hoverParamsSchema, +} from "../tools/hover.js"; /** * Factory class for creating McpServer instances @@ -312,6 +318,13 @@ export class McpServerFactory { pressKeyParamsSchema, handlePressKey(logger, gboxSDK) ); + + server.tool( + HOVER_TOOL, + HOVER_DESCRIPTION, + hoverParamsSchema, + handleHover(logger, gboxSDK) + ); } /** @@ -377,6 +390,13 @@ export class McpServerFactory { closeTabParamsSchema, handleCloseTab(logger, gboxSDK) ); + + server.tool( + HOVER_TOOL, + HOVER_DESCRIPTION, + hoverParamsSchema, + handleHover(logger, gboxSDK) + ); } /** diff --git a/src/tools/hover.ts b/src/tools/hover.ts new file mode 100644 index 0000000..6c0fb80 --- /dev/null +++ b/src/tools/hover.ts @@ -0,0 +1,208 @@ +import { z } from "zod"; +import GboxSDK from "gbox-sdk"; +import type { MCPLogger } from "../logger/logger.js"; +import { attachBox } from "../sdk/index.js"; +import { extractImageInfo } from "../sdk/utils.js"; + +export const HOVER_TOOL = "hover"; + +export const HOVER_DESCRIPTION = + "Hover over a UI element on the browser using natural language description. The element will be detected automatically and the mouse will move to it without clicking."; + +export const hoverParamsSchema = { + boxId: z.string().describe("ID of the browser box"), + target: z + .string() + .describe( + "Description of the element to hover over (e.g. 'login button', 'search field', 'submit button'). MUST be detailed enough to identify the element unambiguously." + ), +}; + +type HoverParams = z.infer>; + +/** + * Call GBOX Handy API to detect element coordinates using SDK + */ +async function detectElementWithHandy( + screenshotUrl: string, + target: string, + logger: MCPLogger, + gboxSDK: GboxSDK +): Promise<{ x: number; y: number } | null> { + try { + await logger.debug("Calling GBOX Handy API", { + target, + screenshotUrl, + }); + + const result = await gboxSDK.model.call({ + model: "gbox-handy-1", + screenshot: screenshotUrl, + action: { + type: "click", + target: target, + }, + }); + + // Check if coordinates are valid (not -1, -1 which indicates no target found) + if ( + result.response.type === "click" && + result.response.coordinates.x !== -1 && + result.response.coordinates.y !== -1 + ) { + const coordinates = { + x: Math.round(result.response.coordinates.x), + y: Math.round(result.response.coordinates.y), + }; + + await logger.info("GBOX Handy detected element", { + target, + coordinates, + requestId: result.id, + }); + + return coordinates; + } + + await logger.warn("GBOX Handy could not find element", { + target, + requestId: result.id, + }); + return null; + } catch (error) { + await logger.error("GBOX Handy detection failed", { + error: error instanceof Error ? error.message : String(error), + target, + }); + return null; + } +} + +export function handleHover(logger: MCPLogger, gboxSDK: GboxSDK) { + return async ({ boxId, target }: HoverParams) => { + const startTime = Date.now(); + + try { + await logger.info("Hover command invoked", { + boxId, + target, + timestamp: new Date().toISOString(), + }); + + // Step 1: Attach to box + const boxAttachStart = Date.now(); + const box = await attachBox(boxId, gboxSDK); + const boxAttachDuration = Date.now() - boxAttachStart; + + await logger.debug("Box attached successfully", { + boxId, + attachDurationMs: boxAttachDuration, + }); + + // Step 2: Take screenshot + await logger.debug("Taking screenshot for element detection"); + const screenshotResult = await box.action.screenshot({ + outputFormat: "storageKey", + }); + + if (!screenshotResult || !screenshotResult.presignedUrl) { + throw new Error("Failed to capture screenshot or get presigned URL"); + } + + // Step 3: Detect element with GBOX Handy using presigned URL + const coordinates = await detectElementWithHandy( + screenshotResult.presignedUrl, + target, + logger, + gboxSDK + ); + + if (!coordinates) { + return { + content: [ + { + type: "text" as const, + text: `Element not found: "${target}". Please provide a more specific description.`, + }, + ], + isError: true, + }; + } + + // Step 4: Execute move action with coordinates + await logger.debug("Moving mouse to coordinates", { + coordinates, + target, + }); + + const moveResult = await box.action.move({ + x: coordinates.x, + y: coordinates.y, + options: { + screenshot: { + phases: ["after"], + outputFormat: "base64", + delay: "500ms", + }, + }, + }); + + const totalDuration = Date.now() - startTime; + + await logger.info("Hover action completed successfully", { + boxId, + target, + coordinates, + totalDurationMs: totalDuration, + actionId: moveResult.actionId, + }); + + return { + content: [ + { + type: "text" as const, + text: `Hover action completed successfully. Moved to "${target}" at (${coordinates.x}, ${coordinates.y})`, + }, + { + type: "image" as const, + ...extractImageInfo(moveResult.screenshot.after.uri), + }, + ], + }; + } catch (error) { + const totalDuration = Date.now() - startTime; + + const errorDetails: Record = { + boxId, + target, + totalDurationMs: totalDuration, + timestamp: new Date().toISOString(), + errorType: error?.constructor?.name, + errorMessage: error instanceof Error ? error.message : String(error), + }; + + if (error && typeof error === "object") { + if ("status" in error) + errorDetails.httpStatus = (error as { status: unknown }).status; + if ("stack" in error) errorDetails.stack = (error as Error).stack; + } + + await logger.error("Failed to run hover action", errorDetails); + + let userMessage = `Error: ${error instanceof Error ? error.message : String(error)}`; + if (errorDetails.httpStatus) { + userMessage += ` (HTTP ${errorDetails.httpStatus})`; + } + + return { + content: [ + { + type: "text" as const, + text: userMessage, + }, + ], + isError: true, + }; + } + }; +} From 7eb0c29d489a3c4df87cb613c5ed0acf4a74c4f1 Mon Sep 17 00:00:00 2001 From: Subash Shibu Date: Mon, 17 Nov 2025 14:50:30 -0800 Subject: [PATCH 2/3] fix: type narrowing for model response coordinates --- src/tools/hover.ts | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/src/tools/hover.ts b/src/tools/hover.ts index 6c0fb80..c0500f1 100644 --- a/src/tools/hover.ts +++ b/src/tools/hover.ts @@ -44,15 +44,24 @@ async function detectElementWithHandy( }, }); + // Check if response is a click action + if (result.response.type !== "click") { + await logger.error("GBOX Handy returned unexpected action type", { + target, + responseType: result.response.type, + requestId: result.id, + }); + return null; + } + + // Type assertion after type guard - we know it's a click response with x, y coordinates + const clickCoords = result.response.coordinates as { x: number; y: number }; + // Check if coordinates are valid (not -1, -1 which indicates no target found) - if ( - result.response.type === "click" && - result.response.coordinates.x !== -1 && - result.response.coordinates.y !== -1 - ) { + if (clickCoords.x !== -1 && clickCoords.y !== -1) { const coordinates = { - x: Math.round(result.response.coordinates.x), - y: Math.round(result.response.coordinates.y), + x: Math.round(clickCoords.x), + y: Math.round(clickCoords.y), }; await logger.info("GBOX Handy detected element", { @@ -64,7 +73,7 @@ async function detectElementWithHandy( return coordinates; } - await logger.warn("GBOX Handy could not find element", { + await logger.info("GBOX Handy could not find element", { target, requestId: result.id, }); From 3f5c1af409e0c31ef7a9a8811b7f4fc5f6e6fdf8 Mon Sep 17 00:00:00 2001 From: Subash Shibu Date: Thu, 27 Nov 2025 18:43:28 -0800 Subject: [PATCH 3/3] Updated to base64 and fixed error handling --- package.json | 2 +- pnpm-lock.yaml | 10 ++++---- src/tools/hover.ts | 63 ++++++++++++++++++++-------------------------- 3 files changed, 33 insertions(+), 42 deletions(-) diff --git a/package.json b/package.json index 3d34f8b..76d8fd7 100644 --- a/package.json +++ b/package.json @@ -44,7 +44,7 @@ "axios": "^1.12.2", "dotenv": "^17.2.3", "express": "^5.1.0", - "gbox-sdk": "^0.42.0", + "gbox-sdk": "^0.43.0", "jose": "^6.1.0", "zod": "^3.25.76" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 78b2102..0395381 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: ^5.1.0 version: 5.1.0 gbox-sdk: - specifier: ^0.42.0 - version: 0.42.0(typescript@5.9.3) + specifier: ^0.43.0 + version: 0.43.0(typescript@5.9.3) jose: specifier: ^6.1.0 version: 6.1.0 @@ -950,8 +950,8 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} - gbox-sdk@0.42.0: - resolution: {integrity: sha512-qXm/XjAfsbWplHO4juThD0SoEqN3Mp9jbPMlKzXm1BfoXHtOjSFSC0pN+1aPMkveQZiiywleKuk/qhKnvOmlQA==} + gbox-sdk@0.43.0: + resolution: {integrity: sha512-cVrKgvM8Vr8vyQIKe2RxUKcjSB0TYMGiUAmtm/nVyL25sJB35g0BN13kDYzwHo122dKaOzYVo+kCMLvRxBADBw==} engines: {node: '>=16.0.0', npm: please-use-yarn, yarn: '>=1.22.0'} get-intrinsic@1.3.0: @@ -2505,7 +2505,7 @@ snapshots: function-bind@1.1.2: {} - gbox-sdk@0.42.0(typescript@5.9.3): + gbox-sdk@0.43.0(typescript@5.9.3): dependencies: smol-toml: 1.4.2 typedoc: 0.28.14(typescript@5.9.3) diff --git a/src/tools/hover.ts b/src/tools/hover.ts index c0500f1..63ee85e 100644 --- a/src/tools/hover.ts +++ b/src/tools/hover.ts @@ -22,22 +22,22 @@ type HoverParams = z.infer>; /** * Call GBOX Handy API to detect element coordinates using SDK + * @throws Error if element detection fails or element not found */ async function detectElementWithHandy( - screenshotUrl: string, + screenshotUri: string, target: string, logger: MCPLogger, gboxSDK: GboxSDK -): Promise<{ x: number; y: number } | null> { +): Promise<{ x: number; y: number }> { try { await logger.debug("Calling GBOX Handy API", { target, - screenshotUrl, }); const result = await gboxSDK.model.call({ model: "gbox-handy-1", - screenshot: screenshotUrl, + screenshot: screenshotUri, action: { type: "click", target: target, @@ -51,39 +51,42 @@ async function detectElementWithHandy( responseType: result.response.type, requestId: result.id, }); - return null; + throw new Error( + `GBOX Handy returned unexpected action type: ${result.response.type}` + ); } - // Type assertion after type guard - we know it's a click response with x, y coordinates const clickCoords = result.response.coordinates as { x: number; y: number }; // Check if coordinates are valid (not -1, -1 which indicates no target found) - if (clickCoords.x !== -1 && clickCoords.y !== -1) { - const coordinates = { - x: Math.round(clickCoords.x), - y: Math.round(clickCoords.y), - }; - - await logger.info("GBOX Handy detected element", { + if (clickCoords.x === -1 && clickCoords.y === -1) { + await logger.info("GBOX Handy could not find element", { target, - coordinates, requestId: result.id, }); - - return coordinates; + throw new Error( + `Element not found: "${target}". Please provide a more specific description.` + ); } - await logger.info("GBOX Handy could not find element", { + const coordinates = { + x: Math.round(clickCoords.x), + y: Math.round(clickCoords.y), + }; + + await logger.info("GBOX Handy detected element", { target, + coordinates, requestId: result.id, }); - return null; + + return coordinates; } catch (error) { await logger.error("GBOX Handy detection failed", { error: error instanceof Error ? error.message : String(error), target, }); - return null; + throw error; } } @@ -111,33 +114,21 @@ export function handleHover(logger: MCPLogger, gboxSDK: GboxSDK) { // Step 2: Take screenshot await logger.debug("Taking screenshot for element detection"); const screenshotResult = await box.action.screenshot({ - outputFormat: "storageKey", + outputFormat: "base64", }); - if (!screenshotResult || !screenshotResult.presignedUrl) { - throw new Error("Failed to capture screenshot or get presigned URL"); + if (!screenshotResult || !screenshotResult.uri) { + throw new Error("Failed to capture screenshot"); } - // Step 3: Detect element with GBOX Handy using presigned URL + // Step 3: Detect element with GBOX Handy using base64 screenshot const coordinates = await detectElementWithHandy( - screenshotResult.presignedUrl, + screenshotResult.uri, target, logger, gboxSDK ); - if (!coordinates) { - return { - content: [ - { - type: "text" as const, - text: `Element not found: "${target}". Please provide a more specific description.`, - }, - ], - isError: true, - }; - } - // Step 4: Execute move action with coordinates await logger.debug("Moving mouse to coordinates", { coordinates,