diff --git a/package.json b/package.json index 186c214..76d8fd7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@gbox.ai/mcp-server", - "version": "0.1.15", + "version": "0.1.17", "description": "MCP server exposing Gbox Android and Linux control tools via Model Context Protocol", "homepage": "https://gbox.ai", "author": "Gbox Team", @@ -44,7 +44,7 @@ "axios": "^1.12.2", "dotenv": "^17.2.3", "express": "^5.1.0", - "gbox-sdk": "^0.37.0", + "gbox-sdk": "^0.43.0", "jose": "^6.1.0", "zod": "^3.25.76" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9dab849..0395381 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -21,8 +21,8 @@ importers: specifier: ^5.1.0 version: 5.1.0 gbox-sdk: - specifier: ^0.37.0 - version: 0.37.0(typescript@5.9.3) + specifier: ^0.43.0 + version: 0.43.0(typescript@5.9.3) jose: specifier: ^6.1.0 version: 6.1.0 @@ -950,8 +950,8 @@ packages: function-bind@1.1.2: resolution: {integrity: sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==} - gbox-sdk@0.37.0: - resolution: {integrity: sha512-v6aQjTxTLvkvGWqsqp/yIDsOJoQDi58+weF29L0ZMWMV9HFXHgXeA5nnch1qAlrMY9Mrr7lfAKuzSH6aU6eo+A==} + gbox-sdk@0.43.0: + resolution: {integrity: sha512-cVrKgvM8Vr8vyQIKe2RxUKcjSB0TYMGiUAmtm/nVyL25sJB35g0BN13kDYzwHo122dKaOzYVo+kCMLvRxBADBw==} engines: {node: '>=16.0.0', npm: please-use-yarn, yarn: '>=1.22.0'} get-intrinsic@1.3.0: @@ -2505,7 +2505,7 @@ snapshots: function-bind@1.1.2: {} - gbox-sdk@0.37.0(typescript@5.9.3): + gbox-sdk@0.43.0(typescript@5.9.3): dependencies: smol-toml: 1.4.2 typedoc: 0.28.14(typescript@5.9.3) diff --git a/src/server/server-factory.ts b/src/server/server-factory.ts index fb69f65..1027daf 100644 --- a/src/server/server-factory.ts +++ b/src/server/server-factory.ts @@ -133,6 +133,12 @@ import { CLOSE_TAB_TOOL, closeTabParamsSchema, } from "../tools/close-tab.js"; +import { + handleHover, + HOVER_DESCRIPTION, + HOVER_TOOL, + hoverParamsSchema, +} from "../tools/hover.js"; /** * Factory class for creating McpServer instances @@ -312,6 +318,13 @@ export class McpServerFactory { pressKeyParamsSchema, handlePressKey(logger, gboxSDK) ); + + server.tool( + HOVER_TOOL, + HOVER_DESCRIPTION, + hoverParamsSchema, + handleHover(logger, gboxSDK) + ); } /** @@ -377,6 +390,13 @@ export class McpServerFactory { closeTabParamsSchema, handleCloseTab(logger, gboxSDK) ); + + server.tool( + HOVER_TOOL, + HOVER_DESCRIPTION, + hoverParamsSchema, + handleHover(logger, gboxSDK) + ); } /** diff --git a/src/tools/hover.ts b/src/tools/hover.ts new file mode 100644 index 0000000..63ee85e --- /dev/null +++ b/src/tools/hover.ts @@ -0,0 +1,208 @@ +import { z } from "zod"; +import GboxSDK from "gbox-sdk"; +import type { MCPLogger } from "../logger/logger.js"; +import { attachBox } from "../sdk/index.js"; +import { extractImageInfo } from "../sdk/utils.js"; + +export const HOVER_TOOL = "hover"; + +export const HOVER_DESCRIPTION = + "Hover over a UI element on the browser using natural language description. The element will be detected automatically and the mouse will move to it without clicking."; + +export const hoverParamsSchema = { + boxId: z.string().describe("ID of the browser box"), + target: z + .string() + .describe( + "Description of the element to hover over (e.g. 'login button', 'search field', 'submit button'). MUST be detailed enough to identify the element unambiguously." + ), +}; + +type HoverParams = z.infer>; + +/** + * Call GBOX Handy API to detect element coordinates using SDK + * @throws Error if element detection fails or element not found + */ +async function detectElementWithHandy( + screenshotUri: string, + target: string, + logger: MCPLogger, + gboxSDK: GboxSDK +): Promise<{ x: number; y: number }> { + try { + await logger.debug("Calling GBOX Handy API", { + target, + }); + + const result = await gboxSDK.model.call({ + model: "gbox-handy-1", + screenshot: screenshotUri, + action: { + type: "click", + target: target, + }, + }); + + // Check if response is a click action + if (result.response.type !== "click") { + await logger.error("GBOX Handy returned unexpected action type", { + target, + responseType: result.response.type, + requestId: result.id, + }); + throw new Error( + `GBOX Handy returned unexpected action type: ${result.response.type}` + ); + } + + const clickCoords = result.response.coordinates as { x: number; y: number }; + + // Check if coordinates are valid (not -1, -1 which indicates no target found) + if (clickCoords.x === -1 && clickCoords.y === -1) { + await logger.info("GBOX Handy could not find element", { + target, + requestId: result.id, + }); + throw new Error( + `Element not found: "${target}". Please provide a more specific description.` + ); + } + + const coordinates = { + x: Math.round(clickCoords.x), + y: Math.round(clickCoords.y), + }; + + await logger.info("GBOX Handy detected element", { + target, + coordinates, + requestId: result.id, + }); + + return coordinates; + } catch (error) { + await logger.error("GBOX Handy detection failed", { + error: error instanceof Error ? error.message : String(error), + target, + }); + throw error; + } +} + +export function handleHover(logger: MCPLogger, gboxSDK: GboxSDK) { + return async ({ boxId, target }: HoverParams) => { + const startTime = Date.now(); + + try { + await logger.info("Hover command invoked", { + boxId, + target, + timestamp: new Date().toISOString(), + }); + + // Step 1: Attach to box + const boxAttachStart = Date.now(); + const box = await attachBox(boxId, gboxSDK); + const boxAttachDuration = Date.now() - boxAttachStart; + + await logger.debug("Box attached successfully", { + boxId, + attachDurationMs: boxAttachDuration, + }); + + // Step 2: Take screenshot + await logger.debug("Taking screenshot for element detection"); + const screenshotResult = await box.action.screenshot({ + outputFormat: "base64", + }); + + if (!screenshotResult || !screenshotResult.uri) { + throw new Error("Failed to capture screenshot"); + } + + // Step 3: Detect element with GBOX Handy using base64 screenshot + const coordinates = await detectElementWithHandy( + screenshotResult.uri, + target, + logger, + gboxSDK + ); + + // Step 4: Execute move action with coordinates + await logger.debug("Moving mouse to coordinates", { + coordinates, + target, + }); + + const moveResult = await box.action.move({ + x: coordinates.x, + y: coordinates.y, + options: { + screenshot: { + phases: ["after"], + outputFormat: "base64", + delay: "500ms", + }, + }, + }); + + const totalDuration = Date.now() - startTime; + + await logger.info("Hover action completed successfully", { + boxId, + target, + coordinates, + totalDurationMs: totalDuration, + actionId: moveResult.actionId, + }); + + return { + content: [ + { + type: "text" as const, + text: `Hover action completed successfully. Moved to "${target}" at (${coordinates.x}, ${coordinates.y})`, + }, + { + type: "image" as const, + ...extractImageInfo(moveResult.screenshot.after.uri), + }, + ], + }; + } catch (error) { + const totalDuration = Date.now() - startTime; + + const errorDetails: Record = { + boxId, + target, + totalDurationMs: totalDuration, + timestamp: new Date().toISOString(), + errorType: error?.constructor?.name, + errorMessage: error instanceof Error ? error.message : String(error), + }; + + if (error && typeof error === "object") { + if ("status" in error) + errorDetails.httpStatus = (error as { status: unknown }).status; + if ("stack" in error) errorDetails.stack = (error as Error).stack; + } + + await logger.error("Failed to run hover action", errorDetails); + + let userMessage = `Error: ${error instanceof Error ? error.message : String(error)}`; + if (errorDetails.httpStatus) { + userMessage += ` (HTTP ${errorDetails.httpStatus})`; + } + + return { + content: [ + { + type: "text" as const, + text: userMessage, + }, + ], + isError: true, + }; + } + }; +}