From f5a4b425daaa695a4d5a75012f38c82097dee179 Mon Sep 17 00:00:00 2001 From: Marco Quinten Date: Mon, 7 Apr 2025 20:35:14 +0700 Subject: [PATCH] feat(browserTool): Implement hover action (#2368) * Implement hover action for the browser action tool * Update snapshots --- .../__tests__/__snapshots__/system.test.ts.snap | 10 ++++++++-- src/core/prompts/tools/browser-action.ts | 5 ++++- src/core/tools/browserActionTool.ts | 6 +++++- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap b/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap index 798aed2976..fe9908aa97 100644 --- a/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap +++ b/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap @@ -2714,6 +2714,9 @@ Parameters: * launch: Launch a new Puppeteer-controlled browser instance at the specified URL. This **must always be the first action**. - Use with the \`url\` parameter to provide the URL. - Ensure the URL is valid and includes the appropriate protocol (e.g. http://localhost:3000/page, file:///path/to/file.html, etc.) + * hover: Move the cursor to a specific x,y coordinate. + - Use with the \`coordinate\` parameter to specify the location. + - Always move to the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. * click: Click at a specific x,y coordinate. - Use with the \`coordinate\` parameter to specify the location. - Always click in the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. @@ -2727,7 +2730,7 @@ Parameters: - Example: \`close\` - url: (optional) Use this for providing the URL for the \`launch\` action. * Example: https://example.com -- coordinate: (optional) The X and Y coordinates for the \`click\` action. Coordinates should be within the **900x600** resolution. +- coordinate: (optional) The X and Y coordinates for the \`click\` and \`hover\` actions. Coordinates should be within the **900x600** resolution. * Example: 450,300 - size: (optional) The width and height for the \`resize\` action. * Example: 1280,720 @@ -3629,6 +3632,9 @@ Parameters: * launch: Launch a new Puppeteer-controlled browser instance at the specified URL. This **must always be the first action**. - Use with the \`url\` parameter to provide the URL. - Ensure the URL is valid and includes the appropriate protocol (e.g. http://localhost:3000/page, file:///path/to/file.html, etc.) + * hover: Move the cursor to a specific x,y coordinate. + - Use with the \`coordinate\` parameter to specify the location. + - Always move to the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. * click: Click at a specific x,y coordinate. - Use with the \`coordinate\` parameter to specify the location. - Always click in the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. @@ -3642,7 +3648,7 @@ Parameters: - Example: \`close\` - url: (optional) Use this for providing the URL for the \`launch\` action. * Example: https://example.com -- coordinate: (optional) The X and Y coordinates for the \`click\` action. Coordinates should be within the **1280x800** resolution. +- coordinate: (optional) The X and Y coordinates for the \`click\` and \`hover\` actions. Coordinates should be within the **1280x800** resolution. * Example: 450,300 - size: (optional) The width and height for the \`resize\` action. * Example: 1280,720 diff --git a/src/core/prompts/tools/browser-action.ts b/src/core/prompts/tools/browser-action.ts index 510bf7b794..e1b33b9d7d 100644 --- a/src/core/prompts/tools/browser-action.ts +++ b/src/core/prompts/tools/browser-action.ts @@ -15,6 +15,9 @@ Parameters: * launch: Launch a new Puppeteer-controlled browser instance at the specified URL. This **must always be the first action**. - Use with the \`url\` parameter to provide the URL. - Ensure the URL is valid and includes the appropriate protocol (e.g. http://localhost:3000/page, file:///path/to/file.html, etc.) + * hover: Move the cursor to a specific x,y coordinate. + - Use with the \`coordinate\` parameter to specify the location. + - Always move to the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. * click: Click at a specific x,y coordinate. - Use with the \`coordinate\` parameter to specify the location. - Always click in the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. @@ -28,7 +31,7 @@ Parameters: - Example: \`close\` - url: (optional) Use this for providing the URL for the \`launch\` action. * Example: https://example.com -- coordinate: (optional) The X and Y coordinates for the \`click\` action. Coordinates should be within the **${args.browserViewportSize}** resolution. +- coordinate: (optional) The X and Y coordinates for the \`click\` and \`hover\` actions. Coordinates should be within the **${args.browserViewportSize}** resolution. * Example: 450,300 - size: (optional) The width and height for the \`resize\` action. * Example: 1280,720 diff --git a/src/core/tools/browserActionTool.ts b/src/core/tools/browserActionTool.ts index de6e8c1c7f..406a9f1fad 100644 --- a/src/core/tools/browserActionTool.ts +++ b/src/core/tools/browserActionTool.ts @@ -73,7 +73,7 @@ export async function browserActionTool( await cline.browserSession.launchBrowser() browserActionResult = await cline.browserSession.navigateToUrl(url) } else { - if (action === "click") { + if (action === "click" || action === "hover") { if (!coordinate) { cline.consecutiveMistakeCount++ pushToolResult(await cline.sayAndCreateMissingParamError("browser_action", "coordinate")) @@ -112,6 +112,9 @@ export async function browserActionTool( case "click": browserActionResult = await cline.browserSession.click(coordinate!) break + case "hover": + browserActionResult = await cline.browserSession.hover(coordinate!) + break case "type": browserActionResult = await cline.browserSession.type(text!) break @@ -133,6 +136,7 @@ export async function browserActionTool( switch (action) { case "launch": case "click": + case "hover": case "type": case "scroll_down": case "scroll_up":