From 59e976a092ffa0e32fbf3ecc6d715deecf13891c Mon Sep 17 00:00:00 2001 From: a8trejo Date: Fri, 13 Dec 2024 11:14:25 -0600 Subject: [PATCH] Adding new explicit browser action snapshot to work with interactive mode to chat while cline analyses the browser --- src/core/Cline.ts | 66 ++++++--- .../parse-assistant-message.ts | 1 + src/core/prompts/system.ts | 11 +- src/services/browser/BrowserSession.ts | 134 +++++++++++++----- src/shared/ExtensionMessage.ts | 2 +- 5 files changed, 152 insertions(+), 62 deletions(-) diff --git a/src/core/Cline.ts b/src/core/Cline.ts index a42996b277..b25fa821c3 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -108,7 +108,7 @@ export class Cline { this.api = buildApiHandler(apiConfiguration) this.terminalManager = new TerminalManager() this.urlContentFetcher = new UrlContentFetcher(provider.context) - this.browserSession = new BrowserSession(provider.context) + this.browserSession = new BrowserSession(provider.context, provider) this.diffViewProvider = new DiffViewProvider(cwd) this.isInteractiveMode = isInteractiveMode ?? false this.browserPort = browserPort ?? "7333" @@ -1498,7 +1498,15 @@ export class Cline { try { if (block.partial) { - if (action === "launch") { + if (action === "snapshot") { + // For snapshot, we just show the streaming UI update + await this.say( + "browser_action", + JSON.stringify({ + action: "snapshot" + } satisfies ClineSayBrowserAction), + ) + } else if (action === "launch") { await this.ask( "browser_action_launch", removeClosingTag("url", url), @@ -1519,25 +1527,32 @@ export class Cline { break } else { let browserActionResult: BrowserActionResult - if (action === "launch") { + if (action === "snapshot") { + await this.say("browser_action",JSON.stringify({action: "snapshot"} satisfies ClineSayBrowserAction)) + browserActionResult = await this.browserSession.takeScreenshot(this.isInteractiveMode, this.browserPort) + } else if (action === "launch") { if (!url) { - this.consecutiveMistakeCount++ - pushToolResult(await this.sayAndCreateMissingParamError("browser_action", "url")) - await this.browserSession.closeBrowser() - break - } - - this.consecutiveMistakeCount = 0 - const didApprove = await askApproval("browser_action_launch", url) - if (!didApprove) { - break - } + if (this.isInteractiveMode) { + browserActionResult = await this.browserSession.takeScreenshot() + } else { + this.consecutiveMistakeCount++ + pushToolResult(await this.sayAndCreateMissingParamError("browser_action", "url")) + await this.browserSession.closeBrowser() + break + } + } else { + this.consecutiveMistakeCount = 0 + const didApprove = await askApproval("browser_action_launch", url) + if (!didApprove) { + break + } - // NOTE: it's okay that we call this message since the partial inspect_site is finished streaming. The only scenario we have to avoid is sending messages WHILE a partial message exists at the end of the messages array. For example the api_req_finished message would interfere with the partial message, so we needed to remove that. - // await this.say("inspect_site_result", "") // no result, starts the loading spinner waiting for result - await this.say("browser_action_result", "") // starts loading spinner - await this.browserSession.launchBrowser(this.isInteractiveMode, this.browserPort) - browserActionResult = await this.browserSession.navigateToUrl(url) + // NOTE: it's okay that we call this message since the partial inspect_site is finished streaming. The only scenario we have to avoid is sending messages WHILE a partial message exists at the end of the messages array. For example the api_req_finished message would interfere with the partial message, so we needed to remove that. + // await this.say("inspect_site_result", "") // no result, starts the loading spinner waiting for result + await this.say("browser_action_result", "") // starts loading spinner + await this.browserSession.launchBrowser(this.isInteractiveMode, this.browserPort) + browserActionResult = await this.browserSession.navigateToUrl(url) + } } else { if (action === "click") { if (!coordinate) { @@ -1592,6 +1607,19 @@ export class Cline { } } switch (action) { + case "snapshot": { + const { screenshot, ...snapshotResult } = browserActionResult + await this.say("browser_action_result", JSON.stringify(snapshotResult)) + pushToolResult( + formatResponse.toolResult( + `The browser action has been executed. The console logs have been captured for your analysis.\n\nConsole logs:\n${ + browserActionResult.logs || "(No new logs)" + }\n\n(REMEMBER: if you need to proceed to using non-\`browser_action\` tools or launch a new browser, you MUST first close this browser. For example, if after analyzing the logs and screenshot you need to edit a file, you must first close the browser before you can use the write_to_file tool.)`, + browserActionResult.screenshot ? [browserActionResult.screenshot] : [], + ), + ) + break + } case "launch": case "click": case "type": diff --git a/src/core/assistant-message/parse-assistant-message.ts b/src/core/assistant-message/parse-assistant-message.ts index e38e8f6458..9a7c31a5bc 100644 --- a/src/core/assistant-message/parse-assistant-message.ts +++ b/src/core/assistant-message/parse-assistant-message.ts @@ -8,6 +8,7 @@ import { ToolUseName, } from "." +// In case of debugging, you may pass 'provider: ClineProvider' as a second argument, and use 'providerRef.deref()?.outputChannel.appendLine' export function parseAssistantMessage(assistantMessage: string) { let contentBlocks: AssistantMessageContent[] = [] let currentTextContent: TextContent | undefined = undefined diff --git a/src/core/prompts/system.ts b/src/core/prompts/system.ts index 531f215bea..788a080e44 100644 --- a/src/core/prompts/system.ts +++ b/src/core/prompts/system.ts @@ -107,12 +107,15 @@ Usage: ## browser_action Description: Request to interact with a Puppeteer-controlled browser. Every action, except \`close\`, will be responded to with a screenshot of the browser's current state, along with any new console logs. You may only perform one browser action per message, and wait for the user's response including a screenshot and logs to determine the next action. -- The sequence of actions **must always start with** launching the browser at a URL, and **must always end with** closing the browser. If you need to visit a new URL that is not possible to navigate to from the current webpage, you must first close the browser, then launch again at the new URL. -- While the browser is active, only the \`browser_action\` tool can be used. No other tools should be called during this time. You may proceed to use other tools only after closing the browser. For example if you run into an error and need to fix a file, you must close the browser, then use other tools to make the necessary changes, then re-launch the browser to verify the result. -- The browser window has a resolution of **900x600** pixels. When performing any click actions, ensure the coordinates are within this resolution range. +- Whenever the user requests for a snapshot, or to interact or view **his browser**, use 'browser_action' with 'snapshot' as the action. +- If the action is \'snapshot\', and \'Interactive Browser Mode\' is enabled, a browser will already be running and you only need to connect to it, respond with a screenshot of the browser's current state and logs. +- For all other actions, the sequence of actions **must always start with** launching the browser at a URL, and **must always end with** closing the browser. If you need to visit a new URL that is not possible to navigate to from the current webpage, you must first close the browser, then launch again at the new URL. +- While the browser is active and \'Interactive Browser Mode\' is disabled, only the \`browser_action\` tool can be used. No other tools should be called during this time. You may proceed to use other tools only after closing the browser. For example if you run into an error and need to fix a file, you must close the browser, then use other tools to make the necessary changes, then re-launch the browser to verify the result. +- The browser window has a resolution of **1440x900** pixels. When performing any click actions, ensure the coordinates are within this resolution range. - Before clicking on any elements such as icons, links, or buttons, you must consult the provided screenshot of the page to determine the coordinates of the element. The click should be targeted at the **center of the element**, not on its edges. Parameters: - action: (required) The action to perform. The available actions are: + * snapshot: Take a screenshot of the current browser state * launch: Launch a new Puppeteer-controlled browser instance at the specified URL. This **must always be the first action**. - Use with the \`url\` parameter to provide the URL. - Ensure the URL is valid and includes the appropriate protocol (e.g. http://localhost:3000/page, file:///path/to/file.html, etc.) @@ -228,7 +231,7 @@ CAPABILITIES - For example, when asked to make edits or improvements you might analyze the file structure in the initial environment_details to get an overview of the project, then use list_code_definition_names to get further insight using source code definitions for files located in relevant directories, then read_file to examine the contents of relevant files, analyze the code and suggest improvements or make necessary edits, then use the write_to_file ${diffStrategy ? "or apply_diff " : ""}tool to apply the changes. If you refactored code that could affect other parts of the codebase, you could use search_files to ensure you update other files as needed. - You can use the execute_command tool to run commands on the user's computer whenever you feel it can help accomplish the user's task. When you need to execute a CLI command, you must provide a clear explanation of what the command does. Prefer to execute complex CLI commands over creating executable scripts, since they are more flexible and easier to run. Interactive and long-running commands are allowed, since the commands are run in the user's VSCode terminal. The user may keep commands running in the background and you will be kept updated on their status along the way. Each command you execute is run in a new terminal instance.${ supportsComputerUse - ? "\n- You can use the browser_action tool to interact with websites (including html files and locally running development servers) through a Puppeteer-controlled browser when you feel it is necessary in accomplishing the user's task. This tool is particularly useful for web development tasks as it allows you to launch a browser, navigate to pages, interact with elements through clicks and keyboard input, and capture the results through screenshots and console logs. This tool may be useful at key stages of web development tasks-such as after implementing new features, making substantial changes, when troubleshooting issues, or to verify the result of your work. You can analyze the provided screenshots to ensure correct rendering or identify errors, and review console logs for runtime issues.\n - For example, if asked to add a component to a react website, you might create the necessary files, use execute_command to run the site locally, then use browser_action to launch the browser, navigate to the local server, and verify the component renders & functions correctly before closing the browser." + ? "\n- You can use the browser_action tool to interact with websites (including html files and locally running development servers) through a Puppeteer-controlled browser when you feel it is necessary in accomplishing the user's task. This tool is particularly useful for web development tasks as it allows you to connect or launch a browser, navigate to pages, interact with elements through clicks and keyboard input, and capture the results through screenshots and console logs. This tool may be useful at key stages of web development tasks-such as after implementing new features, making substantial changes, when troubleshooting issues, or to verify the result of your work. You can analyze the provided screenshots to ensure correct rendering or identify errors, and review console logs for runtime issues.\n - For example, if asked to add a component to a react website, you might create the necessary files, use execute_command to run the site locally, then use browser_action to launch the browser, navigate to the local server, and verify the component renders & functions correctly before closing the browser." : "" } diff --git a/src/services/browser/BrowserSession.ts b/src/services/browser/BrowserSession.ts index 83345d7164..82eae95cf3 100644 --- a/src/services/browser/BrowserSession.ts +++ b/src/services/browser/BrowserSession.ts @@ -1,10 +1,8 @@ import * as vscode from "vscode" -import * as fs from "fs/promises" -import * as path from "path" import { Browser, Page, ScreenshotOptions, TimeoutError, launch, connect } from "puppeteer" import pWaitFor from "p-wait-for" import delay from "delay" -import { fileExistsAtPath } from "../../utils/fs" +import { ClineProvider } from "../../core/webview/ClineProvider" import { BrowserActionResult } from "../../shared/ExtensionMessage" import axios from 'axios' @@ -15,9 +13,11 @@ export class BrowserSession { private currentMousePosition?: string private isInteractive: boolean = false private browserPort: string = '7333' + private providerRef: WeakRef - constructor(context: vscode.ExtensionContext) { + constructor(context: vscode.ExtensionContext, provider: ClineProvider,) { this.context = context + this.providerRef = new WeakRef(provider) } private async puppeteerLaunch() { @@ -33,6 +33,29 @@ export class BrowserSession { }) } + // New method to handle browser connection + private async puppeteerConnect(port: string): Promise { + try { + this.providerRef.deref()?.outputChannel.appendLine(`BrowserSession.ts :: puppeteerConnect :: ${port}`) + const response = await axios.get(`http://127.0.0.1:${port}/json/version`) + const browserWSEndpoint = response.data.webSocketDebuggerUrl + + if (!browserWSEndpoint) { + this.providerRef.deref()?.outputChannel.appendLine(`BrowserSession.ts :: puppeteerConnect :: No webSocketDebuggerUrl found`) + console.log("BrowserSession.ts :: puppeteerConnect :: No webSocketDebuggerUrl found") + return undefined + } + + return await connect({ + browserWSEndpoint, + }) + } catch (error) { + this.providerRef.deref()?.outputChannel.appendLine(`BrowserSession.ts :: puppeteerConnect :: Failed to connect: ${error}`) + console.log("BrowserSession.ts :: puppeteerConnect :: Failed to connect:", error) + return undefined + } + } + async launchBrowser(interactive: boolean = false, port?: string) { console.log("launch browser called") this.isInteractive = interactive @@ -43,24 +66,7 @@ export class BrowserSession { } if (this.isInteractive) { - try { - // Fetch the WebSocket endpoint from Chrome's debugging API using axios - const response = await axios.get(`http://127.0.0.1:${this.browserPort}/json/version`) - const browserWSEndpoint = response.data.webSocketDebuggerUrl - - if (!browserWSEndpoint) { - console.log("BrowserSession.ts :: launchBrowser :: No webSocketDebuggerUrl found, falling back to regular launch mode") - this.isInteractive = false - } else { - this.browser = await connect({ - browserWSEndpoint, - }) - } - } catch (error) { - console.log(`BrowserSession.ts :: launchBrowser :: Failed to connect to browser, falling back to regular launch mode. Error: ${error.message}`) - this.isInteractive = false - } - + this.browser = await this.puppeteerConnect(this.browserPort) // If interactive mode failed, fall back to regular launch mode if (!this.browser) { this.browser = await this.puppeteerLaunch() @@ -69,7 +75,10 @@ export class BrowserSession { this.browser = await this.puppeteerLaunch() } - this.page = await this.browser?.newPage() + // TO DO: Might no longer be needed and this.page = await this.browser?.newPage() could be enough + // Get existing pages or create new one + const pages = await this.browser.pages() + this.page = pages[0] || await this.browser.newPage() await this.page?.setViewport({ width: 1440, height: 900, @@ -78,8 +87,8 @@ export class BrowserSession { }); return { - screenshot: "", - logs: this.isInteractive ? + screenshot: this.isInteractive ? await this.getCurrentScreenshot() : "", + logs: this.isInteractive ? "Connected to browser in remote debugging mode." : "Browser launched successfully.", currentUrl: this.page?.url(), @@ -100,38 +109,65 @@ export class BrowserSession { } private async getCurrentScreenshot(): Promise { - if (!this.page) return undefined - - let options: ScreenshotOptions = { - encoding: "base64", + if (this.isInteractive) { + this.browser = await this.puppeteerConnect(this.browserPort) + + if (this.browser) { + const pages = await this.browser.pages() + this.page = pages[0] || await this.browser.newPage() + } else { + this.providerRef.deref()?.outputChannel.appendLine(`BrowserSession.ts :: getCurrentScreenshot :: Failed to connect to browser for screenshot`) + throw new Error("Failed to connect to browser for screenshot") + } } + if (!this.page) { + this.providerRef.deref()?.outputChannel.appendLine(`BrowserSession.ts :: getCurrentScreenshot :: Failed to take over a page to screenshot`) + throw new Error("Failed to get page for screenshot") + } + + const screenshotType ="webp" try { let screenshotBase64 = await this.page.screenshot({ - ...options, - type: "webp", + encoding: "base64", + type: screenshotType, }) - return `data:image/webp;base64,${screenshotBase64}` + return `data:image/${screenshotType};base64,${screenshotBase64}` } catch (err) { try { let screenshotBase64 = await this.page.screenshot({ - ...options, + encoding: "base64", type: "png", }) return `data:image/png;base64,${screenshotBase64}` } catch (err) { - console.error("Failed to take screenshot:", err) + this.providerRef.deref()?.outputChannel.appendLine(`BrowserSession.ts :: getCurrentScreenshot :: Failed to take ${screenshotType} screenshot`) + console.error(`Failed to take ${screenshotType} screenshot:`, err) return undefined } } } async doAction(action: (page: Page) => Promise): Promise { + try { + if (this.isInteractive && !this.browser) { + this.browser = await this.puppeteerConnect(this.browserPort) + + if (this.browser) { + const pages = await this.browser.pages() + this.page = pages[0] || await this.browser.newPage() + } + } + // Ensure we have a browser and page + if (!this.browser || !this.page) { + throw new Error("Browser is not launched or connected. This may occur if the browser was automatically closed by a non-`browser_action` tool.") + } - if (!this.page) { - throw new Error( - "Browser is not launched or connected. This may occur if the browser was automatically closed by a non-`browser_action` tool.", - ) + }// For interactive mode and snapshot, try to connect first + catch(err) { + this.providerRef.deref()?.outputChannel.appendLine(`BrowserSession.ts :: doAction :: Browser action failed`) + console.error("Browser action failed:", err) + throw err } const logs: string[] = [] @@ -284,6 +320,28 @@ export class BrowserSession { await delay(300) }) } + + async takeScreenshot(interactive: boolean = false, port?: string): Promise { + this.isInteractive = interactive + this.browserPort = port ?? this.browserPort + + // If no browser session exists or we're in interactive mode, try to connect + if ((!this.page || !this.browser) && this.isInteractive) { + this.browser = await this.puppeteerConnect(this.browserPort) + + if (this.browser) { + const pages = await this.browser.pages() + this.page = pages[0] || await this.browser.newPage() + } else { + this.providerRef.deref()?.outputChannel.appendLine(`BrowserSession.ts :: takeScreenshot :: Failed to connect to browser for screenshot`) + throw new Error("Failed to connect to browser for screenshot") + } + } + + return this.doAction(async (page) => { + // doAction will handle actually taking the screenshot + }) + } } diff --git a/src/shared/ExtensionMessage.ts b/src/shared/ExtensionMessage.ts index c2f9077ffa..7788139cd7 100644 --- a/src/shared/ExtensionMessage.ts +++ b/src/shared/ExtensionMessage.ts @@ -106,7 +106,7 @@ export interface ClineSayTool { } // must keep in sync with system prompt -export const browserActions = ["launch", "click", "type", "scroll_down", "scroll_up", "close"] as const +export const browserActions = ["launch", "click", "type", "scroll_down", "scroll_up", "close", "snapshot"] as const export type BrowserAction = (typeof browserActions)[number] export interface ClineSayBrowserAction {