From fe1d823dc36ce443a66f66ce9a4c7ffd5ef91f10 Mon Sep 17 00:00:00 2001 From: a8trejo Date: Wed, 27 Nov 2024 17:55:06 -0800 Subject: [PATCH] Updates to connect puppetter to browser with remote-debugging-port by prompting interactive mode text --- docs/cline-updates.md | 9 +- package-lock.json | 32 ++-- package.json | 4 +- src/core/Cline.ts | 226 ++++++++++++------------- src/core/webview/ClineProvider.ts | 2 +- src/services/browser/BrowserSession.ts | 177 +++++-------------- 6 files changed, 178 insertions(+), 272 deletions(-) diff --git a/docs/cline-updates.md b/docs/cline-updates.md index 9e8257e623..caccf6c497 100644 --- a/docs/cline-updates.md +++ b/docs/cline-updates.md @@ -2,6 +2,11 @@ Remember to rebuild the extension and install it using the cli specified on the Consider that any `export` or `install` command has already been run, so exclude those.ˆ +Before editing the file 'Roo-Cline/src/core/Cline.ts', since it's a really large file, read the original file to understand its full content. +Then create a new file with the -updates suffix containing ONLY the changes you've made to the original file. -First, read the original 'Cline.ts' file to understand its full content -Then create a new file with the -updates suffix containing ONLY the changes \ No newline at end of file + +In interactive mode, browser actions are disabled (if (!this.isInteractive) check in doAction method) +The navigateToUrl method uses doAction, which means navigation won't work in interactive mode + +Can we analyze 'Roo-Cline/src/core/Cline.ts' to see how it handles browser actions and if there's some logic preventing it from using 'doAction' \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 4e27547ad0..bd90377ecd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -32,7 +32,7 @@ "os-name": "^6.0.0", "p-wait-for": "^5.0.2", "pdf-parse": "^1.1.1", - "puppeteer": "^23.4.0", + "puppeteer": "^23.9.0", "serialize-error": "^11.0.3", "strip-ansi": "^7.1.0", "tree-sitter-wasms": "^0.1.11", @@ -4142,7 +4142,7 @@ }, "node_modules/@puppeteer/browsers": { "version": "2.4.1", - "resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.4.1.tgz", + "resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/@puppeteer/browsers/-/browsers-2.4.1.tgz", "integrity": "sha512-0kdAbmic3J09I6dT8e9vE2JOCSt13wHCW5x/ly8TSt2bDtuIWe2TgLZZDHdcziw9AVCzflMAXCrVyRIhIs44Ng==", "dependencies": { "debug": "^4.3.7", @@ -6700,9 +6700,9 @@ } }, "node_modules/bare-stream": { - "version": "2.3.2", - "resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/bare-stream/-/bare-stream-2.3.2.tgz", - "integrity": "sha512-EFZHSIBkDgSHIwj2l2QZfP4U5OcD4xFAOwhSb/vlr9PIqyGJGvB/nfClJbcnh3EY4jtPE4zsb5ztae96bVF79A==", + "version": "2.4.2", + "resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/bare-stream/-/bare-stream-2.4.2.tgz", + "integrity": "sha512-XZ4ln/KV4KT+PXdIWTKjsLY+quqCaEtqqtgGJVPw9AoM73By03ij64YjepK0aQvHSWDb6AfAZwqKaFu68qkrdA==", "optional": true, "dependencies": { "streamx": "^2.20.0" @@ -7655,7 +7655,7 @@ }, "node_modules/devtools-protocol": { "version": "0.0.1367902", - "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1367902.tgz", + "resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/devtools-protocol/-/devtools-protocol-0.0.1367902.tgz", "integrity": "sha512-XxtPuC3PGakY6PD7dG66/o8KwJ/LkH2/EKe19Dcw58w53dv4/vSQEkn/SzuyhHE2q4zPgCkxQBxus3VV4ql+Pg==" }, "node_modules/diff": { @@ -12534,16 +12534,16 @@ } }, "node_modules/puppeteer": { - "version": "23.8.0", - "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-23.8.0.tgz", - "integrity": "sha512-MFWDMWoCcOpwNwQIjA9gPKWrEUbj8bLCzkK56w5lZPMUT6wK4FfpgOEPxKffVmXEMYMZzgcjxzqy15b/Q1ibaw==", + "version": "23.9.0", + "resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/puppeteer/-/puppeteer-23.9.0.tgz", + "integrity": "sha512-WfB8jGwFV+qrD9dcJJVvWPFJBU6kxeu2wxJz9WooDGfM3vIiKLgzImEDBxUQnCBK/2cXB3d4dV6gs/LLpgfLDg==", "hasInstallScript": true, "dependencies": { "@puppeteer/browsers": "2.4.1", "chromium-bidi": "0.8.0", "cosmiconfig": "^9.0.0", "devtools-protocol": "0.0.1367902", - "puppeteer-core": "23.8.0", + "puppeteer-core": "23.9.0", "typed-query-selector": "^2.12.0" }, "bin": { @@ -12554,9 +12554,9 @@ } }, "node_modules/puppeteer-core": { - "version": "23.8.0", - "resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-23.8.0.tgz", - "integrity": "sha512-c2ymGN2M//We7pC+JhP2dE/g4+qnT89BO+EMSZyJmecN3DN6RNqErA7eH7DrWoNIcU75r2nP4VHa4pswAL6NVg==", + "version": "23.9.0", + "resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/puppeteer-core/-/puppeteer-core-23.9.0.tgz", + "integrity": "sha512-hLVrav2HYMVdK0YILtfJwtnkBAwNOztUdR4aJ5YKDvgsbtagNr6urUJk9HyjRA9e+PaLI3jzJ0wM7A4jSZ7Qxw==", "dependencies": { "@puppeteer/browsers": "2.4.1", "chromium-bidi": "0.8.0", @@ -13468,9 +13468,9 @@ } }, "node_modules/streamx": { - "version": "2.20.1", - "resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/streamx/-/streamx-2.20.1.tgz", - "integrity": "sha512-uTa0mU6WUC65iUvzKH4X9hEdvSW7rbPxPtwfWiLMSj3qTdQbAiUboZTxauKfpFuGIGa1C2BYijZ7wgdUXICJhA==", + "version": "2.20.2", + "resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/streamx/-/streamx-2.20.2.tgz", + "integrity": "sha512-aDGDLU+j9tJcUdPGOaHmVF1u/hhI+CsGkT02V3OKlHDV7IukOI+nTWAGkiZEKCO35rWN1wIr4tS7YFr1f4qSvA==", "dependencies": { "fast-fifo": "^1.3.2", "queue-tick": "^1.0.1", diff --git a/package.json b/package.json index 84dd184041..3e8d6c5e02 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "roo-cline", "displayName": "Roo Cline", "description": "Autonomous coding agent right in your IDE, capable of creating/editing files, running commands, using the browser, and more with your permission every step of the way.", - "version": "2.0.4", + "version": "2.1.0", "icon": "assets/icons/icon_Roo.png", "galleryBanner": { "color": "#617A91", @@ -191,7 +191,7 @@ "os-name": "^6.0.0", "p-wait-for": "^5.0.2", "pdf-parse": "^1.1.1", - "puppeteer": "^23.4.0", + "puppeteer": "^23.9.0", "serialize-error": "^11.0.3", "strip-ansi": "^7.1.0", "tree-sitter-wasms": "^0.1.11", diff --git a/src/core/Cline.ts b/src/core/Cline.ts index f6128c17f8..5dfd0e20b5 100644 --- a/src/core/Cline.ts +++ b/src/core/Cline.ts @@ -668,22 +668,43 @@ export class Cline { if (responseImages && responseImages.length > 0) { newUserContent.push(...formatResponse.imageBlocks(responseImages)) } - + const wasInteractiveBrowser = (lastRelevantMessageIndex > 0) ? modifiedClineMessages[lastRelevantMessageIndex - 1].text?.includes("interactive mode") : false await this.overwriteApiConversationHistory(modifiedApiConversationHistory) - await this.initiateTaskLoop(newUserContent) + await this.initiateTaskLoop(newUserContent, wasInteractiveBrowser) } - private async initiateTaskLoop(userContent: UserContent): Promise { - let nextUserContent = userContent - let includeFileDetails = true + private async initiateTaskLoop(userContent: UserContent, wasInteractiveBrowser: boolean = false): Promise { + // Check if any text block contains "interactive mode" + const hasInteractiveMode = userContent.some((block) => { + if (block.type === "text" && typeof block.text === "string") { + this.providerRef.deref()?.outputChannel.appendLine(`initiateTaskLoop :: block.text :: ${block.text.toLowerCase()}`) + + return (block.type === "text" && + typeof block.text === "string" && + block.text.toLowerCase().includes("interactive mode")) + } else { + return false + } + } + ) || wasInteractiveBrowser; + + this.providerRef.deref()?.outputChannel.appendLine(`initiateTaskLoop :: hasInteractiveMode :: ${hasInteractiveMode}`) + + // Set interactive mode flag if found in text + if (hasInteractiveMode) { + this.isInteractiveMode = true; + } + + let nextUserContent = userContent; + let includeFileDetails = true; while (!this.abort) { - const didEndLoop = await this.recursivelyMakeClineRequests(nextUserContent, includeFileDetails) - includeFileDetails = false // we only need file details the first time - - // The way this agentic loop works is that cline will be given a task that he then calls tools to complete. unless there's an attempt_completion call, we keep responding back to him with his tool's responses until he either attempt_completion or does not use anymore tools. If he does not use anymore tools, we ask him to consider if he's completed the task and then call attempt_completion, otherwise proceed with completing the task. - // There is a MAX_REQUESTS_PER_TASK limit to prevent infinite requests, but Cline is prompted to finish the task as efficiently as he can. - - //const totalCost = this.calculateApiCost(totalInputTokens, totalOutputTokens) + const didEndLoop = await this.recursivelyMakeClineRequests( + nextUserContent, + includeFileDetails, + this.isInteractiveMode // Pass the flag to recursivelyMakeClineRequests + ) + includeFileDetails = false + if (didEndLoop) { // For now a task never 'completes'. This will only happen if the user hits max requests and denies resetting the count. //this.say("task_completed", `Task completed. Total API usage cost: ${totalCost}`) @@ -697,7 +718,7 @@ export class Cline { { type: "text", text: formatResponse.noToolsUsed(), - }, + } ] this.consecutiveMistakeCount++ } @@ -1462,130 +1483,95 @@ export class Cline { await this.browserSession.closeBrowser() break } - - // Check if this is an interactive mode request - const isInteractiveRequest = url.toLowerCase().includes("interactive=true") - const actualUrl = isInteractiveRequest ? url.replace(/[?&]interactive=true/i, '') : url - if (isInteractiveRequest) { - this.isInteractiveMode = true - this.consecutiveMistakeCount = 0 - const didApprove = await askApproval("browser_action_launch", - `Launch browser in interactive mode for URL: ${actualUrl}`) - if (!didApprove) { - break - } - - await this.browserSession.launchBrowser(true) - browserActionResult = await this.browserSession.navigateToUrl(actualUrl) - - await this.say("browser_action_result", JSON.stringify(browserActionResult)) - pushToolResult( - formatResponse.toolResult( - `Browser launched in interactive mode. The browser will remain open for manual interaction until explicitly closed. You can:\n` + - `1. Keep browsing\n` + - `2. Take another action\n` + - `3. Close the browser\n\n` + - `Console logs:\n${browserActionResult.logs || "(No new logs)"}`, - browserActionResult.screenshot ? [browserActionResult.screenshot] : [] - ) - ) + this.consecutiveMistakeCount = 0 + const didApprove = await askApproval("browser_action_launch", url) + if (!didApprove) { break - } else { - this.isInteractiveMode = false - this.consecutiveMistakeCount = 0 - const didApprove = await askApproval("browser_action_launch", url) - if (!didApprove) { - break - } + } // NOTE: it's okay that we call this message since the partial inspect_site is finished streaming. The only scenario we have to avoid is sending messages WHILE a partial message exists at the end of the messages array. For example the api_req_finished message would interfere with the partial message, so we needed to remove that. // await this.say("inspect_site_result", "") // no result, starts the loading spinner waiting for result - - await this.say("browser_action_result", "") // starts loading spinner - - await this.browserSession.launchBrowser() - browserActionResult = await this.browserSession.navigateToUrl(actualUrl) - } - - } else if (action === "close") { - // Handle close action for interactive mode - if (this.isInteractiveMode) { - browserActionResult = await this.browserSession.closeBrowser() - if (browserActionResult.logs?.includes("Would you like to:")) { - // Browser is asking for confirmation - await this.say("browser_action_result", JSON.stringify(browserActionResult)) - pushToolResult( - formatResponse.toolResult( - `The browser is in interactive mode. Please confirm if you want to:\n` + - `1. Keep browsing\n` + - `2. Take another action\n` + - `3. Close the browser`, - browserActionResult.screenshot ? [browserActionResult.screenshot] : [] - ) - ) - break - } - this.isInteractiveMode = false - } else { - browserActionResult = await this.browserSession.closeBrowser() - } + await this.say("browser_action_result", "") // starts loading spinner + await this.browserSession.launchBrowser(this.isInteractiveMode) + browserActionResult = await this.browserSession.navigateToUrl(url) } else { - // Handle other actions (click, type, scroll) - if (this.isInteractiveMode) { - pushToolResult( - formatResponse.toolResult( - `Cannot perform browser actions while in interactive mode. The user has manual control. You can:\n` + - `1. Keep browsing\n` + - `2. Take another action\n` + - `3. Close the browser` - ) - ) - break - } - if (action === "click") { if (!coordinate) { this.consecutiveMistakeCount++ - pushToolResult(await this.sayAndCreateMissingParamError("browser_action", "coordinate")) + pushToolResult( + await this.sayAndCreateMissingParamError( + "browser_action", + "coordinate", + ), + ) await this.browserSession.closeBrowser() - break + break // can't be within an inner switch } - browserActionResult = await this.browserSession.click(coordinate) - } else if (action === "type") { + } + if (action === "type") { if (!text) { this.consecutiveMistakeCount++ - pushToolResult(await this.sayAndCreateMissingParamError("browser_action", "text")) + pushToolResult( + await this.sayAndCreateMissingParamError("browser_action", "text"), + ) await this.browserSession.closeBrowser() break } - browserActionResult = await this.browserSession.type(text) - } else if (action === "scroll_down") { - browserActionResult = await this.browserSession.scrollDown() - } else if (action === "scroll_up") { - browserActionResult = await this.browserSession.scrollUp() - } else { - throw new Error(`Unsupported browser action: ${action}`) + } + this.consecutiveMistakeCount = 0 + await this.say( + "browser_action", + JSON.stringify({ + action: action as BrowserAction, + coordinate, + text, + } satisfies ClineSayBrowserAction), + undefined, + false, + ) + switch (action) { + case "click": + browserActionResult = await this.browserSession.click(coordinate!) + break + case "type": + browserActionResult = await this.browserSession.type(text!) + break + case "scroll_down": + browserActionResult = await this.browserSession.scrollDown() + break + case "scroll_up": + browserActionResult = await this.browserSession.scrollUp() + break + case "close": + browserActionResult = await this.browserSession.closeBrowser() + break } } - // Handle results for all actions except interactive mode close confirmation - if (action === "close" && !browserActionResult.logs?.includes("Would you like to:")) { - pushToolResult(formatResponse.toolResult("Browser has been closed.")) - } else if (action !== "close") { - await this.say("browser_action_result", JSON.stringify(browserActionResult)) - pushToolResult( - formatResponse.toolResult( - `The browser action has been executed. The console logs and screenshot have been captured for your analysis.\n\nConsole logs:\n${ - browserActionResult.logs || "(No new logs)" - }\n\n${ - this.isInteractiveMode ? - "The browser is in interactive mode. You can:\n1. Keep browsing\n2. Take another action\n3. Close the browser" : - "(REMEMBER: if you need to proceed to using non-`browser_action` tools or launch a new browser, you MUST first close this browser.)" - }`, - browserActionResult.screenshot ? [browserActionResult.screenshot] : [] + switch (action) { + case "launch": + case "click": + case "type": + case "scroll_down": + case "scroll_up": + await this.say("browser_action_result", JSON.stringify(browserActionResult)) + pushToolResult( + formatResponse.toolResult( + `The browser action has been executed. The console logs and screenshot have been captured for your analysis.\n\nConsole logs:\n${ + browserActionResult.logs || "(No new logs)" + }\n\n(REMEMBER: if you need to proceed to using non-\`browser_action\` tools or launch a new browser, you MUST first close this browser. For example, if after analyzing the logs and screenshot you need to edit a file, you must first close the browser before you can use the write_to_file tool.)`, + browserActionResult.screenshot ? [browserActionResult.screenshot] : [], + ), ) - ) + break + case "close": + pushToolResult( + formatResponse.toolResult( + `The browser has been closed. You may now proceed to using other tools.`, + ), + ) + break } break } @@ -1831,11 +1817,15 @@ export class Cline { async recursivelyMakeClineRequests( userContent: UserContent, includeFileDetails: boolean = false, + isInteractiveMode: boolean = false ): Promise { if (this.abort) { throw new Error("Cline instance aborted") } + // Store interactive mode state + this.isInteractiveMode = isInteractiveMode; + if (this.consecutiveMistakeCount >= 3) { const { response, text, images } = await this.ask( "mistake_limit_reached", @@ -2084,7 +2074,7 @@ export class Cline { this.consecutiveMistakeCount++ } - const recDidEndLoop = await this.recursivelyMakeClineRequests(this.userMessageContent) + const recDidEndLoop = await this.recursivelyMakeClineRequests(this.userMessageContent, false, this.isInteractiveMode) didEndLoop = recDidEndLoop } else { // if there's no assistant_responses, that means we got no text or tool_use content blocks from API which we should assume is an error diff --git a/src/core/webview/ClineProvider.ts b/src/core/webview/ClineProvider.ts index 4c446fbc33..d3d2595b8d 100644 --- a/src/core/webview/ClineProvider.ts +++ b/src/core/webview/ClineProvider.ts @@ -79,7 +79,7 @@ export class ClineProvider implements vscode.WebviewViewProvider { constructor( readonly context: vscode.ExtensionContext, - private readonly outputChannel: vscode.OutputChannel, + readonly outputChannel: vscode.OutputChannel, ) { this.outputChannel.appendLine("ClineProvider instantiated") ClineProvider.activeInstances.add(this) diff --git a/src/services/browser/BrowserSession.ts b/src/services/browser/BrowserSession.ts index 1ecab7bd58..9007848f38 100644 --- a/src/services/browser/BrowserSession.ts +++ b/src/services/browser/BrowserSession.ts @@ -1,7 +1,7 @@ import * as vscode from "vscode" import * as fs from "fs/promises" import * as path from "path" -import { Browser, Page, ScreenshotOptions, TimeoutError, launch } from "puppeteer" +import { Browser, Page, ScreenshotOptions, TimeoutError, launch, connect } from "puppeteer" import pWaitFor from "p-wait-for" import delay from "delay" import { fileExistsAtPath } from "../../utils/fs" @@ -13,10 +13,6 @@ export class BrowserSession { private page?: Page private currentMousePosition?: string private isInteractive: boolean = false - private lastInteractionTime: number = 0 - private readonly TIMEOUT_MINUTES = 15 - private closeRequested: boolean = false - private timeoutCheckInterval?: NodeJS.Timeout constructor(context: vscode.ExtensionContext) { this.context = context @@ -24,45 +20,45 @@ export class BrowserSession { async launchBrowser(interactive: boolean = false) { console.log("launch browser called") + this.isInteractive = interactive + if (this.browser) { await this.closeBrowser() // this may happen when the model launches a browser again after having used it already before } - this.isInteractive = interactive - - this.browser = await launch({ - args: [ - "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", - ], - defaultViewport: this.isInteractive ? { width: 1300, height: 1000 } : { - width: 900, - height: 600, - }, - headless: false, // Always use non-headless mode - }) - this.page = await this.browser?.newPage() - - if (this.isInteractive && this.page) { - // Maximize the window to use full screen dimensions - const session = await this.page.target().createCDPSession() - const { windowId } = await session.send('Browser.getWindowForTarget') - await session.send('Browser.setWindowBounds', { windowId, bounds: { windowState: 'normal' } }) - - // Disable Puppeteer's automation features in interactive mode - await this.page.evaluate(() => { - // @ts-ignore - delete window.navigator.webdriver + if (this.isInteractive) { + try { + // Fetch the WebSocket endpoint from Chrome's debugging API + const response = await fetch('http://127.0.0.1:7333/json/version') + const data = await response.json() + const browserWSEndpoint = data.webSocketDebuggerUrl + + if (!browserWSEndpoint) { + throw new Error('Could not get WebSocket endpoint from Chrome debugging API') + } + + this.browser = await connect({ + browserWSEndpoint, + }) + } catch (error) { + console.error("Failed to connect to browser:", error) + throw new Error(`Failed to connect to browser: ${error.message}`) + } + } else { + this.browser = await launch({ + args: [ + "--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36", + ], + defaultViewport: { + width: 900, + height: 600 + }, + headless: false, // Always use non-headless mode }) + } - // Reset flags - this.closeRequested = false - this.lastInteractionTime = Date.now() - - // Start timeout monitoring if not in interactive mode - if (!this.isInteractive) { - this.startTimeoutMonitoring() - } + this.page = await this.browser?.newPage() return { screenshot: "", @@ -74,55 +70,14 @@ export class BrowserSession { } } - private startTimeoutMonitoring() { - // Clear any existing interval - if (this.timeoutCheckInterval) { - clearInterval(this.timeoutCheckInterval) - } - - // Only start monitoring if not in interactive mode - if (!this.isInteractive) { - this.timeoutCheckInterval = setInterval(async () => { - const timeSinceLastInteraction = Date.now() - this.lastInteractionTime - if (timeSinceLastInteraction > this.TIMEOUT_MINUTES * 60 * 1000) { - if (this.timeoutCheckInterval) { - clearInterval(this.timeoutCheckInterval) - } - console.log("Browser timeout reached, closing...") - await this.closeBrowser() - } - }, 30000) // Check every 30 seconds - } - } - async closeBrowser(): Promise { - // If in interactive mode and close hasn't been requested yet, ask for confirmation - if (this.isInteractive && !this.closeRequested) { - console.log("Close requested but waiting for user confirmation...") - this.closeRequested = true - return { - logs: "Are you done using the browser? Please confirm to close it.", - screenshot: await this.getCurrentScreenshot(), - currentUrl: this.page?.url(), - currentMousePosition: this.currentMousePosition, - } - } - // Only close if explicitly requested or not in interactive mode - if (!this.isInteractive || (this.isInteractive && this.closeRequested)) { + if(this.isInteractive) { + console.log("disconnecting browser...") + await this.browser?.disconnect().catch(() => {}) + } else { console.log("closing browser...") await this.browser?.close().catch(() => {}) - this.browser = undefined - this.page = undefined - this.currentMousePosition = undefined - this.isInteractive = false - this.closeRequested = false - - // Clear timeout monitoring - if (this.timeoutCheckInterval) { - clearInterval(this.timeoutCheckInterval) - this.timeoutCheckInterval = undefined - } } return {} } @@ -154,54 +109,14 @@ export class BrowserSession { } } - async handleNextStep(choice: string): Promise { - const normalizedChoice = choice.toLowerCase().trim() - - if (normalizedChoice === "yes" || normalizedChoice === "done" || normalizedChoice === "confirm") { - await this.browser?.close().catch(() => {}) - this.browser = undefined - this.page = undefined - this.currentMousePosition = undefined - this.isInteractive = false - this.closeRequested = false - - if (this.timeoutCheckInterval) { - clearInterval(this.timeoutCheckInterval) - this.timeoutCheckInterval = undefined - } - return {} - } - - if (normalizedChoice === "no" || normalizedChoice === "continue") { - this.closeRequested = false - this.lastInteractionTime = Date.now() - return { - logs: "Browser session continued. Please confirm when you're done.", - screenshot: await this.getCurrentScreenshot(), - currentUrl: this.page?.url(), - currentMousePosition: this.currentMousePosition, - } - } - - // Default response for unclear input - return { - logs: "Please confirm if you're done using the browser (yes/no).", - screenshot: await this.getCurrentScreenshot(), - currentUrl: this.page?.url(), - currentMousePosition: this.currentMousePosition, - } - } - async doAction(action: (page: Page) => Promise): Promise { + if (!this.page) { throw new Error( - "Browser is not launched. This may occur if the browser was automatically closed by a non-`browser_action` tool.", + "Browser is not launched or connected. This may occur if the browser was automatically closed by a non-`browser_action` tool.", ) } - // Update last interaction time - this.lastInteractionTime = Date.now() - const logs: string[] = [] let lastLogTs = Date.now() @@ -224,31 +139,27 @@ export class BrowserSession { this.page.on("pageerror", errorListener) try { - if (!this.isInteractive) { - await action(this.page) - } else { - console.log("Browser actions cannot be performed in interactive mode. The user has manual control.") - } + await action(this.page) } catch (err) { if (!(err instanceof TimeoutError)) { logs.push(`[Error] ${err.toString()}`) } } - + // Wait for console inactivity, with a timeout await pWaitFor(() => Date.now() - lastLogTs >= 500, { timeout: 3_000, interval: 100, }).catch(() => {}) - + let screenshot = await this.getCurrentScreenshot() if (!screenshot) { throw new Error("Failed to take screenshot.") } - + this.page.off("console", consoleListener) this.page.off("pageerror", errorListener) - + return { screenshot, logs: logs.join("\n"),