scriptoriumadmin/Roo-Code
1
0
Fork 0
mirror of https://github.com/RooVetGit/Roo-Code.git synced 2026-08-28 05:27:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Updates to connect puppetter to browser with remote-debugging-port by prompting interactive mode text

Browse source
This commit is contained in:
a8trejo 2024-11-27 17:55:06 -08:00
parent 22ce1de608
commit fe1d823dc3
6 changed files with 178 additions and 272 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
docs/cline-updates.md
View file

@ -2,6 +2,11 @@ Remember to rebuild the extension and install it using the cli specified on the
Consider that any `export` or `install` command has already been run, so exclude those.ˆ
Before editing the file 'Roo-Cline/src/core/Cline.ts', since it's a really large file, read the original file to understand its full content.
Then create a new file with the -updates suffix containing ONLY the changes you've made to the original file.
First, read the original 'Cline.ts' file to understand its full content
Then create a new file with the -updates suffix containing ONLY the changes
In interactive mode, browser actions are disabled (if (!this.isInteractive) check in doAction method)
The navigateToUrl method uses doAction, which means navigation won't work in interactive mode
Can we analyze 'Roo-Cline/src/core/Cline.ts' to see how it handles browser actions and if there's some logic preventing it from using 'doAction'

32
package-lock.json generated
View file

@ -32,7 +32,7 @@
"os-name": "^6.0.0",
"p-wait-for": "^5.0.2",
"pdf-parse": "^1.1.1",
"puppeteer": "^23.4.0",
"puppeteer": "^23.9.0",
"serialize-error": "^11.0.3",
"strip-ansi": "^7.1.0",
"tree-sitter-wasms": "^0.1.11",
@ -4142,7 +4142,7 @@
},
"node_modules/@puppeteer/browsers": {
"version": "2.4.1",
"resolved": "https://registry.npmjs.org/@puppeteer/browsers/-/browsers-2.4.1.tgz",
"resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/@puppeteer/browsers/-/browsers-2.4.1.tgz",
"integrity": "sha512-0kdAbmic3J09I6dT8e9vE2JOCSt13wHCW5x/ly8TSt2bDtuIWe2TgLZZDHdcziw9AVCzflMAXCrVyRIhIs44Ng==",
"dependencies": {
"debug": "^4.3.7",
@ -6700,9 +6700,9 @@
}
},
"node_modules/bare-stream": {
"version": "2.3.2",
"resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/bare-stream/-/bare-stream-2.3.2.tgz",
"integrity": "sha512-EFZHSIBkDgSHIwj2l2QZfP4U5OcD4xFAOwhSb/vlr9PIqyGJGvB/nfClJbcnh3EY4jtPE4zsb5ztae96bVF79A==",
"version": "2.4.2",
"resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/bare-stream/-/bare-stream-2.4.2.tgz",
"integrity": "sha512-XZ4ln/KV4KT+PXdIWTKjsLY+quqCaEtqqtgGJVPw9AoM73By03ij64YjepK0aQvHSWDb6AfAZwqKaFu68qkrdA==",
"optional": true,
"dependencies": {
"streamx": "^2.20.0"
@ -7655,7 +7655,7 @@
},
"node_modules/devtools-protocol": {
"version": "0.0.1367902",
"resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.1367902.tgz",
"resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/devtools-protocol/-/devtools-protocol-0.0.1367902.tgz",
"integrity": "sha512-XxtPuC3PGakY6PD7dG66/o8KwJ/LkH2/EKe19Dcw58w53dv4/vSQEkn/SzuyhHE2q4zPgCkxQBxus3VV4ql+Pg=="
},
"node_modules/diff": {
@ -12534,16 +12534,16 @@
}
},
"node_modules/puppeteer": {
"version": "23.8.0",
"resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-23.8.0.tgz",
"integrity": "sha512-MFWDMWoCcOpwNwQIjA9gPKWrEUbj8bLCzkK56w5lZPMUT6wK4FfpgOEPxKffVmXEMYMZzgcjxzqy15b/Q1ibaw==",
"version": "23.9.0",
"resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/puppeteer/-/puppeteer-23.9.0.tgz",
"integrity": "sha512-WfB8jGwFV+qrD9dcJJVvWPFJBU6kxeu2wxJz9WooDGfM3vIiKLgzImEDBxUQnCBK/2cXB3d4dV6gs/LLpgfLDg==",
"hasInstallScript": true,
"dependencies": {
"@puppeteer/browsers": "2.4.1",
"chromium-bidi": "0.8.0",
"cosmiconfig": "^9.0.0",
"devtools-protocol": "0.0.1367902",
"puppeteer-core": "23.8.0",
"puppeteer-core": "23.9.0",
"typed-query-selector": "^2.12.0"
},
"bin": {
@ -12554,9 +12554,9 @@
}
},
"node_modules/puppeteer-core": {
"version": "23.8.0",
"resolved": "https://registry.npmjs.org/puppeteer-core/-/puppeteer-core-23.8.0.tgz",
"integrity": "sha512-c2ymGN2M//We7pC+JhP2dE/g4+qnT89BO+EMSZyJmecN3DN6RNqErA7eH7DrWoNIcU75r2nP4VHa4pswAL6NVg==",
"version": "23.9.0",
"resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/puppeteer-core/-/puppeteer-core-23.9.0.tgz",
"integrity": "sha512-hLVrav2HYMVdK0YILtfJwtnkBAwNOztUdR4aJ5YKDvgsbtagNr6urUJk9HyjRA9e+PaLI3jzJ0wM7A4jSZ7Qxw==",
"dependencies": {
"@puppeteer/browsers": "2.4.1",
"chromium-bidi": "0.8.0",
@ -13468,9 +13468,9 @@
}
},
"node_modules/streamx": {
"version": "2.20.1",
"resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/streamx/-/streamx-2.20.1.tgz",
"integrity": "sha512-uTa0mU6WUC65iUvzKH4X9hEdvSW7rbPxPtwfWiLMSj3qTdQbAiUboZTxauKfpFuGIGa1C2BYijZ7wgdUXICJhA==",
"version": "2.20.2",
"resolved": "https://roo-815250993495.d.codeartifact.us-east-1.amazonaws.com/npm/roo-dev/streamx/-/streamx-2.20.2.tgz",
"integrity": "sha512-aDGDLU+j9tJcUdPGOaHmVF1u/hhI+CsGkT02V3OKlHDV7IukOI+nTWAGkiZEKCO35rWN1wIr4tS7YFr1f4qSvA==",
"dependencies": {
"fast-fifo": "^1.3.2",
"queue-tick": "^1.0.1",

4
package.json
View file

@ -2,7 +2,7 @@
"name": "roo-cline",
"displayName": "Roo Cline",
"description": "Autonomous coding agent right in your IDE, capable of creating/editing files, running commands, using the browser, and more with your permission every step of the way.",
"version": "2.0.4",
"version": "2.1.0",
"icon": "assets/icons/icon_Roo.png",
"galleryBanner": {
"color": "#617A91",
@ -191,7 +191,7 @@
"os-name": "^6.0.0",
"p-wait-for": "^5.0.2",
"pdf-parse": "^1.1.1",
"puppeteer": "^23.4.0",
"puppeteer": "^23.9.0",
"serialize-error": "^11.0.3",
"strip-ansi": "^7.1.0",
"tree-sitter-wasms": "^0.1.11",

226
src/core/Cline.ts
View file

@ -668,22 +668,43 @@ export class Cline {
if (responseImages && responseImages.length > 0) {
newUserContent.push(...formatResponse.imageBlocks(responseImages))
}
const wasInteractiveBrowser = (lastRelevantMessageIndex > 0) ? modifiedClineMessages[lastRelevantMessageIndex - 1].text?.includes("interactive mode") : false
await this.overwriteApiConversationHistory(modifiedApiConversationHistory)
await this.initiateTaskLoop(newUserContent)
await this.initiateTaskLoop(newUserContent, wasInteractiveBrowser)
}
private async initiateTaskLoop(userContent: UserContent): Promise<void> {
let nextUserContent = userContent
let includeFileDetails = true
private async initiateTaskLoop(userContent: UserContent, wasInteractiveBrowser: boolean = false): Promise<void> {
// Check if any text block contains "interactive mode"
const hasInteractiveMode = userContent.some((block) => {
if (block.type === "text" && typeof block.text === "string") {
this.providerRef.deref()?.outputChannel.appendLine(`initiateTaskLoop :: block.text :: ${block.text.toLowerCase()}`)
return (block.type === "text" &&
typeof block.text === "string" &&
block.text.toLowerCase().includes("interactive mode"))
} else {
return false
}
}
) || wasInteractiveBrowser;
this.providerRef.deref()?.outputChannel.appendLine(`initiateTaskLoop :: hasInteractiveMode :: ${hasInteractiveMode}`)
// Set interactive mode flag if found in text
if (hasInteractiveMode) {
this.isInteractiveMode = true;
}
let nextUserContent = userContent;
let includeFileDetails = true;
while (!this.abort) {
const didEndLoop = await this.recursivelyMakeClineRequests(nextUserContent, includeFileDetails)
includeFileDetails = false // we only need file details the first time
// The way this agentic loop works is that cline will be given a task that he then calls tools to complete. unless there's an attempt_completion call, we keep responding back to him with his tool's responses until he either attempt_completion or does not use anymore tools. If he does not use anymore tools, we ask him to consider if he's completed the task and then call attempt_completion, otherwise proceed with completing the task.
// There is a MAX_REQUESTS_PER_TASK limit to prevent infinite requests, but Cline is prompted to finish the task as efficiently as he can.
//const totalCost = this.calculateApiCost(totalInputTokens, totalOutputTokens)
const didEndLoop = await this.recursivelyMakeClineRequests(
nextUserContent,
includeFileDetails,
this.isInteractiveMode // Pass the flag to recursivelyMakeClineRequests
)
includeFileDetails = false
if (didEndLoop) {
// For now a task never 'completes'. This will only happen if the user hits max requests and denies resetting the count.
//this.say("task_completed", `Task completed. Total API usage cost: ${totalCost}`)
@ -697,7 +718,7 @@ export class Cline {
{
type: "text",
text: formatResponse.noToolsUsed(),
},
}
]
this.consecutiveMistakeCount++
}
@ -1462,130 +1483,95 @@ export class Cline {
await this.browserSession.closeBrowser()
break
}
// Check if this is an interactive mode request
const isInteractiveRequest = url.toLowerCase().includes("interactive=true")
const actualUrl = isInteractiveRequest ? url.replace(/[?&]interactive=true/i, '') : url
if (isInteractiveRequest) {
this.isInteractiveMode = true
this.consecutiveMistakeCount = 0
const didApprove = await askApproval("browser_action_launch",
`Launch browser in interactive mode for URL: ${actualUrl}`)
if (!didApprove) {
break
}
await this.browserSession.launchBrowser(true)
browserActionResult = await this.browserSession.navigateToUrl(actualUrl)
await this.say("browser_action_result", JSON.stringify(browserActionResult))
pushToolResult(
formatResponse.toolResult(
`Browser launched in interactive mode. The browser will remain open for manual interaction until explicitly closed. You can:\n` +
`1. Keep browsing\n` +
`2. Take another action\n` +
`3. Close the browser\n\n` +
`Console logs:\n${browserActionResult.logs || "(No new logs)"}`,
browserActionResult.screenshot ? [browserActionResult.screenshot] : []
)
)
this.consecutiveMistakeCount = 0
const didApprove = await askApproval("browser_action_launch", url)
if (!didApprove) {
break
} else {
this.isInteractiveMode = false
this.consecutiveMistakeCount = 0
const didApprove = await askApproval("browser_action_launch", url)
if (!didApprove) {
break
}
}
// NOTE: it's okay that we call this message since the partial inspect_site is finished streaming. The only scenario we have to avoid is sending messages WHILE a partial message exists at the end of the messages array. For example the api_req_finished message would interfere with the partial message, so we needed to remove that.
// await this.say("inspect_site_result", "") // no result, starts the loading spinner waiting for result
await this.say("browser_action_result", "") // starts loading spinner
await this.browserSession.launchBrowser()
browserActionResult = await this.browserSession.navigateToUrl(actualUrl)
}
} else if (action === "close") {
// Handle close action for interactive mode
if (this.isInteractiveMode) {
browserActionResult = await this.browserSession.closeBrowser()
if (browserActionResult.logs?.includes("Would you like to:")) {
// Browser is asking for confirmation
await this.say("browser_action_result", JSON.stringify(browserActionResult))
pushToolResult(
formatResponse.toolResult(
`The browser is in interactive mode. Please confirm if you want to:\n` +
`1. Keep browsing\n` +
`2. Take another action\n` +
`3. Close the browser`,
browserActionResult.screenshot ? [browserActionResult.screenshot] : []
)
)
break
}
this.isInteractiveMode = false
} else {
browserActionResult = await this.browserSession.closeBrowser()
}
await this.say("browser_action_result", "") // starts loading spinner
await this.browserSession.launchBrowser(this.isInteractiveMode)
browserActionResult = await this.browserSession.navigateToUrl(url)
} else {
// Handle other actions (click, type, scroll)
if (this.isInteractiveMode) {
pushToolResult(
formatResponse.toolResult(
`Cannot perform browser actions while in interactive mode. The user has manual control. You can:\n` +
`1. Keep browsing\n` +
`2. Take another action\n` +
`3. Close the browser`
)
)
break
}
if (action === "click") {
if (!coordinate) {
this.consecutiveMistakeCount++
pushToolResult(await this.sayAndCreateMissingParamError("browser_action", "coordinate"))
pushToolResult(
await this.sayAndCreateMissingParamError(
"browser_action",
"coordinate",
),
)
await this.browserSession.closeBrowser()
break
break // can't be within an inner switch
}
browserActionResult = await this.browserSession.click(coordinate)
} else if (action === "type") {
}
if (action === "type") {
if (!text) {
this.consecutiveMistakeCount++
pushToolResult(await this.sayAndCreateMissingParamError("browser_action", "text"))
pushToolResult(
await this.sayAndCreateMissingParamError("browser_action", "text"),
)
await this.browserSession.closeBrowser()
break
}
browserActionResult = await this.browserSession.type(text)
} else if (action === "scroll_down") {
browserActionResult = await this.browserSession.scrollDown()
} else if (action === "scroll_up") {
browserActionResult = await this.browserSession.scrollUp()
} else {
throw new Error(`Unsupported browser action: ${action}`)
}
this.consecutiveMistakeCount = 0
await this.say(
"browser_action",
JSON.stringify({
action: action as BrowserAction,
coordinate,
text,
} satisfies ClineSayBrowserAction),
undefined,
false,
)
switch (action) {
case "click":
browserActionResult = await this.browserSession.click(coordinate!)
break
case "type":
browserActionResult = await this.browserSession.type(text!)
break
case "scroll_down":
browserActionResult = await this.browserSession.scrollDown()
break
case "scroll_up":
browserActionResult = await this.browserSession.scrollUp()
break
case "close":
browserActionResult = await this.browserSession.closeBrowser()
break
}
}
// Handle results for all actions except interactive mode close confirmation
if (action === "close" && !browserActionResult.logs?.includes("Would you like to:")) {
pushToolResult(formatResponse.toolResult("Browser has been closed."))
} else if (action !== "close") {
await this.say("browser_action_result", JSON.stringify(browserActionResult))
pushToolResult(
formatResponse.toolResult(
`The browser action has been executed. The console logs and screenshot have been captured for your analysis.\n\nConsole logs:\n${
browserActionResult.logs || "(No new logs)"
}\n\n${
this.isInteractiveMode ?
"The browser is in interactive mode. You can:\n1. Keep browsing\n2. Take another action\n3. Close the browser" :
"(REMEMBER: if you need to proceed to using non-`browser_action` tools or launch a new browser, you MUST first close this browser.)"
}`,
browserActionResult.screenshot ? [browserActionResult.screenshot] : []
switch (action) {
case "launch":
case "click":
case "type":
case "scroll_down":
case "scroll_up":
await this.say("browser_action_result", JSON.stringify(browserActionResult))
pushToolResult(
formatResponse.toolResult(
`The browser action has been executed. The console logs and screenshot have been captured for your analysis.\n\nConsole logs:\n${
browserActionResult.logs || "(No new logs)"
}\n\n(REMEMBER: if you need to proceed to using non-\`browser_action\` tools or launch a new browser, you MUST first close this browser. For example, if after analyzing the logs and screenshot you need to edit a file, you must first close the browser before you can use the write_to_file tool.)`,
browserActionResult.screenshot ? [browserActionResult.screenshot] : [],
),
)
)
break
case "close":
pushToolResult(
formatResponse.toolResult(
`The browser has been closed. You may now proceed to using other tools.`,
),
)
break
}
break
}
@ -1831,11 +1817,15 @@ export class Cline {
async recursivelyMakeClineRequests(
userContent: UserContent,
includeFileDetails: boolean = false,
isInteractiveMode: boolean = false
): Promise<boolean> {
if (this.abort) {
throw new Error("Cline instance aborted")
}
// Store interactive mode state
this.isInteractiveMode = isInteractiveMode;
if (this.consecutiveMistakeCount >= 3) {
const { response, text, images } = await this.ask(
"mistake_limit_reached",
@ -2084,7 +2074,7 @@ export class Cline {
this.consecutiveMistakeCount++
}
const recDidEndLoop = await this.recursivelyMakeClineRequests(this.userMessageContent)
const recDidEndLoop = await this.recursivelyMakeClineRequests(this.userMessageContent, false, this.isInteractiveMode)
didEndLoop = recDidEndLoop
} else {
// if there's no assistant_responses, that means we got no text or tool_use content blocks from API which we should assume is an error

2
src/core/webview/ClineProvider.ts
View file

@ -79,7 +79,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
constructor(
readonly context: vscode.ExtensionContext,
private readonly outputChannel: vscode.OutputChannel,
readonly outputChannel: vscode.OutputChannel,
) {
this.outputChannel.appendLine("ClineProvider instantiated")
ClineProvider.activeInstances.add(this)

177
src/services/browser/BrowserSession.ts
View file

@ -1,7 +1,7 @@
import * as vscode from "vscode"
import * as fs from "fs/promises"
import * as path from "path"
import { Browser, Page, ScreenshotOptions, TimeoutError, launch } from "puppeteer"
import { Browser, Page, ScreenshotOptions, TimeoutError, launch, connect } from "puppeteer"
import pWaitFor from "p-wait-for"
import delay from "delay"
import { fileExistsAtPath } from "../../utils/fs"
@ -13,10 +13,6 @@ export class BrowserSession {
private page?: Page
private currentMousePosition?: string
private isInteractive: boolean = false
private lastInteractionTime: number = 0
private readonly TIMEOUT_MINUTES = 15
private closeRequested: boolean = false
private timeoutCheckInterval?: NodeJS.Timeout
constructor(context: vscode.ExtensionContext) {
this.context = context
@ -24,45 +20,45 @@ export class BrowserSession {
async launchBrowser(interactive: boolean = false) {
console.log("launch browser called")
this.isInteractive = interactive
if (this.browser) {
await this.closeBrowser() // this may happen when the model launches a browser again after having used it already before
}
this.isInteractive = interactive
this.browser = await launch({
args: [
"--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
],
defaultViewport: this.isInteractive ? { width: 1300, height: 1000 } : {
width: 900,
height: 600,
},
headless: false, // Always use non-headless mode
})
this.page = await this.browser?.newPage()
if (this.isInteractive && this.page) {
// Maximize the window to use full screen dimensions
const session = await this.page.target().createCDPSession()
const { windowId } = await session.send('Browser.getWindowForTarget')
await session.send('Browser.setWindowBounds', { windowId, bounds: { windowState: 'normal' } })
// Disable Puppeteer's automation features in interactive mode
await this.page.evaluate(() => {
// @ts-ignore
delete window.navigator.webdriver
if (this.isInteractive) {
try {
// Fetch the WebSocket endpoint from Chrome's debugging API
const response = await fetch('http://127.0.0.1:7333/json/version')
const data = await response.json()
const browserWSEndpoint = data.webSocketDebuggerUrl
if (!browserWSEndpoint) {
throw new Error('Could not get WebSocket endpoint from Chrome debugging API')
}
this.browser = await connect({
browserWSEndpoint,
})
} catch (error) {
console.error("Failed to connect to browser:", error)
throw new Error(`Failed to connect to browser: ${error.message}`)
}
} else {
this.browser = await launch({
args: [
"--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
],
defaultViewport: {
width: 900,
height: 600
},
headless: false, // Always use non-headless mode
})
}
// Reset flags
this.closeRequested = false
this.lastInteractionTime = Date.now()
// Start timeout monitoring if not in interactive mode
if (!this.isInteractive) {
this.startTimeoutMonitoring()
}
this.page = await this.browser?.newPage()
return {
screenshot: "",
@ -74,55 +70,14 @@ export class BrowserSession {
}
}
private startTimeoutMonitoring() {
// Clear any existing interval
if (this.timeoutCheckInterval) {
clearInterval(this.timeoutCheckInterval)
}
// Only start monitoring if not in interactive mode
if (!this.isInteractive) {
this.timeoutCheckInterval = setInterval(async () => {
const timeSinceLastInteraction = Date.now() - this.lastInteractionTime
if (timeSinceLastInteraction > this.TIMEOUT_MINUTES * 60 * 1000) {
if (this.timeoutCheckInterval) {
clearInterval(this.timeoutCheckInterval)
}
console.log("Browser timeout reached, closing...")
await this.closeBrowser()
}
}, 30000) // Check every 30 seconds
}
}
async closeBrowser(): Promise<BrowserActionResult> {
// If in interactive mode and close hasn't been requested yet, ask for confirmation
if (this.isInteractive && !this.closeRequested) {
console.log("Close requested but waiting for user confirmation...")
this.closeRequested = true
return {
logs: "Are you done using the browser? Please confirm to close it.",
screenshot: await this.getCurrentScreenshot(),
currentUrl: this.page?.url(),
currentMousePosition: this.currentMousePosition,
}
}
// Only close if explicitly requested or not in interactive mode
if (!this.isInteractive || (this.isInteractive && this.closeRequested)) {
if(this.isInteractive) {
console.log("disconnecting browser...")
await this.browser?.disconnect().catch(() => {})
} else {
console.log("closing browser...")
await this.browser?.close().catch(() => {})
this.browser = undefined
this.page = undefined
this.currentMousePosition = undefined
this.isInteractive = false
this.closeRequested = false
// Clear timeout monitoring
if (this.timeoutCheckInterval) {
clearInterval(this.timeoutCheckInterval)
this.timeoutCheckInterval = undefined
}
}
return {}
}
@ -154,54 +109,14 @@ export class BrowserSession {
}
}
async handleNextStep(choice: string): Promise<BrowserActionResult> {
const normalizedChoice = choice.toLowerCase().trim()
if (normalizedChoice === "yes" || normalizedChoice === "done" || normalizedChoice === "confirm") {
await this.browser?.close().catch(() => {})
this.browser = undefined
this.page = undefined
this.currentMousePosition = undefined
this.isInteractive = false
this.closeRequested = false
if (this.timeoutCheckInterval) {
clearInterval(this.timeoutCheckInterval)
this.timeoutCheckInterval = undefined
}
return {}
}
if (normalizedChoice === "no" || normalizedChoice === "continue") {
this.closeRequested = false
this.lastInteractionTime = Date.now()
return {
logs: "Browser session continued. Please confirm when you're done.",
screenshot: await this.getCurrentScreenshot(),
currentUrl: this.page?.url(),
currentMousePosition: this.currentMousePosition,
}
}
// Default response for unclear input
return {
logs: "Please confirm if you're done using the browser (yes/no).",
screenshot: await this.getCurrentScreenshot(),
currentUrl: this.page?.url(),
currentMousePosition: this.currentMousePosition,
}
}
async doAction(action: (page: Page) => Promise<void>): Promise<BrowserActionResult> {
if (!this.page) {
throw new Error(
"Browser is not launched. This may occur if the browser was automatically closed by a non-`browser_action` tool.",
"Browser is not launched or connected. This may occur if the browser was automatically closed by a non-`browser_action` tool.",
)
}
// Update last interaction time
this.lastInteractionTime = Date.now()
const logs: string[] = []
let lastLogTs = Date.now()
@ -224,31 +139,27 @@ export class BrowserSession {
this.page.on("pageerror", errorListener)
try {
if (!this.isInteractive) {
await action(this.page)
} else {
console.log("Browser actions cannot be performed in interactive mode. The user has manual control.")
}
await action(this.page)
} catch (err) {
if (!(err instanceof TimeoutError)) {
logs.push(`[Error] ${err.toString()}`)
}
}
// Wait for console inactivity, with a timeout
await pWaitFor(() => Date.now() - lastLogTs >= 500, {
timeout: 3_000,
interval: 100,
}).catch(() => {})
let screenshot = await this.getCurrentScreenshot()
if (!screenshot) {
throw new Error("Failed to take screenshot.")
}
this.page.off("console", consoleListener)
this.page.off("pageerror", errorListener)
return {
screenshot,
logs: logs.join("\n"),