mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-06 08:18:39 +00:00
feat: interactive browser session and supportsImages gating for browser_action
This commit is contained in:
parent
b975ced81b
commit
fe00413533
5 changed files with 226 additions and 504 deletions
|
|
@ -2207,7 +2207,7 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
|
|||
return SYSTEM_PROMPT(
|
||||
provider.context,
|
||||
this.cwd,
|
||||
(this.api.getModel().info.supportsComputerUse ?? false) && (browserToolEnabled ?? true),
|
||||
(this.api.getModel().info.supportsImages ?? false) && (browserToolEnabled ?? true),
|
||||
mcpHub,
|
||||
this.diffStrategy,
|
||||
browserViewportSize,
|
||||
|
|
|
|||
|
|
@ -45,15 +45,15 @@ export const generateSystemPrompt = async (provider: ClineProvider, message: Web
|
|||
const rooIgnoreInstructions = provider.getCurrentTask()?.rooIgnoreController?.getInstructions()
|
||||
|
||||
// Determine if browser tools can be used based on model support, mode, and user settings
|
||||
let modelSupportsComputerUse = false
|
||||
let modelSupportsImages = false
|
||||
|
||||
// Create a temporary API handler to check if the model supports computer use
|
||||
// Create a temporary API handler to check if the model supports images (needed for screenshot-driven browsing)
|
||||
// This avoids relying on an active Cline instance which might not exist during preview
|
||||
try {
|
||||
const tempApiHandler = buildApiHandler(apiConfiguration)
|
||||
modelSupportsComputerUse = tempApiHandler.getModel().info.supportsComputerUse ?? false
|
||||
modelSupportsImages = tempApiHandler.getModel().info.supportsImages ?? false
|
||||
} catch (error) {
|
||||
console.error("Error checking if model supports computer use:", error)
|
||||
console.error("Error checking if model supports images:", error)
|
||||
}
|
||||
|
||||
// Check if the current mode includes the browser tool group
|
||||
|
|
@ -62,7 +62,7 @@ export const generateSystemPrompt = async (provider: ClineProvider, message: Web
|
|||
|
||||
// Only enable browser tools if the model supports it, the mode includes browser tools,
|
||||
// and browser tools are enabled in settings
|
||||
const canUseBrowserTool = modelSupportsComputerUse && modeSupportsBrowser && (browserToolEnabled ?? true)
|
||||
const canUseBrowserTool = modelSupportsImages && modeSupportsBrowser && (browserToolEnabled ?? true)
|
||||
|
||||
const systemPrompt = await SYSTEM_PROMPT(
|
||||
provider.context,
|
||||
|
|
|
|||
|
|
@ -1,560 +1,280 @@
|
|||
import * as vscode from "vscode"
|
||||
import * as fs from "fs/promises"
|
||||
import * as path from "path"
|
||||
import { Browser, Page, ScreenshotOptions, TimeoutError, launch, connect } from "puppeteer-core"
|
||||
import { Browser, Page, launch } from "puppeteer-core"
|
||||
// @ts-ignore
|
||||
import PCR from "puppeteer-chromium-resolver"
|
||||
import pWaitFor from "p-wait-for"
|
||||
import delay from "delay"
|
||||
import { serializeError } from "serialize-error"
|
||||
|
||||
import { fileExistsAtPath } from "../../utils/fs"
|
||||
import { BrowserActionResult } from "../../shared/ExtensionMessage"
|
||||
import { discoverChromeHostUrl, tryChromeHostUrl } from "./browserDiscovery"
|
||||
|
||||
// Timeout constants
|
||||
const BROWSER_NAVIGATION_TIMEOUT = 15_000 // 15 seconds
|
||||
|
||||
interface PCRStats {
|
||||
puppeteer: { launch: typeof launch }
|
||||
executablePath: string
|
||||
}
|
||||
import type { BrowserActionResult } from "../../shared/ExtensionMessage"
|
||||
|
||||
/**
|
||||
* Interactive browser session for the browser_action tool.
|
||||
* - Local Chromium via puppeteer-chromium-resolver (atomic download to global storage).
|
||||
* - Robust navigation (networkidle2 with timeout fallback to domcontentloaded).
|
||||
* - Captures console logs and returns them with every action.
|
||||
* - Returns a screenshot (PNG, base64 data URL) on every action except "close".
|
||||
* - Tracks the current mouse position for debugging/telemetry.
|
||||
*
|
||||
* Note: Viewport defaults to 900x600 to match the prompt description. The model may change it using the "resize" action.
|
||||
*/
|
||||
export class BrowserSession {
|
||||
private context: vscode.ExtensionContext
|
||||
private browser?: Browser
|
||||
private page?: Page
|
||||
private currentMousePosition?: string
|
||||
private lastConnectionAttempt?: number
|
||||
private isUsingRemoteBrowser: boolean = false
|
||||
|
||||
// Logs captured from console events for the current action window
|
||||
private logsBuffer: string[] = []
|
||||
private consoleAttached = false
|
||||
|
||||
// Track last known mouse coordinates for debugging
|
||||
private mouseX: number | null = null
|
||||
private mouseY: number | null = null
|
||||
|
||||
// Default viewport; will be applied on launch and can be changed via resize()
|
||||
private viewport = { width: 900, height: 600 }
|
||||
|
||||
// Timeout constants (aligned with UrlContentFetcher semantics)
|
||||
private static readonly URL_FETCH_TIMEOUT = 30_000
|
||||
private static readonly URL_FETCH_FALLBACK_TIMEOUT = 20_000
|
||||
|
||||
constructor(context: vscode.ExtensionContext) {
|
||||
this.context = context
|
||||
}
|
||||
|
||||
private async ensureChromiumExists(): Promise<PCRStats> {
|
||||
private async ensureChromiumExists(): Promise<{ puppeteer: { launch: typeof launch }; executablePath: string }> {
|
||||
const globalStoragePath = this.context?.globalStorageUri?.fsPath
|
||||
if (!globalStoragePath) {
|
||||
throw new Error("Global storage uri is invalid")
|
||||
}
|
||||
|
||||
const puppeteerDir = path.join(globalStoragePath, "puppeteer")
|
||||
const dirExists = await fileExistsAtPath(puppeteerDir)
|
||||
if (!dirExists) {
|
||||
await fs.mkdir(puppeteerDir, { recursive: true })
|
||||
}
|
||||
|
||||
// if chromium doesn't exist, this will download it to path.join(puppeteerDir, ".chromium-browser-snapshots")
|
||||
// if it does exist it will return the path to existing chromium
|
||||
const stats: PCRStats = await PCR({
|
||||
const stats = await PCR({
|
||||
downloadPath: puppeteerDir,
|
||||
})
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the viewport size from global state or returns default
|
||||
*/
|
||||
private getViewport() {
|
||||
const size = (this.context.globalState.get("browserViewportSize") as string | undefined) || "900x600"
|
||||
const [width, height] = size.split("x").map(Number)
|
||||
return { width, height }
|
||||
}
|
||||
|
||||
/**
|
||||
* Launches a local browser instance
|
||||
*/
|
||||
private async launchLocalBrowser(): Promise<void> {
|
||||
console.log("Launching local browser")
|
||||
const stats = await this.ensureChromiumExists()
|
||||
this.browser = await stats.puppeteer.launch({
|
||||
args: [
|
||||
"--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
|
||||
],
|
||||
executablePath: stats.executablePath,
|
||||
defaultViewport: this.getViewport(),
|
||||
// headless: false,
|
||||
})
|
||||
this.isUsingRemoteBrowser = false
|
||||
}
|
||||
|
||||
/**
|
||||
* Connects to a browser using a WebSocket URL
|
||||
*/
|
||||
private async connectWithChromeHostUrl(chromeHostUrl: string): Promise<boolean> {
|
||||
try {
|
||||
this.browser = await connect({
|
||||
browserURL: chromeHostUrl,
|
||||
defaultViewport: this.getViewport(),
|
||||
})
|
||||
|
||||
// Cache the successful endpoint
|
||||
console.log(`Connected to remote browser at ${chromeHostUrl}`)
|
||||
this.context.globalState.update("cachedChromeHostUrl", chromeHostUrl)
|
||||
this.lastConnectionAttempt = Date.now()
|
||||
this.isUsingRemoteBrowser = true
|
||||
|
||||
return true
|
||||
} catch (error) {
|
||||
console.log(`Failed to connect using WebSocket endpoint: ${error}`)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Attempts to connect to a remote browser using various methods
|
||||
* Returns true if connection was successful, false otherwise
|
||||
*/
|
||||
private async connectToRemoteBrowser(): Promise<boolean> {
|
||||
let remoteBrowserHost = this.context.globalState.get("remoteBrowserHost") as string | undefined
|
||||
let reconnectionAttempted = false
|
||||
|
||||
// Try to connect with cached endpoint first if it exists and is recent (less than 1 hour old)
|
||||
const cachedChromeHostUrl = this.context.globalState.get("cachedChromeHostUrl") as string | undefined
|
||||
if (cachedChromeHostUrl && this.lastConnectionAttempt && Date.now() - this.lastConnectionAttempt < 3_600_000) {
|
||||
console.log(`Attempting to connect using cached Chrome Host Url: ${cachedChromeHostUrl}`)
|
||||
if (await this.connectWithChromeHostUrl(cachedChromeHostUrl)) {
|
||||
return true
|
||||
}
|
||||
|
||||
console.log(`Failed to connect using cached Chrome Host Url: ${cachedChromeHostUrl}`)
|
||||
// Clear the cached endpoint since it's no longer valid
|
||||
this.context.globalState.update("cachedChromeHostUrl", undefined)
|
||||
|
||||
// User wants to give up after one reconnection attempt
|
||||
if (remoteBrowserHost) {
|
||||
reconnectionAttempted = true
|
||||
}
|
||||
}
|
||||
|
||||
// If user provided a remote browser host, try to connect to it
|
||||
else if (remoteBrowserHost && !reconnectionAttempted) {
|
||||
console.log(`Attempting to connect to remote browser at ${remoteBrowserHost}`)
|
||||
private attachConsoleListener(): void {
|
||||
if (this.consoleAttached || !this.page) return
|
||||
this.page.on("console", (msg) => {
|
||||
try {
|
||||
const hostIsValid = await tryChromeHostUrl(remoteBrowserHost)
|
||||
|
||||
if (!hostIsValid) {
|
||||
throw new Error("Could not find chromeHostUrl in the response")
|
||||
// Append newest at end; keep a reasonable limit to avoid unbounded growth
|
||||
const text = msg.text?.() ?? String(msg)
|
||||
this.logsBuffer.push(text)
|
||||
if (this.logsBuffer.length > 200) {
|
||||
this.logsBuffer.splice(0, this.logsBuffer.length - 200)
|
||||
}
|
||||
|
||||
console.log(`Found WebSocket endpoint: ${remoteBrowserHost}`)
|
||||
|
||||
if (await this.connectWithChromeHostUrl(remoteBrowserHost)) {
|
||||
return true
|
||||
}
|
||||
} catch (error) {
|
||||
console.error(`Failed to connect to remote browser: ${error}`)
|
||||
// Fall back to auto-discovery if remote connection fails
|
||||
} catch {
|
||||
// Ignore console parsing errors
|
||||
}
|
||||
})
|
||||
this.consoleAttached = true
|
||||
}
|
||||
|
||||
private resetLogs(): void {
|
||||
this.logsBuffer = []
|
||||
}
|
||||
|
||||
private flushLogs(): string {
|
||||
const joined = this.logsBuffer.join("\n")
|
||||
this.resetLogs()
|
||||
return joined
|
||||
}
|
||||
|
||||
private ensurePage(): Page {
|
||||
if (!this.page) {
|
||||
throw new Error("Browser not initialized")
|
||||
}
|
||||
return this.page
|
||||
}
|
||||
|
||||
private async captureResult(includeScreenshot: boolean = true): Promise<BrowserActionResult> {
|
||||
const page = this.ensurePage()
|
||||
// Small stabilization delay for SPA updates after actions
|
||||
await this.delay(150)
|
||||
|
||||
let screenshot: string | undefined
|
||||
if (includeScreenshot) {
|
||||
const b64 = (await page.screenshot({ type: "png", encoding: "base64", fullPage: false })) as string
|
||||
screenshot = `data:image/png;base64,${b64}`
|
||||
}
|
||||
|
||||
const logs = this.flushLogs()
|
||||
const currentUrl = page.url()
|
||||
const currentMousePosition =
|
||||
this.mouseX != null && this.mouseY != null ? `${this.mouseX},${this.mouseY}` : undefined
|
||||
|
||||
return { screenshot, logs, currentUrl, currentMousePosition }
|
||||
}
|
||||
|
||||
private async navigateWithFallback(url: string): Promise<void> {
|
||||
const page = this.ensurePage()
|
||||
try {
|
||||
console.log("Attempting browser auto-discovery...")
|
||||
const chromeHostUrl = await discoverChromeHostUrl()
|
||||
|
||||
if (chromeHostUrl && (await this.connectWithChromeHostUrl(chromeHostUrl))) {
|
||||
return true
|
||||
}
|
||||
await page.goto(url, {
|
||||
timeout: BrowserSession.URL_FETCH_TIMEOUT,
|
||||
waitUntil: ["domcontentloaded", "networkidle2"],
|
||||
} as any)
|
||||
} catch (error) {
|
||||
console.error(`Auto-discovery failed: ${error}`)
|
||||
// Fall back to local browser if auto-discovery fails
|
||||
}
|
||||
const serialized = serializeError(error)
|
||||
const message = serialized.message || String(error)
|
||||
const name = serialized.name
|
||||
|
||||
return false
|
||||
const shouldRetry =
|
||||
message.includes("timeout") ||
|
||||
message.includes("net::") ||
|
||||
message.includes("NetworkError") ||
|
||||
message.includes("ERR_") ||
|
||||
name === "TimeoutError"
|
||||
|
||||
if (shouldRetry) {
|
||||
await page.goto(url, {
|
||||
timeout: BrowserSession.URL_FETCH_FALLBACK_TIMEOUT,
|
||||
waitUntil: ["domcontentloaded"],
|
||||
} as any)
|
||||
} else {
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async launchBrowser(): Promise<void> {
|
||||
console.log("launch browser called")
|
||||
|
||||
// Check if remote browser connection is enabled
|
||||
const remoteBrowserEnabled = this.context.globalState.get("remoteBrowserEnabled") as boolean | undefined
|
||||
|
||||
if (!remoteBrowserEnabled) {
|
||||
console.log("Launching local browser")
|
||||
if (this.browser) {
|
||||
// throw new Error("Browser already launched")
|
||||
await this.closeBrowser() // this may happen when the model launches a browser again after having used it already before
|
||||
} else {
|
||||
// If browser wasn't open, just reset the state
|
||||
this.resetBrowserState()
|
||||
}
|
||||
await this.launchLocalBrowser()
|
||||
} else {
|
||||
console.log("Connecting to remote browser")
|
||||
// Remote browser connection is enabled
|
||||
const remoteConnected = await this.connectToRemoteBrowser()
|
||||
|
||||
// If all remote connection attempts fail, fall back to local browser
|
||||
if (!remoteConnected) {
|
||||
console.log("Falling back to local browser")
|
||||
await this.launchLocalBrowser()
|
||||
}
|
||||
if (this.browser) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Closes the browser and resets browser state
|
||||
*/
|
||||
async closeBrowser(): Promise<BrowserActionResult> {
|
||||
if (this.browser || this.page) {
|
||||
console.log("closing browser...")
|
||||
|
||||
if (this.isUsingRemoteBrowser && this.browser) {
|
||||
await this.browser.disconnect().catch(() => {})
|
||||
} else {
|
||||
await this.browser?.close().catch(() => {})
|
||||
}
|
||||
this.resetBrowserState()
|
||||
}
|
||||
return {}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets all browser state variables
|
||||
*/
|
||||
private resetBrowserState(): void {
|
||||
this.browser = undefined
|
||||
this.page = undefined
|
||||
this.currentMousePosition = undefined
|
||||
this.isUsingRemoteBrowser = false
|
||||
}
|
||||
|
||||
async doAction(action: (page: Page) => Promise<void>): Promise<BrowserActionResult> {
|
||||
if (!this.page) {
|
||||
throw new Error(
|
||||
"Browser is not launched. This may occur if the browser was automatically closed by a non-`browser_action` tool.",
|
||||
)
|
||||
const stats = await this.ensureChromiumExists()
|
||||
const args: string[] = [
|
||||
"--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-accelerated-2d-canvas",
|
||||
"--no-first-run",
|
||||
"--disable-gpu",
|
||||
"--disable-features=VizDisplayCompositor",
|
||||
]
|
||||
if (process.platform === "linux") {
|
||||
args.push("--no-sandbox")
|
||||
}
|
||||
|
||||
const logs: string[] = []
|
||||
let lastLogTs = Date.now()
|
||||
|
||||
const consoleListener = (msg: any) => {
|
||||
if (msg.type() === "log") {
|
||||
logs.push(msg.text())
|
||||
} else {
|
||||
logs.push(`[${msg.type()}] ${msg.text()}`)
|
||||
}
|
||||
lastLogTs = Date.now()
|
||||
}
|
||||
|
||||
const errorListener = (err: Error) => {
|
||||
logs.push(`[Page Error] ${err.toString()}`)
|
||||
lastLogTs = Date.now()
|
||||
}
|
||||
|
||||
// Add the listeners
|
||||
this.page.on("console", consoleListener)
|
||||
this.page.on("pageerror", errorListener)
|
||||
|
||||
try {
|
||||
await action(this.page)
|
||||
} catch (err) {
|
||||
if (!(err instanceof TimeoutError)) {
|
||||
logs.push(`[Error] ${err.toString()}`)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for console inactivity, with a timeout
|
||||
await pWaitFor(() => Date.now() - lastLogTs >= 500, {
|
||||
timeout: 3_000,
|
||||
interval: 100,
|
||||
}).catch(() => {})
|
||||
|
||||
let options: ScreenshotOptions = {
|
||||
encoding: "base64",
|
||||
|
||||
// clip: {
|
||||
// x: 0,
|
||||
// y: 0,
|
||||
// width: 900,
|
||||
// height: 600,
|
||||
// },
|
||||
}
|
||||
|
||||
let screenshotBase64 = await this.page.screenshot({
|
||||
...options,
|
||||
type: "webp",
|
||||
quality: ((await this.context.globalState.get("screenshotQuality")) as number | undefined) ?? 75,
|
||||
this.browser = await stats.puppeteer.launch({
|
||||
args,
|
||||
executablePath: stats.executablePath,
|
||||
})
|
||||
let screenshot = `data:image/webp;base64,${screenshotBase64}`
|
||||
this.page = await this.browser.newPage()
|
||||
|
||||
if (!screenshotBase64) {
|
||||
console.log("webp screenshot failed, trying png")
|
||||
screenshotBase64 = await this.page.screenshot({
|
||||
...options,
|
||||
type: "png",
|
||||
})
|
||||
screenshot = `data:image/png;base64,${screenshotBase64}`
|
||||
}
|
||||
// Page defaults
|
||||
await this.page.setViewport({ width: this.viewport.width, height: this.viewport.height })
|
||||
await this.page.setExtraHTTPHeaders({ "Accept-Language": "en-US,en;q=0.9" })
|
||||
|
||||
if (!screenshotBase64) {
|
||||
throw new Error("Failed to take screenshot.")
|
||||
}
|
||||
|
||||
// this.page.removeAllListeners() <- causes the page to crash!
|
||||
this.page.off("console", consoleListener)
|
||||
this.page.off("pageerror", errorListener)
|
||||
|
||||
return {
|
||||
screenshot,
|
||||
logs: logs.join("\n"),
|
||||
currentUrl: this.page.url(),
|
||||
currentMousePosition: this.currentMousePosition,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the root domain from a URL
|
||||
* e.g., http://localhost:3000/path -> localhost:3000
|
||||
* e.g., https://example.com/path -> example.com
|
||||
*/
|
||||
private getRootDomain(url: string): string {
|
||||
try {
|
||||
const urlObj = new URL(url)
|
||||
// Remove www. prefix if present
|
||||
return urlObj.host.replace(/^www\./, "")
|
||||
} catch (error) {
|
||||
// If URL parsing fails, return the original URL
|
||||
return url
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate to a URL with standard loading options
|
||||
*/
|
||||
private async navigatePageToUrl(page: Page, url: string): Promise<void> {
|
||||
await page.goto(url, { timeout: BROWSER_NAVIGATION_TIMEOUT, waitUntil: ["domcontentloaded", "networkidle2"] })
|
||||
await this.waitTillHTMLStable(page)
|
||||
}
|
||||
|
||||
/**
|
||||
* Creates a new tab and navigates to the specified URL
|
||||
*/
|
||||
private async createNewTab(url: string): Promise<BrowserActionResult> {
|
||||
if (!this.browser) {
|
||||
throw new Error("Browser is not launched")
|
||||
}
|
||||
|
||||
// Create a new page
|
||||
const newPage = await this.browser.newPage()
|
||||
|
||||
// Set the new page as the active page
|
||||
this.page = newPage
|
||||
|
||||
// Navigate to the URL
|
||||
const result = await this.doAction(async (page) => {
|
||||
await this.navigatePageToUrl(page, url)
|
||||
})
|
||||
|
||||
return result
|
||||
// Attach log capture
|
||||
this.attachConsoleListener()
|
||||
// Reset logs on new launch
|
||||
this.resetLogs()
|
||||
}
|
||||
|
||||
async navigateToUrl(url: string): Promise<BrowserActionResult> {
|
||||
if (!this.browser) {
|
||||
throw new Error("Browser is not launched")
|
||||
}
|
||||
// Remove trailing slash for comparison
|
||||
const normalizedNewUrl = url.replace(/\/$/, "")
|
||||
|
||||
// Extract the root domain from the URL
|
||||
const rootDomain = this.getRootDomain(normalizedNewUrl)
|
||||
|
||||
// Get all current pages
|
||||
const pages = await this.browser.pages()
|
||||
|
||||
// Try to find a page with the same root domain
|
||||
let existingPage: Page | undefined
|
||||
|
||||
for (const page of pages) {
|
||||
try {
|
||||
const pageUrl = page.url()
|
||||
if (pageUrl && this.getRootDomain(pageUrl) === rootDomain) {
|
||||
existingPage = page
|
||||
break
|
||||
}
|
||||
} catch (error) {
|
||||
// Skip pages that might have been closed or have errors
|
||||
console.log(`Error checking page URL: ${error}`)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if (existingPage) {
|
||||
// Tab with the same root domain exists, switch to it
|
||||
console.log(`Tab with domain ${rootDomain} already exists, switching to it`)
|
||||
|
||||
// Update the active page
|
||||
this.page = existingPage
|
||||
existingPage.bringToFront()
|
||||
|
||||
// Navigate to the new URL if it's different]
|
||||
const currentUrl = existingPage.url().replace(/\/$/, "") // Remove trailing / if present
|
||||
if (this.getRootDomain(currentUrl) === rootDomain && currentUrl !== normalizedNewUrl) {
|
||||
console.log(`Navigating to new URL: ${normalizedNewUrl}`)
|
||||
console.log(`Current URL: ${currentUrl}`)
|
||||
console.log(`Root domain: ${this.getRootDomain(currentUrl)}`)
|
||||
console.log(`New URL: ${normalizedNewUrl}`)
|
||||
// Navigate to the new URL
|
||||
return this.doAction(async (page) => {
|
||||
await this.navigatePageToUrl(page, normalizedNewUrl)
|
||||
})
|
||||
} else {
|
||||
console.log(`Tab with domain ${rootDomain} already exists, and URL is the same: ${normalizedNewUrl}`)
|
||||
// URL is the same, just reload the page to ensure it's up to date
|
||||
console.log(`Reloading page: ${normalizedNewUrl}`)
|
||||
console.log(`Current URL: ${currentUrl}`)
|
||||
console.log(`Root domain: ${this.getRootDomain(currentUrl)}`)
|
||||
console.log(`New URL: ${normalizedNewUrl}`)
|
||||
return this.doAction(async (page) => {
|
||||
await page.reload({
|
||||
timeout: BROWSER_NAVIGATION_TIMEOUT,
|
||||
waitUntil: ["domcontentloaded", "networkidle2"],
|
||||
})
|
||||
await this.waitTillHTMLStable(page)
|
||||
})
|
||||
}
|
||||
} else {
|
||||
// No tab with this root domain exists, create a new one
|
||||
console.log(`No tab with domain ${rootDomain} exists, creating a new one`)
|
||||
return this.createNewTab(normalizedNewUrl)
|
||||
}
|
||||
}
|
||||
|
||||
// page.goto { waitUntil: "networkidle0" } may not ever resolve, and not waiting could return page content too early before js has loaded
|
||||
// https://stackoverflow.com/questions/52497252/puppeteer-wait-until-page-is-completely-loaded/61304202#61304202
|
||||
private async waitTillHTMLStable(page: Page, timeout = 5_000) {
|
||||
const checkDurationMsecs = 500 // 1000
|
||||
const maxChecks = timeout / checkDurationMsecs
|
||||
let lastHTMLSize = 0
|
||||
let checkCounts = 1
|
||||
let countStableSizeIterations = 0
|
||||
const minStableSizeIterations = 3
|
||||
|
||||
while (checkCounts++ <= maxChecks) {
|
||||
let html = await page.content()
|
||||
let currentHTMLSize = html.length
|
||||
|
||||
// let bodyHTMLSize = await page.evaluate(() => document.body.innerHTML.length)
|
||||
console.log("last: ", lastHTMLSize, " <> curr: ", currentHTMLSize)
|
||||
|
||||
if (lastHTMLSize !== 0 && currentHTMLSize === lastHTMLSize) {
|
||||
countStableSizeIterations++
|
||||
} else {
|
||||
countStableSizeIterations = 0 //reset the counter
|
||||
}
|
||||
|
||||
if (countStableSizeIterations >= minStableSizeIterations) {
|
||||
console.log("Page rendered fully...")
|
||||
break
|
||||
}
|
||||
|
||||
lastHTMLSize = currentHTMLSize
|
||||
await delay(checkDurationMsecs)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles mouse interaction with network activity monitoring
|
||||
*/
|
||||
private async handleMouseInteraction(
|
||||
page: Page,
|
||||
coordinate: string,
|
||||
action: (x: number, y: number) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const [x, y] = coordinate.split(",").map(Number)
|
||||
|
||||
// Set up network request monitoring
|
||||
let hasNetworkActivity = false
|
||||
const requestListener = () => {
|
||||
hasNetworkActivity = true
|
||||
}
|
||||
page.on("request", requestListener)
|
||||
|
||||
// Perform the mouse action
|
||||
await action(x, y)
|
||||
this.currentMousePosition = coordinate
|
||||
|
||||
// Small delay to check if action triggered any network activity
|
||||
await delay(100)
|
||||
|
||||
if (hasNetworkActivity) {
|
||||
// If we detected network activity, wait for navigation/loading
|
||||
await page
|
||||
.waitForNavigation({
|
||||
waitUntil: ["domcontentloaded", "networkidle2"],
|
||||
timeout: BROWSER_NAVIGATION_TIMEOUT,
|
||||
})
|
||||
.catch(() => {})
|
||||
await this.waitTillHTMLStable(page)
|
||||
}
|
||||
|
||||
// Clean up listener
|
||||
page.off("request", requestListener)
|
||||
const page = this.ensurePage()
|
||||
await this.navigateWithFallback(url)
|
||||
return this.captureResult(true)
|
||||
}
|
||||
|
||||
async click(coordinate: string): Promise<BrowserActionResult> {
|
||||
return this.doAction(async (page) => {
|
||||
await this.handleMouseInteraction(page, coordinate, async (x, y) => {
|
||||
await page.mouse.click(x, y)
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
async type(text: string): Promise<BrowserActionResult> {
|
||||
return this.doAction(async (page) => {
|
||||
await page.keyboard.type(text)
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Scrolls the page by the specified amount
|
||||
*/
|
||||
private async scrollPage(page: Page, direction: "up" | "down"): Promise<void> {
|
||||
const { height } = this.getViewport()
|
||||
const scrollAmount = direction === "down" ? height : -height
|
||||
|
||||
await page.evaluate((scrollHeight) => {
|
||||
window.scrollBy({
|
||||
top: scrollHeight,
|
||||
behavior: "auto",
|
||||
})
|
||||
}, scrollAmount)
|
||||
|
||||
await delay(300)
|
||||
}
|
||||
|
||||
async scrollDown(): Promise<BrowserActionResult> {
|
||||
return this.doAction(async (page) => {
|
||||
await this.scrollPage(page, "down")
|
||||
})
|
||||
}
|
||||
|
||||
async scrollUp(): Promise<BrowserActionResult> {
|
||||
return this.doAction(async (page) => {
|
||||
await this.scrollPage(page, "up")
|
||||
})
|
||||
const page = this.ensurePage()
|
||||
const { x, y } = this.parseCoordinate(coordinate)
|
||||
await page.mouse.move(x, y)
|
||||
await page.mouse.click(x, y, { button: "left", clickCount: 1 })
|
||||
this.mouseX = x
|
||||
this.mouseY = y
|
||||
return this.captureResult(true)
|
||||
}
|
||||
|
||||
async hover(coordinate: string): Promise<BrowserActionResult> {
|
||||
return this.doAction(async (page) => {
|
||||
await this.handleMouseInteraction(page, coordinate, async (x, y) => {
|
||||
await page.mouse.move(x, y)
|
||||
// Small delay to allow any hover effects to appear
|
||||
await delay(300)
|
||||
})
|
||||
const page = this.ensurePage()
|
||||
const { x, y } = this.parseCoordinate(coordinate)
|
||||
await page.mouse.move(x, y)
|
||||
this.mouseX = x
|
||||
this.mouseY = y
|
||||
return this.captureResult(true)
|
||||
}
|
||||
|
||||
async type(text: string): Promise<BrowserActionResult> {
|
||||
const page = this.ensurePage()
|
||||
await page.keyboard.type(text, { delay: 10 })
|
||||
return this.captureResult(true)
|
||||
}
|
||||
|
||||
async scrollDown(): Promise<BrowserActionResult> {
|
||||
const page = this.ensurePage()
|
||||
await page.evaluate(() => {
|
||||
// Scroll by one viewport height
|
||||
window.scrollBy(0, window.innerHeight)
|
||||
})
|
||||
return this.captureResult(true)
|
||||
}
|
||||
|
||||
async scrollUp(): Promise<BrowserActionResult> {
|
||||
const page = this.ensurePage()
|
||||
await page.evaluate(() => {
|
||||
window.scrollBy(0, -window.innerHeight)
|
||||
})
|
||||
return this.captureResult(true)
|
||||
}
|
||||
|
||||
async resize(size: string): Promise<BrowserActionResult> {
|
||||
return this.doAction(async (page) => {
|
||||
const [width, height] = size.split(",").map(Number)
|
||||
const session = await page.createCDPSession()
|
||||
await page.setViewport({ width, height })
|
||||
const { windowId } = await session.send("Browser.getWindowForTarget")
|
||||
await session.send("Browser.setWindowBounds", {
|
||||
bounds: { width, height },
|
||||
windowId,
|
||||
})
|
||||
})
|
||||
const page = this.ensurePage()
|
||||
const { w, h } = this.parseSize(size)
|
||||
this.viewport = { width: w, height: h }
|
||||
await page.setViewport({ width: w, height: h })
|
||||
return this.captureResult(true)
|
||||
}
|
||||
|
||||
async closeBrowser(): Promise<BrowserActionResult> {
|
||||
try {
|
||||
if (this.browser) {
|
||||
await this.browser.close()
|
||||
}
|
||||
} finally {
|
||||
this.browser = undefined
|
||||
this.page = undefined
|
||||
this.consoleAttached = false
|
||||
this.resetLogs()
|
||||
this.mouseX = null
|
||||
this.mouseY = null
|
||||
}
|
||||
// No screenshot on close
|
||||
return {}
|
||||
}
|
||||
|
||||
// Utils
|
||||
|
||||
private parseCoordinate(coordinate: string): { x: number; y: number } {
|
||||
const parts = (coordinate || "").split(",").map((s) => Number(s.trim()))
|
||||
if (parts.length !== 2 || parts.some((n) => Number.isNaN(n))) {
|
||||
throw new Error(`Invalid coordinate: '${coordinate}'. Expected format: "x,y"`)
|
||||
}
|
||||
const [x, y] = parts
|
||||
if (x < 0 || y < 0) {
|
||||
throw new Error(`Invalid coordinate: '${coordinate}'. Coordinates must be non-negative.`)
|
||||
}
|
||||
return { x, y }
|
||||
}
|
||||
|
||||
private parseSize(size: string): { w: number; h: number } {
|
||||
const parts = (size || "").split(",").map((s) => Number(s.trim()))
|
||||
if (parts.length !== 2 || parts.some((n) => Number.isNaN(n))) {
|
||||
throw new Error(`Invalid size: '${size}'. Expected format: "width,height"`)
|
||||
}
|
||||
const [w, h] = parts
|
||||
if (w <= 0 || h <= 0) {
|
||||
throw new Error(`Invalid size: '${size}'. Width and height must be positive integers.`)
|
||||
}
|
||||
return { w, h }
|
||||
}
|
||||
|
||||
private async delay(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
1
tmp/cline
Submodule
1
tmp/cline
Submodule
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 3847a2545cd94979b4fd5d11dcee632b17d228b3
|
||||
1
tmp/cline-cline
Submodule
1
tmp/cline-cline
Submodule
|
|
@ -0,0 +1 @@
|
|||
Subproject commit 0054b21d678175a07989ce30ee66992ff54fdf27
|
||||
Loading…
Add table
Reference in a new issue