scriptoriumadmin/Roo-Code
1
0
Fork 0
mirror of https://github.com/RooVetGit/Roo-Code.git synced 2026-09-09 22:31:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

feat: enable browser use for Gemini models with image support (#5003)

Browse source 
- Replace supportsComputerUse with supportsBrowserUse throughout codebase
- Enable browser use for any model that supports images
- Update Gemini models configuration to include supportsBrowserUse
- Update UI labels from 'computer use' to 'browser use'
- Add comprehensive tests for browser capability logic

This change aligns with Cline's approach where browser interaction
works through screenshot analysis rather than direct computer control.
This commit is contained in:
Daniel Riccio 2025-06-22 17:31:04 -05:00
parent 8eb147346d
commit e9a824cf50
49 changed files with 519 additions and 139 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/types/src/model.ts
View file

@ -32,7 +32,7 @@ export const modelInfoSchema = z.object({
maxThinkingTokens: z.number().nullish(),
contextWindow: z.number(),
supportsImages: z.boolean().optional(),
supportsComputerUse: z.boolean().optional(),
supportsBrowserUse: z.boolean().optional(),
supportsPromptCache: z.boolean(),
supportsReasoningBudget: z.boolean().optional(),
requiredReasoningBudget: z.boolean().optional(),

10
packages/types/src/providers/anthropic.ts
View file

@ -10,7 +10,7 @@ export const anthropicModels = {
maxTokens: 64_000, // Overridden to 8k if `enableReasoningEffort` is false.
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0, // $3 per million input tokens
outputPrice: 15.0, // $15 per million output tokens
@ -22,7 +22,7 @@ export const anthropicModels = {
maxTokens: 32_000, // Overridden to 8k if `enableReasoningEffort` is false.
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 15.0, // $15 per million input tokens
outputPrice: 75.0, // $75 per million output tokens
@ -34,7 +34,7 @@ export const anthropicModels = {
maxTokens: 128_000, // Unlocked by passing `beta` flag to the model. Otherwise, it's 64k.
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0, // $3 per million input tokens
outputPrice: 15.0, // $15 per million output tokens
@ -47,7 +47,7 @@ export const anthropicModels = {
maxTokens: 8192, // Since we already have a `:thinking` virtual model we aren't setting `supportsReasoningBudget: true` here.
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0, // $3 per million input tokens
outputPrice: 15.0, // $15 per million output tokens
@ -58,7 +58,7 @@ export const anthropicModels = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0, // $3 per million input tokens
outputPrice: 15.0, // $15 per million output tokens

46
packages/types/src/providers/bedrock.ts
View file

@ -17,7 +17,7 @@ export const bedrockModels = {
maxTokens: 5000,
contextWindow: 300_000,
supportsImages: true,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: true,
inputPrice: 0.8,
outputPrice: 3.2,
@ -31,7 +31,7 @@ export const bedrockModels = {
maxTokens: 5000,
contextWindow: 300_000,
supportsImages: true,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 1.0,
outputPrice: 4.0,
@ -43,7 +43,7 @@ export const bedrockModels = {
maxTokens: 5000,
contextWindow: 300_000,
supportsImages: true,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: true,
inputPrice: 0.06,
outputPrice: 0.24,
@ -57,7 +57,7 @@ export const bedrockModels = {
maxTokens: 5000,
contextWindow: 128_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: true,
inputPrice: 0.035,
outputPrice: 0.14,
@ -71,7 +71,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
supportsReasoningBudget: true,
inputPrice: 3.0,
@ -86,7 +86,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
supportsReasoningBudget: true,
inputPrice: 15.0,
@ -101,7 +101,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
supportsReasoningBudget: true,
inputPrice: 3.0,
@ -116,7 +116,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0,
outputPrice: 15.0,
@ -210,7 +210,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 128_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.72,
outputPrice: 0.72,
@ -220,7 +220,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 128_000,
supportsImages: true,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.72,
outputPrice: 0.72,
@ -230,7 +230,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 128_000,
supportsImages: true,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.16,
outputPrice: 0.16,
@ -240,7 +240,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 128_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.15,
outputPrice: 0.15,
@ -250,7 +250,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 128_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.1,
outputPrice: 0.1,
@ -260,7 +260,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 128_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 2.4,
outputPrice: 2.4,
@ -270,7 +270,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 128_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.72,
outputPrice: 0.72,
@ -280,7 +280,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 128_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.9,
outputPrice: 0.9,
@ -290,7 +290,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 8_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.22,
outputPrice: 0.22,
@ -300,7 +300,7 @@ export const bedrockModels = {
maxTokens: 2048,
contextWindow: 8_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 2.65,
outputPrice: 3.5,
@ -309,7 +309,7 @@ export const bedrockModels = {
maxTokens: 2048,
contextWindow: 4_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.3,
outputPrice: 0.6,
@ -318,7 +318,7 @@ export const bedrockModels = {
maxTokens: 4096,
contextWindow: 8_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.15,
outputPrice: 0.2,
@ -328,7 +328,7 @@ export const bedrockModels = {
maxTokens: 4096,
contextWindow: 8_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.2,
outputPrice: 0.6,
@ -338,7 +338,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 8_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.1,
description: "Amazon Titan Text Embeddings",
@ -347,7 +347,7 @@ export const bedrockModels = {
maxTokens: 8192,
contextWindow: 8_000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 0.02,
description: "Amazon Titan Text Embeddings V2",

23
packages/types/src/providers/gemini.ts
View file

@ -10,6 +10,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0.15,
outputPrice: 3.5,
@ -21,6 +22,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0.15,
outputPrice: 0.6,
@ -29,6 +31,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 0.15,
outputPrice: 3.5,
@ -42,6 +45,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 0.15,
outputPrice: 0.6,
@ -52,6 +56,7 @@ export const geminiModels = {
maxTokens: 64_000,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 0.3,
outputPrice: 2.5,
@ -64,6 +69,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
@ -72,6 +78,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
outputPrice: 15,
@ -96,6 +103,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
outputPrice: 15,
@ -120,6 +128,7 @@ export const geminiModels = {
maxTokens: 65_535,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
outputPrice: 15,
@ -146,6 +155,7 @@ export const geminiModels = {
maxTokens: 64_000,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 2.5, // This is the pricing for prompts above 200k tokens.
outputPrice: 15,
@ -173,6 +183,7 @@ export const geminiModels = {
maxTokens: 8192,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 0.1,
outputPrice: 0.4,
@ -183,6 +194,7 @@ export const geminiModels = {
maxTokens: 8192,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
@ -191,6 +203,7 @@ export const geminiModels = {
maxTokens: 8192,
contextWindow: 2_097_152,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
@ -199,6 +212,7 @@ export const geminiModels = {
maxTokens: 65_536,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
@ -207,6 +221,7 @@ export const geminiModels = {
maxTokens: 8192,
contextWindow: 32_767,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
@ -215,6 +230,7 @@ export const geminiModels = {
maxTokens: 8192,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
@ -223,6 +239,7 @@ export const geminiModels = {
maxTokens: 8192,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 0.15, // This is the pricing for prompts above 128k tokens.
outputPrice: 0.6,
@ -247,6 +264,7 @@ export const geminiModels = {
maxTokens: 8192,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
@ -255,6 +273,7 @@ export const geminiModels = {
maxTokens: 8192,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
@ -263,6 +282,7 @@ export const geminiModels = {
maxTokens: 8192,
contextWindow: 2_097_152,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
@ -271,6 +291,7 @@ export const geminiModels = {
maxTokens: 8192,
contextWindow: 2_097_152,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
@ -279,6 +300,7 @@ export const geminiModels = {
maxTokens: 8192,
contextWindow: 2_097_152,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 0,
outputPrice: 0,
@ -287,6 +309,7 @@ export const geminiModels = {
maxTokens: 64_000,
contextWindow: 1_048_576,
supportsImages: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 0.1,
outputPrice: 0.4,

2
packages/types/src/providers/glama.ts
View file

@ -7,7 +7,7 @@ export const glamaDefaultModelInfo: ModelInfo = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0,
outputPrice: 15.0,

2
packages/types/src/providers/lite-llm.ts
View file

@ -7,7 +7,7 @@ export const litellmDefaultModelInfo: ModelInfo = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0,
outputPrice: 15.0,

2
packages/types/src/providers/lm-studio.ts
View file

@ -9,7 +9,7 @@ export const lMStudioDefaultModelInfo: ModelInfo = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 0,
outputPrice: 0,

2
packages/types/src/providers/ollama.ts
View file

@ -7,7 +7,7 @@ export const ollamaDefaultModelInfo: ModelInfo = {
maxTokens: 4096,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 0,
outputPrice: 0,

2
packages/types/src/providers/openrouter.ts
View file

@ -7,7 +7,7 @@ export const openRouterDefaultModelInfo: ModelInfo = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0,
outputPrice: 15.0,

2
packages/types/src/providers/requesty.ts
View file

@ -8,7 +8,7 @@ export const requestyDefaultModelInfo: ModelInfo = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0,
outputPrice: 15.0,

10
packages/types/src/providers/vertex.ts
View file

@ -167,7 +167,7 @@ export const vertexModels = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0,
outputPrice: 15.0,
@ -179,7 +179,7 @@ export const vertexModels = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 15.0,
outputPrice: 75.0,
@ -190,7 +190,7 @@ export const vertexModels = {
maxTokens: 64_000,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0,
outputPrice: 15.0,
@ -203,7 +203,7 @@ export const vertexModels = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0,
outputPrice: 15.0,
@ -214,7 +214,7 @@ export const vertexModels = {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0,
outputPrice: 15.0,

2
src/api/providers/__tests__/glama.spec.ts
View file

@ -20,7 +20,7 @@ vitest.mock("../fetchers/modelCache", () => ({
cacheReadsPrice: 0.3,
description: "Claude 3.7 Sonnet",
thinking: false,
supportsComputerUse: true,
supportsBrowserUse: true,
},
"openai/gpt-4o": {
maxTokens: 4096,

4
src/api/providers/__tests__/openrouter.spec.ts
View file

@ -26,7 +26,7 @@ vitest.mock("../fetchers/modelCache", () => ({
cacheReadsPrice: 0.3,
description: "Claude 3.7 Sonnet",
thinking: false,
supportsComputerUse: true,
supportsBrowserUse: true,
},
"anthropic/claude-3.7-sonnet:thinking": {
maxTokens: 128000,
@ -38,7 +38,7 @@ vitest.mock("../fetchers/modelCache", () => ({
cacheWritesPrice: 3.75,
cacheReadsPrice: 0.3,
description: "Claude 3.7 Sonnet with thinking",
supportsComputerUse: true,
supportsBrowserUse: true,
},
})
}),

6
src/api/providers/__tests__/requesty.spec.ts
View file

@ -30,7 +30,7 @@ vitest.mock("../fetchers/modelCache", () => ({
contextWindow: 200000,
supportsImages: true,
supportsPromptCache: true,
supportsComputerUse: true,
supportsBrowserUse: true,
inputPrice: 3,
outputPrice: 15,
cacheWritesPrice: 3.75,
@ -75,7 +75,7 @@ describe("RequestyHandler", () => {
contextWindow: 200000,
supportsImages: true,
supportsPromptCache: true,
supportsComputerUse: true,
supportsBrowserUse: true,
inputPrice: 3,
outputPrice: 15,
cacheWritesPrice: 3.75,
@ -96,7 +96,7 @@ describe("RequestyHandler", () => {
contextWindow: 200000,
supportsImages: true,
supportsPromptCache: true,
supportsComputerUse: true,
supportsBrowserUse: true,
inputPrice: 3,
outputPrice: 15,
cacheWritesPrice: 3.75,

4
src/api/providers/__tests__/unbound.spec.ts
View file

@ -21,7 +21,7 @@ vitest.mock("../fetchers/modelCache", () => ({
cacheReadsPrice: 0.3,
description: "Claude 3.5 Sonnet",
thinking: false,
supportsComputerUse: true,
supportsBrowserUse: true,
},
"anthropic/claude-3-7-sonnet-20250219": {
maxTokens: 8192,
@ -34,7 +34,7 @@ vitest.mock("../fetchers/modelCache", () => ({
cacheReadsPrice: 0.3,
description: "Claude 3.7 Sonnet",
thinking: false,
supportsComputerUse: true,
supportsBrowserUse: true,
},
"openai/gpt-4o": {
maxTokens: 4096,

24
src/api/providers/fetchers/__tests__/litellm.spec.ts
View file

@ -92,7 +92,7 @@ describe("getLiteLLMModels", () => {
maxTokens: 4096,
contextWindow: 200000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: 3,
outputPrice: 15,
@ -102,7 +102,7 @@ describe("getLiteLLMModels", () => {
maxTokens: 8192,
contextWindow: 128000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: 10,
outputPrice: 30,
@ -170,7 +170,7 @@ describe("getLiteLLMModels", () => {
maxTokens: 4096,
contextWindow: 200000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
@ -181,7 +181,7 @@ describe("getLiteLLMModels", () => {
maxTokens: 4096,
contextWindow: 200000,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
@ -316,7 +316,7 @@ describe("getLiteLLMModels", () => {
maxTokens: 4096,
contextWindow: 200000,
supportsImages: true,
supportsComputerUse: true, // Should be true due to fallback
supportsBrowserUse: true, // Should be true due to fallback
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
@ -327,7 +327,7 @@ describe("getLiteLLMModels", () => {
maxTokens: 8192,
contextWindow: 128000,
supportsImages: false,
supportsComputerUse: false, // Should be false as it's not in fallback list
supportsBrowserUse: false, // Should be false as it's not in fallback list
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
@ -390,7 +390,7 @@ describe("getLiteLLMModels", () => {
maxTokens: 4096,
contextWindow: 200000,
supportsImages: true,
supportsComputerUse: false, // False because explicitly set to false (fallback ignored)
supportsBrowserUse: false, // False because explicitly set to false (fallback ignored)
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
@ -401,7 +401,7 @@ describe("getLiteLLMModels", () => {
maxTokens: 8192,
contextWindow: 128000,
supportsImages: false,
supportsComputerUse: true, // True because explicitly set to true
supportsBrowserUse: true, // True because explicitly set to true
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
@ -412,7 +412,7 @@ describe("getLiteLLMModels", () => {
maxTokens: 8192,
contextWindow: 128000,
supportsImages: false,
supportsComputerUse: false, // False because explicitly set to false
supportsBrowserUse: false, // False because explicitly set to false
supportsPromptCache: false,
inputPrice: undefined,
outputPrice: undefined,
@ -468,8 +468,8 @@ describe("getLiteLLMModels", () => {
const result = await getLiteLLMModels("test-api-key", "http://localhost:4000")
expect(result["vertex-claude"].supportsComputerUse).toBe(true)
expect(result["openrouter-claude"].supportsComputerUse).toBe(true)
expect(result["bedrock-claude"].supportsComputerUse).toBe(true)
expect(result["vertex-claude"].supportsBrowserUse).toBe(true)
expect(result["openrouter-claude"].supportsBrowserUse).toBe(true)
expect(result["bedrock-claude"].supportsBrowserUse).toBe(true)
})
})

2
src/api/providers/fetchers/__tests__/lmstudio.test.ts
View file

@ -54,7 +54,7 @@ describe("LMStudio Fetcher", () => {
contextWindow: rawModel.contextLength,
supportsPromptCache: true,
supportsImages: rawModel.vision,
supportsComputerUse: false,
supportsBrowserUse: false,
maxTokens: rawModel.contextLength,
inputPrice: 0,
outputPrice: 0,

4
src/api/providers/fetchers/__tests__/ollama.test.ts
View file

@ -22,7 +22,7 @@ describe("Ollama Fetcher", () => {
maxTokens: 40960,
contextWindow: 40960,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: true,
inputPrice: 0,
outputPrice: 0,
@ -47,7 +47,7 @@ describe("Ollama Fetcher", () => {
maxTokens: 40960,
contextWindow: 40960,
supportsImages: false,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsPromptCache: true,
inputPrice: 0,
outputPrice: 0,

6
src/api/providers/fetchers/__tests__/openrouter.spec.ts
View file

@ -50,7 +50,7 @@ describe("OpenRouter API", () => {
expect(
Object.entries(models)
.filter(([_, model]) => model.supportsComputerUse)
.filter(([_, model]) => model.supportsBrowserUse)
.map(([id, _]) => id)
.sort(),
).toEqual(Array.from(OPEN_ROUTER_COMPUTER_USE_MODELS).sort())
@ -158,7 +158,7 @@ describe("OpenRouter API", () => {
cacheWritesPrice: 3.75,
cacheReadsPrice: 0.3,
description: expect.any(String),
supportsComputerUse: true,
supportsBrowserUse: true,
supportsReasoningBudget: false,
supportsReasoningEffort: false,
supportedParameters: ["max_tokens", "temperature", "reasoning", "include_reasoning"],
@ -174,7 +174,7 @@ describe("OpenRouter API", () => {
cacheWritesPrice: 3.75,
cacheReadsPrice: 0.3,
description: expect.any(String),
supportsComputerUse: true,
supportsBrowserUse: true,
supportsReasoningBudget: true,
requiredReasoningBudget: true,
supportsReasoningEffort: true,

2
src/api/providers/fetchers/glama.ts
View file

@ -16,7 +16,7 @@ export async function getGlamaModels(): Promise<Record<string, ModelInfo>> {
maxTokens: rawModel.maxTokensOutput,
contextWindow: rawModel.maxTokensInput,
supportsImages: rawModel.capabilities?.includes("input:image"),
supportsComputerUse: rawModel.capabilities?.includes("computer_use"),
supportsBrowserUse: rawModel.capabilities?.includes("computer_use"),
supportsPromptCache: rawModel.capabilities?.includes("caching"),
inputPrice: parseApiPrice(rawModel.pricePerToken?.input),
outputPrice: parseApiPrice(rawModel.pricePerToken?.output),

8
src/api/providers/fetchers/litellm.ts
View file

@ -39,12 +39,12 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise
if (!modelName || !modelInfo || !litellmModelName) continue
// Use explicit supports_computer_use if available, otherwise fall back to hardcoded list
let supportsComputerUse: boolean
let supportsBrowserUse: boolean
if (modelInfo.supports_computer_use !== undefined) {
supportsComputerUse = Boolean(modelInfo.supports_computer_use)
supportsBrowserUse = Boolean(modelInfo.supports_computer_use)
} else {
// Fallback for older LiteLLM versions that don't have supports_computer_use field
supportsComputerUse = computerModels.some((computer_model) =>
supportsBrowserUse = computerModels.some((computer_model) =>
litellmModelName.endsWith(computer_model),
)
}
@ -54,7 +54,7 @@ export async function getLiteLLMModels(apiKey: string, baseUrl: string): Promise
contextWindow: modelInfo.max_input_tokens || 200000,
supportsImages: Boolean(modelInfo.supports_vision),
// litellm_params.model may have a prefix like openrouter/
supportsComputerUse,
supportsBrowserUse,
supportsPromptCache: Boolean(modelInfo.supports_prompt_caching),
inputPrice: modelInfo.input_cost_per_token ? modelInfo.input_cost_per_token * 1000000 : undefined,
outputPrice: modelInfo.output_cost_per_token

2
src/api/providers/fetchers/lmstudio.ts
View file

@ -8,7 +8,7 @@ export const parseLMStudioModel = (rawModel: LLMInstanceInfo): ModelInfo => {
contextWindow: rawModel.contextLength,
supportsPromptCache: true,
supportsImages: rawModel.vision,
supportsComputerUse: false,
supportsBrowserUse: false,
maxTokens: rawModel.contextLength,
})

2
src/api/providers/fetchers/ollama.ts
View file

@ -47,7 +47,7 @@ export const parseOllamaModel = (rawModel: OllamaModelInfoResponse): ModelInfo =
contextWindow: contextWindow || ollamaDefaultModelInfo.contextWindow,
supportsPromptCache: true,
supportsImages: rawModel.capabilities?.includes("vision"),
supportsComputerUse: false,
supportsBrowserUse: false,
maxTokens: contextWindow || ollamaDefaultModelInfo.contextWindow,
})

2
src/api/providers/fetchers/openrouter.ts
View file

@ -209,7 +209,7 @@ export const parseOpenRouterModel = ({
// The OpenRouter model definition doesn't give us any hints about
// computer use, so we need to set that manually.
if (OPEN_ROUTER_COMPUTER_USE_MODELS.has(id)) {
modelInfo.supportsComputerUse = true
modelInfo.supportsBrowserUse = true
}
if (OPEN_ROUTER_REASONING_BUDGET_MODELS.has(id)) {

2
src/api/providers/fetchers/requesty.ts
View file

@ -33,7 +33,7 @@ export async function getRequestyModels(apiKey?: string): Promise<Record<string,
contextWindow: rawModel.context_window,
supportsPromptCache: rawModel.supports_caching,
supportsImages: rawModel.supports_vision,
supportsComputerUse: rawModel.supports_computer_use,
supportsBrowserUse: rawModel.supports_computer_use,
supportsReasoningBudget: reasoningBudget,
supportsReasoningEffort: reasoningEffort,
inputPrice: parseApiPrice(rawModel.input_price),

2
src/api/providers/fetchers/unbound.ts
View file

@ -23,7 +23,7 @@ export async function getUnboundModels(apiKey?: string | null): Promise<Record<s
contextWindow: model?.contextWindow ? parseInt(model.contextWindow) : 0,
supportsImages: model?.supportsImages ?? false,
supportsPromptCache: model?.supportsPromptCaching ?? false,
supportsComputerUse: model?.supportsComputerUse ?? false,
supportsBrowserUse: model?.supportsComputerUse ?? false,
inputPrice: model?.inputTokenPrice ? parseFloat(model.inputTokenPrice) : undefined,
outputPrice: model?.outputTokenPrice ? parseFloat(model.outputTokenPrice) : undefined,
cacheWritesPrice: model?.cacheWritePrice ? parseFloat(model.cacheWritePrice) : undefined,

2
src/api/providers/human-relay.ts
View file

@ -70,7 +70,7 @@ export class HumanRelayHandler implements ApiHandler, SingleCompletionHandler {
contextWindow: 100000,
supportsImages: true,
supportsPromptCache: false,
supportsComputerUse: true,
supportsBrowserUse: true,
inputPrice: 0,
outputPrice: 0,
description: "Calling web-side AI model through human relay",

6
src/core/condense/__tests__/index.spec.ts
View file

@ -102,7 +102,7 @@ describe("summarizeConversation", () => {
info: {
contextWindow: 8000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsVision: true,
maxTokens: 4000,
supportsPromptCache: true,
@ -547,7 +547,7 @@ describe("summarizeConversation with custom settings", () => {
info: {
contextWindow: 8000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsVision: true,
maxTokens: 4000,
supportsPromptCache: true,
@ -571,7 +571,7 @@ describe("summarizeConversation with custom settings", () => {
info: {
contextWindow: 4000,
supportsImages: true,
supportsComputerUse: false,
supportsBrowserUse: false,
supportsVision: false,
maxTokens: 2000,
supportsPromptCache: false,

10
src/core/prompts/__tests__/add-custom-instructions.spec.ts
View file

@ -193,7 +193,7 @@ describe("addCustomInstructions", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -216,7 +216,7 @@ describe("addCustomInstructions", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -241,7 +241,7 @@ describe("addCustomInstructions", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
mockMcpHub, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -267,7 +267,7 @@ describe("addCustomInstructions", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
mockMcpHub, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -291,7 +291,7 @@ describe("addCustomInstructions", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize

6
src/core/prompts/__tests__/custom-system-prompt.spec.ts
View file

@ -75,7 +75,7 @@ describe("File-Based Custom System Prompt", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"test/path", // Using a relative path without leading slash
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -112,7 +112,7 @@ describe("File-Based Custom System Prompt", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"test/path", // Using a relative path without leading slash
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -158,7 +158,7 @@ describe("File-Based Custom System Prompt", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"test/path", // Using a relative path without leading slash
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize

26
src/core/prompts/__tests__/system-prompt.spec.ts
View file

@ -207,7 +207,7 @@ describe("SYSTEM_PROMPT", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -226,11 +226,11 @@ describe("SYSTEM_PROMPT", () => {
expect(prompt).toMatchFileSnapshot("./__snapshots__/system-prompt/consistent-system-prompt.snap")
})
it("should include browser actions when supportsComputerUse is true", async () => {
it("should include browser actions when supportsBrowserUse is true", async () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
true, // supportsComputerUse
true, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
"1280x800", // browserViewportSize
@ -255,7 +255,7 @@ describe("SYSTEM_PROMPT", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
mockMcpHub, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -278,7 +278,7 @@ describe("SYSTEM_PROMPT", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // explicitly undefined mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -301,7 +301,7 @@ describe("SYSTEM_PROMPT", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
true, // supportsComputerUse
true, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
"900x600", // different viewport size
@ -324,7 +324,7 @@ describe("SYSTEM_PROMPT", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
new MultiSearchReplaceDiffStrategy(), // Use actual diff strategy from the codebase
undefined, // browserViewportSize
@ -348,7 +348,7 @@ describe("SYSTEM_PROMPT", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
new MultiSearchReplaceDiffStrategy(), // Use actual diff strategy from the codebase
undefined, // browserViewportSize
@ -372,7 +372,7 @@ describe("SYSTEM_PROMPT", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
new MultiSearchReplaceDiffStrategy(), // Use actual diff strategy from the codebase
undefined, // browserViewportSize
@ -423,7 +423,7 @@ describe("SYSTEM_PROMPT", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -484,7 +484,7 @@ describe("SYSTEM_PROMPT", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -522,7 +522,7 @@ describe("SYSTEM_PROMPT", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize
@ -555,7 +555,7 @@ describe("SYSTEM_PROMPT", () => {
const prompt = await SYSTEM_PROMPT(
mockContext,
"/test/path",
false, // supportsComputerUse
false, // supportsBrowserUse
undefined, // mcpHub
undefined, // diffStrategy
undefined, // browserViewportSize

6
src/core/prompts/sections/capabilities.ts
View file

@ -4,7 +4,7 @@ import { CodeIndexManager } from "../../../services/code-index/manager"
export function getCapabilitiesSection(
cwd: string,
supportsComputerUse: boolean,
supportsBrowserUse: boolean,
mcpHub?: McpHub,
diffStrategy?: DiffStrategy,
codeIndexManager?: CodeIndexManager,
@ -14,7 +14,7 @@ export function getCapabilitiesSection(
CAPABILITIES
- You have access to tools that let you execute CLI commands on the user's computer, list files, view source code definitions, regex search${
supportsComputerUse ? ", use the browser" : ""
supportsBrowserUse ? ", use the browser" : ""
}, read and write files, and ask follow-up questions. These tools help you effectively accomplish a wide range of tasks, such as writing code, making edits or improvements to existing files, understanding the current state of a project, performing system operations, and much more.
- When the user initially gives you a task, a recursive list of all filepaths in the current workspace directory ('${cwd}') will be included in environment_details. This provides an overview of the project's file structure, offering key insights into the project from directory/file names (how developers conceptualize and organize their code) and file extensions (the language used). This can also guide decision-making on which files to explore further. If you need to further explore directories such as outside the current workspace directory, you can use the list_files tool. If you pass 'true' for the recursive parameter, it will list files recursively. Otherwise, it will list files at the top level, which is better suited for generic directories where you don't necessarily need the nested structure, like the Desktop.${
codeIndexManager &&
@ -29,7 +29,7 @@ CAPABILITIES
- You can use the list_code_definition_names tool to get an overview of source code definitions for all files at the top level of a specified directory. This can be particularly useful when you need to understand the broader context and relationships between certain parts of the code. You may need to call this tool multiple times to understand various parts of the codebase related to the task.
- For example, when asked to make edits or improvements you might analyze the file structure in the initial environment_details to get an overview of the project, then use list_code_definition_names to get further insight using source code definitions for files located in relevant directories, then read_file to examine the contents of relevant files, analyze the code and suggest improvements or make necessary edits, then use ${diffStrategy ? "the apply_diff or write_to_file" : "the write_to_file"} tool to apply the changes. If you refactored code that could affect other parts of the codebase, you could use search_files to ensure you update other files as needed.
- You can use the execute_command tool to run commands on the user's computer whenever you feel it can help accomplish the user's task. When you need to execute a CLI command, you must provide a clear explanation of what the command does. Prefer to execute complex CLI commands over creating executable scripts, since they are more flexible and easier to run. Interactive and long-running commands are allowed, since the commands are run in the user's VSCode terminal. The user may keep commands running in the background and you will be kept updated on their status along the way. Each command you execute is run in a new terminal instance.${
supportsComputerUse
supportsBrowserUse
? "\n- You can use the browser_action tool to interact with websites (including html files and locally running development servers) through a Puppeteer-controlled browser when you feel it is necessary in accomplishing the user's task. This tool is particularly useful for web development tasks as it allows you to launch a browser, navigate to pages, interact with elements through clicks and keyboard input, and capture the results through screenshots and console logs. This tool may be useful at key stages of web development tasks-such as after implementing new features, making substantial changes, when troubleshooting issues, or to verify the result of your work. You can analyze the provided screenshots to ensure correct rendering or identify errors, and review console logs for runtime issues.\n - For example, if asked to add a component to a react website, you might create the necessary files, use execute_command to run the site locally, then use browser_action to launch the browser, navigate to the local server, and verify the component renders & functions correctly before closing the browser."
: ""
}${

6
src/core/prompts/sections/rules.ts
View file

@ -47,7 +47,7 @@ function getEditingInstructions(diffStrategy?: DiffStrategy): string {
export function getRulesSection(
cwd: string,
supportsComputerUse: boolean,
supportsBrowserUse: boolean,
diffStrategy?: DiffStrategy,
codeIndexManager?: CodeIndexManager,
): string {
@ -82,7 +82,7 @@ ${getEditingInstructions(diffStrategy)}
- When executing commands, if you don't see the expected output, assume the terminal executed the command successfully and proceed with the task. The user's terminal may be unable to stream the output back properly. If you absolutely need to see the actual terminal output, use the ask_followup_question tool to request the user to copy and paste it back to you.
- The user may provide a file's contents directly in their message, in which case you shouldn't use the read_file tool to get the file contents again since you already have it.
- Your goal is to try to accomplish the user's task, NOT engage in a back and forth conversation.${
supportsComputerUse
supportsBrowserUse
? '\n- The user may ask generic non-development tasks, such as "what\'s the latest news" or "look up the weather in San Diego", in which case you might use the browser_action tool to complete the task if it makes sense to do so, rather than trying to create a website or using curl to answer the question. However, if an available MCP server tool or resource can be used instead, you should prefer to use it over browser_action.'
: ""
}
@ -93,7 +93,7 @@ ${getEditingInstructions(diffStrategy)}
- Before executing commands, check the "Actively Running Terminals" section in environment_details. If present, consider how these active processes might impact your task. For example, if a local development server is already running, you wouldn't need to start it again. If no active terminals are listed, proceed with command execution as normal.
- MCP operations should be used one at a time, similar to other tool usage. Wait for confirmation of success before proceeding with additional operations.
- It is critical you wait for the user's response after each tool use, in order to confirm the success of the tool use. For example, if asked to make a todo app, you would create a file, wait for the user's response it was created successfully, then create another file if needed, wait for the user's response it was created successfully, etc.${
supportsComputerUse
supportsBrowserUse
? " Then if you want to test your work, you might use browser_action to launch the site, wait for the user's response confirming the site was launched along with a screenshot, then perhaps e.g., click a button to test functionality if needed, wait for the user's response confirming the button was clicked along with a screenshot of the new state, before finally closing the browser."
: ""
}`

12
src/core/prompts/system.ts
View file

@ -29,7 +29,7 @@ import {
async function generatePrompt(
context: vscode.ExtensionContext,
cwd: string,
supportsComputerUse: boolean,
supportsBrowserUse: boolean,
mode: Mode,
mcpHub?: McpHub,
diffStrategy?: DiffStrategy,
@ -74,7 +74,7 @@ ${getSharedToolUseSection()}
${getToolDescriptionsForMode(
mode,
cwd,
supportsComputerUse,
supportsBrowserUse,
codeIndexManager,
effectiveDiffStrategy,
browserViewportSize,
@ -89,11 +89,11 @@ ${getToolUseGuidelinesSection(codeIndexManager)}
${mcpServersSection}
${getCapabilitiesSection(cwd, supportsComputerUse, mcpHub, effectiveDiffStrategy, codeIndexManager)}
${getCapabilitiesSection(cwd, supportsBrowserUse, mcpHub, effectiveDiffStrategy, codeIndexManager)}
${modesSection}
${getRulesSection(cwd, supportsComputerUse, effectiveDiffStrategy, codeIndexManager)}
${getRulesSection(cwd, supportsBrowserUse, effectiveDiffStrategy, codeIndexManager)}
${getSystemInfoSection(cwd)}
@ -107,7 +107,7 @@ ${await addCustomInstructions(baseInstructions, globalCustomInstructions || "",
export const SYSTEM_PROMPT = async (
context: vscode.ExtensionContext,
cwd: string,
supportsComputerUse: boolean,
supportsBrowserUse: boolean,
mcpHub?: McpHub,
diffStrategy?: DiffStrategy,
browserViewportSize?: string,
@ -180,7 +180,7 @@ ${customInstructions}`
return generatePrompt(
context,
cwd,
supportsComputerUse,
supportsBrowserUse,
currentMode.slug,
mcpHub,
effectiveDiffStrategy,

4
src/core/prompts/tools/__tests__/attempt-completion.spec.ts
View file

@ -4,7 +4,7 @@ describe("getAttemptCompletionDescription", () => {
it("should NOT include command parameter in the description", () => {
const args = {
cwd: "/test/path",
supportsComputerUse: false,
supportsBrowserUse: false,
}
const description = getAttemptCompletionDescription(args)
@ -41,7 +41,7 @@ describe("getAttemptCompletionDescription", () => {
it("should show example without command", () => {
const args = {
cwd: "/test/path",
supportsComputerUse: false,
supportsBrowserUse: false,
}
const description = getAttemptCompletionDescription(args)

2
src/core/prompts/tools/browser-action.ts
View file

@ -1,7 +1,7 @@
import { ToolArgs } from "./types"
export function getBrowserActionDescription(args: ToolArgs): string | undefined {
if (!args.supportsComputerUse) {
if (!args.supportsBrowserUse) {
return undefined
}
return `## browser_action

4
src/core/prompts/tools/index.ts
View file

@ -50,7 +50,7 @@ const toolDescriptionMap: Record<string, (args: ToolArgs) => string | undefined>
export function getToolDescriptionsForMode(
mode: Mode,
cwd: string,
supportsComputerUse: boolean,
supportsBrowserUse: boolean,
codeIndexManager?: CodeIndexManager,
diffStrategy?: DiffStrategy,
browserViewportSize?: string,
@ -63,7 +63,7 @@ export function getToolDescriptionsForMode(
const config = getModeConfig(mode, customModes)
const args: ToolArgs = {
cwd,
supportsComputerUse,
supportsBrowserUse,
diffStrategy,
browserViewportSize,
mcpHub,

2
src/core/prompts/tools/types.ts
View file

@ -3,7 +3,7 @@ import { McpHub } from "../../../services/mcp/McpHub"
export type ToolArgs = {
cwd: string
supportsComputerUse: boolean
supportsBrowserUse: boolean
diffStrategy?: DiffStrategy
browserViewportSize?: string
mcpHub?: McpHub

2
src/core/task/Task.ts
View file

@ -1612,7 +1612,7 @@ export class Task extends EventEmitter<ClineEvents> {
return SYSTEM_PROMPT(
provider.context,
this.cwd,
(this.api.getModel().info.supportsComputerUse ?? false) && (browserToolEnabled ?? true),
(this.api.getModel().info.supportsBrowserUse ?? false) && (browserToolEnabled ?? true),
mcpHub,
this.diffStrategy,
browserViewportSize,

4
src/core/task/__tests__/Task.spec.ts
View file

@ -452,7 +452,7 @@ describe("Cline", () => {
info: {
supportsImages: true,
supportsPromptCache: true,
supportsComputerUse: true,
supportsBrowserUse: true,
contextWindow: 200000,
maxTokens: 4096,
inputPrice: 0.25,
@ -475,7 +475,7 @@ describe("Cline", () => {
info: {
supportsImages: false,
supportsPromptCache: false,
supportsComputerUse: false,
supportsBrowserUse: false,
contextWindow: 16000,
maxTokens: 2048,
inputPrice: 0.1,

2
src/core/webview/__tests__/ClineProvider.spec.ts
View file

@ -282,7 +282,7 @@ vi.mock("../../../api", () => ({
buildApiHandler: vi.fn().mockReturnValue({
getModel: vi.fn().mockReturnValue({
id: "claude-3-sonnet",
info: { supportsComputerUse: false },
info: { supportsBrowserUse: false },
}),
}),
}))

357
src/core/webview/__tests__/generateSystemPrompt.spec.ts Normal file
View file

@ -0,0 +1,357 @@
import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"
import type * as vscode from "vscode"
import type { WebviewMessage } from "../../../shared/WebviewMessage"
import { generateSystemPrompt } from "../generateSystemPrompt"
import type { ClineProvider } from "../ClineProvider"
import { buildApiHandler } from "../../../api"
import { getModeBySlug } from "../../../shared/modes"
import { experiments as experimentsModule, EXPERIMENT_IDS } from "../../../shared/experiments"
// Mock dependencies
vi.mock("../../../api", () => ({
buildApiHandler: vi.fn(),
}))
vi.mock("../../../shared/modes", () => ({
defaultModeSlug: "code",
getModeBySlug: vi.fn(),
getGroupName: vi.fn((group) => (typeof group === "string" ? group : group[0])),
}))
vi.mock("../../../shared/experiments", () => ({
experiments: {
isEnabled: vi.fn(),
},
EXPERIMENT_IDS: {
MULTI_FILE_APPLY_DIFF: "multi_file_apply_diff",
},
}))
vi.mock("../../../core/prompts/system", () => ({
SYSTEM_PROMPT: vi.fn().mockResolvedValue("Generated system prompt"),
}))
vi.mock("../../diff/strategies/multi-search-replace", () => {
const MockMultiSearchReplaceDiffStrategy = vi.fn()
MockMultiSearchReplaceDiffStrategy.mockImplementation(() => ({
getName: () => "multi-search-replace",
}))
return { MultiSearchReplaceDiffStrategy: MockMultiSearchReplaceDiffStrategy }
})
vi.mock("../../diff/strategies/multi-file-search-replace", () => {
const MockMultiFileSearchReplaceDiffStrategy = vi.fn()
MockMultiFileSearchReplaceDiffStrategy.mockImplementation(() => ({
getName: () => "multi-file-search-replace",
}))
return { MultiFileSearchReplaceDiffStrategy: MockMultiFileSearchReplaceDiffStrategy }
})
describe("generateSystemPrompt", () => {
let mockProvider: Partial<ClineProvider>
let mockBuildApiHandler: any
let mockGetModeBySlug: any
let mockIsEnabled: any
beforeEach(() => {
// Reset all mocks
vi.clearAllMocks()
// Setup mock provider
mockProvider = {
getState: vi.fn().mockResolvedValue({
apiConfiguration: { apiProvider: "gemini" },
customModePrompts: {},
customInstructions: "",
browserViewportSize: "900x600",
diffEnabled: true,
mcpEnabled: false,
fuzzyMatchThreshold: 0.8,
experiments: {},
enableMcpServerCreation: false,
browserToolEnabled: true,
language: "en",
maxReadFileLine: 1000,
maxConcurrentFileReads: 10,
}),
cwd: "/test/workspace",
customModesManager: {
getCustomModes: vi.fn().mockResolvedValue([]),
} as any,
getCurrentCline: vi.fn().mockReturnValue(null),
getMcpHub: vi.fn().mockReturnValue(null),
}
// Setup mocked functions
mockBuildApiHandler = vi.mocked(buildApiHandler)
mockGetModeBySlug = vi.mocked(getModeBySlug)
mockIsEnabled = vi.mocked(experimentsModule.isEnabled)
// Default mock implementations
mockIsEnabled.mockReturnValue(false)
})
afterEach(() => {
vi.restoreAllMocks()
})
describe("browser tool support", () => {
it("should enable browser tools when model supports images, mode includes browser group, and browserToolEnabled is true", async () => {
// Setup: Gemini model with supportsImages and supportsBrowserUse
mockBuildApiHandler.mockReturnValue({
getModel: vi.fn().mockReturnValue({
id: "gemini-2.0-flash",
info: {
supportsImages: true,
supportsBrowserUse: true,
},
}),
})
// Setup: Code mode includes browser tool group
mockGetModeBySlug.mockReturnValue({
slug: "code",
groups: ["read", "edit", "browser", "command", "mcp"],
})
const message: WebviewMessage = { type: "getSystemPrompt", mode: "code" }
const result = await generateSystemPrompt(mockProvider as ClineProvider, message)
// Verify SYSTEM_PROMPT was called with canUseBrowserTool = true
const { SYSTEM_PROMPT } = await import("../../../core/prompts/system")
expect(SYSTEM_PROMPT).toHaveBeenCalled()
const callArgs = vi.mocked(SYSTEM_PROMPT).mock.calls[0]
expect(callArgs[2]).toBe(true) // canUseBrowserTool should be true
})
it("should disable browser tools when model does not support browser use", async () => {
// Setup: Model without supportsBrowserUse
mockBuildApiHandler.mockReturnValue({
getModel: vi.fn().mockReturnValue({
id: "claude-3-sonnet",
info: {
supportsImages: false,
supportsBrowserUse: false,
},
}),
})
// Setup: Code mode includes browser tool group
mockGetModeBySlug.mockReturnValue({
slug: "code",
groups: ["read", "edit", "browser", "command", "mcp"],
})
const message: WebviewMessage = { type: "getSystemPrompt", mode: "code" }
const result = await generateSystemPrompt(mockProvider as ClineProvider, message)
// Verify SYSTEM_PROMPT was called with canUseBrowserTool = false
const { SYSTEM_PROMPT } = await import("../../../core/prompts/system")
expect(SYSTEM_PROMPT).toHaveBeenCalled()
const callArgs = vi.mocked(SYSTEM_PROMPT).mock.calls[0]
expect(callArgs[2]).toBe(false) // canUseBrowserTool should be false
})
it("should disable browser tools when mode does not include browser group", async () => {
// Setup: Gemini model with browser support
mockBuildApiHandler.mockReturnValue({
getModel: vi.fn().mockReturnValue({
id: "gemini-2.0-flash",
info: {
supportsImages: true,
supportsBrowserUse: true,
},
}),
})
// Setup: Custom mode without browser tool group
mockGetModeBySlug.mockReturnValue({
slug: "custom-mode",
groups: ["read", "edit"], // No browser group
})
const message: WebviewMessage = { type: "getSystemPrompt", mode: "custom-mode" }
const result = await generateSystemPrompt(mockProvider as ClineProvider, message)
// Verify SYSTEM_PROMPT was called with canUseBrowserTool = false
const { SYSTEM_PROMPT } = await import("../../../core/prompts/system")
expect(SYSTEM_PROMPT).toHaveBeenCalled()
const callArgs = vi.mocked(SYSTEM_PROMPT).mock.calls[0]
expect(callArgs[2]).toBe(false) // canUseBrowserTool should be false
})
it("should disable browser tools when browserToolEnabled setting is false", async () => {
// Setup: Gemini model with browser support
mockBuildApiHandler.mockReturnValue({
getModel: vi.fn().mockReturnValue({
id: "gemini-2.0-flash",
info: {
supportsImages: true,
supportsBrowserUse: true,
},
}),
})
// Setup: Code mode includes browser tool group
mockGetModeBySlug.mockReturnValue({
slug: "code",
groups: ["read", "edit", "browser", "command", "mcp"],
})
// Override provider state to have browserToolEnabled = false
mockProvider.getState = vi.fn().mockResolvedValue({
apiConfiguration: { apiProvider: "gemini" },
customModePrompts: {},
customInstructions: "",
browserViewportSize: "900x600",
diffEnabled: true,
mcpEnabled: false,
fuzzyMatchThreshold: 0.8,
experiments: {},
enableMcpServerCreation: false,
browserToolEnabled: false, // Disabled by user
language: "en",
maxReadFileLine: 1000,
maxConcurrentFileReads: 10,
})
const message: WebviewMessage = { type: "getSystemPrompt", mode: "code" }
const result = await generateSystemPrompt(mockProvider as ClineProvider, message)
// Verify SYSTEM_PROMPT was called with canUseBrowserTool = false
const { SYSTEM_PROMPT } = await import("../../../core/prompts/system")
expect(SYSTEM_PROMPT).toHaveBeenCalled()
const callArgs = vi.mocked(SYSTEM_PROMPT).mock.calls[0]
expect(callArgs[2]).toBe(false) // canUseBrowserTool should be false
})
it("should handle error when checking model support gracefully", async () => {
// Setup: buildApiHandler throws an error
mockBuildApiHandler.mockImplementation(() => {
throw new Error("API configuration error")
})
// Setup: Code mode includes browser tool group
mockGetModeBySlug.mockReturnValue({
slug: "code",
groups: ["read", "edit", "browser", "command", "mcp"],
})
// Spy on console.error
const consoleErrorSpy = vi.spyOn(console, "error").mockImplementation(() => {})
const message: WebviewMessage = { type: "getSystemPrompt", mode: "code" }
const result = await generateSystemPrompt(mockProvider as ClineProvider, message)
// Verify error was logged
expect(consoleErrorSpy).toHaveBeenCalledWith(
"Error checking if model supports browser use:",
expect.any(Error),
)
// Verify SYSTEM_PROMPT was called with canUseBrowserTool = false (fallback)
const { SYSTEM_PROMPT } = await import("../../../core/prompts/system")
expect(SYSTEM_PROMPT).toHaveBeenCalled()
const callArgs = vi.mocked(SYSTEM_PROMPT).mock.calls[0]
expect(callArgs[2]).toBe(false) // canUseBrowserTool should be false on error
consoleErrorSpy.mockRestore()
})
it("should verify all Gemini models with supportsImages have supportsBrowserUse", async () => {
// This test verifies the specific requirement that Gemini models with supportsImages
// should also have supportsBrowserUse set to true
const geminiModelsWithImages = [
"gemini-2.5-flash-preview-04-17:thinking",
"gemini-2.5-flash-preview-04-17",
"gemini-2.5-flash",
"gemini-2.0-flash-001",
"gemini-1.5-flash-002",
]
for (const modelId of geminiModelsWithImages) {
// Setup: Gemini model
mockBuildApiHandler.mockReturnValue({
getModel: vi.fn().mockReturnValue({
id: modelId,
info: {
supportsImages: true,
supportsBrowserUse: true, // This should be true for all Gemini models with images
},
}),
})
// Setup: Code mode includes browser tool group
mockGetModeBySlug.mockReturnValue({
slug: "code",
groups: ["read", "edit", "browser", "command", "mcp"],
})
const message: WebviewMessage = { type: "getSystemPrompt", mode: "code" }
const result = await generateSystemPrompt(mockProvider as ClineProvider, message)
// Verify SYSTEM_PROMPT was called with canUseBrowserTool = true
const { SYSTEM_PROMPT } = await import("../../../core/prompts/system")
expect(SYSTEM_PROMPT).toHaveBeenCalled()
const callArgs = vi.mocked(SYSTEM_PROMPT).mock.calls[0]
expect(callArgs[2]).toBe(true) // canUseBrowserTool should be true for Gemini models with images
vi.clearAllMocks()
}
})
})
describe("diff strategy selection", () => {
it("should use MultiFileSearchReplaceDiffStrategy when experiment is enabled", async () => {
// Enable the multi-file apply diff experiment
mockIsEnabled.mockReturnValue(true)
mockBuildApiHandler.mockReturnValue({
getModel: vi.fn().mockReturnValue({
id: "test-model",
info: { supportsBrowserUse: false },
}),
})
mockGetModeBySlug.mockReturnValue({
slug: "code",
groups: ["read", "edit"],
})
const message: WebviewMessage = { type: "getSystemPrompt", mode: "code" }
await generateSystemPrompt(mockProvider as ClineProvider, message)
// Verify the correct diff strategy was instantiated
const { MultiFileSearchReplaceDiffStrategy } = await import(
"../../diff/strategies/multi-file-search-replace"
)
expect(MultiFileSearchReplaceDiffStrategy).toHaveBeenCalledWith(0.8)
})
it("should use MultiSearchReplaceDiffStrategy when experiment is disabled", async () => {
// Disable the multi-file apply diff experiment
mockIsEnabled.mockReturnValue(false)
mockBuildApiHandler.mockReturnValue({
getModel: vi.fn().mockReturnValue({
id: "test-model",
info: { supportsBrowserUse: false },
}),
})
mockGetModeBySlug.mockReturnValue({
slug: "code",
groups: ["read", "edit"],
})
const message: WebviewMessage = { type: "getSystemPrompt", mode: "code" }
await generateSystemPrompt(mockProvider as ClineProvider, message)
// Verify the correct diff strategy was instantiated
const { MultiSearchReplaceDiffStrategy } = await import("../../diff/strategies/multi-search-replace")
expect(MultiSearchReplaceDiffStrategy).toHaveBeenCalledWith(0.8)
})
})
})

10
src/core/webview/generateSystemPrompt.ts
View file

@ -44,15 +44,15 @@ export const generateSystemPrompt = async (provider: ClineProvider, message: Web
const rooIgnoreInstructions = provider.getCurrentCline()?.rooIgnoreController?.getInstructions()
// Determine if browser tools can be used based on model support, mode, and user settings
let modelSupportsComputerUse = false
let modelSupportsBrowserUse = false
// Create a temporary API handler to check if the model supports computer use
// Create a temporary API handler to check if the model supports browser use
// This avoids relying on an active Cline instance which might not exist during preview
try {
const tempApiHandler = buildApiHandler(apiConfiguration)
modelSupportsComputerUse = tempApiHandler.getModel().info.supportsComputerUse ?? false
modelSupportsBrowserUse = tempApiHandler.getModel().info.supportsBrowserUse ?? false
} catch (error) {
console.error("Error checking if model supports computer use:", error)
console.error("Error checking if model supports browser use:", error)
}
// Check if the current mode includes the browser tool group
@ -61,7 +61,7 @@ export const generateSystemPrompt = async (provider: ClineProvider, message: Web
// Only enable browser tools if the model supports it, the mode includes browser tools,
// and browser tools are enabled in settings
const canUseBrowserTool = modelSupportsComputerUse && modeSupportsBrowser && (browserToolEnabled ?? true)
const canUseBrowserTool = modelSupportsBrowserUse && modeSupportsBrowser && (browserToolEnabled ?? true)
const systemPrompt = await SYSTEM_PROMPT(
provider.context,

2
webview-ui/src/components/settings/ModelInfoView.tsx
View file

@ -32,7 +32,7 @@ export const ModelInfoView = ({
doesNotSupportLabel={t("settings:modelInfo.noImages")}
/>,
<ModelInfoSupportsItem
isSupported={modelInfo?.supportsComputerUse ?? false}
isSupported={modelInfo?.supportsImages ?? false}
supportsLabel={t("settings:modelInfo.supportsComputerUse")}
doesNotSupportLabel={t("settings:modelInfo.noComputerUse")}
/>,

2
webview-ui/src/components/settings/__tests__/ModelPicker.spec.tsx
View file

@ -22,7 +22,7 @@ describe("ModelPicker", () => {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0,
outputPrice: 15.0,

4
webview-ui/src/components/settings/providers/OpenAICompatible.tsx
View file

@ -396,11 +396,11 @@ export const OpenAICompatible = ({
<div>
<div className="flex items-center gap-1">
<Checkbox
checked={apiConfiguration?.openAiCustomModelInfo?.supportsComputerUse ?? false}
checked={apiConfiguration?.openAiCustomModelInfo?.supportsBrowserUse ?? false}
onChange={handleInputChange("openAiCustomModelInfo", (checked) => {
return {
...(apiConfiguration?.openAiCustomModelInfo || openAiModelInfoSaneDefaults),
supportsComputerUse: checked,
supportsBrowserUse: checked,
}
})}>
<span className="font-medium">{t("settings:providers.customModel.computerUse.label")}</span>

6
webview-ui/src/components/ui/hooks/__tests__/useSelectedModel.spec.ts
View file

@ -142,7 +142,7 @@ describe("useSelectedModel", () => {
contextWindow: 8192,
supportsImages: false,
supportsPromptCache: false,
supportsComputerUse: true,
supportsBrowserUse: true,
cacheWritesPrice: 0.1,
cacheReadsPrice: 0.01,
}
@ -189,7 +189,7 @@ describe("useSelectedModel", () => {
// Fields from base model that provider doesn't have
contextWindow: 8192, // From base (provider doesn't override)
supportsPromptCache: false, // From base (provider doesn't override)
supportsComputerUse: true, // From base (provider doesn't have)
supportsBrowserUse: true, // From base (provider doesn't have)
cacheWritesPrice: 0.1, // From base (provider doesn't have)
cacheReadsPrice: 0.01, // From base (provider doesn't have)
@ -251,7 +251,7 @@ describe("useSelectedModel", () => {
maxTokens: 8192,
contextWindow: 200_000,
supportsImages: true,
supportsComputerUse: true,
supportsBrowserUse: true,
supportsPromptCache: true,
inputPrice: 3.0,
outputPrice: 15.0,

2
webview-ui/src/components/ui/hooks/useOpenRouterModelProviders.ts
View file

@ -75,7 +75,7 @@ async function getOpenRouterProvidersForModel(modelId: string) {
// main models endpoint.
switch (true) {
case modelId.startsWith("anthropic/claude-3.7-sonnet"):
modelInfo.supportsComputerUse = true
modelInfo.supportsBrowserUse = true
modelInfo.supportsPromptCache = true
modelInfo.cacheWritesPrice = 3.75
modelInfo.cacheReadsPrice = 0.3

6
webview-ui/src/i18n/locales/en/settings.json
View file

@ -91,7 +91,7 @@
},
"browser": {
"label": "Browser",
"description": "Automatically perform browser actions without requiring approval. Note: Only applies when the model supports computer use"
"description": "Automatically perform browser actions without requiring approval. Note: Only applies when the model supports browser use"
},
"retry": {
"label": "Retry",
@ -266,7 +266,7 @@
"description": "Is this model capable of processing and understanding images?"
},
"computerUse": {
"label": "Computer Use",
"label": "Browser Use",
"description": "Is this model capable of interacting with a browser? (e.g. Claude 3.7 Sonnet)."
},
"promptCache": {
@ -308,7 +308,7 @@
"browser": {
"enable": {
"label": "Enable browser tool",
"description": "When enabled, Roo can use a browser to interact with websites when using models that support computer use. <0>Learn more</0>"
"description": "When enabled, Roo can use a browser to interact with websites when using models that support browser use. <0>Learn more</0>"
},
"viewport": {
"label": "Viewport size",