feat: enable custom API providers for Roo Code Evals

- Remove hardcoded openrouter provider from EVALS_SETTINGS
- Update runTask.ts to support multiple API providers with proper environment variable mapping
- Fix web-evals UI to properly set apiProvider when using OpenRouter mode
- Add tests to verify provider configuration flexibility
- Maintain backward compatibility with OpenRouter as default when no provider specified

Fixes #6817

apps/web-evals/src/app/runs/new/new-run.tsx

@@ -101,7 +101,12 @@ export function NewRun() {
 		async (values: CreateRun) => {
 			try {
 				if (mode === "openrouter") {
-					values.settings = { ...(values.settings || {}), openRouterModelId: model }
+					// Ensure apiProvider is set along with the model ID
+					values.settings = {
+						...(values.settings || {}),
+						apiProvider: "openrouter",
+						openRouterModelId: model,
+					}
 				}
 
 				const { id } = await createRun({ ...values, systemPrompt })

packages/evals/src/cli/__tests__/provider-config.test.ts

@@ -0,0 +1,121 @@
+import { describe, it, expect, vi, beforeEach } from "vitest"
+import { EVALS_SETTINGS } from "@roo-code/types"
+
+describe("Provider Configuration", () => {
+	describe("EVALS_SETTINGS", () => {
+		it("should not have a hardcoded apiProvider", () => {
+			// EVALS_SETTINGS should not have apiProvider set by default
+			// to allow flexibility in choosing providers
+			expect(EVALS_SETTINGS.apiProvider).toBeUndefined()
+		})
+
+		it("should have other necessary settings", () => {
+			// Verify that other important settings are still present
+			expect(EVALS_SETTINGS.autoApprovalEnabled).toBe(true)
+			expect(EVALS_SETTINGS.alwaysAllowWrite).toBe(true)
+			expect(EVALS_SETTINGS.alwaysAllowExecute).toBe(true)
+			expect(EVALS_SETTINGS.mode).toBe("code")
+		})
+	})
+
+	describe("Configuration Merging", () => {
+		it("should allow overriding apiProvider through run settings", () => {
+			const runSettings = {
+				apiProvider: "anthropic" as const,
+				apiKey: "test-key",
+			}
+
+			const mergedConfig = {
+				...EVALS_SETTINGS,
+				...runSettings,
+			}
+
+			expect(mergedConfig.apiProvider).toBe("anthropic")
+			expect(mergedConfig.apiKey).toBe("test-key")
+		})
+
+		it("should support multiple providers", () => {
+			const providers = [
+				{ apiProvider: "openrouter" as const, openRouterApiKey: "key1" },
+				{ apiProvider: "anthropic" as const, apiKey: "key2" },
+				{ apiProvider: "openai" as const, openAiApiKey: "key3" },
+				{ apiProvider: "gemini" as const, geminiApiKey: "key4" },
+				{ apiProvider: "ollama" as const, ollamaModelId: "model1" },
+				{ apiProvider: "litellm" as const, litellmApiKey: "key5" },
+			]
+
+			providers.forEach((providerSettings) => {
+				const mergedConfig = {
+					...EVALS_SETTINGS,
+					...providerSettings,
+				}
+
+				expect(mergedConfig.apiProvider).toBe(providerSettings.apiProvider)
+			})
+		})
+
+		it("should maintain backward compatibility with openrouter as default", () => {
+			// When no apiProvider is specified in run settings,
+			// the implementation should default to openrouter for backward compatibility
+			const runSettings: Record<string, unknown> = {}
+
+			// This simulates the logic in runTask.ts
+			const configuration: Record<string, unknown> = {
+				...EVALS_SETTINGS,
+			}
+
+			if (!runSettings.apiProvider) {
+				configuration.apiProvider = "openrouter"
+				configuration.openRouterApiKey = process.env.OPENROUTER_API_KEY
+			}
+
+			expect(configuration.apiProvider).toBe("openrouter")
+		})
+	})
+
+	describe("Environment Variable Mapping", () => {
+		beforeEach(() => {
+			// Clear environment variables
+			vi.stubEnv("OPENROUTER_API_KEY", "")
+			vi.stubEnv("ANTHROPIC_API_KEY", "")
+			vi.stubEnv("OPENAI_API_KEY", "")
+			vi.stubEnv("GEMINI_API_KEY", "")
+			vi.stubEnv("DEEPSEEK_API_KEY", "")
+			vi.stubEnv("MISTRAL_API_KEY", "")
+			vi.stubEnv("GROQ_API_KEY", "")
+			vi.stubEnv("LITELLM_API_KEY", "")
+		})
+
+		it("should map environment variables based on provider", () => {
+			// Set test environment variables
+			vi.stubEnv("OPENROUTER_API_KEY", "test-openrouter-key")
+			vi.stubEnv("ANTHROPIC_API_KEY", "test-anthropic-key")
+			vi.stubEnv("OPENAI_API_KEY", "test-openai-key")
+
+			const providerKeyMappings = [
+				{ provider: "openrouter", envVar: "OPENROUTER_API_KEY", configKey: "openRouterApiKey" },
+				{ provider: "anthropic", envVar: "ANTHROPIC_API_KEY", configKey: "apiKey" },
+				{ provider: "openai", envVar: "OPENAI_API_KEY", configKey: "openAiApiKey" },
+			]
+
+			providerKeyMappings.forEach(({ provider, envVar, configKey }) => {
+				const configuration: Record<string, unknown> = {}
+
+				// Simulate the switch logic from runTask.ts
+				switch (provider) {
+					case "openrouter":
+						configuration.openRouterApiKey = process.env.OPENROUTER_API_KEY
+						break
+					case "anthropic":
+						configuration.apiKey = process.env.ANTHROPIC_API_KEY
+						break
+					case "openai":
+						configuration.openAiApiKey = process.env.OPENAI_API_KEY
+						break
+				}
+
+				expect(configuration[configKey]).toBe(process.env[envVar])
+			})
+		})
+	})
+})

packages/evals/src/cli/runTask.ts

@@ -302,14 +302,58 @@ export const runTask = async ({ run, task, publish, logger }: RunTaskOptions) =>
 		isClientDisconnected = true
 	})
 
+	// Build configuration with proper precedence:
+	// 1. Base EVALS_SETTINGS (without hardcoded apiProvider)
+	// 2. Environment-based API keys for backward compatibility
+	// 3. run.settings override everything (including apiProvider and keys)
+	const configuration: Record<string, unknown> = {
+		...EVALS_SETTINGS,
+	}
+
+	// Add environment API keys based on the provider if not overridden
+	if (!run.settings?.apiProvider) {
+		// Default to openrouter for backward compatibility if no provider specified
+		configuration.apiProvider = "openrouter"
+		configuration.openRouterApiKey = process.env.OPENROUTER_API_KEY
+	} else {
+		// Set API keys from environment based on the selected provider
+		const provider = run.settings.apiProvider
+		switch (provider) {
+			case "openrouter":
+				configuration.openRouterApiKey = process.env.OPENROUTER_API_KEY
+				break
+			case "anthropic":
+				configuration.apiKey = process.env.ANTHROPIC_API_KEY
+				break
+			case "openai":
+				configuration.openAiApiKey = process.env.OPENAI_API_KEY
+				break
+			case "gemini":
+				configuration.geminiApiKey = process.env.GEMINI_API_KEY
+				break
+			case "deepseek":
+				configuration.deepSeekApiKey = process.env.DEEPSEEK_API_KEY
+				break
+			case "mistral":
+				configuration.mistralApiKey = process.env.MISTRAL_API_KEY
+				break
+			case "groq":
+				configuration.groqApiKey = process.env.GROQ_API_KEY
+				break
+			case "litellm":
+				configuration.litellmApiKey = process.env.LITELLM_API_KEY
+				break
+			// Add more providers as needed
+		}
+	}
+
+	// Override with run.settings (highest priority)
+	Object.assign(configuration, run.settings)
+
 	client.sendCommand({
 		commandName: TaskCommandName.StartNewTask,
 		data: {
-			configuration: {
-				...EVALS_SETTINGS,
-				openRouterApiKey: process.env.OPENROUTER_API_KEY,
-				...run.settings, // Allow the provided settings to override `openRouterApiKey`.
-			},
+			configuration,
 			text: prompt,
 		},
 	})

packages/types/src/__tests__/global-settings.test.ts

@@ -0,0 +1,19 @@
+import { describe, it, expect } from "vitest"
+import { EVALS_SETTINGS } from "../global-settings.js"
+
+describe("EVALS_SETTINGS", () => {
+	it("should not have a hardcoded apiProvider", () => {
+		// EVALS_SETTINGS should not have apiProvider set by default
+		// to allow flexibility in choosing providers
+		expect(EVALS_SETTINGS.apiProvider).toBeUndefined()
+	})
+
+	it("should have other necessary settings", () => {
+		// Verify that other important settings are still present
+		expect(EVALS_SETTINGS.autoApprovalEnabled).toBe(true)
+		expect(EVALS_SETTINGS.alwaysAllowWrite).toBe(true)
+		expect(EVALS_SETTINGS.alwaysAllowExecute).toBe(true)
+		expect(EVALS_SETTINGS.mode).toBe("code")
+		expect(EVALS_SETTINGS.openRouterUseMiddleOutTransform).toBe(false)
+	})
+})

packages/types/src/global-settings.ts

@@ -219,7 +219,8 @@ export const isGlobalStateKey = (key: string): key is Keys<GlobalState> =>
 
 // Default settings when running evals (unless overridden).
 export const EVALS_SETTINGS: RooCodeSettings = {
-	apiProvider: "openrouter",
+	// apiProvider is intentionally not set here to allow flexibility
+	// It should be provided via run.settings or environment configuration
 	openRouterUseMiddleOutTransform: false,
 
 	lastShownAnnouncementId: "jul-09-2025-3-23-0",