From beee73db917f905e387eba71556f3f40b2fa166c Mon Sep 17 00:00:00 2001 From: Roo Code Date: Thu, 7 Aug 2025 16:01:17 +0000 Subject: [PATCH] feat: enable custom API providers for Roo Code Evals - Remove hardcoded openrouter provider from EVALS_SETTINGS - Update runTask.ts to support multiple API providers with proper environment variable mapping - Fix web-evals UI to properly set apiProvider when using OpenRouter mode - Add tests to verify provider configuration flexibility - Maintain backward compatibility with OpenRouter as default when no provider specified Fixes #6817 --- apps/web-evals/src/app/runs/new/new-run.tsx | 7 +- .../src/cli/__tests__/provider-config.test.ts | 121 ++++++++++++++++++ packages/evals/src/cli/runTask.ts | 54 +++++++- .../src/__tests__/global-settings.test.ts | 19 +++ packages/types/src/global-settings.ts | 3 +- 5 files changed, 197 insertions(+), 7 deletions(-) create mode 100644 packages/evals/src/cli/__tests__/provider-config.test.ts create mode 100644 packages/types/src/__tests__/global-settings.test.ts diff --git a/apps/web-evals/src/app/runs/new/new-run.tsx b/apps/web-evals/src/app/runs/new/new-run.tsx index f8633611b6..fc5e99b056 100644 --- a/apps/web-evals/src/app/runs/new/new-run.tsx +++ b/apps/web-evals/src/app/runs/new/new-run.tsx @@ -101,7 +101,12 @@ export function NewRun() { async (values: CreateRun) => { try { if (mode === "openrouter") { - values.settings = { ...(values.settings || {}), openRouterModelId: model } + // Ensure apiProvider is set along with the model ID + values.settings = { + ...(values.settings || {}), + apiProvider: "openrouter", + openRouterModelId: model, + } } const { id } = await createRun({ ...values, systemPrompt }) diff --git a/packages/evals/src/cli/__tests__/provider-config.test.ts b/packages/evals/src/cli/__tests__/provider-config.test.ts new file mode 100644 index 0000000000..d445e91fe5 --- /dev/null +++ b/packages/evals/src/cli/__tests__/provider-config.test.ts @@ -0,0 +1,121 @@ +import { describe, it, expect, vi, beforeEach } from "vitest" +import { EVALS_SETTINGS } from "@roo-code/types" + +describe("Provider Configuration", () => { + describe("EVALS_SETTINGS", () => { + it("should not have a hardcoded apiProvider", () => { + // EVALS_SETTINGS should not have apiProvider set by default + // to allow flexibility in choosing providers + expect(EVALS_SETTINGS.apiProvider).toBeUndefined() + }) + + it("should have other necessary settings", () => { + // Verify that other important settings are still present + expect(EVALS_SETTINGS.autoApprovalEnabled).toBe(true) + expect(EVALS_SETTINGS.alwaysAllowWrite).toBe(true) + expect(EVALS_SETTINGS.alwaysAllowExecute).toBe(true) + expect(EVALS_SETTINGS.mode).toBe("code") + }) + }) + + describe("Configuration Merging", () => { + it("should allow overriding apiProvider through run settings", () => { + const runSettings = { + apiProvider: "anthropic" as const, + apiKey: "test-key", + } + + const mergedConfig = { + ...EVALS_SETTINGS, + ...runSettings, + } + + expect(mergedConfig.apiProvider).toBe("anthropic") + expect(mergedConfig.apiKey).toBe("test-key") + }) + + it("should support multiple providers", () => { + const providers = [ + { apiProvider: "openrouter" as const, openRouterApiKey: "key1" }, + { apiProvider: "anthropic" as const, apiKey: "key2" }, + { apiProvider: "openai" as const, openAiApiKey: "key3" }, + { apiProvider: "gemini" as const, geminiApiKey: "key4" }, + { apiProvider: "ollama" as const, ollamaModelId: "model1" }, + { apiProvider: "litellm" as const, litellmApiKey: "key5" }, + ] + + providers.forEach((providerSettings) => { + const mergedConfig = { + ...EVALS_SETTINGS, + ...providerSettings, + } + + expect(mergedConfig.apiProvider).toBe(providerSettings.apiProvider) + }) + }) + + it("should maintain backward compatibility with openrouter as default", () => { + // When no apiProvider is specified in run settings, + // the implementation should default to openrouter for backward compatibility + const runSettings: Record = {} + + // This simulates the logic in runTask.ts + const configuration: Record = { + ...EVALS_SETTINGS, + } + + if (!runSettings.apiProvider) { + configuration.apiProvider = "openrouter" + configuration.openRouterApiKey = process.env.OPENROUTER_API_KEY + } + + expect(configuration.apiProvider).toBe("openrouter") + }) + }) + + describe("Environment Variable Mapping", () => { + beforeEach(() => { + // Clear environment variables + vi.stubEnv("OPENROUTER_API_KEY", "") + vi.stubEnv("ANTHROPIC_API_KEY", "") + vi.stubEnv("OPENAI_API_KEY", "") + vi.stubEnv("GEMINI_API_KEY", "") + vi.stubEnv("DEEPSEEK_API_KEY", "") + vi.stubEnv("MISTRAL_API_KEY", "") + vi.stubEnv("GROQ_API_KEY", "") + vi.stubEnv("LITELLM_API_KEY", "") + }) + + it("should map environment variables based on provider", () => { + // Set test environment variables + vi.stubEnv("OPENROUTER_API_KEY", "test-openrouter-key") + vi.stubEnv("ANTHROPIC_API_KEY", "test-anthropic-key") + vi.stubEnv("OPENAI_API_KEY", "test-openai-key") + + const providerKeyMappings = [ + { provider: "openrouter", envVar: "OPENROUTER_API_KEY", configKey: "openRouterApiKey" }, + { provider: "anthropic", envVar: "ANTHROPIC_API_KEY", configKey: "apiKey" }, + { provider: "openai", envVar: "OPENAI_API_KEY", configKey: "openAiApiKey" }, + ] + + providerKeyMappings.forEach(({ provider, envVar, configKey }) => { + const configuration: Record = {} + + // Simulate the switch logic from runTask.ts + switch (provider) { + case "openrouter": + configuration.openRouterApiKey = process.env.OPENROUTER_API_KEY + break + case "anthropic": + configuration.apiKey = process.env.ANTHROPIC_API_KEY + break + case "openai": + configuration.openAiApiKey = process.env.OPENAI_API_KEY + break + } + + expect(configuration[configKey]).toBe(process.env[envVar]) + }) + }) + }) +}) diff --git a/packages/evals/src/cli/runTask.ts b/packages/evals/src/cli/runTask.ts index 8b986e2afa..9014f9138e 100644 --- a/packages/evals/src/cli/runTask.ts +++ b/packages/evals/src/cli/runTask.ts @@ -302,14 +302,58 @@ export const runTask = async ({ run, task, publish, logger }: RunTaskOptions) => isClientDisconnected = true }) + // Build configuration with proper precedence: + // 1. Base EVALS_SETTINGS (without hardcoded apiProvider) + // 2. Environment-based API keys for backward compatibility + // 3. run.settings override everything (including apiProvider and keys) + const configuration: Record = { + ...EVALS_SETTINGS, + } + + // Add environment API keys based on the provider if not overridden + if (!run.settings?.apiProvider) { + // Default to openrouter for backward compatibility if no provider specified + configuration.apiProvider = "openrouter" + configuration.openRouterApiKey = process.env.OPENROUTER_API_KEY + } else { + // Set API keys from environment based on the selected provider + const provider = run.settings.apiProvider + switch (provider) { + case "openrouter": + configuration.openRouterApiKey = process.env.OPENROUTER_API_KEY + break + case "anthropic": + configuration.apiKey = process.env.ANTHROPIC_API_KEY + break + case "openai": + configuration.openAiApiKey = process.env.OPENAI_API_KEY + break + case "gemini": + configuration.geminiApiKey = process.env.GEMINI_API_KEY + break + case "deepseek": + configuration.deepSeekApiKey = process.env.DEEPSEEK_API_KEY + break + case "mistral": + configuration.mistralApiKey = process.env.MISTRAL_API_KEY + break + case "groq": + configuration.groqApiKey = process.env.GROQ_API_KEY + break + case "litellm": + configuration.litellmApiKey = process.env.LITELLM_API_KEY + break + // Add more providers as needed + } + } + + // Override with run.settings (highest priority) + Object.assign(configuration, run.settings) + client.sendCommand({ commandName: TaskCommandName.StartNewTask, data: { - configuration: { - ...EVALS_SETTINGS, - openRouterApiKey: process.env.OPENROUTER_API_KEY, - ...run.settings, // Allow the provided settings to override `openRouterApiKey`. - }, + configuration, text: prompt, }, }) diff --git a/packages/types/src/__tests__/global-settings.test.ts b/packages/types/src/__tests__/global-settings.test.ts new file mode 100644 index 0000000000..d823af6b12 --- /dev/null +++ b/packages/types/src/__tests__/global-settings.test.ts @@ -0,0 +1,19 @@ +import { describe, it, expect } from "vitest" +import { EVALS_SETTINGS } from "../global-settings.js" + +describe("EVALS_SETTINGS", () => { + it("should not have a hardcoded apiProvider", () => { + // EVALS_SETTINGS should not have apiProvider set by default + // to allow flexibility in choosing providers + expect(EVALS_SETTINGS.apiProvider).toBeUndefined() + }) + + it("should have other necessary settings", () => { + // Verify that other important settings are still present + expect(EVALS_SETTINGS.autoApprovalEnabled).toBe(true) + expect(EVALS_SETTINGS.alwaysAllowWrite).toBe(true) + expect(EVALS_SETTINGS.alwaysAllowExecute).toBe(true) + expect(EVALS_SETTINGS.mode).toBe("code") + expect(EVALS_SETTINGS.openRouterUseMiddleOutTransform).toBe(false) + }) +}) diff --git a/packages/types/src/global-settings.ts b/packages/types/src/global-settings.ts index f5e9fc32bd..e1de06acdc 100644 --- a/packages/types/src/global-settings.ts +++ b/packages/types/src/global-settings.ts @@ -219,7 +219,8 @@ export const isGlobalStateKey = (key: string): key is Keys => // Default settings when running evals (unless overridden). export const EVALS_SETTINGS: RooCodeSettings = { - apiProvider: "openrouter", + // apiProvider is intentionally not set here to allow flexibility + // It should be provided via run.settings or environment configuration openRouterUseMiddleOutTransform: false, lastShownAnnouncementId: "jul-09-2025-3-23-0",