Evals fixes (#4599)

packages/evals/src/cli/runTask.ts

@@ -294,8 +294,8 @@ export const runTask = async ({ run, task, publish, logger }: RunTaskOptions) =>
 		data: {
 			configuration: {
 				...EVALS_SETTINGS,
-				...run.settings,
 				openRouterApiKey: process.env.OPENROUTER_API_KEY,
+				...run.settings, // Allow the provided settings to override `openRouterApiKey`.
 			},
 			text: prompt,
 			newTab: true,

packages/types/src/experiment.ts

@@ -6,7 +6,13 @@ import type { Keys, Equals, AssertEqual } from "./type-fu.js"
  * ExperimentId
  */
 
-export const experimentIds = ["powerSteering", "concurrentFileReads", "disableCompletionCommand", "marketplace", "multiFileApplyDiff"] as const
+export const experimentIds = [
+	"powerSteering",
+	"concurrentFileReads",
+	"disableCompletionCommand",
+	"marketplace",
+	"multiFileApplyDiff",
+] as const
 
 export const experimentIdsSchema = z.enum(experimentIds)
 
@@ -17,11 +23,11 @@ export type ExperimentId = z.infer<typeof experimentIdsSchema>
  */
 
 export const experimentsSchema = z.object({
-	powerSteering: z.boolean(),
-	marketplace: z.boolean(),
-	concurrentFileReads: z.boolean(),
-	disableCompletionCommand: z.boolean(),
-	multiFileApplyDiff: z.boolean(),
+	powerSteering: z.boolean().optional(),
+	concurrentFileReads: z.boolean().optional(),
+	disableCompletionCommand: z.boolean().optional(),
+	marketplace: z.boolean().optional(),
+	multiFileApplyDiff: z.boolean().optional(),
 })
 
 export type Experiments = z.infer<typeof experimentsSchema>

src/shared/ExtensionMessage.ts

@@ -5,7 +5,7 @@ import type {
 	HistoryItem,
 	ModeConfig,
 	TelemetrySetting,
-	ExperimentId,
+	Experiments,
 	ClineMessage,
 	OrganizationAllowList,
 	CloudUserInfo,
@@ -208,7 +208,7 @@ export type ExtensionState = Pick<
 	showRooIgnoredFiles: boolean // Whether to show .rooignore'd files in listings
 	maxReadFileLine: number // Maximum number of lines to read from a file before truncating
 
-	experiments: Record<ExperimentId, boolean> // Map of experiment IDs to their enabled state
+	experiments: Experiments // Map of experiment IDs to their enabled state
 
 	mcpEnabled: boolean
 	enableMcpServerCreation: boolean

src/shared/experiments.ts

@@ -1,4 +1,4 @@
-import type { AssertEqual, Equals, Keys, Values, ExperimentId } from "@roo-code/types"
+import type { AssertEqual, Equals, Keys, Values, ExperimentId, Experiments } from "@roo-code/types"
 
 export const EXPERIMENT_IDS = {
 	MARKETPLACE: "marketplace",
@@ -33,6 +33,5 @@ export const experimentDefault = Object.fromEntries(
 
 export const experiments = {
 	get: (id: ExperimentKey): ExperimentConfig | undefined => experimentConfigsMap[id],
-	isEnabled: (experimentsConfig: Record<ExperimentId, boolean>, id: ExperimentId) =>
-		experimentsConfig[id] ?? experimentDefault[id],
+	isEnabled: (experimentsConfig: Experiments, id: ExperimentId) => experimentsConfig[id] ?? experimentDefault[id],
 } as const

webview-ui/src/components/settings/ExperimentalSettings.tsx

@@ -1,7 +1,7 @@
 import { HTMLAttributes } from "react"
 import { FlaskConical } from "lucide-react"
 
-import type { ExperimentId, CodebaseIndexConfig, CodebaseIndexModels, ProviderSettings } from "@roo-code/types"
+import type { Experiments, CodebaseIndexConfig, CodebaseIndexModels, ProviderSettings } from "@roo-code/types"
 
 import { EXPERIMENT_IDS, experimentConfigsMap } from "@roo/experiments"
 
@@ -17,7 +17,7 @@ import { CodeIndexSettings } from "./CodeIndexSettings"
 import { ConcurrentFileReadsExperiment } from "./ConcurrentFileReadsExperiment"
 
 type ExperimentalSettingsProps = HTMLAttributes<HTMLDivElement> & {
-	experiments: Record<ExperimentId, boolean>
+	experiments: Experiments
 	setExperimentEnabled: SetExperimentEnabled
 	maxConcurrentFileReads?: number
 	setCachedStateField: SetCachedStateField<"codebaseIndexConfig" | "maxConcurrentFileReads">