More progress

2026-09-06 08:18:39 +00:00 · 2025-03-27 13:11:27 -07:00 · 2025-03-27 13:11:27 -07:00 · 9048ad83f3
commit 9048ad83f3
parent edd3b429c3
30 changed files with 685 additions and 220 deletions
--- a/benchmark/apps/web/package.json
+++ b/benchmark/apps/web/package.json
@ -22,6 +22,7 @@
 		"@radix-ui/react-separator": "^1.1.2",
 		"@radix-ui/react-slot": "^1.1.2",
 		"@radix-ui/react-tabs": "^1.1.3",
+		"@radix-ui/react-tooltip": "^1.1.8",
 		"@tanstack/react-query": "^5.69.0",
 		"class-variance-authority": "^0.7.1",
 		"clsx": "^2.1.1",
--- a/benchmark/apps/web/src/app/globals.css
+++ b/benchmark/apps/web/src/app/globals.css
@ -48,14 +48,14 @@
 	--popover-foreground: oklch(0.985 0 0);
 	--primary: oklch(29.33% 0.0295 276.18);
 	--primary-foreground: var(--accent);
-	--secondary: oklch(0.269 0 0);
-	--secondary-foreground: oklch(0.985 0 0);
+	--secondary: var(--primary);
+	--secondary-foreground: var(--foreground);
 	--muted: oklch(28.27% 0.0207 273.06);
-	--muted-foreground: oklch(75.15% 0.0477 278.41);
+	--muted-foreground: oklch(75.15% 0.0477 278.41 / 75%);
 	--accent: oklch(70.21% 0.1813 328.71);
-	--accent-foreground: rgba(0, 0, 0, 0.5);
+	--accent-foreground: oklch(1 0 0 / 75%);
 	--destructive: oklch(72.14% 0.1616 15.49);
-	--border: oklch(29.33% 0.0295 276.18);
+	--border: var(--primary);
 	--input: var(--primary);
 	--ring: oklch(83.63% 0.1259 176.52);
 	--chart-1: oklch(0.488 0.243 264.376);
--- a/benchmark/apps/web/src/app/runs/new/new-run.tsx
+++ b/benchmark/apps/web/src/app/runs/new/new-run.tsx
@ -1,11 +1,13 @@
 "use client"

-import { useCallback, useRef, useState } from "react"
+import { Fragment, useCallback, useRef, useState } from "react"
 import { useRouter } from "next/navigation"
 import { useForm, FormProvider } from "react-hook-form"
 import { zodResolver } from "@hookform/resolvers/zod"
 import fuzzysort from "fuzzysort"
-import { X, Rocket, Check, ChevronsUpDown } from "lucide-react"
+import { X, Rocket, Check, ChevronsUpDown, HardDriveUpload, CircleCheck } from "lucide-react"
+
+import { ExperimentId, Experiments, globalSettingsSchema, rooCodeDefaults } from "@benchmark/types"

 import { createRun } from "@/lib/server/runs"
 import { createRunSchema as formSchema, type CreateRun as FormValues } from "@/lib/schemas"
@ -18,6 +20,7 @@ import {
 	FormField,
 	FormItem,
 	FormLabel,
+	FormDescription,
 	FormMessage,
 	Textarea,
 	Tabs,
@ -33,7 +36,16 @@ import {
 	Popover,
 	PopoverContent,
 	PopoverTrigger,
+	ScrollArea,
 } from "@/components/ui"
+import { z } from "zod"
+import { SettingsDiff } from "./settings-diff"
+
+const recommendedModels = [
+	"anthropic/claude-3.7-sonnet",
+	"anthropic/claude-3.7-sonnet:thinking",
+	"google/gemini-2.0-flash-001",
+]

 export function NewRun() {
 	const router = useRouter()
@ -53,38 +65,33 @@ export function NewRun() {
 			description: "",
 			suite: "full",
 			exercises: [],
+			settings: undefined,
 		},
 	})

 	const {
 		setValue,
+		setError,
+		clearErrors,
 		watch,
 		formState: { isSubmitting },
 	} = form

-	const [model, suite] = watch(["model", "suite"])
-
-	const selectModel = useCallback(
-		(model: string) => {
-			setValue("model", model)
-			setModelPopoverOpen(false)
-		},
-		[setValue],
-	)
+	const [model, suite, settings] = watch(["model", "suite", "settings"])

 	const onSubmit = useCallback(
 		async (data: FormValues) => {
 			try {
 				const { id } = await createRun(data)
 				router.push(`/runs/${id}`)
-			} catch (_) {
+			} catch (_e) {
 				// Surface error.
 			}
 		},
 		[router],
 	)

-	const onFilter = useCallback(
+	const onFilterModels = useCallback(
 		(value: string, search: string) => {
 			if (modelSearchValueRef.current !== search) {
 				modelSearchValueRef.current = search
@ -105,16 +112,41 @@ export function NewRun() {
 		[models.data],
 	)

-	const recommendedModels = [
-		"anthropic/claude-3.7-sonnet",
-		"anthropic/claude-3.7-sonnet:thinking",
-		"google/gemini-2.0-flash-001",
-	]
+	const onSelectModel = useCallback(
+		(model: string) => {
+			setValue("model", model)
+			setModelPopoverOpen(false)
+		},
+		[setValue],
+	)
+
+	const onImportSettings = useCallback(
+		async (event: React.ChangeEvent<HTMLInputElement>) => {
+			const file = event.target.files?.[0]
+
+			if (!file) {
+				return
+			}
+
+			clearErrors("settings")
+
+			try {
+				const result = z.object({ globalSettings: globalSettingsSchema }).parse(JSON.parse(await file.text()))
+				setValue("settings", result.globalSettings)
+				event.target.value = ""
+			} catch (_error) {
+				setError("settings", { message: "Error parsing JSON file. Please check the file format." })
+			}
+		},
+		[clearErrors, setError, setValue],
+	)

 	return (
 		<>
 			<FormProvider {...form}>
-				<form onSubmit={form.handleSubmit(onSubmit)} className="flex flex-col justify-center gap-6">
+				<form
+					onSubmit={form.handleSubmit(onSubmit)}
+					className="flex flex-col justify-center divide-y divide-primary *:py-5">
 					<FormField
 						control={form.control}
 						name="model"
@ -135,7 +167,7 @@ export function NewRun() {
 										</Button>
 									</PopoverTrigger>
 									<PopoverContent className="p-0 w-[var(--radix-popover-trigger-width)]">
-										<Command filter={onFilter}>
+										<Command filter={onFilterModels}>
 											<CommandInput
 												placeholder="Search"
 												value={modelSearchValue}
@ -146,7 +178,7 @@ export function NewRun() {
 												<CommandEmpty>No model found.</CommandEmpty>
 												<CommandGroup>
 													{models.data?.map(({ id, name }) => (
-														<CommandItem key={id} value={id} onSelect={selectModel}>
+														<CommandItem key={id} value={id} onSelect={onSelectModel}>
 															{name}
 															<Check
 																className={cn(
@ -162,51 +194,77 @@ export function NewRun() {
 									</PopoverContent>
 								</Popover>
 								<FormMessage />
-								<div className="flex flex-wrap items-center gap-2 text-sm">
-									<div>Recommended:</div>
+								<FormDescription className="flex flex-wrap items-center gap-2">
+									<span>Recommended:</span>
 									{recommendedModels.map((modelId) => (
-										<div key={modelId} className="flex items-center gap-2">
-											<Button
-												variant="link"
-												className="break-all px-0!"
-												onClick={(e) => {
-													e.preventDefault()
-													setValue("model", modelId)
-												}}>
-												{modelId}
-											</Button>
-										</div>
+										<Button
+											key={modelId}
+											variant="link"
+											className="break-all px-0!"
+											onClick={(e) => {
+												e.preventDefault()
+												setValue("model", modelId)
+											}}>
+											{modelId}
+										</Button>
 									))}
-								</div>
+								</FormDescription>
 							</FormItem>
 						)}
 					/>

+					<FormItem>
+						<FormLabel>Import Settings</FormLabel>
+						<Button
+							type="button"
+							variant="secondary"
+							size="icon"
+							onClick={() => document.getElementById("json-upload")?.click()}>
+							<HardDriveUpload />
+						</Button>
+						<input
+							id="json-upload"
+							type="file"
+							accept="application/json"
+							className="hidden"
+							onChange={onImportSettings}
+						/>
+						{settings ? (
+							<ScrollArea className="max-h-64 border rounded-sm">
+								<>
+									<div className="flex items-center gap-1 p-2 border-b">
+										<CircleCheck className="size-4 text-ring" />
+										<div className="text-sm">
+											Imported valid Roo Code settings. Showing differences from default settings.
+										</div>
+									</div>
+									<SettingsDiff defaultSettings={rooCodeDefaults} customSettings={settings} />
+								</>
+							</ScrollArea>
+						) : (
+							<FormDescription>
+								Fully configure how Roo Code for this run using a settings file that was exported by Roo
+								Code.
+							</FormDescription>
+						)}
+						<FormMessage />
+					</FormItem>
+
 					<FormField
 						control={form.control}
 						name="suite"
 						render={() => (
 							<FormItem>
-								<FormLabel>Exercise Suite</FormLabel>
+								<FormLabel>Exercises</FormLabel>
 								<Tabs
 									defaultValue="full"
 									onValueChange={(value) => setValue("suite", value as "full" | "partial")}>
-									<TabsList className="grid w-full grid-cols-2">
-										<TabsTrigger value="full">Full</TabsTrigger>
-										<TabsTrigger value="partial">Partial</TabsTrigger>
+									<TabsList>
+										<TabsTrigger value="full">All</TabsTrigger>
+										<TabsTrigger value="partial">Some</TabsTrigger>
 									</TabsList>
 								</Tabs>
-							</FormItem>
-						)}
-					/>
-
-					{suite === "partial" && (
-						<FormField
-							control={form.control}
-							name="exercises"
-							render={() => (
-								<FormItem>
-									<FormLabel>Exercises</FormLabel>
+								{suite === "partial" && (
 									<MultiSelect
 										options={exercises.data?.map((path) => ({ value: path, label: path })) || []}
 										onValueChange={(value) => setValue("exercises", value)}
@ -214,18 +272,18 @@ export function NewRun() {
 										variant="inverted"
 										maxCount={4}
 									/>
-									<FormMessage />
-								</FormItem>
-							)}
-						/>
-					)}
+								)}
+								<FormMessage />
+							</FormItem>
+						)}
+					/>

 					<FormField
 						control={form.control}
 						name="description"
 						render={({ field }) => (
 							<FormItem>
-								<FormLabel>Description</FormLabel>
+								<FormLabel>Description / Notes</FormLabel>
 								<FormControl>
 									<Textarea placeholder="Optional" {...field} />
 								</FormControl>
@ -234,10 +292,12 @@ export function NewRun() {
 						)}
 					/>

-					<Button type="submit" disabled={isSubmitting}>
-						<Rocket className="size-4" />
-						Launch
-					</Button>
+					<div className="flex justify-end">
+						<Button size="lg" type="submit" disabled={isSubmitting}>
+							<Rocket className="size-4" />
+							Launch
+						</Button>
+					</div>
 				</form>
 			</FormProvider>
 			<Button
--- a/benchmark/apps/web/src/app/runs/new/settings-diff.tsx
+++ b/benchmark/apps/web/src/app/runs/new/settings-diff.tsx
@ -0,0 +1,57 @@
+import { Fragment, HTMLAttributes } from "react"
+
+import { RooCodeSettings } from "@benchmark/types"
+
+import { cn } from "@/lib/utils"
+
+type SettingsDiffProps = HTMLAttributes<HTMLDivElement> & {
+	defaultSettings: RooCodeSettings
+	customSettings: RooCodeSettings
+}
+
+export function SettingsDiff({
+	customSettings: { experiments: customExperiments, ...customSettings },
+	defaultSettings: { experiments: defaultExperiments, ...defaultSettings },
+	className,
+	...props
+}: SettingsDiffProps) {
+	const defaults = { ...defaultSettings, ...defaultExperiments }
+	const custom = { ...customSettings, ...customExperiments }
+
+	return (
+		<div className={cn("grid grid-cols-3 gap-2 text-sm p-2", className)} {...props}>
+			<div className="font-medium text-muted-foreground">Setting</div>
+			<div className="font-medium text-muted-foreground">Default</div>
+			<div className="font-medium text-muted-foreground">Custom</div>
+			{Object.entries(defaults).flatMap(([key, defaultValue]) => {
+				const customValue = custom[key as keyof typeof custom]
+				const isDefault = JSON.stringify(defaultValue) === JSON.stringify(customValue)
+
+				return isDefault ? null : (
+					<SetttingDiff
+						key={key}
+						name={key}
+						defaultValue={JSON.stringify(defaultValue, null, 2)}
+						customValue={JSON.stringify(customValue, null, 2)}
+					/>
+				)
+			})}
+		</div>
+	)
+}
+
+type SettingDiffProps = HTMLAttributes<HTMLDivElement> & {
+	name: string
+	defaultValue?: string
+	customValue?: string
+}
+
+export function SetttingDiff({ name, defaultValue, customValue, ...props }: SettingDiffProps) {
+	return (
+		<Fragment {...props}>
+			<div className="overflow-hidden font-mono">{name}</div>
+			<pre className="inline text-rose-500 line-through">{defaultValue}</pre>
+			<pre className="inline text-teal-500">{customValue}</pre>
+		</Fragment>
+	)
+}
--- a/benchmark/apps/web/src/components/ui/button.tsx
+++ b/benchmark/apps/web/src/components/ui/button.tsx
@ -9,20 +9,20 @@ const buttonVariants = cva(
 	{
 		variants: {
 			variant: {
-				default: "bg-primary text-primary-foreground shadow-xs",
+				default: "bg-primary text-primary-foreground shadow-xs [&_svg]:text-accent",
 				destructive:
 					"bg-destructive text-white shadow-xs focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60",
 				outline:
 					"border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input",
-				secondary: "bg-secondary text-secondary-foreground shadow-xs",
-				ghost: "hover:bg-accent hover:text-accent-foreground",
+				secondary: "bg-secondary text-secondary-foreground shadow-xs [&_svg]:text-ring",
+				ghost: "hover:bg-primary hover:text-primary-foreground",
 				link: "text-accent underline-offset-4 hover:underline h-4! px-1! rounded-none",
 				input: "bg-input text-input-foreground active:scale-100 shadow-xs",
 			},
 			size: {
 				default: "h-9 px-4 py-2 has-[>svg]:px-3",
-				sm: "h-8 gap-1.5 px-3 has-[>svg]:px-2.5",
-				lg: "h-10 px-6 has-[>svg]:px-4",
+				sm: "h-8 gap-1.5 px-3 has-[>svg]:px-2.5 text-sm",
+				lg: "h-10 px-6 has-[>svg]:px-4 text-lg",
 				icon: "size-9",
 			},
 		},
--- a/benchmark/apps/web/src/components/ui/index.ts
+++ b/benchmark/apps/web/src/components/ui/index.ts
@ -14,3 +14,4 @@ export * from "./separator"
 export * from "./table"
 export * from "./tabs"
 export * from "./textarea"
+export * from "./tooltip"
--- a/benchmark/apps/web/src/components/ui/tooltip.tsx
+++ b/benchmark/apps/web/src/components/ui/tooltip.tsx
@ -0,0 +1,47 @@
+"use client"
+
+import * as React from "react"
+import * as TooltipPrimitive from "@radix-ui/react-tooltip"
+
+import { cn } from "@/lib/utils"
+
+function TooltipProvider({ delayDuration = 0, ...props }: React.ComponentProps<typeof TooltipPrimitive.Provider>) {
+	return <TooltipPrimitive.Provider data-slot="tooltip-provider" delayDuration={delayDuration} {...props} />
+}
+
+function Tooltip({ ...props }: React.ComponentProps<typeof TooltipPrimitive.Root>) {
+	return (
+		<TooltipProvider>
+			<TooltipPrimitive.Root data-slot="tooltip" {...props} />
+		</TooltipProvider>
+	)
+}
+
+function TooltipTrigger({ ...props }: React.ComponentProps<typeof TooltipPrimitive.Trigger>) {
+	return <TooltipPrimitive.Trigger data-slot="tooltip-trigger" {...props} />
+}
+
+function TooltipContent({
+	className,
+	sideOffset = 0,
+	children,
+	...props
+}: React.ComponentProps<typeof TooltipPrimitive.Content>) {
+	return (
+		<TooltipPrimitive.Portal>
+			<TooltipPrimitive.Content
+				data-slot="tooltip-content"
+				sideOffset={sideOffset}
+				className={cn(
+					"bg-primary text-primary-foreground animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-fit origin-(--radix-tooltip-content-transform-origin) rounded-sm px-3 py-1.5 text-xs text-balance",
+					className,
+				)}
+				{...props}>
+				{children}
+				<TooltipPrimitive.Arrow className="bg-primary fill-primary z-50 size-2.5 translate-y-[calc(-50%_-_2px)] rotate-45 rounded-[2px]" />
+			</TooltipPrimitive.Content>
+		</TooltipPrimitive.Portal>
+	)
+}
+
+export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider }
--- a/benchmark/apps/web/src/lib/schemas.ts
+++ b/benchmark/apps/web/src/lib/schemas.ts
@ -1,5 +1,7 @@
 import { z } from "zod"

+import { globalSettingsSchema } from "@benchmark/types"
+
 /**
 * CreateRun
 */
@ -10,6 +12,7 @@ export const createRunSchema = z
 		description: z.string().optional(),
 		suite: z.enum(["full", "partial"]),
 		exercises: z.array(z.string()).optional(),
+		settings: globalSettingsSchema.optional(),
 	})
 	.refine((data) => data.suite === "full" || (data.exercises || []).length > 0, {
 		message: "Exercises are required when running a partial suite.",
--- a/benchmark/packages/db/README.md
+++ b/benchmark/packages/db/README.md
@ -0,0 +1,15 @@
+## Running Migrations
+
+Update `src/schema.ts` as needed, and then run:
+
+```sh
+pnpm db:generate
+```
+
+Inspect the generated sql in the migration filed added to `drizzle/`.
+
+If it looks okay, then run:
+
+```sh
+pnpm db:migrate
+```
--- a/benchmark/packages/db/drizzle/0001_lush_reavers.sql
+++ b/benchmark/packages/db/drizzle/0001_lush_reavers.sql
@ -0,0 +1 @@
+ALTER TABLE `runs` ADD `settings` blob;
--- a/benchmark/packages/db/drizzle/meta/0001_snapshot.json
+++ b/benchmark/packages/db/drizzle/meta/0001_snapshot.json
@ -0,0 +1,281 @@
+{
+	"version": "6",
+	"dialect": "sqlite",
+	"id": "8906647f-81d6-498a-897c-b1638c04c69a",
+	"prevId": "c0fa8491-b5c0-493d-aa32-ddf280259c30",
+	"tables": {
+		"runs": {
+			"name": "runs",
+			"columns": {
+				"id": {
+					"name": "id",
+					"type": "integer",
+					"primaryKey": true,
+					"notNull": true,
+					"autoincrement": true
+				},
+				"taskMetricsId": {
+					"name": "taskMetricsId",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
+				"model": {
+					"name": "model",
+					"type": "text",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"description": {
+					"name": "description",
+					"type": "text",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
+				"settings": {
+					"name": "settings",
+					"type": "blob",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
+				"pid": {
+					"name": "pid",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
+				"socketPath": {
+					"name": "socketPath",
+					"type": "text",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"passed": {
+					"name": "passed",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false,
+					"default": 0
+				},
+				"failed": {
+					"name": "failed",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false,
+					"default": 0
+				},
+				"createdAt": {
+					"name": "createdAt",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				}
+			},
+			"indexes": {},
+			"foreignKeys": {
+				"runs_taskMetricsId_taskMetrics_id_fk": {
+					"name": "runs_taskMetricsId_taskMetrics_id_fk",
+					"tableFrom": "runs",
+					"tableTo": "taskMetrics",
+					"columnsFrom": ["taskMetricsId"],
+					"columnsTo": ["id"],
+					"onDelete": "no action",
+					"onUpdate": "no action"
+				}
+			},
+			"compositePrimaryKeys": {},
+			"uniqueConstraints": {},
+			"checkConstraints": {}
+		},
+		"taskMetrics": {
+			"name": "taskMetrics",
+			"columns": {
+				"id": {
+					"name": "id",
+					"type": "integer",
+					"primaryKey": true,
+					"notNull": true,
+					"autoincrement": true
+				},
+				"tokensIn": {
+					"name": "tokensIn",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"tokensOut": {
+					"name": "tokensOut",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"tokensContext": {
+					"name": "tokensContext",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"cacheWrites": {
+					"name": "cacheWrites",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"cacheReads": {
+					"name": "cacheReads",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"cost": {
+					"name": "cost",
+					"type": "real",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"duration": {
+					"name": "duration",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"createdAt": {
+					"name": "createdAt",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				}
+			},
+			"indexes": {},
+			"foreignKeys": {},
+			"compositePrimaryKeys": {},
+			"uniqueConstraints": {},
+			"checkConstraints": {}
+		},
+		"tasks": {
+			"name": "tasks",
+			"columns": {
+				"id": {
+					"name": "id",
+					"type": "integer",
+					"primaryKey": true,
+					"notNull": true,
+					"autoincrement": true
+				},
+				"runId": {
+					"name": "runId",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"taskMetricsId": {
+					"name": "taskMetricsId",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
+				"language": {
+					"name": "language",
+					"type": "text",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"exercise": {
+					"name": "exercise",
+					"type": "text",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"passed": {
+					"name": "passed",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
+				"startedAt": {
+					"name": "startedAt",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
+				"finishedAt": {
+					"name": "finishedAt",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
+				"createdAt": {
+					"name": "createdAt",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				}
+			},
+			"indexes": {
+				"tasks_language_exercise_idx": {
+					"name": "tasks_language_exercise_idx",
+					"columns": ["runId", "language", "exercise"],
+					"isUnique": true
+				}
+			},
+			"foreignKeys": {
+				"tasks_runId_runs_id_fk": {
+					"name": "tasks_runId_runs_id_fk",
+					"tableFrom": "tasks",
+					"tableTo": "runs",
+					"columnsFrom": ["runId"],
+					"columnsTo": ["id"],
+					"onDelete": "no action",
+					"onUpdate": "no action"
+				},
+				"tasks_taskMetricsId_taskMetrics_id_fk": {
+					"name": "tasks_taskMetricsId_taskMetrics_id_fk",
+					"tableFrom": "tasks",
+					"tableTo": "taskMetrics",
+					"columnsFrom": ["taskMetricsId"],
+					"columnsTo": ["id"],
+					"onDelete": "no action",
+					"onUpdate": "no action"
+				}
+			},
+			"compositePrimaryKeys": {},
+			"uniqueConstraints": {},
+			"checkConstraints": {}
+		}
+	},
+	"views": {},
+	"enums": {},
+	"_meta": {
+		"schemas": {},
+		"tables": {},
+		"columns": {}
+	},
+	"internal": {
+		"indexes": {}
+	}
+}
--- a/benchmark/packages/db/drizzle/meta/_journal.json
+++ b/benchmark/packages/db/drizzle/meta/_journal.json
@ -8,6 +8,13 @@
 			"when": 1742599919625,
 			"tag": "0000_elite_raza",
 			"breakpoints": true
+		},
+		{
+			"idx": 1,
+			"version": "6",
+			"when": 1743089501047,
+			"tag": "0001_lush_reavers",
+			"breakpoints": true
 		}
 	]
 }
--- a/benchmark/packages/db/package.json
+++ b/benchmark/packages/db/package.json
@ -3,12 +3,12 @@
 	"private": true,
 	"type": "module",
 	"main": "./dist/index.cjs",
-	"module": "./dist/index.js",
-	"types": "./dist/index.d.ts",
+	"module": "./src/index.ts",
+	"types": "./src/index.ts",
 	"exports": {
 		"import": {
-			"types": "./dist/index.d.ts",
-			"import": "./dist/index.js"
+			"types": "./src/index.ts",
+			"import": "./src/index.ts"
 		},
 		"require": {
 			"types": "./dist/index.d.cts",
--- a/benchmark/packages/db/src/index.ts
+++ b/benchmark/packages/db/src/index.ts
@ -1,8 +1,5 @@
-export type { Run, InsertRun, UpdateRun } from "./schema.js"
+export * from "./schema.js"
+
 export * from "./queries/runs.js"
-
-export type { Task, InsertTask, UpdateTask } from "./schema.js"
 export * from "./queries/tasks.js"
-
-export type { TaskMetrics, InsertTaskMetrics, UpdateTaskMetrics } from "./schema.js"
 export * from "./queries/taskMetrics.js"
--- a/benchmark/packages/db/src/schema.ts
+++ b/benchmark/packages/db/src/schema.ts
@ -1,9 +1,8 @@
-import { sqliteTable, text, real, integer, uniqueIndex } from "drizzle-orm/sqlite-core"
+import { sqliteTable, text, real, integer, blob, uniqueIndex } from "drizzle-orm/sqlite-core"
 import { relations } from "drizzle-orm"
 import { createInsertSchema } from "drizzle-zod"
-import { z } from "zod"

-import { exerciseLanguages } from "@benchmark/types"
+import { GlobalSettings, exerciseLanguages, globalSettingsSchema } from "@benchmark/types"

 /**
 * runs
@ -14,6 +13,7 @@ export const runs = sqliteTable("runs", {
 	taskMetricsId: integer({ mode: "number" }).references(() => taskMetrics.id),
 	model: text().notNull(),
 	description: text(),
+	settings: blob({ mode: "json" }).$type<GlobalSettings>(),
 	pid: integer({ mode: "number" }),
 	socketPath: text().notNull(),
 	passed: integer({ mode: "number" }).default(0).notNull(),
@ -27,9 +27,11 @@ export const runsRelations = relations(runs, ({ one }) => ({

 export type Run = typeof runs.$inferSelect

-export const insertRunSchema = createInsertSchema(runs).omit({ id: true, createdAt: true })
+export const insertRunSchema = createInsertSchema(runs).omit({ id: true, createdAt: true, settings: true }).extend({
+	settings: globalSettingsSchema,
+})

-export type InsertRun = z.infer<typeof insertRunSchema>
+export type InsertRun = Omit<typeof runs.$inferInsert, "id" | "createdAt">

 export type UpdateRun = Partial<Omit<Run, "id" | "createdAt">>

@ -64,7 +66,7 @@ export type Task = typeof tasks.$inferSelect

 export const insertTaskSchema = createInsertSchema(tasks).omit({ id: true, createdAt: true })

-export type InsertTask = z.infer<typeof insertTaskSchema>
+export type InsertTask = Omit<typeof tasks.$inferInsert, "id" | "createdAt">

 export type UpdateTask = Partial<Omit<Task, "id" | "createdAt">>

@ -88,7 +90,7 @@ export type TaskMetrics = typeof taskMetrics.$inferSelect

 export const insertTaskMetricsSchema = createInsertSchema(taskMetrics).omit({ id: true, createdAt: true })

-export type InsertTaskMetrics = z.infer<typeof insertTaskMetricsSchema>
+export type InsertTaskMetrics = Omit<typeof taskMetrics.$inferInsert, "id" | "createdAt">

 export type UpdateTaskMetrics = Partial<Omit<TaskMetrics, "id" | "createdAt">>

--- a/benchmark/packages/db/tsup.config.ts
+++ b/benchmark/packages/db/tsup.config.ts
@ -1,20 +1,11 @@
-import { defineConfig, Options } from "tsup"
-
-const shared: Options = {
-	entry: ["src/index.ts"],
-	dts: true,
-	outDir: "dist",
-	clean: true,
-}
+import { defineConfig } from "tsup"

 export default defineConfig([
 	{
-		...shared,
-		format: ["esm"],
-		tsconfig: "./tsconfig.json",
-	},
-	{
-		...shared,
+		entry: ["src/index.ts"],
+		dts: true,
+		outDir: "dist",
+		clean: true,
 		format: ["cjs"],
 		tsconfig: "./tsconfig.cjs.json",
 		outExtension: () => ({ js: ".cjs" }),
--- a/benchmark/packages/ipc/package.json
+++ b/benchmark/packages/ipc/package.json
@ -3,12 +3,12 @@
 	"private": true,
 	"type": "module",
 	"main": "./dist/index.cjs",
-	"module": "./dist/index.js",
-	"types": "./dist/index.d.ts",
+	"module": "./src/index.ts",
+	"types": "./src/index.ts",
 	"exports": {
 		"import": {
-			"types": "./dist/index.d.ts",
-			"import": "./dist/index.js"
+			"types": "./src/index.ts",
+			"import": "./src/index.ts"
 		},
 		"require": {
 			"types": "./dist/index.d.cts",
--- a/benchmark/packages/ipc/tsup.config.ts
+++ b/benchmark/packages/ipc/tsup.config.ts
@ -1,20 +1,11 @@
-import { defineConfig, Options } from "tsup"
-
-const shared: Options = {
-	entry: ["src/index.ts"],
-	dts: true,
-	outDir: "dist",
-	clean: true,
-}
+import { defineConfig } from "tsup"

 export default defineConfig([
 	{
-		...shared,
-		format: ["esm"],
-		tsconfig: "./tsconfig.json",
-	},
-	{
-		...shared,
+		entry: ["src/index.ts"],
+		dts: true,
+		outDir: "dist",
+		clean: true,
 		format: ["cjs"],
 		tsconfig: "./tsconfig.cjs.json",
 		outExtension: () => ({ js: ".cjs" }),
--- a/benchmark/packages/lib/eslint.config.mjs
+++ b/benchmark/packages/lib/eslint.config.mjs
@ -1,4 +0,0 @@
-import { config } from "@benchmark/eslint-config/base"
-
-/** @type {import("eslint").Linter.Config} */
-export default [...config]
--- a/benchmark/packages/lib/package.json
+++ b/benchmark/packages/lib/package.json
@ -1,31 +0,0 @@
-{
-	"name": "@benchmark/lib",
-	"private": true,
-	"type": "module",
-	"main": "./dist/index.cjs",
-	"module": "./dist/index.js",
-	"types": "./dist/index.d.ts",
-	"exports": {
-		"import": {
-			"types": "./dist/index.d.ts",
-			"import": "./dist/index.js"
-		},
-		"require": {
-			"types": "./dist/index.d.cts",
-			"require": "./dist/index.cjs"
-		}
-	},
-	"scripts": {
-		"lint": "eslint src --ext ts --max-warnings=0",
-		"check-types": "tsc --noEmit",
-		"format": "prettier --write src",
-		"build": "tsup"
-	},
-	"dependencies": {
-		"zod": "^3.24.2"
-	},
-	"devDependencies": {
-		"@benchmark/eslint-config": "workspace:^",
-		"@benchmark/typescript-config": "workspace:^"
-	}
-}
--- a/benchmark/packages/lib/src/index.ts
+++ b/benchmark/packages/lib/src/index.ts
@ -1 +0,0 @@
-export const foo = () => "bar"
--- a/benchmark/packages/lib/tsconfig.cjs.json
+++ b/benchmark/packages/lib/tsconfig.cjs.json
@ -1,5 +0,0 @@
-{
-	"extends": "@benchmark/typescript-config/cjs.json",
-	"include": ["src"],
-	"exclude": ["node_modules"]
-}
--- a/benchmark/packages/lib/tsconfig.json
+++ b/benchmark/packages/lib/tsconfig.json
@ -1,5 +0,0 @@
-{
-	"extends": "@benchmark/typescript-config/base.json",
-	"include": ["src"],
-	"exclude": ["node_modules"]
-}
--- a/benchmark/packages/lib/tsup.config.ts
+++ b/benchmark/packages/lib/tsup.config.ts
@ -1,22 +0,0 @@
-import { defineConfig, Options } from "tsup"
-
-const shared: Options = {
-	entry: ["src/index.ts"],
-	dts: true,
-	outDir: "dist",
-	clean: true,
-}
-
-export default defineConfig([
-	{
-		...shared,
-		format: ["esm"],
-		tsconfig: "./tsconfig.json",
-	},
-	{
-		...shared,
-		format: ["cjs"],
-		tsconfig: "./tsconfig.cjs.json",
-		outExtension: () => ({ js: ".cjs" }),
-	},
-])
--- a/benchmark/packages/runner/src/index.ts
+++ b/benchmark/packages/runner/src/index.ts
@ -5,7 +5,7 @@ import * as vscode from "vscode"

 import { RooCodeAPI } from "../../../../src/exports/roo-code.js"

-import { IpcOrigin, IpcMessageType, TaskEventName } from "@benchmark/types"
+import { IpcOrigin, IpcMessageType, TaskEventName, rooCodeDefaults } from "@benchmark/types"
 import { IpcClient } from "@benchmark/ipc"
 import { findTask, findRun, createTaskMetrics, updateTask } from "@benchmark/db"

@ -58,27 +58,16 @@ export async function run() {
 	await waitUntilReady({ api })

 	/**
-	 * Configure Roo Code as needed.
-	 *
-	 * Use Claude 3.7 Sonnet via OpenRouter.
-	 * Don't require approval for anything.
-	 * Run any command without approval.
-	 * Disable checkpoints (for performance).
+	 * Configure Roo Code as needed. First apply defaults, then apply the
+	 * run-specific settings.
 	 */

 	await api.setConfiguration({
 		apiProvider: "openrouter",
 		openRouterApiKey,
 		openRouterModelId,
-		autoApprovalEnabled: true,
-		alwaysAllowReadOnly: true,
-		alwaysAllowWrite: true,
-		alwaysAllowExecute: true,
-		alwaysAllowBrowser: true,
-		alwaysApproveResubmit: true,
-		alwaysAllowMcp: true,
-		alwaysAllowModeSwitch: true,
-		enableCheckpoints: false,
+		...rooCodeDefaults,
+		...Object.fromEntries(Object.entries(run.settings ?? {}).filter(([, value]) => value !== undefined)),
 	})

 	await vscode.workspace
--- a/benchmark/packages/types/package.json
+++ b/benchmark/packages/types/package.json
@ -3,12 +3,12 @@
 	"private": true,
 	"type": "module",
 	"main": "./dist/index.cjs",
-	"module": "./dist/index.js",
-	"types": "./dist/index.d.ts",
+	"module": "./src/index.ts",
+	"types": "./src/index.ts",
 	"exports": {
 		"import": {
-			"types": "./dist/index.d.ts",
-			"import": "./dist/index.js"
+			"types": "./src/index.ts",
+			"import": "./src/index.ts"
 		},
 		"require": {
 			"types": "./dist/index.d.cts",
--- a/benchmark/packages/types/src/index.ts
+++ b/benchmark/packages/types/src/index.ts
@ -1,3 +1,4 @@
 export * from "./exercises.js"
 export * from "./ipc.js"
 export * from "./roo-code.js"
+export * from "./roo-code-defaults.js"
--- a/benchmark/packages/types/src/roo-code-defaults.ts
+++ b/benchmark/packages/types/src/roo-code-defaults.ts
@ -0,0 +1,61 @@
+import { GlobalSettings } from "./roo-code.js"
+
+export const rooCodeDefaults: GlobalSettings = {
+	pinnedApiConfigs: {},
+	lastShownAnnouncementId: "mar-20-2025-3-10",
+
+	autoApprovalEnabled: true,
+	alwaysAllowReadOnly: true,
+	alwaysAllowReadOnlyOutsideWorkspace: false,
+	alwaysAllowWrite: true,
+	alwaysAllowWriteOutsideWorkspace: false,
+	writeDelayMs: 200,
+	alwaysAllowBrowser: true,
+	alwaysApproveResubmit: true,
+	requestDelaySeconds: 5,
+	alwaysAllowMcp: true,
+	alwaysAllowModeSwitch: true,
+	alwaysAllowSubtasks: true,
+	alwaysAllowExecute: true,
+	allowedCommands: ["*"],
+
+	browserToolEnabled: false,
+	browserViewportSize: "900x600",
+	screenshotQuality: 38,
+	remoteBrowserEnabled: true,
+
+	enableCheckpoints: false,
+	checkpointStorage: "task",
+
+	ttsEnabled: false,
+	ttsSpeed: 1,
+	soundEnabled: false,
+	soundVolume: 0.5,
+
+	maxOpenTabsContext: 20,
+	maxWorkspaceFiles: 200,
+	showRooIgnoredFiles: true,
+	maxReadFileLine: 500,
+
+	terminalOutputLineLimit: 500,
+	terminalShellIntegrationTimeout: 5000,
+
+	rateLimitSeconds: 0,
+	diffEnabled: true,
+	fuzzyMatchThreshold: 1.0,
+	experiments: {
+		experimentalDiffStrategy: false, // unified diff
+		multi_search_and_replace: false, // multi-line search and replace
+		search_and_replace: true, // single-line search and replace
+		insert_content: false,
+		powerSteering: false,
+	},
+
+	language: "en",
+
+	telemetrySetting: "enabled",
+
+	mcpEnabled: false,
+	mode: "code",
+	customModes: [],
+}
--- a/benchmark/packages/types/tsup.config.ts
+++ b/benchmark/packages/types/tsup.config.ts
@ -1,20 +1,11 @@
-import { defineConfig, Options } from "tsup"
-
-const shared: Options = {
-	entry: ["src/index.ts"],
-	dts: true,
-	outDir: "dist",
-	clean: true,
-}
+import { defineConfig } from "tsup"

 export default defineConfig([
 	{
-		...shared,
-		format: ["esm"],
-		tsconfig: "./tsconfig.json",
-	},
-	{
-		...shared,
+		entry: ["src/index.ts"],
+		dts: true,
+		outDir: "dist",
+		clean: true,
 		format: ["cjs"],
 		tsconfig: "./tsconfig.cjs.json",
 		outExtension: () => ({ js: ".cjs" }),
--- a/benchmark/pnpm-lock.yaml
+++ b/benchmark/pnpm-lock.yaml
@ -108,6 +108,9 @@ importers:
      '@radix-ui/react-tabs':
        specifier: ^1.1.3
        version: 1.1.3(@types/react-dom@19.0.4(@types/react@19.0.12))(@types/react@19.0.12)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)
+      '@radix-ui/react-tooltip':
+        specifier: ^1.1.8
+        version: 1.1.8(@types/react-dom@19.0.4(@types/react@19.0.12))(@types/react@19.0.12)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)
      '@tanstack/react-query':
        specifier: ^5.69.0
        version: 5.69.0(react@19.0.0)
@ -1402,6 +1405,19 @@ packages:
      '@types/react-dom':
        optional: true

+  '@radix-ui/react-tooltip@1.1.8':
+    resolution: {integrity: sha512-YAA2cu48EkJZdAMHC0dqo9kialOcRStbtiY4nJPaht7Ptrhcvpo+eDChaM6BIs8kL6a8Z5l5poiqLnXcNduOkA==}
+    peerDependencies:
+      '@types/react': '*'
+      '@types/react-dom': '*'
+      react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+      react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc
+    peerDependenciesMeta:
+      '@types/react':
+        optional: true
+      '@types/react-dom':
+        optional: true
+
  '@radix-ui/react-use-callback-ref@1.1.0':
    resolution: {integrity: sha512-CasTfvsy+frcFkbXtSJ2Zu9JHpN8TYKxkgJGWbjiZhFivxaeW7rMeZt7QELGVLaYVfFMsKHjb7Ak0nMEe+2Vfw==}
    peerDependencies:
@ -2958,6 +2974,7 @@ packages:

  libsql@0.4.7:
    resolution: {integrity: sha512-T9eIRCs6b0J1SHKYIvD8+KCJMcWZ900iZyxdnSCdqxN12Z1ijzT+jY5nrk72Jw4B0HGzms2NgpryArlJqvc3Lw==}
+    cpu: [x64, arm64, wasm32]
    os: [darwin, linux, win32]

  lie@3.3.0:
@ -5006,6 +5023,26 @@ snapshots:
      '@types/react': 19.0.12
      '@types/react-dom': 19.0.4(@types/react@19.0.12)

+  '@radix-ui/react-tooltip@1.1.8(@types/react-dom@19.0.4(@types/react@19.0.12))(@types/react@19.0.12)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)':
+    dependencies:
+      '@radix-ui/primitive': 1.1.1
+      '@radix-ui/react-compose-refs': 1.1.1(@types/react@19.0.12)(react@19.0.0)
+      '@radix-ui/react-context': 1.1.1(@types/react@19.0.12)(react@19.0.0)
+      '@radix-ui/react-dismissable-layer': 1.1.5(@types/react-dom@19.0.4(@types/react@19.0.12))(@types/react@19.0.12)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)
+      '@radix-ui/react-id': 1.1.0(@types/react@19.0.12)(react@19.0.0)
+      '@radix-ui/react-popper': 1.2.2(@types/react-dom@19.0.4(@types/react@19.0.12))(@types/react@19.0.12)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)
+      '@radix-ui/react-portal': 1.1.4(@types/react-dom@19.0.4(@types/react@19.0.12))(@types/react@19.0.12)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)
+      '@radix-ui/react-presence': 1.1.2(@types/react-dom@19.0.4(@types/react@19.0.12))(@types/react@19.0.12)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)
+      '@radix-ui/react-primitive': 2.0.2(@types/react-dom@19.0.4(@types/react@19.0.12))(@types/react@19.0.12)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)
+      '@radix-ui/react-slot': 1.1.2(@types/react@19.0.12)(react@19.0.0)
+      '@radix-ui/react-use-controllable-state': 1.1.0(@types/react@19.0.12)(react@19.0.0)
+      '@radix-ui/react-visually-hidden': 1.1.2(@types/react-dom@19.0.4(@types/react@19.0.12))(@types/react@19.0.12)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)
+      react: 19.0.0
+      react-dom: 19.0.0(react@19.0.0)
+    optionalDependencies:
+      '@types/react': 19.0.12
+      '@types/react-dom': 19.0.4(@types/react@19.0.12)
+
  '@radix-ui/react-use-callback-ref@1.1.0(@types/react@19.0.12)(react@19.0.0)':
    dependencies:
      react: 19.0.0