Evals (#2238)

* Evals * Remove redundant line * Fix tsc error * Disable debug mode * Add option to kill run
2026-08-28 05:27:24 +00:00 · 2025-04-02 16:49:41 -07:00 · 2025-04-02 16:49:41 -07:00 · d15f813514
commit d15f813514
parent 24a46698f3
142 changed files with 14162 additions and 3152 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -1,45 +0,0 @@
-# Version control
-# .git/
-# .gitignore
-# .gitattributes
-# .git-blame-ignore-revs
-# .gitconfig
-
-# Build artifacts
-bin/
-dist/
-**/dist/
-out/
-**/out/
-
-# Dependencies
-node_modules/
-**/node_modules/
-
-# Test and development files
-coverage/
-**/.vscode-test/
-
-# Configuration files
-# .env*
-knip.json
-.husky/
-
-# CI/CD
-# .changeset/
-# .github/
-# ellipsis.yaml
-
-# OS specific
-.DS_Store
-
-# Logs
-logs/
-*.log
-
-# Nix
-# flake.lock
-# flake.nix
-
-# Monorepo
-benchmark/exercises/
--- a/.vscodeignore
+++ b/.vscodeignore
@ -5,7 +5,7 @@
 .vscode-test/**
 out/**
 out-integration/**
-benchmark/**
+evals/**
 e2e/**
 node_modules/**
 src/**
--- a/benchmark/.env.local.sample
+++ b/benchmark/.env.local.sample
@ -1,2 +0,0 @@
-OPENROUTER_API_KEY=sk-or-v1-...
-POSTHOG_API_KEY=phc_...
--- a/benchmark/Dockerfile
+++ b/benchmark/Dockerfile
@ -1,89 +0,0 @@
-# docker build -f Dockerfile.base -t roo-code-benchmark-base ..
-# docker build -f Dockerfile -t roo-code-benchmark ..
-# docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark
-# docker exec -it $(docker ps --filter "ancestor=roo-code-benchmark" -q) /bin/bash
-
-FROM ubuntu:latest
-
-# Install dependencies
-RUN apt update && apt install -y sudo curl git vim jq
-
-# Create a `vscode` user
-RUN useradd -m vscode -s /bin/bash && \
-  echo "vscode ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/vscode && \
-  chmod 0440 /etc/sudoers.d/vscode
-
-# Install VS Code
-# https://code.visualstudio.com/docs/setup/linux
-RUN apt install -y wget gpg apt-transport-https
-RUN wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > packages.microsoft.gpg
-RUN install -D -o root -g root -m 644 packages.microsoft.gpg /etc/apt/keyrings/packages.microsoft.gpg
-RUN echo "deb [arch=amd64,arm64,armhf signed-by=/etc/apt/keyrings/packages.microsoft.gpg] https://packages.microsoft.com/repos/code stable main" | tee /etc/apt/sources.list.d/vscode.list > /dev/null
-RUN rm -f packages.microsoft.gpg
-RUN apt update && apt install -y code
-
-# Install Xvfb
-RUN apt install -y xvfb
-
-# [cpp] Install cmake 3.28.3
-RUN apt install -y cmake
-
-# [go] Install Go 1.22.2
-RUN apt install -y golang-go
-
-# [java] Install Java 21
-RUN apt install -y default-jre
-
-# [javascript] Install Node.js v18.20.6
-RUN curl -sL https://deb.nodesource.com/setup_18.x | bash -
-RUN apt update && apt install -y nodejs
-RUN npm install -g corepack@latest
-
-# [python] Install Python 3.12.3 and uv 0.6.6
-RUN apt install -y python3 python3-venv python3-dev python3-pip
-
-# [rust] Install Rust 1.85
-RUN curl https://sh.rustup.rs -sSf | bash -s -- -y
-RUN echo 'source $HOME/.cargo/env' >> $HOME/.bashrc
-
-WORKDIR /home/vscode
-USER vscode
-
-# Enable corepack and install pnpm for the vscode user
-RUN corepack enable
-RUN yes y | pnpm --version
-
-COPY benchmark/entrypoint.sh /usr/local/bin/entrypoint.sh
-
-# Copy and build dependencies
-COPY --chown=vscode:vscode package*.json /home/vscode/repo/
-COPY --chown=vscode:vscode webview-ui/package*.json /home/vscode/repo/webview-ui/
-COPY --chown=vscode:vscode e2e/package*.json /home/vscode/repo/e2e/
-COPY --chown=vscode:vscode benchmark/package*.json /home/vscode/repo/benchmark/
-WORKDIR /home/vscode/repo
-RUN npm run install:all
-
-# Copy and build benchmark runner
-COPY --chown=vscode:vscode . /home/vscode/repo
-WORKDIR /home/vscode/repo/benchmark
-RUN npm run build
-
-# Copy exercises
-WORKDIR /home/vscode
-RUN git clone https://github.com/cte/Roo-Code-Benchmark.git exercises
-
-# Prepare exercises
-WORKDIR /home/vscode/exercises/python
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh
-RUN /home/vscode/.local/bin/uv sync
-
-# Build web-ui
-WORKDIR /home/vscode/exercises/web-ui
-RUN echo "DB_FILE_NAME=file:/tmp/benchmarks.db" > .env
-RUN pnpm install
-RUN npx drizzle-kit push
-
-# Run web-ui
-EXPOSE 3000
-ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
-CMD ["/usr/bin/pnpm", "dev"]
--- a/benchmark/README.md
+++ b/benchmark/README.md
@ -1,51 +0,0 @@
-# Benchmark Harness
-
-Configure ENV vars (OpenRouter, PostHog, etc):
-
-```sh
-cp .env.local.sample .env.local
-# Update ENV vars as needed.
-```
-
-Build and run a Docker image with the development environment needed to run the
-benchmarks (C++, Go, Java, Node.js, Python & Rust):
-
-```sh
-npm run docker:start
-```
-
-Run an exercise:
-
-```sh
-npm run docker:benchmark -- -e exercises/javascript/binary
-```
-
-Select and run an exercise:
-
-```sh
-npm run cli
-```
-
-Select and run an exercise for a specific language:
-
-```sh
-npm run cli -- run rust
-```
-
-Run all exercises for a language:
-
-```sh
-npm run cli -- run rust all
-```
-
-Run all exercises:
-
-```sh
-npm run cli -- run all
-```
-
-Run all exercises using a specific runId (useful for re-trying when an unexpected error occurs):
-
-```sh
-npm run cli -- run all --runId 1
-```
--- a/benchmark/entrypoint.sh
+++ b/benchmark/entrypoint.sh
@ -1,4 +0,0 @@
-#!/bin/bash
-
-npx drizzle-kit push
-exec "$@"
--- a/benchmark/package-lock.json
+++ b/benchmark/package-lock.json
--- a/benchmark/package.json
+++ b/benchmark/package.json
@ -1,30 +0,0 @@
-{
-	"name": "benchmark",
-	"version": "0.1.0",
-	"private": true,
-	"main": "out/run.js",
-	"scripts": {
-		"build": "npm run compile && cd .. && npm run compile && npm run build:webview",
-		"lint": "eslint src --ext ts",
-		"check-types": "tsc --noEmit",
-		"compile": "rm -rf out && tsc -p tsconfig.json",
-		"cli": "npm run compile && npx dotenvx run -f .env.local -- tsx src/cli.ts",
-		"clean": "rimraf out",
-		"clean:exercises": "cd exercises && git checkout -f && git clean -fd",
-		"docker:build": "docker build -f Dockerfile -t roo-code-benchmark ..",
-		"docker:run": "touch /tmp/benchmarks.db && docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark",
-		"docker:start": "npm run docker:build && npm run docker:run",
-		"docker:shell": "docker exec -it $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) /bin/bash",
-		"docker:cli": "docker exec -it -w /home/vscode/repo/benchmark $(docker ps --filter \"ancestor=roo-code-benchmark\" -q) xvfb-run npm run cli --",
-		"docker:stop": "docker stop $(docker ps --filter \"ancestor=roo-code-benchmark\" -q)",
-		"docker:rm": "docker rm $(docker ps -a --filter \"ancestor=roo-code-benchmark\" -q)",
-		"docker:clean": "npm run docker:stop && npm run docker:rm"
-	},
-	"devDependencies": {
-		"@vscode/test-electron": "^2.4.0",
-		"gluegun": "^5.1.2",
-		"tsx": "^4.19.3",
-		"typescript": "^5.4.5",
-		"yargs": "^17.7.2"
-	}
-}
--- a/benchmark/prompts/cpp.md
+++ b/benchmark/prompts/cpp.md
@ -1,17 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, you can compile your code and run the tests with:
-
-```
-mkdir -p build && cd build
-cmake -G "Unix Makefiles" -DEXERCISM_RUN_ALL_TESTS=1 ..
-make
-```
-
-Note that running `make` will compile the tests and generate compile time errors. Once the errors are fixed, running `make` will build and run the tests.
-
-Do not alter the test file; it should be run as-is.
--- a/benchmark/prompts/go.md
+++ b/benchmark/prompts/go.md
@ -1,7 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, run the tests with `go test`. Do not alter the test file; it should be run as-is.
--- a/benchmark/prompts/java.md
+++ b/benchmark/prompts/java.md
@ -1,7 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, run the tests with `./gradlew test`. Do not alter the test file; it should be run as-is.
--- a/benchmark/prompts/javascript.md
+++ b/benchmark/prompts/javascript.md
@ -1,9 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, run the tests with `pnpm test`. Do not alter the test file; it should be run as-is.
-
-Before running the tests make sure your environment is set up by running `pnpm install` to install the dependencies.
--- a/benchmark/prompts/python.md
+++ b/benchmark/prompts/python.md
@ -1,7 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, run the tests with `uv run python3 -m pytest -o markers=task [name]_test.py`. Do not alter the test file; it should be run as-is.
--- a/benchmark/prompts/rust.md
+++ b/benchmark/prompts/rust.md
@ -1,7 +0,0 @@
-Your job is to complete a coding exercise described by `.docs/instructions.md`.
-
-A file with the implementation stubbed out has been created for you, along with a test file.
-
-To successfully complete the exercise, you must pass all the tests in the test file.
-
-To confirm that your solution is correct, run the tests with `cargo test`. Do not alter the test file; it should be run as-is.
--- a/benchmark/src/cli.ts
+++ b/benchmark/src/cli.ts
@ -1,171 +0,0 @@
-import * as fs from "fs"
-import * as path from "path"
-
-import { build, filesystem, GluegunPrompt } from "gluegun"
-import { runTests } from "@vscode/test-electron"
-
-// console.log(__dirname)
-// <...>/Roo-Code/benchmark/src
-
-const extensionDevelopmentPath = path.resolve(__dirname, "../../")
-const extensionTestsPath = path.resolve(__dirname, "../out/runExercise")
-const promptsPath = path.resolve(__dirname, "../prompts")
-const exercisesPath = path.resolve(__dirname, "../../../exercises")
-const languages = ["cpp", "go", "java", "javascript", "python", "rust"]
-
-async function runAll({ runId, model }: { runId: number; model: string }) {
-	for (const language of languages) {
-		await runLanguage({ runId, model, language })
-	}
-}
-
-async function runLanguage({ runId, model, language }: { runId: number; model: string; language: string }) {
-	const languagePath = path.resolve(exercisesPath, language)
-
-	if (!fs.existsSync(languagePath)) {
-		console.error(`Language directory ${languagePath} does not exist`)
-		process.exit(1)
-	}
-
-	const exercises = filesystem
-		.subdirectories(languagePath)
-		.map((exercise) => path.basename(exercise))
-		.filter((exercise) => !exercise.startsWith("."))
-
-	for (const exercise of exercises) {
-		await runExercise({ runId, model, language, exercise })
-	}
-}
-
-async function runExercise({
-	runId,
-	model,
-	language,
-	exercise,
-}: {
-	runId: number
-	model: string
-	language: string
-	exercise: string
-}) {
-	const workspacePath = path.resolve(exercisesPath, language, exercise)
-	const promptPath = path.resolve(promptsPath, `${language}.md`)
-
-	const extensionTestsEnv = {
-		PROMPT_PATH: promptPath,
-		WORKSPACE_PATH: workspacePath,
-		OPENROUTER_MODEL_ID: model,
-		RUN_ID: runId.toString(),
-	}
-
-	if (fs.existsSync(path.resolve(workspacePath, "usage.json"))) {
-		console.log(`Test result exists for ${language} / ${exercise}, skipping`)
-		return
-	}
-
-	console.log(`Running ${language} / ${exercise}`)
-
-	await runTests({
-		extensionDevelopmentPath,
-		extensionTestsPath,
-		launchArgs: [workspacePath, "--disable-extensions"],
-		extensionTestsEnv,
-	})
-}
-
-async function askLanguage(prompt: GluegunPrompt) {
-	const languages = filesystem.subdirectories(exercisesPath)
-
-	if (languages.length === 0) {
-		throw new Error(`No languages found in ${exercisesPath}`)
-	}
-
-	const { language } = await prompt.ask<{ language: string }>({
-		type: "select",
-		name: "language",
-		message: "Which language?",
-		choices: languages.map((language) => path.basename(language)).filter((language) => !language.startsWith(".")),
-	})
-
-	return language
-}
-
-async function askExercise(prompt: GluegunPrompt, language: string) {
-	const exercises = filesystem.subdirectories(path.join(exercisesPath, language))
-
-	if (exercises.length === 0) {
-		throw new Error(`No exercises found for ${language}`)
-	}
-
-	const { exercise } = await prompt.ask<{ exercise: string }>({
-		type: "select",
-		name: "exercise",
-		message: "Which exercise?",
-		choices: exercises.map((exercise) => path.basename(exercise)),
-	})
-
-	return exercise
-}
-
-async function createRun({ model }: { model: string }): Promise<{ id: number; model: string }> {
-	const response = await fetch("http://localhost:3000/api/runs", {
-		method: "POST",
-		body: JSON.stringify({ model }),
-	})
-
-	if (!response.ok) {
-		throw new Error(`Failed to create run: ${response.statusText}`)
-	}
-
-	const {
-		run: [run],
-	} = await response.json()
-	return run
-}
-
-async function main() {
-	const cli = build()
-		.brand("benchmark-runner")
-		.src(__dirname)
-		.help()
-		.version()
-		.command({
-			name: "run",
-			run: ({ config, parameters }) => {
-				config.language = parameters.first
-				config.exercise = parameters.second
-
-				if (parameters.options["runId"]) {
-					config.runId = parameters.options["runId"]
-				}
-			},
-		})
-		.defaultCommand() // Use the default command if no args.
-		.create()
-
-	const { print, prompt, config } = await cli.run(process.argv)
-
-	try {
-		const model = "anthropic/claude-3.7-sonnet"
-		const runId = config.runId ? Number(config.runId) : (await createRun({ model })).id
-
-		if (config.language === "all") {
-			console.log("Running all exercises for all languages")
-			await runAll({ runId, model })
-		} else if (config.exercise === "all") {
-			console.log(`Running all exercises for ${config.language}`)
-			await runLanguage({ runId, model, language: config.language })
-		} else {
-			const language = config.language || (await askLanguage(prompt))
-			const exercise = config.exercise || (await askExercise(prompt, language))
-			await runExercise({ runId, model, language, exercise })
-		}
-
-		process.exit(0)
-	} catch (error) {
-		print.error(error)
-		process.exit(1)
-	}
-}
-
-main()
--- a/benchmark/src/runExercise.ts
+++ b/benchmark/src/runExercise.ts
@ -1,94 +0,0 @@
-import * as fs from "fs/promises"
-import * as path from "path"
-
-import * as vscode from "vscode"
-
-import { RooCodeAPI, TokenUsage } from "../../src/exports/roo-code"
-
-import { waitUntilReady, waitUntilCompleted, sleep } from "./utils"
-
-export async function run() {
-	/**
-	 * Validate environment variables.
-	 */
-
-	const runId = process.env.RUN_ID
-	const openRouterApiKey = process.env.OPENROUTER_API_KEY
-	const openRouterModelId = process.env.OPENROUTER_MODEL_ID
-	const promptPath = process.env.PROMPT_PATH
-	const workspacePath = process.env.WORKSPACE_PATH
-
-	if (!runId || !openRouterApiKey || !openRouterModelId || !promptPath || !workspacePath) {
-		throw new Error("ENV not configured.")
-	}
-
-	const prompt = await fs.readFile(promptPath, "utf-8")
-
-	/**
-	 * Activate the extension.
-	 */
-
-	const extension = vscode.extensions.getExtension<RooCodeAPI>("RooVeterinaryInc.roo-cline")
-
-	if (!extension) {
-		throw new Error("Extension not found.")
-	}
-
-	const api = extension.isActive ? extension.exports : await extension.activate()
-
-	/**
-	 * Wait for the Roo Code to be ready to accept tasks.
-	 */
-
-	await waitUntilReady({ api })
-
-	/**
-	 * Configure Roo Code as needed.
-	 *
-	 * Use Claude 3.7 Sonnet via OpenRouter.
-	 * Don't require approval for anything.
-	 * Run any command without approval.
-	 * Disable checkpoints (for performance).
-	 */
-
-	await api.setConfiguration({
-		apiProvider: "openrouter",
-		openRouterApiKey,
-		openRouterModelId,
-		autoApprovalEnabled: true,
-		alwaysAllowReadOnly: true,
-		alwaysAllowWrite: true,
-		alwaysAllowExecute: true,
-		alwaysAllowBrowser: true,
-		alwaysApproveResubmit: true,
-		alwaysAllowMcp: true,
-		alwaysAllowModeSwitch: true,
-		enableCheckpoints: false,
-	})
-
-	await vscode.workspace
-		.getConfiguration("roo-cline")
-		.update("allowedCommands", ["*"], vscode.ConfigurationTarget.Global)
-
-	await sleep(2_000)
-
-	/**
-	 * Run the task and wait up to 10 minutes for it to complete.
-	 */
-
-	const startTime = Date.now()
-	const taskId = await api.startNewTask(prompt)
-
-	let usage: TokenUsage | undefined = undefined
-
-	try {
-		usage = await waitUntilCompleted({ api, taskId, timeout: 5 * 60 * 1_000 }) // 5m
-	} catch (e) {
-		usage = api.getTokenUsage(taskId)
-	}
-
-	if (usage) {
-		const content = JSON.stringify({ runId: parseInt(runId), ...usage, duration: Date.now() - startTime }, null, 2)
-		await fs.writeFile(path.resolve(workspacePath, "usage.json"), content)
-	}
-}
--- a/benchmark/src/utils.ts
+++ b/benchmark/src/utils.ts
@ -1,111 +0,0 @@
-import * as vscode from "vscode"
-
-import { RooCodeAPI, TokenUsage } from "../../src/exports/roo-code"
-
-type WaitForOptions = {
-	timeout?: number
-	interval?: number
-}
-
-export const waitFor = (
-	condition: (() => Promise<boolean>) | (() => boolean),
-	{ timeout = 30_000, interval = 250 }: WaitForOptions = {},
-) => {
-	let timeoutId: NodeJS.Timeout | undefined = undefined
-
-	return Promise.race([
-		new Promise<void>((resolve) => {
-			const check = async () => {
-				const result = condition()
-				const isSatisfied = result instanceof Promise ? await result : result
-
-				if (isSatisfied) {
-					if (timeoutId) {
-						clearTimeout(timeoutId)
-						timeoutId = undefined
-					}
-
-					resolve()
-				} else {
-					setTimeout(check, interval)
-				}
-			}
-
-			check()
-		}),
-		new Promise((_, reject) => {
-			timeoutId = setTimeout(() => {
-				reject(new Error(`Timeout after ${Math.floor(timeout / 1000)}s`))
-			}, timeout)
-		}),
-	])
-}
-
-type WaitUntilReadyOptions = WaitForOptions & {
-	api: RooCodeAPI
-}
-
-export const waitUntilReady = async ({ api, ...options }: WaitUntilReadyOptions) => {
-	await vscode.commands.executeCommand("roo-cline.SidebarProvider.focus")
-	await waitFor(() => api.isReady(), options)
-}
-
-type WaitUntilAbortedOptions = WaitForOptions & {
-	api: RooCodeAPI
-	taskId: string
-}
-
-export const waitUntilAborted = async ({ api, taskId, ...options }: WaitUntilAbortedOptions) => {
-	const set = new Set<string>()
-	api.on("taskAborted", (taskId) => set.add(taskId))
-	await waitFor(() => set.has(taskId), options)
-}
-
-type WaitUntilCompletedOptions = WaitForOptions & {
-	api: RooCodeAPI
-	taskId: string
-}
-
-export const waitUntilCompleted = async ({ api, taskId, ...options }: WaitUntilCompletedOptions) => {
-	const map = new Map<string, TokenUsage>()
-	api.on("taskCompleted", (taskId, usage) => map.set(taskId, usage))
-	await waitFor(() => map.has(taskId), options)
-	return map.get(taskId)
-}
-
-export const waitForCompletion = async ({
-	api,
-	taskId,
-	...options
-}: WaitUntilReadyOptions & {
-	taskId: string
-}) => waitFor(() => !!getCompletion({ api, taskId }), options)
-
-export const getCompletion = ({ api, taskId }: { api: RooCodeAPI; taskId: string }) =>
-	api.getMessages(taskId).find(({ say, partial }) => say === "completion_result" && partial === false)
-
-type WaitForMessageOptions = WaitUntilReadyOptions & {
-	taskId: string
-	include: string
-	exclude?: string
-}
-
-export const waitForMessage = async ({ api, taskId, include, exclude, ...options }: WaitForMessageOptions) =>
-	waitFor(() => !!getMessage({ api, taskId, include, exclude }), options)
-
-type GetMessageOptions = {
-	api: RooCodeAPI
-	taskId: string
-	include: string
-	exclude?: string
-}
-
-export const getMessage = ({ api, taskId, include, exclude }: GetMessageOptions) =>
-	api
-		.getMessages(taskId)
-		.find(
-			({ type, text }) =>
-				type === "say" && text && text.includes(include) && (!exclude || !text.includes(exclude)),
-		)
-
-export const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
--- a/evals/.env.sample
+++ b/evals/.env.sample
@ -0,0 +1 @@
+BENCHMARKS_DB_PATH=file:/tmp/evals.db
--- a/evals/.gitignore
+++ b/evals/.gitignore
@ -0,0 +1,42 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# Dependencies
+node_modules
+.pnp
+.pnp.js
+
+# Local env files
+.env
+.env.*
+!.env.sample
+
+# Testing
+coverage
+
+# Turbo
+.turbo
+
+# Vercel
+.vercel
+
+# Next.js
+next-env.d.ts
+
+# Build Outputs
+.next/
+out/
+build
+dist
+*.tsbuildinfo
+
+# Debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# Misc
+.DS_Store
+*.pem
+
+# Evals
+evals
--- a/evals/.npmrc
+++ b/evals/.npmrc
@ -0,0 +1,2 @@
+# https://github.com/vercel/next.js/issues/68805
+public-hoist-pattern[]=*libsql*
--- a/evals/.tool-versions
+++ b/evals/.tool-versions
@ -0,0 +1,4 @@
+nodejs v20.18.1
+python 3.13.2
+golang 1.24.2
+rust 1.85.1
--- a/evals/README.md
+++ b/evals/README.md
@ -0,0 +1,21 @@
+# Run Roo Code Evals
+
+## Get Started
+
+NOTE: This is MacOS only for now!
+
+Clone the Roo Code repo:
+
+```sh
+git clone https://github.com/RooVetGit/Roo-Code.git
+cd Roo-Code
+```
+
+Run the setup script:
+
+```sh
+cd evals
+./scripts/setup.sh
+```
+
+Navigate to [localhost:3000](http://localhost:3000/) in your browser.
--- a/evals/apps/cli/eslint.config.mjs
+++ b/evals/apps/cli/eslint.config.mjs
@ -0,0 +1,4 @@
+import { config } from "@evals/eslint-config/base"
+
+/** @type {import("eslint").Linter.Config} */
+export default [...config]
--- a/evals/apps/cli/package.json
+++ b/evals/apps/cli/package.json
@ -0,0 +1,25 @@
+{
+	"name": "@evals/cli",
+	"private": true,
+	"type": "module",
+	"scripts": {
+		"lint": "eslint src --ext ts --max-warnings=0",
+		"check-types": "tsc --noEmit",
+		"format": "prettier --write src",
+		"dev": "dotenvx run -f ../../.env -- tsx src/index.ts"
+	},
+	"dependencies": {
+		"@evals/db": "workspace:^",
+		"@evals/ipc": "workspace:^",
+		"@evals/lib": "workspace:^",
+		"@evals/types": "workspace:^",
+		"execa": "^9.5.2",
+		"gluegun": "^5.1.2",
+		"p-map": "^7.0.3",
+		"p-wait-for": "^5.0.2"
+	},
+	"devDependencies": {
+		"@evals/eslint-config": "workspace:^",
+		"@evals/typescript-config": "workspace:^"
+	}
+}
--- a/evals/apps/cli/src/exercises.ts
+++ b/evals/apps/cli/src/exercises.ts
@ -0,0 +1,31 @@
+import * as path from "path"
+import * as fs from "fs"
+
+import { filesystem } from "gluegun"
+
+import { type ExerciseLanguage, exerciseLanguages } from "@evals/types"
+
+import { exercisesPath } from "./paths.js"
+
+let exercisesByLanguage: Record<ExerciseLanguage, string[]> | null = null
+
+export const getExercises = () => {
+	if (exercisesByLanguage !== null) {
+		return exercisesByLanguage
+	}
+
+	const getLanguageExercises = (language: ExerciseLanguage) =>
+		fs.existsSync(path.resolve(exercisesPath, language))
+			? filesystem
+					.subdirectories(path.resolve(exercisesPath, language))
+					.map((exercise) => path.basename(exercise))
+					.filter((exercise) => !exercise.startsWith("."))
+			: []
+
+	exercisesByLanguage = exerciseLanguages.reduce(
+		(collect, language) => ({ ...collect, [language]: getLanguageExercises(language) }),
+		{} as Record<ExerciseLanguage, string[]>,
+	)
+
+	return exercisesByLanguage
+}
--- a/evals/apps/cli/src/index.ts
+++ b/evals/apps/cli/src/index.ts
@ -0,0 +1,473 @@
+import * as fs from "fs"
+import * as path from "path"
+import * as os from "os"
+
+import pMap from "p-map"
+import pWaitFor from "p-wait-for"
+import { execa, parseCommandString } from "execa"
+import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun"
+
+import {
+	type ExerciseLanguage,
+	exerciseLanguages,
+	RooCodeEventName,
+	IpcOrigin,
+	IpcMessageType,
+	TaskCommandName,
+	rooCodeDefaults,
+} from "@evals/types"
+import {
+	type Run,
+	findRun,
+	createRun,
+	finishRun,
+	type Task,
+	createTask,
+	getTasks,
+	updateTask,
+	createTaskMetrics,
+	updateTaskMetrics,
+} from "@evals/db"
+import { IpcServer, IpcClient } from "@evals/ipc"
+
+import { __dirname, extensionDevelopmentPath, exercisesPath } from "./paths.js"
+import { getExercises } from "./exercises.js"
+
+const maxConcurrency = 2
+const taskTimeLimit = 5 * 60 * 1_000
+
+const testCommands: Record<ExerciseLanguage, { commands: string[]; timeout?: number; cwd?: string }> = {
+	go: { commands: ["go test"] }, // timeout 15s bash -c "cd '$dir' && go test > /dev/null 2>&1"
+	java: { commands: ["./gradlew test"] }, // timeout --foreground 15s bash -c "cd '$dir' && ./gradlew test > /dev/null 2>&1"
+	javascript: { commands: ["pnpm install", "pnpm test"], timeout: 30_000 }, // timeout 30s bash -c "cd '$dir' && pnpm install >/dev/null 2>&1 && pnpm test >/dev/null 2>&1"
+	python: { commands: ["uv run python3 -m pytest -o markers=task *_test.py"] }, // timeout 15s bash -c "cd '$dir' && uv run python3 -m pytest -o markers=task *_test.py"
+	rust: { commands: ["cargo test"] }, // timeout 15s bash -c "cd '$dir' && cargo test > /dev/null 2>&1"
+}
+
+const run = async (toolbox: GluegunToolbox) => {
+	const { config, prompt } = toolbox
+
+	let { language, exercise } = config
+
+	if (![undefined, ...exerciseLanguages, "all"].includes(language)) {
+		throw new Error(`Language is invalid: ${language}`)
+	}
+
+	if (!["undefined", "string"].includes(typeof exercise)) {
+		throw new Error(`Exercise is invalid: ${exercise}`)
+	}
+
+	const id = config.runId ? Number(config.runId) : undefined
+	let run: Run
+
+	if (id) {
+		run = await findRun(id)
+	} else {
+		run = await createRun({
+			model: rooCodeDefaults.openRouterModelId!,
+			pid: process.pid,
+			socketPath: path.resolve(os.tmpdir(), `roo-code-evals-${crypto.randomUUID()}.sock`),
+		})
+
+		if (language === "all") {
+			for (const language of exerciseLanguages) {
+				const exercises = getExercises()[language as ExerciseLanguage]
+
+				await pMap(exercises, (exercise) => createTask({ runId: run.id, language, exercise }), {
+					concurrency: 10,
+				})
+			}
+		} else if (exercise === "all") {
+			const exercises = getExercises()[language as ExerciseLanguage]
+			await pMap(exercises, (exercise) => createTask({ runId: run.id, language, exercise }), { concurrency: 10 })
+		} else {
+			language = language || (await askLanguage(prompt))
+			exercise = exercise || (await askExercise(prompt, language))
+			await createTask({ runId: run.id, language, exercise })
+		}
+	}
+
+	const tasks = await getTasks(run.id)
+
+	if (!tasks[0]) {
+		throw new Error("No tasks found.")
+	}
+
+	console.log(await execa({ cwd: exercisesPath })`git config user.name "Roo Code"`)
+	console.log(await execa({ cwd: exercisesPath })`git config user.email "support@roocode.com"`)
+	console.log(await execa({ cwd: exercisesPath })`git checkout -f`)
+	console.log(await execa({ cwd: exercisesPath })`git clean -fd`)
+	console.log(await execa({ cwd: exercisesPath })`git checkout -b runs/${run.id} main`)
+
+	fs.writeFileSync(
+		path.resolve(exercisesPath, "settings.json"),
+		JSON.stringify({ ...rooCodeDefaults, ...run.settings }, null, 2),
+	)
+
+	const server = new IpcServer(run.socketPath, () => {})
+	server.listen()
+
+	// server.on(IpcMessageType.Connect, (clientId) => {
+	// 	server.send(clientId, {
+	// 		type: IpcMessageType.TaskEvent,
+	// 		origin: IpcOrigin.Server,
+	// 		data: { eventName: RooCodeEventName.Connect, taskId: -1 },
+	// 	})
+	// })
+
+	const runningPromises: Promise<void>[] = []
+
+	const processTask = async (task: Task) => {
+		if (task.finishedAt === null) {
+			await runExercise({ run, task, server })
+		}
+
+		if (task.passed === null) {
+			const passed = await runUnitTest({ task })
+			await updateTask(task.id, { passed })
+		}
+	}
+
+	for (const task of tasks) {
+		const taskPromise = processTask(task)
+		runningPromises.push(taskPromise)
+
+		taskPromise.finally(() => {
+			const index = runningPromises.indexOf(taskPromise)
+
+			if (index > -1) {
+				runningPromises.splice(index, 1)
+			}
+		})
+
+		if (runningPromises.length >= maxConcurrency) {
+			await Promise.race(runningPromises)
+		}
+	}
+
+	await Promise.all(runningPromises)
+
+	const result = await finishRun(run.id)
+	try {
+		console.log("[cli#run]", result)
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	} catch (error) {
+		// console.error(error)
+	}
+
+	console.log(await execa({ cwd: exercisesPath })`git add .`)
+	console.log(await execa({ cwd: exercisesPath })`git commit -m ${`Run #${run.id}`} --no-verify`)
+}
+
+const runExercise = async ({ run, task, server }: { run: Run; task: Task; server: IpcServer }) => {
+	const { language, exercise } = task
+	const prompt = fs.readFileSync(path.resolve(exercisesPath, `prompts/${language}.md`), "utf-8")
+	const dirname = path.dirname(run.socketPath)
+	const taskSocketPath = path.resolve(dirname, `${dirname}/task-${task.id}.sock`)
+
+	const controller = new AbortController()
+	const cancelSignal = controller.signal
+
+	// If debugging:
+	// Use --wait --log trace or --verbose.
+	const codeCommand = `code --disable-workspace-trust`
+
+	await execa({
+		env: {
+			ROO_CODE_IPC_SOCKET_PATH: taskSocketPath,
+		},
+		shell: "/bin/bash",
+		cancelSignal,
+	})`${codeCommand} -n ${path.resolve(exercisesPath, language, exercise)}`
+
+	// If debugging:
+	// Don't await execa and store result as subprocess.
+	// subprocess.stdout.pipe(process.stdout)
+
+	// Give VSCode some time to spawn before connectint to its unix socket.
+	await new Promise((resolve) => setTimeout(resolve, 1_000))
+	console.log(`Connecting to ${taskSocketPath}`)
+
+	const createClient = (taskSocketPath: string) => {
+		const ipcClient = new IpcClient(taskSocketPath)
+
+		ipcClient.on(IpcMessageType.Ack, (ack) => {
+			console.log(`[cli#runExercise | ${language} / ${exercise}] ack`, ack)
+		})
+
+		return ipcClient
+	}
+
+	let tries = 0
+	let client = createClient(taskSocketPath)
+
+	while (++tries < 5) {
+		try {
+			await pWaitFor(() => client.isReady, { interval: 100, timeout: 5_000 })
+			break
+		} catch (error) {
+			console.error(error)
+			client.disconnect()
+			client = createClient(taskSocketPath)
+		}
+	}
+
+	let isTaskFinished = false
+	let isClientDisconnected = false
+
+	client.on(IpcMessageType.Disconnect, async () => {
+		console.log(`[cli#runExercise | ${language} / ${exercise}] disconnect`)
+		isTaskFinished = true
+		isClientDisconnected = true
+	})
+
+	const ignoreEvents: RooCodeEventName[] = [
+		// RooCodeEventName.Message,
+		RooCodeEventName.TaskTokenUsageUpdated,
+		RooCodeEventName.TaskAskResponded,
+	]
+
+	let taskStartedAt = Date.now()
+	let taskMetricsId: number | undefined
+	let rooTaskId: string | undefined
+
+	client.on(IpcMessageType.TaskEvent, async (taskEvent) => {
+		const { eventName, payload } = taskEvent
+
+		server.broadcast({
+			type: IpcMessageType.TaskEvent,
+			origin: IpcOrigin.Server,
+			relayClientId: client.clientId!,
+			data: { ...taskEvent, taskId: task.id },
+		})
+
+		if (!ignoreEvents.includes(eventName)) {
+			console.log(`[cli#runExercise | ${language} / ${exercise}] taskEvent -> ${eventName}`)
+			console.log(payload)
+		}
+
+		if (eventName === RooCodeEventName.TaskStarted) {
+			taskStartedAt = Date.now()
+
+			const taskMetrics = await createTaskMetrics({
+				cost: 0,
+				tokensIn: 0,
+				tokensOut: 0,
+				tokensContext: 0,
+				duration: 0,
+				cacheWrites: 0,
+				cacheReads: 0,
+			})
+
+			await updateTask(task.id, { taskMetricsId: taskMetrics.id, startedAt: new Date() })
+
+			taskStartedAt = Date.now()
+			taskMetricsId = taskMetrics.id
+			rooTaskId = payload[0]
+		}
+
+		if (
+			(eventName === RooCodeEventName.TaskTokenUsageUpdated || eventName === RooCodeEventName.TaskCompleted) &&
+			taskMetricsId
+		) {
+			const duration = Date.now() - taskStartedAt
+
+			const { totalCost, totalTokensIn, totalTokensOut, contextTokens, totalCacheWrites, totalCacheReads } =
+				payload[1]
+
+			await updateTaskMetrics(taskMetricsId, {
+				cost: totalCost,
+				tokensIn: totalTokensIn,
+				tokensOut: totalTokensOut,
+				tokensContext: contextTokens,
+				duration,
+				cacheWrites: totalCacheWrites ?? 0,
+				cacheReads: totalCacheReads ?? 0,
+			})
+		}
+
+		if (eventName === RooCodeEventName.TaskCompleted || eventName === RooCodeEventName.TaskAborted) {
+			await updateTask(task.id, { finishedAt: new Date() })
+			isTaskFinished = true
+		}
+	})
+
+	if (client.isReady) {
+		client.sendMessage({
+			type: IpcMessageType.TaskCommand,
+			origin: IpcOrigin.Client,
+			clientId: client.clientId!,
+			data: {
+				commandName: TaskCommandName.StartNewTask,
+				data: {
+					configuration: {
+						...rooCodeDefaults,
+						openRouterApiKey: process.env.OPENROUTER_API_KEY!,
+						...run.settings,
+					},
+					text: prompt,
+					newTab: true,
+				},
+			},
+		})
+
+		console.log(`[cli#runExercise | ${language} / ${exercise}] starting task`)
+	} else {
+		console.log(`[cli#runExercise | ${language} / ${exercise}] unable to connect`)
+		client.disconnect()
+		isTaskFinished = true
+		isClientDisconnected = true
+	}
+
+	try {
+		await pWaitFor(() => isTaskFinished, { interval: 1_000, timeout: taskTimeLimit })
+		// eslint-disable-next-line @typescript-eslint/no-unused-vars
+	} catch (error) {
+		console.log(`[cli#runExercise | ${language} / ${exercise}] time limit reached`)
+
+		if (rooTaskId && !isClientDisconnected) {
+			client.sendMessage({
+				type: IpcMessageType.TaskCommand,
+				origin: IpcOrigin.Client,
+				clientId: client.clientId!,
+				data: { commandName: TaskCommandName.CancelTask, data: rooTaskId },
+			})
+
+			await new Promise((resolve) => setTimeout(resolve, 2_000))
+		}
+
+		await updateTask(task.id, { finishedAt: new Date() })
+	}
+
+	if (!isClientDisconnected) {
+		try {
+			if (rooTaskId) {
+				client.sendMessage({
+					type: IpcMessageType.TaskCommand,
+					origin: IpcOrigin.Client,
+					clientId: client.clientId!,
+					data: { commandName: TaskCommandName.CloseTask, data: rooTaskId },
+				})
+			}
+
+			client.disconnect()
+		} catch (error) {
+			console.error(error)
+		}
+	}
+
+	// try {
+	// 	console.log(`[cli#runExercise | ${language} / ${exercise}] aborting subprocess`)
+	// 	controller.abort()
+	// 	await subprocess
+	// } catch (error) {
+	// }
+}
+
+const runUnitTest = async ({ task }: { task: Task }) => {
+	const cmd = testCommands[task.language]
+	const exercisePath = path.resolve(exercisesPath, task.language, task.exercise)
+	const cwd = cmd.cwd ? path.resolve(exercisePath, cmd.cwd) : exercisePath
+	const commands = cmd.commands.map((cs) => parseCommandString(cs))
+
+	let passed = true
+
+	for (const command of commands) {
+		// const controller = new AbortController()
+		// const cancelSignal = controller.signal
+		// const timeout = setTimeout(() => controller.abort(), cmd.timeout ?? 15_000)
+
+		try {
+			const result = await execa({ cwd, shell: true, reject: false /* , cancelSignal */ })`${command}`
+			// console.log('[cli#run] execa result =', { ...result, cwd, command })
+
+			// clearTimeout(timeout)
+
+			if (result.failed) {
+				passed = false
+				break
+			}
+		} catch (error) {
+			console.log("[cli#run] execa error =", error)
+			passed = false
+			break
+		}
+	}
+
+	return passed
+}
+
+const askLanguage = async (prompt: GluegunPrompt) => {
+	const { language } = await prompt.ask<{ language: ExerciseLanguage }>({
+		type: "select",
+		name: "language",
+		message: "Which language?",
+		choices: [...exerciseLanguages],
+	})
+
+	return language
+}
+
+const askExercise = async (prompt: GluegunPrompt, language: ExerciseLanguage) => {
+	const exercises = filesystem.subdirectories(path.join(exercisesPath, language))
+
+	if (exercises.length === 0) {
+		throw new Error(`No exercises found for ${language}`)
+	}
+
+	const { exercise } = await prompt.ask<{ exercise: string }>({
+		type: "select",
+		name: "exercise",
+		message: "Which exercise?",
+		choices: exercises.map((exercise) => path.basename(exercise)).filter((exercise) => !exercise.startsWith(".")),
+	})
+
+	return exercise
+}
+
+const main = async () => {
+	const cli = build()
+		.brand("cli")
+		.src(__dirname)
+		.help()
+		.version()
+		.command({
+			name: "run",
+			description: "Run an eval",
+			run: ({ config, parameters }) => {
+				config.language = parameters.first
+				config.exercise = parameters.second
+
+				if (parameters.options["runId"]) {
+					config.runId = parameters.options["runId"]
+				}
+			},
+		})
+		.defaultCommand()
+		.create()
+
+	const toolbox = await cli.run(process.argv)
+	const { command } = toolbox
+
+	switch (command?.name) {
+		case "run":
+			await run(toolbox)
+			break
+	}
+
+	process.exit(0)
+}
+
+if (!fs.existsSync(extensionDevelopmentPath)) {
+	console.error(`"extensionDevelopmentPath" does not exist.`)
+	process.exit(1)
+}
+
+if (!fs.existsSync(exercisesPath)) {
+	console.error(
+		`Exercises path does not exist. Please run "git clone https://github.com/cte/Roo-Code-Benchmark.git exercises".`,
+	)
+	process.exit(1)
+}
+
+main()
--- a/evals/apps/cli/src/paths.ts
+++ b/evals/apps/cli/src/paths.ts
@ -0,0 +1,7 @@
+import * as path from "path"
+import { fileURLToPath } from "url"
+
+export const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export const extensionDevelopmentPath = path.resolve(__dirname, "..", "..", "..", "..")
+export const exercisesPath = path.resolve(extensionDevelopmentPath, "..", "evals")
--- a/evals/apps/cli/tsconfig.json
+++ b/evals/apps/cli/tsconfig.json
@ -0,0 +1,5 @@
+{
+	"extends": "@evals/typescript-config/base.json",
+	"include": ["src"],
+	"exclude": ["node_modules"]
+}
--- a/evals/apps/web/components.json
+++ b/evals/apps/web/components.json
@ -0,0 +1,21 @@
+{
+	"$schema": "https://ui.shadcn.com/schema.json",
+	"style": "new-york",
+	"rsc": true,
+	"tsx": true,
+	"tailwind": {
+		"config": "",
+		"css": "src/app/globals.css",
+		"baseColor": "neutral",
+		"cssVariables": true,
+		"prefix": ""
+	},
+	"aliases": {
+		"components": "@/components",
+		"utils": "@/lib/utils",
+		"ui": "@/components/ui",
+		"lib": "@/lib",
+		"hooks": "@/hooks"
+	},
+	"iconLibrary": "lucide"
+}
--- a/evals/apps/web/eslint.config.mjs
+++ b/evals/apps/web/eslint.config.mjs
@ -0,0 +1,17 @@
+import { nextJsConfig } from "@evals/eslint-config/next-js"
+
+/** @type {import("eslint").Linter.Config} */
+export default [
+	...nextJsConfig,
+	{
+		rules: {
+			"no-unused-vars": "off",
+			"@typescript-eslint/no-unused-vars": [
+				"error",
+				{
+					caughtErrorsIgnorePattern: "^_",
+				},
+			],
+		},
+	},
+]
--- a/evals/apps/web/next.config.ts
+++ b/evals/apps/web/next.config.ts
@ -0,0 +1,7 @@
+import type { NextConfig } from "next"
+
+const nextConfig: NextConfig = {
+	/* config options here */
+}
+
+export default nextConfig
--- a/evals/apps/web/package.json
+++ b/evals/apps/web/package.json
@ -0,0 +1,55 @@
+{
+	"name": "@evals/web",
+	"private": true,
+	"scripts": {
+		"lint": "next lint",
+		"check-types": "tsc -b",
+		"dev": "dotenvx run -f ../../.env -- next dev --turbopack",
+		"format": "prettier --write src",
+		"build": "next build",
+		"start": "next start"
+	},
+	"dependencies": {
+		"@evals/db": "workspace:^",
+		"@evals/ipc": "workspace:^",
+		"@evals/types": "workspace:^",
+		"@hookform/resolvers": "^4.1.3",
+		"@radix-ui/react-dialog": "^1.1.6",
+		"@radix-ui/react-label": "^2.1.2",
+		"@radix-ui/react-popover": "^1.1.6",
+		"@radix-ui/react-scroll-area": "^1.2.3",
+		"@radix-ui/react-select": "^2.1.6",
+		"@radix-ui/react-separator": "^1.1.2",
+		"@radix-ui/react-slot": "^1.1.2",
+		"@radix-ui/react-tabs": "^1.1.3",
+		"@radix-ui/react-tooltip": "^1.1.8",
+		"@tanstack/react-query": "^5.69.0",
+		"class-variance-authority": "^0.7.1",
+		"clsx": "^2.1.1",
+		"cmdk": "^1.1.0",
+		"fuzzysort": "^3.1.0",
+		"lucide-react": "^0.479.0",
+		"next": "15.2.2",
+		"next-themes": "^0.4.6",
+		"p-map": "^7.0.3",
+		"ps-tree": "^1.2.0",
+		"react": "^19.0.0",
+		"react-dom": "^19.0.0",
+		"react-hook-form": "^7.54.2",
+		"react-use": "^17.6.0",
+		"sonner": "^2.0.2",
+		"tailwind-merge": "^3.0.2",
+		"tailwindcss-animate": "^1.0.7",
+		"vaul": "^1.1.2",
+		"zod": "^3.24.2"
+	},
+	"devDependencies": {
+		"@evals/eslint-config": "workspace:^",
+		"@evals/typescript-config": "workspace:^",
+		"@tailwindcss/postcss": "^4",
+		"@types/ps-tree": "^1.1.6",
+		"@types/react": "^19",
+		"@types/react-dom": "^19",
+		"tailwindcss": "^4"
+	}
+}
--- a/evals/apps/web/postcss.config.mjs
+++ b/evals/apps/web/postcss.config.mjs
@ -0,0 +1,5 @@
+const config = {
+	plugins: ["@tailwindcss/postcss"],
+}
+
+export default config
--- a/evals/apps/web/public/.gitkeep
+++ b/evals/apps/web/public/.gitkeep
--- a/evals/apps/web/src/app/api/runs/[id]/stream/route.ts
+++ b/evals/apps/web/src/app/api/runs/[id]/stream/route.ts
@ -0,0 +1,39 @@
+import type { NextRequest } from "next/server"
+
+import { findRun } from "@evals/db"
+import { IpcMessageType } from "@evals/types"
+import { IpcClient } from "@evals/ipc"
+
+import { SSEStream } from "@/lib/server/sse-stream"
+
+export const dynamic = "force-dynamic"
+
+export async function GET(request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
+	const { id } = await params
+	const requestId = crypto.randomUUID()
+	const stream = new SSEStream()
+	const run = await findRun(Number(id))
+	const client = new IpcClient(run.socketPath, () => {})
+
+	const write = async (data: string | object) => {
+		// console.log(`[stream#${requestId}] write`, data)
+		const success = await stream.write(data)
+
+		if (!success) {
+			client.disconnect()
+		}
+	}
+
+	console.log(`[stream#${requestId}] connect`)
+	client.on(IpcMessageType.Connect, () => write("connect"))
+	client.on(IpcMessageType.Disconnect, () => write("disconnect"))
+	client.on(IpcMessageType.TaskEvent, write)
+
+	request.signal.addEventListener("abort", () => {
+		console.log(`[stream#${requestId}] abort`)
+		client.disconnect()
+		stream.close().catch(() => {})
+	})
+
+	return stream.getResponse()
+}
--- a/evals/apps/web/src/app/api/runs/route.ts
+++ b/evals/apps/web/src/app/api/runs/route.ts
@ -0,0 +1,12 @@
+import { NextResponse } from "next/server"
+
+import { createRun } from "@evals/db"
+
+export async function POST(request: Request) {
+	try {
+		const run = await createRun(await request.json())
+		return NextResponse.json({ run }, { status: 201 })
+	} catch (error) {
+		return NextResponse.json({ error: (error as Error).message }, { status: 500 })
+	}
+}
--- a/evals/apps/web/src/app/api/tasks/route.ts
+++ b/evals/apps/web/src/app/api/tasks/route.ts
@ -0,0 +1,12 @@
+import { NextResponse } from "next/server"
+
+import { createTask } from "@evals/db"
+
+export async function POST(request: Request) {
+	try {
+		const task = await createTask(await request.json())
+		return NextResponse.json({ task }, { status: 201 })
+	} catch (error) {
+		return NextResponse.json({ error: (error as Error).message }, { status: 500 })
+	}
+}
--- a/evals/apps/web/src/app/favicon.ico
+++ b/evals/apps/web/src/app/favicon.ico
--- a/evals/apps/web/src/app/globals.css
+++ b/evals/apps/web/src/app/globals.css
@ -0,0 +1,141 @@
+@import "tailwindcss";
+
+@plugin "tailwindcss-animate";
+
+@custom-variant dark (&:is(.dark *));
+
+:root {
+	--radius: 0.625rem;
+	--background: oklch(1 0 0);
+	--foreground: oklch(0.145 0 0);
+	--card: oklch(1 0 0);
+	--card-foreground: oklch(0.145 0 0);
+	--popover: oklch(1 0 0);
+	--popover-foreground: oklch(0.145 0 0);
+	--primary: oklch(0.205 0 0);
+	--primary-foreground: oklch(0.985 0 0);
+	--secondary: oklch(0.97 0 0);
+	--secondary-foreground: oklch(0.205 0 0);
+	--muted: oklch(0.97 0 0);
+	--muted-foreground: oklch(0.556 0 0);
+	--accent: oklch(0.97 0 0);
+	--accent-foreground: oklch(0.205 0 0);
+	--destructive: oklch(0.577 0.245 27.325);
+	--border: oklch(0.922 0 0);
+	--input: oklch(0.922 0 0);
+	--ring: oklch(0.708 0 0);
+	--chart-1: oklch(0.646 0.222 41.116);
+	--chart-2: oklch(0.6 0.118 184.704);
+	--chart-3: oklch(0.398 0.07 227.392);
+	--chart-4: oklch(0.828 0.189 84.429);
+	--chart-5: oklch(0.769 0.188 70.08);
+	--sidebar: oklch(0.985 0 0);
+	--sidebar-foreground: oklch(0.145 0 0);
+	--sidebar-primary: oklch(0.205 0 0);
+	--sidebar-primary-foreground: oklch(0.985 0 0);
+	--sidebar-accent: oklch(0.97 0 0);
+	--sidebar-accent-foreground: oklch(0.205 0 0);
+	--sidebar-border: oklch(0.922 0 0);
+	--sidebar-ring: oklch(0.708 0 0);
+}
+
+.dark {
+	--background: oklch(23.66% 0.0198 271.79);
+	--foreground: oklch(75.15% 0.0477 278.41);
+	--card: oklch(0.205 0 0);
+	--card-foreground: oklch(0.985 0 0);
+	--popover: var(--primary);
+	--popover-foreground: oklch(0.985 0 0);
+	--primary: oklch(29.33% 0.0295 276.18);
+	--primary-foreground: var(--accent);
+	--secondary: var(--primary);
+	--secondary-foreground: var(--foreground);
+	--muted: oklch(28.27% 0.0207 273.06);
+	--muted-foreground: oklch(75.15% 0.0477 278.41 / 75%);
+	--accent: oklch(70.21% 0.1813 328.71);
+	--accent-foreground: oklch(1 0 0 / 75%);
+	--destructive: oklch(72.14% 0.1616 15.49);
+	--border: var(--primary);
+	--input: var(--primary);
+	--ring: oklch(83.63% 0.1259 176.52);
+	--chart-1: oklch(0.488 0.243 264.376);
+	--chart-2: oklch(0.696 0.17 162.48);
+	--chart-3: oklch(0.769 0.188 70.08);
+	--chart-4: oklch(0.627 0.265 303.9);
+	--chart-5: oklch(0.645 0.246 16.439);
+	--sidebar: oklch(0.205 0 0);
+	--sidebar-foreground: oklch(0.985 0 0);
+	--sidebar-primary: oklch(0.488 0.243 264.376);
+	--sidebar-primary-foreground: oklch(0.985 0 0);
+	--sidebar-accent: oklch(0.269 0 0);
+	--sidebar-accent-foreground: oklch(0.985 0 0);
+	--sidebar-border: oklch(1 0 0 / 10%);
+	--sidebar-ring: oklch(0.556 0 0);
+}
+
+@theme inline {
+	--color-background: var(--background);
+	--color-foreground: var(--foreground);
+	--color-sidebar-ring: var(--sidebar-ring);
+	--color-sidebar-border: var(--sidebar-border);
+	--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
+	--color-sidebar-accent: var(--sidebar-accent);
+	--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
+	--color-sidebar-primary: var(--sidebar-primary);
+	--color-sidebar-foreground: var(--sidebar-foreground);
+	--color-sidebar: var(--sidebar);
+	--color-chart-5: var(--chart-5);
+	--color-chart-4: var(--chart-4);
+	--color-chart-3: var(--chart-3);
+	--color-chart-2: var(--chart-2);
+	--color-chart-1: var(--chart-1);
+	--color-ring: var(--ring);
+	--color-input: var(--input);
+	--color-border: var(--border);
+	--color-destructive: var(--destructive);
+	--color-accent-foreground: var(--accent-foreground);
+	--color-accent: var(--accent);
+	--color-muted-foreground: var(--muted-foreground);
+	--color-muted: var(--muted);
+	--color-secondary-foreground: var(--secondary-foreground);
+	--color-secondary: var(--secondary);
+	--color-primary-foreground: var(--primary-foreground);
+	--color-primary: var(--primary);
+	--color-popover-foreground: var(--popover-foreground);
+	--color-popover: var(--popover);
+	--color-card-foreground: var(--card-foreground);
+	--color-card: var(--card);
+	--radius-sm: calc(var(--radius) - 4px);
+	--radius-md: calc(var(--radius) - 2px);
+	--radius-lg: var(--radius);
+	--radius-xl: calc(var(--radius) + 4px);
+
+	--animate-hop: hop 0.8s ease-in-out infinite;
+
+	@keyframes hop {
+		0%,
+		100% {
+			transform: none;
+			animation-timing-function: cubic-bezier(0.8, 0, 1, 1);
+		}
+		50% {
+			transform: translateY(-8px);
+			animation-timing-function: cubic-bezier(0, 0, 0.2, 1);
+		}
+	}
+}
+
+@layer base {
+	* {
+		@apply border-border outline-ring/50;
+	}
+	html,
+	body {
+		height: 100%;
+	}
+	body {
+		@apply bg-background text-foreground;
+		scrollbar-color: rgba(0, 0, 0, 0.2) transparent; /* Firefox */
+		scrollbar-width: thin;
+	}
+}
--- a/evals/apps/web/src/app/home.tsx
+++ b/evals/apps/web/src/app/home.tsx
@ -0,0 +1,72 @@
+"use client"
+
+import { useRouter } from "next/navigation"
+import { Rocket } from "lucide-react"
+
+import type { Run, TaskMetrics } from "@evals/db"
+
+import { formatCurrency, formatDuration } from "@/lib"
+import { Button, Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui"
+import { useMemo } from "react"
+import Link from "next/link"
+
+export function Home({ runs }: { runs: (Run & { taskMetrics: TaskMetrics | null })[] }) {
+	const router = useRouter()
+
+	const visibleRuns = useMemo(() => runs.filter((run) => run.taskMetrics !== null), [runs])
+
+	return (
+		<>
+			<Table className="border border-t-0">
+				<TableHeader>
+					<TableRow>
+						<TableHead>ID</TableHead>
+						<TableHead>Model</TableHead>
+						<TableHead>Timestamp</TableHead>
+						<TableHead>Passed</TableHead>
+						<TableHead>Failed</TableHead>
+						<TableHead>% Correct</TableHead>
+						<TableHead>Cost</TableHead>
+						<TableHead>Duration</TableHead>
+					</TableRow>
+				</TableHeader>
+				<TableBody>
+					{visibleRuns.length ? (
+						visibleRuns.map(({ taskMetrics, ...run }) => (
+							<TableRow key={run.id}>
+								<TableCell>
+									<Button variant="link" asChild>
+										<Link href={`/runs/${run.id}`}>{run.id}</Link>
+									</Button>
+								</TableCell>
+								<TableCell>{run.model}</TableCell>
+								<TableCell>{new Date(run.createdAt).toLocaleString()}</TableCell>
+								<TableCell>{run.passed}</TableCell>
+								<TableCell>{run.failed}</TableCell>
+								<TableCell>{((run.passed / (run.passed + run.failed)) * 100).toFixed(1)}%</TableCell>
+								<TableCell>{formatCurrency(taskMetrics!.cost)}</TableCell>
+								<TableCell>{formatDuration(taskMetrics!.duration)}</TableCell>
+							</TableRow>
+						))
+					) : (
+						<TableRow>
+							<TableCell colSpan={8} className="text-center">
+								No eval runs yet.
+								<Button variant="link" onClick={() => router.push("/runs/new")}>
+									Launch
+								</Button>
+								one now.
+							</TableCell>
+						</TableRow>
+					)}
+				</TableBody>
+			</Table>
+			<Button
+				variant="default"
+				className="absolute top-4 right-12 size-12 rounded-full"
+				onClick={() => router.push("/runs/new")}>
+				<Rocket className="size-6" />
+			</Button>
+		</>
+	)
+}
--- a/evals/apps/web/src/app/layout.tsx
+++ b/evals/apps/web/src/app/layout.tsx
@ -0,0 +1,35 @@
+import type { Metadata } from "next"
+import { Geist, Geist_Mono } from "next/font/google"
+
+import { ThemeProvider, ReactQueryProvider } from "@/components/providers"
+import { Toaster } from "@/components/ui"
+import { Header } from "@/components/layout/header"
+
+import "./globals.css"
+
+const fontSans = Geist({ variable: "--font-sans", subsets: ["latin"] })
+const fontMono = Geist_Mono({ variable: "--font-mono", subsets: ["latin"] })
+
+export const metadata: Metadata = {
+	title: "Roo Code Benchmarks",
+}
+
+export default function RootLayout({
+	children,
+}: Readonly<{
+	children: React.ReactNode
+}>) {
+	return (
+		<html lang="en">
+			<body className={`${fontSans.variable} ${fontMono.variable} font-sans antialiased pb-12`}>
+				<ThemeProvider attribute="class" defaultTheme="system" enableSystem disableTransitionOnChange>
+					<ReactQueryProvider>
+						<Header />
+						{children}
+					</ReactQueryProvider>
+				</ThemeProvider>
+				<Toaster />
+			</body>
+		</html>
+	)
+}
--- a/evals/apps/web/src/app/page.tsx
+++ b/evals/apps/web/src/app/page.tsx
@ -0,0 +1,10 @@
+import { getRuns } from "@evals/db"
+
+import { Home } from "./home"
+
+export const dynamic = "force-dynamic"
+
+export default async function Page() {
+	const runs = await getRuns()
+	return <Home runs={runs} />
+}
--- a/evals/apps/web/src/app/runs/[id]/connection-status.tsx
+++ b/evals/apps/web/src/app/runs/[id]/connection-status.tsx
@ -0,0 +1,69 @@
+"use client"
+
+import { useCallback } from "react"
+import { Skull } from "lucide-react"
+
+import { killProcessTree } from "@/lib/server/processes"
+import { EventSourceStatus } from "@/hooks/use-event-source"
+import { useProcessList } from "@/hooks/use-process-tree"
+import { cn } from "@/lib/utils"
+import { Button } from "@/components/ui"
+
+type ConnectionStatusProps = {
+	status: EventSourceStatus
+	pid: number | null
+}
+
+export const ConnectionStatus = (connectionStatus: ConnectionStatusProps) => {
+	const { data: pids, isLoading } = useProcessList(connectionStatus.pid)
+	const status = isLoading ? "loading" : pids === null ? "dead" : connectionStatus.status
+
+	const onKill = useCallback(async () => {
+		if (connectionStatus.pid) {
+			await killProcessTree(connectionStatus.pid)
+			window.location.reload()
+		}
+	}, [connectionStatus.pid])
+
+	return (
+		<div>
+			<div className="flex items-center gap-2">
+				<div className="flex items-center gap-2">
+					<div>Status:</div>
+					<div className="capitalize">{status}</div>
+				</div>
+				<div className="relative">
+					<div
+						className={cn("absolute size-2.5 rounded-full opacity-50 animate-ping", {
+							"bg-gray-500": status === "loading",
+							"bg-green-500": status === "connected",
+							"bg-amber-500": status === "waiting",
+							"bg-rose-500": status === "error" || status === "dead",
+						})}
+					/>
+					<div
+						className={cn("size-2.5 rounded-full", {
+							"bg-gray-500": status === "loading",
+							"bg-green-500": status === "connected",
+							"bg-amber-500": status === "waiting",
+							"bg-rose-500": status === "error" || status === "dead",
+						})}
+					/>
+				</div>
+			</div>
+			<div className="flex items-center gap-2">
+				<div>PIDs:</div>
+				<div className="font-mono text-sm">{connectionStatus.pid}</div>
+				{status === "connected" && (
+					<>
+						<div className="font-mono text-sm text-muted-foreground">{pids?.join(" ")}</div>
+						<Button variant="ghost" size="sm" onClick={onKill}>
+							Kill
+							<Skull />
+						</Button>
+					</>
+				)}
+			</div>
+		</div>
+	)
+}
--- a/evals/apps/web/src/app/runs/[id]/page.tsx
+++ b/evals/apps/web/src/app/runs/[id]/page.tsx
@ -0,0 +1,14 @@
+import { findRun } from "@evals/db"
+
+import { Run } from "./run"
+
+export default async function Page({ params }: { params: Promise<{ id: string }> }) {
+	const { id } = await params
+	const run = await findRun(Number(id))
+
+	return (
+		<div className="max-w-3xl mx-auto px-12 p-12">
+			<Run run={run} />
+		</div>
+	)
+}
--- a/evals/apps/web/src/app/runs/[id]/run.tsx
+++ b/evals/apps/web/src/app/runs/[id]/run.tsx
@ -0,0 +1,140 @@
+"use client"
+
+import { useState, useRef, useEffect } from "react"
+import { LoaderCircle, SquareTerminal } from "lucide-react"
+
+import * as db from "@evals/db"
+
+import { formatCurrency, formatDuration, formatTokens } from "@/lib"
+import { useRunStatus } from "@/hooks/use-run-status"
+import {
+	Drawer,
+	DrawerContent,
+	DrawerHeader,
+	DrawerTitle,
+	ScrollArea,
+	Separator,
+	Table,
+	TableBody,
+	TableCell,
+	TableHead,
+	TableHeader,
+	TableRow,
+} from "@/components/ui"
+
+import { TaskStatus } from "./task-status"
+import { ConnectionStatus } from "./connection-status"
+
+export function Run({ run }: { run: db.Run }) {
+	const { tasks, status, output, outputCounts } = useRunStatus(run)
+	const scrollAreaRef = useRef<HTMLDivElement>(null)
+	const [selectedTask, setSelectedTask] = useState<db.Task>()
+
+	useEffect(() => {
+		if (selectedTask) {
+			const scrollArea = scrollAreaRef.current
+
+			if (scrollArea) {
+				scrollArea.scrollTo({
+					top: scrollArea.scrollHeight,
+					behavior: "smooth",
+				})
+			}
+		}
+	}, [selectedTask, outputCounts])
+
+	return (
+		<>
+			<div>
+				<div className="mb-2">
+					<div>
+						<div>{run.model}</div>
+						{run.description && <div className="text-sm text-muted-foreground">{run.description}</div>}
+					</div>
+					<ConnectionStatus status={status} pid={run.pid} />
+				</div>
+				{!tasks ? (
+					<LoaderCircle className="size-4 animate-spin" />
+				) : (
+					<Table className="border">
+						<TableHeader>
+							<TableRow>
+								<TableHead>Exercise</TableHead>
+								<TableHead className="text-center">Tokens In / Out</TableHead>
+								<TableHead>Context</TableHead>
+								<TableHead>Duration</TableHead>
+								<TableHead>Cost</TableHead>
+							</TableRow>
+						</TableHeader>
+						<TableBody>
+							{tasks.map((task) => (
+								<TableRow key={task.id}>
+									<TableCell>
+										<div className="flex items-center gap-2">
+											<TaskStatus task={task} />
+											<div>
+												{task.language}/{task.exercise}
+											</div>
+											{(outputCounts[task.id] ?? 0) > 0 && (
+												<div
+													className="flex items-center gap-1 cursor-pointer"
+													onClick={() => setSelectedTask(task)}>
+													<SquareTerminal className="size-4" />
+													<div className="font-mono text-xs text-foreground/50">
+														{outputCounts[task.id]}
+													</div>
+												</div>
+											)}
+										</div>
+									</TableCell>
+									{task.taskMetrics ? (
+										<>
+											<TableCell className="font-mono text-xs">
+												<div className="flex items-center justify-evenly">
+													<div>{formatTokens(task.taskMetrics.tokensIn)}</div>/
+													<div>{formatTokens(task.taskMetrics.tokensOut)}</div>
+												</div>
+											</TableCell>
+											<TableCell className="font-mono text-xs">
+												{formatTokens(task.taskMetrics.tokensContext)}
+											</TableCell>
+											<TableCell className="font-mono text-xs">
+												{formatDuration(task.taskMetrics.duration)}
+											</TableCell>
+											<TableCell className="font-mono text-xs">
+												{formatCurrency(task.taskMetrics.cost)}
+											</TableCell>
+										</>
+									) : (
+										<TableCell colSpan={4} />
+									)}
+								</TableRow>
+							))}
+						</TableBody>
+					</Table>
+				)}
+			</div>
+			<Drawer open={!!selectedTask} onOpenChange={() => setSelectedTask(undefined)}>
+				<DrawerContent>
+					<div className="mx-auto w-full max-w-2xl">
+						<DrawerHeader>
+							<DrawerTitle>
+								{selectedTask?.language}/{selectedTask?.exercise}
+							</DrawerTitle>
+						</DrawerHeader>
+						<div className="font-mono text-xs pb-12">
+							{selectedTask && (
+								<ScrollArea viewportRef={scrollAreaRef} className="h-96 rounded-sm border">
+									<div className="p-4">
+										<h4 className="mb-4 text-sm font-medium leading-none">Tags</h4>
+										{output.get(selectedTask.id)?.map((line, i) => <div key={i}>{line}</div>)}
+									</div>
+								</ScrollArea>
+							)}
+						</div>
+					</div>
+				</DrawerContent>
+			</Drawer>
+		</>
+	)
+}
--- a/evals/apps/web/src/app/runs/[id]/task-status.tsx
+++ b/evals/apps/web/src/app/runs/[id]/task-status.tsx
@ -0,0 +1,21 @@
+import { CircleCheck, CircleDashed, CircleSlash, LoaderCircle } from "lucide-react"
+
+import { type Task } from "@evals/db"
+
+type TaskStatusProps = {
+	task: Task
+}
+
+export const TaskStatus = ({ task }: TaskStatusProps) => {
+	return task.passed === false ? (
+		<CircleSlash className="size-4 text-destructive" />
+	) : task.passed === true ? (
+		<CircleCheck className="size-4 text-green-500" />
+	) : task.startedAt ? (
+		<LoaderCircle className="size-4 animate-spin" />
+	) : task.finishedAt ? (
+		<LoaderCircle className="size-4 animate-spin" />
+	) : (
+		<CircleDashed className="size-4" />
+	)
+}
--- a/evals/apps/web/src/app/runs/new/new-run.tsx
+++ b/evals/apps/web/src/app/runs/new/new-run.tsx
@ -0,0 +1,313 @@
+"use client"
+
+import { useCallback, useRef, useState } from "react"
+import { useRouter } from "next/navigation"
+import { z } from "zod"
+import { useForm, FormProvider } from "react-hook-form"
+import { zodResolver } from "@hookform/resolvers/zod"
+import fuzzysort from "fuzzysort"
+import { toast } from "sonner"
+import { X, Rocket, Check, ChevronsUpDown, HardDriveUpload, CircleCheck } from "lucide-react"
+
+import { globalSettingsSchema, rooCodeDefaults } from "@evals/types"
+
+import { createRun } from "@/lib/server/runs"
+import { createRunSchema as formSchema, type CreateRun as FormValues } from "@/lib/schemas"
+import { cn } from "@/lib/utils"
+import { useOpenRouterModels } from "@/hooks/use-open-router-models"
+import { useExercises } from "@/hooks/use-exercises"
+import {
+	Button,
+	FormControl,
+	FormField,
+	FormItem,
+	FormLabel,
+	FormDescription,
+	FormMessage,
+	Textarea,
+	Tabs,
+	TabsList,
+	TabsTrigger,
+	MultiSelect,
+	Command,
+	CommandEmpty,
+	CommandGroup,
+	CommandInput,
+	CommandItem,
+	CommandList,
+	Popover,
+	PopoverContent,
+	PopoverTrigger,
+	ScrollArea,
+} from "@/components/ui"
+
+import { SettingsDiff } from "./settings-diff"
+
+const recommendedModels = [
+	"anthropic/claude-3.7-sonnet",
+	"anthropic/claude-3.7-sonnet:thinking",
+	"google/gemini-2.0-flash-001",
+]
+
+export function NewRun() {
+	const router = useRouter()
+
+	const [modelSearchValue, setModelSearchValue] = useState("")
+	const [modelPopoverOpen, setModelPopoverOpen] = useState(false)
+	const modelSearchResultsRef = useRef<Map<string, number>>(new Map())
+	const modelSearchValueRef = useRef("")
+	const models = useOpenRouterModels()
+
+	const exercises = useExercises()
+
+	const form = useForm<FormValues>({
+		resolver: zodResolver(formSchema),
+		defaultValues: {
+			model: "",
+			description: "",
+			suite: "full",
+			exercises: [],
+			settings: undefined,
+		},
+	})
+
+	const {
+		setValue,
+		setError,
+		clearErrors,
+		watch,
+		formState: { isSubmitting },
+	} = form
+
+	const [model, suite, settings] = watch(["model", "suite", "settings"])
+
+	const onSubmit = useCallback(
+		async (data: FormValues) => {
+			try {
+				const { id } = await createRun(data)
+				router.push(`/runs/${id}`)
+			} catch (e) {
+				toast.error(e instanceof Error ? e.message : "An unknown error occurred.")
+			}
+		},
+		[router],
+	)
+
+	const onFilterModels = useCallback(
+		(value: string, search: string) => {
+			if (modelSearchValueRef.current !== search) {
+				modelSearchValueRef.current = search
+				modelSearchResultsRef.current.clear()
+
+				for (const {
+					obj: { id },
+					score,
+				} of fuzzysort.go(search, models.data || [], {
+					key: "name",
+				})) {
+					modelSearchResultsRef.current.set(id, score)
+				}
+			}
+
+			return modelSearchResultsRef.current.get(value) ?? 0
+		},
+		[models.data],
+	)
+
+	const onSelectModel = useCallback(
+		(model: string) => {
+			setValue("model", model)
+			setModelPopoverOpen(false)
+		},
+		[setValue],
+	)
+
+	const onImportSettings = useCallback(
+		async (event: React.ChangeEvent<HTMLInputElement>) => {
+			const file = event.target.files?.[0]
+
+			if (!file) {
+				return
+			}
+
+			clearErrors("settings")
+
+			try {
+				const result = z.object({ globalSettings: globalSettingsSchema }).parse(JSON.parse(await file.text()))
+				setValue("settings", result.globalSettings)
+				event.target.value = ""
+			} catch (_error) {
+				setError("settings", { message: "Error parsing JSON file. Please check the file format." })
+			}
+		},
+		[clearErrors, setError, setValue],
+	)
+
+	return (
+		<>
+			<FormProvider {...form}>
+				<form
+					onSubmit={form.handleSubmit(onSubmit)}
+					className="flex flex-col justify-center divide-y divide-primary *:py-5">
+					<FormField
+						control={form.control}
+						name="model"
+						render={() => (
+							<FormItem>
+								<FormLabel>OpenRouter Model</FormLabel>
+								<Popover open={modelPopoverOpen} onOpenChange={setModelPopoverOpen}>
+									<PopoverTrigger asChild>
+										<Button
+											variant="input"
+											role="combobox"
+											aria-expanded={modelPopoverOpen}
+											className="flex items-center justify-between">
+											<div>
+												{models.data?.find(({ id }) => id === model)?.name || model || "Select"}
+											</div>
+											<ChevronsUpDown className="opacity-50" />
+										</Button>
+									</PopoverTrigger>
+									<PopoverContent className="p-0 w-[var(--radix-popover-trigger-width)]">
+										<Command filter={onFilterModels}>
+											<CommandInput
+												placeholder="Search"
+												value={modelSearchValue}
+												onValueChange={setModelSearchValue}
+												className="h-9"
+											/>
+											<CommandList>
+												<CommandEmpty>No model found.</CommandEmpty>
+												<CommandGroup>
+													{models.data?.map(({ id, name }) => (
+														<CommandItem key={id} value={id} onSelect={onSelectModel}>
+															{name}
+															<Check
+																className={cn(
+																	"ml-auto text-accent group-data-[selected=true]:text-accent-foreground size-4",
+																	id === model ? "opacity-100" : "opacity-0",
+																)}
+															/>
+														</CommandItem>
+													))}
+												</CommandGroup>
+											</CommandList>
+										</Command>
+									</PopoverContent>
+								</Popover>
+								<FormMessage />
+								<FormDescription className="flex flex-wrap items-center gap-2">
+									<span>Recommended:</span>
+									{recommendedModels.map((modelId) => (
+										<Button
+											key={modelId}
+											variant="link"
+											className="break-all px-0!"
+											onClick={(e) => {
+												e.preventDefault()
+												setValue("model", modelId)
+											}}>
+											{modelId}
+										</Button>
+									))}
+								</FormDescription>
+							</FormItem>
+						)}
+					/>
+
+					<FormItem>
+						<FormLabel>Import Settings</FormLabel>
+						<Button
+							type="button"
+							variant="secondary"
+							size="icon"
+							onClick={() => document.getElementById("json-upload")?.click()}>
+							<HardDriveUpload />
+						</Button>
+						<input
+							id="json-upload"
+							type="file"
+							accept="application/json"
+							className="hidden"
+							onChange={onImportSettings}
+						/>
+						{settings ? (
+							<ScrollArea className="max-h-64 border rounded-sm">
+								<>
+									<div className="flex items-center gap-1 p-2 border-b">
+										<CircleCheck className="size-4 text-ring" />
+										<div className="text-sm">
+											Imported valid Roo Code settings. Showing differences from default settings.
+										</div>
+									</div>
+									<SettingsDiff defaultSettings={rooCodeDefaults} customSettings={settings} />
+								</>
+							</ScrollArea>
+						) : (
+							<FormDescription>
+								Fully configure how Roo Code for this run using a settings file that was exported by Roo
+								Code.
+							</FormDescription>
+						)}
+						<FormMessage />
+					</FormItem>
+
+					<FormField
+						control={form.control}
+						name="suite"
+						render={() => (
+							<FormItem>
+								<FormLabel>Exercises</FormLabel>
+								<Tabs
+									defaultValue="full"
+									onValueChange={(value) => setValue("suite", value as "full" | "partial")}>
+									<TabsList>
+										<TabsTrigger value="full">All</TabsTrigger>
+										<TabsTrigger value="partial">Some</TabsTrigger>
+									</TabsList>
+								</Tabs>
+								{suite === "partial" && (
+									<MultiSelect
+										options={exercises.data?.map((path) => ({ value: path, label: path })) || []}
+										onValueChange={(value) => setValue("exercises", value)}
+										placeholder="Select"
+										variant="inverted"
+										maxCount={4}
+									/>
+								)}
+								<FormMessage />
+							</FormItem>
+						)}
+					/>
+
+					<FormField
+						control={form.control}
+						name="description"
+						render={({ field }) => (
+							<FormItem>
+								<FormLabel>Description / Notes</FormLabel>
+								<FormControl>
+									<Textarea placeholder="Optional" {...field} />
+								</FormControl>
+								<FormMessage />
+							</FormItem>
+						)}
+					/>
+
+					<div className="flex justify-end">
+						<Button size="lg" type="submit" disabled={isSubmitting}>
+							<Rocket className="size-4" />
+							Launch
+						</Button>
+					</div>
+				</form>
+			</FormProvider>
+			<Button
+				variant="default"
+				className="absolute top-4 right-12 size-12 rounded-full"
+				onClick={() => router.push("/")}>
+				<X className="size-6" />
+			</Button>
+		</>
+	)
+}
--- a/evals/apps/web/src/app/runs/new/page.tsx
+++ b/evals/apps/web/src/app/runs/new/page.tsx
@ -0,0 +1,9 @@
+import { NewRun } from "./new-run"
+
+export default function Page() {
+	return (
+		<div className="max-w-3xl mx-auto px-12 p-12">
+			<NewRun />
+		</div>
+	)
+}
--- a/evals/apps/web/src/app/runs/new/settings-diff.tsx
+++ b/evals/apps/web/src/app/runs/new/settings-diff.tsx
@ -0,0 +1,57 @@
+import { Fragment, HTMLAttributes } from "react"
+
+import { RooCodeSettings } from "@evals/types"
+
+import { cn } from "@/lib/utils"
+
+type SettingsDiffProps = HTMLAttributes<HTMLDivElement> & {
+	defaultSettings: RooCodeSettings
+	customSettings: RooCodeSettings
+}
+
+export function SettingsDiff({
+	customSettings: { experiments: customExperiments, ...customSettings },
+	defaultSettings: { experiments: defaultExperiments, ...defaultSettings },
+	className,
+	...props
+}: SettingsDiffProps) {
+	const defaults = { ...defaultSettings, ...defaultExperiments }
+	const custom = { ...customSettings, ...customExperiments }
+
+	return (
+		<div className={cn("grid grid-cols-3 gap-2 text-sm p-2", className)} {...props}>
+			<div className="font-medium text-muted-foreground">Setting</div>
+			<div className="font-medium text-muted-foreground">Default</div>
+			<div className="font-medium text-muted-foreground">Custom</div>
+			{Object.entries(defaults).flatMap(([key, defaultValue]) => {
+				const customValue = custom[key as keyof typeof custom]
+				const isDefault = JSON.stringify(defaultValue) === JSON.stringify(customValue)
+
+				return isDefault ? null : (
+					<SetttingDiff
+						key={key}
+						name={key}
+						defaultValue={JSON.stringify(defaultValue, null, 2)}
+						customValue={JSON.stringify(customValue, null, 2)}
+					/>
+				)
+			})}
+		</div>
+	)
+}
+
+type SettingDiffProps = HTMLAttributes<HTMLDivElement> & {
+	name: string
+	defaultValue?: string
+	customValue?: string
+}
+
+export function SetttingDiff({ name, defaultValue, customValue, ...props }: SettingDiffProps) {
+	return (
+		<Fragment {...props}>
+			<div className="overflow-hidden font-mono">{name}</div>
+			<pre className="inline text-rose-500 line-through">{defaultValue}</pre>
+			<pre className="inline text-teal-500">{customValue}</pre>
+		</Fragment>
+	)
+}
--- a/evals/apps/web/src/components/layout/header.tsx
+++ b/evals/apps/web/src/components/layout/header.tsx
@ -0,0 +1,7 @@
+import { HoppingLogo } from "./logo"
+
+export const Header = () => (
+	<div className="flex items-center justify-between border-b px-12 py-6">
+		<HoppingLogo />
+	</div>
+)
--- a/evals/apps/web/src/components/layout/logo.tsx
+++ b/evals/apps/web/src/components/layout/logo.tsx
@ -0,0 +1,54 @@
+"use client"
+
+import { SVGProps, useEffect, useRef } from "react"
+import { useRouter } from "next/navigation"
+import { useHover } from "react-use"
+
+import { cn } from "@/lib/utils"
+
+type LogoProps = Omit<SVGProps<SVGSVGElement>, "xmlns" | "viewBox" | "onClick">
+
+export const Logo = ({ width = 50, height = 32, fill = "#fff", className, ...props }: LogoProps) => {
+	const router = useRouter()
+
+	return (
+		<svg
+			xmlns="http://www.w3.org/2000/svg"
+			width={width}
+			height={height}
+			viewBox="90 12 100 64"
+			onClick={() => router.push("/")}
+			className={cn("logo cursor-pointer", className)}
+			{...props}>
+			<path
+				d="M171.633,15.8336l-1.7284,6.2499c-.0915.3309-.4369.5221-.7659.4239l-28.9937-8.6507c-.1928-.0575-.4016-.0167-.5586.1092l-28.7143,23.0269c-.0838.0672-.1839.1112-.2901.1276l-17.0849,2.6329c-.3163.0488-.5419.3327-.5178.6519l.0742.9817c.0237.3136.2809.5583.5953.5664l19.8448.513.2263.0063,14.6634-7.8328c.2053-.1097.455-.0936.6445.0415l10.3884,7.4053c.1629.1161.2589.3045.2571.5045l-.0876,9.826c-.0011.1272.0373.2515.11.3559l14.6133,20.9682c.1146.1644.3024.2624.5028.2624h4.626c.4615,0,.7574-.4908.542-.8989l-10.4155-19.7312c-.1019-.193-.0934-.4255.0221-.6106l5.4305-8.6994c.0591-.0947.143-.1715.2425-.222l19.415-9.8522c.1973-.1001.4332-.0861.6172.0366l5.5481,3.6981c.1007.0671.2189.1029.3399.1029h5.0407c.4881,0,.7804-.5429.5116-.9503l-13.9967-21.2171c-.2898-.4393-.962-.3331-1.1022.1741Z"
+				fill={fill}
+				strokeWidth="0"
+			/>
+		</svg>
+	)
+}
+
+export const HoppingLogo = (props: LogoProps) => {
+	const ref = useRef<SVGSVGElement>(null)
+	const logo = <Logo ref={ref} {...props} />
+	const [hoverable, hovered] = useHover(logo)
+
+	useEffect(() => {
+		const element = ref.current
+		const isHopping = element !== null && element.classList.contains("animate-hop")
+
+		if (hovered && element && !isHopping) {
+			element.classList.add("animate-hop")
+		} else if (element && isHopping) {
+			const onAnimationEnd = () => {
+				element.classList.remove("animate-hop")
+				element.removeEventListener("animationiteration", onAnimationEnd)
+			}
+
+			element.addEventListener("animationiteration", onAnimationEnd)
+		}
+	}, [hovered])
+
+	return hoverable
+}
--- a/evals/apps/web/src/components/providers/index.ts
+++ b/evals/apps/web/src/components/providers/index.ts
@ -0,0 +1,2 @@
+export { ReactQueryProvider } from "./react-query-provider"
+export { ThemeProvider } from "./theme-provider"
--- a/evals/apps/web/src/components/providers/react-query-provider.tsx
+++ b/evals/apps/web/src/components/providers/react-query-provider.tsx
@ -0,0 +1,8 @@
+"use client"
+
+import { QueryClient, QueryClientProvider } from "@tanstack/react-query"
+
+export function ReactQueryProvider({ children }: { children: React.ReactNode }) {
+	const queryClient = new QueryClient()
+	return <QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
+}
--- a/evals/apps/web/src/components/providers/theme-provider.tsx
+++ b/evals/apps/web/src/components/providers/theme-provider.tsx
@ -0,0 +1,13 @@
+"use client"
+
+import * as React from "react"
+import { type ThemeProviderProps } from "next-themes"
+import dynamic from "next/dynamic"
+
+const NextThemesProvider = dynamic(() => import("next-themes").then((e) => e.ThemeProvider), {
+	ssr: false,
+})
+
+export function ThemeProvider({ children, ...props }: ThemeProviderProps) {
+	return <NextThemesProvider {...props}>{children}</NextThemesProvider>
+}
--- a/evals/apps/web/src/components/ui/badge.tsx
+++ b/evals/apps/web/src/components/ui/badge.tsx
@ -0,0 +1,36 @@
+import * as React from "react"
+import { Slot } from "@radix-ui/react-slot"
+import { cva, type VariantProps } from "class-variance-authority"
+
+import { cn } from "@/lib/utils"
+
+const badgeVariants = cva(
+	"inline-flex items-center justify-center rounded-sm border px-2 py-0.5 text-xs font-medium w-fit whitespace-nowrap shrink-0 [&>svg]:size-3 gap-1 [&>svg]:pointer-events-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive transition-[color,box-shadow] overflow-hidden",
+	{
+		variants: {
+			variant: {
+				default: "border-transparent bg-primary text-primary-foreground [a&]:hover:bg-primary/90",
+				secondary: "border-transparent bg-secondary text-secondary-foreground [a&]:hover:bg-secondary/90",
+				destructive:
+					"border-transparent bg-destructive text-white [a&]:hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/70",
+				outline: "text-foreground [a&]:hover:bg-accent [a&]:hover:text-accent-foreground",
+			},
+		},
+		defaultVariants: {
+			variant: "default",
+		},
+	},
+)
+
+function Badge({
+	className,
+	variant,
+	asChild = false,
+	...props
+}: React.ComponentProps<"span"> & VariantProps<typeof badgeVariants> & { asChild?: boolean }) {
+	const Comp = asChild ? Slot : "span"
+
+	return <Comp data-slot="badge" className={cn(badgeVariants({ variant }), className)} {...props} />
+}
+
+export { Badge, badgeVariants }
--- a/evals/apps/web/src/components/ui/button.tsx
+++ b/evals/apps/web/src/components/ui/button.tsx
@ -0,0 +1,51 @@
+import * as React from "react"
+import { Slot } from "@radix-ui/react-slot"
+import { cva, type VariantProps } from "class-variance-authority"
+
+import { cn } from "@/lib/utils"
+
+const buttonVariants = cva(
+	"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive hover:opacity-80 active:scale-95 cursor-pointer",
+	{
+		variants: {
+			variant: {
+				default: "bg-primary text-primary-foreground shadow-xs [&_svg]:text-accent",
+				destructive:
+					"bg-destructive text-white shadow-xs focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60",
+				outline:
+					"border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input",
+				secondary: "bg-secondary text-secondary-foreground shadow-xs [&_svg]:text-ring",
+				ghost: "hover:bg-primary hover:text-primary-foreground",
+				link: "text-accent underline-offset-4 hover:underline h-4! px-1! rounded-none",
+				input: "bg-input text-input-foreground active:scale-100 shadow-xs",
+			},
+			size: {
+				default: "h-9 px-4 py-2 has-[>svg]:px-3",
+				sm: "h-8 gap-1.5 px-3 has-[>svg]:px-2.5 text-sm",
+				lg: "h-10 px-6 has-[>svg]:px-4 text-lg",
+				icon: "size-9",
+			},
+		},
+		defaultVariants: {
+			variant: "default",
+			size: "default",
+		},
+	},
+)
+
+function Button({
+	className,
+	variant,
+	size,
+	asChild = false,
+	...props
+}: React.ComponentProps<"button"> &
+	VariantProps<typeof buttonVariants> & {
+		asChild?: boolean
+	}) {
+	const Comp = asChild ? Slot : "button"
+
+	return <Comp data-slot="button" className={cn(buttonVariants({ variant, size, className }))} {...props} />
+}
+
+export { Button, buttonVariants }
--- a/evals/apps/web/src/components/ui/command.tsx
+++ b/evals/apps/web/src/components/ui/command.tsx
@ -0,0 +1,134 @@
+"use client"
+
+import * as React from "react"
+import { Command as CommandPrimitive } from "cmdk"
+import { SearchIcon } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"
+
+function Command({ className, ...props }: React.ComponentProps<typeof CommandPrimitive>) {
+	return (
+		<CommandPrimitive
+			data-slot="command"
+			className={cn(
+				"bg-popover text-popover-foreground flex h-full w-full flex-col overflow-hidden rounded-sm",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function CommandDialog({
+	title = "Command Palette",
+	description = "Search for a command to run...",
+	children,
+	...props
+}: React.ComponentProps<typeof Dialog> & {
+	title?: string
+	description?: string
+}) {
+	return (
+		<Dialog {...props}>
+			<DialogHeader className="sr-only">
+				<DialogTitle>{title}</DialogTitle>
+				<DialogDescription>{description}</DialogDescription>
+			</DialogHeader>
+			<DialogContent className="overflow-hidden p-0">
+				<Command className="[&_[cmdk-group-heading]]:text-muted-foreground **:data-[slot=command-input-wrapper]:h-12 [&_[cmdk-group-heading]]:px-2 [&_[cmdk-group-heading]]:font-medium [&_[cmdk-group]]:px-2 [&_[cmdk-group]:not([hidden])_~[cmdk-group]]:pt-0 [&_[cmdk-input-wrapper]_svg]:h-5 [&_[cmdk-input-wrapper]_svg]:w-5 [&_[cmdk-input]]:h-12 [&_[cmdk-item]]:px-2 [&_[cmdk-item]]:py-3 [&_[cmdk-item]_svg]:h-5 [&_[cmdk-item]_svg]:w-5">
+					{children}
+				</Command>
+			</DialogContent>
+		</Dialog>
+	)
+}
+
+function CommandInput({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Input>) {
+	return (
+		<div data-slot="command-input-wrapper" className="flex h-9 items-center gap-2 border-b px-3">
+			<SearchIcon className="size-4 shrink-0 opacity-50" />
+			<CommandPrimitive.Input
+				data-slot="command-input"
+				className={cn(
+					"placeholder:text-muted-foreground flex h-10 w-full rounded-sm bg-transparent py-3 outline-hidden disabled:cursor-not-allowed disabled:opacity-50",
+					className,
+				)}
+				{...props}
+			/>
+		</div>
+	)
+}
+
+function CommandList({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.List>) {
+	return (
+		<CommandPrimitive.List
+			data-slot="command-list"
+			className={cn("max-h-[300px] scroll-py-1 overflow-x-hidden overflow-y-auto", className)}
+			{...props}
+		/>
+	)
+}
+
+function CommandEmpty({ ...props }: React.ComponentProps<typeof CommandPrimitive.Empty>) {
+	return <CommandPrimitive.Empty data-slot="command-empty" className="py-6 text-center" {...props} />
+}
+
+function CommandGroup({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Group>) {
+	return (
+		<CommandPrimitive.Group
+			data-slot="command-group"
+			className={cn(
+				"text-foreground [&_[cmdk-group-heading]]:text-muted-foreground overflow-hidden p-1 [&_[cmdk-group-heading]]:px-2 [&_[cmdk-group-heading]]:py-1.5 [&_[cmdk-group-heading]]:text-xs [&_[cmdk-group-heading]]:font-medium",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function CommandSeparator({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Separator>) {
+	return (
+		<CommandPrimitive.Separator
+			data-slot="command-separator"
+			className={cn("bg-accent/5 -mx-1 h-px", className)}
+			{...props}
+		/>
+	)
+}
+
+function CommandItem({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Item>) {
+	return (
+		<CommandPrimitive.Item
+			data-slot="command-item"
+			className={cn(
+				"data-[selected=true]:bg-accent data-[selected=true]:text-accent-foreground [&_svg:not([class*='text-'])]:text-muted-foreground relative flex cursor-default items-center gap-2 rounded-xs px-2 py-1.5 outline-hidden select-none data-[disabled=true]:pointer-events-none data-[disabled=true]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
+				"text-foreground active:opacity-80 cursor-pointer group",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function CommandShortcut({ className, ...props }: React.ComponentProps<"span">) {
+	return (
+		<span
+			data-slot="command-shortcut"
+			className={cn("text-muted-foreground ml-auto text-xs tracking-widest", className)}
+			{...props}
+		/>
+	)
+}
+
+export {
+	Command,
+	CommandDialog,
+	CommandInput,
+	CommandList,
+	CommandEmpty,
+	CommandGroup,
+	CommandItem,
+	CommandShortcut,
+	CommandSeparator,
+}
--- a/evals/apps/web/src/components/ui/dialog.tsx
+++ b/evals/apps/web/src/components/ui/dialog.tsx
@ -0,0 +1,110 @@
+"use client"
+
+import * as React from "react"
+import * as DialogPrimitive from "@radix-ui/react-dialog"
+import { XIcon } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+
+function Dialog({ ...props }: React.ComponentProps<typeof DialogPrimitive.Root>) {
+	return <DialogPrimitive.Root data-slot="dialog" {...props} />
+}
+
+function DialogTrigger({ ...props }: React.ComponentProps<typeof DialogPrimitive.Trigger>) {
+	return <DialogPrimitive.Trigger data-slot="dialog-trigger" {...props} />
+}
+
+function DialogPortal({ ...props }: React.ComponentProps<typeof DialogPrimitive.Portal>) {
+	return <DialogPrimitive.Portal data-slot="dialog-portal" {...props} />
+}
+
+function DialogClose({ ...props }: React.ComponentProps<typeof DialogPrimitive.Close>) {
+	return <DialogPrimitive.Close data-slot="dialog-close" {...props} />
+}
+
+function DialogOverlay({ className, ...props }: React.ComponentProps<typeof DialogPrimitive.Overlay>) {
+	return (
+		<DialogPrimitive.Overlay
+			data-slot="dialog-overlay"
+			className={cn(
+				"data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 fixed inset-0 z-50 bg-black/50",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function DialogContent({ className, children, ...props }: React.ComponentProps<typeof DialogPrimitive.Content>) {
+	return (
+		<DialogPortal data-slot="dialog-portal">
+			<DialogOverlay />
+			<DialogPrimitive.Content
+				data-slot="dialog-content"
+				className={cn(
+					"bg-background data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 fixed top-[50%] left-[50%] z-50 grid w-full max-w-[calc(100%-2rem)] translate-x-[-50%] translate-y-[-50%] gap-4 rounded-lg border p-6 shadow-lg duration-200 sm:max-w-lg",
+					className,
+				)}
+				{...props}>
+				{children}
+				<DialogPrimitive.Close className="ring-offset-background focus:ring-ring data-[state=open]:bg-accent data-[state=open]:text-muted-foreground absolute top-4 right-4 rounded-xs opacity-70 transition-opacity hover:opacity-100 focus:ring-2 focus:ring-offset-2 focus:outline-hidden disabled:pointer-events-none [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4">
+					<XIcon />
+					<span className="sr-only">Close</span>
+				</DialogPrimitive.Close>
+			</DialogPrimitive.Content>
+		</DialogPortal>
+	)
+}
+
+function DialogHeader({ className, ...props }: React.ComponentProps<"div">) {
+	return (
+		<div
+			data-slot="dialog-header"
+			className={cn("flex flex-col gap-2 text-center sm:text-left", className)}
+			{...props}
+		/>
+	)
+}
+
+function DialogFooter({ className, ...props }: React.ComponentProps<"div">) {
+	return (
+		<div
+			data-slot="dialog-footer"
+			className={cn("flex flex-col-reverse gap-2 sm:flex-row sm:justify-end", className)}
+			{...props}
+		/>
+	)
+}
+
+function DialogTitle({ className, ...props }: React.ComponentProps<typeof DialogPrimitive.Title>) {
+	return (
+		<DialogPrimitive.Title
+			data-slot="dialog-title"
+			className={cn("text-lg leading-none font-semibold", className)}
+			{...props}
+		/>
+	)
+}
+
+function DialogDescription({ className, ...props }: React.ComponentProps<typeof DialogPrimitive.Description>) {
+	return (
+		<DialogPrimitive.Description
+			data-slot="dialog-description"
+			className={cn("text-muted-foreground text-sm", className)}
+			{...props}
+		/>
+	)
+}
+
+export {
+	Dialog,
+	DialogClose,
+	DialogContent,
+	DialogDescription,
+	DialogFooter,
+	DialogHeader,
+	DialogOverlay,
+	DialogPortal,
+	DialogTitle,
+	DialogTrigger,
+}
--- a/evals/apps/web/src/components/ui/drawer.tsx
+++ b/evals/apps/web/src/components/ui/drawer.tsx
@ -0,0 +1,98 @@
+"use client"
+
+import * as React from "react"
+import { Drawer as DrawerPrimitive } from "vaul"
+
+import { cn } from "@/lib/utils"
+
+function Drawer({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Root>) {
+	return <DrawerPrimitive.Root data-slot="drawer" {...props} />
+}
+
+function DrawerTrigger({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Trigger>) {
+	return <DrawerPrimitive.Trigger data-slot="drawer-trigger" {...props} />
+}
+
+function DrawerPortal({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Portal>) {
+	return <DrawerPrimitive.Portal data-slot="drawer-portal" {...props} />
+}
+
+function DrawerClose({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Close>) {
+	return <DrawerPrimitive.Close data-slot="drawer-close" {...props} />
+}
+
+function DrawerOverlay({ className, ...props }: React.ComponentProps<typeof DrawerPrimitive.Overlay>) {
+	return (
+		<DrawerPrimitive.Overlay
+			data-slot="drawer-overlay"
+			className={cn(
+				"data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 fixed inset-0 z-50 bg-black/50",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function DrawerContent({ className, children, ...props }: React.ComponentProps<typeof DrawerPrimitive.Content>) {
+	return (
+		<DrawerPortal data-slot="drawer-portal">
+			<DrawerOverlay />
+			<DrawerPrimitive.Content
+				data-slot="drawer-content"
+				className={cn(
+					"group/drawer-content bg-background fixed z-50 flex h-auto flex-col",
+					"data-[vaul-drawer-direction=top]:inset-x-0 data-[vaul-drawer-direction=top]:top-0 data-[vaul-drawer-direction=top]:mb-24 data-[vaul-drawer-direction=top]:max-h-[80vh] data-[vaul-drawer-direction=top]:rounded-b-sm data-[vaul-drawer-direction=top]:border-b",
+					"data-[vaul-drawer-direction=bottom]:inset-x-0 data-[vaul-drawer-direction=bottom]:bottom-0 data-[vaul-drawer-direction=bottom]:mt-24 data-[vaul-drawer-direction=bottom]:max-h-[80vh] data-[vaul-drawer-direction=bottom]:rounded-t-sm data-[vaul-drawer-direction=bottom]:border-t",
+					"data-[vaul-drawer-direction=right]:inset-y-0 data-[vaul-drawer-direction=right]:right-0 data-[vaul-drawer-direction=right]:w-3/4 data-[vaul-drawer-direction=right]:border-l data-[vaul-drawer-direction=right]:sm:max-w-sm",
+					"data-[vaul-drawer-direction=left]:inset-y-0 data-[vaul-drawer-direction=left]:left-0 data-[vaul-drawer-direction=left]:w-3/4 data-[vaul-drawer-direction=left]:border-r data-[vaul-drawer-direction=left]:sm:max-w-sm",
+					className,
+				)}
+				{...props}>
+				<div className="bg-muted mx-auto mt-4 hidden h-2 w-[100px] shrink-0 rounded-full group-data-[vaul-drawer-direction=bottom]/drawer-content:block" />
+				{children}
+			</DrawerPrimitive.Content>
+		</DrawerPortal>
+	)
+}
+
+function DrawerHeader({ className, ...props }: React.ComponentProps<"div">) {
+	return <div data-slot="drawer-header" className={cn("flex flex-col gap-1.5 py-4", className)} {...props} />
+}
+
+function DrawerFooter({ className, ...props }: React.ComponentProps<"div">) {
+	return <div data-slot="drawer-footer" className={cn("mt-auto flex flex-col gap-2 py-4", className)} {...props} />
+}
+
+function DrawerTitle({ className, ...props }: React.ComponentProps<typeof DrawerPrimitive.Title>) {
+	return (
+		<DrawerPrimitive.Title
+			data-slot="drawer-title"
+			className={cn("text-foreground font-semibold", className)}
+			{...props}
+		/>
+	)
+}
+
+function DrawerDescription({ className, ...props }: React.ComponentProps<typeof DrawerPrimitive.Description>) {
+	return (
+		<DrawerPrimitive.Description
+			data-slot="drawer-description"
+			className={cn("text-muted-foreground text-sm", className)}
+			{...props}
+		/>
+	)
+}
+
+export {
+	Drawer,
+	DrawerPortal,
+	DrawerOverlay,
+	DrawerTrigger,
+	DrawerClose,
+	DrawerContent,
+	DrawerHeader,
+	DrawerFooter,
+	DrawerTitle,
+	DrawerDescription,
+}
--- a/evals/apps/web/src/components/ui/form.tsx
+++ b/evals/apps/web/src/components/ui/form.tsx
@ -0,0 +1,138 @@
+"use client"
+
+import * as React from "react"
+import * as LabelPrimitive from "@radix-ui/react-label"
+import { Slot } from "@radix-ui/react-slot"
+import {
+	Controller,
+	FormProvider,
+	useFormContext,
+	useFormState,
+	type ControllerProps,
+	type FieldPath,
+	type FieldValues,
+} from "react-hook-form"
+
+import { cn } from "@/lib/utils"
+import { Label } from "@/components/ui/label"
+
+const Form = FormProvider
+
+type FormFieldContextValue<
+	TFieldValues extends FieldValues = FieldValues,
+	TName extends FieldPath<TFieldValues> = FieldPath<TFieldValues>,
+> = {
+	name: TName
+}
+
+const FormFieldContext = React.createContext<FormFieldContextValue>({} as FormFieldContextValue)
+
+const FormField = <
+	TFieldValues extends FieldValues = FieldValues,
+	TName extends FieldPath<TFieldValues> = FieldPath<TFieldValues>,
+>({
+	...props
+}: ControllerProps<TFieldValues, TName>) => {
+	return (
+		<FormFieldContext.Provider value={{ name: props.name }}>
+			<Controller {...props} />
+		</FormFieldContext.Provider>
+	)
+}
+
+const useFormField = () => {
+	const fieldContext = React.useContext(FormFieldContext)
+	const itemContext = React.useContext(FormItemContext)
+	const { getFieldState } = useFormContext()
+	const formState = useFormState({ name: fieldContext.name })
+	const fieldState = getFieldState(fieldContext.name, formState)
+
+	if (!fieldContext) {
+		throw new Error("useFormField should be used within <FormField>")
+	}
+
+	const { id } = itemContext
+
+	return {
+		id,
+		name: fieldContext.name,
+		formItemId: `${id}-form-item`,
+		formDescriptionId: `${id}-form-item-description`,
+		formMessageId: `${id}-form-item-message`,
+		...fieldState,
+	}
+}
+
+type FormItemContextValue = {
+	id: string
+}
+
+const FormItemContext = React.createContext<FormItemContextValue>({} as FormItemContextValue)
+
+function FormItem({ className, ...props }: React.ComponentProps<"div">) {
+	const id = React.useId()
+
+	return (
+		<FormItemContext.Provider value={{ id }}>
+			<div data-slot="form-item" className={cn("grid gap-2", className)} {...props} />
+		</FormItemContext.Provider>
+	)
+}
+
+function FormLabel({ className, ...props }: React.ComponentProps<typeof LabelPrimitive.Root>) {
+	const { error, formItemId } = useFormField()
+
+	return (
+		<Label
+			data-slot="form-label"
+			data-error={!!error}
+			className={cn("data-[error=true]:text-destructive", className)}
+			htmlFor={formItemId}
+			{...props}
+		/>
+	)
+}
+
+function FormControl({ ...props }: React.ComponentProps<typeof Slot>) {
+	const { error, formItemId, formDescriptionId, formMessageId } = useFormField()
+
+	return (
+		<Slot
+			data-slot="form-control"
+			id={formItemId}
+			aria-describedby={!error ? `${formDescriptionId}` : `${formDescriptionId} ${formMessageId}`}
+			aria-invalid={!!error}
+			{...props}
+		/>
+	)
+}
+
+function FormDescription({ className, ...props }: React.ComponentProps<"p">) {
+	const { formDescriptionId } = useFormField()
+
+	return (
+		<p
+			data-slot="form-description"
+			id={formDescriptionId}
+			className={cn("text-muted-foreground text-sm", className)}
+			{...props}
+		/>
+	)
+}
+
+function FormMessage({ className, ...props }: React.ComponentProps<"p">) {
+	const { error, formMessageId } = useFormField()
+	const body = error ? String(error?.message ?? "") : props.children
+
+	if (!body) {
+		return null
+	}
+
+	return (
+		<p data-slot="form-message" id={formMessageId} className={cn("text-destructive text-sm", className)} {...props}>
+			{body}
+		</p>
+	)
+}
+
+export { useFormField, Form, FormItem, FormLabel, FormControl, FormDescription, FormMessage, FormField }
--- a/evals/apps/web/src/components/ui/index.ts
+++ b/evals/apps/web/src/components/ui/index.ts
@ -0,0 +1,18 @@
+export * from "./badge"
+export * from "./button"
+export * from "./command"
+export * from "./dialog"
+export * from "./drawer"
+export * from "./form"
+export * from "./input"
+export * from "./label"
+export * from "./multi-select"
+export * from "./popover"
+export * from "./scroll-area"
+export * from "./select"
+export * from "./separator"
+export * from "./sonner"
+export * from "./table"
+export * from "./tabs"
+export * from "./textarea"
+export * from "./tooltip"
--- a/evals/apps/web/src/components/ui/input.tsx
+++ b/evals/apps/web/src/components/ui/input.tsx
@ -0,0 +1,22 @@
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+function Input({ className, type, ...props }: React.ComponentProps<"input">) {
+	return (
+		<input
+			type={type}
+			data-slot="input"
+			className={cn(
+				"file:text-foreground placeholder:text-muted-foreground selection:bg-primary selection:text-primary-foreground flex h-9 w-full min-w-0 rounded-sm px-3 py-1 text-base shadow-xs transition-[color,box-shadow] outline-none file:inline-flex file:h-7 file:border-0 file:bg-transparent file:text-sm file:font-medium disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-50",
+				"focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px]",
+				"aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
+				"border border-input bg-input",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+export { Input }
--- a/evals/apps/web/src/components/ui/label.tsx
+++ b/evals/apps/web/src/components/ui/label.tsx
@ -0,0 +1,21 @@
+"use client"
+
+import * as React from "react"
+import * as LabelPrimitive from "@radix-ui/react-label"
+
+import { cn } from "@/lib/utils"
+
+function Label({ className, ...props }: React.ComponentProps<typeof LabelPrimitive.Root>) {
+	return (
+		<LabelPrimitive.Root
+			data-slot="label"
+			className={cn(
+				"flex items-center gap-2 leading-none font-medium select-none group-data-[disabled=true]:pointer-events-none group-data-[disabled=true]:opacity-50 peer-disabled:cursor-not-allowed peer-disabled:opacity-50",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+export { Label }
--- a/evals/apps/web/src/components/ui/multi-select.tsx
+++ b/evals/apps/web/src/components/ui/multi-select.tsx
@ -0,0 +1,272 @@
+import * as React from "react"
+import { cva, type VariantProps } from "class-variance-authority"
+import fuzzysort from "fuzzysort"
+import { Check, X, ChevronsUpDown } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+
+import { Badge } from "./badge"
+import { Popover, PopoverContent, PopoverTrigger } from "./popover"
+import { Command, CommandEmpty, CommandGroup, CommandInput, CommandItem, CommandList } from "./command"
+
+/**
+ * Variants for the multi-select component to handle different styles.
+ * Uses class-variance-authority (cva) to define different styles based on "variant" prop.
+ */
+const multiSelectVariants = cva("px-2 py-1", {
+	variants: {
+		variant: {
+			default: "border-foreground/10 text-foreground bg-card hover:bg-card/80",
+			secondary: "border-foreground/10 bg-secondary text-secondary-foreground hover:bg-secondary/80",
+			destructive: "border-transparent bg-destructive text-destructive-foreground hover:bg-destructive/80",
+			inverted: "bg-background",
+		},
+	},
+	defaultVariants: {
+		variant: "default",
+	},
+})
+
+/**
+ * Props for MultiSelect component
+ */
+interface MultiSelectProps extends React.HTMLAttributes<HTMLDivElement>, VariantProps<typeof multiSelectVariants> {
+	/**
+	 * An array of option objects to be displayed in the multi-select component.
+	 * Each option object has a label and value.
+	 */
+	options: {
+		/** The text to display for the option. */
+		label: string
+		/** The unique value associated with the option. */
+		value: string
+	}[]
+
+	/**
+	 * Callback function triggered when the selected values change.
+	 * Receives an array of the new selected values.
+	 */
+	onValueChange: (value: string[]) => void
+
+	/** The default selected values when the component mounts. */
+	defaultValue?: string[]
+
+	/**
+	 * Placeholder text to be displayed when no values are selected.
+	 * Optional, defaults to "Select options".
+	 */
+	placeholder?: string
+
+	/**
+	 * Maximum number of items to display. Extra selected items will be summarized.
+	 * Optional, defaults to 3.
+	 */
+	maxCount?: number
+
+	/**
+	 * The modality of the popover. When set to true, interaction with outside elements
+	 * will be disabled and only popover content will be visible to screen readers.
+	 * Optional, defaults to false.
+	 */
+	modalPopover?: boolean
+
+	/**
+	 * If true, renders the multi-select component as a child of another component.
+	 * Optional, defaults to false.
+	 */
+	asChild?: boolean
+
+	/**
+	 * Additional class names to apply custom styles to the multi-select component.
+	 * Optional, can be used to add custom styles.
+	 */
+	className?: string
+}
+
+export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
+	(
+		{
+			options,
+			onValueChange,
+			variant,
+			defaultValue = [],
+			placeholder = "Select options",
+			maxCount = 3,
+			modalPopover = false,
+			className,
+			...props
+		},
+		ref,
+	) => {
+		const [selectedValues, setSelectedValues] = React.useState<string[]>(defaultValue)
+		const [isPopoverOpen, setIsPopoverOpen] = React.useState(false)
+
+		const handleInputKeyDown = (event: React.KeyboardEvent<HTMLInputElement>) => {
+			if (event.key === "Enter") {
+				setIsPopoverOpen(true)
+			} else if (event.key === "Backspace" && !event.currentTarget.value) {
+				const newSelectedValues = [...selectedValues]
+				newSelectedValues.pop()
+				setSelectedValues(newSelectedValues)
+				onValueChange(newSelectedValues)
+			}
+		}
+
+		const toggleOption = (option: string) => {
+			const newSelectedValues = selectedValues.includes(option)
+				? selectedValues.filter((value) => value !== option)
+				: [...selectedValues, option]
+			setSelectedValues(newSelectedValues)
+			onValueChange(newSelectedValues)
+		}
+
+		const handleTogglePopover = () => {
+			setIsPopoverOpen((prev) => !prev)
+		}
+
+		const clearExtraOptions = () => {
+			const newSelectedValues = selectedValues.slice(0, maxCount)
+			setSelectedValues(newSelectedValues)
+			onValueChange(newSelectedValues)
+		}
+
+		const searchResultsRef = React.useRef<Map<string, number>>(new Map())
+		const searchValueRef = React.useRef("")
+
+		const onSelectAll = () => {
+			const values = Array.from(searchResultsRef.current.keys())
+
+			if (
+				selectedValues.length === values.length &&
+				selectedValues.sort().join(",") === values.sort().join(",")
+			) {
+				setSelectedValues([])
+				onValueChange([])
+				return
+			}
+
+			setSelectedValues(values)
+			onValueChange(values)
+		}
+
+		const onFilter = React.useCallback(
+			(value: string, search: string) => {
+				if (searchValueRef.current !== search) {
+					searchValueRef.current = search
+					searchResultsRef.current.clear()
+
+					for (const {
+						obj: { value },
+						score,
+					} of fuzzysort.go(search, options, {
+						key: "label",
+					})) {
+						searchResultsRef.current.set(value, score)
+					}
+				}
+
+				if (value === "all") {
+					return searchResultsRef.current.size > 1 ? 0.01 : 0
+				}
+
+				return searchResultsRef.current.get(value) ?? 0
+			},
+			[options],
+		)
+
+		return (
+			<Popover open={isPopoverOpen} onOpenChange={setIsPopoverOpen} modal={modalPopover}>
+				<PopoverTrigger asChild>
+					<div
+						ref={ref}
+						{...props}
+						onClick={handleTogglePopover}
+						className={cn(
+							"flex w-full rounded-sm min-h-9 h-auto items-center justify-between [&_svg]:pointer-events-auto",
+							"font-medium border border-input bg-input hover:opacity-80 cursor-pointer",
+							className,
+						)}>
+						{selectedValues.length > 0 ? (
+							<div className="flex justify-between items-center w-full">
+								<div className="flex flex-wrap items-center gap-1 p-1">
+									{selectedValues.slice(0, maxCount).map((value) => (
+										<Badge key={value} className={cn(multiSelectVariants({ variant }))}>
+											<div className="flex items-center gap-1.5">
+												<div>{options.find((o) => o.value === value)?.label}</div>
+												<div
+													onClick={(event) => {
+														event.stopPropagation()
+														toggleOption(value)
+													}}
+													className="cursor-pointer">
+													<X className="size-4 rounded-full p-0.5 bg-accent/5" />
+												</div>
+											</div>
+										</Badge>
+									))}
+									{selectedValues.length > maxCount && (
+										<Badge className={cn("text-ring", multiSelectVariants({ variant }))}>
+											<div className="flex items-center gap-1.5">
+												<div>{`+ ${selectedValues.length - maxCount} more`}</div>
+												<div
+													onClick={(event) => {
+														event.stopPropagation()
+														clearExtraOptions()
+													}}
+													className="cursor-pointer">
+													<X className="size-4 rounded-full p-0.5 bg-ring/5" />
+												</div>
+											</div>
+										</Badge>
+									)}
+								</div>
+							</div>
+						) : (
+							<div className="flex items-center justify-between w-full mx-auto">
+								<span className="text-muted-foreground mx-3">{placeholder}</span>
+								<ChevronsUpDown className="opacity-50 size-4 mx-2" />
+							</div>
+						)}
+					</div>
+				</PopoverTrigger>
+				<PopoverContent
+					className="p-0 w-[var(--radix-popover-trigger-width)]"
+					align="start"
+					onEscapeKeyDown={() => setIsPopoverOpen(false)}>
+					<Command filter={onFilter}>
+						<CommandInput placeholder="Search" onKeyDown={handleInputKeyDown} />
+						<CommandList>
+							<CommandEmpty>No results found.</CommandEmpty>
+							<CommandGroup>
+								{options.map((option) => (
+									<CommandItem
+										key={option.value}
+										value={option.value}
+										onSelect={() => toggleOption(option.value)}
+										className="flex items-center justify-between">
+										<span>{option.label}</span>
+										<Check
+											className={cn(
+												"text-accent group-data-[selected=true]:text-accent-foreground size-4",
+												{ "opacity-0": !selectedValues.includes(option.value) },
+											)}
+										/>
+									</CommandItem>
+								))}
+								<CommandItem
+									key="all"
+									value="all"
+									onSelect={onSelectAll}
+									className="flex items-center justify-between">
+									<span>Select All</span>
+								</CommandItem>
+							</CommandGroup>
+						</CommandList>
+					</Command>
+				</PopoverContent>
+			</Popover>
+		)
+	},
+)
+
+MultiSelect.displayName = "MultiSelect"
--- a/evals/apps/web/src/components/ui/popover.tsx
+++ b/evals/apps/web/src/components/ui/popover.tsx
@ -0,0 +1,42 @@
+"use client"
+
+import * as React from "react"
+import * as PopoverPrimitive from "@radix-ui/react-popover"
+
+import { cn } from "@/lib/utils"
+
+function Popover({ ...props }: React.ComponentProps<typeof PopoverPrimitive.Root>) {
+	return <PopoverPrimitive.Root data-slot="popover" {...props} />
+}
+
+function PopoverTrigger({ ...props }: React.ComponentProps<typeof PopoverPrimitive.Trigger>) {
+	return <PopoverPrimitive.Trigger data-slot="popover-trigger" {...props} />
+}
+
+function PopoverContent({
+	className,
+	align = "center",
+	sideOffset = 4,
+	...props
+}: React.ComponentProps<typeof PopoverPrimitive.Content>) {
+	return (
+		<PopoverPrimitive.Portal>
+			<PopoverPrimitive.Content
+				data-slot="popover-content"
+				align={align}
+				sideOffset={sideOffset}
+				className={cn(
+					"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-72 origin-(--radix-popover-content-transform-origin) rounded-sm border p-4 shadow-md outline-hidden",
+					className,
+				)}
+				{...props}
+			/>
+		</PopoverPrimitive.Portal>
+	)
+}
+
+function PopoverAnchor({ ...props }: React.ComponentProps<typeof PopoverPrimitive.Anchor>) {
+	return <PopoverPrimitive.Anchor data-slot="popover-anchor" {...props} />
+}
+
+export { Popover, PopoverTrigger, PopoverContent, PopoverAnchor }
--- a/evals/apps/web/src/components/ui/scroll-area.tsx
+++ b/evals/apps/web/src/components/ui/scroll-area.tsx
@ -0,0 +1,51 @@
+"use client"
+
+import * as React from "react"
+import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area"
+
+import { cn } from "@/lib/utils"
+
+type ScrollAreaProps = React.ComponentProps<typeof ScrollAreaPrimitive.Root> & {
+	viewportRef?: React.RefObject<HTMLDivElement | null>
+}
+
+function ScrollArea({ className, children, viewportRef, ...props }: ScrollAreaProps) {
+	return (
+		<ScrollAreaPrimitive.Root data-slot="scroll-area" className={cn("relative", className)} {...props}>
+			<ScrollAreaPrimitive.Viewport
+				ref={viewportRef}
+				data-slot="scroll-area-viewport"
+				className="ring-ring/10 dark:ring-ring/20 dark:outline-ring/40 outline-ring/50 size-full rounded-[inherit] transition-[color,box-shadow] focus-visible:ring-4 focus-visible:outline-1">
+				{children}
+			</ScrollAreaPrimitive.Viewport>
+			<ScrollBar />
+			<ScrollAreaPrimitive.Corner />
+		</ScrollAreaPrimitive.Root>
+	)
+}
+
+function ScrollBar({
+	className,
+	orientation = "vertical",
+	...props
+}: React.ComponentProps<typeof ScrollAreaPrimitive.ScrollAreaScrollbar>) {
+	return (
+		<ScrollAreaPrimitive.ScrollAreaScrollbar
+			data-slot="scroll-area-scrollbar"
+			orientation={orientation}
+			className={cn(
+				"flex touch-none p-px transition-colors select-none",
+				orientation === "vertical" && "h-full w-2.5 border-l border-l-transparent",
+				orientation === "horizontal" && "h-2.5 flex-col border-t border-t-transparent",
+				className,
+			)}
+			{...props}>
+			<ScrollAreaPrimitive.ScrollAreaThumb
+				data-slot="scroll-area-thumb"
+				className="bg-border relative flex-1 rounded-full"
+			/>
+		</ScrollAreaPrimitive.ScrollAreaScrollbar>
+	)
+}
+
+export { ScrollArea, ScrollBar }
--- a/evals/apps/web/src/components/ui/select.tsx
+++ b/evals/apps/web/src/components/ui/select.tsx
@ -0,0 +1,156 @@
+"use client"
+
+import * as React from "react"
+import * as SelectPrimitive from "@radix-ui/react-select"
+import { Check, ChevronDown, ChevronUp } from "lucide-react"
+
+import { cn } from "@/lib/utils"
+
+function Select({ ...props }: React.ComponentProps<typeof SelectPrimitive.Root>) {
+	return <SelectPrimitive.Root data-slot="select" {...props} />
+}
+
+function SelectGroup({ ...props }: React.ComponentProps<typeof SelectPrimitive.Group>) {
+	return <SelectPrimitive.Group data-slot="select-group" {...props} />
+}
+
+function SelectValue({ ...props }: React.ComponentProps<typeof SelectPrimitive.Value>) {
+	return <SelectPrimitive.Value data-slot="select-value" {...props} />
+}
+
+function SelectTrigger({
+	className,
+	size = "default",
+	children,
+	...props
+}: React.ComponentProps<typeof SelectPrimitive.Trigger> & {
+	size?: "sm" | "default"
+}) {
+	return (
+		<SelectPrimitive.Trigger
+			data-slot="select-trigger"
+			data-size={size}
+			className={cn(
+				"data-[placeholder]:text-muted-foreground [&_svg:not([class*='text-'])]:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive flex w-fit items-center justify-between gap-2 rounded-sm px-3 py-2 whitespace-nowrap shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50 data-[size=default]:h-9 data-[size=sm]:h-8 *:data-[slot=select-value]:line-clamp-1 *:data-[slot=select-value]:flex *:data-[slot=select-value]:items-center *:data-[slot=select-value]:gap-2 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
+				"border border-input bg-input hover:opacity-80 cursor-pointer",
+				className,
+			)}
+			{...props}>
+			{children}
+			<SelectPrimitive.Icon asChild>
+				<ChevronDown className="size-4 opacity-50" />
+			</SelectPrimitive.Icon>
+		</SelectPrimitive.Trigger>
+	)
+}
+
+function SelectContent({
+	className,
+	children,
+	position = "popper",
+	...props
+}: React.ComponentProps<typeof SelectPrimitive.Content>) {
+	return (
+		<SelectPrimitive.Portal>
+			<SelectPrimitive.Content
+				data-slot="select-content"
+				className={cn(
+					"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 relative z-50 max-h-(--radix-select-content-available-height) min-w-[8rem] origin-(--radix-select-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-sm shadow-md",
+					position === "popper" &&
+						"data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1",
+					className,
+				)}
+				position={position}
+				{...props}>
+				<SelectScrollUpButton />
+				<SelectPrimitive.Viewport
+					className={cn(
+						"p-1",
+						position === "popper" &&
+							"h-[var(--radix-select-trigger-height)] w-full min-w-[var(--radix-select-trigger-width)] scroll-my-1",
+					)}>
+					{children}
+				</SelectPrimitive.Viewport>
+				<SelectScrollDownButton />
+			</SelectPrimitive.Content>
+		</SelectPrimitive.Portal>
+	)
+}
+
+function SelectLabel({ className, ...props }: React.ComponentProps<typeof SelectPrimitive.Label>) {
+	return (
+		<SelectPrimitive.Label
+			data-slot="select-label"
+			className={cn("text-muted-foreground px-2 py-1.5 text-xs", className)}
+			{...props}
+		/>
+	)
+}
+
+function SelectItem({ className, children, ...props }: React.ComponentProps<typeof SelectPrimitive.Item>) {
+	return (
+		<SelectPrimitive.Item
+			data-slot="select-item"
+			className={cn(
+				"focus:bg-accent focus:text-accent-foreground [&_svg:not([class*='text-'])]:text-muted-foreground relative flex w-full cursor-default items-center gap-2 rounded-xs py-1.5 pr-8 pl-2 outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 *:[span]:last:flex *:[span]:last:items-center *:[span]:last:gap-2",
+				"text-foreground active:opacity-80 cursor-pointer group",
+				className,
+			)}
+			{...props}>
+			<span className="absolute right-2 flex size-3.5 items-center justify-center">
+				<SelectPrimitive.ItemIndicator>
+					<Check className="text-accent group-focus:text-accent-foreground size-4" />
+				</SelectPrimitive.ItemIndicator>
+			</span>
+			<SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
+		</SelectPrimitive.Item>
+	)
+}
+
+function SelectSeparator({ className, ...props }: React.ComponentProps<typeof SelectPrimitive.Separator>) {
+	return (
+		<SelectPrimitive.Separator
+			data-slot="select-separator"
+			className={cn("bg-border pointer-events-none -mx-1 my-1 h-px", className)}
+			{...props}
+		/>
+	)
+}
+
+function SelectScrollUpButton({ className, ...props }: React.ComponentProps<typeof SelectPrimitive.ScrollUpButton>) {
+	return (
+		<SelectPrimitive.ScrollUpButton
+			data-slot="select-scroll-up-button"
+			className={cn("flex cursor-default items-center justify-center py-1", className)}
+			{...props}>
+			<ChevronUp className="size-4" />
+		</SelectPrimitive.ScrollUpButton>
+	)
+}
+
+function SelectScrollDownButton({
+	className,
+	...props
+}: React.ComponentProps<typeof SelectPrimitive.ScrollDownButton>) {
+	return (
+		<SelectPrimitive.ScrollDownButton
+			data-slot="select-scroll-down-button"
+			className={cn("flex cursor-default items-center justify-center py-1", className)}
+			{...props}>
+			<ChevronDown className="size-4" />
+		</SelectPrimitive.ScrollDownButton>
+	)
+}
+
+export {
+	Select,
+	SelectContent,
+	SelectGroup,
+	SelectItem,
+	SelectLabel,
+	SelectScrollDownButton,
+	SelectScrollUpButton,
+	SelectSeparator,
+	SelectTrigger,
+	SelectValue,
+}
--- a/evals/apps/web/src/components/ui/separator.tsx
+++ b/evals/apps/web/src/components/ui/separator.tsx
@ -0,0 +1,28 @@
+"use client"
+
+import * as React from "react"
+import * as SeparatorPrimitive from "@radix-ui/react-separator"
+
+import { cn } from "@/lib/utils"
+
+function Separator({
+	className,
+	orientation = "horizontal",
+	decorative = true,
+	...props
+}: React.ComponentProps<typeof SeparatorPrimitive.Root>) {
+	return (
+		<SeparatorPrimitive.Root
+			data-slot="separator-root"
+			decorative={decorative}
+			orientation={orientation}
+			className={cn(
+				"bg-border shrink-0 data-[orientation=horizontal]:h-px data-[orientation=horizontal]:w-full data-[orientation=vertical]:h-full data-[orientation=vertical]:w-px",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+export { Separator }
--- a/evals/apps/web/src/components/ui/sonner.tsx
+++ b/evals/apps/web/src/components/ui/sonner.tsx
@ -0,0 +1,25 @@
+"use client"
+
+import { useTheme } from "next-themes"
+import { Toaster as Sonner, ToasterProps } from "sonner"
+
+const Toaster = ({ ...props }: ToasterProps) => {
+	const { theme = "system" } = useTheme()
+
+	return (
+		<Sonner
+			theme={theme as ToasterProps["theme"]}
+			className="toaster group"
+			style={
+				{
+					"--normal-bg": "var(--popover)",
+					"--normal-text": "var(--popover-foreground)",
+					"--normal-border": "var(--border)",
+				} as React.CSSProperties
+			}
+			{...props}
+		/>
+	)
+}
+
+export { Toaster }
--- a/evals/apps/web/src/components/ui/table.tsx
+++ b/evals/apps/web/src/components/ui/table.tsx
@ -0,0 +1,75 @@
+"use client"
+
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+function Table({ className, ...props }: React.ComponentProps<"table">) {
+	return (
+		<div data-slot="table-container" className="relative w-full overflow-x-auto">
+			<table data-slot="table" className={cn("w-full caption-bottom text-sm", className)} {...props} />
+		</div>
+	)
+}
+
+function TableHeader({ className, ...props }: React.ComponentProps<"thead">) {
+	return <thead data-slot="table-header" className={cn("[&_tr]:border-b", className)} {...props} />
+}
+
+function TableBody({ className, ...props }: React.ComponentProps<"tbody">) {
+	return <tbody data-slot="table-body" className={cn("[&_tr:last-child]:border-0", className)} {...props} />
+}
+
+function TableFooter({ className, ...props }: React.ComponentProps<"tfoot">) {
+	return (
+		<tfoot
+			data-slot="table-footer"
+			className={cn("bg-muted/50 border-t font-medium [&>tr]:last:border-b-0", className)}
+			{...props}
+		/>
+	)
+}
+
+function TableRow({ className, ...props }: React.ComponentProps<"tr">) {
+	return (
+		<tr
+			data-slot="table-row"
+			className={cn("hover:bg-accent/5 data-[state=selected]:bg-muted border-b transition-colors", className)}
+			{...props}
+		/>
+	)
+}
+
+function TableHead({ className, ...props }: React.ComponentProps<"th">) {
+	return (
+		<th
+			data-slot="table-head"
+			className={cn(
+				"text-foreground h-10 px-2 text-left align-middle font-medium whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function TableCell({ className, ...props }: React.ComponentProps<"td">) {
+	return (
+		<td
+			data-slot="table-cell"
+			className={cn(
+				"p-2 align-middle whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+function TableCaption({ className, ...props }: React.ComponentProps<"caption">) {
+	return (
+		<caption data-slot="table-caption" className={cn("text-muted-foreground mt-4 text-sm", className)} {...props} />
+	)
+}
+
+export { Table, TableHeader, TableBody, TableFooter, TableHead, TableRow, TableCell, TableCaption }
--- a/evals/apps/web/src/components/ui/tabs.tsx
+++ b/evals/apps/web/src/components/ui/tabs.tsx
@ -0,0 +1,122 @@
+"use client"
+
+import * as React from "react"
+import { useEffect, useRef, useState } from "react"
+import * as TabsPrimitive from "@radix-ui/react-tabs"
+
+import { cn } from "@/lib/utils"
+
+const Tabs = TabsPrimitive.Root
+
+const TabsList = React.forwardRef<
+	React.ComponentRef<typeof TabsPrimitive.List>,
+	React.ComponentPropsWithoutRef<typeof TabsPrimitive.List>
+>(({ className, ...props }, ref) => {
+	const [indicatorStyle, setIndicatorStyle] = useState({
+		left: 0,
+		top: 0,
+		width: 0,
+		height: 0,
+	})
+
+	const tabsListRef = useRef<HTMLDivElement | null>(null)
+
+	const updateIndicator = React.useCallback(() => {
+		if (!tabsListRef.current) {
+			return
+		}
+
+		const activeTab = tabsListRef.current.querySelector<HTMLElement>('[data-state="active"]')
+
+		if (!activeTab) {
+			return
+		}
+
+		const activeRect = activeTab.getBoundingClientRect()
+		const tabsRect = tabsListRef.current.getBoundingClientRect()
+
+		requestAnimationFrame(() => {
+			setIndicatorStyle({
+				left: activeRect.left - tabsRect.left,
+				top: activeRect.top - tabsRect.top,
+				width: activeRect.width,
+				height: activeRect.height,
+			})
+		})
+	}, [])
+
+	useEffect(() => {
+		const timeoutId = setTimeout(updateIndicator, 0)
+
+		window.addEventListener("resize", updateIndicator)
+		const observer = new MutationObserver(updateIndicator)
+
+		if (tabsListRef.current) {
+			observer.observe(tabsListRef.current, {
+				attributes: true,
+				childList: true,
+				subtree: true,
+			})
+		}
+
+		return () => {
+			clearTimeout(timeoutId)
+			window.removeEventListener("resize", updateIndicator)
+			observer.disconnect()
+		}
+	}, [updateIndicator])
+
+	return (
+		<div className="relative" ref={tabsListRef}>
+			<TabsPrimitive.List
+				ref={ref}
+				className={cn(
+					"relative inline-flex items-center justify-center rounded-sm bg-primary p-0.5 text-muted-foreground",
+					className,
+				)}
+				{...props}
+			/>
+			<div
+				className={cn(
+					"absolute rounded-sm transition-all duration-300 ease-in-out pointer-events-none",
+					"bg-accent/5",
+				)}
+				style={indicatorStyle}
+			/>
+		</div>
+	)
+})
+TabsList.displayName = TabsPrimitive.List.displayName
+
+const TabsTrigger = React.forwardRef<
+	React.ComponentRef<typeof TabsPrimitive.Trigger>,
+	React.ComponentPropsWithoutRef<typeof TabsPrimitive.Trigger>
+>(({ className, ...props }, ref) => (
+	<TabsPrimitive.Trigger
+		ref={ref}
+		className={cn(
+			"inline-flex items-center justify-center whitespace-nowrap rounded-sm px-3 py-1 ring-offset-background transition-all focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 z-10",
+			"data-[state=active]:text-accent data-[state=active]:font-medium cursor-pointer",
+			className,
+		)}
+		{...props}
+	/>
+))
+TabsTrigger.displayName = TabsPrimitive.Trigger.displayName
+
+const TabsContent = React.forwardRef<
+	React.ComponentRef<typeof TabsPrimitive.Content>,
+	React.ComponentPropsWithoutRef<typeof TabsPrimitive.Content>
+>(({ className, ...props }, ref) => (
+	<TabsPrimitive.Content
+		ref={ref}
+		className={cn(
+			"mt-2 ring-offset-background focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2",
+			className,
+		)}
+		{...props}
+	/>
+))
+TabsContent.displayName = TabsPrimitive.Content.displayName
+
+export { Tabs, TabsContent, TabsList, TabsTrigger }
--- a/evals/apps/web/src/components/ui/textarea.tsx
+++ b/evals/apps/web/src/components/ui/textarea.tsx
@ -0,0 +1,19 @@
+import * as React from "react"
+
+import { cn } from "@/lib/utils"
+
+function Textarea({ className, ...props }: React.ComponentProps<"textarea">) {
+	return (
+		<textarea
+			data-slot="textarea"
+			className={cn(
+				"placeholder:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive flex field-sizing-content min-h-16 w-full rounded-sm px-3 py-2 text-base shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50",
+				"border border-input bg-input",
+				className,
+			)}
+			{...props}
+		/>
+	)
+}
+
+export { Textarea }
--- a/evals/apps/web/src/components/ui/tooltip.tsx
+++ b/evals/apps/web/src/components/ui/tooltip.tsx
@ -0,0 +1,47 @@
+"use client"
+
+import * as React from "react"
+import * as TooltipPrimitive from "@radix-ui/react-tooltip"
+
+import { cn } from "@/lib/utils"
+
+function TooltipProvider({ delayDuration = 0, ...props }: React.ComponentProps<typeof TooltipPrimitive.Provider>) {
+	return <TooltipPrimitive.Provider data-slot="tooltip-provider" delayDuration={delayDuration} {...props} />
+}
+
+function Tooltip({ ...props }: React.ComponentProps<typeof TooltipPrimitive.Root>) {
+	return (
+		<TooltipProvider>
+			<TooltipPrimitive.Root data-slot="tooltip" {...props} />
+		</TooltipProvider>
+	)
+}
+
+function TooltipTrigger({ ...props }: React.ComponentProps<typeof TooltipPrimitive.Trigger>) {
+	return <TooltipPrimitive.Trigger data-slot="tooltip-trigger" {...props} />
+}
+
+function TooltipContent({
+	className,
+	sideOffset = 0,
+	children,
+	...props
+}: React.ComponentProps<typeof TooltipPrimitive.Content>) {
+	return (
+		<TooltipPrimitive.Portal>
+			<TooltipPrimitive.Content
+				data-slot="tooltip-content"
+				sideOffset={sideOffset}
+				className={cn(
+					"bg-primary text-primary-foreground animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-fit origin-(--radix-tooltip-content-transform-origin) rounded-sm px-3 py-1.5 text-xs text-balance",
+					className,
+				)}
+				{...props}>
+				{children}
+				<TooltipPrimitive.Arrow className="bg-primary fill-primary z-50 size-2.5 translate-y-[calc(-50%_-_2px)] rotate-45 rounded-[2px]" />
+			</TooltipPrimitive.Content>
+		</TooltipPrimitive.Portal>
+	)
+}
+
+export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider }
--- a/evals/apps/web/src/hooks/use-event-source.ts
+++ b/evals/apps/web/src/hooks/use-event-source.ts
@ -0,0 +1,57 @@
+import { useCallback, useEffect, useRef, useState } from "react"
+
+export type EventSourceStatus = "waiting" | "connected" | "error"
+
+export type EventSourceEvent = Event & { data: string }
+
+type UseEventSourceOptions = {
+	url: string
+	withCredentials?: boolean
+	onMessage: (event: MessageEvent) => void
+}
+
+export function useEventSource({ url, withCredentials, onMessage }: UseEventSourceOptions) {
+	const sourceRef = useRef<EventSource | null>(null)
+	const statusRef = useRef<EventSourceStatus>("waiting")
+	const [status, setStatus] = useState<EventSourceStatus>("waiting")
+	const handleMessage = useCallback((event: MessageEvent) => onMessage(event), [onMessage])
+
+	const createEventSource = useCallback(() => {
+		sourceRef.current = new EventSource(url, { withCredentials })
+
+		sourceRef.current.onopen = () => {
+			statusRef.current = "connected"
+			setStatus("connected")
+		}
+
+		sourceRef.current.onmessage = (event) => {
+			handleMessage(event)
+		}
+
+		sourceRef.current.onerror = () => {
+			statusRef.current = "error"
+			setStatus("error")
+			// sourceRef.current?.close()
+			// sourceRef.current = null
+		}
+	}, [url, withCredentials, handleMessage])
+
+	useEffect(() => {
+		createEventSource()
+
+		setTimeout(() => {
+			if (statusRef.current === "waiting") {
+				sourceRef.current?.close()
+				sourceRef.current = null
+				createEventSource()
+			}
+		}, 100)
+
+		return () => {
+			sourceRef.current?.close()
+			sourceRef.current = null
+		}
+	}, [createEventSource])
+
+	return status
+}
--- a/evals/apps/web/src/hooks/use-exercises.ts
+++ b/evals/apps/web/src/hooks/use-exercises.ts
@ -0,0 +1,5 @@
+import { useQuery } from "@tanstack/react-query"
+
+import { getExercises } from "@/lib/server/exercises"
+
+export const useExercises = () => useQuery({ queryKey: ["exercises"], queryFn: getExercises })
--- a/evals/apps/web/src/hooks/use-open-router-models.ts
+++ b/evals/apps/web/src/hooks/use-open-router-models.ts
@ -0,0 +1,36 @@
+import { z } from "zod"
+import { useQuery } from "@tanstack/react-query"
+
+export const openRouterModelSchema = z.object({
+	id: z.string(),
+	name: z.string(),
+	description: z.string(),
+	created: z.number(),
+	context_length: z.number(),
+})
+
+export type OpenRouterModel = z.infer<typeof openRouterModelSchema>
+
+export const getOpenRouterModels = async () => {
+	const response = await fetch("https://openrouter.ai/api/v1/models")
+
+	if (!response.ok) {
+		console.error("Failed to fetch OpenRouter models")
+		return []
+	}
+
+	const result = z.object({ data: z.array(openRouterModelSchema) }).safeParse(await response.json())
+
+	if (!result.success) {
+		console.error(result.error)
+		return []
+	}
+
+	return result.data.data.sort((a, b) => a.name.localeCompare(b.name))
+}
+
+export const useOpenRouterModels = () =>
+	useQuery<OpenRouterModel[]>({
+		queryKey: ["getOpenRouterModels"],
+		queryFn: getOpenRouterModels,
+	})
--- a/evals/apps/web/src/hooks/use-process-tree.ts
+++ b/evals/apps/web/src/hooks/use-process-tree.ts
@ -0,0 +1,10 @@
+import { useQuery } from "@tanstack/react-query"
+
+import { getProcessList } from "@/lib/server/processes"
+
+export const useProcessList = (pid: number | null) =>
+	useQuery({
+		queryKey: ["process-tree", pid],
+		queryFn: () => (pid ? getProcessList(pid) : []),
+		enabled: !!pid,
+	})
--- a/evals/apps/web/src/hooks/use-run-status.ts
+++ b/evals/apps/web/src/hooks/use-run-status.ts
@ -0,0 +1,80 @@
+import { useState, useCallback, useRef } from "react"
+import { useQuery, keepPreviousData } from "@tanstack/react-query"
+
+import { RooCodeEventName, taskEventSchema } from "@evals/types"
+import { Run } from "@evals/db"
+
+import { getTasks } from "@/lib/server/tasks"
+import { useEventSource } from "@/hooks/use-event-source"
+
+export const useRunStatus = (run: Run) => {
+	const [tasksUpdatedAt, setTasksUpdatedAt] = useState<number>()
+	const outputRef = useRef<Map<number, string[]>>(new Map())
+	const [outputCounts, setOutputCounts] = useState<Record<number, number>>({})
+
+	const { data: tasks } = useQuery({
+		queryKey: ["run", run.id, tasksUpdatedAt],
+		queryFn: async () => getTasks(run.id),
+		placeholderData: keepPreviousData,
+		refetchInterval: 10_000,
+	})
+
+	const url = `/api/runs/${run.id}/stream`
+
+	const onMessage = useCallback((messageEvent: MessageEvent) => {
+		let data
+
+		try {
+			data = JSON.parse(messageEvent.data)
+		} catch (_) {
+			console.log(`invalid JSON: ${messageEvent.data}`)
+			return
+		}
+
+		const result = taskEventSchema.safeParse(data)
+
+		if (!result.success) {
+			console.log(`unrecognized messageEvent.data: ${messageEvent.data}`)
+			return
+		}
+
+		const { eventName, payload, taskId } = result.data
+
+		if (!taskId) {
+			console.log(`no taskId: ${messageEvent.data}`)
+			return
+		}
+
+		switch (eventName) {
+			case RooCodeEventName.TaskStarted:
+			case RooCodeEventName.TaskCompleted:
+			case RooCodeEventName.TaskAborted:
+				setTasksUpdatedAt(Date.now())
+				break
+			case RooCodeEventName.Message: {
+				const [
+					{
+						message: { text },
+					},
+				] = payload
+
+				if (text) {
+					outputRef.current.set(taskId, [...(outputRef.current.get(taskId) || []), text])
+					const outputCounts: Record<number, number> = {}
+
+					for (const [taskId, messages] of outputRef.current.entries()) {
+						outputCounts[taskId] = messages.length
+					}
+
+					setOutputCounts(outputCounts)
+				}
+
+				break
+			}
+		}
+	}, [])
+
+	const status = useEventSource({ url, onMessage })
+
+	return { tasks, status, output: outputRef.current, outputCounts }
+}
--- a/evals/apps/web/src/lib/format-currency.ts
+++ b/evals/apps/web/src/lib/format-currency.ts
@ -0,0 +1,6 @@
+const formatter = new Intl.NumberFormat("en-US", {
+	style: "currency",
+	currency: "USD",
+})
+
+export const formatCurrency = (amount: number) => formatter.format(amount)
--- a/evals/apps/web/src/lib/format-duration.ts
+++ b/evals/apps/web/src/lib/format-duration.ts
@ -0,0 +1,22 @@
+export const formatDuration = (durationMs: number) => {
+	const seconds = Math.floor(durationMs / 1000)
+	const hours = Math.floor(seconds / 3600)
+	const minutes = Math.floor((seconds % 3600) / 60)
+	const remainingSeconds = seconds % 60
+
+	const parts = []
+
+	if (hours > 0) {
+		parts.push(`${hours}h`)
+	}
+
+	if (minutes > 0) {
+		parts.push(`${minutes}m`)
+	}
+
+	if (remainingSeconds > 0 || parts.length === 0) {
+		parts.push(`${remainingSeconds}s`)
+	}
+
+	return parts.join(" ")
+}
--- a/evals/apps/web/src/lib/format-tokens.ts
+++ b/evals/apps/web/src/lib/format-tokens.ts
@ -0,0 +1,7 @@
+export const formatTokens = (tokens: number) => {
+	if (tokens < 1000) {
+		return tokens.toString()
+	}
+
+	return `${(tokens / 1000).toFixed(1)}k`
+}
--- a/evals/apps/web/src/lib/index.ts
+++ b/evals/apps/web/src/lib/index.ts
@ -0,0 +1,3 @@
+export { formatCurrency } from "./format-currency"
+export { formatDuration } from "./format-duration"
+export { formatTokens } from "./format-tokens"
--- a/evals/apps/web/src/lib/schemas.ts
+++ b/evals/apps/web/src/lib/schemas.ts
@ -0,0 +1,22 @@
+import { z } from "zod"
+
+import { globalSettingsSchema } from "@evals/types"
+
+/**
+ * CreateRun
+ */
+
+export const createRunSchema = z
+	.object({
+		model: z.string().min(1, { message: "Model is required." }),
+		description: z.string().optional(),
+		suite: z.enum(["full", "partial"]),
+		exercises: z.array(z.string()).optional(),
+		settings: globalSettingsSchema.optional(),
+	})
+	.refine((data) => data.suite === "full" || (data.exercises || []).length > 0, {
+		message: "Exercises are required when running a partial suite.",
+		path: ["exercises"],
+	})
+
+export type CreateRun = z.infer<typeof createRunSchema>
--- a/evals/apps/web/src/lib/server/exercises.ts
+++ b/evals/apps/web/src/lib/server/exercises.ts
@ -0,0 +1,38 @@
+"use server"
+
+import * as fs from "fs/promises"
+import * as path from "path"
+import { fileURLToPath } from "url"
+
+import { ExerciseLanguage, exerciseLanguages } from "@evals/types"
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export const listDirectories = async (relativePath: string) => {
+	try {
+		const targetPath = path.resolve(__dirname, relativePath)
+		const entries = await fs.readdir(targetPath, { withFileTypes: true })
+		return entries.filter((entry) => entry.isDirectory() && !entry.name.startsWith(".")).map((entry) => entry.name)
+	} catch (error) {
+		console.error(`Error listing directories at ${relativePath}:`, error)
+		return []
+	}
+}
+
+// __dirname = <repo>/evals/apps/web/src/lib/server
+const EXERCISES_BASE_PATH = path.resolve(__dirname, "../../../../../../../evals")
+
+export const getExercises = async () => {
+	const result = await Promise.all(
+		exerciseLanguages.map(async (language) => {
+			const languagePath = path.join(EXERCISES_BASE_PATH, language)
+			const exercises = await listDirectories(languagePath)
+			return exercises.map((exercise) => `${language}/${exercise}`)
+		}),
+	)
+
+	return result.flat()
+}
+
+export const getExercisesForLanguage = async (language: ExerciseLanguage) =>
+	listDirectories(path.join(EXERCISES_BASE_PATH, language))
--- a/evals/apps/web/src/lib/server/processes.ts
+++ b/evals/apps/web/src/lib/server/processes.ts
@ -0,0 +1,46 @@
+"use server"
+
+import psTree from "ps-tree"
+import { exec } from "child_process"
+
+export const getProcessList = async (pid: number) => {
+	const promise = new Promise<string>((resolve, reject) => {
+		exec(`ps -p ${pid} -o pid=`, (err, stdout, stderr) => {
+			if (err) {
+				reject(stderr)
+			}
+
+			resolve(stdout)
+		})
+	})
+
+	try {
+		await promise
+	} catch (_) {
+		return null
+	}
+
+	return new Promise<number[]>((resolve, reject) => {
+		psTree(pid, (err, children) => {
+			if (err) {
+				reject(err)
+			}
+
+			resolve(children.map((p) => parseInt(p.PID)))
+		})
+	})
+}
+
+export const killProcessTree = async (pid: number) => {
+	const descendants = await getProcessList(pid)
+
+	if (descendants === null) {
+		return
+	}
+
+	if (descendants.length > 0) {
+		await exec(`kill -9 ${descendants.join(" ")}`)
+	}
+
+	await exec(`kill -9 ${pid}`)
+}
--- a/evals/apps/web/src/lib/server/runs.ts
+++ b/evals/apps/web/src/lib/server/runs.ts
@ -0,0 +1,60 @@
+"use server"
+
+import { spawn } from "child_process"
+import path from "path"
+import os from "os"
+import fs from "fs"
+
+import { revalidatePath } from "next/cache"
+import pMap from "p-map"
+
+import { ExerciseLanguage, exerciseLanguages } from "@evals/types"
+import * as db from "@evals/db"
+
+import { CreateRun } from "@/lib/schemas"
+import { getExercisesForLanguage } from "./exercises"
+
+export async function createRun({ suite, exercises = [], ...values }: CreateRun) {
+	const run = await db.createRun({
+		...values,
+		socketPath: path.join(os.tmpdir(), `roo-code-evals-${crypto.randomUUID()}.sock`),
+	})
+
+	if (suite === "partial") {
+		for (const path of exercises) {
+			const [language, exercise] = path.split("/")
+
+			if (!language || !exercise) {
+				throw new Error("Invalid exercise path: " + path)
+			}
+
+			await db.createTask({ ...values, runId: run.id, language: language as ExerciseLanguage, exercise })
+		}
+	} else {
+		for (const language of exerciseLanguages) {
+			const exercises = await getExercisesForLanguage(language)
+
+			await pMap(exercises, (exercise) => db.createTask({ ...values, runId: run.id, language, exercise }), {
+				concurrency: 10,
+			})
+		}
+	}
+
+	revalidatePath("/runs")
+
+	try {
+		const logFile = fs.openSync(`/tmp/roo-code-evals-${run.id}.log`, "a")
+
+		const process = spawn("pnpm", ["--filter", "@evals/cli", "dev", "run", "all", "--runId", run.id.toString()], {
+			detached: true,
+			stdio: ["ignore", logFile, logFile],
+		})
+
+		process.unref()
+		await db.updateRun(run.id, { pid: process.pid })
+	} catch (error) {
+		console.error(error)
+	}
+
+	return run
+}
--- a/evals/apps/web/src/lib/server/sse-stream.ts
+++ b/evals/apps/web/src/lib/server/sse-stream.ts
@ -0,0 +1,37 @@
+export class SSEStream {
+	private readonly _stream: TransformStream
+	private readonly _writer: WritableStreamDefaultWriter
+	private readonly _encoder: TextEncoder
+
+	constructor() {
+		this._stream = new TransformStream()
+		this._writer = this._stream.writable.getWriter()
+		this._encoder = new TextEncoder()
+	}
+
+	public async write(data: string | object) {
+		try {
+			const buffer = typeof data === "object" ? JSON.stringify(data) : data
+			await this._writer.write(this._encoder.encode(`data: ${buffer}\n\n`))
+			return true
+		} catch (error) {
+			console.error("[SSEStream#write]", error)
+			this.close().catch(() => {})
+			return false
+		}
+	}
+
+	public close() {
+		return this._writer.close()
+	}
+
+	public getResponse() {
+		return new Response(this._stream.readable, {
+			headers: {
+				"Content-Type": "text/event-stream",
+				Connection: "keep-alive",
+				"Cache-Control": "no-cache, no-transform",
+			},
+		})
+	}
+}
--- a/evals/apps/web/src/lib/server/tasks.ts
+++ b/evals/apps/web/src/lib/server/tasks.ts
@ -0,0 +1,11 @@
+"use server"
+
+import { revalidatePath } from "next/cache"
+
+import * as db from "@evals/db"
+
+export async function getTasks(runId: number) {
+	const tasks = await db.getTasks(runId)
+	revalidatePath(`/runs/${runId}`)
+	return tasks
+}
--- a/evals/apps/web/src/lib/utils.ts
+++ b/evals/apps/web/src/lib/utils.ts
@ -0,0 +1,6 @@
+import { clsx, type ClassValue } from "clsx"
+import { twMerge } from "tailwind-merge"
+
+export function cn(...inputs: ClassValue[]) {
+	return twMerge(clsx(inputs))
+}
--- a/evals/apps/web/tsconfig.json
+++ b/evals/apps/web/tsconfig.json
@ -0,0 +1,9 @@
+{
+	"extends": "@evals/typescript-config/nextjs.json",
+	"compilerOptions": {
+		"plugins": [{ "name": "next" }],
+		"paths": { "@/*": ["./src/*"] }
+	},
+	"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+	"exclude": ["node_modules"]
+}
--- a/evals/config/eslint/base.js
+++ b/evals/config/eslint/base.js
@ -0,0 +1,32 @@
+import js from "@eslint/js"
+import eslintConfigPrettier from "eslint-config-prettier"
+import turboPlugin from "eslint-plugin-turbo"
+import tseslint from "typescript-eslint"
+import onlyWarn from "eslint-plugin-only-warn"
+
+/**
+ * A shared ESLint configuration for the repository.
+ *
+ * @type {import("eslint").Linter.Config[]}
+ * */
+export const config = [
+	js.configs.recommended,
+	eslintConfigPrettier,
+	...tseslint.configs.recommended,
+	{
+		plugins: {
+			turbo: turboPlugin,
+		},
+		rules: {
+			"turbo/no-undeclared-env-vars": "warn",
+		},
+	},
+	{
+		plugins: {
+			onlyWarn,
+		},
+	},
+	{
+		ignores: ["dist/**"],
+	},
+]
--- a/evals/config/eslint/next.js
+++ b/evals/config/eslint/next.js
@ -0,0 +1,50 @@
+import js from "@eslint/js"
+import eslintConfigPrettier from "eslint-config-prettier"
+import tseslint from "typescript-eslint"
+import pluginReactHooks from "eslint-plugin-react-hooks"
+import pluginReact from "eslint-plugin-react"
+import globals from "globals"
+import pluginNext from "@next/eslint-plugin-next"
+
+import { config as baseConfig } from "./base.js"
+
+/**
+ * A custom ESLint configuration for libraries that use Next.js.
+ *
+ * @type {import("eslint").Linter.Config[]}
+ * */
+export const nextJsConfig = [
+	...baseConfig,
+	js.configs.recommended,
+	eslintConfigPrettier,
+	...tseslint.configs.recommended,
+	{
+		...pluginReact.configs.flat.recommended,
+		languageOptions: {
+			...pluginReact.configs.flat.recommended.languageOptions,
+			globals: {
+				...globals.serviceworker,
+			},
+		},
+	},
+	{
+		plugins: {
+			"@next/next": pluginNext,
+		},
+		rules: {
+			...pluginNext.configs.recommended.rules,
+			...pluginNext.configs["core-web-vitals"].rules,
+		},
+	},
+	{
+		plugins: {
+			"react-hooks": pluginReactHooks,
+		},
+		settings: { react: { version: "detect" } },
+		rules: {
+			...pluginReactHooks.configs.recommended.rules,
+			// React scope no longer necessary with new JSX transform.
+			"react/react-in-jsx-scope": "off",
+		},
+	},
+]
--- a/evals/config/eslint/package.json
+++ b/evals/config/eslint/package.json
@ -0,0 +1,22 @@
+{
+	"name": "@evals/eslint-config",
+	"private": true,
+	"type": "module",
+	"exports": {
+		"./base": "./base.js",
+		"./next-js": "./next.js"
+	},
+	"devDependencies": {
+		"@eslint/js": "^9.22.0",
+		"@next/eslint-plugin-next": "^15.2.1",
+		"eslint": "^9.22.0",
+		"eslint-config-prettier": "^10.1.1",
+		"eslint-plugin-only-warn": "^1.1.0",
+		"eslint-plugin-react": "^7.37.4",
+		"eslint-plugin-react-hooks": "^5.2.0",
+		"eslint-plugin-turbo": "^2.4.4",
+		"globals": "^16.0.0",
+		"typescript": "^5",
+		"typescript-eslint": "^8.26.0"
+	}
+}
--- a/evals/config/typescript/base.json
+++ b/evals/config/typescript/base.json
@ -0,0 +1,19 @@
+{
+	"$schema": "https://json.schemastore.org/tsconfig",
+	"compilerOptions": {
+		"declaration": true,
+		"declarationMap": true,
+		"esModuleInterop": true,
+		"incremental": false,
+		"isolatedModules": true,
+		"lib": ["es2022", "DOM", "DOM.Iterable"],
+		"module": "NodeNext",
+		"moduleDetection": "force",
+		"moduleResolution": "NodeNext",
+		"noUncheckedIndexedAccess": true,
+		"resolveJsonModule": true,
+		"skipLibCheck": true,
+		"strict": true,
+		"target": "ES2022"
+	}
+}
--- a/evals/config/typescript/cjs.json
+++ b/evals/config/typescript/cjs.json
@ -1,4 +1,5 @@
 {
+	"$schema": "https://json.schemastore.org/tsconfig",
 	"compilerOptions": {
 		"module": "CommonJS",
 		"moduleResolution": "Node",
@ -8,9 +9,6 @@
 		"sourceMap": true,
 		"strict": true,
 		"skipLibCheck": true,
-		"useUnknownInCatchVariables": false,
-		"outDir": "out"
-	},
-	"include": ["src", "../src/exports/roo-code.d.ts"],
-	"exclude": ["**/node_modules/**", "out"]
+		"useUnknownInCatchVariables": false
+	}
 }
--- a/evals/config/typescript/nextjs.json
+++ b/evals/config/typescript/nextjs.json
@ -0,0 +1,12 @@
+{
+	"$schema": "https://json.schemastore.org/tsconfig",
+	"extends": "./base.json",
+	"compilerOptions": {
+		"plugins": [{ "name": "next" }],
+		"module": "ESNext",
+		"moduleResolution": "bundler",
+		"allowJs": true,
+		"jsx": "preserve",
+		"noEmit": true
+	}
+}
--- a/evals/config/typescript/package.json
+++ b/evals/config/typescript/package.json
@ -0,0 +1,7 @@
+{
+	"name": "@evals/typescript-config",
+	"private": true,
+	"publishConfig": {
+		"access": "public"
+	}
+}
--- a/evals/package.json
+++ b/evals/package.json
@ -0,0 +1,26 @@
+{
+	"name": "@evals/monorepo",
+	"private": true,
+	"packageManager": "pnpm@10.7.1+sha512.2d92c86b7928dc8284f53494fb4201f983da65f0fb4f0d40baafa5cf628fa31dae3e5968f12466f17df7e97310e30f343a648baea1b9b350685dafafffdf5808",
+	"scripts": {
+		"lint": "turbo lint --log-order grouped --output-logs new-only",
+		"check-types": "turbo check-types --log-order grouped --output-logs new-only",
+		"test": "turbo test --log-order grouped --output-logs new-only",
+		"format": "turbo format --log-order grouped --output-logs new-only",
+		"build": "turbo build --log-order grouped --output-logs new-only",
+		"web": "turbo dev --filter @evals/web",
+		"cli": "turbo dev --filter @evals/cli -- run",
+		"drizzle:studio": "pnpm --filter @evals/db db:studio"
+	},
+	"devDependencies": {
+		"@dotenvx/dotenvx": "^1.39.0",
+		"@eslint/js": "^9.22.0",
+		"eslint": "^9.22.0",
+		"globals": "^16.0.0",
+		"prettier": "^3.5.3",
+		"tsx": "^4.19.3",
+		"turbo": "^2.4.4",
+		"typescript": "^5",
+		"typescript-eslint": "^8.26.0"
+	}
+}
--- a/evals/packages/db/README.md
+++ b/evals/packages/db/README.md
@ -0,0 +1,15 @@
+## Running Migrations
+
+Update `src/schema.ts` as needed, and then run:
+
+```sh
+pnpm db:generate
+```
+
+Inspect the generated sql in the migration filed added to `drizzle/`.
+
+If it looks okay, then run:
+
+```sh
+pnpm db:migrate
+```
--- a/evals/packages/db/drizzle.config.ts
+++ b/evals/packages/db/drizzle.config.ts
@ -0,0 +1,10 @@
+import { defineConfig } from "drizzle-kit"
+
+export default defineConfig({
+	out: "./drizzle",
+	schema: "./src/schema.ts",
+	dialect: "sqlite",
+	dbCredentials: {
+		url: process.env.BENCHMARKS_DB_PATH!,
+	},
+})
--- a/Show more
+++ b/Show more