Move evals into pnpm workspace, switch from SQLite to Postgres (#4278)

apps/web-docs/README.md

@@ -0,0 +1 @@
+TODO

apps/web-evals/.env

@@ -0,0 +1 @@
+DATABASE_URL=postgres://postgres:password@localhost:5432/evals_development

apps/web-evals/.gitignore

@@ -0,0 +1,8 @@
+# .env
+!.env
+
+# next.js
+.next
+
+# typescript
+tsconfig.tsbuildinfo

apps/web-evals/eslint.config.mjs

@@ -1,4 +1,4 @@
-import { nextJsConfig } from "@evals/eslint-config/next-js"
+import { nextJsConfig } from "@roo-code/config-eslint/next-js"
 
 /** @type {import("eslint").Linter.Config} */
 export default [

apps/web-evals/next-env.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

apps/web-evals/package.json

@@ -1,18 +1,15 @@
 {
-	"name": "@evals/web",
+	"name": "@roo-code/web-evals",
 	"private": true,
 	"scripts": {
 		"lint": "next lint",
 		"check-types": "tsc -b",
-		"dev": "dotenvx run -f ../../.env -- next dev --turbopack",
+		"dev": "next dev --turbopack",
 		"format": "prettier --write src",
 		"build": "next build",
 		"start": "next start"
 	},
 	"dependencies": {
-		"@evals/db": "workspace:^",
-		"@evals/ipc": "workspace:^",
-		"@evals/types": "workspace:^",
 		"@hookform/resolvers": "^4.1.3",
 		"@radix-ui/react-alert-dialog": "^1.1.7",
 		"@radix-ui/react-dialog": "^1.1.6",
@@ -26,33 +23,36 @@
 		"@radix-ui/react-slot": "^1.1.2",
 		"@radix-ui/react-tabs": "^1.1.3",
 		"@radix-ui/react-tooltip": "^1.1.8",
+		"@roo-code/evals": "workspace:^",
+		"@roo-code/ipc": "workspace:^",
+		"@roo-code/types": "workspace:^",
 		"@tanstack/react-query": "^5.69.0",
 		"class-variance-authority": "^0.7.1",
 		"clsx": "^2.1.1",
 		"cmdk": "^1.1.0",
 		"fuzzysort": "^3.1.0",
 		"lucide-react": "^0.511.0",
-		"next": "15.3.3",
+		"next": "^15.2.5",
 		"next-themes": "^0.4.6",
 		"p-map": "^7.0.3",
 		"ps-tree": "^1.2.0",
-		"react": "^19.0.0",
-		"react-dom": "^19.0.0",
-		"react-hook-form": "^7.54.2",
+		"react": "^18.3.1",
+		"react-dom": "^18.3.1",
+		"react-hook-form": "^7.57.0",
 		"react-use": "^17.6.0",
-		"sonner": "^2.0.2",
-		"tailwind-merge": "^3.0.2",
+		"sonner": "^2.0.5",
+		"tailwind-merge": "^3.3.0",
 		"tailwindcss-animate": "^1.0.7",
 		"vaul": "^1.1.2",
 		"zod": "^3.24.2"
 	},
 	"devDependencies": {
-		"@evals/eslint-config": "workspace:^",
-		"@evals/typescript-config": "workspace:^",
+		"@roo-code/config-eslint": "workspace:^",
+		"@roo-code/config-typescript": "workspace:^",
 		"@tailwindcss/postcss": "^4",
 		"@types/ps-tree": "^1.1.6",
-		"@types/react": "^19",
-		"@types/react-dom": "^19",
+		"@types/react": "^18.3.18",
+		"@types/react-dom": "^18.3.5",
 		"tailwindcss": "^4"
 	}
 }

apps/web-evals/src/app/api/runs/[id]/stream/route.ts

@@ -1,8 +1,8 @@
 import type { NextRequest } from "next/server"
 
-import { findRun } from "@evals/db"
-import { IpcMessageType } from "@evals/types"
-import { IpcClient } from "@evals/ipc"
+import { findRun } from "@roo-code/evals"
+import { IpcClient } from "@roo-code/ipc"
+import { IpcMessageType } from "@roo-code/types"
 
 import { SSEStream } from "@/lib/server/sse-stream"
 

apps/web-evals/src/app/api/runs/route.ts

@@ -1,6 +1,6 @@
 import { NextResponse } from "next/server"
 
-import { createRun } from "@evals/db"
+import { createRun } from "@roo-code/evals"
 
 export async function POST(request: Request) {
 	try {

apps/web-evals/src/app/api/tasks/route.ts

@@ -1,6 +1,6 @@
 import { NextResponse } from "next/server"
 
-import { createTask } from "@evals/db"
+import { createTask } from "@roo-code/evals"
 
 export async function POST(request: Request) {
 	try {

apps/web-evals/src/app/home.tsx

@@ -5,7 +5,7 @@ import { useRouter } from "next/navigation"
 import Link from "next/link"
 import { Ellipsis, Rocket } from "lucide-react"
 
-import type { Run, TaskMetrics } from "@evals/db"
+import type { Run, TaskMetrics } from "@roo-code/evals"
 
 import { deleteRun } from "@/lib/server/runs"
 import { formatCurrency, formatDuration, formatTokens, formatToolUsageSuccessRate } from "@/lib/formatters"

apps/web-evals/src/app/page.tsx

@@ -1,4 +1,4 @@
-import { getRuns } from "@evals/db"
+import { getRuns } from "@roo-code/evals"
 
 import { Home } from "./home"
 

apps/web-evals/src/app/runs/[id]/page.tsx

@@ -1,4 +1,4 @@
-import { findRun } from "@evals/db"
+import { findRun } from "@roo-code/evals"
 
 import { Run } from "./run"
 

apps/web-evals/src/app/runs/[id]/run.tsx

@@ -3,7 +3,7 @@
 import { useMemo } from "react"
 import { LoaderCircle } from "lucide-react"
 
-import * as db from "@evals/db"
+import type { Run, TaskMetrics as _TaskMetrics } from "@roo-code/evals"
 
 import { formatCurrency, formatDuration, formatTokens } from "@/lib/formatters"
 import { useRunStatus } from "@/hooks/use-run-status"
@@ -12,9 +12,9 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@
 import { TaskStatus } from "./task-status"
 import { ConnectionStatus } from "./connection-status"
 
-type TaskMetrics = Pick<db.TaskMetrics, "tokensIn" | "tokensOut" | "tokensContext" | "duration" | "cost">
+type TaskMetrics = Pick<_TaskMetrics, "tokensIn" | "tokensOut" | "tokensContext" | "duration" | "cost">
 
-export function Run({ run }: { run: db.Run }) {
+export function Run({ run }: { run: Run }) {
 	const { tasks, status, tokenUsage, usageUpdatedAt } = useRunStatus(run)
 
 	const taskMetrics: Record<number, TaskMetrics> = useMemo(() => {

apps/web-evals/src/app/runs/[id]/task-status.tsx

@@ -1,6 +1,6 @@
 import { CircleCheck, CircleDashed, CircleSlash, LoaderCircle } from "lucide-react"
 
-import { type Task } from "@evals/db"
+import type { Task } from "@roo-code/evals"
 
 type TaskStatusProps = {
 	task: Task

apps/web-evals/src/app/runs/new/defaults.ts

@@ -1,4 +1,4 @@
-import { RooCodeSettings } from "./roo-code.js"
+import { RooCodeSettings } from "@roo-code/types"
 
 export const rooCodeDefaults: RooCodeSettings = {
 	apiProvider: "openrouter",

apps/web-evals/src/app/runs/new/new-run.tsx

@@ -9,7 +9,7 @@ import fuzzysort from "fuzzysort"
 import { toast } from "sonner"
 import { X, Rocket, Check, ChevronsUpDown, SlidersHorizontal, Book, CircleCheck } from "lucide-react"
 
-import { globalSettingsSchema, providerSettingsSchema, rooCodeDefaults } from "@evals/types"
+import { globalSettingsSchema, providerSettingsSchema } from "@roo-code/types"
 
 import { createRun } from "@/lib/server/runs"
 import {
@@ -51,6 +51,7 @@ import {
 	DialogFooter,
 } from "@/components/ui"
 
+import { rooCodeDefaults } from "./defaults"
 import { SettingsDiff } from "./settings-diff"
 
 export function NewRun() {

apps/web-evals/src/app/runs/new/settings-diff.tsx

@@ -1,9 +1,11 @@
 import { Fragment, HTMLAttributes } from "react"
 
-import { RooCodeSettings, ROO_CODE_SETTINGS_KEYS } from "@evals/types"
+import { type Keys, type RooCodeSettings, GLOBAL_SETTINGS_KEYS, PROVIDER_SETTINGS_KEYS } from "@roo-code/types"
 
 import { cn } from "@/lib/utils"
 
+export const ROO_CODE_SETTINGS_KEYS = [...GLOBAL_SETTINGS_KEYS, ...PROVIDER_SETTINGS_KEYS] as Keys<RooCodeSettings>[]
+
 type SettingsDiffProps = HTMLAttributes<HTMLDivElement> & {
 	defaultSettings: RooCodeSettings
 	customSettings: RooCodeSettings

apps/web-evals/src/components/ui/scroll-area.tsx

@@ -6,7 +6,7 @@ import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area"
 import { cn } from "@/lib/utils"
 
 type ScrollAreaProps = React.ComponentProps<typeof ScrollAreaPrimitive.Root> & {
-	viewportRef?: React.RefObject<HTMLDivElement | null>
+	viewportRef?: React.RefObject<HTMLDivElement>
 }
 
 function ScrollArea({ className, children, viewportRef, ...props }: ScrollAreaProps) {

apps/web-evals/src/hooks/use-exercises.ts

@@ -2,4 +2,4 @@ import { useQuery } from "@tanstack/react-query"
 
 import { getExercises } from "@/lib/server/exercises"
 
-export const useExercises = () => useQuery({ queryKey: ["exercises"], queryFn: getExercises })
+export const useExercises = () => useQuery({ queryKey: ["exercises"], queryFn: () => getExercises() })

apps/web-evals/src/hooks/use-open-router-models.ts

@@ -0,0 +1,32 @@
+import { z } from "zod"
+import { useQuery } from "@tanstack/react-query"
+
+export const openRouterModelSchema = z.object({
+	id: z.string(),
+	name: z.string(),
+})
+
+export type OpenRouterModel = z.infer<typeof openRouterModelSchema>
+
+export const getOpenRouterModels = async (): Promise<OpenRouterModel[]> => {
+	const response = await fetch("https://openrouter.ai/api/v1/models")
+
+	if (!response.ok) {
+		return []
+	}
+
+	const result = z.object({ data: z.array(openRouterModelSchema) }).safeParse(await response.json())
+
+	if (!result.success) {
+		console.error(result.error)
+		return []
+	}
+
+	return result.data.data.sort((a, b) => a.name.localeCompare(b.name))
+}
+
+export const useOpenRouterModels = () =>
+	useQuery({
+		queryKey: ["getOpenRouterModels"],
+		queryFn: getOpenRouterModels,
+	})

apps/web-evals/src/hooks/use-run-status.ts

@@ -1,8 +1,8 @@
 import { useState, useCallback, useRef } from "react"
 import { useQuery, keepPreviousData } from "@tanstack/react-query"
 
-import { TokenUsage, taskEventSchema, RooCodeEventName, EvalEventName } from "@evals/types"
-import { Run } from "@evals/db"
+import { type TokenUsage, RooCodeEventName, taskEventSchema } from "@roo-code/types"
+import type { Run } from "@roo-code/evals"
 
 import { getTasks } from "@/lib/server/tasks"
 import { useEventSource } from "@/hooks/use-event-source"
@@ -58,8 +58,8 @@ export const useRunStatus = (run: Run) => {
 				setUsageUpdatedAt(Date.now())
 				break
 			}
-			case EvalEventName.Pass:
-			case EvalEventName.Fail:
+			case RooCodeEventName.EvalPass:
+			case RooCodeEventName.EvalFail:
 				setTasksUpdatedAt(Date.now())
 				break
 		}

apps/web-evals/src/lib/schemas.ts

@@ -1,6 +1,6 @@
 import { z } from "zod"
 
-import { rooCodeSettingsSchema } from "@evals/types"
+import { rooCodeSettingsSchema } from "@roo-code/types"
 
 /**
  * CreateRun

apps/web-evals/src/lib/server/exercises.ts

@@ -4,7 +4,7 @@ import * as fs from "fs/promises"
 import * as path from "path"
 import { fileURLToPath } from "url"
 
-import { ExerciseLanguage, exerciseLanguages } from "@evals/types"
+import { type ExerciseLanguage, exerciseLanguages } from "@roo-code/evals"
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url))
 
@@ -20,7 +20,7 @@ export const listDirectories = async (relativePath: string) => {
 }
 
 // __dirname = <repo>/evals/apps/web/src/lib/server
-const EXERCISES_BASE_PATH = path.resolve(__dirname, "../../../../../../../evals")
+const EXERCISES_BASE_PATH = path.resolve(__dirname, "../../../../../../evals")
 
 export const getExercises = async () => {
 	const result = await Promise.all(

apps/web-evals/src/lib/server/runs.ts

@@ -8,14 +8,21 @@ import fs from "fs"
 import { revalidatePath } from "next/cache"
 import pMap from "p-map"
 
-import { ExerciseLanguage, exerciseLanguages } from "@evals/types"
-import * as db from "@evals/db"
+import {
+	type ExerciseLanguage,
+	exerciseLanguages,
+	createRun as _createRun,
+	updateRun as _updateRun,
+	deleteRun as _deleteRun,
+	createTask,
+} from "@roo-code/evals"
 
 import { CreateRun } from "@/lib/schemas"
+
 import { getExercisesForLanguage } from "./exercises"
 
 export async function createRun({ suite, exercises = [], systemPrompt, ...values }: CreateRun) {
-	const run = await db.createRun({
+	const run = await _createRun({
 		...values,
 		socketPath: path.join(os.tmpdir(), `roo-code-evals-${crypto.randomUUID()}.sock`),
 	})
@@ -28,13 +35,13 @@ export async function createRun({ suite, exercises = [], systemPrompt, ...values
 				throw new Error("Invalid exercise path: " + path)
 			}
 
-			await db.createTask({ ...values, runId: run.id, language: language as ExerciseLanguage, exercise })
+			await createTask({ ...values, runId: run.id, language: language as ExerciseLanguage, exercise })
 		}
 	} else {
 		for (const language of exerciseLanguages) {
 			const exercises = await getExercisesForLanguage(language)
 
-			await pMap(exercises, (exercise) => db.createTask({ ...values, runId: run.id, language, exercise }), {
+			await pMap(exercises, (exercise) => createTask({ ...values, runId: run.id, language, exercise }), {
 				concurrency: 10,
 			})
 		}
@@ -49,18 +56,14 @@ export async function createRun({ suite, exercises = [], systemPrompt, ...values
 			? { ...process.env, FOOTGUN_SYSTEM_PROMPT: systemPrompt }
 			: process.env
 
-		const childProcess = spawn(
-			"pnpm",
-			["--filter", "@evals/cli", "dev", "run", "all", "--runId", run.id.toString()],
-			{
-				detached: true,
-				stdio: ["ignore", logFile, logFile],
-				env,
-			},
-		)
+		const childProcess = spawn("pnpm", ["--filter", "@roo-code/evals", "cli", run.id.toString()], {
+			detached: true,
+			stdio: ["ignore", logFile, logFile],
+			env,
+		})
 
 		childProcess.unref()
-		await db.updateRun(run.id, { pid: childProcess.pid })
+		await _updateRun(run.id, { pid: childProcess.pid })
 	} catch (error) {
 		console.error(error)
 	}
@@ -69,6 +72,6 @@ export async function createRun({ suite, exercises = [], systemPrompt, ...values
 }
 
 export async function deleteRun(runId: number) {
-	await db.deleteRun(runId)
+	await _deleteRun(runId)
 	revalidatePath("/runs")
 }

apps/web-evals/src/lib/server/tasks.ts

@@ -2,10 +2,10 @@
 
 import { revalidatePath } from "next/cache"
 
-import * as db from "@evals/db"
+import { getTasks as _getTasks } from "@roo-code/evals"
 
 export async function getTasks(runId: number) {
-	const tasks = await db.getTasks(runId)
+	const tasks = await _getTasks(runId)
 	revalidatePath(`/runs/${runId}`)
 	return tasks
 }

apps/web-evals/tsconfig.json

@@ -1,5 +1,5 @@
 {
-	"extends": "@evals/typescript-config/nextjs.json",
+	"extends": "@roo-code/config-typescript/nextjs.json",
 	"compilerOptions": {
 		"plugins": [{ "name": "next" }],
 		"paths": { "@/*": ["./src/*"] }

apps/web-roo-code/README.md

@@ -0,0 +1 @@
+TODO

evals/.env.sample

@@ -1 +0,0 @@
-BENCHMARKS_DB_PATH=file:/tmp/evals.db

evals/.gitignore

@@ -1,42 +0,0 @@
-# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
-
-# Dependencies
-node_modules
-.pnp
-.pnp.js
-
-# Local env files
-.env
-.env.*
-!.env.sample
-
-# Testing
-coverage
-
-# Turbo
-.turbo
-
-# Vercel
-.vercel
-
-# Next.js
-next-env.d.ts
-
-# Build Outputs
-.next/
-out/
-build
-dist
-*.tsbuildinfo
-
-# Debug
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-
-# Misc
-.DS_Store
-*.pem
-
-# Evals
-evals

evals/.npmrc

@@ -1,2 +0,0 @@
-# https://github.com/vercel/next.js/issues/68805
-public-hoist-pattern[]=*libsql*

evals/.tool-versions

@@ -1,4 +0,0 @@
-python 3.13.2
-golang 1.24.2
-rust 1.85.1
-nodejs 20.19.2

evals/apps/cli/package.json

@@ -1,27 +0,0 @@
-{
-	"name": "@evals/cli",
-	"private": true,
-	"type": "module",
-	"scripts": {
-		"lint": "eslint src/**/*.ts --max-warnings=0",
-		"check-types": "tsc --noEmit",
-		"format": "prettier --write src",
-		"dev": "dotenvx run -f ../../.env -- tsx src/index.ts"
-	},
-	"dependencies": {
-		"@evals/db": "workspace:^",
-		"@evals/ipc": "workspace:^",
-		"@evals/lib": "workspace:^",
-		"@evals/types": "workspace:^",
-		"execa": "^9.5.2",
-		"gluegun": "^5.1.2",
-		"p-map": "^7.0.3",
-		"p-wait-for": "^5.0.2",
-		"ps-tree": "^1.2.0"
-	},
-	"devDependencies": {
-		"@evals/eslint-config": "workspace:^",
-		"@evals/typescript-config": "workspace:^",
-		"@types/ps-tree": "^1.1.6"
-	}
-}

evals/apps/cli/src/exercises.ts

@@ -1,31 +0,0 @@
-import * as path from "path"
-import * as fs from "fs"
-
-import { filesystem } from "gluegun"
-
-import { type ExerciseLanguage, exerciseLanguages } from "@evals/types"
-
-import { exercisesPath } from "./paths.js"
-
-let exercisesByLanguage: Record<ExerciseLanguage, string[]> | null = null
-
-export const getExercises = () => {
-	if (exercisesByLanguage !== null) {
-		return exercisesByLanguage
-	}
-
-	const getLanguageExercises = (language: ExerciseLanguage) =>
-		fs.existsSync(path.resolve(exercisesPath, language))
-			? filesystem
-					.subdirectories(path.resolve(exercisesPath, language))
-					.map((exercise) => path.basename(exercise))
-					.filter((exercise) => !exercise.startsWith("."))
-			: []
-
-	exercisesByLanguage = exerciseLanguages.reduce(
-		(collect, language) => ({ ...collect, [language]: getLanguageExercises(language) }),
-		{} as Record<ExerciseLanguage, string[]>,
-	)
-
-	return exercisesByLanguage
-}

evals/apps/cli/src/paths.ts

@@ -1,7 +0,0 @@
-import * as path from "path"
-import { fileURLToPath } from "url"
-
-export const __dirname = path.dirname(fileURLToPath(import.meta.url))
-
-export const extensionDevelopmentPath = path.resolve(__dirname, "..", "..", "..", "..")
-export const exercisesPath = path.resolve(extensionDevelopmentPath, "..", "evals")

evals/apps/cli/tsconfig.json

@@ -1,5 +0,0 @@
-{
-	"extends": "@evals/typescript-config/base.json",
-	"include": ["src"],
-	"exclude": ["node_modules"]
-}

evals/apps/web/src/hooks/use-open-router-models.ts

@@ -1,75 +0,0 @@
-import { z } from "zod"
-import { useQuery } from "@tanstack/react-query"
-
-import { type ModelInfo } from "@evals/types"
-
-const supportsPromptCache = ["anthropic/claude-3.7-sonnet", "anthropic/claude-3.5-sonnet", "anthropic/claude-3-5-haiku"]
-
-const supportsComputerUse = ["anthropic/claude-3.7-sonnet", "anthropic/claude-3.5-sonnet"]
-
-const supportsThinking = ["anthropic/claude-3.7-sonnet:thinking"]
-
-const parsePrice = (price?: string) => (price ? parseFloat(price) * 1_000_000 : undefined)
-
-export const openRouterModelSchema = z.object({
-	id: z.string(),
-	name: z.string(),
-	description: z.string(),
-	created: z.number(),
-	context_length: z.number(),
-	pricing: z.object({
-		prompt: z.string().optional(),
-		completion: z.string().optional(),
-	}),
-	top_provider: z
-		.object({
-			max_completion_tokens: z.number().nullish(),
-		})
-		.optional(),
-	architecture: z
-		.object({
-			modality: z.string(),
-		})
-		.optional(),
-})
-
-export type OpenRouterModel = z.infer<typeof openRouterModelSchema> & { modelInfo: ModelInfo }
-
-export const getOpenRouterModels = async (): Promise<OpenRouterModel[]> => {
-	const response = await fetch("https://openrouter.ai/api/v1/models")
-
-	if (!response.ok) {
-		console.error("Failed to fetch OpenRouter models")
-		return []
-	}
-
-	const result = z.object({ data: z.array(openRouterModelSchema) }).safeParse(await response.json())
-
-	if (!result.success) {
-		console.error(result.error)
-		return []
-	}
-
-	return result.data.data
-		.sort((a, b) => a.name.localeCompare(b.name))
-		.map((rawModel) => ({
-			...rawModel,
-			modelInfo: {
-				maxTokens: rawModel.top_provider?.max_completion_tokens ?? undefined,
-				contextWindow: rawModel.context_length,
-				supportsImages: rawModel.architecture?.modality?.includes("image"),
-				supportsPromptCache: supportsPromptCache.some((model) => rawModel.id.startsWith(model)),
-				supportsComputerUse: supportsComputerUse.some((model) => rawModel.id.startsWith(model)),
-				inputPrice: parsePrice(rawModel.pricing?.prompt),
-				outputPrice: parsePrice(rawModel.pricing?.completion),
-				description: rawModel.description,
-				thinking: supportsThinking.some((model) => rawModel.id.startsWith(model)),
-			},
-		}))
-}
-
-export const useOpenRouterModels = () =>
-	useQuery<OpenRouterModel[]>({
-		queryKey: ["getOpenRouterModels"],
-		queryFn: getOpenRouterModels,
-	})

evals/config/eslint/base.js

@@ -1,32 +0,0 @@
-import js from "@eslint/js"
-import eslintConfigPrettier from "eslint-config-prettier"
-import turboPlugin from "eslint-plugin-turbo"
-import tseslint from "typescript-eslint"
-import onlyWarn from "eslint-plugin-only-warn"
-
-/**
- * A shared ESLint configuration for the repository.
- *
- * @type {import("eslint").Linter.Config[]}
- * */
-export const config = [
-	js.configs.recommended,
-	eslintConfigPrettier,
-	...tseslint.configs.recommended,
-	{
-		plugins: {
-			turbo: turboPlugin,
-		},
-		rules: {
-			"turbo/no-undeclared-env-vars": "warn",
-		},
-	},
-	{
-		plugins: {
-			onlyWarn,
-		},
-	},
-	{
-		ignores: ["dist/**"],
-	},
-]

evals/config/eslint/next.js

@@ -1,50 +0,0 @@
-import js from "@eslint/js"
-import eslintConfigPrettier from "eslint-config-prettier"
-import tseslint from "typescript-eslint"
-import pluginReactHooks from "eslint-plugin-react-hooks"
-import pluginReact from "eslint-plugin-react"
-import globals from "globals"
-import pluginNext from "@next/eslint-plugin-next"
-
-import { config as baseConfig } from "./base.js"
-
-/**
- * A custom ESLint configuration for libraries that use Next.js.
- *
- * @type {import("eslint").Linter.Config[]}
- * */
-export const nextJsConfig = [
-	...baseConfig,
-	js.configs.recommended,
-	eslintConfigPrettier,
-	...tseslint.configs.recommended,
-	{
-		...pluginReact.configs.flat.recommended,
-		languageOptions: {
-			...pluginReact.configs.flat.recommended.languageOptions,
-			globals: {
-				...globals.serviceworker,
-			},
-		},
-	},
-	{
-		plugins: {
-			"@next/next": pluginNext,
-		},
-		rules: {
-			...pluginNext.configs.recommended.rules,
-			...pluginNext.configs["core-web-vitals"].rules,
-		},
-	},
-	{
-		plugins: {
-			"react-hooks": pluginReactHooks,
-		},
-		settings: { react: { version: "detect" } },
-		rules: {
-			...pluginReactHooks.configs.recommended.rules,
-			// React scope no longer necessary with new JSX transform.
-			"react/react-in-jsx-scope": "off",
-		},
-	},
-]

evals/config/eslint/package.json

@@ -1,21 +0,0 @@
-{
-	"name": "@evals/eslint-config",
-	"private": true,
-	"type": "module",
-	"exports": {
-		"./base": "./base.js",
-		"./next-js": "./next.js"
-	},
-	"devDependencies": {
-		"@eslint/js": "^9.22.0",
-		"@next/eslint-plugin-next": "^15.2.1",
-		"eslint": "^9.22.0",
-		"eslint-config-prettier": "^10.1.1",
-		"eslint-plugin-only-warn": "^1.1.0",
-		"eslint-plugin-react": "^7.37.4",
-		"eslint-plugin-react-hooks": "^5.2.0",
-		"eslint-plugin-turbo": "^2.4.4",
-		"globals": "^16.0.0",
-		"typescript-eslint": "^8.26.0"
-	}
-}

evals/config/typescript/base.json

@@ -1,19 +0,0 @@
-{
-	"$schema": "https://json.schemastore.org/tsconfig",
-	"compilerOptions": {
-		"declaration": true,
-		"declarationMap": true,
-		"esModuleInterop": true,
-		"incremental": false,
-		"isolatedModules": true,
-		"lib": ["es2022", "DOM", "DOM.Iterable"],
-		"module": "NodeNext",
-		"moduleDetection": "force",
-		"moduleResolution": "NodeNext",
-		"noUncheckedIndexedAccess": true,
-		"resolveJsonModule": true,
-		"skipLibCheck": true,
-		"strict": true,
-		"target": "ES2022"
-	}
-}

evals/config/typescript/cjs.json

@@ -1,14 +0,0 @@
-{
-	"$schema": "https://json.schemastore.org/tsconfig",
-	"compilerOptions": {
-		"module": "CommonJS",
-		"moduleResolution": "Node",
-		"esModuleInterop": true,
-		"target": "ES2022",
-		"lib": ["ES2022", "ESNext.Disposable", "DOM"],
-		"sourceMap": true,
-		"strict": true,
-		"skipLibCheck": true,
-		"useUnknownInCatchVariables": false
-	}
-}

evals/config/typescript/nextjs.json

@@ -1,12 +0,0 @@
-{
-	"$schema": "https://json.schemastore.org/tsconfig",
-	"extends": "./base.json",
-	"compilerOptions": {
-		"plugins": [{ "name": "next" }],
-		"module": "ESNext",
-		"moduleResolution": "bundler",
-		"allowJs": true,
-		"jsx": "preserve",
-		"noEmit": true
-	}
-}

evals/config/typescript/package.json

@@ -1,7 +0,0 @@
-{
-	"name": "@evals/typescript-config",
-	"private": true,
-	"publishConfig": {
-		"access": "public"
-	}
-}

evals/package.json

@@ -1,33 +0,0 @@
-{
-	"name": "@evals/monorepo",
-	"private": true,
-	"packageManager": "pnpm@10.8.1",
-	"scripts": {
-		"lint": "turbo lint --log-order grouped --output-logs new-only",
-		"check-types": "turbo check-types --log-order grouped --output-logs new-only",
-		"test": "turbo test --log-order grouped --output-logs new-only",
-		"format": "turbo format --log-order grouped --output-logs new-only",
-		"build": "turbo build --log-order grouped --output-logs new-only",
-		"web": "turbo dev --filter @evals/web",
-		"cli": "turbo dev --filter @evals/cli -- run",
-		"drizzle:studio": "pnpm --filter @evals/db db:studio",
-		"docker:build": "docker build -f Dockerfile -t roo-code-eval --progress=plain ..",
-		"docker:run": "touch /tmp/evals.db && docker run -d -it -p 3000:3000 -v /tmp/evals.db:/tmp/evals.db roo-code-eval",
-		"docker:start": "pnpm docker:build && pnpm docker:run",
-		"docker:shell": "docker exec -it $(docker ps --filter \"ancestor=roo-code-eval\" -q) /bin/bash",
-		"docker:stop": "docker stop $(docker ps --filter \"ancestor=roo-code-eval\" -q)",
-		"docker:rm": "docker rm $(docker ps -a --filter \"ancestor=roo-code-eval\" -q)",
-		"docker:clean": "pnpm docker:stop && pnpm docker:rm"
-	},
-	"devDependencies": {
-		"@dotenvx/dotenvx": "^1.41.0",
-		"@eslint/js": "^9.25.1",
-		"eslint": "^9.25.1",
-		"globals": "^16.0.0",
-		"prettier": "^3.5.3",
-		"tsx": "^4.19.4",
-		"turbo": "^2.5.2",
-		"typescript": "5.8.3",
-		"typescript-eslint": "^8.31.1"
-	}
-}

evals/packages/db/.gitignore

@@ -1 +0,0 @@
-test.db

evals/packages/db/README.md

@@ -1,15 +0,0 @@
-## Running Migrations
-
-Update `src/schema.ts` as needed, and then run:
-
-```sh
-pnpm db:generate
-```
-
-Inspect the sql in the migration file added to `drizzle/`.
-
-If it looks okay, then run:
-
-```sh
-pnpm db:migrate
-```

evals/packages/db/drizzle.config.ts

@@ -1,18 +0,0 @@
-import { defineConfig } from "drizzle-kit"
-
-if ((!process.env.TURSO_CONNECTION_URL || !process.env.TURSO_AUTH_TOKEN) && !process.env.BENCHMARKS_DB_PATH) {
-	throw new Error("TURSO_CONNECTION_URL and TURSO_AUTH_TOKEN or BENCHMARKS_DB_PATH must be set")
-}
-
-const dialect = process.env.BENCHMARKS_DB_PATH ? "sqlite" : "turso"
-
-const dbCredentials = process.env.BENCHMARKS_DB_PATH
-	? { url: process.env.BENCHMARKS_DB_PATH }
-	: { url: process.env.TURSO_CONNECTION_URL!, authToken: process.env.TURSO_AUTH_TOKEN! }
-
-export default defineConfig({
-	out: "./drizzle",
-	schema: "./src/schema.ts",
-	dialect,
-	dbCredentials,
-})

evals/packages/db/drizzle/0000_elite_raza.sql

@@ -1,40 +0,0 @@
-CREATE TABLE `runs` (
-	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-	`taskMetricsId` integer,
-	`model` text NOT NULL,
-	`description` text,
-	`pid` integer,
-	`socketPath` text NOT NULL,
-	`passed` integer DEFAULT 0 NOT NULL,
-	`failed` integer DEFAULT 0 NOT NULL,
-	`createdAt` integer NOT NULL,
-	FOREIGN KEY (`taskMetricsId`) REFERENCES `taskMetrics`(`id`) ON UPDATE no action ON DELETE no action
-);
---> statement-breakpoint
-CREATE TABLE `taskMetrics` (
-	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-	`tokensIn` integer NOT NULL,
-	`tokensOut` integer NOT NULL,
-	`tokensContext` integer NOT NULL,
-	`cacheWrites` integer NOT NULL,
-	`cacheReads` integer NOT NULL,
-	`cost` real NOT NULL,
-	`duration` integer NOT NULL,
-	`createdAt` integer NOT NULL
-);
---> statement-breakpoint
-CREATE TABLE `tasks` (
-	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-	`runId` integer NOT NULL,
-	`taskMetricsId` integer,
-	`language` text NOT NULL,
-	`exercise` text NOT NULL,
-	`passed` integer,
-	`startedAt` integer,
-	`finishedAt` integer,
-	`createdAt` integer NOT NULL,
-	FOREIGN KEY (`runId`) REFERENCES `runs`(`id`) ON UPDATE no action ON DELETE no action,
-	FOREIGN KEY (`taskMetricsId`) REFERENCES `taskMetrics`(`id`) ON UPDATE no action ON DELETE no action
-);
---> statement-breakpoint
-CREATE UNIQUE INDEX `tasks_language_exercise_idx` ON `tasks` (`runId`,`language`,`exercise`);

evals/packages/db/drizzle/0001_lush_reavers.sql

@@ -1 +0,0 @@
-ALTER TABLE `runs` ADD `settings` blob;

evals/packages/db/drizzle/0002_white_flatman.sql

@@ -1 +0,0 @@
-ALTER TABLE `runs` ADD `concurrency` integer DEFAULT 2 NOT NULL;

evals/packages/db/drizzle/0003_sweet_chimera.sql

@@ -1 +0,0 @@
-ALTER TABLE `taskMetrics` ADD `toolUsage` text;

evals/packages/db/drizzle/0004_absent_slapstick.sql

@@ -1,10 +0,0 @@
-CREATE TABLE `toolErrors` (
-	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-	`runId` integer,
-	`taskId` integer,
-	`toolName` text NOT NULL,
-	`error` text NOT NULL,
-	`createdAt` integer NOT NULL,
-	FOREIGN KEY (`runId`) REFERENCES `runs`(`id`) ON UPDATE no action ON DELETE no action,
-	FOREIGN KEY (`taskId`) REFERENCES `tasks`(`id`) ON UPDATE no action ON DELETE no action
-);

evals/packages/db/drizzle/meta/0000_snapshot.json

@@ -1,274 +0,0 @@
-{
-	"version": "6",
-	"dialect": "sqlite",
-	"id": "c0fa8491-b5c0-493d-aa32-ddf280259c30",
-	"prevId": "00000000-0000-0000-0000-000000000000",
-	"tables": {
-		"runs": {
-			"name": "runs",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"taskMetricsId": {
-					"name": "taskMetricsId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"model": {
-					"name": "model",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"description": {
-					"name": "description",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"pid": {
-					"name": "pid",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"socketPath": {
-					"name": "socketPath",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"passed": {
-					"name": "passed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 0
-				},
-				"failed": {
-					"name": "failed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 0
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {},
-			"foreignKeys": {
-				"runs_taskMetricsId_taskMetrics_id_fk": {
-					"name": "runs_taskMetricsId_taskMetrics_id_fk",
-					"tableFrom": "runs",
-					"tableTo": "taskMetrics",
-					"columnsFrom": ["taskMetricsId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				}
-			},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
-		"taskMetrics": {
-			"name": "taskMetrics",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"tokensIn": {
-					"name": "tokensIn",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"tokensOut": {
-					"name": "tokensOut",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"tokensContext": {
-					"name": "tokensContext",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cacheWrites": {
-					"name": "cacheWrites",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cacheReads": {
-					"name": "cacheReads",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cost": {
-					"name": "cost",
-					"type": "real",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"duration": {
-					"name": "duration",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {},
-			"foreignKeys": {},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
-		"tasks": {
-			"name": "tasks",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"runId": {
-					"name": "runId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"taskMetricsId": {
-					"name": "taskMetricsId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"language": {
-					"name": "language",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"exercise": {
-					"name": "exercise",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"passed": {
-					"name": "passed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"startedAt": {
-					"name": "startedAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"finishedAt": {
-					"name": "finishedAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {
-				"tasks_language_exercise_idx": {
-					"name": "tasks_language_exercise_idx",
-					"columns": ["runId", "language", "exercise"],
-					"isUnique": true
-				}
-			},
-			"foreignKeys": {
-				"tasks_runId_runs_id_fk": {
-					"name": "tasks_runId_runs_id_fk",
-					"tableFrom": "tasks",
-					"tableTo": "runs",
-					"columnsFrom": ["runId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				},
-				"tasks_taskMetricsId_taskMetrics_id_fk": {
-					"name": "tasks_taskMetricsId_taskMetrics_id_fk",
-					"tableFrom": "tasks",
-					"tableTo": "taskMetrics",
-					"columnsFrom": ["taskMetricsId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				}
-			},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		}
-	},
-	"views": {},
-	"enums": {},
-	"_meta": {
-		"schemas": {},
-		"tables": {},
-		"columns": {}
-	},
-	"internal": {
-		"indexes": {}
-	}
-}

evals/packages/db/drizzle/meta/0001_snapshot.json

@@ -1,281 +0,0 @@
-{
-	"version": "6",
-	"dialect": "sqlite",
-	"id": "8906647f-81d6-498a-897c-b1638c04c69a",
-	"prevId": "c0fa8491-b5c0-493d-aa32-ddf280259c30",
-	"tables": {
-		"runs": {
-			"name": "runs",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"taskMetricsId": {
-					"name": "taskMetricsId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"model": {
-					"name": "model",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"description": {
-					"name": "description",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"settings": {
-					"name": "settings",
-					"type": "blob",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"pid": {
-					"name": "pid",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"socketPath": {
-					"name": "socketPath",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"passed": {
-					"name": "passed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 0
-				},
-				"failed": {
-					"name": "failed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 0
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {},
-			"foreignKeys": {
-				"runs_taskMetricsId_taskMetrics_id_fk": {
-					"name": "runs_taskMetricsId_taskMetrics_id_fk",
-					"tableFrom": "runs",
-					"tableTo": "taskMetrics",
-					"columnsFrom": ["taskMetricsId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				}
-			},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
-		"taskMetrics": {
-			"name": "taskMetrics",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"tokensIn": {
-					"name": "tokensIn",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"tokensOut": {
-					"name": "tokensOut",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"tokensContext": {
-					"name": "tokensContext",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cacheWrites": {
-					"name": "cacheWrites",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cacheReads": {
-					"name": "cacheReads",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cost": {
-					"name": "cost",
-					"type": "real",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"duration": {
-					"name": "duration",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {},
-			"foreignKeys": {},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
-		"tasks": {
-			"name": "tasks",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"runId": {
-					"name": "runId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"taskMetricsId": {
-					"name": "taskMetricsId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"language": {
-					"name": "language",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"exercise": {
-					"name": "exercise",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"passed": {
-					"name": "passed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"startedAt": {
-					"name": "startedAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"finishedAt": {
-					"name": "finishedAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {
-				"tasks_language_exercise_idx": {
-					"name": "tasks_language_exercise_idx",
-					"columns": ["runId", "language", "exercise"],
-					"isUnique": true
-				}
-			},
-			"foreignKeys": {
-				"tasks_runId_runs_id_fk": {
-					"name": "tasks_runId_runs_id_fk",
-					"tableFrom": "tasks",
-					"tableTo": "runs",
-					"columnsFrom": ["runId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				},
-				"tasks_taskMetricsId_taskMetrics_id_fk": {
-					"name": "tasks_taskMetricsId_taskMetrics_id_fk",
-					"tableFrom": "tasks",
-					"tableTo": "taskMetrics",
-					"columnsFrom": ["taskMetricsId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				}
-			},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		}
-	},
-	"views": {},
-	"enums": {},
-	"_meta": {
-		"schemas": {},
-		"tables": {},
-		"columns": {}
-	},
-	"internal": {
-		"indexes": {}
-	}
-}

evals/packages/db/drizzle/meta/0002_snapshot.json

@@ -1,289 +0,0 @@
-{
-	"version": "6",
-	"dialect": "sqlite",
-	"id": "f49d9b0b-fda9-467a-9adb-c941d6cbf7ce",
-	"prevId": "8906647f-81d6-498a-897c-b1638c04c69a",
-	"tables": {
-		"runs": {
-			"name": "runs",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"taskMetricsId": {
-					"name": "taskMetricsId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"model": {
-					"name": "model",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"description": {
-					"name": "description",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"settings": {
-					"name": "settings",
-					"type": "blob",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"pid": {
-					"name": "pid",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"socketPath": {
-					"name": "socketPath",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"concurrency": {
-					"name": "concurrency",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 2
-				},
-				"passed": {
-					"name": "passed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 0
-				},
-				"failed": {
-					"name": "failed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 0
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {},
-			"foreignKeys": {
-				"runs_taskMetricsId_taskMetrics_id_fk": {
-					"name": "runs_taskMetricsId_taskMetrics_id_fk",
-					"tableFrom": "runs",
-					"tableTo": "taskMetrics",
-					"columnsFrom": ["taskMetricsId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				}
-			},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
-		"taskMetrics": {
-			"name": "taskMetrics",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"tokensIn": {
-					"name": "tokensIn",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"tokensOut": {
-					"name": "tokensOut",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"tokensContext": {
-					"name": "tokensContext",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cacheWrites": {
-					"name": "cacheWrites",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cacheReads": {
-					"name": "cacheReads",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cost": {
-					"name": "cost",
-					"type": "real",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"duration": {
-					"name": "duration",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {},
-			"foreignKeys": {},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
-		"tasks": {
-			"name": "tasks",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"runId": {
-					"name": "runId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"taskMetricsId": {
-					"name": "taskMetricsId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"language": {
-					"name": "language",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"exercise": {
-					"name": "exercise",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"passed": {
-					"name": "passed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"startedAt": {
-					"name": "startedAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"finishedAt": {
-					"name": "finishedAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {
-				"tasks_language_exercise_idx": {
-					"name": "tasks_language_exercise_idx",
-					"columns": ["runId", "language", "exercise"],
-					"isUnique": true
-				}
-			},
-			"foreignKeys": {
-				"tasks_runId_runs_id_fk": {
-					"name": "tasks_runId_runs_id_fk",
-					"tableFrom": "tasks",
-					"tableTo": "runs",
-					"columnsFrom": ["runId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				},
-				"tasks_taskMetricsId_taskMetrics_id_fk": {
-					"name": "tasks_taskMetricsId_taskMetrics_id_fk",
-					"tableFrom": "tasks",
-					"tableTo": "taskMetrics",
-					"columnsFrom": ["taskMetricsId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				}
-			},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		}
-	},
-	"views": {},
-	"enums": {},
-	"_meta": {
-		"schemas": {},
-		"tables": {},
-		"columns": {}
-	},
-	"internal": {
-		"indexes": {}
-	}
-}

evals/packages/db/drizzle/meta/0003_snapshot.json

@@ -1,296 +0,0 @@
-{
-	"version": "6",
-	"dialect": "sqlite",
-	"id": "61d48d20-f662-445d-9962-cf9cb165cbe7",
-	"prevId": "f49d9b0b-fda9-467a-9adb-c941d6cbf7ce",
-	"tables": {
-		"runs": {
-			"name": "runs",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"taskMetricsId": {
-					"name": "taskMetricsId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"model": {
-					"name": "model",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"description": {
-					"name": "description",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"settings": {
-					"name": "settings",
-					"type": "blob",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"pid": {
-					"name": "pid",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"socketPath": {
-					"name": "socketPath",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"concurrency": {
-					"name": "concurrency",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 2
-				},
-				"passed": {
-					"name": "passed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 0
-				},
-				"failed": {
-					"name": "failed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 0
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {},
-			"foreignKeys": {
-				"runs_taskMetricsId_taskMetrics_id_fk": {
-					"name": "runs_taskMetricsId_taskMetrics_id_fk",
-					"tableFrom": "runs",
-					"tableTo": "taskMetrics",
-					"columnsFrom": ["taskMetricsId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				}
-			},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
-		"taskMetrics": {
-			"name": "taskMetrics",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"tokensIn": {
-					"name": "tokensIn",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"tokensOut": {
-					"name": "tokensOut",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"tokensContext": {
-					"name": "tokensContext",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cacheWrites": {
-					"name": "cacheWrites",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cacheReads": {
-					"name": "cacheReads",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cost": {
-					"name": "cost",
-					"type": "real",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"duration": {
-					"name": "duration",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"toolUsage": {
-					"name": "toolUsage",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {},
-			"foreignKeys": {},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
-		"tasks": {
-			"name": "tasks",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"runId": {
-					"name": "runId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"taskMetricsId": {
-					"name": "taskMetricsId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"language": {
-					"name": "language",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"exercise": {
-					"name": "exercise",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"passed": {
-					"name": "passed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"startedAt": {
-					"name": "startedAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"finishedAt": {
-					"name": "finishedAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {
-				"tasks_language_exercise_idx": {
-					"name": "tasks_language_exercise_idx",
-					"columns": ["runId", "language", "exercise"],
-					"isUnique": true
-				}
-			},
-			"foreignKeys": {
-				"tasks_runId_runs_id_fk": {
-					"name": "tasks_runId_runs_id_fk",
-					"tableFrom": "tasks",
-					"tableTo": "runs",
-					"columnsFrom": ["runId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				},
-				"tasks_taskMetricsId_taskMetrics_id_fk": {
-					"name": "tasks_taskMetricsId_taskMetrics_id_fk",
-					"tableFrom": "tasks",
-					"tableTo": "taskMetrics",
-					"columnsFrom": ["taskMetricsId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				}
-			},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		}
-	},
-	"views": {},
-	"enums": {},
-	"_meta": {
-		"schemas": {},
-		"tables": {},
-		"columns": {}
-	},
-	"internal": {
-		"indexes": {}
-	}
-}

evals/packages/db/drizzle/meta/0004_snapshot.json

@@ -1,367 +0,0 @@
-{
-	"version": "6",
-	"dialect": "sqlite",
-	"id": "ae766c54-aff4-4ce6-b492-24813790c279",
-	"prevId": "61d48d20-f662-445d-9962-cf9cb165cbe7",
-	"tables": {
-		"runs": {
-			"name": "runs",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"taskMetricsId": {
-					"name": "taskMetricsId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"model": {
-					"name": "model",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"description": {
-					"name": "description",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"settings": {
-					"name": "settings",
-					"type": "blob",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"pid": {
-					"name": "pid",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"socketPath": {
-					"name": "socketPath",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"concurrency": {
-					"name": "concurrency",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 2
-				},
-				"passed": {
-					"name": "passed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 0
-				},
-				"failed": {
-					"name": "failed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false,
-					"default": 0
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {},
-			"foreignKeys": {
-				"runs_taskMetricsId_taskMetrics_id_fk": {
-					"name": "runs_taskMetricsId_taskMetrics_id_fk",
-					"tableFrom": "runs",
-					"tableTo": "taskMetrics",
-					"columnsFrom": ["taskMetricsId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				}
-			},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
-		"taskMetrics": {
-			"name": "taskMetrics",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"tokensIn": {
-					"name": "tokensIn",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"tokensOut": {
-					"name": "tokensOut",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"tokensContext": {
-					"name": "tokensContext",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cacheWrites": {
-					"name": "cacheWrites",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cacheReads": {
-					"name": "cacheReads",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"cost": {
-					"name": "cost",
-					"type": "real",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"duration": {
-					"name": "duration",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"toolUsage": {
-					"name": "toolUsage",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {},
-			"foreignKeys": {},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
-		"tasks": {
-			"name": "tasks",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"runId": {
-					"name": "runId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"taskMetricsId": {
-					"name": "taskMetricsId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"language": {
-					"name": "language",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"exercise": {
-					"name": "exercise",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"passed": {
-					"name": "passed",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"startedAt": {
-					"name": "startedAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"finishedAt": {
-					"name": "finishedAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {
-				"tasks_language_exercise_idx": {
-					"name": "tasks_language_exercise_idx",
-					"columns": ["runId", "language", "exercise"],
-					"isUnique": true
-				}
-			},
-			"foreignKeys": {
-				"tasks_runId_runs_id_fk": {
-					"name": "tasks_runId_runs_id_fk",
-					"tableFrom": "tasks",
-					"tableTo": "runs",
-					"columnsFrom": ["runId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				},
-				"tasks_taskMetricsId_taskMetrics_id_fk": {
-					"name": "tasks_taskMetricsId_taskMetrics_id_fk",
-					"tableFrom": "tasks",
-					"tableTo": "taskMetrics",
-					"columnsFrom": ["taskMetricsId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				}
-			},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
-		"toolErrors": {
-			"name": "toolErrors",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"runId": {
-					"name": "runId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"taskId": {
-					"name": "taskId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"toolName": {
-					"name": "toolName",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"error": {
-					"name": "error",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {},
-			"foreignKeys": {
-				"toolErrors_runId_runs_id_fk": {
-					"name": "toolErrors_runId_runs_id_fk",
-					"tableFrom": "toolErrors",
-					"tableTo": "runs",
-					"columnsFrom": ["runId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				},
-				"toolErrors_taskId_tasks_id_fk": {
-					"name": "toolErrors_taskId_tasks_id_fk",
-					"tableFrom": "toolErrors",
-					"tableTo": "tasks",
-					"columnsFrom": ["taskId"],
-					"columnsTo": ["id"],
-					"onDelete": "no action",
-					"onUpdate": "no action"
-				}
-			},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		}
-	},
-	"views": {},
-	"enums": {},
-	"_meta": {
-		"schemas": {},
-		"tables": {},
-		"columns": {}
-	},
-	"internal": {
-		"indexes": {}
-	}
-}