More progress

benchmark/apps/cli/package.json

@@ -11,7 +11,8 @@
 	"dependencies": {
 		"@benchmark/db": "workspace:^",
 		"@vscode/test-electron": "^2.4.0",
-		"gluegun": "^5.1.2"
+		"gluegun": "^5.1.2",
+		"p-map": "^7.0.3"
 	},
 	"devDependencies": {
 		"@benchmark/eslint-config": "workspace:^",

benchmark/apps/cli/src/index.ts

@@ -2,19 +2,11 @@ import * as fs from "fs"
 import * as path from "path"
 import * as os from "os"
 
+import pMap from "p-map"
 import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun"
 import { runTests } from "@vscode/test-electron"
 
-import {
-	type Language,
-	languages,
-	type Run,
-	findRun,
-	createRun,
-	getPendingTask,
-	createPendingTask,
-	getTask,
-} from "@benchmark/db"
+import { type Language, languages, type Run, findRun, createRun, getTask, createTask, Task } from "@benchmark/db"
 
 import { __dirname, extensionDevelopmentPath, extensionTestsPath, exercisesPath } from "./paths.js"
 import { getExercises } from "./exercises.js"
@@ -41,34 +33,33 @@ const run = async (toolbox: GluegunToolbox) => {
 
 const runAll = async (id?: number) => {
 	const run = await findOrCreateRun({ id })
-	const exercises = getExercises()
 
-	for (const [language, languageExercises] of Object.entries(exercises)) {
-		await Promise.all(
-			languageExercises.map((exercise) =>
-				findOrCreatePendingTask({ runId: run.id, language: language as Language, exercise }),
-			),
-		)
-	}
+	const entries = Object.entries(getExercises()).flatMap(([language, languageExercises]) =>
+		languageExercises.map((exercise) => ({ language: language as Language, exercise })),
+	)
 
-	for (const [language, languageExercises] of Object.entries(exercises)) {
-		for (const exercise of languageExercises) {
-			await runExercise({ run, language: language as Language, exercise })
-		}
+	const tasks = await pMap(
+		entries,
+		async ({ language, exercise }) => findOrCreateTask({ runId: run.id, language, exercise }),
+		{ concurrency: 10 },
+	)
+
+	for (const task of tasks) {
+		await runExercise({ run, task })
 	}
 }
 
 const runLanguage = async ({ id, language }: { id?: number; language: Language }) => {
 	const run = await findOrCreateRun({ id })
-	const exercises = getExercises()
-	const languageExercises = exercises[language]
 
-	await Promise.all(
-		languageExercises.map((exercise) => findOrCreatePendingTask({ runId: run.id, language, exercise })),
+	const tasks = await pMap(
+		getExercises()[language],
+		async (exercise) => findOrCreateTask({ runId: run.id, language, exercise }),
+		{ concurrency: 10 },
 	)
 
-	for (const exercise of languageExercises) {
-		await runExercise({ run, language, exercise })
+	for (const task of tasks) {
+		await runExercise({ run, task })
 	}
 }
 
@@ -82,11 +73,12 @@ const runLanguageExercise = async ({
 	exercise: string
 }) => {
 	const run = await findOrCreateRun({ id })
-	await findOrCreatePendingTask({ runId: run.id, language, exercise })
-	return runExercise({ run, language, exercise })
+	const task = await findOrCreateTask({ runId: run.id, language, exercise })
+	return runExercise({ run, task })
 }
 
-const runExercise = async ({ run, language, exercise }: { run: Run; language: Language; exercise: string }) => {
+const runExercise = async ({ run, task }: { run: Run; task: Task }) => {
+	const { language, exercise } = task
 	const workspacePath = path.resolve(exercisesPath, language, exercise)
 	const promptPath = path.resolve(exercisesPath, `prompts/${language}.md`)
 
@@ -94,9 +86,7 @@ const runExercise = async ({ run, language, exercise }: { run: Run; language: La
 		throw new Error(`Prompt file does not exist: ${promptPath}`)
 	}
 
-	const task = await getTask({ runId: run.id, language, exercise })
-
-	if (task) {
+	if (task.finishedAt) {
 		console.log(`Test result exists for ${language} / ${exercise}, skipping`)
 		return false
 	}
@@ -108,7 +98,7 @@ const runExercise = async ({ run, language, exercise }: { run: Run; language: La
 		extensionTestsPath,
 		launchArgs: [workspacePath, "--disable-extensions"],
 		extensionTestsEnv: {
-			RUN_ID: run.id.toString(),
+			TASK_ID: task.id.toString(),
 			LANGUAGE: language,
 			EXERCISE: exercise,
 			PROMPT_PATH: promptPath,
@@ -157,7 +147,7 @@ const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: {
 				socketPath: path.resolve(os.tmpdir(), `benchmark-${crypto.randomUUID()}.sock`),
 			})
 
-const findOrCreatePendingTask = async ({
+const findOrCreateTask = async ({
 	runId,
 	language,
 	exercise,
@@ -165,7 +155,7 @@ const findOrCreatePendingTask = async ({
 	runId: number
 	language: Language
 	exercise: string
-}) => (await getPendingTask({ runId, language, exercise })) || (await createPendingTask({ runId, language, exercise }))
+}) => (await getTask({ runId, language, exercise })) || (await createTask({ runId, language, exercise }))
 
 const main = async () => {
 	const cli = build()

benchmark/apps/web/src/app/home.tsx

@@ -3,16 +3,17 @@
 import { useRouter } from "next/navigation"
 import { Rocket } from "lucide-react"
 
-import { getRuns } from "@benchmark/db"
+import type { Run, TaskMetrics } from "@benchmark/db"
 
 import { formatCurrency, formatDuration } from "@/lib"
 import { Button, Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui"
+import { useMemo } from "react"
 
-type Run = Awaited<ReturnType<typeof getRuns>>[number]
-
-export function Home({ runs }: { runs: Run[] }) {
+export function Home({ runs }: { runs: (Run & { taskMetrics: TaskMetrics | null })[] }) {
 	const router = useRouter()
 
+	const visibleRuns = useMemo(() => runs.filter((run) => run.taskMetrics !== null), [runs])
+
 	return (
 		<>
 			<Table className="border">
@@ -29,18 +30,30 @@ export function Home({ runs }: { runs: Run[] }) {
 					</TableRow>
 				</TableHeader>
 				<TableBody>
-					{runs.map((run) => (
-						<TableRow key={run.id}>
-							<TableCell>{run.id}</TableCell>
-							<TableCell>{run.model}</TableCell>
-							<TableCell>{new Date(run.createdAt).toLocaleString()}</TableCell>
-							<TableCell>{run.passed}</TableCell>
-							<TableCell>{run.failed}</TableCell>
-							<TableCell>{(run.rate * 100).toFixed(1)}%</TableCell>
-							<TableCell>{formatCurrency(run.cost)}</TableCell>
-							<TableCell>{formatDuration(run.duration)}</TableCell>
+					{visibleRuns.length ? (
+						visibleRuns.map(({ taskMetrics, ...run }) => (
+							<TableRow key={run.id}>
+								<TableCell>{run.id}</TableCell>
+								<TableCell>{run.model}</TableCell>
+								<TableCell>{new Date(run.createdAt).toLocaleString()}</TableCell>
+								<TableCell>{run.passed}</TableCell>
+								<TableCell>{run.failed}</TableCell>
+								<TableCell>{((run.passed / (run.passed + run.failed)) * 100).toFixed(1)}%</TableCell>
+								<TableCell>{formatCurrency(taskMetrics!.cost)}</TableCell>
+								<TableCell>{formatDuration(taskMetrics!.duration)}</TableCell>
+							</TableRow>
+						))
+					) : (
+						<TableRow>
+							<TableCell colSpan={8} className="text-center">
+								No benchmark runs yet.
+								<Button variant="link" onClick={() => router.push("/runs/new")}>
+									Launch
+								</Button>
+								one now.
+							</TableCell>
 						</TableRow>
-					))}
+					)}
 				</TableBody>
 			</Table>
 			<Button

benchmark/apps/web/src/app/runs/[id]/actions.ts

@@ -0,0 +1,11 @@
+"use server"
+
+import { revalidatePath } from "next/cache"
+
+import * as db from "@benchmark/db"
+
+export async function getTasks(runId: number) {
+	const tasks = await db.getTasks(runId)
+	revalidatePath(`/runs/${runId}`)
+	return tasks
+}

benchmark/apps/web/src/app/runs/[id]/connection-status.tsx

@@ -0,0 +1,36 @@
+import { EventSourceStatus } from "@/hooks/use-event-source"
+import { cn } from "@/lib/utils"
+
+type ConnectionStatusProps = {
+	status: EventSourceStatus
+	clientId?: string
+	pid: number | null
+}
+
+export const ConnectionStatus = ({ status, clientId, pid }: ConnectionStatusProps) => (
+	<div className="flex items-center">
+		<div className="flex flex-col items-end gap-1 font-mono text-xs border-r border-dotted pr-4 mr-4">
+			<div>
+				Status: <span className="capitalize">{status}</span>
+			</div>
+			<div>PID: {pid}</div>
+			{clientId && <div>Client: {clientId}</div>}
+		</div>
+		<div className="relative">
+			<div
+				className={cn("absolute size-2.5 rounded-full opacity-50 animate-ping", {
+					"bg-green-500": status === "connected",
+					"bg-amber-500": status === "waiting",
+					"bg-rose-500": status === "error",
+				})}
+			/>
+			<div
+				className={cn("size-2.5 rounded-full", {
+					"bg-green-500": status === "connected",
+					"bg-amber-500": status === "waiting",
+					"bg-rose-500": status === "error",
+				})}
+			/>
+		</div>
+	</div>
+)

benchmark/apps/web/src/app/runs/[id]/page.tsx

@@ -1,16 +1,9 @@
-import { findRun, getTasks, getPendingTasks } from "@benchmark/db"
+import { findRun } from "@benchmark/db"
 
-import { ShowRun } from "./show-run"
+import { Run } from "./run"
 
 export default async function Page({ params }: { params: Promise<{ id: string }> }) {
 	const { id } = await params
 	const run = await findRun(Number(id))
-	const tasks = await getTasks(run.id)
-	const pendingTasks = await getPendingTasks(run.id)
-
-	if (!run) {
-		return <div>Run not found</div>
-	}
-
-	return <ShowRun run={{ ...run, tasks, pendingTasks }} />
+	return <Run run={run} />
 }

benchmark/apps/web/src/app/runs/[id]/run.tsx

@@ -0,0 +1,40 @@
+"use client"
+
+import { LoaderCircle } from "lucide-react"
+
+import * as db from "@benchmark/db"
+
+import { useRunStatus } from "./use-run-status"
+import { TaskStatus } from "./task-status"
+import { ConnectionStatus } from "./connection-status"
+
+export function Run({ run }: { run: db.Run }) {
+	const { tasks, status, clientId, runningTaskId } = useRunStatus(run)
+
+	return (
+		<>
+			<div className="flex flex-col gap-2">
+				<div className="border-b mb-2 pb-2">
+					<div>Run #{run.id}</div>
+					<div>{run.model}</div>
+					{run.description && <div className="text-sm text-muted-foreground">{run.description}</div>}
+				</div>
+				{!tasks ? (
+					<LoaderCircle className="size-4 animate-spin" />
+				) : (
+					tasks.map((task) => (
+						<div key={task.id} className="flex items-center gap-2">
+							<TaskStatus task={task} runningTaskId={runningTaskId} />
+							<div>
+								{task.language}/{task.exercise}
+							</div>
+						</div>
+					))
+				)}
+			</div>
+			<div className="absolute top-5 right-5">
+				<ConnectionStatus status={status} clientId={clientId} pid={run.pid} />
+			</div>
+		</>
+	)
+}

benchmark/apps/web/src/app/runs/[id]/schemas.ts

@@ -0,0 +1,42 @@
+import { z } from "zod"
+
+export const messageSchema = z.object({
+	type: z.enum(["hello", "data"]),
+	data: z.record(z.string(), z.unknown()),
+})
+
+export const taskSchema = z.object({
+	id: z.number(),
+})
+
+export const taskEventSchema = z.discriminatedUnion("event", [
+	z.object({ event: z.literal("client"), task: taskSchema }),
+	z.object({ event: z.literal("taskStarted"), task: taskSchema }),
+	z.object({ event: z.literal("taskFinished"), task: taskSchema }),
+	z.object({
+		event: z.literal("message"),
+		task: taskSchema,
+		message: z.object({
+			taskId: z.string(),
+			action: z.enum(["created", "updated"]),
+			message: z.object({
+				ts: z.number(),
+				type: z.enum(["ask", "say"]),
+				text: z.string(),
+				partial: z.boolean().optional(),
+			}),
+		}),
+	}),
+	z.object({
+		event: z.literal("taskTokenUsageUpdated"),
+		task: taskSchema,
+		usage: z.object({
+			totalTokensIn: z.number(),
+			totalTokensOut: z.number(),
+			totalCacheWrites: z.number().optional(),
+			totalCacheReads: z.number().optional(),
+			totalCost: z.number(),
+			contextTokens: z.number(),
+		}),
+	}),
+])

benchmark/apps/web/src/app/runs/[id]/show-run.tsx

@@ -1,156 +0,0 @@
-"use client"
-
-import { useCallback, useState } from "react"
-import { z } from "zod"
-import { CircleDashed, CircleCheck, CircleSlash, LoaderCircle, Dot } from "lucide-react"
-
-import { type Run, type PendingTask, Task } from "@benchmark/db"
-
-import { useEventSource } from "@/hooks/use-event-source"
-import { cn } from "@/lib/utils"
-
-const messageSchema = z.object({
-	type: z.enum(["hello", "data"]),
-	data: z.record(z.string(), z.unknown()),
-})
-
-const taskEventSchema = z.discriminatedUnion("event", [
-	// {"event":"client","runId":4,"language":"cpp","exercise":"queen-attack","prompt":"Your job is to complete a coding exercise described by `.docs/instructions.md`...","workspacePath":"/Users/cte/Documents/exercises/cpp/queen-attack"}
-	z.object({
-		event: z.literal("client"),
-		runId: z.number(),
-		language: z.string(),
-		exercise: z.string(),
-		prompt: z.string(),
-		workspacePath: z.string(),
-	}),
-	// {"event":"message","taskId":"acfdc39d-2c21-484b-9fb0-c6670b1a7439","action":"updated","message":{"ts":1742574782085,"type":"say","say":"text","text":"Great! All the tests have passed...","partial":true}}
-	z.object({
-		event: z.literal("message"),
-		taskId: z.string(),
-		action: z.enum(["created", "updated"]),
-		message: z.object({
-			ts: z.number(),
-			type: z.enum(["ask", "say"]),
-			text: z.string(),
-			partial: z.boolean(),
-		}),
-	}),
-	// {"event":"taskTokenUsageUpdated","taskId":"acfdc39d-2c21-484b-9fb0-c6670b1a7439","usage":{"totalTokensIn":102069,"totalTokensOut":1700,"totalCacheWrites":0,"totalCacheReads":0,"totalCost":0.1212111,"contextTokens":19286}}
-	z.object({
-		event: z.literal("taskTokenUsageUpdated"),
-		taskId: z.string(),
-		usage: z.object({
-			totalTokensIn: z.number(),
-			totalTokensOut: z.number(),
-			totalCacheWrites: z.number(),
-			totalCacheReads: z.number(),
-			totalCost: z.number(),
-			contextTokens: z.number(),
-		}),
-	}),
-])
-
-type CurrentRun = Run & { tasks: Task[]; pendingTasks: PendingTask[] }
-
-export function ShowRun({ run }: { run: CurrentRun }) {
-	const [clientId, setClientId] = useState<string>()
-	const [runningTask, setRunningTask] = useState<string>()
-	const url = `/api/runs/${run.id}/stream`
-
-	const onMessage = useCallback((messageEvent: MessageEvent) => {
-		// console.log(messageEvent.data)
-		let data
-
-		try {
-			data = JSON.parse(messageEvent.data)
-		} catch (_) {
-			// console.log(`invalid JSON: ${messageEvent.data}`)
-			return
-		}
-
-		const result = messageSchema.safeParse(data)
-
-		if (!result.success) {
-			// console.log(`unrecognized messageEvent.data: ${messageEvent.data}`)
-			return
-		}
-
-		const payload = result.data
-
-		if (payload.type === "hello") {
-			setClientId(payload.data.clientId as string)
-		} else if (payload.type === "data") {
-			const taskEvent = taskEventSchema.safeParse(payload.data)
-
-			if (!taskEvent.success) {
-				console.log(`unrecognized payload.data`, payload.data, taskEvent.error)
-				return
-			}
-
-			if (taskEvent.data.event === "client") {
-				console.log(`client`, taskEvent.data)
-				const { language, exercise } = taskEvent.data
-				setRunningTask(`${language}/${exercise}`)
-			} else if (taskEvent.data.event === "message") {
-				console.log(`message: ${taskEvent.data.message.text}`)
-			} else if (taskEvent.data.event === "taskTokenUsageUpdated") {
-				console.log(`taskTokenUsageUpdated: ${taskEvent.data.usage}`)
-			}
-		}
-	}, [])
-
-	const status = useEventSource({ url, onMessage })
-
-	const pendingTasks = run.pendingTasks.filter((pendingTask) => !run.tasks.find((task) => task.id === pendingTask.id))
-
-	return (
-		<>
-			<div className="flex flex-col gap-2">
-				<div className="border-b mb-2 pb-2">
-					<div>Run #{run.id}</div>
-					<div>{run.model}</div>
-					{run.description && <div className="text-sm text-muted-foreground">{run.description}</div>}
-				</div>
-				{run.tasks.map((task) => (
-					<div key={task.id} className="flex items-center gap-2">
-						{task.passed ? (
-							<CircleCheck className="size-4 text-green-500" />
-						) : (
-							<CircleSlash className="size-4 text-destructive" />
-						)}
-						<div>
-							{task.language}/{task.exercise}
-						</div>
-					</div>
-				))}
-				{pendingTasks.map((task) => (
-					<div key={task.id} className="flex items-center gap-2">
-						{runningTask === `${task.language}/${task.exercise}` ? (
-							<LoaderCircle className="size-4 animate-spin" />
-						) : (
-							<CircleDashed className="size-4" />
-						)}
-						<div>
-							{task.language}/{task.exercise}
-						</div>
-					</div>
-				))}
-			</div>
-			<div className="absolute top-5 right-5">
-				<div className="flex items-center gap-2">
-					<div className="animate-ping">
-						<div
-							className={cn("size-2 rounded-full", {
-								"bg-green-500": status === "open",
-								"bg-amber-300": status === "init",
-								"bg-rose-500": status === "error",
-							})}
-						/>
-					</div>
-					<div className="font-mono text-xs">{clientId}</div>
-				</div>
-			</div>
-		</>
-	)
-}

benchmark/apps/web/src/app/runs/[id]/task-status.tsx

@@ -0,0 +1,20 @@
+import { CircleCheck, CircleDashed, CircleSlash, LoaderCircle } from "lucide-react"
+
+import { type Task } from "@benchmark/db"
+
+type TaskStatusProps = {
+	task: Task
+	runningTaskId?: number
+}
+
+export const TaskStatus = ({ task, runningTaskId }: TaskStatusProps) => {
+	return runningTaskId === task.id ? (
+		<LoaderCircle className="size-4 animate-spin" />
+	) : !task.finishedAt ? (
+		<CircleDashed className="size-4" />
+	) : !task.passed ? (
+		<CircleSlash className="size-4 text-destructive" />
+	) : (
+		<CircleCheck className="size-4 text-green-500" />
+	)
+}

benchmark/apps/web/src/app/runs/[id]/use-run-status.ts

@@ -0,0 +1,70 @@
+import { useState } from "react"
+import { keepPreviousData, useQuery } from "@tanstack/react-query"
+import { useCallback } from "react"
+import { useEventSource } from "@/hooks/use-event-source"
+
+import { Run } from "@benchmark/db"
+
+import { getTasks } from "./actions"
+import { messageSchema, taskEventSchema } from "./schemas"
+
+export const useRunStatus = (run: Run) => {
+	const [clientId, setClientId] = useState<string>()
+	const [runningTaskId, setRunningTaskId] = useState<number>()
+
+	const { data: tasks } = useQuery({
+		queryKey: ["run", run.id, runningTaskId],
+		queryFn: async () => getTasks(run.id),
+		placeholderData: keepPreviousData,
+	})
+
+	const url = `/api/runs/${run.id}/stream`
+
+	const onMessage = useCallback((messageEvent: MessageEvent) => {
+		let data
+
+		try {
+			data = JSON.parse(messageEvent.data)
+		} catch (_) {
+			console.log(`invalid JSON: ${messageEvent.data}`)
+			return
+		}
+
+		const result = messageSchema.safeParse(data)
+
+		if (!result.success) {
+			console.log(`unrecognized messageEvent.data: ${messageEvent.data}`)
+			return
+		}
+
+		const payload = result.data
+
+		if (payload.type === "hello") {
+			setClientId(payload.data.clientId as string)
+		} else if (payload.type === "data") {
+			const taskEvent = taskEventSchema.safeParse(payload.data)
+
+			if (!taskEvent.success) {
+				console.log(`unrecognized payload.data`, payload.data, taskEvent.error)
+				return
+			}
+
+			if (taskEvent.data.event === "client") {
+				console.log(`client`, taskEvent.data)
+				setRunningTaskId(taskEvent.data.task.id)
+			} else if (taskEvent.data.event === "message") {
+				// console.log(`message: ${taskEvent.data.message.message.text}`)
+			} else if (taskEvent.data.event === "taskTokenUsageUpdated") {
+				console.log(`taskTokenUsageUpdated`, taskEvent.data.usage)
+			} else if (taskEvent.data.event === "taskStarted") {
+				setRunningTaskId(taskEvent.data.task.id)
+			} else if (taskEvent.data.event === "taskFinished") {
+				setRunningTaskId(undefined)
+			}
+		}
+	}, [])
+
+	const status = useEventSource({ url, onMessage })
+
+	return { tasks, status, clientId, runningTaskId }
+}

benchmark/apps/web/src/app/runs/new/new-run.tsx

@@ -107,7 +107,7 @@ export function NewRun() {
 					/>
 					<Button type="submit" disabled={isSubmitting}>
 						<Rocket className="size-4" />
-						Launch Benchmark Run
+						Launch
 					</Button>
 				</form>
 			</FormProvider>

benchmark/apps/web/src/components/ui/button.tsx

@@ -16,7 +16,7 @@ const buttonVariants = cva(
 					"border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input",
 				secondary: "bg-secondary text-secondary-foreground shadow-xs",
 				ghost: "hover:bg-accent hover:text-accent-foreground",
-				link: "text-primary underline-offset-4 hover:underline",
+				link: "text-accent underline-offset-4 hover:underline px-1.5!",
 			},
 			size: {
 				default: "h-9 px-4 py-2 has-[>svg]:px-3",

benchmark/apps/web/src/hooks/use-event-source.ts

@@ -1,6 +1,6 @@
 import { useCallback, useEffect, useRef, useState } from "react"
 
-export type EventSourceStatus = "init" | "open" | "error"
+export type EventSourceStatus = "waiting" | "connected" | "error"
 
 export type EventSourceEvent = Event & { data: string }
 
@@ -12,16 +12,16 @@ type UseEventSourceOptions = {
 
 export function useEventSource({ url, withCredentials, onMessage }: UseEventSourceOptions) {
 	const sourceRef = useRef<EventSource | null>(null)
-	const statusRef = useRef<EventSourceStatus>("init")
-	const [status, setStatus] = useState<EventSourceStatus>("init")
+	const statusRef = useRef<EventSourceStatus>("waiting")
+	const [status, setStatus] = useState<EventSourceStatus>("waiting")
 	const handleMessage = useCallback((event: MessageEvent) => onMessage(event), [onMessage])
 
 	const createEventSource = useCallback(() => {
 		sourceRef.current = new EventSource(url, { withCredentials })
 
 		sourceRef.current.onopen = () => {
-			statusRef.current = "open"
-			setStatus("open")
+			statusRef.current = "connected"
+			setStatus("connected")
 		}
 
 		sourceRef.current.onmessage = (event) => {
@@ -40,7 +40,7 @@ export function useEventSource({ url, withCredentials, onMessage }: UseEventSour
 		createEventSource()
 
 		setTimeout(() => {
-			if (statusRef.current === "init") {
+			if (statusRef.current === "waiting") {
 				sourceRef.current?.close()
 				sourceRef.current = null
 				createEventSource()

benchmark/packages/db/drizzle/0000_elite_raza.sql

@@ -1,27 +1,18 @@
-CREATE TABLE `pendingTasks` (
-	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-	`runId` integer NOT NULL,
-	`language` text NOT NULL,
-	`exercise` text NOT NULL,
-	`pid` integer,
-	`createdAt` integer NOT NULL
-);
---> statement-breakpoint
-CREATE UNIQUE INDEX `pendingTasks_language_exercise_idx` ON `pendingTasks` (`runId`,`language`,`exercise`);--> statement-breakpoint
 CREATE TABLE `runs` (
 	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`taskMetricsId` integer,
 	`model` text NOT NULL,
 	`description` text,
 	`pid` integer,
 	`socketPath` text NOT NULL,
-	`createdAt` integer NOT NULL
+	`passed` integer DEFAULT 0 NOT NULL,
+	`failed` integer DEFAULT 0 NOT NULL,
+	`createdAt` integer NOT NULL,
+	FOREIGN KEY (`taskMetricsId`) REFERENCES `taskMetrics`(`id`) ON UPDATE no action ON DELETE no action
 );
 --> statement-breakpoint
-CREATE TABLE `tasks` (
+CREATE TABLE `taskMetrics` (
 	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
-	`runId` integer NOT NULL,
-	`language` text NOT NULL,
-	`exercise` text NOT NULL,
 	`tokensIn` integer NOT NULL,
 	`tokensOut` integer NOT NULL,
 	`tokensContext` integer NOT NULL,
@@ -29,8 +20,21 @@ CREATE TABLE `tasks` (
 	`cacheReads` integer NOT NULL,
 	`cost` real NOT NULL,
 	`duration` integer NOT NULL,
-	`passed` integer,
 	`createdAt` integer NOT NULL
 );
 --> statement-breakpoint
+CREATE TABLE `tasks` (
+	`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
+	`runId` integer NOT NULL,
+	`taskMetricsId` integer,
+	`language` text NOT NULL,
+	`exercise` text NOT NULL,
+	`passed` integer,
+	`startedAt` integer,
+	`finishedAt` integer,
+	`createdAt` integer NOT NULL,
+	FOREIGN KEY (`runId`) REFERENCES `runs`(`id`) ON UPDATE no action ON DELETE no action,
+	FOREIGN KEY (`taskMetricsId`) REFERENCES `taskMetrics`(`id`) ON UPDATE no action ON DELETE no action
+);
+--> statement-breakpoint
 CREATE UNIQUE INDEX `tasks_language_exercise_idx` ON `tasks` (`runId`,`language`,`exercise`);

benchmark/packages/db/drizzle/meta/0000_snapshot.json

@@ -1,67 +1,9 @@
 {
 	"version": "6",
 	"dialect": "sqlite",
-	"id": "d7bfd73f-dbf5-430e-ab81-f436029d06e5",
+	"id": "c0fa8491-b5c0-493d-aa32-ddf280259c30",
 	"prevId": "00000000-0000-0000-0000-000000000000",
 	"tables": {
-		"pendingTasks": {
-			"name": "pendingTasks",
-			"columns": {
-				"id": {
-					"name": "id",
-					"type": "integer",
-					"primaryKey": true,
-					"notNull": true,
-					"autoincrement": true
-				},
-				"runId": {
-					"name": "runId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"language": {
-					"name": "language",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"exercise": {
-					"name": "exercise",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"pid": {
-					"name": "pid",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": false,
-					"autoincrement": false
-				},
-				"createdAt": {
-					"name": "createdAt",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				}
-			},
-			"indexes": {
-				"pendingTasks_language_exercise_idx": {
-					"name": "pendingTasks_language_exercise_idx",
-					"columns": ["runId", "language", "exercise"],
-					"isUnique": true
-				}
-			},
-			"foreignKeys": {},
-			"compositePrimaryKeys": {},
-			"uniqueConstraints": {},
-			"checkConstraints": {}
-		},
 		"runs": {
 			"name": "runs",
 			"columns": {
@@ -72,6 +14,13 @@
 					"notNull": true,
 					"autoincrement": true
 				},
+				"taskMetricsId": {
+					"name": "taskMetricsId",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
 				"model": {
 					"name": "model",
 					"type": "text",
@@ -100,6 +49,22 @@
 					"notNull": true,
 					"autoincrement": false
 				},
+				"passed": {
+					"name": "passed",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false,
+					"default": 0
+				},
+				"failed": {
+					"name": "failed",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false,
+					"default": 0
+				},
 				"createdAt": {
 					"name": "createdAt",
 					"type": "integer",
@@ -109,13 +74,23 @@
 				}
 			},
 			"indexes": {},
-			"foreignKeys": {},
+			"foreignKeys": {
+				"runs_taskMetricsId_taskMetrics_id_fk": {
+					"name": "runs_taskMetricsId_taskMetrics_id_fk",
+					"tableFrom": "runs",
+					"tableTo": "taskMetrics",
+					"columnsFrom": ["taskMetricsId"],
+					"columnsTo": ["id"],
+					"onDelete": "no action",
+					"onUpdate": "no action"
+				}
+			},
 			"compositePrimaryKeys": {},
 			"uniqueConstraints": {},
 			"checkConstraints": {}
 		},
-		"tasks": {
-			"name": "tasks",
+		"taskMetrics": {
+			"name": "taskMetrics",
 			"columns": {
 				"id": {
 					"name": "id",
@@ -124,27 +99,6 @@
 					"notNull": true,
 					"autoincrement": true
 				},
-				"runId": {
-					"name": "runId",
-					"type": "integer",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"language": {
-					"name": "language",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
-				"exercise": {
-					"name": "exercise",
-					"type": "text",
-					"primaryKey": false,
-					"notNull": true,
-					"autoincrement": false
-				},
 				"tokensIn": {
 					"name": "tokensIn",
 					"type": "integer",
@@ -194,6 +148,58 @@
 					"notNull": true,
 					"autoincrement": false
 				},
+				"createdAt": {
+					"name": "createdAt",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				}
+			},
+			"indexes": {},
+			"foreignKeys": {},
+			"compositePrimaryKeys": {},
+			"uniqueConstraints": {},
+			"checkConstraints": {}
+		},
+		"tasks": {
+			"name": "tasks",
+			"columns": {
+				"id": {
+					"name": "id",
+					"type": "integer",
+					"primaryKey": true,
+					"notNull": true,
+					"autoincrement": true
+				},
+				"runId": {
+					"name": "runId",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"taskMetricsId": {
+					"name": "taskMetricsId",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
+				"language": {
+					"name": "language",
+					"type": "text",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
+				"exercise": {
+					"name": "exercise",
+					"type": "text",
+					"primaryKey": false,
+					"notNull": true,
+					"autoincrement": false
+				},
 				"passed": {
 					"name": "passed",
 					"type": "integer",
@@ -201,6 +207,20 @@
 					"notNull": false,
 					"autoincrement": false
 				},
+				"startedAt": {
+					"name": "startedAt",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
+				"finishedAt": {
+					"name": "finishedAt",
+					"type": "integer",
+					"primaryKey": false,
+					"notNull": false,
+					"autoincrement": false
+				},
 				"createdAt": {
 					"name": "createdAt",
 					"type": "integer",
@@ -216,7 +236,26 @@
 					"isUnique": true
 				}
 			},
-			"foreignKeys": {},
+			"foreignKeys": {
+				"tasks_runId_runs_id_fk": {
+					"name": "tasks_runId_runs_id_fk",
+					"tableFrom": "tasks",
+					"tableTo": "runs",
+					"columnsFrom": ["runId"],
+					"columnsTo": ["id"],
+					"onDelete": "no action",
+					"onUpdate": "no action"
+				},
+				"tasks_taskMetricsId_taskMetrics_id_fk": {
+					"name": "tasks_taskMetricsId_taskMetrics_id_fk",
+					"tableFrom": "tasks",
+					"tableTo": "taskMetrics",
+					"columnsFrom": ["taskMetricsId"],
+					"columnsTo": ["id"],
+					"onDelete": "no action",
+					"onUpdate": "no action"
+				}
+			},
 			"compositePrimaryKeys": {},
 			"uniqueConstraints": {},
 			"checkConstraints": {}

benchmark/packages/db/drizzle/meta/_journal.json

@@ -5,8 +5,8 @@
 		{
 			"idx": 0,
 			"version": "6",
-			"when": 1742568357359,
-			"tag": "0000_legal_cassandra_nova",
+			"when": 1742599919625,
+			"tag": "0000_elite_raza",
 			"breakpoints": true
 		}
 	]

benchmark/packages/db/src/index.ts

@@ -1,6 +1,4 @@
-// export { db } from "./db.js"
 export { type Language, languages } from "./enums.js"
-// export { schema } from "./schema.js"
 
 /**
  * runs
@@ -17,8 +15,8 @@ export type { Task, InsertTask, UpdateTask } from "./schema.js"
 export * from "./queries/tasks.js"
 
 /**
- * pendingTasks
+ * taskMetrics
  */
 
-export type { PendingTask, InsertPendingTask, UpdatePendingTask } from "./schema.js"
-export * from "./queries/pendingTasks.js"
+export type { TaskMetrics, InsertTaskMetrics, UpdateTaskMetrics } from "./schema.js"
+export * from "./queries/taskMetrics.js"

benchmark/packages/db/src/queries/pendingTasks.ts

@@ -1,62 +0,0 @@
-import { and, eq } from "drizzle-orm"
-
-import type { Language } from "../enums.js"
-import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js"
-import type { InsertPendingTask, UpdatePendingTask } from "../schema.js"
-import { insertPendingTaskSchema, pendingTasks } from "../schema.js"
-import { db } from "../db.js"
-
-const table = pendingTasks
-
-export const findPendingTask = async (id: number) => {
-	const run = await db.query.pendingTasks.findFirst({ where: eq(table.id, id) })
-
-	if (!run) {
-		throw new RecordNotFoundError()
-	}
-
-	return run
-}
-
-export const createPendingTask = async (args: InsertPendingTask) => {
-	const records = await db
-		.insert(table)
-		.values({
-			...insertPendingTaskSchema.parse(args),
-			createdAt: new Date(),
-		})
-		.returning()
-
-	const record = records[0]
-
-	if (!record) {
-		throw new RecordNotCreatedError()
-	}
-
-	return record
-}
-
-export const updatePendingTask = async (id: number, values: UpdatePendingTask) => {
-	const records = await db.update(table).set(values).where(eq(table.id, id)).returning()
-	const record = records[0]
-
-	if (!record) {
-		throw new RecordNotFoundError()
-	}
-
-	return record
-}
-
-type GetPendingTask = {
-	runId: number
-	language: Language
-	exercise: string
-}
-
-export const getPendingTask = async ({ runId, language, exercise }: GetPendingTask) =>
-	db.query.pendingTasks.findFirst({
-		where: and(eq(table.runId, runId), eq(table.language, language), eq(table.exercise, exercise)),
-	})
-
-export const getPendingTasks = async (runId: number) =>
-	db.query.pendingTasks.findMany({ where: eq(table.runId, runId) })

benchmark/packages/db/src/queries/runs.ts

@@ -1,11 +1,11 @@
-import { desc, eq, sql } from "drizzle-orm"
+import { desc, eq } from "drizzle-orm"
 
 import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js"
 import type { InsertRun, UpdateRun } from "../schema.js"
-import { insertRunSchema, runs, tasks } from "../schema.js"
+import { insertRunSchema, schema } from "../schema.js"
 import { db } from "../db.js"
 
-const table = runs
+const table = schema.runs
 
 export const findRun = async (id: number) => {
 	const run = await db.query.runs.findFirst({ where: eq(table.id, id) })
@@ -46,20 +46,4 @@ export const updateRun = async (id: number, values: UpdateRun) => {
 	return record
 }
 
-export const getRuns = () =>
-	db
-		.select({
-			id: table.id,
-			model: table.model,
-			description: table.description,
-			createdAt: table.createdAt,
-			passed: sql<number>`sum(${tasks.passed})`,
-			failed: sql<number>`sum(${tasks.passed} = 0)`,
-			total: sql<number>`count(${tasks.id})`,
-			rate: sql<number>`sum(${tasks.passed}) * 1.0 / count(${tasks.id})`,
-			cost: sql<number>`sum(${tasks.cost})`,
-			duration: sql<number>`sum(${tasks.duration})`,
-		})
-		.from(table)
-		.leftJoin(tasks, eq(table.id, tasks.runId))
-		.orderBy(desc(table.id))
+export const getRuns = async () => db.query.runs.findMany({ orderBy: desc(table.id), with: { taskMetrics: true } })

benchmark/packages/db/src/queries/taskMetrics.ts

@@ -0,0 +1,36 @@
+import { eq } from "drizzle-orm"
+
+import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js"
+import type { InsertTaskMetrics } from "../schema.js"
+import { insertTaskMetricsSchema, taskMetrics } from "../schema.js"
+import { db } from "../db.js"
+
+const table = taskMetrics
+
+export const findTaskMetrics = async (id: number) => {
+	const run = await db.query.taskMetrics.findFirst({ where: eq(table.id, id) })
+
+	if (!run) {
+		throw new RecordNotFoundError()
+	}
+
+	return run
+}
+
+export const createTaskMetrics = async (args: InsertTaskMetrics) => {
+	const records = await db
+		.insert(table)
+		.values({
+			...insertTaskMetricsSchema.parse(args),
+			createdAt: new Date(),
+		})
+		.returning()
+
+	const record = records[0]
+
+	if (!record) {
+		throw new RecordNotCreatedError()
+	}
+
+	return record
+}

benchmark/packages/db/src/schema.ts

@@ -1,5 +1,5 @@
-import { sqliteTable, text, real, integer } from "drizzle-orm/sqlite-core"
-import * as t from "drizzle-orm/sqlite-core"
+import { sqliteTable, text, real, integer, uniqueIndex } from "drizzle-orm/sqlite-core"
+import { relations } from "drizzle-orm"
 import { createInsertSchema } from "drizzle-zod"
 import { z } from "zod"
 
@@ -11,19 +11,23 @@ import { languages } from "./enums.js"
 
 export const runs = sqliteTable("runs", {
 	id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }),
+	taskMetricsId: integer({ mode: "number" }).references(() => taskMetrics.id),
 	model: text().notNull(),
 	description: text(),
 	pid: integer({ mode: "number" }),
 	socketPath: text().notNull(),
+	passed: integer({ mode: "number" }).default(0).notNull(),
+	failed: integer({ mode: "number" }).default(0).notNull(),
 	createdAt: integer({ mode: "timestamp" }).notNull(),
 })
 
+export const runsRelations = relations(runs, ({ one }) => ({
+	taskMetrics: one(taskMetrics, { fields: [runs.taskMetricsId], references: [taskMetrics.id] }),
+}))
+
 export type Run = typeof runs.$inferSelect
 
-export const insertRunSchema = createInsertSchema(runs).omit({
-	id: true,
-	createdAt: true,
-})
+export const insertRunSchema = createInsertSchema(runs).omit({ id: true, createdAt: true })
 
 export type InsertRun = z.infer<typeof insertRunSchema>
 
@@ -37,63 +41,59 @@ export const tasks = sqliteTable(
 	"tasks",
 	{
 		id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }),
-		runId: integer({ mode: "number" }).notNull(),
+		runId: integer({ mode: "number" })
+			.references(() => runs.id)
+			.notNull(),
+		taskMetricsId: integer({ mode: "number" }).references(() => taskMetrics.id),
 		language: text({ enum: languages }).notNull(),
 		exercise: text().notNull(),
-		tokensIn: integer({ mode: "number" }).notNull(),
-		tokensOut: integer({ mode: "number" }).notNull(),
-		tokensContext: integer({ mode: "number" }).notNull(),
-		cacheWrites: integer({ mode: "number" }).notNull(),
-		cacheReads: integer({ mode: "number" }).notNull(),
-		cost: real().notNull(),
-		duration: integer({ mode: "number" }).notNull(),
 		passed: integer({ mode: "boolean" }),
+		startedAt: integer({ mode: "timestamp" }),
+		finishedAt: integer({ mode: "timestamp" }),
 		createdAt: integer({ mode: "timestamp" }).notNull(),
 	},
-	(table) => [t.uniqueIndex("tasks_language_exercise_idx").on(table.runId, table.language, table.exercise)],
+	(table) => [uniqueIndex("tasks_language_exercise_idx").on(table.runId, table.language, table.exercise)],
 )
 
+export const tasksRelations = relations(tasks, ({ one }) => ({
+	run: one(runs, { fields: [tasks.runId], references: [runs.id] }),
+	taskMetrics: one(taskMetrics, { fields: [tasks.taskMetricsId], references: [taskMetrics.id] }),
+}))
+
 export type Task = typeof tasks.$inferSelect
 
-export const insertTaskSchema = createInsertSchema(tasks).omit({
-	id: true,
-	createdAt: true,
-})
+export const insertTaskSchema = createInsertSchema(tasks).omit({ id: true, createdAt: true })
 
 export type InsertTask = z.infer<typeof insertTaskSchema>
 
 export type UpdateTask = Partial<Omit<Task, "id" | "createdAt">>
 
 /**
- * pendingTasks
+ * taskMetrics
  */
 
-export const pendingTasks = sqliteTable(
-	"pendingTasks",
-	{
-		id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }),
-		runId: integer({ mode: "number" }).notNull(),
-		language: text({ enum: languages }).notNull(),
-		exercise: text().notNull(),
-		pid: integer({ mode: "number" }),
-		createdAt: integer({ mode: "timestamp" }).notNull(),
-	},
-	(table) => [t.uniqueIndex("pendingTasks_language_exercise_idx").on(table.runId, table.language, table.exercise)],
-)
-
-export type PendingTask = typeof pendingTasks.$inferSelect
-
-export const insertPendingTaskSchema = createInsertSchema(pendingTasks).omit({
-	id: true,
-	createdAt: true,
+export const taskMetrics = sqliteTable("taskMetrics", {
+	id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }),
+	tokensIn: integer({ mode: "number" }).notNull(),
+	tokensOut: integer({ mode: "number" }).notNull(),
+	tokensContext: integer({ mode: "number" }).notNull(),
+	cacheWrites: integer({ mode: "number" }).notNull(),
+	cacheReads: integer({ mode: "number" }).notNull(),
+	cost: real().notNull(),
+	duration: integer({ mode: "number" }).notNull(),
+	createdAt: integer({ mode: "timestamp" }).notNull(),
 })
 
-export type InsertPendingTask = z.infer<typeof insertPendingTaskSchema>
+export type TaskMetrics = typeof taskMetrics.$inferSelect
 
-export type UpdatePendingTask = Partial<Omit<PendingTask, "id" | "createdAt">>
+export const insertTaskMetricsSchema = createInsertSchema(taskMetrics).omit({ id: true, createdAt: true })
+
+export type InsertTaskMetrics = z.infer<typeof insertTaskMetricsSchema>
+
+export type UpdateTaskMetrics = Partial<Omit<TaskMetrics, "id" | "createdAt">>
 
 /**
  * schema
  */
 
-export const schema = { runs, tasks, pendingTasks }
+export const schema = { runs, runsRelations, tasks, tasksRelations, taskMetrics }

benchmark/packages/runner/src/index.ts

@@ -6,7 +6,7 @@ import * as vscode from "vscode"
 import { RooCodeAPI } from "../../../../src/exports/roo-code.js"
 
 import { IpcServer, ServerMessageType } from "@benchmark/ipc"
-import { Language, findRun, createTask } from "@benchmark/db"
+import { Language, findTask, findRun, createTaskMetrics, updateTask } from "@benchmark/db"
 
 import { waitUntilReady, waitUntilCompleted, sleep } from "./utils.js"
 
@@ -15,7 +15,7 @@ export async function run() {
 	 * Validate environment variables.
 	 */
 
-	const runId = process.env.RUN_ID ? parseInt(process.env.RUN_ID) : undefined
+	const tid = process.env.TASK_ID ? parseInt(process.env.TASK_ID) : undefined
 	const language = process.env.LANGUAGE as Language
 	const exercise = process.env.EXERCISE
 	const promptPath = process.env.PROMPT_PATH
@@ -23,13 +23,20 @@ export async function run() {
 	const openRouterApiKey = process.env.OPENROUTER_API_KEY
 	const openRouterModelId = process.env.OPENROUTER_MODEL_ID
 
-	if (!runId || !language || !exercise || !promptPath || !workspacePath || !openRouterApiKey || !openRouterModelId) {
+	if (!tid || !language || !exercise || !promptPath || !workspacePath || !openRouterApiKey || !openRouterModelId) {
 		throw new Error("ENV not configured.")
 	}
 
 	const prompt = await fs.readFile(promptPath, "utf-8")
 
-	const run = await findRun(runId)
+	/**
+	 * Fetch and update the task.
+	 */
+
+	let task = await findTask(tid)
+	task = await updateTask(task.id, { startedAt: new Date() })
+
+	const run = await findRun(task.runId)
 
 	/**
 	 * Activate the extension.
@@ -87,29 +94,19 @@ export async function run() {
 	server.listen()
 
 	server.on("client", (id) => {
-		server.send(id, {
-			type: ServerMessageType.Data,
-			data: {
-				event: "client",
-				runId,
-				language,
-				exercise,
-				prompt,
-				workspacePath,
-			},
-		})
+		server.send(id, { type: ServerMessageType.Data, data: { event: "client", task } })
 	})
 
-	api.on("taskStarted", (taskId) => {
-		server.broadcast({ type: ServerMessageType.Data, data: { event: "taskStarted", taskId } })
+	api.on("taskStarted", () => {
+		server.broadcast({ type: ServerMessageType.Data, data: { event: "taskStarted", task } })
 	})
 
-	api.on("message", ({ taskId, action, message }) => {
-		server.broadcast({ type: ServerMessageType.Data, data: { event: "message", taskId, action, message } })
+	api.on("message", (message) => {
+		server.broadcast({ type: ServerMessageType.Data, data: { event: "message", task, message } })
 	})
 
-	api.on("taskTokenUsageUpdated", (taskId, usage) => {
-		server.broadcast({ type: ServerMessageType.Data, data: { event: "taskTokenUsageUpdated", taskId, usage } })
+	api.on("taskTokenUsageUpdated", (_, usage) => {
+		server.broadcast({ type: ServerMessageType.Data, data: { event: "taskTokenUsageUpdated", task, usage } })
 	})
 
 	/**
@@ -117,20 +114,19 @@ export async function run() {
 	 */
 
 	const startTime = Date.now()
-	const taskId = await api.startNewTask(prompt)
+	const rooTaskId = await api.startNewTask(prompt)
 	let usage
 
 	try {
-		usage = (await waitUntilCompleted({ api, taskId, timeout: 5 * 60 * 1_000 })) || api.getTokenUsage(taskId)
+		usage =
+			(await waitUntilCompleted({ api, taskId: rooTaskId, timeout: 5 * 60 * 1_000 })) ||
+			api.getTokenUsage(rooTaskId)
 	} catch (e: unknown) {
-		usage = api.getTokenUsage(taskId)
+		usage = api.getTokenUsage(rooTaskId)
 		console.error(e)
 	}
 
-	const task = await createTask({
-		runId,
-		language,
-		exercise,
+	const taskMetrics = await createTaskMetrics({
 		duration: Date.now() - startTime,
 		tokensIn: usage.totalTokensIn,
 		tokensOut: usage.totalTokensOut,
@@ -138,8 +134,11 @@ export async function run() {
 		cacheWrites: usage.totalCacheWrites ?? 0,
 		cacheReads: usage.totalCacheReads ?? 0,
 		cost: usage.totalCost,
-		passed: false,
 	})
 
-	await fs.writeFile(path.resolve(workspacePath, "usage.json"), JSON.stringify(task, null, 2))
+	task = await updateTask(task.id, { taskMetricsId: taskMetrics.id, finishedAt: new Date() })
+
+	server.broadcast({ type: ServerMessageType.Data, data: { event: "taskFinished", task, taskMetrics } })
+
+	await fs.writeFile(path.resolve(workspacePath, "usage.json"), JSON.stringify({ ...task, ...taskMetrics }, null, 2))
 }

benchmark/pnpm-lock.yaml

@@ -53,6 +53,9 @@ importers:
       gluegun:
         specifier: ^5.1.2
         version: 5.2.0
+      p-map:
+        specifier: ^7.0.3
+        version: 7.0.3
     devDependencies:
       '@benchmark/eslint-config':
         specifier: workspace:^
@@ -2707,6 +2710,7 @@ packages:
 
   libsql@0.4.7:
     resolution: {integrity: sha512-T9eIRCs6b0J1SHKYIvD8+KCJMcWZ900iZyxdnSCdqxN12Z1ijzT+jY5nrk72Jw4B0HGzms2NgpryArlJqvc3Lw==}
+    cpu: [x64, arm64, wasm32]
     os: [darwin, linux, win32]
 
   lie@3.3.0:
@@ -3045,6 +3049,10 @@ packages:
     resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
     engines: {node: '>=10'}
 
+  p-map@7.0.3:
+    resolution: {integrity: sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==}
+    engines: {node: '>=18'}
+
   package-json-from-dist@1.0.1:
     resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==}
 
@@ -6476,6 +6484,8 @@ snapshots:
     dependencies:
       p-limit: 3.1.0
 
+  p-map@7.0.3: {}
+
   package-json-from-dist@1.0.1: {}
 
   pako@1.0.11: {}

benchmark/turbo.json

@@ -4,7 +4,7 @@
 	"globalEnv": [
 		"NODE_ENV",
 		"NEXT_RUNTIME",
-		"RUN_ID",
+		"TASK_ID",
 		"LANGUAGE",
 		"EXERCISE",
 		"OPENROUTER_API_KEY",