From d79c72ec2e6218a18c0bab26fe0d84a747a85b38 Mon Sep 17 00:00:00 2001 From: cte Date: Mon, 24 Mar 2025 01:12:34 -0700 Subject: [PATCH] More progress --- benchmark/apps/cli/src/index.ts | 113 ++++----- benchmark/apps/web/package.json | 1 + benchmark/apps/web/src/app/globals.css | 1 + benchmark/apps/web/src/app/runs/[id]/run.tsx | 3 +- .../apps/web/src/app/runs/new/actions.ts | 33 --- .../apps/web/src/app/runs/new/new-run.tsx | 228 +++++++++++------- .../apps/web/src/components/ui/button.tsx | 3 +- .../web/src/components/ui/multi-select.tsx | 114 +++++---- .../apps/web/src/components/ui/select.tsx | 10 +- benchmark/apps/web/src/hooks/use-exercises.ts | 5 + .../runs/[id] => hooks}/use-run-status.ts | 7 +- .../{formatCurrency.ts => format-currency.ts} | 0 .../{formatDuration.ts => format-duration.ts} | 0 benchmark/apps/web/src/lib/index.ts | 4 +- .../web/src/{app/runs/[id] => lib}/schemas.ts | 23 ++ .../apps/web/src/lib/server/exercises.ts | 35 +++ benchmark/apps/web/src/lib/server/runs.ts | 51 ++++ .../[id]/actions.ts => lib/server/tasks.ts} | 0 benchmark/package.json | 4 +- benchmark/packages/db/src/queries/runs.ts | 35 ++- benchmark/packages/runner/src/index.ts | 10 +- benchmark/pnpm-lock.yaml | 9 + 22 files changed, 427 insertions(+), 262 deletions(-) delete mode 100644 benchmark/apps/web/src/app/runs/new/actions.ts create mode 100644 benchmark/apps/web/src/hooks/use-exercises.ts rename benchmark/apps/web/src/{app/runs/[id] => hooks}/use-run-status.ts (93%) rename benchmark/apps/web/src/lib/{formatCurrency.ts => format-currency.ts} (100%) rename benchmark/apps/web/src/lib/{formatDuration.ts => format-duration.ts} (100%) rename benchmark/apps/web/src/{app/runs/[id] => lib}/schemas.ts (51%) create mode 100644 benchmark/apps/web/src/lib/server/exercises.ts create mode 100644 benchmark/apps/web/src/lib/server/runs.ts rename benchmark/apps/web/src/{app/runs/[id]/actions.ts => lib/server/tasks.ts} (100%) diff --git a/benchmark/apps/cli/src/index.ts b/benchmark/apps/cli/src/index.ts index 0006e5b1f5..192c7b7d45 100644 --- a/benchmark/apps/cli/src/index.ts +++ b/benchmark/apps/cli/src/index.ts @@ -7,7 +7,7 @@ import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun" import { runTests } from "@vscode/test-electron" import { type Language, languages } from "@benchmark/types" -import { type Run, findRun, createRun, getTask, createTask, Task } from "@benchmark/db" +import { type Run, findRun, createRun, finishRun, createTask, Task, getTasks } from "@benchmark/db" import { __dirname, extensionDevelopmentPath, extensionTestsPath, exercisesPath } from "./paths.js" import { getExercises } from "./exercises.js" @@ -16,68 +16,56 @@ export const isLanguage = (language: string): language is Language => languages. const run = async (toolbox: GluegunToolbox) => { const { config, prompt } = toolbox - const id = config.runId ? Number(config.runId) : undefined + let { language, exercise } = config - if (language === "all") { - await runAll(id) - } else if (exercise === "all") { - await runLanguage({ id, language }) + if (![undefined, ...languages, "all"].includes(language)) { + throw new Error(`Language is invalid: ${language}`) + } + + if (!["undefined", "string"].includes(typeof exercise)) { + throw new Error(`Exercise is invalid: ${exercise}`) + } + + const id = config.runId ? Number(config.runId) : undefined + let run: Run + + if (id) { + run = await findRun(id) } else { - language = language || (await askLanguage(prompt)) - exercise = exercise || (await askExercise(prompt, language)) - await runLanguageExercise({ id, language, exercise }) + run = await createRun({ + model: "anthropic/claude-3.7-sonnet", + pid: process.pid, + socketPath: path.resolve(os.tmpdir(), `benchmark-${crypto.randomUUID()}.sock`), + }) + + if (language === "all") { + for (const language of languages) { + const exercises = getExercises()[language as Language] + await pMap(exercises, async (exercise) => createTask({ runId: run.id, language, exercise }), { + concurrency: 10, + }) + } + } else if (exercise === "all") { + const exercises = getExercises()[language as Language] + await pMap(exercises, async (exercise) => createTask({ runId: run.id, language, exercise }), { + concurrency: 10, + }) + } else { + language = language || (await askLanguage(prompt)) + exercise = exercise || (await askExercise(prompt, language)) + await createTask({ runId: run.id, language, exercise }) + } } -} -const runAll = async (id?: number) => { - const run = await findOrCreateRun({ id }) - - const entries = Object.entries(getExercises()).flatMap(([language, languageExercises]) => - languageExercises.map((exercise) => ({ language: language as Language, exercise })), - ) - - const tasks = await pMap( - entries, - async ({ language, exercise }) => findOrCreateTask({ runId: run.id, language, exercise }), - { concurrency: 10 }, - ) + const tasks = await getTasks(run.id) for (const task of tasks) { await runExercise({ run, task }) } -} -const runLanguage = async ({ id, language }: { id?: number; language: Language }) => { - const run = await findOrCreateRun({ id }) - - const tasks = await pMap( - getExercises()[language], - async (exercise) => findOrCreateTask({ runId: run.id, language, exercise }), - { concurrency: 10 }, - ) - - for (const task of tasks) { - await runExercise({ run, task }) - } -} - -const runLanguageExercise = async ({ - id, - language, - exercise, -}: { - id?: number - language: Language - exercise: string -}) => { - if (!getExercises()[language].includes(exercise)) { - throw new Error(`Exercise ${exercise} not found for language ${language}`) - } - - const run = await findOrCreateRun({ id }) - const task = await findOrCreateTask({ runId: run.id, language, exercise }) - return runExercise({ run, task }) + const result = await finishRun(run.id) + console.log(result) } const runExercise = async ({ run, task }: { run: Run; task: Task }) => { @@ -139,25 +127,6 @@ const askExercise = async (prompt: GluegunPrompt, language: Language) => { return exercise } -const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: { id?: number; model?: string }) => - id - ? findRun(id) - : createRun({ - model, - pid: process.pid, - socketPath: path.resolve(os.tmpdir(), `benchmark-${crypto.randomUUID()}.sock`), - }) - -const findOrCreateTask = async ({ - runId, - language, - exercise, -}: { - runId: number - language: Language - exercise: string -}) => (await getTask({ runId, language, exercise })) || (await createTask({ runId, language, exercise })) - const main = async () => { const cli = build() .brand("cli") diff --git a/benchmark/apps/web/package.json b/benchmark/apps/web/package.json index d47b35f1fe..c078ac48fb 100644 --- a/benchmark/apps/web/package.json +++ b/benchmark/apps/web/package.json @@ -26,6 +26,7 @@ "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "cmdk": "1.0.0", + "fuzzysort": "^3.1.0", "lucide-react": "^0.479.0", "next": "15.2.2", "next-themes": "^0.4.6", diff --git a/benchmark/apps/web/src/app/globals.css b/benchmark/apps/web/src/app/globals.css index 6cff4c4208..b9d4caf20c 100644 --- a/benchmark/apps/web/src/app/globals.css +++ b/benchmark/apps/web/src/app/globals.css @@ -135,5 +135,6 @@ } body { @apply bg-background text-foreground; + scrollbar-color: var(--color-background) transparent; /* Firefox */ } } diff --git a/benchmark/apps/web/src/app/runs/[id]/run.tsx b/benchmark/apps/web/src/app/runs/[id]/run.tsx index 9d55577785..48b9d60696 100644 --- a/benchmark/apps/web/src/app/runs/[id]/run.tsx +++ b/benchmark/apps/web/src/app/runs/[id]/run.tsx @@ -4,7 +4,8 @@ import { LoaderCircle } from "lucide-react" import * as db from "@benchmark/db" -import { useRunStatus } from "./use-run-status" +import { useRunStatus } from "@/hooks/use-run-status" + import { TaskStatus } from "./task-status" import { ConnectionStatus } from "./connection-status" diff --git a/benchmark/apps/web/src/app/runs/new/actions.ts b/benchmark/apps/web/src/app/runs/new/actions.ts deleted file mode 100644 index e41b7381f5..0000000000 --- a/benchmark/apps/web/src/app/runs/new/actions.ts +++ /dev/null @@ -1,33 +0,0 @@ -"use server" - -import { spawn } from "child_process" -import path from "path" -import os from "os" - -import { revalidatePath } from "next/cache" - -import * as db from "@benchmark/db" - -export async function createRun(data: Omit) { - const socketPath = path.join(os.tmpdir(), `benchmark-${crypto.randomUUID()}.sock`) - const run = await db.createRun({ ...data, socketPath }) - revalidatePath("/runs") - - try { - const process = spawn( - "pnpm", - ["--filter", "@benchmark/cli", "dev", "run", "all", "--runId", run.id.toString()], - { - detached: true, - stdio: "ignore", - }, - ) - - process.unref() - await db.updateRun(run.id, { pid: process.pid }) - } catch (error) { - console.error(error) - } - - return run -} diff --git a/benchmark/apps/web/src/app/runs/new/new-run.tsx b/benchmark/apps/web/src/app/runs/new/new-run.tsx index 9dd23aedfb..2752c4948f 100644 --- a/benchmark/apps/web/src/app/runs/new/new-run.tsx +++ b/benchmark/apps/web/src/app/runs/new/new-run.tsx @@ -1,15 +1,17 @@ "use client" -import { useState } from "react" +import { useCallback, useEffect, useRef, useState } from "react" import { useRouter } from "next/navigation" import { useForm, FormProvider } from "react-hook-form" import { zodResolver } from "@hookform/resolvers/zod" -import { z } from "zod" -import { X, Loader2, Rocket } from "lucide-react" - -import { languages } from "@benchmark/types" +import fuzzysort from "fuzzysort" +import { X, Rocket, Check, ChevronsUpDown } from "lucide-react" +import { createRun } from "@/lib/server/runs" +import { createRunSchema as formSchema, type CreateRun as FormValues } from "@/lib/schemas" +import { cn } from "@/lib/utils" import { useOpenRouterModels } from "@/hooks/use-open-router-models" +import { useExercises } from "@/hooks/use-exercises" import { Button, FormControl, @@ -17,56 +19,70 @@ import { FormItem, FormLabel, FormMessage, - FormDescription, - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, Textarea, Tabs, - TabsContent, TabsList, TabsTrigger, MultiSelect, + Command, + CommandEmpty, + CommandGroup, + CommandInput, + CommandItem, + CommandList, + Popover, + PopoverContent, + PopoverTrigger, } from "@/components/ui" -import { createRun } from "./actions" -import { cn } from "@/lib/utils" - -const formSchema = z.object({ - model: z.string(), - description: z.string().optional(), -}) - -type FormValues = z.infer - export function NewRun() { const router = useRouter() - const { data: models, isLoading, isError } = useOpenRouterModels() - const [isSubmitting, setIsSubmitting] = useState(false) + + const [modelSearchValue, setModelSearchValue] = useState("") + const [modelPopoverOpen, setModelPopoverOpen] = useState(false) + const modelSearchResultsRef = useRef>(new Map()) + const modelSearchValueRef = useRef("") + const models = useOpenRouterModels() + + const exercises = useExercises() const form = useForm({ resolver: zodResolver(formSchema), defaultValues: { - model: "anthropic/claude-3.7-sonnet", + model: "", description: "", + suite: "full", + exercises: [], }, }) - async function onSubmit(data: FormValues) { - setIsSubmitting(true) + const { + setValue, + watch, + formState: { isSubmitting }, + } = form - try { - const run = await createRun(data) - router.push(`/runs/${run.id}`) - } catch (_) { - setIsSubmitting(false) - } - } + const [model, suite] = watch(["model", "suite"]) - const [selectedSuite, setSelectedSuite] = useState<"full" | "partial">("full") - const [selectedExercises, setSelectedExercises] = useState([]) + const selectModel = useCallback( + (model: string) => { + setValue("model", model) + setModelPopoverOpen(false) + }, + [setValue], + ) + + const onSubmit = useCallback( + async (data: FormValues) => { + try { + const { id } = await createRun(data) + router.push(`/runs/${id}`) + } catch (_) { + // Surface error. + } + }, + [router], + ) return ( <> @@ -75,63 +91,107 @@ export function NewRun() { ( + render={() => ( OpenRouter Model - + + + + + { + if (modelSearchValueRef.current !== search) { + modelSearchValueRef.current = search + modelSearchResultsRef.current.clear() + + for (const { + obj: { id }, + score, + } of fuzzysort.go(search, models.data || [], { + key: "name", + })) { + modelSearchResultsRef.current.set(id, score) + } + } + + return modelSearchResultsRef.current.get(value) ?? 0 + }}> + + + No model found. + + {models.data?.map(({ id, name }) => ( + + {name} + + + ))} + + + + + + )} /> - - Exercise Suite - setSelectedSuite(value as "full" | "partial")}> - - Full - Partial - - - + ( + + Exercise Suite + setValue("suite", value as "full" | "partial")}> + + Full + Partial + + + + )} + /> - {selectedSuite === "partial" && ( - - Exercises - ({ - value: language, - label: language, - }))} - onValueChange={setSelectedExercises} - defaultValue={selectedExercises} - placeholder="Select" - variant="inverted" - maxCount={10} - /> - + {suite === "partial" && ( + ( + + Exercises + ({ value: path, label: path })) || []} + onValueChange={(value) => setValue("exercises", value)} + placeholder="Select" + variant="inverted" + maxCount={4} + /> + + )} + /> )} svg]:px-3", diff --git a/benchmark/apps/web/src/components/ui/multi-select.tsx b/benchmark/apps/web/src/components/ui/multi-select.tsx index b7396787db..e7264e5360 100644 --- a/benchmark/apps/web/src/components/ui/multi-select.tsx +++ b/benchmark/apps/web/src/components/ui/multi-select.tsx @@ -1,20 +1,13 @@ import * as React from "react" import { cva, type VariantProps } from "class-variance-authority" -import { CheckIcon, X, ChevronDown } from "lucide-react" +import fuzzysort from "fuzzysort" +import { Check, X, ChevronsUpDown } from "lucide-react" import { cn } from "@/lib/utils" import { Badge } from "./badge" import { Popover, PopoverContent, PopoverTrigger } from "./popover" -import { - Command, - CommandEmpty, - CommandGroup, - CommandInput, - CommandItem, - CommandList, - CommandSeparator, -} from "./command" +import { Command, CommandEmpty, CommandGroup, CommandInput, CommandItem, CommandList } from "./command" /** * Variants for the multi-select component to handle different styles. @@ -127,11 +120,6 @@ export const MultiSelect = React.forwardRef( onValueChange(newSelectedValues) } - const handleClear = () => { - setSelectedValues([]) - onValueChange([]) - } - const handleTogglePopover = () => { setIsPopoverOpen((prev) => !prev) } @@ -142,16 +130,50 @@ export const MultiSelect = React.forwardRef( onValueChange(newSelectedValues) } - const toggleAll = () => { - if (selectedValues.length === options.length) { - handleClear() - } else { - const allValues = options.map((option) => option.value) - setSelectedValues(allValues) - onValueChange(allValues) + const searchResultsRef = React.useRef>(new Map()) + const searchValueRef = React.useRef("") + + const onSelectAll = () => { + const values = Array.from(searchResultsRef.current.keys()) + + if ( + selectedValues.length === values.length && + selectedValues.sort().join(",") === values.sort().join(",") + ) { + setSelectedValues([]) + onValueChange([]) + return } + + setSelectedValues(values) + onValueChange(values) } + const onFilter = React.useCallback( + (value: string, search: string) => { + if (searchValueRef.current !== search) { + searchValueRef.current = search + searchResultsRef.current.clear() + + for (const { + obj: { value }, + score, + } of fuzzysort.go(search, options, { + key: "label", + })) { + searchResultsRef.current.set(value, score) + } + } + + if (value === "all") { + return searchResultsRef.current.size > 1 ? 0.01 : 0 + } + + return searchResultsRef.current.get(value) ?? 0 + }, + [options], + ) + return ( @@ -160,13 +182,13 @@ export const MultiSelect = React.forwardRef( {...props} onClick={handleTogglePopover} className={cn( - "flex w-full p-1 rounded-sm border min-h-10 h-auto items-center justify-between [&_svg]:pointer-events-auto pl-2 pr-3", + "flex w-full rounded-sm border min-h-9 h-auto items-center justify-between [&_svg]:pointer-events-auto", "border border-input bg-input hover:opacity-80 cursor-pointer", className, )}> {selectedValues.length > 0 ? (
-
+
{selectedValues.slice(0, maxCount).map((value) => (
@@ -202,39 +224,28 @@ export const MultiSelect = React.forwardRef( ) : (
{placeholder} - +
)}
- setIsPopoverOpen(false)}> - + setIsPopoverOpen(false)}> + No results found. - - Select All - - - {options.map((option) => ( toggleOption(option.value)} className="flex items-center justify-between"> {option.label} - ( /> ))} - {selectedValues.length > 0 && ( - <> - - - Select None - - - )} + + Select All + diff --git a/benchmark/apps/web/src/components/ui/select.tsx b/benchmark/apps/web/src/components/ui/select.tsx index 3fd27c554c..1ad884f873 100644 --- a/benchmark/apps/web/src/components/ui/select.tsx +++ b/benchmark/apps/web/src/components/ui/select.tsx @@ -2,7 +2,7 @@ import * as React from "react" import * as SelectPrimitive from "@radix-ui/react-select" -import { CheckIcon, ChevronDownIcon, ChevronUpIcon } from "lucide-react" +import { Check, ChevronDown, ChevronUp } from "lucide-react" import { cn } from "@/lib/utils" @@ -38,7 +38,7 @@ function SelectTrigger({ {...props}> {children} - + ) @@ -99,7 +99,7 @@ function SelectItem({ className, children, ...props }: React.ComponentProps - + {children} @@ -123,7 +123,7 @@ function SelectScrollUpButton({ className, ...props }: React.ComponentProps - + ) } @@ -137,7 +137,7 @@ function SelectScrollDownButton({ data-slot="select-scroll-down-button" className={cn("flex cursor-default items-center justify-center py-1", className)} {...props}> - + ) } diff --git a/benchmark/apps/web/src/hooks/use-exercises.ts b/benchmark/apps/web/src/hooks/use-exercises.ts new file mode 100644 index 0000000000..2149f70cda --- /dev/null +++ b/benchmark/apps/web/src/hooks/use-exercises.ts @@ -0,0 +1,5 @@ +import { useQuery } from "@tanstack/react-query" + +import { getExercises } from "@/lib/server/exercises" + +export const useExercises = () => useQuery({ queryKey: ["exercises"], queryFn: getExercises }) diff --git a/benchmark/apps/web/src/app/runs/[id]/use-run-status.ts b/benchmark/apps/web/src/hooks/use-run-status.ts similarity index 93% rename from benchmark/apps/web/src/app/runs/[id]/use-run-status.ts rename to benchmark/apps/web/src/hooks/use-run-status.ts index 344b6efca6..6d80e86162 100644 --- a/benchmark/apps/web/src/app/runs/[id]/use-run-status.ts +++ b/benchmark/apps/web/src/hooks/use-run-status.ts @@ -1,12 +1,11 @@ import { useState, useCallback } from "react" import { useQuery, keepPreviousData } from "@tanstack/react-query" -import { useEventSource } from "@/hooks/use-event-source" - import { Run } from "@benchmark/db" -import { getTasks } from "./actions" -import { ipcServerMessageSchema } from "./schemas" +import { getTasks } from "@/lib/server/tasks" +import { ipcServerMessageSchema } from "@/lib/schemas" +import { useEventSource } from "@/hooks/use-event-source" export const useRunStatus = (run: Run) => { const [clientId, setClientId] = useState() diff --git a/benchmark/apps/web/src/lib/formatCurrency.ts b/benchmark/apps/web/src/lib/format-currency.ts similarity index 100% rename from benchmark/apps/web/src/lib/formatCurrency.ts rename to benchmark/apps/web/src/lib/format-currency.ts diff --git a/benchmark/apps/web/src/lib/formatDuration.ts b/benchmark/apps/web/src/lib/format-duration.ts similarity index 100% rename from benchmark/apps/web/src/lib/formatDuration.ts rename to benchmark/apps/web/src/lib/format-duration.ts diff --git a/benchmark/apps/web/src/lib/index.ts b/benchmark/apps/web/src/lib/index.ts index b64fcc6b02..e2d57baac5 100644 --- a/benchmark/apps/web/src/lib/index.ts +++ b/benchmark/apps/web/src/lib/index.ts @@ -1,2 +1,2 @@ -export { formatCurrency } from "./formatCurrency" -export { formatDuration } from "./formatDuration" +export { formatCurrency } from "./format-currency" +export { formatDuration } from "./format-duration" diff --git a/benchmark/apps/web/src/app/runs/[id]/schemas.ts b/benchmark/apps/web/src/lib/schemas.ts similarity index 51% rename from benchmark/apps/web/src/app/runs/[id]/schemas.ts rename to benchmark/apps/web/src/lib/schemas.ts index 8fc15181a7..b827e64d1a 100644 --- a/benchmark/apps/web/src/app/runs/[id]/schemas.ts +++ b/benchmark/apps/web/src/lib/schemas.ts @@ -1,5 +1,26 @@ import { z } from "zod" +/** + * CreateRun + */ + +export const createRunSchema = z + .object({ + model: z.string().min(1), + description: z.string().optional(), + suite: z.enum(["full", "partial"]), + exercises: z.array(z.string()).optional(), + }) + .refine((data) => data.suite === "full" || (data.exercises || []).length > 0, { + message: "Exercises are required for partial suite.", + }) + +export type CreateRun = z.infer + +/** + * IpcServerMessage + */ + export const ipcServerMessageSchema = z.discriminatedUnion("type", [ z.object({ type: z.literal("Ack"), @@ -18,3 +39,5 @@ export const ipcServerMessageSchema = z.discriminatedUnion("type", [ }), }), ]) + +export type IpcServerMessage = z.infer diff --git a/benchmark/apps/web/src/lib/server/exercises.ts b/benchmark/apps/web/src/lib/server/exercises.ts new file mode 100644 index 0000000000..b508de9186 --- /dev/null +++ b/benchmark/apps/web/src/lib/server/exercises.ts @@ -0,0 +1,35 @@ +"use server" + +import * as fs from "fs/promises" +import * as path from "path" +import { fileURLToPath } from "url" + +import { languages } from "@benchmark/types" + +const __dirname = path.dirname(fileURLToPath(import.meta.url)) + +const listDirectories = async (relativePath: string) => { + try { + const targetPath = path.resolve(__dirname, relativePath) + const entries = await fs.readdir(targetPath, { withFileTypes: true }) + return entries.filter((entry) => entry.isDirectory() && !entry.name.startsWith(".")).map((entry) => entry.name) + } catch (error) { + console.error(`Error listing directories at ${relativePath}:`, error) + return [] + } +} + +// __dirname = /benchmark/apps/web/src/lib/server +const EXERCISES_BASE_PATH = path.resolve(__dirname, "../../../../../../../exercises") + +export const getExercises = async () => { + const result = await Promise.all( + languages.map(async (language) => { + const languagePath = path.join(EXERCISES_BASE_PATH, language) + const exercises = await listDirectories(languagePath) + return exercises.map((exercise) => `${language}/${exercise}`) + }), + ) + + return result.flat() +} diff --git a/benchmark/apps/web/src/lib/server/runs.ts b/benchmark/apps/web/src/lib/server/runs.ts new file mode 100644 index 0000000000..987f7a97a5 --- /dev/null +++ b/benchmark/apps/web/src/lib/server/runs.ts @@ -0,0 +1,51 @@ +"use server" + +import { spawn } from "child_process" +import path from "path" +import os from "os" + +import { revalidatePath } from "next/cache" + +import { Language } from "@benchmark/types" +import * as db from "@benchmark/db" + +import { CreateRun } from "@/lib/schemas" + +export async function createRun({ suite, exercises = [], ...values }: CreateRun) { + const run = await db.createRun({ + ...values, + socketPath: path.join(os.tmpdir(), `benchmark-${crypto.randomUUID()}.sock`), + }) + + if (suite === "partial") { + for (const path of exercises) { + const [language, exercise] = path.split("/") + + if (!language || !exercise) { + throw new Error("Invalid exercise path: " + path) + } + + await db.createTask({ ...values, runId: run.id, language: language as Language, exercise }) + } + } + + revalidatePath("/runs") + + try { + const process = spawn( + "pnpm", + ["--filter", "@benchmark/cli", "dev", "run", "all", "--runId", run.id.toString()], + { + detached: true, + stdio: "ignore", + }, + ) + + process.unref() + await db.updateRun(run.id, { pid: process.pid }) + } catch (error) { + console.error(error) + } + + return run +} diff --git a/benchmark/apps/web/src/app/runs/[id]/actions.ts b/benchmark/apps/web/src/lib/server/tasks.ts similarity index 100% rename from benchmark/apps/web/src/app/runs/[id]/actions.ts rename to benchmark/apps/web/src/lib/server/tasks.ts diff --git a/benchmark/package.json b/benchmark/package.json index 87ddb202cf..d9763a4619 100644 --- a/benchmark/package.json +++ b/benchmark/package.json @@ -7,8 +7,8 @@ "check-types": "turbo check-types --log-order grouped --output-logs new-only", "format": "turbo format --log-order grouped --output-logs new-only", "build": "turbo build --log-order grouped --output-logs new-only", - "web": "turbo dev --filter @benchmark/web --log-order grouped --output-logs new-only", - "cli": "turbo dev --filter @benchmark/cli --ui tui -- run", + "web": "turbo dev --filter @benchmark/web --output-logs new-only --ui tui", + "cli": "turbo dev --filter @benchmark/cli --output-logs new-only --ui tui -- run", "drizzle:studio": "pnpm --filter @benchmark/db db:studio", "docker:build": "docker build -f Dockerfile -t roo-code-benchmark ..", "docker:run": "touch /tmp/benchmarks.db && docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark", diff --git a/benchmark/packages/db/src/queries/runs.ts b/benchmark/packages/db/src/queries/runs.ts index 20c8027ea5..faed9b67ff 100644 --- a/benchmark/packages/db/src/queries/runs.ts +++ b/benchmark/packages/db/src/queries/runs.ts @@ -1,9 +1,10 @@ -import { desc, eq } from "drizzle-orm" +import { desc, eq, sum } from "drizzle-orm" import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js" import type { InsertRun, UpdateRun } from "../schema.js" import { insertRunSchema, schema } from "../schema.js" import { db } from "../db.js" +import { createTaskMetrics } from "./taskMetrics.js" const table = schema.runs @@ -47,3 +48,35 @@ export const updateRun = async (id: number, values: UpdateRun) => { } export const getRuns = async () => db.query.runs.findMany({ orderBy: desc(table.id), with: { taskMetrics: true } }) + +export const finishRun = async (runId: number) => { + const [values] = await db + .select({ + tokensIn: sum(schema.taskMetrics.tokensIn).mapWith(Number), + tokensOut: sum(schema.taskMetrics.tokensOut).mapWith(Number), + tokensContext: sum(schema.taskMetrics.tokensContext).mapWith(Number), + cacheWrites: sum(schema.taskMetrics.cacheWrites).mapWith(Number), + cacheReads: sum(schema.taskMetrics.cacheReads).mapWith(Number), + cost: sum(schema.taskMetrics.cost).mapWith(Number), + duration: sum(schema.taskMetrics.duration).mapWith(Number), + }) + .from(schema.taskMetrics) + .innerJoin(schema.tasks, eq(schema.taskMetrics.id, schema.tasks.taskMetricsId)) + .innerJoin(schema.runs, eq(schema.tasks.runId, schema.runs.id)) + .where(eq(schema.runs.id, runId)) + + if (!values) { + throw new RecordNotFoundError() + } + + const taskMetrics = await createTaskMetrics(values) + await updateRun(runId, { taskMetricsId: taskMetrics.id }) + + const run = await db.query.runs.findFirst({ where: eq(table.id, runId), with: { taskMetrics: true } }) + + if (!run) { + throw new RecordNotFoundError() + } + + return run +} diff --git a/benchmark/packages/runner/src/index.ts b/benchmark/packages/runner/src/index.ts index 0a1c03ba03..411a35ee58 100644 --- a/benchmark/packages/runner/src/index.ts +++ b/benchmark/packages/runner/src/index.ts @@ -15,14 +15,16 @@ export async function run() { * Validate environment variables. */ - const tid = process.env.TASK_ID ? parseInt(process.env.TASK_ID) : undefined + const taskId = process.env.TASK_ID ? parseInt(process.env.TASK_ID) : undefined const promptPath = process.env.PROMPT_PATH const workspacePath = process.env.WORKSPACE_PATH const openRouterApiKey = process.env.OPENROUTER_API_KEY const openRouterModelId = process.env.OPENROUTER_MODEL_ID - if (!tid || !promptPath || !workspacePath || !openRouterApiKey || !openRouterModelId) { - throw new Error("ENV not configured.") + if (!taskId || !promptPath || !workspacePath || !openRouterApiKey || !openRouterModelId) { + throw new Error( + `ENV not configured. ${JSON.stringify({ taskId, promptPath, workspacePath, openRouterApiKey, openRouterModelId })}`, + ) } const prompt = await fs.readFile(promptPath, "utf-8") @@ -31,7 +33,7 @@ export async function run() { * Fetch and update the task. */ - let task = await findTask(tid) + let task = await findTask(taskId) task = await updateTask(task.id, { startedAt: new Date() }) const run = await findRun(task.runId) diff --git a/benchmark/pnpm-lock.yaml b/benchmark/pnpm-lock.yaml index 41a1bbf605..9e0b033eb4 100644 --- a/benchmark/pnpm-lock.yaml +++ b/benchmark/pnpm-lock.yaml @@ -114,6 +114,9 @@ importers: cmdk: specifier: 1.0.0 version: 1.0.0(@types/react-dom@19.0.4(@types/react@19.0.12))(@types/react@19.0.12)(react-dom@19.0.0(react@19.0.0))(react@19.0.0) + fuzzysort: + specifier: ^3.1.0 + version: 3.1.0 lucide-react: specifier: ^0.479.0 version: 0.479.0(react@19.0.0) @@ -2689,6 +2692,9 @@ packages: functions-have-names@1.2.3: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} + fuzzysort@3.1.0: + resolution: {integrity: sha512-sR9BNCjBg6LNgwvxlBd0sBABvQitkLzoVY9MYYROQVX/FvfJ4Mai9LsGhDgd8qYdds0bY77VzYd5iuB+v5rwQQ==} + gel@2.0.1: resolution: {integrity: sha512-gfem3IGvqKqXwEq7XseBogyaRwGsQGuE7Cw/yQsjLGdgiyqX92G1xENPCE0ltunPGcsJIa6XBOTx/PK169mOqw==} engines: {node: '>= 18.0.0'} @@ -3061,6 +3067,7 @@ packages: libsql@0.4.7: resolution: {integrity: sha512-T9eIRCs6b0J1SHKYIvD8+KCJMcWZ900iZyxdnSCdqxN12Z1ijzT+jY5nrk72Jw4B0HGzms2NgpryArlJqvc3Lw==} + cpu: [x64, arm64, wasm32] os: [darwin, linux, win32] lie@3.3.0: @@ -6483,6 +6490,8 @@ snapshots: functions-have-names@1.2.3: {} + fuzzysort@3.1.0: {} + gel@2.0.1: dependencies: '@petamoriken/float16': 3.9.2