mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-06 08:18:39 +00:00
More progress
This commit is contained in:
cte
parent
2012901dae
commit
d79c72ec2e
22 changed files with 427 additions and 262 deletions
|
|
@ -7,7 +7,7 @@ import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun"
|
|||
import { runTests } from "@vscode/test-electron"
|
||||
|
||||
import { type Language, languages } from "@benchmark/types"
|
||||
import { type Run, findRun, createRun, getTask, createTask, Task } from "@benchmark/db"
|
||||
import { type Run, findRun, createRun, finishRun, createTask, Task, getTasks } from "@benchmark/db"
|
||||
|
||||
import { __dirname, extensionDevelopmentPath, extensionTestsPath, exercisesPath } from "./paths.js"
|
||||
import { getExercises } from "./exercises.js"
|
||||
|
|
@ -16,68 +16,56 @@ export const isLanguage = (language: string): language is Language => languages.
|
|||
|
||||
const run = async (toolbox: GluegunToolbox) => {
|
||||
const { config, prompt } = toolbox
|
||||
const id = config.runId ? Number(config.runId) : undefined
|
||||
|
||||
let { language, exercise } = config
|
||||
|
||||
if (language === "all") {
|
||||
await runAll(id)
|
||||
} else if (exercise === "all") {
|
||||
await runLanguage({ id, language })
|
||||
if (![undefined, ...languages, "all"].includes(language)) {
|
||||
throw new Error(`Language is invalid: ${language}`)
|
||||
}
|
||||
|
||||
if (!["undefined", "string"].includes(typeof exercise)) {
|
||||
throw new Error(`Exercise is invalid: ${exercise}`)
|
||||
}
|
||||
|
||||
const id = config.runId ? Number(config.runId) : undefined
|
||||
let run: Run
|
||||
|
||||
if (id) {
|
||||
run = await findRun(id)
|
||||
} else {
|
||||
language = language || (await askLanguage(prompt))
|
||||
exercise = exercise || (await askExercise(prompt, language))
|
||||
await runLanguageExercise({ id, language, exercise })
|
||||
run = await createRun({
|
||||
model: "anthropic/claude-3.7-sonnet",
|
||||
pid: process.pid,
|
||||
socketPath: path.resolve(os.tmpdir(), `benchmark-${crypto.randomUUID()}.sock`),
|
||||
})
|
||||
|
||||
if (language === "all") {
|
||||
for (const language of languages) {
|
||||
const exercises = getExercises()[language as Language]
|
||||
await pMap(exercises, async (exercise) => createTask({ runId: run.id, language, exercise }), {
|
||||
concurrency: 10,
|
||||
})
|
||||
}
|
||||
} else if (exercise === "all") {
|
||||
const exercises = getExercises()[language as Language]
|
||||
await pMap(exercises, async (exercise) => createTask({ runId: run.id, language, exercise }), {
|
||||
concurrency: 10,
|
||||
})
|
||||
} else {
|
||||
language = language || (await askLanguage(prompt))
|
||||
exercise = exercise || (await askExercise(prompt, language))
|
||||
await createTask({ runId: run.id, language, exercise })
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const runAll = async (id?: number) => {
|
||||
const run = await findOrCreateRun({ id })
|
||||
|
||||
const entries = Object.entries(getExercises()).flatMap(([language, languageExercises]) =>
|
||||
languageExercises.map((exercise) => ({ language: language as Language, exercise })),
|
||||
)
|
||||
|
||||
const tasks = await pMap(
|
||||
entries,
|
||||
async ({ language, exercise }) => findOrCreateTask({ runId: run.id, language, exercise }),
|
||||
{ concurrency: 10 },
|
||||
)
|
||||
const tasks = await getTasks(run.id)
|
||||
|
||||
for (const task of tasks) {
|
||||
await runExercise({ run, task })
|
||||
}
|
||||
}
|
||||
|
||||
const runLanguage = async ({ id, language }: { id?: number; language: Language }) => {
|
||||
const run = await findOrCreateRun({ id })
|
||||
|
||||
const tasks = await pMap(
|
||||
getExercises()[language],
|
||||
async (exercise) => findOrCreateTask({ runId: run.id, language, exercise }),
|
||||
{ concurrency: 10 },
|
||||
)
|
||||
|
||||
for (const task of tasks) {
|
||||
await runExercise({ run, task })
|
||||
}
|
||||
}
|
||||
|
||||
const runLanguageExercise = async ({
|
||||
id,
|
||||
language,
|
||||
exercise,
|
||||
}: {
|
||||
id?: number
|
||||
language: Language
|
||||
exercise: string
|
||||
}) => {
|
||||
if (!getExercises()[language].includes(exercise)) {
|
||||
throw new Error(`Exercise ${exercise} not found for language ${language}`)
|
||||
}
|
||||
|
||||
const run = await findOrCreateRun({ id })
|
||||
const task = await findOrCreateTask({ runId: run.id, language, exercise })
|
||||
return runExercise({ run, task })
|
||||
const result = await finishRun(run.id)
|
||||
console.log(result)
|
||||
}
|
||||
|
||||
const runExercise = async ({ run, task }: { run: Run; task: Task }) => {
|
||||
|
|
@ -139,25 +127,6 @@ const askExercise = async (prompt: GluegunPrompt, language: Language) => {
|
|||
return exercise
|
||||
}
|
||||
|
||||
const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: { id?: number; model?: string }) =>
|
||||
id
|
||||
? findRun(id)
|
||||
: createRun({
|
||||
model,
|
||||
pid: process.pid,
|
||||
socketPath: path.resolve(os.tmpdir(), `benchmark-${crypto.randomUUID()}.sock`),
|
||||
})
|
||||
|
||||
const findOrCreateTask = async ({
|
||||
runId,
|
||||
language,
|
||||
exercise,
|
||||
}: {
|
||||
runId: number
|
||||
language: Language
|
||||
exercise: string
|
||||
}) => (await getTask({ runId, language, exercise })) || (await createTask({ runId, language, exercise }))
|
||||
|
||||
const main = async () => {
|
||||
const cli = build()
|
||||
.brand("cli")
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@
|
|||
"class-variance-authority": "^0.7.1",
|
||||
"clsx": "^2.1.1",
|
||||
"cmdk": "1.0.0",
|
||||
"fuzzysort": "^3.1.0",
|
||||
"lucide-react": "^0.479.0",
|
||||
"next": "15.2.2",
|
||||
"next-themes": "^0.4.6",
|
||||
|
|
|
|||
|
|
@ -135,5 +135,6 @@
|
|||
}
|
||||
body {
|
||||
@apply bg-background text-foreground;
|
||||
scrollbar-color: var(--color-background) transparent; /* Firefox */
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ import { LoaderCircle } from "lucide-react"
|
|||
|
||||
import * as db from "@benchmark/db"
|
||||
|
||||
import { useRunStatus } from "./use-run-status"
|
||||
import { useRunStatus } from "@/hooks/use-run-status"
|
||||
|
||||
import { TaskStatus } from "./task-status"
|
||||
import { ConnectionStatus } from "./connection-status"
|
||||
|
||||
|
|
|
|||
|
|
@ -1,33 +0,0 @@
|
|||
"use server"
|
||||
|
||||
import { spawn } from "child_process"
|
||||
import path from "path"
|
||||
import os from "os"
|
||||
|
||||
import { revalidatePath } from "next/cache"
|
||||
|
||||
import * as db from "@benchmark/db"
|
||||
|
||||
export async function createRun(data: Omit<db.InsertRun, "socketPath">) {
|
||||
const socketPath = path.join(os.tmpdir(), `benchmark-${crypto.randomUUID()}.sock`)
|
||||
const run = await db.createRun({ ...data, socketPath })
|
||||
revalidatePath("/runs")
|
||||
|
||||
try {
|
||||
const process = spawn(
|
||||
"pnpm",
|
||||
["--filter", "@benchmark/cli", "dev", "run", "all", "--runId", run.id.toString()],
|
||||
{
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
},
|
||||
)
|
||||
|
||||
process.unref()
|
||||
await db.updateRun(run.id, { pid: process.pid })
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
}
|
||||
|
||||
return run
|
||||
}
|
||||
|
|
@ -1,15 +1,17 @@
|
|||
"use client"
|
||||
|
||||
import { useState } from "react"
|
||||
import { useCallback, useEffect, useRef, useState } from "react"
|
||||
import { useRouter } from "next/navigation"
|
||||
import { useForm, FormProvider } from "react-hook-form"
|
||||
import { zodResolver } from "@hookform/resolvers/zod"
|
||||
import { z } from "zod"
|
||||
import { X, Loader2, Rocket } from "lucide-react"
|
||||
|
||||
import { languages } from "@benchmark/types"
|
||||
import fuzzysort from "fuzzysort"
|
||||
import { X, Rocket, Check, ChevronsUpDown } from "lucide-react"
|
||||
|
||||
import { createRun } from "@/lib/server/runs"
|
||||
import { createRunSchema as formSchema, type CreateRun as FormValues } from "@/lib/schemas"
|
||||
import { cn } from "@/lib/utils"
|
||||
import { useOpenRouterModels } from "@/hooks/use-open-router-models"
|
||||
import { useExercises } from "@/hooks/use-exercises"
|
||||
import {
|
||||
Button,
|
||||
FormControl,
|
||||
|
|
@ -17,56 +19,70 @@ import {
|
|||
FormItem,
|
||||
FormLabel,
|
||||
FormMessage,
|
||||
FormDescription,
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
Textarea,
|
||||
Tabs,
|
||||
TabsContent,
|
||||
TabsList,
|
||||
TabsTrigger,
|
||||
MultiSelect,
|
||||
Command,
|
||||
CommandEmpty,
|
||||
CommandGroup,
|
||||
CommandInput,
|
||||
CommandItem,
|
||||
CommandList,
|
||||
Popover,
|
||||
PopoverContent,
|
||||
PopoverTrigger,
|
||||
} from "@/components/ui"
|
||||
|
||||
import { createRun } from "./actions"
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
const formSchema = z.object({
|
||||
model: z.string(),
|
||||
description: z.string().optional(),
|
||||
})
|
||||
|
||||
type FormValues = z.infer<typeof formSchema>
|
||||
|
||||
export function NewRun() {
|
||||
const router = useRouter()
|
||||
const { data: models, isLoading, isError } = useOpenRouterModels()
|
||||
const [isSubmitting, setIsSubmitting] = useState(false)
|
||||
|
||||
const [modelSearchValue, setModelSearchValue] = useState("")
|
||||
const [modelPopoverOpen, setModelPopoverOpen] = useState(false)
|
||||
const modelSearchResultsRef = useRef<Map<string, number>>(new Map())
|
||||
const modelSearchValueRef = useRef("")
|
||||
const models = useOpenRouterModels()
|
||||
|
||||
const exercises = useExercises()
|
||||
|
||||
const form = useForm<FormValues>({
|
||||
resolver: zodResolver(formSchema),
|
||||
defaultValues: {
|
||||
model: "anthropic/claude-3.7-sonnet",
|
||||
model: "",
|
||||
description: "",
|
||||
suite: "full",
|
||||
exercises: [],
|
||||
},
|
||||
})
|
||||
|
||||
async function onSubmit(data: FormValues) {
|
||||
setIsSubmitting(true)
|
||||
const {
|
||||
setValue,
|
||||
watch,
|
||||
formState: { isSubmitting },
|
||||
} = form
|
||||
|
||||
try {
|
||||
const run = await createRun(data)
|
||||
router.push(`/runs/${run.id}`)
|
||||
} catch (_) {
|
||||
setIsSubmitting(false)
|
||||
}
|
||||
}
|
||||
const [model, suite] = watch(["model", "suite"])
|
||||
|
||||
const [selectedSuite, setSelectedSuite] = useState<"full" | "partial">("full")
|
||||
const [selectedExercises, setSelectedExercises] = useState<string[]>([])
|
||||
const selectModel = useCallback(
|
||||
(model: string) => {
|
||||
setValue("model", model)
|
||||
setModelPopoverOpen(false)
|
||||
},
|
||||
[setValue],
|
||||
)
|
||||
|
||||
const onSubmit = useCallback(
|
||||
async (data: FormValues) => {
|
||||
try {
|
||||
const { id } = await createRun(data)
|
||||
router.push(`/runs/${id}`)
|
||||
} catch (_) {
|
||||
// Surface error.
|
||||
}
|
||||
},
|
||||
[router],
|
||||
)
|
||||
|
||||
return (
|
||||
<>
|
||||
|
|
@ -75,63 +91,107 @@ export function NewRun() {
|
|||
<FormField
|
||||
control={form.control}
|
||||
name="model"
|
||||
render={({ field }) => (
|
||||
render={() => (
|
||||
<FormItem>
|
||||
<FormLabel>OpenRouter Model</FormLabel>
|
||||
<Select onValueChange={field.onChange} defaultValue={field.value}>
|
||||
<FormControl>
|
||||
<SelectTrigger className="w-full">
|
||||
<SelectValue placeholder="Select" />
|
||||
</SelectTrigger>
|
||||
</FormControl>
|
||||
<SelectContent>
|
||||
{isLoading ? (
|
||||
<Loader2 className="size-4 m-2 animate-spin" />
|
||||
) : isError ? (
|
||||
<div className="m-2 text-center text-destructive">
|
||||
Failed to load models.
|
||||
<Popover open={modelPopoverOpen} onOpenChange={setModelPopoverOpen}>
|
||||
<PopoverTrigger asChild>
|
||||
<Button
|
||||
variant="input"
|
||||
role="combobox"
|
||||
aria-expanded={modelPopoverOpen}
|
||||
className="flex items-center justify-between">
|
||||
<div>
|
||||
{models.data?.find(({ id }) => id === model)?.name || model || "Select"}
|
||||
</div>
|
||||
) : (
|
||||
models?.map((model) => (
|
||||
<SelectItem key={model.id} value={model.id}>
|
||||
{model.name}
|
||||
</SelectItem>
|
||||
))
|
||||
)}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
<ChevronsUpDown className="opacity-50" />
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="p-0 w-[var(--radix-popover-trigger-width)]">
|
||||
<Command
|
||||
filter={(value, search) => {
|
||||
if (modelSearchValueRef.current !== search) {
|
||||
modelSearchValueRef.current = search
|
||||
modelSearchResultsRef.current.clear()
|
||||
|
||||
for (const {
|
||||
obj: { id },
|
||||
score,
|
||||
} of fuzzysort.go(search, models.data || [], {
|
||||
key: "name",
|
||||
})) {
|
||||
modelSearchResultsRef.current.set(id, score)
|
||||
}
|
||||
}
|
||||
|
||||
return modelSearchResultsRef.current.get(value) ?? 0
|
||||
}}>
|
||||
<CommandInput
|
||||
placeholder="Search"
|
||||
value={modelSearchValue}
|
||||
onValueChange={setModelSearchValue}
|
||||
className="h-9"
|
||||
/>
|
||||
<CommandList>
|
||||
<CommandEmpty>No model found.</CommandEmpty>
|
||||
<CommandGroup>
|
||||
{models.data?.map(({ id, name }) => (
|
||||
<CommandItem key={id} value={id} onSelect={selectModel}>
|
||||
{name}
|
||||
<Check
|
||||
className={cn(
|
||||
"ml-auto text-accent group-data-[selected=true]:text-accent-foreground size-4",
|
||||
id === model ? "opacity-100" : "opacity-0",
|
||||
)}
|
||||
/>
|
||||
</CommandItem>
|
||||
))}
|
||||
</CommandGroup>
|
||||
</CommandList>
|
||||
</Command>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
|
||||
<FormMessage />
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
<FormItem>
|
||||
<FormLabel>Exercise Suite</FormLabel>
|
||||
<Tabs
|
||||
defaultValue="full"
|
||||
onValueChange={(value) => setSelectedSuite(value as "full" | "partial")}>
|
||||
<TabsList className="grid w-full grid-cols-2">
|
||||
<TabsTrigger value="full">Full</TabsTrigger>
|
||||
<TabsTrigger value="partial">Partial</TabsTrigger>
|
||||
</TabsList>
|
||||
</Tabs>
|
||||
</FormItem>
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="suite"
|
||||
render={() => (
|
||||
<FormItem>
|
||||
<FormLabel>Exercise Suite</FormLabel>
|
||||
<Tabs
|
||||
defaultValue="full"
|
||||
onValueChange={(value) => setValue("suite", value as "full" | "partial")}>
|
||||
<TabsList className="grid w-full grid-cols-2">
|
||||
<TabsTrigger value="full">Full</TabsTrigger>
|
||||
<TabsTrigger value="partial">Partial</TabsTrigger>
|
||||
</TabsList>
|
||||
</Tabs>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
|
||||
{selectedSuite === "partial" && (
|
||||
<FormItem>
|
||||
<FormLabel>Exercises</FormLabel>
|
||||
<MultiSelect
|
||||
options={languages.map((language) => ({
|
||||
value: language,
|
||||
label: language,
|
||||
}))}
|
||||
onValueChange={setSelectedExercises}
|
||||
defaultValue={selectedExercises}
|
||||
placeholder="Select"
|
||||
variant="inverted"
|
||||
maxCount={10}
|
||||
/>
|
||||
</FormItem>
|
||||
{suite === "partial" && (
|
||||
<FormField
|
||||
control={form.control}
|
||||
name="exercises"
|
||||
render={() => (
|
||||
<FormItem>
|
||||
<FormLabel>Exercises</FormLabel>
|
||||
<MultiSelect
|
||||
options={exercises.data?.map((path) => ({ value: path, label: path })) || []}
|
||||
onValueChange={(value) => setValue("exercises", value)}
|
||||
placeholder="Select"
|
||||
variant="inverted"
|
||||
maxCount={4}
|
||||
/>
|
||||
</FormItem>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
|
||||
<FormField
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import { cva, type VariantProps } from "class-variance-authority"
|
|||
import { cn } from "@/lib/utils"
|
||||
|
||||
const buttonVariants = cva(
|
||||
"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-sm text-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive hover:opacity-80 active:scale-95 cursor-pointer",
|
||||
"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive hover:opacity-80 active:scale-95 cursor-pointer",
|
||||
{
|
||||
variants: {
|
||||
variant: {
|
||||
|
|
@ -17,6 +17,7 @@ const buttonVariants = cva(
|
|||
secondary: "bg-secondary text-secondary-foreground shadow-xs",
|
||||
ghost: "hover:bg-accent hover:text-accent-foreground",
|
||||
link: "text-accent underline-offset-4 hover:underline px-1.5!",
|
||||
input: "bg-input text-input-foreground active:scale-100 shadow-xs",
|
||||
},
|
||||
size: {
|
||||
default: "h-9 px-4 py-2 has-[>svg]:px-3",
|
||||
|
|
|
|||
|
|
@ -1,20 +1,13 @@
|
|||
import * as React from "react"
|
||||
import { cva, type VariantProps } from "class-variance-authority"
|
||||
import { CheckIcon, X, ChevronDown } from "lucide-react"
|
||||
import fuzzysort from "fuzzysort"
|
||||
import { Check, X, ChevronsUpDown } from "lucide-react"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
import { Badge } from "./badge"
|
||||
import { Popover, PopoverContent, PopoverTrigger } from "./popover"
|
||||
import {
|
||||
Command,
|
||||
CommandEmpty,
|
||||
CommandGroup,
|
||||
CommandInput,
|
||||
CommandItem,
|
||||
CommandList,
|
||||
CommandSeparator,
|
||||
} from "./command"
|
||||
import { Command, CommandEmpty, CommandGroup, CommandInput, CommandItem, CommandList } from "./command"
|
||||
|
||||
/**
|
||||
* Variants for the multi-select component to handle different styles.
|
||||
|
|
@ -127,11 +120,6 @@ export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
|
|||
onValueChange(newSelectedValues)
|
||||
}
|
||||
|
||||
const handleClear = () => {
|
||||
setSelectedValues([])
|
||||
onValueChange([])
|
||||
}
|
||||
|
||||
const handleTogglePopover = () => {
|
||||
setIsPopoverOpen((prev) => !prev)
|
||||
}
|
||||
|
|
@ -142,16 +130,50 @@ export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
|
|||
onValueChange(newSelectedValues)
|
||||
}
|
||||
|
||||
const toggleAll = () => {
|
||||
if (selectedValues.length === options.length) {
|
||||
handleClear()
|
||||
} else {
|
||||
const allValues = options.map((option) => option.value)
|
||||
setSelectedValues(allValues)
|
||||
onValueChange(allValues)
|
||||
const searchResultsRef = React.useRef<Map<string, number>>(new Map())
|
||||
const searchValueRef = React.useRef("")
|
||||
|
||||
const onSelectAll = () => {
|
||||
const values = Array.from(searchResultsRef.current.keys())
|
||||
|
||||
if (
|
||||
selectedValues.length === values.length &&
|
||||
selectedValues.sort().join(",") === values.sort().join(",")
|
||||
) {
|
||||
setSelectedValues([])
|
||||
onValueChange([])
|
||||
return
|
||||
}
|
||||
|
||||
setSelectedValues(values)
|
||||
onValueChange(values)
|
||||
}
|
||||
|
||||
const onFilter = React.useCallback(
|
||||
(value: string, search: string) => {
|
||||
if (searchValueRef.current !== search) {
|
||||
searchValueRef.current = search
|
||||
searchResultsRef.current.clear()
|
||||
|
||||
for (const {
|
||||
obj: { value },
|
||||
score,
|
||||
} of fuzzysort.go(search, options, {
|
||||
key: "label",
|
||||
})) {
|
||||
searchResultsRef.current.set(value, score)
|
||||
}
|
||||
}
|
||||
|
||||
if (value === "all") {
|
||||
return searchResultsRef.current.size > 1 ? 0.01 : 0
|
||||
}
|
||||
|
||||
return searchResultsRef.current.get(value) ?? 0
|
||||
},
|
||||
[options],
|
||||
)
|
||||
|
||||
return (
|
||||
<Popover open={isPopoverOpen} onOpenChange={setIsPopoverOpen} modal={modalPopover}>
|
||||
<PopoverTrigger asChild>
|
||||
|
|
@ -160,13 +182,13 @@ export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
|
|||
{...props}
|
||||
onClick={handleTogglePopover}
|
||||
className={cn(
|
||||
"flex w-full p-1 rounded-sm border min-h-10 h-auto items-center justify-between [&_svg]:pointer-events-auto pl-2 pr-3",
|
||||
"flex w-full rounded-sm border min-h-9 h-auto items-center justify-between [&_svg]:pointer-events-auto",
|
||||
"border border-input bg-input hover:opacity-80 cursor-pointer",
|
||||
className,
|
||||
)}>
|
||||
{selectedValues.length > 0 ? (
|
||||
<div className="flex justify-between items-center w-full">
|
||||
<div className="flex flex-wrap items-center gap-1">
|
||||
<div className="flex flex-wrap items-center gap-1 p-1">
|
||||
{selectedValues.slice(0, maxCount).map((value) => (
|
||||
<Badge key={value} className={cn(multiSelectVariants({ variant }))}>
|
||||
<div className="flex items-center gap-1.5">
|
||||
|
|
@ -202,39 +224,28 @@ export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
|
|||
) : (
|
||||
<div className="flex items-center justify-between w-full mx-auto">
|
||||
<span className="text-muted-foreground mx-3">{placeholder}</span>
|
||||
<ChevronDown className="h-4 cursor-pointer text-muted-foreground mx-2" />
|
||||
<ChevronsUpDown className="opacity-50 size-4 mx-2" />
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="w-auto p-0" align="start" onEscapeKeyDown={() => setIsPopoverOpen(false)}>
|
||||
<Command>
|
||||
<PopoverContent
|
||||
className="p-0 w-[var(--radix-popover-trigger-width)]"
|
||||
align="start"
|
||||
onEscapeKeyDown={() => setIsPopoverOpen(false)}>
|
||||
<Command filter={onFilter}>
|
||||
<CommandInput placeholder="Search" onKeyDown={handleInputKeyDown} />
|
||||
<CommandList>
|
||||
<CommandEmpty>No results found.</CommandEmpty>
|
||||
<CommandGroup>
|
||||
<CommandItem
|
||||
key="all"
|
||||
onSelect={toggleAll}
|
||||
className="flex items-center justify-between">
|
||||
<span>Select All</span>
|
||||
<CheckIcon
|
||||
className={cn(
|
||||
"text-accent group-data-[selected=true]:text-accent-foreground size-4",
|
||||
{
|
||||
"opacity-0": selectedValues.length !== options.length,
|
||||
},
|
||||
)}
|
||||
/>
|
||||
</CommandItem>
|
||||
<CommandSeparator />
|
||||
{options.map((option) => (
|
||||
<CommandItem
|
||||
key={option.value}
|
||||
value={option.value}
|
||||
onSelect={() => toggleOption(option.value)}
|
||||
className="flex items-center justify-between">
|
||||
<span>{option.label}</span>
|
||||
<CheckIcon
|
||||
<Check
|
||||
className={cn(
|
||||
"text-accent group-data-[selected=true]:text-accent-foreground size-4",
|
||||
{ "opacity-0": !selectedValues.includes(option.value) },
|
||||
|
|
@ -242,16 +253,13 @@ export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
|
|||
/>
|
||||
</CommandItem>
|
||||
))}
|
||||
{selectedValues.length > 0 && (
|
||||
<>
|
||||
<CommandSeparator />
|
||||
<CommandItem
|
||||
className="flex items-center justify-between"
|
||||
onSelect={handleClear}>
|
||||
Select None
|
||||
</CommandItem>
|
||||
</>
|
||||
)}
|
||||
<CommandItem
|
||||
key="all"
|
||||
value="all"
|
||||
onSelect={onSelectAll}
|
||||
className="flex items-center justify-between">
|
||||
<span>Select All</span>
|
||||
</CommandItem>
|
||||
</CommandGroup>
|
||||
</CommandList>
|
||||
</Command>
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
import * as React from "react"
|
||||
import * as SelectPrimitive from "@radix-ui/react-select"
|
||||
import { CheckIcon, ChevronDownIcon, ChevronUpIcon } from "lucide-react"
|
||||
import { Check, ChevronDown, ChevronUp } from "lucide-react"
|
||||
|
||||
import { cn } from "@/lib/utils"
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ function SelectTrigger({
|
|||
{...props}>
|
||||
{children}
|
||||
<SelectPrimitive.Icon asChild>
|
||||
<ChevronDownIcon className="size-4 opacity-50" />
|
||||
<ChevronDown className="size-4 opacity-50" />
|
||||
</SelectPrimitive.Icon>
|
||||
</SelectPrimitive.Trigger>
|
||||
)
|
||||
|
|
@ -99,7 +99,7 @@ function SelectItem({ className, children, ...props }: React.ComponentProps<type
|
|||
{...props}>
|
||||
<span className="absolute right-2 flex size-3.5 items-center justify-center">
|
||||
<SelectPrimitive.ItemIndicator>
|
||||
<CheckIcon className="text-accent group-focus:text-accent-foreground size-4" />
|
||||
<Check className="text-accent group-focus:text-accent-foreground size-4" />
|
||||
</SelectPrimitive.ItemIndicator>
|
||||
</span>
|
||||
<SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
|
||||
|
|
@ -123,7 +123,7 @@ function SelectScrollUpButton({ className, ...props }: React.ComponentProps<type
|
|||
data-slot="select-scroll-up-button"
|
||||
className={cn("flex cursor-default items-center justify-center py-1", className)}
|
||||
{...props}>
|
||||
<ChevronUpIcon className="size-4" />
|
||||
<ChevronUp className="size-4" />
|
||||
</SelectPrimitive.ScrollUpButton>
|
||||
)
|
||||
}
|
||||
|
|
@ -137,7 +137,7 @@ function SelectScrollDownButton({
|
|||
data-slot="select-scroll-down-button"
|
||||
className={cn("flex cursor-default items-center justify-center py-1", className)}
|
||||
{...props}>
|
||||
<ChevronDownIcon className="size-4" />
|
||||
<ChevronDown className="size-4" />
|
||||
</SelectPrimitive.ScrollDownButton>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
5
benchmark/apps/web/src/hooks/use-exercises.ts
Normal file
5
benchmark/apps/web/src/hooks/use-exercises.ts
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
import { useQuery } from "@tanstack/react-query"
|
||||
|
||||
import { getExercises } from "@/lib/server/exercises"
|
||||
|
||||
export const useExercises = () => useQuery({ queryKey: ["exercises"], queryFn: getExercises })
|
||||
|
|
@ -1,12 +1,11 @@
|
|||
import { useState, useCallback } from "react"
|
||||
import { useQuery, keepPreviousData } from "@tanstack/react-query"
|
||||
|
||||
import { useEventSource } from "@/hooks/use-event-source"
|
||||
|
||||
import { Run } from "@benchmark/db"
|
||||
|
||||
import { getTasks } from "./actions"
|
||||
import { ipcServerMessageSchema } from "./schemas"
|
||||
import { getTasks } from "@/lib/server/tasks"
|
||||
import { ipcServerMessageSchema } from "@/lib/schemas"
|
||||
import { useEventSource } from "@/hooks/use-event-source"
|
||||
|
||||
export const useRunStatus = (run: Run) => {
|
||||
const [clientId, setClientId] = useState<string>()
|
||||
|
|
@ -1,2 +1,2 @@
|
|||
export { formatCurrency } from "./formatCurrency"
|
||||
export { formatDuration } from "./formatDuration"
|
||||
export { formatCurrency } from "./format-currency"
|
||||
export { formatDuration } from "./format-duration"
|
||||
|
|
|
|||
|
|
@ -1,5 +1,26 @@
|
|||
import { z } from "zod"
|
||||
|
||||
/**
|
||||
* CreateRun
|
||||
*/
|
||||
|
||||
export const createRunSchema = z
|
||||
.object({
|
||||
model: z.string().min(1),
|
||||
description: z.string().optional(),
|
||||
suite: z.enum(["full", "partial"]),
|
||||
exercises: z.array(z.string()).optional(),
|
||||
})
|
||||
.refine((data) => data.suite === "full" || (data.exercises || []).length > 0, {
|
||||
message: "Exercises are required for partial suite.",
|
||||
})
|
||||
|
||||
export type CreateRun = z.infer<typeof createRunSchema>
|
||||
|
||||
/**
|
||||
* IpcServerMessage
|
||||
*/
|
||||
|
||||
export const ipcServerMessageSchema = z.discriminatedUnion("type", [
|
||||
z.object({
|
||||
type: z.literal("Ack"),
|
||||
|
|
@ -18,3 +39,5 @@ export const ipcServerMessageSchema = z.discriminatedUnion("type", [
|
|||
}),
|
||||
}),
|
||||
])
|
||||
|
||||
export type IpcServerMessage = z.infer<typeof ipcServerMessageSchema>
|
||||
35
benchmark/apps/web/src/lib/server/exercises.ts
Normal file
35
benchmark/apps/web/src/lib/server/exercises.ts
Normal file
|
|
@ -0,0 +1,35 @@
|
|||
"use server"
|
||||
|
||||
import * as fs from "fs/promises"
|
||||
import * as path from "path"
|
||||
import { fileURLToPath } from "url"
|
||||
|
||||
import { languages } from "@benchmark/types"
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url))
|
||||
|
||||
const listDirectories = async (relativePath: string) => {
|
||||
try {
|
||||
const targetPath = path.resolve(__dirname, relativePath)
|
||||
const entries = await fs.readdir(targetPath, { withFileTypes: true })
|
||||
return entries.filter((entry) => entry.isDirectory() && !entry.name.startsWith(".")).map((entry) => entry.name)
|
||||
} catch (error) {
|
||||
console.error(`Error listing directories at ${relativePath}:`, error)
|
||||
return []
|
||||
}
|
||||
}
|
||||
|
||||
// __dirname = <repo>/benchmark/apps/web/src/lib/server
|
||||
const EXERCISES_BASE_PATH = path.resolve(__dirname, "../../../../../../../exercises")
|
||||
|
||||
export const getExercises = async () => {
|
||||
const result = await Promise.all(
|
||||
languages.map(async (language) => {
|
||||
const languagePath = path.join(EXERCISES_BASE_PATH, language)
|
||||
const exercises = await listDirectories(languagePath)
|
||||
return exercises.map((exercise) => `${language}/${exercise}`)
|
||||
}),
|
||||
)
|
||||
|
||||
return result.flat()
|
||||
}
|
||||
51
benchmark/apps/web/src/lib/server/runs.ts
Normal file
51
benchmark/apps/web/src/lib/server/runs.ts
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
"use server"
|
||||
|
||||
import { spawn } from "child_process"
|
||||
import path from "path"
|
||||
import os from "os"
|
||||
|
||||
import { revalidatePath } from "next/cache"
|
||||
|
||||
import { Language } from "@benchmark/types"
|
||||
import * as db from "@benchmark/db"
|
||||
|
||||
import { CreateRun } from "@/lib/schemas"
|
||||
|
||||
export async function createRun({ suite, exercises = [], ...values }: CreateRun) {
|
||||
const run = await db.createRun({
|
||||
...values,
|
||||
socketPath: path.join(os.tmpdir(), `benchmark-${crypto.randomUUID()}.sock`),
|
||||
})
|
||||
|
||||
if (suite === "partial") {
|
||||
for (const path of exercises) {
|
||||
const [language, exercise] = path.split("/")
|
||||
|
||||
if (!language || !exercise) {
|
||||
throw new Error("Invalid exercise path: " + path)
|
||||
}
|
||||
|
||||
await db.createTask({ ...values, runId: run.id, language: language as Language, exercise })
|
||||
}
|
||||
}
|
||||
|
||||
revalidatePath("/runs")
|
||||
|
||||
try {
|
||||
const process = spawn(
|
||||
"pnpm",
|
||||
["--filter", "@benchmark/cli", "dev", "run", "all", "--runId", run.id.toString()],
|
||||
{
|
||||
detached: true,
|
||||
stdio: "ignore",
|
||||
},
|
||||
)
|
||||
|
||||
process.unref()
|
||||
await db.updateRun(run.id, { pid: process.pid })
|
||||
} catch (error) {
|
||||
console.error(error)
|
||||
}
|
||||
|
||||
return run
|
||||
}
|
||||
|
|
@ -7,8 +7,8 @@
|
|||
"check-types": "turbo check-types --log-order grouped --output-logs new-only",
|
||||
"format": "turbo format --log-order grouped --output-logs new-only",
|
||||
"build": "turbo build --log-order grouped --output-logs new-only",
|
||||
"web": "turbo dev --filter @benchmark/web --log-order grouped --output-logs new-only",
|
||||
"cli": "turbo dev --filter @benchmark/cli --ui tui -- run",
|
||||
"web": "turbo dev --filter @benchmark/web --output-logs new-only --ui tui",
|
||||
"cli": "turbo dev --filter @benchmark/cli --output-logs new-only --ui tui -- run",
|
||||
"drizzle:studio": "pnpm --filter @benchmark/db db:studio",
|
||||
"docker:build": "docker build -f Dockerfile -t roo-code-benchmark ..",
|
||||
"docker:run": "touch /tmp/benchmarks.db && docker run -d -it -p 3000:3000 -v /tmp/benchmarks.db:/tmp/benchmarks.db roo-code-benchmark",
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
import { desc, eq } from "drizzle-orm"
|
||||
import { desc, eq, sum } from "drizzle-orm"
|
||||
|
||||
import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js"
|
||||
import type { InsertRun, UpdateRun } from "../schema.js"
|
||||
import { insertRunSchema, schema } from "../schema.js"
|
||||
import { db } from "../db.js"
|
||||
import { createTaskMetrics } from "./taskMetrics.js"
|
||||
|
||||
const table = schema.runs
|
||||
|
||||
|
|
@ -47,3 +48,35 @@ export const updateRun = async (id: number, values: UpdateRun) => {
|
|||
}
|
||||
|
||||
export const getRuns = async () => db.query.runs.findMany({ orderBy: desc(table.id), with: { taskMetrics: true } })
|
||||
|
||||
export const finishRun = async (runId: number) => {
|
||||
const [values] = await db
|
||||
.select({
|
||||
tokensIn: sum(schema.taskMetrics.tokensIn).mapWith(Number),
|
||||
tokensOut: sum(schema.taskMetrics.tokensOut).mapWith(Number),
|
||||
tokensContext: sum(schema.taskMetrics.tokensContext).mapWith(Number),
|
||||
cacheWrites: sum(schema.taskMetrics.cacheWrites).mapWith(Number),
|
||||
cacheReads: sum(schema.taskMetrics.cacheReads).mapWith(Number),
|
||||
cost: sum(schema.taskMetrics.cost).mapWith(Number),
|
||||
duration: sum(schema.taskMetrics.duration).mapWith(Number),
|
||||
})
|
||||
.from(schema.taskMetrics)
|
||||
.innerJoin(schema.tasks, eq(schema.taskMetrics.id, schema.tasks.taskMetricsId))
|
||||
.innerJoin(schema.runs, eq(schema.tasks.runId, schema.runs.id))
|
||||
.where(eq(schema.runs.id, runId))
|
||||
|
||||
if (!values) {
|
||||
throw new RecordNotFoundError()
|
||||
}
|
||||
|
||||
const taskMetrics = await createTaskMetrics(values)
|
||||
await updateRun(runId, { taskMetricsId: taskMetrics.id })
|
||||
|
||||
const run = await db.query.runs.findFirst({ where: eq(table.id, runId), with: { taskMetrics: true } })
|
||||
|
||||
if (!run) {
|
||||
throw new RecordNotFoundError()
|
||||
}
|
||||
|
||||
return run
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,14 +15,16 @@ export async function run() {
|
|||
* Validate environment variables.
|
||||
*/
|
||||
|
||||
const tid = process.env.TASK_ID ? parseInt(process.env.TASK_ID) : undefined
|
||||
const taskId = process.env.TASK_ID ? parseInt(process.env.TASK_ID) : undefined
|
||||
const promptPath = process.env.PROMPT_PATH
|
||||
const workspacePath = process.env.WORKSPACE_PATH
|
||||
const openRouterApiKey = process.env.OPENROUTER_API_KEY
|
||||
const openRouterModelId = process.env.OPENROUTER_MODEL_ID
|
||||
|
||||
if (!tid || !promptPath || !workspacePath || !openRouterApiKey || !openRouterModelId) {
|
||||
throw new Error("ENV not configured.")
|
||||
if (!taskId || !promptPath || !workspacePath || !openRouterApiKey || !openRouterModelId) {
|
||||
throw new Error(
|
||||
`ENV not configured. ${JSON.stringify({ taskId, promptPath, workspacePath, openRouterApiKey, openRouterModelId })}`,
|
||||
)
|
||||
}
|
||||
|
||||
const prompt = await fs.readFile(promptPath, "utf-8")
|
||||
|
|
@ -31,7 +33,7 @@ export async function run() {
|
|||
* Fetch and update the task.
|
||||
*/
|
||||
|
||||
let task = await findTask(tid)
|
||||
let task = await findTask(taskId)
|
||||
task = await updateTask(task.id, { startedAt: new Date() })
|
||||
|
||||
const run = await findRun(task.runId)
|
||||
|
|
|
|||
9
benchmark/pnpm-lock.yaml
generated
9
benchmark/pnpm-lock.yaml
generated
|
|
@ -114,6 +114,9 @@ importers:
|
|||
cmdk:
|
||||
specifier: 1.0.0
|
||||
version: 1.0.0(@types/react-dom@19.0.4(@types/react@19.0.12))(@types/react@19.0.12)(react-dom@19.0.0(react@19.0.0))(react@19.0.0)
|
||||
fuzzysort:
|
||||
specifier: ^3.1.0
|
||||
version: 3.1.0
|
||||
lucide-react:
|
||||
specifier: ^0.479.0
|
||||
version: 0.479.0(react@19.0.0)
|
||||
|
|
@ -2689,6 +2692,9 @@ packages:
|
|||
functions-have-names@1.2.3:
|
||||
resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==}
|
||||
|
||||
fuzzysort@3.1.0:
|
||||
resolution: {integrity: sha512-sR9BNCjBg6LNgwvxlBd0sBABvQitkLzoVY9MYYROQVX/FvfJ4Mai9LsGhDgd8qYdds0bY77VzYd5iuB+v5rwQQ==}
|
||||
|
||||
gel@2.0.1:
|
||||
resolution: {integrity: sha512-gfem3IGvqKqXwEq7XseBogyaRwGsQGuE7Cw/yQsjLGdgiyqX92G1xENPCE0ltunPGcsJIa6XBOTx/PK169mOqw==}
|
||||
engines: {node: '>= 18.0.0'}
|
||||
|
|
@ -3061,6 +3067,7 @@ packages:
|
|||
|
||||
libsql@0.4.7:
|
||||
resolution: {integrity: sha512-T9eIRCs6b0J1SHKYIvD8+KCJMcWZ900iZyxdnSCdqxN12Z1ijzT+jY5nrk72Jw4B0HGzms2NgpryArlJqvc3Lw==}
|
||||
cpu: [x64, arm64, wasm32]
|
||||
os: [darwin, linux, win32]
|
||||
|
||||
lie@3.3.0:
|
||||
|
|
@ -6483,6 +6490,8 @@ snapshots:
|
|||
|
||||
functions-have-names@1.2.3: {}
|
||||
|
||||
fuzzysort@3.1.0: {}
|
||||
|
||||
gel@2.0.1:
|
||||
dependencies:
|
||||
'@petamoriken/float16': 3.9.2
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue