More progress

This commit is contained in:
cte 2025-03-21 07:51:22 -07:00
parent 9466f0511e
commit 6b9ed895a7
21 changed files with 554 additions and 196 deletions

View file

@ -27,6 +27,7 @@
"react-hook-form": "^7.54.2",
"tailwind-merge": "^3.0.2",
"tailwindcss-animate": "^1.0.7",
"uuid": "^11.1.0",
"zod": "^3.24.2"
},
"devDependencies": {

View file

@ -1,4 +1,5 @@
import type { NextRequest } from "next/server"
import { v4 as uuidv4 } from "uuid"
import { findRun } from "@benchmark/db"
import { IpcClient } from "@benchmark/ipc"
@ -9,9 +10,10 @@ export const dynamic = "force-dynamic"
export async function GET(request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
const { id } = await params
const requestId = uuidv4()
const stream = new SSEStream()
const run = await findRun(Number(id))
const client = new IpcClient(`/tmp/benchmark-${run.id}.sock`, () => {})
const client = new IpcClient(run.socketPath, () => {})
const write = async (data: string | object) => {
const success = await stream.write(data)
@ -21,12 +23,13 @@ export async function GET(request: NextRequest, { params }: { params: Promise<{
}
}
console.log(`[stream#${requestId}] connect`)
client.on("connect", () => write("connect"))
client.on("disconnect", () => write("disconnect"))
client.on("message", write)
request.signal.addEventListener("abort", () => {
console.log(`abort`)
console.log(`[stream#${requestId}] abort`)
client.disconnect()
stream.close().catch(() => {})
})

View file

@ -4,46 +4,6 @@
@custom-variant dark (&:is(.dark *));
@theme inline {
--color-background: var(--background);
--color-foreground: var(--foreground);
--font-sans: var(--font-geist-sans);
--font-mono: var(--font-geist-mono);
--color-sidebar-ring: var(--sidebar-ring);
--color-sidebar-border: var(--sidebar-border);
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
--color-sidebar-accent: var(--sidebar-accent);
--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
--color-sidebar-primary: var(--sidebar-primary);
--color-sidebar-foreground: var(--sidebar-foreground);
--color-sidebar: var(--sidebar);
--color-chart-5: var(--chart-5);
--color-chart-4: var(--chart-4);
--color-chart-3: var(--chart-3);
--color-chart-2: var(--chart-2);
--color-chart-1: var(--chart-1);
--color-ring: var(--ring);
--color-input: var(--input);
--color-border: var(--border);
--color-destructive: var(--destructive);
--color-accent-foreground: var(--accent-foreground);
--color-accent: var(--accent);
--color-muted-foreground: var(--muted-foreground);
--color-muted: var(--muted);
--color-secondary-foreground: var(--secondary-foreground);
--color-secondary: var(--secondary);
--color-primary-foreground: var(--primary-foreground);
--color-primary: var(--primary);
--color-popover-foreground: var(--popover-foreground);
--color-popover: var(--popover);
--color-card-foreground: var(--card-foreground);
--color-card: var(--card);
--radius-sm: calc(var(--radius) - 4px);
--radius-md: calc(var(--radius) - 2px);
--radius-lg: var(--radius);
--radius-xl: calc(var(--radius) + 4px);
}
:root {
--radius: 0.625rem;
--background: oklch(1 0 0);
@ -80,24 +40,24 @@
}
.dark {
--background: oklch(0.145 0 0);
--foreground: oklch(0.985 0 0);
--background: oklch(23.66% 0.0198 271.79);
--foreground: oklch(75.15% 0.0477 278.41);
--card: oklch(0.205 0 0);
--card-foreground: oklch(0.985 0 0);
--popover: oklch(0.205 0 0);
--popover: var(--primary);
--popover-foreground: oklch(0.985 0 0);
--primary: oklch(0.922 0 0);
--primary-foreground: oklch(0.205 0 0);
--primary: oklch(29.33% 0.0295 276.18);
--primary-foreground: var(--accent);
--secondary: oklch(0.269 0 0);
--secondary-foreground: oklch(0.985 0 0);
--muted: oklch(0.269 0 0);
--muted-foreground: oklch(0.708 0 0);
--accent: oklch(0.269 0 0);
--accent-foreground: oklch(0.985 0 0);
--destructive: oklch(0.704 0.191 22.216);
--border: oklch(1 0 0 / 10%);
--input: oklch(1 0 0 / 15%);
--ring: oklch(0.556 0 0);
--muted: oklch(28.27% 0.0207 273.06);
--muted-foreground: oklch(75.15% 0.0477 278.41);
--accent: oklch(70.21% 0.1813 328.71);
--accent-foreground: rgba(0, 0, 0, 0.5);
--destructive: oklch(72.14% 0.1616 15.49);
--border: oklch(29.33% 0.0295 276.18);
--input: var(--primary);
--ring: oklch(83.63% 0.1259 176.52);
--chart-1: oklch(0.488 0.243 264.376);
--chart-2: oklch(0.696 0.17 162.48);
--chart-3: oklch(0.769 0.188 70.08);
@ -113,10 +73,52 @@
--sidebar-ring: oklch(0.556 0 0);
}
@theme inline {
--color-background: var(--background);
--color-foreground: var(--foreground);
--color-sidebar-ring: var(--sidebar-ring);
--color-sidebar-border: var(--sidebar-border);
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
--color-sidebar-accent: var(--sidebar-accent);
--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
--color-sidebar-primary: var(--sidebar-primary);
--color-sidebar-foreground: var(--sidebar-foreground);
--color-sidebar: var(--sidebar);
--color-chart-5: var(--chart-5);
--color-chart-4: var(--chart-4);
--color-chart-3: var(--chart-3);
--color-chart-2: var(--chart-2);
--color-chart-1: var(--chart-1);
--color-ring: var(--ring);
--color-input: var(--input);
--color-border: var(--border);
--color-destructive: var(--destructive);
--color-accent-foreground: var(--accent-foreground);
--color-accent: var(--accent);
--color-muted-foreground: var(--muted-foreground);
--color-muted: var(--muted);
--color-secondary-foreground: var(--secondary-foreground);
--color-secondary: var(--secondary);
--color-primary-foreground: var(--primary-foreground);
--color-primary: var(--primary);
--color-popover-foreground: var(--popover-foreground);
--color-popover: var(--popover);
--color-card-foreground: var(--card-foreground);
--color-card: var(--card);
--radius-sm: calc(var(--radius) - 4px);
--radius-md: calc(var(--radius) - 2px);
--radius-lg: var(--radius);
--radius-xl: calc(var(--radius) + 4px);
}
@layer base {
* {
@apply border-border outline-ring/50;
}
html,
body {
height: 100%;
}
body {
@apply bg-background text-foreground;
}

View file

@ -1,43 +1,56 @@
"use client"
import { useRouter } from "next/navigation"
import { Rocket } from "lucide-react"
import { getRuns } from "@benchmark/db"
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui"
import { Button, Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui"
import { formatCurrency, formatDuration } from "@/lib"
export const dynamic = "force-dynamic"
type Run = Awaited<ReturnType<typeof getRuns>>[number]
export async function Home() {
const runs = await getRuns()
export function Home({ runs }: { runs: Run[] }) {
const router = useRouter()
return (
<div className="mx-auto my-20 w-3xl">
<Table className="border">
<TableHeader>
<TableRow>
<TableHead>ID</TableHead>
<TableHead>Model</TableHead>
<TableHead>Timestamp</TableHead>
<TableHead>Passed</TableHead>
<TableHead>Failed</TableHead>
<TableHead>% Correct</TableHead>
<TableHead>Cost</TableHead>
<TableHead>Duration</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{runs.map((run) => (
<TableRow key={run.id}>
<TableCell>{run.id}</TableCell>
<TableCell>{run.model}</TableCell>
<TableCell>{new Date(run.createdAt).toLocaleString()}</TableCell>
<TableCell>{run.passed}</TableCell>
<TableCell>{run.failed}</TableCell>
<TableCell>{(run.rate * 100).toFixed(1)}%</TableCell>
<TableCell>{formatCurrency(run.cost)}</TableCell>
<TableCell>{formatDuration(run.duration)}</TableCell>
<>
<div className="max-w-4xl px-12 mx-auto py-24">
<Table className="border">
<TableHeader>
<TableRow>
<TableHead>ID</TableHead>
<TableHead>Model</TableHead>
<TableHead>Timestamp</TableHead>
<TableHead>Passed</TableHead>
<TableHead>Failed</TableHead>
<TableHead>% Correct</TableHead>
<TableHead>Cost</TableHead>
<TableHead>Duration</TableHead>
</TableRow>
))}
</TableBody>
</Table>
</div>
</TableHeader>
<TableBody>
{runs.map((run) => (
<TableRow key={run.id}>
<TableCell>{run.id}</TableCell>
<TableCell>{run.model}</TableCell>
<TableCell>{new Date(run.createdAt).toLocaleString()}</TableCell>
<TableCell>{run.passed}</TableCell>
<TableCell>{run.failed}</TableCell>
<TableCell>{(run.rate * 100).toFixed(1)}%</TableCell>
<TableCell>{formatCurrency(run.cost)}</TableCell>
<TableCell>{formatDuration(run.duration)}</TableCell>
</TableRow>
))}
</TableBody>
</Table>
</div>
<Button
variant="default"
className="absolute top-5 right-5 size-12 rounded-full"
onClick={() => router.push("/runs/new")}>
<Rocket className="size-6" />
</Button>
</>
)
}

View file

@ -5,8 +5,8 @@ import { ThemeProvider, ReactQueryProvider } from "@/components/providers"
import "./globals.css"
const geistSans = Geist({ variable: "--font-geist-sans", subsets: ["latin"] })
const geistMono = Geist_Mono({ variable: "--font-geist-mono", subsets: ["latin"] })
const fontSans = Geist({ variable: "--font-sans", subsets: ["latin"] })
const fontMono = Geist_Mono({ variable: "--font-mono", subsets: ["latin"] })
export const metadata: Metadata = {
title: "Roo Code Benchmarks",
@ -19,7 +19,7 @@ export default function RootLayout({
}>) {
return (
<html lang="en">
<body className={`${geistSans.variable} ${geistMono.variable} antialiased`}>
<body className={`${fontSans.variable} ${fontMono.variable} font-sans antialiased`}>
<ThemeProvider attribute="class" defaultTheme="system" enableSystem disableTransitionOnChange>
<ReactQueryProvider>{children}</ReactQueryProvider>
</ThemeProvider>

View file

@ -1,5 +1,12 @@
import { getRuns } from "@benchmark/db"
import { Home } from "./home"
export const dynamic = "force-dynamic"
export default async function Page() {
return <Home />
const runs = await getRuns()
console.log(runs)
return <Home runs={runs} />
}

View file

@ -10,6 +10,7 @@ export function ShowRun({ run }: { run: Run }) {
const url = `/api/runs/${run.id}/stream`
const onMessage = useCallback(({ data }: MessageEvent) => console.log(data), [])
const status = useEventSource({ url, onMessage })
return (
<div>
Show Run: {run.id} | {status}

View file

@ -1,5 +1,6 @@
"use server"
import { spawn } from "child_process"
import { revalidatePath } from "next/cache"
import * as db from "@benchmark/db"
@ -7,5 +8,20 @@ import * as db from "@benchmark/db"
export async function createRun(data: db.InsertRun) {
const run = await db.createRun(data)
revalidatePath("/runs")
return run
try {
const process = spawn(
"pnpm",
["--filter", "@benchmark/cli", "dev", "run", "all", "--runId", run.id.toString()],
{
detached: true,
stdio: "ignore",
},
)
process.unref()
return process.pid
} catch (_) {
return undefined
}
}

View file

@ -5,12 +5,12 @@ import { useRouter } from "next/navigation"
import { useForm, FormProvider } from "react-hook-form"
import { zodResolver } from "@hookform/resolvers/zod"
import { z } from "zod"
import { X, Loader2, Rocket } from "lucide-react"
import { useOpenRouterModels } from "@/hooks/use-open-router-models"
import {
Button,
FormControl,
FormDescription,
FormField,
FormItem,
FormLabel,
@ -26,9 +26,7 @@ import {
import { createRun } from "./actions"
const formSchema = z.object({
model: z.string({
required_error: "Please select a model",
}),
model: z.string(),
description: z.string().optional(),
})
@ -36,11 +34,15 @@ type FormValues = z.infer<typeof formSchema>
export function NewRun() {
const router = useRouter()
const { data: models, isLoading, error } = useOpenRouterModels()
const { data: models, isLoading, isError } = useOpenRouterModels()
const [isSubmitting, setIsSubmitting] = useState(false)
const form = useForm<FormValues>({
resolver: zodResolver(formSchema),
defaultValues: {
model: "anthropic/claude-3.7-sonnet",
description: "",
},
})
async function onSubmit(data: FormValues) {
@ -56,77 +58,69 @@ export function NewRun() {
}
return (
<div className="space-y-6 max-w-2xl mx-auto p-6">
<div className="space-y-2">
<h1 className="text-2xl font-bold">Create New Run</h1>
<p className="text-muted-foreground">
Create a new run by selecting a model and providing an optional description.
</p>
</div>
<div>
<FormProvider {...form}>
<form onSubmit={form.handleSubmit(onSubmit)} className="space-y-6">
<FormField
control={form.control}
name="model"
render={({ field }) => (
<FormItem>
<FormLabel>Model</FormLabel>
<Select onValueChange={field.onChange} defaultValue={field.value}>
<FormControl>
<SelectTrigger>
<SelectValue placeholder="Select a model" />
</SelectTrigger>
</FormControl>
<SelectContent>
{isLoading ? (
<div className="p-2 text-center text-muted-foreground">
Loading models...
</div>
) : error ? (
<div className="p-2 text-center text-destructive">
Error loading models. Using fallback options.
</div>
) : (
models?.map((model) => (
<SelectItem key={model.id} value={model.id}>
{model.name}
</SelectItem>
))
)}
</SelectContent>
</Select>
<FormDescription>Select the model to use for this run.</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="description"
render={({ field }) => (
<FormItem>
<FormLabel>Description</FormLabel>
<>
<FormProvider {...form}>
<form
onSubmit={form.handleSubmit(onSubmit)}
className="flex flex-col justify-center gap-6 h-dvh max-w-xl mx-auto">
<div>__dirname: {__dirname}</div>
<FormField
control={form.control}
name="model"
render={({ field }) => (
<FormItem>
<FormLabel>OpenRouter Model</FormLabel>
<Select onValueChange={field.onChange} defaultValue={field.value}>
<FormControl>
<Textarea
placeholder="Enter a description for this run (optional)"
className="resize-none"
{...field}
/>
<SelectTrigger className="w-full">
<SelectValue placeholder="Select" />
</SelectTrigger>
</FormControl>
<FormDescription>
Provide an optional description to help identify this run.
</FormDescription>
<FormMessage />
</FormItem>
)}
/>
<Button type="submit" disabled={isSubmitting}>
{isSubmitting ? "Creating..." : "Create Run"}
</Button>
</form>
</FormProvider>
</div>
</div>
<SelectContent>
{isLoading ? (
<Loader2 className="size-4 m-2 animate-spin" />
) : isError ? (
<div className="m-2 text-center text-destructive">
Failed to load models.
</div>
) : (
models?.map((model) => (
<SelectItem key={model.id} value={model.id}>
{model.name}
</SelectItem>
))
)}
</SelectContent>
</Select>
<FormMessage />
</FormItem>
)}
/>
<FormField
control={form.control}
name="description"
render={({ field }) => (
<FormItem>
<FormLabel>Description</FormLabel>
<FormControl>
<Textarea placeholder="Optional" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
<Button type="submit" disabled={isSubmitting}>
<Rocket className="size-4" />
Launch Benchmark Run
</Button>
</form>
</FormProvider>
<Button
variant="default"
className="absolute top-5 right-5 size-12 rounded-full"
onClick={() => router.push("/")}>
<X className="size-6" />
</Button>
</>
)
}

View file

@ -5,23 +5,23 @@ import { cva, type VariantProps } from "class-variance-authority"
import { cn } from "@/lib/utils"
const buttonVariants = cva(
"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-sm text-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive hover:opacity-80 active:scale-95 cursor-pointer",
{
variants: {
variant: {
default: "bg-primary text-primary-foreground shadow-xs hover:bg-primary/90",
default: "bg-primary text-primary-foreground shadow-xs",
destructive:
"bg-destructive text-white shadow-xs hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60",
"bg-destructive text-white shadow-xs focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60",
outline:
"border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input dark:hover:bg-input/50",
secondary: "bg-secondary text-secondary-foreground shadow-xs hover:bg-secondary/80",
ghost: "hover:bg-accent hover:text-accent-foreground dark:hover:bg-accent/50",
"border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input",
secondary: "bg-secondary text-secondary-foreground shadow-xs",
ghost: "hover:bg-accent hover:text-accent-foreground",
link: "text-primary underline-offset-4 hover:underline",
},
size: {
default: "h-9 px-4 py-2 has-[>svg]:px-3",
sm: "h-8 rounded-md gap-1.5 px-3 has-[>svg]:px-2.5",
lg: "h-10 rounded-md px-6 has-[>svg]:px-4",
sm: "h-8 gap-1.5 px-3 has-[>svg]:px-2.5",
lg: "h-10 px-6 has-[>svg]:px-4",
icon: "size-9",
},
},

View file

@ -8,9 +8,10 @@ function Input({ className, type, ...props }: React.ComponentProps<"input">) {
type={type}
data-slot="input"
className={cn(
"file:text-foreground placeholder:text-muted-foreground selection:bg-primary selection:text-primary-foreground dark:bg-input/30 border-input flex h-9 w-full min-w-0 rounded-md border bg-transparent px-3 py-1 text-base shadow-xs transition-[color,box-shadow] outline-none file:inline-flex file:h-7 file:border-0 file:bg-transparent file:text-sm file:font-medium disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-50 md:text-sm",
"file:text-foreground placeholder:text-muted-foreground selection:bg-primary selection:text-primary-foreground flex h-9 w-full min-w-0 rounded-sm px-3 py-1 text-base shadow-xs transition-[color,box-shadow] outline-none file:inline-flex file:h-7 file:border-0 file:bg-transparent file:text-sm file:font-medium disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-50",
"focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px]",
"aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
"border border-input bg-input",
className,
)}
{...props}

View file

@ -31,7 +31,8 @@ function SelectTrigger({
data-slot="select-trigger"
data-size={size}
className={cn(
"border-input data-[placeholder]:text-muted-foreground [&_svg:not([class*='text-'])]:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive dark:bg-input/30 dark:hover:bg-input/50 flex w-fit items-center justify-between gap-2 rounded-md border bg-transparent px-3 py-2 text-sm whitespace-nowrap shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50 data-[size=default]:h-9 data-[size=sm]:h-8 *:data-[slot=select-value]:line-clamp-1 *:data-[slot=select-value]:flex *:data-[slot=select-value]:items-center *:data-[slot=select-value]:gap-2 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
"data-[placeholder]:text-muted-foreground [&_svg:not([class*='text-'])]:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive flex w-fit items-center justify-between gap-2 rounded-sm px-3 py-2 whitespace-nowrap shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50 data-[size=default]:h-9 data-[size=sm]:h-8 *:data-[slot=select-value]:line-clamp-1 *:data-[slot=select-value]:flex *:data-[slot=select-value]:items-center *:data-[slot=select-value]:gap-2 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
"border border-input bg-input hover:opacity-80 cursor-pointer",
className,
)}
{...props}>
@ -54,7 +55,7 @@ function SelectContent({
<SelectPrimitive.Content
data-slot="select-content"
className={cn(
"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 relative z-50 max-h-(--radix-select-content-available-height) min-w-[8rem] origin-(--radix-select-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-md border shadow-md",
"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 relative z-50 max-h-(--radix-select-content-available-height) min-w-[8rem] origin-(--radix-select-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-sm shadow-md",
position === "popper" &&
"data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1",
className,
@ -91,13 +92,14 @@ function SelectItem({ className, children, ...props }: React.ComponentProps<type
<SelectPrimitive.Item
data-slot="select-item"
className={cn(
"focus:bg-accent focus:text-accent-foreground [&_svg:not([class*='text-'])]:text-muted-foreground relative flex w-full cursor-default items-center gap-2 rounded-sm py-1.5 pr-8 pl-2 text-sm outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 *:[span]:last:flex *:[span]:last:items-center *:[span]:last:gap-2",
"focus:bg-accent focus:text-accent-foreground [&_svg:not([class*='text-'])]:text-muted-foreground relative flex w-full cursor-default items-center gap-2 rounded-xs py-1.5 pr-8 pl-2 outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 *:[span]:last:flex *:[span]:last:items-center *:[span]:last:gap-2",
"text-foreground active:opacity-80 cursor-pointer group",
className,
)}
{...props}>
<span className="absolute right-2 flex size-3.5 items-center justify-center">
<SelectPrimitive.ItemIndicator>
<CheckIcon className="size-4" />
<CheckIcon className="text-accent group-focus:text-accent-foreground size-4" />
</SelectPrimitive.ItemIndicator>
</span>
<SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>

View file

@ -34,7 +34,7 @@ function TableRow({ className, ...props }: React.ComponentProps<"tr">) {
return (
<tr
data-slot="table-row"
className={cn("hover:bg-muted/50 data-[state=selected]:bg-muted border-b transition-colors", className)}
className={cn("hover:bg-accent/5 data-[state=selected]:bg-muted border-b transition-colors", className)}
{...props}
/>
)
@ -45,7 +45,7 @@ function TableHead({ className, ...props }: React.ComponentProps<"th">) {
<th
data-slot="table-head"
className={cn(
"text-muted-foreground h-10 px-2 text-left align-middle font-medium whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
"text-foreground h-10 px-2 text-left align-middle font-medium whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
className,
)}
{...props}

View file

@ -7,7 +7,8 @@ function Textarea({ className, ...props }: React.ComponentProps<"textarea">) {
<textarea
data-slot="textarea"
className={cn(
"border-input placeholder:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive dark:bg-input/30 flex field-sizing-content min-h-16 w-full rounded-md border bg-transparent px-3 py-2 text-base shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50 md:text-sm",
"placeholder:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive flex field-sizing-content min-h-16 w-full rounded-sm px-3 py-2 text-base shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50",
"border border-input bg-input",
className,
)}
{...props}

View file

@ -17,22 +17,18 @@ export function useEventSource({ url, withCredentials, onMessage }: UseEventSour
const handleMessage = useCallback((event: MessageEvent) => onMessage(event), [onMessage])
const createEventSource = useCallback(() => {
console.log("connecting")
sourceRef.current = new EventSource(url, { withCredentials })
sourceRef.current.onopen = (event) => {
console.log("onopen", event)
sourceRef.current.onopen = () => {
statusRef.current = "open"
setStatus("open")
}
sourceRef.current.onmessage = (event) => {
// console.log("onmessage", event)
handleMessage(event)
}
sourceRef.current.onerror = (event) => {
console.log("onerror", event)
sourceRef.current.onerror = () => {
statusRef.current = "error"
setStatus("error")
// sourceRef.current?.close()
@ -45,7 +41,6 @@ export function useEventSource({ url, withCredentials, onMessage }: UseEventSour
setTimeout(() => {
if (statusRef.current === "init") {
console.log("timeout -> close")
sourceRef.current?.close()
sourceRef.current = null
createEventSource()
@ -53,7 +48,6 @@ export function useEventSource({ url, withCredentials, onMessage }: UseEventSour
}, 100)
return () => {
console.log("unmounting -> close")
sourceRef.current?.close()
sourceRef.current = null
}

View file

@ -0,0 +1,36 @@
CREATE TABLE `pendingTasks` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`runId` integer NOT NULL,
`language` text NOT NULL,
`exercise` text NOT NULL,
`pid` integer,
`createdAt` integer NOT NULL
);
--> statement-breakpoint
CREATE UNIQUE INDEX `pendingTasks_language_exercise_idx` ON `pendingTasks` (`runId`,`language`,`exercise`);--> statement-breakpoint
CREATE TABLE `runs` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`model` text NOT NULL,
`description` text,
`pid` integer,
`socketPath` text NOT NULL,
`createdAt` integer NOT NULL
);
--> statement-breakpoint
CREATE TABLE `tasks` (
`id` integer PRIMARY KEY AUTOINCREMENT NOT NULL,
`runId` integer NOT NULL,
`language` text NOT NULL,
`exercise` text NOT NULL,
`tokensIn` integer NOT NULL,
`tokensOut` integer NOT NULL,
`tokensContext` integer NOT NULL,
`cacheWrites` integer NOT NULL,
`cacheReads` integer NOT NULL,
`cost` real NOT NULL,
`duration` integer NOT NULL,
`passed` integer,
`createdAt` integer NOT NULL
);
--> statement-breakpoint
CREATE UNIQUE INDEX `tasks_language_exercise_idx` ON `tasks` (`runId`,`language`,`exercise`);

View file

@ -0,0 +1,235 @@
{
"version": "6",
"dialect": "sqlite",
"id": "d7bfd73f-dbf5-430e-ab81-f436029d06e5",
"prevId": "00000000-0000-0000-0000-000000000000",
"tables": {
"pendingTasks": {
"name": "pendingTasks",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"runId": {
"name": "runId",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"language": {
"name": "language",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"exercise": {
"name": "exercise",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"pid": {
"name": "pid",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"createdAt": {
"name": "createdAt",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"pendingTasks_language_exercise_idx": {
"name": "pendingTasks_language_exercise_idx",
"columns": ["runId", "language", "exercise"],
"isUnique": true
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"runs": {
"name": "runs",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"model": {
"name": "model",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"description": {
"name": "description",
"type": "text",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"pid": {
"name": "pid",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"socketPath": {
"name": "socketPath",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"createdAt": {
"name": "createdAt",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
},
"tasks": {
"name": "tasks",
"columns": {
"id": {
"name": "id",
"type": "integer",
"primaryKey": true,
"notNull": true,
"autoincrement": true
},
"runId": {
"name": "runId",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"language": {
"name": "language",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"exercise": {
"name": "exercise",
"type": "text",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"tokensIn": {
"name": "tokensIn",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"tokensOut": {
"name": "tokensOut",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"tokensContext": {
"name": "tokensContext",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"cacheWrites": {
"name": "cacheWrites",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"cacheReads": {
"name": "cacheReads",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"cost": {
"name": "cost",
"type": "real",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"duration": {
"name": "duration",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
},
"passed": {
"name": "passed",
"type": "integer",
"primaryKey": false,
"notNull": false,
"autoincrement": false
},
"createdAt": {
"name": "createdAt",
"type": "integer",
"primaryKey": false,
"notNull": true,
"autoincrement": false
}
},
"indexes": {
"tasks_language_exercise_idx": {
"name": "tasks_language_exercise_idx",
"columns": ["runId", "language", "exercise"],
"isUnique": true
}
},
"foreignKeys": {},
"compositePrimaryKeys": {},
"uniqueConstraints": {},
"checkConstraints": {}
}
},
"views": {},
"enums": {},
"_meta": {
"schemas": {},
"tables": {},
"columns": {}
},
"internal": {
"indexes": {}
}
}

View file

@ -0,0 +1,13 @@
{
"version": "7",
"dialect": "sqlite",
"entries": [
{
"idx": 0,
"version": "6",
"when": 1742568357359,
"tag": "0000_legal_cassandra_nova",
"breakpoints": true
}
]
}

View file

@ -13,6 +13,8 @@ export const runs = sqliteTable("runs", {
id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }),
model: text().notNull(),
description: text(),
pid: integer({ mode: "number" }),
socketPath: text().notNull(),
createdAt: integer({ mode: "timestamp" }).notNull(),
})
@ -43,10 +45,10 @@ export const tasks = sqliteTable(
cacheReads: integer({ mode: "number" }).notNull(),
cost: real().notNull(),
duration: integer({ mode: "number" }).notNull(),
passed: integer({ mode: "boolean" }).notNull(),
passed: integer({ mode: "boolean" }),
createdAt: integer({ mode: "timestamp" }).notNull(),
},
(table) => [t.uniqueIndex("language_exercise_idx").on(table.runId, table.language, table.exercise)],
(table) => [t.uniqueIndex("tasks_language_exercise_idx").on(table.runId, table.language, table.exercise)],
)
export type Task = typeof tasks.$inferSelect
@ -58,6 +60,32 @@ export const insertTaskSchema = createInsertSchema(tasks).omit({
export type InsertTask = z.infer<typeof insertTaskSchema>
/**
* pendingTasks
*/
export const pendingTasks = sqliteTable(
"pendingTasks",
{
id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }),
runId: integer({ mode: "number" }).notNull(),
language: text({ enum: languages }).notNull(),
exercise: text().notNull(),
pid: integer({ mode: "number" }),
createdAt: integer({ mode: "timestamp" }).notNull(),
},
(table) => [t.uniqueIndex("pendingTasks_language_exercise_idx").on(table.runId, table.language, table.exercise)],
)
export type PendingTask = typeof pendingTasks.$inferSelect
export const insertPendingTaskSchema = createInsertSchema(pendingTasks).omit({
id: true,
createdAt: true,
})
export type InsertPendingTask = z.infer<typeof insertPendingTaskSchema>
/**
* schema
*/

View file

@ -6,7 +6,7 @@ import * as vscode from "vscode"
import { RooCodeAPI } from "../../../../src/exports/roo-code.js"
import { IpcServer, ServerMessageType } from "@benchmark/ipc"
import { Language, createTask } from "@benchmark/db"
import { Language, findRun, createTask } from "@benchmark/db"
import { waitUntilReady, waitUntilCompleted, sleep } from "./utils.js"
@ -29,6 +29,8 @@ export async function run() {
const prompt = await fs.readFile(promptPath, "utf-8")
const run = await findRun(runId)
/**
* Activate the extension.
*/
@ -81,7 +83,7 @@ export async function run() {
* Start the IPC server.
*/
const server = new IpcServer(`/tmp/benchmark-${runId}.sock`, () => {})
const server = new IpcServer(run.socketPath, () => {})
server.listen()
server.on("client", (id) => {

View file

@ -117,6 +117,9 @@ importers:
tailwindcss-animate:
specifier: ^1.0.7
version: 1.0.7(tailwindcss@4.0.14)
uuid:
specifier: ^11.1.0
version: 11.1.0
zod:
specifier: ^3.24.2
version: 3.24.2
@ -3602,6 +3605,10 @@ packages:
util-deprecate@1.0.2:
resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==}
uuid@11.1.0:
resolution: {integrity: sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A==}
hasBin: true
wcwidth@1.0.1:
resolution: {integrity: sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==}
@ -6958,6 +6965,8 @@ snapshots:
util-deprecate@1.0.2: {}
uuid@11.1.0: {}
wcwidth@1.0.1:
dependencies:
defaults: 1.0.4