diff --git a/benchmark/apps/web/package.json b/benchmark/apps/web/package.json index 677a436bc4..6f3f2ee985 100644 --- a/benchmark/apps/web/package.json +++ b/benchmark/apps/web/package.json @@ -27,6 +27,7 @@ "react-hook-form": "^7.54.2", "tailwind-merge": "^3.0.2", "tailwindcss-animate": "^1.0.7", + "uuid": "^11.1.0", "zod": "^3.24.2" }, "devDependencies": { diff --git a/benchmark/apps/web/src/app/api/runs/[id]/stream/route.ts b/benchmark/apps/web/src/app/api/runs/[id]/stream/route.ts index e94b457e45..a2f13c7907 100644 --- a/benchmark/apps/web/src/app/api/runs/[id]/stream/route.ts +++ b/benchmark/apps/web/src/app/api/runs/[id]/stream/route.ts @@ -1,4 +1,5 @@ import type { NextRequest } from "next/server" +import { v4 as uuidv4 } from "uuid" import { findRun } from "@benchmark/db" import { IpcClient } from "@benchmark/ipc" @@ -9,9 +10,10 @@ export const dynamic = "force-dynamic" export async function GET(request: NextRequest, { params }: { params: Promise<{ id: string }> }) { const { id } = await params + const requestId = uuidv4() const stream = new SSEStream() const run = await findRun(Number(id)) - const client = new IpcClient(`/tmp/benchmark-${run.id}.sock`, () => {}) + const client = new IpcClient(run.socketPath, () => {}) const write = async (data: string | object) => { const success = await stream.write(data) @@ -21,12 +23,13 @@ export async function GET(request: NextRequest, { params }: { params: Promise<{ } } + console.log(`[stream#${requestId}] connect`) client.on("connect", () => write("connect")) client.on("disconnect", () => write("disconnect")) client.on("message", write) request.signal.addEventListener("abort", () => { - console.log(`abort`) + console.log(`[stream#${requestId}] abort`) client.disconnect() stream.close().catch(() => {}) }) diff --git a/benchmark/apps/web/src/app/globals.css b/benchmark/apps/web/src/app/globals.css index e93a4205a4..22506c2526 100644 --- a/benchmark/apps/web/src/app/globals.css +++ b/benchmark/apps/web/src/app/globals.css @@ -4,46 +4,6 @@ @custom-variant dark (&:is(.dark *)); -@theme inline { - --color-background: var(--background); - --color-foreground: var(--foreground); - --font-sans: var(--font-geist-sans); - --font-mono: var(--font-geist-mono); - --color-sidebar-ring: var(--sidebar-ring); - --color-sidebar-border: var(--sidebar-border); - --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); - --color-sidebar-accent: var(--sidebar-accent); - --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); - --color-sidebar-primary: var(--sidebar-primary); - --color-sidebar-foreground: var(--sidebar-foreground); - --color-sidebar: var(--sidebar); - --color-chart-5: var(--chart-5); - --color-chart-4: var(--chart-4); - --color-chart-3: var(--chart-3); - --color-chart-2: var(--chart-2); - --color-chart-1: var(--chart-1); - --color-ring: var(--ring); - --color-input: var(--input); - --color-border: var(--border); - --color-destructive: var(--destructive); - --color-accent-foreground: var(--accent-foreground); - --color-accent: var(--accent); - --color-muted-foreground: var(--muted-foreground); - --color-muted: var(--muted); - --color-secondary-foreground: var(--secondary-foreground); - --color-secondary: var(--secondary); - --color-primary-foreground: var(--primary-foreground); - --color-primary: var(--primary); - --color-popover-foreground: var(--popover-foreground); - --color-popover: var(--popover); - --color-card-foreground: var(--card-foreground); - --color-card: var(--card); - --radius-sm: calc(var(--radius) - 4px); - --radius-md: calc(var(--radius) - 2px); - --radius-lg: var(--radius); - --radius-xl: calc(var(--radius) + 4px); -} - :root { --radius: 0.625rem; --background: oklch(1 0 0); @@ -80,24 +40,24 @@ } .dark { - --background: oklch(0.145 0 0); - --foreground: oklch(0.985 0 0); + --background: oklch(23.66% 0.0198 271.79); + --foreground: oklch(75.15% 0.0477 278.41); --card: oklch(0.205 0 0); --card-foreground: oklch(0.985 0 0); - --popover: oklch(0.205 0 0); + --popover: var(--primary); --popover-foreground: oklch(0.985 0 0); - --primary: oklch(0.922 0 0); - --primary-foreground: oklch(0.205 0 0); + --primary: oklch(29.33% 0.0295 276.18); + --primary-foreground: var(--accent); --secondary: oklch(0.269 0 0); --secondary-foreground: oklch(0.985 0 0); - --muted: oklch(0.269 0 0); - --muted-foreground: oklch(0.708 0 0); - --accent: oklch(0.269 0 0); - --accent-foreground: oklch(0.985 0 0); - --destructive: oklch(0.704 0.191 22.216); - --border: oklch(1 0 0 / 10%); - --input: oklch(1 0 0 / 15%); - --ring: oklch(0.556 0 0); + --muted: oklch(28.27% 0.0207 273.06); + --muted-foreground: oklch(75.15% 0.0477 278.41); + --accent: oklch(70.21% 0.1813 328.71); + --accent-foreground: rgba(0, 0, 0, 0.5); + --destructive: oklch(72.14% 0.1616 15.49); + --border: oklch(29.33% 0.0295 276.18); + --input: var(--primary); + --ring: oklch(83.63% 0.1259 176.52); --chart-1: oklch(0.488 0.243 264.376); --chart-2: oklch(0.696 0.17 162.48); --chart-3: oklch(0.769 0.188 70.08); @@ -113,10 +73,52 @@ --sidebar-ring: oklch(0.556 0 0); } +@theme inline { + --color-background: var(--background); + --color-foreground: var(--foreground); + --color-sidebar-ring: var(--sidebar-ring); + --color-sidebar-border: var(--sidebar-border); + --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); + --color-sidebar-accent: var(--sidebar-accent); + --color-sidebar-primary-foreground: var(--sidebar-primary-foreground); + --color-sidebar-primary: var(--sidebar-primary); + --color-sidebar-foreground: var(--sidebar-foreground); + --color-sidebar: var(--sidebar); + --color-chart-5: var(--chart-5); + --color-chart-4: var(--chart-4); + --color-chart-3: var(--chart-3); + --color-chart-2: var(--chart-2); + --color-chart-1: var(--chart-1); + --color-ring: var(--ring); + --color-input: var(--input); + --color-border: var(--border); + --color-destructive: var(--destructive); + --color-accent-foreground: var(--accent-foreground); + --color-accent: var(--accent); + --color-muted-foreground: var(--muted-foreground); + --color-muted: var(--muted); + --color-secondary-foreground: var(--secondary-foreground); + --color-secondary: var(--secondary); + --color-primary-foreground: var(--primary-foreground); + --color-primary: var(--primary); + --color-popover-foreground: var(--popover-foreground); + --color-popover: var(--popover); + --color-card-foreground: var(--card-foreground); + --color-card: var(--card); + --radius-sm: calc(var(--radius) - 4px); + --radius-md: calc(var(--radius) - 2px); + --radius-lg: var(--radius); + --radius-xl: calc(var(--radius) + 4px); +} + @layer base { * { @apply border-border outline-ring/50; } + html, + body { + height: 100%; + } body { @apply bg-background text-foreground; } diff --git a/benchmark/apps/web/src/app/home.tsx b/benchmark/apps/web/src/app/home.tsx index 1f5333b2cf..58d99d33d9 100644 --- a/benchmark/apps/web/src/app/home.tsx +++ b/benchmark/apps/web/src/app/home.tsx @@ -1,43 +1,56 @@ +"use client" + +import { useRouter } from "next/navigation" +import { Rocket } from "lucide-react" + import { getRuns } from "@benchmark/db" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui" +import { Button, Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui" import { formatCurrency, formatDuration } from "@/lib" -export const dynamic = "force-dynamic" +type Run = Awaited>[number] -export async function Home() { - const runs = await getRuns() +export function Home({ runs }: { runs: Run[] }) { + const router = useRouter() return ( -
- - - - ID - Model - Timestamp - Passed - Failed - % Correct - Cost - Duration - - - - {runs.map((run) => ( - - {run.id} - {run.model} - {new Date(run.createdAt).toLocaleString()} - {run.passed} - {run.failed} - {(run.rate * 100).toFixed(1)}% - {formatCurrency(run.cost)} - {formatDuration(run.duration)} + <> +
+
+ + + ID + Model + Timestamp + Passed + Failed + % Correct + Cost + Duration - ))} - -
-
+ + + {runs.map((run) => ( + + {run.id} + {run.model} + {new Date(run.createdAt).toLocaleString()} + {run.passed} + {run.failed} + {(run.rate * 100).toFixed(1)}% + {formatCurrency(run.cost)} + {formatDuration(run.duration)} + + ))} + + + + + ) } diff --git a/benchmark/apps/web/src/app/layout.tsx b/benchmark/apps/web/src/app/layout.tsx index f95e35e961..7362b413b1 100644 --- a/benchmark/apps/web/src/app/layout.tsx +++ b/benchmark/apps/web/src/app/layout.tsx @@ -5,8 +5,8 @@ import { ThemeProvider, ReactQueryProvider } from "@/components/providers" import "./globals.css" -const geistSans = Geist({ variable: "--font-geist-sans", subsets: ["latin"] }) -const geistMono = Geist_Mono({ variable: "--font-geist-mono", subsets: ["latin"] }) +const fontSans = Geist({ variable: "--font-sans", subsets: ["latin"] }) +const fontMono = Geist_Mono({ variable: "--font-mono", subsets: ["latin"] }) export const metadata: Metadata = { title: "Roo Code Benchmarks", @@ -19,7 +19,7 @@ export default function RootLayout({ }>) { return ( - + {children} diff --git a/benchmark/apps/web/src/app/page.tsx b/benchmark/apps/web/src/app/page.tsx index 9bb31b3de0..f3299e278d 100644 --- a/benchmark/apps/web/src/app/page.tsx +++ b/benchmark/apps/web/src/app/page.tsx @@ -1,5 +1,12 @@ +import { getRuns } from "@benchmark/db" + import { Home } from "./home" +export const dynamic = "force-dynamic" + export default async function Page() { - return + const runs = await getRuns() + console.log(runs) + + return } diff --git a/benchmark/apps/web/src/app/runs/[id]/show-run.tsx b/benchmark/apps/web/src/app/runs/[id]/show-run.tsx index 4ac0b2c6d0..ac0f3a2add 100644 --- a/benchmark/apps/web/src/app/runs/[id]/show-run.tsx +++ b/benchmark/apps/web/src/app/runs/[id]/show-run.tsx @@ -10,6 +10,7 @@ export function ShowRun({ run }: { run: Run }) { const url = `/api/runs/${run.id}/stream` const onMessage = useCallback(({ data }: MessageEvent) => console.log(data), []) const status = useEventSource({ url, onMessage }) + return (
Show Run: {run.id} | {status} diff --git a/benchmark/apps/web/src/app/runs/new/actions.ts b/benchmark/apps/web/src/app/runs/new/actions.ts index 925653d09a..f06b8086f8 100644 --- a/benchmark/apps/web/src/app/runs/new/actions.ts +++ b/benchmark/apps/web/src/app/runs/new/actions.ts @@ -1,5 +1,6 @@ "use server" +import { spawn } from "child_process" import { revalidatePath } from "next/cache" import * as db from "@benchmark/db" @@ -7,5 +8,20 @@ import * as db from "@benchmark/db" export async function createRun(data: db.InsertRun) { const run = await db.createRun(data) revalidatePath("/runs") - return run + + try { + const process = spawn( + "pnpm", + ["--filter", "@benchmark/cli", "dev", "run", "all", "--runId", run.id.toString()], + { + detached: true, + stdio: "ignore", + }, + ) + + process.unref() + return process.pid + } catch (_) { + return undefined + } } diff --git a/benchmark/apps/web/src/app/runs/new/new-run.tsx b/benchmark/apps/web/src/app/runs/new/new-run.tsx index bc9a91881a..6ce415c679 100644 --- a/benchmark/apps/web/src/app/runs/new/new-run.tsx +++ b/benchmark/apps/web/src/app/runs/new/new-run.tsx @@ -5,12 +5,12 @@ import { useRouter } from "next/navigation" import { useForm, FormProvider } from "react-hook-form" import { zodResolver } from "@hookform/resolvers/zod" import { z } from "zod" +import { X, Loader2, Rocket } from "lucide-react" import { useOpenRouterModels } from "@/hooks/use-open-router-models" import { Button, FormControl, - FormDescription, FormField, FormItem, FormLabel, @@ -26,9 +26,7 @@ import { import { createRun } from "./actions" const formSchema = z.object({ - model: z.string({ - required_error: "Please select a model", - }), + model: z.string(), description: z.string().optional(), }) @@ -36,11 +34,15 @@ type FormValues = z.infer export function NewRun() { const router = useRouter() - const { data: models, isLoading, error } = useOpenRouterModels() + const { data: models, isLoading, isError } = useOpenRouterModels() const [isSubmitting, setIsSubmitting] = useState(false) const form = useForm({ resolver: zodResolver(formSchema), + defaultValues: { + model: "anthropic/claude-3.7-sonnet", + description: "", + }, }) async function onSubmit(data: FormValues) { @@ -56,77 +58,69 @@ export function NewRun() { } return ( -
-
-

Create New Run

-

- Create a new run by selecting a model and providing an optional description. -

-
-
- -
- ( - - Model - - Select the model to use for this run. - - - )} - /> - ( - - Description + <> + + +
__dirname: {__dirname}
+ ( + + OpenRouter Model +