diff --git a/benchmark/README.md b/benchmark/README.md index ec8674022b..a40d7a3bdd 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -30,60 +30,8 @@ cp packages/server/.env.sample packages/server/.env pnpm --filter @benchmark/server db:migrate ``` -Run the benchmark server: +Run the web ui: ```sh -pnpm dev -``` - -################################################################################ - -Configure ENV vars (OpenRouter, PostHog, etc): - -```sh -cp .env.local.sample .env.local -# Update ENV vars as needed. -``` - -Build and run a Docker image with the development environment needed to run the -benchmarks (C++, Go, Java, Node.js, Python & Rust): - -```sh -npm run docker:start -``` - -Run an exercise: - -```sh -npm run docker:benchmark -- -e exercises/javascript/binary -``` - -Select and run an exercise: - -```sh -npm run cli -``` - -Select and run an exercise for a specific language: - -```sh -npm run cli -- run rust -``` - -Run all exercises for a language: - -```sh -npm run cli -- run rust all -``` - -Run all exercises: - -```sh -npm run cli -- run all -``` - -Run all exercises using a specific runId (useful for re-trying when an unexpected error occurs): - -```sh -npm run cli -- run all --runId 1 +pnpm web ``` diff --git a/benchmark/packages/db/src/index.ts b/benchmark/packages/db/src/index.ts index 980e881b6a..ab36bafa14 100644 --- a/benchmark/packages/db/src/index.ts +++ b/benchmark/packages/db/src/index.ts @@ -6,12 +6,12 @@ export { schema } from "./schema.js" * runs */ -export { type Run, insertRunSchema } from "./schema.js" +export { type Run, type InsertRun, insertRunSchema } from "./schema.js" export { findRun, createRun, getRuns } from "./queries/runs.js" /** * tasks */ -export { type Task, insertTaskSchema } from "./schema.js" +export { type Task, type InsertTask, insertTaskSchema } from "./schema.js" export { findTask, createTask, getTask } from "./queries/tasks.js" diff --git a/benchmark/packages/web/package.json b/benchmark/packages/web/package.json index 06cf72eeb6..4fecd92b0e 100644 --- a/benchmark/packages/web/package.json +++ b/benchmark/packages/web/package.json @@ -13,7 +13,11 @@ }, "dependencies": { "@benchmark/db": "workspace:^", + "@hookform/resolvers": "^4.1.3", + "@radix-ui/react-label": "^2.1.2", + "@radix-ui/react-select": "^2.1.6", "@radix-ui/react-slot": "^1.1.2", + "@tanstack/react-query": "^5.69.0", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "lucide-react": "^0.479.0", @@ -21,6 +25,7 @@ "next-themes": "^0.4.6", "react": "^19.0.0", "react-dom": "^19.0.0", + "react-hook-form": "^7.54.2", "tailwind-merge": "^3.0.2", "tailwindcss-animate": "^1.0.7", "zod": "^3.24.2" diff --git a/benchmark/packages/web/src/app/home.tsx b/benchmark/packages/web/src/app/home.tsx new file mode 100644 index 0000000000..1f5333b2cf --- /dev/null +++ b/benchmark/packages/web/src/app/home.tsx @@ -0,0 +1,43 @@ +import { getRuns } from "@benchmark/db" + +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui" +import { formatCurrency, formatDuration } from "@/lib" + +export const dynamic = "force-dynamic" + +export async function Home() { + const runs = await getRuns() + + return ( +
+ + + + ID + Model + Timestamp + Passed + Failed + % Correct + Cost + Duration + + + + {runs.map((run) => ( + + {run.id} + {run.model} + {new Date(run.createdAt).toLocaleString()} + {run.passed} + {run.failed} + {(run.rate * 100).toFixed(1)}% + {formatCurrency(run.cost)} + {formatDuration(run.duration)} + + ))} + +
+
+ ) +} diff --git a/benchmark/packages/web/src/app/layout.tsx b/benchmark/packages/web/src/app/layout.tsx index 8369358768..f95e35e961 100644 --- a/benchmark/packages/web/src/app/layout.tsx +++ b/benchmark/packages/web/src/app/layout.tsx @@ -1,20 +1,12 @@ import type { Metadata } from "next" import { Geist, Geist_Mono } from "next/font/google" -import { ThemeProvider } from "@/components/theme-provider" +import { ThemeProvider, ReactQueryProvider } from "@/components/providers" import "./globals.css" -const geistSans = Geist({ - variable: "--font-geist-sans", - subsets: ["latin"], -}) - -const geistMono = Geist_Mono({ - variable: "--font-geist-mono", - - subsets: ["latin"], -}) +const geistSans = Geist({ variable: "--font-geist-sans", subsets: ["latin"] }) +const geistMono = Geist_Mono({ variable: "--font-geist-mono", subsets: ["latin"] }) export const metadata: Metadata = { title: "Roo Code Benchmarks", @@ -29,7 +21,7 @@ export default function RootLayout({ - {children} + {children} diff --git a/benchmark/packages/web/src/app/page.tsx b/benchmark/packages/web/src/app/page.tsx index 954cc5a0cb..9bb31b3de0 100644 --- a/benchmark/packages/web/src/app/page.tsx +++ b/benchmark/packages/web/src/app/page.tsx @@ -1,43 +1,5 @@ -import { getRuns } from "@benchmark/db" +import { Home } from "./home" -import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui" -import { formatCurrency, formatDuration } from "@/lib" - -export const dynamic = "force-dynamic" - -export default async function Home() { - const runs = await getRuns() - - return ( -
- - - - ID - Model - Timestamp - Passed - Failed - % Correct - Cost - Duration - - - - {runs.map((run) => ( - - {run.id} - {run.model} - {new Date(run.createdAt).toLocaleString()} - {run.passed} - {run.failed} - {(run.rate * 100).toFixed(1)}% - {formatCurrency(run.cost)} - {formatDuration(run.duration)} - - ))} - -
-
- ) +export default async function Page() { + return } diff --git a/benchmark/packages/web/src/app/runs/[id]/page.tsx b/benchmark/packages/web/src/app/runs/[id]/page.tsx new file mode 100644 index 0000000000..88e145e18e --- /dev/null +++ b/benchmark/packages/web/src/app/runs/[id]/page.tsx @@ -0,0 +1,5 @@ +import { ShowRun } from "./show-run" + +export default async function Page() { + return +} diff --git a/benchmark/packages/web/src/app/runs/[id]/show-run.tsx b/benchmark/packages/web/src/app/runs/[id]/show-run.tsx new file mode 100644 index 0000000000..cb2486ce3d --- /dev/null +++ b/benchmark/packages/web/src/app/runs/[id]/show-run.tsx @@ -0,0 +1,3 @@ +export async function ShowRun() { + return
Show Run
+} diff --git a/benchmark/packages/web/src/app/runs/new/actions.ts b/benchmark/packages/web/src/app/runs/new/actions.ts new file mode 100644 index 0000000000..925653d09a --- /dev/null +++ b/benchmark/packages/web/src/app/runs/new/actions.ts @@ -0,0 +1,11 @@ +"use server" + +import { revalidatePath } from "next/cache" + +import * as db from "@benchmark/db" + +export async function createRun(data: db.InsertRun) { + const run = await db.createRun(data) + revalidatePath("/runs") + return run +} diff --git a/benchmark/packages/web/src/app/runs/new/new-run.tsx b/benchmark/packages/web/src/app/runs/new/new-run.tsx new file mode 100644 index 0000000000..bc9a91881a --- /dev/null +++ b/benchmark/packages/web/src/app/runs/new/new-run.tsx @@ -0,0 +1,132 @@ +"use client" + +import { useState } from "react" +import { useRouter } from "next/navigation" +import { useForm, FormProvider } from "react-hook-form" +import { zodResolver } from "@hookform/resolvers/zod" +import { z } from "zod" + +import { useOpenRouterModels } from "@/hooks/use-open-router-models" +import { + Button, + FormControl, + FormDescription, + FormField, + FormItem, + FormLabel, + FormMessage, + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, + Textarea, +} from "@/components/ui" + +import { createRun } from "./actions" + +const formSchema = z.object({ + model: z.string({ + required_error: "Please select a model", + }), + description: z.string().optional(), +}) + +type FormValues = z.infer + +export function NewRun() { + const router = useRouter() + const { data: models, isLoading, error } = useOpenRouterModels() + const [isSubmitting, setIsSubmitting] = useState(false) + + const form = useForm({ + resolver: zodResolver(formSchema), + }) + + async function onSubmit(data: FormValues) { + setIsSubmitting(true) + + try { + const run = await createRun(data) + router.push(`/runs/${run.id}`) + } catch (error) { + console.error("Error creating run:", error) + setIsSubmitting(false) + } + } + + return ( +
+
+

Create New Run

+

+ Create a new run by selecting a model and providing an optional description. +

+
+
+ +
+ ( + + Model + + Select the model to use for this run. + + + )} + /> + ( + + Description + +