diff --git a/benchmark/apps/cli/src/index.ts b/benchmark/apps/cli/src/index.ts index a23298f06a..583cfea2af 100644 --- a/benchmark/apps/cli/src/index.ts +++ b/benchmark/apps/cli/src/index.ts @@ -7,9 +7,9 @@ import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun" import { runTests } from "@vscode/test-electron" import { execa, parseCommandString } from "execa" -import { type Language, languages, TaskEventName } from "@benchmark/types" +import { type Language, languages, IpcOrigin, IpcMessageType, TaskEventName } from "@benchmark/types" import { type Run, findRun, createRun, finishRun, createTask, Task, getTasks, updateTask } from "@benchmark/db" -import { IpcServer, IpcMessageType, IpcOrigin } from "@benchmark/ipc" +import { IpcServer } from "@benchmark/ipc" import { __dirname, extensionDevelopmentPath, extensionTestsPath, exercisesPath } from "./paths.js" import { getExercises } from "./exercises.js" @@ -67,21 +67,29 @@ const run = async (toolbox: GluegunToolbox) => { } const tasks = await getTasks(run.id) + let currentTask = tasks[0] - if (tasks.length === 0) { + if (!currentTask) { throw new Error("No tasks found.") } const server = new IpcServer(run.socketPath, () => {}) server.listen() - let currentTask = tasks[0] - server.on("connect", (clientId) => { server.send(clientId, { type: IpcMessageType.TaskEvent, origin: IpcOrigin.Server, - data: { eventName: TaskEventName.Connect, data: { task: currentTask } }, + data: { eventName: TaskEventName.Connect, data: { task: currentTask! } }, + }) + }) + + server.on("taskEvent", (relayClientId, data) => { + server.broadcast({ + type: IpcMessageType.TaskEvent, + origin: IpcOrigin.Server, + relayClientId, + data, }) }) diff --git a/benchmark/apps/web/src/app/runs/new/new-run.tsx b/benchmark/apps/web/src/app/runs/new/new-run.tsx index dab90249fd..587ab91737 100644 --- a/benchmark/apps/web/src/app/runs/new/new-run.tsx +++ b/benchmark/apps/web/src/app/runs/new/new-run.tsx @@ -84,6 +84,33 @@ export function NewRun() { [router], ) + const onFilter = useCallback( + (value: string, search: string) => { + if (modelSearchValueRef.current !== search) { + modelSearchValueRef.current = search + modelSearchResultsRef.current.clear() + + for (const { + obj: { id }, + score, + } of fuzzysort.go(search, models.data || [], { + key: "name", + })) { + modelSearchResultsRef.current.set(id, score) + } + } + + return modelSearchResultsRef.current.get(value) ?? 0 + }, + [models.data], + ) + + const recommendedModels = [ + "anthropic/claude-3.7-sonnet", + "anthropic/claude-3.7-sonnet:thinking", + "google/gemini-2.0-flash-001", + ] + return ( <> @@ -108,24 +135,7 @@ export function NewRun() { - { - if (modelSearchValueRef.current !== search) { - modelSearchValueRef.current = search - modelSearchResultsRef.current.clear() - - for (const { - obj: { id }, - score, - } of fuzzysort.go(search, models.data || [], { - key: "name", - })) { - modelSearchResultsRef.current.set(id, score) - } - } - - return modelSearchResultsRef.current.get(value) ?? 0 - }}> + +
+
Recommended:
+ {recommendedModels.map((modelId) => ( +
+ +
+ ))} +
)} /> diff --git a/benchmark/apps/web/src/components/ui/button.tsx b/benchmark/apps/web/src/components/ui/button.tsx index 7e0f88032d..f53d07c893 100644 --- a/benchmark/apps/web/src/components/ui/button.tsx +++ b/benchmark/apps/web/src/components/ui/button.tsx @@ -16,7 +16,7 @@ const buttonVariants = cva( "border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input", secondary: "bg-secondary text-secondary-foreground shadow-xs", ghost: "hover:bg-accent hover:text-accent-foreground", - link: "text-accent underline-offset-4 hover:underline px-1.5!", + link: "text-accent underline-offset-4 hover:underline h-4! px-1! rounded-none", input: "bg-input text-input-foreground active:scale-100 shadow-xs", }, size: { diff --git a/benchmark/apps/web/src/hooks/use-run-status.ts b/benchmark/apps/web/src/hooks/use-run-status.ts index 3216aa05f6..4c02b4c7db 100644 --- a/benchmark/apps/web/src/hooks/use-run-status.ts +++ b/benchmark/apps/web/src/hooks/use-run-status.ts @@ -1,14 +1,13 @@ import { useState, useCallback, useRef } from "react" import { useQuery, keepPreviousData } from "@tanstack/react-query" +import { TaskEventName, taskEventSchema } from "@benchmark/types" import { Run } from "@benchmark/db" import { getTasks } from "@/lib/server/tasks" -import { ipcServerMessageSchema } from "@/lib/schemas" import { useEventSource } from "@/hooks/use-event-source" export const useRunStatus = (run: Run) => { - const [clientId, setClientId] = useState() const [runningTaskId, setRunningTaskId] = useState() const outputRef = useRef>(new Map()) const [outputCounts, setOutputCounts] = useState>({}) @@ -32,7 +31,7 @@ export const useRunStatus = (run: Run) => { return } - const result = ipcServerMessageSchema.safeParse(data) + const result = taskEventSchema.safeParse(data) if (!result.success) { console.log(`unrecognized messageEvent.data: ${messageEvent.data}`) @@ -40,32 +39,36 @@ export const useRunStatus = (run: Run) => { } const payload = result.data + const taskId = payload.data.task.id - if (payload.type === "Ack") { - setClientId(payload.data.clientId as string) - } else if (payload.type === "TaskEvent") { - const taskId = payload.data.data.task.id - - if (payload.data.eventName === "connect" || payload.data.eventName === "taskStarted") { + switch (payload.eventName) { + case TaskEventName.Connect: + case TaskEventName.TaskStarted: setRunningTaskId(taskId) - } else if (payload.data.eventName === "taskFinished") { + break + case TaskEventName.TaskFinished: setRunningTaskId(undefined) - } else if (payload.data.eventName === "message") { - const { text } = payload.data.data.message.message - console.log(`message: ${taskId} ->`, text) - outputRef.current.set(taskId, [...(outputRef.current.get(taskId) || []), text]) - const outputCounts: Record = {} + break + case TaskEventName.Message: { + const text = payload.data.message.message.text - for (const [taskId, messages] of outputRef.current.entries()) { - outputCounts[taskId] = messages.length + if (text) { + outputRef.current.set(taskId, [...(outputRef.current.get(taskId) || []), text]) + const outputCounts: Record = {} + + for (const [taskId, messages] of outputRef.current.entries()) { + outputCounts[taskId] = messages.length + } + + setOutputCounts(outputCounts) } - setOutputCounts(outputCounts) + break } } }, []) const status = useEventSource({ url, onMessage }) - return { tasks, status, clientId, runningTaskId, output: outputRef.current, outputCounts } + return { tasks, status, runningTaskId, output: outputRef.current, outputCounts } } diff --git a/benchmark/apps/web/src/lib/schemas.ts b/benchmark/apps/web/src/lib/schemas.ts index a82a1641d5..4036da7945 100644 --- a/benchmark/apps/web/src/lib/schemas.ts +++ b/benchmark/apps/web/src/lib/schemas.ts @@ -17,62 +17,3 @@ export const createRunSchema = z }) export type CreateRun = z.infer - -/** - * TaskEvent - */ - -export const taskEventSchema = z.discriminatedUnion("eventName", [ - z.object({ - eventName: z.literal("connect"), - data: z.object({ task: z.object({ id: z.number() }) }), - }), - z.object({ - eventName: z.literal("taskStarted"), - data: z.object({ task: z.object({ id: z.number() }) }), - }), - z.object({ - eventName: z.literal("message"), - data: z.object({ - task: z.object({ id: z.number() }), - message: z.object({ - taskId: z.string(), - action: z.enum(["created", "updated"]), - message: z.object({ - ask: z.string().optional(), - say: z.string().optional(), - partial: z.boolean(), - text: z.string(), - }), - }), - }), - }), - z.object({ - eventName: z.literal("taskTokenUsageUpdated"), - data: z.object({ - task: z.object({ id: z.number() }), - usage: z.object({}), - }), - }), - z.object({ - eventName: z.literal("taskFinished"), - data: z.object({ task: z.object({ id: z.number() }) }), - }), -]) - -/** - * IpcServerMessage - */ - -export const ipcServerMessageSchema = z.discriminatedUnion("type", [ - z.object({ - type: z.literal("Ack"), - data: z.object({ clientId: z.string() }), - }), - z.object({ - type: z.literal("TaskEvent"), - data: taskEventSchema, - }), -]) - -export type IpcServerMessage = z.infer diff --git a/benchmark/packages/ipc/src/index.ts b/benchmark/packages/ipc/src/index.ts index 140ed6b7ad..20ba9671a9 100644 --- a/benchmark/packages/ipc/src/index.ts +++ b/benchmark/packages/ipc/src/index.ts @@ -3,47 +3,8 @@ import { Socket } from "node:net" import * as crypto from "node:crypto" import ipc from "node-ipc" -import { z } from "zod" -import { TaskCommand, taskCommandSchema, TaskEvent, taskEventSchema } from "@benchmark/types" - -/** - * IpcMessage - */ - -export enum IpcMessageType { - Ack = "Ack", - TaskCommand = "TaskCommand", - TaskEvent = "TaskEvent", -} - -export enum IpcOrigin { - Client = "client", - Server = "server", - Relay = "relay", -} - -export const ipcMessageSchema = z.discriminatedUnion("type", [ - z.object({ - type: z.literal(IpcMessageType.Ack), - origin: z.literal(IpcOrigin.Server), - data: z.object({ clientId: z.string() }), - }), - z.object({ - type: z.literal(IpcMessageType.TaskCommand), - origin: z.literal(IpcOrigin.Client), - clientId: z.string(), - data: taskCommandSchema, - }), - z.object({ - type: z.literal(IpcMessageType.TaskEvent), - origin: z.union([z.literal(IpcOrigin.Server), z.literal(IpcOrigin.Relay)]), - relayClientId: z.string().optional(), - data: taskEventSchema, - }), -]) - -export type IpcMessage = z.infer +import { IpcOrigin, IpcMessageType, IpcMessage, ipcMessageSchema, TaskCommand, TaskEvent } from "@benchmark/types" /** * IpcClient diff --git a/benchmark/packages/runner/src/index.ts b/benchmark/packages/runner/src/index.ts index 008631030a..0197f32332 100644 --- a/benchmark/packages/runner/src/index.ts +++ b/benchmark/packages/runner/src/index.ts @@ -5,8 +5,8 @@ import * as vscode from "vscode" import { RooCodeAPI } from "../../../../src/exports/roo-code.js" -import { TaskEventName } from "@benchmark/types" -import { IpcMessageType, IpcOrigin, IpcClient } from "@benchmark/ipc" +import { IpcOrigin, IpcMessageType, TaskEventName } from "@benchmark/types" +import { IpcClient } from "@benchmark/ipc" import { findTask, findRun, createTaskMetrics, updateTask } from "@benchmark/db" import { waitUntilReady, waitUntilCompleted, sleep } from "./utils.js" diff --git a/benchmark/packages/types/src/index.ts b/benchmark/packages/types/src/index.ts index ff0b111d7f..184b5a43cf 100644 --- a/benchmark/packages/types/src/index.ts +++ b/benchmark/packages/types/src/index.ts @@ -22,10 +22,47 @@ export enum TaskEventName { TaskFinished = "TaskFinished", } -export const taskEventSchema = z.object({ - eventName: z.nativeEnum(TaskEventName), - data: z.unknown(), -}) +export const taskEventSchema = z.discriminatedUnion("eventName", [ + z.object({ + eventName: z.literal(TaskEventName.Connect), + data: z.object({ task: z.object({ id: z.number() }) }), + }), + z.object({ + eventName: z.literal(TaskEventName.TaskStarted), + data: z.object({ task: z.object({ id: z.number() }) }), + }), + z.object({ + eventName: z.literal(TaskEventName.Message), + data: z.object({ + task: z.object({ id: z.number() }), + message: z.object({ + taskId: z.string(), + action: z.enum(["created", "updated"]), + message: z.object({ + // See ClineMessage. + ts: z.number(), + type: z.enum(["ask", "say"]), + ask: z.string().optional(), + say: z.string().optional(), + partial: z.boolean().optional(), + text: z.string().optional(), + reasoning: z.string().optional(), + }), + }), + }), + }), + z.object({ + eventName: z.literal(TaskEventName.TaskTokenUsageUpdated), + data: z.object({ + task: z.object({ id: z.number() }), + usage: z.object({}), + }), + }), + z.object({ + eventName: z.literal(TaskEventName.TaskFinished), + data: z.object({ task: z.object({ id: z.number() }), taskMetrics: z.unknown() }), + }), +]) export type TaskEvent = z.infer @@ -48,3 +85,41 @@ export const taskCommandSchema = z.discriminatedUnion("commandName", [ ]) export type TaskCommand = z.infer + +/** + * IpcMessage + */ + +export enum IpcMessageType { + Ack = "Ack", + TaskCommand = "TaskCommand", + TaskEvent = "TaskEvent", +} + +export enum IpcOrigin { + Client = "client", + Server = "server", + Relay = "relay", +} + +export const ipcMessageSchema = z.discriminatedUnion("type", [ + z.object({ + type: z.literal(IpcMessageType.Ack), + origin: z.literal(IpcOrigin.Server), + data: z.object({ clientId: z.string() }), + }), + z.object({ + type: z.literal(IpcMessageType.TaskCommand), + origin: z.literal(IpcOrigin.Client), + clientId: z.string(), + data: taskCommandSchema, + }), + z.object({ + type: z.literal(IpcMessageType.TaskEvent), + origin: z.union([z.literal(IpcOrigin.Server), z.literal(IpcOrigin.Relay)]), + relayClientId: z.string().optional(), + data: taskEventSchema, + }), +]) + +export type IpcMessage = z.infer