Move IPC server to the cli

benchmark/apps/cli/src/index.ts

@@ -7,9 +7,9 @@ import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun"
 import { runTests } from "@vscode/test-electron"
 import { execa, parseCommandString } from "execa"
 
-import { type Language, languages, TaskEventName } from "@benchmark/types"
+import { type Language, languages, IpcOrigin, IpcMessageType, TaskEventName } from "@benchmark/types"
 import { type Run, findRun, createRun, finishRun, createTask, Task, getTasks, updateTask } from "@benchmark/db"
-import { IpcServer, IpcMessageType, IpcOrigin } from "@benchmark/ipc"
+import { IpcServer } from "@benchmark/ipc"
 
 import { __dirname, extensionDevelopmentPath, extensionTestsPath, exercisesPath } from "./paths.js"
 import { getExercises } from "./exercises.js"
@@ -67,21 +67,29 @@ const run = async (toolbox: GluegunToolbox) => {
 	}
 
 	const tasks = await getTasks(run.id)
+	let currentTask = tasks[0]
 
-	if (tasks.length === 0) {
+	if (!currentTask) {
 		throw new Error("No tasks found.")
 	}
 
 	const server = new IpcServer(run.socketPath, () => {})
 	server.listen()
 
-	let currentTask = tasks[0]
-
 	server.on("connect", (clientId) => {
 		server.send(clientId, {
 			type: IpcMessageType.TaskEvent,
 			origin: IpcOrigin.Server,
-			data: { eventName: TaskEventName.Connect, data: { task: currentTask } },
+			data: { eventName: TaskEventName.Connect, data: { task: currentTask! } },
+		})
+	})
+
+	server.on("taskEvent", (relayClientId, data) => {
+		server.broadcast({
+			type: IpcMessageType.TaskEvent,
+			origin: IpcOrigin.Server,
+			relayClientId,
+			data,
 		})
 	})
 

benchmark/apps/web/src/app/runs/new/new-run.tsx

@@ -84,6 +84,33 @@ export function NewRun() {
 		[router],
 	)
 
+	const onFilter = useCallback(
+		(value: string, search: string) => {
+			if (modelSearchValueRef.current !== search) {
+				modelSearchValueRef.current = search
+				modelSearchResultsRef.current.clear()
+
+				for (const {
+					obj: { id },
+					score,
+				} of fuzzysort.go(search, models.data || [], {
+					key: "name",
+				})) {
+					modelSearchResultsRef.current.set(id, score)
+				}
+			}
+
+			return modelSearchResultsRef.current.get(value) ?? 0
+		},
+		[models.data],
+	)
+
+	const recommendedModels = [
+		"anthropic/claude-3.7-sonnet",
+		"anthropic/claude-3.7-sonnet:thinking",
+		"google/gemini-2.0-flash-001",
+	]
+
 	return (
 		<>
 			<FormProvider {...form}>
@@ -108,24 +135,7 @@ export function NewRun() {
 										</Button>
 									</PopoverTrigger>
 									<PopoverContent className="p-0 w-[var(--radix-popover-trigger-width)]">
-										<Command
-											filter={(value, search) => {
-												if (modelSearchValueRef.current !== search) {
-													modelSearchValueRef.current = search
-													modelSearchResultsRef.current.clear()
-
-													for (const {
-														obj: { id },
-														score,
-													} of fuzzysort.go(search, models.data || [], {
-														key: "name",
-													})) {
-														modelSearchResultsRef.current.set(id, score)
-													}
-												}
-
-												return modelSearchResultsRef.current.get(value) ?? 0
-											}}>
+										<Command filter={onFilter}>
 											<CommandInput
 												placeholder="Search"
 												value={modelSearchValue}
@@ -152,6 +162,22 @@ export function NewRun() {
 									</PopoverContent>
 								</Popover>
 								<FormMessage />
+								<div className="flex flex-wrap items-center gap-2 text-sm">
+									<div>Recommended:</div>
+									{recommendedModels.map((modelId) => (
+										<div key={modelId} className="flex items-center gap-2">
+											<Button
+												variant="link"
+												className="break-all px-0!"
+												onClick={(e) => {
+													e.preventDefault()
+													setValue("model", modelId)
+												}}>
+												{modelId}
+											</Button>
+										</div>
+									))}
+								</div>
 							</FormItem>
 						)}
 					/>

benchmark/apps/web/src/components/ui/button.tsx

@@ -16,7 +16,7 @@ const buttonVariants = cva(
 					"border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input",
 				secondary: "bg-secondary text-secondary-foreground shadow-xs",
 				ghost: "hover:bg-accent hover:text-accent-foreground",
-				link: "text-accent underline-offset-4 hover:underline px-1.5!",
+				link: "text-accent underline-offset-4 hover:underline h-4! px-1! rounded-none",
 				input: "bg-input text-input-foreground active:scale-100 shadow-xs",
 			},
 			size: {

benchmark/apps/web/src/hooks/use-run-status.ts

@@ -1,14 +1,13 @@
 import { useState, useCallback, useRef } from "react"
 import { useQuery, keepPreviousData } from "@tanstack/react-query"
 
+import { TaskEventName, taskEventSchema } from "@benchmark/types"
 import { Run } from "@benchmark/db"
 
 import { getTasks } from "@/lib/server/tasks"
-import { ipcServerMessageSchema } from "@/lib/schemas"
 import { useEventSource } from "@/hooks/use-event-source"
 
 export const useRunStatus = (run: Run) => {
-	const [clientId, setClientId] = useState<string>()
 	const [runningTaskId, setRunningTaskId] = useState<number>()
 	const outputRef = useRef<Map<number, string[]>>(new Map())
 	const [outputCounts, setOutputCounts] = useState<Record<number, number>>({})
@@ -32,7 +31,7 @@ export const useRunStatus = (run: Run) => {
 			return
 		}
 
-		const result = ipcServerMessageSchema.safeParse(data)
+		const result = taskEventSchema.safeParse(data)
 
 		if (!result.success) {
 			console.log(`unrecognized messageEvent.data: ${messageEvent.data}`)
@@ -40,32 +39,36 @@ export const useRunStatus = (run: Run) => {
 		}
 
 		const payload = result.data
+		const taskId = payload.data.task.id
 
-		if (payload.type === "Ack") {
-			setClientId(payload.data.clientId as string)
-		} else if (payload.type === "TaskEvent") {
-			const taskId = payload.data.data.task.id
-
-			if (payload.data.eventName === "connect" || payload.data.eventName === "taskStarted") {
+		switch (payload.eventName) {
+			case TaskEventName.Connect:
+			case TaskEventName.TaskStarted:
 				setRunningTaskId(taskId)
-			} else if (payload.data.eventName === "taskFinished") {
+				break
+			case TaskEventName.TaskFinished:
 				setRunningTaskId(undefined)
-			} else if (payload.data.eventName === "message") {
-				const { text } = payload.data.data.message.message
-				console.log(`message: ${taskId} ->`, text)
-				outputRef.current.set(taskId, [...(outputRef.current.get(taskId) || []), text])
-				const outputCounts: Record<number, number> = {}
+				break
+			case TaskEventName.Message: {
+				const text = payload.data.message.message.text
 
-				for (const [taskId, messages] of outputRef.current.entries()) {
-					outputCounts[taskId] = messages.length
+				if (text) {
+					outputRef.current.set(taskId, [...(outputRef.current.get(taskId) || []), text])
+					const outputCounts: Record<number, number> = {}
+
+					for (const [taskId, messages] of outputRef.current.entries()) {
+						outputCounts[taskId] = messages.length
+					}
+
+					setOutputCounts(outputCounts)
 				}
 
-				setOutputCounts(outputCounts)
+				break
 			}
 		}
 	}, [])
 
 	const status = useEventSource({ url, onMessage })
 
-	return { tasks, status, clientId, runningTaskId, output: outputRef.current, outputCounts }
+	return { tasks, status, runningTaskId, output: outputRef.current, outputCounts }
 }

benchmark/apps/web/src/lib/schemas.ts

@@ -17,62 +17,3 @@ export const createRunSchema = z
 	})
 
 export type CreateRun = z.infer<typeof createRunSchema>
-
-/**
- * TaskEvent
- */
-
-export const taskEventSchema = z.discriminatedUnion("eventName", [
-	z.object({
-		eventName: z.literal("connect"),
-		data: z.object({ task: z.object({ id: z.number() }) }),
-	}),
-	z.object({
-		eventName: z.literal("taskStarted"),
-		data: z.object({ task: z.object({ id: z.number() }) }),
-	}),
-	z.object({
-		eventName: z.literal("message"),
-		data: z.object({
-			task: z.object({ id: z.number() }),
-			message: z.object({
-				taskId: z.string(),
-				action: z.enum(["created", "updated"]),
-				message: z.object({
-					ask: z.string().optional(),
-					say: z.string().optional(),
-					partial: z.boolean(),
-					text: z.string(),
-				}),
-			}),
-		}),
-	}),
-	z.object({
-		eventName: z.literal("taskTokenUsageUpdated"),
-		data: z.object({
-			task: z.object({ id: z.number() }),
-			usage: z.object({}),
-		}),
-	}),
-	z.object({
-		eventName: z.literal("taskFinished"),
-		data: z.object({ task: z.object({ id: z.number() }) }),
-	}),
-])
-
-/**
- * IpcServerMessage
- */
-
-export const ipcServerMessageSchema = z.discriminatedUnion("type", [
-	z.object({
-		type: z.literal("Ack"),
-		data: z.object({ clientId: z.string() }),
-	}),
-	z.object({
-		type: z.literal("TaskEvent"),
-		data: taskEventSchema,
-	}),
-])
-
-export type IpcServerMessage = z.infer<typeof ipcServerMessageSchema>

benchmark/packages/ipc/src/index.ts

@@ -3,47 +3,8 @@ import { Socket } from "node:net"
 import * as crypto from "node:crypto"
 
 import ipc from "node-ipc"
-import { z } from "zod"
 
-import { TaskCommand, taskCommandSchema, TaskEvent, taskEventSchema } from "@benchmark/types"
-
-/**
- * IpcMessage
- */
-
-export enum IpcMessageType {
-	Ack = "Ack",
-	TaskCommand = "TaskCommand",
-	TaskEvent = "TaskEvent",
-}
-
-export enum IpcOrigin {
-	Client = "client",
-	Server = "server",
-	Relay = "relay",
-}
-
-export const ipcMessageSchema = z.discriminatedUnion("type", [
-	z.object({
-		type: z.literal(IpcMessageType.Ack),
-		origin: z.literal(IpcOrigin.Server),
-		data: z.object({ clientId: z.string() }),
-	}),
-	z.object({
-		type: z.literal(IpcMessageType.TaskCommand),
-		origin: z.literal(IpcOrigin.Client),
-		clientId: z.string(),
-		data: taskCommandSchema,
-	}),
-	z.object({
-		type: z.literal(IpcMessageType.TaskEvent),
-		origin: z.union([z.literal(IpcOrigin.Server), z.literal(IpcOrigin.Relay)]),
-		relayClientId: z.string().optional(),
-		data: taskEventSchema,
-	}),
-])
-
-export type IpcMessage = z.infer<typeof ipcMessageSchema>
+import { IpcOrigin, IpcMessageType, IpcMessage, ipcMessageSchema, TaskCommand, TaskEvent } from "@benchmark/types"
 
 /**
  * IpcClient

benchmark/packages/runner/src/index.ts

@@ -5,8 +5,8 @@ import * as vscode from "vscode"
 
 import { RooCodeAPI } from "../../../../src/exports/roo-code.js"
 
-import { TaskEventName } from "@benchmark/types"
-import { IpcMessageType, IpcOrigin, IpcClient } from "@benchmark/ipc"
+import { IpcOrigin, IpcMessageType, TaskEventName } from "@benchmark/types"
+import { IpcClient } from "@benchmark/ipc"
 import { findTask, findRun, createTaskMetrics, updateTask } from "@benchmark/db"
 
 import { waitUntilReady, waitUntilCompleted, sleep } from "./utils.js"

benchmark/packages/types/src/index.ts

@@ -22,10 +22,47 @@ export enum TaskEventName {
 	TaskFinished = "TaskFinished",
 }
 
-export const taskEventSchema = z.object({
-	eventName: z.nativeEnum(TaskEventName),
-	data: z.unknown(),
-})
+export const taskEventSchema = z.discriminatedUnion("eventName", [
+	z.object({
+		eventName: z.literal(TaskEventName.Connect),
+		data: z.object({ task: z.object({ id: z.number() }) }),
+	}),
+	z.object({
+		eventName: z.literal(TaskEventName.TaskStarted),
+		data: z.object({ task: z.object({ id: z.number() }) }),
+	}),
+	z.object({
+		eventName: z.literal(TaskEventName.Message),
+		data: z.object({
+			task: z.object({ id: z.number() }),
+			message: z.object({
+				taskId: z.string(),
+				action: z.enum(["created", "updated"]),
+				message: z.object({
+					// See ClineMessage.
+					ts: z.number(),
+					type: z.enum(["ask", "say"]),
+					ask: z.string().optional(),
+					say: z.string().optional(),
+					partial: z.boolean().optional(),
+					text: z.string().optional(),
+					reasoning: z.string().optional(),
+				}),
+			}),
+		}),
+	}),
+	z.object({
+		eventName: z.literal(TaskEventName.TaskTokenUsageUpdated),
+		data: z.object({
+			task: z.object({ id: z.number() }),
+			usage: z.object({}),
+		}),
+	}),
+	z.object({
+		eventName: z.literal(TaskEventName.TaskFinished),
+		data: z.object({ task: z.object({ id: z.number() }), taskMetrics: z.unknown() }),
+	}),
+])
 
 export type TaskEvent = z.infer<typeof taskEventSchema>
 
@@ -48,3 +85,41 @@ export const taskCommandSchema = z.discriminatedUnion("commandName", [
 ])
 
 export type TaskCommand = z.infer<typeof taskCommandSchema>
+
+/**
+ * IpcMessage
+ */
+
+export enum IpcMessageType {
+	Ack = "Ack",
+	TaskCommand = "TaskCommand",
+	TaskEvent = "TaskEvent",
+}
+
+export enum IpcOrigin {
+	Client = "client",
+	Server = "server",
+	Relay = "relay",
+}
+
+export const ipcMessageSchema = z.discriminatedUnion("type", [
+	z.object({
+		type: z.literal(IpcMessageType.Ack),
+		origin: z.literal(IpcOrigin.Server),
+		data: z.object({ clientId: z.string() }),
+	}),
+	z.object({
+		type: z.literal(IpcMessageType.TaskCommand),
+		origin: z.literal(IpcOrigin.Client),
+		clientId: z.string(),
+		data: taskCommandSchema,
+	}),
+	z.object({
+		type: z.literal(IpcMessageType.TaskEvent),
+		origin: z.union([z.literal(IpcOrigin.Server), z.literal(IpcOrigin.Relay)]),
+		relayClientId: z.string().optional(),
+		data: taskEventSchema,
+	}),
+])
+
+export type IpcMessage = z.infer<typeof ipcMessageSchema>