diff --git a/benchmark/apps/cli/src/index.ts b/benchmark/apps/cli/src/index.ts index 8410c43277..29e6307d0d 100644 --- a/benchmark/apps/cli/src/index.ts +++ b/benchmark/apps/cli/src/index.ts @@ -21,11 +21,12 @@ import { findRun, createRun, finishRun, + type Task, createTask, - Task, getTasks, updateTask, createTaskMetrics, + updateTaskMetrics, } from "@benchmark/db" import { inChunksOf } from "@benchmark/lib" import { IpcServer, IpcClient } from "@benchmark/ipc" @@ -62,7 +63,7 @@ const run = async (toolbox: GluegunToolbox) => { run = await findRun(id) } else { run = await createRun({ - model: "anthropic/claude-3.7-sonnet", + model: rooCodeDefaults.openRouterModelId!, pid: process.pid, socketPath: path.resolve(os.tmpdir(), `benchmark-${crypto.randomUUID()}.sock`), }) @@ -103,7 +104,7 @@ const run = async (toolbox: GluegunToolbox) => { }) }) - const chunks = inChunksOf(tasks, 2) + const chunks = inChunksOf(tasks, 3) for (const chunk of chunks) { await Promise.all( @@ -144,7 +145,7 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server while (++tries < 5) { try { - await pWaitFor(() => client.isConnected, { interval: 100, timeout: 2_000 }) + await pWaitFor(() => client.isReady, { interval: 100, timeout: 2_000 }) break } catch (error) { console.error(error) @@ -153,6 +154,12 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server } } + if (!client.isReady) { + client.disconnect() + console.log(`[cli#runExercise | ${language} / ${exercise}] unable to connect`) + return false + } + let isTaskFinished = false let isTaskAborted = false @@ -168,6 +175,7 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server ] let taskStartedAt = Date.now() + let taskMetricsId: number | undefined client.on(IpcMessageType.TaskEvent, async (taskEvent) => { const { eventName, payload } = taskEvent @@ -193,16 +201,33 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server if (eventName === RooCodeEventName.TaskStarted) { taskStartedAt = Date.now() - await updateTask(task.id, { startedAt: new Date() }) + + const taskMetrics = await createTaskMetrics({ + cost: 0, + tokensIn: 0, + tokensOut: 0, + tokensContext: 0, + duration: 0, + cacheWrites: 0, + cacheReads: 0, + }) + + await updateTask(task.id, { taskMetricsId: taskMetrics.id, startedAt: new Date() }) + + taskStartedAt = Date.now() + taskMetricsId = taskMetrics.id } - if (eventName === RooCodeEventName.TaskCompleted) { + if ( + (eventName === RooCodeEventName.TaskTokenUsageUpdated || eventName === RooCodeEventName.TaskCompleted) && + taskMetricsId + ) { const duration = Date.now() - taskStartedAt const { totalCost, totalTokensIn, totalTokensOut, contextTokens, totalCacheWrites, totalCacheReads } = payload[1] - const taskMetrics = await createTaskMetrics({ + await updateTaskMetrics(taskMetricsId, { cost: totalCost, tokensIn: totalTokensIn, tokensOut: totalTokensOut, @@ -211,8 +236,10 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server cacheWrites: totalCacheWrites ?? 0, cacheReads: totalCacheReads ?? 0, }) + } - await updateTask(task.id, { taskMetricsId: taskMetrics.id, finishedAt: new Date() }) + if (eventName === RooCodeEventName.TaskCompleted) { + await updateTask(task.id, { finishedAt: new Date() }) isTaskFinished = true } @@ -239,12 +266,13 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server }, }) - console.log(`[cli#runExercise | ${language} / ${exercise}] StartNewTask`) + console.log(`[cli#runExercise | ${language} / ${exercise}] starting task`) try { - await pWaitFor(() => isTaskFinished || isTaskAborted, { interval: 1_000, timeout: 300 * 1_000 }) + await pWaitFor(() => isTaskFinished || isTaskAborted, { interval: 1_000, timeout: 1 * 60 * 1_000 }) + // eslint-disable-next-line @typescript-eslint/no-unused-vars } catch (error) { - console.error(error) + console.log(`[cli#runExercise | ${language} / ${exercise}] time limit reached`) } try { @@ -255,8 +283,6 @@ const runExercise = async ({ run, task, server }: { run: Run; task: Task; server data: "workbench.action.closeWindow", }) - console.log(`[cli#runExercise | ${language} / ${exercise}] VSCodeCommand (workbench.action.closeWindow)`) - client.disconnect() } catch (error) { console.error(error) diff --git a/benchmark/packages/db/src/queries/taskMetrics.ts b/benchmark/packages/db/src/queries/taskMetrics.ts index 75c13d5db9..135285ce9d 100644 --- a/benchmark/packages/db/src/queries/taskMetrics.ts +++ b/benchmark/packages/db/src/queries/taskMetrics.ts @@ -1,7 +1,7 @@ import { eq } from "drizzle-orm" import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js" -import type { InsertTaskMetrics } from "../schema.js" +import type { InsertTaskMetrics, UpdateTaskMetrics } from "../schema.js" import { insertTaskMetricsSchema, taskMetrics } from "../schema.js" import { db } from "../db.js" @@ -34,3 +34,14 @@ export const createTaskMetrics = async (args: InsertTaskMetrics) => { return record } + +export const updateTaskMetrics = async (id: number, values: UpdateTaskMetrics) => { + const records = await db.update(table).set(values).where(eq(table.id, id)).returning() + const record = records[0] + + if (!record) { + throw new RecordNotFoundError() + } + + return record +} diff --git a/benchmark/packages/db/src/queries/tasks.ts b/benchmark/packages/db/src/queries/tasks.ts index caa386675a..95ca6179ba 100644 --- a/benchmark/packages/db/src/queries/tasks.ts +++ b/benchmark/packages/db/src/queries/tasks.ts @@ -47,6 +47,7 @@ export const updateTask = async (id: number, values: UpdateTask) => { return record } + type GetTask = { runId: number language: ExerciseLanguage