diff --git a/benchmark/apps/cli/package.json b/benchmark/apps/cli/package.json index 1f45ceaf14..686f5b5e9d 100644 --- a/benchmark/apps/cli/package.json +++ b/benchmark/apps/cli/package.json @@ -11,7 +11,8 @@ "dependencies": { "@benchmark/db": "workspace:^", "@vscode/test-electron": "^2.4.0", - "gluegun": "^5.1.2" + "gluegun": "^5.1.2", + "p-map": "^7.0.3" }, "devDependencies": { "@benchmark/eslint-config": "workspace:^", diff --git a/benchmark/apps/cli/src/index.ts b/benchmark/apps/cli/src/index.ts index a2e0907d02..3ddc110df6 100644 --- a/benchmark/apps/cli/src/index.ts +++ b/benchmark/apps/cli/src/index.ts @@ -2,19 +2,11 @@ import * as fs from "fs" import * as path from "path" import * as os from "os" +import pMap from "p-map" import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun" import { runTests } from "@vscode/test-electron" -import { - type Language, - languages, - type Run, - findRun, - createRun, - getPendingTask, - createPendingTask, - getTask, -} from "@benchmark/db" +import { type Language, languages, type Run, findRun, createRun, getTask, createTask, Task } from "@benchmark/db" import { __dirname, extensionDevelopmentPath, extensionTestsPath, exercisesPath } from "./paths.js" import { getExercises } from "./exercises.js" @@ -41,34 +33,33 @@ const run = async (toolbox: GluegunToolbox) => { const runAll = async (id?: number) => { const run = await findOrCreateRun({ id }) - const exercises = getExercises() - for (const [language, languageExercises] of Object.entries(exercises)) { - await Promise.all( - languageExercises.map((exercise) => - findOrCreatePendingTask({ runId: run.id, language: language as Language, exercise }), - ), - ) - } + const entries = Object.entries(getExercises()).flatMap(([language, languageExercises]) => + languageExercises.map((exercise) => ({ language: language as Language, exercise })), + ) - for (const [language, languageExercises] of Object.entries(exercises)) { - for (const exercise of languageExercises) { - await runExercise({ run, language: language as Language, exercise }) - } + const tasks = await pMap( + entries, + async ({ language, exercise }) => findOrCreateTask({ runId: run.id, language, exercise }), + { concurrency: 10 }, + ) + + for (const task of tasks) { + await runExercise({ run, task }) } } const runLanguage = async ({ id, language }: { id?: number; language: Language }) => { const run = await findOrCreateRun({ id }) - const exercises = getExercises() - const languageExercises = exercises[language] - await Promise.all( - languageExercises.map((exercise) => findOrCreatePendingTask({ runId: run.id, language, exercise })), + const tasks = await pMap( + getExercises()[language], + async (exercise) => findOrCreateTask({ runId: run.id, language, exercise }), + { concurrency: 10 }, ) - for (const exercise of languageExercises) { - await runExercise({ run, language, exercise }) + for (const task of tasks) { + await runExercise({ run, task }) } } @@ -82,11 +73,12 @@ const runLanguageExercise = async ({ exercise: string }) => { const run = await findOrCreateRun({ id }) - await findOrCreatePendingTask({ runId: run.id, language, exercise }) - return runExercise({ run, language, exercise }) + const task = await findOrCreateTask({ runId: run.id, language, exercise }) + return runExercise({ run, task }) } -const runExercise = async ({ run, language, exercise }: { run: Run; language: Language; exercise: string }) => { +const runExercise = async ({ run, task }: { run: Run; task: Task }) => { + const { language, exercise } = task const workspacePath = path.resolve(exercisesPath, language, exercise) const promptPath = path.resolve(exercisesPath, `prompts/${language}.md`) @@ -94,9 +86,7 @@ const runExercise = async ({ run, language, exercise }: { run: Run; language: La throw new Error(`Prompt file does not exist: ${promptPath}`) } - const task = await getTask({ runId: run.id, language, exercise }) - - if (task) { + if (task.finishedAt) { console.log(`Test result exists for ${language} / ${exercise}, skipping`) return false } @@ -108,7 +98,7 @@ const runExercise = async ({ run, language, exercise }: { run: Run; language: La extensionTestsPath, launchArgs: [workspacePath, "--disable-extensions"], extensionTestsEnv: { - RUN_ID: run.id.toString(), + TASK_ID: task.id.toString(), LANGUAGE: language, EXERCISE: exercise, PROMPT_PATH: promptPath, @@ -157,7 +147,7 @@ const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: { socketPath: path.resolve(os.tmpdir(), `benchmark-${crypto.randomUUID()}.sock`), }) -const findOrCreatePendingTask = async ({ +const findOrCreateTask = async ({ runId, language, exercise, @@ -165,7 +155,7 @@ const findOrCreatePendingTask = async ({ runId: number language: Language exercise: string -}) => (await getPendingTask({ runId, language, exercise })) || (await createPendingTask({ runId, language, exercise })) +}) => (await getTask({ runId, language, exercise })) || (await createTask({ runId, language, exercise })) const main = async () => { const cli = build() diff --git a/benchmark/apps/web/src/app/home.tsx b/benchmark/apps/web/src/app/home.tsx index 314c9f7788..2562fa1e01 100644 --- a/benchmark/apps/web/src/app/home.tsx +++ b/benchmark/apps/web/src/app/home.tsx @@ -3,16 +3,17 @@ import { useRouter } from "next/navigation" import { Rocket } from "lucide-react" -import { getRuns } from "@benchmark/db" +import type { Run, TaskMetrics } from "@benchmark/db" import { formatCurrency, formatDuration } from "@/lib" import { Button, Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui" +import { useMemo } from "react" -type Run = Awaited>[number] - -export function Home({ runs }: { runs: Run[] }) { +export function Home({ runs }: { runs: (Run & { taskMetrics: TaskMetrics | null })[] }) { const router = useRouter() + const visibleRuns = useMemo(() => runs.filter((run) => run.taskMetrics !== null), [runs]) + return ( <> @@ -29,18 +30,30 @@ export function Home({ runs }: { runs: Run[] }) { - {runs.map((run) => ( - - {run.id} - {run.model} - {new Date(run.createdAt).toLocaleString()} - {run.passed} - {run.failed} - {(run.rate * 100).toFixed(1)}% - {formatCurrency(run.cost)} - {formatDuration(run.duration)} + {visibleRuns.length ? ( + visibleRuns.map(({ taskMetrics, ...run }) => ( + + {run.id} + {run.model} + {new Date(run.createdAt).toLocaleString()} + {run.passed} + {run.failed} + {((run.passed / (run.passed + run.failed)) * 100).toFixed(1)}% + {formatCurrency(taskMetrics!.cost)} + {formatDuration(taskMetrics!.duration)} + + )) + ) : ( + + + No benchmark runs yet. + + one now. + - ))} + )}
diff --git a/benchmark/apps/web/src/components/ui/button.tsx b/benchmark/apps/web/src/components/ui/button.tsx index d5a84b741f..3a0277d12d 100644 --- a/benchmark/apps/web/src/components/ui/button.tsx +++ b/benchmark/apps/web/src/components/ui/button.tsx @@ -16,7 +16,7 @@ const buttonVariants = cva( "border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input", secondary: "bg-secondary text-secondary-foreground shadow-xs", ghost: "hover:bg-accent hover:text-accent-foreground", - link: "text-primary underline-offset-4 hover:underline", + link: "text-accent underline-offset-4 hover:underline px-1.5!", }, size: { default: "h-9 px-4 py-2 has-[>svg]:px-3", diff --git a/benchmark/apps/web/src/hooks/use-event-source.ts b/benchmark/apps/web/src/hooks/use-event-source.ts index 51382a5483..d076e68a5a 100644 --- a/benchmark/apps/web/src/hooks/use-event-source.ts +++ b/benchmark/apps/web/src/hooks/use-event-source.ts @@ -1,6 +1,6 @@ import { useCallback, useEffect, useRef, useState } from "react" -export type EventSourceStatus = "init" | "open" | "error" +export type EventSourceStatus = "waiting" | "connected" | "error" export type EventSourceEvent = Event & { data: string } @@ -12,16 +12,16 @@ type UseEventSourceOptions = { export function useEventSource({ url, withCredentials, onMessage }: UseEventSourceOptions) { const sourceRef = useRef(null) - const statusRef = useRef("init") - const [status, setStatus] = useState("init") + const statusRef = useRef("waiting") + const [status, setStatus] = useState("waiting") const handleMessage = useCallback((event: MessageEvent) => onMessage(event), [onMessage]) const createEventSource = useCallback(() => { sourceRef.current = new EventSource(url, { withCredentials }) sourceRef.current.onopen = () => { - statusRef.current = "open" - setStatus("open") + statusRef.current = "connected" + setStatus("connected") } sourceRef.current.onmessage = (event) => { @@ -40,7 +40,7 @@ export function useEventSource({ url, withCredentials, onMessage }: UseEventSour createEventSource() setTimeout(() => { - if (statusRef.current === "init") { + if (statusRef.current === "waiting") { sourceRef.current?.close() sourceRef.current = null createEventSource() diff --git a/benchmark/packages/db/drizzle/0000_legal_cassandra_nova.sql b/benchmark/packages/db/drizzle/0000_elite_raza.sql similarity index 59% rename from benchmark/packages/db/drizzle/0000_legal_cassandra_nova.sql rename to benchmark/packages/db/drizzle/0000_elite_raza.sql index 76411c8ee0..4af2be3564 100644 --- a/benchmark/packages/db/drizzle/0000_legal_cassandra_nova.sql +++ b/benchmark/packages/db/drizzle/0000_elite_raza.sql @@ -1,27 +1,18 @@ -CREATE TABLE `pendingTasks` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `runId` integer NOT NULL, - `language` text NOT NULL, - `exercise` text NOT NULL, - `pid` integer, - `createdAt` integer NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `pendingTasks_language_exercise_idx` ON `pendingTasks` (`runId`,`language`,`exercise`);--> statement-breakpoint CREATE TABLE `runs` ( `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `taskMetricsId` integer, `model` text NOT NULL, `description` text, `pid` integer, `socketPath` text NOT NULL, - `createdAt` integer NOT NULL + `passed` integer DEFAULT 0 NOT NULL, + `failed` integer DEFAULT 0 NOT NULL, + `createdAt` integer NOT NULL, + FOREIGN KEY (`taskMetricsId`) REFERENCES `taskMetrics`(`id`) ON UPDATE no action ON DELETE no action ); --> statement-breakpoint -CREATE TABLE `tasks` ( +CREATE TABLE `taskMetrics` ( `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `runId` integer NOT NULL, - `language` text NOT NULL, - `exercise` text NOT NULL, `tokensIn` integer NOT NULL, `tokensOut` integer NOT NULL, `tokensContext` integer NOT NULL, @@ -29,8 +20,21 @@ CREATE TABLE `tasks` ( `cacheReads` integer NOT NULL, `cost` real NOT NULL, `duration` integer NOT NULL, - `passed` integer, `createdAt` integer NOT NULL ); --> statement-breakpoint +CREATE TABLE `tasks` ( + `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, + `runId` integer NOT NULL, + `taskMetricsId` integer, + `language` text NOT NULL, + `exercise` text NOT NULL, + `passed` integer, + `startedAt` integer, + `finishedAt` integer, + `createdAt` integer NOT NULL, + FOREIGN KEY (`runId`) REFERENCES `runs`(`id`) ON UPDATE no action ON DELETE no action, + FOREIGN KEY (`taskMetricsId`) REFERENCES `taskMetrics`(`id`) ON UPDATE no action ON DELETE no action +); +--> statement-breakpoint CREATE UNIQUE INDEX `tasks_language_exercise_idx` ON `tasks` (`runId`,`language`,`exercise`); \ No newline at end of file diff --git a/benchmark/packages/db/drizzle/meta/0000_snapshot.json b/benchmark/packages/db/drizzle/meta/0000_snapshot.json index 212257208d..1b8c44283a 100644 --- a/benchmark/packages/db/drizzle/meta/0000_snapshot.json +++ b/benchmark/packages/db/drizzle/meta/0000_snapshot.json @@ -1,67 +1,9 @@ { "version": "6", "dialect": "sqlite", - "id": "d7bfd73f-dbf5-430e-ab81-f436029d06e5", + "id": "c0fa8491-b5c0-493d-aa32-ddf280259c30", "prevId": "00000000-0000-0000-0000-000000000000", "tables": { - "pendingTasks": { - "name": "pendingTasks", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "runId": { - "name": "runId", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "language": { - "name": "language", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "exercise": { - "name": "exercise", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "pid": { - "name": "pid", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "createdAt": { - "name": "createdAt", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "pendingTasks_language_exercise_idx": { - "name": "pendingTasks_language_exercise_idx", - "columns": ["runId", "language", "exercise"], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, "runs": { "name": "runs", "columns": { @@ -72,6 +14,13 @@ "notNull": true, "autoincrement": true }, + "taskMetricsId": { + "name": "taskMetricsId", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, "model": { "name": "model", "type": "text", @@ -100,6 +49,22 @@ "notNull": true, "autoincrement": false }, + "passed": { + "name": "passed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "failed": { + "name": "failed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, "createdAt": { "name": "createdAt", "type": "integer", @@ -109,13 +74,23 @@ } }, "indexes": {}, - "foreignKeys": {}, + "foreignKeys": { + "runs_taskMetricsId_taskMetrics_id_fk": { + "name": "runs_taskMetricsId_taskMetrics_id_fk", + "tableFrom": "runs", + "tableTo": "taskMetrics", + "columnsFrom": ["taskMetricsId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, "compositePrimaryKeys": {}, "uniqueConstraints": {}, "checkConstraints": {} }, - "tasks": { - "name": "tasks", + "taskMetrics": { + "name": "taskMetrics", "columns": { "id": { "name": "id", @@ -124,27 +99,6 @@ "notNull": true, "autoincrement": true }, - "runId": { - "name": "runId", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "language": { - "name": "language", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "exercise": { - "name": "exercise", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, "tokensIn": { "name": "tokensIn", "type": "integer", @@ -194,6 +148,58 @@ "notNull": true, "autoincrement": false }, + "createdAt": { + "name": "createdAt", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "tasks": { + "name": "tasks", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": true + }, + "runId": { + "name": "runId", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "taskMetricsId": { + "name": "taskMetricsId", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "language": { + "name": "language", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "exercise": { + "name": "exercise", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, "passed": { "name": "passed", "type": "integer", @@ -201,6 +207,20 @@ "notNull": false, "autoincrement": false }, + "startedAt": { + "name": "startedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finishedAt": { + "name": "finishedAt", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, "createdAt": { "name": "createdAt", "type": "integer", @@ -216,7 +236,26 @@ "isUnique": true } }, - "foreignKeys": {}, + "foreignKeys": { + "tasks_runId_runs_id_fk": { + "name": "tasks_runId_runs_id_fk", + "tableFrom": "tasks", + "tableTo": "runs", + "columnsFrom": ["runId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + }, + "tasks_taskMetricsId_taskMetrics_id_fk": { + "name": "tasks_taskMetricsId_taskMetrics_id_fk", + "tableFrom": "tasks", + "tableTo": "taskMetrics", + "columnsFrom": ["taskMetricsId"], + "columnsTo": ["id"], + "onDelete": "no action", + "onUpdate": "no action" + } + }, "compositePrimaryKeys": {}, "uniqueConstraints": {}, "checkConstraints": {} diff --git a/benchmark/packages/db/drizzle/meta/_journal.json b/benchmark/packages/db/drizzle/meta/_journal.json index 91db75ffed..bc133625a4 100644 --- a/benchmark/packages/db/drizzle/meta/_journal.json +++ b/benchmark/packages/db/drizzle/meta/_journal.json @@ -5,8 +5,8 @@ { "idx": 0, "version": "6", - "when": 1742568357359, - "tag": "0000_legal_cassandra_nova", + "when": 1742599919625, + "tag": "0000_elite_raza", "breakpoints": true } ] diff --git a/benchmark/packages/db/src/index.ts b/benchmark/packages/db/src/index.ts index 6200d48676..18e27f0214 100644 --- a/benchmark/packages/db/src/index.ts +++ b/benchmark/packages/db/src/index.ts @@ -1,6 +1,4 @@ -// export { db } from "./db.js" export { type Language, languages } from "./enums.js" -// export { schema } from "./schema.js" /** * runs @@ -17,8 +15,8 @@ export type { Task, InsertTask, UpdateTask } from "./schema.js" export * from "./queries/tasks.js" /** - * pendingTasks + * taskMetrics */ -export type { PendingTask, InsertPendingTask, UpdatePendingTask } from "./schema.js" -export * from "./queries/pendingTasks.js" +export type { TaskMetrics, InsertTaskMetrics, UpdateTaskMetrics } from "./schema.js" +export * from "./queries/taskMetrics.js" diff --git a/benchmark/packages/db/src/queries/pendingTasks.ts b/benchmark/packages/db/src/queries/pendingTasks.ts deleted file mode 100644 index 75b1a761fd..0000000000 --- a/benchmark/packages/db/src/queries/pendingTasks.ts +++ /dev/null @@ -1,62 +0,0 @@ -import { and, eq } from "drizzle-orm" - -import type { Language } from "../enums.js" -import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js" -import type { InsertPendingTask, UpdatePendingTask } from "../schema.js" -import { insertPendingTaskSchema, pendingTasks } from "../schema.js" -import { db } from "../db.js" - -const table = pendingTasks - -export const findPendingTask = async (id: number) => { - const run = await db.query.pendingTasks.findFirst({ where: eq(table.id, id) }) - - if (!run) { - throw new RecordNotFoundError() - } - - return run -} - -export const createPendingTask = async (args: InsertPendingTask) => { - const records = await db - .insert(table) - .values({ - ...insertPendingTaskSchema.parse(args), - createdAt: new Date(), - }) - .returning() - - const record = records[0] - - if (!record) { - throw new RecordNotCreatedError() - } - - return record -} - -export const updatePendingTask = async (id: number, values: UpdatePendingTask) => { - const records = await db.update(table).set(values).where(eq(table.id, id)).returning() - const record = records[0] - - if (!record) { - throw new RecordNotFoundError() - } - - return record -} - -type GetPendingTask = { - runId: number - language: Language - exercise: string -} - -export const getPendingTask = async ({ runId, language, exercise }: GetPendingTask) => - db.query.pendingTasks.findFirst({ - where: and(eq(table.runId, runId), eq(table.language, language), eq(table.exercise, exercise)), - }) - -export const getPendingTasks = async (runId: number) => - db.query.pendingTasks.findMany({ where: eq(table.runId, runId) }) diff --git a/benchmark/packages/db/src/queries/runs.ts b/benchmark/packages/db/src/queries/runs.ts index 07c5723716..20c8027ea5 100644 --- a/benchmark/packages/db/src/queries/runs.ts +++ b/benchmark/packages/db/src/queries/runs.ts @@ -1,11 +1,11 @@ -import { desc, eq, sql } from "drizzle-orm" +import { desc, eq } from "drizzle-orm" import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js" import type { InsertRun, UpdateRun } from "../schema.js" -import { insertRunSchema, runs, tasks } from "../schema.js" +import { insertRunSchema, schema } from "../schema.js" import { db } from "../db.js" -const table = runs +const table = schema.runs export const findRun = async (id: number) => { const run = await db.query.runs.findFirst({ where: eq(table.id, id) }) @@ -46,20 +46,4 @@ export const updateRun = async (id: number, values: UpdateRun) => { return record } -export const getRuns = () => - db - .select({ - id: table.id, - model: table.model, - description: table.description, - createdAt: table.createdAt, - passed: sql`sum(${tasks.passed})`, - failed: sql`sum(${tasks.passed} = 0)`, - total: sql`count(${tasks.id})`, - rate: sql`sum(${tasks.passed}) * 1.0 / count(${tasks.id})`, - cost: sql`sum(${tasks.cost})`, - duration: sql`sum(${tasks.duration})`, - }) - .from(table) - .leftJoin(tasks, eq(table.id, tasks.runId)) - .orderBy(desc(table.id)) +export const getRuns = async () => db.query.runs.findMany({ orderBy: desc(table.id), with: { taskMetrics: true } }) diff --git a/benchmark/packages/db/src/queries/taskMetrics.ts b/benchmark/packages/db/src/queries/taskMetrics.ts new file mode 100644 index 0000000000..75c13d5db9 --- /dev/null +++ b/benchmark/packages/db/src/queries/taskMetrics.ts @@ -0,0 +1,36 @@ +import { eq } from "drizzle-orm" + +import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js" +import type { InsertTaskMetrics } from "../schema.js" +import { insertTaskMetricsSchema, taskMetrics } from "../schema.js" +import { db } from "../db.js" + +const table = taskMetrics + +export const findTaskMetrics = async (id: number) => { + const run = await db.query.taskMetrics.findFirst({ where: eq(table.id, id) }) + + if (!run) { + throw new RecordNotFoundError() + } + + return run +} + +export const createTaskMetrics = async (args: InsertTaskMetrics) => { + const records = await db + .insert(table) + .values({ + ...insertTaskMetricsSchema.parse(args), + createdAt: new Date(), + }) + .returning() + + const record = records[0] + + if (!record) { + throw new RecordNotCreatedError() + } + + return record +} diff --git a/benchmark/packages/db/src/schema.ts b/benchmark/packages/db/src/schema.ts index b93021beaa..f48add48ab 100644 --- a/benchmark/packages/db/src/schema.ts +++ b/benchmark/packages/db/src/schema.ts @@ -1,5 +1,5 @@ -import { sqliteTable, text, real, integer } from "drizzle-orm/sqlite-core" -import * as t from "drizzle-orm/sqlite-core" +import { sqliteTable, text, real, integer, uniqueIndex } from "drizzle-orm/sqlite-core" +import { relations } from "drizzle-orm" import { createInsertSchema } from "drizzle-zod" import { z } from "zod" @@ -11,19 +11,23 @@ import { languages } from "./enums.js" export const runs = sqliteTable("runs", { id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }), + taskMetricsId: integer({ mode: "number" }).references(() => taskMetrics.id), model: text().notNull(), description: text(), pid: integer({ mode: "number" }), socketPath: text().notNull(), + passed: integer({ mode: "number" }).default(0).notNull(), + failed: integer({ mode: "number" }).default(0).notNull(), createdAt: integer({ mode: "timestamp" }).notNull(), }) +export const runsRelations = relations(runs, ({ one }) => ({ + taskMetrics: one(taskMetrics, { fields: [runs.taskMetricsId], references: [taskMetrics.id] }), +})) + export type Run = typeof runs.$inferSelect -export const insertRunSchema = createInsertSchema(runs).omit({ - id: true, - createdAt: true, -}) +export const insertRunSchema = createInsertSchema(runs).omit({ id: true, createdAt: true }) export type InsertRun = z.infer @@ -37,63 +41,59 @@ export const tasks = sqliteTable( "tasks", { id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }), - runId: integer({ mode: "number" }).notNull(), + runId: integer({ mode: "number" }) + .references(() => runs.id) + .notNull(), + taskMetricsId: integer({ mode: "number" }).references(() => taskMetrics.id), language: text({ enum: languages }).notNull(), exercise: text().notNull(), - tokensIn: integer({ mode: "number" }).notNull(), - tokensOut: integer({ mode: "number" }).notNull(), - tokensContext: integer({ mode: "number" }).notNull(), - cacheWrites: integer({ mode: "number" }).notNull(), - cacheReads: integer({ mode: "number" }).notNull(), - cost: real().notNull(), - duration: integer({ mode: "number" }).notNull(), passed: integer({ mode: "boolean" }), + startedAt: integer({ mode: "timestamp" }), + finishedAt: integer({ mode: "timestamp" }), createdAt: integer({ mode: "timestamp" }).notNull(), }, - (table) => [t.uniqueIndex("tasks_language_exercise_idx").on(table.runId, table.language, table.exercise)], + (table) => [uniqueIndex("tasks_language_exercise_idx").on(table.runId, table.language, table.exercise)], ) +export const tasksRelations = relations(tasks, ({ one }) => ({ + run: one(runs, { fields: [tasks.runId], references: [runs.id] }), + taskMetrics: one(taskMetrics, { fields: [tasks.taskMetricsId], references: [taskMetrics.id] }), +})) + export type Task = typeof tasks.$inferSelect -export const insertTaskSchema = createInsertSchema(tasks).omit({ - id: true, - createdAt: true, -}) +export const insertTaskSchema = createInsertSchema(tasks).omit({ id: true, createdAt: true }) export type InsertTask = z.infer export type UpdateTask = Partial> /** - * pendingTasks + * taskMetrics */ -export const pendingTasks = sqliteTable( - "pendingTasks", - { - id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }), - runId: integer({ mode: "number" }).notNull(), - language: text({ enum: languages }).notNull(), - exercise: text().notNull(), - pid: integer({ mode: "number" }), - createdAt: integer({ mode: "timestamp" }).notNull(), - }, - (table) => [t.uniqueIndex("pendingTasks_language_exercise_idx").on(table.runId, table.language, table.exercise)], -) - -export type PendingTask = typeof pendingTasks.$inferSelect - -export const insertPendingTaskSchema = createInsertSchema(pendingTasks).omit({ - id: true, - createdAt: true, +export const taskMetrics = sqliteTable("taskMetrics", { + id: integer({ mode: "number" }).primaryKey({ autoIncrement: true }), + tokensIn: integer({ mode: "number" }).notNull(), + tokensOut: integer({ mode: "number" }).notNull(), + tokensContext: integer({ mode: "number" }).notNull(), + cacheWrites: integer({ mode: "number" }).notNull(), + cacheReads: integer({ mode: "number" }).notNull(), + cost: real().notNull(), + duration: integer({ mode: "number" }).notNull(), + createdAt: integer({ mode: "timestamp" }).notNull(), }) -export type InsertPendingTask = z.infer +export type TaskMetrics = typeof taskMetrics.$inferSelect -export type UpdatePendingTask = Partial> +export const insertTaskMetricsSchema = createInsertSchema(taskMetrics).omit({ id: true, createdAt: true }) + +export type InsertTaskMetrics = z.infer + +export type UpdateTaskMetrics = Partial> /** * schema */ -export const schema = { runs, tasks, pendingTasks } +export const schema = { runs, runsRelations, tasks, tasksRelations, taskMetrics } diff --git a/benchmark/packages/runner/src/index.ts b/benchmark/packages/runner/src/index.ts index 5b0359f75f..dc72d320ba 100644 --- a/benchmark/packages/runner/src/index.ts +++ b/benchmark/packages/runner/src/index.ts @@ -6,7 +6,7 @@ import * as vscode from "vscode" import { RooCodeAPI } from "../../../../src/exports/roo-code.js" import { IpcServer, ServerMessageType } from "@benchmark/ipc" -import { Language, findRun, createTask } from "@benchmark/db" +import { Language, findTask, findRun, createTaskMetrics, updateTask } from "@benchmark/db" import { waitUntilReady, waitUntilCompleted, sleep } from "./utils.js" @@ -15,7 +15,7 @@ export async function run() { * Validate environment variables. */ - const runId = process.env.RUN_ID ? parseInt(process.env.RUN_ID) : undefined + const tid = process.env.TASK_ID ? parseInt(process.env.TASK_ID) : undefined const language = process.env.LANGUAGE as Language const exercise = process.env.EXERCISE const promptPath = process.env.PROMPT_PATH @@ -23,13 +23,20 @@ export async function run() { const openRouterApiKey = process.env.OPENROUTER_API_KEY const openRouterModelId = process.env.OPENROUTER_MODEL_ID - if (!runId || !language || !exercise || !promptPath || !workspacePath || !openRouterApiKey || !openRouterModelId) { + if (!tid || !language || !exercise || !promptPath || !workspacePath || !openRouterApiKey || !openRouterModelId) { throw new Error("ENV not configured.") } const prompt = await fs.readFile(promptPath, "utf-8") - const run = await findRun(runId) + /** + * Fetch and update the task. + */ + + let task = await findTask(tid) + task = await updateTask(task.id, { startedAt: new Date() }) + + const run = await findRun(task.runId) /** * Activate the extension. @@ -87,29 +94,19 @@ export async function run() { server.listen() server.on("client", (id) => { - server.send(id, { - type: ServerMessageType.Data, - data: { - event: "client", - runId, - language, - exercise, - prompt, - workspacePath, - }, - }) + server.send(id, { type: ServerMessageType.Data, data: { event: "client", task } }) }) - api.on("taskStarted", (taskId) => { - server.broadcast({ type: ServerMessageType.Data, data: { event: "taskStarted", taskId } }) + api.on("taskStarted", () => { + server.broadcast({ type: ServerMessageType.Data, data: { event: "taskStarted", task } }) }) - api.on("message", ({ taskId, action, message }) => { - server.broadcast({ type: ServerMessageType.Data, data: { event: "message", taskId, action, message } }) + api.on("message", (message) => { + server.broadcast({ type: ServerMessageType.Data, data: { event: "message", task, message } }) }) - api.on("taskTokenUsageUpdated", (taskId, usage) => { - server.broadcast({ type: ServerMessageType.Data, data: { event: "taskTokenUsageUpdated", taskId, usage } }) + api.on("taskTokenUsageUpdated", (_, usage) => { + server.broadcast({ type: ServerMessageType.Data, data: { event: "taskTokenUsageUpdated", task, usage } }) }) /** @@ -117,20 +114,19 @@ export async function run() { */ const startTime = Date.now() - const taskId = await api.startNewTask(prompt) + const rooTaskId = await api.startNewTask(prompt) let usage try { - usage = (await waitUntilCompleted({ api, taskId, timeout: 5 * 60 * 1_000 })) || api.getTokenUsage(taskId) + usage = + (await waitUntilCompleted({ api, taskId: rooTaskId, timeout: 5 * 60 * 1_000 })) || + api.getTokenUsage(rooTaskId) } catch (e: unknown) { - usage = api.getTokenUsage(taskId) + usage = api.getTokenUsage(rooTaskId) console.error(e) } - const task = await createTask({ - runId, - language, - exercise, + const taskMetrics = await createTaskMetrics({ duration: Date.now() - startTime, tokensIn: usage.totalTokensIn, tokensOut: usage.totalTokensOut, @@ -138,8 +134,11 @@ export async function run() { cacheWrites: usage.totalCacheWrites ?? 0, cacheReads: usage.totalCacheReads ?? 0, cost: usage.totalCost, - passed: false, }) - await fs.writeFile(path.resolve(workspacePath, "usage.json"), JSON.stringify(task, null, 2)) + task = await updateTask(task.id, { taskMetricsId: taskMetrics.id, finishedAt: new Date() }) + + server.broadcast({ type: ServerMessageType.Data, data: { event: "taskFinished", task, taskMetrics } }) + + await fs.writeFile(path.resolve(workspacePath, "usage.json"), JSON.stringify({ ...task, ...taskMetrics }, null, 2)) } diff --git a/benchmark/pnpm-lock.yaml b/benchmark/pnpm-lock.yaml index deda92dc4d..c2fbce2f3e 100644 --- a/benchmark/pnpm-lock.yaml +++ b/benchmark/pnpm-lock.yaml @@ -53,6 +53,9 @@ importers: gluegun: specifier: ^5.1.2 version: 5.2.0 + p-map: + specifier: ^7.0.3 + version: 7.0.3 devDependencies: '@benchmark/eslint-config': specifier: workspace:^ @@ -2707,6 +2710,7 @@ packages: libsql@0.4.7: resolution: {integrity: sha512-T9eIRCs6b0J1SHKYIvD8+KCJMcWZ900iZyxdnSCdqxN12Z1ijzT+jY5nrk72Jw4B0HGzms2NgpryArlJqvc3Lw==} + cpu: [x64, arm64, wasm32] os: [darwin, linux, win32] lie@3.3.0: @@ -3045,6 +3049,10 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + p-map@7.0.3: + resolution: {integrity: sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==} + engines: {node: '>=18'} + package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} @@ -6476,6 +6484,8 @@ snapshots: dependencies: p-limit: 3.1.0 + p-map@7.0.3: {} + package-json-from-dist@1.0.1: {} pako@1.0.11: {} diff --git a/benchmark/turbo.json b/benchmark/turbo.json index bb13159c45..aac7916a4f 100644 --- a/benchmark/turbo.json +++ b/benchmark/turbo.json @@ -4,7 +4,7 @@ "globalEnv": [ "NODE_ENV", "NEXT_RUNTIME", - "RUN_ID", + "TASK_ID", "LANGUAGE", "EXERCISE", "OPENROUTER_API_KEY",