More progress

benchmark/.env.sample

@@ -0,0 +1,2 @@
+BENCHMARKS_DB_PATH=file:/tmp/benchmarks.db
+OPENROUTER_API_KEY=sk-or-v1-...

benchmark/.gitignore

@@ -7,10 +7,8 @@ node_modules
 
 # Local env files
 .env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
+.env.*
+!.env.sample
 
 # Testing
 coverage
@@ -36,3 +34,7 @@ yarn-error.log*
 # Misc
 .DS_Store
 *.pem
+
+# Roo-Code-Benchmark
+exercises
+

benchmark/package.json

@@ -5,7 +5,8 @@
 		"lint": "turbo lint",
 		"check-types": "turbo check-types",
 		"format": "turbo format",
-		"dev": "pnpm --filter @benchmark/server dev"
+		"cli": "pnpm --filter @benchmark/cli dev",
+		"web": "pnpm --filter @benchmark/web dev"
 	},
 	"devDependencies": {
 		"@dotenvx/dotenvx": "^1.39.0",

benchmark/packages/cli/package.json

@@ -2,6 +2,7 @@
 	"name": "@benchmark/cli",
 	"version": "0.1.0",
 	"private": true,
+	"type": "module",
 	"packageManager": "pnpm@10.6.5+sha512.cdf928fca20832cd59ec53826492b7dc25dc524d4370b6b4adbf65803d32efaa6c1c88147c0ae4e8d579a6c9eec715757b50d4fa35eea179d868eada4ed043af",
 	"scripts": {
 		"lint": "eslint src --ext ts --max-warnings=0",
@@ -11,10 +12,13 @@
 		"dev": "dotenvx run -f ../../.env -- tsx src/index.ts"
 	},
 	"dependencies": {
+		"@benchmark/db": "workspace:^",
 		"@vscode/test-electron": "^2.4.0",
-		"gluegun": "^5.1.2"
+		"gluegun": "^5.1.2",
+		"p-map": "^7.0.3"
 	},
 	"devDependencies": {
-		"@benchmark/eslint-config": "workspace:^"
+		"@benchmark/eslint-config": "workspace:^",
+		"@benchmark/typescript-config": "workspace:^"
 	}
 }

benchmark/packages/cli/src/index.ts

@@ -1,25 +1,45 @@
 import * as fs from "fs"
 import * as path from "path"
+import { fileURLToPath } from "url"
+import pMap from "p-map"
 
-import { build, filesystem, GluegunPrompt } from "gluegun"
+import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun"
 import { runTests } from "@vscode/test-electron"
 
-// console.log(__dirname)
-// <...>/Roo-Code/benchmark/src
+import { type Language, languages, type Run, findRun, createRun, getTask } from "@benchmark/db"
 
-const extensionDevelopmentPath = path.resolve(__dirname, "../../")
-const extensionTestsPath = path.resolve(__dirname, "../out/run.js")
-const promptsPath = path.resolve(__dirname, "../prompts")
-const exercisesPath = path.resolve(__dirname, "../../../exercises")
-const languages = ["cpp", "go", "java", "javascript", "python", "rust"]
+const __dirname = path.dirname(fileURLToPath(import.meta.url))
+const extensionDevelopmentPath = path.resolve(__dirname, "../../../..")
+const extensionTestsPath = path.resolve(extensionDevelopmentPath, "benchmark/packages/runner/dist/run.js")
+const exercisesPath = path.resolve(extensionDevelopmentPath, "benchmark/exercises")
 
-async function runAll({ runId, model }: { runId: number; model: string }) {
-	for (const language of languages) {
-		await runLanguage({ runId, model, language })
+export const isLanguage = (language: string): language is Language => languages.includes(language as Language)
+
+const run = async (toolbox: GluegunToolbox) => {
+	const { config, prompt } = toolbox
+	const id = config.runId ? Number(config.runId) : undefined
+	let { language, exercise } = config
+
+	if (language === "all") {
+		const run = await findOrCreateRun({ id })
+		await runAll(run)
+	} else if (exercise === "all") {
+		const run = await findOrCreateRun({ id })
+		await runLanguage({ run, language })
+	} else {
+		language = language || (await askLanguage(prompt))
+		exercise = exercise || (await askExercise(prompt, language))
+		const run = await findOrCreateRun({ id })
+		await runExercise({ run, language, exercise })
 	}
 }
 
-async function runLanguage({ runId, model, language }: { runId: number; model: string; language: string }) {
+const runAll = async (run: Run) =>
+	(await pMap(languages, (language) => runLanguage({ run, language }), { concurrency: 1 })).flatMap(
+		(language) => language,
+	)
+
+const runLanguage = async ({ run, language }: { run: Run; language: Language }) => {
 	const languagePath = path.resolve(exercisesPath, language)
 
 	if (!fs.existsSync(languagePath)) {
@@ -32,65 +52,64 @@ async function runLanguage({ runId, model, language }: { runId: number; model: s
 		.map((exercise) => path.basename(exercise))
 		.filter((exercise) => !exercise.startsWith("."))
 
-	for (const exercise of exercises) {
-		await runExercise({ runId, model, language, exercise })
-	}
+	const results = await pMap(
+		exercises,
+		async (exercise) => ({
+			language,
+			exercise,
+			result: await runExercise({ run, language, exercise }),
+		}),
+		{ concurrency: 1 },
+	)
+
+	return results
 }
 
-async function runExercise({
-	runId,
-	model,
-	language,
-	exercise,
-}: {
-	runId: number
-	model: string
-	language: string
-	exercise: string
-}) {
+const runExercise = async ({ run, language, exercise }: { run: Run; language: Language; exercise: string }) => {
 	const workspacePath = path.resolve(exercisesPath, language, exercise)
-	const promptPath = path.resolve(promptsPath, `${language}.md`)
+	const promptPath = path.resolve(exercisesPath, `prompts/${language}.md`)
 
-	const extensionTestsEnv = {
-		PROMPT_PATH: promptPath,
-		WORKSPACE_PATH: workspacePath,
-		OPENROUTER_MODEL_ID: model,
-		RUN_ID: runId.toString(),
+	if (!fs.existsSync(promptPath)) {
+		throw new Error(`Prompt file does not exist: ${promptPath}`)
 	}
 
-	if (fs.existsSync(path.resolve(workspacePath, "usage.json"))) {
+	const task = await getTask({ runId: run.id, language, exercise })
+
+	if (task) {
 		console.log(`Test result exists for ${language} / ${exercise}, skipping`)
-		return
+		return false
 	}
 
 	console.log(`Running ${language} / ${exercise}`)
+	return true
 
 	await runTests({
 		extensionDevelopmentPath,
 		extensionTestsPath,
 		launchArgs: [workspacePath, "--disable-extensions"],
-		extensionTestsEnv,
+		extensionTestsEnv: {
+			PROMPT_PATH: promptPath,
+			WORKSPACE_PATH: workspacePath,
+			OPENROUTER_MODEL_ID: run.model,
+			RUN_ID: run.id.toString(),
+		},
 	})
+
+	return true
 }
 
-async function askLanguage(prompt: GluegunPrompt) {
-	const languages = filesystem.subdirectories(exercisesPath)
-
-	if (languages.length === 0) {
-		throw new Error(`No languages found in ${exercisesPath}`)
-	}
-
-	const { language } = await prompt.ask<{ language: string }>({
+const askLanguage = async (prompt: GluegunPrompt) => {
+	const { language } = await prompt.ask<{ language: Language }>({
 		type: "select",
 		name: "language",
 		message: "Which language?",
-		choices: languages.map((language) => path.basename(language)).filter((language) => !language.startsWith(".")),
+		choices: [...languages],
 	})
 
 	return language
 }
 
-async function askExercise(prompt: GluegunPrompt, language: string) {
+const askExercise = async (prompt: GluegunPrompt, language: Language) => {
 	const exercises = filesystem.subdirectories(path.join(exercisesPath, language))
 
 	if (exercises.length === 0) {
@@ -107,30 +126,20 @@ async function askExercise(prompt: GluegunPrompt, language: string) {
 	return exercise
 }
 
-async function createRun({ model }: { model: string }): Promise<{ id: number; model: string }> {
-	const response = await fetch("http://localhost:3000/api/runs", {
-		method: "POST",
-		body: JSON.stringify({ model }),
-	})
+type FindOrCreateRun = { id?: number; model?: string }
 
-	if (!response.ok) {
-		throw new Error(`Failed to create run: ${response.statusText}`)
-	}
+const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: FindOrCreateRun) =>
+	id ? findRun(id) : createRun({ model })
 
-	const {
-		run: [run],
-	} = await response.json()
-	return run
-}
-
-async function main() {
+const main = async () => {
 	const cli = build()
-		.brand("benchmark-cli")
+		.brand("cli")
 		.src(__dirname)
 		.help()
 		.version()
 		.command({
 			name: "run",
+			description: "Run a benchmark",
 			run: ({ config, parameters }) => {
 				config.language = parameters.first
 				config.exercise = parameters.second
@@ -140,32 +149,41 @@ async function main() {
 				}
 			},
 		})
-		.defaultCommand() // Use the default command if no args.
+		.defaultCommand()
 		.create()
 
-	const { print, prompt, config } = await cli.run(process.argv)
+	const toolbox = await cli.run(process.argv)
+	const { print, command } = toolbox
 
 	try {
-		const model = "anthropic/claude-3.7-sonnet"
-		const runId = config.runId ? Number(config.runId) : (await createRun({ model })).id
-
-		if (config.language === "all") {
-			console.log("Running all exercises for all languages")
-			await runAll({ runId, model })
-		} else if (config.exercise === "all") {
-			console.log(`Running all exercises for ${config.language}`)
-			await runLanguage({ runId, model, language: config.language })
-		} else {
-			const language = config.language || (await askLanguage(prompt))
-			const exercise = config.exercise || (await askExercise(prompt, language))
-			await runExercise({ runId, model, language, exercise })
+		switch (command?.name) {
+			case "run":
+				await run(toolbox)
+				break
 		}
 
 		process.exit(0)
-	} catch (error) {
-		print.error(error)
+	} catch (error: unknown) {
+		print.error(error instanceof Error ? error.message : String(error))
 		process.exit(1)
 	}
 }
 
+if (!fs.existsSync(extensionDevelopmentPath)) {
+	console.error(`"extensionDevelopmentPath" does not exist.`)
+	process.exit(1)
+}
+
+if (!fs.existsSync(extensionTestsPath)) {
+	console.error(`"extensionTestsPath" does not exist. Please run "pnpm --filter @benchmark/runner build".`)
+	process.exit(1)
+}
+
+if (!fs.existsSync(exercisesPath)) {
+	console.error(
+		`Exercises path does not exist. Please run "git clone https://github.com/cte/Roo-Code-Benchmark.git exercises".`,
+	)
+	process.exit(1)
+}
+
 main()

benchmark/packages/cli/tsconfig.json

@@ -1,22 +1,5 @@
 {
-	"$schema": "https://json.schemastore.org/tsconfig",
-	"display": "Default",
-	"compilerOptions": {
-		"composite": false,
-		"declaration": true,
-		"declarationMap": true,
-		"esModuleInterop": true,
-		"forceConsistentCasingInFileNames": true,
-		"inlineSources": false,
-		"isolatedModules": true,
-		"noUnusedLocals": false,
-		"noUnusedParameters": false,
-		"preserveWatchOutput": true,
-		"skipLibCheck": true,
-		"strict": true,
-		"module": "ESNext",
-		"moduleResolution": "Node",
-		"target": "es2020"
-	},
+	"extends": "@benchmark/typescript-config/base.json",
+	"include": ["src"],
 	"exclude": ["node_modules"]
 }

benchmark/packages/db/package.json

@@ -2,6 +2,7 @@
 	"name": "@benchmark/db",
 	"version": "0.1.0",
 	"private": true,
+	"type": "module",
 	"packageManager": "pnpm@10.6.5",
 	"exports": {
 		".": "./src/index.ts"
@@ -21,6 +22,7 @@
 	},
 	"devDependencies": {
 		"@benchmark/eslint-config": "workspace:^",
+		"@benchmark/typescript-config": "workspace:^",
 		"drizzle-kit": "^0.30.5"
 	}
 }

benchmark/packages/db/src/db.ts

@@ -1,5 +1,8 @@
 import { drizzle } from "drizzle-orm/libsql"
 
-export * from "./schema"
+import { schema } from "./schema"
 
-export const db = drizzle({ connection: { url: process.env.BENCHMARKS_DB_PATH! } })
+export const db = drizzle({
+	schema,
+	connection: { url: process.env.BENCHMARKS_DB_PATH! },
+})

benchmark/packages/db/src/enums.ts

@@ -0,0 +1,7 @@
+/**
+ * languages
+ */
+
+export const languages = ["cpp", "go", "java", "javascript", "python", "rust"] as const
+
+export type Language = (typeof languages)[number]

benchmark/packages/db/src/index.ts

@@ -1,7 +1,15 @@
 export { db } from "./db"
+export { type Language, languages } from "./enums"
+export { schema } from "./schema"
 
-export { type Language, languages } from "./schema"
-export { type Run, insertRunSchema, runs } from "./schema"
-export { type Task, insertTaskSchema, tasks } from "./schema"
+/**
+ * runs
+ */
+export { type Run, insertRunSchema } from "./schema"
+export { findRun, createRun, getRuns } from "./queries/runs"
 
-export { getRuns } from "./queries"
+/**
+ * tasks
+ */
+export { type Task, insertTaskSchema } from "./schema"
+export { findTask, createTask, getTask } from "./queries/tasks"

benchmark/packages/db/src/queries/errors.ts

@@ -0,0 +1 @@
+export class RecordNotFoundError extends Error {}

benchmark/packages/db/src/queries/runs.ts

@@ -1,6 +1,31 @@
 import { desc, eq, sql } from "drizzle-orm"
 
-import { db, runs, tasks } from "./db"
+import { db } from "../db"
+import { InsertRun, insertRunSchema, runs, tasks } from "../schema"
+
+import { RecordNotFoundError } from "./errors"
+
+export const findRun = async (id: number) => {
+	const run = await db.query.runs.findFirst({ where: eq(runs.id, id) })
+
+	if (!run) {
+		throw new RecordNotFoundError()
+	}
+
+	return run
+}
+
+export const createRun = async (args: InsertRun) => {
+	const result = await db
+		.insert(runs)
+		.values({
+			...insertRunSchema.parse(args),
+			createdAt: new Date(),
+		})
+		.returning()
+
+	return result[0]
+}
 
 export const getRuns = () =>
 	db

benchmark/packages/db/src/queries/tasks.ts

@@ -0,0 +1,34 @@
+import { and, eq } from "drizzle-orm"
+
+import { db } from "../db"
+import { InsertTask, insertTaskSchema, tasks } from "../schema"
+
+import { RecordNotFoundError } from "./errors"
+import { Language } from "../enums"
+
+export const findTask = async (id: number) => {
+	const run = await db.query.tasks.findFirst({ where: eq(tasks.id, id) })
+
+	if (!run) {
+		throw new RecordNotFoundError()
+	}
+
+	return run
+}
+
+export const createTask = async (args: InsertTask) => {
+	const result = await db
+		.insert(tasks)
+		.values({
+			...insertTaskSchema.parse(args),
+			createdAt: new Date(),
+		})
+		.returning()
+
+	return result[0]
+}
+
+export const getTask = async ({ runId, language, exercise }: { runId: number; language: Language; exercise: string }) =>
+	db.query.tasks.findFirst({
+		where: and(eq(tasks.runId, runId), eq(tasks.language, language), eq(tasks.exercise, exercise)),
+	})

benchmark/packages/db/src/schema.ts

@@ -1,14 +1,9 @@
 import { sqliteTable, text, real, integer } from "drizzle-orm/sqlite-core"
 import * as t from "drizzle-orm/sqlite-core"
 import { createInsertSchema } from "drizzle-zod"
+import { z } from "zod"
 
-/**
- * languages
- */
-
-export const languages = ["cpp", "go", "java", "javascript", "python", "rust"] as const
-
-export type Language = (typeof languages)[number]
+import { languages } from "./enums"
 
 /**
  * runs
@@ -28,6 +23,8 @@ export const insertRunSchema = createInsertSchema(runs).omit({
 	createdAt: true,
 })
 
+export type InsertRun = z.infer<typeof insertRunSchema>
+
 /**
  * tasks
  */
@@ -58,3 +55,11 @@ export const insertTaskSchema = createInsertSchema(tasks).omit({
 	id: true,
 	createdAt: true,
 })
+
+export type InsertTask = z.infer<typeof insertTaskSchema>
+
+/**
+ * schema
+ */
+
+export const schema = { runs, tasks }

benchmark/packages/db/tsconfig.json

@@ -1,23 +1,5 @@
 {
-	"$schema": "https://json.schemastore.org/tsconfig",
-	"display": "Default",
-	"compilerOptions": {
-		"composite": false,
-		"declaration": true,
-		"declarationMap": true,
-		"esModuleInterop": true,
-		"forceConsistentCasingInFileNames": true,
-		"inlineSources": false,
-		"isolatedModules": true,
-		"noUnusedLocals": false,
-		"noUnusedParameters": false,
-		"preserveWatchOutput": true,
-		"skipLibCheck": true,
-		"strict": true,
-		"module": "ESNext",
-		"moduleResolution": "Node",
-		"target": "es2020",
-		"types": ["node"]
-	},
+	"extends": "@benchmark/typescript-config/base.json",
+	"include": ["src"],
 	"exclude": ["node_modules"]
 }

benchmark/packages/ipc/package.json

@@ -2,6 +2,7 @@
 	"name": "@benchmark/ipc",
 	"version": "0.1.0",
 	"private": true,
+	"type": "module",
 	"packageManager": "pnpm@10.6.5",
 	"scripts": {
 		"lint": "eslint src --ext ts --max-warnings=0",
@@ -16,6 +17,7 @@
 	},
 	"devDependencies": {
 		"@benchmark/eslint-config": "workspace:^",
+		"@benchmark/typescript-config": "workspace:^",
 		"@types/node-ipc": "^9.2.3"
 	}
 }

benchmark/packages/ipc/scripts/server.ts

@@ -4,6 +4,7 @@ async function main() {
 	try {
 		const server = new IpcServer()
 		server.listen()
+		console.log(`listening @ ${server.socketPath}`)
 
 		while (server.isListening) {
 			await new Promise((resolve) => setTimeout(resolve, 1000))

benchmark/packages/ipc/tsconfig.json

@@ -1,22 +1,5 @@
 {
-	"$schema": "https://json.schemastore.org/tsconfig",
-	"display": "Default",
-	"compilerOptions": {
-		"composite": false,
-		"declaration": true,
-		"declarationMap": true,
-		"esModuleInterop": true,
-		"forceConsistentCasingInFileNames": true,
-		"inlineSources": false,
-		"isolatedModules": true,
-		"noUnusedLocals": false,
-		"noUnusedParameters": false,
-		"preserveWatchOutput": true,
-		"skipLibCheck": true,
-		"strict": true,
-		"module": "ESNext",
-		"moduleResolution": "Node",
-		"target": "es2020"
-	},
+	"extends": "@benchmark/typescript-config/base.json",
+	"include": ["src"],
 	"exclude": ["node_modules"]
 }

benchmark/packages/runner/package.json

@@ -1,5 +1,5 @@
 {
-	"name": "@benchmark/client",
+	"name": "@benchmark/runner",
 	"version": "0.1.0",
 	"private": true,
 	"packageManager": "pnpm@10.6.5",
@@ -7,9 +7,8 @@
 		"lint": "eslint src --ext ts --max-warnings=0",
 		"check-types": "tsc --noEmit",
 		"format": "prettier --write .",
-		"compile": "pnpm clean && tsc",
-		"build": "pnpm compile && cd ../../.. && npm run compile && npm run build:webview",
-		"clean": "rimraf out"
+		"build": "rimraf dist && tsc",
+		"vscode-test": "pnpm build && cd ../../.. && npm run vscode-test"
 	},
 	"devDependencies": {
 		"@benchmark/eslint-config": "workspace:^",

benchmark/packages/runner/tsconfig.json

@@ -9,7 +9,7 @@
 		"strict": true,
 		"skipLibCheck": true,
 		"useUnknownInCatchVariables": false,
-		"outDir": "out"
+		"outDir": "dist"
 	},
-	"include": ["src", "../../../../src/exports/roo-code.d.ts"]
+	"include": ["src"]
 }

benchmark/packages/server/.gitignore

@@ -1,45 +0,0 @@
-# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
-
-# dependencies
-/node_modules
-/.pnp
-.pnp.*
-.yarn/*
-!.yarn/patches
-!.yarn/plugins
-!.yarn/releases
-!.yarn/versions
-
-# testing
-/coverage
-
-# next.js
-/.next/
-/out/
-
-# production
-/build
-
-# misc
-.DS_Store
-*.pem
-
-# debug
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-.pnpm-debug.log*
-
-# env files (can opt-in for committing if needed)
-.env*
-!.env.sample
-
-# vercel
-.vercel
-
-# typescript
-*.tsbuildinfo
-next-env.d.ts
-
-# sqlite
-local.db

benchmark/packages/server/tsconfig.json

@@ -1,35 +0,0 @@
-{
-	"compilerOptions": {
-		"target": "ES2017",
-		"lib": ["dom", "dom.iterable", "esnext"],
-		"allowJs": true,
-		"skipLibCheck": true,
-		"strict": true,
-		"noEmit": true,
-		"esModuleInterop": true,
-		"module": "esnext",
-		"moduleResolution": "bundler",
-		"resolveJsonModule": true,
-		"isolatedModules": true,
-		"jsx": "preserve",
-		"incremental": true,
-		"plugins": [
-			{
-				"name": "next"
-			}
-		],
-		"paths": {
-			"@/*": ["./src/*"]
-		}
-	},
-	"include": [
-		"next-env.d.ts",
-		"**/*.ts",
-		"**/*.tsx",
-		".next/types/**/*.ts",
-		"../db/src/index.ts",
-		"../db/src/queries.ts",
-		"../db/src/schema.ts"
-	],
-	"exclude": ["node_modules"]
-}

benchmark/packages/typescript-config/base.json

@@ -0,0 +1,19 @@
+{
+	"$schema": "https://json.schemastore.org/tsconfig",
+	"compilerOptions": {
+		"declaration": true,
+		"declarationMap": true,
+		"esModuleInterop": true,
+		"incremental": false,
+		"isolatedModules": true,
+		"lib": ["es2022", "DOM", "DOM.Iterable"],
+		"module": "NodeNext",
+		"moduleDetection": "force",
+		"moduleResolution": "NodeNext",
+		"noUncheckedIndexedAccess": true,
+		"resolveJsonModule": true,
+		"skipLibCheck": true,
+		"strict": true,
+		"target": "ES2022"
+	}
+}

benchmark/packages/typescript-config/nextjs.json

@@ -0,0 +1,12 @@
+{
+	"$schema": "https://json.schemastore.org/tsconfig",
+	"extends": "./base.json",
+	"compilerOptions": {
+		"plugins": [{ "name": "next" }],
+		"module": "ESNext",
+		"moduleResolution": "bundler",
+		"allowJs": true,
+		"jsx": "preserve",
+		"noEmit": true
+	}
+}

benchmark/packages/typescript-config/package.json

@@ -0,0 +1,7 @@
+{
+	"name": "@benchmark/typescript-config",
+	"private": true,
+	"publishConfig": {
+		"access": "public"
+	}
+}

benchmark/packages/web/next-env.d.ts

@@ -0,0 +1,5 @@
+/// <reference types="next" />
+/// <reference types="next/image-types/global" />
+
+// NOTE: This file should not be edited
+// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

benchmark/packages/web/package.json

@@ -1,5 +1,5 @@
 {
-	"name": "@benchmark/server",
+	"name": "@benchmark/web",
 	"version": "0.1.0",
 	"private": true,
 	"packageManager": "pnpm@10.6.5",
@@ -27,6 +27,7 @@
 	},
 	"devDependencies": {
 		"@benchmark/eslint-config": "workspace:^",
+		"@benchmark/typescript-config": "workspace:^",
 		"@tailwindcss/postcss": "^4",
 		"@types/react": "^19",
 		"@types/react-dom": "^19",

benchmark/packages/web/src/app/api/runs/route.ts

@@ -1,17 +1,10 @@
 import { NextResponse } from "next/server"
 
-import { db, runs, insertRunSchema } from "@benchmark/db"
+import { createRun } from "@benchmark/db"
 
 export async function POST(request: Request) {
 	try {
-		const run = await db
-			.insert(runs)
-			.values({
-				...insertRunSchema.parse(await request.json()),
-				createdAt: new Date(),
-			})
-			.returning()
-
+		const run = await createRun(await request.json())
 		return NextResponse.json({ run }, { status: 201 })
 	} catch (error) {
 		return NextResponse.json({ error: (error as Error).message }, { status: 500 })

benchmark/packages/web/src/app/api/tasks/route.ts

@@ -1,17 +1,10 @@
 import { NextResponse } from "next/server"
 
-import { db, tasks, insertTaskSchema } from "@benchmark/db"
+import { createTask } from "@benchmark/db"
 
 export async function POST(request: Request) {
 	try {
-		const task = await db
-			.insert(tasks)
-			.values({
-				...insertTaskSchema.parse(await request.json()),
-				createdAt: new Date(),
-			})
-			.returning()
-
+		const task = await createTask(await request.json())
 		return NextResponse.json({ task }, { status: 201 })
 	} catch (error) {
 		return NextResponse.json({ error: (error as Error).message }, { status: 500 })

benchmark/packages/web/tsconfig.json

@@ -0,0 +1,9 @@
+{
+	"extends": "@benchmark/typescript-config/nextjs.json",
+	"compilerOptions": {
+		"plugins": [{ "name": "next" }],
+		"paths": { "@/*": ["./src/*"] }
+	},
+	"include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+	"exclude": ["node_modules"]
+}

benchmark/pnpm-lock.yaml

@@ -41,25 +41,25 @@ importers:
 
   packages/cli:
     dependencies:
+      '@benchmark/db':
+        specifier: workspace:^
+        version: link:../db
       '@vscode/test-electron':
         specifier: ^2.4.0
         version: 2.4.1
       gluegun:
         specifier: ^5.1.2
         version: 5.2.0
+      p-map:
+        specifier: ^7.0.3
+        version: 7.0.3
     devDependencies:
       '@benchmark/eslint-config':
         specifier: workspace:^
         version: link:../eslint-config
-
-  packages/client:
-    devDependencies:
-      '@benchmark/eslint-config':
+      '@benchmark/typescript-config':
         specifier: workspace:^
-        version: link:../eslint-config
-      '@types/vscode':
-        specifier: ^1.98.0
-        version: 1.98.0
+        version: link:../typescript-config
 
   packages/db:
     dependencies:
@@ -82,6 +82,9 @@ importers:
       '@benchmark/eslint-config':
         specifier: workspace:^
         version: link:../eslint-config
+      '@benchmark/typescript-config':
+        specifier: workspace:^
+        version: link:../typescript-config
       drizzle-kit:
         specifier: ^0.30.5
         version: 0.30.5
@@ -136,11 +139,25 @@ importers:
       '@benchmark/eslint-config':
         specifier: workspace:^
         version: link:../eslint-config
+      '@benchmark/typescript-config':
+        specifier: workspace:^
+        version: link:../typescript-config
       '@types/node-ipc':
         specifier: ^9.2.3
         version: 9.2.3
 
-  packages/server:
+  packages/runner:
+    devDependencies:
+      '@benchmark/eslint-config':
+        specifier: workspace:^
+        version: link:../eslint-config
+      '@types/vscode':
+        specifier: ^1.98.0
+        version: 1.98.0
+
+  packages/typescript-config: {}
+
+  packages/web:
     dependencies:
       '@benchmark/db':
         specifier: workspace:^
@@ -182,6 +199,9 @@ importers:
       '@benchmark/eslint-config':
         specifier: workspace:^
         version: link:../eslint-config
+      '@benchmark/typescript-config':
+        specifier: workspace:^
+        version: link:../typescript-config
       '@tailwindcss/postcss':
         specifier: ^4
         version: 4.0.14
@@ -2512,6 +2532,10 @@ packages:
     resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
     engines: {node: '>=10'}
 
+  p-map@7.0.3:
+    resolution: {integrity: sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==}
+    engines: {node: '>=18'}
+
   package-json-from-dist@1.0.1:
     resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==}
 
@@ -5317,6 +5341,8 @@ snapshots:
     dependencies:
       p-limit: 3.1.0
 
+  p-map@7.0.3: {}
+
   package-json-from-dist@1.0.1: {}
 
   pako@1.0.11: {}

e2e/package.json

@@ -3,13 +3,12 @@
 	"version": "0.1.0",
 	"private": true,
 	"scripts": {
-		"build": "cd .. && npm run compile && npm run build:webview",
-		"compile": "rm -rf out && tsc -p tsconfig.json",
 		"lint": "eslint src --ext ts",
 		"check-types": "tsc --noEmit",
-		"test": "npm run compile && npx dotenvx run -f .env.local -- node ./out/runTest.js",
-		"ci": "npm run build && npm run test",
-		"clean": "rimraf out"
+		"test": "npm run build && npx dotenvx run -f .env.local -- node ./out/runTest.js",
+		"ci": "npm run vscode-test && npm run test",
+		"build": "rimraf out && tsc -p tsconfig.json",
+		"vscode-test": "cd .. && npm run vscode-test"
 	},
 	"dependencies": {},
 	"devDependencies": {

package.json

@@ -279,23 +279,22 @@
 		"build:webview": "cd webview-ui && npm run build",
 		"build:esbuild": "node esbuild.js --production",
 		"compile": "tsc -p . --outDir out && node esbuild.js",
-		"install:all": "npm install npm-run-all && npm run install:_all",
-		"install:_all": "npm-run-all -p install-*",
+		"install:all": "npm install npm-run-all && npm-run-all -l -p install-*",
 		"install-extension": "npm install",
 		"install-webview": "cd webview-ui && npm install",
 		"install-e2e": "cd e2e && npm install",
-		"lint": "npm-run-all -p lint:*",
+		"lint": "npm-run-all -l -p lint:*",
 		"lint:extension": "eslint src --ext ts",
 		"lint:webview": "cd webview-ui && npm run lint",
 		"lint:e2e": "cd e2e && npm run lint",
-		"check-types": "npm-run-all -p check-types:*",
+		"check-types": "npm-run-all -l -p check-types:*",
 		"check-types:extension": "tsc --noEmit",
 		"check-types:webview": "cd webview-ui && npm run check-types",
 		"check-types:e2e": "cd e2e && npm run check-types",
-		"package": "npm-run-all -p build:webview build:esbuild check-types lint",
+		"package": "npm-run-all -l -p build:webview build:esbuild check-types lint",
 		"pretest": "npm run compile",
 		"dev": "cd webview-ui && npm run dev",
-		"test": "npm-run-all -p test:*",
+		"test": "npm-run-all -l -p test:*",
 		"test:extension": "jest",
 		"test:webview": "cd webview-ui && npm run test",
 		"prepare": "husky",
@@ -304,16 +303,19 @@
 		"version-packages": "changeset version && npm install --package-lock-only",
 		"vscode:prepublish": "npm run package",
 		"vsix": "rimraf bin && mkdirp bin && npx vsce package --out bin",
-		"watch": "npm-run-all -p watch:*",
+		"watch": "npm-run-all -l -p watch:*",
 		"watch:esbuild": "node esbuild.js --watch",
 		"watch:tsc": "tsc --noEmit --watch --project tsconfig.json",
 		"watch-tests": "tsc -p . -w --outDir out",
 		"changeset": "changeset",
 		"knip": "knip --include files",
-		"clean": "npm-run-all -p clean:*",
+		"clean": "npm-run-all -l -p clean:*",
 		"clean:extension": "rimraf bin dist out",
 		"clean:webview": "cd webview-ui && npm run clean",
 		"clean:e2e": "cd e2e && npm run clean",
+		"vscode-test": "npm-run-all -l -p vscode-test:*",
+		"vscode-test:extension": "tsc -p . --outDir out && node esbuild.js",
+		"vscode-test:webview": "cd webview-ui && npm run build",
 		"update-contributors": "node scripts/update-contributors.js"
 	},
 	"dependencies": {