diff --git a/benchmark/.env.sample b/benchmark/.env.sample new file mode 100644 index 0000000000..7a456f389c --- /dev/null +++ b/benchmark/.env.sample @@ -0,0 +1,2 @@ +BENCHMARKS_DB_PATH=file:/tmp/benchmarks.db +OPENROUTER_API_KEY=sk-or-v1-... diff --git a/benchmark/.gitignore b/benchmark/.gitignore index 96fab4fed3..33952e49e9 100644 --- a/benchmark/.gitignore +++ b/benchmark/.gitignore @@ -7,10 +7,8 @@ node_modules # Local env files .env -.env.local -.env.development.local -.env.test.local -.env.production.local +.env.* +!.env.sample # Testing coverage @@ -36,3 +34,7 @@ yarn-error.log* # Misc .DS_Store *.pem + +# Roo-Code-Benchmark +exercises + diff --git a/benchmark/package.json b/benchmark/package.json index 31c8a57c01..37c0e47198 100644 --- a/benchmark/package.json +++ b/benchmark/package.json @@ -5,7 +5,8 @@ "lint": "turbo lint", "check-types": "turbo check-types", "format": "turbo format", - "dev": "pnpm --filter @benchmark/server dev" + "cli": "pnpm --filter @benchmark/cli dev", + "web": "pnpm --filter @benchmark/web dev" }, "devDependencies": { "@dotenvx/dotenvx": "^1.39.0", diff --git a/benchmark/packages/cli/package.json b/benchmark/packages/cli/package.json index 52126e8f76..396799e526 100644 --- a/benchmark/packages/cli/package.json +++ b/benchmark/packages/cli/package.json @@ -2,6 +2,7 @@ "name": "@benchmark/cli", "version": "0.1.0", "private": true, + "type": "module", "packageManager": "pnpm@10.6.5+sha512.cdf928fca20832cd59ec53826492b7dc25dc524d4370b6b4adbf65803d32efaa6c1c88147c0ae4e8d579a6c9eec715757b50d4fa35eea179d868eada4ed043af", "scripts": { "lint": "eslint src --ext ts --max-warnings=0", @@ -11,10 +12,13 @@ "dev": "dotenvx run -f ../../.env -- tsx src/index.ts" }, "dependencies": { + "@benchmark/db": "workspace:^", "@vscode/test-electron": "^2.4.0", - "gluegun": "^5.1.2" + "gluegun": "^5.1.2", + "p-map": "^7.0.3" }, "devDependencies": { - "@benchmark/eslint-config": "workspace:^" + "@benchmark/eslint-config": "workspace:^", + "@benchmark/typescript-config": "workspace:^" } } diff --git a/benchmark/packages/cli/src/index.ts b/benchmark/packages/cli/src/index.ts index f62a8b7379..f5e09d1f80 100644 --- a/benchmark/packages/cli/src/index.ts +++ b/benchmark/packages/cli/src/index.ts @@ -1,25 +1,45 @@ import * as fs from "fs" import * as path from "path" +import { fileURLToPath } from "url" +import pMap from "p-map" -import { build, filesystem, GluegunPrompt } from "gluegun" +import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun" import { runTests } from "@vscode/test-electron" -// console.log(__dirname) -// <...>/Roo-Code/benchmark/src +import { type Language, languages, type Run, findRun, createRun, getTask } from "@benchmark/db" -const extensionDevelopmentPath = path.resolve(__dirname, "../../") -const extensionTestsPath = path.resolve(__dirname, "../out/run.js") -const promptsPath = path.resolve(__dirname, "../prompts") -const exercisesPath = path.resolve(__dirname, "../../../exercises") -const languages = ["cpp", "go", "java", "javascript", "python", "rust"] +const __dirname = path.dirname(fileURLToPath(import.meta.url)) +const extensionDevelopmentPath = path.resolve(__dirname, "../../../..") +const extensionTestsPath = path.resolve(extensionDevelopmentPath, "benchmark/packages/runner/dist/run.js") +const exercisesPath = path.resolve(extensionDevelopmentPath, "benchmark/exercises") -async function runAll({ runId, model }: { runId: number; model: string }) { - for (const language of languages) { - await runLanguage({ runId, model, language }) +export const isLanguage = (language: string): language is Language => languages.includes(language as Language) + +const run = async (toolbox: GluegunToolbox) => { + const { config, prompt } = toolbox + const id = config.runId ? Number(config.runId) : undefined + let { language, exercise } = config + + if (language === "all") { + const run = await findOrCreateRun({ id }) + await runAll(run) + } else if (exercise === "all") { + const run = await findOrCreateRun({ id }) + await runLanguage({ run, language }) + } else { + language = language || (await askLanguage(prompt)) + exercise = exercise || (await askExercise(prompt, language)) + const run = await findOrCreateRun({ id }) + await runExercise({ run, language, exercise }) } } -async function runLanguage({ runId, model, language }: { runId: number; model: string; language: string }) { +const runAll = async (run: Run) => + (await pMap(languages, (language) => runLanguage({ run, language }), { concurrency: 1 })).flatMap( + (language) => language, + ) + +const runLanguage = async ({ run, language }: { run: Run; language: Language }) => { const languagePath = path.resolve(exercisesPath, language) if (!fs.existsSync(languagePath)) { @@ -32,65 +52,64 @@ async function runLanguage({ runId, model, language }: { runId: number; model: s .map((exercise) => path.basename(exercise)) .filter((exercise) => !exercise.startsWith(".")) - for (const exercise of exercises) { - await runExercise({ runId, model, language, exercise }) - } + const results = await pMap( + exercises, + async (exercise) => ({ + language, + exercise, + result: await runExercise({ run, language, exercise }), + }), + { concurrency: 1 }, + ) + + return results } -async function runExercise({ - runId, - model, - language, - exercise, -}: { - runId: number - model: string - language: string - exercise: string -}) { +const runExercise = async ({ run, language, exercise }: { run: Run; language: Language; exercise: string }) => { const workspacePath = path.resolve(exercisesPath, language, exercise) - const promptPath = path.resolve(promptsPath, `${language}.md`) + const promptPath = path.resolve(exercisesPath, `prompts/${language}.md`) - const extensionTestsEnv = { - PROMPT_PATH: promptPath, - WORKSPACE_PATH: workspacePath, - OPENROUTER_MODEL_ID: model, - RUN_ID: runId.toString(), + if (!fs.existsSync(promptPath)) { + throw new Error(`Prompt file does not exist: ${promptPath}`) } - if (fs.existsSync(path.resolve(workspacePath, "usage.json"))) { + const task = await getTask({ runId: run.id, language, exercise }) + + if (task) { console.log(`Test result exists for ${language} / ${exercise}, skipping`) - return + return false } console.log(`Running ${language} / ${exercise}`) + return true await runTests({ extensionDevelopmentPath, extensionTestsPath, launchArgs: [workspacePath, "--disable-extensions"], - extensionTestsEnv, + extensionTestsEnv: { + PROMPT_PATH: promptPath, + WORKSPACE_PATH: workspacePath, + OPENROUTER_MODEL_ID: run.model, + RUN_ID: run.id.toString(), + }, }) + + return true } -async function askLanguage(prompt: GluegunPrompt) { - const languages = filesystem.subdirectories(exercisesPath) - - if (languages.length === 0) { - throw new Error(`No languages found in ${exercisesPath}`) - } - - const { language } = await prompt.ask<{ language: string }>({ +const askLanguage = async (prompt: GluegunPrompt) => { + const { language } = await prompt.ask<{ language: Language }>({ type: "select", name: "language", message: "Which language?", - choices: languages.map((language) => path.basename(language)).filter((language) => !language.startsWith(".")), + choices: [...languages], }) return language } -async function askExercise(prompt: GluegunPrompt, language: string) { +const askExercise = async (prompt: GluegunPrompt, language: Language) => { const exercises = filesystem.subdirectories(path.join(exercisesPath, language)) if (exercises.length === 0) { @@ -107,30 +126,20 @@ async function askExercise(prompt: GluegunPrompt, language: string) { return exercise } -async function createRun({ model }: { model: string }): Promise<{ id: number; model: string }> { - const response = await fetch("http://localhost:3000/api/runs", { - method: "POST", - body: JSON.stringify({ model }), - }) +type FindOrCreateRun = { id?: number; model?: string } - if (!response.ok) { - throw new Error(`Failed to create run: ${response.statusText}`) - } +const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: FindOrCreateRun) => + id ? findRun(id) : createRun({ model }) - const { - run: [run], - } = await response.json() - return run -} - -async function main() { +const main = async () => { const cli = build() - .brand("benchmark-cli") + .brand("cli") .src(__dirname) .help() .version() .command({ name: "run", + description: "Run a benchmark", run: ({ config, parameters }) => { config.language = parameters.first config.exercise = parameters.second @@ -140,32 +149,41 @@ async function main() { } }, }) - .defaultCommand() // Use the default command if no args. + .defaultCommand() .create() - const { print, prompt, config } = await cli.run(process.argv) + const toolbox = await cli.run(process.argv) + const { print, command } = toolbox try { - const model = "anthropic/claude-3.7-sonnet" - const runId = config.runId ? Number(config.runId) : (await createRun({ model })).id - - if (config.language === "all") { - console.log("Running all exercises for all languages") - await runAll({ runId, model }) - } else if (config.exercise === "all") { - console.log(`Running all exercises for ${config.language}`) - await runLanguage({ runId, model, language: config.language }) - } else { - const language = config.language || (await askLanguage(prompt)) - const exercise = config.exercise || (await askExercise(prompt, language)) - await runExercise({ runId, model, language, exercise }) + switch (command?.name) { + case "run": + await run(toolbox) + break } process.exit(0) - } catch (error) { - print.error(error) + } catch (error: unknown) { + print.error(error instanceof Error ? error.message : String(error)) process.exit(1) } } +if (!fs.existsSync(extensionDevelopmentPath)) { + console.error(`"extensionDevelopmentPath" does not exist.`) + process.exit(1) +} + +if (!fs.existsSync(extensionTestsPath)) { + console.error(`"extensionTestsPath" does not exist. Please run "pnpm --filter @benchmark/runner build".`) + process.exit(1) +} + +if (!fs.existsSync(exercisesPath)) { + console.error( + `Exercises path does not exist. Please run "git clone https://github.com/cte/Roo-Code-Benchmark.git exercises".`, + ) + process.exit(1) +} + main() diff --git a/benchmark/packages/cli/tsconfig.json b/benchmark/packages/cli/tsconfig.json index bd342ec15c..5440710515 100644 --- a/benchmark/packages/cli/tsconfig.json +++ b/benchmark/packages/cli/tsconfig.json @@ -1,22 +1,5 @@ { - "$schema": "https://json.schemastore.org/tsconfig", - "display": "Default", - "compilerOptions": { - "composite": false, - "declaration": true, - "declarationMap": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "inlineSources": false, - "isolatedModules": true, - "noUnusedLocals": false, - "noUnusedParameters": false, - "preserveWatchOutput": true, - "skipLibCheck": true, - "strict": true, - "module": "ESNext", - "moduleResolution": "Node", - "target": "es2020" - }, + "extends": "@benchmark/typescript-config/base.json", + "include": ["src"], "exclude": ["node_modules"] } diff --git a/benchmark/packages/db/package.json b/benchmark/packages/db/package.json index 65b6ea8629..cb1f532cfd 100644 --- a/benchmark/packages/db/package.json +++ b/benchmark/packages/db/package.json @@ -2,6 +2,7 @@ "name": "@benchmark/db", "version": "0.1.0", "private": true, + "type": "module", "packageManager": "pnpm@10.6.5", "exports": { ".": "./src/index.ts" @@ -21,6 +22,7 @@ }, "devDependencies": { "@benchmark/eslint-config": "workspace:^", + "@benchmark/typescript-config": "workspace:^", "drizzle-kit": "^0.30.5" } } diff --git a/benchmark/packages/db/src/db.ts b/benchmark/packages/db/src/db.ts index 1ee0dda3bf..357af93987 100644 --- a/benchmark/packages/db/src/db.ts +++ b/benchmark/packages/db/src/db.ts @@ -1,5 +1,8 @@ import { drizzle } from "drizzle-orm/libsql" -export * from "./schema" +import { schema } from "./schema" -export const db = drizzle({ connection: { url: process.env.BENCHMARKS_DB_PATH! } }) +export const db = drizzle({ + schema, + connection: { url: process.env.BENCHMARKS_DB_PATH! }, +}) diff --git a/benchmark/packages/db/src/enums.ts b/benchmark/packages/db/src/enums.ts new file mode 100644 index 0000000000..df7bcf2780 --- /dev/null +++ b/benchmark/packages/db/src/enums.ts @@ -0,0 +1,7 @@ +/** + * languages + */ + +export const languages = ["cpp", "go", "java", "javascript", "python", "rust"] as const + +export type Language = (typeof languages)[number] diff --git a/benchmark/packages/db/src/index.ts b/benchmark/packages/db/src/index.ts index 19271cb60a..1ad559e594 100644 --- a/benchmark/packages/db/src/index.ts +++ b/benchmark/packages/db/src/index.ts @@ -1,7 +1,15 @@ export { db } from "./db" +export { type Language, languages } from "./enums" +export { schema } from "./schema" -export { type Language, languages } from "./schema" -export { type Run, insertRunSchema, runs } from "./schema" -export { type Task, insertTaskSchema, tasks } from "./schema" +/** + * runs + */ +export { type Run, insertRunSchema } from "./schema" +export { findRun, createRun, getRuns } from "./queries/runs" -export { getRuns } from "./queries" +/** + * tasks + */ +export { type Task, insertTaskSchema } from "./schema" +export { findTask, createTask, getTask } from "./queries/tasks" diff --git a/benchmark/packages/db/src/queries/errors.ts b/benchmark/packages/db/src/queries/errors.ts new file mode 100644 index 0000000000..e1c9cb13d5 --- /dev/null +++ b/benchmark/packages/db/src/queries/errors.ts @@ -0,0 +1 @@ +export class RecordNotFoundError extends Error {} diff --git a/benchmark/packages/db/src/queries.ts b/benchmark/packages/db/src/queries/runs.ts similarity index 52% rename from benchmark/packages/db/src/queries.ts rename to benchmark/packages/db/src/queries/runs.ts index 21d283fe8d..c303c76de0 100644 --- a/benchmark/packages/db/src/queries.ts +++ b/benchmark/packages/db/src/queries/runs.ts @@ -1,6 +1,31 @@ import { desc, eq, sql } from "drizzle-orm" -import { db, runs, tasks } from "./db" +import { db } from "../db" +import { InsertRun, insertRunSchema, runs, tasks } from "../schema" + +import { RecordNotFoundError } from "./errors" + +export const findRun = async (id: number) => { + const run = await db.query.runs.findFirst({ where: eq(runs.id, id) }) + + if (!run) { + throw new RecordNotFoundError() + } + + return run +} + +export const createRun = async (args: InsertRun) => { + const result = await db + .insert(runs) + .values({ + ...insertRunSchema.parse(args), + createdAt: new Date(), + }) + .returning() + + return result[0] +} export const getRuns = () => db diff --git a/benchmark/packages/db/src/queries/tasks.ts b/benchmark/packages/db/src/queries/tasks.ts new file mode 100644 index 0000000000..44655e5968 --- /dev/null +++ b/benchmark/packages/db/src/queries/tasks.ts @@ -0,0 +1,34 @@ +import { and, eq } from "drizzle-orm" + +import { db } from "../db" +import { InsertTask, insertTaskSchema, tasks } from "../schema" + +import { RecordNotFoundError } from "./errors" +import { Language } from "../enums" + +export const findTask = async (id: number) => { + const run = await db.query.tasks.findFirst({ where: eq(tasks.id, id) }) + + if (!run) { + throw new RecordNotFoundError() + } + + return run +} + +export const createTask = async (args: InsertTask) => { + const result = await db + .insert(tasks) + .values({ + ...insertTaskSchema.parse(args), + createdAt: new Date(), + }) + .returning() + + return result[0] +} + +export const getTask = async ({ runId, language, exercise }: { runId: number; language: Language; exercise: string }) => + db.query.tasks.findFirst({ + where: and(eq(tasks.runId, runId), eq(tasks.language, language), eq(tasks.exercise, exercise)), + }) diff --git a/benchmark/packages/db/src/schema.ts b/benchmark/packages/db/src/schema.ts index a845873623..b02aa94c15 100644 --- a/benchmark/packages/db/src/schema.ts +++ b/benchmark/packages/db/src/schema.ts @@ -1,14 +1,9 @@ import { sqliteTable, text, real, integer } from "drizzle-orm/sqlite-core" import * as t from "drizzle-orm/sqlite-core" import { createInsertSchema } from "drizzle-zod" +import { z } from "zod" -/** - * languages - */ - -export const languages = ["cpp", "go", "java", "javascript", "python", "rust"] as const - -export type Language = (typeof languages)[number] +import { languages } from "./enums" /** * runs @@ -28,6 +23,8 @@ export const insertRunSchema = createInsertSchema(runs).omit({ createdAt: true, }) +export type InsertRun = z.infer + /** * tasks */ @@ -58,3 +55,11 @@ export const insertTaskSchema = createInsertSchema(tasks).omit({ id: true, createdAt: true, }) + +export type InsertTask = z.infer + +/** + * schema + */ + +export const schema = { runs, tasks } diff --git a/benchmark/packages/db/tsconfig.json b/benchmark/packages/db/tsconfig.json index 6632f20aff..5440710515 100644 --- a/benchmark/packages/db/tsconfig.json +++ b/benchmark/packages/db/tsconfig.json @@ -1,23 +1,5 @@ { - "$schema": "https://json.schemastore.org/tsconfig", - "display": "Default", - "compilerOptions": { - "composite": false, - "declaration": true, - "declarationMap": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "inlineSources": false, - "isolatedModules": true, - "noUnusedLocals": false, - "noUnusedParameters": false, - "preserveWatchOutput": true, - "skipLibCheck": true, - "strict": true, - "module": "ESNext", - "moduleResolution": "Node", - "target": "es2020", - "types": ["node"] - }, + "extends": "@benchmark/typescript-config/base.json", + "include": ["src"], "exclude": ["node_modules"] } diff --git a/benchmark/packages/ipc/package.json b/benchmark/packages/ipc/package.json index 4f642b8441..b457cb2aa7 100644 --- a/benchmark/packages/ipc/package.json +++ b/benchmark/packages/ipc/package.json @@ -2,6 +2,7 @@ "name": "@benchmark/ipc", "version": "0.1.0", "private": true, + "type": "module", "packageManager": "pnpm@10.6.5", "scripts": { "lint": "eslint src --ext ts --max-warnings=0", @@ -16,6 +17,7 @@ }, "devDependencies": { "@benchmark/eslint-config": "workspace:^", + "@benchmark/typescript-config": "workspace:^", "@types/node-ipc": "^9.2.3" } } diff --git a/benchmark/packages/ipc/scripts/server.ts b/benchmark/packages/ipc/scripts/server.ts index 8893836b21..e6a59ec281 100644 --- a/benchmark/packages/ipc/scripts/server.ts +++ b/benchmark/packages/ipc/scripts/server.ts @@ -4,6 +4,7 @@ async function main() { try { const server = new IpcServer() server.listen() + console.log(`listening @ ${server.socketPath}`) while (server.isListening) { await new Promise((resolve) => setTimeout(resolve, 1000)) diff --git a/benchmark/packages/ipc/tsconfig.json b/benchmark/packages/ipc/tsconfig.json index bd342ec15c..5440710515 100644 --- a/benchmark/packages/ipc/tsconfig.json +++ b/benchmark/packages/ipc/tsconfig.json @@ -1,22 +1,5 @@ { - "$schema": "https://json.schemastore.org/tsconfig", - "display": "Default", - "compilerOptions": { - "composite": false, - "declaration": true, - "declarationMap": true, - "esModuleInterop": true, - "forceConsistentCasingInFileNames": true, - "inlineSources": false, - "isolatedModules": true, - "noUnusedLocals": false, - "noUnusedParameters": false, - "preserveWatchOutput": true, - "skipLibCheck": true, - "strict": true, - "module": "ESNext", - "moduleResolution": "Node", - "target": "es2020" - }, + "extends": "@benchmark/typescript-config/base.json", + "include": ["src"], "exclude": ["node_modules"] } diff --git a/benchmark/packages/client/eslint.config.mjs b/benchmark/packages/runner/eslint.config.mjs similarity index 100% rename from benchmark/packages/client/eslint.config.mjs rename to benchmark/packages/runner/eslint.config.mjs diff --git a/benchmark/packages/client/package.json b/benchmark/packages/runner/package.json similarity index 64% rename from benchmark/packages/client/package.json rename to benchmark/packages/runner/package.json index 9fa57c7626..bbb65401d7 100644 --- a/benchmark/packages/client/package.json +++ b/benchmark/packages/runner/package.json @@ -1,5 +1,5 @@ { - "name": "@benchmark/client", + "name": "@benchmark/runner", "version": "0.1.0", "private": true, "packageManager": "pnpm@10.6.5", @@ -7,9 +7,8 @@ "lint": "eslint src --ext ts --max-warnings=0", "check-types": "tsc --noEmit", "format": "prettier --write .", - "compile": "pnpm clean && tsc", - "build": "pnpm compile && cd ../../.. && npm run compile && npm run build:webview", - "clean": "rimraf out" + "build": "rimraf dist && tsc", + "vscode-test": "pnpm build && cd ../../.. && npm run vscode-test" }, "devDependencies": { "@benchmark/eslint-config": "workspace:^", diff --git a/benchmark/packages/client/src/roo-code.d.ts b/benchmark/packages/runner/src/roo-code.d.ts similarity index 100% rename from benchmark/packages/client/src/roo-code.d.ts rename to benchmark/packages/runner/src/roo-code.d.ts diff --git a/benchmark/packages/client/src/run.ts b/benchmark/packages/runner/src/run.ts similarity index 100% rename from benchmark/packages/client/src/run.ts rename to benchmark/packages/runner/src/run.ts diff --git a/benchmark/packages/client/src/utils.ts b/benchmark/packages/runner/src/utils.ts similarity index 100% rename from benchmark/packages/client/src/utils.ts rename to benchmark/packages/runner/src/utils.ts diff --git a/benchmark/packages/client/tsconfig.json b/benchmark/packages/runner/tsconfig.json similarity index 78% rename from benchmark/packages/client/tsconfig.json rename to benchmark/packages/runner/tsconfig.json index e59aa681b3..d11199c83c 100644 --- a/benchmark/packages/client/tsconfig.json +++ b/benchmark/packages/runner/tsconfig.json @@ -9,7 +9,7 @@ "strict": true, "skipLibCheck": true, "useUnknownInCatchVariables": false, - "outDir": "out" + "outDir": "dist" }, - "include": ["src", "../../../../src/exports/roo-code.d.ts"] + "include": ["src"] } diff --git a/benchmark/packages/server/.gitignore b/benchmark/packages/server/.gitignore deleted file mode 100644 index 68b26e8d57..0000000000 --- a/benchmark/packages/server/.gitignore +++ /dev/null @@ -1,45 +0,0 @@ -# See https://help.github.com/articles/ignoring-files/ for more about ignoring files. - -# dependencies -/node_modules -/.pnp -.pnp.* -.yarn/* -!.yarn/patches -!.yarn/plugins -!.yarn/releases -!.yarn/versions - -# testing -/coverage - -# next.js -/.next/ -/out/ - -# production -/build - -# misc -.DS_Store -*.pem - -# debug -npm-debug.log* -yarn-debug.log* -yarn-error.log* -.pnpm-debug.log* - -# env files (can opt-in for committing if needed) -.env* -!.env.sample - -# vercel -.vercel - -# typescript -*.tsbuildinfo -next-env.d.ts - -# sqlite -local.db diff --git a/benchmark/packages/server/tsconfig.json b/benchmark/packages/server/tsconfig.json deleted file mode 100644 index 8f96fa185d..0000000000 --- a/benchmark/packages/server/tsconfig.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "compilerOptions": { - "target": "ES2017", - "lib": ["dom", "dom.iterable", "esnext"], - "allowJs": true, - "skipLibCheck": true, - "strict": true, - "noEmit": true, - "esModuleInterop": true, - "module": "esnext", - "moduleResolution": "bundler", - "resolveJsonModule": true, - "isolatedModules": true, - "jsx": "preserve", - "incremental": true, - "plugins": [ - { - "name": "next" - } - ], - "paths": { - "@/*": ["./src/*"] - } - }, - "include": [ - "next-env.d.ts", - "**/*.ts", - "**/*.tsx", - ".next/types/**/*.ts", - "../db/src/index.ts", - "../db/src/queries.ts", - "../db/src/schema.ts" - ], - "exclude": ["node_modules"] -} diff --git a/benchmark/packages/typescript-config/base.json b/benchmark/packages/typescript-config/base.json new file mode 100644 index 0000000000..0756a8cdef --- /dev/null +++ b/benchmark/packages/typescript-config/base.json @@ -0,0 +1,19 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "esModuleInterop": true, + "incremental": false, + "isolatedModules": true, + "lib": ["es2022", "DOM", "DOM.Iterable"], + "module": "NodeNext", + "moduleDetection": "force", + "moduleResolution": "NodeNext", + "noUncheckedIndexedAccess": true, + "resolveJsonModule": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2022" + } +} diff --git a/benchmark/packages/typescript-config/nextjs.json b/benchmark/packages/typescript-config/nextjs.json new file mode 100644 index 0000000000..fe3784d2f1 --- /dev/null +++ b/benchmark/packages/typescript-config/nextjs.json @@ -0,0 +1,12 @@ +{ + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "./base.json", + "compilerOptions": { + "plugins": [{ "name": "next" }], + "module": "ESNext", + "moduleResolution": "bundler", + "allowJs": true, + "jsx": "preserve", + "noEmit": true + } +} diff --git a/benchmark/packages/typescript-config/package.json b/benchmark/packages/typescript-config/package.json new file mode 100644 index 0000000000..8ec6df5ff9 --- /dev/null +++ b/benchmark/packages/typescript-config/package.json @@ -0,0 +1,7 @@ +{ + "name": "@benchmark/typescript-config", + "private": true, + "publishConfig": { + "access": "public" + } +} diff --git a/benchmark/packages/server/README.md b/benchmark/packages/web/README.md similarity index 100% rename from benchmark/packages/server/README.md rename to benchmark/packages/web/README.md diff --git a/benchmark/packages/server/components.json b/benchmark/packages/web/components.json similarity index 100% rename from benchmark/packages/server/components.json rename to benchmark/packages/web/components.json diff --git a/benchmark/packages/server/eslint.config.mjs b/benchmark/packages/web/eslint.config.mjs similarity index 100% rename from benchmark/packages/server/eslint.config.mjs rename to benchmark/packages/web/eslint.config.mjs diff --git a/benchmark/packages/web/next-env.d.ts b/benchmark/packages/web/next-env.d.ts new file mode 100644 index 0000000000..1b3be0840f --- /dev/null +++ b/benchmark/packages/web/next-env.d.ts @@ -0,0 +1,5 @@ +/// +/// + +// NOTE: This file should not be edited +// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/benchmark/packages/server/next.config.ts b/benchmark/packages/web/next.config.ts similarity index 100% rename from benchmark/packages/server/next.config.ts rename to benchmark/packages/web/next.config.ts diff --git a/benchmark/packages/server/package.json b/benchmark/packages/web/package.json similarity index 91% rename from benchmark/packages/server/package.json rename to benchmark/packages/web/package.json index a133b26684..fe02a903b2 100644 --- a/benchmark/packages/server/package.json +++ b/benchmark/packages/web/package.json @@ -1,5 +1,5 @@ { - "name": "@benchmark/server", + "name": "@benchmark/web", "version": "0.1.0", "private": true, "packageManager": "pnpm@10.6.5", @@ -27,6 +27,7 @@ }, "devDependencies": { "@benchmark/eslint-config": "workspace:^", + "@benchmark/typescript-config": "workspace:^", "@tailwindcss/postcss": "^4", "@types/react": "^19", "@types/react-dom": "^19", diff --git a/benchmark/packages/server/postcss.config.mjs b/benchmark/packages/web/postcss.config.mjs similarity index 100% rename from benchmark/packages/server/postcss.config.mjs rename to benchmark/packages/web/postcss.config.mjs diff --git a/benchmark/packages/server/public/.gitkeep b/benchmark/packages/web/public/.gitkeep similarity index 100% rename from benchmark/packages/server/public/.gitkeep rename to benchmark/packages/web/public/.gitkeep diff --git a/benchmark/packages/server/src/app/api/runs/route.ts b/benchmark/packages/web/src/app/api/runs/route.ts similarity index 54% rename from benchmark/packages/server/src/app/api/runs/route.ts rename to benchmark/packages/web/src/app/api/runs/route.ts index 2e1ac75e7a..f8f4349aef 100644 --- a/benchmark/packages/server/src/app/api/runs/route.ts +++ b/benchmark/packages/web/src/app/api/runs/route.ts @@ -1,17 +1,10 @@ import { NextResponse } from "next/server" -import { db, runs, insertRunSchema } from "@benchmark/db" +import { createRun } from "@benchmark/db" export async function POST(request: Request) { try { - const run = await db - .insert(runs) - .values({ - ...insertRunSchema.parse(await request.json()), - createdAt: new Date(), - }) - .returning() - + const run = await createRun(await request.json()) return NextResponse.json({ run }, { status: 201 }) } catch (error) { return NextResponse.json({ error: (error as Error).message }, { status: 500 }) diff --git a/benchmark/packages/server/src/app/api/tasks/route.ts b/benchmark/packages/web/src/app/api/tasks/route.ts similarity index 54% rename from benchmark/packages/server/src/app/api/tasks/route.ts rename to benchmark/packages/web/src/app/api/tasks/route.ts index 62582d9f3e..b096309792 100644 --- a/benchmark/packages/server/src/app/api/tasks/route.ts +++ b/benchmark/packages/web/src/app/api/tasks/route.ts @@ -1,17 +1,10 @@ import { NextResponse } from "next/server" -import { db, tasks, insertTaskSchema } from "@benchmark/db" +import { createTask } from "@benchmark/db" export async function POST(request: Request) { try { - const task = await db - .insert(tasks) - .values({ - ...insertTaskSchema.parse(await request.json()), - createdAt: new Date(), - }) - .returning() - + const task = await createTask(await request.json()) return NextResponse.json({ task }, { status: 201 }) } catch (error) { return NextResponse.json({ error: (error as Error).message }, { status: 500 }) diff --git a/benchmark/packages/server/src/app/favicon.ico b/benchmark/packages/web/src/app/favicon.ico similarity index 100% rename from benchmark/packages/server/src/app/favicon.ico rename to benchmark/packages/web/src/app/favicon.ico diff --git a/benchmark/packages/server/src/app/globals.css b/benchmark/packages/web/src/app/globals.css similarity index 100% rename from benchmark/packages/server/src/app/globals.css rename to benchmark/packages/web/src/app/globals.css diff --git a/benchmark/packages/server/src/app/layout.tsx b/benchmark/packages/web/src/app/layout.tsx similarity index 100% rename from benchmark/packages/server/src/app/layout.tsx rename to benchmark/packages/web/src/app/layout.tsx diff --git a/benchmark/packages/server/src/app/page.tsx b/benchmark/packages/web/src/app/page.tsx similarity index 100% rename from benchmark/packages/server/src/app/page.tsx rename to benchmark/packages/web/src/app/page.tsx diff --git a/benchmark/packages/server/src/components/theme-provider.tsx b/benchmark/packages/web/src/components/theme-provider.tsx similarity index 100% rename from benchmark/packages/server/src/components/theme-provider.tsx rename to benchmark/packages/web/src/components/theme-provider.tsx diff --git a/benchmark/packages/server/src/components/ui/button.tsx b/benchmark/packages/web/src/components/ui/button.tsx similarity index 100% rename from benchmark/packages/server/src/components/ui/button.tsx rename to benchmark/packages/web/src/components/ui/button.tsx diff --git a/benchmark/packages/server/src/components/ui/index.ts b/benchmark/packages/web/src/components/ui/index.ts similarity index 100% rename from benchmark/packages/server/src/components/ui/index.ts rename to benchmark/packages/web/src/components/ui/index.ts diff --git a/benchmark/packages/server/src/components/ui/table.tsx b/benchmark/packages/web/src/components/ui/table.tsx similarity index 100% rename from benchmark/packages/server/src/components/ui/table.tsx rename to benchmark/packages/web/src/components/ui/table.tsx diff --git a/benchmark/packages/server/src/lib/formatCurrency.ts b/benchmark/packages/web/src/lib/formatCurrency.ts similarity index 100% rename from benchmark/packages/server/src/lib/formatCurrency.ts rename to benchmark/packages/web/src/lib/formatCurrency.ts diff --git a/benchmark/packages/server/src/lib/formatDuration.ts b/benchmark/packages/web/src/lib/formatDuration.ts similarity index 100% rename from benchmark/packages/server/src/lib/formatDuration.ts rename to benchmark/packages/web/src/lib/formatDuration.ts diff --git a/benchmark/packages/server/src/lib/index.ts b/benchmark/packages/web/src/lib/index.ts similarity index 100% rename from benchmark/packages/server/src/lib/index.ts rename to benchmark/packages/web/src/lib/index.ts diff --git a/benchmark/packages/server/src/lib/utils.ts b/benchmark/packages/web/src/lib/utils.ts similarity index 100% rename from benchmark/packages/server/src/lib/utils.ts rename to benchmark/packages/web/src/lib/utils.ts diff --git a/benchmark/packages/web/tsconfig.json b/benchmark/packages/web/tsconfig.json new file mode 100644 index 0000000000..22f2ee20fd --- /dev/null +++ b/benchmark/packages/web/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "@benchmark/typescript-config/nextjs.json", + "compilerOptions": { + "plugins": [{ "name": "next" }], + "paths": { "@/*": ["./src/*"] } + }, + "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"], + "exclude": ["node_modules"] +} diff --git a/benchmark/pnpm-lock.yaml b/benchmark/pnpm-lock.yaml index da35ad2849..86c59a00dd 100644 --- a/benchmark/pnpm-lock.yaml +++ b/benchmark/pnpm-lock.yaml @@ -41,25 +41,25 @@ importers: packages/cli: dependencies: + '@benchmark/db': + specifier: workspace:^ + version: link:../db '@vscode/test-electron': specifier: ^2.4.0 version: 2.4.1 gluegun: specifier: ^5.1.2 version: 5.2.0 + p-map: + specifier: ^7.0.3 + version: 7.0.3 devDependencies: '@benchmark/eslint-config': specifier: workspace:^ version: link:../eslint-config - - packages/client: - devDependencies: - '@benchmark/eslint-config': + '@benchmark/typescript-config': specifier: workspace:^ - version: link:../eslint-config - '@types/vscode': - specifier: ^1.98.0 - version: 1.98.0 + version: link:../typescript-config packages/db: dependencies: @@ -82,6 +82,9 @@ importers: '@benchmark/eslint-config': specifier: workspace:^ version: link:../eslint-config + '@benchmark/typescript-config': + specifier: workspace:^ + version: link:../typescript-config drizzle-kit: specifier: ^0.30.5 version: 0.30.5 @@ -136,11 +139,25 @@ importers: '@benchmark/eslint-config': specifier: workspace:^ version: link:../eslint-config + '@benchmark/typescript-config': + specifier: workspace:^ + version: link:../typescript-config '@types/node-ipc': specifier: ^9.2.3 version: 9.2.3 - packages/server: + packages/runner: + devDependencies: + '@benchmark/eslint-config': + specifier: workspace:^ + version: link:../eslint-config + '@types/vscode': + specifier: ^1.98.0 + version: 1.98.0 + + packages/typescript-config: {} + + packages/web: dependencies: '@benchmark/db': specifier: workspace:^ @@ -182,6 +199,9 @@ importers: '@benchmark/eslint-config': specifier: workspace:^ version: link:../eslint-config + '@benchmark/typescript-config': + specifier: workspace:^ + version: link:../typescript-config '@tailwindcss/postcss': specifier: ^4 version: 4.0.14 @@ -2512,6 +2532,10 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + p-map@7.0.3: + resolution: {integrity: sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==} + engines: {node: '>=18'} + package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} @@ -5317,6 +5341,8 @@ snapshots: dependencies: p-limit: 3.1.0 + p-map@7.0.3: {} + package-json-from-dist@1.0.1: {} pako@1.0.11: {} diff --git a/e2e/package.json b/e2e/package.json index d4932f8a07..d6a2c7af00 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -3,13 +3,12 @@ "version": "0.1.0", "private": true, "scripts": { - "build": "cd .. && npm run compile && npm run build:webview", - "compile": "rm -rf out && tsc -p tsconfig.json", "lint": "eslint src --ext ts", "check-types": "tsc --noEmit", - "test": "npm run compile && npx dotenvx run -f .env.local -- node ./out/runTest.js", - "ci": "npm run build && npm run test", - "clean": "rimraf out" + "test": "npm run build && npx dotenvx run -f .env.local -- node ./out/runTest.js", + "ci": "npm run vscode-test && npm run test", + "build": "rimraf out && tsc -p tsconfig.json", + "vscode-test": "cd .. && npm run vscode-test" }, "dependencies": {}, "devDependencies": { diff --git a/package.json b/package.json index 9d9cb1407d..7a8f770d2f 100644 --- a/package.json +++ b/package.json @@ -279,23 +279,22 @@ "build:webview": "cd webview-ui && npm run build", "build:esbuild": "node esbuild.js --production", "compile": "tsc -p . --outDir out && node esbuild.js", - "install:all": "npm install npm-run-all && npm run install:_all", - "install:_all": "npm-run-all -p install-*", + "install:all": "npm install npm-run-all && npm-run-all -l -p install-*", "install-extension": "npm install", "install-webview": "cd webview-ui && npm install", "install-e2e": "cd e2e && npm install", - "lint": "npm-run-all -p lint:*", + "lint": "npm-run-all -l -p lint:*", "lint:extension": "eslint src --ext ts", "lint:webview": "cd webview-ui && npm run lint", "lint:e2e": "cd e2e && npm run lint", - "check-types": "npm-run-all -p check-types:*", + "check-types": "npm-run-all -l -p check-types:*", "check-types:extension": "tsc --noEmit", "check-types:webview": "cd webview-ui && npm run check-types", "check-types:e2e": "cd e2e && npm run check-types", - "package": "npm-run-all -p build:webview build:esbuild check-types lint", + "package": "npm-run-all -l -p build:webview build:esbuild check-types lint", "pretest": "npm run compile", "dev": "cd webview-ui && npm run dev", - "test": "npm-run-all -p test:*", + "test": "npm-run-all -l -p test:*", "test:extension": "jest", "test:webview": "cd webview-ui && npm run test", "prepare": "husky", @@ -304,16 +303,19 @@ "version-packages": "changeset version && npm install --package-lock-only", "vscode:prepublish": "npm run package", "vsix": "rimraf bin && mkdirp bin && npx vsce package --out bin", - "watch": "npm-run-all -p watch:*", + "watch": "npm-run-all -l -p watch:*", "watch:esbuild": "node esbuild.js --watch", "watch:tsc": "tsc --noEmit --watch --project tsconfig.json", "watch-tests": "tsc -p . -w --outDir out", "changeset": "changeset", "knip": "knip --include files", - "clean": "npm-run-all -p clean:*", + "clean": "npm-run-all -l -p clean:*", "clean:extension": "rimraf bin dist out", "clean:webview": "cd webview-ui && npm run clean", "clean:e2e": "cd e2e && npm run clean", + "vscode-test": "npm-run-all -l -p vscode-test:*", + "vscode-test:extension": "tsc -p . --outDir out && node esbuild.js", + "vscode-test:webview": "cd webview-ui && npm run build", "update-contributors": "node scripts/update-contributors.js" }, "dependencies": {