diff --git a/benchmark/apps/cli/package.json b/benchmark/apps/cli/package.json index 686f5b5e9d..0263052d27 100644 --- a/benchmark/apps/cli/package.json +++ b/benchmark/apps/cli/package.json @@ -12,7 +12,7 @@ "@benchmark/db": "workspace:^", "@vscode/test-electron": "^2.4.0", "gluegun": "^5.1.2", - "p-map": "^7.0.3" + "uuid": "^11.1.0" }, "devDependencies": { "@benchmark/eslint-config": "workspace:^", diff --git a/benchmark/apps/cli/src/exercises.ts b/benchmark/apps/cli/src/exercises.ts new file mode 100644 index 0000000000..e1a76ab198 --- /dev/null +++ b/benchmark/apps/cli/src/exercises.ts @@ -0,0 +1,31 @@ +import * as path from "path" +import * as fs from "fs" + +import { filesystem } from "gluegun" + +import { type Language, languages } from "@benchmark/db" + +import { exercisesPath } from "./paths.js" + +let exercisesByLanguage: Record | null = null + +export const getExercises = () => { + if (exercisesByLanguage !== null) { + return exercisesByLanguage + } + + const getLanguageExercises = (language: Language) => + fs.existsSync(path.resolve(exercisesPath, language)) + ? filesystem + .subdirectories(path.resolve(exercisesPath, language)) + .map((exercise) => path.basename(exercise)) + .filter((exercise) => !exercise.startsWith(".")) + : [] + + exercisesByLanguage = languages.reduce( + (collect, language) => ({ ...collect, [language]: getLanguageExercises(language) }), + {} as Record, + ) + + return exercisesByLanguage +} diff --git a/benchmark/apps/cli/src/index.ts b/benchmark/apps/cli/src/index.ts index c45543f502..9333677f42 100644 --- a/benchmark/apps/cli/src/index.ts +++ b/benchmark/apps/cli/src/index.ts @@ -1,68 +1,90 @@ import * as fs from "fs" import * as path from "path" -import { fileURLToPath } from "url" -import pMap from "p-map" +import * as os from "os" +import { v4 as uuidv4 } from "uuid" import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun" import { runTests } from "@vscode/test-electron" -import { type Language, languages, type Run, findRun, createRun, getTask } from "@benchmark/db" +import { + type Language, + languages, + type Run, + findRun, + createRun, + getPendingTask, + createPendingTask, + getTask, +} from "@benchmark/db" -const __dirname = path.dirname(fileURLToPath(import.meta.url)) -const extensionDevelopmentPath = path.resolve(__dirname, "..", "..", "..", "..") -const extensionTestsPath = path.resolve(extensionDevelopmentPath, "benchmark/packages/runner/dist") -const exercisesPath = path.resolve(extensionDevelopmentPath, "..", "exercises") +import { __dirname, extensionDevelopmentPath, extensionTestsPath, exercisesPath } from "./paths.js" +import { getExercises } from "./exercises.js" export const isLanguage = (language: string): language is Language => languages.includes(language as Language) const run = async (toolbox: GluegunToolbox) => { const { config, prompt } = toolbox const id = config.runId ? Number(config.runId) : undefined - let { language, exercise } = config + const { language, exercise } = config if (language === "all") { - const run = await findOrCreateRun({ id }) - await runAll(run) + await runAll(id) } else if (exercise === "all") { - const run = await findOrCreateRun({ id }) - await runLanguage({ run, language }) + await runLanguage({ id, language }) } else { - language = language || (await askLanguage(prompt)) - exercise = exercise || (await askExercise(prompt, language)) - const run = await findOrCreateRun({ id }) + await runLanguageExercise({ + id, + language: language || (await askLanguage(prompt)), + exercise: exercise || (await askExercise(prompt, language)), + }) + } +} + +const runAll = async (id?: number) => { + const run = await findOrCreateRun({ id }) + const exercises = getExercises() + + for (const [language, languageExercises] of Object.entries(exercises)) { + await Promise.all( + languageExercises.map((exercise) => + findOrCreatePendingTask({ runId: run.id, language: language as Language, exercise }), + ), + ) + } + + for (const [language, languageExercises] of Object.entries(exercises)) { + for (const exercise of languageExercises) { + await runExercise({ run, language: language as Language, exercise }) + } + } +} + +const runLanguage = async ({ id, language }: { id?: number; language: Language }) => { + const run = await findOrCreateRun({ id }) + const exercises = getExercises() + const languageExercises = exercises[language] + + await Promise.all( + languageExercises.map((exercise) => findOrCreatePendingTask({ runId: run.id, language, exercise })), + ) + + for (const exercise of languageExercises) { await runExercise({ run, language, exercise }) } } -const runAll = async (run: Run) => - (await pMap(languages, (language) => runLanguage({ run, language }), { concurrency: 1 })).flatMap( - (language) => language, - ) - -const runLanguage = async ({ run, language }: { run: Run; language: Language }) => { - const languagePath = path.resolve(exercisesPath, language) - - if (!fs.existsSync(languagePath)) { - console.error(`Language directory ${languagePath} does not exist`) - process.exit(1) - } - - const exercises = filesystem - .subdirectories(languagePath) - .map((exercise) => path.basename(exercise)) - .filter((exercise) => !exercise.startsWith(".")) - - const results = await pMap( - exercises, - async (exercise) => ({ - language, - exercise, - result: await runExercise({ run, language, exercise }), - }), - { concurrency: 1 }, - ) - - return results +const runLanguageExercise = async ({ + id, + language, + exercise, +}: { + id?: number + language: Language + exercise: string +}) => { + const run = await findOrCreateRun({ id }) + await findOrCreatePendingTask({ runId: run.id, language, exercise }) + return runExercise({ run, language, exercise }) } const runExercise = async ({ run, language, exercise }: { run: Run; language: Language; exercise: string }) => { @@ -127,10 +149,20 @@ const askExercise = async (prompt: GluegunPrompt, language: Language) => { return exercise } -type FindOrCreateRun = { id?: number; model?: string } +const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: { id?: number; model?: string }) => + id + ? findRun(id) + : createRun({ model, pid: process.pid, socketPath: path.resolve(os.tmpdir(), `benchmark-${uuidv4()}.sock`) }) -const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: FindOrCreateRun) => - id ? findRun(id) : createRun({ model }) +const findOrCreatePendingTask = async ({ + runId, + language, + exercise, +}: { + runId: number + language: Language + exercise: string +}) => (await getPendingTask({ runId, language, exercise })) || (await createPendingTask({ runId, language, exercise })) const main = async () => { const cli = build() diff --git a/benchmark/apps/cli/src/paths.ts b/benchmark/apps/cli/src/paths.ts new file mode 100644 index 0000000000..5f046510a8 --- /dev/null +++ b/benchmark/apps/cli/src/paths.ts @@ -0,0 +1,8 @@ +import * as path from "path" +import { fileURLToPath } from "url" + +export const __dirname = path.dirname(fileURLToPath(import.meta.url)) + +export const extensionDevelopmentPath = path.resolve(__dirname, "..", "..", "..", "..") +export const extensionTestsPath = path.resolve(extensionDevelopmentPath, "benchmark/packages/runner/dist") +export const exercisesPath = path.resolve(extensionDevelopmentPath, "..", "exercises") diff --git a/benchmark/packages/db/src/index.ts b/benchmark/packages/db/src/index.ts index ab36bafa14..b15bf68231 100644 --- a/benchmark/packages/db/src/index.ts +++ b/benchmark/packages/db/src/index.ts @@ -15,3 +15,10 @@ export { findRun, createRun, getRuns } from "./queries/runs.js" export { type Task, type InsertTask, insertTaskSchema } from "./schema.js" export { findTask, createTask, getTask } from "./queries/tasks.js" + +/** + * pendingTasks + */ + +export { type PendingTask, type InsertPendingTask, insertPendingTaskSchema } from "./schema.js" +export { findPendingTask, createPendingTask, getPendingTask } from "./queries/pendingTasks.js" diff --git a/benchmark/packages/db/src/queries/pendingTasks.ts b/benchmark/packages/db/src/queries/pendingTasks.ts new file mode 100644 index 0000000000..08bc62e6bc --- /dev/null +++ b/benchmark/packages/db/src/queries/pendingTasks.ts @@ -0,0 +1,50 @@ +import { and, eq } from "drizzle-orm" + +import { db } from "../db.js" +import { type InsertPendingTask, insertPendingTaskSchema, pendingTasks } from "../schema.js" + +import { Language } from "../enums.js" +import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js" + +export const findPendingTask = async (id: number) => { + const run = await db.query.pendingTasks.findFirst({ where: eq(pendingTasks.id, id) }) + + if (!run) { + throw new RecordNotFoundError() + } + + return run +} + +export const createPendingTask = async (args: InsertPendingTask) => { + const result = await db + .insert(pendingTasks) + .values({ + ...insertPendingTaskSchema.parse(args), + createdAt: new Date(), + }) + .returning() + + const task = result[0] + + if (!task) { + throw new RecordNotCreatedError() + } + + return task +} + +type GetPendingTask = { + runId: number + language: Language + exercise: string +} + +export const getPendingTask = async ({ runId, language, exercise }: GetPendingTask) => + db.query.pendingTasks.findFirst({ + where: and( + eq(pendingTasks.runId, runId), + eq(pendingTasks.language, language), + eq(pendingTasks.exercise, exercise), + ), + }) diff --git a/benchmark/packages/db/src/schema.ts b/benchmark/packages/db/src/schema.ts index 51b3eb38a3..7c4d57ec61 100644 --- a/benchmark/packages/db/src/schema.ts +++ b/benchmark/packages/db/src/schema.ts @@ -90,4 +90,4 @@ export type InsertPendingTask = z.infer * schema */ -export const schema = { runs, tasks } +export const schema = { runs, tasks, pendingTasks } diff --git a/benchmark/pnpm-lock.yaml b/benchmark/pnpm-lock.yaml index 0383e6099f..27867bad6f 100644 --- a/benchmark/pnpm-lock.yaml +++ b/benchmark/pnpm-lock.yaml @@ -53,9 +53,9 @@ importers: gluegun: specifier: ^5.1.2 version: 5.2.0 - p-map: - specifier: ^7.0.3 - version: 7.0.3 + uuid: + specifier: ^11.1.0 + version: 11.1.0 devDependencies: '@benchmark/eslint-config': specifier: workspace:^ @@ -2972,10 +2972,6 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} - p-map@7.0.3: - resolution: {integrity: sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==} - engines: {node: '>=18'} - package-json-from-dist@1.0.1: resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==} @@ -6274,8 +6270,6 @@ snapshots: dependencies: p-limit: 3.1.0 - p-map@7.0.3: {} - package-json-from-dist@1.0.1: {} pako@1.0.11: {}