Create pending tasks

benchmark/apps/cli/package.json

@@ -12,7 +12,7 @@
 		"@benchmark/db": "workspace:^",
 		"@vscode/test-electron": "^2.4.0",
 		"gluegun": "^5.1.2",
-		"p-map": "^7.0.3"
+		"uuid": "^11.1.0"
 	},
 	"devDependencies": {
 		"@benchmark/eslint-config": "workspace:^",

benchmark/apps/cli/src/exercises.ts

@@ -0,0 +1,31 @@
+import * as path from "path"
+import * as fs from "fs"
+
+import { filesystem } from "gluegun"
+
+import { type Language, languages } from "@benchmark/db"
+
+import { exercisesPath } from "./paths.js"
+
+let exercisesByLanguage: Record<Language, string[]> | null = null
+
+export const getExercises = () => {
+	if (exercisesByLanguage !== null) {
+		return exercisesByLanguage
+	}
+
+	const getLanguageExercises = (language: Language) =>
+		fs.existsSync(path.resolve(exercisesPath, language))
+			? filesystem
+					.subdirectories(path.resolve(exercisesPath, language))
+					.map((exercise) => path.basename(exercise))
+					.filter((exercise) => !exercise.startsWith("."))
+			: []
+
+	exercisesByLanguage = languages.reduce(
+		(collect, language) => ({ ...collect, [language]: getLanguageExercises(language) }),
+		{} as Record<Language, string[]>,
+	)
+
+	return exercisesByLanguage
+}

benchmark/apps/cli/src/index.ts

@@ -1,68 +1,90 @@
 import * as fs from "fs"
 import * as path from "path"
-import { fileURLToPath } from "url"
-import pMap from "p-map"
+import * as os from "os"
 
+import { v4 as uuidv4 } from "uuid"
 import { build, filesystem, GluegunPrompt, GluegunToolbox } from "gluegun"
 import { runTests } from "@vscode/test-electron"
 
-import { type Language, languages, type Run, findRun, createRun, getTask } from "@benchmark/db"
+import {
+	type Language,
+	languages,
+	type Run,
+	findRun,
+	createRun,
+	getPendingTask,
+	createPendingTask,
+	getTask,
+} from "@benchmark/db"
 
-const __dirname = path.dirname(fileURLToPath(import.meta.url))
-const extensionDevelopmentPath = path.resolve(__dirname, "..", "..", "..", "..")
-const extensionTestsPath = path.resolve(extensionDevelopmentPath, "benchmark/packages/runner/dist")
-const exercisesPath = path.resolve(extensionDevelopmentPath, "..", "exercises")
+import { __dirname, extensionDevelopmentPath, extensionTestsPath, exercisesPath } from "./paths.js"
+import { getExercises } from "./exercises.js"
 
 export const isLanguage = (language: string): language is Language => languages.includes(language as Language)
 
 const run = async (toolbox: GluegunToolbox) => {
 	const { config, prompt } = toolbox
 	const id = config.runId ? Number(config.runId) : undefined
-	let { language, exercise } = config
+	const { language, exercise } = config
 
 	if (language === "all") {
-		const run = await findOrCreateRun({ id })
-		await runAll(run)
+		await runAll(id)
 	} else if (exercise === "all") {
-		const run = await findOrCreateRun({ id })
-		await runLanguage({ run, language })
+		await runLanguage({ id, language })
 	} else {
-		language = language || (await askLanguage(prompt))
-		exercise = exercise || (await askExercise(prompt, language))
-		const run = await findOrCreateRun({ id })
+		await runLanguageExercise({
+			id,
+			language: language || (await askLanguage(prompt)),
+			exercise: exercise || (await askExercise(prompt, language)),
+		})
+	}
+}
+
+const runAll = async (id?: number) => {
+	const run = await findOrCreateRun({ id })
+	const exercises = getExercises()
+
+	for (const [language, languageExercises] of Object.entries(exercises)) {
+		await Promise.all(
+			languageExercises.map((exercise) =>
+				findOrCreatePendingTask({ runId: run.id, language: language as Language, exercise }),
+			),
+		)
+	}
+
+	for (const [language, languageExercises] of Object.entries(exercises)) {
+		for (const exercise of languageExercises) {
+			await runExercise({ run, language: language as Language, exercise })
+		}
+	}
+}
+
+const runLanguage = async ({ id, language }: { id?: number; language: Language }) => {
+	const run = await findOrCreateRun({ id })
+	const exercises = getExercises()
+	const languageExercises = exercises[language]
+
+	await Promise.all(
+		languageExercises.map((exercise) => findOrCreatePendingTask({ runId: run.id, language, exercise })),
+	)
+
+	for (const exercise of languageExercises) {
 		await runExercise({ run, language, exercise })
 	}
 }
 
-const runAll = async (run: Run) =>
-	(await pMap(languages, (language) => runLanguage({ run, language }), { concurrency: 1 })).flatMap(
-		(language) => language,
-	)
-
-const runLanguage = async ({ run, language }: { run: Run; language: Language }) => {
-	const languagePath = path.resolve(exercisesPath, language)
-
-	if (!fs.existsSync(languagePath)) {
-		console.error(`Language directory ${languagePath} does not exist`)
-		process.exit(1)
-	}
-
-	const exercises = filesystem
-		.subdirectories(languagePath)
-		.map((exercise) => path.basename(exercise))
-		.filter((exercise) => !exercise.startsWith("."))
-
-	const results = await pMap(
-		exercises,
-		async (exercise) => ({
-			language,
-			exercise,
-			result: await runExercise({ run, language, exercise }),
-		}),
-		{ concurrency: 1 },
-	)
-
-	return results
+const runLanguageExercise = async ({
+	id,
+	language,
+	exercise,
+}: {
+	id?: number
+	language: Language
+	exercise: string
+}) => {
+	const run = await findOrCreateRun({ id })
+	await findOrCreatePendingTask({ runId: run.id, language, exercise })
+	return runExercise({ run, language, exercise })
 }
 
 const runExercise = async ({ run, language, exercise }: { run: Run; language: Language; exercise: string }) => {
@@ -127,10 +149,20 @@ const askExercise = async (prompt: GluegunPrompt, language: Language) => {
 	return exercise
 }
 
-type FindOrCreateRun = { id?: number; model?: string }
+const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: { id?: number; model?: string }) =>
+	id
+		? findRun(id)
+		: createRun({ model, pid: process.pid, socketPath: path.resolve(os.tmpdir(), `benchmark-${uuidv4()}.sock`) })
 
-const findOrCreateRun = async ({ id, model = "anthropic/claude-3.7-sonnet" }: FindOrCreateRun) =>
-	id ? findRun(id) : createRun({ model })
+const findOrCreatePendingTask = async ({
+	runId,
+	language,
+	exercise,
+}: {
+	runId: number
+	language: Language
+	exercise: string
+}) => (await getPendingTask({ runId, language, exercise })) || (await createPendingTask({ runId, language, exercise }))
 
 const main = async () => {
 	const cli = build()

benchmark/apps/cli/src/paths.ts

@@ -0,0 +1,8 @@
+import * as path from "path"
+import { fileURLToPath } from "url"
+
+export const __dirname = path.dirname(fileURLToPath(import.meta.url))
+
+export const extensionDevelopmentPath = path.resolve(__dirname, "..", "..", "..", "..")
+export const extensionTestsPath = path.resolve(extensionDevelopmentPath, "benchmark/packages/runner/dist")
+export const exercisesPath = path.resolve(extensionDevelopmentPath, "..", "exercises")

benchmark/packages/db/src/index.ts

@@ -15,3 +15,10 @@ export { findRun, createRun, getRuns } from "./queries/runs.js"
 
 export { type Task, type InsertTask, insertTaskSchema } from "./schema.js"
 export { findTask, createTask, getTask } from "./queries/tasks.js"
+
+/**
+ * pendingTasks
+ */
+
+export { type PendingTask, type InsertPendingTask, insertPendingTaskSchema } from "./schema.js"
+export { findPendingTask, createPendingTask, getPendingTask } from "./queries/pendingTasks.js"

benchmark/packages/db/src/queries/pendingTasks.ts

@@ -0,0 +1,50 @@
+import { and, eq } from "drizzle-orm"
+
+import { db } from "../db.js"
+import { type InsertPendingTask, insertPendingTaskSchema, pendingTasks } from "../schema.js"
+
+import { Language } from "../enums.js"
+import { RecordNotFoundError, RecordNotCreatedError } from "./errors.js"
+
+export const findPendingTask = async (id: number) => {
+	const run = await db.query.pendingTasks.findFirst({ where: eq(pendingTasks.id, id) })
+
+	if (!run) {
+		throw new RecordNotFoundError()
+	}
+
+	return run
+}
+
+export const createPendingTask = async (args: InsertPendingTask) => {
+	const result = await db
+		.insert(pendingTasks)
+		.values({
+			...insertPendingTaskSchema.parse(args),
+			createdAt: new Date(),
+		})
+		.returning()
+
+	const task = result[0]
+
+	if (!task) {
+		throw new RecordNotCreatedError()
+	}
+
+	return task
+}
+
+type GetPendingTask = {
+	runId: number
+	language: Language
+	exercise: string
+}
+
+export const getPendingTask = async ({ runId, language, exercise }: GetPendingTask) =>
+	db.query.pendingTasks.findFirst({
+		where: and(
+			eq(pendingTasks.runId, runId),
+			eq(pendingTasks.language, language),
+			eq(pendingTasks.exercise, exercise),
+		),
+	})

benchmark/packages/db/src/schema.ts

@@ -90,4 +90,4 @@ export type InsertPendingTask = z.infer<typeof insertPendingTaskSchema>
  * schema
  */
 
-export const schema = { runs, tasks }
+export const schema = { runs, tasks, pendingTasks }

benchmark/pnpm-lock.yaml

@@ -53,9 +53,9 @@ importers:
       gluegun:
         specifier: ^5.1.2
         version: 5.2.0
-      p-map:
-        specifier: ^7.0.3
-        version: 7.0.3
+      uuid:
+        specifier: ^11.1.0
+        version: 11.1.0
     devDependencies:
       '@benchmark/eslint-config':
         specifier: workspace:^
@@ -2972,10 +2972,6 @@ packages:
     resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==}
     engines: {node: '>=10'}
 
-  p-map@7.0.3:
-    resolution: {integrity: sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==}
-    engines: {node: '>=18'}
-
   package-json-from-dist@1.0.1:
     resolution: {integrity: sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==}
 
@@ -6274,8 +6270,6 @@ snapshots:
     dependencies:
       p-limit: 3.1.0
 
-  p-map@7.0.3: {}
-
   package-json-from-dist@1.0.1: {}
 
   pako@1.0.11: {}