More small evals tweaks (#2620)

evals/apps/cli/src/index.ts

@@ -372,7 +372,7 @@ const runUnitTest = async ({ task }: { task: Task }) => {
 				})
 
 				console.log(
-					`${Date.now()} [cli#runUnitTest | ${task.language} / ${task.exercise}] "${command.join(" ")}": ${subprocess.pid} -> ${JSON.stringify(descendants)}`,
+					`${Date.now()} [cli#runUnitTest | ${task.language} / ${task.exercise}] "${command.join(" ")}": unit tests timed out, killing ${subprocess.pid} + ${JSON.stringify(descendants)}`,
 				)
 
 				if (descendants.length > 0) {
@@ -384,7 +384,10 @@ const runUnitTest = async ({ task }: { task: Task }) => {
 
 							await execa`kill -9 ${descendant}`
 						} catch (error) {
-							console.error("Error killing descendant processes:", error)
+							console.error(
+								`${Date.now()} [cli#runUnitTest | ${task.language} / ${task.exercise}] Error killing descendant processes:`,
+								error,
+							)
 						}
 					}
 				}
@@ -396,7 +399,10 @@ const runUnitTest = async ({ task }: { task: Task }) => {
 				try {
 					await execa`kill -9 ${subprocess.pid!}`
 				} catch (error) {
-					console.error("Error killing process:", error)
+					console.error(
+						`${Date.now()} [cli#runUnitTest | ${task.language} / ${task.exercise}] Error killing process:`,
+						error,
+					)
 				}
 			}, UNIT_TEST_TIMEOUT)
 

evals/packages/db/package.json

@@ -23,6 +23,7 @@
 		"@libsql/client": "^0.14.0",
 		"drizzle-orm": "^0.40.0",
 		"drizzle-zod": "^0.7.0",
+		"p-map": "^7.0.3",
 		"zod": "^3.24.2"
 	},
 	"devDependencies": {

evals/packages/db/scripts/copy-run.mts

@@ -1,5 +1,6 @@
 import { drizzle } from "drizzle-orm/libsql"
 import { eq } from "drizzle-orm"
+import pMap from "p-map"
 
 import { db as sourceDb } from "../src/db.js"
 import { schema } from "../src/schema.js"
@@ -52,29 +53,33 @@ const copyRun = async (runId: number) => {
 
 	console.log(`Copying ${tasks.length} tasks`)
 
-	for (const task of tasks) {
-		// eslint-disable-next-line @typescript-eslint/no-unused-vars
-		const { id: _, ...newTaskMetricsValues } = task.taskMetrics!
-		const [newTaskMetrics] = await destDb.insert(schema.taskMetrics).values(newTaskMetricsValues).returning()
+	await pMap(
+		tasks,
+		async (task) => {
+			// eslint-disable-next-line @typescript-eslint/no-unused-vars
+			const { id: _, ...newTaskMetricsValues } = task.taskMetrics!
+			const [newTaskMetrics] = await destDb.insert(schema.taskMetrics).values(newTaskMetricsValues).returning()
 
-		if (!newTaskMetrics) {
-			throw new Error(`Failed to insert taskMetrics for task ${task.id}`)
-		}
+			if (!newTaskMetrics) {
+				throw new Error(`Failed to insert taskMetrics for task ${task.id}`)
+			}
 
-		// eslint-disable-next-line @typescript-eslint/no-unused-vars
-		const { id: __, ...newTaskValues } = task
+			// eslint-disable-next-line @typescript-eslint/no-unused-vars
+			const { id: __, ...newTaskValues } = task
 
-		const [newTask] = await destDb
-			.insert(schema.tasks)
-			.values({ ...newTaskValues, runId: newRun.id, taskMetricsId: newTaskMetrics.id })
-			.returning()
+			const [newTask] = await destDb
+				.insert(schema.tasks)
+				.values({ ...newTaskValues, runId: newRun.id, taskMetricsId: newTaskMetrics.id })
+				.returning()
 
-		if (!newTask) {
-			throw new Error(`Failed to insert task ${task.id}`)
-		}
-	}
+			if (!newTask) {
+				throw new Error(`Failed to insert task ${task.id}`)
+			}
+		},
+		{ concurrency: 25 },
+	)
 
-	console.log(`Successfully copied run ${runId} with ${tasks.length} tasks`)
+	console.log(`\nSuccessfully copied run ${runId} with ${tasks.length} tasks`)
 }
 
 const main = async () => {

evals/packages/types/src/roo-code-defaults.ts

@@ -6,6 +6,7 @@ export const rooCodeDefaults: RooCodeSettings = {
 
 	// modelTemperature: null,
 	// reasoningEffort: "high",
+	rateLimitSeconds: 0,
 
 	pinnedApiConfigs: {},
 	lastShownAnnouncementId: "apr-04-2025-boomerang",
@@ -45,11 +46,11 @@ export const rooCodeDefaults: RooCodeSettings = {
 
 	terminalOutputLineLimit: 500,
 	terminalShellIntegrationTimeout: 15_000,
-	// terminalCommandDelay: 0,
-	// terminalPowershellCounter: false,
+	terminalCommandDelay: 0,
+	terminalPowershellCounter: false,
 	terminalZshClearEolMark: true,
 	terminalZshOhMy: true,
-	// terminalZshP10k: false,
+	terminalZshP10k: false,
 	terminalZdotdir: true,
 
 	diffEnabled: true,

evals/pnpm-lock.yaml

@@ -258,6 +258,9 @@ importers:
       drizzle-zod:
         specifier: ^0.7.0
         version: 0.7.0(drizzle-orm@0.40.1(@libsql/client@0.14.0)(gel@2.0.1))(zod@3.24.2)
+      p-map:
+        specifier: ^7.0.3
+        version: 7.0.3
       zod:
         specifier: ^3.24.2
         version: 3.24.2