Roo-Code/packages/evals/src/cli/runEvals.ts

import PQueue from "p-queue"

import { findRun, finishRun, getTasks } from "../db/index.js"
import { EVALS_REPO_PATH } from "../exercises/index.js"

import { Logger, getTag, isDockerContainer, resetEvalsRepo, commitEvalsRepoChanges } from "./utils.js"
import { startHeartbeat, stopHeartbeat } from "./redis.js"
import { processTask, processTaskInContainer } from "./runTask.js"

export const runEvals = async (runId: number) => {
	const run = await findRun(runId)

	if (run.taskMetricsId) {
		throw new Error(`Run ${run.id} already finished.`)
	}

	const tasks = await getTasks(runId)

	if (tasks.length === 0) {
		throw new Error(`Run ${run.id} has no tasks.`)
	}

	const containerized = isDockerContainer()

	const logger = new Logger({
		logDir: containerized ? `/var/log/evals/runs/${run.id}` : `/tmp/evals/runs/${run.id}`,
		filename: `controller.log`,
		tag: getTag("runEvals", { run }),
	})

	logger.info(`running ${tasks.length} task(s)`)

	if (!containerized) {
		await resetEvalsRepo({ run, cwd: EVALS_REPO_PATH })
	}

	const heartbeat = await startHeartbeat(run.id)
	const queue = new PQueue({ concurrency: run.concurrency })

	try {
		await queue.addAll(
			tasks
				.filter((task) => task.finishedAt === null)
				.map((task) => async () => {
					try {
						if (containerized) {
							await processTaskInContainer({ taskId: task.id, logger })
						} else {
							await processTask({ taskId: task.id, logger })
						}
					} catch (error) {
						logger.error("error processing task", error)
					}
				}),
		)

		logger.info("finishRun")
		const result = await finishRun(run.id)
		logger.info("result ->", result)

		// There's no need to commit the changes in the container since they
		// will lost when the container is destroyed. I think we should
		// store the diffs in the database instead.
		if (!containerized) {
			await commitEvalsRepoChanges({ run, cwd: EVALS_REPO_PATH })
		}
	} finally {
		logger.info("cleaning up")
		stopHeartbeat(run.id, heartbeat)
		logger.close()
	}
}