feat: implement global FIFO queue for Evals runs

- Add Redis-based queue management with run-queue, active-run, and dispatcher lock
- Modify createRun to enqueue runs instead of immediate spawning
- Implement auto-advance mechanism when runs complete
- Add UI status column showing Running/Queued/Completed states
- Add queue position display for queued runs
- Add cancel button for queued runs
- Preserve per-run task concurrency via PQueue

Addresses issue #7966

apps/web-evals/src/actions/runs.ts

@@ -15,12 +15,118 @@ import {
 	deleteRun as _deleteRun,
 	createTask,
 	getExercisesForLanguage,
+	findRun,
 } from "@roo-code/evals"
 
 import { CreateRun } from "@/lib/schemas"
+import { redisClient } from "@/lib/server/redis"
 
 const EVALS_REPO_PATH = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../../../evals")
 
+// Queue management keys (matching the ones in packages/evals/src/cli/redis.ts)
+const getRunQueueKey = () => `evals:run-queue`
+const getActiveRunKey = () => `evals:active-run`
+const getDispatcherLockKey = () => `evals:dispatcher:lock`
+
+async function spawnController(runId: number) {
+	const isRunningInDocker = fs.existsSync("/.dockerenv")
+
+	const dockerArgs = [
+		`--name evals-controller-${runId}`,
+		"--rm",
+		"--network evals_default",
+		"-v /var/run/docker.sock:/var/run/docker.sock",
+		"-v /tmp/evals:/var/log/evals",
+		"-e HOST_EXECUTION_METHOD=docker",
+	]
+
+	const cliCommand = `pnpm --filter @roo-code/evals cli --runId ${runId}`
+
+	const command = isRunningInDocker
+		? `docker run ${dockerArgs.join(" ")} evals-runner sh -c "${cliCommand}"`
+		: cliCommand
+
+	console.log("spawn ->", command)
+
+	const childProcess = spawn("sh", ["-c", command], {
+		detached: true,
+		stdio: ["ignore", "pipe", "pipe"],
+	})
+
+	const logStream = fs.createWriteStream("/tmp/roo-code-evals.log", { flags: "a" })
+
+	if (childProcess.stdout) {
+		childProcess.stdout.pipe(logStream)
+	}
+
+	if (childProcess.stderr) {
+		childProcess.stderr.pipe(logStream)
+	}
+
+	childProcess.unref()
+}
+
+export async function dispatchNextRun() {
+	const redis = await redisClient()
+
+	// Try to acquire dispatcher lock (10 second TTL)
+	const lockAcquired = await redis.set(getDispatcherLockKey(), Date.now().toString(), {
+		NX: true,
+		EX: 10,
+	})
+
+	if (lockAcquired !== "OK") {
+		console.log("Dispatcher lock already held, skipping dispatch")
+		return
+	}
+
+	try {
+		// Check if there's already an active run
+		const activeRunId = await redis.get(getActiveRunKey())
+		if (activeRunId) {
+			console.log(`Run ${activeRunId} is already active, skipping dispatch`)
+			return
+		}
+
+		// Pop the next run from the queue
+		const nextRunId = await redis.lPop(getRunQueueKey())
+		if (!nextRunId) {
+			console.log("No runs in queue")
+			return
+		}
+
+		const runId = parseInt(nextRunId, 10)
+		console.log(`Dispatching run ${runId}`)
+
+		// Set as active run with generous TTL (1 hour default, will be cleared when run completes)
+		const setActive = await redis.set(getActiveRunKey(), runId.toString(), {
+			NX: true,
+			EX: 3600,
+		})
+
+		if (setActive !== "OK") {
+			// Another process may have set an active run, put this run back in the queue
+			console.log("Failed to set active run, requeueing")
+			await redis.lPush(getRunQueueKey(), runId.toString())
+			return
+		}
+
+		// Spawn the controller for this run
+		try {
+			await spawnController(runId)
+			console.log(`Successfully spawned controller for run ${runId}`)
+		} catch (error) {
+			console.error(`Failed to spawn controller for run ${runId}:`, error)
+			// Clear active run and requeue on spawn failure
+			await redis.del(getActiveRunKey())
+			await redis.lPush(getRunQueueKey(), runId.toString())
+		}
+	} finally {
+		// Release dispatcher lock
+		await redis.del(getDispatcherLockKey())
+	}
+}
+
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
 export async function createRun({ suite, exercises = [], systemPrompt, timeout, ...values }: CreateRun) {
 	const run = await _createRun({
@@ -51,44 +157,16 @@ export async function createRun({ suite, exercises = [], systemPrompt, timeout,
 
 	revalidatePath("/runs")
 
+	// Add run to queue
+	const redis = await redisClient()
+	await redis.rPush(getRunQueueKey(), run.id.toString())
+	console.log(`Run ${run.id} added to queue`)
+
+	// Try to dispatch if no active run
 	try {
-		const isRunningInDocker = fs.existsSync("/.dockerenv")
-
-		const dockerArgs = [
-			`--name evals-controller-${run.id}`,
-			"--rm",
-			"--network evals_default",
-			"-v /var/run/docker.sock:/var/run/docker.sock",
-			"-v /tmp/evals:/var/log/evals",
-			"-e HOST_EXECUTION_METHOD=docker",
-		]
-
-		const cliCommand = `pnpm --filter @roo-code/evals cli --runId ${run.id}`
-
-		const command = isRunningInDocker
-			? `docker run ${dockerArgs.join(" ")} evals-runner sh -c "${cliCommand}"`
-			: cliCommand
-
-		console.log("spawn ->", command)
-
-		const childProcess = spawn("sh", ["-c", command], {
-			detached: true,
-			stdio: ["ignore", "pipe", "pipe"],
-		})
-
-		const logStream = fs.createWriteStream("/tmp/roo-code-evals.log", { flags: "a" })
-
-		if (childProcess.stdout) {
-			childProcess.stdout.pipe(logStream)
-		}
-
-		if (childProcess.stderr) {
-			childProcess.stderr.pipe(logStream)
-		}
-
-		childProcess.unref()
+		await dispatchNextRun()
 	} catch (error) {
-		console.error(error)
+		console.error("Error dispatching run:", error)
 	}
 
 	return run
@@ -98,3 +176,51 @@ export async function deleteRun(runId: number) {
 	await _deleteRun(runId)
 	revalidatePath("/runs")
 }
+
+export async function cancelQueuedRun(runId: number) {
+	const redis = await redisClient()
+
+	// Remove from queue
+	const removed = await redis.lRem(getRunQueueKey(), 1, runId.toString())
+
+	if (removed > 0) {
+		console.log(`Removed run ${runId} from queue`)
+		// Delete the run from database
+		await deleteRun(runId)
+		return true
+	}
+
+	return false
+}
+
+export async function getRunQueueStatus(runId: number) {
+	const redis = await redisClient()
+
+	// Check if run is active
+	const activeRunId = await redis.get(getActiveRunKey())
+	if (activeRunId === runId.toString()) {
+		return { status: "running" as const, position: null }
+	}
+
+	// Check position in queue
+	const queue = await redis.lRange(getRunQueueKey(), 0, -1)
+	const position = queue.indexOf(runId.toString())
+
+	if (position !== -1) {
+		return { status: "queued" as const, position: position + 1 }
+	}
+
+	// Check if run has a heartbeat (running but not marked as active - edge case)
+	const heartbeat = await redis.get(`heartbeat:${runId}`)
+	if (heartbeat) {
+		return { status: "running" as const, position: null }
+	}
+
+	// Run is completed or not found
+	const run = await findRun(runId)
+	if (run?.taskMetricsId) {
+		return { status: "completed" as const, position: null }
+	}
+
+	return { status: "unknown" as const, position: null }
+}

apps/web-evals/src/components/home/run.tsx

@@ -1,10 +1,11 @@
-import { useCallback, useState, useRef } from "react"
+import { useCallback, useState, useRef, useEffect } from "react"
 import Link from "next/link"
-import { Ellipsis, ClipboardList, Copy, Check, LoaderCircle, Trash } from "lucide-react"
+import { Ellipsis, ClipboardList, Copy, Check, LoaderCircle, Trash, X, Clock, Play, CheckCircle } from "lucide-react"
 
 import type { Run as EvalsRun, TaskMetrics as EvalsTaskMetrics } from "@roo-code/evals"
 
-import { deleteRun } from "@/actions/runs"
+import { deleteRun, cancelQueuedRun, getRunQueueStatus } from "@/actions/runs"
+import { getHeartbeat } from "@/actions/heartbeat"
 import { formatCurrency, formatDuration, formatTokens, formatToolUsageSuccessRate } from "@/lib/formatters"
 import { useCopyRun } from "@/hooks/use-copy-run"
 import {
@@ -23,6 +24,7 @@ import {
 	AlertDialogFooter,
 	AlertDialogHeader,
 	AlertDialogTitle,
+	Badge,
 } from "@/components/ui"
 
 type RunProps = {
@@ -30,11 +32,57 @@ type RunProps = {
 	taskMetrics: EvalsTaskMetrics | null
 }
 
+type RunStatus = {
+	status: "running" | "queued" | "completed" | "unknown"
+	position: number | null
+}
+
 export function Run({ run, taskMetrics }: RunProps) {
 	const [deleteRunId, setDeleteRunId] = useState<number>()
+	const [runStatus, setRunStatus] = useState<RunStatus>({ status: "unknown", position: null })
+	const [isLoadingStatus, setIsLoadingStatus] = useState(true)
+	const [isCancelling, setIsCancelling] = useState(false)
 	const continueRef = useRef<HTMLButtonElement>(null)
 	const { isPending, copyRun, copied } = useCopyRun(run.id)
 
+	// Fetch run status on mount and periodically
+	useEffect(() => {
+		const fetchStatus = async () => {
+			try {
+				// First check if run is completed
+				if (run.taskMetricsId) {
+					setRunStatus({ status: "completed", position: null })
+					setIsLoadingStatus(false)
+					return
+				}
+
+				// Check heartbeat for running status
+				const heartbeat = await getHeartbeat(run.id)
+				if (heartbeat) {
+					setRunStatus({ status: "running", position: null })
+					setIsLoadingStatus(false)
+					return
+				}
+
+				// Get queue status
+				const status = await getRunQueueStatus(run.id)
+				setRunStatus(status)
+			} catch (error) {
+				console.error("Error fetching run status:", error)
+			} finally {
+				setIsLoadingStatus(false)
+			}
+		}
+
+		fetchStatus()
+		// Refresh status every 5 seconds for non-completed runs
+		const interval = !run.taskMetricsId ? setInterval(fetchStatus, 5000) : null
+
+		return () => {
+			if (interval) clearInterval(interval)
+		}
+	}, [run.id, run.taskMetricsId])
+
 	const onConfirmDelete = useCallback(async () => {
 		if (!deleteRunId) {
 			return
@@ -48,9 +96,57 @@ export function Run({ run, taskMetrics }: RunProps) {
 		}
 	}, [deleteRunId])
 
+	const handleCancelQueued = useCallback(async () => {
+		setIsCancelling(true)
+		try {
+			const cancelled = await cancelQueuedRun(run.id)
+			if (cancelled) {
+				// Refresh the page to update the list
+				window.location.reload()
+			}
+		} catch (error) {
+			console.error("Error cancelling queued run:", error)
+		} finally {
+			setIsCancelling(false)
+		}
+	}, [run.id])
+
+	const getStatusBadge = () => {
+		if (isLoadingStatus) {
+			return <Badge variant="secondary">Loading...</Badge>
+		}
+
+		switch (runStatus.status) {
+			case "running":
+				return (
+					<Badge variant="default" className="bg-green-600">
+						<Play className="size-3 mr-1" />
+						Running
+					</Badge>
+				)
+			case "queued":
+				return (
+					<Badge variant="secondary">
+						<Clock className="size-3 mr-1" />
+						Queued #{runStatus.position}
+					</Badge>
+				)
+			case "completed":
+				return (
+					<Badge variant="outline">
+						<CheckCircle className="size-3 mr-1" />
+						Completed
+					</Badge>
+				)
+			default:
+				return <Badge variant="outline">Unknown</Badge>
+		}
+	}
+
 	return (
 		<>
 			<TableRow>
+				<TableCell>{getStatusBadge()}</TableCell>
 				<TableCell>{run.model}</TableCell>
 				<TableCell>{run.passed}</TableCell>
 				<TableCell>{run.failed}</TableCell>
@@ -79,55 +175,71 @@ export function Run({ run, taskMetrics }: RunProps) {
 				<TableCell>{taskMetrics && formatCurrency(taskMetrics.cost)}</TableCell>
 				<TableCell>{taskMetrics && formatDuration(taskMetrics.duration)}</TableCell>
 				<TableCell>
-					<DropdownMenu>
-						<Button variant="ghost" size="icon" asChild>
-							<DropdownMenuTrigger>
-								<Ellipsis />
-							</DropdownMenuTrigger>
-						</Button>
-						<DropdownMenuContent align="end">
-							<DropdownMenuItem asChild>
-								<Link href={`/runs/${run.id}`}>
+					<div className="flex items-center gap-2">
+						{runStatus.status === "queued" && (
+							<Button
+								variant="ghost"
+								size="icon"
+								onClick={handleCancelQueued}
+								disabled={isCancelling}
+								title="Cancel queued run">
+								{isCancelling ? (
+									<LoaderCircle className="animate-spin size-4" />
+								) : (
+									<X className="size-4" />
+								)}
+							</Button>
+						)}
+						<DropdownMenu>
+							<Button variant="ghost" size="icon" asChild>
+								<DropdownMenuTrigger>
+									<Ellipsis />
+								</DropdownMenuTrigger>
+							</Button>
+							<DropdownMenuContent align="end">
+								<DropdownMenuItem asChild>
+									<Link href={`/runs/${run.id}`}>
+										<div className="flex items-center gap-1">
+											<ClipboardList />
+											<div>View Tasks</div>
+										</div>
+									</Link>
+								</DropdownMenuItem>
+								{run.taskMetricsId && (
+									<DropdownMenuItem onClick={() => copyRun()} disabled={isPending || copied}>
+										<div className="flex items-center gap-1">
+											{isPending ? (
+												<>
+													<LoaderCircle className="animate-spin" />
+													Copying...
+												</>
+											) : copied ? (
+												<>
+													<Check />
+													Copied!
+												</>
+											) : (
+												<>
+													<Copy />
+													Copy to Production
+												</>
+											)}
+										</div>
+									</DropdownMenuItem>
+								)}
+								<DropdownMenuItem
+									onClick={() => {
+										setDeleteRunId(run.id)
+										setTimeout(() => continueRef.current?.focus(), 0)
+									}}>
 									<div className="flex items-center gap-1">
-										<ClipboardList />
-										<div>View Tasks</div>
-									</div>
-								</Link>
-							</DropdownMenuItem>
-							{run.taskMetricsId && (
-								<DropdownMenuItem onClick={() => copyRun()} disabled={isPending || copied}>
-									<div className="flex items-center gap-1">
-										{isPending ? (
-											<>
-												<LoaderCircle className="animate-spin" />
-												Copying...
-											</>
-										) : copied ? (
-											<>
-												<Check />
-												Copied!
-											</>
-										) : (
-											<>
-												<Copy />
-												Copy to Production
-											</>
-										)}
+										<Trash />
+										<div>Delete</div>
 									</div>
 								</DropdownMenuItem>
-							)}
-							<DropdownMenuItem
-								onClick={() => {
-									setDeleteRunId(run.id)
-									setTimeout(() => continueRef.current?.focus(), 0)
-								}}>
-								<div className="flex items-center gap-1">
-									<Trash />
-									<div>Delete</div>
-								</div>
-							</DropdownMenuItem>
-						</DropdownMenuContent>
-					</DropdownMenu>
+							</DropdownMenuContent>
+						</DropdownMenu>
+					</div>
 				</TableCell>
 			</TableRow>
 			<AlertDialog open={!!deleteRunId} onOpenChange={() => setDeleteRunId(undefined)}>

apps/web-evals/src/components/home/runs.tsx

@@ -18,6 +18,7 @@ export function Runs({ runs }: { runs: RunWithTaskMetrics[] }) {
 			<Table className="border border-t-0">
 				<TableHeader>
 					<TableRow>
+						<TableHead>Status</TableHead>
 						<TableHead>Model</TableHead>
 						<TableHead>Passed</TableHead>
 						<TableHead>Failed</TableHead>
@@ -34,7 +35,7 @@ export function Runs({ runs }: { runs: RunWithTaskMetrics[] }) {
 						runs.map(({ taskMetrics, ...run }) => <Row key={run.id} run={run} taskMetrics={taskMetrics} />)
 					) : (
 						<TableRow>
-							<TableCell colSpan={9} className="text-center">
+							<TableCell colSpan={10} className="text-center">
 								No eval runs yet.
 								<Button variant="link" onClick={() => router.push("/runs/new")}>
 									Launch

packages/evals/src/cli/redis.ts

@@ -16,6 +16,11 @@ export const getPubSubKey = (runId: number) => `evals:${runId}`
 export const getRunnersKey = (runId: number) => `runners:${runId}`
 export const getHeartbeatKey = (runId: number) => `heartbeat:${runId}`
 
+// Queue management keys
+export const getRunQueueKey = () => `evals:run-queue`
+export const getActiveRunKey = () => `evals:active-run`
+export const getDispatcherLockKey = () => `evals:dispatcher:lock`
+
 export const registerRunner = async ({
 	runId,
 	taskId,
@@ -61,3 +66,69 @@ export const stopHeartbeat = async (runId: number, heartbeat: NodeJS.Timeout) =>
 		console.error("redis.del failed:", error)
 	}
 }
+
+// Queue management functions
+export const enqueueRun = async (runId: number) => {
+	const redis = await redisClient()
+	await redis.rPush(getRunQueueKey(), runId.toString())
+}
+
+export const dequeueRun = async (): Promise<number | null> => {
+	const redis = await redisClient()
+	const runId = await redis.lPop(getRunQueueKey())
+	return runId ? parseInt(runId, 10) : null
+}
+
+export const getQueuePosition = async (runId: number): Promise<number | null> => {
+	const redis = await redisClient()
+	const position = await redis.lPos(getRunQueueKey(), runId.toString())
+	return position !== null ? position : null
+}
+
+export const removeFromQueue = async (runId: number): Promise<boolean> => {
+	const redis = await redisClient()
+	const removed = await redis.lRem(getRunQueueKey(), 1, runId.toString())
+	return removed > 0
+}
+
+export const getActiveRun = async (): Promise<number | null> => {
+	const redis = await redisClient()
+	const activeRunId = await redis.get(getActiveRunKey())
+	return activeRunId ? parseInt(activeRunId, 10) : null
+}
+
+export const setActiveRun = async (runId: number, ttlSeconds: number = 3600): Promise<boolean> => {
+	const redis = await redisClient()
+	// Use SET NX (set if not exists) with EX (expiry) for crash safety
+	const result = await redis.set(getActiveRunKey(), runId.toString(), {
+		NX: true,
+		EX: ttlSeconds,
+	})
+	return result === "OK"
+}
+
+export const clearActiveRun = async (): Promise<void> => {
+	const redis = await redisClient()
+	await redis.del(getActiveRunKey())
+}
+
+export const acquireDispatcherLock = async (ttlSeconds: number = 10): Promise<boolean> => {
+	const redis = await redisClient()
+	const lockId = Date.now().toString()
+	const result = await redis.set(getDispatcherLockKey(), lockId, {
+		NX: true,
+		EX: ttlSeconds,
+	})
+	return result === "OK"
+}
+
+export const releaseDispatcherLock = async (): Promise<void> => {
+	const redis = await redisClient()
+	await redis.del(getDispatcherLockKey())
+}
+
+export const getQueuedRunIds = async (): Promise<number[]> => {
+	const redis = await redisClient()
+	const runIds = await redis.lRange(getRunQueueKey(), 0, -1)
+	return runIds.map((id) => parseInt(id, 10))
+}

packages/evals/src/cli/runEvals.ts

@@ -1,12 +1,94 @@
 import PQueue from "p-queue"
+import { spawn } from "child_process"
 
 import { findRun, finishRun, getTasks } from "../db/index.js"
 import { EVALS_REPO_PATH } from "../exercises/index.js"
 
 import { Logger, getTag, isDockerContainer, resetEvalsRepo, commitEvalsRepoChanges } from "./utils.js"
-import { startHeartbeat, stopHeartbeat } from "./redis.js"
+import {
+	startHeartbeat,
+	stopHeartbeat,
+	clearActiveRun,
+	dequeueRun,
+	setActiveRun,
+	acquireDispatcherLock,
+	releaseDispatcherLock,
+} from "./redis.js"
 import { processTask, processTaskInContainer } from "./runTask.js"
 
+const dispatchNextRun = async (logger: Logger) => {
+	// Try to acquire dispatcher lock
+	const lockAcquired = await acquireDispatcherLock(10)
+
+	if (!lockAcquired) {
+		logger.info("Dispatcher lock already held, skipping dispatch")
+		return
+	}
+
+	try {
+		// Pop the next run from the queue
+		const nextRunId = await dequeueRun()
+		if (!nextRunId) {
+			logger.info("No runs in queue")
+			return
+		}
+
+		logger.info(`Dispatching next run: ${nextRunId}`)
+
+		// Set as active run with generous TTL (1 hour)
+		const setActive = await setActiveRun(nextRunId, 3600)
+
+		if (!setActive) {
+			// This shouldn't happen but handle it gracefully
+			logger.error(`Failed to set run ${nextRunId} as active`)
+			return
+		}
+
+		// Spawn the controller for this run
+		const containerized = isDockerContainer()
+		const cliCommand = `pnpm --filter @roo-code/evals cli --runId ${nextRunId}`
+
+		if (containerized) {
+			// When running in a container, spawn a new container for the next run
+			const dockerArgs = [
+				`--name evals-controller-${nextRunId}`,
+				"--rm",
+				"--network evals_default",
+				"-v /var/run/docker.sock:/var/run/docker.sock",
+				"-v /tmp/evals:/var/log/evals",
+				"-e HOST_EXECUTION_METHOD=docker",
+			]
+
+			const command = `docker run ${dockerArgs.join(" ")} evals-runner sh -c "${cliCommand}"`
+			logger.info(`Spawning next controller: ${command}`)
+
+			const childProcess = spawn("sh", ["-c", command], {
+				detached: true,
+				stdio: ["ignore", "ignore", "ignore"],
+			})
+
+			childProcess.unref()
+		} else {
+			// When not in a container, spawn the CLI directly
+			logger.info(`Spawning next controller: ${cliCommand}`)
+
+			const childProcess = spawn("sh", ["-c", cliCommand], {
+				detached: true,
+				stdio: ["ignore", "ignore", "ignore"],
+			})
+
+			childProcess.unref()
+		}
+
+		logger.info(`Successfully dispatched run ${nextRunId}`)
+	} catch (error) {
+		logger.error("Error dispatching next run:", error)
+	} finally {
+		// Release dispatcher lock
+		await releaseDispatcherLock()
+	}
+}
+
 export const runEvals = async (runId: number) => {
 	const run = await findRun(runId)
 
@@ -67,6 +149,14 @@ export const runEvals = async (runId: number) => {
 	} finally {
 		logger.info("cleaning up")
 		stopHeartbeat(run.id, heartbeat)
+
+		// Clear active run status
+		await clearActiveRun()
+		logger.info("Cleared active run status")
+
+		// Dispatch the next run in queue
+		await dispatchNextRun(logger)
+
 		logger.close()
 	}
 }