mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-08-28 05:27:24 +00:00
feat(web-evals): enhance dashboard with dynamic tool columns and UX improvements (#9592)
Co-authored-by: Roo Code <roomote@roocode.com>
This commit is contained in:
Hannes Rudolph
GitHub
parent
3c989d3591
commit
4442397507
7 changed files with 727 additions and 62 deletions
|
|
@ -5,15 +5,36 @@ import { LoaderCircle } from "lucide-react"
|
|||
|
||||
import type { Run, TaskMetrics as _TaskMetrics } from "@roo-code/evals"
|
||||
|
||||
import { formatCurrency, formatDuration, formatTokens } from "@/lib/formatters"
|
||||
import { formatCurrency, formatDuration, formatTokens, formatToolUsageSuccessRate } from "@/lib/formatters"
|
||||
import { useRunStatus } from "@/hooks/use-run-status"
|
||||
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui"
|
||||
import {
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui"
|
||||
|
||||
import { TaskStatus } from "./task-status"
|
||||
import { RunStatus } from "./run-status"
|
||||
|
||||
type TaskMetrics = Pick<_TaskMetrics, "tokensIn" | "tokensOut" | "tokensContext" | "duration" | "cost">
|
||||
|
||||
type ToolUsageEntry = { attempts: number; failures: number }
|
||||
type ToolUsage = Record<string, ToolUsageEntry>
|
||||
|
||||
// Generate abbreviation from tool name (e.g., "read_file" -> "RF", "list_code_definition_names" -> "LCDN")
|
||||
function getToolAbbreviation(toolName: string): string {
|
||||
return toolName
|
||||
.split("_")
|
||||
.map((word) => word[0]?.toUpperCase() ?? "")
|
||||
.join("")
|
||||
}
|
||||
|
||||
export function Run({ run }: { run: Run }) {
|
||||
const runStatus = useRunStatus(run)
|
||||
const { tasks, tokenUsage, usageUpdatedAt } = runStatus
|
||||
|
|
@ -41,16 +62,170 @@ export function Run({ run }: { run: Run }) {
|
|||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [tasks, tokenUsage, usageUpdatedAt])
|
||||
|
||||
// Compute aggregate stats
|
||||
const stats = useMemo(() => {
|
||||
if (!tasks) return null
|
||||
|
||||
const passed = tasks.filter((t) => t.passed === true).length
|
||||
const failed = tasks.filter((t) => t.passed === false).length
|
||||
// Count running tasks exactly like TaskStatus shows spinner:
|
||||
// - passed is not true and not false (null/undefined)
|
||||
// - AND has activity (startedAt or tokenUsage)
|
||||
const running = tasks.filter(
|
||||
(t) => t.passed !== true && t.passed !== false && (t.startedAt || tokenUsage.get(t.id)),
|
||||
).length
|
||||
const pending = tasks.filter(
|
||||
(t) => t.passed !== true && t.passed !== false && !t.startedAt && !tokenUsage.get(t.id),
|
||||
).length
|
||||
const total = tasks.length
|
||||
const completed = passed + failed
|
||||
|
||||
let totalTokensIn = 0
|
||||
let totalTokensOut = 0
|
||||
let totalCost = 0
|
||||
let totalDuration = 0
|
||||
|
||||
// Aggregate tool usage from completed tasks
|
||||
const toolUsage: ToolUsage = {}
|
||||
|
||||
for (const task of tasks) {
|
||||
const metrics = taskMetrics[task.id]
|
||||
if (metrics) {
|
||||
totalTokensIn += metrics.tokensIn
|
||||
totalTokensOut += metrics.tokensOut
|
||||
totalCost += metrics.cost
|
||||
totalDuration += metrics.duration
|
||||
}
|
||||
|
||||
// Aggregate tool usage from finished tasks with taskMetrics
|
||||
if (task.finishedAt && task.taskMetrics?.toolUsage) {
|
||||
for (const [key, usage] of Object.entries(task.taskMetrics.toolUsage)) {
|
||||
const tool = key as keyof ToolUsage
|
||||
if (!toolUsage[tool]) {
|
||||
toolUsage[tool] = { attempts: 0, failures: 0 }
|
||||
}
|
||||
toolUsage[tool].attempts += usage.attempts
|
||||
toolUsage[tool].failures += usage.failures
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
passed,
|
||||
failed,
|
||||
running,
|
||||
pending,
|
||||
total,
|
||||
completed,
|
||||
passRate: completed > 0 ? ((passed / completed) * 100).toFixed(1) : null,
|
||||
totalTokensIn,
|
||||
totalTokensOut,
|
||||
totalCost,
|
||||
totalDuration,
|
||||
toolUsage,
|
||||
}
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [tasks, taskMetrics, tokenUsage, usageUpdatedAt])
|
||||
|
||||
return (
|
||||
<>
|
||||
<div>
|
||||
<div className="mb-2">
|
||||
<div className="mb-4">
|
||||
<div>
|
||||
<div className="font-mono">{run.model}</div>
|
||||
{run.description && <div className="text-sm text-muted-foreground">{run.description}</div>}
|
||||
</div>
|
||||
{!run.taskMetricsId && <RunStatus runStatus={runStatus} />}
|
||||
</div>
|
||||
|
||||
{stats && (
|
||||
<div className="mb-4 p-4 border rounded-lg bg-muted/50">
|
||||
{/* Main Stats Row */}
|
||||
<div className="flex flex-wrap items-start justify-between gap-x-6 gap-y-3">
|
||||
{/* Passed/Failed */}
|
||||
<div className="text-center">
|
||||
<div className="text-2xl font-bold whitespace-nowrap">
|
||||
<span className="text-green-600">{stats.passed}</span>
|
||||
<span className="text-muted-foreground mx-1">/</span>
|
||||
<span className="text-red-600">{stats.failed}</span>
|
||||
{stats.running > 0 && (
|
||||
<span className="text-yellow-600 text-sm ml-2">({stats.running})</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="text-xs text-muted-foreground">Passed / Failed</div>
|
||||
</div>
|
||||
|
||||
{/* Pass Rate */}
|
||||
<div className="text-center">
|
||||
<div className="text-2xl font-bold">{stats.passRate ? `${stats.passRate}%` : "-"}</div>
|
||||
<div className="text-xs text-muted-foreground">Pass Rate</div>
|
||||
</div>
|
||||
|
||||
{/* Tokens */}
|
||||
<div className="text-center">
|
||||
<div className="text-xl font-bold font-mono whitespace-nowrap">
|
||||
{formatTokens(stats.totalTokensIn)}
|
||||
<span className="text-muted-foreground mx-1">/</span>
|
||||
{formatTokens(stats.totalTokensOut)}
|
||||
</div>
|
||||
<div className="text-xs text-muted-foreground">Tokens In / Out</div>
|
||||
</div>
|
||||
|
||||
{/* Cost */}
|
||||
<div className="text-center">
|
||||
<div className="text-2xl font-bold font-mono">{formatCurrency(stats.totalCost)}</div>
|
||||
<div className="text-xs text-muted-foreground">Cost</div>
|
||||
</div>
|
||||
|
||||
{/* Duration */}
|
||||
<div className="text-center">
|
||||
<div className="text-2xl font-bold font-mono whitespace-nowrap">
|
||||
{stats.totalDuration > 0 ? formatDuration(stats.totalDuration) : "-"}
|
||||
</div>
|
||||
<div className="text-xs text-muted-foreground">Duration</div>
|
||||
</div>
|
||||
|
||||
{/* Tool Usage - Inline */}
|
||||
{Object.keys(stats.toolUsage).length > 0 && (
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
{Object.entries(stats.toolUsage)
|
||||
.sort(([, a], [, b]) => b.attempts - a.attempts)
|
||||
.map(([toolName, usage]) => {
|
||||
const abbr = getToolAbbreviation(toolName)
|
||||
const successRate =
|
||||
usage.attempts > 0
|
||||
? ((usage.attempts - usage.failures) / usage.attempts) * 100
|
||||
: 100
|
||||
const rateColor =
|
||||
successRate === 100
|
||||
? "text-green-500"
|
||||
: successRate >= 80
|
||||
? "text-yellow-500"
|
||||
: "text-red-500"
|
||||
return (
|
||||
<Tooltip key={toolName}>
|
||||
<TooltipTrigger asChild>
|
||||
<div className="flex items-center gap-1 px-2 py-1 rounded bg-background/50 border border-border/50 hover:border-border transition-colors cursor-default text-xs">
|
||||
<span className="font-medium text-muted-foreground">
|
||||
{abbr}
|
||||
</span>
|
||||
<span className="font-bold tabular-nums">
|
||||
{usage.attempts}
|
||||
</span>
|
||||
<span className={`${rateColor}`}>
|
||||
{formatToolUsageSuccessRate(usage)}
|
||||
</span>
|
||||
</div>
|
||||
</TooltipTrigger>
|
||||
<TooltipContent side="bottom">{toolName}</TooltipContent>
|
||||
</Tooltip>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
{!tasks ? (
|
||||
<LoaderCircle className="size-4 animate-spin" />
|
||||
) : (
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"use client"
|
||||
|
||||
import { useCallback, useState } from "react"
|
||||
import { useCallback, useEffect, useMemo, useState } from "react"
|
||||
import { useRouter } from "next/navigation"
|
||||
import { z } from "zod"
|
||||
import { useQuery } from "@tanstack/react-query"
|
||||
|
|
@ -9,7 +9,14 @@ import { zodResolver } from "@hookform/resolvers/zod"
|
|||
import { toast } from "sonner"
|
||||
import { X, Rocket, Check, ChevronsUpDown, SlidersHorizontal } from "lucide-react"
|
||||
|
||||
import { globalSettingsSchema, providerSettingsSchema, EVALS_SETTINGS, getModelId } from "@roo-code/types"
|
||||
import {
|
||||
globalSettingsSchema,
|
||||
providerSettingsSchema,
|
||||
EVALS_SETTINGS,
|
||||
getModelId,
|
||||
type ProviderSettings,
|
||||
type GlobalSettings,
|
||||
} from "@roo-code/types"
|
||||
|
||||
import { createRun } from "@/actions/runs"
|
||||
import { getExercises } from "@/actions/exercises"
|
||||
|
|
@ -59,6 +66,12 @@ import {
|
|||
|
||||
import { SettingsDiff } from "./settings-diff"
|
||||
|
||||
type ImportedSettings = {
|
||||
apiConfigs: Record<string, ProviderSettings>
|
||||
globalSettings: GlobalSettings
|
||||
currentApiConfigName: string
|
||||
}
|
||||
|
||||
export function NewRun() {
|
||||
const router = useRouter()
|
||||
|
||||
|
|
@ -66,6 +79,11 @@ export function NewRun() {
|
|||
const [modelPopoverOpen, setModelPopoverOpen] = useState(false)
|
||||
const [useNativeToolProtocol, setUseNativeToolProtocol] = useState(true)
|
||||
|
||||
// State for imported settings with config selection
|
||||
const [importedSettings, setImportedSettings] = useState<ImportedSettings | null>(null)
|
||||
const [selectedConfigName, setSelectedConfigName] = useState<string>("")
|
||||
const [configPopoverOpen, setConfigPopoverOpen] = useState(false)
|
||||
|
||||
const openRouter = useOpenRouterModels()
|
||||
const rooCodeCloud = useRooCodeCloudModels()
|
||||
const models = provider === "openrouter" ? openRouter.data : rooCodeCloud.data
|
||||
|
|
@ -75,6 +93,9 @@ export function NewRun() {
|
|||
|
||||
const exercises = useQuery({ queryKey: ["getExercises"], queryFn: () => getExercises() })
|
||||
|
||||
// State for selected exercises (needed for language toggle buttons)
|
||||
const [selectedExercises, setSelectedExercises] = useState<string[]>([])
|
||||
|
||||
const form = useForm<CreateRun>({
|
||||
resolver: zodResolver(createRunSchema),
|
||||
defaultValues: {
|
||||
|
|
@ -98,6 +119,88 @@ export function NewRun() {
|
|||
|
||||
const [model, suite, settings] = watch(["model", "suite", "settings", "concurrency"])
|
||||
|
||||
// Load concurrency and timeout from localStorage on mount
|
||||
useEffect(() => {
|
||||
const savedConcurrency = localStorage.getItem("evals-concurrency")
|
||||
if (savedConcurrency) {
|
||||
const parsed = parseInt(savedConcurrency, 10)
|
||||
if (!isNaN(parsed) && parsed >= CONCURRENCY_MIN && parsed <= CONCURRENCY_MAX) {
|
||||
setValue("concurrency", parsed)
|
||||
}
|
||||
}
|
||||
const savedTimeout = localStorage.getItem("evals-timeout")
|
||||
if (savedTimeout) {
|
||||
const parsed = parseInt(savedTimeout, 10)
|
||||
if (!isNaN(parsed) && parsed >= TIMEOUT_MIN && parsed <= TIMEOUT_MAX) {
|
||||
setValue("timeout", parsed)
|
||||
}
|
||||
}
|
||||
}, [setValue])
|
||||
|
||||
// Extract unique languages from exercises
|
||||
const languages = useMemo(() => {
|
||||
if (!exercises.data) return []
|
||||
const langs = new Set<string>()
|
||||
for (const path of exercises.data) {
|
||||
const lang = path.split("/")[0]
|
||||
if (lang) langs.add(lang)
|
||||
}
|
||||
return Array.from(langs).sort()
|
||||
}, [exercises.data])
|
||||
|
||||
// Get exercises for a specific language
|
||||
const getExercisesForLanguage = useCallback(
|
||||
(lang: string) => {
|
||||
if (!exercises.data) return []
|
||||
return exercises.data.filter((path) => path.startsWith(`${lang}/`))
|
||||
},
|
||||
[exercises.data],
|
||||
)
|
||||
|
||||
// Toggle all exercises for a language
|
||||
const toggleLanguage = useCallback(
|
||||
(lang: string) => {
|
||||
const langExercises = getExercisesForLanguage(lang)
|
||||
const allSelected = langExercises.every((ex) => selectedExercises.includes(ex))
|
||||
|
||||
let newSelected: string[]
|
||||
if (allSelected) {
|
||||
// Remove all exercises for this language
|
||||
newSelected = selectedExercises.filter((ex) => !ex.startsWith(`${lang}/`))
|
||||
} else {
|
||||
// Add all exercises for this language (avoiding duplicates)
|
||||
const existing = new Set(selectedExercises)
|
||||
for (const ex of langExercises) {
|
||||
existing.add(ex)
|
||||
}
|
||||
newSelected = Array.from(existing)
|
||||
}
|
||||
|
||||
setSelectedExercises(newSelected)
|
||||
setValue("exercises", newSelected)
|
||||
},
|
||||
[getExercisesForLanguage, selectedExercises, setValue],
|
||||
)
|
||||
|
||||
// Check if all exercises for a language are selected
|
||||
const isLanguageSelected = useCallback(
|
||||
(lang: string) => {
|
||||
const langExercises = getExercisesForLanguage(lang)
|
||||
return langExercises.length > 0 && langExercises.every((ex) => selectedExercises.includes(ex))
|
||||
},
|
||||
[getExercisesForLanguage, selectedExercises],
|
||||
)
|
||||
|
||||
// Check if some (but not all) exercises for a language are selected
|
||||
const isLanguagePartiallySelected = useCallback(
|
||||
(lang: string) => {
|
||||
const langExercises = getExercisesForLanguage(lang)
|
||||
const selectedCount = langExercises.filter((ex) => selectedExercises.includes(ex)).length
|
||||
return selectedCount > 0 && selectedCount < langExercises.length
|
||||
},
|
||||
[getExercisesForLanguage, selectedExercises],
|
||||
)
|
||||
|
||||
const onSubmit = useCallback(
|
||||
async (values: CreateRun) => {
|
||||
try {
|
||||
|
|
@ -155,8 +258,19 @@ export function NewRun() {
|
|||
})
|
||||
.parse(JSON.parse(await file.text()))
|
||||
|
||||
const providerSettings = providerProfiles.apiConfigs[providerProfiles.currentApiConfigName] ?? {}
|
||||
// Store all imported configs for user selection
|
||||
setImportedSettings({
|
||||
apiConfigs: providerProfiles.apiConfigs,
|
||||
globalSettings,
|
||||
currentApiConfigName: providerProfiles.currentApiConfigName,
|
||||
})
|
||||
|
||||
// Default to the current config
|
||||
const defaultConfigName = providerProfiles.currentApiConfigName
|
||||
setSelectedConfigName(defaultConfigName)
|
||||
|
||||
// Apply the default config
|
||||
const providerSettings = providerProfiles.apiConfigs[defaultConfigName] ?? {}
|
||||
setValue("model", getModelId(providerSettings) ?? "")
|
||||
setValue("settings", { ...EVALS_SETTINGS, ...providerSettings, ...globalSettings })
|
||||
|
||||
|
|
@ -169,6 +283,22 @@ export function NewRun() {
|
|||
[clearErrors, setValue],
|
||||
)
|
||||
|
||||
const onSelectConfig = useCallback(
|
||||
(configName: string) => {
|
||||
if (!importedSettings) {
|
||||
return
|
||||
}
|
||||
|
||||
setSelectedConfigName(configName)
|
||||
setConfigPopoverOpen(false)
|
||||
|
||||
const providerSettings = importedSettings.apiConfigs[configName] ?? {}
|
||||
setValue("model", getModelId(providerSettings) ?? "")
|
||||
setValue("settings", { ...EVALS_SETTINGS, ...providerSettings, ...importedSettings.globalSettings })
|
||||
},
|
||||
[importedSettings, setValue],
|
||||
)
|
||||
|
||||
return (
|
||||
<>
|
||||
<FormProvider {...form}>
|
||||
|
|
@ -207,6 +337,63 @@ export function NewRun() {
|
|||
className="hidden"
|
||||
onChange={onImportSettings}
|
||||
/>
|
||||
|
||||
{importedSettings && Object.keys(importedSettings.apiConfigs).length > 1 && (
|
||||
<div className="space-y-1">
|
||||
<Label>API Config</Label>
|
||||
<Popover open={configPopoverOpen} onOpenChange={setConfigPopoverOpen}>
|
||||
<PopoverTrigger asChild>
|
||||
<Button
|
||||
variant="input"
|
||||
role="combobox"
|
||||
aria-expanded={configPopoverOpen}
|
||||
className="flex items-center justify-between w-full">
|
||||
<div>{selectedConfigName || "Select config"}</div>
|
||||
<ChevronsUpDown className="opacity-50" />
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent className="p-0 w-[var(--radix-popover-trigger-width)]">
|
||||
<Command>
|
||||
<CommandInput
|
||||
placeholder="Search configs..."
|
||||
className="h-9"
|
||||
/>
|
||||
<CommandList>
|
||||
<CommandEmpty>No config found.</CommandEmpty>
|
||||
<CommandGroup>
|
||||
{Object.keys(importedSettings.apiConfigs).map(
|
||||
(configName) => (
|
||||
<CommandItem
|
||||
key={configName}
|
||||
value={configName}
|
||||
onSelect={onSelectConfig}>
|
||||
{configName}
|
||||
{configName ===
|
||||
importedSettings.currentApiConfigName && (
|
||||
<span className="ml-2 text-xs text-muted-foreground">
|
||||
(default)
|
||||
</span>
|
||||
)}
|
||||
<Check
|
||||
className={cn(
|
||||
"ml-auto size-4",
|
||||
configName ===
|
||||
selectedConfigName
|
||||
? "opacity-100"
|
||||
: "opacity-0",
|
||||
)}
|
||||
/>
|
||||
</CommandItem>
|
||||
),
|
||||
)}
|
||||
</CommandGroup>
|
||||
</CommandList>
|
||||
</Command>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{settings && (
|
||||
<SettingsDiff defaultSettings={EVALS_SETTINGS} customSettings={settings} />
|
||||
)}
|
||||
|
|
@ -306,18 +493,51 @@ export function NewRun() {
|
|||
render={() => (
|
||||
<FormItem>
|
||||
<FormLabel>Exercises</FormLabel>
|
||||
<Tabs
|
||||
defaultValue="full"
|
||||
onValueChange={(value) => setValue("suite", value as "full" | "partial")}>
|
||||
<TabsList>
|
||||
<TabsTrigger value="full">All</TabsTrigger>
|
||||
<TabsTrigger value="partial">Some</TabsTrigger>
|
||||
</TabsList>
|
||||
</Tabs>
|
||||
<div className="flex items-center gap-2 flex-wrap">
|
||||
<Tabs
|
||||
defaultValue="full"
|
||||
onValueChange={(value) => {
|
||||
setValue("suite", value as "full" | "partial")
|
||||
if (value === "full") {
|
||||
setSelectedExercises([])
|
||||
setValue("exercises", [])
|
||||
}
|
||||
}}>
|
||||
<TabsList>
|
||||
<TabsTrigger value="full">All</TabsTrigger>
|
||||
<TabsTrigger value="partial">Some</TabsTrigger>
|
||||
</TabsList>
|
||||
</Tabs>
|
||||
{suite === "partial" && languages.length > 0 && (
|
||||
<div className="flex items-center gap-1 flex-wrap">
|
||||
{languages.map((lang) => (
|
||||
<Button
|
||||
key={lang}
|
||||
type="button"
|
||||
variant={
|
||||
isLanguageSelected(lang)
|
||||
? "default"
|
||||
: isLanguagePartiallySelected(lang)
|
||||
? "secondary"
|
||||
: "outline"
|
||||
}
|
||||
size="sm"
|
||||
onClick={() => toggleLanguage(lang)}
|
||||
className="text-xs capitalize">
|
||||
{lang}
|
||||
</Button>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{suite === "partial" && (
|
||||
<MultiSelect
|
||||
options={exercises.data?.map((path) => ({ value: path, label: path })) || []}
|
||||
onValueChange={(value) => setValue("exercises", value)}
|
||||
value={selectedExercises}
|
||||
onValueChange={(value) => {
|
||||
setSelectedExercises(value)
|
||||
setValue("exercises", value)
|
||||
}}
|
||||
placeholder="Select"
|
||||
variant="inverted"
|
||||
maxCount={4}
|
||||
|
|
@ -337,11 +557,14 @@ export function NewRun() {
|
|||
<FormControl>
|
||||
<div className="flex flex-row items-center gap-2">
|
||||
<Slider
|
||||
defaultValue={[field.value]}
|
||||
value={[field.value]}
|
||||
min={CONCURRENCY_MIN}
|
||||
max={CONCURRENCY_MAX}
|
||||
step={1}
|
||||
onValueChange={(value) => field.onChange(value[0])}
|
||||
onValueChange={(value) => {
|
||||
field.onChange(value[0])
|
||||
localStorage.setItem("evals-concurrency", String(value[0]))
|
||||
}}
|
||||
/>
|
||||
<div>{field.value}</div>
|
||||
</div>
|
||||
|
|
@ -360,11 +583,14 @@ export function NewRun() {
|
|||
<FormControl>
|
||||
<div className="flex flex-row items-center gap-2">
|
||||
<Slider
|
||||
defaultValue={[field.value]}
|
||||
value={[field.value]}
|
||||
min={TIMEOUT_MIN}
|
||||
max={TIMEOUT_MAX}
|
||||
step={1}
|
||||
onValueChange={(value) => field.onChange(value[0])}
|
||||
onValueChange={(value) => {
|
||||
field.onChange(value[0])
|
||||
localStorage.setItem("evals-timeout", String(value[0]))
|
||||
}}
|
||||
/>
|
||||
<div>{field.value}</div>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,11 +1,19 @@
|
|||
import { useCallback, useState, useRef } from "react"
|
||||
import Link from "next/link"
|
||||
import { Ellipsis, ClipboardList, Copy, Check, LoaderCircle, Trash } from "lucide-react"
|
||||
import { useRouter } from "next/navigation"
|
||||
import { Ellipsis, ClipboardList, Copy, Check, LoaderCircle, Trash, Settings } from "lucide-react"
|
||||
|
||||
import type { Run as EvalsRun, TaskMetrics as EvalsTaskMetrics } from "@roo-code/evals"
|
||||
import type { ToolName } from "@roo-code/types"
|
||||
|
||||
import { deleteRun } from "@/actions/runs"
|
||||
import { formatCurrency, formatDuration, formatTokens, formatToolUsageSuccessRate } from "@/lib/formatters"
|
||||
import {
|
||||
formatCurrency,
|
||||
formatDateTime,
|
||||
formatDuration,
|
||||
formatTokens,
|
||||
formatToolUsageSuccessRate,
|
||||
} from "@/lib/formatters"
|
||||
import { useCopyRun } from "@/hooks/use-copy-run"
|
||||
import {
|
||||
Button,
|
||||
|
|
@ -23,15 +31,23 @@ import {
|
|||
AlertDialogFooter,
|
||||
AlertDialogHeader,
|
||||
AlertDialogTitle,
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
ScrollArea,
|
||||
} from "@/components/ui"
|
||||
|
||||
type RunProps = {
|
||||
run: EvalsRun
|
||||
taskMetrics: EvalsTaskMetrics | null
|
||||
toolColumns: ToolName[]
|
||||
}
|
||||
|
||||
export function Run({ run, taskMetrics }: RunProps) {
|
||||
export function Run({ run, taskMetrics, toolColumns }: RunProps) {
|
||||
const router = useRouter()
|
||||
const [deleteRunId, setDeleteRunId] = useState<number>()
|
||||
const [showSettings, setShowSettings] = useState(false)
|
||||
const continueRef = useRef<HTMLButtonElement>(null)
|
||||
const { isPending, copyRun, copied } = useCopyRun(run.id)
|
||||
|
||||
|
|
@ -48,10 +64,25 @@ export function Run({ run, taskMetrics }: RunProps) {
|
|||
}
|
||||
}, [deleteRunId])
|
||||
|
||||
const handleRowClick = useCallback(
|
||||
(e: React.MouseEvent) => {
|
||||
// Don't navigate if clicking on the dropdown menu
|
||||
if ((e.target as HTMLElement).closest("[data-dropdown-trigger]")) {
|
||||
return
|
||||
}
|
||||
router.push(`/runs/${run.id}`)
|
||||
},
|
||||
[router, run.id],
|
||||
)
|
||||
|
||||
return (
|
||||
<>
|
||||
<TableRow>
|
||||
<TableCell>{run.model}</TableCell>
|
||||
<TableRow className="cursor-pointer hover:bg-muted/50" onClick={handleRowClick}>
|
||||
<TableCell className="max-w-[200px] truncate">{run.model}</TableCell>
|
||||
<TableCell>{run.settings?.apiProvider ?? "-"}</TableCell>
|
||||
<TableCell className="text-sm text-muted-foreground whitespace-nowrap">
|
||||
{formatDateTime(run.createdAt)}
|
||||
</TableCell>
|
||||
<TableCell>{run.passed}</TableCell>
|
||||
<TableCell>{run.failed}</TableCell>
|
||||
<TableCell>
|
||||
|
|
@ -61,27 +92,33 @@ export function Run({ run, taskMetrics }: RunProps) {
|
|||
</TableCell>
|
||||
<TableCell>
|
||||
{taskMetrics && (
|
||||
<div className="flex items-center gap-1.5">
|
||||
<div>{formatTokens(taskMetrics.tokensIn)}</div>/
|
||||
<div>{formatTokens(taskMetrics.tokensOut)}</div>
|
||||
</div>
|
||||
)}
|
||||
</TableCell>
|
||||
<TableCell>
|
||||
{taskMetrics?.toolUsage?.apply_diff && (
|
||||
<div className="flex flex-row items-center gap-1.5">
|
||||
<div>{taskMetrics.toolUsage.apply_diff.attempts}</div>
|
||||
<div>/</div>
|
||||
<div>{formatToolUsageSuccessRate(taskMetrics.toolUsage.apply_diff)}</div>
|
||||
<div className="flex items-center gap-1">
|
||||
<span>{formatTokens(taskMetrics.tokensIn)}</span>/
|
||||
<span>{formatTokens(taskMetrics.tokensOut)}</span>
|
||||
</div>
|
||||
)}
|
||||
</TableCell>
|
||||
{toolColumns.map((toolName) => {
|
||||
const usage = taskMetrics?.toolUsage?.[toolName]
|
||||
return (
|
||||
<TableCell key={toolName} className="text-xs text-center">
|
||||
{usage ? (
|
||||
<div className="flex flex-col items-center">
|
||||
<span className="font-medium">{usage.attempts}</span>
|
||||
<span className="text-muted-foreground">{formatToolUsageSuccessRate(usage)}</span>
|
||||
</div>
|
||||
) : (
|
||||
<span className="text-muted-foreground">-</span>
|
||||
)}
|
||||
</TableCell>
|
||||
)
|
||||
})}
|
||||
<TableCell>{taskMetrics && formatCurrency(taskMetrics.cost)}</TableCell>
|
||||
<TableCell>{taskMetrics && formatDuration(taskMetrics.duration)}</TableCell>
|
||||
<TableCell>
|
||||
<TableCell onClick={(e) => e.stopPropagation()}>
|
||||
<DropdownMenu>
|
||||
<Button variant="ghost" size="icon" asChild>
|
||||
<DropdownMenuTrigger>
|
||||
<DropdownMenuTrigger data-dropdown-trigger>
|
||||
<Ellipsis />
|
||||
</DropdownMenuTrigger>
|
||||
</Button>
|
||||
|
|
@ -94,6 +131,14 @@ export function Run({ run, taskMetrics }: RunProps) {
|
|||
</div>
|
||||
</Link>
|
||||
</DropdownMenuItem>
|
||||
{run.settings && (
|
||||
<DropdownMenuItem onClick={() => setShowSettings(true)}>
|
||||
<div className="flex items-center gap-1">
|
||||
<Settings />
|
||||
<div>View Settings</div>
|
||||
</div>
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{run.taskMetricsId && (
|
||||
<DropdownMenuItem onClick={() => copyRun()} disabled={isPending || copied}>
|
||||
<div className="flex items-center gap-1">
|
||||
|
|
@ -144,6 +189,18 @@ export function Run({ run, taskMetrics }: RunProps) {
|
|||
</AlertDialogFooter>
|
||||
</AlertDialogContent>
|
||||
</AlertDialog>
|
||||
<Dialog open={showSettings} onOpenChange={setShowSettings}>
|
||||
<DialogContent className="max-w-2xl max-h-[80vh]">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Run Settings</DialogTitle>
|
||||
</DialogHeader>
|
||||
<ScrollArea className="max-h-[60vh]">
|
||||
<pre className="text-xs font-mono bg-muted p-4 rounded-md overflow-auto">
|
||||
{JSON.stringify(run.settings, null, 2)}
|
||||
</pre>
|
||||
</ScrollArea>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,40 +1,224 @@
|
|||
"use client"
|
||||
|
||||
import { useMemo, useState } from "react"
|
||||
import { useRouter } from "next/navigation"
|
||||
import { Rocket } from "lucide-react"
|
||||
import { ArrowDown, ArrowUp, ArrowUpDown, Rocket } from "lucide-react"
|
||||
|
||||
import type { Run, TaskMetrics } from "@roo-code/evals"
|
||||
import type { ToolName } from "@roo-code/types"
|
||||
|
||||
import { Button, Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui"
|
||||
import {
|
||||
Button,
|
||||
Table,
|
||||
TableBody,
|
||||
TableCell,
|
||||
TableHead,
|
||||
TableHeader,
|
||||
TableRow,
|
||||
Tooltip,
|
||||
TooltipContent,
|
||||
TooltipTrigger,
|
||||
} from "@/components/ui"
|
||||
import { Run as Row } from "@/components/home/run"
|
||||
|
||||
type RunWithTaskMetrics = Run & { taskMetrics: TaskMetrics | null }
|
||||
|
||||
type SortColumn = "model" | "provider" | "passed" | "failed" | "percent" | "cost" | "duration" | "createdAt"
|
||||
type SortDirection = "asc" | "desc"
|
||||
|
||||
// Generate abbreviation from tool name (e.g., "read_file" -> "RF", "list_code_definition_names" -> "LCDN")
|
||||
function getToolAbbreviation(toolName: string): string {
|
||||
return toolName
|
||||
.split("_")
|
||||
.map((word) => word[0]?.toUpperCase() ?? "")
|
||||
.join("")
|
||||
}
|
||||
|
||||
function SortIcon({
|
||||
column,
|
||||
sortColumn,
|
||||
sortDirection,
|
||||
}: {
|
||||
column: SortColumn
|
||||
sortColumn: SortColumn | null
|
||||
sortDirection: SortDirection
|
||||
}) {
|
||||
if (sortColumn !== column) {
|
||||
return <ArrowUpDown className="ml-1 h-3 w-3 opacity-50" />
|
||||
}
|
||||
return sortDirection === "asc" ? <ArrowUp className="ml-1 h-3 w-3" /> : <ArrowDown className="ml-1 h-3 w-3" />
|
||||
}
|
||||
|
||||
export function Runs({ runs }: { runs: RunWithTaskMetrics[] }) {
|
||||
const router = useRouter()
|
||||
const [sortColumn, setSortColumn] = useState<SortColumn | null>("createdAt")
|
||||
const [sortDirection, setSortDirection] = useState<SortDirection>("desc")
|
||||
|
||||
const handleSort = (column: SortColumn) => {
|
||||
if (sortColumn === column) {
|
||||
setSortDirection(sortDirection === "asc" ? "desc" : "asc")
|
||||
} else {
|
||||
setSortColumn(column)
|
||||
setSortDirection("desc")
|
||||
}
|
||||
}
|
||||
|
||||
// Collect all unique tool names from all runs and sort by total attempts
|
||||
const toolColumns = useMemo<ToolName[]>(() => {
|
||||
const toolTotals = new Map<ToolName, number>()
|
||||
|
||||
for (const run of runs) {
|
||||
if (run.taskMetrics?.toolUsage) {
|
||||
for (const [toolName, usage] of Object.entries(run.taskMetrics.toolUsage)) {
|
||||
const tool = toolName as ToolName
|
||||
const current = toolTotals.get(tool) ?? 0
|
||||
toolTotals.set(tool, current + usage.attempts)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by total attempts descending
|
||||
return Array.from(toolTotals.entries())
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.map(([name]): ToolName => name)
|
||||
}, [runs])
|
||||
|
||||
// Sort runs based on current sort column and direction
|
||||
const sortedRuns = useMemo(() => {
|
||||
if (!sortColumn) return runs
|
||||
|
||||
return [...runs].sort((a, b) => {
|
||||
let aVal: string | number | Date | null = null
|
||||
let bVal: string | number | Date | null = null
|
||||
|
||||
switch (sortColumn) {
|
||||
case "model":
|
||||
aVal = a.model
|
||||
bVal = b.model
|
||||
break
|
||||
case "provider":
|
||||
aVal = a.settings?.apiProvider ?? ""
|
||||
bVal = b.settings?.apiProvider ?? ""
|
||||
break
|
||||
case "passed":
|
||||
aVal = a.passed
|
||||
bVal = b.passed
|
||||
break
|
||||
case "failed":
|
||||
aVal = a.failed
|
||||
bVal = b.failed
|
||||
break
|
||||
case "percent":
|
||||
aVal = a.passed + a.failed > 0 ? a.passed / (a.passed + a.failed) : 0
|
||||
bVal = b.passed + b.failed > 0 ? b.passed / (b.passed + b.failed) : 0
|
||||
break
|
||||
case "cost":
|
||||
aVal = a.taskMetrics?.cost ?? 0
|
||||
bVal = b.taskMetrics?.cost ?? 0
|
||||
break
|
||||
case "duration":
|
||||
aVal = a.taskMetrics?.duration ?? 0
|
||||
bVal = b.taskMetrics?.duration ?? 0
|
||||
break
|
||||
case "createdAt":
|
||||
aVal = a.createdAt
|
||||
bVal = b.createdAt
|
||||
break
|
||||
}
|
||||
|
||||
if (aVal === null || bVal === null) return 0
|
||||
|
||||
let comparison = 0
|
||||
if (typeof aVal === "string" && typeof bVal === "string") {
|
||||
comparison = aVal.localeCompare(bVal)
|
||||
} else if (aVal instanceof Date && bVal instanceof Date) {
|
||||
comparison = aVal.getTime() - bVal.getTime()
|
||||
} else {
|
||||
comparison = (aVal as number) - (bVal as number)
|
||||
}
|
||||
|
||||
return sortDirection === "asc" ? comparison : -comparison
|
||||
})
|
||||
}, [runs, sortColumn, sortDirection])
|
||||
|
||||
// Calculate colSpan for empty state (7 base columns + dynamic tools + 3 end columns)
|
||||
const totalColumns = 7 + toolColumns.length + 3
|
||||
|
||||
return (
|
||||
<>
|
||||
<Table className="border border-t-0">
|
||||
<TableHeader>
|
||||
<TableRow>
|
||||
<TableHead>Model</TableHead>
|
||||
<TableHead>Passed</TableHead>
|
||||
<TableHead>Failed</TableHead>
|
||||
<TableHead>% Correct</TableHead>
|
||||
<TableHead>Tokens In / Out</TableHead>
|
||||
<TableHead>Diff Edits</TableHead>
|
||||
<TableHead>Cost</TableHead>
|
||||
<TableHead>Duration</TableHead>
|
||||
<TableHead />
|
||||
<TableHead
|
||||
className="max-w-[200px] cursor-pointer select-none"
|
||||
onClick={() => handleSort("model")}>
|
||||
<div className="flex items-center">
|
||||
Model
|
||||
<SortIcon column="model" sortColumn={sortColumn} sortDirection={sortDirection} />
|
||||
</div>
|
||||
</TableHead>
|
||||
<TableHead className="cursor-pointer select-none" onClick={() => handleSort("provider")}>
|
||||
<div className="flex items-center">
|
||||
Provider
|
||||
<SortIcon column="provider" sortColumn={sortColumn} sortDirection={sortDirection} />
|
||||
</div>
|
||||
</TableHead>
|
||||
<TableHead className="cursor-pointer select-none" onClick={() => handleSort("createdAt")}>
|
||||
<div className="flex items-center">
|
||||
Created
|
||||
<SortIcon column="createdAt" sortColumn={sortColumn} sortDirection={sortDirection} />
|
||||
</div>
|
||||
</TableHead>
|
||||
<TableHead className="cursor-pointer select-none" onClick={() => handleSort("passed")}>
|
||||
<div className="flex items-center">
|
||||
Passed
|
||||
<SortIcon column="passed" sortColumn={sortColumn} sortDirection={sortDirection} />
|
||||
</div>
|
||||
</TableHead>
|
||||
<TableHead className="cursor-pointer select-none" onClick={() => handleSort("failed")}>
|
||||
<div className="flex items-center">
|
||||
Failed
|
||||
<SortIcon column="failed" sortColumn={sortColumn} sortDirection={sortDirection} />
|
||||
</div>
|
||||
</TableHead>
|
||||
<TableHead className="cursor-pointer select-none" onClick={() => handleSort("percent")}>
|
||||
<div className="flex items-center">
|
||||
%
|
||||
<SortIcon column="percent" sortColumn={sortColumn} sortDirection={sortDirection} />
|
||||
</div>
|
||||
</TableHead>
|
||||
<TableHead>Tokens</TableHead>
|
||||
{toolColumns.map((toolName) => (
|
||||
<TableHead key={toolName} className="text-xs text-center">
|
||||
<Tooltip>
|
||||
<TooltipTrigger>{getToolAbbreviation(toolName)}</TooltipTrigger>
|
||||
<TooltipContent>{toolName}</TooltipContent>
|
||||
</Tooltip>
|
||||
</TableHead>
|
||||
))}
|
||||
<TableHead className="cursor-pointer select-none" onClick={() => handleSort("cost")}>
|
||||
<div className="flex items-center">
|
||||
Cost
|
||||
<SortIcon column="cost" sortColumn={sortColumn} sortDirection={sortDirection} />
|
||||
</div>
|
||||
</TableHead>
|
||||
<TableHead className="cursor-pointer select-none" onClick={() => handleSort("duration")}>
|
||||
<div className="flex items-center">
|
||||
Duration
|
||||
<SortIcon column="duration" sortColumn={sortColumn} sortDirection={sortDirection} />
|
||||
</div>
|
||||
</TableHead>
|
||||
<TableHead></TableHead>
|
||||
</TableRow>
|
||||
</TableHeader>
|
||||
<TableBody>
|
||||
{runs.length ? (
|
||||
runs.map(({ taskMetrics, ...run }) => <Row key={run.id} run={run} taskMetrics={taskMetrics} />)
|
||||
{sortedRuns.length ? (
|
||||
sortedRuns.map(({ taskMetrics, ...run }) => (
|
||||
<Row key={run.id} run={run} taskMetrics={taskMetrics} toolColumns={toolColumns} />
|
||||
))
|
||||
) : (
|
||||
<TableRow>
|
||||
<TableCell colSpan={9} className="text-center">
|
||||
<TableCell colSpan={totalColumns} className="text-center">
|
||||
No eval runs yet.
|
||||
<Button variant="link" onClick={() => router.push("/runs/new")}>
|
||||
Launch
|
||||
|
|
|
|||
|
|
@ -48,7 +48,10 @@ interface MultiSelectProps extends React.HTMLAttributes<HTMLDivElement>, Variant
|
|||
*/
|
||||
onValueChange: (value: string[]) => void
|
||||
|
||||
/** The default selected values when the component mounts. */
|
||||
/** The controlled selected values. When provided, the component becomes controlled. */
|
||||
value?: string[]
|
||||
|
||||
/** The default selected values when the component mounts (uncontrolled mode). */
|
||||
defaultValue?: string[]
|
||||
|
||||
/**
|
||||
|
|
@ -89,6 +92,7 @@ export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
|
|||
options,
|
||||
onValueChange,
|
||||
variant,
|
||||
value,
|
||||
defaultValue = [],
|
||||
placeholder = "Select options",
|
||||
maxCount = 3,
|
||||
|
|
@ -98,17 +102,30 @@ export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
|
|||
},
|
||||
ref,
|
||||
) => {
|
||||
const [selectedValues, setSelectedValues] = React.useState<string[]>(defaultValue)
|
||||
const [internalSelectedValues, setInternalSelectedValues] = React.useState<string[]>(defaultValue)
|
||||
const [isPopoverOpen, setIsPopoverOpen] = React.useState(false)
|
||||
|
||||
// Use controlled value if provided, otherwise use internal state
|
||||
const isControlled = value !== undefined
|
||||
const selectedValues = isControlled ? value : internalSelectedValues
|
||||
|
||||
const setSelectedValues = React.useCallback(
|
||||
(newValues: string[]) => {
|
||||
if (!isControlled) {
|
||||
setInternalSelectedValues(newValues)
|
||||
}
|
||||
onValueChange(newValues)
|
||||
},
|
||||
[isControlled, onValueChange],
|
||||
)
|
||||
|
||||
const handleInputKeyDown = (event: React.KeyboardEvent<HTMLInputElement>) => {
|
||||
if (event.key === "Enter") {
|
||||
setIsPopoverOpen(true)
|
||||
} else if (event.key === "Backspace" && !event.currentTarget.value) {
|
||||
const newSelectedValues = [...selectedValues]
|
||||
newSelectedValues.pop()
|
||||
if (!selectedValues.length) return
|
||||
const newSelectedValues = selectedValues.slice(0, -1)
|
||||
setSelectedValues(newSelectedValues)
|
||||
onValueChange(newSelectedValues)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -117,7 +134,6 @@ export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
|
|||
? selectedValues.filter((value) => value !== option)
|
||||
: [...selectedValues, option]
|
||||
setSelectedValues(newSelectedValues)
|
||||
onValueChange(newSelectedValues)
|
||||
}
|
||||
|
||||
const handleTogglePopover = () => {
|
||||
|
|
@ -127,7 +143,6 @@ export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
|
|||
const clearExtraOptions = () => {
|
||||
const newSelectedValues = selectedValues.slice(0, maxCount)
|
||||
setSelectedValues(newSelectedValues)
|
||||
onValueChange(newSelectedValues)
|
||||
}
|
||||
|
||||
const searchResultsRef = React.useRef<Map<string, number>>(new Map())
|
||||
|
|
@ -141,12 +156,10 @@ export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
|
|||
selectedValues.sort().join(",") === values.sort().join(",")
|
||||
) {
|
||||
setSelectedValues([])
|
||||
onValueChange([])
|
||||
return
|
||||
}
|
||||
|
||||
setSelectedValues(values)
|
||||
onValueChange(values)
|
||||
}
|
||||
|
||||
const onFilter = React.useCallback(
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ export const getRooCodeCloudModels = async (): Promise<RooCodeCloudModel[]> => {
|
|||
return []
|
||||
}
|
||||
|
||||
return result.data.data.sort((a, b) => a.name.localeCompare(b.name))
|
||||
return result.data.data.filter((model) => !model.deprecated).sort((a, b) => a.name.localeCompare(b.name))
|
||||
}
|
||||
|
||||
export const useRooCodeCloudModels = () => {
|
||||
|
|
|
|||
|
|
@ -46,3 +46,13 @@ export const formatTokens = (tokens: number) => {
|
|||
|
||||
export const formatToolUsageSuccessRate = (usage: { attempts: number; failures: number }) =>
|
||||
usage.attempts === 0 ? "0%" : `${(((usage.attempts - usage.failures) / usage.attempts) * 100).toFixed(1)}%`
|
||||
|
||||
export const formatDateTime = (date: Date) => {
|
||||
return new Intl.DateTimeFormat("en-US", {
|
||||
month: "short",
|
||||
day: "numeric",
|
||||
hour: "numeric",
|
||||
minute: "2-digit",
|
||||
hour12: true,
|
||||
}).format(date)
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue