scriptoriumadmin/Roo-Code
1
0
Fork 0
mirror of https://github.com/RooVetGit/Roo-Code.git synced 2026-08-28 05:27:24 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Remove Roo Code Cloud and evals (#12328)
Some checks are pending
Code QA Roo Code / check-translations (push) Waiting to run
Details
Code QA Roo Code / knip (push) Waiting to run
Details
Code QA Roo Code / compile (push) Waiting to run
Details
Code QA Roo Code / platform-unit-test (ubuntu-latest) (push) Waiting to run
Details
Code QA Roo Code / platform-unit-test (windows-latest) (push) Waiting to run
Details
CodeQL Advanced / Analyze (javascript-typescript) (push) Waiting to run
Details
Nightly Publish / publish-nightly (push) Waiting to run
Details
Deploy roocode.com / check-secrets (push) Waiting to run
Details
Deploy roocode.com / deploy (push) Blocked by required conditions
Details

Browse source 
* Remove Roo Code Cloud and evals

* Remove unused onboarding and web helper files

* Update ChatView welcome tests after cloud removal
This commit is contained in:
Matt Rubens 2026-05-11 22:43:45 -04:00 • committed by GitHub
parent 22d845cecb
commit 8922418600
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
455 changed files with 393 additions and 37017 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

74
.github/workflows/evals.yml vendored
View file

@ -1,74 +0,0 @@
name: Evals
on:
pull_request:
types: [labeled]
workflow_dispatch:
env:
DOCKER_BUILDKIT: 1
COMPOSE_DOCKER_CLI_BUILD: 1
jobs:
evals:
# Run if triggered manually or if PR has 'evals' label.
if: github.event_name == 'workflow_dispatch' || contains(github.event.label.name, 'evals')
runs-on: blacksmith-16vcpu-ubuntu-2404
timeout-minutes: 45
defaults:
run:
working-directory: packages/evals
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Create environment
run: |
cat > .env.local << EOF
OPENROUTER_API_KEY=${{ secrets.OPENROUTER_API_KEY || 'test-key-for-build' }}
EOF
cat > .env.development << EOF
NODE_ENV=development
DATABASE_URL=postgresql://postgres:password@db:5432/evals_development
REDIS_URL=redis://redis:6379
HOST_EXECUTION_METHOD=docker
EOF
- name: Build image
uses: docker/build-push-action@v6
with:
context: .
file: packages/evals/Dockerfile.runner
tags: evals-runner:latest
cache-from: type=gha
cache-to: type=gha,mode=max
push: false
load: true
- name: Tag image
run: docker tag evals-runner:latest evals-runner
- name: Start containers
run: |
docker compose up -d db redis
timeout 60 bash -c 'until docker compose exec -T db pg_isready -U postgres; do sleep 2; done'
timeout 60 bash -c 'until docker compose exec -T redis redis-cli ping | grep -q PONG; do sleep 2; done'
docker compose run --rm runner sh -c 'nc -z db 5432 && echo "✓ Runner -> Database connection successful"'
docker compose run --rm runner sh -c 'nc -z redis 6379 && echo "✓ Runner -> Redis connection successful"'
docker compose run --rm runner docker ps
- name: Run database migrations
run: docker compose run --rm runner pnpm --filter @roo-code/evals db:migrate
- name: Run evals
run: docker compose run --rm runner pnpm --filter @roo-code/evals cli --ci
- name: Cleanup
if: always()
run: docker compose down -v --remove-orphans

4
PRIVACY.md
View file

@ -6,9 +6,9 @@ Roo Code respects your privacy and is committed to transparency about how we han
### **Where Your Data Goes (And Where It Doesn’t)**
- **Code & Files**: Roo Code accesses files on your local machine when needed for AI-assisted features. When you send commands to Roo Code, relevant files may be transmitted to your chosen AI model provider (e.g., OpenAI, Anthropic, OpenRouter) to generate responses. If you select Roo Code Cloud as the model provider (proxy mode), your code may transit Roo Code servers only to forward it to the upstream provider. We do not store your code; it is deleted immediately after forwarding. Otherwise, your code is sent directly to the provider. AI providers may store data per their privacy policies.
- **Code & Files**: Roo Code accesses files on your local machine when needed for AI-assisted features. When you send commands to Roo Code, relevant files may be transmitted to your chosen AI model provider (e.g., OpenAI, Anthropic, OpenRouter) to generate responses. AI providers may store data per their privacy policies.
- **Commands**: Any commands executed through Roo Code happen on your local environment. However, when you use AI-powered features, the relevant code and context from your commands may be transmitted to your chosen AI model provider (e.g., OpenAI, Anthropic, OpenRouter) to generate responses. We do not have access to or store this data, but AI providers may process it per their privacy policies.
- **Prompts & AI Requests**: When you use AI-powered features, your prompts and relevant project context are sent to your chosen AI model provider (e.g., OpenAI, Anthropic, OpenRouter) to generate responses. We do not store or process this data. These AI providers have their own privacy policies and may store data per their terms of service. If you choose Roo Code Cloud as the provider (proxy mode), prompts may transit Roo Code servers only to forward them to the upstream model and are not stored.
- **Prompts & AI Requests**: When you use AI-powered features, your prompts and relevant project context are sent to your chosen AI model provider (e.g., OpenAI, Anthropic, OpenRouter) to generate responses. We do not store or process this data. These AI providers have their own privacy policies and may store data per their terms of service.
- **API Keys & Credentials**: If you enter an API key (e.g., to connect an AI model), it is stored locally on your device and never sent to us or any third party, except the provider you have chosen.
- **Telemetry (Usage Data)**: We collect anonymous feature usage and error data to help us improve Roo Code. This telemetry is powered by PostHog and includes your VS Code machine ID, feature usage patterns, and exception reports. This telemetry does **not** collect personally identifiable information, your code, or AI prompts. You can opt out of this telemetry at any time through the settings.

109
apps/cli/README.md
View file

@ -130,81 +130,29 @@ printf '{"command":"start","requestId":"1","prompt":"1+1=?"}\n' | roo --print --
printf '{"command":"start","requestId":"1","taskId":"018f7fc8-7c96-7f7c-98aa-2ec4ff7f6d87","prompt":"1+1=?"}\n' | roo --print --stdin-prompt-stream --output-format stream-json
```
### Roo Code Cloud Authentication
To use Roo Code Cloud features (like the provider proxy), you need to authenticate:
```bash
# Log in to Roo Code Cloud (opens browser)
roo auth login
# Check authentication status
roo auth status
# Log out
roo auth logout
```
The `auth login` command:
1. Opens your browser to authenticate with Roo Code Cloud
2. Receives a secure token via localhost callback
3. Stores the token in `~/.config/roo/credentials.json`
Tokens are valid for 90 days. The CLI will prompt you to re-authenticate when your token expires.
**Authentication Flow:**
```
┌──────┐ ┌─────────┐ ┌───────────────┐
│ CLI │ │ Browser │ │ Roo Code Cloud│
└──┬───┘ └────┬────┘ └───────┬───────┘
│ │ │
│ Open auth URL │ │
│─────────────────>│ │
│ │ │
│ │ Authenticate │
│ │─────────────────────>│
│ │ │
│ │<─────────────────────│
│ │ Token via callback │
│<─────────────────│ │
│ │ │
│ Store token │ │
│ │ │
```
## Options
| Option | Description | Default |
| --------------------------------------- | --------------------------------------------------------------------------------------- | ---------------------------------------- |
| `[prompt]` | Your prompt (positional argument, optional) | None |
| `--prompt-file <path>` | Read prompt from a file instead of command line argument | None |
| `--create-with-session-id <session-id>` | Create a new task using the provided session ID (UUID) | None |
| `-w, --workspace <path>` | Workspace path to operate in | Current directory |
| `-p, --print` | Print response and exit (non-interactive mode) | `false` |
| `--stdin-prompt-stream` | Read NDJSON control commands from stdin (requires `--print`) | `false` |
| `-e, --extension <path>` | Path to the extension bundle directory | Auto-detected |
| `-d, --debug` | Enable debug output (includes detailed debug information, prompts, paths, etc) | `false` |
| `-a, --require-approval` | Require manual approval before actions execute | `false` |
| `-k, --api-key <key>` | API key for the LLM provider | From env var |
| `--provider <provider>` | API provider (roo, anthropic, openai, openrouter, etc.) | `openrouter` (or `roo` if authenticated) |
| `-m, --model <model>` | Model to use | `anthropic/claude-opus-4.6` |
| `--mode <mode>` | Mode to start in (code, architect, ask, debug, etc.) | `code` |
| `--terminal-shell <path>` | Absolute shell path for inline terminal command execution | Auto-detected shell |
| `-r, --reasoning-effort <effort>` | Reasoning effort level (unspecified, disabled, none, minimal, low, medium, high, xhigh) | `medium` |
| `--consecutive-mistake-limit <n>` | Consecutive error/repetition limit before guidance prompt (`0` disables the limit) | `10` |
| `--ephemeral` | Run without persisting state (uses temporary storage) | `false` |
| `--oneshot` | Exit upon task completion | `false` |
| `--output-format <format>` | Output format with `--print`: `text`, `json`, or `stream-json` | `text` |
## Auth Commands
| Command | Description |
| ----------------- | ---------------------------------- |
| `roo auth login` | Authenticate with Roo Code Cloud |
| `roo auth logout` | Clear stored authentication token |
| `roo auth status` | Show current authentication status |
| Option | Description | Default |
| --------------------------------------- | --------------------------------------------------------------------------------------- | --------------------------- |
| `[prompt]` | Your prompt (positional argument, optional) | None |
| `--prompt-file <path>` | Read prompt from a file instead of command line argument | None |
| `--create-with-session-id <session-id>` | Create a new task using the provided session ID (UUID) | None |
| `-w, --workspace <path>` | Workspace path to operate in | Current directory |
| `-p, --print` | Print response and exit (non-interactive mode) | `false` |
| `--stdin-prompt-stream` | Read NDJSON control commands from stdin (requires `--print`) | `false` |
| `-e, --extension <path>` | Path to the extension bundle directory | Auto-detected |
| `-d, --debug` | Enable debug output (includes detailed debug information, prompts, paths, etc) | `false` |
| `-a, --require-approval` | Require manual approval before actions execute | `false` |
| `-k, --api-key <key>` | API key for the LLM provider | From env var |
| `--provider <provider>` | API provider (anthropic, openai, openrouter, etc.) | `openrouter` |
| `-m, --model <model>` | Model to use | `anthropic/claude-opus-4.6` |
| `--mode <mode>` | Mode to start in (code, architect, ask, debug, etc.) | `code` |
| `--terminal-shell <path>` | Absolute shell path for inline terminal command execution | Auto-detected shell |
| `-r, --reasoning-effort <effort>` | Reasoning effort level (unspecified, disabled, none, minimal, low, medium, high, xhigh) | `medium` |
| `--consecutive-mistake-limit <n>` | Consecutive error/repetition limit before guidance prompt (`0` disables the limit) | `10` |
| `--ephemeral` | Run without persisting state (uses temporary storage) | `false` |
| `--oneshot` | Exit upon task completion | `false` |
| `--output-format <format>` | Output format with `--print`: `text`, `json`, or `stream-json` | `text` |
## Environment Variables
@ -212,19 +160,12 @@ The CLI will look for API keys in environment variables if not provided via `--a
| Provider | Environment Variable |
| ----------------- | --------------------------- |
| roo | `ROO_API_KEY` |
| anthropic | `ANTHROPIC_API_KEY` |
| openai-native | `OPENAI_API_KEY` |
| openrouter | `OPENROUTER_API_KEY` |
| gemini | `GOOGLE_API_KEY` |
| vercel-ai-gateway | `VERCEL_AI_GATEWAY_API_KEY` |
**Authentication Environment Variables:**
| Variable | Description |
| ----------------- | -------------------------------------------------------------------- |
| `ROO_WEB_APP_URL` | Override the Roo Code Cloud URL (default: `https://app.roocode.com`) |
## Architecture
```
@ -268,7 +209,7 @@ The CLI will look for API keys in environment variables if not provided via `--a
```bash
# Run directly from source (no build required)
pnpm dev --provider roo --api-key $ROO_API_KEY --print "Hello"
pnpm dev --provider openrouter --api-key $OPENROUTER_API_KEY --print "Hello"
# Run tests
pnpm test
@ -280,12 +221,6 @@ pnpm check-types
pnpm lint
```
By default the `start` script points `ROO_CODE_PROVIDER_URL` at `http://localhost:8080/proxy` for local development. To point at the production API instead, override the environment variable:
```bash
ROO_CODE_PROVIDER_URL=https://api.roocode.com/proxy pnpm dev --provider roo --api-key $ROO_API_KEY --print "Hello"
```
## Releasing
Official releases are created via the GitHub Actions workflow at `.github/workflows/cli-release.yml`.

4
apps/cli/package.json
View file

@ -16,8 +16,8 @@
"test:integration": "tsx scripts/integration/run.ts",
"build": "tsup",
"build:extension": "pnpm --filter roo-cline bundle",
"dev": "ROO_AUTH_BASE_URL=https://app.roocode.com ROO_SDK_BASE_URL=https://cloud-api.roocode.com ROO_CODE_PROVIDER_URL=https://api.roocode.com/proxy tsx src/index.ts",
"dev:local": "ROO_AUTH_BASE_URL=http://localhost:3000 ROO_SDK_BASE_URL=http://localhost:3001 ROO_CODE_PROVIDER_URL=http://localhost:8080/proxy tsx src/index.ts",
"dev": "tsx src/index.ts",
"dev:local": "tsx src/index.ts",
"clean": "rimraf dist .turbo"
},
"dependencies": {

4
apps/cli/scripts/integration/cases/create-with-session-id-resume-loads-correct-session.ts
View file

@ -88,7 +88,7 @@ async function createSessionWithCustomId(
"dev",
"--print",
"--provider",
"roo",
"openrouter",
"--output-format",
"stream-json",
"--workspace",
@ -148,7 +148,7 @@ async function resumeSessionAndSendMarker(
"--print",
"--stdin-prompt-stream",
"--provider",
"roo",
"openrouter",
"--output-format",
"stream-json",
"--workspace",

2
apps/cli/scripts/integration/lib/stream-harness.ts
View file

@ -69,7 +69,7 @@ export async function runStreamCase(options: RunStreamCaseOptions): Promise<void
const child = execa(
"pnpm",
["dev", "--print", "--stdin-prompt-stream", "--provider", "roo", "--output-format", "stream-json"],
["dev", "--print", "--stdin-prompt-stream", "--provider", "openrouter", "--output-format", "stream-json"],
{
cwd: cliRoot,
stdin: "pipe",

2
apps/cli/src/agent/extension-host.ts
View file

@ -446,7 +446,7 @@ export class ExtensionHost extends EventEmitter implements ExtensionHostInterfac
// Apply CLI settings to the runtime config and context proxy BEFORE
// sending webviewDidLaunch. This prevents a race condition where the
// webviewDidLaunch handler's first-time init sync reads default state
// (apiProvider: "anthropic") instead of the CLI-provided settings.
// instead of the CLI-provided settings.
setRuntimeConfigValues("roo-cline", this.initialSettings as Record<string, unknown>)
this.sendToExtension({ type: "updateSettings", updatedSettings: this.initialSettings })

3
apps/cli/src/commands/auth/index.ts
View file

@ -1,3 +0,0 @@
export * from "./login.js"
export * from "./logout.js"
export * from "./status.js"

177
apps/cli/src/commands/auth/login.ts
View file

@ -1,177 +0,0 @@
import http from "http"
import { randomBytes } from "crypto"
import net from "net"
import { exec } from "child_process"
import { AUTH_BASE_URL } from "@/types/index.js"
import { saveToken } from "@/lib/storage/index.js"
export interface LoginOptions {
timeout?: number
verbose?: boolean
}
export type LoginResult =
| {
success: true
token: string
}
| {
success: false
error: string
}
const LOCALHOST = "127.0.0.1"
export async function login({ timeout = 5 * 60 * 1000, verbose = false }: LoginOptions = {}): Promise<LoginResult> {
const state = randomBytes(16).toString("hex")
const port = await getAvailablePort()
const host = `http://${LOCALHOST}:${port}`
if (verbose) {
console.log(`[Auth] Starting local callback server on port ${port}`)
}
// Create promise that will be resolved when we receive the callback.
const tokenPromise = new Promise<{ token: string; state: string }>((resolve, reject) => {
const server = http.createServer((req, res) => {
const url = new URL(req.url!, host)
if (url.pathname === "/callback") {
const receivedState = url.searchParams.get("state")
const token = url.searchParams.get("token")
const error = url.searchParams.get("error")
if (error) {
const errorUrl = new URL(`${AUTH_BASE_URL}/cli/sign-in?error=error-in-callback`)
errorUrl.searchParams.set("message", error)
res.writeHead(302, { Location: errorUrl.toString() })
res.end(() => {
server.close()
reject(new Error(error))
})
} else if (!token) {
const errorUrl = new URL(`${AUTH_BASE_URL}/cli/sign-in?error=missing-token`)
errorUrl.searchParams.set("message", "Missing token in callback")
res.writeHead(302, { Location: errorUrl.toString() })
res.end(() => {
server.close()
reject(new Error("Missing token in callback"))
})
} else if (receivedState !== state) {
const errorUrl = new URL(`${AUTH_BASE_URL}/cli/sign-in?error=invalid-state-parameter`)
errorUrl.searchParams.set("message", "Invalid state parameter")
res.writeHead(302, { Location: errorUrl.toString() })
res.end(() => {
server.close()
reject(new Error("Invalid state parameter"))
})
} else {
res.writeHead(302, { Location: `${AUTH_BASE_URL}/cli/sign-in?success=true` })
res.end(() => {
server.close()
resolve({ token, state: receivedState })
})
}
} else {
res.writeHead(404, { "Content-Type": "text/plain" })
res.end("Not found")
}
})
server.listen(port, LOCALHOST)
const timeoutId = setTimeout(() => {
server.close()
reject(new Error("Authentication timed out"))
}, timeout)
server.on("close", () => {
clearTimeout(timeoutId)
})
})
const authUrl = new URL(`${AUTH_BASE_URL}/cli/sign-in`)
authUrl.searchParams.set("state", state)
authUrl.searchParams.set("callback", `${host}/callback`)
console.log("Opening browser for authentication...")
console.log(`If the browser doesn't open, visit: ${authUrl.toString()}`)
try {
await openBrowser(authUrl.toString())
} catch (error) {
if (verbose) {
console.warn("[Auth] Failed to open browser automatically:", error)
}
console.log("Please open the URL above in your browser manually.")
}
try {
const { token } = await tokenPromise
await saveToken(token)
console.log("✓ Successfully authenticated!")
return { success: true, token }
} catch (error) {
const message = error instanceof Error ? error.message : String(error)
console.error(`✗ Authentication failed: ${message}`)
return { success: false, error: message }
}
}
async function getAvailablePort(startPort = 49152, endPort = 65535): Promise<number> {
return new Promise((resolve, reject) => {
const server = net.createServer()
let port = startPort
const tryPort = () => {
server.once("error", (err: NodeJS.ErrnoException) => {
if (err.code === "EADDRINUSE" && port < endPort) {
port++
tryPort()
} else {
reject(err)
}
})
server.once("listening", () => {
server.close(() => {
resolve(port)
})
})
server.listen(port, LOCALHOST)
}
tryPort()
})
}
function openBrowser(url: string): Promise<void> {
return new Promise((resolve, reject) => {
const platform = process.platform
let command: string
switch (platform) {
case "darwin":
command = `open "${url}"`
break
case "win32":
command = `start "" "${url}"`
break
default:
// Linux and other Unix-like systems.
command = `xdg-open "${url}"`
break
}
exec(command, (error) => {
if (error) {
reject(error)
} else {
resolve()
}
})
})
}

27
apps/cli/src/commands/auth/logout.ts
View file

@ -1,27 +0,0 @@
import { clearToken, hasToken, getCredentialsPath } from "@/lib/storage/index.js"
export interface LogoutOptions {
verbose?: boolean
}
export interface LogoutResult {
success: boolean
wasLoggedIn: boolean
}
export async function logout({ verbose = false }: LogoutOptions = {}): Promise<LogoutResult> {
const wasLoggedIn = await hasToken()
if (!wasLoggedIn) {
console.log("You are not currently logged in.")
return { success: true, wasLoggedIn: false }
}
if (verbose) {
console.log(`[Auth] Removing credentials from ${getCredentialsPath()}`)
}
await clearToken()
console.log("✓ Successfully logged out")
return { success: true, wasLoggedIn: true }
}

97
apps/cli/src/commands/auth/status.ts
View file

@ -1,97 +0,0 @@
import { loadToken, loadCredentials, getCredentialsPath } from "@/lib/storage/index.js"
import { isTokenExpired, isTokenValid, getTokenExpirationDate } from "@/lib/auth/index.js"
export interface StatusOptions {
verbose?: boolean
}
export interface StatusResult {
authenticated: boolean
expired?: boolean
expiringSoon?: boolean
userId?: string
orgId?: string | null
expiresAt?: Date
createdAt?: Date
}
export async function status(options: StatusOptions = {}): Promise<StatusResult> {
const { verbose = false } = options
const token = await loadToken()
if (!token) {
console.log("✗ Not authenticated")
console.log("")
console.log("Run: roo auth login")
return { authenticated: false }
}
const expiresAt = getTokenExpirationDate(token)
const expired = !isTokenValid(token)
const expiringSoon = isTokenExpired(token, 24 * 60 * 60) && !expired
const credentials = await loadCredentials()
const createdAt = credentials?.createdAt ? new Date(credentials.createdAt) : undefined
if (expired) {
console.log("✗ Authentication token expired")
console.log("")
console.log("Run: roo auth login")
return {
authenticated: false,
expired: true,
expiresAt: expiresAt ?? undefined,
}
}
if (expiringSoon) {
console.log("⚠ Expires soon; refresh with `roo auth login`")
} else {
console.log("✓ Authenticated")
}
if (expiresAt) {
const remaining = getTimeRemaining(expiresAt)
console.log(` Expires: ${formatDate(expiresAt)} (${remaining})`)
}
if (createdAt && verbose) {
console.log(` Created: ${formatDate(createdAt)}`)
}
if (verbose) {
console.log(` Credentials: ${getCredentialsPath()}`)
}
return {
authenticated: true,
expired: false,
expiringSoon,
expiresAt: expiresAt ?? undefined,
createdAt,
}
}
function formatDate(date: Date): string {
return date.toLocaleDateString("en-US", { year: "numeric", month: "long", day: "numeric" })
}
function getTimeRemaining(date: Date): string {
const now = new Date()
const diff = date.getTime() - now.getTime()
if (diff <= 0) {
return "expired"
}
const days = Math.floor(diff / (1000 * 60 * 60 * 24))
const hours = Math.floor((diff % (1000 * 60 * 60 * 24)) / (1000 * 60 * 60))
if (days > 0) {
return `${days} day${days === 1 ? "" : "s"}`
}
return `${hours} hour${hours === 1 ? "" : "s"}`
}

34
apps/cli/src/commands/cli/list.ts
View file

@ -6,11 +6,10 @@ import pWaitFor from "p-wait-for"
import type { TaskSessionEntry } from "@roo-code/core/cli"
import type { Command, ModelRecord, WebviewMessage } from "@roo-code/types"
import { getProviderDefaultModelId } from "@roo-code/types"
import { openRouterDefaultModelId } from "@roo-code/types"
import { ExtensionHost, type ExtensionHostOptions } from "@/agent/index.js"
import { readWorkspaceTaskSessions } from "@/lib/task-history/index.js"
import { loadToken } from "@/lib/storage/index.js"
import { getDefaultExtensionPath } from "@/lib/utils/extension.js"
import { getApiKeyFromEnv } from "@/lib/utils/provider.js"
import { isRecord } from "@/lib/utils/guards.js"
@ -106,14 +105,14 @@ function outputSessionsText(sessions: SessionLike[]): void {
async function createListHost(options: BaseListOptions, hostOptions: ListHostOptions): Promise<ExtensionHost> {
const workspacePath = resolveWorkspacePath(options.workspace)
const extensionPath = resolveExtensionPath(options.extension)
const apiKey = options.apiKey || (await loadToken()) || getApiKeyFromEnv("roo")
const apiKey = options.apiKey || getApiKeyFromEnv("openrouter")
const extensionHostOptions: ExtensionHostOptions = {
mode: "code",
reasoningEffort: undefined,
user: null,
provider: "roo",
model: getProviderDefaultModelId("roo"),
provider: "openrouter",
model: openRouterDefaultModelId,
apiKey,
workspacePath,
extensionPath,
@ -215,26 +214,15 @@ function requestModes(host: ExtensionHost): Promise<ModeLike[]> {
})
}
function requestRooModels(host: ExtensionHost): Promise<ModelRecord> {
return requestFromExtension(host, "requestRooModels", (message) => {
if (message.type !== "singleRouterModelFetchResponse") {
function requestOpenRouterModels(host: ExtensionHost): Promise<ModelRecord> {
return requestFromExtension(host, "requestRouterModels", (message) => {
if (message.type !== "routerModels") {
return undefined
}
const values = isRecord(message.values) ? message.values : undefined
if (values?.provider !== "roo") {
return undefined
}
if (message.success === false) {
const errorMessage =
typeof message.error === "string" && message.error.length > 0
? message.error
: "Failed to fetch Roo models"
throw new Error(errorMessage)
}
return isRecord(values.models) ? (values.models as ModelRecord) : {}
const routerModels = isRecord(message.routerModels) ? message.routerModels : {}
const openRouterModels = routerModels.openrouter
return isRecord(openRouterModels) ? (openRouterModels as ModelRecord) : {}
})
}
@ -299,7 +287,7 @@ export async function listModels(options: BaseListOptions): Promise<void> {
const format = parseFormat(options.format)
await withHostAndSignalHandlers(options, { ephemeral: true }, async (host) => {
const models = await requestRooModels(host)
const models = await requestOpenRouterModels(host)
if (format === "json") {
outputJson({ models })

132
apps/cli/src/commands/cli/run.ts
View file

@ -10,22 +10,17 @@ import { setLogger } from "@roo-code/vscode-shim"
import {
FlagOptions,
isSupportedProvider,
OnboardingProviderChoice,
supportedProviders,
DEFAULT_FLAGS,
REASONING_EFFORTS,
SDK_BASE_URL,
OutputFormat,
} from "@/types/index.js"
import { isValidOutputFormat } from "@/types/json-events.js"
import { JsonEventEmitter } from "@/agent/json-event-emitter.js"
import { createClient } from "@/lib/sdk/index.js"
import { loadToken, loadSettings } from "@/lib/storage/index.js"
import { loadSettings } from "@/lib/storage/index.js"
import { readWorkspaceTaskSessions, resolveWorkspaceResumeSessionId } from "@/lib/task-history/index.js"
import { isRecord } from "@/lib/utils/guards.js"
import { getEnvVarName, getApiKeyFromEnv } from "@/lib/utils/provider.js"
import { runOnboarding } from "@/lib/utils/onboarding.js"
import { validateTerminalShellPath } from "@/lib/utils/shell.js"
import { getDefaultExtensionPath } from "@/lib/utils/extension.js"
import { isValidSessionId } from "@/lib/utils/session-id.js"
@ -36,7 +31,6 @@ import { isExpectedControlFlowError } from "./cancellation.js"
import { runStdinStreamMode } from "./stdin-stream.js"
const __dirname = path.dirname(fileURLToPath(import.meta.url))
const ROO_MODEL_WARMUP_TIMEOUT_MS = 10_000
const SIGNAL_ONLY_EXIT_KEEPALIVE_MS = 60_000
const STREAM_RESUME_WAIT_TIMEOUT_MS = 2_000
@ -54,59 +48,6 @@ function normalizeError(error: unknown): Error {
return error instanceof Error ? error : new Error(String(error))
}
async function warmRooModels(host: ExtensionHost): Promise<void> {
await new Promise<void>((resolve, reject) => {
let settled = false
const cleanup = () => {
clearTimeout(timeoutId)
host.off("extensionWebviewMessage", onMessage)
}
const finish = (fn: () => void) => {
if (settled) return
settled = true
cleanup()
fn()
}
const onMessage = (message: unknown) => {
if (!isRecord(message)) {
return
}
if (message.type !== "singleRouterModelFetchResponse") {
return
}
const values = isRecord(message.values) ? message.values : undefined
if (values?.provider !== "roo") {
return
}
if (message.success === false) {
const errorMessage =
typeof message.error === "string" && message.error.length > 0
? message.error
: "failed to refresh Roo models"
finish(() => reject(new Error(errorMessage)))
return
}
finish(() => resolve())
}
const timeoutId = setTimeout(() => {
finish(() => reject(new Error(`timed out waiting for Roo models after ${ROO_MODEL_WARMUP_TIMEOUT_MS}ms`)))
}, ROO_MODEL_WARMUP_TIMEOUT_MS)
host.on("extensionWebviewMessage", onMessage)
host.sendToExtension({ type: "requestRooModels" })
})
}
export async function run(promptArg: string | undefined, flagOptions: FlagOptions) {
setLogger({
info: () => {},
@ -169,19 +110,17 @@ export async function run(promptArg: string | undefined, flagOptions: FlagOption
// Options
let rooToken = await loadToken()
const settings = await loadSettings()
const isTuiSupported = process.stdin.isTTY && process.stdout.isTTY
const isTuiEnabled = !flagOptions.print && isTuiSupported
const isOnboardingEnabled = isTuiEnabled && !rooToken && !flagOptions.provider && !settings.provider
// Determine effective values: CLI flags > settings file > DEFAULT_FLAGS.
const effectiveMode = flagOptions.mode || settings.mode || DEFAULT_FLAGS.mode
const effectiveModel = flagOptions.model || settings.model || DEFAULT_FLAGS.model
const effectiveReasoningEffort =
flagOptions.reasoningEffort || settings.reasoningEffort || DEFAULT_FLAGS.reasoningEffort
const effectiveProvider = flagOptions.provider ?? settings.provider ?? (rooToken ? "roo" : "openrouter")
const effectiveProvider = flagOptions.provider ?? settings.provider ?? "openrouter"
const effectiveWorkspacePath = flagOptions.workspace ? path.resolve(flagOptions.workspace) : process.cwd()
const legacyRequireApprovalFromSettings =
settings.requireApproval ??
@ -229,49 +168,6 @@ export async function run(promptArg: string | undefined, flagOptions: FlagOption
terminalShell,
}
// Roo Code Cloud Authentication
if (isOnboardingEnabled) {
let { onboardingProviderChoice } = settings
if (!onboardingProviderChoice) {
const { choice, token } = await runOnboarding()
onboardingProviderChoice = choice
rooToken = token ?? null
}
if (onboardingProviderChoice === OnboardingProviderChoice.Roo) {
extensionHostOptions.provider = "roo"
}
}
if (extensionHostOptions.provider === "roo") {
if (rooToken) {
try {
const client = createClient({ url: SDK_BASE_URL, authToken: rooToken })
const me = await client.auth.me.query()
if (me?.type !== "user") {
throw new Error("Invalid token")
}
extensionHostOptions.apiKey = rooToken
extensionHostOptions.user = me.user
} catch {
// If an explicit API key was provided via flag or env var, fall through
// to the general API key resolution below instead of exiting.
if (!flagOptions.apiKey && !getApiKeyFromEnv(extensionHostOptions.provider)) {
console.error("[CLI] Your Roo Code Router token is not valid.")
console.error("[CLI] Please run: roo auth login")
console.error("[CLI] Or use --api-key or set ROO_API_KEY to provide your own API key.")
process.exit(1)
}
}
}
// If no rooToken, fall through to the general API key resolution below
// which will check flagOptions.apiKey and ROO_API_KEY env var.
}
// Validations
// TODO: Validate the API key for the chosen provider.
// TODO: Validate the model for the chosen provider.
@ -287,18 +183,8 @@ export async function run(promptArg: string | undefined, flagOptions: FlagOption
extensionHostOptions.apiKey || flagOptions.apiKey || getApiKeyFromEnv(extensionHostOptions.provider)
if (!extensionHostOptions.apiKey) {
if (extensionHostOptions.provider === "roo") {
console.error("[CLI] Error: Authentication with Roo Code Cloud failed or was cancelled.")
console.error("[CLI] Please run: roo auth login")
console.error("[CLI] Or use --api-key to provide your own API key.")
} else {
console.error(
`[CLI] Error: No API key provided. Use --api-key or set the appropriate environment variable.`,
)
console.error(
`[CLI] For ${extensionHostOptions.provider}, set ${getEnvVarName(extensionHostOptions.provider)}`,
)
}
console.error(`[CLI] Error: No API key provided. Use --api-key or set the appropriate environment variable.`)
console.error(`[CLI] For ${extensionHostOptions.provider}, set ${getEnvVarName(extensionHostOptions.provider)}`)
process.exit(1)
}
@ -606,16 +492,6 @@ export async function run(promptArg: string | undefined, flagOptions: FlagOption
try {
await host.activate()
if (extensionHostOptions.provider === "roo") {
try {
await warmRooModels(host)
} catch (warmupError) {
if (flagOptions.debug) {
const message = warmupError instanceof Error ? warmupError.message : String(warmupError)
console.error(`[CLI] Warning: Roo model warmup failed: ${message}`)
}
}
}
if (jsonEmitter) {
jsonEmitter.attachToClient(host.client)

1
apps/cli/src/commands/index.ts
View file

@ -1,2 +1 @@
export * from "./auth/index.js"
export * from "./cli/index.js"

47
apps/cli/src/index.ts
View file

@ -2,17 +2,7 @@ import { Command } from "commander"
import { DEFAULT_FLAGS } from "@/types/constants.js"
import { VERSION } from "@/lib/utils/version.js"
import {
run,
login,
logout,
status,
listCommands,
listModes,
listModels,
listSessions,
upgrade,
} from "@/commands/index.js"
import { run, listCommands, listModes, listModels, listSessions, upgrade } from "@/commands/index.js"
const program = new Command()
@ -45,7 +35,7 @@ program
.option("-d, --debug", "Enable debug output (includes detailed debug information)", false)
.option("-a, --require-approval", "Require manual approval for actions", false)
.option("-k, --api-key <key>", "API key for the LLM provider")
.option("--provider <provider>", "API provider (roo, anthropic, openai, openrouter, etc.)")
.option("--provider <provider>", "API provider (anthropic, openai, openrouter, etc.)")
.option("-m, --model <model>", "Model to use", DEFAULT_FLAGS.model)
.option("--mode <mode>", "Mode to start in (code, architect, ask, debug, etc.)", DEFAULT_FLAGS.mode)
.option("--terminal-shell <path>", "Absolute path to shell executable for inline terminal commands")
@ -79,7 +69,7 @@ const applyListOptions = (command: Command) =>
command
.option("-w, --workspace <path>", "Workspace directory path (defaults to current working directory)")
.option("-e, --extension <path>", "Path to the extension bundle directory")
.option("-k, --api-key <key>", "Roo API key (falls back to saved login/session token)")
.option("-k, --api-key <key>", "API key for the LLM provider")
.option("--format <format>", 'Output format: "json" (default) or "text"', "json")
.option("-d, --debug", "Enable debug output", false)
@ -117,7 +107,7 @@ applyListOptions(listCommand.command("modes").description("List available modes"
},
)
applyListOptions(listCommand.command("models").description("List available Roo models")).action(
applyListOptions(listCommand.command("models").description("List available models")).action(
async (options: Parameters<typeof listModels>[0]) => {
await runListAction(() => listModels(options))
},
@ -136,33 +126,4 @@ program
await runUpgradeAction(() => upgrade())
})
const authCommand = program.command("auth").description("Manage authentication for Roo Code Cloud")
authCommand
.command("login")
.description("Authenticate with Roo Code Cloud")
.option("-v, --verbose", "Enable verbose output", false)
.action(async (options: { verbose: boolean }) => {
const result = await login({ verbose: options.verbose })
process.exit(result.success ? 0 : 1)
})
authCommand
.command("logout")
.description("Log out from Roo Code Cloud")
.option("-v, --verbose", "Enable verbose output", false)
.action(async (options: { verbose: boolean }) => {
const result = await logout({ verbose: options.verbose })
process.exit(result.success ? 0 : 1)
})
authCommand
.command("status")
.description("Show authentication status")
.option("-v, --verbose", "Enable verbose output", false)
.action(async (options: { verbose: boolean }) => {
const result = await status({ verbose: options.verbose })
process.exit(result.authenticated ? 0 : 1)
})
program.parse()

1
apps/cli/src/lib/auth/index.ts
View file

@ -1 +0,0 @@
export * from "./token.js"

61
apps/cli/src/lib/auth/token.ts
View file

@ -1,61 +0,0 @@
export interface DecodedToken {
iss: string
sub: string
exp: number
iat: number
nbf: number
v: number
r?: {
u?: string
o?: string
t: string
}
}
function decodeToken(token: string): DecodedToken | null {
try {
const parts = token.split(".")
if (parts.length !== 3) {
return null
}
const payload = parts[1]
if (!payload) {
return null
}
const padded = payload + "=".repeat((4 - (payload.length % 4)) % 4)
const decoded = Buffer.from(padded, "base64url").toString("utf-8")
return JSON.parse(decoded) as DecodedToken
} catch {
return null
}
}
export function isTokenExpired(token: string, bufferSeconds = 24 * 60 * 60): boolean {
const decoded = decodeToken(token)
if (!decoded?.exp) {
return true
}
const expiresAt = decoded.exp
const bufferTime = Math.floor(Date.now() / 1000) + bufferSeconds
return expiresAt < bufferTime
}
export function isTokenValid(token: string): boolean {
return !isTokenExpired(token, 0)
}
export function getTokenExpirationDate(token: string): Date | null {
const decoded = decodeToken(token)
if (!decoded?.exp) {
return null
}
return new Date(decoded.exp * 1000)
}

152
apps/cli/src/lib/storage/__tests__/credentials.test.ts
View file

@ -1,152 +0,0 @@
import fs from "fs/promises"
import path from "path"
// Use vi.hoisted to make the test directory available to the mock
// This must return the path synchronously since CREDENTIALS_FILE is computed at import time
const { getTestConfigDir } = vi.hoisted(() => {
// eslint-disable-next-line @typescript-eslint/no-require-imports
const os = require("os")
// eslint-disable-next-line @typescript-eslint/no-require-imports
const path = require("path")
const testRunId = Date.now().toString()
const testConfigDir = path.join(os.tmpdir(), `roo-cli-test-${testRunId}`)
return { getTestConfigDir: () => testConfigDir }
})
vi.mock("../config-dir.js", () => ({
getConfigDir: getTestConfigDir,
}))
// Import after mocking
import { saveToken, loadToken, loadCredentials, clearToken, hasToken, getCredentialsPath } from "../credentials.js"
// Re-derive the test config dir for use in tests (must match the hoisted one)
const actualTestConfigDir = getTestConfigDir()
describe("Token Storage", () => {
const expectedCredentialsFile = path.join(actualTestConfigDir, "cli-credentials.json")
beforeEach(async () => {
// Clear test directory before each test
await fs.rm(actualTestConfigDir, { recursive: true, force: true })
})
afterAll(async () => {
// Clean up test directory
await fs.rm(actualTestConfigDir, { recursive: true, force: true })
})
describe("getCredentialsPath", () => {
it("should return the correct credentials file path", () => {
expect(getCredentialsPath()).toBe(expectedCredentialsFile)
})
})
describe("saveToken", () => {
it("should save token to disk", async () => {
const token = "test-token-123"
await saveToken(token)
const savedData = await fs.readFile(expectedCredentialsFile, "utf-8")
const credentials = JSON.parse(savedData)
expect(credentials.token).toBe(token)
expect(credentials.createdAt).toBeDefined()
})
it("should save token with user info", async () => {
const token = "test-token-456"
await saveToken(token, { userId: "user_123", orgId: "org_456" })
const savedData = await fs.readFile(expectedCredentialsFile, "utf-8")
const credentials = JSON.parse(savedData)
expect(credentials.token).toBe(token)
expect(credentials.userId).toBe("user_123")
expect(credentials.orgId).toBe("org_456")
})
it("should create config directory if it doesn't exist", async () => {
const token = "test-token-789"
await saveToken(token)
const dirStats = await fs.stat(actualTestConfigDir)
expect(dirStats.isDirectory()).toBe(true)
})
// Unix file permissions don't apply on Windows - skip this test
it.skipIf(process.platform === "win32")("should set restrictive file permissions", async () => {
const token = "test-token-perms"
await saveToken(token)
const stats = await fs.stat(expectedCredentialsFile)
// Check that only owner has read/write (mode 0o600)
const mode = stats.mode & 0o777
expect(mode).toBe(0o600)
})
})
describe("loadToken", () => {
it("should load saved token", async () => {
const token = "test-token-abc"
await saveToken(token)
const loaded = await loadToken()
expect(loaded).toBe(token)
})
it("should return null if no token exists", async () => {
const loaded = await loadToken()
expect(loaded).toBeNull()
})
})
describe("loadCredentials", () => {
it("should load full credentials", async () => {
const token = "test-token-def"
await saveToken(token, { userId: "user_789" })
const credentials = await loadCredentials()
expect(credentials).not.toBeNull()
expect(credentials?.token).toBe(token)
expect(credentials?.userId).toBe("user_789")
expect(credentials?.createdAt).toBeDefined()
})
it("should return null if no credentials exist", async () => {
const credentials = await loadCredentials()
expect(credentials).toBeNull()
})
})
describe("clearToken", () => {
it("should remove saved token", async () => {
const token = "test-token-ghi"
await saveToken(token)
await clearToken()
const loaded = await loadToken()
expect(loaded).toBeNull()
})
it("should not throw if no token exists", async () => {
await expect(clearToken()).resolves.not.toThrow()
})
})
describe("hasToken", () => {
it("should return true if token exists", async () => {
await saveToken("test-token-jkl")
const exists = await hasToken()
expect(exists).toBe(true)
})
it("should return false if no token exists", async () => {
const exists = await hasToken()
expect(exists).toBe(false)
})
})
})

4
apps/cli/src/lib/storage/__tests__/settings.test.ts
View file

@ -51,7 +51,7 @@ describe("Settings Storage", () => {
it("should load saved settings", async () => {
const settingsData = {
onboardingProviderChoice: OnboardingProviderChoice.Roo,
onboardingProviderChoice: OnboardingProviderChoice.Byok,
mode: "architect",
provider: "anthropic" as const,
model: "claude-sonnet-4-20250514",
@ -138,7 +138,7 @@ describe("Settings Storage", () => {
describe("resetOnboarding", () => {
it("should reset onboarding provider choice", async () => {
await saveSettings({ onboardingProviderChoice: OnboardingProviderChoice.Roo })
await saveSettings({ onboardingProviderChoice: OnboardingProviderChoice.Byok })
await resetOnboarding()

72
apps/cli/src/lib/storage/credentials.ts
View file

@ -1,72 +0,0 @@
import fs from "fs/promises"
import path from "path"
import { getConfigDir } from "./index.js"
const CREDENTIALS_FILE = path.join(getConfigDir(), "cli-credentials.json")
export interface Credentials {
token: string
createdAt: string
userId?: string
orgId?: string
}
export async function saveToken(token: string, options?: { userId?: string; orgId?: string }): Promise<void> {
await fs.mkdir(getConfigDir(), { recursive: true })
const credentials: Credentials = {
token,
createdAt: new Date().toISOString(),
userId: options?.userId,
orgId: options?.orgId,
}
await fs.writeFile(CREDENTIALS_FILE, JSON.stringify(credentials, null, 2), {
mode: 0o600, // Read/write for owner only
})
}
export async function loadToken(): Promise<string | null> {
try {
const data = await fs.readFile(CREDENTIALS_FILE, "utf-8")
const credentials: Credentials = JSON.parse(data)
return credentials.token
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return null
}
throw error
}
}
export async function loadCredentials(): Promise<Credentials | null> {
try {
const data = await fs.readFile(CREDENTIALS_FILE, "utf-8")
return JSON.parse(data) as Credentials
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
return null
}
throw error
}
}
export async function clearToken(): Promise<void> {
try {
await fs.unlink(CREDENTIALS_FILE)
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
throw error
}
}
}
export async function hasToken(): Promise<boolean> {
const token = await loadToken()
return token !== null
}
export function getCredentialsPath(): string {
return CREDENTIALS_FILE
}

1
apps/cli/src/lib/storage/index.ts
View file

@ -1,4 +1,3 @@
export * from "./config-dir.js"
export * from "./settings.js"
export * from "./credentials.js"
export * from "./ephemeral.js"

38
apps/cli/src/lib/utils/onboarding.ts
View file

@ -1,38 +0,0 @@
import { createElement } from "react"
import { type OnboardingResult, OnboardingProviderChoice } from "@/types/index.js"
import { login } from "@/commands/index.js"
import { saveSettings } from "@/lib/storage/index.js"
export async function runOnboarding(): Promise<OnboardingResult> {
const { render } = await import("ink")
const { OnboardingScreen } = await import("../../ui/components/onboarding/index.js")
return new Promise<OnboardingResult>((resolve) => {
const onSelect = async (choice: OnboardingProviderChoice) => {
await saveSettings({ onboardingProviderChoice: choice })
app.unmount()
console.log("")
if (choice === OnboardingProviderChoice.Roo) {
const result = await login()
await saveSettings({ onboardingProviderChoice: choice })
resolve({
choice: OnboardingProviderChoice.Roo,
token: result.success ? result.token : undefined,
skipped: false,
})
} else {
console.log("Using your own API key.")
console.log("Set your API key via --api-key or environment variable.")
console.log("")
resolve({ choice: OnboardingProviderChoice.Byok, skipped: false })
}
}
const app = render(createElement(OnboardingScreen, { onSelect }))
})
}

5
apps/cli/src/lib/utils/provider.ts
View file

@ -8,7 +8,6 @@ const envVarMap: Record<SupportedProvider, string> = {
gemini: "GOOGLE_API_KEY",
openrouter: "OPENROUTER_API_KEY",
"vercel-ai-gateway": "VERCEL_AI_GATEWAY_API_KEY",
roo: "ROO_API_KEY",
}
export function getEnvVarName(provider: SupportedProvider): string {
@ -48,10 +47,6 @@ export function getProviderSettings(
if (apiKey) config.vercelAiGatewayApiKey = apiKey
if (model) config.vercelAiGatewayModelId = model
break
case "roo":
if (apiKey) config.rooApiKey = apiKey
if (model) config.apiModelId = model
break
default:
if (apiKey) config.apiKey = apiKey
if (model) config.apiModelId = model

4
apps/cli/src/types/constants.ts
View file

@ -21,7 +21,3 @@ export const ASCII_ROO = ` _,' ___
\\,\\ / \\\\
// \\\\
,/' \`\\_,`
export const AUTH_BASE_URL = process.env.ROO_AUTH_BASE_URL ?? "https://app.roocode.com"
export const SDK_BASE_URL = process.env.ROO_SDK_BASE_URL ?? "https://cloud-api.roocode.com"

2
apps/cli/src/types/types.ts
View file

@ -7,7 +7,6 @@ export const supportedProviders = [
"gemini",
"openrouter",
"vercel-ai-gateway",
"roo",
] as const satisfies ProviderName[]
export type SupportedProvider = (typeof supportedProviders)[number]
@ -44,7 +43,6 @@ export type FlagOptions = {
}
export enum OnboardingProviderChoice {
Roo = "roo",
Byok = "byok",
}

28
apps/cli/src/ui/components/onboarding/OnboardingScreen.tsx
View file

@ -1,28 +0,0 @@
import { Box, Text } from "ink"
import { Select } from "@inkjs/ui"
import { OnboardingProviderChoice, ASCII_ROO } from "@/types/index.js"
export interface OnboardingScreenProps {
onSelect: (choice: OnboardingProviderChoice) => void
}
export function OnboardingScreen({ onSelect }: OnboardingScreenProps) {
return (
<Box flexDirection="column" gap={1}>
<Text bold color="cyan">
{ASCII_ROO}
</Text>
<Text dimColor>Welcome! How would you like to connect to an LLM provider?</Text>
<Select
options={[
{ label: "Connect to Roo Code Cloud", value: OnboardingProviderChoice.Roo },
{ label: "Bring your own API key", value: OnboardingProviderChoice.Byok },
]}
onChange={(value: string) => {
onSelect(value as OnboardingProviderChoice)
}}
/>
</Box>
)
}

1
apps/cli/src/ui/components/onboarding/index.ts
View file

@ -1 +0,0 @@
export * from "./OnboardingScreen.js"

1
apps/web-evals/.env
View file

@ -1 +0,0 @@
DATABASE_URL=postgres://postgres:password@localhost:5433/evals_development

8
apps/web-evals/.gitignore vendored
View file

@ -1,8 +0,0 @@
# .env
!.env
# next.js
.next
# typescript
tsconfig.tsbuildinfo

3
apps/web-evals/CHANGELOG.md
View file

@ -1,3 +0,0 @@
# @roo-code/web-evals
## 0.0.1

21
apps/web-evals/components.json
View file

@ -1,21 +0,0 @@
{
"$schema": "https://ui.shadcn.com/schema.json",
"style": "new-york",
"rsc": true,
"tsx": true,
"tailwind": {
"config": "",
"css": "src/app/globals.css",
"baseColor": "neutral",
"cssVariables": true,
"prefix": ""
},
"aliases": {
"components": "@/components",
"utils": "@/lib/utils",
"ui": "@/components/ui",
"lib": "@/lib",
"hooks": "@/hooks"
},
"iconLibrary": "lucide"
}

17
apps/web-evals/eslint.config.mjs
View file

@ -1,17 +0,0 @@
import { nextJsConfig } from "@roo-code/config-eslint/next-js"
/** @type {import("eslint").Linter.Config} */
export default [
...nextJsConfig,
{
rules: {
"no-unused-vars": "off",
"@typescript-eslint/no-unused-vars": [
"error",
{
caughtErrorsIgnorePattern: "^_",
},
],
},
},
]

6
apps/web-evals/next-env.d.ts vendored
View file

@ -1,6 +0,0 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />
import "./.next/dev/types/routes.d.ts"
// NOTE: This file should not be edited
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

7
apps/web-evals/next.config.ts
View file

@ -1,7 +0,0 @@
import type { NextConfig } from "next"
const nextConfig: NextConfig = {
turbopack: {},
}
export default nextConfig

63
apps/web-evals/package.json
View file

@ -1,63 +0,0 @@
{
"name": "@roo-code/web-evals",
"version": "0.0.1",
"type": "module",
"scripts": {
"lint": "eslint src --ext=ts,tsx --max-warnings=0",
"check-types": "tsc -b",
"dev": "scripts/check-services.sh && next dev -p 3446",
"format": "prettier --write src",
"build": "next build",
"start": "next start -p 3446",
"clean": "rimraf tsconfig.tsbuildinfo .next .turbo"
},
"dependencies": {
"@hookform/resolvers": "^5.1.1",
"@radix-ui/react-alert-dialog": "^1.1.7",
"@radix-ui/react-checkbox": "^1.1.5",
"@radix-ui/react-dialog": "^1.1.6",
"@radix-ui/react-dropdown-menu": "^2.1.7",
"@radix-ui/react-label": "^2.1.2",
"@radix-ui/react-popover": "^1.1.6",
"@radix-ui/react-scroll-area": "^1.2.3",
"@radix-ui/react-select": "^2.1.6",
"@radix-ui/react-separator": "^1.1.2",
"@radix-ui/react-slider": "^1.2.4",
"@radix-ui/react-slot": "^1.1.2",
"@radix-ui/react-tabs": "^1.1.3",
"@radix-ui/react-tooltip": "^1.2.8",
"@roo-code/evals": "workspace:^",
"@roo-code/types": "workspace:^",
"@tanstack/react-query": "^5.69.0",
"archiver": "^7.0.1",
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"cmdk": "^1.1.0",
"fuzzysort": "^3.1.0",
"lucide-react": "^0.518.0",
"next": "^16.1.6",
"next-themes": "^0.4.6",
"p-map": "^7.0.3",
"react": "^18.3.1",
"react-dom": "^18.3.1",
"react-hook-form": "^7.57.0",
"react-use": "^17.6.0",
"redis": "^5.5.5",
"sonner": "^2.0.5",
"tailwind-merge": "^3.3.0",
"tailwindcss-animate": "^1.0.7",
"vaul": "^1.1.2",
"zod": "^3.25.61"
},
"devDependencies": {
"@roo-code/config-eslint": "workspace:^",
"@roo-code/config-typescript": "workspace:^",
"@tailwindcss/postcss": "^4",
"@types/archiver": "^7.0.0",
"@types/ps-tree": "^1.1.6",
"@types/react": "^18.3.23",
"@types/react-dom": "^18.3.5",
"tailwindcss": "^4",
"vitest": "^3.2.3"
}
}

5
apps/web-evals/postcss.config.mjs
View file

@ -1,5 +0,0 @@
const config = {
plugins: ["@tailwindcss/postcss"],
}
export default config

0
apps/web-evals/public/.gitkeep
View file

20
apps/web-evals/scripts/check-services.sh
View file

@ -1,20 +0,0 @@
#!/bin/bash
if ! docker info &> /dev/null; then
echo "❌ Docker is not running. Please start Docker Desktop and try again."
exit 1
fi
if ! nc -z postgres 5433 2>/dev/null; then
echo "❌ PostgreSQL is not running on port 5432"
echo "💡 Start it with: pnpm --filter @roo-code/evals db:up"
exit 1
fi
if ! nc -z redis 6380 2>/dev/null; then
echo "❌ Redis is not running on port 6379"
echo "💡 Start it with: pnpm --filter @roo-code/evals redis:up"
exit 1
fi
echo "✅ All required services are running"

207
apps/web-evals/src/actions/__tests__/killRun.spec.ts
View file

@ -1,207 +0,0 @@
// npx vitest run src/actions/__tests__/killRun.spec.ts
import { execFileSync } from "child_process"
// Mock child_process
vi.mock("child_process", () => ({
execFileSync: vi.fn(),
spawn: vi.fn(),
}))
// Mock next/cache
vi.mock("next/cache", () => ({
revalidatePath: vi.fn(),
}))
// Mock redis client
vi.mock("@/lib/server/redis", () => ({
redisClient: vi.fn().mockResolvedValue({
del: vi.fn().mockResolvedValue(1),
}),
}))
// Mock @roo-code/evals
vi.mock("@roo-code/evals", () => ({
createRun: vi.fn(),
deleteRun: vi.fn(),
createTask: vi.fn(),
exerciseLanguages: [],
getExercisesForLanguage: vi.fn().mockResolvedValue([]),
}))
// Mock timers to speed up tests
vi.useFakeTimers()
// Import after mocks
import { killRun } from "../runs"
const mockExecFileSync = execFileSync as ReturnType<typeof vi.fn>
describe("killRun", () => {
beforeEach(() => {
vi.clearAllMocks()
})
afterEach(() => {
vi.clearAllTimers()
})
it("should kill controller first, wait, then kill task containers", async () => {
const runId = 123
// execFileSync is used for all docker commands
mockExecFileSync
.mockReturnValueOnce("") // docker kill controller
.mockReturnValueOnce("evals-task-123-456.0\nevals-task-123-789.1\n") // docker ps
.mockReturnValueOnce("") // docker kill evals-task-123-456.0
.mockReturnValueOnce("") // docker kill evals-task-123-789.1
const resultPromise = killRun(runId)
// Fast-forward past the 10 second sleep
await vi.advanceTimersByTimeAsync(10000)
const result = await resultPromise
expect(result.success).toBe(true)
expect(result.killedContainers).toContain("evals-controller-123")
expect(result.killedContainers).toContain("evals-task-123-456.0")
expect(result.killedContainers).toContain("evals-task-123-789.1")
expect(result.errors).toHaveLength(0)
// Verify execFileSync was called for docker kill
expect(mockExecFileSync).toHaveBeenNthCalledWith(
1,
"docker",
["kill", "evals-controller-123"],
expect.any(Object),
)
// Verify execFileSync was called for docker ps with run-specific filter
expect(mockExecFileSync).toHaveBeenNthCalledWith(
2,
"docker",
["ps", "--format", "{{.Names}}", "--filter", "name=evals-task-123-"],
expect.any(Object),
)
})
it("should continue killing runners even if controller is not running", async () => {
const runId = 456
mockExecFileSync
.mockImplementationOnce(() => {
throw new Error("No such container")
}) // controller kill fails
.mockReturnValueOnce("evals-task-456-100.0\n") // docker ps
.mockReturnValueOnce("") // docker kill task
const resultPromise = killRun(runId)
await vi.advanceTimersByTimeAsync(10000)
const result = await resultPromise
expect(result.success).toBe(true)
expect(result.killedContainers).toContain("evals-task-456-100.0")
// Controller not in list since it failed
expect(result.killedContainers).not.toContain("evals-controller-456")
})
it("should clear Redis state after killing containers", async () => {
const runId = 789
const mockDel = vi.fn().mockResolvedValue(1)
const { redisClient } = await import("@/lib/server/redis")
vi.mocked(redisClient).mockResolvedValue({ del: mockDel } as never)
mockExecFileSync
.mockReturnValueOnce("") // controller kill
.mockReturnValueOnce("") // docker ps (no tasks)
const resultPromise = killRun(runId)
await vi.advanceTimersByTimeAsync(10000)
await resultPromise
expect(mockDel).toHaveBeenCalledWith("heartbeat:789")
expect(mockDel).toHaveBeenCalledWith("runners:789")
})
it("should handle docker ps failure gracefully", async () => {
const runId = 111
mockExecFileSync
.mockReturnValueOnce("") // controller kill succeeds
.mockImplementationOnce(() => {
throw new Error("Docker error")
}) // docker ps fails
const resultPromise = killRun(runId)
await vi.advanceTimersByTimeAsync(10000)
const result = await resultPromise
// Should still be successful because controller was killed
expect(result.success).toBe(true)
expect(result.killedContainers).toContain("evals-controller-111")
expect(result.errors).toContain("Failed to list Docker task containers")
})
it("should handle individual task kill failures", async () => {
const runId = 222
mockExecFileSync
.mockReturnValueOnce("") // controller kill
.mockReturnValueOnce("evals-task-222-300.0\nevals-task-222-400.0\n") // docker ps
.mockImplementationOnce(() => {
throw new Error("Kill failed")
}) // first task kill fails
.mockReturnValueOnce("") // second task kill succeeds
const resultPromise = killRun(runId)
await vi.advanceTimersByTimeAsync(10000)
const result = await resultPromise
expect(result.success).toBe(true)
expect(result.killedContainers).toContain("evals-controller-222")
expect(result.killedContainers).toContain("evals-task-222-400.0")
expect(result.errors.length).toBe(1)
expect(result.errors[0]).toContain("evals-task-222-300.0")
})
it("should return success with no containers when nothing is running", async () => {
const runId = 333
mockExecFileSync
.mockImplementationOnce(() => {
throw new Error("No such container")
}) // controller not running
.mockReturnValueOnce("") // no task containers
const resultPromise = killRun(runId)
await vi.advanceTimersByTimeAsync(10000)
const result = await resultPromise
expect(result.success).toBe(true)
expect(result.killedContainers).toHaveLength(0)
expect(result.errors).toHaveLength(0)
})
it("should only kill containers belonging to the specific run", async () => {
const runId = 555
mockExecFileSync
.mockReturnValueOnce("") // controller kill
.mockReturnValueOnce("evals-task-555-100.0\n") // docker ps
.mockReturnValueOnce("") // docker kill task
const resultPromise = killRun(runId)
await vi.advanceTimersByTimeAsync(10000)
const result = await resultPromise
expect(result.success).toBe(true)
// Verify execFileSync was called for docker ps with run-specific filter
expect(mockExecFileSync).toHaveBeenNthCalledWith(
2,
"docker",
["ps", "--format", "{{.Names}}", "--filter", "name=evals-task-555-"],
expect.any(Object),
)
})
})

22
apps/web-evals/src/actions/exercises.ts
View file

@ -1,22 +0,0 @@
"use server"
import * as path from "path"
import { fileURLToPath } from "url"
import { exerciseLanguages, listDirectories } from "@roo-code/evals"
const __dirname = path.dirname(fileURLToPath(import.meta.url)) // <repo>/apps/web-evals/src/actions
const EVALS_REPO_PATH = path.resolve(__dirname, "../../../../../evals")
export const getExercises = async () => {
const result = await Promise.all(
exerciseLanguages.map(async (language) => {
const languagePath = path.join(EVALS_REPO_PATH, language)
const exercises = await listDirectories(__dirname, languagePath)
return exercises.map((exercise) => `${language}/${exercise}`)
}),
)
return result.flat()
}

8
apps/web-evals/src/actions/heartbeat.ts
View file

@ -1,8 +0,0 @@
"use server"
import { redisClient } from "@/lib/server/redis"
export const getHeartbeat = async (runId: number) => {
const redis = await redisClient()
return redis.get(`heartbeat:${runId}`)
}

8
apps/web-evals/src/actions/runners.ts
View file

@ -1,8 +0,0 @@
"use server"
import { redisClient } from "@/lib/server/redis"
export const getRunners = async (runId: number) => {
const redis = await redisClient()
return redis.sMembers(`runners:${runId}`)
}

377
apps/web-evals/src/actions/runs.ts
View file

@ -1,377 +0,0 @@
"use server"
import * as path from "path"
import fs from "fs"
import { fileURLToPath } from "url"
import { spawn, execFileSync } from "child_process"
import { revalidatePath } from "next/cache"
import pMap from "p-map"
import {
type ExerciseLanguage,
exerciseLanguages,
createRun as _createRun,
deleteRun as _deleteRun,
updateRun as _updateRun,
getIncompleteRuns as _getIncompleteRuns,
deleteRunsByIds as _deleteRunsByIds,
createTask,
getExercisesForLanguage,
} from "@roo-code/evals"
import { CreateRun } from "@/lib/schemas"
import { redisClient } from "@/lib/server/redis"
// Storage base path for eval logs
const EVALS_STORAGE_PATH = "/tmp/evals/runs"
const EVALS_REPO_PATH = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../../../../../evals")
export async function createRun({
suite,
exercises = [],
timeout,
iterations = 1,
executionMethod = "vscode",
...values
}: CreateRun) {
const run = await _createRun({
...values,
timeout,
executionMethod,
socketPath: "", // TODO: Get rid of this.
})
if (suite === "partial") {
for (const path of exercises) {
const [language, exercise] = path.split("/")
if (!language || !exercise) {
throw new Error("Invalid exercise path: " + path)
}
// Create multiple tasks for each iteration
for (let iteration = 1; iteration <= iterations; iteration++) {
await createTask({
...values,
runId: run.id,
language: language as ExerciseLanguage,
exercise,
iteration,
})
}
}
} else {
for (const language of exerciseLanguages) {
const languageExercises = await getExercisesForLanguage(EVALS_REPO_PATH, language)
// Create tasks for all iterations of each exercise
const tasksToCreate: Array<{ language: ExerciseLanguage; exercise: string; iteration: number }> = []
for (const exercise of languageExercises) {
for (let iteration = 1; iteration <= iterations; iteration++) {
tasksToCreate.push({ language, exercise, iteration })
}
}
await pMap(
tasksToCreate,
({ language, exercise, iteration }) => createTask({ runId: run.id, language, exercise, iteration }),
{ concurrency: 10 },
)
}
}
revalidatePath("/runs")
try {
const isRunningInDocker = fs.existsSync("/.dockerenv")
const dockerArgs = [
`--name evals-controller-${run.id}`,
"--rm",
"--network evals_default",
"-v /var/run/docker.sock:/var/run/docker.sock",
"-v /tmp/evals:/var/log/evals",
"-e HOST_EXECUTION_METHOD=docker",
]
const cliCommand = `pnpm --filter @roo-code/evals cli --runId ${run.id}`
const command = isRunningInDocker
? `docker run ${dockerArgs.join(" ")} evals-runner sh -c "${cliCommand}"`
: cliCommand
console.log("spawn ->", command)
const childProcess = spawn("sh", ["-c", command], {
detached: true,
stdio: ["ignore", "pipe", "pipe"],
})
const logStream = fs.createWriteStream("/tmp/roo-code-evals.log", { flags: "a" })
if (childProcess.stdout) {
childProcess.stdout.pipe(logStream)
}
if (childProcess.stderr) {
childProcess.stderr.pipe(logStream)
}
childProcess.unref()
} catch (error) {
console.error(error)
}
return run
}
export async function deleteRun(runId: number) {
await _deleteRun(runId)
revalidatePath("/runs")
}
export type KillRunResult = {
success: boolean
killedContainers: string[]
errors: string[]
}
const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms))
/**
* Kill all Docker containers associated with a run (controller and task runners).
* Kills the controller first, waits 10 seconds, then kills runners.
* Also clears Redis state for heartbeat and runners.
*
* Container naming conventions:
* - Controller: evals-controller-{runId}
* - Task runners: evals-task-{runId}-{taskId}.{attempt}
*/
export async function killRun(runId: number): Promise<KillRunResult> {
const killedContainers: string[] = []
const errors: string[] = []
const controllerPattern = `evals-controller-${runId}`
const taskPattern = `evals-task-${runId}-`
try {
// Step 1: Kill the controller first
console.log(`Killing controller: ${controllerPattern}`)
try {
execFileSync("docker", ["kill", controllerPattern], { encoding: "utf-8", timeout: 10000 })
killedContainers.push(controllerPattern)
console.log(`Killed controller container: ${controllerPattern}`)
} catch (_error) {
// Controller might not be running - that's ok, continue to kill runners
console.log(`Controller ${controllerPattern} not running or already stopped`)
}
// Step 2: Wait 10 seconds before killing runners
console.log("Waiting 10 seconds before killing runners...")
await sleep(10000)
// Step 3: Find and kill all task runner containers for THIS run only
let taskContainerNames: string[] = []
try {
const output = execFileSync("docker", ["ps", "--format", "{{.Names}}", "--filter", `name=${taskPattern}`], {
encoding: "utf-8",
timeout: 10000,
})
taskContainerNames = output
.split("\n")
.map((name) => name.trim())
.filter((name) => name.length > 0 && name.startsWith(taskPattern))
} catch (error) {
console.error("Failed to list task containers:", error)
errors.push("Failed to list Docker task containers")
}
// Kill each task runner container
for (const containerName of taskContainerNames) {
try {
execFileSync("docker", ["kill", containerName], { encoding: "utf-8", timeout: 10000 })
killedContainers.push(containerName)
console.log(`Killed task container: ${containerName}`)
} catch (error) {
// Container might have already stopped
console.error(`Failed to kill container ${containerName}:`, error)
errors.push(`Failed to kill container: ${containerName}`)
}
}
// Step 4: Clear Redis state
try {
const redis = await redisClient()
const heartbeatKey = `heartbeat:${runId}`
const runnersKey = `runners:${runId}`
await redis.del(heartbeatKey)
await redis.del(runnersKey)
console.log(`Cleared Redis keys: ${heartbeatKey}, ${runnersKey}`)
} catch (error) {
console.error("Failed to clear Redis state:", error)
errors.push("Failed to clear Redis state")
}
} catch (error) {
console.error("Error in killRun:", error)
errors.push("Unexpected error while killing containers")
}
revalidatePath(`/runs/${runId}`)
revalidatePath("/runs")
return {
success: killedContainers.length > 0 || errors.length === 0,
killedContainers,
errors,
}
}
export type DeleteIncompleteRunsResult = {
success: boolean
deletedCount: number
deletedRunIds: number[]
storageErrors: string[]
}
/**
* Delete all incomplete runs (runs without a taskMetricsId/final score).
* Removes both database records and storage folders.
*/
export async function deleteIncompleteRuns(): Promise<DeleteIncompleteRunsResult> {
const storageErrors: string[] = []
// Get all incomplete runs
const incompleteRuns = await _getIncompleteRuns()
const runIds = incompleteRuns.map((run) => run.id)
if (runIds.length === 0) {
return {
success: true,
deletedCount: 0,
deletedRunIds: [],
storageErrors: [],
}
}
// Delete storage folders for each run
for (const runId of runIds) {
const storagePath = path.join(EVALS_STORAGE_PATH, String(runId))
try {
if (fs.existsSync(storagePath)) {
fs.rmSync(storagePath, { recursive: true, force: true })
console.log(`Deleted storage folder: ${storagePath}`)
}
} catch (error) {
console.error(`Failed to delete storage folder ${storagePath}:`, error)
storageErrors.push(`Failed to delete storage for run ${runId}`)
}
// Also try to clear Redis state for any potentially running incomplete runs
try {
const redis = await redisClient()
await redis.del(`heartbeat:${runId}`)
await redis.del(`runners:${runId}`)
} catch (error) {
// Non-critical error, just log it
console.error(`Failed to clear Redis state for run ${runId}:`, error)
}
}
// Delete from database
await _deleteRunsByIds(runIds)
revalidatePath("/runs")
return {
success: true,
deletedCount: runIds.length,
deletedRunIds: runIds,
storageErrors,
}
}
/**
* Get count of incomplete runs (for UI display)
*/
export async function getIncompleteRunsCount(): Promise<number> {
const incompleteRuns = await _getIncompleteRuns()
return incompleteRuns.length
}
/**
* Delete all runs older than 30 days.
* Removes both database records and storage folders.
*/
export async function deleteOldRuns(): Promise<DeleteIncompleteRunsResult> {
const storageErrors: string[] = []
// Get all runs older than 30 days
const thirtyDaysAgo = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000)
const { getRuns } = await import("@roo-code/evals")
const allRuns = await getRuns()
const oldRuns = allRuns.filter((run) => run.createdAt < thirtyDaysAgo)
const runIds = oldRuns.map((run) => run.id)
if (runIds.length === 0) {
return {
success: true,
deletedCount: 0,
deletedRunIds: [],
storageErrors: [],
}
}
// Delete storage folders for each run
for (const runId of runIds) {
const storagePath = path.join(EVALS_STORAGE_PATH, String(runId))
try {
if (fs.existsSync(storagePath)) {
fs.rmSync(storagePath, { recursive: true, force: true })
console.log(`Deleted storage folder: ${storagePath}`)
}
} catch (error) {
console.error(`Failed to delete storage folder ${storagePath}:`, error)
storageErrors.push(`Failed to delete storage for run ${runId}`)
}
// Also try to clear Redis state
try {
const redis = await redisClient()
await redis.del(`heartbeat:${runId}`)
await redis.del(`runners:${runId}`)
} catch (error) {
// Non-critical error, just log it
console.error(`Failed to clear Redis state for run ${runId}:`, error)
}
}
// Delete from database
await _deleteRunsByIds(runIds)
revalidatePath("/runs")
return {
success: true,
deletedCount: runIds.length,
deletedRunIds: runIds,
storageErrors,
}
}
/**
* Update the description of a run.
*/
export async function updateRunDescription(runId: number, description: string | null): Promise<{ success: boolean }> {
try {
await _updateRun(runId, { description })
revalidatePath("/runs")
revalidatePath(`/runs/${runId}`)
return { success: true }
} catch (error) {
console.error("Failed to update run description:", error)
return { success: false }
}
}

11
apps/web-evals/src/actions/tasks.ts
View file

@ -1,11 +0,0 @@
"use server"
import { revalidatePath } from "next/cache"
import { getTasks as _getTasks } from "@roo-code/evals"
export async function getTasks(runId: number) {
const tasks = await _getTasks(runId)
revalidatePath(`/runs/${runId}`)
return tasks
}

74
apps/web-evals/src/app/api/runs/[id]/logs/[taskId]/route.ts
View file

@ -1,74 +0,0 @@
import { NextResponse } from "next/server"
import type { NextRequest } from "next/server"
import * as fs from "node:fs/promises"
import * as path from "node:path"
import { findTask, findRun } from "@roo-code/evals"
export const dynamic = "force-dynamic"
const LOG_BASE_PATH = "/tmp/evals/runs"
// Sanitize path components to prevent path traversal attacks
function sanitizePathComponent(component: string): string {
// Remove any path separators, null bytes, and other dangerous characters
return component.replace(/[/\\:\0*?"<>|]/g, "_")
}
export async function GET(request: NextRequest, { params }: { params: Promise<{ id: string; taskId: string }> }) {
const { id, taskId } = await params
try {
const runId = Number(id)
const taskIdNum = Number(taskId)
if (isNaN(runId) || isNaN(taskIdNum)) {
return NextResponse.json({ error: "Invalid run ID or task ID" }, { status: 400 })
}
// Verify the run exists
await findRun(runId)
// Get the task to find its language and exercise
const task = await findTask(taskIdNum)
// Verify the task belongs to this run
if (task.runId !== runId) {
return NextResponse.json({ error: "Task does not belong to this run" }, { status: 404 })
}
// Sanitize language and exercise to prevent path traversal
const safeLanguage = sanitizePathComponent(task.language)
const safeExercise = sanitizePathComponent(task.exercise)
// Construct the log file path
const logFileName = `${safeLanguage}-${safeExercise}.log`
const logFilePath = path.join(LOG_BASE_PATH, String(runId), logFileName)
// Verify the resolved path is within the expected directory (defense in depth)
const resolvedPath = path.resolve(logFilePath)
const expectedBase = path.resolve(LOG_BASE_PATH)
if (!resolvedPath.startsWith(expectedBase)) {
return NextResponse.json({ error: "Invalid log path" }, { status: 400 })
}
// Check if the log file exists and read it (async)
try {
const logContent = await fs.readFile(logFilePath, "utf-8")
return NextResponse.json({ logContent })
} catch (err) {
if ((err as NodeJS.ErrnoException).code === "ENOENT") {
return NextResponse.json({ error: "Log file not found", logContent: null }, { status: 200 })
}
throw err
}
} catch (error) {
console.error("Error reading task log:", error)
if (error instanceof Error && error.name === "RecordNotFoundError") {
return NextResponse.json({ error: "Task or run not found" }, { status: 404 })
}
return NextResponse.json({ error: "Failed to read log file" }, { status: 500 })
}
}

147
apps/web-evals/src/app/api/runs/[id]/logs/failed/route.ts
View file

@ -1,147 +0,0 @@
import { NextResponse } from "next/server"
import type { NextRequest } from "next/server"
import * as fs from "node:fs"
import * as path from "node:path"
import archiver from "archiver"
import { findRun, getTasks } from "@roo-code/evals"
export const dynamic = "force-dynamic"
const LOG_BASE_PATH = "/tmp/evals/runs"
// Sanitize path components to prevent path traversal attacks
function sanitizePathComponent(component: string): string {
// Remove any path separators, null bytes, and other dangerous characters
return component.replace(/[/\\:\0*?"<>|]/g, "_")
}
export async function GET(request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
const { id } = await params
try {
const runId = Number(id)
if (isNaN(runId)) {
return NextResponse.json({ error: "Invalid run ID" }, { status: 400 })
}
// Verify the run exists
await findRun(runId)
// Get all tasks for this run
const tasks = await getTasks(runId)
// Filter for failed tasks only
const failedTasks = tasks.filter((task) => task.passed === false)
if (failedTasks.length === 0) {
return NextResponse.json({ error: "No failed tasks to export" }, { status: 400 })
}
// Create a zip archive
const archive = archiver("zip", { zlib: { level: 9 } })
// Collect chunks to build the response
const chunks: Buffer[] = []
archive.on("data", (chunk: Buffer) => {
chunks.push(chunk)
})
// Track archive errors
let archiveError: Error | null = null
archive.on("error", (err: Error) => {
archiveError = err
})
// Set up the end promise before finalizing (proper event listener ordering)
const archiveEndPromise = new Promise<void>((resolve, reject) => {
archive.on("end", resolve)
archive.on("error", reject)
})
// Add each failed task's log file and history files to the archive
const logDir = path.join(LOG_BASE_PATH, String(runId))
let filesAdded = 0
for (const task of failedTasks) {
// Sanitize language and exercise to prevent path traversal
const safeLanguage = sanitizePathComponent(task.language)
const safeExercise = sanitizePathComponent(task.exercise)
const expectedBase = path.resolve(LOG_BASE_PATH)
// Add the log file
const logFileName = `${safeLanguage}-${safeExercise}.log`
const logFilePath = path.join(logDir, logFileName)
// Verify the resolved path is within the expected directory (defense in depth)
const resolvedLogPath = path.resolve(logFilePath)
if (resolvedLogPath.startsWith(expectedBase) && fs.existsSync(logFilePath)) {
archive.file(logFilePath, { name: logFileName })
filesAdded++
}
// Add the API conversation history file
// Format: {language}-{exercise}.{iteration}_api_conversation_history.json
const apiHistoryFileName = `${safeLanguage}-${safeExercise}.${task.iteration}_api_conversation_history.json`
const apiHistoryFilePath = path.join(logDir, apiHistoryFileName)
const resolvedApiHistoryPath = path.resolve(apiHistoryFilePath)
if (resolvedApiHistoryPath.startsWith(expectedBase) && fs.existsSync(apiHistoryFilePath)) {
archive.file(apiHistoryFilePath, { name: apiHistoryFileName })
filesAdded++
}
// Add the UI messages file
// Format: {language}-{exercise}.{iteration}_ui_messages.json
const uiMessagesFileName = `${safeLanguage}-${safeExercise}.${task.iteration}_ui_messages.json`
const uiMessagesFilePath = path.join(logDir, uiMessagesFileName)
const resolvedUiMessagesPath = path.resolve(uiMessagesFilePath)
if (resolvedUiMessagesPath.startsWith(expectedBase) && fs.existsSync(uiMessagesFilePath)) {
archive.file(uiMessagesFilePath, { name: uiMessagesFileName })
filesAdded++
}
}
// Check if any files were actually added
if (filesAdded === 0) {
archive.abort()
return NextResponse.json(
{ error: "No log files found - they may have been cleared from disk" },
{ status: 404 },
)
}
// Finalize the archive
await archive.finalize()
// Wait for all data to be collected
await archiveEndPromise
// Check for archive errors
if (archiveError) {
throw archiveError
}
// Combine all chunks into a single buffer
const zipBuffer = Buffer.concat(chunks)
// Return the zip file
return new NextResponse(zipBuffer, {
status: 200,
headers: {
"Content-Type": "application/zip",
"Content-Disposition": `attachment; filename="run-${runId}-failed-logs.zip"`,
"Content-Length": String(zipBuffer.length),
},
})
} catch (error) {
console.error("Error exporting failed logs:", error)
if (error instanceof Error && error.name === "RecordNotFoundError") {
return NextResponse.json({ error: "Run not found" }, { status: 404 })
}
return NextResponse.json({ error: "Failed to export logs" }, { status: 500 })
}
}

71
apps/web-evals/src/app/api/runs/[id]/stream/route.ts
View file

@ -1,71 +0,0 @@
import type { NextRequest } from "next/server"
import { taskEventSchema } from "@roo-code/types"
import { findRun } from "@roo-code/evals"
import { SSEStream } from "@/lib/server/sse-stream"
import { redisClient } from "@/lib/server/redis"
export const dynamic = "force-dynamic"
export async function GET(request: NextRequest, { params }: { params: Promise<{ id: string }> }) {
const { id } = await params
const requestId = crypto.randomUUID()
const stream = new SSEStream()
const run = await findRun(Number(id))
const redis = await redisClient()
let isStreamClosed = false
const channelName = `evals:${run.id}`
const onMessage = async (data: string) => {
if (isStreamClosed || stream.isClosed) {
return
}
try {
const taskEvent = taskEventSchema.parse(JSON.parse(data))
// console.log(`[stream#${requestId}] task event -> ${taskEvent.eventName}`)
const writeSuccess = await stream.write(JSON.stringify(taskEvent))
if (!writeSuccess) {
await disconnect()
}
} catch (_error) {
console.error(`[stream#${requestId}] invalid task event:`, data)
}
}
const disconnect = async () => {
if (isStreamClosed) {
return
}
isStreamClosed = true
try {
await redis.unsubscribe(channelName)
console.log(`[stream#${requestId}] unsubscribed from ${channelName}`)
} catch (error) {
console.error(`[stream#${requestId}] error unsubscribing:`, error)
}
try {
await stream.close()
} catch (error) {
console.error(`[stream#${requestId}] error closing stream:`, error)
}
}
await redis.subscribe(channelName, onMessage)
request.signal.addEventListener("abort", () => {
console.log(`[stream#${requestId}] abort`)
disconnect().catch((error) => {
console.error(`[stream#${requestId}] cleanup error:`, error)
})
})
return stream.getResponse()
}

BIN
apps/web-evals/src/app/favicon.ico
View file

Binary file not shown.
Side by side

Before

Width:  |  Height:  |  Size: 25 KiB

141
apps/web-evals/src/app/globals.css
View file

@ -1,141 +0,0 @@
@import "tailwindcss";
@plugin "tailwindcss-animate";
@custom-variant dark (&:is(.dark *));
:root {
--radius: 0.625rem;
--background: oklch(1 0 0);
--foreground: oklch(0.145 0 0);
--card: oklch(1 0 0);
--card-foreground: oklch(0.145 0 0);
--popover: oklch(1 0 0);
--popover-foreground: oklch(0.145 0 0);
--primary: oklch(0.205 0 0);
--primary-foreground: oklch(0.985 0 0);
--secondary: oklch(0.97 0 0);
--secondary-foreground: oklch(0.205 0 0);
--muted: oklch(0.97 0 0);
--muted-foreground: oklch(0.556 0 0);
--accent: oklch(0.97 0 0);
--accent-foreground: oklch(0.205 0 0);
--destructive: oklch(0.577 0.245 27.325);
--border: oklch(0.922 0 0);
--input: oklch(0.922 0 0);
--ring: oklch(0.708 0 0);
--chart-1: oklch(0.646 0.222 41.116);
--chart-2: oklch(0.6 0.118 184.704);
--chart-3: oklch(0.398 0.07 227.392);
--chart-4: oklch(0.828 0.189 84.429);
--chart-5: oklch(0.769 0.188 70.08);
--sidebar: oklch(0.985 0 0);
--sidebar-foreground: oklch(0.145 0 0);
--sidebar-primary: oklch(0.205 0 0);
--sidebar-primary-foreground: oklch(0.985 0 0);
--sidebar-accent: oklch(0.97 0 0);
--sidebar-accent-foreground: oklch(0.205 0 0);
--sidebar-border: oklch(0.922 0 0);
--sidebar-ring: oklch(0.708 0 0);
}
.dark {
--background: oklch(23.66% 0.0198 271.79);
--foreground: oklch(75.15% 0.0477 278.41);
--card: oklch(0.205 0 0);
--card-foreground: oklch(0.985 0 0);
--popover: var(--primary);
--popover-foreground: oklch(0.985 0 0);
--primary: oklch(29.33% 0.0295 276.18);
--primary-foreground: var(--accent);
--secondary: var(--primary);
--secondary-foreground: var(--foreground);
--muted: oklch(28.27% 0.0207 273.06);
--muted-foreground: oklch(75.15% 0.0477 278.41 / 75%);
--accent: oklch(70.21% 0.1813 328.71);
--accent-foreground: oklch(1 0 0 / 75%);
--destructive: oklch(72.14% 0.1616 15.49);
--border: var(--primary);
--input: var(--primary);
--ring: oklch(83.63% 0.1259 176.52);
--chart-1: oklch(0.488 0.243 264.376);
--chart-2: oklch(0.696 0.17 162.48);
--chart-3: oklch(0.769 0.188 70.08);
--chart-4: oklch(0.627 0.265 303.9);
--chart-5: oklch(0.645 0.246 16.439);
--sidebar: oklch(0.205 0 0);
--sidebar-foreground: oklch(0.985 0 0);
--sidebar-primary: oklch(0.488 0.243 264.376);
--sidebar-primary-foreground: oklch(0.985 0 0);
--sidebar-accent: oklch(0.269 0 0);
--sidebar-accent-foreground: oklch(0.985 0 0);
--sidebar-border: oklch(1 0 0 / 10%);
--sidebar-ring: oklch(0.556 0 0);
}
@theme inline {
--color-background: var(--background);
--color-foreground: var(--foreground);
--color-sidebar-ring: var(--sidebar-ring);
--color-sidebar-border: var(--sidebar-border);
--color-sidebar-accent-foreground: var(--sidebar-accent-foreground);
--color-sidebar-accent: var(--sidebar-accent);
--color-sidebar-primary-foreground: var(--sidebar-primary-foreground);
--color-sidebar-primary: var(--sidebar-primary);
--color-sidebar-foreground: var(--sidebar-foreground);
--color-sidebar: var(--sidebar);
--color-chart-5: var(--chart-5);
--color-chart-4: var(--chart-4);
--color-chart-3: var(--chart-3);
--color-chart-2: var(--chart-2);
--color-chart-1: var(--chart-1);
--color-ring: var(--ring);
--color-input: var(--input);
--color-border: var(--border);
--color-destructive: var(--destructive);
--color-accent-foreground: var(--accent-foreground);
--color-accent: var(--accent);
--color-muted-foreground: var(--muted-foreground);
--color-muted: var(--muted);
--color-secondary-foreground: var(--secondary-foreground);
--color-secondary: var(--secondary);
--color-primary-foreground: var(--primary-foreground);
--color-primary: var(--primary);
--color-popover-foreground: var(--popover-foreground);
--color-popover: var(--popover);
--color-card-foreground: var(--card-foreground);
--color-card: var(--card);
--radius-sm: calc(var(--radius) - 4px);
--radius-md: calc(var(--radius) - 2px);
--radius-lg: var(--radius);
--radius-xl: calc(var(--radius) + 4px);
--animate-hop: hop 0.8s ease-in-out infinite;
@keyframes hop {
0%,
100% {
transform: none;
animation-timing-function: cubic-bezier(0.8, 0, 1, 1);
}
50% {
transform: translateY(-8px);
animation-timing-function: cubic-bezier(0, 0, 0.2, 1);
}
}
}
@layer base {
* {
@apply border-border outline-ring/50;
}
html,
body {
height: 100%;
}
body {
@apply bg-background text-foreground;
scrollbar-color: rgba(0, 0, 0, 0.2) transparent; /* Firefox */
scrollbar-width: thin;
}
}

35
apps/web-evals/src/app/layout.tsx
View file

@ -1,35 +0,0 @@
import type { Metadata } from "next"
import { Geist, Geist_Mono } from "next/font/google"
import { ThemeProvider, ReactQueryProvider } from "@/components/providers"
import { Toaster } from "@/components/ui"
import { Header } from "@/components/layout/header"
import "./globals.css"
const fontSans = Geist({ variable: "--font-sans", subsets: ["latin"] })
const fontMono = Geist_Mono({ variable: "--font-mono", subsets: ["latin"] })
export const metadata: Metadata = {
title: "Roo Code Evals",
}
export default function RootLayout({
children,
}: Readonly<{
children: React.ReactNode
}>) {
return (
<html lang="en">
<body className={`${fontSans.variable} ${fontMono.variable} font-sans antialiased pb-12`}>
<ThemeProvider attribute="class" forcedTheme="dark" disableTransitionOnChange>
<ReactQueryProvider>
<Header />
{children}
</ReactQueryProvider>
</ThemeProvider>
<Toaster />
</body>
</html>
)
}

10
apps/web-evals/src/app/page.tsx
View file

@ -1,10 +0,0 @@
import { getRuns } from "@roo-code/evals"
import { Runs } from "@/components/home/runs"
export const dynamic = "force-dynamic"
export default async function Page() {
const runs = await getRuns()
return <Runs runs={runs} />
}

14
apps/web-evals/src/app/runs/[id]/page.tsx
View file

@ -1,14 +0,0 @@
import { findRun } from "@roo-code/evals"
import { Run } from "./run"
export default async function Page({ params }: { params: Promise<{ id: string }> }) {
const { id } = await params
const run = await findRun(Number(id))
return (
<div className="w-full px-6 py-12">
<Run run={run} />
</div>
)
}

79
apps/web-evals/src/app/runs/[id]/run-status.tsx
View file

@ -1,79 +0,0 @@
"use client"
import { Link2, Link2Off, CheckCircle2 } from "lucide-react"
import type { RunStatus as _RunStatus } from "@/hooks/use-run-status"
import { cn } from "@/lib/utils"
import { Tooltip, TooltipContent, TooltipTrigger } from "@/components/ui"
function StreamIcon({ status }: { status: "connected" | "waiting" | "error" }) {
if (status === "connected") {
return <Link2 className="size-4 text-green-500" />
}
return <Link2Off className={cn("size-4", status === "waiting" ? "text-amber-500" : "text-rose-500")} />
}
export const RunStatus = ({
runStatus: { sseStatus, heartbeat, runners = [] },
isComplete = false,
}: {
runStatus: _RunStatus
isComplete?: boolean
}) => {
// For completed runs, show a simple "Complete" badge
if (isComplete) {
return (
<Tooltip>
<TooltipTrigger asChild>
<div className="flex items-center gap-1 cursor-default text-muted-foreground">
<CheckCircle2 className="size-4" />
</div>
</TooltipTrigger>
<TooltipContent side="bottom" className="font-mono text-xs">
Run complete
</TooltipContent>
</Tooltip>
)
}
return (
<Tooltip>
<TooltipTrigger asChild>
<div className="flex items-center gap-2 cursor-default text-xs font-mono">
{/* Task Stream status icon */}
<StreamIcon status={sseStatus} />
{/* Task Controller ID */}
<span className={heartbeat ? "text-green-500" : "text-rose-500"}>{heartbeat ?? "-"}</span>
{/* Task Runners count */}
<span className={runners.length > 0 ? "text-green-500" : "text-rose-500"}>
{runners.length > 0 ? `${runners.length}r` : "0r"}
</span>
</div>
</TooltipTrigger>
<TooltipContent side="bottom" className="font-mono text-xs max-w-md">
<div className="space-y-1">
<div className="flex items-center gap-2">
<StreamIcon status={sseStatus} />
<span>Task Stream: {sseStatus}</span>
</div>
<div className="flex items-center gap-2">
<span className={heartbeat ? "text-green-500" : "text-rose-500"}>●</span>
<span>Task Controller: {heartbeat ?? "dead"}</span>
</div>
<div className="flex items-center gap-2">
<span className={runners.length > 0 ? "text-green-500" : "text-rose-500"}>●</span>
<span>Task Runners: {runners.length > 0 ? runners.length : "none"}</span>
</div>
{runners.length > 0 && (
<div className="mt-2 pt-2 border-t border-border text-muted-foreground space-y-0.5">
{runners.map((runner) => (
<div key={runner}>{runner}</div>
))}
</div>
)}
</div>
</TooltipContent>
</Tooltip>
)
}

1058
apps/web-evals/src/app/runs/[id]/run.tsx
View file

File diff suppressed because it is too large Load diff

20
apps/web-evals/src/app/runs/[id]/task-status.tsx
View file

@ -1,20 +0,0 @@
import { CircleCheck, CircleDashed, CircleSlash, LoaderCircle } from "lucide-react"
import type { Task } from "@roo-code/evals"
type TaskStatusProps = {
task: Task
running: boolean
}
export const TaskStatus = ({ task, running }: TaskStatusProps) => {
return task.passed === false ? (
<CircleSlash className="size-4 text-destructive" />
) : task.passed === true ? (
<CircleCheck className="size-4 text-green-500" />
) : running ? (
<LoaderCircle className="size-4 animate-spin" />
) : (
<CircleDashed className="size-4" />
)
}

1083
apps/web-evals/src/app/runs/new/new-run.tsx
View file

File diff suppressed because it is too large Load diff

9
apps/web-evals/src/app/runs/new/page.tsx
View file

@ -1,9 +0,0 @@
import { NewRun } from "./new-run"
export default function Page() {
return (
<div className="max-w-3xl mx-auto px-12 p-12">
<NewRun />
</div>
)
}

58
apps/web-evals/src/app/runs/new/settings-diff.tsx
View file

@ -1,58 +0,0 @@
import { type Keys, type RooCodeSettings, GLOBAL_SETTINGS_KEYS, PROVIDER_SETTINGS_KEYS } from "@roo-code/types"
import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui"
export const ROO_CODE_SETTINGS_KEYS = [
...new Set([...GLOBAL_SETTINGS_KEYS, ...PROVIDER_SETTINGS_KEYS]),
] as Keys<RooCodeSettings>[]
type SettingsDiffProps = {
defaultSettings: RooCodeSettings
customSettings: RooCodeSettings
}
export function SettingsDiff({
customSettings: { experiments: customExperiments, ...customSettings },
defaultSettings: { experiments: defaultExperiments, ...defaultSettings },
}: SettingsDiffProps) {
const defaults = { ...defaultSettings, ...defaultExperiments }
const custom = { ...customSettings, ...customExperiments }
return (
<div className="border rounded-sm">
<Table>
<TableHeader>
<TableRow className="font-medium text-muted-foreground">
<TableHead>Setting</TableHead>
<TableHead>Default</TableHead>
<TableHead>Custom</TableHead>
</TableRow>
</TableHeader>
<TableBody>
{ROO_CODE_SETTINGS_KEYS.map((key) => {
const defaultValue = JSON.stringify(defaults[key as keyof typeof defaults], null, 2)
const customValue = JSON.stringify(custom[key as keyof typeof custom], null, 2)
return defaultValue === customValue ||
(isEmpty(defaultValue) && isEmpty(customValue)) ? null : (
<TableRow key={key}>
<TableCell className="font-mono" title={key}>
{key}
</TableCell>
<TableCell className="font-mono text-rose-500 line-through" title={defaultValue}>
{defaultValue}
</TableCell>
<TableCell className="font-mono text-teal-500" title={customValue}>
{customValue}
</TableCell>
</TableRow>
)
})}
</TableBody>
</Table>
</div>
)
}
const isEmpty = (value: string | undefined) =>
value === undefined || value === "" || value === "null" || value === '""' || value === "[]" || value === "{}"

433
apps/web-evals/src/components/home/run.tsx
View file

@ -1,433 +0,0 @@
import { useCallback, useState, useRef } from "react"
import Link from "next/link"
import { useRouter } from "next/navigation"
import { toast } from "sonner"
import { Ellipsis, ClipboardList, Copy, Check, LoaderCircle, Trash, Settings, FileDown, StickyNote } from "lucide-react"
import type { Run as EvalsRun, TaskMetrics as EvalsTaskMetrics } from "@roo-code/evals"
import type { ToolName } from "@roo-code/types"
import { deleteRun, updateRunDescription } from "@/actions/runs"
import {
formatCurrency,
formatDateTime,
formatDuration,
formatTokens,
formatToolUsageSuccessRate,
} from "@/lib/formatters"
import { useCopyRun } from "@/hooks/use-copy-run"
import {
Button,
TableCell,
TableRow,
Textarea,
Tooltip,
TooltipContent,
TooltipTrigger,
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
Dialog,
DialogContent,
DialogFooter,
DialogHeader,
DialogTitle,
ScrollArea,
} from "@/components/ui"
// Tool group type (same as in runs.tsx)
type ToolGroup = {
id: string
name: string
icon: string
tools: string[]
}
type RunProps = {
run: EvalsRun
taskMetrics: EvalsTaskMetrics | null
toolColumns: ToolName[]
toolGroups: ToolGroup[]
}
export function Run({ run, taskMetrics, toolColumns, toolGroups }: RunProps) {
const router = useRouter()
const [deleteRunId, setDeleteRunId] = useState<number>()
const [showSettings, setShowSettings] = useState(false)
const [isExportingLogs, setIsExportingLogs] = useState(false)
const [showNotesDialog, setShowNotesDialog] = useState(false)
const [editingDescription, setEditingDescription] = useState(run.description ?? "")
const [isSavingNotes, setIsSavingNotes] = useState(false)
const continueRef = useRef<HTMLButtonElement>(null)
const { isPending, copyRun, copied } = useCopyRun(run.id)
const hasDescription = Boolean(run.description && run.description.trim().length > 0)
const handleSaveDescription = useCallback(async () => {
setIsSavingNotes(true)
try {
const result = await updateRunDescription(run.id, editingDescription.trim() || null)
if (result.success) {
toast.success("Description saved")
setShowNotesDialog(false)
router.refresh()
} else {
toast.error("Failed to save description")
}
} catch (error) {
console.error("Error saving description:", error)
toast.error("Failed to save description")
} finally {
setIsSavingNotes(false)
}
}, [run.id, editingDescription, router])
const onExportFailedLogs = useCallback(async () => {
if (run.failed === 0) {
toast.error("No failed tasks to export")
return
}
setIsExportingLogs(true)
try {
const response = await fetch(`/api/runs/${run.id}/logs/failed`)
if (!response.ok) {
const error = await response.json()
toast.error(error.error || "Failed to export logs")
return
}
// Download the zip file
const blob = await response.blob()
const url = window.URL.createObjectURL(blob)
const a = document.createElement("a")
a.href = url
a.download = `run-${run.id}-failed-logs.zip`
document.body.appendChild(a)
a.click()
window.URL.revokeObjectURL(url)
document.body.removeChild(a)
toast.success("Failed logs exported successfully")
} catch (error) {
console.error("Error exporting logs:", error)
toast.error("Failed to export logs")
} finally {
setIsExportingLogs(false)
}
}, [run.id, run.failed])
const onConfirmDelete = useCallback(async () => {
if (!deleteRunId) {
return
}
try {
await deleteRun(deleteRunId)
setDeleteRunId(undefined)
} catch (error) {
console.error(error)
}
}, [deleteRunId])
const handleRowClick = useCallback(
(e: React.MouseEvent) => {
// Don't navigate if clicking on the dropdown menu
if ((e.target as HTMLElement).closest("[data-dropdown-trigger]")) {
return
}
router.push(`/runs/${run.id}`)
},
[router, run.id],
)
// Helper to render a tool group cell
const renderToolGroupCell = (group: ToolGroup) => {
if (!taskMetrics?.toolUsage) {
return <span className="text-muted-foreground">-</span>
}
let totalAttempts = 0
let totalFailures = 0
const breakdown: Array<{ tool: string; attempts: number; rate: string }> = []
for (const toolName of group.tools) {
const usage = taskMetrics.toolUsage[toolName as ToolName]
if (usage) {
totalAttempts += usage.attempts
totalFailures += usage.failures
const rate =
usage.attempts > 0
? `${Math.round(((usage.attempts - usage.failures) / usage.attempts) * 100)}%`
: "0%"
breakdown.push({ tool: toolName, attempts: usage.attempts, rate })
}
}
if (totalAttempts === 0) {
return <span className="text-muted-foreground">-</span>
}
const successRate = ((totalAttempts - totalFailures) / totalAttempts) * 100
const rateColor =
successRate === 100 ? "text-muted-foreground" : successRate >= 80 ? "text-yellow-500" : "text-red-500"
return (
<Tooltip>
<TooltipTrigger>
<div className="flex flex-col items-center">
<span className="font-medium">{totalAttempts}</span>
<span className={rateColor}>{Math.round(successRate)}%</span>
</div>
</TooltipTrigger>
<TooltipContent>
<div className="text-xs">
<div className="font-semibold mb-1">{group.name}</div>
{breakdown.map(({ tool, attempts, rate }) => (
<div key={tool} className="flex justify-between gap-4">
<span>{tool}:</span>
<span>
{attempts} ({rate})
</span>
</div>
))}
</div>
</TooltipContent>
</Tooltip>
)
}
return (
<>
<TableRow className="cursor-pointer hover:bg-muted/50" onClick={handleRowClick}>
<TableCell className="max-w-[200px] truncate">{run.model}</TableCell>
<TableCell>{run.settings?.apiProvider ?? "-"}</TableCell>
<TableCell className="text-sm text-muted-foreground whitespace-nowrap">
{formatDateTime(run.createdAt)}
</TableCell>
<TableCell>{run.passed}</TableCell>
<TableCell>{run.failed}</TableCell>
<TableCell>
{run.passed + run.failed > 0 &&
(() => {
const percent = (run.passed / (run.passed + run.failed)) * 100
const colorClass =
percent === 100 ? "text-green-500" : percent >= 80 ? "text-yellow-500" : "text-red-500"
return <span className={colorClass}>{percent.toFixed(1)}%</span>
})()}
</TableCell>
<TableCell>
{taskMetrics && (
<div className="flex items-center gap-1">
<span>{formatTokens(taskMetrics.tokensIn)}</span>/
<span>{formatTokens(taskMetrics.tokensOut)}</span>
</div>
)}
</TableCell>
{/* Tool Group Columns */}
{toolGroups.map((group) => (
<TableCell key={group.id} className="text-xs text-center">
{renderToolGroupCell(group)}
</TableCell>
))}
{toolColumns.map((toolName) => {
const usage = taskMetrics?.toolUsage?.[toolName]
const successRate =
usage && usage.attempts > 0 ? ((usage.attempts - usage.failures) / usage.attempts) * 100 : 100
const rateColor =
successRate === 100
? "text-muted-foreground"
: successRate >= 80
? "text-yellow-500"
: "text-red-500"
return (
<TableCell key={toolName} className="text-xs text-center">
{usage ? (
<div className="flex flex-col items-center">
<span className="font-medium">{usage.attempts}</span>
<span className={rateColor}>{formatToolUsageSuccessRate(usage)}</span>
</div>
) : (
<span className="text-muted-foreground">-</span>
)}
</TableCell>
)
})}
<TableCell>{taskMetrics && formatCurrency(taskMetrics.cost)}</TableCell>
<TableCell>{taskMetrics && formatDuration(taskMetrics.duration)}</TableCell>
<TableCell onClick={(e) => e.stopPropagation()}>
<div className="flex items-center gap-1">
{/* Note Icon */}
<Tooltip>
<TooltipTrigger asChild>
<Button
variant="ghost"
size="icon"
className={hasDescription ? "" : "opacity-30 hover:opacity-60"}
onClick={(e) => {
e.stopPropagation()
setEditingDescription(run.description ?? "")
setShowNotesDialog(true)
}}>
<StickyNote className="h-4 w-4" />
</Button>
</TooltipTrigger>
<TooltipContent className="max-w-[300px]">
{hasDescription ? (
<div className="whitespace-pre-wrap">{run.description}</div>
) : (
<div className="text-muted-foreground">No description. Click to add one.</div>
)}
</TooltipContent>
</Tooltip>
{/* More Actions Menu */}
<DropdownMenu>
<Button variant="ghost" size="icon" asChild>
<DropdownMenuTrigger data-dropdown-trigger>
<Ellipsis />
</DropdownMenuTrigger>
</Button>
<DropdownMenuContent align="end">
<DropdownMenuItem asChild>
<Link href={`/runs/${run.id}`}>
<div className="flex items-center gap-1">
<ClipboardList />
<div>View Tasks</div>
</div>
</Link>
</DropdownMenuItem>
{run.settings && (
<DropdownMenuItem onClick={() => setShowSettings(true)}>
<div className="flex items-center gap-1">
<Settings />
<div>View Settings</div>
</div>
</DropdownMenuItem>
)}
{run.taskMetricsId && (
<DropdownMenuItem onClick={() => copyRun()} disabled={isPending || copied}>
<div className="flex items-center gap-1">
{isPending ? (
<>
<LoaderCircle className="animate-spin" />
Copying...
</>
) : copied ? (
<>
<Check />
Copied!
</>
) : (
<>
<Copy />
Copy to Production
</>
)}
</div>
</DropdownMenuItem>
)}
{run.failed > 0 && (
<DropdownMenuItem onClick={onExportFailedLogs} disabled={isExportingLogs}>
<div className="flex items-center gap-1">
{isExportingLogs ? (
<>
<LoaderCircle className="animate-spin" />
Exporting...
</>
) : (
<>
<FileDown />
Export Failed Logs
</>
)}
</div>
</DropdownMenuItem>
)}
<DropdownMenuItem
onClick={() => {
setDeleteRunId(run.id)
setTimeout(() => continueRef.current?.focus(), 0)
}}>
<div className="flex items-center gap-1">
<Trash />
<div>Delete</div>
</div>
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
</TableCell>
</TableRow>
<AlertDialog open={!!deleteRunId} onOpenChange={() => setDeleteRunId(undefined)}>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>Are you sure?</AlertDialogTitle>
<AlertDialogDescription>This action cannot be undone.</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel>Cancel</AlertDialogCancel>
<AlertDialogAction ref={continueRef} onClick={onConfirmDelete}>
Continue
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
<Dialog open={showSettings} onOpenChange={setShowSettings}>
<DialogContent className="max-w-2xl max-h-[80vh]">
<DialogHeader>
<DialogTitle>Run Settings</DialogTitle>
</DialogHeader>
<ScrollArea className="max-h-[60vh]">
<pre className="text-xs font-mono bg-muted p-4 rounded-md overflow-auto">
{JSON.stringify(run.settings, null, 2)}
</pre>
</ScrollArea>
</DialogContent>
</Dialog>
{/* Notes/Description Dialog */}
<Dialog open={showNotesDialog} onOpenChange={setShowNotesDialog}>
<DialogContent className="max-w-lg">
<DialogHeader>
<DialogTitle>Run Description</DialogTitle>
</DialogHeader>
<div className="space-y-4">
<Textarea
placeholder="Add a description or notes for this run..."
value={editingDescription}
onChange={(e) => setEditingDescription(e.target.value)}
rows={4}
className="resize-none"
/>
</div>
<DialogFooter>
<Button variant="outline" onClick={() => setShowNotesDialog(false)}>
Cancel
</Button>
<Button onClick={handleSaveDescription} disabled={isSavingNotes}>
{isSavingNotes ? (
<>
<LoaderCircle className="h-4 w-4 mr-2 animate-spin" />
Saving...
</>
) : (
"Save"
)}
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
</>
)
}

1024
apps/web-evals/src/components/home/runs.tsx
View file

File diff suppressed because it is too large Load diff

7
apps/web-evals/src/components/layout/header.tsx
View file

@ -1,7 +0,0 @@
import { HoppingLogo } from "./logo"
export const Header = () => (
<div className="flex items-center justify-between border-b px-12 py-6">
<HoppingLogo />
</div>
)

54
apps/web-evals/src/components/layout/logo.tsx
View file

@ -1,54 +0,0 @@
"use client"
import { SVGProps, useEffect, useRef } from "react"
import { useRouter } from "next/navigation"
import { useHover } from "react-use"
import { cn } from "@/lib/utils"
type LogoProps = Omit<SVGProps<SVGSVGElement>, "xmlns" | "viewBox" | "onClick">
export const Logo = ({ width = 50, height = 32, fill = "#fff", className, ...props }: LogoProps) => {
const router = useRouter()
return (
<svg
xmlns="http://www.w3.org/2000/svg"
width={width}
height={height}
viewBox="90 12 100 64"
onClick={() => router.push("/")}
className={cn("logo cursor-pointer", className)}
{...props}>
<path
d="M171.633,15.8336l-1.7284,6.2499c-.0915.3309-.4369.5221-.7659.4239l-28.9937-8.6507c-.1928-.0575-.4016-.0167-.5586.1092l-28.7143,23.0269c-.0838.0672-.1839.1112-.2901.1276l-17.0849,2.6329c-.3163.0488-.5419.3327-.5178.6519l.0742.9817c.0237.3136.2809.5583.5953.5664l19.8448.513.2263.0063,14.6634-7.8328c.2053-.1097.455-.0936.6445.0415l10.3884,7.4053c.1629.1161.2589.3045.2571.5045l-.0876,9.826c-.0011.1272.0373.2515.11.3559l14.6133,20.9682c.1146.1644.3024.2624.5028.2624h4.626c.4615,0,.7574-.4908.542-.8989l-10.4155-19.7312c-.1019-.193-.0934-.4255.0221-.6106l5.4305-8.6994c.0591-.0947.143-.1715.2425-.222l19.415-9.8522c.1973-.1001.4332-.0861.6172.0366l5.5481,3.6981c.1007.0671.2189.1029.3399.1029h5.0407c.4881,0,.7804-.5429.5116-.9503l-13.9967-21.2171c-.2898-.4393-.962-.3331-1.1022.1741Z"
fill={fill}
strokeWidth="0"
/>
</svg>
)
}
export const HoppingLogo = (props: LogoProps) => {
const ref = useRef<SVGSVGElement>(null)
const logo = <Logo ref={ref} {...props} />
const [hoverable, hovered] = useHover(logo)
useEffect(() => {
const element = ref.current
const isHopping = element !== null && element.classList.contains("animate-hop")
if (hovered && element && !isHopping) {
element.classList.add("animate-hop")
} else if (element && isHopping) {
const onAnimationEnd = () => {
element.classList.remove("animate-hop")
element.removeEventListener("animationiteration", onAnimationEnd)
}
element.addEventListener("animationiteration", onAnimationEnd)
}
}, [hovered])
return hoverable
}

2
apps/web-evals/src/components/providers/index.ts
View file

@ -1,2 +0,0 @@
export { ReactQueryProvider } from "./react-query-provider"
export { ThemeProvider } from "./theme-provider"

8
apps/web-evals/src/components/providers/react-query-provider.tsx
View file

@ -1,8 +0,0 @@
"use client"
import { QueryClient, QueryClientProvider } from "@tanstack/react-query"
export function ReactQueryProvider({ children }: { children: React.ReactNode }) {
const queryClient = new QueryClient()
return <QueryClientProvider client={queryClient}>{children}</QueryClientProvider>
}

13
apps/web-evals/src/components/providers/theme-provider.tsx
View file

@ -1,13 +0,0 @@
"use client"
import * as React from "react"
import { type ThemeProviderProps } from "next-themes"
import dynamic from "next/dynamic"
const NextThemesProvider = dynamic(() => import("next-themes").then((e) => e.ThemeProvider), {
ssr: false,
})
export function ThemeProvider({ children, ...props }: ThemeProviderProps) {
return <NextThemesProvider {...props}>{children}</NextThemesProvider>
}

113
apps/web-evals/src/components/ui/alert-dialog.tsx
View file

@ -1,113 +0,0 @@
"use client"
import * as React from "react"
import * as AlertDialogPrimitive from "@radix-ui/react-alert-dialog"
import { cn } from "@/lib/utils"
import { buttonVariants } from "@/components/ui/button"
function AlertDialog({ ...props }: React.ComponentProps<typeof AlertDialogPrimitive.Root>) {
return <AlertDialogPrimitive.Root data-slot="alert-dialog" {...props} />
}
function AlertDialogTrigger({ ...props }: React.ComponentProps<typeof AlertDialogPrimitive.Trigger>) {
return <AlertDialogPrimitive.Trigger data-slot="alert-dialog-trigger" {...props} />
}
function AlertDialogPortal({ ...props }: React.ComponentProps<typeof AlertDialogPrimitive.Portal>) {
return <AlertDialogPrimitive.Portal data-slot="alert-dialog-portal" {...props} />
}
function AlertDialogOverlay({ className, ...props }: React.ComponentProps<typeof AlertDialogPrimitive.Overlay>) {
return (
<AlertDialogPrimitive.Overlay
data-slot="alert-dialog-overlay"
className={cn(
"data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 fixed inset-0 z-50 bg-black/50",
className,
)}
{...props}
/>
)
}
function AlertDialogContent({ className, ...props }: React.ComponentProps<typeof AlertDialogPrimitive.Content>) {
return (
<AlertDialogPortal>
<AlertDialogOverlay />
<AlertDialogPrimitive.Content
data-slot="alert-dialog-content"
className={cn(
"bg-background data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 fixed top-[50%] left-[50%] z-50 grid w-full max-w-[calc(100%-2rem)] translate-x-[-50%] translate-y-[-50%] gap-4 rounded-lg border p-6 shadow-lg duration-200 sm:max-w-lg",
className,
)}
{...props}
/>
</AlertDialogPortal>
)
}
function AlertDialogHeader({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="alert-dialog-header"
className={cn("flex flex-col gap-2 text-center sm:text-left", className)}
{...props}
/>
)
}
function AlertDialogFooter({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="alert-dialog-footer"
className={cn("flex flex-col-reverse gap-2 sm:flex-row sm:justify-end", className)}
{...props}
/>
)
}
function AlertDialogTitle({ className, ...props }: React.ComponentProps<typeof AlertDialogPrimitive.Title>) {
return (
<AlertDialogPrimitive.Title
data-slot="alert-dialog-title"
className={cn("text-lg font-semibold", className)}
{...props}
/>
)
}
function AlertDialogDescription({
className,
...props
}: React.ComponentProps<typeof AlertDialogPrimitive.Description>) {
return (
<AlertDialogPrimitive.Description
data-slot="alert-dialog-description"
className={cn("text-muted-foreground text-sm", className)}
{...props}
/>
)
}
function AlertDialogAction({ className, ...props }: React.ComponentProps<typeof AlertDialogPrimitive.Action>) {
return <AlertDialogPrimitive.Action className={cn(buttonVariants(), className)} {...props} />
}
function AlertDialogCancel({ className, ...props }: React.ComponentProps<typeof AlertDialogPrimitive.Cancel>) {
return <AlertDialogPrimitive.Cancel className={cn(buttonVariants({ variant: "outline" }), className)} {...props} />
}
export {
AlertDialog,
AlertDialogPortal,
AlertDialogOverlay,
AlertDialogTrigger,
AlertDialogContent,
AlertDialogHeader,
AlertDialogFooter,
AlertDialogTitle,
AlertDialogDescription,
AlertDialogAction,
AlertDialogCancel,
}

36
apps/web-evals/src/components/ui/badge.tsx
View file

@ -1,36 +0,0 @@
import * as React from "react"
import { Slot } from "@radix-ui/react-slot"
import { cva, type VariantProps } from "class-variance-authority"
import { cn } from "@/lib/utils"
const badgeVariants = cva(
"inline-flex items-center justify-center rounded-sm border px-2 py-0.5 text-xs font-medium w-fit whitespace-nowrap shrink-0 [&>svg]:size-3 gap-1 [&>svg]:pointer-events-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive transition-[color,box-shadow] overflow-hidden",
{
variants: {
variant: {
default: "border-transparent bg-primary text-primary-foreground [a&]:hover:bg-primary/90",
secondary: "border-transparent bg-secondary text-secondary-foreground [a&]:hover:bg-secondary/90",
destructive:
"border-transparent bg-destructive text-white [a&]:hover:bg-destructive/90 focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/70",
outline: "text-foreground [a&]:hover:bg-accent [a&]:hover:text-accent-foreground",
},
},
defaultVariants: {
variant: "default",
},
},
)
function Badge({
className,
variant,
asChild = false,
...props
}: React.ComponentProps<"span"> & VariantProps<typeof badgeVariants> & { asChild?: boolean }) {
const Comp = asChild ? Slot : "span"
return <Comp data-slot="badge" className={cn(badgeVariants({ variant }), className)} {...props} />
}
export { Badge, badgeVariants }

51
apps/web-evals/src/components/ui/button.tsx
View file

@ -1,51 +0,0 @@
import * as React from "react"
import { Slot } from "@radix-ui/react-slot"
import { cva, type VariantProps } from "class-variance-authority"
import { cn } from "@/lib/utils"
const buttonVariants = cva(
"inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-sm font-medium transition-all disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg:not([class*='size-'])]:size-4 shrink-0 [&_svg]:shrink-0 outline-none focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px] aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive hover:opacity-80 active:scale-95 cursor-pointer",
{
variants: {
variant: {
default: "bg-primary text-primary-foreground shadow-xs [&_svg]:text-accent",
destructive:
"bg-destructive text-white shadow-xs focus-visible:ring-destructive/20 dark:focus-visible:ring-destructive/40 dark:bg-destructive/60",
outline:
"border bg-background shadow-xs hover:bg-accent hover:text-accent-foreground dark:bg-input/30 dark:border-input",
secondary: "bg-secondary text-secondary-foreground shadow-xs [&_svg]:text-ring",
ghost: "hover:bg-primary hover:text-primary-foreground",
link: "text-accent underline-offset-4 hover:underline h-4! px-1! rounded-none",
input: "bg-input text-input-foreground active:scale-100 shadow-xs",
},
size: {
default: "h-9 px-4 py-2 has-[>svg]:px-3",
sm: "h-8 gap-1.5 px-3 has-[>svg]:px-2.5 text-sm",
lg: "h-10 px-6 has-[>svg]:px-4 text-lg",
icon: "size-9",
},
},
defaultVariants: {
variant: "default",
size: "default",
},
},
)
function Button({
className,
variant,
size,
asChild = false,
...props
}: React.ComponentProps<"button"> &
VariantProps<typeof buttonVariants> & {
asChild?: boolean
}) {
const Comp = asChild ? Slot : "button"
return <Comp data-slot="button" className={cn(buttonVariants({ variant, size, className }))} {...props} />
}
export { Button, buttonVariants }

27
apps/web-evals/src/components/ui/checkbox.tsx
View file

@ -1,27 +0,0 @@
"use client"
import * as React from "react"
import * as CheckboxPrimitive from "@radix-ui/react-checkbox"
import { CheckIcon } from "lucide-react"
import { cn } from "@/lib/utils"
function Checkbox({ className, ...props }: React.ComponentProps<typeof CheckboxPrimitive.Root>) {
return (
<CheckboxPrimitive.Root
data-slot="checkbox"
className={cn(
"peer border-input dark:bg-input/30 data-[state=checked]:bg-primary data-[state=checked]:text-primary-foreground dark:data-[state=checked]:bg-primary data-[state=checked]:border-primary focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive size-4 shrink-0 rounded-[4px] border shadow-xs transition-shadow outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50",
className,
)}
{...props}>
<CheckboxPrimitive.Indicator
data-slot="checkbox-indicator"
className="grid place-content-center text-current transition-none">
<CheckIcon className="size-3.5" />
</CheckboxPrimitive.Indicator>
</CheckboxPrimitive.Root>
)
}
export { Checkbox }

134
apps/web-evals/src/components/ui/command.tsx
View file

@ -1,134 +0,0 @@
"use client"
import * as React from "react"
import { Command as CommandPrimitive } from "cmdk"
import { SearchIcon } from "lucide-react"
import { cn } from "@/lib/utils"
import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle } from "@/components/ui/dialog"
function Command({ className, ...props }: React.ComponentProps<typeof CommandPrimitive>) {
return (
<CommandPrimitive
data-slot="command"
className={cn(
"bg-popover text-popover-foreground flex h-full w-full flex-col overflow-hidden rounded-sm",
className,
)}
{...props}
/>
)
}
function CommandDialog({
title = "Command Palette",
description = "Search for a command to run...",
children,
...props
}: React.ComponentProps<typeof Dialog> & {
title?: string
description?: string
}) {
return (
<Dialog {...props}>
<DialogHeader className="sr-only">
<DialogTitle>{title}</DialogTitle>
<DialogDescription>{description}</DialogDescription>
</DialogHeader>
<DialogContent className="overflow-hidden p-0">
<Command className="[&_[cmdk-group-heading]]:text-muted-foreground **:data-[slot=command-input-wrapper]:h-12 [&_[cmdk-group-heading]]:px-2 [&_[cmdk-group-heading]]:font-medium [&_[cmdk-group]]:px-2 [&_[cmdk-group]:not([hidden])_~[cmdk-group]]:pt-0 [&_[cmdk-input-wrapper]_svg]:h-5 [&_[cmdk-input-wrapper]_svg]:w-5 [&_[cmdk-input]]:h-12 [&_[cmdk-item]]:px-2 [&_[cmdk-item]]:py-3 [&_[cmdk-item]_svg]:h-5 [&_[cmdk-item]_svg]:w-5">
{children}
</Command>
</DialogContent>
</Dialog>
)
}
function CommandInput({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Input>) {
return (
<div data-slot="command-input-wrapper" className="flex h-9 items-center gap-2 border-b px-3">
<SearchIcon className="size-4 shrink-0 opacity-50" />
<CommandPrimitive.Input
data-slot="command-input"
className={cn(
"placeholder:text-muted-foreground flex h-10 w-full rounded-sm bg-transparent py-3 outline-hidden disabled:cursor-not-allowed disabled:opacity-50",
className,
)}
{...props}
/>
</div>
)
}
function CommandList({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.List>) {
return (
<CommandPrimitive.List
data-slot="command-list"
className={cn("max-h-[300px] scroll-py-1 overflow-x-hidden overflow-y-auto", className)}
{...props}
/>
)
}
function CommandEmpty({ ...props }: React.ComponentProps<typeof CommandPrimitive.Empty>) {
return <CommandPrimitive.Empty data-slot="command-empty" className="py-6 text-center" {...props} />
}
function CommandGroup({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Group>) {
return (
<CommandPrimitive.Group
data-slot="command-group"
className={cn(
"text-foreground [&_[cmdk-group-heading]]:text-muted-foreground overflow-hidden p-1 [&_[cmdk-group-heading]]:px-2 [&_[cmdk-group-heading]]:py-1.5 [&_[cmdk-group-heading]]:text-xs [&_[cmdk-group-heading]]:font-medium",
className,
)}
{...props}
/>
)
}
function CommandSeparator({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Separator>) {
return (
<CommandPrimitive.Separator
data-slot="command-separator"
className={cn("bg-accent/5 -mx-1 h-px", className)}
{...props}
/>
)
}
function CommandItem({ className, ...props }: React.ComponentProps<typeof CommandPrimitive.Item>) {
return (
<CommandPrimitive.Item
data-slot="command-item"
className={cn(
"data-[selected=true]:bg-accent data-[selected=true]:text-accent-foreground [&_svg:not([class*='text-'])]:text-muted-foreground relative flex cursor-default items-center gap-2 rounded-xs px-2 py-1.5 outline-hidden select-none data-[disabled=true]:pointer-events-none data-[disabled=true]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
"text-foreground active:opacity-80 cursor-pointer group",
className,
)}
{...props}
/>
)
}
function CommandShortcut({ className, ...props }: React.ComponentProps<"span">) {
return (
<span
data-slot="command-shortcut"
className={cn("text-muted-foreground ml-auto text-xs tracking-widest", className)}
{...props}
/>
)
}
export {
Command,
CommandDialog,
CommandInput,
CommandList,
CommandEmpty,
CommandGroup,
CommandItem,
CommandShortcut,
CommandSeparator,
}

110
apps/web-evals/src/components/ui/dialog.tsx
View file

@ -1,110 +0,0 @@
"use client"
import * as React from "react"
import * as DialogPrimitive from "@radix-ui/react-dialog"
import { XIcon } from "lucide-react"
import { cn } from "@/lib/utils"
function Dialog({ ...props }: React.ComponentProps<typeof DialogPrimitive.Root>) {
return <DialogPrimitive.Root data-slot="dialog" {...props} />
}
function DialogTrigger({ ...props }: React.ComponentProps<typeof DialogPrimitive.Trigger>) {
return <DialogPrimitive.Trigger data-slot="dialog-trigger" {...props} />
}
function DialogPortal({ ...props }: React.ComponentProps<typeof DialogPrimitive.Portal>) {
return <DialogPrimitive.Portal data-slot="dialog-portal" {...props} />
}
function DialogClose({ ...props }: React.ComponentProps<typeof DialogPrimitive.Close>) {
return <DialogPrimitive.Close data-slot="dialog-close" {...props} />
}
function DialogOverlay({ className, ...props }: React.ComponentProps<typeof DialogPrimitive.Overlay>) {
return (
<DialogPrimitive.Overlay
data-slot="dialog-overlay"
className={cn(
"data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 fixed inset-0 z-50 bg-black/50",
className,
)}
{...props}
/>
)
}
function DialogContent({ className, children, ...props }: React.ComponentProps<typeof DialogPrimitive.Content>) {
return (
<DialogPortal data-slot="dialog-portal">
<DialogOverlay />
<DialogPrimitive.Content
data-slot="dialog-content"
className={cn(
"bg-background data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 fixed top-[50%] left-[50%] z-50 grid w-full max-w-[calc(100%-2rem)] translate-x-[-50%] translate-y-[-50%] gap-4 rounded-lg border p-6 shadow-lg duration-200 sm:max-w-lg",
className,
)}
{...props}>
{children}
<DialogPrimitive.Close className="ring-offset-background focus:ring-ring data-[state=open]:bg-accent data-[state=open]:text-muted-foreground absolute top-4 right-4 rounded-xs opacity-70 transition-opacity hover:opacity-100 focus:ring-2 focus:ring-offset-2 focus:outline-hidden disabled:pointer-events-none [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4">
<XIcon />
<span className="sr-only">Close</span>
</DialogPrimitive.Close>
</DialogPrimitive.Content>
</DialogPortal>
)
}
function DialogHeader({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="dialog-header"
className={cn("flex flex-col gap-2 text-center sm:text-left", className)}
{...props}
/>
)
}
function DialogFooter({ className, ...props }: React.ComponentProps<"div">) {
return (
<div
data-slot="dialog-footer"
className={cn("flex flex-col-reverse gap-2 sm:flex-row sm:justify-end", className)}
{...props}
/>
)
}
function DialogTitle({ className, ...props }: React.ComponentProps<typeof DialogPrimitive.Title>) {
return (
<DialogPrimitive.Title
data-slot="dialog-title"
className={cn("text-lg leading-none font-semibold", className)}
{...props}
/>
)
}
function DialogDescription({ className, ...props }: React.ComponentProps<typeof DialogPrimitive.Description>) {
return (
<DialogPrimitive.Description
data-slot="dialog-description"
className={cn("text-muted-foreground text-sm", className)}
{...props}
/>
)
}
export {
Dialog,
DialogClose,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogOverlay,
DialogPortal,
DialogTitle,
DialogTrigger,
}

98
apps/web-evals/src/components/ui/drawer.tsx
View file

@ -1,98 +0,0 @@
"use client"
import * as React from "react"
import { Drawer as DrawerPrimitive } from "vaul"
import { cn } from "@/lib/utils"
function Drawer({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Root>) {
return <DrawerPrimitive.Root data-slot="drawer" {...props} />
}
function DrawerTrigger({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Trigger>) {
return <DrawerPrimitive.Trigger data-slot="drawer-trigger" {...props} />
}
function DrawerPortal({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Portal>) {
return <DrawerPrimitive.Portal data-slot="drawer-portal" {...props} />
}
function DrawerClose({ ...props }: React.ComponentProps<typeof DrawerPrimitive.Close>) {
return <DrawerPrimitive.Close data-slot="drawer-close" {...props} />
}
function DrawerOverlay({ className, ...props }: React.ComponentProps<typeof DrawerPrimitive.Overlay>) {
return (
<DrawerPrimitive.Overlay
data-slot="drawer-overlay"
className={cn(
"data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 fixed inset-0 z-50 bg-black/50",
className,
)}
{...props}
/>
)
}
function DrawerContent({ className, children, ...props }: React.ComponentProps<typeof DrawerPrimitive.Content>) {
return (
<DrawerPortal data-slot="drawer-portal">
<DrawerOverlay />
<DrawerPrimitive.Content
data-slot="drawer-content"
className={cn(
"group/drawer-content bg-background fixed z-50 flex h-auto flex-col",
"data-[vaul-drawer-direction=top]:inset-x-0 data-[vaul-drawer-direction=top]:top-0 data-[vaul-drawer-direction=top]:mb-24 data-[vaul-drawer-direction=top]:max-h-[80vh] data-[vaul-drawer-direction=top]:rounded-b-sm data-[vaul-drawer-direction=top]:border-b",
"data-[vaul-drawer-direction=bottom]:inset-x-0 data-[vaul-drawer-direction=bottom]:bottom-0 data-[vaul-drawer-direction=bottom]:mt-24 data-[vaul-drawer-direction=bottom]:max-h-[80vh] data-[vaul-drawer-direction=bottom]:rounded-t-sm data-[vaul-drawer-direction=bottom]:border-t",
"data-[vaul-drawer-direction=right]:inset-y-0 data-[vaul-drawer-direction=right]:right-0 data-[vaul-drawer-direction=right]:w-3/4 data-[vaul-drawer-direction=right]:border-l data-[vaul-drawer-direction=right]:sm:max-w-sm",
"data-[vaul-drawer-direction=left]:inset-y-0 data-[vaul-drawer-direction=left]:left-0 data-[vaul-drawer-direction=left]:w-3/4 data-[vaul-drawer-direction=left]:border-r data-[vaul-drawer-direction=left]:sm:max-w-sm",
className,
)}
{...props}>
<div className="bg-muted mx-auto mt-4 hidden h-2 w-[100px] shrink-0 rounded-full group-data-[vaul-drawer-direction=bottom]/drawer-content:block" />
{children}
</DrawerPrimitive.Content>
</DrawerPortal>
)
}
function DrawerHeader({ className, ...props }: React.ComponentProps<"div">) {
return <div data-slot="drawer-header" className={cn("flex flex-col gap-1.5 py-4", className)} {...props} />
}
function DrawerFooter({ className, ...props }: React.ComponentProps<"div">) {
return <div data-slot="drawer-footer" className={cn("mt-auto flex flex-col gap-2 py-4", className)} {...props} />
}
function DrawerTitle({ className, ...props }: React.ComponentProps<typeof DrawerPrimitive.Title>) {
return (
<DrawerPrimitive.Title
data-slot="drawer-title"
className={cn("text-foreground font-semibold", className)}
{...props}
/>
)
}
function DrawerDescription({ className, ...props }: React.ComponentProps<typeof DrawerPrimitive.Description>) {
return (
<DrawerPrimitive.Description
data-slot="drawer-description"
className={cn("text-muted-foreground text-sm", className)}
{...props}
/>
)
}
export {
Drawer,
DrawerPortal,
DrawerOverlay,
DrawerTrigger,
DrawerClose,
DrawerContent,
DrawerHeader,
DrawerFooter,
DrawerTitle,
DrawerDescription,
}

171
apps/web-evals/src/components/ui/dropdown-menu.tsx
View file

@ -1,171 +0,0 @@
"use client"
import * as React from "react"
import * as DropdownMenuPrimitive from "@radix-ui/react-dropdown-menu"
import { CheckIcon, CircleIcon } from "lucide-react"
import { cn } from "@/lib/utils"
function DropdownMenu({ ...props }: React.ComponentProps<typeof DropdownMenuPrimitive.Root>) {
return <DropdownMenuPrimitive.Root data-slot="dropdown-menu" {...props} />
}
function DropdownMenuPortal({ ...props }: React.ComponentProps<typeof DropdownMenuPrimitive.Portal>) {
return <DropdownMenuPrimitive.Portal data-slot="dropdown-menu-portal" {...props} />
}
function DropdownMenuTrigger({ ...props }: React.ComponentProps<typeof DropdownMenuPrimitive.Trigger>) {
return <DropdownMenuPrimitive.Trigger data-slot="dropdown-menu-trigger" {...props} />
}
function DropdownMenuContent({
className,
sideOffset = 4,
...props
}: React.ComponentProps<typeof DropdownMenuPrimitive.Content>) {
return (
<DropdownMenuPrimitive.Portal>
<DropdownMenuPrimitive.Content
data-slot="dropdown-menu-content"
sideOffset={sideOffset}
className={cn(
"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 max-h-(--radix-dropdown-menu-content-available-height) min-w-[8rem] origin-(--radix-dropdown-menu-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-md border p-1 shadow-md",
className,
)}
{...props}
/>
</DropdownMenuPrimitive.Portal>
)
}
function DropdownMenuGroup({ ...props }: React.ComponentProps<typeof DropdownMenuPrimitive.Group>) {
return <DropdownMenuPrimitive.Group data-slot="dropdown-menu-group" {...props} />
}
function DropdownMenuItem({
className,
inset,
variant = "default",
...props
}: React.ComponentProps<typeof DropdownMenuPrimitive.Item> & {
inset?: boolean
variant?: "default" | "destructive"
}) {
return (
<DropdownMenuPrimitive.Item
data-slot="dropdown-menu-item"
data-inset={inset}
data-variant={variant}
className={cn(
"focus:bg-accent/5 focus:text-accent-foreground data-[variant=destructive]:text-destructive data-[variant=destructive]:focus:bg-destructive/10 dark:data-[variant=destructive]:focus:bg-destructive/20 data-[variant=destructive]:focus:text-destructive data-[variant=destructive]:*:[svg]:!text-destructive [&_svg:not([class*='text-'])]:text-muted-foreground relative flex cursor-default items-center gap-2 rounded-sm px-2 py-1.5 text-sm outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 data-[inset]:pl-8 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
"cursor-pointer",
className,
)}
{...props}
/>
)
}
function DropdownMenuCheckboxItem({
className,
children,
checked,
...props
}: React.ComponentProps<typeof DropdownMenuPrimitive.CheckboxItem>) {
return (
<DropdownMenuPrimitive.CheckboxItem
data-slot="dropdown-menu-checkbox-item"
className={cn(
"focus:bg-accent focus:text-accent-foreground relative flex cursor-default items-center gap-2 rounded-sm py-1.5 pr-2 pl-8 text-sm outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
className,
)}
checked={checked}
{...props}>
<span className="pointer-events-none absolute left-2 flex size-3.5 items-center justify-center">
<DropdownMenuPrimitive.ItemIndicator>
<CheckIcon className="size-4" />
</DropdownMenuPrimitive.ItemIndicator>
</span>
{children}
</DropdownMenuPrimitive.CheckboxItem>
)
}
function DropdownMenuRadioGroup({ ...props }: React.ComponentProps<typeof DropdownMenuPrimitive.RadioGroup>) {
return <DropdownMenuPrimitive.RadioGroup data-slot="dropdown-menu-radio-group" {...props} />
}
function DropdownMenuRadioItem({
className,
children,
...props
}: React.ComponentProps<typeof DropdownMenuPrimitive.RadioItem>) {
return (
<DropdownMenuPrimitive.RadioItem
data-slot="dropdown-menu-radio-item"
className={cn(
"focus:bg-accent focus:text-accent-foreground relative flex cursor-default items-center gap-2 rounded-sm py-1.5 pr-2 pl-8 text-sm outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
className,
)}
{...props}>
<span className="pointer-events-none absolute left-2 flex size-3.5 items-center justify-center">
<DropdownMenuPrimitive.ItemIndicator>
<CircleIcon className="size-2 fill-current" />
</DropdownMenuPrimitive.ItemIndicator>
</span>
{children}
</DropdownMenuPrimitive.RadioItem>
)
}
function DropdownMenuLabel({
className,
inset,
...props
}: React.ComponentProps<typeof DropdownMenuPrimitive.Label> & {
inset?: boolean
}) {
return (
<DropdownMenuPrimitive.Label
data-slot="dropdown-menu-label"
data-inset={inset}
className={cn("px-2 py-1.5 text-sm font-medium data-[inset]:pl-8", className)}
{...props}
/>
)
}
function DropdownMenuSeparator({ className, ...props }: React.ComponentProps<typeof DropdownMenuPrimitive.Separator>) {
return (
<DropdownMenuPrimitive.Separator
data-slot="dropdown-menu-separator"
className={cn("bg-border -mx-1 my-1 h-px", className)}
{...props}
/>
)
}
function DropdownMenuShortcut({ className, ...props }: React.ComponentProps<"span">) {
return (
<span
data-slot="dropdown-menu-shortcut"
className={cn("text-muted-foreground ml-auto text-xs tracking-widest", className)}
{...props}
/>
)
}
export {
DropdownMenu,
DropdownMenuPortal,
DropdownMenuTrigger,
DropdownMenuContent,
DropdownMenuGroup,
DropdownMenuLabel,
DropdownMenuItem,
DropdownMenuCheckboxItem,
DropdownMenuRadioGroup,
DropdownMenuRadioItem,
DropdownMenuSeparator,
DropdownMenuShortcut,
}

138
apps/web-evals/src/components/ui/form.tsx
View file

@ -1,138 +0,0 @@
"use client"
import * as React from "react"
import * as LabelPrimitive from "@radix-ui/react-label"
import { Slot } from "@radix-ui/react-slot"
import {
Controller,
FormProvider,
useFormContext,
useFormState,
type ControllerProps,
type FieldPath,
type FieldValues,
} from "react-hook-form"
import { cn } from "@/lib/utils"
import { Label } from "@/components/ui/label"
const Form = FormProvider
type FormFieldContextValue<
TFieldValues extends FieldValues = FieldValues,
TName extends FieldPath<TFieldValues> = FieldPath<TFieldValues>,
> = {
name: TName
}
const FormFieldContext = React.createContext<FormFieldContextValue>({} as FormFieldContextValue)
const FormField = <
TFieldValues extends FieldValues = FieldValues,
TName extends FieldPath<TFieldValues> = FieldPath<TFieldValues>,
>({
...props
}: ControllerProps<TFieldValues, TName>) => {
return (
<FormFieldContext.Provider value={{ name: props.name }}>
<Controller {...props} />
</FormFieldContext.Provider>
)
}
const useFormField = () => {
const fieldContext = React.useContext(FormFieldContext)
const itemContext = React.useContext(FormItemContext)
const { getFieldState } = useFormContext()
const formState = useFormState({ name: fieldContext.name })
const fieldState = getFieldState(fieldContext.name, formState)
if (!fieldContext) {
throw new Error("useFormField should be used within <FormField>")
}
const { id } = itemContext
return {
id,
name: fieldContext.name,
formItemId: `${id}-form-item`,
formDescriptionId: `${id}-form-item-description`,
formMessageId: `${id}-form-item-message`,
...fieldState,
}
}
type FormItemContextValue = {
id: string
}
const FormItemContext = React.createContext<FormItemContextValue>({} as FormItemContextValue)
function FormItem({ className, ...props }: React.ComponentProps<"div">) {
const id = React.useId()
return (
<FormItemContext.Provider value={{ id }}>
<div data-slot="form-item" className={cn("grid gap-2", className)} {...props} />
</FormItemContext.Provider>
)
}
function FormLabel({ className, ...props }: React.ComponentProps<typeof LabelPrimitive.Root>) {
const { error, formItemId } = useFormField()
return (
<Label
data-slot="form-label"
data-error={!!error}
className={cn("data-[error=true]:text-destructive", className)}
htmlFor={formItemId}
{...props}
/>
)
}
function FormControl({ ...props }: React.ComponentProps<typeof Slot>) {
const { error, formItemId, formDescriptionId, formMessageId } = useFormField()
return (
<Slot
data-slot="form-control"
id={formItemId}
aria-describedby={!error ? `${formDescriptionId}` : `${formDescriptionId} ${formMessageId}`}
aria-invalid={!!error}
{...props}
/>
)
}
function FormDescription({ className, ...props }: React.ComponentProps<"p">) {
const { formDescriptionId } = useFormField()
return (
<p
data-slot="form-description"
id={formDescriptionId}
className={cn("text-muted-foreground text-sm", className)}
{...props}
/>
)
}
function FormMessage({ className, ...props }: React.ComponentProps<"p">) {
const { error, formMessageId } = useFormField()
const body = error ? String(error?.message ?? "") : props.children
if (!body) {
return null
}
return (
<p data-slot="form-message" id={formMessageId} className={cn("text-destructive text-sm", className)} {...props}>
{body}
</p>
)
}
export { useFormField, Form, FormItem, FormLabel, FormControl, FormDescription, FormMessage, FormField }

22
apps/web-evals/src/components/ui/index.ts
View file

@ -1,22 +0,0 @@
export * from "./alert-dialog"
export * from "./badge"
export * from "./button"
export * from "./checkbox"
export * from "./command"
export * from "./dialog"
export * from "./drawer"
export * from "./dropdown-menu"
export * from "./form"
export * from "./input"
export * from "./label"
export * from "./multi-select"
export * from "./popover"
export * from "./scroll-area"
export * from "./select"
export * from "./separator"
export * from "./slider"
export * from "./sonner"
export * from "./table"
export * from "./tabs"
export * from "./textarea"
export * from "./tooltip"

22
apps/web-evals/src/components/ui/input.tsx
View file

@ -1,22 +0,0 @@
import * as React from "react"
import { cn } from "@/lib/utils"
function Input({ className, type, ...props }: React.ComponentProps<"input">) {
return (
<input
type={type}
data-slot="input"
className={cn(
"file:text-foreground placeholder:text-muted-foreground selection:bg-primary selection:text-primary-foreground flex h-9 w-full min-w-0 rounded-sm px-3 py-1 text-base shadow-xs transition-[color,box-shadow] outline-none file:inline-flex file:h-7 file:border-0 file:bg-transparent file:text-sm file:font-medium disabled:pointer-events-none disabled:cursor-not-allowed disabled:opacity-50",
"focus-visible:border-ring focus-visible:ring-ring/50 focus-visible:ring-[3px]",
"aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive",
"border border-input bg-input",
className,
)}
{...props}
/>
)
}
export { Input }

21
apps/web-evals/src/components/ui/label.tsx
View file

@ -1,21 +0,0 @@
"use client"
import * as React from "react"
import * as LabelPrimitive from "@radix-ui/react-label"
import { cn } from "@/lib/utils"
function Label({ className, ...props }: React.ComponentProps<typeof LabelPrimitive.Root>) {
return (
<LabelPrimitive.Root
data-slot="label"
className={cn(
"flex items-center gap-2 leading-none font-medium select-none group-data-[disabled=true]:pointer-events-none group-data-[disabled=true]:opacity-50 peer-disabled:cursor-not-allowed peer-disabled:opacity-50",
className,
)}
{...props}
/>
)
}
export { Label }

300
apps/web-evals/src/components/ui/multi-select.tsx
View file

@ -1,300 +0,0 @@
import * as React from "react"
import { cva, type VariantProps } from "class-variance-authority"
import fuzzysort from "fuzzysort"
import { Check, X, ChevronsUpDown } from "lucide-react"
import { cn } from "@/lib/utils"
import { Badge } from "./badge"
import { Popover, PopoverContent, PopoverTrigger } from "./popover"
import { Command, CommandEmpty, CommandGroup, CommandInput, CommandItem, CommandList } from "./command"
/**
* Variants for the multi-select component to handle different styles.
* Uses class-variance-authority (cva) to define different styles based on "variant" prop.
*/
const multiSelectVariants = cva("px-2 py-1", {
variants: {
variant: {
default: "border-foreground/10 text-foreground bg-card hover:bg-card/80",
secondary: "border-foreground/10 bg-secondary text-secondary-foreground hover:bg-secondary/80",
destructive: "border-transparent bg-destructive text-destructive-foreground hover:bg-destructive/80",
inverted: "bg-background",
},
},
defaultVariants: {
variant: "default",
},
})
/**
* Props for MultiSelect component
*/
interface MultiSelectProps extends React.HTMLAttributes<HTMLDivElement>, VariantProps<typeof multiSelectVariants> {
/**
* An array of option objects to be displayed in the multi-select component.
* Each option object has a label and value.
*/
options: {
/** The text to display for the option. */
label: string
/** The unique value associated with the option. */
value: string
}[]
/**
* Callback function triggered when the selected values change.
* Receives an array of the new selected values.
*/
onValueChange: (value: string[]) => void
/** The controlled selected values. When provided, the component becomes controlled. */
value?: string[]
/** The default selected values when the component mounts (uncontrolled mode). */
defaultValue?: string[]
/**
* Placeholder text to be displayed when no values are selected.
* Optional, defaults to "Select options".
*/
placeholder?: string
/**
* Maximum number of items to display. Extra selected items will be summarized.
* Optional, defaults to 3.
*/
maxCount?: number
/**
* The modality of the popover. When set to true, interaction with outside elements
* will be disabled and only popover content will be visible to screen readers.
* Optional, defaults to false.
*/
modalPopover?: boolean
/**
* If true, renders the multi-select component as a child of another component.
* Optional, defaults to false.
*/
asChild?: boolean
/**
* Additional class names to apply custom styles to the multi-select component.
* Optional, can be used to add custom styles.
*/
className?: string
/**
* If true, popover width will auto-size to content instead of matching trigger width.
* Optional, defaults to false.
*/
popoverAutoWidth?: boolean
/**
* Optional footer content to render at the bottom of the popover.
* Useful for adding reset buttons or other actions.
*/
footer?: React.ReactNode
}
export const MultiSelect = React.forwardRef<HTMLDivElement, MultiSelectProps>(
(
{
options,
onValueChange,
variant,
value,
defaultValue = [],
placeholder = "Select options",
maxCount = 3,
modalPopover = false,
popoverAutoWidth = false,
footer,
className,
...props
},
ref,
) => {
const [internalSelectedValues, setInternalSelectedValues] = React.useState<string[]>(defaultValue)
const [isPopoverOpen, setIsPopoverOpen] = React.useState(false)
// Use controlled value if provided, otherwise use internal state
const isControlled = value !== undefined
const selectedValues = isControlled ? value : internalSelectedValues
const setSelectedValues = React.useCallback(
(newValues: string[]) => {
if (!isControlled) {
setInternalSelectedValues(newValues)
}
onValueChange(newValues)
},
[isControlled, onValueChange],
)
const handleInputKeyDown = (event: React.KeyboardEvent<HTMLInputElement>) => {
if (event.key === "Enter") {
setIsPopoverOpen(true)
} else if (event.key === "Backspace" && !event.currentTarget.value) {
if (!selectedValues.length) return
const newSelectedValues = selectedValues.slice(0, -1)
setSelectedValues(newSelectedValues)
}
}
const toggleOption = (option: string) => {
const newSelectedValues = selectedValues.includes(option)
? selectedValues.filter((value) => value !== option)
: [...selectedValues, option]
setSelectedValues(newSelectedValues)
}
const handleTogglePopover = () => {
setIsPopoverOpen((prev) => !prev)
}
const clearExtraOptions = () => {
const newSelectedValues = selectedValues.slice(0, maxCount)
setSelectedValues(newSelectedValues)
}
const searchResultsRef = React.useRef<Map<string, number>>(new Map())
const searchValueRef = React.useRef("")
const onSelectAll = () => {
const values = Array.from(searchResultsRef.current.keys())
if (
selectedValues.length === values.length &&
selectedValues.sort().join(",") === values.sort().join(",")
) {
setSelectedValues([])
return
}
setSelectedValues(values)
}
const onFilter = React.useCallback(
(value: string, search: string) => {
if (searchValueRef.current !== search) {
searchValueRef.current = search
searchResultsRef.current.clear()
for (const {
obj: { value },
score,
} of fuzzysort.go(search, options, {
key: "label",
})) {
searchResultsRef.current.set(value, score)
}
}
if (value === "all") {
return searchResultsRef.current.size > 1 ? 0.01 : 0
}
return searchResultsRef.current.get(value) ?? 0
},
[options],
)
return (
<Popover open={isPopoverOpen} onOpenChange={setIsPopoverOpen} modal={modalPopover}>
<PopoverTrigger asChild>
<div
ref={ref}
{...props}
onClick={handleTogglePopover}
className={cn(
"flex w-full rounded-sm min-h-9 h-auto items-center justify-between [&_svg]:pointer-events-auto",
"font-medium border border-input bg-input hover:opacity-80 cursor-pointer",
className,
)}>
{selectedValues.length > 0 ? (
<div className="flex justify-between items-center w-full">
<div className="flex flex-wrap items-center gap-1 p-1">
{selectedValues.slice(0, maxCount).map((value) => (
<Badge key={value} className={cn(multiSelectVariants({ variant }))}>
<div className="flex items-center gap-1.5">
<div>{options.find((o) => o.value === value)?.label}</div>
<div
onClick={(event) => {
event.stopPropagation()
toggleOption(value)
}}
className="cursor-pointer">
<X className="size-4 rounded-full p-0.5 bg-accent/5" />
</div>
</div>
</Badge>
))}
{selectedValues.length > maxCount && (
<Badge className={cn("text-ring", multiSelectVariants({ variant }))}>
<div className="flex items-center gap-1.5">
<div>{`+ ${selectedValues.length - maxCount} more`}</div>
<div
onClick={(event) => {
event.stopPropagation()
clearExtraOptions()
}}
className="cursor-pointer">
<X className="size-4 rounded-full p-0.5 bg-ring/5" />
</div>
</div>
</Badge>
)}
</div>
</div>
) : (
<div className="flex items-center justify-between w-full mx-auto">
<span className="text-muted-foreground mx-3">{placeholder}</span>
<ChevronsUpDown className="opacity-50 size-4 mx-2" />
</div>
)}
</div>
</PopoverTrigger>
<PopoverContent
className={cn("p-0", popoverAutoWidth ? "w-auto" : "w-[var(--radix-popover-trigger-width)]")}
align="start"
onEscapeKeyDown={() => setIsPopoverOpen(false)}>
<Command filter={onFilter}>
<CommandInput placeholder="Search" onKeyDown={handleInputKeyDown} />
<CommandList>
<CommandEmpty>No results found.</CommandEmpty>
<CommandGroup>
{options.map((option) => (
<CommandItem
key={option.value}
value={option.value}
onSelect={() => toggleOption(option.value)}
className="flex items-center justify-between">
<span>{option.label}</span>
<Check
className={cn(
"text-accent group-data-[selected=true]:text-accent-foreground size-4",
{ "opacity-0": !selectedValues.includes(option.value) },
)}
/>
</CommandItem>
))}
<CommandItem
key="all"
value="all"
onSelect={onSelectAll}
className="flex items-center justify-between">
<span>Select All</span>
</CommandItem>
</CommandGroup>
</CommandList>
</Command>
{footer && <div className="border-t p-2">{footer}</div>}
</PopoverContent>
</Popover>
)
},
)
MultiSelect.displayName = "MultiSelect"

42
apps/web-evals/src/components/ui/popover.tsx
View file

@ -1,42 +0,0 @@
"use client"
import * as React from "react"
import * as PopoverPrimitive from "@radix-ui/react-popover"
import { cn } from "@/lib/utils"
function Popover({ ...props }: React.ComponentProps<typeof PopoverPrimitive.Root>) {
return <PopoverPrimitive.Root data-slot="popover" {...props} />
}
function PopoverTrigger({ ...props }: React.ComponentProps<typeof PopoverPrimitive.Trigger>) {
return <PopoverPrimitive.Trigger data-slot="popover-trigger" {...props} />
}
function PopoverContent({
className,
align = "center",
sideOffset = 4,
...props
}: React.ComponentProps<typeof PopoverPrimitive.Content>) {
return (
<PopoverPrimitive.Portal>
<PopoverPrimitive.Content
data-slot="popover-content"
align={align}
sideOffset={sideOffset}
className={cn(
"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-72 origin-(--radix-popover-content-transform-origin) rounded-sm border p-4 shadow-md outline-hidden",
className,
)}
{...props}
/>
</PopoverPrimitive.Portal>
)
}
function PopoverAnchor({ ...props }: React.ComponentProps<typeof PopoverPrimitive.Anchor>) {
return <PopoverPrimitive.Anchor data-slot="popover-anchor" {...props} />
}
export { Popover, PopoverTrigger, PopoverContent, PopoverAnchor }

51
apps/web-evals/src/components/ui/scroll-area.tsx
View file

@ -1,51 +0,0 @@
"use client"
import * as React from "react"
import * as ScrollAreaPrimitive from "@radix-ui/react-scroll-area"
import { cn } from "@/lib/utils"
type ScrollAreaProps = React.ComponentProps<typeof ScrollAreaPrimitive.Root> & {
viewportRef?: React.RefObject<HTMLDivElement>
}
function ScrollArea({ className, children, viewportRef, ...props }: ScrollAreaProps) {
return (
<ScrollAreaPrimitive.Root data-slot="scroll-area" className={cn("relative", className)} {...props}>
<ScrollAreaPrimitive.Viewport
ref={viewportRef}
data-slot="scroll-area-viewport"
className="ring-ring/10 dark:ring-ring/20 dark:outline-ring/40 outline-ring/50 size-full rounded-[inherit] transition-[color,box-shadow] focus-visible:ring-4 focus-visible:outline-1">
{children}
</ScrollAreaPrimitive.Viewport>
<ScrollBar />
<ScrollAreaPrimitive.Corner />
</ScrollAreaPrimitive.Root>
)
}
function ScrollBar({
className,
orientation = "vertical",
...props
}: React.ComponentProps<typeof ScrollAreaPrimitive.ScrollAreaScrollbar>) {
return (
<ScrollAreaPrimitive.ScrollAreaScrollbar
data-slot="scroll-area-scrollbar"
orientation={orientation}
className={cn(
"flex touch-none p-px transition-colors select-none",
orientation === "vertical" && "h-full w-2.5 border-l border-l-transparent",
orientation === "horizontal" && "h-2.5 flex-col border-t border-t-transparent",
className,
)}
{...props}>
<ScrollAreaPrimitive.ScrollAreaThumb
data-slot="scroll-area-thumb"
className="bg-border relative flex-1 rounded-full"
/>
</ScrollAreaPrimitive.ScrollAreaScrollbar>
)
}
export { ScrollArea, ScrollBar }

156
apps/web-evals/src/components/ui/select.tsx
View file

@ -1,156 +0,0 @@
"use client"
import * as React from "react"
import * as SelectPrimitive from "@radix-ui/react-select"
import { Check, ChevronDown, ChevronUp } from "lucide-react"
import { cn } from "@/lib/utils"
function Select({ ...props }: React.ComponentProps<typeof SelectPrimitive.Root>) {
return <SelectPrimitive.Root data-slot="select" {...props} />
}
function SelectGroup({ ...props }: React.ComponentProps<typeof SelectPrimitive.Group>) {
return <SelectPrimitive.Group data-slot="select-group" {...props} />
}
function SelectValue({ ...props }: React.ComponentProps<typeof SelectPrimitive.Value>) {
return <SelectPrimitive.Value data-slot="select-value" {...props} />
}
function SelectTrigger({
className,
size = "default",
children,
...props
}: React.ComponentProps<typeof SelectPrimitive.Trigger> & {
size?: "sm" | "default"
}) {
return (
<SelectPrimitive.Trigger
data-slot="select-trigger"
data-size={size}
className={cn(
"data-[placeholder]:text-muted-foreground [&_svg:not([class*='text-'])]:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive flex w-fit items-center justify-between gap-2 rounded-sm px-3 py-2 whitespace-nowrap shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50 data-[size=default]:h-9 data-[size=sm]:h-8 *:data-[slot=select-value]:line-clamp-1 *:data-[slot=select-value]:flex *:data-[slot=select-value]:items-center *:data-[slot=select-value]:gap-2 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4",
"border border-input bg-input hover:opacity-80 cursor-pointer",
className,
)}
{...props}>
{children}
<SelectPrimitive.Icon asChild>
<ChevronDown className="size-4 opacity-50" />
</SelectPrimitive.Icon>
</SelectPrimitive.Trigger>
)
}
function SelectContent({
className,
children,
position = "popper",
...props
}: React.ComponentProps<typeof SelectPrimitive.Content>) {
return (
<SelectPrimitive.Portal>
<SelectPrimitive.Content
data-slot="select-content"
className={cn(
"bg-popover text-popover-foreground data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 relative z-50 max-h-(--radix-select-content-available-height) min-w-[8rem] origin-(--radix-select-content-transform-origin) overflow-x-hidden overflow-y-auto rounded-sm shadow-md",
position === "popper" &&
"data-[side=bottom]:translate-y-1 data-[side=left]:-translate-x-1 data-[side=right]:translate-x-1 data-[side=top]:-translate-y-1",
className,
)}
position={position}
{...props}>
<SelectScrollUpButton />
<SelectPrimitive.Viewport
className={cn(
"p-1",
position === "popper" &&
"h-[var(--radix-select-trigger-height)] w-full min-w-[var(--radix-select-trigger-width)] scroll-my-1",
)}>
{children}
</SelectPrimitive.Viewport>
<SelectScrollDownButton />
</SelectPrimitive.Content>
</SelectPrimitive.Portal>
)
}
function SelectLabel({ className, ...props }: React.ComponentProps<typeof SelectPrimitive.Label>) {
return (
<SelectPrimitive.Label
data-slot="select-label"
className={cn("text-muted-foreground px-2 py-1.5 text-xs", className)}
{...props}
/>
)
}
function SelectItem({ className, children, ...props }: React.ComponentProps<typeof SelectPrimitive.Item>) {
return (
<SelectPrimitive.Item
data-slot="select-item"
className={cn(
"focus:bg-accent focus:text-accent-foreground [&_svg:not([class*='text-'])]:text-muted-foreground relative flex w-full cursor-default items-center gap-2 rounded-xs py-1.5 pr-8 pl-2 outline-hidden select-none data-[disabled]:pointer-events-none data-[disabled]:opacity-50 [&_svg]:pointer-events-none [&_svg]:shrink-0 [&_svg:not([class*='size-'])]:size-4 *:[span]:last:flex *:[span]:last:items-center *:[span]:last:gap-2",
"text-foreground active:opacity-80 cursor-pointer group",
className,
)}
{...props}>
<span className="absolute right-2 flex size-3.5 items-center justify-center">
<SelectPrimitive.ItemIndicator>
<Check className="text-accent group-focus:text-accent-foreground size-4" />
</SelectPrimitive.ItemIndicator>
</span>
<SelectPrimitive.ItemText>{children}</SelectPrimitive.ItemText>
</SelectPrimitive.Item>
)
}
function SelectSeparator({ className, ...props }: React.ComponentProps<typeof SelectPrimitive.Separator>) {
return (
<SelectPrimitive.Separator
data-slot="select-separator"
className={cn("bg-border pointer-events-none -mx-1 my-1 h-px", className)}
{...props}
/>
)
}
function SelectScrollUpButton({ className, ...props }: React.ComponentProps<typeof SelectPrimitive.ScrollUpButton>) {
return (
<SelectPrimitive.ScrollUpButton
data-slot="select-scroll-up-button"
className={cn("flex cursor-default items-center justify-center py-1", className)}
{...props}>
<ChevronUp className="size-4" />
</SelectPrimitive.ScrollUpButton>
)
}
function SelectScrollDownButton({
className,
...props
}: React.ComponentProps<typeof SelectPrimitive.ScrollDownButton>) {
return (
<SelectPrimitive.ScrollDownButton
data-slot="select-scroll-down-button"
className={cn("flex cursor-default items-center justify-center py-1", className)}
{...props}>
<ChevronDown className="size-4" />
</SelectPrimitive.ScrollDownButton>
)
}
export {
Select,
SelectContent,
SelectGroup,
SelectItem,
SelectLabel,
SelectScrollDownButton,
SelectScrollUpButton,
SelectSeparator,
SelectTrigger,
SelectValue,
}

28
apps/web-evals/src/components/ui/separator.tsx
View file

@ -1,28 +0,0 @@
"use client"
import * as React from "react"
import * as SeparatorPrimitive from "@radix-ui/react-separator"
import { cn } from "@/lib/utils"
function Separator({
className,
orientation = "horizontal",
decorative = true,
...props
}: React.ComponentProps<typeof SeparatorPrimitive.Root>) {
return (
<SeparatorPrimitive.Root
data-slot="separator-root"
decorative={decorative}
orientation={orientation}
className={cn(
"bg-border shrink-0 data-[orientation=horizontal]:h-px data-[orientation=horizontal]:w-full data-[orientation=vertical]:h-full data-[orientation=vertical]:w-px",
className,
)}
{...props}
/>
)
}
export { Separator }

56
apps/web-evals/src/components/ui/slider.tsx
View file

@ -1,56 +0,0 @@
"use client"
import * as React from "react"
import * as SliderPrimitive from "@radix-ui/react-slider"
import { cn } from "@/lib/utils"
function Slider({
className,
defaultValue,
value,
min = 0,
max = 100,
...props
}: React.ComponentProps<typeof SliderPrimitive.Root>) {
const _values = React.useMemo(
() => (Array.isArray(value) ? value : Array.isArray(defaultValue) ? defaultValue : [min, max]),
[value, defaultValue, min, max],
)
return (
<SliderPrimitive.Root
data-slot="slider"
defaultValue={defaultValue}
value={value}
min={min}
max={max}
className={cn(
"relative flex w-full touch-none items-center select-none data-[disabled]:opacity-50 data-[orientation=vertical]:h-full data-[orientation=vertical]:min-h-44 data-[orientation=vertical]:w-auto data-[orientation=vertical]:flex-col",
className,
)}
{...props}>
<SliderPrimitive.Track
data-slot="slider-track"
className={cn(
"bg-muted relative grow overflow-hidden rounded-full data-[orientation=horizontal]:h-1.5 data-[orientation=horizontal]:w-full data-[orientation=vertical]:h-full data-[orientation=vertical]:w-1.5",
)}>
<SliderPrimitive.Range
data-slot="slider-range"
className={cn(
"bg-primary absolute data-[orientation=horizontal]:h-full data-[orientation=vertical]:w-full",
)}
/>
</SliderPrimitive.Track>
{Array.from({ length: _values.length }, (_, index) => (
<SliderPrimitive.Thumb
data-slot="slider-thumb"
key={index}
className="border-primary bg-accent block size-4 shrink-0 rounded-full border shadow-sm transition-[color,box-shadow] focus-visible:outline-hidden disabled:pointer-events-none disabled:opacity-50 cursor-pointer"
/>
))}
</SliderPrimitive.Root>
)
}
export { Slider }

25
apps/web-evals/src/components/ui/sonner.tsx
View file

@ -1,25 +0,0 @@
"use client"
import { useTheme } from "next-themes"
import { Toaster as Sonner, ToasterProps } from "sonner"
const Toaster = ({ ...props }: ToasterProps) => {
const { theme = "system" } = useTheme()
return (
<Sonner
theme={theme as ToasterProps["theme"]}
className="toaster group"
style={
{
"--normal-bg": "var(--popover)",
"--normal-text": "var(--popover-foreground)",
"--normal-border": "var(--border)",
} as React.CSSProperties
}
{...props}
/>
)
}
export { Toaster }

75
apps/web-evals/src/components/ui/table.tsx
View file

@ -1,75 +0,0 @@
"use client"
import * as React from "react"
import { cn } from "@/lib/utils"
function Table({ className, ...props }: React.ComponentProps<"table">) {
return (
<div data-slot="table-container" className="relative w-full overflow-x-auto">
<table data-slot="table" className={cn("w-full caption-bottom text-sm", className)} {...props} />
</div>
)
}
function TableHeader({ className, ...props }: React.ComponentProps<"thead">) {
return <thead data-slot="table-header" className={cn("[&_tr]:border-b", className)} {...props} />
}
function TableBody({ className, ...props }: React.ComponentProps<"tbody">) {
return <tbody data-slot="table-body" className={cn("[&_tr:last-child]:border-0", className)} {...props} />
}
function TableFooter({ className, ...props }: React.ComponentProps<"tfoot">) {
return (
<tfoot
data-slot="table-footer"
className={cn("bg-muted/50 border-t font-medium [&>tr]:last:border-b-0", className)}
{...props}
/>
)
}
function TableRow({ className, ...props }: React.ComponentProps<"tr">) {
return (
<tr
data-slot="table-row"
className={cn("hover:bg-accent/5 data-[state=selected]:bg-muted border-b transition-colors", className)}
{...props}
/>
)
}
function TableHead({ className, ...props }: React.ComponentProps<"th">) {
return (
<th
data-slot="table-head"
className={cn(
"text-foreground h-10 px-2 text-left align-middle font-medium whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
className,
)}
{...props}
/>
)
}
function TableCell({ className, ...props }: React.ComponentProps<"td">) {
return (
<td
data-slot="table-cell"
className={cn(
"p-2 align-middle whitespace-nowrap [&:has([role=checkbox])]:pr-0 [&>[role=checkbox]]:translate-y-[2px]",
className,
)}
{...props}
/>
)
}
function TableCaption({ className, ...props }: React.ComponentProps<"caption">) {
return (
<caption data-slot="table-caption" className={cn("text-muted-foreground mt-4 text-sm", className)} {...props} />
)
}
export { Table, TableHeader, TableBody, TableFooter, TableHead, TableRow, TableCell, TableCaption }

122
apps/web-evals/src/components/ui/tabs.tsx
View file

@ -1,122 +0,0 @@
"use client"
import * as React from "react"
import { useEffect, useRef, useState } from "react"
import * as TabsPrimitive from "@radix-ui/react-tabs"
import { cn } from "@/lib/utils"
const Tabs = TabsPrimitive.Root
const TabsList = React.forwardRef<
React.ComponentRef<typeof TabsPrimitive.List>,
React.ComponentPropsWithoutRef<typeof TabsPrimitive.List>
>(({ className, ...props }, ref) => {
const [indicatorStyle, setIndicatorStyle] = useState({
left: 0,
top: 0,
width: 0,
height: 0,
})
const tabsListRef = useRef<HTMLDivElement | null>(null)
const updateIndicator = React.useCallback(() => {
if (!tabsListRef.current) {
return
}
const activeTab = tabsListRef.current.querySelector<HTMLElement>('[data-state="active"]')
if (!activeTab) {
return
}
const activeRect = activeTab.getBoundingClientRect()
const tabsRect = tabsListRef.current.getBoundingClientRect()
requestAnimationFrame(() => {
setIndicatorStyle({
left: activeRect.left - tabsRect.left,
top: activeRect.top - tabsRect.top,
width: activeRect.width,
height: activeRect.height,
})
})
}, [])
useEffect(() => {
const timeoutId = setTimeout(updateIndicator, 0)
window.addEventListener("resize", updateIndicator)
const observer = new MutationObserver(updateIndicator)
if (tabsListRef.current) {
observer.observe(tabsListRef.current, {
attributes: true,
childList: true,
subtree: true,
})
}
return () => {
clearTimeout(timeoutId)
window.removeEventListener("resize", updateIndicator)
observer.disconnect()
}
}, [updateIndicator])
return (
<div className="relative" ref={tabsListRef}>
<TabsPrimitive.List
ref={ref}
className={cn(
"relative inline-flex items-center justify-center rounded-sm bg-primary p-0.5 text-muted-foreground",
className,
)}
{...props}
/>
<div
className={cn(
"absolute rounded-sm transition-all duration-300 ease-in-out pointer-events-none",
"bg-accent/5",
)}
style={indicatorStyle}
/>
</div>
)
})
TabsList.displayName = TabsPrimitive.List.displayName
const TabsTrigger = React.forwardRef<
React.ComponentRef<typeof TabsPrimitive.Trigger>,
React.ComponentPropsWithoutRef<typeof TabsPrimitive.Trigger>
>(({ className, ...props }, ref) => (
<TabsPrimitive.Trigger
ref={ref}
className={cn(
"inline-flex items-center justify-center whitespace-nowrap rounded-sm px-3 py-1 ring-offset-background transition-all focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2 disabled:pointer-events-none disabled:opacity-50 z-10",
"data-[state=active]:text-accent data-[state=active]:font-medium cursor-pointer",
className,
)}
{...props}
/>
))
TabsTrigger.displayName = TabsPrimitive.Trigger.displayName
const TabsContent = React.forwardRef<
React.ComponentRef<typeof TabsPrimitive.Content>,
React.ComponentPropsWithoutRef<typeof TabsPrimitive.Content>
>(({ className, ...props }, ref) => (
<TabsPrimitive.Content
ref={ref}
className={cn(
"mt-2 ring-offset-background focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring focus-visible:ring-offset-2",
className,
)}
{...props}
/>
))
TabsContent.displayName = TabsPrimitive.Content.displayName
export { Tabs, TabsContent, TabsList, TabsTrigger }

19
apps/web-evals/src/components/ui/textarea.tsx
View file

@ -1,19 +0,0 @@
import * as React from "react"
import { cn } from "@/lib/utils"
function Textarea({ className, ...props }: React.ComponentProps<"textarea">) {
return (
<textarea
data-slot="textarea"
className={cn(
"placeholder:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive flex field-sizing-content min-h-16 w-full rounded-sm px-3 py-2 text-base shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50",
"border border-input bg-input",
className,
)}
{...props}
/>
)
}
export { Textarea }

47
apps/web-evals/src/components/ui/tooltip.tsx
View file

@ -1,47 +0,0 @@
"use client"
import * as React from "react"
import * as TooltipPrimitive from "@radix-ui/react-tooltip"
import { cn } from "@/lib/utils"
function TooltipProvider({ delayDuration = 0, ...props }: React.ComponentProps<typeof TooltipPrimitive.Provider>) {
return <TooltipPrimitive.Provider data-slot="tooltip-provider" delayDuration={delayDuration} {...props} />
}
function Tooltip({ ...props }: React.ComponentProps<typeof TooltipPrimitive.Root>) {
return (
<TooltipProvider>
<TooltipPrimitive.Root data-slot="tooltip" {...props} />
</TooltipProvider>
)
}
function TooltipTrigger({ ...props }: React.ComponentProps<typeof TooltipPrimitive.Trigger>) {
return <TooltipPrimitive.Trigger data-slot="tooltip-trigger" {...props} />
}
function TooltipContent({
className,
sideOffset = 0,
children,
...props
}: React.ComponentProps<typeof TooltipPrimitive.Content>) {
return (
<TooltipPrimitive.Portal>
<TooltipPrimitive.Content
data-slot="tooltip-content"
sideOffset={sideOffset}
className={cn(
"bg-primary text-primary-foreground animate-in fade-in-0 zoom-in-95 data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=closed]:zoom-out-95 data-[side=bottom]:slide-in-from-top-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2 data-[side=top]:slide-in-from-bottom-2 z-50 w-fit origin-(--radix-tooltip-content-transform-origin) rounded-sm px-3 py-1.5 text-xs text-balance",
className,
)}
{...props}>
{children}
<TooltipPrimitive.Arrow className="bg-primary fill-primary z-50 size-2.5 translate-y-[calc(-50%_-_2px)] rotate-45 rounded-[2px]" />
</TooltipPrimitive.Content>
</TooltipPrimitive.Portal>
)
}
export { Tooltip, TooltipTrigger, TooltipContent, TooltipProvider }

28
apps/web-evals/src/hooks/use-copy-run.ts
View file

@ -1,28 +0,0 @@
import { useState } from "react"
import { useMutation } from "@tanstack/react-query"
import { toast } from "sonner"
import { copyRunToProduction } from "@/lib/actions"
export function useCopyRun(runId: number) {
const [copied, setCopied] = useState(false)
const { isPending, mutate: copyRun } = useMutation({
mutationFn: () => copyRunToProduction(runId),
onSuccess: (result) => {
if (result.success) {
toast.success(result.message)
setCopied(true)
setTimeout(() => setCopied(false), 3000)
} else {
toast.error(result.error)
}
},
onError: (error) => {
console.error("Copy to production failed:", error)
toast.error("Failed to copy run to production")
},
})
return { isPending, copyRun, copied }
}

101
apps/web-evals/src/hooks/use-event-source.ts
View file

@ -1,101 +0,0 @@
import { useCallback, useEffect, useRef, useState } from "react"
export type EventSourceStatus = "waiting" | "connected" | "error"
export type EventSourceEvent = Event & { data: string }
type UseEventSourceOptions = {
url: string
withCredentials?: boolean
onMessage: (event: MessageEvent) => void
}
export function useEventSource({ url, withCredentials, onMessage }: UseEventSourceOptions) {
const sourceRef = useRef<EventSource | null>(null)
const statusRef = useRef<EventSourceStatus>("waiting")
const [status, setStatus] = useState<EventSourceStatus>("waiting")
const reconnectTimeoutRef = useRef<NodeJS.Timeout | null>(null)
const isUnmountedRef = useRef(false)
const handleMessage = useCallback((event: MessageEvent) => onMessage(event), [onMessage])
const cleanup = useCallback(() => {
if (reconnectTimeoutRef.current) {
clearTimeout(reconnectTimeoutRef.current)
reconnectTimeoutRef.current = null
}
if (sourceRef.current) {
sourceRef.current.close()
sourceRef.current = null
}
}, [])
const createEventSource = useCallback(() => {
if (isUnmountedRef.current) {
return
}
cleanup()
statusRef.current = "waiting"
setStatus("waiting")
sourceRef.current = new EventSource(url, { withCredentials })
sourceRef.current.onopen = () => {
if (isUnmountedRef.current) {
return
}
statusRef.current = "connected"
setStatus("connected")
}
sourceRef.current.onmessage = (event) => {
if (isUnmountedRef.current) {
return
}
handleMessage(event)
}
sourceRef.current.onerror = () => {
if (isUnmountedRef.current) {
return
}
statusRef.current = "error"
setStatus("error")
// Clean up current connection.
cleanup()
// Attempt to reconnect after a delay.
reconnectTimeoutRef.current = setTimeout(() => {
if (!isUnmountedRef.current) {
createEventSource()
}
}, 1000)
}
}, [url, withCredentials, handleMessage, cleanup])
useEffect(() => {
isUnmountedRef.current = false
createEventSource()
// Initial connection timeout.
const initialTimeout = setTimeout(() => {
if (statusRef.current === "waiting" && !isUnmountedRef.current) {
createEventSource()
}
}, 5000)
return () => {
isUnmountedRef.current = true
clearTimeout(initialTimeout)
cleanup()
}
}, [createEventSource, cleanup])
return status
}

37
apps/web-evals/src/hooks/use-fuzzy-model-search.ts
View file

@ -1,37 +0,0 @@
import { useCallback, useRef, useState } from "react"
import fuzzysort from "fuzzysort"
interface ModelWithId {
id: string
name: string
}
export const useFuzzyModelSearch = <T extends ModelWithId>(data: T[] | undefined) => {
const [searchValue, setSearchValue] = useState("")
const searchResultsRef = useRef<Map<string, number>>(new Map())
const searchValueRef = useRef("")
const onFilter = useCallback(
(value: string, search: string) => {
if (searchValueRef.current !== search) {
searchValueRef.current = search
searchResultsRef.current.clear()
for (const {
obj: { id },
score,
} of fuzzysort.go(search, data || [], {
key: "name",
})) {
searchResultsRef.current.set(id, score)
}
}
return searchResultsRef.current.get(value) ?? 0
},
[data],
)
return { searchValue, setSearchValue, onFilter }
}

38
apps/web-evals/src/hooks/use-open-router-models.ts
View file

@ -1,38 +0,0 @@
import { z } from "zod"
import { useQuery } from "@tanstack/react-query"
import { useFuzzyModelSearch } from "./use-fuzzy-model-search"
export const openRouterModelSchema = z.object({
id: z.string(),
name: z.string(),
})
export type OpenRouterModel = z.infer<typeof openRouterModelSchema>
export const getOpenRouterModels = async (): Promise<OpenRouterModel[]> => {
const response = await fetch("https://openrouter.ai/api/v1/models")
if (!response.ok) {
return []
}
const result = z.object({ data: z.array(openRouterModelSchema) }).safeParse(await response.json())
if (!result.success) {
console.error(result.error)
return []
}
return result.data.data.sort((a, b) => a.name.localeCompare(b.name))
}
export const useOpenRouterModels = () => {
const query = useQuery({
queryKey: ["getOpenRouterModels"],
queryFn: getOpenRouterModels,
})
const { searchValue, setSearchValue, onFilter } = useFuzzyModelSearch(query.data)
return { ...query, searchValue, setSearchValue, onFilter }
}

66
apps/web-evals/src/hooks/use-roo-code-cloud-models.ts
View file

@ -1,66 +0,0 @@
import { z } from "zod"
import { useQuery } from "@tanstack/react-query"
import { useFuzzyModelSearch } from "./use-fuzzy-model-search"
export const rooCodeCloudModelSchema = z.object({
object: z.literal("model"),
id: z.string(),
name: z.string(),
description: z.string().optional(),
context_window: z.number(),
max_tokens: z.number(),
supports_images: z.boolean().optional(),
supports_prompt_cache: z.boolean().optional(),
type: z.literal("language"),
tags: z.array(z.string()).optional(),
deprecationMessage: z.string().optional(),
owned_by: z.string(),
pricing: z.object({
input: z.string(),
output: z.string(),
input_cache_read: z.string().optional(),
input_cache_write: z.string().optional(),
}),
evals: z
.object({
score: z.number().min(0).max(100),
})
.optional(),
created: z.number(),
deprecated: z.boolean().optional(),
})
export type RooCodeCloudModel = z.infer<typeof rooCodeCloudModelSchema>
export const getRooCodeCloudModels = async (): Promise<RooCodeCloudModel[]> => {
const response = await fetch("https://api.roocode.com/proxy/v1/models")
if (!response.ok) {
return []
}
const result = z
.object({
object: z.literal("list"),
data: z.array(rooCodeCloudModelSchema),
})
.safeParse(await response.json())
if (!result.success) {
console.error(result.error)
return []
}
return result.data.data.filter((model) => !model.deprecated).sort((a, b) => a.name.localeCompare(b.name))
}
export const useRooCodeCloudModels = () => {
const query = useQuery({
queryKey: ["getRooCodeCloudModels"],
queryFn: getRooCodeCloudModels,
})
const { searchValue, setSearchValue, onFilter } = useFuzzyModelSearch(query.data)
return { ...query, searchValue, setSearchValue, onFilter }
}

110
apps/web-evals/src/hooks/use-run-status.ts
View file

@ -1,110 +0,0 @@
import { useState, useCallback, useRef } from "react"
import { useQuery, keepPreviousData } from "@tanstack/react-query"
import { type TokenUsage, type ToolUsage, RooCodeEventName, taskEventSchema } from "@roo-code/types"
import type { Run, Task, TaskMetrics } from "@roo-code/evals"
import { getHeartbeat } from "@/actions/heartbeat"
import { getRunners } from "@/actions/runners"
import { getTasks } from "@/actions/tasks"
import { type EventSourceStatus, useEventSource } from "@/hooks/use-event-source"
export type RunStatus = {
sseStatus: EventSourceStatus
heartbeat: string | null | undefined
runners: string[] | undefined
tasks: (Task & { taskMetrics: TaskMetrics | null })[] | undefined
tokenUsage: Map<number, TokenUsage & { duration?: number }>
toolUsage: Map<number, ToolUsage>
usageUpdatedAt: number | undefined
}
export const useRunStatus = (run: Run): RunStatus => {
const [tasksUpdatedAt, setTasksUpdatedAt] = useState<number>()
const [usageUpdatedAt, setUsageUpdatedAt] = useState<number>()
const tokenUsage = useRef<Map<number, TokenUsage & { duration?: number }>>(new Map())
const toolUsage = useRef<Map<number, ToolUsage>>(new Map())
const startTimes = useRef<Map<number, number>>(new Map())
const { data: heartbeat } = useQuery({
queryKey: ["getHeartbeat", run.id],
queryFn: () => getHeartbeat(run.id),
refetchInterval: 10_000,
})
const { data: runners } = useQuery({
queryKey: ["getRunners", run.id],
queryFn: () => getRunners(run.id),
refetchInterval: 10_000,
})
const { data: tasks } = useQuery({
queryKey: ["getTasks", run.id, tasksUpdatedAt],
queryFn: async () => getTasks(run.id),
placeholderData: keepPreviousData,
refetchInterval: 30_000,
})
const url = `/api/runs/${run.id}/stream`
const onMessage = useCallback((messageEvent: MessageEvent) => {
let data
try {
data = JSON.parse(messageEvent.data)
} catch (_) {
console.log(`invalid JSON: ${messageEvent.data}`)
return
}
const result = taskEventSchema.safeParse(data)
if (!result.success) {
console.log(`unrecognized messageEvent.data: ${messageEvent.data}`)
return
}
const { eventName, payload, taskId } = result.data
if (!taskId) {
console.log(`no taskId: ${messageEvent.data}`)
return
}
switch (eventName) {
case RooCodeEventName.TaskStarted:
startTimes.current.set(taskId, Date.now())
break
case RooCodeEventName.TaskTokenUsageUpdated: {
const startTime = startTimes.current.get(taskId)
const duration = startTime ? Date.now() - startTime : undefined
tokenUsage.current.set(taskId, { ...payload[1], duration })
// Track tool usage from streaming updates
if (payload[2]) {
toolUsage.current.set(taskId, payload[2])
}
setUsageUpdatedAt(Date.now())
break
}
case RooCodeEventName.EvalPass:
case RooCodeEventName.EvalFail:
setTasksUpdatedAt(Date.now())
break
}
}, [])
const sseStatus = useEventSource({ url, onMessage })
return {
sseStatus,
heartbeat,
runners,
tasks,
tokenUsage: tokenUsage.current,
toolUsage: toolUsage.current,
usageUpdatedAt,
}
}

30
apps/web-evals/src/lib/__tests__/formatters.spec.ts
View file

@ -1,30 +0,0 @@
import { formatDuration, formatTokens } from "../formatters"
describe("formatDuration()", () => {
it("formats as H:MM:SS", () => {
expect(formatDuration(0)).toBe("0:00:00")
expect(formatDuration(1_000)).toBe("0:00:01")
expect(formatDuration(61_000)).toBe("0:01:01")
expect(formatDuration(3_661_000)).toBe("1:01:01")
})
})
describe("formatTokens()", () => {
it("formats small numbers without suffix", () => {
expect(formatTokens(0)).toBe("0")
expect(formatTokens(999)).toBe("999")
})
it("formats thousands without decimals and clamps to 1.0M at boundary", () => {
expect(formatTokens(1_000)).toBe("1k")
expect(formatTokens(72_500)).toBe("73k")
expect(formatTokens(999_499)).toBe("999k")
expect(formatTokens(999_500)).toBe("1.0M")
})
it("formats millions with one decimal and clamps to 1.0B at boundary", () => {
expect(formatTokens(1_000_000)).toBe("1.0M")
expect(formatTokens(3_240_000)).toBe("3.2M")
expect(formatTokens(999_950_000)).toBe("1.0B")
})
})

65
apps/web-evals/src/lib/__tests__/normalize-create-run.spec.ts
View file

@ -1,65 +0,0 @@
import { normalizeCreateRunForSubmit } from "../normalize-create-run"
describe("normalizeCreateRunForSubmit", () => {
it("uses selectedExercises for partial suite", () => {
const result = normalizeCreateRunForSubmit(
{
model: "roo/model-a",
description: "",
suite: "partial",
exercises: [],
settings: undefined,
concurrency: 1,
timeout: 5,
iterations: 1,
jobToken: "",
executionMethod: "vscode",
},
["js/foo", "py/bar"],
)
expect(result.suite).toBe("partial")
expect(result.exercises).toEqual(["js/foo", "py/bar"])
})
it("dedupes selectedExercises for partial suite", () => {
const result = normalizeCreateRunForSubmit(
{
model: "roo/model-a",
description: "",
suite: "partial",
exercises: [],
settings: undefined,
concurrency: 1,
timeout: 5,
iterations: 1,
jobToken: "",
executionMethod: "vscode",
},
["js/foo", "js/foo", "py/bar"],
)
expect(result.exercises).toEqual(["js/foo", "py/bar"])
})
it("clears exercises for full suite", () => {
const result = normalizeCreateRunForSubmit(
{
model: "roo/model-a",
description: "",
suite: "full",
exercises: ["js/foo"],
settings: undefined,
concurrency: 1,
timeout: 5,
iterations: 1,
jobToken: "",
executionMethod: "vscode",
},
["js/foo"],
)
expect(result.suite).toBe("full")
expect(result.exercises).toEqual([])
})
})

78
apps/web-evals/src/lib/__tests__/roo-last-model-selection.spec.ts
View file

@ -1,78 +0,0 @@
import {
loadRooLastModelSelection,
ROO_LAST_MODEL_SELECTION_KEY,
saveRooLastModelSelection,
} from "../roo-last-model-selection"
class LocalStorageMock implements Storage {
private store = new Map<string, string>()
get length(): number {
return this.store.size
}
clear(): void {
this.store.clear()
}
getItem(key: string): string | null {
return this.store.get(key) ?? null
}
key(index: number): string | null {
return Array.from(this.store.keys())[index] ?? null
}
removeItem(key: string): void {
this.store.delete(key)
}
setItem(key: string, value: string): void {
this.store.set(key, value)
}
}
beforeEach(() => {
Object.defineProperty(globalThis, "localStorage", {
value: new LocalStorageMock(),
configurable: true,
})
})
describe("roo-last-model-selection", () => {
it("saves and loads (deduped + trimmed)", () => {
saveRooLastModelSelection([" roo/model-a ", "roo/model-a", "roo/model-b"])
expect(loadRooLastModelSelection()).toEqual(["roo/model-a", "roo/model-b"])
})
it("ignores invalid JSON", () => {
localStorage.setItem(ROO_LAST_MODEL_SELECTION_KEY, "{this is not json")
expect(loadRooLastModelSelection()).toEqual([])
})
it("clears when empty", () => {
localStorage.setItem(ROO_LAST_MODEL_SELECTION_KEY, JSON.stringify(["roo/model-a"]))
saveRooLastModelSelection([])
expect(localStorage.getItem(ROO_LAST_MODEL_SELECTION_KEY)).toBeNull()
})
it("does not throw if localStorage access fails", () => {
Object.defineProperty(globalThis, "localStorage", {
value: {
getItem: () => {
throw new Error("blocked")
},
setItem: () => {
throw new Error("blocked")
},
removeItem: () => {
throw new Error("blocked")
},
},
configurable: true,
})
expect(() => loadRooLastModelSelection()).not.toThrow()
expect(() => saveRooLastModelSelection(["roo/model-a"])).not.toThrow()
})
})

19
apps/web-evals/src/lib/actions.ts
View file

@ -1,19 +0,0 @@
"use server"
import { client, getProductionClient, copyRun } from "@roo-code/evals"
export async function copyRunToProduction(runId: number) {
try {
await copyRun({ sourceDb: client, targetDb: getProductionClient(), runId })
return {
success: true,
message: `Run ${runId} successfully copied to production.`,
}
} catch (error) {
return {
success: false,
error: `Failed to copy run ${runId} to production: ${error instanceof Error ? error.message : "Unknown error"}.`,
}
}
}

59
apps/web-evals/src/lib/formatters.ts
View file

@ -1,59 +0,0 @@
const formatter = new Intl.NumberFormat("en-US", {
style: "currency",
currency: "USD",
})
export const formatCurrency = (amount: number) => formatter.format(amount)
export const formatDuration = (durationMs: number) => {
const seconds = Math.floor(durationMs / 1000)
const hours = Math.floor(seconds / 3600)
const minutes = Math.floor((seconds % 3600) / 60)
const remainingSeconds = seconds % 60
// Format as H:MM:SS
const mm = minutes.toString().padStart(2, "0")
const ss = remainingSeconds.toString().padStart(2, "0")
return `${hours}:${mm}:${ss}`
}
export const formatTokens = (tokens: number) => {
if (tokens < 1000) {
return tokens.toString()
}
if (tokens < 1000000) {
// No decimal for thousands (e.g., 72k not 72.5k)
const rounded = Math.round(tokens / 1000)
// If rounding crosses the boundary to 1000k, show as 1.0M instead
if (rounded >= 1000) {
return "1.0M"
}
return `${rounded}k`
}
if (tokens < 1000000000) {
// Keep decimal for millions (e.g., 3.2M)
const rounded = Math.round(tokens / 100000) / 10 // Round to 1 decimal
// If rounding crosses the boundary to 1000M, show as 1.0B instead
if (rounded >= 1000) {
return "1.0B"
}
return `${rounded.toFixed(1)}M`
}
return `${(tokens / 1000000000).toFixed(1)}B`
}
export const formatToolUsageSuccessRate = (usage: { attempts: number; failures: number }) =>
usage.attempts === 0 ? "0%" : `${Math.round(((usage.attempts - usage.failures) / usage.attempts) * 100)}%`
export const formatDateTime = (date: Date) => {
return new Intl.DateTimeFormat("en-US", {
month: "short",
day: "numeric",
hour: "numeric",
minute: "2-digit",
hour12: true,
}).format(date)
}

20
apps/web-evals/src/lib/normalize-create-run.ts
View file

@ -1,20 +0,0 @@
import type { CreateRun } from "./schemas"
/**
* The New Run UI keeps exercise selection in component state.
* This normalizer ensures we submit the *visible/selected* exercises when suite is partial.
*/
export function normalizeCreateRunForSubmit(
values: CreateRun,
selectedExercises: string[],
suiteOverride?: CreateRun["suite"],
): CreateRun {
const suite = suiteOverride ?? values.suite
const normalizedSelectedExercises = Array.from(new Set(selectedExercises))
return {
...values,
suite,
exercises: suite === "partial" ? normalizedSelectedExercises : [],
}
}

Some files were not shown because too many files have changed in this diff Show more