diff --git a/apps/web-roo-code/src/app/evals/methodology/methodology-content.tsx b/apps/web-roo-code/src/app/evals/methodology/methodology-content.tsx index 654b1740fe..852356c6c8 100644 --- a/apps/web-roo-code/src/app/evals/methodology/methodology-content.tsx +++ b/apps/web-roo-code/src/app/evals/methodology/methodology-content.tsx @@ -1,931 +1,517 @@ "use client" import { motion } from "framer-motion" -import { - ArrowRight, - FlaskConical, - Code, - AlertTriangle, - BarChart3, - Terminal, - ExternalLink, - CheckCircle2, - Beaker, - Timer, - DollarSign, - Zap, - Trophy, - Scale, -} from "lucide-react" +import { AlertTriangle, CheckCircle2, Scale, Timer, Zap } from "lucide-react" import Link from "next/link" -// ── Framer Motion Variants ────────────────────────────────────────────────── - const containerVariants = { hidden: { opacity: 0 }, visible: { opacity: 1, transition: { staggerChildren: 0.12, - delayChildren: 0.1, + delayChildren: 0.08, }, }, } const fadeUpVariants = { - hidden: { opacity: 0, y: 20 }, + hidden: { opacity: 0, y: 18 }, visible: { opacity: 1, y: 0, - transition: { - duration: 0.6, - ease: [0.21, 0.45, 0.27, 0.9] as const, - }, + transition: { duration: 0.55, ease: [0.21, 0.45, 0.27, 0.9] as const }, }, } const backgroundVariants = { hidden: { opacity: 0 }, - visible: { - opacity: 1, - transition: { - duration: 1.2, - ease: "easeOut" as const, - }, - }, + visible: { opacity: 1, transition: { duration: 1.1, ease: "easeOut" as const } }, } -const cardVariants = { - hidden: { opacity: 0, y: 30 }, - visible: { - opacity: 1, - y: 0, - transition: { - duration: 0.6, - ease: [0.21, 0.45, 0.27, 0.9] as const, - }, - }, -} - -// ── Section Number Marker ─────────────────────────────────────────────────── - -function SectionNumber({ num }: { num: string }) { +function InlineArrow() { return ( - - {num} + + → ) } -// ── Process Step Icon ─────────────────────────────────────────────────────── +function Chip({ icon: Icon, label }: { icon: React.ComponentType<{ className?: string }>; label: string }) { + return ( + + + {label} + + ) +} -function ProcessStep({ +function Callout({ icon: Icon, - label, - isLast, + title, + body, + tone = "neutral", }: { icon: React.ComponentType<{ className?: string }> - label: string - isLast?: boolean + title: string + body: string + tone?: "neutral" | "warning" | "success" }) { + const toneClasses = + tone === "warning" + ? "border-amber-500/20 bg-amber-500/5" + : tone === "success" + ? "border-emerald-500/20 bg-emerald-500/5" + : "border-border/50 bg-card/40" + return ( -
-
-
+
+
+
- {label} -
- {!isLast && ( -
-
- +
+

{title}

+

{body}

- )} +
) } -// ── Language Card ─────────────────────────────────────────────────────────── - -function LanguageCard({ name, color }: { name: string; color: string }) { +function Step({ num, title, body }: { num: string; title: string; body: string }) { return ( -
-
- {name.slice(0, 2).toUpperCase()} +
+ + {num} + +
+

{title}

+

{body}

- {name}
) } -// ── Scoring Bar Component ─────────────────────────────────────────────────── - -function ScoringBar({ - label, - icon: Icon, - color, - bgColor, - weight, - description, -}: { - label: string - icon: React.ComponentType<{ className?: string }> - color: string - bgColor: string - weight: number - description: string -}) { +function SmallLink({ href, label }: { href: string; label: string }) { return ( -
-
- -
-
-
-

{label}

- {weight}% -
-

{description}

-
- -
-
-
+ + {label} + ) } -// ── Main Content Component ────────────────────────────────────────────────── - export function MethodologyContent() { return ( <> - {/* ════════════════════════════════════════════════════════════════ - HERO SECTION - ════════════════════════════════════════════════════════════════ */} -
- {/* Atmospheric blur background */} + {/* Hero */} +
+ variants={backgroundVariants} + className="absolute inset-0"> +
-
-
-
+
+
+
+
-
+
- {/* Breadcrumb */} - - - Evals - - / - - Build with Roo Code Cloud - - / - Methodology - + variants={containerVariants} + className="mx-auto max-w-6xl"> +
+
+ + + Evals + + / + + Recommendations + + / + Methodology + - {/* Heading */} - - How We Run{" "} - - Evals - - + + Read this before you compare models + - {/* Subtitle */} - - Same exercises, same environment, same scoring for every model. Every step is documented and - every eval run is reproducible. - + + How we run evals + - {/* Pill badge links */} - - - - View recommendations - - - - - Raw eval data - - - + + We keep tasks, environment, and scoring constant across models. Use our results to + pick a default for a specific objective, then validate in your repo. + + + + + + + + + + + View recommendations + + + Raw eval data + + + + + Jump to: + + What we hold constant + + + Scoring + + + Limitations + + +
+ + +
+
+ +
+

+ Methodology at a glance +

+

+ Comparable results, not universal truth +

+ +
+
+

+ We hold constant +

+

+ Same exercises, same tools, same time limit, same scoring. +

+
+
+

+ We measure +

+

+ Pass rate, latency, and cost signals across multiple languages. +

+
+
+

+ We recommend +

+

+ A default model and agent lineup for an objective. It’s a + baseline, not a guarantee. +

+
+
+ +
+ + Objective-first + + + Optimized for: Quality / Speed / Cost + + + Validate in your repo + +
+
+
+ +
- {/* ════════════════════════════════════════════════════════════════ - SECTION 01: THE INTERVIEW PROCESS - ════════════════════════════════════════════════════════════════ */} - -
- - - - - - The Eval Process - - + {/* Body */} +
+
-

- We don't test models in isolation. We test them as they work inside Roo Code. Each - model gets the same exercises, same time limit, same tools. We measure what matters. -

- - - {/* Process flow */} - - - - - - - - {/* Key principles */} - - {[ - { - title: "Identical Environment", - desc: "Docker container with VS Code, Roo Code extension, and a fresh workspace per exercise.", - }, - { - title: "No Cherry-Picking", - desc: "Every model gets the exact same eval run. No curated demos, no special treatment.", - }, - { - title: "Real Metrics", - desc: "Does it pass the tests? How much does it cost? How fast does it deliver?", - }, - ].map((item) => ( - -

{item.title}

-

{item.desc}

- - ))} - -
- - - {/* ════════════════════════════════════════════════════════════════ - SECTION 02: THE EVAL SUITE - ════════════════════════════════════════════════════════════════ */} - - {/* Subtle background glow */} - -
-
-
- - -
- - - - - - The Eval Suite - - - - Hundreds of coding exercises across 5 languages and{" "} - 3 difficulty tiers. From single-file fixes to - complex architecture decisions. - - - {/* Language cards */} - - - - - - - - - - - - - - - - - - - {/* Difficulty tiers */} - -

- Difficulty Tiers -

-
- {/* Easy */} -
-
- E -
-
-

Easy

-

- Single-file fixes, straightforward implementations, basic debugging -

-
-
- - 90–95% - -
- -
-
-
- {/* Medium */} -
-
- M -
-
-

Medium

-

- Multi-file changes, refactoring, cross-file understanding -

-
-
- - 60–80% - -
- -
-
-
- {/* Hard */} -
-
- H -
-
-

Hard

-

- Architecture decisions, ambiguous requirements, complex system design -

-
-
- - 30–50% - -
- -
-
-
-
- -
- - - {/* ════════════════════════════════════════════════════════════════ - SECTION 03: ENGINEER ROLES - ════════════════════════════════════════════════════════════════ */} - -
- - - - - - Engineer Roles - - - -

- Each role represents a different engineering seniority level. We test models against - exercises matched to that role's complexity, then score using role-specific weights. -

- - - {/* How weights differ */} - - -

Different Roles, Different Weights

-

- Each role has its own scoring weights. A model that's great for simple tasks might - not rank for architecture decisions. -

- - -

Matched Exercises

-

- Budget roles get simpler exercises. Complex roles get harder ones. The difficulty and - scoring shift together so recommendations stay relevant. -

- - - - {/* Budget vs Complex comparison */} - -

- How Scoring Weights Shift -

-
- {/* Budget roles */} -
-
- -
-
-

Budget Roles

-

- Cost and speed matter most. Simpler exercises where many models succeed, so - efficiency breaks the tie. -

-
-
-
-
-
-
-
-
-
- Success - Quality - Cost ↑ - Speed -
-
-
- {/* Complex roles */} -
-
- -
-
-

Complex Roles

-

- Reasoning quality and success rate matter most. Harder exercises where only the - best models deliver. -

-
-
-
-
-
-
-
-
-
- - Success ↑ - - - Quality ↑ - - Cost - Speed -
-
-
-
- - - {/* Link to roles page */} - - - Browse all engineer roles - - - -
- - - {/* ════════════════════════════════════════════════════════════════ - SECTION 04: SCORING - ════════════════════════════════════════════════════════════════ */} - - {/* Background glow */} - -
-
-
- - -
- - - - - - Scoring - - - - Each model receives a composite score, a weighted - sum of four dimensions normalized to a 0–100 scale. - - - {/* Scoring formula components */} - - - - - - - - - - - - - - - - {/* Tier classification */} - -

- Recommendation Tiers -

-

- Composite scores are mapped to recommendation tiers: -

- - - - {/* Best */} - -
- ≥85 -
-
- - Best - -

Top Performer

-

Highly recommended for this role.

-
- - - {/* Recommended */} - -
- 70–84 -
-
- - Recommended - -

Solid Choice

-

- Reliable for most tasks at this level. -

-
- - - {/* Situational */} - -
- 50–69 -
-
- - Situational - -

Usable with Caveats

-

May struggle in specific areas.

-
- - - {/* Not Recommended */} - -
- <50 -
-
- - Not Recommended - -

High Failure Rate

-

Not suitable for this role.

-
- - - - - Per-language breakdowns reveal where each model excels or struggles. A model might score well - overall but underperform in Rust, or dominate in Python but lag in Go. - -
- - - {/* ════════════════════════════════════════════════════════════════ - SECTION 05: RUN YOUR OWN EVALS - ════════════════════════════════════════════════════════════════ */} - -
- - - - - - Run Your Own Evals - - - - Our evaluation framework is fully open source. Run the exact same evals on your own - infrastructure, with your own API keys, against any model. - - - {/* Terminal card */} - - {/* Terminal header */} -
-
-
-
-
-
-
- - terminal -
-
- {/* Terminal body */} -
-
- ${" "} - git clone{" "} - https://github.com/RooCodeInc/Roo-Code-Evals.git -
-
- $ cd{" "} - Roo-Code-Evals -
-
- ${" "} - # Follow the README for setup instructions -
-
- - - {/* GitHub link */} - - - - - - View on GitHub - - - -
- - - {/* ════════════════════════════════════════════════════════════════ - SECTION 06: LIMITATIONS - ════════════════════════════════════════════════════════════════ */} - -
- - - - - - Limitations - - - - Every evaluation has blind spots. These are ours. - - - - {[ - { - title: "Single test environment", - description: - "All evals run in Docker + VS Code. Results may differ in other IDEs or environments.", - }, - { - title: "Expanding exercise coverage", - description: - "Hundreds of exercises, but the suite is continuously growing. Some niche patterns may be underrepresented.", - }, - { - title: "API changes affect results", - description: - "Providers update their models. A model that scored well last month may behave differently after an update.", - }, - { - title: "Point-in-time snapshots", - description: - 'Each eval run captures performance at a specific point. We re-run regularly; check the "last updated" date.', - }, - ].map((item) => ( - - -
-

{item.title}

-

- {item.description} -

-
- - ))} - -
- - - {/* ════════════════════════════════════════════════════════════════ - BOTTOM NAVIGATION - ════════════════════════════════════════════════════════════════ */} -
-
- - - Ready to see the results? - - - - - View recommendations - - - - - Raw eval data - - + viewport={{ once: true, margin: "-120px" }} + variants={containerVariants} + className="mx-auto grid max-w-6xl grid-cols-1 gap-10 lg:grid-cols-12"> + {/* Left rail */} + +
+

+ How to read results responsibly +

+

+ Evals help you pick a better default. They don’t predict how a model behaves + in your repo, with your tests, tooling, and constraints. +

+ +
+ + +
+ + + +
+

+ Quick definitions +

+
    +
  • + + + Objective: + the workflow you want to ship (for example,{" "} + Issue → PR). + +
    • +
  • + + + + Optimized for + + : the tradeoff you care about most ( + Quality,{" "} + Speed,{" "} + Cost). + +
    • +
  • + + + Pass rate: + percent of exercises a model completes within the limit. + +
    • +
+
+
+ + {/* Main column */} +
+ +

+ Methodology +

+

+ How to use these evals +

+

+ The recommendations page is organized around what you’re trying to ship. You + pick the objective and tradeoff. We show the best default setup based on the signal + we have. +

+ + +
+ + + + + + + + + +
+ + +

+ Scoring and signals +

+

+ What we measure +

+
+ + + + +
+ + + +

+ Tradeoffs +

+

+ Quality, speed, cost: pick one to optimize +

+
+

+ Choosing an optimization mode is how you tell the system what matters most for + your objective. If you care about merge confidence, optimize for Quality. If you + care about throughput, Speed and Cost matter. +

+

+ When two models are close on pass rate, the most practical tie-breakers are + latency and $/task. +

+
+ + + +

+ Limitations +

+

+ What these evals don’t tell you +

+
+

+ A model can score well on this suite and still struggle in your repo because + your stack, tests, dependencies, and CI constraints are different. +

+

+ The right move is to treat our results as a starting point, then run your + objective end-to-end in Roo Code Cloud and inspect the PR output. +

+
+ + + +

+ Links +

+
+ + +
+
+

+ If something feels off in the recommendations, that’s a signal too. The + fastest path is to run your objective in Roo Code Cloud and compare the PR + output. +

+
+ +
diff --git a/apps/web-roo-code/src/app/evals/methodology/page.tsx b/apps/web-roo-code/src/app/evals/methodology/page.tsx index 039d7e6fd7..e723cabd67 100644 --- a/apps/web-roo-code/src/app/evals/methodology/page.tsx +++ b/apps/web-roo-code/src/app/evals/methodology/page.tsx @@ -7,9 +7,10 @@ import { MethodologyContent } from "./methodology-content" // ── SEO Metadata ──────────────────────────────────────────────────────────── -const TITLE = "Methodology | Roo Code Evals" -const DESCRIPTION = "Our methodology for evaluating AI coding models. Transparent, reproducible, evidence-based." -const OG_DESCRIPTION = "Our methodology for evaluating AI coding models" +const TITLE = "Methodology | Roo Code Cloud Evals" +const DESCRIPTION = + "How we run Roo Code Cloud evals and how to interpret outcomes-first recommendations. Same tasks, same limits, clear tradeoffs." +const OG_DESCRIPTION = "How we run Roo Code Cloud evals" const PATH = "/evals/methodology" export const metadata: Metadata = { diff --git a/apps/web-roo-code/src/app/evals/workers-v2/[roleId]/compare/page.tsx b/apps/web-roo-code/src/app/evals/workers-v2/[roleId]/compare/page.tsx index 2176eaae88..ceb688d8a7 100644 --- a/apps/web-roo-code/src/app/evals/workers-v2/[roleId]/compare/page.tsx +++ b/apps/web-roo-code/src/app/evals/workers-v2/[roleId]/compare/page.tsx @@ -1,83 +1,14 @@ -import { notFound } from "next/navigation" -import type { Metadata } from "next" +import { permanentRedirect } from "next/navigation" -import { SEO } from "@/lib/seo" -import { ogImageUrl } from "@/lib/og" -import { getEngineerRole, getRoleRecommendation } from "@/lib/mock-recommendations" +import { buildQueryString, type RedirectSearchParams } from "../../../_redirect-utils" -import { ComparisonChart } from "../../../workers/[roleId]/compare/comparison-chart" - -type PageProps = { params: Promise<{ roleId: string }> } - -export async function generateMetadata({ params }: PageProps): Promise { - const { roleId } = await params - const role = getEngineerRole(roleId) - - if (!role) { - return { - title: "Role Not Found | Roo Code Evals", - description: "The requested role was not found.", - } - } - - const title = `Compare Models — ${role.name} (V2 Preview) | Roo Code Evals` - const description = `Outcome-first comparison of AI models for ${role.name}. Compare composite score, success rate, cost efficiency, and speed.` - const ogDescription = `Compare Models — ${role.name} (V2 Preview)` - const path = `/evals/workers-v2/${roleId}/compare` - - return { - title, - description, - alternates: { - canonical: `${SEO.url}${path}`, - }, - openGraph: { - title, - description, - url: `${SEO.url}${path}`, - siteName: SEO.name, - images: [ - { - url: ogImageUrl(title, ogDescription), - width: 1200, - height: 630, - alt: title, - }, - ], - locale: SEO.locale, - type: "website", - }, - twitter: { - card: SEO.twitterCard, - title, - description, - images: [ogImageUrl(title, ogDescription)], - }, - keywords: [ - ...SEO.keywords, - "AI coding", - "model comparison", - "coding evals", - role.name.toLowerCase(), - "outcome-first", - ], - } +type PageProps = { + params: Promise<{ roleId: string }> + searchParams?: Promise } -export default async function WorkersV2ComparePage({ params }: PageProps) { +export default async function WorkersV2ComparePage({ params, searchParams }: PageProps) { const { roleId } = await params - const recommendation = getRoleRecommendation(roleId) - - if (!recommendation) { - notFound() - } - - return ( - - ) + const sp = (await searchParams) ?? {} + permanentRedirect(`/evals/workers/${roleId}/compare${buildQueryString(sp)}`) } diff --git a/apps/web-roo-code/src/app/evals/workers-v2/[roleId]/page.tsx b/apps/web-roo-code/src/app/evals/workers-v2/[roleId]/page.tsx index 8afef01a58..affc235515 100644 --- a/apps/web-roo-code/src/app/evals/workers-v2/[roleId]/page.tsx +++ b/apps/web-roo-code/src/app/evals/workers-v2/[roleId]/page.tsx @@ -1,100 +1,14 @@ -import { notFound } from "next/navigation" -import type { Metadata } from "next" +import { permanentRedirect } from "next/navigation" -import { SEO } from "@/lib/seo" -import { ogImageUrl } from "@/lib/og" -import { getRoleRecommendation, getCloudSetupUrl } from "@/lib/mock-recommendations" +import { buildQueryString, type RedirectSearchParams } from "../../_redirect-utils" -import { CandidatesContent } from "../../workers/[roleId]/candidates-content" - -type PageProps = { params: Promise<{ roleId: string }> } - -export async function generateMetadata({ params }: PageProps): Promise { - const { roleId } = await params - const recommendation = getRoleRecommendation(roleId) - - if (!recommendation) { - return { - title: "Role Not Found | Roo Code Evals", - description: "The requested role was not found.", - } - } - - const { role } = recommendation - const title = `${role.name} — Recommended Models (V2 Preview) | Roo Code Evals` - const description = `Outcome-first recommendations for ${role.name}. Compare models by success rate, cost, and speed across 5 languages.` - const ogDescription = `${role.name} — Recommended Models (V2 Preview)` - const path = `/evals/workers-v2/${roleId}` - - return { - title, - description, - alternates: { - canonical: `${SEO.url}${path}`, - }, - openGraph: { - title, - description, - url: `${SEO.url}${path}`, - siteName: SEO.name, - images: [ - { - url: ogImageUrl(title, ogDescription), - width: 1200, - height: 630, - alt: title, - }, - ], - locale: SEO.locale, - type: "website", - }, - twitter: { - card: SEO.twitterCard, - title, - description, - images: [ogImageUrl(title, ogDescription)], - }, - keywords: [ - ...SEO.keywords, - "AI coding", - "coding agents", - "model recommendations", - "coding evals", - role.name.toLowerCase(), - "outcome-first", - ], - } +type PageProps = { + params: Promise<{ roleId: string }> + searchParams?: Promise } -export default async function WorkersV2RolePage({ params }: PageProps) { +export default async function WorkersV2RolePage({ params, searchParams }: PageProps) { const { roleId } = await params - const recommendation = getRoleRecommendation(roleId) - - if (!recommendation) { - notFound() - } - - const { role, best, budgetHire, speedHire, allCandidates, totalEvalRuns, totalExercises, lastUpdated } = - recommendation - - const cloudUrls: Record = {} - for (const candidate of allCandidates) { - cloudUrls[candidate.modelId] = getCloudSetupUrl(candidate) - } - - return ( - - ) + const sp = (await searchParams) ?? {} + permanentRedirect(`/evals/workers/${roleId}${buildQueryString(sp)}`) } diff --git a/apps/web-roo-code/src/app/evals/workers-v2/_redirect-utils.ts b/apps/web-roo-code/src/app/evals/workers-v2/_redirect-utils.ts new file mode 100644 index 0000000000..a8884e591a --- /dev/null +++ b/apps/web-roo-code/src/app/evals/workers-v2/_redirect-utils.ts @@ -0,0 +1,11 @@ +export type RedirectSearchParams = Record + +export function buildQueryString(searchParams: RedirectSearchParams): string { + const params = new URLSearchParams() + for (const [key, value] of Object.entries(searchParams)) { + if (typeof value === "string") params.set(key, value) + else if (Array.isArray(value)) value.forEach((v) => params.append(key, v)) + } + const qs = params.toString() + return qs ? `?${qs}` : "" +} diff --git a/apps/web-roo-code/src/app/evals/workers-v2/page.tsx b/apps/web-roo-code/src/app/evals/workers-v2/page.tsx index 5196214f68..e8559e09ca 100644 --- a/apps/web-roo-code/src/app/evals/workers-v2/page.tsx +++ b/apps/web-roo-code/src/app/evals/workers-v2/page.tsx @@ -1,91 +1,12 @@ -import type { Metadata } from "next" -import { Fraunces, IBM_Plex_Sans } from "next/font/google" +import { permanentRedirect } from "next/navigation" -import { SEO } from "@/lib/seo" -import { ogImageUrl } from "@/lib/og" -import { getEngineerRoles, getAllRecommendations } from "@/lib/mock-recommendations" +import { buildQueryString, type RedirectSearchParams } from "./_redirect-utils" -import { WorkersContent } from "../workers/workers-content" - -const TITLE = "Build with Roo Code Cloud (V2 Preview) | Roo Code Evals" -const DESCRIPTION = - "Outcome-first, eval-backed recommendations for shipping production code. Start from what you need to ship and pick a setup." -const OG_DESCRIPTION = "Outcome-first recommendations for shipping production code" -const PATH = "/evals/workers-v2" - -const display = Fraunces({ subsets: ["latin"], variable: "--font-display" }) -const body = IBM_Plex_Sans({ subsets: ["latin"], weight: ["400", "500", "600"], variable: "--font-body" }) - -export const metadata: Metadata = { - title: TITLE, - description: DESCRIPTION, - alternates: { - canonical: `${SEO.url}${PATH}`, - }, - openGraph: { - title: TITLE, - description: DESCRIPTION, - url: `${SEO.url}${PATH}`, - siteName: SEO.name, - images: [ - { - url: ogImageUrl(TITLE, OG_DESCRIPTION), - width: 1200, - height: 630, - alt: TITLE, - }, - ], - locale: SEO.locale, - type: "website", - }, - twitter: { - card: SEO.twitterCard, - title: TITLE, - description: DESCRIPTION, - images: [ogImageUrl(TITLE, OG_DESCRIPTION)], - }, - keywords: [ - ...SEO.keywords, - "AI coding", - "coding agents", - "roo code cloud", - "model recommendations", - "coding evals", - "shipping code", - "prototype", - "outcome-first", - ], +type PageProps = { + searchParams?: Promise } -export default function WorkersV2Page() { - const roles = getEngineerRoles() - const recommendations = getAllRecommendations() - - const totalEvalRuns = recommendations.reduce((sum, recommendation) => sum + recommendation.totalEvalRuns, 0) - const totalExercises = recommendations.reduce((sum, recommendation) => sum + recommendation.totalExercises, 0) - const uniqueModels = new Set( - recommendations.flatMap((recommendation) => recommendation.allCandidates.map((candidate) => candidate.modelId)), - ) - const totalModels = uniqueModels.size - const lastUpdated = recommendations - .map((recommendation) => recommendation.lastUpdated) - .sort() - .pop() - - return ( -
- -
- ) +export default async function WorkersV2Page({ searchParams }: PageProps) { + const sp = (await searchParams) ?? {} + permanentRedirect(`/evals/workers${buildQueryString(sp)}`) } diff --git a/apps/web-roo-code/src/app/evals/workers/[roleId]/candidates-content.tsx b/apps/web-roo-code/src/app/evals/workers/[roleId]/candidates-content.tsx index 9d48a2b095..0b1105650f 100644 --- a/apps/web-roo-code/src/app/evals/workers/[roleId]/candidates-content.tsx +++ b/apps/web-roo-code/src/app/evals/workers/[roleId]/candidates-content.tsx @@ -665,8 +665,6 @@ export function CandidatesContent({ const searchParams = useSearchParams() const theme = ROLE_THEMES[roleId] ?? DEFAULT_THEME const IconComponent = ICON_MAP[role.icon] ?? Code - const alternateWorkersRootPath = workersRootPath === "/evals/workers-v2" ? "/evals/workers" : "/evals/workers-v2" - const alternateVersionLabel = workersRootPath === "/evals/workers-v2" ? "View baseline" : "View V2 preview" const setupQuery = (() => { const outcome = searchParams.get("outcome") if (!outcome) return "" @@ -773,12 +771,6 @@ export function CandidatesContent({ Methodology -
- - {alternateVersionLabel} - {/* Strengths + Trade-offs grid */} diff --git a/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/comparison-chart.tsx b/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/comparison-chart.tsx index d03f83fe67..8c005cdbd5 100644 --- a/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/comparison-chart.tsx +++ b/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/comparison-chart.tsx @@ -446,8 +446,6 @@ export function ComparisonChart({ const searchParams = useSearchParams() const { allCandidates } = recommendation const theme = ROLE_THEMES[roleId] ?? DEFAULT_THEME - const alternateWorkersRootPath = workersRootPath === "/evals/workers-v2" ? "/evals/workers" : "/evals/workers-v2" - const alternateVersionLabel = workersRootPath === "/evals/workers-v2" ? "View baseline" : "View V2 preview" const setupQuery = (() => { const outcome = searchParams.get("outcome") if (!outcome) return "" @@ -593,12 +591,6 @@ export function ComparisonChart({ / Compare Models - / - - {alternateVersionLabel} - {/* Title row */} diff --git a/apps/web-roo-code/src/app/evals/workers/[roleId]/copy-settings-button.tsx b/apps/web-roo-code/src/app/evals/workers/[roleId]/copy-settings-button.tsx index 06adacff5a..b330ba7653 100644 --- a/apps/web-roo-code/src/app/evals/workers/[roleId]/copy-settings-button.tsx +++ b/apps/web-roo-code/src/app/evals/workers/[roleId]/copy-settings-button.tsx @@ -34,7 +34,7 @@ export function CopySettingsButton({ settings }: CopySettingsButtonProps) { ) : ( <> - 🔧 Configure Extension + Copy Roo Code Cloud Config )} diff --git a/apps/web-roo-code/src/app/evals/workers/page.tsx b/apps/web-roo-code/src/app/evals/workers/page.tsx index a2b95a1bd8..718011b56d 100644 --- a/apps/web-roo-code/src/app/evals/workers/page.tsx +++ b/apps/web-roo-code/src/app/evals/workers/page.tsx @@ -1,4 +1,5 @@ import type { Metadata } from "next" +import { Fraunces, IBM_Plex_Sans } from "next/font/google" import { SEO } from "@/lib/seo" import { ogImageUrl } from "@/lib/og" @@ -10,10 +11,13 @@ import { WorkersContent } from "./workers-content" const TITLE = "Build with Roo Code Cloud | Roo Code Evals" const DESCRIPTION = - "Eval-backed model recommendations for shipping production code. Pick a setup based on the work you're doing: single-file fixes, multi-file changes, review, and autonomous runs." -const OG_DESCRIPTION = "Eval-backed model recommendations for shipping production code" + "Outcome-first, eval-backed recommendations for shipping production code. Start from your objective and pick a tradeoff." +const OG_DESCRIPTION = "Outcome-first recommendations for shipping production code" const PATH = "/evals/workers" +const display = Fraunces({ subsets: ["latin"], variable: "--font-display" }) +const body = IBM_Plex_Sans({ subsets: ["latin"], weight: ["400", "500", "600"], variable: "--font-body" }) + export const metadata: Metadata = { title: TITLE, description: DESCRIPTION, @@ -62,11 +66,11 @@ export default function WorkersPage() { const recommendations = getAllRecommendations() // Aggregate totals - const totalEvalRuns = recommendations.reduce((sum, r) => sum + r.totalEvalRuns, 0) - const totalExercises = recommendations.reduce((sum, r) => sum + r.totalExercises, 0) - - // Unique model count across all roles - const uniqueModels = new Set(recommendations.flatMap((r) => r.allCandidates.map((c) => c.modelId))) + const totalEvalRuns = recommendations.reduce((sum, recommendation) => sum + recommendation.totalEvalRuns, 0) + const totalExercises = recommendations.reduce((sum, recommendation) => sum + recommendation.totalExercises, 0) + const uniqueModels = new Set( + recommendations.flatMap((recommendation) => recommendation.allCandidates.map((candidate) => candidate.modelId)), + ) const totalModels = uniqueModels.size const lastUpdated = recommendations @@ -75,17 +79,16 @@ export default function WorkersPage() { .pop() return ( - +
+ +
) } diff --git a/apps/web-roo-code/src/app/evals/workers/workers-content.tsx b/apps/web-roo-code/src/app/evals/workers/workers-content.tsx index e5c770ea6d..9189190120 100644 --- a/apps/web-roo-code/src/app/evals/workers/workers-content.tsx +++ b/apps/web-roo-code/src/app/evals/workers/workers-content.tsx @@ -2,23 +2,7 @@ import { useCallback, useMemo } from "react" import { motion } from "framer-motion" -import { - Code, - GitBranch, - Building2, - Search, - Bot, - ArrowRight, - ChevronDown, - CheckCircle2, - AlertTriangle, - Users, - FlaskConical, - Beaker, - Globe, - TrendingUp, -} from "lucide-react" -import type { LucideIcon } from "lucide-react" +import { ArrowRight, FlaskConical, Beaker } from "lucide-react" import Link from "next/link" import { usePathname, useRouter, useSearchParams } from "next/navigation" import { ScatterChart, Scatter, XAxis, YAxis, ZAxis, Tooltip, ResponsiveContainer, Cell, ReferenceLine } from "recharts" @@ -26,120 +10,7 @@ import { ScatterChart, Scatter, XAxis, YAxis, ZAxis, Tooltip, ResponsiveContaine import type { EngineerRole, RoleRecommendation } from "@/lib/mock-recommendations" import { TASKS_PER_DAY, MODEL_TIMELINE } from "@/lib/mock-recommendations" import { EVAL_OUTCOMES, isEvalOutcomeId, type EvalOutcomeId } from "@/lib/eval-outcomes" - -// ── Icon Mapping ──────────────────────────────────────────────────────────── - -const ICON_MAP: Record = { - Code, - GitBranch, - Building2, - Search, - Bot, -} - -// ── Color Themes per Role ─────────────────────────────────────────────────── - -type RoleTheme = { - accent: string - accentLight: string - accentDark: string - iconBg: string - iconText: string - badgeBg: string - badgeText: string - borderHover: string - shadowHover: string - buttonBg: string - buttonHover: string - glowColor: string - dotColor: string - strengthColor: string -} - -const ROLE_THEMES: Record = { - junior: { - accent: "emerald", - accentLight: "text-emerald-600", - accentDark: "dark:text-emerald-400", - iconBg: "bg-emerald-100 dark:bg-emerald-900/30", - iconText: "text-emerald-700 dark:text-emerald-300", - badgeBg: "bg-emerald-100 dark:bg-emerald-900/30", - badgeText: "text-emerald-700 dark:text-emerald-300", - borderHover: "hover:border-emerald-500/40 dark:hover:border-emerald-400/30", - shadowHover: "hover:shadow-emerald-500/10 dark:hover:shadow-emerald-400/10", - buttonBg: "bg-emerald-600 dark:bg-emerald-600", - buttonHover: "hover:bg-emerald-700 dark:hover:bg-emerald-500", - glowColor: "bg-emerald-500/8 dark:bg-emerald-600/15", - dotColor: "bg-emerald-500", - strengthColor: "text-emerald-600 dark:text-emerald-400", - }, - senior: { - accent: "blue", - accentLight: "text-blue-600", - accentDark: "dark:text-blue-400", - iconBg: "bg-blue-100 dark:bg-blue-900/30", - iconText: "text-blue-700 dark:text-blue-300", - badgeBg: "bg-blue-100 dark:bg-blue-900/30", - badgeText: "text-blue-700 dark:text-blue-300", - borderHover: "hover:border-blue-500/40 dark:hover:border-blue-400/30", - shadowHover: "hover:shadow-blue-500/10 dark:hover:shadow-blue-400/10", - buttonBg: "bg-blue-600 dark:bg-blue-600", - buttonHover: "hover:bg-blue-700 dark:hover:bg-blue-500", - glowColor: "bg-blue-500/8 dark:bg-blue-600/15", - dotColor: "bg-blue-500", - strengthColor: "text-blue-600 dark:text-blue-400", - }, - staff: { - accent: "amber", - accentLight: "text-amber-600", - accentDark: "dark:text-amber-400", - iconBg: "bg-amber-100 dark:bg-amber-900/30", - iconText: "text-amber-700 dark:text-amber-300", - badgeBg: "bg-amber-100 dark:bg-amber-900/30", - badgeText: "text-amber-700 dark:text-amber-300", - borderHover: "hover:border-amber-500/40 dark:hover:border-amber-400/30", - shadowHover: "hover:shadow-amber-500/10 dark:hover:shadow-amber-400/10", - buttonBg: "bg-amber-600 dark:bg-amber-600", - buttonHover: "hover:bg-amber-700 dark:hover:bg-amber-500", - glowColor: "bg-amber-500/8 dark:bg-amber-600/15", - dotColor: "bg-amber-500", - strengthColor: "text-amber-600 dark:text-amber-400", - }, - reviewer: { - accent: "violet", - accentLight: "text-violet-600", - accentDark: "dark:text-violet-400", - iconBg: "bg-violet-100 dark:bg-violet-900/30", - iconText: "text-violet-700 dark:text-violet-300", - badgeBg: "bg-violet-100 dark:bg-violet-900/30", - badgeText: "text-violet-700 dark:text-violet-300", - borderHover: "hover:border-violet-500/40 dark:hover:border-violet-400/30", - shadowHover: "hover:shadow-violet-500/10 dark:hover:shadow-violet-400/10", - buttonBg: "bg-violet-600 dark:bg-violet-600", - buttonHover: "hover:bg-violet-700 dark:hover:bg-violet-500", - glowColor: "bg-violet-500/8 dark:bg-violet-600/15", - dotColor: "bg-violet-500", - strengthColor: "text-violet-600 dark:text-violet-400", - }, - autonomous: { - accent: "cyan", - accentLight: "text-cyan-600", - accentDark: "dark:text-cyan-400", - iconBg: "bg-cyan-100 dark:bg-cyan-900/30", - iconText: "text-cyan-700 dark:text-cyan-300", - badgeBg: "bg-cyan-100 dark:bg-cyan-900/30", - badgeText: "text-cyan-700 dark:text-cyan-300", - borderHover: "hover:border-cyan-500/40 dark:hover:border-cyan-400/30", - shadowHover: "hover:shadow-cyan-500/10 dark:hover:shadow-cyan-400/10", - buttonBg: "bg-cyan-600 dark:bg-cyan-600", - buttonHover: "hover:bg-cyan-700 dark:hover:bg-cyan-500", - glowColor: "bg-cyan-500/8 dark:bg-cyan-600/15", - dotColor: "bg-cyan-500", - strengthColor: "text-cyan-600 dark:text-cyan-400", - }, -} - -const DEFAULT_THEME = ROLE_THEMES.senior! +import { pickObjectiveDefaultModelV1 } from "@/lib/objective-default-models-v1" // ── Outcome Layer: Optimization Modes ────────────────────────────────────── @@ -150,9 +21,9 @@ const OPTIMIZATION_MODES: Array<{ label: string description: string }> = [ - { id: "best", label: "Best", description: "Best overall quality across our eval suite." }, - { id: "fastest", label: "Fastest", description: "Lower latency per task when speed matters." }, - { id: "cost", label: "Most cost-effective", description: "Lower cost per task for high-volume work." }, + { id: "best", label: "Quality", description: "Maximize pass rate and overall quality across our eval suite." }, + { id: "fastest", label: "Speed", description: "Lower latency per task when speed matters." }, + { id: "cost", label: "Cost", description: "Lower cost per task for high-volume work." }, ] function isEvalOptimizationMode(value: string): value is EvalOptimizationMode { @@ -167,9 +38,20 @@ function getModeCandidate(rec: RoleRecommendation | undefined, mode: EvalOptimiz } function getModeLabel(mode: EvalOptimizationMode) { - if (mode === "fastest") return "Fastest" - if (mode === "cost") return "Most cost-effective" - return "Best" + if (mode === "fastest") return "Speed" + if (mode === "cost") return "Cost" + return "Quality" +} + +function formatModelIdForUi(modelId: string) { + if (modelId.startsWith("claude-opus-")) { + const rest = modelId.replace(/^claude-opus-/, "") + const parts = rest.split("-").filter(Boolean) + if (parts.length >= 2) return `Opus ${parts[0]}.${parts[1]}` + if (parts.length === 1) return `Opus ${parts[0]}` + } + if (modelId === "kimi-k2-0905") return "Kimi K2" + return modelId } // ── Framer Motion Variants ────────────────────────────────────────────────── @@ -298,16 +180,6 @@ function TimelineTooltip({ // ── Sub-Components ────────────────────────────────────────────────────────── -function StatPill({ icon: Icon, value, label }: { icon: LucideIcon; value: string; label: string }) { - return ( -
- - {value} - {label} -
- ) -} - // ── Main Content Component ────────────────────────────────────────────────── type WorkersContentProps = { @@ -318,23 +190,21 @@ type WorkersContentProps = { totalModels: number lastUpdated: string | undefined workersRootPath?: string - enableOutcomeLayer?: boolean - alternateVersionHref?: string - alternateVersionLabel?: string } +// Outcomes-first is canonical. Baseline/V1 is removed from the UI. +const ENABLE_OUTCOME_LAYER = true + export function WorkersContent({ roles, recommendations, totalEvalRuns, - totalExercises, - totalModels, + totalExercises: _totalExercises, + totalModels: _totalModels, lastUpdated, workersRootPath = "/evals/workers", - enableOutcomeLayer = false, - alternateVersionHref, - alternateVersionLabel, }: WorkersContentProps) { + const enableOutcomeLayer = ENABLE_OUTCOME_LAYER const router = useRouter() const pathname = usePathname() const searchParams = useSearchParams() @@ -374,6 +244,13 @@ export function WorkersContent({ [pathname, router, searchParams], ) + const scrollToOutcomes = useCallback(() => { + if (typeof document === "undefined") return + const el = document.getElementById("outcomes") + if (!el) return + el.scrollIntoView({ behavior: "smooth", block: "start" }) + }, []) + const recByRole = new Map(recommendations.map((r) => [r.roleId, r])) const roleById = useMemo(() => new Map(roles.map((r) => [r.id, r])), [roles]) @@ -383,19 +260,39 @@ export function WorkersContent({ }, [selectedOutcomeId]) const setupQuery = useMemo(() => { - if (!enableOutcomeLayer || !selectedOutcomeId) return "" + if (!selectedOutcomeId) return "" const params = new URLSearchParams() params.set("outcome", selectedOutcomeId) params.set("mode", selectedMode) const query = params.toString() return query ? `?${query}` : "" - }, [enableOutcomeLayer, selectedOutcomeId, selectedMode]) + }, [selectedOutcomeId, selectedMode]) + + const isProfileView = useMemo(() => { + return searchParams.get("view") === "profile" + }, [searchParams]) - const profileTitle = selectedOutcome?.builderProfile?.title ?? "Your Builder Profile" const profileDescription = selectedOutcome?.builderProfile?.description ?? "A default setup built from our eval signals. It’s a baseline, not a guarantee." const profileHowItWorks = selectedOutcome?.builderProfile?.howItWorks ?? selectedOutcome?.whyItWorks ?? [] + const objectiveDefaultModel = useMemo(() => { + if (!selectedOutcomeId) return null + return pickObjectiveDefaultModelV1(selectedOutcomeId, selectedMode) + }, [selectedOutcomeId, selectedMode]) + const objectiveDefaultModelLabel = useMemo(() => { + if (!objectiveDefaultModel?.modelId) return "—" + return formatModelIdForUi(objectiveDefaultModel.modelId) + }, [objectiveDefaultModel]) + const examplePrompt = selectedOutcome?.builderProfile?.examplePrompt ?? "" + const cloudSetupHref = useMemo(() => { + if (!selectedOutcomeId) return "/cloud-agents/setup" + const params = new URLSearchParams() + params.set("outcome", selectedOutcomeId) + params.set("mode", selectedMode) + if (examplePrompt) params.set("prompt", examplePrompt) + return `/cloud-agents/setup?${params.toString()}` + }, [examplePrompt, selectedMode, selectedOutcomeId]) const profileCapabilities = useMemo(() => { if (!selectedOutcome) return [] @@ -414,7 +311,7 @@ export function WorkersContent({ const agentCapabilities = useMemo(() => profileCapabilities.filter((c) => Boolean(c.roleId)), [profileCapabilities]) - const builtInCapabilities = useMemo(() => profileCapabilities.filter((c) => !c.roleId), [profileCapabilities]) + const skillCapabilities = useMemo(() => profileCapabilities.filter((c) => !c.roleId), [profileCapabilities]) // ── Timeline scatter data ────────────────────────────────────────────── const timelineData = useMemo(() => { @@ -460,255 +357,189 @@ export function WorkersContent({
- {/* Blueprint grid overlay (V2) */} - {enableOutcomeLayer ? ( -
- ) : null} + {/* Blueprint grid overlay */} +
{/* Gradient fade from hero atmosphere to cards */}
- {/* Badge */} - -
- - - How we run evals - - - {alternateVersionHref && alternateVersionLabel ? ( - - {alternateVersionLabel} - - ) : null} + {enableOutcomeLayer ? ( +
+
+
+ {/* Badge */} + +
+ + + How we run evals + + +
+ + + + Outcomes over artifacts + + + {/* Heading */} + + You’re the Builder + + Ship{" "} + + Real Code + + + + + {/* Subheading */} + + Pick an objective. We’ll suggest an agent lineup and default model + based on eval results. Treat it as a baseline for your repo. + + + + + +
+
- - - {enableOutcomeLayer ? ( - - Outcomes over artifacts - ) : null} - - {/* Heading */} - - {enableOutcomeLayer ? ( - <> - Build from outcomes. -
Ship{" "} - - real code - - . - - ) : ( - <> - Build with{" "} - - Roo Code Cloud - - - )} - - - {/* Subheading */} - - {enableOutcomeLayer ? ( - <> - Pick what you're trying to ship. We assemble a Builder Profile: the - capabilities you need, plus a default model recommendation backed by eval data. - - ) : ( - <> - Outcomes over artifacts: start from the production codebase and ship as a reviewable - PR. Every model runs the same tasks, same tools, and the same time limit. Your repo - will differ—treat this as a baseline. - - )} - - - {enableOutcomeLayer ? ( - - - Start with Prototype → PR - - - - Browse outcomes - - - ) : null} - - {/* Stats bar */} - - -
- -
- -
- -
{/* ── Outcomes Overlay ───────────────────────────────────────── */} {enableOutcomeLayer ? ( -
-
+
+
- -

- Start with an outcome -

-

- Pick what you're trying to ship. We assemble a Builder Profile: capabilities - plus a default model recommendation. It's a baseline, not a guarantee. -

- - - - {EVAL_OUTCOMES.map((outcome) => { - const Icon = outcome.icon - const isSelected = outcome.id === selectedOutcomeId - const isFeatured = outcome.id === "prototype_to_pr" - - return ( - setOutcome(isSelected ? null : outcome.id)} - className={[ - "group rounded-2xl border bg-card/40 p-5 text-left backdrop-blur-sm transition-all duration-200 hover:bg-card/60", - isSelected - ? "border-foreground/20 ring-1 ring-foreground/15" - : "border-border/50 hover:border-border", - isFeatured ? "lg:col-span-2" : "", - ].join(" ")}> -
-
- + {isProfileView ? ( + + {selectedOutcome ? ( + <> +
+
+

+ Profile +

+

+ {selectedOutcome.name} +

+

+ {profileDescription} +

-
- {isFeatured ? ( - - Recommended starting point - + +
+ {examplePrompt ? ( +
+

+ Example prompt +

+ 
+																{examplePrompt}
+
+
) : null} -

- {outcome.name} -

-

- {outcome.description} -

+ + Start in Roo Code Cloud + + + + Back to objectives + +
- - ) - })} - - {selectedOutcome ? ( - -
-
-

- {profileTitle} -

-

- {selectedOutcome.name} -

-

- {profileDescription} -

- - {profileHowItWorks.length > 0 ? ( -
-

- {selectedOutcome.builderProfile - ? "How it works" - : "Why it works"} -

-
    - {profileHowItWorks.map((line) => ( -
  • - - {line} -
    • - ))} -
+
+
+
+

+ Optimized for +

+

+ {getModeLabel(selectedMode)} +

+
+
+

+ Default model +

+

+ {objectiveDefaultModelLabel} +

+
+
+

+ Agents +

+

+ {agentCapabilities.length} +

+
+
+

+ Skills +

+

+ {skillCapabilities.length} +

+
- ) : null} +
- {selectedOutcome.builderProfile?.howItWorks ? ( -
-

- Why it works -

-
    - {selectedOutcome.whyItWorks.map((line) => ( -
  • - - {line} -
    • - ))} -
-
- ) : null} -
- -
-
-
- - Optimize for - +
+

+ Optimize for +

+
{OPTIMIZATION_MODES.map((mode) => { const isSelected = mode.id === selectedMode return ( @@ -719,59 +550,53 @@ export function WorkersContent({ title={mode.description} onClick={() => setMode(mode.id)} className={[ - "inline-flex items-center rounded-full border px-3 py-1.5 text-xs font-semibold transition-colors", + "rounded-full px-3 py-1.5 text-xs font-semibold transition-colors", isSelected - ? "border-foreground/20 bg-foreground/5 text-foreground" - : "border-border/50 bg-background/20 text-foreground/75 hover:border-border hover:text-foreground", + ? "bg-foreground/10 text-foreground" + : "text-muted-foreground hover:text-foreground", ].join(" ")}> {mode.label} ) })}
- - Capability set -
-
-
+
+

- Agents + Agent lineup

-

- Click for candidates & settings -

-
-
- {agentCapabilities.map((capability) => { - const roleId = capability.roleId! - const rec = recByRole.get(roleId) - const candidate = getModeCandidate(rec, selectedMode) +
+ {agentCapabilities.map((capability) => { + const roleId = capability.roleId! + const rec = recByRole.get(roleId) + const candidate = getModeCandidate(rec, selectedMode) - return ( - -
+ const providerColor = candidate + ? (PROVIDER_COLORS[candidate.provider] ?? "#94a3b8") + : "#94a3b8" + + return ( +
-

+

{capability.name}

- {capability.description ? ( -

- {capability.description} -

- ) : null} -

+

+ {candidate ? ( - <> - - {getModeLabel(selectedMode)}: - {" "} + {candidate.displayName} - + ) : ( View models @@ -779,228 +604,368 @@ export function WorkersContent({ )}

- + + + ) + })} +
+
+ +
+

+ Skills included +

+
+ {skillCapabilities.length > 0 ? ( + skillCapabilities.map((capability) => ( +
+

+ {capability.name} +

+

+ {capability.description} +

- - ) - })} + )) + ) : ( +

+ No skills listed for this profile yet. +

+ )} +
- {builtInCapabilities.length > 0 ? ( -
+ {profileHowItWorks.length > 0 || selectedOutcome.whyItWorks.length > 0 ? ( +

- Built-ins + Rationale

-
- {builtInCapabilities.map((capability) => ( -
-
- -
-
-

- {capability.name} -

- - Built-in - -
-

- {capability.description} -

-
-
-
- ))} +
+
+

+ {selectedOutcome.builderProfile + ? "How it works" + : "Why it works"} +

+
    + {profileHowItWorks.map((line) => ( +
  • + + {line} +
    • + ))} +
+
+
+

+ Why it works +

+
    + {selectedOutcome.whyItWorks.map((line) => ( +
  • + + {line} +
    • + ))} +
+
) : null} + + ) : ( +
+
+

+ No objective selected +

+

+ Pick an objective first, then open the profile view. +

+
+ + Back to objectives + +
-
+ )} ) : null} - -
-
- ) : null} - {/* ── Role Cards Grid (baseline only) ────────────────────────── */} - {!enableOutcomeLayer ? ( -
- {/* Subtle section background */} - -
-
-
- +
+ {/* Left rail: objective + mode */} +
+ +

+ Select your objective +

+ -
- {/* Section connector */} - -

- Choose a setup for the work -

- - - - - {roles.map((role) => { - const rec = recByRole.get(role.id) - const IconComponent = ICON_MAP[role.icon] ?? Code - const candidateCount = rec?.allCandidates.length ?? 0 - const exerciseCount = rec?.totalExercises ?? 0 - const theme = ROLE_THEMES[role.id] ?? DEFAULT_THEME - const topModel = rec?.best[0] ?? null - - return ( -
- {/* Subtle glow on hover */} -
+ className="mt-6 flex flex-wrap items-center justify-center gap-2 lg:justify-start" + variants={fadeUpVariants}> + + Optimize for + +
+ {OPTIMIZATION_MODES.map((mode) => { + const isSelected = mode.id === selectedMode + return ( + + ) + })} +
+ -
- {/* Header: Icon + role badge */} -
-
- + + {EVAL_OUTCOMES.map((outcome) => { + const Icon = outcome.icon + const isSelected = outcome.id === selectedOutcomeId + + return ( + setOutcome(isSelected ? null : outcome.id)} + className={[ + "group w-full rounded-2xl border bg-card/35 p-4 text-left backdrop-blur-sm transition-all duration-200 hover:bg-card/55", + isSelected + ? "border-foreground/20 ring-1 ring-foreground/15" + : "border-border/50 hover:border-border", + ].join(" ")}> +
+
+ +
+
+
+

+ {outcome.name} +

+
+

+ {outcome.description} +

+
- {topModel && ( - - Top: {topModel.displayName} + + ) + })} + +
+ + {/* Right rail: profile snapshot */} +
+ +
+
+ +
+
+
+
+

+ Profile snapshot +

+

+ {selectedOutcome + ? selectedOutcome.name + : "Pick an objective"} +

+

+ {selectedOutcome + ? profileDescription + : "Select an objective to see the suggested lineup and default model."} +

+
+ +
+ + Optimized for: {getModeLabel(selectedMode)} +
+
+ +
+
+
+

+ Signal +

+

+ {totalEvalRuns.toLocaleString()} runs +

+
+
+

+ Agents +

+

+ {agentCapabilities.length} +

+
+
+

+ Default model +

+

+ {objectiveDefaultModelLabel} +

+
+
+
+
+ +
+
+

+ Agent lineup +

+

+ Open candidates & settings +

+
+ +
+ {selectedOutcome ? ( +
+ {agentCapabilities.map((capability) => { + const roleId = capability.roleId! + const rec = recByRole.get(roleId) + const candidate = getModeCandidate(rec, selectedMode) + + const providerColor = candidate + ? (PROVIDER_COLORS[candidate.provider] ?? "#94a3b8") + : "#94a3b8" + + return ( + + +
+
+

+ {capability.name} +

+

+ + {candidate ? ( + + {candidate.displayName} + + ) : ( + + View models + + )} +

+
+ +
+ + ) + })} +
+ ) : ( +
+

+ No objective selected +

+

+ Pick an objective to see the recommended agent lineup. +

+
)}
- {/* Profile name + descriptor */} -

{role.name}

-

- {role.salaryRange} -

- - {/* Description */} -

- {role.description} -

- - {/* Best for */} -
-

- Best for -

-
- {role.bestFor.map((item) => ( - - {item} - - ))} -
-
- - {/* Strengths & Weaknesses side by side */} -
- {/* Strengths */} -
-

- Strengths -

-
    - {role.strengths.map((item) => ( -
  • - - {item} -
    • - ))} -
-
- - {/* Weaknesses */} -
-

- Trade-offs -

-
    - {role.weaknesses.map((item) => ( -
  • - - {item} -
    • - ))} -
-
-
- - {/* Bottom stats + CTA */} -
-
- - - {candidateCount} models - - - - {exerciseCount.toLocaleString()} exercises - -
- - - View models - - -
+ {selectedOutcome ? ( + <> + {examplePrompt ? ( +
+

+ Example prompt +

+ 
+																	{examplePrompt}
+
+
+ ) : null} +
+ + Start in Roo Code Cloud + + + + Learn more / customize + + +
+ + ) : null}
- ) - })} +
+
) : null} {/* ── AI Coding Capability Over Time ─────────────────────────── */} -
+
{/* Subtle atmospheric background */}

- Pass rates on our eval suite, by model release date. The best ones now score 100%. + Pass rates on our eval suite by model release date. Several current models hit 100% on + this suite.

diff --git a/apps/web-roo-code/src/lib/eval-outcomes.ts b/apps/web-roo-code/src/lib/eval-outcomes.ts index 85ced98347..4ff9e9b588 100644 --- a/apps/web-roo-code/src/lib/eval-outcomes.ts +++ b/apps/web-roo-code/src/lib/eval-outcomes.ts @@ -23,6 +23,11 @@ export type EvalOutcomeCapability = { export type EvalOutcomeProfile = { title: string description: string + /** + * Starter prompt shown in the UI to help users understand what to ask for. + * This is product copy, not an eval artifact. + */ + examplePrompt?: string capabilities: EvalOutcomeCapability[] howItWorks: string[] } @@ -46,6 +51,69 @@ export type EvalOutcome = { } export const EVAL_OUTCOMES: EvalOutcome[] = [ + { + id: "review_guardrails", + name: "Idea → Prototype", + description: "Turn a vague idea into a working demo in your real codebase.", + icon: Sparkles, + recommendedRoleIds: ["autonomous", "senior"], + whyItWorks: [ + "Optimizes for momentum: map the codebase fast, then build a working slice.", + "Senior builder keeps the prototype grounded in production constraints.", + ], + builderProfile: { + title: "Your Builder Profile", + description: "For turning an idea into a working demo in your repo.", + examplePrompt: `Objective: Idea → Prototype + +In this repo, turn this idea into a working demo: . + +Constraints: +- Keep scope small and demo-first. +- Use the existing stack and patterns in this codebase. + +Deliver: +- A reviewable PR +- A short walkthrough (how to run it, what works, what’s next)`, + capabilities: [ + { + id: "autonomous_researcher", + name: "Autonomous Researcher", + description: "Maps the codebase, constraints, and best path forward before implementation starts.", + roleId: "autonomous", + }, + { + id: "multi_file_builder", + name: "Senior Builder", + description: "Builds a working prototype directly in your repo across the files it touches.", + roleId: "senior", + }, + { + id: "discovery_loop", + name: "Discovery loop", + description: + "Maps the codebase and constraints before making changes (so the prototype fits reality).", + }, + { + id: "prototype_scaffold", + name: "Prototype scaffold", + description: "Creates the smallest working slice you can demo and build on.", + }, + { + id: "demo_ready_output", + name: "Demo-ready output", + description: + "Delivers a reviewable diff plus a clear walkthrough of what’s working and what’s next.", + }, + ], + howItWorks: [ + "Clarify the objective and success criteria.", + "Explore the codebase and pick the smallest viable implementation path.", + "Build the prototype directly in the repo (no throwaway export/import step).", + "Deliver a demo-ready diff with notes for the next iteration.", + ], + }, + }, { id: "prototype_to_pr", name: "Prototype → PR", @@ -58,8 +126,17 @@ export const EVAL_OUTCOMES: EvalOutcome[] = [ ], builderProfile: { title: "Your Builder Profile", - description: - "A default set of capabilities for turning a working prototype into a reviewable PR—on the production codebase.", + description: "For turning a prototype into a reviewable PR on the production codebase.", + examplePrompt: `Objective: Prototype → PR + +Take the current prototype implementation and turn it into a reviewable PR. + +Do: +- Tighten scope to the smallest shippable diff +- Add/adjust tests, lint, and typechecks as needed + +Deliver: +- A PR-ready diff with a plain-English summary and review notes`, capabilities: [ { id: "multi_file_builder", @@ -109,50 +186,6 @@ export const EVAL_OUTCOMES: EvalOutcome[] = [ ], }, }, - { - id: "paper_cuts", - name: "Paper cuts & small fixes", - description: "Fix the small stuff without dragging engineers off big projects.", - icon: CheckCircle2, - recommendedRoleIds: ["junior", "reviewer"], - whyItWorks: [ - "Small diffs are high-leverage when the work is well-scoped.", - "Reviewer keeps the quality bar and reduces surprise.", - ], - }, - { - id: "sentry_triage", - name: "Sentry triage", - description: "Turn recurring errors into concrete fixes with proof before review.", - icon: Bug, - recommendedRoleIds: ["autonomous", "reviewer"], - whyItWorks: [ - "Autonomous runs handle multi-step investigation and iteration.", - "Reviewer focuses on safety, correctness, and “does this hold up?”.", - ], - }, - { - id: "repro_to_fix", - name: "Bug repro → fix", - description: "Make the handoff less lossy: reproduce, patch, and validate in one loop.", - icon: Workflow, - recommendedRoleIds: ["senior", "reviewer"], - whyItWorks: [ - "Good default for ambiguous bugs that touch a few files.", - "Reviewer helps catch cross-team assumptions early.", - ], - }, - { - id: "review_guardrails", - name: "Guardrails & review", - description: "Raise the quality bar without becoming the blocker.", - icon: GitPullRequest, - recommendedRoleIds: ["reviewer"], - whyItWorks: [ - "Works alongside CI, linters, and team review.", - "Scales judgement through fast, consistent feedback.", - ], - }, { id: "issue_to_pr", name: "Issue → PR", @@ -163,6 +196,251 @@ export const EVAL_OUTCOMES: EvalOutcome[] = [ "Handles out-of-band work while humans stay on the roadmap.", "Pairs autonomy with guardrails for merge safety.", ], + builderProfile: { + title: "Your Builder Profile", + description: "For turning an issue into a reviewable PR.", + examplePrompt: `Objective: Issue → PR + +Fix this issue in the repo: . + +Requirements: +- Define “done” in 2-3 acceptance criteria +- Implement the fix and validate it (tests/lint/typechecks) + +Deliver: +- A reviewable PR with context and any follow-ups`, + capabilities: [ + { + id: "autonomous_executor", + name: "Autonomous Executor", + description: "Runs the full loop (investigate → implement → validate) while you stay unblocked.", + roleId: "autonomous", + }, + { + id: "reviewer_guardrails", + name: "Reviewer & Guardrails", + description: "Reviews the diff for correctness, edge cases, and merge safety.", + roleId: "reviewer", + }, + { + id: "issue_intake", + name: "Issue intake", + description: + "Translates a request into scoped tasks, acceptance criteria, and a safe plan of attack.", + }, + { + id: "validation_loop", + name: "Validation loop", + description: "Runs tests/lint/typechecks and iterates until it’s clean (or flags what’s blocked).", + }, + { + id: "pr_ready_output", + name: "PR-ready output", + description: "Produces a focused diff plus a plain-English summary and review notes.", + }, + ], + howItWorks: [ + "Clarify the issue and define what “done” means.", + "Implement in the background with frequent validation checkpoints.", + "Run a reviewer pass to reduce merge risk.", + "Deliver a PR-ready result with context and next steps.", + ], + }, + }, + { + id: "sentry_triage", + name: "Customer Escalation → Resolved", + description: "Triage a customer-blocking issue and ship the smallest safe fix.", + icon: Bug, + recommendedRoleIds: ["autonomous", "senior", "reviewer"], + whyItWorks: [ + "Autonomous runs handle multi-step investigation and iteration.", + "Senior builder makes the final fix precise and production-safe.", + "Reviewer focuses on safety, correctness, and “does this hold up?”.", + ], + builderProfile: { + title: "Your Builder Profile", + description: "For resolving a customer escalation quickly and safely.", + examplePrompt: `Objective: Customer Escalation → Resolved + +We have a customer-blocking escalation: +- Symptoms: +- Context: + +Do: +- Find the smallest safe fix with a clear blast-radius assessment +- Add guardrails/tests where it makes sense + +Deliver: +- A PR with the fix and a short “risk + rollout” note`, + capabilities: [ + { + id: "autonomous_triage", + name: "Autonomous Triage", + description: "Investigates logs, context, and repro steps to converge on a fix quickly.", + roleId: "autonomous", + }, + { + id: "senior_fixer", + name: "Senior Builder", + description: "Implements the smallest production-safe fix when the blast radius is unclear.", + roleId: "senior", + }, + { + id: "reviewer_guardrails", + name: "Reviewer & Guardrails", + description: "Double-checks safety and correctness so speed doesn’t create regressions.", + roleId: "reviewer", + }, + { + id: "repro_first", + name: "Repro-first", + description: "Prioritizes a minimal reproduction so we know the fix actually fixes the issue.", + }, + { + id: "minimal_fix", + name: "Minimal safe fix", + description: "Ships the smallest change that unblocks customers, with a clear rollback story.", + }, + { + id: "verification_artifacts", + name: "Verification artifacts", + description: "Provides proof (tests/logs/steps) that the fix works and what it covers.", + }, + ], + howItWorks: [ + "Gather context and reproduce the customer issue.", + "Implement the smallest safe fix with verification.", + "Run a reviewer pass to catch edge cases.", + "Deliver a PR-ready result plus rollout notes.", + ], + }, + }, + { + id: "repro_to_fix", + name: "Bug Report → Fix", + description: "Reproduce, isolate, patch, and validate in one loop.", + icon: Workflow, + recommendedRoleIds: ["senior", "reviewer"], + whyItWorks: [ + "Good default for ambiguous bugs that touch a few files.", + "Reviewer helps catch cross-team assumptions early.", + ], + builderProfile: { + title: "Your Builder Profile", + description: "For turning a bug report into a verified fix.", + examplePrompt: `Objective: Bug Report → Fix + +Fix this bug: +- Report: +- Expected vs actual: + +Do: +- Reproduce if possible, then implement the fix +- Validate with tests/lint/typechecks (or explain what’s blocked) + +Deliver: +- A PR with the fix and verification notes`, + capabilities: [ + { + id: "bug_fixer", + name: "Bug Fixer", + description: "Reproduces and fixes bugs efficiently across the files involved.", + roleId: "senior", + }, + { + id: "reviewer_guardrails", + name: "Reviewer & Guardrails", + description: "Reviews the diff for correctness and regression risk before it ships.", + roleId: "reviewer", + }, + { + id: "repro_harness", + name: "Repro harness", + description: + "Creates a minimal reproduction path (tests or steps) to prevent “can’t repro” stalls.", + }, + { + id: "fix_with_tests", + name: "Fix with tests", + description: "Pairs the fix with verification so it doesn’t regress on the next change.", + }, + { + id: "validation_loop", + name: "Validation loop", + description: "Runs tests/lint/typechecks and iterates until it’s clean (or flags what’s blocked).", + }, + ], + howItWorks: [ + "Reproduce the issue and isolate the root cause.", + "Implement a targeted fix with verification.", + "Run a reviewer pass to reduce regression risk.", + "Deliver a PR-ready result with steps to validate.", + ], + }, + }, + { + id: "paper_cuts", + name: "Paper Cuts → Shipped", + description: "Fix the small stuff without dragging engineers off big projects.", + icon: CheckCircle2, + recommendedRoleIds: ["junior", "reviewer"], + whyItWorks: [ + "Small diffs are high-leverage when the work is well-scoped.", + "Reviewer keeps the quality bar and reduces surprise.", + ], + builderProfile: { + title: "Your Builder Profile", + description: "For shipping small fixes quickly, cleanly, and safely.", + examplePrompt: `Objective: Paper Cuts → Shipped + +Ship these small fixes in this repo: +- +- +- + +Constraints: +- Keep diffs small and easy to review +- Don’t change behavior unless it’s clearly a bug + +Deliver: +- A PR with grouped, well-scoped commits and a short summary`, + capabilities: [ + { + id: "small_diff_builder", + name: "Small-diff Builder", + description: "Ships focused fixes with low review surface area and minimal risk.", + roleId: "junior", + }, + { + id: "reviewer_guardrails", + name: "Reviewer & Guardrails", + description: "Catches edge cases and keeps changes aligned with team conventions.", + roleId: "reviewer", + }, + { + id: "scope_control", + name: "Scope control", + description: "Keeps changes tight: fewer surprises, faster reviews, easier merges.", + }, + { + id: "quick_validation", + name: "Quick validation", + description: "Runs the relevant checks and flags what’s safe to skip (and what’s not).", + }, + { + id: "pr_ready_output", + name: "PR-ready output", + description: "Produces a focused diff plus a plain-English summary and review notes.", + }, + ], + howItWorks: [ + "Pick the smallest fix that moves the needle.", + "Implement with tight scope control.", + "Validate quickly and review for conventions.", + "Deliver a PR-ready result you can merge confidently.", + ], + }, }, ] diff --git a/apps/web-roo-code/src/lib/mock-recommendations.ts b/apps/web-roo-code/src/lib/mock-recommendations.ts index 9a08487ca7..2e30b5ec0b 100644 --- a/apps/web-roo-code/src/lib/mock-recommendations.ts +++ b/apps/web-roo-code/src/lib/mock-recommendations.ts @@ -302,7 +302,7 @@ const seniorCandidates: ModelCandidate[] = [ { provider: "moonshot", modelId: "kimi-k2-0905", - displayName: "Kimi K2 0905", + displayName: "Kimi K2", compositeScore: 95, tier: "best", tags: ["budget-hire", "best-value"], @@ -439,7 +439,7 @@ const staffCandidates: ModelCandidate[] = [ { provider: "anthropic", modelId: "claude-opus-4-6", - displayName: "Claude Opus 4.6", + displayName: "Opus 4.6", compositeScore: 98, tier: "best", tags: ["speed-hire", "top-performer"], @@ -467,7 +467,7 @@ const staffCandidates: ModelCandidate[] = [ { provider: "anthropic", modelId: "claude-opus-4-5", - displayName: "Claude Opus 4.5", + displayName: "Opus 4.5", compositeScore: 96, tier: "recommended", tags: [], @@ -495,7 +495,7 @@ const staffCandidates: ModelCandidate[] = [ { provider: "anthropic", modelId: "claude-opus-4-1", - displayName: "Claude Opus 4.1", + displayName: "Opus 4.1", compositeScore: 73, tier: "situational", tags: [], @@ -525,7 +525,7 @@ const staffCandidates: ModelCandidate[] = [ { provider: "anthropic", modelId: "claude-opus-4", - displayName: "Claude Opus 4", + displayName: "Opus 4", compositeScore: 57, tier: "not-recommended", tags: [], @@ -565,7 +565,7 @@ const reviewerCandidates: ModelCandidate[] = [ { provider: "anthropic", modelId: "claude-opus-4-6", - displayName: "Claude Opus 4.6", + displayName: "Opus 4.6", compositeScore: 95, tier: "best", tags: ["speed-hire", "top-performer"], @@ -731,7 +731,7 @@ const autonomousCandidates: ModelCandidate[] = [ { provider: "moonshot", modelId: "kimi-k2-0905", - displayName: "Kimi K2 0905", + displayName: "Kimi K2", compositeScore: 86, tier: "recommended", tags: [], @@ -885,6 +885,6 @@ export const MODEL_TIMELINE: ModelTimelineEntry[] = [ { modelName: "GPT-5 Mini", provider: "openai", releaseDate: "2025-12-01", score: 99, costPerRun: 3.34 }, { modelName: "Claude Sonnet 4.5", provider: "anthropic", releaseDate: "2026-01-15", score: 100, costPerRun: 38.43 }, { modelName: "GPT 5.2 (Med)", provider: "openai", releaseDate: "2026-01-20", score: 100, costPerRun: 12.5 }, - { modelName: "Claude Opus 4.6", provider: "anthropic", releaseDate: "2026-02-01", score: 100, costPerRun: 49.48 }, + { modelName: "Opus 4.6", provider: "anthropic", releaseDate: "2026-02-01", score: 100, costPerRun: 49.48 }, { modelName: "Gemini 3 Pro", provider: "google", releaseDate: "2026-02-05", score: 100, costPerRun: 33.06 }, ] diff --git a/apps/web-roo-code/src/lib/objective-default-models-v1.ts b/apps/web-roo-code/src/lib/objective-default-models-v1.ts new file mode 100644 index 0000000000..e60480d908 --- /dev/null +++ b/apps/web-roo-code/src/lib/objective-default-models-v1.ts @@ -0,0 +1,227 @@ +import type { EvalOutcomeId } from "./eval-outcomes" + +type ObjectiveMetric = { score: number; costUsd: number; runtimeS: number } + +type ModelObjectiveMetrics = { + modelId: string + issueResolution: ObjectiveMetric + frontend: ObjectiveMetric + greenfield: ObjectiveMetric + testing: ObjectiveMetric + infoGathering: ObjectiveMetric +} + +type EvalOptimizationModeV1 = "best" | "fastest" | "cost" + +type ObjectiveWeights = { + issueResolution: number + frontend: number + greenfield: number + testing: number + infoGathering: number +} + +type WeightedObjectiveMetrics = { score: number; costUsd: number; runtimeS: number } + +export type ObjectiveDefaultModelV1 = { + modelId: string + weighted: WeightedObjectiveMetrics +} + +const MODEL_METRICS_V1: ModelObjectiveMetrics[] = [ + { + modelId: "claude-opus-4-6", + issueResolution: { score: 74.8, costUsd: 0.56, runtimeS: 178 }, + frontend: { score: 41.8, costUsd: 2.37, runtimeS: 602 }, + greenfield: { score: 43.8, costUsd: 2.5, runtimeS: 388 }, + testing: { score: 78.8, costUsd: 0.43, runtimeS: 138 }, + infoGathering: { score: 80, costUsd: 1.33, runtimeS: 526 }, + }, + { + modelId: "GPT-5.2-Codex", + issueResolution: { score: 73.8, costUsd: 0.94, runtimeS: 438 }, + frontend: { score: 35.9, costUsd: 2.97, runtimeS: 1434 }, + greenfield: { score: 62.5, costUsd: 2.5, runtimeS: 838 }, + testing: { score: 62.5, costUsd: 0.66, runtimeS: 343 }, + infoGathering: { score: 70.9, costUsd: 1.66, runtimeS: 799 }, + }, + { + modelId: "claude-opus-4-5", + issueResolution: { score: 76.6, costUsd: 1.82, runtimeS: 325 }, + frontend: { score: 41.2, costUsd: 2.54, runtimeS: 671 }, + greenfield: { score: 37.5, costUsd: 4.65, runtimeS: 495 }, + testing: { score: 78.5, costUsd: 1.38, runtimeS: 268 }, + infoGathering: { score: 69.1, costUsd: 0.55, runtimeS: 97 }, + }, + { + modelId: "MiniMax-M2.5", + issueResolution: { score: 72.6, costUsd: 0.1, runtimeS: 455 }, + frontend: { score: 25, costUsd: 0.15, runtimeS: 611 }, + greenfield: { score: 50, costUsd: 0.16, runtimeS: 376 }, + testing: { score: 68.1, costUsd: 0.07, runtimeS: 389 }, + infoGathering: { score: 47.9, costUsd: 0.06, runtimeS: 716 }, + }, + { + modelId: "GPT-5.2", + issueResolution: { score: 74.6, costUsd: 0.86, runtimeS: 476 }, + frontend: { score: 30.9, costUsd: 2.77, runtimeS: 1571 }, + greenfield: { score: 18.8, costUsd: 0.71, runtimeS: 397 }, + testing: { score: 73.2, costUsd: 0.56, runtimeS: 347 }, + infoGathering: { score: 65.5, costUsd: 0.48, runtimeS: 189 }, + }, + { + modelId: "claude-sonnet-4-5", + issueResolution: { score: 74.2, costUsd: 1.19, runtimeS: 534 }, + frontend: { score: 36.8, costUsd: 1.89, runtimeS: 787 }, + greenfield: { score: 12.5, costUsd: 2.65, runtimeS: 744 }, + testing: { score: 68.8, costUsd: 0.98, runtimeS: 488 }, + infoGathering: { score: 58.8, costUsd: 0.38, runtimeS: 126 }, + }, + { + modelId: "Kimi-K2.5", + issueResolution: { score: 68.8, costUsd: 0.48, runtimeS: 707 }, + frontend: { score: 32.8, costUsd: 1.58, runtimeS: 921 }, + greenfield: { score: 18.8, costUsd: 0.96, runtimeS: 814 }, + testing: { score: 61.9, costUsd: 0.42, runtimeS: 385 }, + infoGathering: { score: 63.6, costUsd: 0.39, runtimeS: 602 }, + }, + { + modelId: "Gemini-3-Flash", + issueResolution: { score: 74.6, costUsd: 0.42, runtimeS: 343 }, + frontend: { score: 22.1, costUsd: 0.8, runtimeS: 1152 }, + greenfield: { score: 18.8, costUsd: 0.82, runtimeS: 399 }, + testing: { score: 70.7, costUsd: 0.3, runtimeS: 213 }, + infoGathering: { score: 58.8, costUsd: 0.38, runtimeS: 398 }, + }, + { + modelId: "DeepSeek-V3.2-Reasoner", + issueResolution: { score: 71.6, costUsd: 0.16, runtimeS: 1429 }, + frontend: { score: 27.9, costUsd: 0.19, runtimeS: 1515 }, + greenfield: { score: 31.2, costUsd: 0.12, runtimeS: 1411 }, + testing: { score: 53.6, costUsd: 0.12, runtimeS: 1215 }, + infoGathering: { score: 50.3, costUsd: 0.06, runtimeS: 427 }, + }, + { + modelId: "Gemini-3-Pro", + issueResolution: { score: 70.6, costUsd: 0.95, runtimeS: 343 }, + frontend: { score: 36.8, costUsd: 1.46, runtimeS: 710 }, + greenfield: { score: 12.5, costUsd: 2.68, runtimeS: 554 }, + testing: { score: 68.6, costUsd: 1.01, runtimeS: 386 }, + infoGathering: { score: 44.2, costUsd: 1.5, runtimeS: 1775 }, + }, + { + modelId: "MiniMax-M2.1", + issueResolution: { score: 68.8, costUsd: 0.14, runtimeS: 579 }, + frontend: { score: 16.2, costUsd: 0.21, runtimeS: 1417 }, + greenfield: { score: 25, costUsd: 0.33, runtimeS: 826 }, + testing: { score: 61.4, costUsd: 0.11, runtimeS: 473 }, + infoGathering: { score: 40.6, costUsd: 0.06, runtimeS: 641 }, + }, + { + modelId: "GLM-4.7", + issueResolution: { score: 73.4, costUsd: 0.56, runtimeS: 1007 }, + frontend: { score: 22.1, costUsd: 0.66, runtimeS: 1519 }, + greenfield: { score: 12.5, costUsd: 0.54, runtimeS: 578 }, + testing: { score: 49.4, costUsd: 0.37, runtimeS: 744 }, + infoGathering: { score: 53.9, costUsd: 0.46, runtimeS: 1138 }, + }, + { + modelId: "Kimi-K2-Thinking", + issueResolution: { score: 69.2, costUsd: 2, runtimeS: 1325 }, + frontend: { score: 32.4, costUsd: 2.31, runtimeS: 1641 }, + greenfield: { score: 18.8, costUsd: 6.78, runtimeS: 2314 }, + testing: { score: 47.3, costUsd: 1.39, runtimeS: 1253 }, + infoGathering: { score: 43.6, costUsd: 0.65, runtimeS: 279 }, + }, + { + modelId: "Qwen3-Coder-480B", + issueResolution: { score: 62.4, costUsd: 1.26, runtimeS: 680 }, + frontend: { score: 23.5, costUsd: 2.09, runtimeS: 1006 }, + greenfield: { score: 0, costUsd: 1.79, runtimeS: 924 }, + testing: { score: 34.9, costUsd: 0.97, runtimeS: 626 }, + infoGathering: { score: 33.9, costUsd: 0.28, runtimeS: 197 }, + }, +] + +function getOutcomeWeights(outcomeId: EvalOutcomeId): ObjectiveWeights { + // These are intentionally opinionated. They exist to make the prototype feel realistic + // before we wire real Roo Code Cloud evals. + switch (outcomeId) { + // Idea → Prototype + case "review_guardrails": + return { greenfield: 0.5, infoGathering: 0.35, frontend: 0.1, testing: 0.05, issueResolution: 0 } + // Prototype → PR + case "prototype_to_pr": + return { greenfield: 0.35, testing: 0.35, issueResolution: 0.2, frontend: 0.1, infoGathering: 0 } + // Issue → PR + case "issue_to_pr": + return { issueResolution: 0.4, testing: 0.3, infoGathering: 0.2, frontend: 0.1, greenfield: 0 } + // Customer Escalation → Resolved + case "sentry_triage": + return { issueResolution: 0.55, infoGathering: 0.25, testing: 0.2, frontend: 0, greenfield: 0 } + // Bug Report → Fix + case "repro_to_fix": + return { issueResolution: 0.45, testing: 0.4, infoGathering: 0.15, frontend: 0, greenfield: 0 } + // Paper Cuts → Shipped + case "paper_cuts": + return { frontend: 0.6, issueResolution: 0.2, testing: 0.2, greenfield: 0, infoGathering: 0 } + } +} + +function getWeightedMetrics(row: ModelObjectiveMetrics, weights: ObjectiveWeights): WeightedObjectiveMetrics { + const score = + row.issueResolution.score * weights.issueResolution + + row.frontend.score * weights.frontend + + row.greenfield.score * weights.greenfield + + row.testing.score * weights.testing + + row.infoGathering.score * weights.infoGathering + const costUsd = + row.issueResolution.costUsd * weights.issueResolution + + row.frontend.costUsd * weights.frontend + + row.greenfield.costUsd * weights.greenfield + + row.testing.costUsd * weights.testing + + row.infoGathering.costUsd * weights.infoGathering + const runtimeS = + row.issueResolution.runtimeS * weights.issueResolution + + row.frontend.runtimeS * weights.frontend + + row.greenfield.runtimeS * weights.greenfield + + row.testing.runtimeS * weights.testing + + row.infoGathering.runtimeS * weights.infoGathering + return { score, costUsd, runtimeS } +} + +function pickByMode( + rows: Array<{ modelId: string; weighted: WeightedObjectiveMetrics }>, + mode: EvalOptimizationModeV1, +): { modelId: string; weighted: WeightedObjectiveMetrics } { + const bestByQuality = rows.reduce((best, cur) => (cur.weighted.score > best.weighted.score ? cur : best)) + + // For speed/cost modes, don't pick a model that is dramatically worse on quality. + // This keeps the v1 prototype recommendations feeling credible even when a model is + // extremely cheap or fast but underperforms for the selected objective. + const QUALITY_FLOOR = 0.85 + const qualityThreshold = bestByQuality.weighted.score * QUALITY_FLOOR + const qualityGated = rows.filter((r) => r.weighted.score >= qualityThreshold) + const pool = qualityGated.length > 0 ? qualityGated : rows + + if (mode === "fastest") { + return pool.reduce((best, cur) => (cur.weighted.runtimeS < best.weighted.runtimeS ? cur : best)) + } + if (mode === "cost") { + return pool.reduce((best, cur) => (cur.weighted.costUsd < best.weighted.costUsd ? cur : best)) + } + return bestByQuality +} + +export function pickObjectiveDefaultModelV1( + outcomeId: EvalOutcomeId, + mode: EvalOptimizationModeV1, +): ObjectiveDefaultModelV1 | null { + const weights = getOutcomeWeights(outcomeId) + const candidates = MODEL_METRICS_V1.map((row) => ({ + modelId: row.modelId, + weighted: getWeightedMetrics(row, weights), + })) + if (candidates.length === 0) return null + return pickByMode(candidates, mode) +}