diff --git a/apps/web-roo-code/src/app/evals/methodology/methodology-content.tsx b/apps/web-roo-code/src/app/evals/methodology/methodology-content.tsx new file mode 100644 index 0000000000..18d9bcef3a --- /dev/null +++ b/apps/web-roo-code/src/app/evals/methodology/methodology-content.tsx @@ -0,0 +1,949 @@ +"use client" + +import { motion } from "framer-motion" +import { + ArrowRight, + FlaskConical, + Code, + GitBranch, + Building2, + AlertTriangle, + BarChart3, + Terminal, + ExternalLink, + CheckCircle2, + Beaker, + Timer, + DollarSign, + Zap, + Trophy, +} from "lucide-react" +import Link from "next/link" + +// ── Framer Motion Variants ────────────────────────────────────────────────── + +const containerVariants = { + hidden: { opacity: 0 }, + visible: { + opacity: 1, + transition: { + staggerChildren: 0.12, + delayChildren: 0.1, + }, + }, +} + +const fadeUpVariants = { + hidden: { opacity: 0, y: 20 }, + visible: { + opacity: 1, + y: 0, + transition: { + duration: 0.6, + ease: [0.21, 0.45, 0.27, 0.9] as const, + }, + }, +} + +const backgroundVariants = { + hidden: { opacity: 0 }, + visible: { + opacity: 1, + transition: { + duration: 1.2, + ease: "easeOut" as const, + }, + }, +} + +const cardVariants = { + hidden: { opacity: 0, y: 30 }, + visible: { + opacity: 1, + y: 0, + transition: { + duration: 0.6, + ease: [0.21, 0.45, 0.27, 0.9] as const, + }, + }, +} + +// ── Section Number Marker ─────────────────────────────────────────────────── + +function SectionNumber({ num }: { num: string }) { + return ( + + {num} + + ) +} + +// ── Process Step Icon ─────────────────────────────────────────────────────── + +function ProcessStep({ + icon: Icon, + label, + isLast, +}: { + icon: React.ComponentType<{ className?: string }> + label: string + isLast?: boolean +}) { + return ( +
+
+
+ +
+ {label} +
+ {!isLast && ( +
+
+ +
+ )} +
+ ) +} + +// ── Language Card ─────────────────────────────────────────────────────────── + +function LanguageCard({ name, color }: { name: string; color: string }) { + return ( +
+
+ {name.slice(0, 2).toUpperCase()} +
+ {name} +
+ ) +} + +// ── Scoring Bar Component ─────────────────────────────────────────────────── + +function ScoringBar({ + label, + icon: Icon, + color, + bgColor, + weight, + description, +}: { + label: string + icon: React.ComponentType<{ className?: string }> + color: string + bgColor: string + weight: number + description: string +}) { + return ( +
+
+ +
+
+
+

{label}

+ {weight}% +
+

{description}

+
+ +
+
+
+ ) +} + +// ── Main Content Component ────────────────────────────────────────────────── + +export function MethodologyContent() { + return ( + <> + {/* ════════════════════════════════════════════════════════════════ + HERO SECTION + ════════════════════════════════════════════════════════════════ */} +
+ {/* Atmospheric blur background */} + +
+
+
+
+
+ + +
+ + {/* Breadcrumb */} + + + Evals + + / + + Hire an AI Engineer + + / + How We Interview + + + {/* Heading */} + + How We Interview{" "} + + AI Models + + + + {/* Subtitle */} + + Same exercises, same environment, same scoring for every model. Every step is documented and + every eval run is reproducible. + + + {/* Pill badge links */} + + + + View recommendations + + + + + Raw eval data + + + + +
+
+ + {/* ════════════════════════════════════════════════════════════════ + SECTION 01: THE INTERVIEW PROCESS + ════════════════════════════════════════════════════════════════ */} + +
+ + + + + + The Interview Process + + + +

+ We don't test models in isolation. We test them as they work inside Roo Code. Each + model gets the same exercises, same time limit, same tools. We measure what matters. +

+ + + {/* Process flow */} + + + + + + + + {/* Key principles */} + + {[ + { + title: "Identical Environment", + desc: "Docker container with VS Code, Roo Code extension, and a fresh workspace per exercise.", + }, + { + title: "No Cherry-Picking", + desc: "Every model gets the exact same interview. No curated demos, no special treatment.", + }, + { + title: "Real Metrics", + desc: "Does it pass the tests? How much does it cost? How fast does it deliver?", + }, + ].map((item) => ( + +

{item.title}

+

{item.desc}

+ + ))} + +
+ + + {/* ════════════════════════════════════════════════════════════════ + SECTION 02: THE INTERVIEW SUITE + ════════════════════════════════════════════════════════════════ */} + + {/* Subtle background glow */} + +
+
+
+ + +
+ + + + + + The Interview Suite + + + + Hundreds of coding exercises across 5 languages and{" "} + 3 difficulty tiers. From single-file fixes to + complex architecture decisions. + + + {/* Language cards */} + + + + + + + + + + + + + + + + + + + {/* Difficulty tiers */} + +

+ Difficulty Tiers +

+
+ {/* Easy */} +
+
+ E +
+
+

Easy

+

+ Single-file fixes, straightforward implementations, basic debugging +

+
+
+ + 90–95% + +
+ +
+
+
+ {/* Medium */} +
+
+ M +
+
+

Medium

+

+ Multi-file changes, refactoring, cross-file understanding +

+
+
+ + 60–80% + +
+ +
+
+
+ {/* Hard */} +
+
+ H +
+
+

Hard

+

+ Architecture decisions, ambiguous requirements, complex system design +

+
+
+ + 30–50% + +
+ +
+
+
+
+ +
+ + + {/* ════════════════════════════════════════════════════════════════ + SECTION 03: ENGINEER ROLES + ════════════════════════════════════════════════════════════════ */} + +
+ + + + + + Engineer Roles + + + + Not every task needs the same level of engineering. Three role tiers, each with different + exercise difficulty and scoring weights. + + + {/* Role cards */} + + {/* Junior */} + +
+
+
+ +
+

Junior Engineer

+

+ Easy + Medium exercises. Boilerplate, simple bug fixes, test generation. Scoring + emphasizes{" "} + cost efficiency. +

+ {/* Weight breakdown */} +
+

+ Scoring Weights +

+
+
+
+
+
+
+
+ Success 35% + Quality 15% + Cost 35% + Speed 15% +
+
+
+ + + {/* Senior */} + +
+
+
+ +
+

Senior Engineer

+

+ Medium exercises. Feature development, debugging, code review. Balanced scoring with + emphasis on{" "} + success rate + quality + . +

+ {/* Weight breakdown */} +
+

+ Scoring Weights +

+
+
+
+
+
+
+
+ Success 40% + Quality 25% + Cost 20% + Speed 15% +
+
+
+ + + {/* Staff */} + +
+
+
+ +
+

Staff Engineer

+

+ Hard exercises. Architecture, ambiguous requirements, system design. Scoring + prioritizes{" "} + + reasoning quality + correctness + + . +

+ {/* Weight breakdown */} +
+

+ Scoring Weights +

+
+
+
+
+
+
+
+ Success 45% + Quality 30% + Cost 10% + Speed 15% +
+
+
+ + +
+ + + {/* ════════════════════════════════════════════════════════════════ + SECTION 04: SCORING + ════════════════════════════════════════════════════════════════ */} + + {/* Background glow */} + +
+
+
+ + +
+ + + + + + Scoring + + + + Each model receives a composite score, a weighted + sum of four dimensions normalized to a 0–100 scale. + + + {/* Scoring formula components */} + + + + + + + + + + + + + + + + {/* Tier classification */} + +

+ Recommendation Tiers +

+

+ Composite scores are mapped to recommendation tiers: +

+ + + + {/* Best */} + +
+ ≥85 +
+
+ + Best + +

Top Performer

+

Highly recommended for this role.

+
+ + + {/* Recommended */} + +
+ 70–84 +
+
+ + Recommended + +

Solid Choice

+

+ Reliable for most tasks at this level. +

+
+ + + {/* Situational */} + +
+ 50–69 +
+
+ + Situational + +

Usable with Caveats

+

May struggle in specific areas.

+
+ + + {/* Not Recommended */} + +
+ <50 +
+
+ + Not Recommended + +

High Failure Rate

+

Not suitable for this role.

+
+ + + + + Per-language breakdowns reveal where each model excels or struggles. A model might score well + overall but underperform in Rust, or dominate in Python but lag in Go. + +
+ + + {/* ════════════════════════════════════════════════════════════════ + SECTION 05: RUN YOUR OWN INTERVIEWS + ════════════════════════════════════════════════════════════════ */} + +
+ + + + + + Run Your Own Interviews + + + + Our evaluation framework is fully open source. Run the exact same interviews on your own + infrastructure, with your own API keys, against any model. + + + {/* Terminal card */} + + {/* Terminal header */} +
+
+
+
+
+
+
+ + terminal +
+
+ {/* Terminal body */} +
+
+ ${" "} + git clone{" "} + https://github.com/RooCodeInc/Roo-Code-Evals.git +
+
+ $ cd{" "} + Roo-Code-Evals +
+
+ ${" "} + # Follow the README for setup instructions +
+
+ + + {/* GitHub link */} + + + + + + View on GitHub + + + +
+ + + {/* ════════════════════════════════════════════════════════════════ + SECTION 06: LIMITATIONS + ════════════════════════════════════════════════════════════════ */} + +
+ + + + + + Limitations + + + + Every evaluation has blind spots. These are ours. + + + + {[ + { + title: "Single test environment", + description: + "All evals run in Docker + VS Code. Results may differ in other IDEs or environments.", + }, + { + title: "Expanding exercise coverage", + description: + "Hundreds of exercises, but the suite is continuously growing. Some niche patterns may be underrepresented.", + }, + { + title: "API changes affect results", + description: + "Providers update their models. A model that scored well last month may behave differently after an update.", + }, + { + title: "Point-in-time snapshots", + description: + 'Each eval run captures performance at a specific point. We re-run regularly; check the "last updated" date.', + }, + ].map((item) => ( + + +
+

{item.title}

+

+ {item.description} +

+
+ + ))} + +
+ + + {/* ════════════════════════════════════════════════════════════════ + BOTTOM NAVIGATION + ════════════════════════════════════════════════════════════════ */} +
+
+ + + Ready to see the results? + + + + + View recommendations + + + + + Raw eval data + + + + +
+
+ + ) +} diff --git a/apps/web-roo-code/src/app/evals/methodology/page.tsx b/apps/web-roo-code/src/app/evals/methodology/page.tsx new file mode 100644 index 0000000000..8a0960142c --- /dev/null +++ b/apps/web-roo-code/src/app/evals/methodology/page.tsx @@ -0,0 +1,58 @@ +import type { Metadata } from "next" + +import { SEO } from "@/lib/seo" +import { ogImageUrl } from "@/lib/og" + +import { MethodologyContent } from "./methodology-content" + +// ── SEO Metadata ──────────────────────────────────────────────────────────── + +const TITLE = "How We Interview AI Models | Roo Code Evals" +const DESCRIPTION = "Our methodology for evaluating AI coding models. Transparent, reproducible, evidence-based." +const OG_DESCRIPTION = "Our methodology for evaluating AI coding models" +const PATH = "/evals/methodology" + +export const metadata: Metadata = { + title: TITLE, + description: DESCRIPTION, + alternates: { + canonical: `${SEO.url}${PATH}`, + }, + openGraph: { + title: TITLE, + description: DESCRIPTION, + url: `${SEO.url}${PATH}`, + siteName: SEO.name, + images: [ + { + url: ogImageUrl(TITLE, OG_DESCRIPTION), + width: 1200, + height: 630, + alt: TITLE, + }, + ], + locale: SEO.locale, + type: "website", + }, + twitter: { + card: SEO.twitterCard, + title: TITLE, + description: DESCRIPTION, + images: [ogImageUrl(TITLE, OG_DESCRIPTION)], + }, + keywords: [ + ...SEO.keywords, + "AI evaluation", + "model benchmarking", + "coding evals", + "methodology", + "interview process", + "transparent evaluation", + ], +} + +// ── Page Component ────────────────────────────────────────────────────────── + +export default function MethodologyPage() { + return +} diff --git a/apps/web-roo-code/src/app/evals/workers/[roleId]/candidates-content.tsx b/apps/web-roo-code/src/app/evals/workers/[roleId]/candidates-content.tsx new file mode 100644 index 0000000000..c4a42a71f8 --- /dev/null +++ b/apps/web-roo-code/src/app/evals/workers/[roleId]/candidates-content.tsx @@ -0,0 +1,1010 @@ +"use client" + +import { motion } from "framer-motion" +import { + Code, + GitBranch, + Building2, + Search, + Bot, + ArrowRight, + ArrowLeft, + Trophy, + DollarSign, + Zap, + ExternalLink, + CheckCircle2, + AlertTriangle, + FlaskConical, + BarChart3, + Beaker, +} from "lucide-react" +import type { LucideIcon } from "lucide-react" +import Link from "next/link" + +import type { ModelCandidate, LanguageScores, EngineerRole } from "@/lib/mock-recommendations" + +import { CopySettingsButton } from "./copy-settings-button" + +// ── Icon Mapping ──────────────────────────────────────────────────────────── + +const ICON_MAP: Record = { + Code, + GitBranch, + Building2, + Search, + Bot, +} + +// ── Role Color Themes ─────────────────────────────────────────────────────── + +type RoleTheme = { + accent: string + accentLight: string + accentDark: string + iconBg: string + iconText: string + badgeBg: string + badgeText: string + borderHover: string + shadowHover: string + buttonBg: string + buttonHover: string + glowColor: string + gradientFrom: string + gradientVia: string + ringColor: string + scoreText: string + scoreBg: string + blurBg1: string + blurBg2: string + methodologyBorder: string +} + +const ROLE_THEMES: Record = { + junior: { + accent: "emerald", + accentLight: "text-emerald-600", + accentDark: "dark:text-emerald-400", + iconBg: "bg-emerald-100 dark:bg-emerald-900/30", + iconText: "text-emerald-700 dark:text-emerald-300", + badgeBg: "bg-emerald-100 dark:bg-emerald-900/30", + badgeText: "text-emerald-700 dark:text-emerald-300", + borderHover: "hover:border-emerald-500/40 dark:hover:border-emerald-400/30", + shadowHover: "hover:shadow-emerald-500/10 dark:hover:shadow-emerald-400/10", + buttonBg: "bg-emerald-600 dark:bg-emerald-600", + buttonHover: "hover:bg-emerald-700 dark:hover:bg-emerald-500", + glowColor: "bg-emerald-500/8 dark:bg-emerald-600/15", + gradientFrom: "from-emerald-500", + gradientVia: "via-emerald-400", + ringColor: "ring-emerald-500/30", + scoreText: "text-emerald-400", + scoreBg: "bg-emerald-500/10 border-emerald-500/20", + blurBg1: "bg-emerald-500/10 dark:bg-emerald-600/20", + blurBg2: "bg-emerald-400/5 dark:bg-emerald-500/10", + methodologyBorder: "border-emerald-500/30 hover:border-emerald-500/50", + }, + senior: { + accent: "blue", + accentLight: "text-blue-600", + accentDark: "dark:text-blue-400", + iconBg: "bg-blue-100 dark:bg-blue-900/30", + iconText: "text-blue-700 dark:text-blue-300", + badgeBg: "bg-blue-100 dark:bg-blue-900/30", + badgeText: "text-blue-700 dark:text-blue-300", + borderHover: "hover:border-blue-500/40 dark:hover:border-blue-400/30", + shadowHover: "hover:shadow-blue-500/10 dark:hover:shadow-blue-400/10", + buttonBg: "bg-blue-600 dark:bg-blue-600", + buttonHover: "hover:bg-blue-700 dark:hover:bg-blue-500", + glowColor: "bg-blue-500/8 dark:bg-blue-600/15", + gradientFrom: "from-blue-500", + gradientVia: "via-blue-400", + ringColor: "ring-blue-500/30", + scoreText: "text-blue-400", + scoreBg: "bg-blue-500/10 border-blue-500/20", + blurBg1: "bg-blue-500/10 dark:bg-blue-600/20", + blurBg2: "bg-blue-400/5 dark:bg-blue-500/10", + methodologyBorder: "border-blue-500/30 hover:border-blue-500/50", + }, + staff: { + accent: "amber", + accentLight: "text-amber-600", + accentDark: "dark:text-amber-400", + iconBg: "bg-amber-100 dark:bg-amber-900/30", + iconText: "text-amber-700 dark:text-amber-300", + badgeBg: "bg-amber-100 dark:bg-amber-900/30", + badgeText: "text-amber-700 dark:text-amber-300", + borderHover: "hover:border-amber-500/40 dark:hover:border-amber-400/30", + shadowHover: "hover:shadow-amber-500/10 dark:hover:shadow-amber-400/10", + buttonBg: "bg-amber-600 dark:bg-amber-600", + buttonHover: "hover:bg-amber-700 dark:hover:bg-amber-500", + glowColor: "bg-amber-500/8 dark:bg-amber-600/15", + gradientFrom: "from-amber-500", + gradientVia: "via-amber-400", + ringColor: "ring-amber-500/30", + scoreText: "text-amber-400", + scoreBg: "bg-amber-500/10 border-amber-500/20", + blurBg1: "bg-amber-500/10 dark:bg-amber-600/20", + blurBg2: "bg-amber-400/5 dark:bg-amber-500/10", + methodologyBorder: "border-amber-500/30 hover:border-amber-500/50", + }, + reviewer: { + accent: "violet", + accentLight: "text-violet-600", + accentDark: "dark:text-violet-400", + iconBg: "bg-violet-100 dark:bg-violet-900/30", + iconText: "text-violet-700 dark:text-violet-300", + badgeBg: "bg-violet-100 dark:bg-violet-900/30", + badgeText: "text-violet-700 dark:text-violet-300", + borderHover: "hover:border-violet-500/40 dark:hover:border-violet-400/30", + shadowHover: "hover:shadow-violet-500/10 dark:hover:shadow-violet-400/10", + buttonBg: "bg-violet-600 dark:bg-violet-600", + buttonHover: "hover:bg-violet-700 dark:hover:bg-violet-500", + glowColor: "bg-violet-500/8 dark:bg-violet-600/15", + gradientFrom: "from-violet-500", + gradientVia: "via-violet-400", + ringColor: "ring-violet-500/30", + scoreText: "text-violet-400", + scoreBg: "bg-violet-500/10 border-violet-500/20", + blurBg1: "bg-violet-500/10 dark:bg-violet-600/20", + blurBg2: "bg-violet-400/5 dark:bg-violet-500/10", + methodologyBorder: "border-violet-500/30 hover:border-violet-500/50", + }, + autonomous: { + accent: "cyan", + accentLight: "text-cyan-600", + accentDark: "dark:text-cyan-400", + iconBg: "bg-cyan-100 dark:bg-cyan-900/30", + iconText: "text-cyan-700 dark:text-cyan-300", + badgeBg: "bg-cyan-100 dark:bg-cyan-900/30", + badgeText: "text-cyan-700 dark:text-cyan-300", + borderHover: "hover:border-cyan-500/40 dark:hover:border-cyan-400/30", + shadowHover: "hover:shadow-cyan-500/10 dark:hover:shadow-cyan-400/10", + buttonBg: "bg-cyan-600 dark:bg-cyan-600", + buttonHover: "hover:bg-cyan-700 dark:hover:bg-cyan-500", + glowColor: "bg-cyan-500/8 dark:bg-cyan-600/15", + gradientFrom: "from-cyan-500", + gradientVia: "via-cyan-400", + ringColor: "ring-cyan-500/30", + scoreText: "text-cyan-400", + scoreBg: "bg-cyan-500/10 border-cyan-500/20", + blurBg1: "bg-cyan-500/10 dark:bg-cyan-600/20", + blurBg2: "bg-cyan-400/5 dark:bg-cyan-500/10", + methodologyBorder: "border-cyan-500/30 hover:border-cyan-500/50", + }, +} + +const DEFAULT_THEME = ROLE_THEMES.senior! + +// ── Framer Motion Variants ────────────────────────────────────────────────── + +const containerVariants = { + hidden: { opacity: 0 }, + visible: { + opacity: 1, + transition: { + staggerChildren: 0.12, + delayChildren: 0.1, + }, + }, +} + +const cardVariants = { + hidden: { opacity: 0, y: 30 }, + visible: { + opacity: 1, + y: 0, + transition: { + duration: 0.6, + ease: [0.21, 0.45, 0.27, 0.9] as const, + }, + }, +} + +const fadeUpVariants = { + hidden: { opacity: 0, y: 20 }, + visible: { + opacity: 1, + y: 0, + transition: { + duration: 0.6, + ease: [0.21, 0.45, 0.27, 0.9] as const, + }, + }, +} + +const backgroundVariants = { + hidden: { opacity: 0 }, + visible: { + opacity: 1, + transition: { + duration: 1.2, + ease: "easeOut" as const, + }, + }, +} + +const tableRowVariants = { + hidden: { opacity: 0, x: -10 }, + visible: { + opacity: 1, + x: 0, + transition: { + duration: 0.4, + ease: [0.21, 0.45, 0.27, 0.9] as const, + }, + }, +} + +// ── Helpers ───────────────────────────────────────────────────────────────── + +function scoreBadgeColor(score: number): string { + if (score >= 85) return "bg-green-500/10 text-green-400 border border-green-500/20" + if (score >= 70) return "bg-blue-500/10 text-blue-400 border border-blue-500/20" + if (score >= 50) return "bg-yellow-500/10 text-yellow-400 border border-yellow-500/20" + return "bg-red-500/10 text-red-400 border border-red-500/20" +} + +function tierBadge(tier: ModelCandidate["tier"]): { label: string; className: string } { + switch (tier) { + case "best": + return { + label: "Best", + className: "bg-green-500/10 text-green-400 border border-green-500/20", + } + case "recommended": + return { + label: "Recommended", + className: "bg-blue-500/10 text-blue-400 border border-blue-500/20", + } + case "situational": + return { + label: "Situational", + className: "bg-yellow-500/10 text-yellow-400 border border-yellow-500/20", + } + case "not-recommended": + return { + label: "Not Recommended", + className: "bg-red-500/10 text-red-400 border border-red-500/20", + } + } +} + +const RANK_BADGES = ["🥇", "🥈", "🥉"] + +const LANGUAGE_CONFIG: { key: keyof LanguageScores; label: string; color: string; bgColor: string }[] = [ + { key: "python", label: "Python", color: "bg-green-500", bgColor: "bg-green-500/20" }, + { key: "javascript", label: "JS", color: "bg-yellow-500", bgColor: "bg-yellow-500/20" }, + { key: "java", label: "Java", color: "bg-orange-500", bgColor: "bg-orange-500/20" }, + { key: "go", label: "Go", color: "bg-cyan-500", bgColor: "bg-cyan-500/20" }, + { key: "rust", label: "Rust", color: "bg-red-500", bgColor: "bg-red-500/20" }, +] + +function settingsLabel(candidate: ModelCandidate): string { + const parts = [`temp=${candidate.settings.temperature}`] + if (candidate.settings.reasoningEffort) { + parts.push(`reasoning=${candidate.settings.reasoningEffort}`) + } + return parts.join(", ") +} + +// ── Language Score Bars ───────────────────────────────────────────────────── + +function LanguageBars({ scores }: { scores: LanguageScores }) { + return ( +
+ {LANGUAGE_CONFIG.map(({ key, label, color, bgColor }) => { + const value = scores[key] + return ( +
+ {label} +
+
+ +
+
+ {value} +
+ ) + })} +
+ ) +} + +// ── Composite Score Ring ──────────────────────────────────────────────────── + +function ScoreRing({ score, theme }: { score: number; theme: RoleTheme }) { + const circumference = 2 * Math.PI * 40 + const strokeDashoffset = circumference - (score / 100) * circumference + + return ( +
+ + + + +
+ {score} +
+
+ ) +} + +// ── Candidate Card ────────────────────────────────────────────────────────── + +function CandidateCard({ + candidate, + rank, + theme, + cloudUrl, + highlight, +}: { + candidate: ModelCandidate + rank?: number + theme: RoleTheme + cloudUrl: string + highlight?: "cost" | "speed" +}) { + const tier = tierBadge(candidate.tier) + const copySettings = { + provider: candidate.provider, + model: candidate.modelId, + temperature: candidate.settings.temperature, + ...(candidate.settings.reasoningEffort ? { reasoningEffort: candidate.settings.reasoningEffort } : {}), + } + + return ( + + {/* Subtle glow on hover */} +
+ +
+ {/* Rank badge */} + {rank !== undefined && rank < 3 && ( +
+ {RANK_BADGES[rank]} +
+ )} + + {/* Provider label */} +

+ {candidate.provider} +

+ + {/* Model name */} +

{candidate.displayName}

+ + {/* Tier pill */} +
+ + {tier.label} + +
+ + {/* Score ring */} +
+ +
+ + {/* Key metrics grid */} +
+
+

Success

+

{candidate.successRate}%

+
+
+

+ Daily Cost +

+

+ ~${Math.round(candidate.estimatedDailyCost)}/day +

+

+ (${candidate.avgCostPerTask.toFixed(3)}/task) +

+
+
+

Avg Time

+

+ {candidate.avgTimePerTask.toFixed(1)}s +

+
+
+ + {/* Per-language breakdown */} +
+

+ Language Scores +

+ +
+ + {/* Recommended settings */} +
+ {settingsLabel(candidate)} +
+ + {/* Caveats */} + {candidate.caveats && candidate.caveats.length > 0 && ( +
+ {candidate.caveats.map((caveat) => ( +

+ + {caveat} +

+ ))} +
+ )} + + {/* CTAs */} + +
+ + ) +} + +// ── Compact Card (Budget / Speed) ─────────────────────────────────────────── + +function CompactCard({ + candidate, + label, + icon: IconComp, + highlight, + theme, + cloudUrl, +}: { + candidate: ModelCandidate + label: string + icon: LucideIcon + highlight: "cost" | "speed" + theme: RoleTheme + cloudUrl: string +}) { + const tier = tierBadge(candidate.tier) + const copySettings = { + provider: candidate.provider, + model: candidate.modelId, + temperature: candidate.settings.temperature, + ...(candidate.settings.reasoningEffort ? { reasoningEffort: candidate.settings.reasoningEffort } : {}), + } + + return ( + + {/* Subtle glow on hover */} +
+ +
+ {/* Label header */} +
+
+ +
+

{label}

+
+ +
+ {/* Left: model info + score */} +
+

+ {candidate.provider} +

+

{candidate.displayName}

+ + {/* Score + tier */} +
+ {candidate.compositeScore} + + {tier.label} + +
+
+ + {/* Right: highlighted metric */} +
+

+ {highlight === "cost" ? "Daily Cost" : "Avg Time"} +

+

+ {highlight === "cost" + ? `~$${Math.round(candidate.estimatedDailyCost)}/day` + : `${candidate.avgTimePerTask.toFixed(1)}s`} +

+ {highlight === "cost" && ( +

+ (${candidate.avgCostPerTask.toFixed(3)}/task) +

+ )} +
+
+ + {/* Metrics row */} +
+
+

Success

+

{candidate.successRate}%

+
+
+

+ Daily Cost +

+

+ ~${Math.round(candidate.estimatedDailyCost)} +

+

+ (${candidate.avgCostPerTask.toFixed(3)}/task) +

+
+
+

Time

+

{candidate.avgTimePerTask.toFixed(1)}s

+
+
+ + {/* Language bars */} +
+ +
+ + {/* Settings */} +
+ {settingsLabel(candidate)} +
+ + {/* CTAs */} + +
+ + ) +} + +// ── Props ─────────────────────────────────────────────────────────────────── + +export type CandidatesContentProps = { + roleId: string + role: EngineerRole + best: ModelCandidate[] + budgetHire: ModelCandidate | null + speedHire: ModelCandidate | null + allCandidates: ModelCandidate[] + totalEvalRuns: number + totalExercises: number + lastUpdated: string + cloudUrls: Record +} + +// ── Main Content Component ────────────────────────────────────────────────── + +export function CandidatesContent({ + roleId, + role, + best, + budgetHire, + speedHire, + allCandidates, + totalEvalRuns, + totalExercises, + lastUpdated, + cloudUrls, +}: CandidatesContentProps) { + const theme = ROLE_THEMES[roleId] ?? DEFAULT_THEME + const IconComponent = ICON_MAP[role.icon] ?? Code + + return ( + <> + {/* ── Role Header ────────────────────────────────────────────── */} +
+ {/* Atmospheric blur gradient background */} + +
+
+
+
+
+ + +
+ + {/* Breadcrumb */} + + + Evals + + / + + Hire an AI Engineer + + / + {role.name} + + + {/* Icon + Title row */} + +
+ +
+
+

{role.name}

+ + {role.salaryRange} + +

+ {role.description} +

+
+ + + {/* Stats bar */} + + + + + {totalEvalRuns.toLocaleString()} + + eval runs + +
+ + + + {totalExercises.toLocaleString()} + + exercises + +
+ + Updated{" "} + + {new Date(lastUpdated).toLocaleDateString("en-US", { + month: "short", + day: "numeric", + year: "numeric", + })} + + +
+ + + How we interview + + + + + {/* Strengths + Trade-offs grid */} + +
+

+ Strengths +

+ {role.strengths.map((s) => ( +
+ + {s} +
+ ))} +
+
+

+ Trade-offs +

+ {role.weaknesses.map((w) => ( +
+ + {w} +
+ ))} +
+ + +
+
+ + {/* ── Top Candidates: Best Overall ────────────────────────────── */} +
+
+ + + + Top Candidates + + + + {best.map((candidate, i) => ( + + ))} + + +
+
+ + {/* ── Budget & Speed Hire ─────────────────────────────────────── */} + {(budgetHire || speedHire) && ( +
+
+ + {budgetHire && ( + + )} + {speedHire && ( + + )} + +
+
+ )} + + {/* ── All Candidates Table ────────────────────────────────────── */} +
+ {/* Subtle background */} + +
+
+
+ + +
+ + + All Candidates + + + + + + + + + + + + + + + + + + {allCandidates.map((candidate, i) => { + const tier = tierBadge(candidate.tier) + return ( + + + + + + + + + + + ) + })} + +
+ # + + Model + + Provider + + Score + + Tier + + Success + + Daily Cost + + Time +
+ {i + 1} + {candidate.displayName} + {candidate.provider} + + + {candidate.compositeScore} + + + + {tier.label} + + + {candidate.successRate}% + + ~${Math.round(candidate.estimatedDailyCost)} + + (${candidate.avgCostPerTask.toFixed(3)}) + + + {candidate.avgTimePerTask.toFixed(1)}s +
+ + + {/* Compare link */} + + + 📊 Compare all candidates + + + + +
+
+ + {/* ── Bottom Navigation ───────────────────────────────────────── */} +
+
+ + + + + Back to all roles + + + + + 📊 Compare candidates + + + 📋 Raw eval data + + + + +
+
+ + ) +} diff --git a/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/comparison-chart.tsx b/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/comparison-chart.tsx new file mode 100644 index 0000000000..4b87bbb1cf --- /dev/null +++ b/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/comparison-chart.tsx @@ -0,0 +1,443 @@ +"use client" + +import { useState, useMemo, useCallback } from "react" +import Link from "next/link" +import { ArrowLeft, Copy, Check, FileJson, FileSpreadsheet } from "lucide-react" +import { BarChart, Bar, XAxis, YAxis, Tooltip, ResponsiveContainer, Legend } from "recharts" + +import type { ModelCandidate, LanguageScores, EngineerRole, RoleRecommendation } from "@/lib/mock-recommendations" +import { TASKS_PER_DAY } from "@/lib/mock-recommendations" + +// ── Constants ─────────────────────────────────────────────────────────────── + +const LANGUAGES: { key: keyof LanguageScores; label: string }[] = [ + { key: "go", label: "Go" }, + { key: "java", label: "Java" }, + { key: "javascript", label: "JavaScript" }, + { key: "python", label: "Python" }, + { key: "rust", label: "Rust" }, +] + +const PROVIDERS = ["anthropic", "openai", "google", "deepseek", "groq", "alibaba", "mistral"] as const + +const PROVIDER_LABELS: Record = { + anthropic: "Anthropic", + openai: "OpenAI", + google: "Google", + deepseek: "DeepSeek", + groq: "Meta/Groq", + alibaba: "Alibaba", + mistral: "Mistral", +} + +const DIMENSION_COLORS = { + composite: "#3b82f6", // blue + success: "#22c55e", // green + cost: "#f59e0b", // amber + speed: "#a855f7", // purple +} + +// ── Helpers ───────────────────────────────────────────────────────────────── + +/** Normalize cost: lower cost → higher bar (0–100). */ +function normalizeCost(cost: number, maxCost: number): number { + if (maxCost === 0) return 100 + return Math.round((1 - cost / maxCost) * 100) +} + +/** Normalize speed: lower time → higher bar (0–100). */ +function normalizeSpeed(time: number, maxTime: number): number { + if (maxTime === 0) return 100 + return Math.round((1 - time / maxTime) * 100) +} + +function buildChartData( + candidates: ModelCandidate[], + language: keyof LanguageScores | "all", + maxCost: number, + maxTime: number, +) { + return candidates.map((c) => ({ + name: c.displayName, + composite: language === "all" ? c.compositeScore : c.languageScores[language], + success: c.successRate, + costEfficiency: normalizeCost(c.avgCostPerTask, maxCost), + speed: normalizeSpeed(c.avgTimePerTask, maxTime), + // raw daily cost for tooltip display + dailyCost: Math.round(c.estimatedDailyCost), + costPerTask: c.avgCostPerTask, + // raw data for export + _raw: c, + })) +} + +function candidateToCsvRow(c: ModelCandidate): string { + return [ + c.provider, + c.modelId, + c.displayName, + c.compositeScore, + c.successRate, + c.avgCostPerTask, + Math.round(c.estimatedDailyCost), + c.avgTimePerTask, + c.languageScores.go, + c.languageScores.java, + c.languageScores.javascript, + c.languageScores.python, + c.languageScores.rust, + c.tier, + `"${c.settings.temperature}"`, + `"${c.settings.reasoningEffort ?? ""}"`, + ].join(",") +} + +function downloadBlob(content: string, filename: string, mimeType: string) { + const blob = new Blob([content], { type: mimeType }) + const url = URL.createObjectURL(blob) + const a = document.createElement("a") + a.href = url + a.download = filename + document.body.appendChild(a) + a.click() + document.body.removeChild(a) + URL.revokeObjectURL(url) +} + +// ── Custom Tooltip ────────────────────────────────────────────────────────── + +function CustomTooltip({ + active, + payload, + label, +}: { + active?: boolean + // eslint-disable-next-line @typescript-eslint/no-explicit-any + payload?: any[] + label?: string +}) { + if (!active || !payload || !payload.length) return null + + // Extract raw daily cost from first payload entry's data + const rawData = payload[0]?.payload as { dailyCost?: number; costPerTask?: number } | undefined + const dailyCost = rawData?.dailyCost + const costPerTask = rawData?.costPerTask + + return ( +
+

{label}

+ {payload.map( + ( + entry: { + name: string + value: number + color: string + dataKey: string + }, + index: number, + ) => ( +
+ + {entry.name}: + + {entry.dataKey === "costEfficiency" && dailyCost !== undefined + ? `${entry.value} (~$${dailyCost}/day · $${costPerTask?.toFixed(3)}/task)` + : entry.value} + +
+ ), + )} +
+ ) +} + +// ── Main Component ────────────────────────────────────────────────────────── + +interface ComparisonChartProps { + recommendation: RoleRecommendation + role: EngineerRole + roleId: string +} + +export function ComparisonChart({ recommendation, role, roleId }: ComparisonChartProps) { + const { allCandidates } = recommendation + + // ── State ─────────────────────────────────────────────────────────────── + const [selectedLanguage, setSelectedLanguage] = useState("all") + const [enabledProviders, setEnabledProviders] = useState>(() => new Set(PROVIDERS)) + const [minSuccessRate, setMinSuccessRate] = useState(0) + const [copiedSettings, setCopiedSettings] = useState(false) + + // ── Derived ───────────────────────────────────────────────────────────── + + const filteredCandidates = useMemo( + () => allCandidates.filter((c) => enabledProviders.has(c.provider) && c.successRate >= minSuccessRate), + [allCandidates, enabledProviders, minSuccessRate], + ) + + const maxCost = useMemo(() => Math.max(...allCandidates.map((c) => c.avgCostPerTask), 0.001), [allCandidates]) + + const maxTime = useMemo(() => Math.max(...allCandidates.map((c) => c.avgTimePerTask), 0.1), [allCandidates]) + + const chartData = useMemo( + () => buildChartData(filteredCandidates, selectedLanguage, maxCost, maxTime), + [filteredCandidates, selectedLanguage, maxCost, maxTime], + ) + + const chartHeight = Math.max(300, chartData.length * 60 + 80) + + // ── Handlers ──────────────────────────────────────────────────────────── + + const toggleProvider = useCallback((provider: string) => { + setEnabledProviders((prev) => { + const next = new Set(prev) + if (next.has(provider)) { + next.delete(provider) + } else { + next.add(provider) + } + return next + }) + }, []) + + const handleCopySettings = useCallback(async () => { + const settings = filteredCandidates.map((c) => ({ + provider: c.provider, + model: c.modelId, + displayName: c.displayName, + temperature: c.settings.temperature, + ...(c.settings.reasoningEffort ? { reasoningEffort: c.settings.reasoningEffort } : {}), + })) + await navigator.clipboard.writeText(JSON.stringify(settings, null, 2)) + setCopiedSettings(true) + setTimeout(() => setCopiedSettings(false), 2000) + }, [filteredCandidates]) + + const handleExportCsv = useCallback(() => { + const header = + "Provider,Model ID,Display Name,Composite Score,Success Rate,Avg Cost/Task,Est. Daily Cost,Avg Time/Task,Go,Java,JavaScript,Python,Rust,Tier,Temperature,Reasoning Effort" + const rows = filteredCandidates.map(candidateToCsvRow) + const csv = [header, ...rows].join("\n") + downloadBlob(csv, `${roleId}-comparison.csv`, "text/csv") + }, [filteredCandidates, roleId]) + + const handleExportJson = useCallback(() => { + const json = JSON.stringify(filteredCandidates, null, 2) + downloadBlob(json, `${roleId}-comparison.json`, "application/json") + }, [filteredCandidates, roleId]) + + // ── Render ────────────────────────────────────────────────────────────── + + return ( +
+ {/* ── Breadcrumb ─────────────────────────────────────────────── */} + + + {/* ── Page Header ────────────────────────────────────────────── */} +
+

Compare Candidates — {role.name}

+

+ Interactive comparison across composite score, success rate, cost efficiency, and speed. +

+
+ + {/* ── Language Toggle ─────────────────────────────────────────── */} +
+

+ Score View +

+
+ + {LANGUAGES.map(({ key, label }) => ( + + ))} +
+
+ + {/* ── Filters ────────────────────────────────────────────────── */} +
+ {/* Provider checkboxes */} +
+

+ Providers +

+
+ {PROVIDERS.map((p) => ( + + ))} +
+
+ + {/* Min success rate slider */} +
+

+ Min Success Rate +

+
+ setMinSuccessRate(Number(e.target.value))} + className="h-2 flex-1 cursor-pointer accent-blue-600" + /> + {minSuccessRate}% +
+
+
+ + {/* ── Chart ──────────────────────────────────────────────────── */} +
+

+ {selectedLanguage === "all" + ? "Composite Score" + : `${LANGUAGES.find((l) => l.key === selectedLanguage)?.label} Score`}{" "} + Comparison +

+

+ Cost Efficiency and Speed are inverted — higher bars mean cheaper / faster. Daily costs assume ~ + {TASKS_PER_DAY} tasks per agent per day (~6 productive hours). +

+ + {chartData.length === 0 ? ( +
+ No candidates match the current filters. +
+ ) : ( + + + `${v}`} /> + + } /> + + l.key === selectedLanguage)?.label ?? "Language"} Score` + } + fill={DIMENSION_COLORS.composite} + radius={[0, 4, 4, 0]} + barSize={12} + /> + + + + + + )} +
+ + {/* ── Export Buttons ──────────────────────────────────────────── */} +
+ + + +
+ + {/* ── Bottom Navigation ───────────────────────────────────────── */} + +
+ ) +} diff --git a/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/page.tsx b/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/page.tsx new file mode 100644 index 0000000000..d79b22eedd --- /dev/null +++ b/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/page.tsx @@ -0,0 +1,81 @@ +import { notFound } from "next/navigation" +import type { Metadata } from "next" + +import { SEO } from "@/lib/seo" +import { ogImageUrl } from "@/lib/og" +import { getEngineerRole, getRoleRecommendation } from "@/lib/mock-recommendations" + +import { ComparisonChart } from "./comparison-chart" + +// ── SEO Metadata ──────────────────────────────────────────────────────────── + +type PageProps = { params: Promise<{ roleId: string }> } + +export async function generateMetadata({ params }: PageProps): Promise { + const { roleId } = await params + const role = getEngineerRole(roleId) + + if (!role) { + return { + title: "Role Not Found | Roo Code Evals", + description: "The requested engineer role was not found.", + } + } + + const title = `Compare Candidates — ${role.name} | Roo Code Evals` + const description = `Interactive comparison of AI model candidates for the ${role.name} role. Compare composite score, success rate, cost efficiency, and speed.` + const ogDescription = `Compare Candidates — ${role.name}` + const path = `/evals/workers/${roleId}/compare` + + return { + title, + description, + alternates: { + canonical: `${SEO.url}${path}`, + }, + openGraph: { + title, + description, + url: `${SEO.url}${path}`, + siteName: SEO.name, + images: [ + { + url: ogImageUrl(title, ogDescription), + width: 1200, + height: 630, + alt: title, + }, + ], + locale: SEO.locale, + type: "website", + }, + twitter: { + card: SEO.twitterCard, + title, + description, + images: [ogImageUrl(title, ogDescription)], + }, + keywords: [ + ...SEO.keywords, + "AI engineer", + "model comparison", + "coding evals", + role.name.toLowerCase(), + "bar chart", + "candidate comparison", + ], + } +} + +// ── Page Component ────────────────────────────────────────────────────────── + +export default async function CompareCandidatesPage({ params }: PageProps) { + const { roleId } = await params + const recommendation = getRoleRecommendation(roleId) + + if (!recommendation) { + notFound() + } + + return +} diff --git a/apps/web-roo-code/src/app/evals/workers/[roleId]/copy-settings-button.tsx b/apps/web-roo-code/src/app/evals/workers/[roleId]/copy-settings-button.tsx new file mode 100644 index 0000000000..06adacff5a --- /dev/null +++ b/apps/web-roo-code/src/app/evals/workers/[roleId]/copy-settings-button.tsx @@ -0,0 +1,42 @@ +"use client" + +import { useState } from "react" +import { Copy, Check } from "lucide-react" + +interface CopySettingsButtonProps { + settings: { + provider: string + model: string + temperature: number + reasoningEffort?: string + } +} + +export function CopySettingsButton({ settings }: CopySettingsButtonProps) { + const [copied, setCopied] = useState(false) + + const handleCopy = async () => { + const json = JSON.stringify(settings, null, 2) + await navigator.clipboard.writeText(json) + setCopied(true) + setTimeout(() => setCopied(false), 2000) + } + + return ( + + ) +} diff --git a/apps/web-roo-code/src/app/evals/workers/[roleId]/page.tsx b/apps/web-roo-code/src/app/evals/workers/[roleId]/page.tsx new file mode 100644 index 0000000000..8daa554cd2 --- /dev/null +++ b/apps/web-roo-code/src/app/evals/workers/[roleId]/page.tsx @@ -0,0 +1,104 @@ +import { notFound } from "next/navigation" +import type { Metadata } from "next" + +import { SEO } from "@/lib/seo" +import { ogImageUrl } from "@/lib/og" +import { getRoleRecommendation, getCloudSetupUrl } from "@/lib/mock-recommendations" + +import { CandidatesContent } from "./candidates-content" + +// ── SEO Metadata ──────────────────────────────────────────────────────────── + +type PageProps = { params: Promise<{ roleId: string }> } + +export async function generateMetadata({ params }: PageProps): Promise { + const { roleId } = await params + const recommendation = getRoleRecommendation(roleId) + + if (!recommendation) { + return { + title: "Role Not Found | Roo Code Evals", + description: "The requested engineer role was not found.", + } + } + + const { role } = recommendation + const title = `${role.name} — AI Engineer Candidates | Roo Code Evals` + const description = `Interview results for ${role.name} AI candidates. Compare models by success rate, cost, and speed across 5 languages.` + const ogDescription = `${role.name} — AI Engineer Candidates` + const path = `/evals/workers/${roleId}` + + return { + title, + description, + alternates: { + canonical: `${SEO.url}${path}`, + }, + openGraph: { + title, + description, + url: `${SEO.url}${path}`, + siteName: SEO.name, + images: [ + { + url: ogImageUrl(title, ogDescription), + width: 1200, + height: 630, + alt: title, + }, + ], + locale: SEO.locale, + type: "website", + }, + twitter: { + card: SEO.twitterCard, + title, + description, + images: [ogImageUrl(title, ogDescription)], + }, + keywords: [ + ...SEO.keywords, + "AI engineer", + "model recommendations", + "coding evals", + role.name.toLowerCase(), + "model comparison", + ], + } +} + +// ── Page Component ────────────────────────────────────────────────────────── + +export default async function RoleCandidatesPage({ params }: PageProps) { + const { roleId } = await params + const recommendation = getRoleRecommendation(roleId) + + if (!recommendation) { + notFound() + } + + const { role, best, budgetHire, speedHire, allCandidates, totalEvalRuns, totalExercises, lastUpdated } = + recommendation + + // Pre-compute cloud URLs on the server so the client component receives + // only serializable data (no functions). + const cloudUrls: Record = {} + for (const candidate of allCandidates) { + cloudUrls[candidate.modelId] = getCloudSetupUrl(candidate) + } + + return ( + + ) +} diff --git a/apps/web-roo-code/src/app/evals/workers/page.tsx b/apps/web-roo-code/src/app/evals/workers/page.tsx new file mode 100644 index 0000000000..d7a8f5cc17 --- /dev/null +++ b/apps/web-roo-code/src/app/evals/workers/page.tsx @@ -0,0 +1,85 @@ +import type { Metadata } from "next" + +import { SEO } from "@/lib/seo" +import { ogImageUrl } from "@/lib/og" +import { getEngineerRoles, getAllRecommendations } from "@/lib/mock-recommendations" + +import { WorkersContent } from "./workers-content" + +// ── SEO Metadata ──────────────────────────────────────────────────────────── + +const TITLE = "Hire an AI Engineer | Roo Code Evals" +const DESCRIPTION = + "Find the right AI coding model for your team. Compare interview results across Junior, Senior, and Staff Engineer roles." +const OG_DESCRIPTION = "Find the right AI coding model for your team" +const PATH = "/evals/workers" + +export const metadata: Metadata = { + title: TITLE, + description: DESCRIPTION, + alternates: { + canonical: `${SEO.url}${PATH}`, + }, + openGraph: { + title: TITLE, + description: DESCRIPTION, + url: `${SEO.url}${PATH}`, + siteName: SEO.name, + images: [ + { + url: ogImageUrl(TITLE, OG_DESCRIPTION), + width: 1200, + height: 630, + alt: TITLE, + }, + ], + locale: SEO.locale, + type: "website", + }, + twitter: { + card: SEO.twitterCard, + title: TITLE, + description: DESCRIPTION, + images: [ogImageUrl(TITLE, OG_DESCRIPTION)], + }, + keywords: [ + ...SEO.keywords, + "AI engineer", + "model recommendations", + "coding evals", + "model comparison", + "hire AI", + "talent marketplace", + ], +} + +// ── Page Component ────────────────────────────────────────────────────────── + +export default function HireAnAIEngineerPage() { + const roles = getEngineerRoles() + const recommendations = getAllRecommendations() + + // Aggregate totals + const totalEvalRuns = recommendations.reduce((sum, r) => sum + r.totalEvalRuns, 0) + const totalExercises = recommendations.reduce((sum, r) => sum + r.totalExercises, 0) + + // Unique model count across all roles + const uniqueModels = new Set(recommendations.flatMap((r) => r.allCandidates.map((c) => c.modelId))) + const totalModels = uniqueModels.size + + const lastUpdated = recommendations + .map((r) => r.lastUpdated) + .sort() + .pop() + + return ( + + ) +} diff --git a/apps/web-roo-code/src/app/evals/workers/workers-content.tsx b/apps/web-roo-code/src/app/evals/workers/workers-content.tsx new file mode 100644 index 0000000000..ee0618eb73 --- /dev/null +++ b/apps/web-roo-code/src/app/evals/workers/workers-content.tsx @@ -0,0 +1,519 @@ +"use client" + +import { motion } from "framer-motion" +import { + Code, + GitBranch, + Building2, + Search, + Bot, + ArrowRight, + ChevronDown, + CheckCircle2, + AlertTriangle, + Users, + FlaskConical, + Beaker, + Globe, + TrendingUp, +} from "lucide-react" +import type { LucideIcon } from "lucide-react" +import Link from "next/link" + +import type { EngineerRole, RoleRecommendation } from "@/lib/mock-recommendations" +import { TASKS_PER_DAY } from "@/lib/mock-recommendations" + +// ── Icon Mapping ──────────────────────────────────────────────────────────── + +const ICON_MAP: Record = { + Code, + GitBranch, + Building2, + Search, + Bot, +} + +// ── Color Themes per Role ─────────────────────────────────────────────────── + +type RoleTheme = { + accent: string + accentLight: string + accentDark: string + iconBg: string + iconText: string + badgeBg: string + badgeText: string + borderHover: string + shadowHover: string + buttonBg: string + buttonHover: string + glowColor: string + dotColor: string + strengthColor: string +} + +const ROLE_THEMES: Record = { + junior: { + accent: "emerald", + accentLight: "text-emerald-600", + accentDark: "dark:text-emerald-400", + iconBg: "bg-emerald-100 dark:bg-emerald-900/30", + iconText: "text-emerald-700 dark:text-emerald-300", + badgeBg: "bg-emerald-100 dark:bg-emerald-900/30", + badgeText: "text-emerald-700 dark:text-emerald-300", + borderHover: "hover:border-emerald-500/40 dark:hover:border-emerald-400/30", + shadowHover: "hover:shadow-emerald-500/10 dark:hover:shadow-emerald-400/10", + buttonBg: "bg-emerald-600 dark:bg-emerald-600", + buttonHover: "hover:bg-emerald-700 dark:hover:bg-emerald-500", + glowColor: "bg-emerald-500/8 dark:bg-emerald-600/15", + dotColor: "bg-emerald-500", + strengthColor: "text-emerald-600 dark:text-emerald-400", + }, + senior: { + accent: "blue", + accentLight: "text-blue-600", + accentDark: "dark:text-blue-400", + iconBg: "bg-blue-100 dark:bg-blue-900/30", + iconText: "text-blue-700 dark:text-blue-300", + badgeBg: "bg-blue-100 dark:bg-blue-900/30", + badgeText: "text-blue-700 dark:text-blue-300", + borderHover: "hover:border-blue-500/40 dark:hover:border-blue-400/30", + shadowHover: "hover:shadow-blue-500/10 dark:hover:shadow-blue-400/10", + buttonBg: "bg-blue-600 dark:bg-blue-600", + buttonHover: "hover:bg-blue-700 dark:hover:bg-blue-500", + glowColor: "bg-blue-500/8 dark:bg-blue-600/15", + dotColor: "bg-blue-500", + strengthColor: "text-blue-600 dark:text-blue-400", + }, + staff: { + accent: "amber", + accentLight: "text-amber-600", + accentDark: "dark:text-amber-400", + iconBg: "bg-amber-100 dark:bg-amber-900/30", + iconText: "text-amber-700 dark:text-amber-300", + badgeBg: "bg-amber-100 dark:bg-amber-900/30", + badgeText: "text-amber-700 dark:text-amber-300", + borderHover: "hover:border-amber-500/40 dark:hover:border-amber-400/30", + shadowHover: "hover:shadow-amber-500/10 dark:hover:shadow-amber-400/10", + buttonBg: "bg-amber-600 dark:bg-amber-600", + buttonHover: "hover:bg-amber-700 dark:hover:bg-amber-500", + glowColor: "bg-amber-500/8 dark:bg-amber-600/15", + dotColor: "bg-amber-500", + strengthColor: "text-amber-600 dark:text-amber-400", + }, + reviewer: { + accent: "violet", + accentLight: "text-violet-600", + accentDark: "dark:text-violet-400", + iconBg: "bg-violet-100 dark:bg-violet-900/30", + iconText: "text-violet-700 dark:text-violet-300", + badgeBg: "bg-violet-100 dark:bg-violet-900/30", + badgeText: "text-violet-700 dark:text-violet-300", + borderHover: "hover:border-violet-500/40 dark:hover:border-violet-400/30", + shadowHover: "hover:shadow-violet-500/10 dark:hover:shadow-violet-400/10", + buttonBg: "bg-violet-600 dark:bg-violet-600", + buttonHover: "hover:bg-violet-700 dark:hover:bg-violet-500", + glowColor: "bg-violet-500/8 dark:bg-violet-600/15", + dotColor: "bg-violet-500", + strengthColor: "text-violet-600 dark:text-violet-400", + }, + autonomous: { + accent: "cyan", + accentLight: "text-cyan-600", + accentDark: "dark:text-cyan-400", + iconBg: "bg-cyan-100 dark:bg-cyan-900/30", + iconText: "text-cyan-700 dark:text-cyan-300", + badgeBg: "bg-cyan-100 dark:bg-cyan-900/30", + badgeText: "text-cyan-700 dark:text-cyan-300", + borderHover: "hover:border-cyan-500/40 dark:hover:border-cyan-400/30", + shadowHover: "hover:shadow-cyan-500/10 dark:hover:shadow-cyan-400/10", + buttonBg: "bg-cyan-600 dark:bg-cyan-600", + buttonHover: "hover:bg-cyan-700 dark:hover:bg-cyan-500", + glowColor: "bg-cyan-500/8 dark:bg-cyan-600/15", + dotColor: "bg-cyan-500", + strengthColor: "text-cyan-600 dark:text-cyan-400", + }, +} + +const DEFAULT_THEME = ROLE_THEMES.senior! + +// ── Framer Motion Variants ────────────────────────────────────────────────── + +const containerVariants = { + hidden: { opacity: 0 }, + visible: { + opacity: 1, + transition: { + staggerChildren: 0.15, + delayChildren: 0.2, + }, + }, +} + +const cardVariants = { + hidden: { opacity: 0, y: 30 }, + visible: { + opacity: 1, + y: 0, + transition: { + duration: 0.6, + ease: [0.21, 0.45, 0.27, 0.9] as const, + }, + }, +} + +const fadeUpVariants = { + hidden: { opacity: 0, y: 20 }, + visible: { + opacity: 1, + y: 0, + transition: { + duration: 0.6, + ease: [0.21, 0.45, 0.27, 0.9] as const, + }, + }, +} + +const backgroundVariants = { + hidden: { opacity: 0 }, + visible: { + opacity: 1, + transition: { + duration: 1.2, + ease: "easeOut" as const, + }, + }, +} + +// ── Sub-Components ────────────────────────────────────────────────────────── + +function StatPill({ icon: Icon, value, label }: { icon: LucideIcon; value: string; label: string }) { + return ( +
+ + {value} + {label} +
+ ) +} + +// ── Main Content Component ────────────────────────────────────────────────── + +type WorkersContentProps = { + roles: EngineerRole[] + recommendations: RoleRecommendation[] + totalEvalRuns: number + totalExercises: number + totalModels: number + lastUpdated: string | undefined +} + +export function WorkersContent({ + roles, + recommendations, + totalEvalRuns, + totalExercises, + totalModels, + lastUpdated, +}: WorkersContentProps) { + const recByRole = new Map(recommendations.map((r) => [r.roleId, r])) + + return ( + <> + {/* ── Hero Section ───────────────────────────────────────────── */} +
+ {/* Atmospheric blur background */} + +
+
+
+
+
+ + + {/* Gradient fade from hero atmosphere to cards */} +
+ +
+ + {/* Badge */} + + + + How we interview AI models + + + + + {/* Heading */} + + Hire an{" "} + + AI Engineer + + + + {/* Subheading */} + + Every model runs the same coding tasks, same tools, same time limit. Pick the right + candidate for your team and budget. + + + {/* Stats bar */} + + +
+ +
+ +
+ + + +
+
+ + {/* ── Role Cards Grid ────────────────────────────────────────── */} +
+ {/* Subtle section background */} + +
+
+
+ + +
+ {/* Section connector */} + +

+ Choose your agentic team member +

+ + + + + {roles.map((role) => { + const rec = recByRole.get(role.id) + const IconComponent = ICON_MAP[role.icon] ?? Code + const candidateCount = rec?.allCandidates.length ?? 0 + const exerciseCount = rec?.totalExercises ?? 0 + const theme = ROLE_THEMES[role.id] ?? DEFAULT_THEME + const topModel = rec?.best[0] + + return ( + +
+ {/* Subtle glow on hover */} +
+ +
+ {/* Header: Icon + role badge */} +
+
+ +
+ {topModel && ( + + Top: {topModel.displayName} + + )} +
+ + {/* Role name + salary */} +

{role.name}

+

+ {role.salaryRange} +

+ + {/* Description */} +

+ {role.description} +

+ + {/* Best for */} +
+

+ Best for +

+
+ {role.bestFor.map((item) => ( + + {item} + + ))} +
+
+ + {/* Strengths & Weaknesses side by side */} +
+ {/* Strengths */} +
+

+ Strengths +

+
    + {role.strengths.map((item) => ( +
  • + + {item} +
    • + ))} +
+
+ + {/* Weaknesses */} +
+

+ Trade-offs +

+
    + {role.weaknesses.map((item) => ( +
  • + + {item} +
    • + ))} +
+
+
+ + {/* Bottom stats + CTA */} +
+
+ + + {candidateCount} candidates + + + + {exerciseCount.toLocaleString()} exercises + +
+ + + View Candidates + + +
+
+
+ + ) + })} + +
+
+ + {/* ── Footer / Methodology Section ───────────────────────────── */} +
+
+ + {/* Stats summary */} + + + {totalEvalRuns.toLocaleString()}+ + {" "} + eval runs + • + 5 languages + • + Last updated:{" "} + + {lastUpdated + ? new Date(lastUpdated).toLocaleDateString("en-US", { + year: "numeric", + month: "long", + day: "numeric", + }) + : "N/A"} + + + + {/* Assumption note */} + + Daily costs assume ~{TASKS_PER_DAY} tasks per agent per day (~6 productive hours including + overhead). + + + {/* Links */} + + + + Our methodology + + + + Raw eval data + + + + +
+
+ + ) +} diff --git a/apps/web-roo-code/src/lib/mock-recommendations.ts b/apps/web-roo-code/src/lib/mock-recommendations.ts new file mode 100644 index 0000000000..214d12bd27 --- /dev/null +++ b/apps/web-roo-code/src/lib/mock-recommendations.ts @@ -0,0 +1,866 @@ +// --------------------------------------------------------------------------- +// Eval Recommendations: Types + Mock Data (S1.1a) +// --------------------------------------------------------------------------- +// This file defines the API contract for the AI Engineer Talent Marketplace. +// The backend (Sprint 3-4) will produce data matching these exact types. +// --------------------------------------------------------------------------- + +// ── Constants ────────────────────────────────────────────────────────────── + +/** + * Estimated tasks per agent per day. + * Assumes ~6 productive hours with overhead for setup, review, and iteration. + * One human engineer typically manages 2-3 agents throughout a workday. + */ +export const TASKS_PER_DAY = 80 + +// ── Types ────────────────────────────────────────────────────────────────── + +/** Engineer role definition: maps task complexity to a hiring tier. */ +export type EngineerRole = { + id: string + name: string + /** Daily salary range string, e.g. "~$3–38/day" */ + salaryRange: string + description: string + bestFor: string[] + strengths: string[] + weaknesses: string[] + icon: string +} + +/** Language-specific eval scores (0–100). */ +export type LanguageScores = { + go: number + java: number + javascript: number + python: number + rust: number +} + +/** Model inference settings used during evaluation. */ +export type ModelSettings = { + temperature: number + reasoningEffort?: string +} + +/** A model candidate evaluated for a specific role. */ +export type ModelCandidate = { + provider: string + modelId: string + displayName: string + compositeScore: number + tier: "best" | "recommended" | "situational" | "not-recommended" + tags: string[] + successRate: number + avgCostPerTask: number + /** Estimated daily cost: avgCostPerTask × TASKS_PER_DAY */ + estimatedDailyCost: number + avgTimePerTask: number + languageScores: LanguageScores + settings: ModelSettings + caveats?: string[] +} + +/** Full recommendation payload for a single role. */ +export type RoleRecommendation = { + roleId: string + role: EngineerRole + lastUpdated: string + totalEvalRuns: number + totalExercises: number + best: ModelCandidate[] + budgetHire: ModelCandidate | null + speedHire: ModelCandidate | null + allCandidates: ModelCandidate[] +} + +// ── Engineer Role Configs ────────────────────────────────────────────────── + +const ENGINEER_ROLES: EngineerRole[] = [ + { + id: "junior", + name: "Junior Engineer", + salaryRange: "~$2–10/day", + description: + "Handles well-scoped, single-file tasks: boilerplate, simple bug fixes, and test generation at the lowest cost per task.", + bestFor: ["Single-file fixes", "Boilerplate generation", "Test generation", "Simple implementations"], + strengths: ["Cheap", "High throughput", "Best cost-to-quality ratio on simple tasks"], + weaknesses: ["Struggles with multi-file changes", "Limited reasoning depth", "May miss edge cases"], + icon: "Code", + }, + { + id: "senior", + name: "Senior Engineer", + salaryRange: "~$10–26/day", + description: + "The sweet spot for most engineering work. Senior-tier models balance cost and quality across multi-file refactors, feature development, and debugging.", + bestFor: ["Multi-file refactors", "Feature development", "Debugging", "Code review"], + strengths: [ + "Balanced cost/quality", + "Handles multi-file changes and cross-cutting refactors", + "Consistent pass rates across all five languages", + ], + weaknesses: ["More expensive than junior", "Overkill for trivial tasks"], + icon: "GitBranch", + }, + { + id: "staff", + name: "Staff Engineer", + salaryRange: "~$8–34/day", + description: + "For architecture decisions, system design, and complex refactors. Staff-tier models handle ambiguous requirements and cross-cutting changes where other tiers fail.", + bestFor: ["Architecture decisions", "Complex features", "System design", "Ambiguous requirements"], + strengths: [ + "Handles multi-step reasoning and ambiguous specs", + "Passes existing test suites consistently", + "Resolves underspecified requirements", + ], + weaknesses: ["Most expensive", "Overkill for simple tasks", "Diminishing returns on easy work"], + icon: "Building2", + }, + { + id: "reviewer", + name: "Architecture Reviewer", + salaryRange: "~$15–40/day", + description: + "For code review, PR feedback, security analysis, and design critique. Reviewer-tier models catch issues other models miss and provide actionable, context-aware suggestions.", + bestFor: ["Code review", "PR feedback", "Security analysis", "Design critique", "Refactor guidance"], + strengths: [ + "Catches subtle bugs and logic errors", + "Provides actionable suggestions with context", + "Understands cross-file impact of changes", + ], + weaknesses: [ + "Not for writing code from scratch", + "More expensive than running linters", + "Review quality varies by codebase size", + ], + icon: "Search", + }, + { + id: "autonomous", + name: "Autonomous Agent", + salaryRange: "~$5–30/day", + description: + "For issue-to-PR workflows, long-running tasks, and multi-step debugging with minimal supervision. Autonomous-tier models complete tasks end-to-end and recover from errors without human intervention.", + bestFor: [ + "Issue-to-PR workflows", + "Multi-step debugging", + "Feature implementation from spec", + "Long-running tasks", + "Batch operations", + ], + strengths: [ + "Completes tasks end-to-end with minimal guidance", + "Recovers from errors and retries automatically", + "Handles ambiguous requirements independently", + ], + weaknesses: [ + "Higher cost per completed task due to retries", + "May take unexpected approaches without oversight", + "Results need review before merging", + ], + icon: "Bot", + }, +] + +// ── Model Candidates (derived from roocode.com/evals data) ───────────────── +// Cost per task = total run cost ÷ 120 exercises +// Time per task = total duration (seconds) ÷ 120 exercises +// Composite scores computed using role-specific weights: +// Junior: success 50%, speed 20%, cost 25%, quality 5% +// Senior: success 40%, quality 25%, cost 20%, speed 15% +// Staff: success 40%, quality 35%, cost 15%, speed 10% +// Quality = consistency across languages (lower variance → higher score) + +// --- Junior Role Candidates ------------------------------------------------- + +const juniorCandidates: ModelCandidate[] = [ + { + provider: "xai", + modelId: "grok-4-fast", + displayName: "Grok 4 Fast", + compositeScore: 94, + tier: "best", + tags: ["best-value"], + successRate: 97, + avgCostPerTask: 0.029, + estimatedDailyCost: 0.029 * TASKS_PER_DAY, + avgTimePerTask: 144.0, + languageScores: { go: 97, java: 96, javascript: 98, python: 100, rust: 97 }, + settings: { temperature: 0 }, + }, + { + provider: "openai", + modelId: "gpt-5-mini", + displayName: "GPT-5 Mini", + compositeScore: 92, + tier: "best", + tags: [], + successRate: 99, + avgCostPerTask: 0.028, + estimatedDailyCost: 0.028 * TASKS_PER_DAY, + avgTimePerTask: 173.0, + languageScores: { go: 100, java: 98, javascript: 100, python: 100, rust: 97 }, + settings: { temperature: 0 }, + }, + { + provider: "xai", + modelId: "grok-code-fast-1", + displayName: "Grok Code Fast 1", + compositeScore: 85, + tier: "best", + tags: [], + successRate: 90, + avgCostPerTask: 0.057, + estimatedDailyCost: 0.057 * TASKS_PER_DAY, + avgTimePerTask: 146.0, + languageScores: { go: 92, java: 91, javascript: 88, python: 94, rust: 83 }, + settings: { temperature: 0 }, + caveats: ["Weaker on Rust (83%): consider alternatives for Rust-heavy tasks"], + }, + { + provider: "google", + modelId: "gemini-2.5-flash", + displayName: "Gemini 2.5 Flash", + compositeScore: 82, + tier: "recommended", + tags: ["speed-hire"], + successRate: 90, + avgCostPerTask: 0.118, + estimatedDailyCost: 0.118 * TASKS_PER_DAY, + avgTimePerTask: 109.5, + languageScores: { go: 89, java: 91, javascript: 92, python: 85, rust: 90 }, + settings: { temperature: 0 }, + }, + { + provider: "openai", + modelId: "gpt-4.1-mini", + displayName: "GPT-4.1 Mini", + compositeScore: 77, + tier: "recommended", + tags: [], + successRate: 83, + avgCostPerTask: 0.073, + estimatedDailyCost: 0.073 * TASKS_PER_DAY, + avgTimePerTask: 158.5, + languageScores: { go: 81, java: 84, javascript: 94, python: 76, rust: 70 }, + settings: { temperature: 0 }, + caveats: ["Inconsistent across languages: Python (76%) to JavaScript (94%)"], + }, + { + provider: "anthropic", + modelId: "claude-haiku-4-5", + displayName: "Claude Haiku 4.5", + compositeScore: 77, + tier: "recommended", + tags: [], + successRate: 95, + avgCostPerTask: 0.159, + estimatedDailyCost: 0.159 * TASKS_PER_DAY, + avgTimePerTask: 139.0, + languageScores: { go: 92, java: 93, javascript: 94, python: 97, rust: 100 }, + settings: { temperature: 0 }, + caveats: ["Most expensive in junior tier. Consider Grok 4 Fast for better cost-to-quality ratio."], + }, + { + provider: "openai", + modelId: "gpt-5-nano", + displayName: "GPT-5 Nano", + compositeScore: 73, + tier: "situational", + tags: ["budget-hire"], + successRate: 78, + avgCostPerTask: 0.013, + estimatedDailyCost: 0.013 * TASKS_PER_DAY, + avgTimePerTask: 276.5, + languageScores: { go: 86, java: 73, javascript: 76, python: 79, rust: 77 }, + settings: { temperature: 0 }, + caveats: ["Cheapest option but slowest: 4.6 min/task average"], + }, + { + provider: "deepseek", + modelId: "deepseek-v3", + displayName: "DeepSeek V3", + compositeScore: 66, + tier: "situational", + tags: [], + successRate: 77, + avgCostPerTask: 0.107, + estimatedDailyCost: 0.107 * TASKS_PER_DAY, + avgTimePerTask: 216.0, + languageScores: { go: 83, java: 76, javascript: 82, python: 76, rust: 67 }, + settings: { temperature: 0 }, + caveats: ["Weakest on Rust (67%)", "Open-source model, self-hostable"], + }, +] + +// --- Senior Role Candidates ------------------------------------------------- + +const seniorCandidates: ModelCandidate[] = [ + { + provider: "moonshot", + modelId: "kimi-k2-0905", + displayName: "Kimi K2 0905", + compositeScore: 95, + tier: "best", + tags: ["budget-hire", "best-value"], + successRate: 94, + avgCostPerTask: 0.127, + estimatedDailyCost: 0.127 * TASKS_PER_DAY, + avgTimePerTask: 112.0, + languageScores: { go: 94, java: 91, javascript: 96, python: 97, rust: 93 }, + settings: { temperature: 0 }, + caveats: ["Tested via Groq; latency may vary by provider"], + }, + { + provider: "openai", + modelId: "gpt-4.1", + displayName: "GPT-4.1", + compositeScore: 87, + tier: "best", + tags: [], + successRate: 91, + avgCostPerTask: 0.322, + estimatedDailyCost: 0.322 * TASKS_PER_DAY, + avgTimePerTask: 139.5, + languageScores: { go: 92, java: 91, javascript: 90, python: 94, rust: 90 }, + settings: { temperature: 0 }, + }, + { + provider: "anthropic", + modelId: "claude-sonnet-4", + displayName: "Claude Sonnet 4", + compositeScore: 84, + tier: "best", + tags: ["top-performer"], + successRate: 98, + avgCostPerTask: 0.33, + estimatedDailyCost: 0.33 * TASKS_PER_DAY, + avgTimePerTask: 167.5, + languageScores: { go: 94, java: 100, javascript: 98, python: 100, rust: 97 }, + settings: { temperature: 0 }, + }, + { + provider: "openai", + modelId: "gpt-5-medium", + displayName: "GPT-5 (Medium)", + compositeScore: 81, + tier: "recommended", + tags: [], + successRate: 98, + avgCostPerTask: 0.193, + estimatedDailyCost: 0.193 * TASKS_PER_DAY, + avgTimePerTask: 260.0, + languageScores: { go: 97, java: 98, javascript: 100, python: 100, rust: 93 }, + settings: { temperature: 0, reasoningEffort: "medium" }, + caveats: ["Slowest in tier: 4.3 min/task average"], + }, + { + provider: "anthropic", + modelId: "claude-3.7-sonnet", + displayName: "Claude 3.7 Sonnet", + compositeScore: 79, + tier: "recommended", + tags: [], + successRate: 95, + avgCostPerTask: 0.313, + estimatedDailyCost: 0.313 * TASKS_PER_DAY, + avgTimePerTask: 176.5, + languageScores: { go: 92, java: 98, javascript: 94, python: 100, rust: 93 }, + settings: { temperature: 0 }, + }, + { + provider: "anthropic", + modelId: "claude-3.5-sonnet", + displayName: "Claude 3.5 Sonnet", + compositeScore: 78, + tier: "recommended", + tags: ["speed-hire"], + successRate: 90, + avgCostPerTask: 0.208, + estimatedDailyCost: 0.208 * TASKS_PER_DAY, + avgTimePerTask: 108.5, + languageScores: { go: 94, java: 91, javascript: 92, python: 88, rust: 80 }, + settings: { temperature: 0 }, + caveats: ["Previous generation; weaker on Rust (80%)"], + }, + { + provider: "openai", + modelId: "gpt-5-low", + displayName: "GPT-5 (Low)", + compositeScore: 76, + tier: "situational", + tags: [], + successRate: 95, + avgCostPerTask: 0.135, + estimatedDailyCost: 0.135 * TASKS_PER_DAY, + avgTimePerTask: 175.0, + languageScores: { go: 100, java: 96, javascript: 86, python: 100, rust: 100 }, + settings: { temperature: 0, reasoningEffort: "low" }, + caveats: ["Weak on JavaScript (86%) compared to other languages"], + }, + { + provider: "google", + modelId: "gemini-2.5-pro", + displayName: "Gemini 2.5 Pro", + compositeScore: 73, + tier: "situational", + tags: [], + successRate: 96, + avgCostPerTask: 0.482, + estimatedDailyCost: 0.482 * TASKS_PER_DAY, + avgTimePerTask: 188.5, + languageScores: { go: 97, java: 91, javascript: 96, python: 100, rust: 97 }, + settings: { temperature: 0 }, + caveats: ["Most expensive in this tier: ~$39/day ($0.48/task)"], + }, +] + +// --- Staff Role Candidates -------------------------------------------------- + +const staffCandidates: ModelCandidate[] = [ + { + provider: "openai", + modelId: "gpt-5.2-med", + displayName: "GPT 5.2 (Med)", + compositeScore: 99, + tier: "best", + tags: ["budget-hire", "best-value"], + successRate: 100, + avgCostPerTask: 0.104, + estimatedDailyCost: 0.104 * TASKS_PER_DAY, + avgTimePerTask: 105.5, + languageScores: { go: 100, java: 100, javascript: 100, python: 100, rust: 100 }, + settings: { temperature: 0, reasoningEffort: "medium" }, + caveats: ["100% pass rate at ~$8/day ($0.10/task): best cost-to-quality ratio in this role"], + }, + { + provider: "anthropic", + modelId: "claude-opus-4-6", + displayName: "Claude Opus 4.6", + compositeScore: 98, + tier: "best", + tags: ["speed-hire", "top-performer"], + successRate: 100, + avgCostPerTask: 0.412, + estimatedDailyCost: 0.412 * TASKS_PER_DAY, + avgTimePerTask: 76.5, + languageScores: { go: 100, java: 100, javascript: 100, python: 100, rust: 100 }, + settings: { temperature: 0 }, + }, + { + provider: "anthropic", + modelId: "claude-sonnet-4-5", + displayName: "Claude Sonnet 4.5", + compositeScore: 97, + tier: "best", + tags: [], + successRate: 100, + avgCostPerTask: 0.32, + estimatedDailyCost: 0.32 * TASKS_PER_DAY, + avgTimePerTask: 103.0, + languageScores: { go: 100, java: 100, javascript: 100, python: 100, rust: 100 }, + settings: { temperature: 0 }, + }, + { + provider: "anthropic", + modelId: "claude-opus-4-5", + displayName: "Claude Opus 4.5", + compositeScore: 96, + tier: "recommended", + tags: [], + successRate: 100, + avgCostPerTask: 0.419, + estimatedDailyCost: 0.419 * TASKS_PER_DAY, + avgTimePerTask: 124.0, + languageScores: { go: 100, java: 100, javascript: 100, python: 100, rust: 100 }, + settings: { temperature: 0 }, + }, + { + provider: "google", + modelId: "gemini-3-pro-preview", + displayName: "Gemini 3 Pro Preview", + compositeScore: 95, + tier: "recommended", + tags: [], + successRate: 100, + avgCostPerTask: 0.276, + estimatedDailyCost: 0.276 * TASKS_PER_DAY, + avgTimePerTask: 164.0, + languageScores: { go: 100, java: 100, javascript: 100, python: 100, rust: 100 }, + settings: { temperature: 0 }, + }, + { + provider: "anthropic", + modelId: "claude-opus-4-1", + displayName: "Claude Opus 4.1", + compositeScore: 73, + tier: "situational", + tags: [], + successRate: 98, + avgCostPerTask: 1.168, + estimatedDailyCost: 1.168 * TASKS_PER_DAY, + avgTimePerTask: 211.5, + languageScores: { go: 97, java: 96, javascript: 98, python: 100, rust: 100 }, + settings: { temperature: 0 }, + caveats: ["~$93/day ($1.17/task), 11× the cost of the top pick"], + }, + { + provider: "openai", + modelId: "gpt-5-medium", + displayName: "GPT-5 (Medium)", + compositeScore: 71, + tier: "situational", + tags: [], + successRate: 98, + avgCostPerTask: 0.193, + estimatedDailyCost: 0.193 * TASKS_PER_DAY, + avgTimePerTask: 260.0, + languageScores: { go: 97, java: 98, javascript: 100, python: 100, rust: 93 }, + settings: { temperature: 0, reasoningEffort: "medium" }, + caveats: ["Slowest in tier: 4.3 min/task average"], + }, + { + provider: "anthropic", + modelId: "claude-opus-4", + displayName: "Claude Opus 4", + compositeScore: 57, + tier: "not-recommended", + tags: [], + successRate: 94, + avgCostPerTask: 1.436, + estimatedDailyCost: 1.436 * TASKS_PER_DAY, + avgTimePerTask: 235.0, + languageScores: { go: 92, java: 91, javascript: 94, python: 94, rust: 100 }, + settings: { temperature: 0 }, + caveats: [ + "Most expensive model tested: ~$115/day ($1.44/task)", + "Lower success rate (94%) despite highest cost", + ], + }, +] + +// --- Architecture Reviewer Candidates --------------------------------------- +// Composite scoring: quality 50%, success 30%, cost 15%, speed 5% +// Quality = consistency across languages (lower variance → higher score) + +const reviewerCandidates: ModelCandidate[] = [ + { + provider: "openai", + modelId: "gpt-5.2-med", + displayName: "GPT 5.2 (Med)", + compositeScore: 98, + tier: "best", + tags: ["budget-hire", "best-value"], + successRate: 100, + avgCostPerTask: 0.104, + estimatedDailyCost: 0.104 * TASKS_PER_DAY, + avgTimePerTask: 105.5, + languageScores: { go: 100, java: 100, javascript: 100, python: 100, rust: 100 }, + settings: { temperature: 0, reasoningEffort: "medium" }, + caveats: ["100% consistency across all languages: ideal reviewer at ~$8/day"], + }, + { + provider: "anthropic", + modelId: "claude-opus-4-6", + displayName: "Claude Opus 4.6", + compositeScore: 95, + tier: "best", + tags: ["speed-hire", "top-performer"], + successRate: 100, + avgCostPerTask: 0.412, + estimatedDailyCost: 0.412 * TASKS_PER_DAY, + avgTimePerTask: 76.5, + languageScores: { go: 100, java: 100, javascript: 100, python: 100, rust: 100 }, + settings: { temperature: 0 }, + }, + { + provider: "anthropic", + modelId: "claude-sonnet-4-5", + displayName: "Claude Sonnet 4.5", + compositeScore: 94, + tier: "best", + tags: [], + successRate: 100, + avgCostPerTask: 0.32, + estimatedDailyCost: 0.32 * TASKS_PER_DAY, + avgTimePerTask: 103.0, + languageScores: { go: 100, java: 100, javascript: 100, python: 100, rust: 100 }, + settings: { temperature: 0 }, + }, + { + provider: "anthropic", + modelId: "claude-sonnet-4", + displayName: "Claude Sonnet 4", + compositeScore: 90, + tier: "recommended", + tags: ["top-performer"], + successRate: 98, + avgCostPerTask: 0.33, + estimatedDailyCost: 0.33 * TASKS_PER_DAY, + avgTimePerTask: 167.5, + languageScores: { go: 94, java: 100, javascript: 98, python: 100, rust: 97 }, + settings: { temperature: 0 }, + }, + { + provider: "anthropic", + modelId: "claude-haiku-4-5", + displayName: "Claude Haiku 4.5", + compositeScore: 88, + tier: "recommended", + tags: [], + successRate: 95, + avgCostPerTask: 0.159, + estimatedDailyCost: 0.159 * TASKS_PER_DAY, + avgTimePerTask: 139.0, + languageScores: { go: 92, java: 93, javascript: 94, python: 97, rust: 100 }, + settings: { temperature: 0 }, + caveats: ["Budget reviewer option: good consistency at lower cost"], + }, + { + provider: "anthropic", + modelId: "claude-3.7-sonnet", + displayName: "Claude 3.7 Sonnet", + compositeScore: 86, + tier: "recommended", + tags: [], + successRate: 95, + avgCostPerTask: 0.313, + estimatedDailyCost: 0.313 * TASKS_PER_DAY, + avgTimePerTask: 176.5, + languageScores: { go: 92, java: 98, javascript: 94, python: 100, rust: 93 }, + settings: { temperature: 0 }, + }, + { + provider: "google", + modelId: "gemini-2.5-pro", + displayName: "Gemini 2.5 Pro", + compositeScore: 82, + tier: "situational", + tags: [], + successRate: 96, + avgCostPerTask: 0.482, + estimatedDailyCost: 0.482 * TASKS_PER_DAY, + avgTimePerTask: 188.5, + languageScores: { go: 97, java: 91, javascript: 96, python: 100, rust: 97 }, + settings: { temperature: 0 }, + caveats: ["Most expensive reviewer: ~$39/day ($0.48/task)", "More variable across languages than top picks"], + }, + { + provider: "openai", + modelId: "gpt-4.1", + displayName: "GPT-4.1", + compositeScore: 80, + tier: "situational", + tags: [], + successRate: 91, + avgCostPerTask: 0.322, + estimatedDailyCost: 0.322 * TASKS_PER_DAY, + avgTimePerTask: 139.5, + languageScores: { go: 92, java: 91, javascript: 90, python: 94, rust: 90 }, + settings: { temperature: 0 }, + caveats: ["Lower consistency across languages than Anthropic alternatives"], + }, +] + +// --- Autonomous Agent Candidates -------------------------------------------- +// Composite scoring: success 35%, quality 35%, cost 20%, speed 10% +// Focused on end-to-end task completion and error recovery + +const autonomousCandidates: ModelCandidate[] = [ + { + provider: "openai", + modelId: "gpt-5.2-med", + displayName: "GPT 5.2 (Med)", + compositeScore: 97, + tier: "best", + tags: ["best-value", "speed-hire"], + successRate: 100, + avgCostPerTask: 0.104, + estimatedDailyCost: 0.104 * TASKS_PER_DAY, + avgTimePerTask: 105.5, + languageScores: { go: 100, java: 100, javascript: 100, python: 100, rust: 100 }, + settings: { temperature: 0, reasoningEffort: "medium" }, + caveats: ["Perfect success rate + fast completion: ideal autonomous agent at ~$8/day"], + }, + { + provider: "openai", + modelId: "gpt-5-mini", + displayName: "GPT-5 Mini", + compositeScore: 93, + tier: "best", + tags: ["budget-hire"], + successRate: 99, + avgCostPerTask: 0.028, + estimatedDailyCost: 0.028 * TASKS_PER_DAY, + avgTimePerTask: 173.0, + languageScores: { go: 100, java: 98, javascript: 100, python: 100, rust: 97 }, + settings: { temperature: 0 }, + caveats: ["Cheapest autonomous option at ~$2/day with near-perfect success"], + }, + { + provider: "xai", + modelId: "grok-4-fast", + displayName: "Grok 4 Fast", + compositeScore: 92, + tier: "best", + tags: [], + successRate: 97, + avgCostPerTask: 0.029, + estimatedDailyCost: 0.029 * TASKS_PER_DAY, + avgTimePerTask: 144.0, + languageScores: { go: 97, java: 96, javascript: 98, python: 100, rust: 97 }, + settings: { temperature: 0 }, + }, + { + provider: "anthropic", + modelId: "claude-sonnet-4", + displayName: "Claude Sonnet 4", + compositeScore: 87, + tier: "recommended", + tags: ["top-performer"], + successRate: 98, + avgCostPerTask: 0.33, + estimatedDailyCost: 0.33 * TASKS_PER_DAY, + avgTimePerTask: 167.5, + languageScores: { go: 94, java: 100, javascript: 98, python: 100, rust: 97 }, + settings: { temperature: 0 }, + }, + { + provider: "moonshot", + modelId: "kimi-k2-0905", + displayName: "Kimi K2 0905", + compositeScore: 86, + tier: "recommended", + tags: [], + successRate: 94, + avgCostPerTask: 0.127, + estimatedDailyCost: 0.127 * TASKS_PER_DAY, + avgTimePerTask: 112.0, + languageScores: { go: 94, java: 91, javascript: 96, python: 97, rust: 93 }, + settings: { temperature: 0 }, + caveats: ["Tested via Groq; latency may vary by provider"], + }, + { + provider: "anthropic", + modelId: "claude-haiku-4-5", + displayName: "Claude Haiku 4.5", + compositeScore: 85, + tier: "recommended", + tags: [], + successRate: 95, + avgCostPerTask: 0.159, + estimatedDailyCost: 0.159 * TASKS_PER_DAY, + avgTimePerTask: 139.0, + languageScores: { go: 92, java: 93, javascript: 94, python: 97, rust: 100 }, + settings: { temperature: 0 }, + }, + { + provider: "openai", + modelId: "gpt-5-low", + displayName: "GPT-5 (Low)", + compositeScore: 82, + tier: "situational", + tags: [], + successRate: 95, + avgCostPerTask: 0.135, + estimatedDailyCost: 0.135 * TASKS_PER_DAY, + avgTimePerTask: 175.0, + languageScores: { go: 100, java: 96, javascript: 86, python: 100, rust: 100 }, + settings: { temperature: 0, reasoningEffort: "low" }, + caveats: ["Weak on JavaScript (86%) compared to other languages"], + }, + { + provider: "openai", + modelId: "gpt-5-medium", + displayName: "GPT-5 (Medium)", + compositeScore: 80, + tier: "situational", + tags: [], + successRate: 98, + avgCostPerTask: 0.193, + estimatedDailyCost: 0.193 * TASKS_PER_DAY, + avgTimePerTask: 260.0, + languageScores: { go: 97, java: 98, javascript: 100, python: 100, rust: 93 }, + settings: { temperature: 0, reasoningEffort: "medium" }, + caveats: ["Slowest in tier: 4.3 min/task average"], + }, +] + +// ── Recommendation Builders ──────────────────────────────────────────────── + +function findBudgetHire(candidates: ModelCandidate[]): ModelCandidate | null { + const budget = candidates + .filter((c) => c.tags.includes("budget-hire")) + .sort((a, b) => a.avgCostPerTask - b.avgCostPerTask) + return budget[0] ?? null +} + +function findSpeedHire(candidates: ModelCandidate[]): ModelCandidate | null { + const fast = [...candidates] + .filter((c) => c.tier !== "not-recommended") + .sort((a, b) => a.avgTimePerTask - b.avgTimePerTask) + return fast[0] ?? null +} + +function buildRecommendation( + role: EngineerRole, + candidates: ModelCandidate[], + totalEvalRuns: number, + totalExercises: number, +): RoleRecommendation { + const sorted = [...candidates].sort((a, b) => b.compositeScore - a.compositeScore) + return { + roleId: role.id, + role, + lastUpdated: "2026-02-11T00:00:00Z", + totalEvalRuns, + totalExercises, + best: sorted.filter((c) => c.tier === "best").slice(0, 3), + budgetHire: findBudgetHire(sorted), + speedHire: findSpeedHire(sorted), + allCandidates: sorted, + } +} + +// ── Pre-built Recommendations ────────────────────────────────────────────── + +const RECOMMENDATIONS: Record = { + junior: buildRecommendation(ENGINEER_ROLES[0]!, juniorCandidates, 27, 120), + senior: buildRecommendation(ENGINEER_ROLES[1]!, seniorCandidates, 27, 120), + staff: buildRecommendation(ENGINEER_ROLES[2]!, staffCandidates, 27, 120), + reviewer: buildRecommendation(ENGINEER_ROLES[3]!, reviewerCandidates, 27, 120), + autonomous: buildRecommendation(ENGINEER_ROLES[4]!, autonomousCandidates, 27, 120), +} + +// ── Public API ───────────────────────────────────────────────────────────── + +/** Returns all engineer role configurations. */ +export function getEngineerRoles(): EngineerRole[] { + return ENGINEER_ROLES +} + +/** Returns a single engineer role by id, or `undefined` if not found. */ +export function getEngineerRole(roleId: string): EngineerRole | undefined { + return ENGINEER_ROLES.find((r) => r.id === roleId) +} + +/** Returns the full recommendation payload for a role, or `undefined` if not found. */ +export function getRoleRecommendation(roleId: string): RoleRecommendation | undefined { + return RECOMMENDATIONS[roleId] +} + +/** Returns recommendation payloads for all roles. */ +export function getAllRecommendations(): RoleRecommendation[] { + return Object.values(RECOMMENDATIONS) +} + +/** Generates a Cloud signup URL pre-configured with the candidate's model settings. */ +export function getCloudSetupUrl(candidate: ModelCandidate): string { + const params = new URLSearchParams({ + redirect_url: `/cloud-agents/setup?model=${candidate.modelId}&provider=${candidate.provider}&temperature=${candidate.settings.temperature}`, + }) + return `https://app.roocode.com/sign-up?${params.toString()}` +}