diff --git a/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/comparison-chart.tsx b/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/comparison-chart.tsx index bc9b9cb8df..8e5aa85b7f 100644 --- a/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/comparison-chart.tsx +++ b/apps/web-roo-code/src/app/evals/workers/[roleId]/compare/comparison-chart.tsx @@ -15,7 +15,20 @@ import { Download, FlaskConical, } from "lucide-react" -import { BarChart, Bar, XAxis, YAxis, Tooltip, ResponsiveContainer, Legend } from "recharts" +import { + BarChart, + Bar, + XAxis, + YAxis, + Tooltip, + ResponsiveContainer, + Legend, + ScatterChart, + Scatter, + ZAxis, + Cell, + ReferenceArea, +} from "recharts" import type { ModelCandidate, LanguageScores, EngineerRole, RoleRecommendation } from "@/lib/mock-recommendations" import { TASKS_PER_DAY } from "@/lib/mock-recommendations" @@ -225,6 +238,20 @@ const DIMENSION_COLORS = { speed: "#a855f7", // purple } +const TIER_COLORS: Record = { + best: "#22c55e", // green + recommended: "#3b82f6", // blue + situational: "#eab308", // yellow + "not-recommended": "#ef4444", // red +} + +const TIER_LABELS: Record = { + best: "Best", + recommended: "Recommended", + situational: "Situational", + "not-recommended": "Not Recommended", +} + // ── Helpers ───────────────────────────────────────────────────────────────── /** Normalize cost: lower cost → higher bar (0–100). */ @@ -344,6 +371,62 @@ function CustomTooltip({ ) } +// ── Scatter Tooltip ───────────────────────────────────────────────────────── + +function ScatterTooltip({ + active, + payload, +}: { + active?: boolean + // eslint-disable-next-line @typescript-eslint/no-explicit-any + payload?: any[] +}) { + if (!active || !payload || !payload.length) return null + + const data = payload[0]?.payload as + | { + name?: string + dailyCost?: number + score?: number + successRate?: number + tier?: string + } + | undefined + + if (!data) return null + + return ( +
+

{data.name}

+
+
+ + Tier: + {TIER_LABELS[data.tier ?? "situational"]} +
+
+ + Daily Salary: + ${data.dailyCost}/day +
+
+ + Interview Score: + {data.score} +
+
+ + Success Rate: + {data.successRate}% +
+
+
+ ) +} + // ── Main Component ────────────────────────────────────────────────────────── interface ComparisonChartProps { @@ -380,6 +463,25 @@ export function ComparisonChart({ recommendation, role, roleId }: ComparisonChar const chartHeight = Math.max(400, chartData.length * 100) + // Scatter plot data: value map of daily cost vs composite score + const scatterData = useMemo( + () => + filteredCandidates.map((c) => ({ + name: c.displayName, + dailyCost: Math.round(c.estimatedDailyCost), + score: c.compositeScore, + successRate: c.successRate, + tier: c.tier, + // ZAxis size: map success rate to dot size (60–400 range) + dotSize: Math.round(60 + (c.successRate / 100) * 340), + })), + [filteredCandidates], + ) + + // Determine axis domains for scatter plot + const scatterMaxCost = useMemo(() => Math.max(...scatterData.map((d) => d.dailyCost), 10), [scatterData]) + const scatterMinScore = useMemo(() => Math.min(...scatterData.map((d) => d.score), 50), [scatterData]) + // Providers that actually appear in data const activeProviders = useMemo(() => { const providers = new Set(allCandidates.map((c) => c.provider)) @@ -624,6 +726,124 @@ export function ComparisonChart({ recommendation, role, roleId }: ComparisonChar + {/* ── Value Map Scatter Chart ────────────────────────────── */} + +
+

Value Map: Salary vs Interview Score

+
+

+ Upper-left = best value. Each dot is a candidate model. Size reflects success rate. +

+ + {/* Tier legend */} +
+ {Object.entries(TIER_COLORS).map(([tier, color]) => ( +
+ + {TIER_LABELS[tier]} +
+ ))} +
+ + {scatterData.length === 0 ? ( +
+ +

No candidates match the current filters.

+

+ Try adjusting the provider or success rate filters. +

+
+ ) : ( +
+ + + `$${v}`} + stroke="hsl(var(--muted-foreground))" + strokeOpacity={0.3} + tick={{ fontSize: 11, fill: "hsl(var(--muted-foreground))" }} + axisLine={false} + label={{ + value: "Daily Salary ($)", + position: "insideBottom", + offset: -10, + style: { fontSize: 11, fill: "hsl(var(--muted-foreground))" }, + }} + /> + + + {/* Sweet spot reference zone: upper-left quadrant */} + + } + cursor={{ + strokeDasharray: "3 3", + stroke: "hsl(var(--muted-foreground))", + strokeOpacity: 0.3, + }} + /> + + {scatterData.map((entry, index) => ( + + ))} + + + +
+ )} + + {/* ── Chart Section ──────────────────────────────────────── */} = { + anthropic: "#fb923c", // orange-400 + openai: "#4ade80", // green-400 + google: "#60a5fa", // blue-400 + xai: "#c084fc", // purple-400 + deepseek: "#22d3ee", // cyan-400 + moonshot: "#f472b6", // pink-400 +} + +const PROVIDER_DISPLAY: Record = { + anthropic: "Anthropic", + openai: "OpenAI", + google: "Google", + xai: "xAI", + deepseek: "DeepSeek", + moonshot: "Moonshot", +} + +// ── Timeline Tooltip ──────────────────────────────────────────────────────── + +function TimelineTooltip({ + active, + payload, +}: { + active?: boolean + // eslint-disable-next-line @typescript-eslint/no-explicit-any + payload?: any[] +}) { + if (!active || !payload || !payload.length) return null + + const data = payload[0]?.payload as + | { + modelName?: string + provider?: string + score?: number + costPerRun?: number + dateLabel?: string + } + | undefined + + if (!data) return null + + return ( +
+

{data.modelName}

+
+
+ + Provider: + {PROVIDER_DISPLAY[data.provider ?? ""] ?? data.provider} +
+
+ + Release: + {data.dateLabel} +
+
+ + Eval Score: + {data.score}% +
+
+ + Cost per Run: + ${data.costPerRun?.toFixed(2)} +
+
+
+ ) +} + // ── Sub-Components ────────────────────────────────────────────────────────── function StatPill({ icon: Icon, value, label }: { icon: LucideIcon; value: string; label: string }) { @@ -218,6 +296,33 @@ export function WorkersContent({ }: WorkersContentProps) { const recByRole = new Map(recommendations.map((r) => [r.roleId, r])) + // ── Timeline scatter data ────────────────────────────────────────────── + const timelineData = useMemo(() => { + const maxCost = Math.max(...MODEL_TIMELINE.map((m) => m.costPerRun)) + return MODEL_TIMELINE.map((m) => { + const date = new Date(m.releaseDate) + return { + modelName: m.modelName, + provider: m.provider, + score: m.score, + costPerRun: m.costPerRun, + // numeric X for scatter: days since epoch + dateNum: date.getTime(), + dateLabel: date.toLocaleDateString("en-US", { month: "short", year: "numeric" }), + // Dot size: inversely proportional to cost (cheaper = bigger dot) + dotSize: Math.round(60 + (1 - m.costPerRun / maxCost) * 340), + } + }).sort((a, b) => a.dateNum - b.dateNum) + }, []) + + // Trend line endpoints for the timeline + const trendLine = useMemo(() => { + if (timelineData.length < 2) return null + const first = timelineData[0]! + const last = timelineData[timelineData.length - 1]! + return { x1: first.dateNum, y1: first.score, x2: last.dateNum, y2: last.score } + }, [timelineData]) + return ( <> {/* ── Hero Section ───────────────────────────────────────────── */} @@ -455,6 +560,164 @@ export function WorkersContent({ + {/* ── AI Coding Capability Over Time ─────────────────────────── */} +
+ {/* Subtle atmospheric background */} + +
+
+
+ + +
+ + {/* Section header */} + +

+ AI Coding Capability{" "} + + Over Time + +

+

+ Pass rates on our eval suite, by model release date. The best ones now score 100%. +

+ + + {/* Chart container */} + + {/* Provider legend */} +
+ {Object.entries(PROVIDER_COLORS) + .filter(([provider]) => MODEL_TIMELINE.some((m) => m.provider === provider)) + .map(([provider, color]) => ( +
+ + {PROVIDER_DISPLAY[provider] ?? provider} +
+ ))} +
+ ● + Bigger dot = lower cost +
+
+ +
+ + + { + const d = new Date(v) + return d.toLocaleDateString("en-US", { + month: "short", + year: "2-digit", + }) + }} + stroke="hsl(var(--muted-foreground))" + strokeOpacity={0.3} + tick={{ fontSize: 11, fill: "hsl(var(--muted-foreground))" }} + axisLine={false} + label={{ + value: "Release Date", + position: "insideBottom", + offset: -10, + style: { fontSize: 11, fill: "hsl(var(--muted-foreground))" }, + }} + /> + `${v}%`} + stroke="hsl(var(--muted-foreground))" + strokeOpacity={0.3} + tick={{ fontSize: 11, fill: "hsl(var(--muted-foreground))" }} + axisLine={false} + label={{ + value: "Eval Score (%)", + angle: -90, + position: "insideLeft", + offset: 10, + style: { fontSize: 11, fill: "hsl(var(--muted-foreground))" }, + }} + /> + + {/* Trend line: dashed line from first to last */} + {trendLine && ( + + )} + {/* 100% reference line */} + + } + cursor={{ + strokeDasharray: "3 3", + stroke: "hsl(var(--muted-foreground))", + strokeOpacity: 0.3, + }} + /> + + {timelineData.map((entry, index) => ( + + ))} + + + +
+ + +
+
+ {/* ── Footer / Methodology Section ───────────────────────────── */}
diff --git a/apps/web-roo-code/src/lib/mock-recommendations.ts b/apps/web-roo-code/src/lib/mock-recommendations.ts index 214d12bd27..8c57d6c147 100644 --- a/apps/web-roo-code/src/lib/mock-recommendations.ts +++ b/apps/web-roo-code/src/lib/mock-recommendations.ts @@ -864,3 +864,27 @@ export function getCloudSetupUrl(candidate: ModelCandidate): string { }) return `https://app.roocode.com/sign-up?${params.toString()}` } + +// ── Model Timeline Data ──────────────────────────────────────────────────── +// Historical model performance over time for the landing page chart. + +export type ModelTimelineEntry = { + modelName: string + provider: string + releaseDate: string // ISO date + score: number // our eval score (total %) + costPerRun: number // total cost for the full eval run +} + +export const MODEL_TIMELINE: ModelTimelineEntry[] = [ + { modelName: "Claude 3.5 Sonnet", provider: "anthropic", releaseDate: "2025-06-20", score: 90, costPerRun: 24.98 }, + { modelName: "GPT-4.1", provider: "openai", releaseDate: "2025-08-14", score: 91, costPerRun: 38.64 }, + { modelName: "Claude 3.7 Sonnet", provider: "anthropic", releaseDate: "2025-09-15", score: 95, costPerRun: 37.58 }, + { modelName: "Gemini 2.5 Pro", provider: "google", releaseDate: "2025-10-01", score: 96, costPerRun: 57.8 }, + { modelName: "Claude Sonnet 4", provider: "anthropic", releaseDate: "2025-11-01", score: 98, costPerRun: 39.61 }, + { modelName: "GPT-5 Mini", provider: "openai", releaseDate: "2025-12-01", score: 99, costPerRun: 3.34 }, + { modelName: "Claude Sonnet 4.5", provider: "anthropic", releaseDate: "2026-01-15", score: 100, costPerRun: 38.43 }, + { modelName: "GPT 5.2 (Med)", provider: "openai", releaseDate: "2026-01-20", score: 100, costPerRun: 12.5 }, + { modelName: "Claude Opus 4.6", provider: "anthropic", releaseDate: "2026-02-01", score: 100, costPerRun: 49.48 }, + { modelName: "Gemini 3 Pro", provider: "google", releaseDate: "2026-02-05", score: 100, costPerRun: 33.06 }, +]