mirror of
https://github.com/RooVetGit/Roo-Code.git
synced 2026-09-07 08:26:51 +00:00
Make workers outcomes-first canonical; redirect v2; refresh methodology
This commit is contained in:
Michael Preuss
parent
310b10cdc2
commit
d91c035e99
14 changed files with 1630 additions and 1808 deletions
File diff suppressed because it is too large
Load diff
|
|
@ -7,9 +7,10 @@ import { MethodologyContent } from "./methodology-content"
|
|||
|
||||
// ── SEO Metadata ────────────────────────────────────────────────────────────
|
||||
|
||||
const TITLE = "Methodology | Roo Code Evals"
|
||||
const DESCRIPTION = "Our methodology for evaluating AI coding models. Transparent, reproducible, evidence-based."
|
||||
const OG_DESCRIPTION = "Our methodology for evaluating AI coding models"
|
||||
const TITLE = "Methodology | Roo Code Cloud Evals"
|
||||
const DESCRIPTION =
|
||||
"How we run Roo Code Cloud evals and how to interpret outcomes-first recommendations. Same tasks, same limits, clear tradeoffs."
|
||||
const OG_DESCRIPTION = "How we run Roo Code Cloud evals"
|
||||
const PATH = "/evals/methodology"
|
||||
|
||||
export const metadata: Metadata = {
|
||||
|
|
|
|||
|
|
@ -1,83 +1,14 @@
|
|||
import { notFound } from "next/navigation"
|
||||
import type { Metadata } from "next"
|
||||
import { permanentRedirect } from "next/navigation"
|
||||
|
||||
import { SEO } from "@/lib/seo"
|
||||
import { ogImageUrl } from "@/lib/og"
|
||||
import { getEngineerRole, getRoleRecommendation } from "@/lib/mock-recommendations"
|
||||
import { buildQueryString, type RedirectSearchParams } from "../../../_redirect-utils"
|
||||
|
||||
import { ComparisonChart } from "../../../workers/[roleId]/compare/comparison-chart"
|
||||
|
||||
type PageProps = { params: Promise<{ roleId: string }> }
|
||||
|
||||
export async function generateMetadata({ params }: PageProps): Promise<Metadata> {
|
||||
const { roleId } = await params
|
||||
const role = getEngineerRole(roleId)
|
||||
|
||||
if (!role) {
|
||||
return {
|
||||
title: "Role Not Found | Roo Code Evals",
|
||||
description: "The requested role was not found.",
|
||||
}
|
||||
}
|
||||
|
||||
const title = `Compare Models — ${role.name} (V2 Preview) | Roo Code Evals`
|
||||
const description = `Outcome-first comparison of AI models for ${role.name}. Compare composite score, success rate, cost efficiency, and speed.`
|
||||
const ogDescription = `Compare Models — ${role.name} (V2 Preview)`
|
||||
const path = `/evals/workers-v2/${roleId}/compare`
|
||||
|
||||
return {
|
||||
title,
|
||||
description,
|
||||
alternates: {
|
||||
canonical: `${SEO.url}${path}`,
|
||||
},
|
||||
openGraph: {
|
||||
title,
|
||||
description,
|
||||
url: `${SEO.url}${path}`,
|
||||
siteName: SEO.name,
|
||||
images: [
|
||||
{
|
||||
url: ogImageUrl(title, ogDescription),
|
||||
width: 1200,
|
||||
height: 630,
|
||||
alt: title,
|
||||
},
|
||||
],
|
||||
locale: SEO.locale,
|
||||
type: "website",
|
||||
},
|
||||
twitter: {
|
||||
card: SEO.twitterCard,
|
||||
title,
|
||||
description,
|
||||
images: [ogImageUrl(title, ogDescription)],
|
||||
},
|
||||
keywords: [
|
||||
...SEO.keywords,
|
||||
"AI coding",
|
||||
"model comparison",
|
||||
"coding evals",
|
||||
role.name.toLowerCase(),
|
||||
"outcome-first",
|
||||
],
|
||||
}
|
||||
type PageProps = {
|
||||
params: Promise<{ roleId: string }>
|
||||
searchParams?: Promise<RedirectSearchParams>
|
||||
}
|
||||
|
||||
export default async function WorkersV2ComparePage({ params }: PageProps) {
|
||||
export default async function WorkersV2ComparePage({ params, searchParams }: PageProps) {
|
||||
const { roleId } = await params
|
||||
const recommendation = getRoleRecommendation(roleId)
|
||||
|
||||
if (!recommendation) {
|
||||
notFound()
|
||||
}
|
||||
|
||||
return (
|
||||
<ComparisonChart
|
||||
recommendation={recommendation}
|
||||
role={recommendation.role}
|
||||
roleId={roleId}
|
||||
workersRootPath="/evals/workers-v2"
|
||||
/>
|
||||
)
|
||||
const sp = (await searchParams) ?? {}
|
||||
permanentRedirect(`/evals/workers/${roleId}/compare${buildQueryString(sp)}`)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,100 +1,14 @@
|
|||
import { notFound } from "next/navigation"
|
||||
import type { Metadata } from "next"
|
||||
import { permanentRedirect } from "next/navigation"
|
||||
|
||||
import { SEO } from "@/lib/seo"
|
||||
import { ogImageUrl } from "@/lib/og"
|
||||
import { getRoleRecommendation, getCloudSetupUrl } from "@/lib/mock-recommendations"
|
||||
import { buildQueryString, type RedirectSearchParams } from "../../_redirect-utils"
|
||||
|
||||
import { CandidatesContent } from "../../workers/[roleId]/candidates-content"
|
||||
|
||||
type PageProps = { params: Promise<{ roleId: string }> }
|
||||
|
||||
export async function generateMetadata({ params }: PageProps): Promise<Metadata> {
|
||||
const { roleId } = await params
|
||||
const recommendation = getRoleRecommendation(roleId)
|
||||
|
||||
if (!recommendation) {
|
||||
return {
|
||||
title: "Role Not Found | Roo Code Evals",
|
||||
description: "The requested role was not found.",
|
||||
}
|
||||
}
|
||||
|
||||
const { role } = recommendation
|
||||
const title = `${role.name} — Recommended Models (V2 Preview) | Roo Code Evals`
|
||||
const description = `Outcome-first recommendations for ${role.name}. Compare models by success rate, cost, and speed across 5 languages.`
|
||||
const ogDescription = `${role.name} — Recommended Models (V2 Preview)`
|
||||
const path = `/evals/workers-v2/${roleId}`
|
||||
|
||||
return {
|
||||
title,
|
||||
description,
|
||||
alternates: {
|
||||
canonical: `${SEO.url}${path}`,
|
||||
},
|
||||
openGraph: {
|
||||
title,
|
||||
description,
|
||||
url: `${SEO.url}${path}`,
|
||||
siteName: SEO.name,
|
||||
images: [
|
||||
{
|
||||
url: ogImageUrl(title, ogDescription),
|
||||
width: 1200,
|
||||
height: 630,
|
||||
alt: title,
|
||||
},
|
||||
],
|
||||
locale: SEO.locale,
|
||||
type: "website",
|
||||
},
|
||||
twitter: {
|
||||
card: SEO.twitterCard,
|
||||
title,
|
||||
description,
|
||||
images: [ogImageUrl(title, ogDescription)],
|
||||
},
|
||||
keywords: [
|
||||
...SEO.keywords,
|
||||
"AI coding",
|
||||
"coding agents",
|
||||
"model recommendations",
|
||||
"coding evals",
|
||||
role.name.toLowerCase(),
|
||||
"outcome-first",
|
||||
],
|
||||
}
|
||||
type PageProps = {
|
||||
params: Promise<{ roleId: string }>
|
||||
searchParams?: Promise<RedirectSearchParams>
|
||||
}
|
||||
|
||||
export default async function WorkersV2RolePage({ params }: PageProps) {
|
||||
export default async function WorkersV2RolePage({ params, searchParams }: PageProps) {
|
||||
const { roleId } = await params
|
||||
const recommendation = getRoleRecommendation(roleId)
|
||||
|
||||
if (!recommendation) {
|
||||
notFound()
|
||||
}
|
||||
|
||||
const { role, best, budgetHire, speedHire, allCandidates, totalEvalRuns, totalExercises, lastUpdated } =
|
||||
recommendation
|
||||
|
||||
const cloudUrls: Record<string, string> = {}
|
||||
for (const candidate of allCandidates) {
|
||||
cloudUrls[candidate.modelId] = getCloudSetupUrl(candidate)
|
||||
}
|
||||
|
||||
return (
|
||||
<CandidatesContent
|
||||
roleId={roleId}
|
||||
role={role}
|
||||
best={best}
|
||||
budgetHire={budgetHire}
|
||||
speedHire={speedHire}
|
||||
allCandidates={allCandidates}
|
||||
totalEvalRuns={totalEvalRuns}
|
||||
totalExercises={totalExercises}
|
||||
lastUpdated={lastUpdated}
|
||||
cloudUrls={cloudUrls}
|
||||
workersRootPath="/evals/workers-v2"
|
||||
/>
|
||||
)
|
||||
const sp = (await searchParams) ?? {}
|
||||
permanentRedirect(`/evals/workers/${roleId}${buildQueryString(sp)}`)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,11 @@
|
|||
export type RedirectSearchParams = Record<string, string | string[] | undefined>
|
||||
|
||||
export function buildQueryString(searchParams: RedirectSearchParams): string {
|
||||
const params = new URLSearchParams()
|
||||
for (const [key, value] of Object.entries(searchParams)) {
|
||||
if (typeof value === "string") params.set(key, value)
|
||||
else if (Array.isArray(value)) value.forEach((v) => params.append(key, v))
|
||||
}
|
||||
const qs = params.toString()
|
||||
return qs ? `?${qs}` : ""
|
||||
}
|
||||
|
|
@ -1,91 +1,12 @@
|
|||
import type { Metadata } from "next"
|
||||
import { Fraunces, IBM_Plex_Sans } from "next/font/google"
|
||||
import { permanentRedirect } from "next/navigation"
|
||||
|
||||
import { SEO } from "@/lib/seo"
|
||||
import { ogImageUrl } from "@/lib/og"
|
||||
import { getEngineerRoles, getAllRecommendations } from "@/lib/mock-recommendations"
|
||||
import { buildQueryString, type RedirectSearchParams } from "./_redirect-utils"
|
||||
|
||||
import { WorkersContent } from "../workers/workers-content"
|
||||
|
||||
const TITLE = "Build with Roo Code Cloud (V2 Preview) | Roo Code Evals"
|
||||
const DESCRIPTION =
|
||||
"Outcome-first, eval-backed recommendations for shipping production code. Start from what you need to ship and pick a setup."
|
||||
const OG_DESCRIPTION = "Outcome-first recommendations for shipping production code"
|
||||
const PATH = "/evals/workers-v2"
|
||||
|
||||
const display = Fraunces({ subsets: ["latin"], variable: "--font-display" })
|
||||
const body = IBM_Plex_Sans({ subsets: ["latin"], weight: ["400", "500", "600"], variable: "--font-body" })
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: TITLE,
|
||||
description: DESCRIPTION,
|
||||
alternates: {
|
||||
canonical: `${SEO.url}${PATH}`,
|
||||
},
|
||||
openGraph: {
|
||||
title: TITLE,
|
||||
description: DESCRIPTION,
|
||||
url: `${SEO.url}${PATH}`,
|
||||
siteName: SEO.name,
|
||||
images: [
|
||||
{
|
||||
url: ogImageUrl(TITLE, OG_DESCRIPTION),
|
||||
width: 1200,
|
||||
height: 630,
|
||||
alt: TITLE,
|
||||
},
|
||||
],
|
||||
locale: SEO.locale,
|
||||
type: "website",
|
||||
},
|
||||
twitter: {
|
||||
card: SEO.twitterCard,
|
||||
title: TITLE,
|
||||
description: DESCRIPTION,
|
||||
images: [ogImageUrl(TITLE, OG_DESCRIPTION)],
|
||||
},
|
||||
keywords: [
|
||||
...SEO.keywords,
|
||||
"AI coding",
|
||||
"coding agents",
|
||||
"roo code cloud",
|
||||
"model recommendations",
|
||||
"coding evals",
|
||||
"shipping code",
|
||||
"prototype",
|
||||
"outcome-first",
|
||||
],
|
||||
type PageProps = {
|
||||
searchParams?: Promise<RedirectSearchParams>
|
||||
}
|
||||
|
||||
export default function WorkersV2Page() {
|
||||
const roles = getEngineerRoles()
|
||||
const recommendations = getAllRecommendations()
|
||||
|
||||
const totalEvalRuns = recommendations.reduce((sum, recommendation) => sum + recommendation.totalEvalRuns, 0)
|
||||
const totalExercises = recommendations.reduce((sum, recommendation) => sum + recommendation.totalExercises, 0)
|
||||
const uniqueModels = new Set(
|
||||
recommendations.flatMap((recommendation) => recommendation.allCandidates.map((candidate) => candidate.modelId)),
|
||||
)
|
||||
const totalModels = uniqueModels.size
|
||||
const lastUpdated = recommendations
|
||||
.map((recommendation) => recommendation.lastUpdated)
|
||||
.sort()
|
||||
.pop()
|
||||
|
||||
return (
|
||||
<div className={`${body.variable} ${display.variable} [font-family:var(--font-body)]`}>
|
||||
<WorkersContent
|
||||
roles={roles}
|
||||
recommendations={recommendations}
|
||||
totalEvalRuns={totalEvalRuns}
|
||||
totalExercises={totalExercises}
|
||||
totalModels={totalModels}
|
||||
lastUpdated={lastUpdated}
|
||||
workersRootPath="/evals/workers-v2"
|
||||
enableOutcomeLayer
|
||||
alternateVersionHref="/evals/workers"
|
||||
alternateVersionLabel="View baseline"
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
export default async function WorkersV2Page({ searchParams }: PageProps) {
|
||||
const sp = (await searchParams) ?? {}
|
||||
permanentRedirect(`/evals/workers${buildQueryString(sp)}`)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -665,8 +665,6 @@ export function CandidatesContent({
|
|||
const searchParams = useSearchParams()
|
||||
const theme = ROLE_THEMES[roleId] ?? DEFAULT_THEME
|
||||
const IconComponent = ICON_MAP[role.icon] ?? Code
|
||||
const alternateWorkersRootPath = workersRootPath === "/evals/workers-v2" ? "/evals/workers" : "/evals/workers-v2"
|
||||
const alternateVersionLabel = workersRootPath === "/evals/workers-v2" ? "View baseline" : "View V2 preview"
|
||||
const setupQuery = (() => {
|
||||
const outcome = searchParams.get("outcome")
|
||||
if (!outcome) return ""
|
||||
|
|
@ -773,12 +771,6 @@ export function CandidatesContent({
|
|||
Methodology
|
||||
<ArrowRight className="size-3 transition-transform duration-200 group-hover:translate-x-0.5" />
|
||||
</Link>
|
||||
<div className="hidden h-4 w-px bg-border sm:block" />
|
||||
<Link
|
||||
href={`${alternateWorkersRootPath}/${roleId}${setupQuery}`}
|
||||
className="inline-flex items-center gap-1.5 text-sm font-medium text-muted-foreground transition-colors hover:text-foreground">
|
||||
{alternateVersionLabel}
|
||||
</Link>
|
||||
</motion.div>
|
||||
|
||||
{/* Strengths + Trade-offs grid */}
|
||||
|
|
|
|||
|
|
@ -446,8 +446,6 @@ export function ComparisonChart({
|
|||
const searchParams = useSearchParams()
|
||||
const { allCandidates } = recommendation
|
||||
const theme = ROLE_THEMES[roleId] ?? DEFAULT_THEME
|
||||
const alternateWorkersRootPath = workersRootPath === "/evals/workers-v2" ? "/evals/workers" : "/evals/workers-v2"
|
||||
const alternateVersionLabel = workersRootPath === "/evals/workers-v2" ? "View baseline" : "View V2 preview"
|
||||
const setupQuery = (() => {
|
||||
const outcome = searchParams.get("outcome")
|
||||
if (!outcome) return ""
|
||||
|
|
@ -593,12 +591,6 @@ export function ComparisonChart({
|
|||
</Link>
|
||||
<span className="text-border">/</span>
|
||||
<span className="font-medium text-foreground">Compare Models</span>
|
||||
<span className="text-border">/</span>
|
||||
<Link
|
||||
href={`${alternateWorkersRootPath}/${roleId}/compare${setupQuery}`}
|
||||
className="font-medium text-muted-foreground transition-colors hover:text-foreground">
|
||||
{alternateVersionLabel}
|
||||
</Link>
|
||||
</motion.nav>
|
||||
|
||||
{/* Title row */}
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ export function CopySettingsButton({ settings }: CopySettingsButtonProps) {
|
|||
) : (
|
||||
<>
|
||||
<Copy className="size-4 text-muted-foreground" />
|
||||
🔧 Configure Extension
|
||||
Copy Roo Code Cloud Config
|
||||
</>
|
||||
)}
|
||||
</button>
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import type { Metadata } from "next"
|
||||
import { Fraunces, IBM_Plex_Sans } from "next/font/google"
|
||||
|
||||
import { SEO } from "@/lib/seo"
|
||||
import { ogImageUrl } from "@/lib/og"
|
||||
|
|
@ -10,10 +11,13 @@ import { WorkersContent } from "./workers-content"
|
|||
|
||||
const TITLE = "Build with Roo Code Cloud | Roo Code Evals"
|
||||
const DESCRIPTION =
|
||||
"Eval-backed model recommendations for shipping production code. Pick a setup based on the work you're doing: single-file fixes, multi-file changes, review, and autonomous runs."
|
||||
const OG_DESCRIPTION = "Eval-backed model recommendations for shipping production code"
|
||||
"Outcome-first, eval-backed recommendations for shipping production code. Start from your objective and pick a tradeoff."
|
||||
const OG_DESCRIPTION = "Outcome-first recommendations for shipping production code"
|
||||
const PATH = "/evals/workers"
|
||||
|
||||
const display = Fraunces({ subsets: ["latin"], variable: "--font-display" })
|
||||
const body = IBM_Plex_Sans({ subsets: ["latin"], weight: ["400", "500", "600"], variable: "--font-body" })
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: TITLE,
|
||||
description: DESCRIPTION,
|
||||
|
|
@ -62,11 +66,11 @@ export default function WorkersPage() {
|
|||
const recommendations = getAllRecommendations()
|
||||
|
||||
// Aggregate totals
|
||||
const totalEvalRuns = recommendations.reduce((sum, r) => sum + r.totalEvalRuns, 0)
|
||||
const totalExercises = recommendations.reduce((sum, r) => sum + r.totalExercises, 0)
|
||||
|
||||
// Unique model count across all roles
|
||||
const uniqueModels = new Set(recommendations.flatMap((r) => r.allCandidates.map((c) => c.modelId)))
|
||||
const totalEvalRuns = recommendations.reduce((sum, recommendation) => sum + recommendation.totalEvalRuns, 0)
|
||||
const totalExercises = recommendations.reduce((sum, recommendation) => sum + recommendation.totalExercises, 0)
|
||||
const uniqueModels = new Set(
|
||||
recommendations.flatMap((recommendation) => recommendation.allCandidates.map((candidate) => candidate.modelId)),
|
||||
)
|
||||
const totalModels = uniqueModels.size
|
||||
|
||||
const lastUpdated = recommendations
|
||||
|
|
@ -75,17 +79,16 @@ export default function WorkersPage() {
|
|||
.pop()
|
||||
|
||||
return (
|
||||
<WorkersContent
|
||||
roles={roles}
|
||||
recommendations={recommendations}
|
||||
totalEvalRuns={totalEvalRuns}
|
||||
totalExercises={totalExercises}
|
||||
totalModels={totalModels}
|
||||
lastUpdated={lastUpdated}
|
||||
workersRootPath="/evals/workers"
|
||||
enableOutcomeLayer={false}
|
||||
alternateVersionHref="/evals/workers-v2"
|
||||
alternateVersionLabel="View V2 preview"
|
||||
/>
|
||||
<div className={`${body.variable} ${display.variable} [font-family:var(--font-body)]`}>
|
||||
<WorkersContent
|
||||
roles={roles}
|
||||
recommendations={recommendations}
|
||||
totalEvalRuns={totalEvalRuns}
|
||||
totalExercises={totalExercises}
|
||||
totalModels={totalModels}
|
||||
lastUpdated={lastUpdated}
|
||||
workersRootPath="/evals/workers"
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -23,6 +23,11 @@ export type EvalOutcomeCapability = {
|
|||
export type EvalOutcomeProfile = {
|
||||
title: string
|
||||
description: string
|
||||
/**
|
||||
* Starter prompt shown in the UI to help users understand what to ask for.
|
||||
* This is product copy, not an eval artifact.
|
||||
*/
|
||||
examplePrompt?: string
|
||||
capabilities: EvalOutcomeCapability[]
|
||||
howItWorks: string[]
|
||||
}
|
||||
|
|
@ -46,6 +51,69 @@ export type EvalOutcome = {
|
|||
}
|
||||
|
||||
export const EVAL_OUTCOMES: EvalOutcome[] = [
|
||||
{
|
||||
id: "review_guardrails",
|
||||
name: "Idea → Prototype",
|
||||
description: "Turn a vague idea into a working demo in your real codebase.",
|
||||
icon: Sparkles,
|
||||
recommendedRoleIds: ["autonomous", "senior"],
|
||||
whyItWorks: [
|
||||
"Optimizes for momentum: map the codebase fast, then build a working slice.",
|
||||
"Senior builder keeps the prototype grounded in production constraints.",
|
||||
],
|
||||
builderProfile: {
|
||||
title: "Your Builder Profile",
|
||||
description: "For turning an idea into a working demo in your repo.",
|
||||
examplePrompt: `Objective: Idea → Prototype
|
||||
|
||||
In this repo, turn this idea into a working demo: <describe the idea>.
|
||||
|
||||
Constraints:
|
||||
- Keep scope small and demo-first.
|
||||
- Use the existing stack and patterns in this codebase.
|
||||
|
||||
Deliver:
|
||||
- A reviewable PR
|
||||
- A short walkthrough (how to run it, what works, what’s next)`,
|
||||
capabilities: [
|
||||
{
|
||||
id: "autonomous_researcher",
|
||||
name: "Autonomous Researcher",
|
||||
description: "Maps the codebase, constraints, and best path forward before implementation starts.",
|
||||
roleId: "autonomous",
|
||||
},
|
||||
{
|
||||
id: "multi_file_builder",
|
||||
name: "Senior Builder",
|
||||
description: "Builds a working prototype directly in your repo across the files it touches.",
|
||||
roleId: "senior",
|
||||
},
|
||||
{
|
||||
id: "discovery_loop",
|
||||
name: "Discovery loop",
|
||||
description:
|
||||
"Maps the codebase and constraints before making changes (so the prototype fits reality).",
|
||||
},
|
||||
{
|
||||
id: "prototype_scaffold",
|
||||
name: "Prototype scaffold",
|
||||
description: "Creates the smallest working slice you can demo and build on.",
|
||||
},
|
||||
{
|
||||
id: "demo_ready_output",
|
||||
name: "Demo-ready output",
|
||||
description:
|
||||
"Delivers a reviewable diff plus a clear walkthrough of what’s working and what’s next.",
|
||||
},
|
||||
],
|
||||
howItWorks: [
|
||||
"Clarify the objective and success criteria.",
|
||||
"Explore the codebase and pick the smallest viable implementation path.",
|
||||
"Build the prototype directly in the repo (no throwaway export/import step).",
|
||||
"Deliver a demo-ready diff with notes for the next iteration.",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "prototype_to_pr",
|
||||
name: "Prototype → PR",
|
||||
|
|
@ -58,8 +126,17 @@ export const EVAL_OUTCOMES: EvalOutcome[] = [
|
|||
],
|
||||
builderProfile: {
|
||||
title: "Your Builder Profile",
|
||||
description:
|
||||
"A default set of capabilities for turning a working prototype into a reviewable PR—on the production codebase.",
|
||||
description: "For turning a prototype into a reviewable PR on the production codebase.",
|
||||
examplePrompt: `Objective: Prototype → PR
|
||||
|
||||
Take the current prototype implementation and turn it into a reviewable PR.
|
||||
|
||||
Do:
|
||||
- Tighten scope to the smallest shippable diff
|
||||
- Add/adjust tests, lint, and typechecks as needed
|
||||
|
||||
Deliver:
|
||||
- A PR-ready diff with a plain-English summary and review notes`,
|
||||
capabilities: [
|
||||
{
|
||||
id: "multi_file_builder",
|
||||
|
|
@ -109,50 +186,6 @@ export const EVAL_OUTCOMES: EvalOutcome[] = [
|
|||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "paper_cuts",
|
||||
name: "Paper cuts & small fixes",
|
||||
description: "Fix the small stuff without dragging engineers off big projects.",
|
||||
icon: CheckCircle2,
|
||||
recommendedRoleIds: ["junior", "reviewer"],
|
||||
whyItWorks: [
|
||||
"Small diffs are high-leverage when the work is well-scoped.",
|
||||
"Reviewer keeps the quality bar and reduces surprise.",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "sentry_triage",
|
||||
name: "Sentry triage",
|
||||
description: "Turn recurring errors into concrete fixes with proof before review.",
|
||||
icon: Bug,
|
||||
recommendedRoleIds: ["autonomous", "reviewer"],
|
||||
whyItWorks: [
|
||||
"Autonomous runs handle multi-step investigation and iteration.",
|
||||
"Reviewer focuses on safety, correctness, and “does this hold up?”.",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "repro_to_fix",
|
||||
name: "Bug repro → fix",
|
||||
description: "Make the handoff less lossy: reproduce, patch, and validate in one loop.",
|
||||
icon: Workflow,
|
||||
recommendedRoleIds: ["senior", "reviewer"],
|
||||
whyItWorks: [
|
||||
"Good default for ambiguous bugs that touch a few files.",
|
||||
"Reviewer helps catch cross-team assumptions early.",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "review_guardrails",
|
||||
name: "Guardrails & review",
|
||||
description: "Raise the quality bar without becoming the blocker.",
|
||||
icon: GitPullRequest,
|
||||
recommendedRoleIds: ["reviewer"],
|
||||
whyItWorks: [
|
||||
"Works alongside CI, linters, and team review.",
|
||||
"Scales judgement through fast, consistent feedback.",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "issue_to_pr",
|
||||
name: "Issue → PR",
|
||||
|
|
@ -163,6 +196,251 @@ export const EVAL_OUTCOMES: EvalOutcome[] = [
|
|||
"Handles out-of-band work while humans stay on the roadmap.",
|
||||
"Pairs autonomy with guardrails for merge safety.",
|
||||
],
|
||||
builderProfile: {
|
||||
title: "Your Builder Profile",
|
||||
description: "For turning an issue into a reviewable PR.",
|
||||
examplePrompt: `Objective: Issue → PR
|
||||
|
||||
Fix this issue in the repo: <describe the problem and expected behavior>.
|
||||
|
||||
Requirements:
|
||||
- Define “done” in 2-3 acceptance criteria
|
||||
- Implement the fix and validate it (tests/lint/typechecks)
|
||||
|
||||
Deliver:
|
||||
- A reviewable PR with context and any follow-ups`,
|
||||
capabilities: [
|
||||
{
|
||||
id: "autonomous_executor",
|
||||
name: "Autonomous Executor",
|
||||
description: "Runs the full loop (investigate → implement → validate) while you stay unblocked.",
|
||||
roleId: "autonomous",
|
||||
},
|
||||
{
|
||||
id: "reviewer_guardrails",
|
||||
name: "Reviewer & Guardrails",
|
||||
description: "Reviews the diff for correctness, edge cases, and merge safety.",
|
||||
roleId: "reviewer",
|
||||
},
|
||||
{
|
||||
id: "issue_intake",
|
||||
name: "Issue intake",
|
||||
description:
|
||||
"Translates a request into scoped tasks, acceptance criteria, and a safe plan of attack.",
|
||||
},
|
||||
{
|
||||
id: "validation_loop",
|
||||
name: "Validation loop",
|
||||
description: "Runs tests/lint/typechecks and iterates until it’s clean (or flags what’s blocked).",
|
||||
},
|
||||
{
|
||||
id: "pr_ready_output",
|
||||
name: "PR-ready output",
|
||||
description: "Produces a focused diff plus a plain-English summary and review notes.",
|
||||
},
|
||||
],
|
||||
howItWorks: [
|
||||
"Clarify the issue and define what “done” means.",
|
||||
"Implement in the background with frequent validation checkpoints.",
|
||||
"Run a reviewer pass to reduce merge risk.",
|
||||
"Deliver a PR-ready result with context and next steps.",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "sentry_triage",
|
||||
name: "Customer Escalation → Resolved",
|
||||
description: "Triage a customer-blocking issue and ship the smallest safe fix.",
|
||||
icon: Bug,
|
||||
recommendedRoleIds: ["autonomous", "senior", "reviewer"],
|
||||
whyItWorks: [
|
||||
"Autonomous runs handle multi-step investigation and iteration.",
|
||||
"Senior builder makes the final fix precise and production-safe.",
|
||||
"Reviewer focuses on safety, correctness, and “does this hold up?”.",
|
||||
],
|
||||
builderProfile: {
|
||||
title: "Your Builder Profile",
|
||||
description: "For resolving a customer escalation quickly and safely.",
|
||||
examplePrompt: `Objective: Customer Escalation → Resolved
|
||||
|
||||
We have a customer-blocking escalation:
|
||||
- Symptoms: <what the customer sees>
|
||||
- Context: <logs/errors/links if available>
|
||||
|
||||
Do:
|
||||
- Find the smallest safe fix with a clear blast-radius assessment
|
||||
- Add guardrails/tests where it makes sense
|
||||
|
||||
Deliver:
|
||||
- A PR with the fix and a short “risk + rollout” note`,
|
||||
capabilities: [
|
||||
{
|
||||
id: "autonomous_triage",
|
||||
name: "Autonomous Triage",
|
||||
description: "Investigates logs, context, and repro steps to converge on a fix quickly.",
|
||||
roleId: "autonomous",
|
||||
},
|
||||
{
|
||||
id: "senior_fixer",
|
||||
name: "Senior Builder",
|
||||
description: "Implements the smallest production-safe fix when the blast radius is unclear.",
|
||||
roleId: "senior",
|
||||
},
|
||||
{
|
||||
id: "reviewer_guardrails",
|
||||
name: "Reviewer & Guardrails",
|
||||
description: "Double-checks safety and correctness so speed doesn’t create regressions.",
|
||||
roleId: "reviewer",
|
||||
},
|
||||
{
|
||||
id: "repro_first",
|
||||
name: "Repro-first",
|
||||
description: "Prioritizes a minimal reproduction so we know the fix actually fixes the issue.",
|
||||
},
|
||||
{
|
||||
id: "minimal_fix",
|
||||
name: "Minimal safe fix",
|
||||
description: "Ships the smallest change that unblocks customers, with a clear rollback story.",
|
||||
},
|
||||
{
|
||||
id: "verification_artifacts",
|
||||
name: "Verification artifacts",
|
||||
description: "Provides proof (tests/logs/steps) that the fix works and what it covers.",
|
||||
},
|
||||
],
|
||||
howItWorks: [
|
||||
"Gather context and reproduce the customer issue.",
|
||||
"Implement the smallest safe fix with verification.",
|
||||
"Run a reviewer pass to catch edge cases.",
|
||||
"Deliver a PR-ready result plus rollout notes.",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "repro_to_fix",
|
||||
name: "Bug Report → Fix",
|
||||
description: "Reproduce, isolate, patch, and validate in one loop.",
|
||||
icon: Workflow,
|
||||
recommendedRoleIds: ["senior", "reviewer"],
|
||||
whyItWorks: [
|
||||
"Good default for ambiguous bugs that touch a few files.",
|
||||
"Reviewer helps catch cross-team assumptions early.",
|
||||
],
|
||||
builderProfile: {
|
||||
title: "Your Builder Profile",
|
||||
description: "For turning a bug report into a verified fix.",
|
||||
examplePrompt: `Objective: Bug Report → Fix
|
||||
|
||||
Fix this bug:
|
||||
- Report: <paste or summarize>
|
||||
- Expected vs actual: <what should happen vs what happens>
|
||||
|
||||
Do:
|
||||
- Reproduce if possible, then implement the fix
|
||||
- Validate with tests/lint/typechecks (or explain what’s blocked)
|
||||
|
||||
Deliver:
|
||||
- A PR with the fix and verification notes`,
|
||||
capabilities: [
|
||||
{
|
||||
id: "bug_fixer",
|
||||
name: "Bug Fixer",
|
||||
description: "Reproduces and fixes bugs efficiently across the files involved.",
|
||||
roleId: "senior",
|
||||
},
|
||||
{
|
||||
id: "reviewer_guardrails",
|
||||
name: "Reviewer & Guardrails",
|
||||
description: "Reviews the diff for correctness and regression risk before it ships.",
|
||||
roleId: "reviewer",
|
||||
},
|
||||
{
|
||||
id: "repro_harness",
|
||||
name: "Repro harness",
|
||||
description:
|
||||
"Creates a minimal reproduction path (tests or steps) to prevent “can’t repro” stalls.",
|
||||
},
|
||||
{
|
||||
id: "fix_with_tests",
|
||||
name: "Fix with tests",
|
||||
description: "Pairs the fix with verification so it doesn’t regress on the next change.",
|
||||
},
|
||||
{
|
||||
id: "validation_loop",
|
||||
name: "Validation loop",
|
||||
description: "Runs tests/lint/typechecks and iterates until it’s clean (or flags what’s blocked).",
|
||||
},
|
||||
],
|
||||
howItWorks: [
|
||||
"Reproduce the issue and isolate the root cause.",
|
||||
"Implement a targeted fix with verification.",
|
||||
"Run a reviewer pass to reduce regression risk.",
|
||||
"Deliver a PR-ready result with steps to validate.",
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
id: "paper_cuts",
|
||||
name: "Paper Cuts → Shipped",
|
||||
description: "Fix the small stuff without dragging engineers off big projects.",
|
||||
icon: CheckCircle2,
|
||||
recommendedRoleIds: ["junior", "reviewer"],
|
||||
whyItWorks: [
|
||||
"Small diffs are high-leverage when the work is well-scoped.",
|
||||
"Reviewer keeps the quality bar and reduces surprise.",
|
||||
],
|
||||
builderProfile: {
|
||||
title: "Your Builder Profile",
|
||||
description: "For shipping small fixes quickly, cleanly, and safely.",
|
||||
examplePrompt: `Objective: Paper Cuts → Shipped
|
||||
|
||||
Ship these small fixes in this repo:
|
||||
- <paper cut 1>
|
||||
- <paper cut 2>
|
||||
- <paper cut 3>
|
||||
|
||||
Constraints:
|
||||
- Keep diffs small and easy to review
|
||||
- Don’t change behavior unless it’s clearly a bug
|
||||
|
||||
Deliver:
|
||||
- A PR with grouped, well-scoped commits and a short summary`,
|
||||
capabilities: [
|
||||
{
|
||||
id: "small_diff_builder",
|
||||
name: "Small-diff Builder",
|
||||
description: "Ships focused fixes with low review surface area and minimal risk.",
|
||||
roleId: "junior",
|
||||
},
|
||||
{
|
||||
id: "reviewer_guardrails",
|
||||
name: "Reviewer & Guardrails",
|
||||
description: "Catches edge cases and keeps changes aligned with team conventions.",
|
||||
roleId: "reviewer",
|
||||
},
|
||||
{
|
||||
id: "scope_control",
|
||||
name: "Scope control",
|
||||
description: "Keeps changes tight: fewer surprises, faster reviews, easier merges.",
|
||||
},
|
||||
{
|
||||
id: "quick_validation",
|
||||
name: "Quick validation",
|
||||
description: "Runs the relevant checks and flags what’s safe to skip (and what’s not).",
|
||||
},
|
||||
{
|
||||
id: "pr_ready_output",
|
||||
name: "PR-ready output",
|
||||
description: "Produces a focused diff plus a plain-English summary and review notes.",
|
||||
},
|
||||
],
|
||||
howItWorks: [
|
||||
"Pick the smallest fix that moves the needle.",
|
||||
"Implement with tight scope control.",
|
||||
"Validate quickly and review for conventions.",
|
||||
"Deliver a PR-ready result you can merge confidently.",
|
||||
],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -302,7 +302,7 @@ const seniorCandidates: ModelCandidate[] = [
|
|||
{
|
||||
provider: "moonshot",
|
||||
modelId: "kimi-k2-0905",
|
||||
displayName: "Kimi K2 0905",
|
||||
displayName: "Kimi K2",
|
||||
compositeScore: 95,
|
||||
tier: "best",
|
||||
tags: ["budget-hire", "best-value"],
|
||||
|
|
@ -439,7 +439,7 @@ const staffCandidates: ModelCandidate[] = [
|
|||
{
|
||||
provider: "anthropic",
|
||||
modelId: "claude-opus-4-6",
|
||||
displayName: "Claude Opus 4.6",
|
||||
displayName: "Opus 4.6",
|
||||
compositeScore: 98,
|
||||
tier: "best",
|
||||
tags: ["speed-hire", "top-performer"],
|
||||
|
|
@ -467,7 +467,7 @@ const staffCandidates: ModelCandidate[] = [
|
|||
{
|
||||
provider: "anthropic",
|
||||
modelId: "claude-opus-4-5",
|
||||
displayName: "Claude Opus 4.5",
|
||||
displayName: "Opus 4.5",
|
||||
compositeScore: 96,
|
||||
tier: "recommended",
|
||||
tags: [],
|
||||
|
|
@ -495,7 +495,7 @@ const staffCandidates: ModelCandidate[] = [
|
|||
{
|
||||
provider: "anthropic",
|
||||
modelId: "claude-opus-4-1",
|
||||
displayName: "Claude Opus 4.1",
|
||||
displayName: "Opus 4.1",
|
||||
compositeScore: 73,
|
||||
tier: "situational",
|
||||
tags: [],
|
||||
|
|
@ -525,7 +525,7 @@ const staffCandidates: ModelCandidate[] = [
|
|||
{
|
||||
provider: "anthropic",
|
||||
modelId: "claude-opus-4",
|
||||
displayName: "Claude Opus 4",
|
||||
displayName: "Opus 4",
|
||||
compositeScore: 57,
|
||||
tier: "not-recommended",
|
||||
tags: [],
|
||||
|
|
@ -565,7 +565,7 @@ const reviewerCandidates: ModelCandidate[] = [
|
|||
{
|
||||
provider: "anthropic",
|
||||
modelId: "claude-opus-4-6",
|
||||
displayName: "Claude Opus 4.6",
|
||||
displayName: "Opus 4.6",
|
||||
compositeScore: 95,
|
||||
tier: "best",
|
||||
tags: ["speed-hire", "top-performer"],
|
||||
|
|
@ -731,7 +731,7 @@ const autonomousCandidates: ModelCandidate[] = [
|
|||
{
|
||||
provider: "moonshot",
|
||||
modelId: "kimi-k2-0905",
|
||||
displayName: "Kimi K2 0905",
|
||||
displayName: "Kimi K2",
|
||||
compositeScore: 86,
|
||||
tier: "recommended",
|
||||
tags: [],
|
||||
|
|
@ -885,6 +885,6 @@ export const MODEL_TIMELINE: ModelTimelineEntry[] = [
|
|||
{ modelName: "GPT-5 Mini", provider: "openai", releaseDate: "2025-12-01", score: 99, costPerRun: 3.34 },
|
||||
{ modelName: "Claude Sonnet 4.5", provider: "anthropic", releaseDate: "2026-01-15", score: 100, costPerRun: 38.43 },
|
||||
{ modelName: "GPT 5.2 (Med)", provider: "openai", releaseDate: "2026-01-20", score: 100, costPerRun: 12.5 },
|
||||
{ modelName: "Claude Opus 4.6", provider: "anthropic", releaseDate: "2026-02-01", score: 100, costPerRun: 49.48 },
|
||||
{ modelName: "Opus 4.6", provider: "anthropic", releaseDate: "2026-02-01", score: 100, costPerRun: 49.48 },
|
||||
{ modelName: "Gemini 3 Pro", provider: "google", releaseDate: "2026-02-05", score: 100, costPerRun: 33.06 },
|
||||
]
|
||||
|
|
|
|||
227
apps/web-roo-code/src/lib/objective-default-models-v1.ts
Normal file
227
apps/web-roo-code/src/lib/objective-default-models-v1.ts
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
import type { EvalOutcomeId } from "./eval-outcomes"
|
||||
|
||||
type ObjectiveMetric = { score: number; costUsd: number; runtimeS: number }
|
||||
|
||||
type ModelObjectiveMetrics = {
|
||||
modelId: string
|
||||
issueResolution: ObjectiveMetric
|
||||
frontend: ObjectiveMetric
|
||||
greenfield: ObjectiveMetric
|
||||
testing: ObjectiveMetric
|
||||
infoGathering: ObjectiveMetric
|
||||
}
|
||||
|
||||
type EvalOptimizationModeV1 = "best" | "fastest" | "cost"
|
||||
|
||||
type ObjectiveWeights = {
|
||||
issueResolution: number
|
||||
frontend: number
|
||||
greenfield: number
|
||||
testing: number
|
||||
infoGathering: number
|
||||
}
|
||||
|
||||
type WeightedObjectiveMetrics = { score: number; costUsd: number; runtimeS: number }
|
||||
|
||||
export type ObjectiveDefaultModelV1 = {
|
||||
modelId: string
|
||||
weighted: WeightedObjectiveMetrics
|
||||
}
|
||||
|
||||
const MODEL_METRICS_V1: ModelObjectiveMetrics[] = [
|
||||
{
|
||||
modelId: "claude-opus-4-6",
|
||||
issueResolution: { score: 74.8, costUsd: 0.56, runtimeS: 178 },
|
||||
frontend: { score: 41.8, costUsd: 2.37, runtimeS: 602 },
|
||||
greenfield: { score: 43.8, costUsd: 2.5, runtimeS: 388 },
|
||||
testing: { score: 78.8, costUsd: 0.43, runtimeS: 138 },
|
||||
infoGathering: { score: 80, costUsd: 1.33, runtimeS: 526 },
|
||||
},
|
||||
{
|
||||
modelId: "GPT-5.2-Codex",
|
||||
issueResolution: { score: 73.8, costUsd: 0.94, runtimeS: 438 },
|
||||
frontend: { score: 35.9, costUsd: 2.97, runtimeS: 1434 },
|
||||
greenfield: { score: 62.5, costUsd: 2.5, runtimeS: 838 },
|
||||
testing: { score: 62.5, costUsd: 0.66, runtimeS: 343 },
|
||||
infoGathering: { score: 70.9, costUsd: 1.66, runtimeS: 799 },
|
||||
},
|
||||
{
|
||||
modelId: "claude-opus-4-5",
|
||||
issueResolution: { score: 76.6, costUsd: 1.82, runtimeS: 325 },
|
||||
frontend: { score: 41.2, costUsd: 2.54, runtimeS: 671 },
|
||||
greenfield: { score: 37.5, costUsd: 4.65, runtimeS: 495 },
|
||||
testing: { score: 78.5, costUsd: 1.38, runtimeS: 268 },
|
||||
infoGathering: { score: 69.1, costUsd: 0.55, runtimeS: 97 },
|
||||
},
|
||||
{
|
||||
modelId: "MiniMax-M2.5",
|
||||
issueResolution: { score: 72.6, costUsd: 0.1, runtimeS: 455 },
|
||||
frontend: { score: 25, costUsd: 0.15, runtimeS: 611 },
|
||||
greenfield: { score: 50, costUsd: 0.16, runtimeS: 376 },
|
||||
testing: { score: 68.1, costUsd: 0.07, runtimeS: 389 },
|
||||
infoGathering: { score: 47.9, costUsd: 0.06, runtimeS: 716 },
|
||||
},
|
||||
{
|
||||
modelId: "GPT-5.2",
|
||||
issueResolution: { score: 74.6, costUsd: 0.86, runtimeS: 476 },
|
||||
frontend: { score: 30.9, costUsd: 2.77, runtimeS: 1571 },
|
||||
greenfield: { score: 18.8, costUsd: 0.71, runtimeS: 397 },
|
||||
testing: { score: 73.2, costUsd: 0.56, runtimeS: 347 },
|
||||
infoGathering: { score: 65.5, costUsd: 0.48, runtimeS: 189 },
|
||||
},
|
||||
{
|
||||
modelId: "claude-sonnet-4-5",
|
||||
issueResolution: { score: 74.2, costUsd: 1.19, runtimeS: 534 },
|
||||
frontend: { score: 36.8, costUsd: 1.89, runtimeS: 787 },
|
||||
greenfield: { score: 12.5, costUsd: 2.65, runtimeS: 744 },
|
||||
testing: { score: 68.8, costUsd: 0.98, runtimeS: 488 },
|
||||
infoGathering: { score: 58.8, costUsd: 0.38, runtimeS: 126 },
|
||||
},
|
||||
{
|
||||
modelId: "Kimi-K2.5",
|
||||
issueResolution: { score: 68.8, costUsd: 0.48, runtimeS: 707 },
|
||||
frontend: { score: 32.8, costUsd: 1.58, runtimeS: 921 },
|
||||
greenfield: { score: 18.8, costUsd: 0.96, runtimeS: 814 },
|
||||
testing: { score: 61.9, costUsd: 0.42, runtimeS: 385 },
|
||||
infoGathering: { score: 63.6, costUsd: 0.39, runtimeS: 602 },
|
||||
},
|
||||
{
|
||||
modelId: "Gemini-3-Flash",
|
||||
issueResolution: { score: 74.6, costUsd: 0.42, runtimeS: 343 },
|
||||
frontend: { score: 22.1, costUsd: 0.8, runtimeS: 1152 },
|
||||
greenfield: { score: 18.8, costUsd: 0.82, runtimeS: 399 },
|
||||
testing: { score: 70.7, costUsd: 0.3, runtimeS: 213 },
|
||||
infoGathering: { score: 58.8, costUsd: 0.38, runtimeS: 398 },
|
||||
},
|
||||
{
|
||||
modelId: "DeepSeek-V3.2-Reasoner",
|
||||
issueResolution: { score: 71.6, costUsd: 0.16, runtimeS: 1429 },
|
||||
frontend: { score: 27.9, costUsd: 0.19, runtimeS: 1515 },
|
||||
greenfield: { score: 31.2, costUsd: 0.12, runtimeS: 1411 },
|
||||
testing: { score: 53.6, costUsd: 0.12, runtimeS: 1215 },
|
||||
infoGathering: { score: 50.3, costUsd: 0.06, runtimeS: 427 },
|
||||
},
|
||||
{
|
||||
modelId: "Gemini-3-Pro",
|
||||
issueResolution: { score: 70.6, costUsd: 0.95, runtimeS: 343 },
|
||||
frontend: { score: 36.8, costUsd: 1.46, runtimeS: 710 },
|
||||
greenfield: { score: 12.5, costUsd: 2.68, runtimeS: 554 },
|
||||
testing: { score: 68.6, costUsd: 1.01, runtimeS: 386 },
|
||||
infoGathering: { score: 44.2, costUsd: 1.5, runtimeS: 1775 },
|
||||
},
|
||||
{
|
||||
modelId: "MiniMax-M2.1",
|
||||
issueResolution: { score: 68.8, costUsd: 0.14, runtimeS: 579 },
|
||||
frontend: { score: 16.2, costUsd: 0.21, runtimeS: 1417 },
|
||||
greenfield: { score: 25, costUsd: 0.33, runtimeS: 826 },
|
||||
testing: { score: 61.4, costUsd: 0.11, runtimeS: 473 },
|
||||
infoGathering: { score: 40.6, costUsd: 0.06, runtimeS: 641 },
|
||||
},
|
||||
{
|
||||
modelId: "GLM-4.7",
|
||||
issueResolution: { score: 73.4, costUsd: 0.56, runtimeS: 1007 },
|
||||
frontend: { score: 22.1, costUsd: 0.66, runtimeS: 1519 },
|
||||
greenfield: { score: 12.5, costUsd: 0.54, runtimeS: 578 },
|
||||
testing: { score: 49.4, costUsd: 0.37, runtimeS: 744 },
|
||||
infoGathering: { score: 53.9, costUsd: 0.46, runtimeS: 1138 },
|
||||
},
|
||||
{
|
||||
modelId: "Kimi-K2-Thinking",
|
||||
issueResolution: { score: 69.2, costUsd: 2, runtimeS: 1325 },
|
||||
frontend: { score: 32.4, costUsd: 2.31, runtimeS: 1641 },
|
||||
greenfield: { score: 18.8, costUsd: 6.78, runtimeS: 2314 },
|
||||
testing: { score: 47.3, costUsd: 1.39, runtimeS: 1253 },
|
||||
infoGathering: { score: 43.6, costUsd: 0.65, runtimeS: 279 },
|
||||
},
|
||||
{
|
||||
modelId: "Qwen3-Coder-480B",
|
||||
issueResolution: { score: 62.4, costUsd: 1.26, runtimeS: 680 },
|
||||
frontend: { score: 23.5, costUsd: 2.09, runtimeS: 1006 },
|
||||
greenfield: { score: 0, costUsd: 1.79, runtimeS: 924 },
|
||||
testing: { score: 34.9, costUsd: 0.97, runtimeS: 626 },
|
||||
infoGathering: { score: 33.9, costUsd: 0.28, runtimeS: 197 },
|
||||
},
|
||||
]
|
||||
|
||||
function getOutcomeWeights(outcomeId: EvalOutcomeId): ObjectiveWeights {
|
||||
// These are intentionally opinionated. They exist to make the prototype feel realistic
|
||||
// before we wire real Roo Code Cloud evals.
|
||||
switch (outcomeId) {
|
||||
// Idea → Prototype
|
||||
case "review_guardrails":
|
||||
return { greenfield: 0.5, infoGathering: 0.35, frontend: 0.1, testing: 0.05, issueResolution: 0 }
|
||||
// Prototype → PR
|
||||
case "prototype_to_pr":
|
||||
return { greenfield: 0.35, testing: 0.35, issueResolution: 0.2, frontend: 0.1, infoGathering: 0 }
|
||||
// Issue → PR
|
||||
case "issue_to_pr":
|
||||
return { issueResolution: 0.4, testing: 0.3, infoGathering: 0.2, frontend: 0.1, greenfield: 0 }
|
||||
// Customer Escalation → Resolved
|
||||
case "sentry_triage":
|
||||
return { issueResolution: 0.55, infoGathering: 0.25, testing: 0.2, frontend: 0, greenfield: 0 }
|
||||
// Bug Report → Fix
|
||||
case "repro_to_fix":
|
||||
return { issueResolution: 0.45, testing: 0.4, infoGathering: 0.15, frontend: 0, greenfield: 0 }
|
||||
// Paper Cuts → Shipped
|
||||
case "paper_cuts":
|
||||
return { frontend: 0.6, issueResolution: 0.2, testing: 0.2, greenfield: 0, infoGathering: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
function getWeightedMetrics(row: ModelObjectiveMetrics, weights: ObjectiveWeights): WeightedObjectiveMetrics {
|
||||
const score =
|
||||
row.issueResolution.score * weights.issueResolution +
|
||||
row.frontend.score * weights.frontend +
|
||||
row.greenfield.score * weights.greenfield +
|
||||
row.testing.score * weights.testing +
|
||||
row.infoGathering.score * weights.infoGathering
|
||||
const costUsd =
|
||||
row.issueResolution.costUsd * weights.issueResolution +
|
||||
row.frontend.costUsd * weights.frontend +
|
||||
row.greenfield.costUsd * weights.greenfield +
|
||||
row.testing.costUsd * weights.testing +
|
||||
row.infoGathering.costUsd * weights.infoGathering
|
||||
const runtimeS =
|
||||
row.issueResolution.runtimeS * weights.issueResolution +
|
||||
row.frontend.runtimeS * weights.frontend +
|
||||
row.greenfield.runtimeS * weights.greenfield +
|
||||
row.testing.runtimeS * weights.testing +
|
||||
row.infoGathering.runtimeS * weights.infoGathering
|
||||
return { score, costUsd, runtimeS }
|
||||
}
|
||||
|
||||
function pickByMode(
|
||||
rows: Array<{ modelId: string; weighted: WeightedObjectiveMetrics }>,
|
||||
mode: EvalOptimizationModeV1,
|
||||
): { modelId: string; weighted: WeightedObjectiveMetrics } {
|
||||
const bestByQuality = rows.reduce((best, cur) => (cur.weighted.score > best.weighted.score ? cur : best))
|
||||
|
||||
// For speed/cost modes, don't pick a model that is dramatically worse on quality.
|
||||
// This keeps the v1 prototype recommendations feeling credible even when a model is
|
||||
// extremely cheap or fast but underperforms for the selected objective.
|
||||
const QUALITY_FLOOR = 0.85
|
||||
const qualityThreshold = bestByQuality.weighted.score * QUALITY_FLOOR
|
||||
const qualityGated = rows.filter((r) => r.weighted.score >= qualityThreshold)
|
||||
const pool = qualityGated.length > 0 ? qualityGated : rows
|
||||
|
||||
if (mode === "fastest") {
|
||||
return pool.reduce((best, cur) => (cur.weighted.runtimeS < best.weighted.runtimeS ? cur : best))
|
||||
}
|
||||
if (mode === "cost") {
|
||||
return pool.reduce((best, cur) => (cur.weighted.costUsd < best.weighted.costUsd ? cur : best))
|
||||
}
|
||||
return bestByQuality
|
||||
}
|
||||
|
||||
export function pickObjectiveDefaultModelV1(
|
||||
outcomeId: EvalOutcomeId,
|
||||
mode: EvalOptimizationModeV1,
|
||||
): ObjectiveDefaultModelV1 | null {
|
||||
const weights = getOutcomeWeights(outcomeId)
|
||||
const candidates = MODEL_METRICS_V1.map((row) => ({
|
||||
modelId: row.modelId,
|
||||
weighted: getWeightedMetrics(row, weights),
|
||||
}))
|
||||
if (candidates.length === 0) return null
|
||||
return pickByMode(candidates, mode)
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue