diff --git a/cookbook/auto_router_roi_training/ABLATIONS.md b/cookbook/auto_router_roi_training/ABLATIONS.md new file mode 100644 index 00000000000..19ca39b609b --- /dev/null +++ b/cookbook/auto_router_roi_training/ABLATIONS.md @@ -0,0 +1,44 @@ +# Card and calibration ablations + +These comparisons hold the boundary at the original default: capability base 0.5 with step 0.1, or V2 quality gap 0.05. All coefficients were fitted on the training split. This table reports every raw card and the fixed middle regularization strength of 10 for calibrated variants; the JSON contains all strengths. No variant is chosen using these held-out results + +| Pair | Classifier | Card | Adjustment | Solved | Cost | Savings | Efficient Brier | +|---|---|---|---|---|---:|---:|---:| +| sonnet_opus | v2 | original | none | 23/25 | $8.619 | 0.9% | 0.080 | +| sonnet_opus | v2 | original | per_model | 23/25 | $8.716 | -0.2% | 0.279 | +| sonnet_opus | v2 | original | task_conditioned | 23/25 | $8.716 | -0.2% | 0.296 | +| sonnet_opus | cap | original | none | 24/25 | $7.439 | 14.5% | 0.074 | +| sonnet_opus | cap | original | per_model | 23/25 | $8.716 | -0.2% | 0.288 | +| sonnet_opus | cap | original | task_conditioned | 23/25 | $8.716 | -0.2% | 0.288 | +| sonnet_opus | v2 | research | none | 23/25 | $8.380 | 3.7% | 0.076 | +| sonnet_opus | v2 | research | per_model | 23/25 | $8.716 | -0.2% | 0.288 | +| sonnet_opus | v2 | research | task_conditioned | 23/25 | $8.716 | -0.2% | 0.297 | +| sonnet_opus | cap | research | none | 24/25 | $7.433 | 14.6% | 0.066 | +| sonnet_opus | cap | research | per_model | 23/25 | $8.710 | -0.1% | 0.288 | +| sonnet_opus | cap | research | task_conditioned | 23/25 | $8.710 | -0.1% | 0.303 | +| sonnet_opus | v2 | trained_card | none | 23/25 | $8.711 | -0.1% | 0.169 | +| sonnet_opus | v2 | trained_card | per_model | 23/25 | $8.711 | -0.1% | 0.288 | +| sonnet_opus | v2 | trained_card | task_conditioned | 23/25 | $8.711 | -0.1% | 0.304 | +| sonnet_opus | cap | trained_card | none | 24/25 | $7.705 | 11.4% | 0.125 | +| sonnet_opus | cap | trained_card | per_model | 23/25 | $8.710 | -0.1% | 0.288 | +| sonnet_opus | cap | trained_card | task_conditioned | 23/25 | $8.710 | -0.1% | 0.299 | +| luna_sol | v2 | original | none | 25/25 | $5.887 | 23.5% | 0.114 | +| luna_sol | v2 | original | per_model | 25/25 | $7.713 | -0.2% | 0.293 | +| luna_sol | v2 | original | task_conditioned | 25/25 | $7.713 | -0.2% | 0.309 | +| luna_sol | cap | original | none | 23/25 | $0.454 | 94.1% | 0.095 | +| luna_sol | cap | original | per_model | 25/25 | $7.548 | 1.9% | 0.265 | +| luna_sol | cap | original | task_conditioned | 25/25 | $7.504 | 2.5% | 0.276 | +| luna_sol | v2 | research | none | 25/25 | $7.132 | 7.3% | 0.114 | +| luna_sol | v2 | research | per_model | 25/25 | $7.714 | -0.2% | 0.268 | +| luna_sol | v2 | research | task_conditioned | 25/25 | $7.714 | -0.2% | 0.274 | +| luna_sol | cap | research | none | 23/25 | $0.454 | 94.1% | 0.080 | +| luna_sol | cap | research | per_model | 25/25 | $7.707 | -0.1% | 0.293 | +| luna_sol | cap | research | task_conditioned | 25/25 | $7.707 | -0.1% | 0.315 | +| luna_sol | v2 | trained_card | none | 25/25 | $7.708 | -0.1% | 0.228 | +| luna_sol | v2 | trained_card | per_model | 25/25 | $7.708 | -0.1% | 0.272 | +| luna_sol | v2 | trained_card | task_conditioned | 25/25 | $7.708 | -0.1% | 0.250 | +| luna_sol | cap | trained_card | none | 23/25 | $1.936 | 74.9% | 0.131 | +| luna_sol | cap | trained_card | per_model | 25/25 | $7.707 | -0.1% | 0.293 | +| luna_sol | cap | trained_card | task_conditioned | 25/25 | $7.707 | -0.1% | 0.304 | + +Lower Brier means more accurate probabilities on these realized attempts. A lower Brier score can still yield worse routing at a fixed boundary. Compare these fixed-boundary controls with the separately frozen validation-selected profiles in REPORT.md. These exploratory comparisons reuse the same 25 tasks, so selecting a new winner here would require another holdout diff --git a/cookbook/auto_router_roi_training/BENCHMARK.md b/cookbook/auto_router_roi_training/BENCHMARK.md new file mode 100644 index 00000000000..317e11186a1 --- /dev/null +++ b/cookbook/auto_router_roi_training/BENCHMARK.md @@ -0,0 +1,55 @@ +# Auto Router training experiment + +Profiles were fitted on 83 DeepSWE tasks and selected on 30 repository-disjoint validation tasks before inspecting live grades. The live comparison uses 25 native-image-eligible SWE-bench Verified tasks and four current solver models at high effort. Each solver runs once per task; frozen task-pinned policies reuse those attempts and add their judge cost + +The baselines use the original classifier prompts and boundaries under the same judge settings: capability base 0.5 with step 0.1, and V2 gap 0.05 with neutral model-name profiles. This isolates classifier changes. It does not reproduce a production router that reclassifies every turn or escalates during an attempt + +| Pair | Policy | Solved | Cost | Savings vs capable | Efficient tasks | Lost / gained | +|---|---|---:|---:|---:|---:|---:| +| Sonnet / Opus | Sonnet only | 24/25 | $7.422 | 14.7% | 25 | 1 / 2 | +| Sonnet / Opus | Opus only | 23/25 | $8.699 | 0.0% | 0 | 0 / 0 | +| Sonnet / Opus | Original capability | 24/25 | $7.439 | 14.5% | 25 | 1 / 2 | +| Sonnet / Opus | Original V2 | 23/25 | $8.619 | 0.9% | 1 | 0 / 0 | +| Sonnet / Opus | Trained capability, 0 pp validation allowance | 24/25 | $7.705 | 11.4% | 22 | 1 / 2 | +| Sonnet / Opus | Trained capability, 2 pp validation allowance | 24/25 | $7.633 | 12.3% | 22 | 1 / 2 | +| Sonnet / Opus | Trained capability, 5 pp validation allowance | 24/25 | $7.633 | 12.3% | 22 | 1 / 2 | +| Sonnet / Opus | Trained V2, 0 pp validation allowance | 23/25 | $8.711 | -0.1% | 0 | 0 / 0 | +| Sonnet / Opus | Trained V2, 2 pp validation allowance | 23/25 | $8.711 | -0.1% | 0 | 0 / 0 | +| Sonnet / Opus | Trained V2, 5 pp validation allowance | 23/25 | $8.711 | -0.1% | 0 | 0 / 0 | +| Luna / Sol | Luna only | 23/25 | $0.444 | 94.2% | 25 | 2 / 0 | +| Luna / Sol | Sol only | 25/25 | $7.697 | 0.0% | 0 | 0 / 0 | +| Luna / Sol | Original capability | 23/25 | $0.454 | 94.1% | 25 | 2 / 0 | +| Luna / Sol | Original V2 | 25/25 | $5.887 | 23.5% | 7 | 0 / 0 | +| Luna / Sol | Trained capability, 0 pp validation allowance | 23/25 | $1.234 | 84.0% | 23 | 2 / 0 | +| Luna / Sol | Trained capability, 2 pp validation allowance | 24/25 | $5.643 | 26.7% | 8 | 1 / 0 | +| Luna / Sol | Trained capability, 5 pp validation allowance | 23/25 | $1.234 | 84.0% | 23 | 2 / 0 | +| Luna / Sol | Trained V2, 0 pp validation allowance | 23/25 | $2.086 | 72.9% | 21 | 2 / 0 | +| Luna / Sol | Trained V2, 2 pp validation allowance | 23/25 | $0.891 | 88.4% | 23 | 2 / 0 | +| Luna / Sol | Trained V2, 5 pp validation allowance | 25/25 | $7.713 | -0.2% | 0 | 0 / 0 | + +The fitted settings below were selected on validation. The allowance is a constraint on average net validation loss, not the V2 gap threshold and not a production guarantee + +| Pair | Classifier | Validation allowance | Card | Probability adjustment | Boundary | +|---|---|---:|---|---|---| +| sonnet_opus | cap | 0 pp | trained_card | none | base_threshold=0.5, threshold_step=0 | +| sonnet_opus | cap | 2 pp | research | none | base_threshold=0.72, threshold_step=0 | +| sonnet_opus | cap | 5 pp | research | none | base_threshold=0.72, threshold_step=0 | +| sonnet_opus | v2 | 0 pp | trained_card | task_conditioned (regularization 1) | max_quality_gap=0.1649 | +| sonnet_opus | v2 | 2 pp | trained_card | task_conditioned (regularization 1) | max_quality_gap=0.1649 | +| sonnet_opus | v2 | 5 pp | trained_card | task_conditioned (regularization 1) | max_quality_gap=0.1649 | +| luna_sol | cap | 0 pp | original | task_conditioned (regularization 100) | base_threshold=0.4565, threshold_step=0 | +| luna_sol | cap | 2 pp | original | task_conditioned (regularization 1) | base_threshold=0.5, threshold_step=0 | +| luna_sol | cap | 5 pp | original | task_conditioned (regularization 10) | base_threshold=0.4544, threshold_step=0 | +| luna_sol | v2 | 0 pp | original | none | max_quality_gap=0.08 | +| luna_sol | v2 | 2 pp | research | none | max_quality_gap=0.09 | +| luna_sol | v2 | 5 pp | original | per_model (regularization 1) | max_quality_gap=0.2561 | + +Equal solve counts can hide different successful tasks. Twenty-five tasks cannot establish small quality differences; the JSON includes paired repository-cluster intervals. These intervals are exploratory with few repositories. If every observed paired difference is zero, the empirical bootstrap interval is also zero and cannot estimate unseen failures. Even zero lost successes in 25 independent trials permits an 11.3% one-sided 95% binomial upper bound on that event rate; repository dependence weakens that inference. Public-data training uses different budgets and serving configurations, so this is a transfer test + +The initial x86 runs were excluded because of environment activation and emulator startup failures. Native tasks were selected by the same seeded repository/task ordering, skipping unavailable ARM images before any live grade inspection. The sample is not representative of every SWE-bench platform or repository + +The OpenAI adapter pilot was excluded because the gateway split sequential output blocks across choices and the stock harness discarded its tool calls. A later accounting correction restarted all three in-flight Anthropic attempts to capture the billed cost of malformed responses. Four completed Anthropic attempts were retained after confirming complete accounting. Grading container-name conflicts were retried with model-specific containers, preserving the solver attempts. One incomplete Sol attempt was archived and retried after gateway rate limits exhausted transport retries before submission; its $0.345 cost is excluded from the policy comparison and recorded separately. The final Sol grade exceeded 900 seconds during host slowdown; an unmodified-image CLI control also took 102 seconds. Only the saved patch was regraded with a 3600-second infrastructure deadline. These amendments, excluded attempts and unchanged frozen-selection hash are recorded in protocol.json + +Costs include billed solver responses, including malformed replies, plus the applicable classifier forecast. Host compute, image downloads and excluded infrastructure pilots are separate experiment expenses. The replay does not demonstrate mid-task escalation, inherited-state rescue, or a new repeated stochastic router run + +The JSON also reports held-out Brier scores for per-model probabilities and mean squared error for the V2 predicted gap. Those diagnostics measure probability accuracy separately from the routing threshold. One attempt per model per task does not reveal a task's true solve probability diff --git a/cookbook/auto_router_roi_training/FINDINGS.md b/cookbook/auto_router_roi_training/FINDINGS.md new file mode 100644 index 00000000000..64a052b27ac --- /dev/null +++ b/cookbook/auto_router_roi_training/FINDINGS.md @@ -0,0 +1,33 @@ +# What the training experiment showed + +Both implementations now support runnable, opt-in trained profiles. Training covered three cards, raw probabilities, per-model calibration, task-dependent calibration, and pair-specific boundaries, including combinations. All fitting and policy selection used the DeepSWE training and validation splits. The 25 fresh SWE-bench tasks were used only for evaluation + +The selected profiles did not establish a consistent improvement over the original routers. Most paid more for the same solve count or traded away solves for savings. They remain experimental configurations, with the original defaults preserved + +## Comparison with the existing routers + +For Sonnet 5 / Opus 5, Sonnet-only solved 24/25 for $7.422 and Opus-only solved 23/25 for $8.699. Original capability matched Sonnet's 24 solves at $7.439. The primary trained capability profile also solved 24, but cost $7.705. Original V2 solved 23 for $8.619; trained V2 selected Opus for every task, solved 23, and cost $8.711 + +For Luna / Sol, Luna-only solved 23/25 for $0.444 and Sol-only solved 25/25 for $7.697. Original V2 solved the same 25 tasks for $5.887, saving 23.5% versus Sol. The primary trained V2 profile kept the original card and raw probabilities but raised the gap boundary from 0.05 to 0.08. It cost $2.086 and solved 23, losing both tasks Luna failed. Its lower cost therefore came with an observed quality loss. The primary trained capability profile also solved 23 and cost $1.234, versus original capability's 23 for $0.454 + +The other validation allowances are reported in BENCHMARK.md in the PRs and REPORT.md in the full bundle. The Luna/Sol capability profile selected with a two-point validation allowance solved 24 for $5.643. That is a quality/cost tradeoff, not a profile that maximizes both. The most permissive selected V2 calibration chose Sol everywhere and added judge cost + +## Cards, probability calibration, and boundaries had different effects + +The research rewrite improved the capability classifier's efficient-model Brier score from 0.0736 to 0.0662 for Sonnet, and from 0.0953 to 0.0804 for Luna. With the original boundary, it still routed all 25 tasks to the efficient solver. This is a descriptive signal for better card wording, without an observed solve-rate improvement or a meaningful solver-cost advantage. It was not promoted to a new winner using the held-out outcomes + +The training-derived priors and learned probability adjustments transferred poorly. The primary trained V2 profile predicted a mean 37-point Opus advantage, while the realized paired difference was a 4-point Sonnet advantage. Its Sonnet probability averaged 0.434 against observed success of 0.96. The primary Luna capability calibration averaged 0.463 against observed success of 0.92. These estimates and Brier scores show a calibration problem on this workload, independently of the threshold + +Threshold tuning cannot repair that probability error. The Luna/Sol 0.05-to-0.08 comparison also shows that a boundary can be too permissive even when the mean predicted gap is close to the observed average gap. Request-level ranking and the placement of harmful downgrades matter, as well as average calibration + +## What to train next + +Use paired outcomes collected with the same agent, model effort, tools and budget as the intended deployment, with a new repository-held-out evaluation set. Treat these 25 tasks as used evaluation data. DeepSWE supplied useful fitting data, but this experiment does not establish whether its domain, task difficulty, budget or serving differences caused the failed transfer + +Train a model-pair estimate of the expected quality difference and the risk that only the stronger solver succeeds, alongside per-model expected total attempt cost. Some Sonnet attempts needed 71-84 model calls, so nominal token prices alone do not capture the routing opportunity. Keep task labels and model-specific probabilities as inputs, and test shrinking learned corrections toward the original probabilities when matching evidence is sparse + +Keep the research-only card rewrite as a candidate for that next evaluation. Do not increase the boundary merely to make these held-out results look better. The unchanged Luna/Sol V2 configuration is a useful quality-preserving control on this sample, and the efficient-only model is an essential savings control + +## Scope of the evidence + +These results come from 100 fresh solver attempts, one attempt per model per task, plus 300 fresh classifier forecasts. The policies are frozen, task-pinned paired replays over those attempts, with measured judge cost added. They do not measure per-turn reclassification, escalation or additional independently sampled router runs. Twenty-five tasks cannot establish a small production quality-loss guarantee diff --git a/cookbook/auto_router_roi_training/README.md b/cookbook/auto_router_roi_training/README.md index eed9e35af0d..56f6b07dbc4 100644 --- a/cookbook/auto_router_roi_training/README.md +++ b/cookbook/auto_router_roi_training/README.md @@ -28,6 +28,8 @@ curl http://localhost:4000/v1/chat/completions \ `profiles.json` contains the exact cards, coefficients, thresholds and validation results for each alias. `manifest.json` records model versions, effort, sources and limitations. `training_records.jsonl` retains the numerical forecasts and paired outcome evidence used for fitting and selection +For controlled comparisons, `fixed_boundary_profiles.json` contains 18 additional configurations: each model pair, each of the three cards, and raw, per-model, or task-dependent calibration. Calibration uses the fixed middle regularization strength of 10. These keep the original routing boundary unchanged and were not selected using live outcomes. Replace one alias's `complexity_router_config` with the chosen entry to benchmark it + ## What was fitted DeepSWE v1.1 supplies 113 tasks, with repeated attempts for each solver. We split repositories into 83 training tasks and 30 validation tasks. Within each task, repeated outcomes are averaged; each task receives equal fitting weight. Missing bills remain missing and are excluded as matched pairs only from cost calculations @@ -45,3 +47,9 @@ The raw model probabilities and calibrated probabilities are separate outputs. B These are small-sample experimental snapshots. Model, judge, prompt, effort, task distribution and harness changes can invalidate the coefficients. Published DeepSWE outcomes use different budgets and serving configurations from the live SWE-bench pilot, so that pilot measures transfer. Zero observed validation loss is a point estimate, not statistical noninferiority The capability classifier uses an absolute efficient-model probability threshold plus its boundary step. V2 compares the two models' predicted probabilities. Their threshold values have different meanings and should not be copied between classifiers + +## Fresh benchmark results + +Read [FINDINGS.md](FINDINGS.md) for the interpretation and next experiments. See [BENCHMARK.md](BENCHMARK.md) for the 25-task comparison against the original classifiers and each fixed model. [ABLATIONS.md](ABLATIONS.md) holds the boundary fixed to isolate card and calibration changes. The JSON files include per-task routes, outcomes, cost, lost and gained solves, probability diagnostics, and the frozen selection hash + +The learned Sonnet/Opus V2 profiles choose Opus on every live task and add classifier cost. That configuration provides no savings on this sample. Use the complete tables to compare other profiles with both fixed-model baselines; validation gains do not establish transfer to this workload diff --git a/cookbook/auto_router_roi_training/ablation_diagnostics.json b/cookbook/auto_router_roi_training/ablation_diagnostics.json new file mode 100644 index 00000000000..a18d5f5b4f0 --- /dev/null +++ b/cookbook/auto_router_roi_training/ablation_diagnostics.json @@ -0,0 +1,2399 @@ +{ + "scope": "Descriptive held-out ablations using previously fitted coefficients and original fixed routing boundaries; no held-out fitting or selection", + "results": [ + { + "key": "v2_sonnet_opus_original_none_0", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "original", + "adjustment": "none", + "alpha": 0, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.079748, + "mean_probability": 0.7604, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.07612000000000001, + "mean_probability": 0.8552000000000001, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.135972 + } + }, + "solved": 23, + "cost": 8.61870845, + "efficient_tasks": 1, + "savings_vs_capable": 0.009278491477484052, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_original_per_model_1", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "original", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.27794028029002865, + "mean_probability": 0.4704427158195999, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.1021595524175915, + "mean_probability": 0.7510042628607855, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.22122896815327736 + } + }, + "solved": 23, + "cost": 8.71597085, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019018323737678422, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_original_per_model_10", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "original", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.2789359079612613, + "mean_probability": 0.46943972279160173, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.10215955213956023, + "mean_probability": 0.7510042636833839, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.22186515742232807 + } + }, + "solved": 23, + "cost": 8.71597085, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019018323737678422, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_original_per_model_100", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "original", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.28332120934015553, + "mean_probability": 0.4650475018973117, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.10215954294862443, + "mean_probability": 0.7510042908761915, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.22467652549216838 + } + }, + "solved": 23, + "cost": 8.71597085, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019018323737678422, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_original_task_conditioned_1", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.36308527194686313, + "mean_probability": 0.39628343342870315, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.12591444705552035, + "mean_probability": 0.6941896559242194, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.24382206765927505 + } + }, + "solved": 23, + "cost": 8.71597085, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019018323737678422, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_original_task_conditioned_10", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.2962169385227655, + "mean_probability": 0.4523888476514996, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.11243251125742348, + "mean_probability": 0.7229117308308468, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.2163712407775607 + } + }, + "solved": 23, + "cost": 8.71597085, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019018323737678422, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_original_task_conditioned_100", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.2879036723967363, + "mean_probability": 0.4604415584252592, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.1035792913710479, + "mean_probability": 0.7467504230527484, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.2250556370374381 + } + }, + "solved": 23, + "cost": 8.71597085, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019018323737678422, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_original_none_0", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "original", + "adjustment": "none", + "alpha": 0, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.07364000000000001, + "mean_probability": 0.7728, + "mean_success": 0.96 + } + }, + "solved": 24, + "cost": 7.438772450000001, + "efficient_tasks": 25, + "savings_vs_capable": 0.1449122677749085, + "lost_capable_successes": 1, + "gained_efficient_successes": 2 + }, + { + "key": "cap_sonnet_opus_original_per_model_1", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "original", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2875570809540929, + "mean_probability": 0.4608436242573206, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.71599345, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019044302463169238, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_original_per_model_10", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "original", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.28755716634579204, + "mean_probability": 0.46084353872130845, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.71599345, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019044302463169238, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_original_per_model_100", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "original", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.28755712405378614, + "mean_probability": 0.4608435810847865, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.71599345, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019044302463169238, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_original_task_conditioned_1", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.283923378065796, + "mean_probability": 0.4633263645657801, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.6398926, + "efficient_tasks": 2, + "savings_vs_capable": 0.006843371045400337, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_original_task_conditioned_10", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2876105405621741, + "mean_probability": 0.4596932157480619, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.71599345, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019044302463169238, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_original_task_conditioned_100", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2875419515812453, + "mean_probability": 0.4606315304463149, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.71599345, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019044302463169238, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_research_none_0", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "research", + "adjustment": "none", + "alpha": 0, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.075772, + "mean_probability": 0.7812, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.083056, + "mean_probability": 0.8544000000000002, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.12974000000000005 + } + }, + "solved": 23, + "cost": 8.3804996, + "efficient_tasks": 3, + "savings_vs_capable": 0.03666062565507222, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_research_per_model_1", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "research", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.28755702357624857, + "mean_probability": 0.46084368841979556, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.10215956766580786, + "mean_probability": 0.7510042466318165, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.22740599175241083 + } + }, + "solved": 23, + "cost": 8.71649185, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019617213825369184, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_research_per_model_10", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "research", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.2875570115229681, + "mean_probability": 0.46084370049344897, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.10215956733306003, + "mean_probability": 0.7510042476163024, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.2274059844299993 + } + }, + "solved": 23, + "cost": 8.71649185, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019617213825369184, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_research_per_model_100", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "research", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.287557065393447, + "mean_probability": 0.46084364653191817, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.1021595622748158, + "mean_probability": 0.7510042625818982, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.2274060299440398 + } + }, + "solved": 23, + "cost": 8.71649185, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019617213825369184, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_research_task_conditioned_1", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.3059615497884397, + "mean_probability": 0.4439284142823081, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.1108453339392709, + "mean_probability": 0.7387745895860501, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.24709080789014723 + } + }, + "solved": 23, + "cost": 8.71649185, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019617213825369184, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_research_task_conditioned_10", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.29692233140450003, + "mean_probability": 0.45113431655284353, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.1055804190608191, + "mean_probability": 0.7430961515361965, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.23297743453461459 + } + }, + "solved": 23, + "cost": 8.71649185, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019617213825369184, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_research_task_conditioned_100", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.2893380643292629, + "mean_probability": 0.4588383860815747, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.10270450787421566, + "mean_probability": 0.7494100259237025, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.22810834000624067 + } + }, + "solved": 23, + "cost": 8.71649185, + "efficient_tasks": 0, + "savings_vs_capable": -0.0019617213825369184, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_research_none_0", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "research", + "adjustment": "none", + "alpha": 0, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.06624400000000001, + "mean_probability": 0.8051999999999999, + "mean_success": 0.96 + } + }, + "solved": 24, + "cost": 7.43261425, + "efficient_tasks": 25, + "savings_vs_capable": 0.14562015355955693, + "lost_capable_successes": 1, + "gained_efficient_successes": 2 + }, + { + "key": "cap_sonnet_opus_research_per_model_1", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "research", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2875569829466434, + "mean_probability": 0.4608437244101924, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.709835250000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011965444616688359, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_research_per_model_10", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "research", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2875569423532278, + "mean_probability": 0.46084376507222474, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.709835250000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011965444616688359, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_research_per_model_100", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "research", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2875569358163707, + "mean_probability": 0.46084377162013174, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.709835250000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011965444616688359, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_research_task_conditioned_1", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.37508996121022137, + "mean_probability": 0.37788648108842726, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.709835250000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011965444616688359, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_research_task_conditioned_10", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.30318514851583234, + "mean_probability": 0.44492506046090563, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.709835250000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011965444616688359, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_research_task_conditioned_100", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.28925395756469163, + "mean_probability": 0.45908749013264605, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.709835250000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011965444616688359, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_trained_card_none_0", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "trained_card", + "adjustment": "none", + "alpha": 0, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.168676, + "mean_probability": 0.6076, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.08659999999999998, + "mean_probability": 0.8144, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.17989200000000005 + } + }, + "solved": 23, + "cost": 8.71103165, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013340707766233972, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_trained_card_per_model_1", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.2875571114859761, + "mean_probability": 0.4608435974111398, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.1021595934786992, + "mean_probability": 0.7510041548091273, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.22740599218430954 + } + }, + "solved": 23, + "cost": 8.71103165, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013340707766233972, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_trained_card_per_model_10", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.2875571151138243, + "mean_probability": 0.4608435937771604, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.10215959351502482, + "mean_probability": 0.7510041547016525, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.22740599451293494 + } + }, + "solved": 23, + "cost": 8.71103165, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013340707766233972, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_trained_card_per_model_100", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.28755711158000513, + "mean_probability": 0.46084359731695196, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.10215959379624227, + "mean_probability": 0.7510041538696277, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.22740599162613237 + } + }, + "solved": 23, + "cost": 8.71103165, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013340707766233972, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_trained_card_task_conditioned_1", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.31665228081145946, + "mean_probability": 0.43411072591813205, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.09095517149287767, + "mean_probability": 0.8041056413999473, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.2890826011480608 + } + }, + "solved": 23, + "cost": 8.71103165, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013340707766233972, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_trained_card_task_conditioned_10", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.3042932741747288, + "mean_probability": 0.44520031083683287, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.10161513251587312, + "mean_probability": 0.7539819279976726, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.24017321465535685 + } + }, + "solved": 23, + "cost": 8.71103165, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013340707766233972, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_sonnet_opus_trained_card_task_conditioned_100", + "pair": "sonnet_opus", + "classifier": "v2", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.29072851803787375, + "mean_probability": 0.4578249125015566, + "mean_success": 0.96 + }, + "capable": { + "brier": 0.10326274868600573, + "mean_probability": 0.7478517316497024, + "mean_success": 0.92 + }, + "gap": { + "mse": 0.22729947565695874 + } + }, + "solved": 23, + "cost": 8.71103165, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013340707766233972, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_trained_card_none_0", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "trained_card", + "adjustment": "none", + "alpha": 0, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.124676, + "mean_probability": 0.6747999999999998, + "mean_success": 0.96 + } + }, + "solved": 24, + "cost": 7.705032350000001, + "efficient_tasks": 22, + "savings_vs_capable": 0.11430566223564653, + "lost_capable_successes": 1, + "gained_efficient_successes": 2 + }, + { + "key": "cap_sonnet_opus_trained_card_per_model_1", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2875569606247018, + "mean_probability": 0.46084374591258964, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.70978845, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011911647963895877, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_trained_card_per_model_10", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2875570001194668, + "mean_probability": 0.46084370635106653, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.70978845, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011911647963895877, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_trained_card_per_model_100", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.28755696326011687, + "mean_probability": 0.4608437432727199, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.70978845, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011911647963895877, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_trained_card_task_conditioned_1", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.3757063567782923, + "mean_probability": 0.3806681938876079, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.70978845, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011911647963895877, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_trained_card_task_conditioned_10", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.29926571087895887, + "mean_probability": 0.4493445442023018, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.70978845, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011911647963895877, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_sonnet_opus_trained_card_task_conditioned_100", + "pair": "sonnet_opus", + "classifier": "cap", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2887678608024657, + "mean_probability": 0.45963984818377995, + "mean_success": 0.96 + } + }, + "solved": 23, + "cost": 8.70978845, + "efficient_tasks": 0, + "savings_vs_capable": -0.0011911647963895877, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_original_none_0", + "pair": "luna_sol", + "classifier": "v2", + "card": "original", + "adjustment": "none", + "alpha": 0, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.11399200000000001, + "mean_probability": 0.7480000000000001, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.03844, + "mean_probability": 0.8144, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.07523200000000001 + } + }, + "solved": 25, + "cost": 5.8865393500000005, + "efficient_tasks": 7, + "savings_vs_capable": 0.23518233993843451, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_original_per_model_1", + "pair": "luna_sol", + "classifier": "v2", + "card": "original", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.29280415326290876, + "mean_probability": 0.4518075740486838, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.08206787123595165, + "mean_probability": 0.7135364774862528, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.10650573917319446 + } + }, + "solved": 25, + "cost": 7.713418050000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.002177679161805779, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_original_per_model_10", + "pair": "luna_sol", + "classifier": "v2", + "card": "original", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.2928041632313154, + "mean_probability": 0.45180756340305395, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.08266238242441645, + "mean_probability": 0.7124978475859192, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.10614467277206435 + } + }, + "solved": 25, + "cost": 7.713418050000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.002177679161805779, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_original_per_model_100", + "pair": "luna_sol", + "classifier": "v2", + "card": "original", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.2928041587755105, + "mean_probability": 0.45180756816157264, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.08487634767677435, + "mean_probability": 0.7086660809032587, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.10483330138416896 + } + }, + "solved": 25, + "cost": 7.713418050000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.002177679161805779, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_original_task_conditioned_1", + "pair": "luna_sol", + "classifier": "v2", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.3548002775195893, + "mean_probability": 0.3908214122713399, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.07432745676793391, + "mean_probability": 0.7331384746941513, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.14686729512308752 + } + }, + "solved": 25, + "cost": 7.713418050000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.002177679161805779, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_original_task_conditioned_10", + "pair": "luna_sol", + "classifier": "v2", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.3092058232917819, + "mean_probability": 0.43386824542874547, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.07860497482303502, + "mean_probability": 0.720351528896477, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.11728110929571926 + } + }, + "solved": 25, + "cost": 7.713418050000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.002177679161805779, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_original_task_conditioned_100", + "pair": "luna_sol", + "classifier": "v2", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.29509588080407867, + "mean_probability": 0.44921772442254926, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.08591911659001744, + "mean_probability": 0.706894247137934, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.10530011684393965 + } + }, + "solved": 25, + "cost": 7.713418050000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.002177679161805779, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_original_none_0", + "pair": "luna_sol", + "classifier": "cap", + "card": "original", + "adjustment": "none", + "alpha": 0, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.095296, + "mean_probability": 0.7759999999999999, + "mean_success": 0.92 + } + }, + "solved": 23, + "cost": 0.45423660999999993, + "efficient_tasks": 25, + "savings_vs_capable": 0.9409826112562217, + "lost_capable_successes": 2, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_original_per_model_1", + "pair": "luna_sol", + "classifier": "cap", + "card": "original", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.25978692916010465, + "mean_probability": 0.4878249735872592, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.19860084, + "efficient_tasks": 3, + "savings_vs_capable": 0.06471073701970254, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_original_per_model_10", + "pair": "luna_sol", + "classifier": "cap", + "card": "original", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.26459478741986797, + "mean_probability": 0.48236844476001733, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.548072479999999, + "efficient_tasks": 1, + "savings_vs_capable": 0.01930509780271894, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_original_per_model_100", + "pair": "luna_sol", + "classifier": "cap", + "card": "original", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2812914321660473, + "mean_probability": 0.4639982675367723, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.70707553, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013536175159263308, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_original_task_conditioned_1", + "pair": "luna_sol", + "classifier": "cap", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.28972178243797075, + "mean_probability": 0.4601118755343589, + "mean_success": 0.92 + } + }, + "solved": 24, + "cost": 5.6430830300000006, + "efficient_tasks": 8, + "savings_vs_capable": 0.2668137759857616, + "lost_capable_successes": 1, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_original_task_conditioned_10", + "pair": "luna_sol", + "classifier": "cap", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2758875769644403, + "mean_probability": 0.47150335677677147, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.503500099999999, + "efficient_tasks": 1, + "savings_vs_capable": 0.025096232686574682, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_original_task_conditioned_100", + "pair": "luna_sol", + "classifier": "cap", + "card": "original", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2820900941429093, + "mean_probability": 0.46335458127953316, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.70707553, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013536175159263308, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_research_none_0", + "pair": "luna_sol", + "classifier": "v2", + "card": "research", + "adjustment": "none", + "alpha": 0, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.114424, + "mean_probability": 0.7167999999999999, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.047700000000000006, + "mean_probability": 0.79, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.072164 + } + }, + "solved": 25, + "cost": 7.1324348, + "efficient_tasks": 3, + "savings_vs_capable": 0.0733074613222997, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_research_per_model_1", + "pair": "luna_sol", + "classifier": "v2", + "card": "research", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.26405453900921133, + "mean_probability": 0.4828215141408515, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.0865397854427683, + "mean_probability": 0.7058235470967105, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.09337909637214345 + } + }, + "solved": 25, + "cost": 7.7137271, + "efficient_tasks": 0, + "savings_vs_capable": -0.0022178329574038536, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_research_per_model_10", + "pair": "luna_sol", + "classifier": "v2", + "card": "research", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.26784512929721227, + "mean_probability": 0.4785993032872986, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.08653978536907665, + "mean_probability": 0.7058235472219613, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.09468553688217077 + } + }, + "solved": 25, + "cost": 7.7137271, + "efficient_tasks": 0, + "savings_vs_capable": -0.0022178329574038536, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_research_per_model_100", + "pair": "luna_sol", + "classifier": "v2", + "card": "research", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.2820450714621263, + "mean_probability": 0.4631518773294203, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.08653978613424879, + "mean_probability": 0.7058235459214286, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.09979694144591417 + } + }, + "solved": 25, + "cost": 7.7137271, + "efficient_tasks": 0, + "savings_vs_capable": -0.0022178329574038536, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_research_task_conditioned_1", + "pair": "luna_sol", + "classifier": "v2", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.25110147734567884, + "mean_probability": 0.5130557551354863, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.16534084147514008, + "mean_probability": 0.5948591698590449, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.09253588406341894 + } + }, + "solved": 23, + "cost": 3.6107769100000002, + "efficient_tasks": 15, + "savings_vs_capable": 0.5308642679317976, + "lost_capable_successes": 2, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_research_task_conditioned_10", + "pair": "luna_sol", + "classifier": "v2", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.27438937075247344, + "mean_probability": 0.47320616011023775, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.11565025341690252, + "mean_probability": 0.6601758116179052, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.08792931435780871 + } + }, + "solved": 25, + "cost": 7.7137271, + "efficient_tasks": 0, + "savings_vs_capable": -0.0022178329574038536, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_research_task_conditioned_100", + "pair": "luna_sol", + "classifier": "v2", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.2839548637673906, + "mean_probability": 0.4612566479475081, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.09097174761309613, + "mean_probability": 0.6983925888719742, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.09846970893099971 + } + }, + "solved": 25, + "cost": 7.7137271, + "efficient_tasks": 0, + "savings_vs_capable": -0.0022178329574038536, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_research_none_0", + "pair": "luna_sol", + "classifier": "cap", + "card": "research", + "adjustment": "none", + "alpha": 0, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.08042400000000001, + "mean_probability": 0.8223999999999999, + "mean_success": 0.92 + } + }, + "solved": 23, + "cost": 0.45399943000000004, + "efficient_tasks": 25, + "savings_vs_capable": 0.9410134272317597, + "lost_capable_successes": 2, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_research_per_model_1", + "pair": "luna_sol", + "classifier": "cap", + "card": "research", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2928040337273977, + "mean_probability": 0.4518076822531272, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.706838350000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013228015403883298, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_research_per_model_10", + "pair": "luna_sol", + "classifier": "cap", + "card": "research", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2928040337214044, + "mean_probability": 0.4518076822595278, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.706838350000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013228015403883298, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_research_per_model_100", + "pair": "luna_sol", + "classifier": "cap", + "card": "research", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.29280403370983843, + "mean_probability": 0.45180768227187945, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.706838350000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013228015403883298, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_research_task_conditioned_1", + "pair": "luna_sol", + "classifier": "cap", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.4308246215137724, + "mean_probability": 0.3255628214768662, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.706838350000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013228015403883298, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_research_task_conditioned_10", + "pair": "luna_sol", + "classifier": "cap", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.31503252924010744, + "mean_probability": 0.42922186517312366, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.706838350000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013228015403883298, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_research_task_conditioned_100", + "pair": "luna_sol", + "classifier": "cap", + "card": "research", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.29517455314184166, + "mean_probability": 0.44934382803078093, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.706838350000001, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013228015403883298, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_trained_card_none_0", + "pair": "luna_sol", + "classifier": "v2", + "card": "trained_card", + "adjustment": "none", + "alpha": 0, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.22795200000000002, + "mean_probability": 0.5264000000000001, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.066516, + "mean_probability": 0.7467999999999999, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.094084 + } + }, + "solved": 25, + "cost": 7.70803095, + "efficient_tasks": 0, + "savings_vs_capable": -0.0014777519258621918, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_trained_card_per_model_1", + "pair": "luna_sol", + "classifier": "v2", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.26574854498536066, + "mean_probability": 0.480752500039812, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.08653989674606706, + "mean_probability": 0.7058233579189824, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.09385573801418495 + } + }, + "solved": 25, + "cost": 7.70803095, + "efficient_tasks": 0, + "savings_vs_capable": -0.0014777519258621918, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_trained_card_per_model_10", + "pair": "luna_sol", + "classifier": "v2", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.27229598868635685, + "mean_probability": 0.47355221449889184, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.0865398568799184, + "mean_probability": 0.7058234256778516, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.09616689107102508 + } + }, + "solved": 25, + "cost": 7.70803095, + "efficient_tasks": 0, + "savings_vs_capable": -0.0014777519258621918, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_trained_card_per_model_100", + "pair": "luna_sol", + "classifier": "v2", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.28680638149963433, + "mean_probability": 0.45804748847938365, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.08653991827232044, + "mean_probability": 0.7058233213316926, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.10155499528533848 + } + }, + "solved": 25, + "cost": 7.70803095, + "efficient_tasks": 0, + "savings_vs_capable": -0.0014777519258621918, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_trained_card_task_conditioned_1", + "pair": "luna_sol", + "classifier": "v2", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.1978650333150751, + "mean_probability": 0.5987047835778408, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.06335276259701819, + "mean_probability": 0.7580793531241258, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.08560298031106717 + } + }, + "solved": 25, + "cost": 7.70803095, + "efficient_tasks": 0, + "savings_vs_capable": -0.0014777519258621918, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_trained_card_task_conditioned_10", + "pair": "luna_sol", + "classifier": "v2", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.250447683016267, + "mean_probability": 0.5046505444351115, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.08401050700413831, + "mean_probability": 0.7107567452841864, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.09157589219865582 + } + }, + "solved": 25, + "cost": 7.70803095, + "efficient_tasks": 0, + "savings_vs_capable": -0.0014777519258621918, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "v2_luna_sol_trained_card_task_conditioned_100", + "pair": "luna_sol", + "classifier": "v2", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "max_quality_gap": 0.05 + }, + "metrics": { + "efficient": { + "brier": 0.28255272326922126, + "mean_probability": 0.4632572658097204, + "mean_success": 0.92 + }, + "capable": { + "brier": 0.08651103723383644, + "mean_probability": 0.7058809401461426, + "mean_success": 1.0 + }, + "gap": { + "mse": 0.10023060064624457 + } + }, + "solved": 25, + "cost": 7.70803095, + "efficient_tasks": 0, + "savings_vs_capable": -0.0014777519258621918, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_trained_card_none_0", + "pair": "luna_sol", + "classifier": "cap", + "card": "trained_card", + "adjustment": "none", + "alpha": 0, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.13101200000000002, + "mean_probability": 0.682, + "mean_success": 0.92 + } + }, + "solved": 23, + "cost": 1.9355552000000003, + "efficient_tasks": 22, + "savings_vs_capable": 0.7485200198340651, + "lost_capable_successes": 2, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_trained_card_per_model_1", + "pair": "luna_sol", + "classifier": "cap", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.29280410701784, + "mean_probability": 0.4518076068566819, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.70728115, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013803330100241595, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_trained_card_per_model_10", + "pair": "luna_sol", + "classifier": "cap", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2928041070399462, + "mean_probability": 0.45180760683307397, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.70728115, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013803330100241595, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_trained_card_per_model_100", + "pair": "luna_sol", + "classifier": "cap", + "card": "trained_card", + "adjustment": "per_model", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2928041104565612, + "mean_probability": 0.4518076031843442, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.70728115, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013803330100241595, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_trained_card_task_conditioned_1", + "pair": "luna_sol", + "classifier": "cap", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 1, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.37414614819844033, + "mean_probability": 0.37326969735166743, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.70728115, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013803330100241595, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_trained_card_task_conditioned_10", + "pair": "luna_sol", + "classifier": "cap", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 10, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.30360542420143477, + "mean_probability": 0.44057025407348505, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.70728115, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013803330100241595, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + }, + { + "key": "cap_luna_sol_trained_card_task_conditioned_100", + "pair": "luna_sol", + "classifier": "cap", + "card": "trained_card", + "adjustment": "task_conditioned", + "alpha": 100, + "boundary": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "metrics": { + "efficient": { + "brier": 0.2939207425617313, + "mean_probability": 0.45063199845906105, + "mean_success": 0.92 + } + }, + "solved": 25, + "cost": 7.70728115, + "efficient_tasks": 0, + "savings_vs_capable": -0.0013803330100241595, + "lost_capable_successes": 0, + "gained_efficient_successes": 0 + } + ] +} diff --git a/cookbook/auto_router_roi_training/benchmark_results.json b/cookbook/auto_router_roi_training/benchmark_results.json new file mode 100644 index 00000000000..12c9ec87f5c --- /dev/null +++ b/cookbook/auto_router_roi_training/benchmark_results.json @@ -0,0 +1,4783 @@ +{ + "protocol": { + "created_on": "2026-09-14", + "training_source": "DeepSWE v1.1 published repeated trials", + "training_task_revision": "0b9fabbb63b9104d678fe965e1632f2dd9eaa2ea", + "train_tasks": 83, + "validation_tasks": 30, + "live_holdout": "25 SWE-bench Verified tasks, repository-disjoint from DeepSWE", + "live_ids": [ + "pallets__flask-5014", + "sympy__sympy-16792", + "django__django-13964", + "mwaskom__seaborn-3069", + "sphinx-doc__sphinx-8621", + "pylint-dev__pylint-6903", + "astropy__astropy-14369", + "psf__requests-1921", + "matplotlib__matplotlib-20676", + "pytest-dev__pytest-6202", + "scikit-learn__scikit-learn-25973", + "sympy__sympy-15345", + "django__django-16560", + "sphinx-doc__sphinx-8035", + "pylint-dev__pylint-7277", + "astropy__astropy-14995", + "psf__requests-1724", + "matplotlib__matplotlib-20859", + "pytest-dev__pytest-7521", + "scikit-learn__scikit-learn-26323", + "sympy__sympy-22080", + "django__django-12858", + "sphinx-doc__sphinx-9320", + "pylint-dev__pylint-6528", + "astropy__astropy-13236" + ], + "live_dataset_sha256": "a45b1fe4e2f0c8390b2b2938ac83e92ed5979000856808f3679c07812e9e6dcd", + "judge_model": "openai/gpt-5.6-luna", + "judge_effort": "low", + "solver_models": [ + "anthropic/claude-sonnet-5", + "anthropic/claude-opus-5", + "openai/gpt-5.6-luna", + "openai/gpt-5.6-sol" + ], + "solver_effort": "high", + "solver_max_completion_tokens": 8192, + "solver_step_limit": 150, + "solver_cost_limit_usd": 5, + "solver_parallelism": 3, + "harness": "mini-swe-agent==2.0.0", + "grader": "swebench==4.1.0", + "variations": [ + "original_card", + "research_card", + "training_conditioned_card" + ], + "adjustments": [ + "none", + "per_model_positive_logit", + "task_conditioned_ridge_logit" + ], + "policy_selection": "minimum validation solver cost with <= 0 net loss versus capable; also report 0.02 and 0.05 regret sensitivity", + "fitting": "train only; hyperparameters and cards selected on validation; freeze before opening live grades", + "paired_replay": "Each live solver runs once per task. Frozen task-pinned policies reuse matched attempts, plus their measured judge costs. No claim of independent live router arms.", + "limits": "DeepSWE to SWE-bench is distribution transfer. Research card changes are hypotheses. Twenty-five tasks cannot establish small quality differences.", + "harness_correction": "Pilot runs excluded before grading: preactivate the testbed Conda environment for every tool command. Full 25-task trial restarts on all models with this correction.", + "baseline_settings": { + "capability": { + "base_threshold": 0.5, + "threshold_step": 0.1 + }, + "llm_v2": { + "max_quality_gap": 0.05 + } + }, + "selection_cost": "published contestant solver cost plus gateway-reported judge cost; rate-based fallback if cost header absent", + "threshold_resolution": 0.0001, + "openai_parallelism": 2, + "openai_extension": "Added after frozen public-data selection showed stronger savings, before inspecting any live quality grades", + "superseded_x86_ids": [ + "pallets__flask-5014", + "sympy__sympy-16792", + "django__django-13964", + "mwaskom__seaborn-3069", + "sphinx-doc__sphinx-7757", + "pylint-dev__pylint-6386", + "astropy__astropy-8707", + "psf__requests-1921", + "matplotlib__matplotlib-24177", + "pytest-dev__pytest-6202", + "scikit-learn__scikit-learn-12682", + "pydata__xarray-6599", + "sympy__sympy-15345", + "django__django-16560", + "mwaskom__seaborn-3187", + "sphinx-doc__sphinx-8621", + "pylint-dev__pylint-6903", + "astropy__astropy-14369", + "psf__requests-1724", + "matplotlib__matplotlib-23476", + "pytest-dev__pytest-7324", + "scikit-learn__scikit-learn-13142", + "pydata__xarray-7229", + "sympy__sympy-22080", + "django__django-12858" + ], + "native_amendment": "Native ARM64 images selected by the same seeded repository/task order, skipping missing images. No live grades were inspected. All x86 attempts excluded due emulator startup failures. This limits generalization to platform-eligible tasks.", + "architecture": "arm64", + "openai_adapter_amendment": { + "time_utc": "2026-09-14T23:37:57.375320+00:00", + "reason": "Gateway Responses bridge returns sequential output blocks in separate Chat Completion choices. mini-SWE-agent read only choices[0] and discarded all tool calls. Archived the OpenAI pilot trajectories, all with zero executed tool messages, independent of grades. Concatenate sequential resp_ output blocks for single-sample OpenAI requests. Log all responses and bill format errors. Policies and tasks unchanged." + }, + "cost_accounting_amendment": { + "time_utc": "2026-09-14T23:43:28.751453+00:00", + "reason": "The stock harness loses billed cost when response parsing raises a FormatError. Model-level metering now records every successful gateway response before parsing and enforces the same USD 5 budget using all billed replies. All in-flight Anthropic attempts restarted without inspecting their hidden grades; completed attempts retained only after confirming no parse-error replies. Frozen policies and task identities unchanged.", + "retained_complete": [ + "claude-sonnet-5/pallets__flask-5014", + "claude-sonnet-5/sympy__sympy-16792", + "claude-opus-5/pallets__flask-5014", + "claude-opus-5/sympy__sympy-16792" + ], + "excluded_inflight": [ + "claude-sonnet-5/sphinx-doc__sphinx-8621", + "claude-sonnet-5/mwaskom__seaborn-3069", + "claude-sonnet-5/django__django-13964" + ] + }, + "live_dataset_revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a", + "live_dataset_source": "https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified/tree/c104f840cc67f8b6eec6f759ebc8b2693d585d4a", + "grading_isolation_correction": { + "time_utc": "2026-09-15T00:21:16.270656+00:00", + "reason": "Official grader names containers by task/run, not model. Parallel per-model evaluations could conflict with HTTP 409. Completed reports retained only after checking their model patch matches the saved solver patch; incomplete grades retried with a model-specific run ID. Grading uses one worker per family process to avoid shared logger interference. No solver attempts repeated and no grade infrastructure error counted as a model failure.", + "retained_completed_grades": 72, + "archived_incomplete_grades": [ + "grading_infrastructure_retries/claude-opus-5/astropy__astropy-14995/grade.json", + "grading_infrastructure_retries/claude-opus-5/psf__requests-1724/grade.json" + ] + }, + "solver_rate_limit_retry": { + "time_utc": "2026-09-15T01:37:13.281068+00:00", + "model": "openai/gpt-5.6-sol", + "task": "pylint-dev__pylint-6528", + "reason": "Gateway HTTP 429 exhausted four transport attempts before a patch was submitted. Archived the incomplete trajectory and billed responses, then retried only this task from the same image and settings. No solver grade existed for this attempt; completed solver runs and frozen policies were preserved.", + "excluded_attempt_path": "solver_infrastructure_retries/gpt-5.6-sol/pylint-dev__pylint-6528", + "excluded_billed_cost_usd": 0.3449082 + }, + "grading_slow_host_retry": { + "time_utc": "2026-09-15T01:57:36.188181+00:00", + "task": "pylint-dev__pylint-6528", + "model": "gpt-5.6-sol", + "patch_sha256": "ae4aafb976402102db8f49a9b1b31cf4a5dd40cdf0b358058e87752218cf4bbe", + "reason": "The official test suite exceeded its 900-second infrastructure deadline while other host jobs were active. The same simple CLI command on the unmodified official image took 101.66 seconds and succeeded, independently of the solver patch. Retry only grading of the identical saved patch under a fresh run ID with a 3600-second deadline. No new solver attempt or changed solver budget.", + "control_file": "grader_cli_control.json", + "original_grade": "grading_infrastructure_retries/slow_host/gpt-5.6-sol/pylint-dev__pylint-6528/grade.json", + "retry_timeout_seconds": 3600 + } + }, + "source": "100 fresh native ARM64 SWE-bench solver attempts plus fresh classifier forecasts", + "policy_evaluation": "Task-pinned paired replay over the same fresh attempts, not independent full router arms", + "selection_sha256": "ba2b00700aa338080077936a1fd794fd3bfa85fde28baef1fc5560f319ab1f32", + "results": [ + { + "pair": "sonnet_opus", + "policy": "always_efficient", + "solved": 24, + "tasks": 25, + "cost": 7.422205000000001, + "judge_cost": 0.0, + "savings_vs_capable": 0.14681669802122588, + "efficient_tasks": 25, + "lost_capable_successes": 1, + "gained_efficient_successes": 2, + "quality_difference_pp": 4.0, + "quality_difference_cluster_ci95_pp": [ + -8.695652173913043, + 17.391304347826086 + ], + "savings_cluster_ci95": [ + -0.24866662016843258, + 0.4434814724713227 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0420538 + }, + { + "task": "sympy__sympy-16792", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.19676100000000002 + }, + { + "task": "django__django-13964", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.9597126 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 1.2149754000000001 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.2626202 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.14270520000000003 + }, + { + "task": "astropy__astropy-14369", + "selected": "claude-sonnet-5", + "solved": false, + "cost": 0.5075477999999999 + }, + { + "task": "psf__requests-1921", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.039054700000000005 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 1.0838079000000005 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.10625200000000003 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.1266076 + }, + { + "task": "sympy__sympy-15345", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0728639 + }, + { + "task": "django__django-16560", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.5436054 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.44484859999999987 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.061800600000000004 + }, + { + "task": "astropy__astropy-14995", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0404853 + }, + { + "task": "psf__requests-1724", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0869914 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.1183156 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0494126 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.04162500000000001 + }, + { + "task": "sympy__sympy-22080", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.48527220000000004 + }, + { + "task": "django__django-12858", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.12366310000000003 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.04595890000000001 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.3607400999999999 + }, + { + "task": "astropy__astropy-13236", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.26452410000000004 + } + ], + "savings_vs_efficient": 0.0 + }, + { + "pair": "sonnet_opus", + "policy": "always_capable", + "solved": 23, + "tasks": 25, + "cost": 8.699426000000003, + "judge_cost": 0.0, + "savings_vs_capable": 0.0, + "efficient_tasks": 0, + "lost_capable_successes": 0, + "gained_efficient_successes": 0, + "quality_difference_pp": 0.0, + "quality_difference_cluster_ci95_pp": [ + 0.0, + 0.0 + ], + "savings_cluster_ci95": [ + 0.0, + 0.0 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.21910125000000003 + }, + { + "task": "sympy__sympy-16792", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.43841425 + }, + { + "task": "django__django-13964", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.132728 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.9364415000000001 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.19985625 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.18283075000000001 + }, + { + "task": "astropy__astropy-14369", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.6680605000000001 + }, + { + "task": "psf__requests-1921", + "selected": "claude-opus-5", + "solved": false, + "cost": 0.09872325 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.2870272499999999 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.0981345 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16898575000000002 + }, + { + "task": "sympy__sympy-15345", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16719575000000003 + }, + { + "task": "django__django-16560", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7521627499999999 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7252282499999999 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.159063 + }, + { + "task": "astropy__astropy-14995", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.08700474999999998 + }, + { + "task": "psf__requests-1724", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.11912225000000001 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.3029935 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09132525000000001 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16044425 + }, + { + "task": "sympy__sympy-22080", + "selected": "claude-opus-5", + "solved": false, + "cost": 1.3774870000000001 + }, + { + "task": "django__django-12858", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09448024999999999 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.13017725 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.5672952499999999 + }, + { + "task": "astropy__astropy-13236", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.53514325 + } + ], + "savings_vs_efficient": -0.17208107294260966 + }, + { + "pair": "sonnet_opus", + "policy": "baseline_cap", + "solved": 24, + "tasks": 25, + "cost": 7.438772450000001, + "judge_cost": 0.01656745, + "savings_vs_capable": 0.1449122677749085, + "efficient_tasks": 25, + "lost_capable_successes": 1, + "gained_efficient_successes": 2, + "quality_difference_pp": 4.0, + "quality_difference_cluster_ci95_pp": [ + -8.695652173913043, + 17.391304347826086 + ], + "savings_cluster_ci95": [ + -0.2508189402535668, + 0.4414604801154941 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.042666100000000005 + }, + { + "task": "sympy__sympy-16792", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.19733695 + }, + { + "task": "django__django-13964", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.9603764 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 1.21552845 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.2632781 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.14346800000000004 + }, + { + "task": "astropy__astropy-14369", + "selected": "claude-sonnet-5", + "solved": false, + "cost": 0.5083022999999999 + }, + { + "task": "psf__requests-1921", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.039545800000000006 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 1.0844305500000004 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.10686150000000003 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.12725864999999997 + }, + { + "task": "sympy__sympy-15345", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0734232 + }, + { + "task": "django__django-16560", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.5442966 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.44547494999999987 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.062428750000000005 + }, + { + "task": "astropy__astropy-14995", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0411542 + }, + { + "task": "psf__requests-1724", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.08817065 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.118901 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.05018565 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.04234545000000001 + }, + { + "task": "sympy__sympy-22080", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.48591750000000006 + }, + { + "task": "django__django-12858", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.12419665000000002 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.04654485000000001 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.3614648999999999 + }, + { + "task": "astropy__astropy-13236", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.26521530000000004 + } + ], + "savings_vs_efficient": -0.0022321466464481077 + }, + { + "pair": "sonnet_opus", + "policy": "baseline_v2", + "solved": 23, + "tasks": 25, + "cost": 8.61870845, + "judge_cost": 0.01654485, + "savings_vs_capable": 0.009278491477484052, + "efficient_tasks": 1, + "lost_capable_successes": 0, + "gained_efficient_successes": 0, + "quality_difference_pp": 0.0, + "quality_difference_cluster_ci95_pp": [ + 0.0, + 0.0 + ], + "savings_cluster_ci95": [ + -0.002646288654394835, + 0.0364068556414913 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.21954465000000004 + }, + { + "task": "sympy__sympy-16792", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.4390243 + }, + { + "task": "django__django-13964", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.13334095 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.9370243000000001 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.2005373 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.1835632 + }, + { + "task": "astropy__astropy-14369", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.6687612500000001 + }, + { + "task": "psf__requests-1921", + "selected": "claude-opus-5", + "solved": false, + "cost": 0.09927695 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.2876613499999999 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.0988219 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16984780000000002 + }, + { + "task": "sympy__sympy-15345", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16776220000000003 + }, + { + "task": "django__django-16560", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7527215999999999 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7257540499999999 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0623789 + }, + { + "task": "astropy__astropy-14995", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.08770084999999998 + }, + { + "task": "psf__requests-1724", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.12020850000000001 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.3037189 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09218505 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16119625 + }, + { + "task": "sympy__sympy-22080", + "selected": "claude-opus-5", + "solved": false, + "cost": 1.3780946 + }, + { + "task": "django__django-12858", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09505214999999999 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.1307673 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.56796615 + }, + { + "task": "astropy__astropy-13236", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.535798 + } + ], + "savings_vs_efficient": -0.16120592869639117 + }, + { + "pair": "sonnet_opus", + "policy": "trained_cap_validation_regret_0", + "solved": 24, + "tasks": 25, + "cost": 7.705032350000001, + "judge_cost": 0.010362449999999999, + "savings_vs_capable": 0.11430566223564653, + "efficient_tasks": 22, + "lost_capable_successes": 1, + "gained_efficient_successes": 2, + "quality_difference_pp": 4.0, + "quality_difference_cluster_ci95_pp": [ + -8.695652173913043, + 17.391304347826086 + ], + "savings_cluster_ci95": [ + -0.25029247740141247, + 0.4037151806140097 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.042297180000000004 + }, + { + "task": "sympy__sympy-16792", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.19710963 + }, + { + "task": "django__django-13964", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.96003748 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.9367744300000002 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.26299318000000005 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.14310508000000002 + }, + { + "task": "astropy__astropy-14369", + "selected": "claude-sonnet-5", + "solved": false, + "cost": 0.5080857799999999 + }, + { + "task": "psf__requests-1921", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.03949848 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 1.0841444300000005 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.10663538000000003 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.12742973 + }, + { + "task": "sympy__sympy-15345", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.07317428 + }, + { + "task": "django__django-16560", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.54396248 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7254904799999999 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.062092230000000005 + }, + { + "task": "astropy__astropy-14995", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.04095448 + }, + { + "task": "psf__requests-1724", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.08774293 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.11867608 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.050033930000000004 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.04209293000000001 + }, + { + "task": "sympy__sympy-22080", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.48563018 + }, + { + "task": "django__django-12858", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.12401253000000002 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.04626233000000001 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.36127357999999987 + }, + { + "task": "astropy__astropy-13236", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.53552313 + } + ], + "savings_vs_efficient": -0.03810556970603751, + "versus_original_classifier": { + "solve_difference": 0, + "savings": -0.035793526659092745, + "lost_successes": 0, + "gained_successes": 0 + } + }, + { + "pair": "sonnet_opus", + "policy": "trained_cap_validation_regret_0.02", + "solved": 24, + "tasks": 25, + "cost": 7.63325685, + "judge_cost": 0.010409249999999998, + "savings_vs_capable": 0.12255626405696207, + "efficient_tasks": 22, + "lost_capable_successes": 1, + "gained_efficient_successes": 2, + "quality_difference_pp": 4.0, + "quality_difference_cluster_ci95_pp": [ + -8.695652173913043, + 17.391304347826086 + ], + "savings_cluster_ci95": [ + -0.2832854868327926, + 0.4268841187043486 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.042306300000000005 + }, + { + "task": "sympy__sympy-16792", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.19707795000000003 + }, + { + "task": "django__django-13964", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.960163 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.9367403500000001 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.2630059 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.14314540000000003 + }, + { + "task": "astropy__astropy-14369", + "selected": "claude-sonnet-5", + "solved": false, + "cost": 0.5079244999999999 + }, + { + "task": "psf__requests-1921", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.03966000000000001 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 1.0841919500000006 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.10665890000000003 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.12728045 + }, + { + "task": "sympy__sympy-15345", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0732302 + }, + { + "task": "django__django-16560", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7524929499999998 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.44514874999999987 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.062158950000000004 + }, + { + "task": "astropy__astropy-14995", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0409288 + }, + { + "task": "psf__requests-1724", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.08771005 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.11866 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.05001665 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.04202525000000001 + }, + { + "task": "sympy__sympy-22080", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.4856825 + }, + { + "task": "django__django-12858", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.12414045000000003 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.04623905000000001 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.3611746999999999 + }, + { + "task": "astropy__astropy-13236", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.5354938499999999 + } + ], + "savings_vs_efficient": -0.028435195470887686, + "versus_original_classifier": { + "solve_difference": 0, + "savings": -0.026144690042239294, + "lost_successes": 0, + "gained_successes": 0 + } + }, + { + "pair": "sonnet_opus", + "policy": "trained_cap_validation_regret_0.05", + "solved": 24, + "tasks": 25, + "cost": 7.63325685, + "judge_cost": 0.010409249999999998, + "savings_vs_capable": 0.12255626405696207, + "efficient_tasks": 22, + "lost_capable_successes": 1, + "gained_efficient_successes": 2, + "quality_difference_pp": 4.0, + "quality_difference_cluster_ci95_pp": [ + -8.695652173913043, + 17.391304347826086 + ], + "savings_cluster_ci95": [ + -0.2832854868327926, + 0.4268841187043486 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.042306300000000005 + }, + { + "task": "sympy__sympy-16792", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.19707795000000003 + }, + { + "task": "django__django-13964", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.960163 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.9367403500000001 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.2630059 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.14314540000000003 + }, + { + "task": "astropy__astropy-14369", + "selected": "claude-sonnet-5", + "solved": false, + "cost": 0.5079244999999999 + }, + { + "task": "psf__requests-1921", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.03966000000000001 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 1.0841919500000006 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.10665890000000003 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.12728045 + }, + { + "task": "sympy__sympy-15345", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0732302 + }, + { + "task": "django__django-16560", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7524929499999998 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.44514874999999987 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.062158950000000004 + }, + { + "task": "astropy__astropy-14995", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.0409288 + }, + { + "task": "psf__requests-1724", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.08771005 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.11866 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.05001665 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.04202525000000001 + }, + { + "task": "sympy__sympy-22080", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.4856825 + }, + { + "task": "django__django-12858", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.12414045000000003 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.04623905000000001 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "claude-sonnet-5", + "solved": true, + "cost": 0.3611746999999999 + }, + { + "task": "astropy__astropy-13236", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.5354938499999999 + } + ], + "savings_vs_efficient": -0.028435195470887686, + "versus_original_classifier": { + "solve_difference": 0, + "savings": -0.026144690042239294, + "lost_successes": 0, + "gained_successes": 0 + } + }, + { + "pair": "sonnet_opus", + "policy": "trained_v2_validation_regret_0", + "solved": 23, + "tasks": 25, + "cost": 8.71103165, + "judge_cost": 0.01160565, + "savings_vs_capable": -0.0013340707766233972, + "efficient_tasks": 0, + "lost_capable_successes": 0, + "gained_efficient_successes": 0, + "quality_difference_pp": 0.0, + "quality_difference_cluster_ci95_pp": [ + 0.0, + 0.0 + ], + "savings_cluster_ci95": [ + -0.0020970951011087978, + -0.0009375550391474476 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.21941553000000003 + }, + { + "task": "sympy__sympy-16792", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.43889398 + }, + { + "task": "django__django-13964", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.13313263 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.9368279800000001 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.20037098 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.18338368000000002 + }, + { + "task": "astropy__astropy-14369", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.66859013 + }, + { + "task": "psf__requests-1921", + "selected": "claude-opus-5", + "solved": false, + "cost": 0.09908663 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.2874782299999999 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09866038 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16943308 + }, + { + "task": "sympy__sympy-15345", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16752388000000004 + }, + { + "task": "django__django-16560", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7525336799999999 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7255853299999999 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.15946658 + }, + { + "task": "astropy__astropy-14995", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.08749612999999998 + }, + { + "task": "psf__requests-1724", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.11999538000000001 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.30349138000000003 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09198393 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16104553000000002 + }, + { + "task": "sympy__sympy-22080", + "selected": "claude-opus-5", + "solved": false, + "cost": 1.3778778800000002 + }, + { + "task": "django__django-12858", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09483542999999998 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.13056257999999998 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.56777103 + }, + { + "task": "astropy__astropy-13236", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.53558968 + } + ], + "savings_vs_efficient": -0.17364471204985565, + "versus_original_classifier": { + "solve_difference": 0, + "savings": -0.0107119530189006, + "lost_successes": 0, + "gained_successes": 0 + } + }, + { + "pair": "sonnet_opus", + "policy": "trained_v2_validation_regret_0.02", + "solved": 23, + "tasks": 25, + "cost": 8.71103165, + "judge_cost": 0.01160565, + "savings_vs_capable": -0.0013340707766233972, + "efficient_tasks": 0, + "lost_capable_successes": 0, + "gained_efficient_successes": 0, + "quality_difference_pp": 0.0, + "quality_difference_cluster_ci95_pp": [ + 0.0, + 0.0 + ], + "savings_cluster_ci95": [ + -0.0020970951011087978, + -0.0009375550391474476 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.21941553000000003 + }, + { + "task": "sympy__sympy-16792", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.43889398 + }, + { + "task": "django__django-13964", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.13313263 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.9368279800000001 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.20037098 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.18338368000000002 + }, + { + "task": "astropy__astropy-14369", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.66859013 + }, + { + "task": "psf__requests-1921", + "selected": "claude-opus-5", + "solved": false, + "cost": 0.09908663 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.2874782299999999 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09866038 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16943308 + }, + { + "task": "sympy__sympy-15345", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16752388000000004 + }, + { + "task": "django__django-16560", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7525336799999999 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7255853299999999 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.15946658 + }, + { + "task": "astropy__astropy-14995", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.08749612999999998 + }, + { + "task": "psf__requests-1724", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.11999538000000001 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.30349138000000003 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09198393 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16104553000000002 + }, + { + "task": "sympy__sympy-22080", + "selected": "claude-opus-5", + "solved": false, + "cost": 1.3778778800000002 + }, + { + "task": "django__django-12858", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09483542999999998 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.13056257999999998 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.56777103 + }, + { + "task": "astropy__astropy-13236", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.53558968 + } + ], + "savings_vs_efficient": -0.17364471204985565, + "versus_original_classifier": { + "solve_difference": 0, + "savings": -0.0107119530189006, + "lost_successes": 0, + "gained_successes": 0 + } + }, + { + "pair": "sonnet_opus", + "policy": "trained_v2_validation_regret_0.05", + "solved": 23, + "tasks": 25, + "cost": 8.71103165, + "judge_cost": 0.01160565, + "savings_vs_capable": -0.0013340707766233972, + "efficient_tasks": 0, + "lost_capable_successes": 0, + "gained_efficient_successes": 0, + "quality_difference_pp": 0.0, + "quality_difference_cluster_ci95_pp": [ + 0.0, + 0.0 + ], + "savings_cluster_ci95": [ + -0.0020970951011087978, + -0.0009375550391474476 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.21941553000000003 + }, + { + "task": "sympy__sympy-16792", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.43889398 + }, + { + "task": "django__django-13964", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.13313263 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.9368279800000001 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.20037098 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.18338368000000002 + }, + { + "task": "astropy__astropy-14369", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.66859013 + }, + { + "task": "psf__requests-1921", + "selected": "claude-opus-5", + "solved": false, + "cost": 0.09908663 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.2874782299999999 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09866038 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16943308 + }, + { + "task": "sympy__sympy-15345", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16752388000000004 + }, + { + "task": "django__django-16560", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7525336799999999 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.7255853299999999 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.15946658 + }, + { + "task": "astropy__astropy-14995", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.08749612999999998 + }, + { + "task": "psf__requests-1724", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.11999538000000001 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.30349138000000003 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09198393 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.16104553000000002 + }, + { + "task": "sympy__sympy-22080", + "selected": "claude-opus-5", + "solved": false, + "cost": 1.3778778800000002 + }, + { + "task": "django__django-12858", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.09483542999999998 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.13056257999999998 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.56777103 + }, + { + "task": "astropy__astropy-13236", + "selected": "claude-opus-5", + "solved": true, + "cost": 0.53558968 + } + ], + "savings_vs_efficient": -0.17364471204985565, + "versus_original_classifier": { + "solve_difference": 0, + "savings": -0.0107119530189006, + "lost_successes": 0, + "gained_successes": 0 + } + }, + { + "pair": "luna_sol", + "policy": "always_efficient", + "solved": 23, + "tasks": 25, + "cost": 0.44381828, + "judge_cost": 0.0, + "savings_vs_capable": 0.942336228772148, + "efficient_tasks": 25, + "lost_capable_successes": 2, + "gained_efficient_successes": 0, + "quality_difference_pp": -8.0, + "quality_difference_cluster_ci95_pp": [ + -19.230769230769234, + 0.0 + ], + "savings_cluster_ci95": [ + 0.9359842363728711, + 0.947990729908217 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.006036549999999999 + }, + { + "task": "sympy__sympy-16792", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.026326870000000002 + }, + { + "task": "django__django-13964", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01957996 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.04791672 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.013940949999999995 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00832655 + }, + { + "task": "astropy__astropy-14369", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.024806449999999997 + }, + { + "task": "psf__requests-1921", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00863327 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.0314039 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.015579949999999997 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01486182 + }, + { + "task": "sympy__sympy-15345", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.0107484 + }, + { + "task": "django__django-16560", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.015834799999999996 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.028651450000000002 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.011006789999999999 + }, + { + "task": "astropy__astropy-14995", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01503533 + }, + { + "task": "psf__requests-1724", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.011925660000000001 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00656957 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01015545 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.012308079999999999 + }, + { + "task": "sympy__sympy-22080", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.03821885999999999 + }, + { + "task": "django__django-12858", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.019858380000000002 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.013937859999999998 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.022390809999999997 + }, + { + "task": "astropy__astropy-13236", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00976385 + } + ], + "savings_vs_efficient": 0.0 + }, + { + "pair": "luna_sol", + "policy": "always_capable", + "solved": 25, + "tasks": 25, + "cost": 7.6966572, + "judge_cost": 0.0, + "savings_vs_capable": 0.0, + "efficient_tasks": 0, + "lost_capable_successes": 0, + "gained_efficient_successes": 0, + "quality_difference_pp": 0.0, + "quality_difference_cluster_ci95_pp": [ + 0.0, + 0.0 + ], + "savings_cluster_ci95": [ + 0.0, + 0.0 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.1650396 + }, + { + "task": "sympy__sympy-16792", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.36961879999999997 + }, + { + "task": "django__django-13964", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.3488006 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.6197701999999999 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.22146640000000004 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.19231320000000002 + }, + { + "task": "astropy__astropy-14369", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.43219620000000003 + }, + { + "task": "psf__requests-1921", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.23612840000000002 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.3209066 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.21509500000000004 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.24569480000000005 + }, + { + "task": "sympy__sympy-15345", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.249051 + }, + { + "task": "django__django-16560", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.46097499999999997 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.47764600000000007 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.15690300000000001 + }, + { + "task": "astropy__astropy-14995", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.242054 + }, + { + "task": "psf__requests-1724", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.22139700000000004 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.210145 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.21148460000000002 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.17821340000000002 + }, + { + "task": "sympy__sympy-22080", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.4984838 + }, + { + "task": "django__django-12858", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.3036818 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.3727256 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.5114310000000001 + }, + { + "task": "astropy__astropy-13236", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.23543620000000004 + } + ], + "savings_vs_efficient": -16.341911198430132 + }, + { + "pair": "luna_sol", + "policy": "baseline_cap", + "solved": 23, + "tasks": 25, + "cost": 0.45423660999999993, + "judge_cost": 0.01041833, + "savings_vs_capable": 0.9409826112562217, + "efficient_tasks": 25, + "lost_capable_successes": 2, + "gained_efficient_successes": 0, + "quality_difference_pp": -8.0, + "quality_difference_cluster_ci95_pp": [ + -19.230769230769234, + 0.0 + ], + "savings_cluster_ci95": [ + 0.9346189512291398, + 0.946625478409905 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.006349029999999999 + }, + { + "task": "sympy__sympy-16792", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.02690282 + }, + { + "task": "django__django-13964", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01988096 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.04846977 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.014231109999999995 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00866091 + }, + { + "task": "astropy__astropy-14369", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.025159249999999998 + }, + { + "task": "psf__requests-1921", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00889175 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.031748359999999996 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.016189449999999998 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01551287 + }, + { + "task": "sympy__sympy-15345", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.0113077 + }, + { + "task": "django__django-16560", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.016335559999999996 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.029277800000000003 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.011286889999999999 + }, + { + "task": "astropy__astropy-14995", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01570423 + }, + { + "task": "psf__requests-1724", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01240916 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.006834010000000001 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01053887 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.012613899999999999 + }, + { + "task": "sympy__sympy-22080", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.03845285999999999 + }, + { + "task": "django__django-12858", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.020391930000000003 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.014286439999999997 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.022744929999999997 + }, + { + "task": "astropy__astropy-13236", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.010056049999999999 + } + ], + "savings_vs_efficient": -0.02347431475783268 + }, + { + "pair": "luna_sol", + "policy": "baseline_v2", + "solved": 25, + "tasks": 25, + "cost": 5.8865393500000005, + "judge_cost": 0.01676085, + "savings_vs_capable": 0.23518233993843451, + "efficient_tasks": 7, + "lost_capable_successes": 0, + "gained_efficient_successes": 0, + "quality_difference_pp": 0.0, + "quality_difference_cluster_ci95_pp": [ + 0.0, + 0.0 + ], + "savings_cluster_ci95": [ + 0.0882773516679589, + 0.3920524578911113 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.006489549999999999 + }, + { + "task": "sympy__sympy-16792", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.37028194999999997 + }, + { + "task": "django__django-13964", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.34941385 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.6203713 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.22213455000000004 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.0090677 + }, + { + "task": "astropy__astropy-14369", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.43292845 + }, + { + "task": "psf__requests-1921", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.23667760000000002 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.32155059999999996 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.016361249999999997 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.24635915000000005 + }, + { + "task": "sympy__sympy-15345", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01127555 + }, + { + "task": "django__django-16560", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.46158455 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.4782105000000001 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.1574792 + }, + { + "task": "astropy__astropy-14995", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01578213 + }, + { + "task": "psf__requests-1724", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.22250635000000005 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.2108539 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.21231710000000004 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.1789813 + }, + { + "task": "sympy__sympy-22080", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.03884595999999999 + }, + { + "task": "django__django-12858", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.3042804 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.014557009999999999 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.5121562000000001 + }, + { + "task": "astropy__astropy-13236", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.23607325000000004 + } + ], + "savings_vs_efficient": -12.263399943778792 + }, + { + "pair": "luna_sol", + "policy": "trained_cap_validation_regret_0", + "solved": 23, + "tasks": 25, + "cost": 1.23361554, + "judge_cost": 0.01041833, + "savings_vs_capable": 0.8397206075385558, + "efficient_tasks": 23, + "lost_capable_successes": 2, + "gained_efficient_successes": 0, + "quality_difference_pp": -8.0, + "quality_difference_cluster_ci95_pp": [ + -19.230769230769234, + 0.0 + ], + "savings_cluster_ci95": [ + 0.6648193163218975, + 0.9443655815307329 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.006349029999999999 + }, + { + "task": "sympy__sympy-16792", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.02690282 + }, + { + "task": "django__django-13964", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01988096 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.62032325 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.22175656000000005 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00866091 + }, + { + "task": "astropy__astropy-14369", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.025159249999999998 + }, + { + "task": "psf__requests-1921", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00889175 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.031748359999999996 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.016189449999999998 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01551287 + }, + { + "task": "sympy__sympy-15345", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.0113077 + }, + { + "task": "django__django-16560", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.016335559999999996 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.029277800000000003 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.011286889999999999 + }, + { + "task": "astropy__astropy-14995", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01570423 + }, + { + "task": "psf__requests-1724", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01240916 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.006834010000000001 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01053887 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.012613899999999999 + }, + { + "task": "sympy__sympy-22080", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.03845285999999999 + }, + { + "task": "django__django-12858", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.020391930000000003 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.014286439999999997 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.022744929999999997 + }, + { + "task": "astropy__astropy-13236", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.010056049999999999 + } + ], + "savings_vs_efficient": -1.7795509910046965, + "versus_original_classifier": { + "solve_difference": 0, + "savings": -1.7157994596692685, + "lost_successes": 0, + "gained_successes": 0 + } + }, + { + "pair": "luna_sol", + "policy": "trained_cap_validation_regret_0.02", + "solved": 24, + "tasks": 25, + "cost": 5.6430830300000006, + "judge_cost": 0.01041833, + "savings_vs_capable": 0.2668137759857616, + "efficient_tasks": 8, + "lost_capable_successes": 1, + "gained_efficient_successes": 0, + "quality_difference_pp": -4.0, + "quality_difference_cluster_ci95_pp": [ + -13.636363636363635, + 0.0 + ], + "savings_cluster_ci95": [ + 0.07236860546374389, + 0.5300537443946068 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.16535208 + }, + { + "task": "sympy__sympy-16792", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.37019474999999996 + }, + { + "task": "django__django-13964", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.3491016 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.62032325 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.22175656000000005 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.19264756000000002 + }, + { + "task": "astropy__astropy-14369", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.025159249999999998 + }, + { + "task": "psf__requests-1921", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00889175 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.031748359999999996 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.016189449999999998 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.24634585000000003 + }, + { + "task": "sympy__sympy-15345", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.2496103 + }, + { + "task": "django__django-16560", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.46147575999999996 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.47827235000000007 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.15718310000000002 + }, + { + "task": "astropy__astropy-14995", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01570423 + }, + { + "task": "psf__requests-1724", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.22188050000000004 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.006834010000000001 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.21186802000000002 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.17851922000000003 + }, + { + "task": "sympy__sympy-22080", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.4987178 + }, + { + "task": "django__django-12858", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.020391930000000003 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.37307418 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.5117851200000001 + }, + { + "task": "astropy__astropy-13236", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.010056049999999999 + } + ], + "savings_vs_efficient": -11.714850388767223, + "versus_original_classifier": { + "solve_difference": 1, + "savings": -11.423223724745572, + "lost_successes": 0, + "gained_successes": 1 + } + }, + { + "pair": "luna_sol", + "policy": "trained_cap_validation_regret_0.05", + "solved": 23, + "tasks": 25, + "cost": 1.23361554, + "judge_cost": 0.01041833, + "savings_vs_capable": 0.8397206075385558, + "efficient_tasks": 23, + "lost_capable_successes": 2, + "gained_efficient_successes": 0, + "quality_difference_pp": -8.0, + "quality_difference_cluster_ci95_pp": [ + -19.230769230769234, + 0.0 + ], + "savings_cluster_ci95": [ + 0.6648193163218975, + 0.9443655815307329 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.006349029999999999 + }, + { + "task": "sympy__sympy-16792", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.02690282 + }, + { + "task": "django__django-13964", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01988096 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.62032325 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.22175656000000005 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00866091 + }, + { + "task": "astropy__astropy-14369", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.025159249999999998 + }, + { + "task": "psf__requests-1921", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00889175 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.031748359999999996 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.016189449999999998 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01551287 + }, + { + "task": "sympy__sympy-15345", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.0113077 + }, + { + "task": "django__django-16560", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.016335559999999996 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.029277800000000003 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.011286889999999999 + }, + { + "task": "astropy__astropy-14995", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01570423 + }, + { + "task": "psf__requests-1724", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01240916 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.006834010000000001 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01053887 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.012613899999999999 + }, + { + "task": "sympy__sympy-22080", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.03845285999999999 + }, + { + "task": "django__django-12858", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.020391930000000003 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.014286439999999997 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.022744929999999997 + }, + { + "task": "astropy__astropy-13236", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.010056049999999999 + } + ], + "savings_vs_efficient": -1.7795509910046965, + "versus_original_classifier": { + "solve_difference": 0, + "savings": -1.7157994596692685, + "lost_successes": 0, + "gained_successes": 0 + } + }, + { + "pair": "luna_sol", + "policy": "trained_v2_validation_regret_0", + "solved": 23, + "tasks": 25, + "cost": 2.08629171, + "judge_cost": 0.01676085, + "savings_vs_capable": 0.7289353474129003, + "efficient_tasks": 21, + "lost_capable_successes": 2, + "gained_efficient_successes": 0, + "quality_difference_pp": -8.0, + "quality_difference_cluster_ci95_pp": [ + -19.230769230769234, + 0.0 + ], + "savings_cluster_ci95": [ + 0.5359810465888823, + 0.8880320406858081 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.006489549999999999 + }, + { + "task": "sympy__sympy-16792", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.026990020000000003 + }, + { + "task": "django__django-13964", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.02019321 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.6203713 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.014609099999999995 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.0090677 + }, + { + "task": "astropy__astropy-14369", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.43292845 + }, + { + "task": "psf__requests-1921", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00918247 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.0320479 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.016361249999999997 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.015526169999999999 + }, + { + "task": "sympy__sympy-15345", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01127555 + }, + { + "task": "django__django-16560", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.46158455 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.02921595 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01158299 + }, + { + "task": "astropy__astropy-14995", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01578213 + }, + { + "task": "psf__requests-1724", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.013035010000000001 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00727847 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.21231710000000004 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01307598 + }, + { + "task": "sympy__sympy-22080", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.03884595999999999 + }, + { + "task": "django__django-12858", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.020456980000000003 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.014557009999999999 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.023116009999999996 + }, + { + "task": "astropy__astropy-13236", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.0104009 + } + ], + "savings_vs_efficient": -3.7007791341987977, + "versus_original_classifier": { + "solve_difference": -2, + "savings": 0.6455826444105908, + "lost_successes": 2, + "gained_successes": 0 + } + }, + { + "pair": "luna_sol", + "policy": "trained_v2_validation_regret_0.02", + "solved": 23, + "tasks": 25, + "cost": 0.8912362100000001, + "judge_cost": 0.0170699, + "savings_vs_capable": 0.8842047674930877, + "efficient_tasks": 23, + "lost_capable_successes": 2, + "gained_efficient_successes": 0, + "quality_difference_pp": -8.0, + "quality_difference_cluster_ci95_pp": [ + -19.230769230769234, + 0.0 + ], + "savings_cluster_ci95": [ + 0.7790813182256774, + 0.9431699893434674 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.006566999999999999 + }, + { + "task": "sympy__sympy-16792", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.027013170000000003 + }, + { + "task": "django__django-13964", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.02023316 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.04851697 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.014611849999999996 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00908245 + }, + { + "task": "astropy__astropy-14369", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.025527049999999996 + }, + { + "task": "psf__requests-1921", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.00937242 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.03206145 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.21582865000000004 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.24639070000000005 + }, + { + "task": "sympy__sympy-15345", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.0113155 + }, + { + "task": "django__django-16560", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.016403899999999996 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.029229500000000002 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.011631339999999999 + }, + { + "task": "astropy__astropy-14995", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01570688 + }, + { + "task": "psf__requests-1724", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01302696 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.0072536200000000006 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.0110219 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01307273 + }, + { + "task": "sympy__sympy-22080", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.03883551 + }, + { + "task": "django__django-12858", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.02044893 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.014551359999999998 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "gpt-5.6-luna", + "solved": false, + "cost": 0.023113959999999996 + }, + { + "task": "astropy__astropy-13236", + "selected": "gpt-5.6-luna", + "solved": true, + "cost": 0.01041925 + } + ], + "savings_vs_efficient": -1.0081106393364423, + "versus_original_classifier": { + "solve_difference": -2, + "savings": 0.848597595801343, + "lost_successes": 2, + "gained_successes": 0 + } + }, + { + "pair": "luna_sol", + "policy": "trained_v2_validation_regret_0.05", + "solved": 25, + "tasks": 25, + "cost": 7.713418050000001, + "judge_cost": 0.01676085, + "savings_vs_capable": -0.002177679161805779, + "efficient_tasks": 0, + "lost_capable_successes": 0, + "gained_efficient_successes": 0, + "quality_difference_pp": 0.0, + "quality_difference_cluster_ci95_pp": [ + 0.0, + 0.0 + ], + "savings_cluster_ci95": [ + -0.0027083820068265763, + -0.0018042435215650577 + ], + "per_task": [ + { + "task": "pallets__flask-5014", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.16549260000000002 + }, + { + "task": "sympy__sympy-16792", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.37028194999999997 + }, + { + "task": "django__django-13964", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.34941385 + }, + { + "task": "mwaskom__seaborn-3069", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.6203713 + }, + { + "task": "sphinx-doc__sphinx-8621", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.22213455000000004 + }, + { + "task": "pylint-dev__pylint-6903", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.19305435 + }, + { + "task": "astropy__astropy-14369", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.43292845 + }, + { + "task": "psf__requests-1921", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.23667760000000002 + }, + { + "task": "matplotlib__matplotlib-20676", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.32155059999999996 + }, + { + "task": "pytest-dev__pytest-6202", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.21587630000000005 + }, + { + "task": "scikit-learn__scikit-learn-25973", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.24635915000000005 + }, + { + "task": "sympy__sympy-15345", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.24957815 + }, + { + "task": "django__django-16560", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.46158455 + }, + { + "task": "sphinx-doc__sphinx-8035", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.4782105000000001 + }, + { + "task": "pylint-dev__pylint-7277", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.1574792 + }, + { + "task": "astropy__astropy-14995", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.24280079999999998 + }, + { + "task": "psf__requests-1724", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.22250635000000005 + }, + { + "task": "matplotlib__matplotlib-20859", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.2108539 + }, + { + "task": "pytest-dev__pytest-7521", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.21231710000000004 + }, + { + "task": "scikit-learn__scikit-learn-26323", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.1789813 + }, + { + "task": "sympy__sympy-22080", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.49911089999999997 + }, + { + "task": "django__django-12858", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.3042804 + }, + { + "task": "sphinx-doc__sphinx-9320", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.37334475 + }, + { + "task": "pylint-dev__pylint-6528", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.5121562000000001 + }, + { + "task": "astropy__astropy-13236", + "selected": "gpt-5.6-sol", + "solved": true, + "cost": 0.23607325000000004 + } + ], + "savings_vs_efficient": -16.379676317072835, + "versus_original_classifier": { + "solve_difference": 0, + "savings": -0.31034850722606655, + "lost_successes": 0, + "gained_successes": 0 + } + } + ], + "solver_results": { + "claude-sonnet-5": [ + { + "task": "pallets__flask-5014", + "solved": true, + "cost": 0.0420538, + "calls": 11, + "seconds": 64.96246719360352, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-16792", + "solved": true, + "cost": 0.19676100000000002, + "calls": 27, + "seconds": 221.51862812042236, + "exit_status": "Submitted" + }, + { + "task": "django__django-13964", + "solved": true, + "cost": 0.9597126, + "calls": 75, + "seconds": 1227.2067458629608, + "exit_status": "Submitted" + }, + { + "task": "mwaskom__seaborn-3069", + "solved": true, + "cost": 1.2149754000000001, + "calls": 84, + "seconds": 761.6485688686371, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-8621", + "solved": true, + "cost": 0.2626202, + "calls": 22, + "seconds": 227.73902487754822, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-6903", + "solved": true, + "cost": 0.14270520000000003, + "calls": 23, + "seconds": 108.67690300941467, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-14369", + "solved": false, + "cost": 0.5075477999999999, + "calls": 36, + "seconds": 343.32984590530396, + "exit_status": "Submitted" + }, + { + "task": "psf__requests-1921", + "solved": true, + "cost": 0.039054700000000005, + "calls": 9, + "seconds": 33.61791515350342, + "exit_status": "Submitted" + }, + { + "task": "matplotlib__matplotlib-20676", + "solved": true, + "cost": 1.0838079000000005, + "calls": 71, + "seconds": 545.636461019516, + "exit_status": "Submitted" + }, + { + "task": "pytest-dev__pytest-6202", + "solved": true, + "cost": 0.10625200000000003, + "calls": 21, + "seconds": 150.41276788711548, + "exit_status": "Submitted" + }, + { + "task": "scikit-learn__scikit-learn-25973", + "solved": true, + "cost": 0.1266076, + "calls": 21, + "seconds": 97.22808909416199, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-15345", + "solved": true, + "cost": 0.0728639, + "calls": 15, + "seconds": 45.69230794906616, + "exit_status": "Submitted" + }, + { + "task": "django__django-16560", + "solved": true, + "cost": 0.5436054, + "calls": 47, + "seconds": 173.06891322135925, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-8035", + "solved": true, + "cost": 0.44484859999999987, + "calls": 57, + "seconds": 233.74119400978088, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-7277", + "solved": true, + "cost": 0.061800600000000004, + "calls": 13, + "seconds": 57.77951407432556, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-14995", + "solved": true, + "cost": 0.0404853, + "calls": 7, + "seconds": 47.72135400772095, + "exit_status": "Submitted" + }, + { + "task": "psf__requests-1724", + "solved": true, + "cost": 0.0869914, + "calls": 19, + "seconds": 115.3474051952362, + "exit_status": "Submitted" + }, + { + "task": "matplotlib__matplotlib-20859", + "solved": true, + "cost": 0.1183156, + "calls": 24, + "seconds": 245.59339427947998, + "exit_status": "Submitted" + }, + { + "task": "pytest-dev__pytest-7521", + "solved": true, + "cost": 0.0494126, + "calls": 12, + "seconds": 88.70717406272888, + "exit_status": "Submitted" + }, + { + "task": "scikit-learn__scikit-learn-26323", + "solved": true, + "cost": 0.04162500000000001, + "calls": 8, + "seconds": 80.6632342338562, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-22080", + "solved": true, + "cost": 0.48527220000000004, + "calls": 46, + "seconds": 312.1787061691284, + "exit_status": "Submitted" + }, + { + "task": "django__django-12858", + "solved": true, + "cost": 0.12366310000000003, + "calls": 16, + "seconds": 106.6139121055603, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-9320", + "solved": true, + "cost": 0.04595890000000001, + "calls": 10, + "seconds": 27.077608823776245, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-6528", + "solved": true, + "cost": 0.3607400999999999, + "calls": 43, + "seconds": 210.1569790840149, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-13236", + "solved": true, + "cost": 0.26452410000000004, + "calls": 40, + "seconds": 223.09438824653625, + "exit_status": "Submitted" + } + ], + "claude-opus-5": [ + { + "task": "pallets__flask-5014", + "solved": true, + "cost": 0.21910125000000003, + "calls": 11, + "seconds": 80.48021674156189, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-16792", + "solved": true, + "cost": 0.43841425, + "calls": 22, + "seconds": 230.8556559085846, + "exit_status": "Submitted" + }, + { + "task": "django__django-13964", + "solved": true, + "cost": 0.132728, + "calls": 8, + "seconds": 115.08643412590027, + "exit_status": "Submitted" + }, + { + "task": "mwaskom__seaborn-3069", + "solved": true, + "cost": 0.9364415000000001, + "calls": 26, + "seconds": 459.0944368839264, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-8621", + "solved": true, + "cost": 0.19985625, + "calls": 13, + "seconds": 135.37433886528015, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-6903", + "solved": true, + "cost": 0.18283075000000001, + "calls": 14, + "seconds": 97.46161198616028, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-14369", + "solved": true, + "cost": 0.6680605000000001, + "calls": 22, + "seconds": 296.15429496765137, + "exit_status": "Submitted" + }, + { + "task": "psf__requests-1921", + "solved": false, + "cost": 0.09872325, + "calls": 8, + "seconds": 30.57511281967163, + "exit_status": "Submitted" + }, + { + "task": "matplotlib__matplotlib-20676", + "solved": true, + "cost": 0.2870272499999999, + "calls": 10, + "seconds": 89.92236089706421, + "exit_status": "Submitted" + }, + { + "task": "pytest-dev__pytest-6202", + "solved": true, + "cost": 0.0981345, + "calls": 7, + "seconds": 41.59011101722717, + "exit_status": "Submitted" + }, + { + "task": "scikit-learn__scikit-learn-25973", + "solved": true, + "cost": 0.16898575000000002, + "calls": 9, + "seconds": 60.229105949401855, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-15345", + "solved": true, + "cost": 0.16719575000000003, + "calls": 6, + "seconds": 45.88169598579407, + "exit_status": "Submitted" + }, + { + "task": "django__django-16560", + "solved": true, + "cost": 0.7521627499999999, + "calls": 26, + "seconds": 175.14391589164734, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-8035", + "solved": true, + "cost": 0.7252282499999999, + "calls": 21, + "seconds": 287.1081998348236, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-7277", + "solved": true, + "cost": 0.159063, + "calls": 10, + "seconds": 69.97761678695679, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-14995", + "solved": true, + "cost": 0.08700474999999998, + "calls": 4, + "seconds": 38.71838688850403, + "exit_status": "Submitted" + }, + { + "task": "psf__requests-1724", + "solved": true, + "cost": 0.11912225000000001, + "calls": 8, + "seconds": 46.86448574066162, + "exit_status": "Submitted" + }, + { + "task": "matplotlib__matplotlib-20859", + "solved": true, + "cost": 0.3029935, + "calls": 17, + "seconds": 129.98464274406433, + "exit_status": "Submitted" + }, + { + "task": "pytest-dev__pytest-7521", + "solved": true, + "cost": 0.09132525000000001, + "calls": 7, + "seconds": 47.895599126815796, + "exit_status": "Submitted" + }, + { + "task": "scikit-learn__scikit-learn-26323", + "solved": true, + "cost": 0.16044425, + "calls": 8, + "seconds": 57.09654712677002, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-22080", + "solved": false, + "cost": 1.3774870000000001, + "calls": 32, + "seconds": 639.588751077652, + "exit_status": "Submitted" + }, + { + "task": "django__django-12858", + "solved": true, + "cost": 0.09448024999999999, + "calls": 7, + "seconds": 30.875277042388916, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-9320", + "solved": true, + "cost": 0.13017725, + "calls": 6, + "seconds": 28.189494132995605, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-6528", + "solved": true, + "cost": 0.5672952499999999, + "calls": 26, + "seconds": 317.3973047733307, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-13236", + "solved": true, + "cost": 0.53514325, + "calls": 22, + "seconds": 359.5103678703308, + "exit_status": "Submitted" + } + ], + "gpt-5.6-luna": [ + { + "task": "pallets__flask-5014", + "solved": true, + "cost": 0.006036549999999999, + "calls": 9, + "seconds": 33.020166873931885, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-16792", + "solved": true, + "cost": 0.026326870000000002, + "calls": 20, + "seconds": 106.58319997787476, + "exit_status": "Submitted" + }, + { + "task": "django__django-13964", + "solved": true, + "cost": 0.01957996, + "calls": 18, + "seconds": 100.13061499595642, + "exit_status": "Submitted" + }, + { + "task": "mwaskom__seaborn-3069", + "solved": true, + "cost": 0.04791672, + "calls": 26, + "seconds": 153.3259298801422, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-8621", + "solved": true, + "cost": 0.013940949999999995, + "calls": 14, + "seconds": 59.73096799850464, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-6903", + "solved": true, + "cost": 0.00832655, + "calls": 9, + "seconds": 62.57857298851013, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-14369", + "solved": true, + "cost": 0.024806449999999997, + "calls": 16, + "seconds": 135.53946113586426, + "exit_status": "Submitted" + }, + { + "task": "psf__requests-1921", + "solved": true, + "cost": 0.00863327, + "calls": 9, + "seconds": 42.352205753326416, + "exit_status": "Submitted" + }, + { + "task": "matplotlib__matplotlib-20676", + "solved": false, + "cost": 0.0314039, + "calls": 18, + "seconds": 196.67904686927795, + "exit_status": "Submitted" + }, + { + "task": "pytest-dev__pytest-6202", + "solved": true, + "cost": 0.015579949999999997, + "calls": 14, + "seconds": 98.11653208732605, + "exit_status": "Submitted" + }, + { + "task": "scikit-learn__scikit-learn-25973", + "solved": true, + "cost": 0.01486182, + "calls": 13, + "seconds": 67.90426182746887, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-15345", + "solved": true, + "cost": 0.0107484, + "calls": 14, + "seconds": 80.64220380783081, + "exit_status": "Submitted" + }, + { + "task": "django__django-16560", + "solved": true, + "cost": 0.015834799999999996, + "calls": 11, + "seconds": 94.99836587905884, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-8035", + "solved": true, + "cost": 0.028651450000000002, + "calls": 22, + "seconds": 155.87903714179993, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-7277", + "solved": true, + "cost": 0.011006789999999999, + "calls": 12, + "seconds": 86.04285311698914, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-14995", + "solved": true, + "cost": 0.01503533, + "calls": 12, + "seconds": 79.58418989181519, + "exit_status": "Submitted" + }, + { + "task": "psf__requests-1724", + "solved": true, + "cost": 0.011925660000000001, + "calls": 12, + "seconds": 59.012890100479126, + "exit_status": "Submitted" + }, + { + "task": "matplotlib__matplotlib-20859", + "solved": true, + "cost": 0.00656957, + "calls": 10, + "seconds": 57.28176832199097, + "exit_status": "Submitted" + }, + { + "task": "pytest-dev__pytest-7521", + "solved": true, + "cost": 0.01015545, + "calls": 10, + "seconds": 47.45153212547302, + "exit_status": "Submitted" + }, + { + "task": "scikit-learn__scikit-learn-26323", + "solved": true, + "cost": 0.012308079999999999, + "calls": 11, + "seconds": 57.70905089378357, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-22080", + "solved": true, + "cost": 0.03821885999999999, + "calls": 24, + "seconds": 340.3521111011505, + "exit_status": "Submitted" + }, + { + "task": "django__django-12858", + "solved": true, + "cost": 0.019858380000000002, + "calls": 20, + "seconds": 215.69341325759888, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-9320", + "solved": true, + "cost": 0.013937859999999998, + "calls": 14, + "seconds": 82.3595199584961, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-6528", + "solved": false, + "cost": 0.022390809999999997, + "calls": 16, + "seconds": 192.10791659355164, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-13236", + "solved": true, + "cost": 0.00976385, + "calls": 11, + "seconds": 67.40134692192078, + "exit_status": "Submitted" + } + ], + "gpt-5.6-sol": [ + { + "task": "pallets__flask-5014", + "solved": true, + "cost": 0.1650396, + "calls": 10, + "seconds": 58.31565284729004, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-16792", + "solved": true, + "cost": 0.36961879999999997, + "calls": 15, + "seconds": 144.69103384017944, + "exit_status": "Submitted" + }, + { + "task": "django__django-13964", + "solved": true, + "cost": 0.3488006, + "calls": 15, + "seconds": 136.48035287857056, + "exit_status": "Submitted" + }, + { + "task": "mwaskom__seaborn-3069", + "solved": true, + "cost": 0.6197701999999999, + "calls": 19, + "seconds": 184.12896609306335, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-8621", + "solved": true, + "cost": 0.22146640000000004, + "calls": 14, + "seconds": 96.26454091072083, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-6903", + "solved": true, + "cost": 0.19231320000000002, + "calls": 10, + "seconds": 103.44035696983337, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-14369", + "solved": true, + "cost": 0.43219620000000003, + "calls": 13, + "seconds": 178.15541315078735, + "exit_status": "Submitted" + }, + { + "task": "psf__requests-1921", + "solved": true, + "cost": 0.23612840000000002, + "calls": 11, + "seconds": 97.04425311088562, + "exit_status": "Submitted" + }, + { + "task": "matplotlib__matplotlib-20676", + "solved": true, + "cost": 0.3209066, + "calls": 15, + "seconds": 148.63819217681885, + "exit_status": "Submitted" + }, + { + "task": "pytest-dev__pytest-6202", + "solved": true, + "cost": 0.21509500000000004, + "calls": 11, + "seconds": 136.20845484733582, + "exit_status": "Submitted" + }, + { + "task": "scikit-learn__scikit-learn-25973", + "solved": true, + "cost": 0.24569480000000005, + "calls": 12, + "seconds": 155.78092169761658, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-15345", + "solved": true, + "cost": 0.249051, + "calls": 13, + "seconds": 120.23954916000366, + "exit_status": "Submitted" + }, + { + "task": "django__django-16560", + "solved": true, + "cost": 0.46097499999999997, + "calls": 19, + "seconds": 187.19075989723206, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-8035", + "solved": true, + "cost": 0.47764600000000007, + "calls": 18, + "seconds": 170.83832812309265, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-7277", + "solved": true, + "cost": 0.15690300000000001, + "calls": 10, + "seconds": 80.81029510498047, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-14995", + "solved": true, + "cost": 0.242054, + "calls": 11, + "seconds": 66.18213605880737, + "exit_status": "Submitted" + }, + { + "task": "psf__requests-1724", + "solved": true, + "cost": 0.22139700000000004, + "calls": 12, + "seconds": 69.80437207221985, + "exit_status": "Submitted" + }, + { + "task": "matplotlib__matplotlib-20859", + "solved": true, + "cost": 0.210145, + "calls": 14, + "seconds": 138.85214495658875, + "exit_status": "Submitted" + }, + { + "task": "pytest-dev__pytest-7521", + "solved": true, + "cost": 0.21148460000000002, + "calls": 12, + "seconds": 79.6021478176117, + "exit_status": "Submitted" + }, + { + "task": "scikit-learn__scikit-learn-26323", + "solved": true, + "cost": 0.17821340000000002, + "calls": 10, + "seconds": 102.16119194030762, + "exit_status": "Submitted" + }, + { + "task": "sympy__sympy-22080", + "solved": true, + "cost": 0.4984838, + "calls": 18, + "seconds": 210.76050400733948, + "exit_status": "Submitted" + }, + { + "task": "django__django-12858", + "solved": true, + "cost": 0.3036818, + "calls": 14, + "seconds": 169.38917303085327, + "exit_status": "Submitted" + }, + { + "task": "sphinx-doc__sphinx-9320", + "solved": true, + "cost": 0.3727256, + "calls": 15, + "seconds": 131.0103030204773, + "exit_status": "Submitted" + }, + { + "task": "pylint-dev__pylint-6528", + "solved": true, + "cost": 0.5114310000000001, + "calls": 20, + "seconds": 184.7115581035614, + "exit_status": "Submitted" + }, + { + "task": "astropy__astropy-13236", + "solved": true, + "cost": 0.23543620000000004, + "calls": 13, + "seconds": 222.15648198127747, + "exit_status": "Submitted" + } + ] + }, + "calibration_diagnostics": [ + { + "pair": "sonnet_opus", + "policy": "baseline_cap", + "metrics": { + "efficient": { + "mean_prediction": 0.7728, + "observed_success": 0.96, + "brier": 0.07364000000000001 + } + } + }, + { + "pair": "sonnet_opus", + "policy": "baseline_v2", + "metrics": { + "efficient": { + "mean_prediction": 0.7604, + "observed_success": 0.96, + "brier": 0.079748 + }, + "capable": { + "mean_prediction": 0.8552000000000001, + "observed_success": 0.92, + "brier": 0.07612000000000001 + }, + "gap": { + "mean_prediction": 0.09480000000000001, + "observed_difference": -0.04, + "mse": 0.135972 + } + } + }, + { + "pair": "sonnet_opus", + "policy": "trained_cap_validation_regret_0", + "metrics": { + "efficient": { + "mean_prediction": 0.6747999999999998, + "observed_success": 0.96, + "brier": 0.124676 + } + } + }, + { + "pair": "sonnet_opus", + "policy": "trained_cap_validation_regret_0.02", + "metrics": { + "efficient": { + "mean_prediction": 0.8051999999999999, + "observed_success": 0.96, + "brier": 0.06624400000000001 + } + } + }, + { + "pair": "sonnet_opus", + "policy": "trained_cap_validation_regret_0.05", + "metrics": { + "efficient": { + "mean_prediction": 0.8051999999999999, + "observed_success": 0.96, + "brier": 0.06624400000000001 + } + } + }, + { + "pair": "sonnet_opus", + "policy": "trained_v2_validation_regret_0", + "metrics": { + "efficient": { + "mean_prediction": 0.43411072591813205, + "observed_success": 0.96, + "brier": 0.31665228081145946 + }, + "capable": { + "mean_prediction": 0.8041056413999473, + "observed_success": 0.92, + "brier": 0.09095517149287767 + }, + "gap": { + "mean_prediction": 0.3699949154818153, + "observed_difference": -0.04, + "mse": 0.2890826011480608 + } + } + }, + { + "pair": "sonnet_opus", + "policy": "trained_v2_validation_regret_0.02", + "metrics": { + "efficient": { + "mean_prediction": 0.43411072591813205, + "observed_success": 0.96, + "brier": 0.31665228081145946 + }, + "capable": { + "mean_prediction": 0.8041056413999473, + "observed_success": 0.92, + "brier": 0.09095517149287767 + }, + "gap": { + "mean_prediction": 0.3699949154818153, + "observed_difference": -0.04, + "mse": 0.2890826011480608 + } + } + }, + { + "pair": "sonnet_opus", + "policy": "trained_v2_validation_regret_0.05", + "metrics": { + "efficient": { + "mean_prediction": 0.43411072591813205, + "observed_success": 0.96, + "brier": 0.31665228081145946 + }, + "capable": { + "mean_prediction": 0.8041056413999473, + "observed_success": 0.92, + "brier": 0.09095517149287767 + }, + "gap": { + "mean_prediction": 0.3699949154818153, + "observed_difference": -0.04, + "mse": 0.2890826011480608 + } + } + }, + { + "pair": "luna_sol", + "policy": "baseline_cap", + "metrics": { + "efficient": { + "mean_prediction": 0.7759999999999999, + "observed_success": 0.92, + "brier": 0.095296 + } + } + }, + { + "pair": "luna_sol", + "policy": "baseline_v2", + "metrics": { + "efficient": { + "mean_prediction": 0.7480000000000001, + "observed_success": 0.92, + "brier": 0.11399200000000001 + }, + "capable": { + "mean_prediction": 0.8144, + "observed_success": 1.0, + "brier": 0.03844 + }, + "gap": { + "mean_prediction": 0.06639999999999999, + "observed_difference": 0.08, + "mse": 0.07523200000000001 + } + } + }, + { + "pair": "luna_sol", + "policy": "trained_cap_validation_regret_0", + "metrics": { + "efficient": { + "mean_prediction": 0.46335458127953316, + "observed_success": 0.92, + "brier": 0.2820900941429093 + } + } + }, + { + "pair": "luna_sol", + "policy": "trained_cap_validation_regret_0.02", + "metrics": { + "efficient": { + "mean_prediction": 0.4601118755343589, + "observed_success": 0.92, + "brier": 0.28972178243797075 + } + } + }, + { + "pair": "luna_sol", + "policy": "trained_cap_validation_regret_0.05", + "metrics": { + "efficient": { + "mean_prediction": 0.47150335677677147, + "observed_success": 0.92, + "brier": 0.2758875769644403 + } + } + }, + { + "pair": "luna_sol", + "policy": "trained_v2_validation_regret_0", + "metrics": { + "efficient": { + "mean_prediction": 0.7480000000000001, + "observed_success": 0.92, + "brier": 0.11399200000000001 + }, + "capable": { + "mean_prediction": 0.8144, + "observed_success": 1.0, + "brier": 0.03844 + }, + "gap": { + "mean_prediction": 0.06639999999999999, + "observed_difference": 0.08, + "mse": 0.07523200000000001 + } + } + }, + { + "pair": "luna_sol", + "policy": "trained_v2_validation_regret_0.02", + "metrics": { + "efficient": { + "mean_prediction": 0.7167999999999999, + "observed_success": 0.92, + "brier": 0.114424 + }, + "capable": { + "mean_prediction": 0.79, + "observed_success": 1.0, + "brier": 0.047700000000000006 + }, + "gap": { + "mean_prediction": 0.07320000000000002, + "observed_difference": 0.08, + "mse": 0.072164 + } + } + }, + { + "pair": "luna_sol", + "policy": "trained_v2_validation_regret_0.05", + "metrics": { + "efficient": { + "mean_prediction": 0.4518075740486838, + "observed_success": 0.92, + "brier": 0.29280415326290876 + }, + "capable": { + "mean_prediction": 0.7135364774862528, + "observed_success": 1.0, + "brier": 0.08206787123595165 + }, + "gap": { + "mean_prediction": 0.261728903437569, + "observed_difference": 0.08, + "mse": 0.10650573917319446 + } + } + } + ] +} diff --git a/cookbook/auto_router_roi_training/fixed_boundary_profiles.json b/cookbook/auto_router_roi_training/fixed_boundary_profiles.json new file mode 100644 index 00000000000..be94454a036 --- /dev/null +++ b/cookbook/auto_router_roi_training/fixed_boundary_profiles.json @@ -0,0 +1,1958 @@ +{ + "sonnet_opus_original_none_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "sonnet_opus", + "definition_id": "v2_sonnet_opus_original", + "variant": "original", + "adjustment": "none", + "alpha": 0, + "key": "v2_sonnet_opus_original_none_0", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + }, + "capable": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "claude-sonnet-5" + ], + "REASONING": [ + "claude-opus-5" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "General coding model: claude-sonnet-5, high reasoning effort.", + "capable_profile": "General coding model: claude-opus-5, high reasoning effort.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": null + } + } + }, + "sonnet_opus_original_per_model_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "sonnet_opus", + "definition_id": "v2_sonnet_opus_original", + "variant": "original", + "adjustment": "per_model", + "alpha": 10, + "key": "v2_sonnet_opus_original_per_model_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + -0.15106471238105254, + 0.024046997094613942 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + 1.103973729478556, + 1e-06 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "claude-sonnet-5" + ], + "REASONING": [ + "claude-opus-5" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "General coding model: claude-sonnet-5, high reasoning effort.", + "capable_profile": "General coding model: claude-opus-5, high reasoning effort.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_sonnet_opus_original_per_model_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.15106471238105254, + "slope": 0.024046997094613942, + "offsets": [] + }, + "capable": { + "intercept": 1.103973729478556, + "slope": 1e-06, + "offsets": [] + } + } + } + } + }, + "sonnet_opus_original_task_conditioned_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "sonnet_opus", + "definition_id": "v2_sonnet_opus_original", + "variant": "original", + "adjustment": "task_conditioned", + "alpha": 10, + "key": "v2_sonnet_opus_original_task_conditioned_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial", + "verification:relevant" + ], + "coefficients": [ + -0.19870733699698753, + 0.1154153042726698, + -0.022310526826368113, + 0.022311725781833932, + 0.10587564721920996, + -0.10587444826374405, + 0.01949014056426, + -0.019488941608794257, + 0.04066688704084502, + -0.04066568808537909 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial", + "verification:relevant" + ], + "coefficients": [ + 1.0441019309853248, + 1e-06, + -0.022086256476281534, + 0.02210418618535574, + 0.1123412719736076, + -0.11235689519315654, + -0.04468483306247375, + 0.044697798307619004, + -0.002753391346423374, + 0.0027430749676260404 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "claude-sonnet-5" + ], + "REASONING": [ + "claude-opus-5" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "General coding model: claude-sonnet-5, high reasoning effort.", + "capable_profile": "General coding model: claude-opus-5, high reasoning effort.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_sonnet_opus_original_task_conditioned_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.19870733699698753, + "slope": 0.1154153042726698, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": -0.022310526826368113 + }, + { + "feature": "reasoning:open_ended", + "intercept": 0.022311725781833932 + }, + { + "feature": "scope:broad", + "intercept": 0.10587564721920996 + }, + { + "feature": "scope:coupled", + "intercept": -0.10587444826374405 + }, + { + "feature": "specification:ambiguous", + "intercept": 0.01949014056426 + }, + { + "feature": "specification:clear", + "intercept": -0.019488941608794257 + }, + { + "feature": "verification:partial", + "intercept": 0.04066688704084502 + }, + { + "feature": "verification:relevant", + "intercept": -0.04066568808537909 + } + ] + }, + "capable": { + "intercept": 1.0441019309853248, + "slope": 1e-06, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": -0.022086256476281534 + }, + { + "feature": "reasoning:open_ended", + "intercept": 0.02210418618535574 + }, + { + "feature": "scope:broad", + "intercept": 0.1123412719736076 + }, + { + "feature": "scope:coupled", + "intercept": -0.11235689519315654 + }, + { + "feature": "specification:ambiguous", + "intercept": -0.04468483306247375 + }, + { + "feature": "specification:clear", + "intercept": 0.044697798307619004 + }, + { + "feature": "verification:partial", + "intercept": -0.002753391346423374 + }, + { + "feature": "verification:relevant", + "intercept": 0.0027430749676260404 + } + ] + } + } + } + } + }, + "sonnet_opus_research_none_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "sonnet_opus", + "definition_id": "v2_sonnet_opus_research", + "variant": "research", + "adjustment": "none", + "alpha": 0, + "key": "v2_sonnet_opus_research_none_0", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + }, + "capable": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "claude-sonnet-5" + ], + "REASONING": [ + "claude-opus-5" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "Claude Sonnet 5 at high effort is a general coding solver. Evaluate its fit to the actual mechanism and scope. Clear instructions and runnable tests help, but do not establish complete coverage or guarantee success. Large repository size alone is not difficult scope. Public research does not establish a universal Sonnet advantage for a task family.", + "capable_profile": "Claude Opus 5 at high effort is a general coding solver, with research suggesting strengths in ambiguous requirements, coordinating coupled changes and migrations. Treat those as hypotheses, not guaranteed wins. It can still fail bounded tasks and share the same inaccessible-information or environment limits as the efficient solver.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": null + } + } + }, + "sonnet_opus_research_per_model_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "sonnet_opus", + "definition_id": "v2_sonnet_opus_research", + "variant": "research", + "adjustment": "per_model", + "alpha": 10, + "key": "v2_sonnet_opus_research_per_model_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + -0.15694788395290932, + 1e-06 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + 1.1039736429304146, + 1e-06 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "claude-sonnet-5" + ], + "REASONING": [ + "claude-opus-5" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "Claude Sonnet 5 at high effort is a general coding solver. Evaluate its fit to the actual mechanism and scope. Clear instructions and runnable tests help, but do not establish complete coverage or guarantee success. Large repository size alone is not difficult scope. Public research does not establish a universal Sonnet advantage for a task family.", + "capable_profile": "Claude Opus 5 at high effort is a general coding solver, with research suggesting strengths in ambiguous requirements, coordinating coupled changes and migrations. Treat those as hypotheses, not guaranteed wins. It can still fail bounded tasks and share the same inaccessible-information or environment limits as the efficient solver.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_sonnet_opus_research_per_model_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.15694788395290932, + "slope": 1e-06, + "offsets": [] + }, + "capable": { + "intercept": 1.1039736429304146, + "slope": 1e-06, + "offsets": [] + } + } + } + } + }, + "sonnet_opus_research_task_conditioned_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "sonnet_opus", + "definition_id": "v2_sonnet_opus_research", + "variant": "research", + "adjustment": "task_conditioned", + "alpha": 10, + "key": "v2_sonnet_opus_research_task_conditioned_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial" + ], + "coefficients": [ + -0.061388029436848145, + 1e-06, + -0.0817937774144661, + 0.0817919988869514, + -0.01157242844296772, + 0.011572843879506899, + 0.11953063257375798, + -0.11953178893236181, + 1.0703511821894975e-06 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial" + ], + "coefficients": [ + 1.1761243470757616, + 1e-06, + -0.05006513343308526, + 0.050065733767740594, + 0.1666478691645122, + -0.16664827409511887, + -0.031655137226720656, + 0.03165618982123785, + -4.432577135708213e-07 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "claude-sonnet-5" + ], + "REASONING": [ + "claude-opus-5" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "Claude Sonnet 5 at high effort is a general coding solver. Evaluate its fit to the actual mechanism and scope. Clear instructions and runnable tests help, but do not establish complete coverage or guarantee success. Large repository size alone is not difficult scope. Public research does not establish a universal Sonnet advantage for a task family.", + "capable_profile": "Claude Opus 5 at high effort is a general coding solver, with research suggesting strengths in ambiguous requirements, coordinating coupled changes and migrations. Treat those as hypotheses, not guaranteed wins. It can still fail bounded tasks and share the same inaccessible-information or environment limits as the efficient solver.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_sonnet_opus_research_task_conditioned_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.061388029436848145, + "slope": 1e-06, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": -0.0817937774144661 + }, + { + "feature": "reasoning:open_ended", + "intercept": 0.0817919988869514 + }, + { + "feature": "scope:broad", + "intercept": -0.01157242844296772 + }, + { + "feature": "scope:coupled", + "intercept": 0.011572843879506899 + }, + { + "feature": "specification:ambiguous", + "intercept": 0.11953063257375798 + }, + { + "feature": "specification:clear", + "intercept": -0.11953178893236181 + }, + { + "feature": "verification:partial", + "intercept": 1.0703511821894975e-06 + } + ] + }, + "capable": { + "intercept": 1.1761243470757616, + "slope": 1e-06, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": -0.05006513343308526 + }, + { + "feature": "reasoning:open_ended", + "intercept": 0.050065733767740594 + }, + { + "feature": "scope:broad", + "intercept": 0.1666478691645122 + }, + { + "feature": "scope:coupled", + "intercept": -0.16664827409511887 + }, + { + "feature": "specification:ambiguous", + "intercept": -0.031655137226720656 + }, + { + "feature": "specification:clear", + "intercept": 0.03165618982123785 + }, + { + "feature": "verification:partial", + "intercept": -4.432577135708213e-07 + } + ] + } + } + } + } + }, + "sonnet_opus_trained_card_none_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "sonnet_opus", + "definition_id": "v2_sonnet_opus_trained_card", + "variant": "trained_card", + "adjustment": "none", + "alpha": 0, + "key": "v2_sonnet_opus_trained_card_none_0", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + }, + "capable": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "claude-sonnet-5" + ], + "REASONING": [ + "claude-opus-5" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "Claude Sonnet 5 at high effort is a general coding solver. Evaluate its fit to the actual mechanism and scope. Clear instructions and runnable tests help, but do not establish complete coverage or guarantee success. Large repository size alone is not difficult scope. Public research does not establish a universal Sonnet advantage for a task family.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.461. go: 23 tasks, success estimate shrunk toward the overall mean 0.425. python: 27 tasks, success estimate shrunk toward the overall mean 0.477. rust: 5 tasks, success estimate shrunk toward the overall mean 0.437. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.483. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "capable_profile": "Claude Opus 5 at high effort is a general coding solver, with research suggesting strengths in ambiguous requirements, coordinating coupled changes and migrations. Treat those as hypotheses, not guaranteed wins. It can still fail bounded tasks and share the same inaccessible-information or environment limits as the efficient solver.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.751. go: 23 tasks, success estimate shrunk toward the overall mean 0.788. python: 27 tasks, success estimate shrunk toward the overall mean 0.736. rust: 5 tasks, success estimate shrunk toward the overall mean 0.731. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.748. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": null + } + } + }, + "sonnet_opus_trained_card_per_model_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "sonnet_opus", + "definition_id": "v2_sonnet_opus_trained_card", + "variant": "trained_card", + "adjustment": "per_model", + "alpha": 10, + "key": "v2_sonnet_opus_trained_card_per_model_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + -0.15694745947746486, + 1e-06 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + 1.1039734479091954, + 1e-06 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "claude-sonnet-5" + ], + "REASONING": [ + "claude-opus-5" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "Claude Sonnet 5 at high effort is a general coding solver. Evaluate its fit to the actual mechanism and scope. Clear instructions and runnable tests help, but do not establish complete coverage or guarantee success. Large repository size alone is not difficult scope. Public research does not establish a universal Sonnet advantage for a task family.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.461. go: 23 tasks, success estimate shrunk toward the overall mean 0.425. python: 27 tasks, success estimate shrunk toward the overall mean 0.477. rust: 5 tasks, success estimate shrunk toward the overall mean 0.437. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.483. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "capable_profile": "Claude Opus 5 at high effort is a general coding solver, with research suggesting strengths in ambiguous requirements, coordinating coupled changes and migrations. Treat those as hypotheses, not guaranteed wins. It can still fail bounded tasks and share the same inaccessible-information or environment limits as the efficient solver.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.751. go: 23 tasks, success estimate shrunk toward the overall mean 0.788. python: 27 tasks, success estimate shrunk toward the overall mean 0.736. rust: 5 tasks, success estimate shrunk toward the overall mean 0.731. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.748. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_sonnet_opus_trained_card_per_model_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.15694745947746486, + "slope": 1e-06, + "offsets": [] + }, + "capable": { + "intercept": 1.1039734479091954, + "slope": 1e-06, + "offsets": [] + } + } + } + } + }, + "sonnet_opus_trained_card_task_conditioned_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "sonnet_opus", + "definition_id": "v2_sonnet_opus_trained_card", + "variant": "trained_card", + "adjustment": "task_conditioned", + "alpha": 10, + "key": "v2_sonnet_opus_trained_card_task_conditioned_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial" + ], + "coefficients": [ + -0.10777549226033059, + 1e-06, + -0.027765989289143762, + 0.02775681699433656, + 0.08614759664525455, + -0.08613347375985544, + 0.04852049775771622, + -0.04852967005252337, + 4.087873744178555e-07 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial" + ], + "coefficients": [ + 0.9939813315016867, + 0.16547448827283578, + 0.018908118952849138, + -0.018894722719430025, + 0.18542223003930738, + -0.18540897609036602, + 0.04310276547766364, + -0.043089369244244356, + 1.3358604466297135e-05 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "claude-sonnet-5" + ], + "REASONING": [ + "claude-opus-5" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "Claude Sonnet 5 at high effort is a general coding solver. Evaluate its fit to the actual mechanism and scope. Clear instructions and runnable tests help, but do not establish complete coverage or guarantee success. Large repository size alone is not difficult scope. Public research does not establish a universal Sonnet advantage for a task family.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.461. go: 23 tasks, success estimate shrunk toward the overall mean 0.425. python: 27 tasks, success estimate shrunk toward the overall mean 0.477. rust: 5 tasks, success estimate shrunk toward the overall mean 0.437. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.483. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "capable_profile": "Claude Opus 5 at high effort is a general coding solver, with research suggesting strengths in ambiguous requirements, coordinating coupled changes and migrations. Treat those as hypotheses, not guaranteed wins. It can still fail bounded tasks and share the same inaccessible-information or environment limits as the efficient solver.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.751. go: 23 tasks, success estimate shrunk toward the overall mean 0.788. python: 27 tasks, success estimate shrunk toward the overall mean 0.736. rust: 5 tasks, success estimate shrunk toward the overall mean 0.731. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.748. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_sonnet_opus_trained_card_task_conditioned_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.10777549226033059, + "slope": 1e-06, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": -0.027765989289143762 + }, + { + "feature": "reasoning:open_ended", + "intercept": 0.02775681699433656 + }, + { + "feature": "scope:broad", + "intercept": 0.08614759664525455 + }, + { + "feature": "scope:coupled", + "intercept": -0.08613347375985544 + }, + { + "feature": "specification:ambiguous", + "intercept": 0.04852049775771622 + }, + { + "feature": "specification:clear", + "intercept": -0.04852967005252337 + }, + { + "feature": "verification:partial", + "intercept": 4.087873744178555e-07 + } + ] + }, + "capable": { + "intercept": 0.9939813315016867, + "slope": 0.16547448827283578, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": 0.018908118952849138 + }, + { + "feature": "reasoning:open_ended", + "intercept": -0.018894722719430025 + }, + { + "feature": "scope:broad", + "intercept": 0.18542223003930738 + }, + { + "feature": "scope:coupled", + "intercept": -0.18540897609036602 + }, + { + "feature": "specification:ambiguous", + "intercept": 0.04310276547766364 + }, + { + "feature": "specification:clear", + "intercept": -0.043089369244244356 + }, + { + "feature": "verification:partial", + "intercept": 1.3358604466297135e-05 + } + ] + } + } + } + } + }, + "luna_sol_original_none_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "luna_sol", + "definition_id": "v2_luna_sol_original", + "variant": "original", + "adjustment": "none", + "alpha": 0, + "key": "v2_luna_sol_original_none_0", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + }, + "capable": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "gpt-5.6-luna" + ], + "REASONING": [ + "gpt-5.6-sol" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "General coding model: gpt-5.6-luna, high reasoning effort.", + "capable_profile": "General coding model: gpt-5.6-sol, high reasoning effort.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": null + } + } + }, + "luna_sol_original_per_model_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "luna_sol", + "definition_id": "v2_luna_sol_original", + "variant": "original", + "adjustment": "per_model", + "alpha": 10, + "key": "v2_luna_sol_original_per_model_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + -0.19337116823324155, + 1e-06 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + 0.8675435678375288, + 0.026134712667179476 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "gpt-5.6-luna" + ], + "REASONING": [ + "gpt-5.6-sol" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "General coding model: gpt-5.6-luna, high reasoning effort.", + "capable_profile": "General coding model: gpt-5.6-sol, high reasoning effort.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_luna_sol_original_per_model_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.19337116823324155, + "slope": 1e-06, + "offsets": [] + }, + "capable": { + "intercept": 0.8675435678375288, + "slope": 0.026134712667179476, + "offsets": [] + } + } + } + } + }, + "luna_sol_original_task_conditioned_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "luna_sol", + "definition_id": "v2_luna_sol_original", + "variant": "original", + "adjustment": "task_conditioned", + "alpha": 10, + "key": "v2_luna_sol_original_task_conditioned_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial", + "verification:relevant" + ], + "coefficients": [ + -0.2530165488190076, + 1e-06, + 0.041465032046650446, + -0.041472907982567565, + 0.043538136984809704, + -0.043546012920726865, + 0.03388315910954777, + -0.0338910350454649, + 0.03400286969808646, + -0.034010745634003636 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial", + "verification:relevant" + ], + "coefficients": [ + 0.8913823204417615, + 0.18058612579460084, + -0.1059187519513455, + 0.10591901379724131, + 0.07580225637774672, + -0.07580312331250551, + 0.11341421856829113, + -0.11341395672239529, + 0.035612364739125296, + -0.035613176707174045 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "gpt-5.6-luna" + ], + "REASONING": [ + "gpt-5.6-sol" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "General coding model: gpt-5.6-luna, high reasoning effort.", + "capable_profile": "General coding model: gpt-5.6-sol, high reasoning effort.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_luna_sol_original_task_conditioned_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.2530165488190076, + "slope": 1e-06, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": 0.041465032046650446 + }, + { + "feature": "reasoning:open_ended", + "intercept": -0.041472907982567565 + }, + { + "feature": "scope:broad", + "intercept": 0.043538136984809704 + }, + { + "feature": "scope:coupled", + "intercept": -0.043546012920726865 + }, + { + "feature": "specification:ambiguous", + "intercept": 0.03388315910954777 + }, + { + "feature": "specification:clear", + "intercept": -0.0338910350454649 + }, + { + "feature": "verification:partial", + "intercept": 0.03400286969808646 + }, + { + "feature": "verification:relevant", + "intercept": -0.034010745634003636 + } + ] + }, + "capable": { + "intercept": 0.8913823204417615, + "slope": 0.18058612579460084, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": -0.1059187519513455 + }, + { + "feature": "reasoning:open_ended", + "intercept": 0.10591901379724131 + }, + { + "feature": "scope:broad", + "intercept": 0.07580225637774672 + }, + { + "feature": "scope:coupled", + "intercept": -0.07580312331250551 + }, + { + "feature": "specification:ambiguous", + "intercept": 0.11341421856829113 + }, + { + "feature": "specification:clear", + "intercept": -0.11341395672239529 + }, + { + "feature": "verification:partial", + "intercept": 0.035612364739125296 + }, + { + "feature": "verification:relevant", + "intercept": -0.035613176707174045 + } + ] + } + } + } + } + }, + "luna_sol_research_none_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "luna_sol", + "definition_id": "v2_luna_sol_research", + "variant": "research", + "adjustment": "none", + "alpha": 0, + "key": "v2_luna_sol_research_none_0", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + }, + "capable": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "gpt-5.6-luna" + ], + "REASONING": [ + "gpt-5.6-sol" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "GPT-5.6 Luna at high effort is a low-cost general coding solver. Public research suggests useful document and scientific subproblem capability, but results at max effort do not establish high-effort task success. Check whether the core algorithm and integration demands fit; executable feedback does not remove missing reasoning.", + "capable_profile": "GPT-5.6 Sol at high effort is a general coding solver. Research suggests useful implementation and security repair ability in particular harnesses. Evidence at other efforts is not a guaranteed advantage here. It can share the efficient solver's ambiguity, environment and hidden-behavior failures.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": null + } + } + }, + "luna_sol_research_per_model_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "luna_sol", + "definition_id": "v2_luna_sol_research", + "variant": "research", + "adjustment": "per_model", + "alpha": 10, + "key": "v2_luna_sol_research_per_model_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + -0.16292866356447414, + 0.08008403448757105 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + 0.8751841361311286, + 1e-06 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "gpt-5.6-luna" + ], + "REASONING": [ + "gpt-5.6-sol" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "GPT-5.6 Luna at high effort is a low-cost general coding solver. Public research suggests useful document and scientific subproblem capability, but results at max effort do not establish high-effort task success. Check whether the core algorithm and integration demands fit; executable feedback does not remove missing reasoning.", + "capable_profile": "GPT-5.6 Sol at high effort is a general coding solver. Research suggests useful implementation and security repair ability in particular harnesses. Evidence at other efforts is not a guaranteed advantage here. It can share the efficient solver's ambiguity, environment and hidden-behavior failures.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_luna_sol_research_per_model_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.16292866356447414, + "slope": 0.08008403448757105, + "offsets": [] + }, + "capable": { + "intercept": 0.8751841361311286, + "slope": 1e-06, + "offsets": [] + } + } + } + } + }, + "luna_sol_research_task_conditioned_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "luna_sol", + "definition_id": "v2_luna_sol_research", + "variant": "research", + "adjustment": "task_conditioned", + "alpha": 10, + "key": "v2_luna_sol_research_task_conditioned_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial", + "verification:relevant" + ], + "coefficients": [ + -0.22998308641102425, + 0.10974128448800057, + 0.03386085422119013, + -0.03385291557105156, + 0.095768412892066, + -0.0957460839861836, + -0.07727010463496342, + 0.07727804328510196, + -0.032835396562853184, + 0.03284333521299173 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial", + "verification:relevant" + ], + "coefficients": [ + 0.8937404190361302, + 1e-06, + -0.11224557487124642, + 0.11227311912819139, + 0.12206991990089967, + -0.12208273826072996, + 0.011803984873682734, + -0.011779549259434646, + 0.02992395092570415, + -0.02993572478158832 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "gpt-5.6-luna" + ], + "REASONING": [ + "gpt-5.6-sol" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "GPT-5.6 Luna at high effort is a low-cost general coding solver. Public research suggests useful document and scientific subproblem capability, but results at max effort do not establish high-effort task success. Check whether the core algorithm and integration demands fit; executable feedback does not remove missing reasoning.", + "capable_profile": "GPT-5.6 Sol at high effort is a general coding solver. Research suggests useful implementation and security repair ability in particular harnesses. Evidence at other efforts is not a guaranteed advantage here. It can share the efficient solver's ambiguity, environment and hidden-behavior failures.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_luna_sol_research_task_conditioned_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.22998308641102425, + "slope": 0.10974128448800057, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": 0.03386085422119013 + }, + { + "feature": "reasoning:open_ended", + "intercept": -0.03385291557105156 + }, + { + "feature": "scope:broad", + "intercept": 0.095768412892066 + }, + { + "feature": "scope:coupled", + "intercept": -0.0957460839861836 + }, + { + "feature": "specification:ambiguous", + "intercept": -0.07727010463496342 + }, + { + "feature": "specification:clear", + "intercept": 0.07727804328510196 + }, + { + "feature": "verification:partial", + "intercept": -0.032835396562853184 + }, + { + "feature": "verification:relevant", + "intercept": 0.03284333521299173 + } + ] + }, + "capable": { + "intercept": 0.8937404190361302, + "slope": 1e-06, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": -0.11224557487124642 + }, + { + "feature": "reasoning:open_ended", + "intercept": 0.11227311912819139 + }, + { + "feature": "scope:broad", + "intercept": 0.12206991990089967 + }, + { + "feature": "scope:coupled", + "intercept": -0.12208273826072996 + }, + { + "feature": "specification:ambiguous", + "intercept": 0.011803984873682734 + }, + { + "feature": "specification:clear", + "intercept": -0.011779549259434646 + }, + { + "feature": "verification:partial", + "intercept": 0.02992395092570415 + }, + { + "feature": "verification:relevant", + "intercept": -0.02993572478158832 + } + ] + } + } + } + } + }, + "luna_sol_trained_card_none_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "luna_sol", + "definition_id": "v2_luna_sol_trained_card", + "variant": "trained_card", + "adjustment": "none", + "alpha": 0, + "key": "v2_luna_sol_trained_card_none_0", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + }, + "capable": { + "adjustment": "none", + "categories": [], + "coefficients": [ + 0.0, + 1.0 + ], + "alpha": 0.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "gpt-5.6-luna" + ], + "REASONING": [ + "gpt-5.6-sol" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "GPT-5.6 Luna at high effort is a low-cost general coding solver. Public research suggests useful document and scientific subproblem capability, but results at max effort do not establish high-effort task success. Check whether the core algorithm and integration demands fit; executable feedback does not remove missing reasoning.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.452. go: 23 tasks, success estimate shrunk toward the overall mean 0.479. python: 27 tasks, success estimate shrunk toward the overall mean 0.453. rust: 5 tasks, success estimate shrunk toward the overall mean 0.355. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.473. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "capable_profile": "GPT-5.6 Sol at high effort is a general coding solver. Research suggests useful implementation and security repair ability in particular harnesses. Evidence at other efforts is not a guaranteed advantage here. It can share the efficient solver's ambiguity, environment and hidden-behavior failures.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.706. go: 23 tasks, success estimate shrunk toward the overall mean 0.747. python: 27 tasks, success estimate shrunk toward the overall mean 0.699. rust: 5 tasks, success estimate shrunk toward the overall mean 0.627. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.686. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": null + } + } + }, + "luna_sol_trained_card_per_model_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "luna_sol", + "definition_id": "v2_luna_sol_trained_card", + "variant": "trained_card", + "adjustment": "per_model", + "alpha": 10, + "key": "v2_luna_sol_trained_card_per_model_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + -0.11883110924268137, + 0.11503598755943051 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "per_model", + "categories": [], + "coefficients": [ + 0.8751838170104753, + 1e-06 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "gpt-5.6-luna" + ], + "REASONING": [ + "gpt-5.6-sol" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "GPT-5.6 Luna at high effort is a low-cost general coding solver. Public research suggests useful document and scientific subproblem capability, but results at max effort do not establish high-effort task success. Check whether the core algorithm and integration demands fit; executable feedback does not remove missing reasoning.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.452. go: 23 tasks, success estimate shrunk toward the overall mean 0.479. python: 27 tasks, success estimate shrunk toward the overall mean 0.453. rust: 5 tasks, success estimate shrunk toward the overall mean 0.355. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.473. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "capable_profile": "GPT-5.6 Sol at high effort is a general coding solver. Research suggests useful implementation and security repair ability in particular harnesses. Evidence at other efforts is not a guaranteed advantage here. It can share the efficient solver's ambiguity, environment and hidden-behavior failures.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.706. go: 23 tasks, success estimate shrunk toward the overall mean 0.747. python: 27 tasks, success estimate shrunk toward the overall mean 0.699. rust: 5 tasks, success estimate shrunk toward the overall mean 0.627. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.686. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_luna_sol_trained_card_per_model_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.11883110924268137, + "slope": 0.11503598755943051, + "offsets": [] + }, + "capable": { + "intercept": 0.8751838170104753, + "slope": 1e-06, + "offsets": [] + } + } + } + } + }, + "luna_sol_trained_card_task_conditioned_default_boundary": { + "selection": { + "classifier": "v2", + "pair": "luna_sol", + "definition_id": "v2_luna_sol_trained_card", + "variant": "trained_card", + "adjustment": "task_conditioned", + "alpha": 10, + "key": "v2_luna_sol_trained_card_task_conditioned_10", + "boundary": { + "max_quality_gap": 0.05 + }, + "fitted": { + "efficient": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial", + "verification:relevant" + ], + "coefficients": [ + -0.1263631936070021, + 0.07617904485653157, + -0.024977926668820625, + 0.024967982876796725, + -0.038702067776608215, + 0.03869982221277211, + -0.13026844674084445, + 0.1302741181048604, + -0.07490299356680703, + 0.07490219142075599 + ], + "alpha": 10.0 + }, + "capable": { + "adjustment": "task_conditioned", + "categories": [ + "reasoning:multistep", + "reasoning:open_ended", + "scope:broad", + "scope:coupled", + "specification:ambiguous", + "specification:clear", + "verification:partial", + "verification:relevant" + ], + "coefficients": [ + 0.9060021529493855, + 1e-06, + -0.06234152111590296, + 0.06228905300426797, + 0.029236942967245612, + -0.02920388542690915, + -0.06274228598717928, + 0.06275101971823942, + -0.05915115169852544, + 0.05916033379473363 + ], + "alpha": 10.0 + } + } + }, + "complexity_router_config": { + "classifier_type": "llm_v2", + "classifier_llm_config": { + "model": "router-judge", + "timeout_ms": 30000, + "reasoning_effort": "low" + }, + "tiers": { + "SIMPLE": [ + "gpt-5.6-luna" + ], + "REASONING": [ + "gpt-5.6-sol" + ] + }, + "adaptive": false, + "route_housekeeping_to_cheapest_tier": false, + "escalation_keywords": [], + "plan_mode_min_tier": null, + "enable_context_window_escalation": false, + "return_raw_model_name": true, + "max_tokens_from_tier_model": false, + "llm_v2_config": { + "efficient_tier": "SIMPLE", + "capable_tier": "REASONING", + "efficient_profile": "GPT-5.6 Luna at high effort is a low-cost general coding solver. Public research suggests useful document and scientific subproblem capability, but results at max effort do not establish high-effort task success. Check whether the core algorithm and integration demands fit; executable feedback does not remove missing reasoning.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.452. go: 23 tasks, success estimate shrunk toward the overall mean 0.479. python: 27 tasks, success estimate shrunk toward the overall mean 0.453. rust: 5 tasks, success estimate shrunk toward the overall mean 0.355. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.473. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "capable_profile": "GPT-5.6 Sol at high effort is a general coding solver. Research suggests useful implementation and security repair ability in particular harnesses. Evidence at other efforts is not a guaranteed advantage here. It can share the efficient solver's ambiguity, environment and hidden-behavior failures.\nTraining evidence: 83 distinct DeepSWE tasks, repeated attempts averaged within task; mean success 0.706. go: 23 tasks, success estimate shrunk toward the overall mean 0.747. python: 27 tasks, success estimate shrunk toward the overall mean 0.699. rust: 5 tasks, success estimate shrunk toward the overall mean 0.627. typescript: 25 tasks, success estimate shrunk toward the overall mean 0.686. These are task-family priors from a different budget/configuration, not the probability for this request. Use visible demands to update them. Do not identify or recall a benchmark task or its published answer.", + "harness": "mini-SWE-agent 2.0.0, Linux shell, repository inspection and edits, existing tests and self-written reproductions; one fresh attempt, high native reasoning effort, 150 model calls and USD 5 inference limit. Final hidden tests are unavailable to the solver. No retry or inherited patch from another model.", + "max_quality_gap": 0.05, + "max_output_tokens": 2048, + "response_format": "json_schema", + "calibration": { + "version": "roi-20260914-v2_luna_sol_trained_card_task_conditioned_10", + "prompt_version": "llm-v2-1", + "efficient": { + "intercept": -0.1263631936070021, + "slope": 0.07617904485653157, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": -0.024977926668820625 + }, + { + "feature": "reasoning:open_ended", + "intercept": 0.024967982876796725 + }, + { + "feature": "scope:broad", + "intercept": -0.038702067776608215 + }, + { + "feature": "scope:coupled", + "intercept": 0.03869982221277211 + }, + { + "feature": "specification:ambiguous", + "intercept": -0.13026844674084445 + }, + { + "feature": "specification:clear", + "intercept": 0.1302741181048604 + }, + { + "feature": "verification:partial", + "intercept": -0.07490299356680703 + }, + { + "feature": "verification:relevant", + "intercept": 0.07490219142075599 + } + ] + }, + "capable": { + "intercept": 0.9060021529493855, + "slope": 1e-06, + "offsets": [ + { + "feature": "reasoning:multistep", + "intercept": -0.06234152111590296 + }, + { + "feature": "reasoning:open_ended", + "intercept": 0.06228905300426797 + }, + { + "feature": "scope:broad", + "intercept": 0.029236942967245612 + }, + { + "feature": "scope:coupled", + "intercept": -0.02920388542690915 + }, + { + "feature": "specification:ambiguous", + "intercept": -0.06274228598717928 + }, + { + "feature": "specification:clear", + "intercept": 0.06275101971823942 + }, + { + "feature": "verification:partial", + "intercept": -0.05915115169852544 + }, + { + "feature": "verification:relevant", + "intercept": 0.05916033379473363 + } + ] + } + } + } + } + } +} diff --git a/cookbook/auto_router_roi_training/manifest.json b/cookbook/auto_router_roi_training/manifest.json index 5e6ea6343ac..99fe504f8f7 100644 --- a/cookbook/auto_router_roi_training/manifest.json +++ b/cookbook/auto_router_roi_training/manifest.json @@ -25,7 +25,31 @@ "83 fitting tasks and 30 validation tasks support only exploratory estimates", "Published training outcomes use different task budgets and serving configurations from the live pilot", "Missing training cost records remain missing and are excluded as pairs only from cost metrics", - "Luna/Sol profiles have public-data validation only unless a corresponding fresh live evaluation is listed", - "Training retains the source included_in_score filter; excluded provider/infrastructure failures are not modeled as solver failures" - ] + "Training retains the source included_in_score filter; excluded provider/infrastructure failures are not modeled as solver failures", + "One attempt per solver per task; 25 tasks do not establish a small production quality-loss bound" + ], + "live_evaluation": { + "tasks": 25, + "solver_attempts": 100, + "judge_forecasts": 300, + "dataset": "SWE-bench Verified", + "dataset_revision": "c104f840cc67f8b6eec6f759ebc8b2693d585d4a", + "architecture": "native ARM64", + "harness": "mini-swe-agent==2.0.0", + "grader": "swebench==4.1.0", + "solver_models": [ + "anthropic/claude-sonnet-5", + "anthropic/claude-opus-5", + "openai/gpt-5.6-luna", + "openai/gpt-5.6-sol" + ], + "comparison": "Task-pinned paired replay of frozen policies, with recorded solver and judge cost", + "files": [ + "FINDINGS.md", + "BENCHMARK.md", + "benchmark_results.json", + "ABLATIONS.md", + "ablation_diagnostics.json" + ] + } }