From 002468316dbde1e602fa8c8eb4f7c24cd08e55bc Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 16 Mar 2026 11:23:23 -0400 Subject: [PATCH] Record Sonnet 4.6 baseline: 55.7% on SWE-Bench-Lite ($39.78) Co-Authored-By: Claude Opus 4.6 (1M context) --- evals/swe-bench/scoreboard/leaderboard.json | 13 + .../sonnet-baseline-20260316/README.md | 38 +++ .../sonnet-baseline-20260316/instances.jsonl | 300 ++++++++++++++++++ .../sonnet-baseline-20260316/meta.json | 94 ++++++ 4 files changed, 445 insertions(+) create mode 100644 evals/swe-bench/scoreboard/sonnet-baseline-20260316/README.md create mode 100644 evals/swe-bench/scoreboard/sonnet-baseline-20260316/instances.jsonl create mode 100644 evals/swe-bench/scoreboard/sonnet-baseline-20260316/meta.json diff --git a/evals/swe-bench/scoreboard/leaderboard.json b/evals/swe-bench/scoreboard/leaderboard.json index 83de0fb69..1b391c7cf 100644 --- a/evals/swe-bench/scoreboard/leaderboard.json +++ b/evals/swe-bench/scoreboard/leaderboard.json @@ -1,4 +1,17 @@ [ + { + "run_name": "sonnet-baseline-20260316", + "date": "2026-03-16", + "model": "claude-sonnet-4-6", + "provider": "anthropic", + "resolved": 167, + "total": 300, + "resolve_pct": 55.7, + "patched": 281, + "patch_pct": 93.7, + "total_cost_usd": 39.78, + "avg_cost_usd": 0.1326 + }, { "run_name": "haiku-baseline-20260316", "date": "2026-03-16", diff --git a/evals/swe-bench/scoreboard/sonnet-baseline-20260316/README.md b/evals/swe-bench/scoreboard/sonnet-baseline-20260316/README.md new file mode 100644 index 000000000..788afb90e --- /dev/null +++ b/evals/swe-bench/scoreboard/sonnet-baseline-20260316/README.md @@ -0,0 +1,38 @@ +# sonnet-baseline-20260316 + +**Date:** 2026-03-16 +**Model:** claude-sonnet-4-6 (anthropic) +**Fabro:** fabro 0.5.0 (4c6ad4d 2026-03-16) + +## Description + +Sonnet 4.6 baseline, default prompt, 2 CPU / 4 GB, 10min timeout + +## Results + +| Metric | Value | +|--------|-------| +| Instances | 300 | +| Patched | 281 (93.7%) | +| **Resolved** | **167 (55.7%)** | +| Total gen cost | $39.78 | +| Avg gen cost | $0.1326/instance | +| Gen wall time | 993.6s | +| Eval wall time | 642.3s | + +## Per-repo breakdown + +| Repo | Resolved | Total | Rate | +|------|----------|-------|------| +| astropy/astropy | 1 | 6 | 16.7% | +| django/django | 86 | 114 | 75.4% | +| matplotlib/matplotlib | 0 | 23 | 0.0% | +| mwaskom/seaborn | 2 | 4 | 50.0% | +| pallets/flask | 0 | 3 | 0.0% | +| psf/requests | 5 | 6 | 83.3% | +| pydata/xarray | 1 | 5 | 20.0% | +| pylint-dev/pylint | 0 | 6 | 0.0% | +| pytest-dev/pytest | 11 | 17 | 64.7% | +| scikit-learn/scikit-learn | 14 | 23 | 60.9% | +| sphinx-doc/sphinx | 0 | 16 | 0.0% | +| sympy/sympy | 47 | 77 | 61.0% | diff --git a/evals/swe-bench/scoreboard/sonnet-baseline-20260316/instances.jsonl b/evals/swe-bench/scoreboard/sonnet-baseline-20260316/instances.jsonl new file mode 100644 index 000000000..917eba1b8 --- /dev/null +++ b/evals/swe-bench/scoreboard/sonnet-baseline-20260316/instances.jsonl @@ -0,0 +1,300 @@ +{"instance_id": "astropy__astropy-12907", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 112.5, "eval_duration_s": 78.5, "gen_cost_usd": 0.072702} +{"instance_id": "astropy__astropy-14182", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 196.7, "eval_duration_s": 59.6, "gen_cost_usd": 0.175827} +{"instance_id": "astropy__astropy-14365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 117.2, "eval_duration_s": 63.9, "gen_cost_usd": 0.091416} +{"instance_id": "astropy__astropy-14995", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 78.2, "eval_duration_s": 67.8, "gen_cost_usd": 0.03861} +{"instance_id": "astropy__astropy-6938", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 102.3, "eval_duration_s": 142.3, "gen_cost_usd": 0.013179} +{"instance_id": "astropy__astropy-7746", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 125.5, "eval_duration_s": 92.8, "gen_cost_usd": 0.044217} +{"instance_id": "django__django-10914", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 203.0, "eval_duration_s": 23.3, "gen_cost_usd": 0.136578} +{"instance_id": "django__django-10924", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.1, "eval_duration_s": 24.3, "gen_cost_usd": 0.094581} +{"instance_id": "django__django-11001", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 142.5, "eval_duration_s": 51.0, "gen_cost_usd": 0.105576} +{"instance_id": "django__django-11019", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 605.7, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "django__django-11039", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 85.5, "eval_duration_s": 34.8, "gen_cost_usd": 0.09552} +{"instance_id": "django__django-11049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 45.3, "eval_duration_s": 31.3, "gen_cost_usd": 0.052521} +{"instance_id": "django__django-11099", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 38.0, "eval_duration_s": 118.9, "gen_cost_usd": 0.014364} +{"instance_id": "django__django-11133", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 42.3, "eval_duration_s": 22.9, "gen_cost_usd": 0.021831} +{"instance_id": "django__django-11179", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.3, "eval_duration_s": 31.4, "gen_cost_usd": 0.015576} +{"instance_id": "django__django-11283", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 201.0, "eval_duration_s": 25.2, "gen_cost_usd": 0.234933} +{"instance_id": "django__django-11422", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.5, "eval_duration_s": 30.9, "gen_cost_usd": 0.09819} +{"instance_id": "django__django-11564", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 333.9, "eval_duration_s": 19.2, "gen_cost_usd": 0.413628} +{"instance_id": "django__django-11583", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.8, "eval_duration_s": 36.3, "gen_cost_usd": 0.021516} +{"instance_id": "django__django-11620", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 74.3, "eval_duration_s": 27.8, "gen_cost_usd": 0.07572} +{"instance_id": "django__django-11630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 193.0, "eval_duration_s": 24.0, "gen_cost_usd": 0.1971} +{"instance_id": "django__django-11742", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 236.1, "eval_duration_s": 30.8, "gen_cost_usd": 0.20424} +{"instance_id": "django__django-11797", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 481.1, "eval_duration_s": 20.1, "gen_cost_usd": 0.446673} +{"instance_id": "django__django-11815", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.8, "eval_duration_s": 35.9, "gen_cost_usd": 0.104976} +{"instance_id": "django__django-11848", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 84.5, "eval_duration_s": 37.4, "gen_cost_usd": 0.051981} +{"instance_id": "django__django-11905", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 93.1, "eval_duration_s": 36.4, "gen_cost_usd": 0.108054} +{"instance_id": "django__django-11910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 462.7, "eval_duration_s": 16.1, "gen_cost_usd": 0.55728} +{"instance_id": "django__django-11964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 195.6, "eval_duration_s": 20.4, "gen_cost_usd": 0.212775} +{"instance_id": "django__django-11999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 68.3, "eval_duration_s": 66.6, "gen_cost_usd": 0.045507} +{"instance_id": "django__django-12113", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 165.1, "eval_duration_s": 30.0, "gen_cost_usd": 0.20745} +{"instance_id": "django__django-12125", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 120.6, "eval_duration_s": 40.4, "gen_cost_usd": 0.127506} +{"instance_id": "django__django-12184", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 183.9, "eval_duration_s": 48.3, "gen_cost_usd": 0.177834} +{"instance_id": "django__django-12284", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 75.6, "eval_duration_s": 25.4, "gen_cost_usd": 0.053568} +{"instance_id": "django__django-12286", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 66.3, "eval_duration_s": 29.2, "gen_cost_usd": 0.045819} +{"instance_id": "django__django-12308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 133.7, "eval_duration_s": 74.7, "gen_cost_usd": 0.166785} +{"instance_id": "django__django-12453", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.2, "eval_duration_s": 25.5, "gen_cost_usd": 0.050874} +{"instance_id": "django__django-12470", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 513.6, "eval_duration_s": 18.5, "gen_cost_usd": 0.505338} +{"instance_id": "django__django-12497", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 90.6, "eval_duration_s": 30.5, "gen_cost_usd": 0.062343} +{"instance_id": "django__django-12589", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 251.3, "eval_duration_s": 48.6, "gen_cost_usd": 0.272889} +{"instance_id": "django__django-12700", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 63.2, "eval_duration_s": 25.7, "gen_cost_usd": 0.057297} +{"instance_id": "django__django-12708", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 142.6, "eval_duration_s": 35.9, "gen_cost_usd": 0.130812} +{"instance_id": "django__django-12747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 75.3, "eval_duration_s": 43.9, "gen_cost_usd": 0.049929} +{"instance_id": "django__django-12856", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 169.4, "eval_duration_s": 26.9, "gen_cost_usd": 0.169635} +{"instance_id": "django__django-12908", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 108.8, "eval_duration_s": 35.1, "gen_cost_usd": 0.104679} +{"instance_id": "django__django-12915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 94.6, "eval_duration_s": 32.4, "gen_cost_usd": 0.04815} +{"instance_id": "django__django-12983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 96.8, "eval_duration_s": 24.7, "gen_cost_usd": 0.09093} +{"instance_id": "django__django-13028", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 30.1, "eval_duration_s": 37.1, "gen_cost_usd": 0.016263} +{"instance_id": "django__django-13033", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 29.2, "eval_duration_s": 32.8, "gen_cost_usd": 0.020088} +{"instance_id": "django__django-13158", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 124.3, "eval_duration_s": 26.0, "gen_cost_usd": 0.114822} +{"instance_id": "django__django-13220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 127.3, "eval_duration_s": 23.6, "gen_cost_usd": 0.124545} +{"instance_id": "django__django-13230", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 117.4, "eval_duration_s": 27.9, "gen_cost_usd": 0.13749} +{"instance_id": "django__django-13265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 213.4, "eval_duration_s": 21.9, "gen_cost_usd": 0.222177} +{"instance_id": "django__django-13315", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 239.5, "eval_duration_s": 19.6, "gen_cost_usd": 0.26547} +{"instance_id": "django__django-13321", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 49.8, "eval_duration_s": 25.1, "gen_cost_usd": 0.030936} +{"instance_id": "django__django-13401", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.3, "eval_duration_s": 27.9, "gen_cost_usd": 0.074103} +{"instance_id": "django__django-13447", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 179.6, "eval_duration_s": 34.9, "gen_cost_usd": 0.133173} +{"instance_id": "django__django-13448", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 264.6, "eval_duration_s": 23.4, "gen_cost_usd": 0.309393} +{"instance_id": "django__django-13551", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.6, "eval_duration_s": 30.7, "gen_cost_usd": 0.05235} +{"instance_id": "django__django-13590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.0, "eval_duration_s": 39.4, "gen_cost_usd": 0.071469} +{"instance_id": "django__django-13658", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 38.4, "eval_duration_s": 61.3, "gen_cost_usd": 0.01923} +{"instance_id": "django__django-13660", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 64.3, "eval_duration_s": 29.5, "gen_cost_usd": 0.041499} +{"instance_id": "django__django-13710", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 139.6, "eval_duration_s": 39.5, "gen_cost_usd": 0.148041} +{"instance_id": "django__django-13757", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 134.1, "eval_duration_s": 28.2, "gen_cost_usd": 0.113307} +{"instance_id": "django__django-13768", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.3, "eval_duration_s": 37.1, "gen_cost_usd": 0.053163} +{"instance_id": "django__django-13925", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 190.4, "eval_duration_s": 22.7, "gen_cost_usd": 0.156345} +{"instance_id": "django__django-13933", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 381.2, "eval_duration_s": 17.8, "gen_cost_usd": 0.384363} +{"instance_id": "django__django-13964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.8, "eval_duration_s": 45.6, "gen_cost_usd": 0.100275} +{"instance_id": "django__django-14016", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 187.3, "eval_duration_s": 28.8, "gen_cost_usd": 0.19587} +{"instance_id": "django__django-14017", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 260.1, "eval_duration_s": 25.4, "gen_cost_usd": 0.29916} +{"instance_id": "django__django-14155", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 66.9, "eval_duration_s": 39.2, "gen_cost_usd": 0.05961} +{"instance_id": "django__django-14238", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.7, "eval_duration_s": 28.3, "gen_cost_usd": 0.041748} +{"instance_id": "django__django-14382", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 30.3, "eval_duration_s": 59.9, "gen_cost_usd": 0.013485} +{"instance_id": "django__django-14411", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 88.7, "eval_duration_s": 31.9, "gen_cost_usd": 0.076698} +{"instance_id": "django__django-14534", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 142.3, "eval_duration_s": 41.9, "gen_cost_usd": 0.140226} +{"instance_id": "django__django-14580", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.3, "eval_duration_s": 40.2, "gen_cost_usd": 0.015012} +{"instance_id": "django__django-14608", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 173.9, "eval_duration_s": 52.9, "gen_cost_usd": 0.168522} +{"instance_id": "django__django-14667", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 256.6, "eval_duration_s": 22.6, "gen_cost_usd": 0.282657} +{"instance_id": "django__django-14672", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.0, "eval_duration_s": 52.9, "gen_cost_usd": 0.0327} +{"instance_id": "django__django-14730", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 469.6, "eval_duration_s": 48.0, "gen_cost_usd": 0.52137} +{"instance_id": "django__django-14752", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.9, "eval_duration_s": 52.4, "gen_cost_usd": 0.057282} +{"instance_id": "django__django-14787", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 68.0, "eval_duration_s": 73.4, "gen_cost_usd": 0.04998} +{"instance_id": "django__django-14855", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 45.8, "eval_duration_s": 145.2, "gen_cost_usd": 0.023643} +{"instance_id": "django__django-14915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.7, "eval_duration_s": 40.4, "gen_cost_usd": 0.017049} +{"instance_id": "django__django-14997", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 283.3, "eval_duration_s": 23.1, "gen_cost_usd": 0.312906} +{"instance_id": "django__django-14999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 206.2, "eval_duration_s": 26.7, "gen_cost_usd": 0.224217} +{"instance_id": "django__django-15061", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 124.0, "eval_duration_s": 101.9, "gen_cost_usd": 0.106311} +{"instance_id": "django__django-15202", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 71.0, "eval_duration_s": 37.2, "gen_cost_usd": 0.052821} +{"instance_id": "django__django-15213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 330.7, "eval_duration_s": 19.5, "gen_cost_usd": 0.338769} +{"instance_id": "django__django-15252", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 133.1, "eval_duration_s": 40.3, "gen_cost_usd": 0.13539} +{"instance_id": "django__django-15320", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.0, "eval_duration_s": 38.6, "gen_cost_usd": 0.057657} +{"instance_id": "django__django-15347", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 75.6, "eval_duration_s": 29.4, "gen_cost_usd": 0.05088} +{"instance_id": "django__django-15388", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 59.0, "eval_duration_s": 26.8, "gen_cost_usd": 0.05031} +{"instance_id": "django__django-15400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.2, "eval_duration_s": 37.6, "gen_cost_usd": 0.04971} +{"instance_id": "django__django-15498", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.1, "eval_duration_s": 33.4, "gen_cost_usd": 0.114474} +{"instance_id": "django__django-15695", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 353.0, "eval_duration_s": 26.3, "gen_cost_usd": 0.380646} +{"instance_id": "django__django-15738", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 615.1, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "django__django-15781", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.3, "eval_duration_s": 52.6, "gen_cost_usd": 0.020385} +{"instance_id": "django__django-15789", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 76.6, "eval_duration_s": 38.4, "gen_cost_usd": 0.07986} +{"instance_id": "django__django-15790", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 85.6, "eval_duration_s": 41.0, "gen_cost_usd": 0.080313} +{"instance_id": "django__django-15814", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.3, "eval_duration_s": 32.2, "gen_cost_usd": 0.050913} +{"instance_id": "django__django-15819", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 235.7, "eval_duration_s": 43.9, "gen_cost_usd": 0.297603} +{"instance_id": "django__django-15851", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.3, "eval_duration_s": 33.4, "gen_cost_usd": 0.054147} +{"instance_id": "django__django-15902", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 47.2, "eval_duration_s": 39.2, "gen_cost_usd": 0.036603} +{"instance_id": "django__django-15996", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 159.7, "eval_duration_s": 23.8, "gen_cost_usd": 0.129498} +{"instance_id": "django__django-16041", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 46.9, "eval_duration_s": 41.7, "gen_cost_usd": 0.038313} +{"instance_id": "django__django-16046", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 61.0, "eval_duration_s": 26.4, "gen_cost_usd": 0.049527} +{"instance_id": "django__django-16139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 44.6, "eval_duration_s": 27.2, "gen_cost_usd": 0.017859} +{"instance_id": "django__django-16229", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 268.5, "eval_duration_s": 20.4, "gen_cost_usd": 0.270381} +{"instance_id": "django__django-16255", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.4, "eval_duration_s": 25.5, "gen_cost_usd": 0.012861} +{"instance_id": "django__django-16379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 35.5, "eval_duration_s": 113.1, "gen_cost_usd": 0.022248} +{"instance_id": "django__django-16400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 87.5, "eval_duration_s": 91.9, "gen_cost_usd": 0.083976} +{"instance_id": "django__django-16408", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 408.4, "eval_duration_s": 19.3, "gen_cost_usd": 0.461664} +{"instance_id": "django__django-16527", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 25.5, "eval_duration_s": 31.0, "gen_cost_usd": 0.013059} +{"instance_id": "django__django-16595", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 58.1, "eval_duration_s": 43.8, "gen_cost_usd": 0.053478} +{"instance_id": "django__django-16816", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 434.3, "eval_duration_s": 20.7, "gen_cost_usd": 0.429663} +{"instance_id": "django__django-16820", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 601.5, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "django__django-16873", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 74.8, "eval_duration_s": 28.4, "gen_cost_usd": 0.060855} +{"instance_id": "django__django-16910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 500.8, "eval_duration_s": 23.7, "gen_cost_usd": 0.607005} +{"instance_id": "django__django-17051", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.9, "eval_duration_s": 29.4, "gen_cost_usd": 0.041154} +{"instance_id": "django__django-17087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 96.1, "eval_duration_s": 35.6, "gen_cost_usd": 0.064275} +{"instance_id": "matplotlib__matplotlib-18869", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 282.7, "eval_duration_s": 75.5, "gen_cost_usd": 0.105255} +{"instance_id": "matplotlib__matplotlib-22711", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 226.4, "eval_duration_s": 79.4, "gen_cost_usd": 0.089361} +{"instance_id": "matplotlib__matplotlib-22835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 211.4, "eval_duration_s": 103.5, "gen_cost_usd": 0.078297} +{"instance_id": "matplotlib__matplotlib-23299", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 534.7, "eval_duration_s": 91.4, "gen_cost_usd": 0.396213} +{"instance_id": "matplotlib__matplotlib-23314", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 306.3, "eval_duration_s": 82.1, "gen_cost_usd": 0.093924} +{"instance_id": "matplotlib__matplotlib-23476", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 440.0, "eval_duration_s": 94.1, "gen_cost_usd": 0.407541} +{"instance_id": "matplotlib__matplotlib-23562", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 332.3, "eval_duration_s": 81.6, "gen_cost_usd": 0.198681} +{"instance_id": "matplotlib__matplotlib-23563", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 313.1, "eval_duration_s": 77.5, "gen_cost_usd": 0.309069} +{"instance_id": "matplotlib__matplotlib-23913", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 214.3, "eval_duration_s": 83.6, "gen_cost_usd": 0.114642} +{"instance_id": "matplotlib__matplotlib-23964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 130.1, "eval_duration_s": 100.8, "gen_cost_usd": 0.017466} +{"instance_id": "matplotlib__matplotlib-23987", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 219.7, "eval_duration_s": 195.6, "gen_cost_usd": 0.084459} +{"instance_id": "matplotlib__matplotlib-24149", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 169.6, "eval_duration_s": 204.2, "gen_cost_usd": 0.036435} +{"instance_id": "matplotlib__matplotlib-24265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 185.0, "eval_duration_s": 89.9, "gen_cost_usd": 0.080229} +{"instance_id": "matplotlib__matplotlib-24334", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 215.0, "eval_duration_s": 92.7, "gen_cost_usd": 0.065604} +{"instance_id": "matplotlib__matplotlib-24970", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 208.3, "eval_duration_s": 93.4, "gen_cost_usd": 0.0777} +{"instance_id": "matplotlib__matplotlib-25079", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 506.8, "eval_duration_s": 91.5, "gen_cost_usd": 0.401136} +{"instance_id": "matplotlib__matplotlib-25311", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 601.0, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "matplotlib__matplotlib-25332", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 246.2, "eval_duration_s": 83.1, "gen_cost_usd": 0.089643} +{"instance_id": "matplotlib__matplotlib-25433", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.4, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "matplotlib__matplotlib-25442", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.3, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "matplotlib__matplotlib-25498", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 580.8, "eval_duration_s": 90.2, "gen_cost_usd": 0.482124} +{"instance_id": "matplotlib__matplotlib-26011", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 239.9, "eval_duration_s": 83.5, "gen_cost_usd": 0.140001} +{"instance_id": "matplotlib__matplotlib-26020", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 184.2, "eval_duration_s": 186.1, "gen_cost_usd": 0.066654} +{"instance_id": "mwaskom__seaborn-2848", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 508.3, "eval_duration_s": 21.5, "gen_cost_usd": 0.552411} +{"instance_id": "mwaskom__seaborn-3010", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 46.9, "eval_duration_s": 33.3, "gen_cost_usd": 0.052899} +{"instance_id": "mwaskom__seaborn-3190", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 42.2, "eval_duration_s": 30.9, "gen_cost_usd": 0.04521} +{"instance_id": "mwaskom__seaborn-3407", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 222.4, "eval_duration_s": 73.0, "gen_cost_usd": 0.2136} +{"instance_id": "pallets__flask-4045", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 53.1, "eval_duration_s": 24.1, "gen_cost_usd": 0.076347} +{"instance_id": "pallets__flask-4992", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 39.3, "eval_duration_s": 19.1, "gen_cost_usd": 0.045555} +{"instance_id": "pallets__flask-5063", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 127.4, "eval_duration_s": 8.9, "gen_cost_usd": 0.154236} +{"instance_id": "psf__requests-1963", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.4, "eval_duration_s": 31.3, "gen_cost_usd": 0.035673} +{"instance_id": "psf__requests-2148", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 42.8, "eval_duration_s": 61.7, "gen_cost_usd": 0.057693} +{"instance_id": "psf__requests-2317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "timeout", "gen_duration_s": 280.7, "eval_duration_s": 600.2, "gen_cost_usd": 0.07626} +{"instance_id": "psf__requests-2674", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 171.8, "eval_duration_s": 82.9, "gen_cost_usd": 0.187542} +{"instance_id": "psf__requests-3362", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 62.4, "eval_duration_s": 13.7, "gen_cost_usd": 0.058596} +{"instance_id": "psf__requests-863", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 57.5, "eval_duration_s": 36.4, "gen_cost_usd": 0.053295} +{"instance_id": "pydata__xarray-3364", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.9, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "pydata__xarray-4094", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 194.7, "eval_duration_s": 16.9, "gen_cost_usd": 0.190782} +{"instance_id": "pydata__xarray-4248", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 108.5, "eval_duration_s": 13.8, "gen_cost_usd": 0.130917} +{"instance_id": "pydata__xarray-4493", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 86.2, "eval_duration_s": 48.8, "gen_cost_usd": 0.121611} +{"instance_id": "pydata__xarray-5131", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 36.7, "eval_duration_s": 42.4, "gen_cost_usd": 0.032739} +{"instance_id": "pylint-dev__pylint-5859", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 175.4, "eval_duration_s": 9.1, "gen_cost_usd": 0.236112} +{"instance_id": "pylint-dev__pylint-6506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 71.5, "eval_duration_s": 13.6, "gen_cost_usd": 0.098181} +{"instance_id": "pylint-dev__pylint-7080", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 136.2, "eval_duration_s": 15.5, "gen_cost_usd": 0.171075} +{"instance_id": "pylint-dev__pylint-7114", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 440.6, "eval_duration_s": 5.2, "gen_cost_usd": 0.589275} +{"instance_id": "pylint-dev__pylint-7228", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 90.9, "eval_duration_s": 10.0, "gen_cost_usd": 0.109128} +{"instance_id": "pylint-dev__pylint-7993", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 77.7, "eval_duration_s": 11.9, "gen_cost_usd": 0.091728} +{"instance_id": "pytest-dev__pytest-11143", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 45.9, "eval_duration_s": 24.3, "gen_cost_usd": 0.050079} +{"instance_id": "pytest-dev__pytest-11148", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 263.1, "eval_duration_s": 13.2, "gen_cost_usd": 0.329841} +{"instance_id": "pytest-dev__pytest-5103", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.9, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "pytest-dev__pytest-5221", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 263.9, "eval_duration_s": 13.5, "gen_cost_usd": 0.255471} +{"instance_id": "pytest-dev__pytest-5227", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.3, "eval_duration_s": 34.5, "gen_cost_usd": 0.020928} +{"instance_id": "pytest-dev__pytest-5413", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.0, "eval_duration_s": 14.5, "gen_cost_usd": 0.057057} +{"instance_id": "pytest-dev__pytest-5495", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 69.3, "eval_duration_s": 15.8, "gen_cost_usd": 0.059319} +{"instance_id": "pytest-dev__pytest-5692", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 49.9, "eval_duration_s": 27.0, "gen_cost_usd": 0.072726} +{"instance_id": "pytest-dev__pytest-6116", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 30.6, "eval_duration_s": 20.5, "gen_cost_usd": 0.028284} +{"instance_id": "pytest-dev__pytest-7168", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 26.7, "eval_duration_s": 25.1, "gen_cost_usd": 0.034452} +{"instance_id": "pytest-dev__pytest-7220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 235.4, "eval_duration_s": 9.7, "gen_cost_usd": 0.241527} +{"instance_id": "pytest-dev__pytest-7373", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.6, "eval_duration_s": 13.7, "gen_cost_usd": 0.051297} +{"instance_id": "pytest-dev__pytest-7432", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.3, "eval_duration_s": 16.1, "gen_cost_usd": 0.063561} +{"instance_id": "pytest-dev__pytest-7490", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 100.3, "eval_duration_s": 12.8, "gen_cost_usd": 0.146775} +{"instance_id": "pytest-dev__pytest-8365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 68.6, "eval_duration_s": 13.5, "gen_cost_usd": 0.072432} +{"instance_id": "pytest-dev__pytest-8906", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 51.3, "eval_duration_s": 16.2, "gen_cost_usd": 0.056994} +{"instance_id": "pytest-dev__pytest-9359", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 423.0, "eval_duration_s": 9.3, "gen_cost_usd": 0.513519} +{"instance_id": "scikit-learn__scikit-learn-10297", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 179.9, "eval_duration_s": 132.3, "gen_cost_usd": 0.059958} +{"instance_id": "scikit-learn__scikit-learn-10508", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 142.8, "eval_duration_s": 133.3, "gen_cost_usd": 0.020919} +{"instance_id": "scikit-learn__scikit-learn-10949", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 329.8, "eval_duration_s": 141.7, "gen_cost_usd": 0.267798} +{"instance_id": "scikit-learn__scikit-learn-11040", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 213.0, "eval_duration_s": 133.7, "gen_cost_usd": 0.104217} +{"instance_id": "scikit-learn__scikit-learn-11281", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 227.0, "eval_duration_s": 136.0, "gen_cost_usd": 0.11433} +{"instance_id": "scikit-learn__scikit-learn-12471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 184.9, "eval_duration_s": 151.4, "gen_cost_usd": 0.043431} +{"instance_id": "scikit-learn__scikit-learn-13142", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 196.2, "eval_duration_s": 156.3, "gen_cost_usd": 0.034335} +{"instance_id": "scikit-learn__scikit-learn-13241", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 451.3, "eval_duration_s": 182.0, "gen_cost_usd": 0.351219} +{"instance_id": "scikit-learn__scikit-learn-13439", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 205.7, "eval_duration_s": 174.3, "gen_cost_usd": 0.019278} +{"instance_id": "scikit-learn__scikit-learn-13496", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 225.5, "eval_duration_s": 175.9, "gen_cost_usd": 0.092478} +{"instance_id": "scikit-learn__scikit-learn-13497", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 243.3, "eval_duration_s": 174.6, "gen_cost_usd": 0.078357} +{"instance_id": "scikit-learn__scikit-learn-13584", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 254.5, "eval_duration_s": 179.9, "gen_cost_usd": 0.074706} +{"instance_id": "scikit-learn__scikit-learn-13779", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 229.1, "eval_duration_s": 203.7, "gen_cost_usd": 0.020601} +{"instance_id": "scikit-learn__scikit-learn-14087", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.8, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "scikit-learn__scikit-learn-14092", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 283.2, "eval_duration_s": 200.9, "gen_cost_usd": 0.110145} +{"instance_id": "scikit-learn__scikit-learn-14894", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 228.1, "eval_duration_s": 204.3, "gen_cost_usd": 0.033885} +{"instance_id": "scikit-learn__scikit-learn-14983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 339.9, "eval_duration_s": 208.0, "gen_cost_usd": 0.120936} +{"instance_id": "scikit-learn__scikit-learn-15512", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 306.4, "eval_duration_s": 194.3, "gen_cost_usd": 0.133101} +{"instance_id": "scikit-learn__scikit-learn-15535", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 233.3, "eval_duration_s": 203.0, "gen_cost_usd": 0.047589} +{"instance_id": "scikit-learn__scikit-learn-25500", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 451.4, "eval_duration_s": 16.0, "gen_cost_usd": 0.092292} +{"instance_id": "scikit-learn__scikit-learn-25570", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.3, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "scikit-learn__scikit-learn-25638", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.2, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "scikit-learn__scikit-learn-25747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 328.1, "eval_duration_s": 10.6, "gen_cost_usd": 0.084288} +{"instance_id": "sphinx-doc__sphinx-10325", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 250.0, "eval_duration_s": 14.8, "gen_cost_usd": 0.278871} +{"instance_id": "sphinx-doc__sphinx-10451", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 464.4, "eval_duration_s": 59.7, "gen_cost_usd": 0.54561} +{"instance_id": "sphinx-doc__sphinx-11445", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 77.7, "eval_duration_s": 21.1, "gen_cost_usd": 0.093483} +{"instance_id": "sphinx-doc__sphinx-7686", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 152.1, "eval_duration_s": 15.1, "gen_cost_usd": 0.187743} +{"instance_id": "sphinx-doc__sphinx-7738", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 151.5, "eval_duration_s": 15.6, "gen_cost_usd": 0.160485} +{"instance_id": "sphinx-doc__sphinx-7975", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 141.0, "eval_duration_s": 14.5, "gen_cost_usd": 0.180909} +{"instance_id": "sphinx-doc__sphinx-8273", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 73.2, "eval_duration_s": 16.9, "gen_cost_usd": 0.111054} +{"instance_id": "sphinx-doc__sphinx-8282", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 112.6, "eval_duration_s": 17.5, "gen_cost_usd": 0.153642} +{"instance_id": "sphinx-doc__sphinx-8435", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 326.2, "eval_duration_s": 13.2, "gen_cost_usd": 0.407775} +{"instance_id": "sphinx-doc__sphinx-8474", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.2, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sphinx-doc__sphinx-8506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 139.2, "eval_duration_s": 14.5, "gen_cost_usd": 0.145218} +{"instance_id": "sphinx-doc__sphinx-8595", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 75.1, "eval_duration_s": 17.5, "gen_cost_usd": 0.069192} +{"instance_id": "sphinx-doc__sphinx-8627", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 232.6, "eval_duration_s": 16.6, "gen_cost_usd": 0.294498} +{"instance_id": "sphinx-doc__sphinx-8713", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 49.8, "eval_duration_s": 17.5, "gen_cost_usd": 0.058677} +{"instance_id": "sphinx-doc__sphinx-8721", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 99.5, "eval_duration_s": 15.5, "gen_cost_usd": 0.120267} +{"instance_id": "sphinx-doc__sphinx-8801", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 313.3, "eval_duration_s": 13.2, "gen_cost_usd": 0.361779} +{"instance_id": "sympy__sympy-11400", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 87.7, "eval_duration_s": 20.3, "gen_cost_usd": 0.085452} +{"instance_id": "sympy__sympy-11870", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 120.9, "eval_duration_s": 500.6, "gen_cost_usd": 0.106254} +{"instance_id": "sympy__sympy-11897", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.5, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sympy__sympy-12171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 103.5, "eval_duration_s": 43.1, "gen_cost_usd": 0.093951} +{"instance_id": "sympy__sympy-12236", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 361.8, "eval_duration_s": 29.7, "gen_cost_usd": 0.390981} +{"instance_id": "sympy__sympy-12419", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 99.3, "eval_duration_s": 59.6, "gen_cost_usd": 0.097242} +{"instance_id": "sympy__sympy-12454", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 52.7, "eval_duration_s": 38.3, "gen_cost_usd": 0.03927} +{"instance_id": "sympy__sympy-12481", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 151.2, "eval_duration_s": 27.8, "gen_cost_usd": 0.154821} +{"instance_id": "sympy__sympy-13031", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 186.4, "eval_duration_s": 28.7, "gen_cost_usd": 0.200958} +{"instance_id": "sympy__sympy-13043", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 233.7, "eval_duration_s": 24.5, "gen_cost_usd": 0.234324} +{"instance_id": "sympy__sympy-13146", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.1, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sympy__sympy-13177", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 57.8, "eval_duration_s": 20.8, "gen_cost_usd": 0.041253} +{"instance_id": "sympy__sympy-13437", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 108.1, "eval_duration_s": 30.6, "gen_cost_usd": 0.111075} +{"instance_id": "sympy__sympy-13471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 88.5, "eval_duration_s": 26.7, "gen_cost_usd": 0.060327} +{"instance_id": "sympy__sympy-13480", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 44.3, "eval_duration_s": 28.1, "gen_cost_usd": 0.017181} +{"instance_id": "sympy__sympy-13647", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 49.2, "eval_duration_s": 24.7, "gen_cost_usd": 0.03015} +{"instance_id": "sympy__sympy-13773", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 123.7, "eval_duration_s": 23.8, "gen_cost_usd": 0.100059} +{"instance_id": "sympy__sympy-13895", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.4, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sympy__sympy-13915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 280.7, "eval_duration_s": 21.3, "gen_cost_usd": 0.258333} +{"instance_id": "sympy__sympy-13971", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.9, "eval_duration_s": 21.6, "gen_cost_usd": 0.065253} +{"instance_id": "sympy__sympy-14024", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 504.7, "eval_duration_s": 27.6, "gen_cost_usd": 0.63942} +{"instance_id": "sympy__sympy-14308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 399.1, "eval_duration_s": 23.8, "gen_cost_usd": 0.501777} +{"instance_id": "sympy__sympy-14317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.7, "eval_duration_s": 25.0, "gen_cost_usd": 0.230826} +{"instance_id": "sympy__sympy-14396", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.4, "eval_duration_s": 64.7, "gen_cost_usd": 0.073446} +{"instance_id": "sympy__sympy-14774", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 28.0, "eval_duration_s": 24.2, "gen_cost_usd": 0.011418} +{"instance_id": "sympy__sympy-14817", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 176.8, "eval_duration_s": 24.8, "gen_cost_usd": 0.141678} +{"instance_id": "sympy__sympy-15011", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 359.9, "eval_duration_s": 23.6, "gen_cost_usd": 0.382536} +{"instance_id": "sympy__sympy-15308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 101.0, "eval_duration_s": 45.5, "gen_cost_usd": 0.080235} +{"instance_id": "sympy__sympy-15345", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 90.5, "eval_duration_s": 21.5, "gen_cost_usd": 0.055428} +{"instance_id": "sympy__sympy-15346", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 72.0, "eval_duration_s": 42.5, "gen_cost_usd": 0.040305} +{"instance_id": "sympy__sympy-15609", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 91.8, "eval_duration_s": 22.8, "gen_cost_usd": 0.044412} +{"instance_id": "sympy__sympy-15678", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 265.4, "eval_duration_s": 27.1, "gen_cost_usd": 0.271749} +{"instance_id": "sympy__sympy-16106", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 136.7, "eval_duration_s": 23.0, "gen_cost_usd": 0.181446} +{"instance_id": "sympy__sympy-16281", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.7, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sympy__sympy-16503", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.1, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sympy__sympy-16792", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 205.4, "eval_duration_s": 22.9, "gen_cost_usd": 0.194646} +{"instance_id": "sympy__sympy-16988", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.2, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sympy__sympy-17022", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 243.2, "eval_duration_s": 18.7, "gen_cost_usd": 0.260619} +{"instance_id": "sympy__sympy-17139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 91.4, "eval_duration_s": 118.4, "gen_cost_usd": 0.061002} +{"instance_id": "sympy__sympy-17630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 508.8, "eval_duration_s": 47.8, "gen_cost_usd": 0.607587} +{"instance_id": "sympy__sympy-17655", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 168.3, "eval_duration_s": 34.1, "gen_cost_usd": 0.147522} +{"instance_id": "sympy__sympy-18057", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 124.7, "eval_duration_s": 115.2, "gen_cost_usd": 0.085764} +{"instance_id": "sympy__sympy-18087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 526.6, "eval_duration_s": 30.7, "gen_cost_usd": 0.547908} +{"instance_id": "sympy__sympy-18189", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 61.4, "eval_duration_s": 41.2, "gen_cost_usd": 0.020223} +{"instance_id": "sympy__sympy-18199", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 90.8, "eval_duration_s": 90.1, "gen_cost_usd": 0.075705} +{"instance_id": "sympy__sympy-18532", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 131.7, "eval_duration_s": 25.8, "gen_cost_usd": 0.089325} +{"instance_id": "sympy__sympy-18621", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.9, "eval_duration_s": 24.9, "gen_cost_usd": 0.033663} +{"instance_id": "sympy__sympy-18698", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 242.7, "eval_duration_s": 28.2, "gen_cost_usd": 0.215727} +{"instance_id": "sympy__sympy-18835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 86.0, "eval_duration_s": 21.0, "gen_cost_usd": 0.061776} +{"instance_id": "sympy__sympy-19007", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 226.1, "eval_duration_s": 22.7, "gen_cost_usd": 0.21489} +{"instance_id": "sympy__sympy-19254", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 375.9, "eval_duration_s": 54.3, "gen_cost_usd": 0.429435} +{"instance_id": "sympy__sympy-19487", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.0, "eval_duration_s": 28.3, "gen_cost_usd": 0.059358} +{"instance_id": "sympy__sympy-20049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 276.0, "eval_duration_s": 21.7, "gen_cost_usd": 0.301704} +{"instance_id": "sympy__sympy-20154", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.8, "eval_duration_s": 20.9, "gen_cost_usd": 0.05976} +{"instance_id": "sympy__sympy-20212", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 181.0, "eval_duration_s": 35.0, "gen_cost_usd": 0.157905} +{"instance_id": "sympy__sympy-20322", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 95.4, "eval_duration_s": 34.1, "gen_cost_usd": 0.067455} +{"instance_id": "sympy__sympy-20442", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.0, "eval_duration_s": 22.7, "gen_cost_usd": 0.080367} +{"instance_id": "sympy__sympy-20590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 52.0, "eval_duration_s": 28.1, "gen_cost_usd": 0.030279} +{"instance_id": "sympy__sympy-20639", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 177.2, "eval_duration_s": 35.7, "gen_cost_usd": 0.192159} +{"instance_id": "sympy__sympy-21055", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 101.9, "eval_duration_s": 26.8, "gen_cost_usd": 0.095226} +{"instance_id": "sympy__sympy-21171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 79.0, "eval_duration_s": 25.2, "gen_cost_usd": 0.057663} +{"instance_id": "sympy__sympy-21379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 82.4, "eval_duration_s": 38.3, "gen_cost_usd": 0.064461} +{"instance_id": "sympy__sympy-21612", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 72.3, "eval_duration_s": 22.2, "gen_cost_usd": 0.041337} +{"instance_id": "sympy__sympy-21614", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 102.2, "eval_duration_s": 23.2, "gen_cost_usd": 0.081867} +{"instance_id": "sympy__sympy-21627", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 59.3, "eval_duration_s": 26.2, "gen_cost_usd": 0.041439} +{"instance_id": "sympy__sympy-21847", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.1, "eval_duration_s": 22.0, "gen_cost_usd": 0.036519} +{"instance_id": "sympy__sympy-22005", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.8, "eval_duration_s": 26.9, "gen_cost_usd": 0.154881} +{"instance_id": "sympy__sympy-22714", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 117.6, "eval_duration_s": 24.2, "gen_cost_usd": 0.078186} +{"instance_id": "sympy__sympy-22840", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 221.2, "eval_duration_s": 28.3, "gen_cost_usd": 0.27489} +{"instance_id": "sympy__sympy-23117", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 57.5, "eval_duration_s": 43.0, "gen_cost_usd": 0.033213} +{"instance_id": "sympy__sympy-23191", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.3, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sympy__sympy-23262", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 124.1, "eval_duration_s": 20.9, "gen_cost_usd": 0.160128} +{"instance_id": "sympy__sympy-24066", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 121.7, "eval_duration_s": 24.1, "gen_cost_usd": 0.109968} +{"instance_id": "sympy__sympy-24102", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 120.4, "eval_duration_s": 20.7, "gen_cost_usd": 0.151842} +{"instance_id": "sympy__sympy-24152", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.5, "eval_duration_s": 21.5, "gen_cost_usd": 0.075042} +{"instance_id": "sympy__sympy-24213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 58.7, "eval_duration_s": 23.3, "gen_cost_usd": 0.047781} +{"instance_id": "sympy__sympy-24909", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 191.4, "eval_duration_s": 26.7, "gen_cost_usd": 0.200436} diff --git a/evals/swe-bench/scoreboard/sonnet-baseline-20260316/meta.json b/evals/swe-bench/scoreboard/sonnet-baseline-20260316/meta.json new file mode 100644 index 000000000..912ee5d52 --- /dev/null +++ b/evals/swe-bench/scoreboard/sonnet-baseline-20260316/meta.json @@ -0,0 +1,94 @@ +{ + "run_name": "sonnet-baseline-20260316", + "timestamp": "2026-03-16T15:18:35.681947+00:00", + "model": "claude-sonnet-4-6", + "provider": "anthropic", + "fabro_version": "fabro 0.5.0 (4c6ad4d 2026-03-16)", + "description": "Sonnet 4.6 baseline, default prompt, 2 CPU / 4 GB, 10min timeout", + "notes": "", + "total_instances": 300, + "patched": 281, + "patch_pct": 93.7, + "resolved": 167, + "resolve_pct": 55.7, + "total_gen_cost_usd": 39.78, + "avg_gen_cost_usd": 0.1326, + "gen_wall_time_s": 993.6, + "eval_wall_time_s": 642.3, + "avg_gen_duration_s": 189.7, + "avg_eval_duration_s": 47.8, + "per_repo": { + "astropy/astropy": { + "total": 6, + "patched": 6, + "resolved": 1, + "resolve_pct": 16.7 + }, + "django/django": { + "total": 114, + "patched": 111, + "resolved": 86, + "resolve_pct": 75.4 + }, + "matplotlib/matplotlib": { + "total": 23, + "patched": 20, + "resolved": 0, + "resolve_pct": 0.0 + }, + "mwaskom/seaborn": { + "total": 4, + "patched": 4, + "resolved": 2, + "resolve_pct": 50.0 + }, + "pallets/flask": { + "total": 3, + "patched": 3, + "resolved": 0, + "resolve_pct": 0.0 + }, + "psf/requests": { + "total": 6, + "patched": 6, + "resolved": 5, + "resolve_pct": 83.3 + }, + "pydata/xarray": { + "total": 5, + "patched": 4, + "resolved": 1, + "resolve_pct": 20.0 + }, + "pylint-dev/pylint": { + "total": 6, + "patched": 6, + "resolved": 0, + "resolve_pct": 0.0 + }, + "pytest-dev/pytest": { + "total": 17, + "patched": 16, + "resolved": 11, + "resolve_pct": 64.7 + }, + "scikit-learn/scikit-learn": { + "total": 23, + "patched": 20, + "resolved": 14, + "resolve_pct": 60.9 + }, + "sphinx-doc/sphinx": { + "total": 16, + "patched": 15, + "resolved": 0, + "resolve_pct": 0.0 + }, + "sympy/sympy": { + "total": 77, + "patched": 70, + "resolved": 47, + "resolve_pct": 61.0 + } + } +}