diff --git a/evals/swe-bench/scoreboard/leaderboard.json b/evals/swe-bench/scoreboard/leaderboard.json index ba3f0205a..ec540827e 100644 --- a/evals/swe-bench/scoreboard/leaderboard.json +++ b/evals/swe-bench/scoreboard/leaderboard.json @@ -1,4 +1,17 @@ [ + { + "run_name": "opus-20min-20260316", + "date": "2026-03-16", + "model": "claude-opus-4-6", + "provider": "anthropic", + "resolved": 174, + "total": 300, + "resolve_pct": 58.0, + "patched": 300, + "patch_pct": 100.0, + "total_cost_usd": 218.65, + "avg_cost_usd": 0.7288 + }, { "run_name": "sonnet-20min-20260316", "date": "2026-03-16", diff --git a/evals/swe-bench/scoreboard/opus-20min-20260316/README.md b/evals/swe-bench/scoreboard/opus-20min-20260316/README.md new file mode 100644 index 000000000..fe7631871 --- /dev/null +++ b/evals/swe-bench/scoreboard/opus-20min-20260316/README.md @@ -0,0 +1,38 @@ +# opus-20min-20260316 + +**Date:** 2026-03-16 +**Model:** claude-opus-4-6 (anthropic) +**Fabro:** fabro 0.5.0 (4c6ad4d 2026-03-16) + +## Description + +Opus 4.6, 20min timeout, 4 CPU / 8 GB, default prompt + +## Results + +| Metric | Value | +|--------|-------| +| Instances | 300 | +| Patched | 300 (100.0%) | +| **Resolved** | **174 (58.0%)** | +| Total gen cost | $218.65 | +| Avg gen cost | $0.7288/instance | +| Gen wall time | 1210.0s | +| Eval wall time | 626.4s | + +## Per-repo breakdown + +| Repo | Resolved | Total | Rate | +|------|----------|-------|------| +| astropy/astropy | 1 | 6 | 16.7% | +| django/django | 88 | 114 | 77.2% | +| matplotlib/matplotlib | 0 | 23 | 0.0% | +| mwaskom/seaborn | 1 | 4 | 25.0% | +| pallets/flask | 0 | 3 | 0.0% | +| psf/requests | 5 | 6 | 83.3% | +| pydata/xarray | 2 | 5 | 40.0% | +| pylint-dev/pylint | 0 | 6 | 0.0% | +| pytest-dev/pytest | 12 | 17 | 70.6% | +| scikit-learn/scikit-learn | 15 | 23 | 65.2% | +| sphinx-doc/sphinx | 0 | 16 | 0.0% | +| sympy/sympy | 50 | 77 | 64.9% | diff --git a/evals/swe-bench/scoreboard/opus-20min-20260316/instances.jsonl b/evals/swe-bench/scoreboard/opus-20min-20260316/instances.jsonl new file mode 100644 index 000000000..5404a42b3 --- /dev/null +++ b/evals/swe-bench/scoreboard/opus-20min-20260316/instances.jsonl @@ -0,0 +1,300 @@ +{"instance_id": "astropy__astropy-12907", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 89.3, "eval_duration_s": 86.9, "gen_cost_usd": 0.284535} +{"instance_id": "astropy__astropy-14182", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 149.0, "eval_duration_s": 60.3, "gen_cost_usd": 0.58434} +{"instance_id": "astropy__astropy-14365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 130.3, "eval_duration_s": 53.2, "gen_cost_usd": 0.469515} +{"instance_id": "astropy__astropy-14995", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 76.5, "eval_duration_s": 43.3, "gen_cost_usd": 0.24747} +{"instance_id": "astropy__astropy-6938", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 176.7, "eval_duration_s": 100.8, "gen_cost_usd": 0.61686} +{"instance_id": "astropy__astropy-7746", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 130.0, "eval_duration_s": 98.3, "gen_cost_usd": 0.336255} +{"instance_id": "django__django-10914", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 157.9, "eval_duration_s": 18.4, "gen_cost_usd": 0.741015} +{"instance_id": "django__django-10924", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.1, "eval_duration_s": 28.4, "gen_cost_usd": 0.52605} +{"instance_id": "django__django-11001", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 66.9, "eval_duration_s": 27.6, "gen_cost_usd": 0.277995} +{"instance_id": "django__django-11019", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 304.4, "eval_duration_s": 20.7, "gen_cost_usd": 1.95222} +{"instance_id": "django__django-11039", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 131.4, "eval_duration_s": 17.5, "gen_cost_usd": 0.42363} +{"instance_id": "django__django-11049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 102.5, "eval_duration_s": 17.5, "gen_cost_usd": 0.44535} +{"instance_id": "django__django-11099", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 60.0, "eval_duration_s": 19.0, "gen_cost_usd": 0.213735} +{"instance_id": "django__django-11133", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.4, "eval_duration_s": 28.6, "gen_cost_usd": 0.33831} +{"instance_id": "django__django-11179", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 49.4, "eval_duration_s": 19.5, "gen_cost_usd": 0.21846} +{"instance_id": "django__django-11283", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 137.5, "eval_duration_s": 37.3, "gen_cost_usd": 0.519855} +{"instance_id": "django__django-11422", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 82.6, "eval_duration_s": 26.8, "gen_cost_usd": 0.371205} +{"instance_id": "django__django-11564", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 401.0, "eval_duration_s": 35.0, "gen_cost_usd": 1.851525} +{"instance_id": "django__django-11583", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 85.5, "eval_duration_s": 30.9, "gen_cost_usd": 0.316965} +{"instance_id": "django__django-11620", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 68.6, "eval_duration_s": 20.4, "gen_cost_usd": 0.318915} +{"instance_id": "django__django-11630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 155.1, "eval_duration_s": 24.4, "gen_cost_usd": 0.888495} +{"instance_id": "django__django-11742", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 124.5, "eval_duration_s": 16.5, "gen_cost_usd": 0.68478} +{"instance_id": "django__django-11797", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.2, "eval_duration_s": 17.4, "gen_cost_usd": 0.22053} +{"instance_id": "django__django-11815", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 104.2, "eval_duration_s": 39.1, "gen_cost_usd": 0.379785} +{"instance_id": "django__django-11848", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 45.5, "eval_duration_s": 22.5, "gen_cost_usd": 0.170115} +{"instance_id": "django__django-11905", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 145.3, "eval_duration_s": 26.4, "gen_cost_usd": 0.60399} +{"instance_id": "django__django-11910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 433.9, "eval_duration_s": 56.9, "gen_cost_usd": 2.2692} +{"instance_id": "django__django-11964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 283.4, "eval_duration_s": 20.4, "gen_cost_usd": 1.22238} +{"instance_id": "django__django-11999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 78.3, "eval_duration_s": 61.7, "gen_cost_usd": 0.215535} +{"instance_id": "django__django-12113", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 98.7, "eval_duration_s": 40.9, "gen_cost_usd": 0.508635} +{"instance_id": "django__django-12125", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.6, "eval_duration_s": 16.8, "gen_cost_usd": 0.472545} +{"instance_id": "django__django-12184", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 97.3, "eval_duration_s": 22.8, "gen_cost_usd": 0.46932} +{"instance_id": "django__django-12284", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 158.4, "eval_duration_s": 17.7, "gen_cost_usd": 0.747405} +{"instance_id": "django__django-12286", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.1, "eval_duration_s": 27.4, "gen_cost_usd": 0.3603} +{"instance_id": "django__django-12308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 76.1, "eval_duration_s": 23.5, "gen_cost_usd": 0.569145} +{"instance_id": "django__django-12453", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 47.3, "eval_duration_s": 23.9, "gen_cost_usd": 0.223515} +{"instance_id": "django__django-12470", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 343.9, "eval_duration_s": 33.5, "gen_cost_usd": 1.586655} +{"instance_id": "django__django-12497", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 66.3, "eval_duration_s": 41.9, "gen_cost_usd": 0.28761} +{"instance_id": "django__django-12589", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 580.5, "eval_duration_s": 18.6, "gen_cost_usd": 2.54202} +{"instance_id": "django__django-12700", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.2, "eval_duration_s": 25.0, "gen_cost_usd": 0.221505} +{"instance_id": "django__django-12708", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 103.0, "eval_duration_s": 18.4, "gen_cost_usd": 0.438105} +{"instance_id": "django__django-12747", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 147.0, "eval_duration_s": 17.6, "gen_cost_usd": 0.822} +{"instance_id": "django__django-12856", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 125.0, "eval_duration_s": 16.7, "gen_cost_usd": 0.599145} +{"instance_id": "django__django-12908", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.4, "eval_duration_s": 28.7, "gen_cost_usd": 0.305805} +{"instance_id": "django__django-12915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 97.1, "eval_duration_s": 20.5, "gen_cost_usd": 0.41772} +{"instance_id": "django__django-12983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 46.5, "eval_duration_s": 17.4, "gen_cost_usd": 0.309615} +{"instance_id": "django__django-13028", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.6, "eval_duration_s": 18.6, "gen_cost_usd": 0.215565} +{"instance_id": "django__django-13033", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.5, "eval_duration_s": 19.2, "gen_cost_usd": 0.304545} +{"instance_id": "django__django-13158", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 109.9, "eval_duration_s": 18.0, "gen_cost_usd": 0.53832} +{"instance_id": "django__django-13220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 134.6, "eval_duration_s": 17.0, "gen_cost_usd": 0.669645} +{"instance_id": "django__django-13230", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.8, "eval_duration_s": 22.4, "gen_cost_usd": 0.486225} +{"instance_id": "django__django-13265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.8, "eval_duration_s": 24.0, "gen_cost_usd": 0.989775} +{"instance_id": "django__django-13315", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 143.5, "eval_duration_s": 20.6, "gen_cost_usd": 0.65895} +{"instance_id": "django__django-13321", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 58.2, "eval_duration_s": 21.6, "gen_cost_usd": 0.23115} +{"instance_id": "django__django-13401", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 66.2, "eval_duration_s": 22.6, "gen_cost_usd": 0.308175} +{"instance_id": "django__django-13447", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 90.7, "eval_duration_s": 22.0, "gen_cost_usd": 0.398145} +{"instance_id": "django__django-13448", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 151.3, "eval_duration_s": 16.5, "gen_cost_usd": 0.563925} +{"instance_id": "django__django-13551", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.7, "eval_duration_s": 85.2, "gen_cost_usd": 0.285285} +{"instance_id": "django__django-13590", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 57.9, "eval_duration_s": 23.6, "gen_cost_usd": 0.25767} +{"instance_id": "django__django-13658", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 63.9, "eval_duration_s": 46.3, "gen_cost_usd": 0.22407} +{"instance_id": "django__django-13660", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 62.1, "eval_duration_s": 18.9, "gen_cost_usd": 0.234135} +{"instance_id": "django__django-13710", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 125.6, "eval_duration_s": 23.6, "gen_cost_usd": 0.505305} +{"instance_id": "django__django-13757", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 166.2, "eval_duration_s": 17.8, "gen_cost_usd": 0.981255} +{"instance_id": "django__django-13768", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.6, "eval_duration_s": 32.7, "gen_cost_usd": 0.38901} +{"instance_id": "django__django-13925", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 127.6, "eval_duration_s": 20.8, "gen_cost_usd": 0.726075} +{"instance_id": "django__django-13933", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 260.6, "eval_duration_s": 64.7, "gen_cost_usd": 1.075485} +{"instance_id": "django__django-13964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.5, "eval_duration_s": 23.3, "gen_cost_usd": 0.20028} +{"instance_id": "django__django-14016", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.7, "eval_duration_s": 32.6, "gen_cost_usd": 0.268725} +{"instance_id": "django__django-14017", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.6, "eval_duration_s": 21.2, "gen_cost_usd": 0.56922} +{"instance_id": "django__django-14155", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 102.5, "eval_duration_s": 21.8, "gen_cost_usd": 0.520395} +{"instance_id": "django__django-14238", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 53.4, "eval_duration_s": 26.9, "gen_cost_usd": 0.22512} +{"instance_id": "django__django-14382", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 30.3, "eval_duration_s": 55.2, "gen_cost_usd": 0.109305} +{"instance_id": "django__django-14411", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.8, "eval_duration_s": 21.8, "gen_cost_usd": 0.26427} +{"instance_id": "django__django-14534", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 123.2, "eval_duration_s": 27.1, "gen_cost_usd": 0.481905} +{"instance_id": "django__django-14580", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 44.3, "eval_duration_s": 25.4, "gen_cost_usd": 0.15237} +{"instance_id": "django__django-14608", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.7, "eval_duration_s": 40.7, "gen_cost_usd": 0.480225} +{"instance_id": "django__django-14667", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 105.4, "eval_duration_s": 25.6, "gen_cost_usd": 0.58335} +{"instance_id": "django__django-14672", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 50.1, "eval_duration_s": 30.5, "gen_cost_usd": 0.22401} +{"instance_id": "django__django-14730", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 131.6, "eval_duration_s": 21.8, "gen_cost_usd": 0.63759} +{"instance_id": "django__django-14752", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 51.5, "eval_duration_s": 25.9, "gen_cost_usd": 0.227025} +{"instance_id": "django__django-14787", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 79.0, "eval_duration_s": 23.6, "gen_cost_usd": 0.29388} +{"instance_id": "django__django-14855", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 163.6, "eval_duration_s": 134.6, "gen_cost_usd": 0.68208} +{"instance_id": "django__django-14915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 59.1, "eval_duration_s": 35.0, "gen_cost_usd": 0.1893} +{"instance_id": "django__django-14997", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 118.9, "eval_duration_s": 25.0, "gen_cost_usd": 0.635475} +{"instance_id": "django__django-14999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 289.2, "eval_duration_s": 22.6, "gen_cost_usd": 1.326255} +{"instance_id": "django__django-15061", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 145.1, "eval_duration_s": 18.7, "gen_cost_usd": 0.542115} +{"instance_id": "django__django-15202", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 72.4, "eval_duration_s": 21.2, "gen_cost_usd": 0.280995} +{"instance_id": "django__django-15213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 423.3, "eval_duration_s": 19.5, "gen_cost_usd": 1.867005} +{"instance_id": "django__django-15252", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 133.0, "eval_duration_s": 23.7, "gen_cost_usd": 0.72942} +{"instance_id": "django__django-15320", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 68.0, "eval_duration_s": 19.9, "gen_cost_usd": 0.250185} +{"instance_id": "django__django-15347", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 50.1, "eval_duration_s": 31.4, "gen_cost_usd": 0.22737} +{"instance_id": "django__django-15388", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 75.6, "eval_duration_s": 22.4, "gen_cost_usd": 0.31479} +{"instance_id": "django__django-15400", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 72.6, "eval_duration_s": 28.0, "gen_cost_usd": 0.30222} +{"instance_id": "django__django-15498", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 87.9, "eval_duration_s": 273.7, "gen_cost_usd": 0.6765} +{"instance_id": "django__django-15695", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 113.9, "eval_duration_s": 56.8, "gen_cost_usd": 0.52794} +{"instance_id": "django__django-15738", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 364.6, "eval_duration_s": 20.4, "gen_cost_usd": 1.85304} +{"instance_id": "django__django-15781", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 32.7, "eval_duration_s": 20.4, "gen_cost_usd": 0.154275} +{"instance_id": "django__django-15789", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 83.2, "eval_duration_s": 32.5, "gen_cost_usd": 0.33726} +{"instance_id": "django__django-15790", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 88.0, "eval_duration_s": 20.8, "gen_cost_usd": 0.434655} +{"instance_id": "django__django-15814", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 62.7, "eval_duration_s": 25.0, "gen_cost_usd": 0.265665} +{"instance_id": "django__django-15819", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 183.7, "eval_duration_s": 20.5, "gen_cost_usd": 0.974025} +{"instance_id": "django__django-15851", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 61.7, "eval_duration_s": 26.0, "gen_cost_usd": 0.23775} +{"instance_id": "django__django-15902", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.2, "eval_duration_s": 20.5, "gen_cost_usd": 0.2793} +{"instance_id": "django__django-15996", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 138.5, "eval_duration_s": 18.7, "gen_cost_usd": 0.516285} +{"instance_id": "django__django-16041", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.0, "eval_duration_s": 18.7, "gen_cost_usd": 0.36036} +{"instance_id": "django__django-16046", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 81.1, "eval_duration_s": 22.9, "gen_cost_usd": 0.27381} +{"instance_id": "django__django-16139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.4, "eval_duration_s": 20.8, "gen_cost_usd": 0.249465} +{"instance_id": "django__django-16229", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 320.1, "eval_duration_s": 18.6, "gen_cost_usd": 1.374135} +{"instance_id": "django__django-16255", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.1, "eval_duration_s": 19.6, "gen_cost_usd": 0.17004} +{"instance_id": "django__django-16379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.7, "eval_duration_s": 105.6, "gen_cost_usd": 0.223515} +{"instance_id": "django__django-16400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 57.3, "eval_duration_s": 19.6, "gen_cost_usd": 0.270885} +{"instance_id": "django__django-16408", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.7, "eval_duration_s": 19.6, "gen_cost_usd": 0.47247} +{"instance_id": "django__django-16527", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 52.1, "eval_duration_s": 20.0, "gen_cost_usd": 0.18885} +{"instance_id": "django__django-16595", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 51.6, "eval_duration_s": 24.1, "gen_cost_usd": 0.264015} +{"instance_id": "django__django-16816", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 248.5, "eval_duration_s": 20.6, "gen_cost_usd": 1.20255} +{"instance_id": "django__django-16820", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 615.7, "eval_duration_s": 18.1, "gen_cost_usd": 3.29649} +{"instance_id": "django__django-16873", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.0, "eval_duration_s": 20.1, "gen_cost_usd": 0.227835} +{"instance_id": "django__django-16910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 654.5, "eval_duration_s": 18.0, "gen_cost_usd": 3.61026} +{"instance_id": "django__django-17051", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.8, "eval_duration_s": 20.3, "gen_cost_usd": 0.385755} +{"instance_id": "django__django-17087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.5, "eval_duration_s": 19.9, "gen_cost_usd": 0.287835} +{"instance_id": "matplotlib__matplotlib-18869", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 313.0, "eval_duration_s": 82.2, "gen_cost_usd": 0.74742} +{"instance_id": "matplotlib__matplotlib-22711", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 431.3, "eval_duration_s": 80.9, "gen_cost_usd": 1.31733} +{"instance_id": "matplotlib__matplotlib-22835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 325.3, "eval_duration_s": 78.3, "gen_cost_usd": 0.51057} +{"instance_id": "matplotlib__matplotlib-23299", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 288.7, "eval_duration_s": 86.6, "gen_cost_usd": 0.51285} +{"instance_id": "matplotlib__matplotlib-23314", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 284.1, "eval_duration_s": 82.8, "gen_cost_usd": 0.411885} +{"instance_id": "matplotlib__matplotlib-23476", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 224.0, "eval_duration_s": 142.8, "gen_cost_usd": 0.361485} +{"instance_id": "matplotlib__matplotlib-23562", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 271.5, "eval_duration_s": 128.6, "gen_cost_usd": 0.883065} +{"instance_id": "matplotlib__matplotlib-23563", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 673.4, "eval_duration_s": 78.5, "gen_cost_usd": 3.56103} +{"instance_id": "matplotlib__matplotlib-23913", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 287.4, "eval_duration_s": 81.7, "gen_cost_usd": 0.7575} +{"instance_id": "matplotlib__matplotlib-23964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 160.8, "eval_duration_s": 84.6, "gen_cost_usd": 0.202215} +{"instance_id": "matplotlib__matplotlib-23987", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 281.6, "eval_duration_s": 77.7, "gen_cost_usd": 0.34662} +{"instance_id": "matplotlib__matplotlib-24149", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 178.4, "eval_duration_s": 79.9, "gen_cost_usd": 0.32202} +{"instance_id": "matplotlib__matplotlib-24265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 276.4, "eval_duration_s": 78.7, "gen_cost_usd": 0.485025} +{"instance_id": "matplotlib__matplotlib-24334", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 151.1, "eval_duration_s": 88.7, "gen_cost_usd": 0.27198} +{"instance_id": "matplotlib__matplotlib-24970", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 223.4, "eval_duration_s": 152.5, "gen_cost_usd": 0.461745} +{"instance_id": "matplotlib__matplotlib-25079", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 999.2, "eval_duration_s": 84.8, "gen_cost_usd": 4.17225} +{"instance_id": "matplotlib__matplotlib-25311", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 589.9, "eval_duration_s": 81.7, "gen_cost_usd": 1.654695} +{"instance_id": "matplotlib__matplotlib-25332", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 245.1, "eval_duration_s": 189.5, "gen_cost_usd": 0.52548} +{"instance_id": "matplotlib__matplotlib-25433", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 389.9, "eval_duration_s": 107.4, "gen_cost_usd": 0.81474} +{"instance_id": "matplotlib__matplotlib-25442", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 258.7, "eval_duration_s": 111.6, "gen_cost_usd": 0.5304} +{"instance_id": "matplotlib__matplotlib-25498", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 384.2, "eval_duration_s": 291.5, "gen_cost_usd": 1.03152} +{"instance_id": "matplotlib__matplotlib-26011", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 298.2, "eval_duration_s": 81.9, "gen_cost_usd": 0.78435} +{"instance_id": "matplotlib__matplotlib-26020", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 219.3, "eval_duration_s": 88.9, "gen_cost_usd": 0.58329} +{"instance_id": "mwaskom__seaborn-2848", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 353.3, "eval_duration_s": 31.5, "gen_cost_usd": 1.25613} +{"instance_id": "mwaskom__seaborn-3010", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 46.6, "eval_duration_s": 18.1, "gen_cost_usd": 0.218205} +{"instance_id": "mwaskom__seaborn-3190", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 134.0, "eval_duration_s": 14.1, "gen_cost_usd": 0.641295} +{"instance_id": "mwaskom__seaborn-3407", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 155.4, "eval_duration_s": 83.7, "gen_cost_usd": 0.53895} +{"instance_id": "pallets__flask-4045", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 54.7, "eval_duration_s": 7.1, "gen_cost_usd": 0.298815} +{"instance_id": "pallets__flask-4992", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 93.3, "eval_duration_s": 6.5, "gen_cost_usd": 0.40863} +{"instance_id": "pallets__flask-5063", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 84.4, "eval_duration_s": 17.1, "gen_cost_usd": 0.466095} +{"instance_id": "psf__requests-1963", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 109.9, "eval_duration_s": 34.7, "gen_cost_usd": 0.836325} +{"instance_id": "psf__requests-2148", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 52.4, "eval_duration_s": 25.0, "gen_cost_usd": 0.271395} +{"instance_id": "psf__requests-2317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 32.1, "eval_duration_s": 624.2, "gen_cost_usd": 0.21918} +{"instance_id": "psf__requests-2674", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 125.5, "eval_duration_s": 31.5, "gen_cost_usd": 0.834435} +{"instance_id": "psf__requests-3362", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 140.2, "eval_duration_s": 146.6, "gen_cost_usd": 0.72411} +{"instance_id": "psf__requests-863", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.1, "eval_duration_s": 36.0, "gen_cost_usd": 0.29136} +{"instance_id": "pydata__xarray-3364", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 1097.1, "eval_duration_s": 16.5, "gen_cost_usd": 5.58348} +{"instance_id": "pydata__xarray-4094", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 64.8, "eval_duration_s": 18.0, "gen_cost_usd": 0.36246} +{"instance_id": "pydata__xarray-4248", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 299.6, "eval_duration_s": 14.1, "gen_cost_usd": 1.81785} +{"instance_id": "pydata__xarray-4493", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 125.0, "eval_duration_s": 47.7, "gen_cost_usd": 0.39585} +{"instance_id": "pydata__xarray-5131", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 44.1, "eval_duration_s": 15.2, "gen_cost_usd": 0.27705} +{"instance_id": "pylint-dev__pylint-5859", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 87.2, "eval_duration_s": 13.0, "gen_cost_usd": 0.57327} +{"instance_id": "pylint-dev__pylint-6506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 97.2, "eval_duration_s": 17.7, "gen_cost_usd": 0.74064} +{"instance_id": "pylint-dev__pylint-7080", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 191.5, "eval_duration_s": 15.5, "gen_cost_usd": 1.093935} +{"instance_id": "pylint-dev__pylint-7114", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 482.2, "eval_duration_s": 5.4, "gen_cost_usd": 2.48145} +{"instance_id": "pylint-dev__pylint-7228", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 198.2, "eval_duration_s": 8.6, "gen_cost_usd": 1.00035} +{"instance_id": "pylint-dev__pylint-7993", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 187.0, "eval_duration_s": 13.8, "gen_cost_usd": 0.868035} +{"instance_id": "pytest-dev__pytest-11143", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 52.5, "eval_duration_s": 13.0, "gen_cost_usd": 0.264195} +{"instance_id": "pytest-dev__pytest-11148", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 165.9, "eval_duration_s": 18.5, "gen_cost_usd": 0.80727} +{"instance_id": "pytest-dev__pytest-5103", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 727.5, "eval_duration_s": 11.1, "gen_cost_usd": 4.16271} +{"instance_id": "pytest-dev__pytest-5221", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 56.2, "eval_duration_s": 18.0, "gen_cost_usd": 0.256725} +{"instance_id": "pytest-dev__pytest-5227", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.7, "eval_duration_s": 10.3, "gen_cost_usd": 0.790275} +{"instance_id": "pytest-dev__pytest-5413", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 101.7, "eval_duration_s": 13.9, "gen_cost_usd": 0.399915} +{"instance_id": "pytest-dev__pytest-5495", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 153.0, "eval_duration_s": 14.1, "gen_cost_usd": 0.572055} +{"instance_id": "pytest-dev__pytest-5692", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.9, "eval_duration_s": 10.8, "gen_cost_usd": 0.258585} +{"instance_id": "pytest-dev__pytest-6116", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 38.7, "eval_duration_s": 14.6, "gen_cost_usd": 0.18537} +{"instance_id": "pytest-dev__pytest-7168", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 36.5, "eval_duration_s": 10.5, "gen_cost_usd": 0.238875} +{"instance_id": "pytest-dev__pytest-7220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 201.4, "eval_duration_s": 12.2, "gen_cost_usd": 0.828255} +{"instance_id": "pytest-dev__pytest-7373", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 41.8, "eval_duration_s": 11.4, "gen_cost_usd": 0.264735} +{"instance_id": "pytest-dev__pytest-7432", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 49.7, "eval_duration_s": 22.9, "gen_cost_usd": 0.30717} +{"instance_id": "pytest-dev__pytest-7490", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 181.0, "eval_duration_s": 16.1, "gen_cost_usd": 1.08102} +{"instance_id": "pytest-dev__pytest-8365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 48.1, "eval_duration_s": 8.9, "gen_cost_usd": 0.2511} +{"instance_id": "pytest-dev__pytest-8906", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 38.1, "eval_duration_s": 13.5, "gen_cost_usd": 0.21246} +{"instance_id": "pytest-dev__pytest-9359", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 192.7, "eval_duration_s": 10.1, "gen_cost_usd": 1.074225} +{"instance_id": "scikit-learn__scikit-learn-10297", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 212.2, "eval_duration_s": 136.1, "gen_cost_usd": 0.514125} +{"instance_id": "scikit-learn__scikit-learn-10508", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 152.7, "eval_duration_s": 136.0, "gen_cost_usd": 0.218415} +{"instance_id": "scikit-learn__scikit-learn-10949", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 204.2, "eval_duration_s": 176.0, "gen_cost_usd": 0.52761} +{"instance_id": "scikit-learn__scikit-learn-11040", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 207.6, "eval_duration_s": 137.4, "gen_cost_usd": 0.44247} +{"instance_id": "scikit-learn__scikit-learn-11281", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 204.2, "eval_duration_s": 144.0, "gen_cost_usd": 0.57006} +{"instance_id": "scikit-learn__scikit-learn-12471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 187.6, "eval_duration_s": 153.2, "gen_cost_usd": 0.25494} +{"instance_id": "scikit-learn__scikit-learn-13142", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 238.2, "eval_duration_s": 288.6, "gen_cost_usd": 0.442305} +{"instance_id": "scikit-learn__scikit-learn-13241", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 327.9, "eval_duration_s": 176.6, "gen_cost_usd": 1.01706} +{"instance_id": "scikit-learn__scikit-learn-13439", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 210.2, "eval_duration_s": 164.8, "gen_cost_usd": 0.313305} +{"instance_id": "scikit-learn__scikit-learn-13496", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 227.0, "eval_duration_s": 168.9, "gen_cost_usd": 0.59988} +{"instance_id": "scikit-learn__scikit-learn-13497", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 239.4, "eval_duration_s": 177.9, "gen_cost_usd": 0.38997} +{"instance_id": "scikit-learn__scikit-learn-13584", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 227.6, "eval_duration_s": 177.2, "gen_cost_usd": 0.32181} +{"instance_id": "scikit-learn__scikit-learn-13779", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 227.9, "eval_duration_s": 202.9, "gen_cost_usd": 0.213465} +{"instance_id": "scikit-learn__scikit-learn-14087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 546.7, "eval_duration_s": 211.0, "gen_cost_usd": 1.4427} +{"instance_id": "scikit-learn__scikit-learn-14092", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 273.2, "eval_duration_s": 212.8, "gen_cost_usd": 0.40689} +{"instance_id": "scikit-learn__scikit-learn-14894", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 268.7, "eval_duration_s": 197.7, "gen_cost_usd": 0.433005} +{"instance_id": "scikit-learn__scikit-learn-14983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 310.6, "eval_duration_s": 195.5, "gen_cost_usd": 0.480105} +{"instance_id": "scikit-learn__scikit-learn-15512", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 309.0, "eval_duration_s": 193.8, "gen_cost_usd": 0.721515} +{"instance_id": "scikit-learn__scikit-learn-15535", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 233.5, "eval_duration_s": 194.7, "gen_cost_usd": 0.27201} +{"instance_id": "scikit-learn__scikit-learn-25500", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 355.0, "eval_duration_s": 14.1, "gen_cost_usd": 0.5682} +{"instance_id": "scikit-learn__scikit-learn-25570", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 515.7, "eval_duration_s": 14.3, "gen_cost_usd": 1.30029} +{"instance_id": "scikit-learn__scikit-learn-25638", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 404.1, "eval_duration_s": 10.8, "gen_cost_usd": 0.781845} +{"instance_id": "scikit-learn__scikit-learn-25747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 372.5, "eval_duration_s": 10.8, "gen_cost_usd": 0.609105} +{"instance_id": "sphinx-doc__sphinx-10325", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 190.9, "eval_duration_s": 20.2, "gen_cost_usd": 0.74217} +{"instance_id": "sphinx-doc__sphinx-10451", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 385.7, "eval_duration_s": 15.6, "gen_cost_usd": 2.093775} +{"instance_id": "sphinx-doc__sphinx-11445", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 106.4, "eval_duration_s": 14.7, "gen_cost_usd": 0.55665} +{"instance_id": "sphinx-doc__sphinx-7686", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 436.7, "eval_duration_s": 13.8, "gen_cost_usd": 2.02653} +{"instance_id": "sphinx-doc__sphinx-7738", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 116.9, "eval_duration_s": 21.8, "gen_cost_usd": 0.55476} +{"instance_id": "sphinx-doc__sphinx-7975", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 127.7, "eval_duration_s": 19.1, "gen_cost_usd": 0.706575} +{"instance_id": "sphinx-doc__sphinx-8273", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 104.2, "eval_duration_s": 16.8, "gen_cost_usd": 0.54189} +{"instance_id": "sphinx-doc__sphinx-8282", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 168.9, "eval_duration_s": 17.6, "gen_cost_usd": 0.9495} +{"instance_id": "sphinx-doc__sphinx-8435", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 236.2, "eval_duration_s": 16.3, "gen_cost_usd": 1.29648} +{"instance_id": "sphinx-doc__sphinx-8474", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 232.8, "eval_duration_s": 19.6, "gen_cost_usd": 1.158885} +{"instance_id": "sphinx-doc__sphinx-8506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 191.8, "eval_duration_s": 17.2, "gen_cost_usd": 0.95802} +{"instance_id": "sphinx-doc__sphinx-8595", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 67.1, "eval_duration_s": 17.6, "gen_cost_usd": 0.343575} +{"instance_id": "sphinx-doc__sphinx-8627", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 397.9, "eval_duration_s": 14.9, "gen_cost_usd": 2.52297} +{"instance_id": "sphinx-doc__sphinx-8713", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 63.6, "eval_duration_s": 16.6, "gen_cost_usd": 0.284115} +{"instance_id": "sphinx-doc__sphinx-8721", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 71.5, "eval_duration_s": 15.5, "gen_cost_usd": 0.388095} +{"instance_id": "sphinx-doc__sphinx-8801", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 197.1, "eval_duration_s": 17.1, "gen_cost_usd": 1.00809} +{"instance_id": "sympy__sympy-11400", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 104.9, "eval_duration_s": 24.1, "gen_cost_usd": 0.61914} +{"instance_id": "sympy__sympy-11870", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 129.1, "eval_duration_s": 482.5, "gen_cost_usd": 0.513795} +{"instance_id": "sympy__sympy-11897", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 744.8, "eval_duration_s": 23.7, "gen_cost_usd": 3.87081} +{"instance_id": "sympy__sympy-12171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 93.2, "eval_duration_s": 24.7, "gen_cost_usd": 0.463095} +{"instance_id": "sympy__sympy-12236", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 236.1, "eval_duration_s": 31.5, "gen_cost_usd": 1.03428} +{"instance_id": "sympy__sympy-12419", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.6, "eval_duration_s": 27.2, "gen_cost_usd": 0.4194} +{"instance_id": "sympy__sympy-12454", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 95.1, "eval_duration_s": 35.4, "gen_cost_usd": 0.274605} +{"instance_id": "sympy__sympy-12481", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 96.5, "eval_duration_s": 38.4, "gen_cost_usd": 0.443235} +{"instance_id": "sympy__sympy-13031", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 187.3, "eval_duration_s": 34.5, "gen_cost_usd": 1.052745} +{"instance_id": "sympy__sympy-13043", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 198.1, "eval_duration_s": 25.3, "gen_cost_usd": 1.114275} +{"instance_id": "sympy__sympy-13146", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 970.6, "eval_duration_s": 30.9, "gen_cost_usd": 4.576065} +{"instance_id": "sympy__sympy-13177", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 61.1, "eval_duration_s": 19.6, "gen_cost_usd": 0.219585} +{"instance_id": "sympy__sympy-13437", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 72.4, "eval_duration_s": 27.7, "gen_cost_usd": 0.304905} +{"instance_id": "sympy__sympy-13471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 86.4, "eval_duration_s": 29.4, "gen_cost_usd": 0.307395} +{"instance_id": "sympy__sympy-13480", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 39.6, "eval_duration_s": 26.7, "gen_cost_usd": 0.11295} +{"instance_id": "sympy__sympy-13647", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 44.6, "eval_duration_s": 33.9, "gen_cost_usd": 0.15882} +{"instance_id": "sympy__sympy-13773", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 99.1, "eval_duration_s": 23.4, "gen_cost_usd": 0.45621} +{"instance_id": "sympy__sympy-13895", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 648.9, "eval_duration_s": 22.4, "gen_cost_usd": 3.764025} +{"instance_id": "sympy__sympy-13915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 507.7, "eval_duration_s": 20.1, "gen_cost_usd": 1.906905} +{"instance_id": "sympy__sympy-13971", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.0, "eval_duration_s": 27.5, "gen_cost_usd": 0.38655} +{"instance_id": "sympy__sympy-14024", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 330.7, "eval_duration_s": 21.2, "gen_cost_usd": 1.7703} +{"instance_id": "sympy__sympy-14308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 240.1, "eval_duration_s": 29.6, "gen_cost_usd": 1.23525} +{"instance_id": "sympy__sympy-14317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 200.7, "eval_duration_s": 22.6, "gen_cost_usd": 1.03587} +{"instance_id": "sympy__sympy-14396", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 74.9, "eval_duration_s": 22.4, "gen_cost_usd": 0.37422} +{"instance_id": "sympy__sympy-14774", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.2, "eval_duration_s": 24.9, "gen_cost_usd": 0.17235} +{"instance_id": "sympy__sympy-14817", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 221.4, "eval_duration_s": 24.7, "gen_cost_usd": 0.9978} +{"instance_id": "sympy__sympy-15011", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 240.9, "eval_duration_s": 22.6, "gen_cost_usd": 0.97833} +{"instance_id": "sympy__sympy-15308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 98.8, "eval_duration_s": 24.5, "gen_cost_usd": 0.37335} +{"instance_id": "sympy__sympy-15345", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 87.2, "eval_duration_s": 21.9, "gen_cost_usd": 0.25809} +{"instance_id": "sympy__sympy-15346", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 97.4, "eval_duration_s": 55.9, "gen_cost_usd": 0.342135} +{"instance_id": "sympy__sympy-15609", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 91.5, "eval_duration_s": 22.4, "gen_cost_usd": 0.28065} +{"instance_id": "sympy__sympy-15678", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 129.4, "eval_duration_s": 26.3, "gen_cost_usd": 0.525135} +{"instance_id": "sympy__sympy-16106", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 116.0, "eval_duration_s": 21.2, "gen_cost_usd": 0.800535} +{"instance_id": "sympy__sympy-16281", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 258.3, "eval_duration_s": 25.9, "gen_cost_usd": 1.20744} +{"instance_id": "sympy__sympy-16503", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 851.1, "eval_duration_s": 26.4, "gen_cost_usd": 4.423005} +{"instance_id": "sympy__sympy-16792", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 207.3, "eval_duration_s": 21.7, "gen_cost_usd": 0.880905} +{"instance_id": "sympy__sympy-16988", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 872.2, "eval_duration_s": 78.8, "gen_cost_usd": 4.234695} +{"instance_id": "sympy__sympy-17022", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 207.4, "eval_duration_s": 20.4, "gen_cost_usd": 0.825225} +{"instance_id": "sympy__sympy-17139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.3, "eval_duration_s": 99.5, "gen_cost_usd": 0.23529} +{"instance_id": "sympy__sympy-17630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 111.8, "eval_duration_s": 26.7, "gen_cost_usd": 0.596655} +{"instance_id": "sympy__sympy-17655", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 166.1, "eval_duration_s": 28.6, "gen_cost_usd": 0.411285} +{"instance_id": "sympy__sympy-18057", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 95.9, "eval_duration_s": 114.1, "gen_cost_usd": 0.34194} +{"instance_id": "sympy__sympy-18087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 206.6, "eval_duration_s": 43.4, "gen_cost_usd": 1.019595} +{"instance_id": "sympy__sympy-18189", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 32.7, "eval_duration_s": 41.9, "gen_cost_usd": 0.131385} +{"instance_id": "sympy__sympy-18199", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 133.1, "eval_duration_s": 86.9, "gen_cost_usd": 0.65052} +{"instance_id": "sympy__sympy-18532", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 127.7, "eval_duration_s": 41.8, "gen_cost_usd": 0.49362} +{"instance_id": "sympy__sympy-18621", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.9, "eval_duration_s": 23.7, "gen_cost_usd": 0.253065} +{"instance_id": "sympy__sympy-18698", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 111.8, "eval_duration_s": 31.5, "gen_cost_usd": 0.508215} +{"instance_id": "sympy__sympy-18835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 83.7, "eval_duration_s": 24.9, "gen_cost_usd": 0.320685} +{"instance_id": "sympy__sympy-19007", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 215.4, "eval_duration_s": 28.2, "gen_cost_usd": 1.00479} +{"instance_id": "sympy__sympy-19254", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 230.7, "eval_duration_s": 28.7, "gen_cost_usd": 1.603245} +{"instance_id": "sympy__sympy-19487", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 90.7, "eval_duration_s": 37.5, "gen_cost_usd": 0.255645} +{"instance_id": "sympy__sympy-20049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 388.3, "eval_duration_s": 19.2, "gen_cost_usd": 2.12964} +{"instance_id": "sympy__sympy-20154", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 103.7, "eval_duration_s": 20.9, "gen_cost_usd": 0.39435} +{"instance_id": "sympy__sympy-20212", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 94.5, "eval_duration_s": 37.1, "gen_cost_usd": 0.470625} +{"instance_id": "sympy__sympy-20322", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 162.3, "eval_duration_s": 29.2, "gen_cost_usd": 0.680145} +{"instance_id": "sympy__sympy-20442", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 131.7, "eval_duration_s": 21.7, "gen_cost_usd": 0.608295} +{"instance_id": "sympy__sympy-20590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.1, "eval_duration_s": 25.1, "gen_cost_usd": 0.183615} +{"instance_id": "sympy__sympy-20639", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.1, "eval_duration_s": 29.7, "gen_cost_usd": 0.95748} +{"instance_id": "sympy__sympy-21055", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 108.3, "eval_duration_s": 23.8, "gen_cost_usd": 0.57885} +{"instance_id": "sympy__sympy-21171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 55.2, "eval_duration_s": 24.2, "gen_cost_usd": 0.275235} +{"instance_id": "sympy__sympy-21379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 65.5, "eval_duration_s": 36.1, "gen_cost_usd": 0.350145} +{"instance_id": "sympy__sympy-21612", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 65.5, "eval_duration_s": 24.6, "gen_cost_usd": 0.247335} +{"instance_id": "sympy__sympy-21614", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 86.5, "eval_duration_s": 24.0, "gen_cost_usd": 0.38304} +{"instance_id": "sympy__sympy-21627", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.9, "eval_duration_s": 26.0, "gen_cost_usd": 0.245145} +{"instance_id": "sympy__sympy-21847", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 50.6, "eval_duration_s": 21.2, "gen_cost_usd": 0.1923} +{"instance_id": "sympy__sympy-22005", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 80.1, "eval_duration_s": 28.4, "gen_cost_usd": 0.474135} +{"instance_id": "sympy__sympy-22714", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 84.0, "eval_duration_s": 29.4, "gen_cost_usd": 0.31401} +{"instance_id": "sympy__sympy-22840", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 134.2, "eval_duration_s": 26.0, "gen_cost_usd": 0.803835} +{"instance_id": "sympy__sympy-23117", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.5, "eval_duration_s": 24.5, "gen_cost_usd": 0.22491} +{"instance_id": "sympy__sympy-23191", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 314.6, "eval_duration_s": 21.6, "gen_cost_usd": 1.63155} +{"instance_id": "sympy__sympy-23262", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 78.2, "eval_duration_s": 23.2, "gen_cost_usd": 0.29304} +{"instance_id": "sympy__sympy-24066", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.8, "eval_duration_s": 21.1, "gen_cost_usd": 0.36447} +{"instance_id": "sympy__sympy-24102", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 117.3, "eval_duration_s": 22.8, "gen_cost_usd": 0.67569} +{"instance_id": "sympy__sympy-24152", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 72.8, "eval_duration_s": 41.9, "gen_cost_usd": 0.290565} +{"instance_id": "sympy__sympy-24213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 72.4, "eval_duration_s": 20.4, "gen_cost_usd": 0.259845} +{"instance_id": "sympy__sympy-24909", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 88.3, "eval_duration_s": 22.8, "gen_cost_usd": 0.376725} diff --git a/evals/swe-bench/scoreboard/opus-20min-20260316/meta.json b/evals/swe-bench/scoreboard/opus-20min-20260316/meta.json new file mode 100644 index 000000000..323f09379 --- /dev/null +++ b/evals/swe-bench/scoreboard/opus-20min-20260316/meta.json @@ -0,0 +1,97 @@ +{ + "run_name": "opus-20min-20260316", + "timestamp": "2026-03-16T18:42:11.799807+00:00", + "model": "claude-opus-4-6", + "provider": "anthropic", + "fabro_version": "fabro 0.5.0 (4c6ad4d 2026-03-16)", + "timeout_s": 1200, + "sandbox_cpu": 4, + "sandbox_memory_gb": 8, + "description": "Opus 4.6, 20min timeout, 4 CPU / 8 GB, default prompt", + "notes": "", + "total_instances": 300, + "patched": 300, + "patch_pct": 100.0, + "resolved": 174, + "resolve_pct": 58.0, + "total_gen_cost_usd": 218.65, + "avg_gen_cost_usd": 0.7288, + "gen_wall_time_s": 1210.0, + "eval_wall_time_s": 626.4, + "avg_gen_duration_s": 175.6, + "avg_eval_duration_s": 47.8, + "per_repo": { + "astropy/astropy": { + "total": 6, + "patched": 6, + "resolved": 1, + "resolve_pct": 16.7 + }, + "django/django": { + "total": 114, + "patched": 114, + "resolved": 88, + "resolve_pct": 77.2 + }, + "matplotlib/matplotlib": { + "total": 23, + "patched": 23, + "resolved": 0, + "resolve_pct": 0.0 + }, + "mwaskom/seaborn": { + "total": 4, + "patched": 4, + "resolved": 1, + "resolve_pct": 25.0 + }, + "pallets/flask": { + "total": 3, + "patched": 3, + "resolved": 0, + "resolve_pct": 0.0 + }, + "psf/requests": { + "total": 6, + "patched": 6, + "resolved": 5, + "resolve_pct": 83.3 + }, + "pydata/xarray": { + "total": 5, + "patched": 5, + "resolved": 2, + "resolve_pct": 40.0 + }, + "pylint-dev/pylint": { + "total": 6, + "patched": 6, + "resolved": 0, + "resolve_pct": 0.0 + }, + "pytest-dev/pytest": { + "total": 17, + "patched": 17, + "resolved": 12, + "resolve_pct": 70.6 + }, + "scikit-learn/scikit-learn": { + "total": 23, + "patched": 23, + "resolved": 15, + "resolve_pct": 65.2 + }, + "sphinx-doc/sphinx": { + "total": 16, + "patched": 16, + "resolved": 0, + "resolve_pct": 0.0 + }, + "sympy/sympy": { + "total": 77, + "patched": 77, + "resolved": 50, + "resolve_pct": 64.9 + } + } +}