mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-07 08:27:12 +00:00
Record Opus 4.6 20min timeout: 58.0% on SWE-Bench-Lite ($218.65)
100% patch rate, 0 timeouts. Only +1 instance over Sonnet at 4x the cost. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp
parent
d13972e195
commit
dfcece9a0d
4 changed files with 448 additions and 0 deletions
|
|
@ -1,4 +1,17 @@
|
|||
[
|
||||
{
|
||||
"run_name": "opus-20min-20260316",
|
||||
"date": "2026-03-16",
|
||||
"model": "claude-opus-4-6",
|
||||
"provider": "anthropic",
|
||||
"resolved": 174,
|
||||
"total": 300,
|
||||
"resolve_pct": 58.0,
|
||||
"patched": 300,
|
||||
"patch_pct": 100.0,
|
||||
"total_cost_usd": 218.65,
|
||||
"avg_cost_usd": 0.7288
|
||||
},
|
||||
{
|
||||
"run_name": "sonnet-20min-20260316",
|
||||
"date": "2026-03-16",
|
||||
|
|
|
|||
38
evals/swe-bench/scoreboard/opus-20min-20260316/README.md
Normal file
38
evals/swe-bench/scoreboard/opus-20min-20260316/README.md
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
# opus-20min-20260316
|
||||
|
||||
**Date:** 2026-03-16
|
||||
**Model:** claude-opus-4-6 (anthropic)
|
||||
**Fabro:** fabro 0.5.0 (4c6ad4d 2026-03-16)
|
||||
|
||||
## Description
|
||||
|
||||
Opus 4.6, 20min timeout, 4 CPU / 8 GB, default prompt
|
||||
|
||||
## Results
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Instances | 300 |
|
||||
| Patched | 300 (100.0%) |
|
||||
| **Resolved** | **174 (58.0%)** |
|
||||
| Total gen cost | $218.65 |
|
||||
| Avg gen cost | $0.7288/instance |
|
||||
| Gen wall time | 1210.0s |
|
||||
| Eval wall time | 626.4s |
|
||||
|
||||
## Per-repo breakdown
|
||||
|
||||
| Repo | Resolved | Total | Rate |
|
||||
|------|----------|-------|------|
|
||||
| astropy/astropy | 1 | 6 | 16.7% |
|
||||
| django/django | 88 | 114 | 77.2% |
|
||||
| matplotlib/matplotlib | 0 | 23 | 0.0% |
|
||||
| mwaskom/seaborn | 1 | 4 | 25.0% |
|
||||
| pallets/flask | 0 | 3 | 0.0% |
|
||||
| psf/requests | 5 | 6 | 83.3% |
|
||||
| pydata/xarray | 2 | 5 | 40.0% |
|
||||
| pylint-dev/pylint | 0 | 6 | 0.0% |
|
||||
| pytest-dev/pytest | 12 | 17 | 70.6% |
|
||||
| scikit-learn/scikit-learn | 15 | 23 | 65.2% |
|
||||
| sphinx-doc/sphinx | 0 | 16 | 0.0% |
|
||||
| sympy/sympy | 50 | 77 | 64.9% |
|
||||
300
evals/swe-bench/scoreboard/opus-20min-20260316/instances.jsonl
Normal file
300
evals/swe-bench/scoreboard/opus-20min-20260316/instances.jsonl
Normal file
|
|
@ -0,0 +1,300 @@
|
|||
{"instance_id": "astropy__astropy-12907", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 89.3, "eval_duration_s": 86.9, "gen_cost_usd": 0.284535}
|
||||
{"instance_id": "astropy__astropy-14182", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 149.0, "eval_duration_s": 60.3, "gen_cost_usd": 0.58434}
|
||||
{"instance_id": "astropy__astropy-14365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 130.3, "eval_duration_s": 53.2, "gen_cost_usd": 0.469515}
|
||||
{"instance_id": "astropy__astropy-14995", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 76.5, "eval_duration_s": 43.3, "gen_cost_usd": 0.24747}
|
||||
{"instance_id": "astropy__astropy-6938", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 176.7, "eval_duration_s": 100.8, "gen_cost_usd": 0.61686}
|
||||
{"instance_id": "astropy__astropy-7746", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 130.0, "eval_duration_s": 98.3, "gen_cost_usd": 0.336255}
|
||||
{"instance_id": "django__django-10914", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 157.9, "eval_duration_s": 18.4, "gen_cost_usd": 0.741015}
|
||||
{"instance_id": "django__django-10924", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.1, "eval_duration_s": 28.4, "gen_cost_usd": 0.52605}
|
||||
{"instance_id": "django__django-11001", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 66.9, "eval_duration_s": 27.6, "gen_cost_usd": 0.277995}
|
||||
{"instance_id": "django__django-11019", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 304.4, "eval_duration_s": 20.7, "gen_cost_usd": 1.95222}
|
||||
{"instance_id": "django__django-11039", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 131.4, "eval_duration_s": 17.5, "gen_cost_usd": 0.42363}
|
||||
{"instance_id": "django__django-11049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 102.5, "eval_duration_s": 17.5, "gen_cost_usd": 0.44535}
|
||||
{"instance_id": "django__django-11099", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 60.0, "eval_duration_s": 19.0, "gen_cost_usd": 0.213735}
|
||||
{"instance_id": "django__django-11133", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.4, "eval_duration_s": 28.6, "gen_cost_usd": 0.33831}
|
||||
{"instance_id": "django__django-11179", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 49.4, "eval_duration_s": 19.5, "gen_cost_usd": 0.21846}
|
||||
{"instance_id": "django__django-11283", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 137.5, "eval_duration_s": 37.3, "gen_cost_usd": 0.519855}
|
||||
{"instance_id": "django__django-11422", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 82.6, "eval_duration_s": 26.8, "gen_cost_usd": 0.371205}
|
||||
{"instance_id": "django__django-11564", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 401.0, "eval_duration_s": 35.0, "gen_cost_usd": 1.851525}
|
||||
{"instance_id": "django__django-11583", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 85.5, "eval_duration_s": 30.9, "gen_cost_usd": 0.316965}
|
||||
{"instance_id": "django__django-11620", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 68.6, "eval_duration_s": 20.4, "gen_cost_usd": 0.318915}
|
||||
{"instance_id": "django__django-11630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 155.1, "eval_duration_s": 24.4, "gen_cost_usd": 0.888495}
|
||||
{"instance_id": "django__django-11742", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 124.5, "eval_duration_s": 16.5, "gen_cost_usd": 0.68478}
|
||||
{"instance_id": "django__django-11797", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.2, "eval_duration_s": 17.4, "gen_cost_usd": 0.22053}
|
||||
{"instance_id": "django__django-11815", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 104.2, "eval_duration_s": 39.1, "gen_cost_usd": 0.379785}
|
||||
{"instance_id": "django__django-11848", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 45.5, "eval_duration_s": 22.5, "gen_cost_usd": 0.170115}
|
||||
{"instance_id": "django__django-11905", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 145.3, "eval_duration_s": 26.4, "gen_cost_usd": 0.60399}
|
||||
{"instance_id": "django__django-11910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 433.9, "eval_duration_s": 56.9, "gen_cost_usd": 2.2692}
|
||||
{"instance_id": "django__django-11964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 283.4, "eval_duration_s": 20.4, "gen_cost_usd": 1.22238}
|
||||
{"instance_id": "django__django-11999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 78.3, "eval_duration_s": 61.7, "gen_cost_usd": 0.215535}
|
||||
{"instance_id": "django__django-12113", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 98.7, "eval_duration_s": 40.9, "gen_cost_usd": 0.508635}
|
||||
{"instance_id": "django__django-12125", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.6, "eval_duration_s": 16.8, "gen_cost_usd": 0.472545}
|
||||
{"instance_id": "django__django-12184", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 97.3, "eval_duration_s": 22.8, "gen_cost_usd": 0.46932}
|
||||
{"instance_id": "django__django-12284", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 158.4, "eval_duration_s": 17.7, "gen_cost_usd": 0.747405}
|
||||
{"instance_id": "django__django-12286", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.1, "eval_duration_s": 27.4, "gen_cost_usd": 0.3603}
|
||||
{"instance_id": "django__django-12308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 76.1, "eval_duration_s": 23.5, "gen_cost_usd": 0.569145}
|
||||
{"instance_id": "django__django-12453", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 47.3, "eval_duration_s": 23.9, "gen_cost_usd": 0.223515}
|
||||
{"instance_id": "django__django-12470", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 343.9, "eval_duration_s": 33.5, "gen_cost_usd": 1.586655}
|
||||
{"instance_id": "django__django-12497", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 66.3, "eval_duration_s": 41.9, "gen_cost_usd": 0.28761}
|
||||
{"instance_id": "django__django-12589", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 580.5, "eval_duration_s": 18.6, "gen_cost_usd": 2.54202}
|
||||
{"instance_id": "django__django-12700", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.2, "eval_duration_s": 25.0, "gen_cost_usd": 0.221505}
|
||||
{"instance_id": "django__django-12708", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 103.0, "eval_duration_s": 18.4, "gen_cost_usd": 0.438105}
|
||||
{"instance_id": "django__django-12747", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 147.0, "eval_duration_s": 17.6, "gen_cost_usd": 0.822}
|
||||
{"instance_id": "django__django-12856", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 125.0, "eval_duration_s": 16.7, "gen_cost_usd": 0.599145}
|
||||
{"instance_id": "django__django-12908", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.4, "eval_duration_s": 28.7, "gen_cost_usd": 0.305805}
|
||||
{"instance_id": "django__django-12915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 97.1, "eval_duration_s": 20.5, "gen_cost_usd": 0.41772}
|
||||
{"instance_id": "django__django-12983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 46.5, "eval_duration_s": 17.4, "gen_cost_usd": 0.309615}
|
||||
{"instance_id": "django__django-13028", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.6, "eval_duration_s": 18.6, "gen_cost_usd": 0.215565}
|
||||
{"instance_id": "django__django-13033", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.5, "eval_duration_s": 19.2, "gen_cost_usd": 0.304545}
|
||||
{"instance_id": "django__django-13158", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 109.9, "eval_duration_s": 18.0, "gen_cost_usd": 0.53832}
|
||||
{"instance_id": "django__django-13220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 134.6, "eval_duration_s": 17.0, "gen_cost_usd": 0.669645}
|
||||
{"instance_id": "django__django-13230", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.8, "eval_duration_s": 22.4, "gen_cost_usd": 0.486225}
|
||||
{"instance_id": "django__django-13265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.8, "eval_duration_s": 24.0, "gen_cost_usd": 0.989775}
|
||||
{"instance_id": "django__django-13315", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 143.5, "eval_duration_s": 20.6, "gen_cost_usd": 0.65895}
|
||||
{"instance_id": "django__django-13321", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 58.2, "eval_duration_s": 21.6, "gen_cost_usd": 0.23115}
|
||||
{"instance_id": "django__django-13401", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 66.2, "eval_duration_s": 22.6, "gen_cost_usd": 0.308175}
|
||||
{"instance_id": "django__django-13447", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 90.7, "eval_duration_s": 22.0, "gen_cost_usd": 0.398145}
|
||||
{"instance_id": "django__django-13448", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 151.3, "eval_duration_s": 16.5, "gen_cost_usd": 0.563925}
|
||||
{"instance_id": "django__django-13551", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.7, "eval_duration_s": 85.2, "gen_cost_usd": 0.285285}
|
||||
{"instance_id": "django__django-13590", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 57.9, "eval_duration_s": 23.6, "gen_cost_usd": 0.25767}
|
||||
{"instance_id": "django__django-13658", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 63.9, "eval_duration_s": 46.3, "gen_cost_usd": 0.22407}
|
||||
{"instance_id": "django__django-13660", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 62.1, "eval_duration_s": 18.9, "gen_cost_usd": 0.234135}
|
||||
{"instance_id": "django__django-13710", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 125.6, "eval_duration_s": 23.6, "gen_cost_usd": 0.505305}
|
||||
{"instance_id": "django__django-13757", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 166.2, "eval_duration_s": 17.8, "gen_cost_usd": 0.981255}
|
||||
{"instance_id": "django__django-13768", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.6, "eval_duration_s": 32.7, "gen_cost_usd": 0.38901}
|
||||
{"instance_id": "django__django-13925", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 127.6, "eval_duration_s": 20.8, "gen_cost_usd": 0.726075}
|
||||
{"instance_id": "django__django-13933", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 260.6, "eval_duration_s": 64.7, "gen_cost_usd": 1.075485}
|
||||
{"instance_id": "django__django-13964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.5, "eval_duration_s": 23.3, "gen_cost_usd": 0.20028}
|
||||
{"instance_id": "django__django-14016", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.7, "eval_duration_s": 32.6, "gen_cost_usd": 0.268725}
|
||||
{"instance_id": "django__django-14017", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.6, "eval_duration_s": 21.2, "gen_cost_usd": 0.56922}
|
||||
{"instance_id": "django__django-14155", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 102.5, "eval_duration_s": 21.8, "gen_cost_usd": 0.520395}
|
||||
{"instance_id": "django__django-14238", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 53.4, "eval_duration_s": 26.9, "gen_cost_usd": 0.22512}
|
||||
{"instance_id": "django__django-14382", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 30.3, "eval_duration_s": 55.2, "gen_cost_usd": 0.109305}
|
||||
{"instance_id": "django__django-14411", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.8, "eval_duration_s": 21.8, "gen_cost_usd": 0.26427}
|
||||
{"instance_id": "django__django-14534", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 123.2, "eval_duration_s": 27.1, "gen_cost_usd": 0.481905}
|
||||
{"instance_id": "django__django-14580", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 44.3, "eval_duration_s": 25.4, "gen_cost_usd": 0.15237}
|
||||
{"instance_id": "django__django-14608", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.7, "eval_duration_s": 40.7, "gen_cost_usd": 0.480225}
|
||||
{"instance_id": "django__django-14667", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 105.4, "eval_duration_s": 25.6, "gen_cost_usd": 0.58335}
|
||||
{"instance_id": "django__django-14672", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 50.1, "eval_duration_s": 30.5, "gen_cost_usd": 0.22401}
|
||||
{"instance_id": "django__django-14730", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 131.6, "eval_duration_s": 21.8, "gen_cost_usd": 0.63759}
|
||||
{"instance_id": "django__django-14752", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 51.5, "eval_duration_s": 25.9, "gen_cost_usd": 0.227025}
|
||||
{"instance_id": "django__django-14787", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 79.0, "eval_duration_s": 23.6, "gen_cost_usd": 0.29388}
|
||||
{"instance_id": "django__django-14855", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 163.6, "eval_duration_s": 134.6, "gen_cost_usd": 0.68208}
|
||||
{"instance_id": "django__django-14915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 59.1, "eval_duration_s": 35.0, "gen_cost_usd": 0.1893}
|
||||
{"instance_id": "django__django-14997", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 118.9, "eval_duration_s": 25.0, "gen_cost_usd": 0.635475}
|
||||
{"instance_id": "django__django-14999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 289.2, "eval_duration_s": 22.6, "gen_cost_usd": 1.326255}
|
||||
{"instance_id": "django__django-15061", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 145.1, "eval_duration_s": 18.7, "gen_cost_usd": 0.542115}
|
||||
{"instance_id": "django__django-15202", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 72.4, "eval_duration_s": 21.2, "gen_cost_usd": 0.280995}
|
||||
{"instance_id": "django__django-15213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 423.3, "eval_duration_s": 19.5, "gen_cost_usd": 1.867005}
|
||||
{"instance_id": "django__django-15252", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 133.0, "eval_duration_s": 23.7, "gen_cost_usd": 0.72942}
|
||||
{"instance_id": "django__django-15320", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 68.0, "eval_duration_s": 19.9, "gen_cost_usd": 0.250185}
|
||||
{"instance_id": "django__django-15347", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 50.1, "eval_duration_s": 31.4, "gen_cost_usd": 0.22737}
|
||||
{"instance_id": "django__django-15388", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 75.6, "eval_duration_s": 22.4, "gen_cost_usd": 0.31479}
|
||||
{"instance_id": "django__django-15400", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 72.6, "eval_duration_s": 28.0, "gen_cost_usd": 0.30222}
|
||||
{"instance_id": "django__django-15498", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 87.9, "eval_duration_s": 273.7, "gen_cost_usd": 0.6765}
|
||||
{"instance_id": "django__django-15695", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 113.9, "eval_duration_s": 56.8, "gen_cost_usd": 0.52794}
|
||||
{"instance_id": "django__django-15738", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 364.6, "eval_duration_s": 20.4, "gen_cost_usd": 1.85304}
|
||||
{"instance_id": "django__django-15781", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 32.7, "eval_duration_s": 20.4, "gen_cost_usd": 0.154275}
|
||||
{"instance_id": "django__django-15789", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 83.2, "eval_duration_s": 32.5, "gen_cost_usd": 0.33726}
|
||||
{"instance_id": "django__django-15790", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 88.0, "eval_duration_s": 20.8, "gen_cost_usd": 0.434655}
|
||||
{"instance_id": "django__django-15814", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 62.7, "eval_duration_s": 25.0, "gen_cost_usd": 0.265665}
|
||||
{"instance_id": "django__django-15819", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 183.7, "eval_duration_s": 20.5, "gen_cost_usd": 0.974025}
|
||||
{"instance_id": "django__django-15851", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 61.7, "eval_duration_s": 26.0, "gen_cost_usd": 0.23775}
|
||||
{"instance_id": "django__django-15902", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.2, "eval_duration_s": 20.5, "gen_cost_usd": 0.2793}
|
||||
{"instance_id": "django__django-15996", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 138.5, "eval_duration_s": 18.7, "gen_cost_usd": 0.516285}
|
||||
{"instance_id": "django__django-16041", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.0, "eval_duration_s": 18.7, "gen_cost_usd": 0.36036}
|
||||
{"instance_id": "django__django-16046", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 81.1, "eval_duration_s": 22.9, "gen_cost_usd": 0.27381}
|
||||
{"instance_id": "django__django-16139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.4, "eval_duration_s": 20.8, "gen_cost_usd": 0.249465}
|
||||
{"instance_id": "django__django-16229", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 320.1, "eval_duration_s": 18.6, "gen_cost_usd": 1.374135}
|
||||
{"instance_id": "django__django-16255", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.1, "eval_duration_s": 19.6, "gen_cost_usd": 0.17004}
|
||||
{"instance_id": "django__django-16379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.7, "eval_duration_s": 105.6, "gen_cost_usd": 0.223515}
|
||||
{"instance_id": "django__django-16400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 57.3, "eval_duration_s": 19.6, "gen_cost_usd": 0.270885}
|
||||
{"instance_id": "django__django-16408", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.7, "eval_duration_s": 19.6, "gen_cost_usd": 0.47247}
|
||||
{"instance_id": "django__django-16527", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 52.1, "eval_duration_s": 20.0, "gen_cost_usd": 0.18885}
|
||||
{"instance_id": "django__django-16595", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 51.6, "eval_duration_s": 24.1, "gen_cost_usd": 0.264015}
|
||||
{"instance_id": "django__django-16816", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 248.5, "eval_duration_s": 20.6, "gen_cost_usd": 1.20255}
|
||||
{"instance_id": "django__django-16820", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 615.7, "eval_duration_s": 18.1, "gen_cost_usd": 3.29649}
|
||||
{"instance_id": "django__django-16873", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.0, "eval_duration_s": 20.1, "gen_cost_usd": 0.227835}
|
||||
{"instance_id": "django__django-16910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 654.5, "eval_duration_s": 18.0, "gen_cost_usd": 3.61026}
|
||||
{"instance_id": "django__django-17051", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.8, "eval_duration_s": 20.3, "gen_cost_usd": 0.385755}
|
||||
{"instance_id": "django__django-17087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.5, "eval_duration_s": 19.9, "gen_cost_usd": 0.287835}
|
||||
{"instance_id": "matplotlib__matplotlib-18869", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 313.0, "eval_duration_s": 82.2, "gen_cost_usd": 0.74742}
|
||||
{"instance_id": "matplotlib__matplotlib-22711", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 431.3, "eval_duration_s": 80.9, "gen_cost_usd": 1.31733}
|
||||
{"instance_id": "matplotlib__matplotlib-22835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 325.3, "eval_duration_s": 78.3, "gen_cost_usd": 0.51057}
|
||||
{"instance_id": "matplotlib__matplotlib-23299", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 288.7, "eval_duration_s": 86.6, "gen_cost_usd": 0.51285}
|
||||
{"instance_id": "matplotlib__matplotlib-23314", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 284.1, "eval_duration_s": 82.8, "gen_cost_usd": 0.411885}
|
||||
{"instance_id": "matplotlib__matplotlib-23476", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 224.0, "eval_duration_s": 142.8, "gen_cost_usd": 0.361485}
|
||||
{"instance_id": "matplotlib__matplotlib-23562", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 271.5, "eval_duration_s": 128.6, "gen_cost_usd": 0.883065}
|
||||
{"instance_id": "matplotlib__matplotlib-23563", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 673.4, "eval_duration_s": 78.5, "gen_cost_usd": 3.56103}
|
||||
{"instance_id": "matplotlib__matplotlib-23913", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 287.4, "eval_duration_s": 81.7, "gen_cost_usd": 0.7575}
|
||||
{"instance_id": "matplotlib__matplotlib-23964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 160.8, "eval_duration_s": 84.6, "gen_cost_usd": 0.202215}
|
||||
{"instance_id": "matplotlib__matplotlib-23987", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 281.6, "eval_duration_s": 77.7, "gen_cost_usd": 0.34662}
|
||||
{"instance_id": "matplotlib__matplotlib-24149", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 178.4, "eval_duration_s": 79.9, "gen_cost_usd": 0.32202}
|
||||
{"instance_id": "matplotlib__matplotlib-24265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 276.4, "eval_duration_s": 78.7, "gen_cost_usd": 0.485025}
|
||||
{"instance_id": "matplotlib__matplotlib-24334", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 151.1, "eval_duration_s": 88.7, "gen_cost_usd": 0.27198}
|
||||
{"instance_id": "matplotlib__matplotlib-24970", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 223.4, "eval_duration_s": 152.5, "gen_cost_usd": 0.461745}
|
||||
{"instance_id": "matplotlib__matplotlib-25079", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 999.2, "eval_duration_s": 84.8, "gen_cost_usd": 4.17225}
|
||||
{"instance_id": "matplotlib__matplotlib-25311", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 589.9, "eval_duration_s": 81.7, "gen_cost_usd": 1.654695}
|
||||
{"instance_id": "matplotlib__matplotlib-25332", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 245.1, "eval_duration_s": 189.5, "gen_cost_usd": 0.52548}
|
||||
{"instance_id": "matplotlib__matplotlib-25433", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 389.9, "eval_duration_s": 107.4, "gen_cost_usd": 0.81474}
|
||||
{"instance_id": "matplotlib__matplotlib-25442", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 258.7, "eval_duration_s": 111.6, "gen_cost_usd": 0.5304}
|
||||
{"instance_id": "matplotlib__matplotlib-25498", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 384.2, "eval_duration_s": 291.5, "gen_cost_usd": 1.03152}
|
||||
{"instance_id": "matplotlib__matplotlib-26011", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 298.2, "eval_duration_s": 81.9, "gen_cost_usd": 0.78435}
|
||||
{"instance_id": "matplotlib__matplotlib-26020", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 219.3, "eval_duration_s": 88.9, "gen_cost_usd": 0.58329}
|
||||
{"instance_id": "mwaskom__seaborn-2848", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 353.3, "eval_duration_s": 31.5, "gen_cost_usd": 1.25613}
|
||||
{"instance_id": "mwaskom__seaborn-3010", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 46.6, "eval_duration_s": 18.1, "gen_cost_usd": 0.218205}
|
||||
{"instance_id": "mwaskom__seaborn-3190", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 134.0, "eval_duration_s": 14.1, "gen_cost_usd": 0.641295}
|
||||
{"instance_id": "mwaskom__seaborn-3407", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 155.4, "eval_duration_s": 83.7, "gen_cost_usd": 0.53895}
|
||||
{"instance_id": "pallets__flask-4045", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 54.7, "eval_duration_s": 7.1, "gen_cost_usd": 0.298815}
|
||||
{"instance_id": "pallets__flask-4992", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 93.3, "eval_duration_s": 6.5, "gen_cost_usd": 0.40863}
|
||||
{"instance_id": "pallets__flask-5063", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 84.4, "eval_duration_s": 17.1, "gen_cost_usd": 0.466095}
|
||||
{"instance_id": "psf__requests-1963", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 109.9, "eval_duration_s": 34.7, "gen_cost_usd": 0.836325}
|
||||
{"instance_id": "psf__requests-2148", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 52.4, "eval_duration_s": 25.0, "gen_cost_usd": 0.271395}
|
||||
{"instance_id": "psf__requests-2317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 32.1, "eval_duration_s": 624.2, "gen_cost_usd": 0.21918}
|
||||
{"instance_id": "psf__requests-2674", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 125.5, "eval_duration_s": 31.5, "gen_cost_usd": 0.834435}
|
||||
{"instance_id": "psf__requests-3362", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 140.2, "eval_duration_s": 146.6, "gen_cost_usd": 0.72411}
|
||||
{"instance_id": "psf__requests-863", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.1, "eval_duration_s": 36.0, "gen_cost_usd": 0.29136}
|
||||
{"instance_id": "pydata__xarray-3364", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 1097.1, "eval_duration_s": 16.5, "gen_cost_usd": 5.58348}
|
||||
{"instance_id": "pydata__xarray-4094", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 64.8, "eval_duration_s": 18.0, "gen_cost_usd": 0.36246}
|
||||
{"instance_id": "pydata__xarray-4248", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 299.6, "eval_duration_s": 14.1, "gen_cost_usd": 1.81785}
|
||||
{"instance_id": "pydata__xarray-4493", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 125.0, "eval_duration_s": 47.7, "gen_cost_usd": 0.39585}
|
||||
{"instance_id": "pydata__xarray-5131", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 44.1, "eval_duration_s": 15.2, "gen_cost_usd": 0.27705}
|
||||
{"instance_id": "pylint-dev__pylint-5859", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 87.2, "eval_duration_s": 13.0, "gen_cost_usd": 0.57327}
|
||||
{"instance_id": "pylint-dev__pylint-6506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 97.2, "eval_duration_s": 17.7, "gen_cost_usd": 0.74064}
|
||||
{"instance_id": "pylint-dev__pylint-7080", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 191.5, "eval_duration_s": 15.5, "gen_cost_usd": 1.093935}
|
||||
{"instance_id": "pylint-dev__pylint-7114", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 482.2, "eval_duration_s": 5.4, "gen_cost_usd": 2.48145}
|
||||
{"instance_id": "pylint-dev__pylint-7228", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 198.2, "eval_duration_s": 8.6, "gen_cost_usd": 1.00035}
|
||||
{"instance_id": "pylint-dev__pylint-7993", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 187.0, "eval_duration_s": 13.8, "gen_cost_usd": 0.868035}
|
||||
{"instance_id": "pytest-dev__pytest-11143", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 52.5, "eval_duration_s": 13.0, "gen_cost_usd": 0.264195}
|
||||
{"instance_id": "pytest-dev__pytest-11148", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 165.9, "eval_duration_s": 18.5, "gen_cost_usd": 0.80727}
|
||||
{"instance_id": "pytest-dev__pytest-5103", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 727.5, "eval_duration_s": 11.1, "gen_cost_usd": 4.16271}
|
||||
{"instance_id": "pytest-dev__pytest-5221", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 56.2, "eval_duration_s": 18.0, "gen_cost_usd": 0.256725}
|
||||
{"instance_id": "pytest-dev__pytest-5227", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.7, "eval_duration_s": 10.3, "gen_cost_usd": 0.790275}
|
||||
{"instance_id": "pytest-dev__pytest-5413", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 101.7, "eval_duration_s": 13.9, "gen_cost_usd": 0.399915}
|
||||
{"instance_id": "pytest-dev__pytest-5495", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 153.0, "eval_duration_s": 14.1, "gen_cost_usd": 0.572055}
|
||||
{"instance_id": "pytest-dev__pytest-5692", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.9, "eval_duration_s": 10.8, "gen_cost_usd": 0.258585}
|
||||
{"instance_id": "pytest-dev__pytest-6116", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 38.7, "eval_duration_s": 14.6, "gen_cost_usd": 0.18537}
|
||||
{"instance_id": "pytest-dev__pytest-7168", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 36.5, "eval_duration_s": 10.5, "gen_cost_usd": 0.238875}
|
||||
{"instance_id": "pytest-dev__pytest-7220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 201.4, "eval_duration_s": 12.2, "gen_cost_usd": 0.828255}
|
||||
{"instance_id": "pytest-dev__pytest-7373", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 41.8, "eval_duration_s": 11.4, "gen_cost_usd": 0.264735}
|
||||
{"instance_id": "pytest-dev__pytest-7432", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 49.7, "eval_duration_s": 22.9, "gen_cost_usd": 0.30717}
|
||||
{"instance_id": "pytest-dev__pytest-7490", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 181.0, "eval_duration_s": 16.1, "gen_cost_usd": 1.08102}
|
||||
{"instance_id": "pytest-dev__pytest-8365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 48.1, "eval_duration_s": 8.9, "gen_cost_usd": 0.2511}
|
||||
{"instance_id": "pytest-dev__pytest-8906", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 38.1, "eval_duration_s": 13.5, "gen_cost_usd": 0.21246}
|
||||
{"instance_id": "pytest-dev__pytest-9359", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 192.7, "eval_duration_s": 10.1, "gen_cost_usd": 1.074225}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10297", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 212.2, "eval_duration_s": 136.1, "gen_cost_usd": 0.514125}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10508", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 152.7, "eval_duration_s": 136.0, "gen_cost_usd": 0.218415}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10949", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 204.2, "eval_duration_s": 176.0, "gen_cost_usd": 0.52761}
|
||||
{"instance_id": "scikit-learn__scikit-learn-11040", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 207.6, "eval_duration_s": 137.4, "gen_cost_usd": 0.44247}
|
||||
{"instance_id": "scikit-learn__scikit-learn-11281", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 204.2, "eval_duration_s": 144.0, "gen_cost_usd": 0.57006}
|
||||
{"instance_id": "scikit-learn__scikit-learn-12471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 187.6, "eval_duration_s": 153.2, "gen_cost_usd": 0.25494}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13142", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 238.2, "eval_duration_s": 288.6, "gen_cost_usd": 0.442305}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13241", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 327.9, "eval_duration_s": 176.6, "gen_cost_usd": 1.01706}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13439", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 210.2, "eval_duration_s": 164.8, "gen_cost_usd": 0.313305}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13496", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 227.0, "eval_duration_s": 168.9, "gen_cost_usd": 0.59988}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13497", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 239.4, "eval_duration_s": 177.9, "gen_cost_usd": 0.38997}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13584", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 227.6, "eval_duration_s": 177.2, "gen_cost_usd": 0.32181}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13779", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 227.9, "eval_duration_s": 202.9, "gen_cost_usd": 0.213465}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 546.7, "eval_duration_s": 211.0, "gen_cost_usd": 1.4427}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14092", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 273.2, "eval_duration_s": 212.8, "gen_cost_usd": 0.40689}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14894", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 268.7, "eval_duration_s": 197.7, "gen_cost_usd": 0.433005}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 310.6, "eval_duration_s": 195.5, "gen_cost_usd": 0.480105}
|
||||
{"instance_id": "scikit-learn__scikit-learn-15512", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 309.0, "eval_duration_s": 193.8, "gen_cost_usd": 0.721515}
|
||||
{"instance_id": "scikit-learn__scikit-learn-15535", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 233.5, "eval_duration_s": 194.7, "gen_cost_usd": 0.27201}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25500", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 355.0, "eval_duration_s": 14.1, "gen_cost_usd": 0.5682}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25570", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 515.7, "eval_duration_s": 14.3, "gen_cost_usd": 1.30029}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25638", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 404.1, "eval_duration_s": 10.8, "gen_cost_usd": 0.781845}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 372.5, "eval_duration_s": 10.8, "gen_cost_usd": 0.609105}
|
||||
{"instance_id": "sphinx-doc__sphinx-10325", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 190.9, "eval_duration_s": 20.2, "gen_cost_usd": 0.74217}
|
||||
{"instance_id": "sphinx-doc__sphinx-10451", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 385.7, "eval_duration_s": 15.6, "gen_cost_usd": 2.093775}
|
||||
{"instance_id": "sphinx-doc__sphinx-11445", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 106.4, "eval_duration_s": 14.7, "gen_cost_usd": 0.55665}
|
||||
{"instance_id": "sphinx-doc__sphinx-7686", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 436.7, "eval_duration_s": 13.8, "gen_cost_usd": 2.02653}
|
||||
{"instance_id": "sphinx-doc__sphinx-7738", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 116.9, "eval_duration_s": 21.8, "gen_cost_usd": 0.55476}
|
||||
{"instance_id": "sphinx-doc__sphinx-7975", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 127.7, "eval_duration_s": 19.1, "gen_cost_usd": 0.706575}
|
||||
{"instance_id": "sphinx-doc__sphinx-8273", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 104.2, "eval_duration_s": 16.8, "gen_cost_usd": 0.54189}
|
||||
{"instance_id": "sphinx-doc__sphinx-8282", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 168.9, "eval_duration_s": 17.6, "gen_cost_usd": 0.9495}
|
||||
{"instance_id": "sphinx-doc__sphinx-8435", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 236.2, "eval_duration_s": 16.3, "gen_cost_usd": 1.29648}
|
||||
{"instance_id": "sphinx-doc__sphinx-8474", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 232.8, "eval_duration_s": 19.6, "gen_cost_usd": 1.158885}
|
||||
{"instance_id": "sphinx-doc__sphinx-8506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 191.8, "eval_duration_s": 17.2, "gen_cost_usd": 0.95802}
|
||||
{"instance_id": "sphinx-doc__sphinx-8595", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 67.1, "eval_duration_s": 17.6, "gen_cost_usd": 0.343575}
|
||||
{"instance_id": "sphinx-doc__sphinx-8627", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 397.9, "eval_duration_s": 14.9, "gen_cost_usd": 2.52297}
|
||||
{"instance_id": "sphinx-doc__sphinx-8713", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 63.6, "eval_duration_s": 16.6, "gen_cost_usd": 0.284115}
|
||||
{"instance_id": "sphinx-doc__sphinx-8721", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 71.5, "eval_duration_s": 15.5, "gen_cost_usd": 0.388095}
|
||||
{"instance_id": "sphinx-doc__sphinx-8801", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 197.1, "eval_duration_s": 17.1, "gen_cost_usd": 1.00809}
|
||||
{"instance_id": "sympy__sympy-11400", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 104.9, "eval_duration_s": 24.1, "gen_cost_usd": 0.61914}
|
||||
{"instance_id": "sympy__sympy-11870", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 129.1, "eval_duration_s": 482.5, "gen_cost_usd": 0.513795}
|
||||
{"instance_id": "sympy__sympy-11897", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 744.8, "eval_duration_s": 23.7, "gen_cost_usd": 3.87081}
|
||||
{"instance_id": "sympy__sympy-12171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 93.2, "eval_duration_s": 24.7, "gen_cost_usd": 0.463095}
|
||||
{"instance_id": "sympy__sympy-12236", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 236.1, "eval_duration_s": 31.5, "gen_cost_usd": 1.03428}
|
||||
{"instance_id": "sympy__sympy-12419", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.6, "eval_duration_s": 27.2, "gen_cost_usd": 0.4194}
|
||||
{"instance_id": "sympy__sympy-12454", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 95.1, "eval_duration_s": 35.4, "gen_cost_usd": 0.274605}
|
||||
{"instance_id": "sympy__sympy-12481", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 96.5, "eval_duration_s": 38.4, "gen_cost_usd": 0.443235}
|
||||
{"instance_id": "sympy__sympy-13031", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 187.3, "eval_duration_s": 34.5, "gen_cost_usd": 1.052745}
|
||||
{"instance_id": "sympy__sympy-13043", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 198.1, "eval_duration_s": 25.3, "gen_cost_usd": 1.114275}
|
||||
{"instance_id": "sympy__sympy-13146", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 970.6, "eval_duration_s": 30.9, "gen_cost_usd": 4.576065}
|
||||
{"instance_id": "sympy__sympy-13177", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 61.1, "eval_duration_s": 19.6, "gen_cost_usd": 0.219585}
|
||||
{"instance_id": "sympy__sympy-13437", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 72.4, "eval_duration_s": 27.7, "gen_cost_usd": 0.304905}
|
||||
{"instance_id": "sympy__sympy-13471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 86.4, "eval_duration_s": 29.4, "gen_cost_usd": 0.307395}
|
||||
{"instance_id": "sympy__sympy-13480", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 39.6, "eval_duration_s": 26.7, "gen_cost_usd": 0.11295}
|
||||
{"instance_id": "sympy__sympy-13647", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 44.6, "eval_duration_s": 33.9, "gen_cost_usd": 0.15882}
|
||||
{"instance_id": "sympy__sympy-13773", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 99.1, "eval_duration_s": 23.4, "gen_cost_usd": 0.45621}
|
||||
{"instance_id": "sympy__sympy-13895", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 648.9, "eval_duration_s": 22.4, "gen_cost_usd": 3.764025}
|
||||
{"instance_id": "sympy__sympy-13915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 507.7, "eval_duration_s": 20.1, "gen_cost_usd": 1.906905}
|
||||
{"instance_id": "sympy__sympy-13971", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.0, "eval_duration_s": 27.5, "gen_cost_usd": 0.38655}
|
||||
{"instance_id": "sympy__sympy-14024", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 330.7, "eval_duration_s": 21.2, "gen_cost_usd": 1.7703}
|
||||
{"instance_id": "sympy__sympy-14308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 240.1, "eval_duration_s": 29.6, "gen_cost_usd": 1.23525}
|
||||
{"instance_id": "sympy__sympy-14317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 200.7, "eval_duration_s": 22.6, "gen_cost_usd": 1.03587}
|
||||
{"instance_id": "sympy__sympy-14396", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 74.9, "eval_duration_s": 22.4, "gen_cost_usd": 0.37422}
|
||||
{"instance_id": "sympy__sympy-14774", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.2, "eval_duration_s": 24.9, "gen_cost_usd": 0.17235}
|
||||
{"instance_id": "sympy__sympy-14817", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 221.4, "eval_duration_s": 24.7, "gen_cost_usd": 0.9978}
|
||||
{"instance_id": "sympy__sympy-15011", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 240.9, "eval_duration_s": 22.6, "gen_cost_usd": 0.97833}
|
||||
{"instance_id": "sympy__sympy-15308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 98.8, "eval_duration_s": 24.5, "gen_cost_usd": 0.37335}
|
||||
{"instance_id": "sympy__sympy-15345", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 87.2, "eval_duration_s": 21.9, "gen_cost_usd": 0.25809}
|
||||
{"instance_id": "sympy__sympy-15346", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 97.4, "eval_duration_s": 55.9, "gen_cost_usd": 0.342135}
|
||||
{"instance_id": "sympy__sympy-15609", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 91.5, "eval_duration_s": 22.4, "gen_cost_usd": 0.28065}
|
||||
{"instance_id": "sympy__sympy-15678", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 129.4, "eval_duration_s": 26.3, "gen_cost_usd": 0.525135}
|
||||
{"instance_id": "sympy__sympy-16106", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 116.0, "eval_duration_s": 21.2, "gen_cost_usd": 0.800535}
|
||||
{"instance_id": "sympy__sympy-16281", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 258.3, "eval_duration_s": 25.9, "gen_cost_usd": 1.20744}
|
||||
{"instance_id": "sympy__sympy-16503", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 851.1, "eval_duration_s": 26.4, "gen_cost_usd": 4.423005}
|
||||
{"instance_id": "sympy__sympy-16792", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 207.3, "eval_duration_s": 21.7, "gen_cost_usd": 0.880905}
|
||||
{"instance_id": "sympy__sympy-16988", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 872.2, "eval_duration_s": 78.8, "gen_cost_usd": 4.234695}
|
||||
{"instance_id": "sympy__sympy-17022", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 207.4, "eval_duration_s": 20.4, "gen_cost_usd": 0.825225}
|
||||
{"instance_id": "sympy__sympy-17139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.3, "eval_duration_s": 99.5, "gen_cost_usd": 0.23529}
|
||||
{"instance_id": "sympy__sympy-17630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 111.8, "eval_duration_s": 26.7, "gen_cost_usd": 0.596655}
|
||||
{"instance_id": "sympy__sympy-17655", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 166.1, "eval_duration_s": 28.6, "gen_cost_usd": 0.411285}
|
||||
{"instance_id": "sympy__sympy-18057", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 95.9, "eval_duration_s": 114.1, "gen_cost_usd": 0.34194}
|
||||
{"instance_id": "sympy__sympy-18087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 206.6, "eval_duration_s": 43.4, "gen_cost_usd": 1.019595}
|
||||
{"instance_id": "sympy__sympy-18189", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 32.7, "eval_duration_s": 41.9, "gen_cost_usd": 0.131385}
|
||||
{"instance_id": "sympy__sympy-18199", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 133.1, "eval_duration_s": 86.9, "gen_cost_usd": 0.65052}
|
||||
{"instance_id": "sympy__sympy-18532", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 127.7, "eval_duration_s": 41.8, "gen_cost_usd": 0.49362}
|
||||
{"instance_id": "sympy__sympy-18621", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.9, "eval_duration_s": 23.7, "gen_cost_usd": 0.253065}
|
||||
{"instance_id": "sympy__sympy-18698", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 111.8, "eval_duration_s": 31.5, "gen_cost_usd": 0.508215}
|
||||
{"instance_id": "sympy__sympy-18835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 83.7, "eval_duration_s": 24.9, "gen_cost_usd": 0.320685}
|
||||
{"instance_id": "sympy__sympy-19007", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 215.4, "eval_duration_s": 28.2, "gen_cost_usd": 1.00479}
|
||||
{"instance_id": "sympy__sympy-19254", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 230.7, "eval_duration_s": 28.7, "gen_cost_usd": 1.603245}
|
||||
{"instance_id": "sympy__sympy-19487", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 90.7, "eval_duration_s": 37.5, "gen_cost_usd": 0.255645}
|
||||
{"instance_id": "sympy__sympy-20049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 388.3, "eval_duration_s": 19.2, "gen_cost_usd": 2.12964}
|
||||
{"instance_id": "sympy__sympy-20154", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 103.7, "eval_duration_s": 20.9, "gen_cost_usd": 0.39435}
|
||||
{"instance_id": "sympy__sympy-20212", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 94.5, "eval_duration_s": 37.1, "gen_cost_usd": 0.470625}
|
||||
{"instance_id": "sympy__sympy-20322", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 162.3, "eval_duration_s": 29.2, "gen_cost_usd": 0.680145}
|
||||
{"instance_id": "sympy__sympy-20442", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 131.7, "eval_duration_s": 21.7, "gen_cost_usd": 0.608295}
|
||||
{"instance_id": "sympy__sympy-20590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.1, "eval_duration_s": 25.1, "gen_cost_usd": 0.183615}
|
||||
{"instance_id": "sympy__sympy-20639", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.1, "eval_duration_s": 29.7, "gen_cost_usd": 0.95748}
|
||||
{"instance_id": "sympy__sympy-21055", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 108.3, "eval_duration_s": 23.8, "gen_cost_usd": 0.57885}
|
||||
{"instance_id": "sympy__sympy-21171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 55.2, "eval_duration_s": 24.2, "gen_cost_usd": 0.275235}
|
||||
{"instance_id": "sympy__sympy-21379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 65.5, "eval_duration_s": 36.1, "gen_cost_usd": 0.350145}
|
||||
{"instance_id": "sympy__sympy-21612", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 65.5, "eval_duration_s": 24.6, "gen_cost_usd": 0.247335}
|
||||
{"instance_id": "sympy__sympy-21614", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 86.5, "eval_duration_s": 24.0, "gen_cost_usd": 0.38304}
|
||||
{"instance_id": "sympy__sympy-21627", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.9, "eval_duration_s": 26.0, "gen_cost_usd": 0.245145}
|
||||
{"instance_id": "sympy__sympy-21847", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 50.6, "eval_duration_s": 21.2, "gen_cost_usd": 0.1923}
|
||||
{"instance_id": "sympy__sympy-22005", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 80.1, "eval_duration_s": 28.4, "gen_cost_usd": 0.474135}
|
||||
{"instance_id": "sympy__sympy-22714", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 84.0, "eval_duration_s": 29.4, "gen_cost_usd": 0.31401}
|
||||
{"instance_id": "sympy__sympy-22840", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 134.2, "eval_duration_s": 26.0, "gen_cost_usd": 0.803835}
|
||||
{"instance_id": "sympy__sympy-23117", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.5, "eval_duration_s": 24.5, "gen_cost_usd": 0.22491}
|
||||
{"instance_id": "sympy__sympy-23191", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 314.6, "eval_duration_s": 21.6, "gen_cost_usd": 1.63155}
|
||||
{"instance_id": "sympy__sympy-23262", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 78.2, "eval_duration_s": 23.2, "gen_cost_usd": 0.29304}
|
||||
{"instance_id": "sympy__sympy-24066", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.8, "eval_duration_s": 21.1, "gen_cost_usd": 0.36447}
|
||||
{"instance_id": "sympy__sympy-24102", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 117.3, "eval_duration_s": 22.8, "gen_cost_usd": 0.67569}
|
||||
{"instance_id": "sympy__sympy-24152", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 72.8, "eval_duration_s": 41.9, "gen_cost_usd": 0.290565}
|
||||
{"instance_id": "sympy__sympy-24213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 72.4, "eval_duration_s": 20.4, "gen_cost_usd": 0.259845}
|
||||
{"instance_id": "sympy__sympy-24909", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 88.3, "eval_duration_s": 22.8, "gen_cost_usd": 0.376725}
|
||||
97
evals/swe-bench/scoreboard/opus-20min-20260316/meta.json
Normal file
97
evals/swe-bench/scoreboard/opus-20min-20260316/meta.json
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
{
|
||||
"run_name": "opus-20min-20260316",
|
||||
"timestamp": "2026-03-16T18:42:11.799807+00:00",
|
||||
"model": "claude-opus-4-6",
|
||||
"provider": "anthropic",
|
||||
"fabro_version": "fabro 0.5.0 (4c6ad4d 2026-03-16)",
|
||||
"timeout_s": 1200,
|
||||
"sandbox_cpu": 4,
|
||||
"sandbox_memory_gb": 8,
|
||||
"description": "Opus 4.6, 20min timeout, 4 CPU / 8 GB, default prompt",
|
||||
"notes": "",
|
||||
"total_instances": 300,
|
||||
"patched": 300,
|
||||
"patch_pct": 100.0,
|
||||
"resolved": 174,
|
||||
"resolve_pct": 58.0,
|
||||
"total_gen_cost_usd": 218.65,
|
||||
"avg_gen_cost_usd": 0.7288,
|
||||
"gen_wall_time_s": 1210.0,
|
||||
"eval_wall_time_s": 626.4,
|
||||
"avg_gen_duration_s": 175.6,
|
||||
"avg_eval_duration_s": 47.8,
|
||||
"per_repo": {
|
||||
"astropy/astropy": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 1,
|
||||
"resolve_pct": 16.7
|
||||
},
|
||||
"django/django": {
|
||||
"total": 114,
|
||||
"patched": 114,
|
||||
"resolved": 88,
|
||||
"resolve_pct": 77.2
|
||||
},
|
||||
"matplotlib/matplotlib": {
|
||||
"total": 23,
|
||||
"patched": 23,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"mwaskom/seaborn": {
|
||||
"total": 4,
|
||||
"patched": 4,
|
||||
"resolved": 1,
|
||||
"resolve_pct": 25.0
|
||||
},
|
||||
"pallets/flask": {
|
||||
"total": 3,
|
||||
"patched": 3,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"psf/requests": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 5,
|
||||
"resolve_pct": 83.3
|
||||
},
|
||||
"pydata/xarray": {
|
||||
"total": 5,
|
||||
"patched": 5,
|
||||
"resolved": 2,
|
||||
"resolve_pct": 40.0
|
||||
},
|
||||
"pylint-dev/pylint": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"pytest-dev/pytest": {
|
||||
"total": 17,
|
||||
"patched": 17,
|
||||
"resolved": 12,
|
||||
"resolve_pct": 70.6
|
||||
},
|
||||
"scikit-learn/scikit-learn": {
|
||||
"total": 23,
|
||||
"patched": 23,
|
||||
"resolved": 15,
|
||||
"resolve_pct": 65.2
|
||||
},
|
||||
"sphinx-doc/sphinx": {
|
||||
"total": 16,
|
||||
"patched": 16,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"sympy/sympy": {
|
||||
"total": 77,
|
||||
"patched": 77,
|
||||
"resolved": 50,
|
||||
"resolve_pct": 64.9
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue