mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-10-07 03:00:29 +00:00
Record Sonnet 4.6 20min timeout: 57.7% on SWE-Bench-Lite ($55.22)
20min timeout vs 10min: 173 vs 167 resolved (+6), patch rate 99% vs 94%. Also fixes: revert to v4 snapshots, concurrency default to 100, preflight uses actual 4 CPU per sandbox. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp
parent
0cc02c294d
commit
165742d7fa
4 changed files with 448 additions and 0 deletions
|
|
@ -1,4 +1,17 @@
|
|||
[
|
||||
{
|
||||
"run_name": "sonnet-20min-20260316",
|
||||
"date": "2026-03-16",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"provider": "anthropic",
|
||||
"resolved": 173,
|
||||
"total": 300,
|
||||
"resolve_pct": 57.7,
|
||||
"patched": 297,
|
||||
"patch_pct": 99.0,
|
||||
"total_cost_usd": 55.22,
|
||||
"avg_cost_usd": 0.1841
|
||||
},
|
||||
{
|
||||
"run_name": "sonnet-baseline-20260316",
|
||||
"date": "2026-03-16",
|
||||
|
|
|
|||
38
evals/swe-bench/scoreboard/sonnet-20min-20260316/README.md
Normal file
38
evals/swe-bench/scoreboard/sonnet-20min-20260316/README.md
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
# sonnet-20min-20260316
|
||||
|
||||
**Date:** 2026-03-16
|
||||
**Model:** claude-sonnet-4-6 (anthropic)
|
||||
**Fabro:** fabro 0.5.0 (4c6ad4d 2026-03-16)
|
||||
|
||||
## Description
|
||||
|
||||
Sonnet 4.6, 20min timeout, 4 CPU / 8 GB, default prompt
|
||||
|
||||
## Results
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Instances | 300 |
|
||||
| Patched | 297 (99.0%) |
|
||||
| **Resolved** | **173 (57.7%)** |
|
||||
| Total gen cost | $55.22 |
|
||||
| Avg gen cost | $0.1841/instance |
|
||||
| Gen wall time | 1466.0s |
|
||||
| Eval wall time | 682.3s |
|
||||
|
||||
## Per-repo breakdown
|
||||
|
||||
| Repo | Resolved | Total | Rate |
|
||||
|------|----------|-------|------|
|
||||
| astropy/astropy | 1 | 6 | 16.7% |
|
||||
| django/django | 88 | 114 | 77.2% |
|
||||
| matplotlib/matplotlib | 0 | 23 | 0.0% |
|
||||
| mwaskom/seaborn | 3 | 4 | 75.0% |
|
||||
| pallets/flask | 0 | 3 | 0.0% |
|
||||
| psf/requests | 5 | 6 | 83.3% |
|
||||
| pydata/xarray | 1 | 5 | 20.0% |
|
||||
| pylint-dev/pylint | 0 | 6 | 0.0% |
|
||||
| pytest-dev/pytest | 11 | 17 | 64.7% |
|
||||
| scikit-learn/scikit-learn | 15 | 23 | 65.2% |
|
||||
| sphinx-doc/sphinx | 0 | 16 | 0.0% |
|
||||
| sympy/sympy | 49 | 77 | 63.6% |
|
||||
300
evals/swe-bench/scoreboard/sonnet-20min-20260316/instances.jsonl
Normal file
300
evals/swe-bench/scoreboard/sonnet-20min-20260316/instances.jsonl
Normal file
|
|
@ -0,0 +1,300 @@
|
|||
{"instance_id": "astropy__astropy-12907", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 106.8, "eval_duration_s": 56.1, "gen_cost_usd": 0.064464}
|
||||
{"instance_id": "astropy__astropy-14182", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 164.7, "eval_duration_s": 64.8, "gen_cost_usd": 0.148908}
|
||||
{"instance_id": "astropy__astropy-14365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 97.7, "eval_duration_s": 42.3, "gen_cost_usd": 0.078402}
|
||||
{"instance_id": "astropy__astropy-14995", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 71.1, "eval_duration_s": 47.7, "gen_cost_usd": 0.037638}
|
||||
{"instance_id": "astropy__astropy-6938", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.9, "eval_duration_s": 81.7, "gen_cost_usd": 0.012978}
|
||||
{"instance_id": "astropy__astropy-7746", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 149.2, "eval_duration_s": 84.5, "gen_cost_usd": 0.058356}
|
||||
{"instance_id": "django__django-10914", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 136.8, "eval_duration_s": 25.4, "gen_cost_usd": 0.118077}
|
||||
{"instance_id": "django__django-10924", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.7, "eval_duration_s": 25.3, "gen_cost_usd": 0.134157}
|
||||
{"instance_id": "django__django-11001", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 161.8, "eval_duration_s": 20.9, "gen_cost_usd": 0.121002}
|
||||
{"instance_id": "django__django-11019", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 783.1, "eval_duration_s": 607.9, "gen_cost_usd": 0.576756}
|
||||
{"instance_id": "django__django-11039", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 121.5, "eval_duration_s": 24.9, "gen_cost_usd": 0.141678}
|
||||
{"instance_id": "django__django-11049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.4, "eval_duration_s": 21.6, "gen_cost_usd": 0.056811}
|
||||
{"instance_id": "django__django-11099", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 25.6, "eval_duration_s": 104.9, "gen_cost_usd": 0.013665}
|
||||
{"instance_id": "django__django-11133", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 42.1, "eval_duration_s": 86.4, "gen_cost_usd": 0.018816}
|
||||
{"instance_id": "django__django-11179", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 29.2, "eval_duration_s": 20.7, "gen_cost_usd": 0.017781}
|
||||
{"instance_id": "django__django-11283", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 136.9, "eval_duration_s": 21.8, "gen_cost_usd": 0.136041}
|
||||
{"instance_id": "django__django-11422", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 105.8, "eval_duration_s": 25.0, "gen_cost_usd": 0.121464}
|
||||
{"instance_id": "django__django-11564", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 266.0, "eval_duration_s": 17.6, "gen_cost_usd": 0.337134}
|
||||
{"instance_id": "django__django-11583", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 25.1, "eval_duration_s": 22.3, "gen_cost_usd": 0.018714}
|
||||
{"instance_id": "django__django-11620", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 126.9, "eval_duration_s": 23.1, "gen_cost_usd": 0.126471}
|
||||
{"instance_id": "django__django-11630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 238.1, "eval_duration_s": 35.7, "gen_cost_usd": 0.259635}
|
||||
{"instance_id": "django__django-11742", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 232.0, "eval_duration_s": 64.7, "gen_cost_usd": 0.268938}
|
||||
{"instance_id": "django__django-11797", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 79.5, "eval_duration_s": 34.0, "gen_cost_usd": 0.066111}
|
||||
{"instance_id": "django__django-11815", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 146.8, "eval_duration_s": 22.0, "gen_cost_usd": 0.134709}
|
||||
{"instance_id": "django__django-11848", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 83.2, "eval_duration_s": 21.5, "gen_cost_usd": 0.069561}
|
||||
{"instance_id": "django__django-11905", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 225.0, "eval_duration_s": 20.8, "gen_cost_usd": 0.132522}
|
||||
{"instance_id": "django__django-11910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 527.1, "eval_duration_s": 22.7, "gen_cost_usd": 0.708078}
|
||||
{"instance_id": "django__django-11964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 94.7, "eval_duration_s": 21.3, "gen_cost_usd": 0.0783}
|
||||
{"instance_id": "django__django-11999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 80.2, "eval_duration_s": 33.1, "gen_cost_usd": 0.063777}
|
||||
{"instance_id": "django__django-12113", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.0, "eval_duration_s": 22.5, "gen_cost_usd": 0.164424}
|
||||
{"instance_id": "django__django-12125", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.3, "eval_duration_s": 24.6, "gen_cost_usd": 0.104868}
|
||||
{"instance_id": "django__django-12184", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 145.3, "eval_duration_s": 24.6, "gen_cost_usd": 0.155193}
|
||||
{"instance_id": "django__django-12284", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.3, "eval_duration_s": 24.0, "gen_cost_usd": 0.106953}
|
||||
{"instance_id": "django__django-12286", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 60.7, "eval_duration_s": 21.1, "gen_cost_usd": 0.05334}
|
||||
{"instance_id": "django__django-12308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 305.2, "eval_duration_s": 16.6, "gen_cost_usd": 0.18663}
|
||||
{"instance_id": "django__django-12453", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.3, "eval_duration_s": 77.9, "gen_cost_usd": 0.114987}
|
||||
{"instance_id": "django__django-12470", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 588.1, "eval_duration_s": 18.8, "gen_cost_usd": 0.611658}
|
||||
{"instance_id": "django__django-12497", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.1, "eval_duration_s": 28.6, "gen_cost_usd": 0.044742}
|
||||
{"instance_id": "django__django-12589", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 898.9, "eval_duration_s": 17.9, "gen_cost_usd": 0.996936}
|
||||
{"instance_id": "django__django-12700", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 59.6, "eval_duration_s": 22.6, "gen_cost_usd": 0.039462}
|
||||
{"instance_id": "django__django-12708", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.9, "eval_duration_s": 25.3, "gen_cost_usd": 0.109767}
|
||||
{"instance_id": "django__django-12747", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 207.4, "eval_duration_s": 17.1, "gen_cost_usd": 0.199233}
|
||||
{"instance_id": "django__django-12856", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.5, "eval_duration_s": 21.5, "gen_cost_usd": 0.12063}
|
||||
{"instance_id": "django__django-12908", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 79.3, "eval_duration_s": 55.4, "gen_cost_usd": 0.076269}
|
||||
{"instance_id": "django__django-12915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 121.1, "eval_duration_s": 38.9, "gen_cost_usd": 0.111195}
|
||||
{"instance_id": "django__django-12983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.7, "eval_duration_s": 21.8, "gen_cost_usd": 0.062208}
|
||||
{"instance_id": "django__django-13028", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.8, "eval_duration_s": 23.4, "gen_cost_usd": 0.015219}
|
||||
{"instance_id": "django__django-13033", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 88.1, "eval_duration_s": 24.9, "gen_cost_usd": 0.050868}
|
||||
{"instance_id": "django__django-13158", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 175.5, "eval_duration_s": 150.1, "gen_cost_usd": 0.19146}
|
||||
{"instance_id": "django__django-13220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 174.4, "eval_duration_s": 22.3, "gen_cost_usd": 0.147219}
|
||||
{"instance_id": "django__django-13230", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 99.7, "eval_duration_s": 61.9, "gen_cost_usd": 0.126726}
|
||||
{"instance_id": "django__django-13265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 447.1, "eval_duration_s": 16.9, "gen_cost_usd": 0.499551}
|
||||
{"instance_id": "django__django-13315", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 180.5, "eval_duration_s": 18.1, "gen_cost_usd": 0.182493}
|
||||
{"instance_id": "django__django-13321", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 43.0, "eval_duration_s": 25.2, "gen_cost_usd": 0.030351}
|
||||
{"instance_id": "django__django-13401", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.6, "eval_duration_s": 24.9, "gen_cost_usd": 0.065265}
|
||||
{"instance_id": "django__django-13447", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.5, "eval_duration_s": 27.2, "gen_cost_usd": 0.079938}
|
||||
{"instance_id": "django__django-13448", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 165.0, "eval_duration_s": 26.1, "gen_cost_usd": 0.155982}
|
||||
{"instance_id": "django__django-13551", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.0, "eval_duration_s": 21.0, "gen_cost_usd": 0.063585}
|
||||
{"instance_id": "django__django-13590", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 40.6, "eval_duration_s": 20.7, "gen_cost_usd": 0.019314}
|
||||
{"instance_id": "django__django-13658", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 36.5, "eval_duration_s": 47.4, "gen_cost_usd": 0.018885}
|
||||
{"instance_id": "django__django-13660", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 50.2, "eval_duration_s": 23.0, "gen_cost_usd": 0.044526}
|
||||
{"instance_id": "django__django-13710", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 247.2, "eval_duration_s": 22.2, "gen_cost_usd": 0.280179}
|
||||
{"instance_id": "django__django-13757", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 125.1, "eval_duration_s": 39.9, "gen_cost_usd": 0.1221}
|
||||
{"instance_id": "django__django-13768", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 81.1, "eval_duration_s": 21.6, "gen_cost_usd": 0.061737}
|
||||
{"instance_id": "django__django-13925", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 269.1, "eval_duration_s": 17.8, "gen_cost_usd": 0.279765}
|
||||
{"instance_id": "django__django-13933", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 249.7, "eval_duration_s": 21.5, "gen_cost_usd": 0.292293}
|
||||
{"instance_id": "django__django-13964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 41.2, "eval_duration_s": 23.4, "gen_cost_usd": 0.02547}
|
||||
{"instance_id": "django__django-14016", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 280.0, "eval_duration_s": 20.8, "gen_cost_usd": 0.329361}
|
||||
{"instance_id": "django__django-14017", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 330.9, "eval_duration_s": 21.4, "gen_cost_usd": 0.397476}
|
||||
{"instance_id": "django__django-14155", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 88.5, "eval_duration_s": 28.8, "gen_cost_usd": 0.083484}
|
||||
{"instance_id": "django__django-14238", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.9, "eval_duration_s": 38.7, "gen_cost_usd": 0.044787}
|
||||
{"instance_id": "django__django-14382", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 129.2, "eval_duration_s": 55.2, "gen_cost_usd": 0.013203}
|
||||
{"instance_id": "django__django-14411", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 65.1, "eval_duration_s": 25.0, "gen_cost_usd": 0.056901}
|
||||
{"instance_id": "django__django-14534", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 36.6, "eval_duration_s": 24.3, "gen_cost_usd": 0.017343}
|
||||
{"instance_id": "django__django-14580", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 31.3, "eval_duration_s": 24.2, "gen_cost_usd": 0.015474}
|
||||
{"instance_id": "django__django-14608", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 160.7, "eval_duration_s": 46.9, "gen_cost_usd": 0.188115}
|
||||
{"instance_id": "django__django-14667", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 184.3, "eval_duration_s": 20.8, "gen_cost_usd": 0.19143}
|
||||
{"instance_id": "django__django-14672", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.7, "eval_duration_s": 28.9, "gen_cost_usd": 0.032706}
|
||||
{"instance_id": "django__django-14730", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 290.3, "eval_duration_s": 38.3, "gen_cost_usd": 0.306333}
|
||||
{"instance_id": "django__django-14752", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.3, "eval_duration_s": 26.0, "gen_cost_usd": 0.038664}
|
||||
{"instance_id": "django__django-14787", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 65.5, "eval_duration_s": 37.9, "gen_cost_usd": 0.049047}
|
||||
{"instance_id": "django__django-14855", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 32.5, "eval_duration_s": 135.5, "gen_cost_usd": 0.020211}
|
||||
{"instance_id": "django__django-14915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.9, "eval_duration_s": 24.0, "gen_cost_usd": 0.016164}
|
||||
{"instance_id": "django__django-14997", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 217.3, "eval_duration_s": 51.7, "gen_cost_usd": 0.24462}
|
||||
{"instance_id": "django__django-14999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.8, "eval_duration_s": 27.2, "gen_cost_usd": 0.163908}
|
||||
{"instance_id": "django__django-15061", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 138.0, "eval_duration_s": 26.1, "gen_cost_usd": 0.115611}
|
||||
{"instance_id": "django__django-15202", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 165.4, "eval_duration_s": 26.6, "gen_cost_usd": 0.163416}
|
||||
{"instance_id": "django__django-15213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 302.3, "eval_duration_s": 19.6, "gen_cost_usd": 0.33372}
|
||||
{"instance_id": "django__django-15252", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 1106.4, "eval_duration_s": 19.3, "gen_cost_usd": 1.374828}
|
||||
{"instance_id": "django__django-15320", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.6, "eval_duration_s": 28.4, "gen_cost_usd": 0.075711}
|
||||
{"instance_id": "django__django-15347", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 52.4, "eval_duration_s": 27.0, "gen_cost_usd": 0.035637}
|
||||
{"instance_id": "django__django-15388", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.9, "eval_duration_s": 26.4, "gen_cost_usd": 0.048576}
|
||||
{"instance_id": "django__django-15400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.0, "eval_duration_s": 23.5, "gen_cost_usd": 0.060801}
|
||||
{"instance_id": "django__django-15498", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 108.4, "eval_duration_s": 50.8, "gen_cost_usd": 0.098769}
|
||||
{"instance_id": "django__django-15695", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 429.9, "eval_duration_s": 39.8, "gen_cost_usd": 0.473772}
|
||||
{"instance_id": "django__django-15738", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 944.5, "eval_duration_s": 19.9, "gen_cost_usd": 1.124073}
|
||||
{"instance_id": "django__django-15781", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 36.8, "eval_duration_s": 23.4, "gen_cost_usd": 0.021132}
|
||||
{"instance_id": "django__django-15789", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 43.5, "eval_duration_s": 25.6, "gen_cost_usd": 0.031632}
|
||||
{"instance_id": "django__django-15790", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.0, "eval_duration_s": 26.0, "gen_cost_usd": 0.115029}
|
||||
{"instance_id": "django__django-15814", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 74.7, "eval_duration_s": 25.6, "gen_cost_usd": 0.056643}
|
||||
{"instance_id": "django__django-15819", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 249.1, "eval_duration_s": 19.1, "gen_cost_usd": 0.335946}
|
||||
{"instance_id": "django__django-15851", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.0, "eval_duration_s": 29.7, "gen_cost_usd": 0.047985}
|
||||
{"instance_id": "django__django-15902", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 101.8, "eval_duration_s": 27.1, "gen_cost_usd": 0.08235}
|
||||
{"instance_id": "django__django-15996", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.6, "eval_duration_s": 17.5, "gen_cost_usd": 0.204909}
|
||||
{"instance_id": "django__django-16041", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 144.1, "eval_duration_s": 25.9, "gen_cost_usd": 0.038502}
|
||||
{"instance_id": "django__django-16046", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 65.0, "eval_duration_s": 27.8, "gen_cost_usd": 0.05763}
|
||||
{"instance_id": "django__django-16139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.5, "eval_duration_s": 23.4, "gen_cost_usd": 0.017829}
|
||||
{"instance_id": "django__django-16229", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 924.9, "eval_duration_s": 18.3, "gen_cost_usd": 1.006125}
|
||||
{"instance_id": "django__django-16255", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 39.9, "eval_duration_s": 25.6, "gen_cost_usd": 0.013146}
|
||||
{"instance_id": "django__django-16379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 31.8, "eval_duration_s": 98.6, "gen_cost_usd": 0.026895}
|
||||
{"instance_id": "django__django-16400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.9, "eval_duration_s": 25.0, "gen_cost_usd": 0.064098}
|
||||
{"instance_id": "django__django-16408", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 132.7, "eval_duration_s": 45.8, "gen_cost_usd": 0.148632}
|
||||
{"instance_id": "django__django-16527", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 28.4, "eval_duration_s": 27.3, "gen_cost_usd": 0.013221}
|
||||
{"instance_id": "django__django-16595", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 63.1, "eval_duration_s": 26.4, "gen_cost_usd": 0.06387}
|
||||
{"instance_id": "django__django-16816", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 224.9, "eval_duration_s": 23.8, "gen_cost_usd": 0.267636}
|
||||
{"instance_id": "django__django-16820", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 393.5, "eval_duration_s": 20.5, "gen_cost_usd": 0.541443}
|
||||
{"instance_id": "django__django-16873", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 58.8, "eval_duration_s": 26.6, "gen_cost_usd": 0.053043}
|
||||
{"instance_id": "django__django-16910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 495.2, "eval_duration_s": 19.5, "gen_cost_usd": 0.58017}
|
||||
{"instance_id": "django__django-17051", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 122.6, "eval_duration_s": 24.6, "gen_cost_usd": 0.137709}
|
||||
{"instance_id": "django__django-17087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 81.8, "eval_duration_s": 28.3, "gen_cost_usd": 0.063789}
|
||||
{"instance_id": "matplotlib__matplotlib-18869", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 289.3, "eval_duration_s": 89.9, "gen_cost_usd": 0.166473}
|
||||
{"instance_id": "matplotlib__matplotlib-22711", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 196.1, "eval_duration_s": 107.9, "gen_cost_usd": 0.081615}
|
||||
{"instance_id": "matplotlib__matplotlib-22835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 401.4, "eval_duration_s": 85.4, "gen_cost_usd": 0.139341}
|
||||
{"instance_id": "matplotlib__matplotlib-23299", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 405.3, "eval_duration_s": 71.5, "gen_cost_usd": 0.309069}
|
||||
{"instance_id": "matplotlib__matplotlib-23314", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 112.8, "eval_duration_s": 71.4, "gen_cost_usd": 0.031626}
|
||||
{"instance_id": "matplotlib__matplotlib-23476", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 400.6, "eval_duration_s": 83.1, "gen_cost_usd": 0.279963}
|
||||
{"instance_id": "matplotlib__matplotlib-23562", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 233.3, "eval_duration_s": 135.2, "gen_cost_usd": 0.093468}
|
||||
{"instance_id": "matplotlib__matplotlib-23563", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 328.5, "eval_duration_s": 291.7, "gen_cost_usd": 0.365733}
|
||||
{"instance_id": "matplotlib__matplotlib-23913", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 218.2, "eval_duration_s": 81.6, "gen_cost_usd": 0.122436}
|
||||
{"instance_id": "matplotlib__matplotlib-23964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 387.2, "eval_duration_s": 85.9, "gen_cost_usd": 0.02988}
|
||||
{"instance_id": "matplotlib__matplotlib-23987", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 271.8, "eval_duration_s": 92.2, "gen_cost_usd": 0.089511}
|
||||
{"instance_id": "matplotlib__matplotlib-24149", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 174.2, "eval_duration_s": 186.8, "gen_cost_usd": 0.037743}
|
||||
{"instance_id": "matplotlib__matplotlib-24265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 212.6, "eval_duration_s": 83.6, "gen_cost_usd": 0.153486}
|
||||
{"instance_id": "matplotlib__matplotlib-24334", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 237.7, "eval_duration_s": 85.3, "gen_cost_usd": 0.059715}
|
||||
{"instance_id": "matplotlib__matplotlib-24970", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 283.9, "eval_duration_s": 268.5, "gen_cost_usd": 0.071205}
|
||||
{"instance_id": "matplotlib__matplotlib-25079", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 1211.9, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "matplotlib__matplotlib-25311", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 693.0, "eval_duration_s": 109.6, "gen_cost_usd": 0.600099}
|
||||
{"instance_id": "matplotlib__matplotlib-25332", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 250.7, "eval_duration_s": 373.9, "gen_cost_usd": 0.129576}
|
||||
{"instance_id": "matplotlib__matplotlib-25433", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 552.5, "eval_duration_s": 83.1, "gen_cost_usd": 0.488568}
|
||||
{"instance_id": "matplotlib__matplotlib-25442", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 735.9, "eval_duration_s": 87.7, "gen_cost_usd": 0.611346}
|
||||
{"instance_id": "matplotlib__matplotlib-25498", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 421.1, "eval_duration_s": 90.8, "gen_cost_usd": 0.28884}
|
||||
{"instance_id": "matplotlib__matplotlib-26011", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 248.2, "eval_duration_s": 193.0, "gen_cost_usd": 0.100854}
|
||||
{"instance_id": "matplotlib__matplotlib-26020", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 515.1, "eval_duration_s": 77.8, "gen_cost_usd": 0.081144}
|
||||
{"instance_id": "mwaskom__seaborn-2848", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 443.3, "eval_duration_s": 31.5, "gen_cost_usd": 0.477777}
|
||||
{"instance_id": "mwaskom__seaborn-3010", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 57.3, "eval_duration_s": 8.9, "gen_cost_usd": 0.06033}
|
||||
{"instance_id": "mwaskom__seaborn-3190", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 119.0, "eval_duration_s": 10.5, "gen_cost_usd": 0.113079}
|
||||
{"instance_id": "mwaskom__seaborn-3407", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 273.2, "eval_duration_s": 79.9, "gen_cost_usd": 0.263928}
|
||||
{"instance_id": "pallets__flask-4045", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 76.0, "eval_duration_s": 9.1, "gen_cost_usd": 0.07935}
|
||||
{"instance_id": "pallets__flask-4992", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 50.8, "eval_duration_s": 5.3, "gen_cost_usd": 0.049701}
|
||||
{"instance_id": "pallets__flask-5063", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 116.6, "eval_duration_s": 8.8, "gen_cost_usd": 0.138921}
|
||||
{"instance_id": "psf__requests-1963", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.3, "eval_duration_s": 40.6, "gen_cost_usd": 0.064146}
|
||||
{"instance_id": "psf__requests-2148", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 74.6, "eval_duration_s": 20.8, "gen_cost_usd": 0.072666}
|
||||
{"instance_id": "psf__requests-2317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 259.6, "eval_duration_s": 608.0, "gen_cost_usd": 0.073623}
|
||||
{"instance_id": "psf__requests-2674", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 128.8, "eval_duration_s": 47.2, "gen_cost_usd": 0.168309}
|
||||
{"instance_id": "psf__requests-3362", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 48.8, "eval_duration_s": 6.3, "gen_cost_usd": 0.058566}
|
||||
{"instance_id": "psf__requests-863", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.6, "eval_duration_s": 41.4, "gen_cost_usd": 0.04485}
|
||||
{"instance_id": "pydata__xarray-3364", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 1212.0, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "pydata__xarray-4094", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 106.3, "eval_duration_s": 24.5, "gen_cost_usd": 0.09525}
|
||||
{"instance_id": "pydata__xarray-4248", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 124.3, "eval_duration_s": 10.9, "gen_cost_usd": 0.169662}
|
||||
{"instance_id": "pydata__xarray-4493", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 82.4, "eval_duration_s": 58.7, "gen_cost_usd": 0.075021}
|
||||
{"instance_id": "pydata__xarray-5131", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.7, "eval_duration_s": 10.2, "gen_cost_usd": 0.032568}
|
||||
{"instance_id": "pylint-dev__pylint-5859", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 94.9, "eval_duration_s": 8.9, "gen_cost_usd": 0.136593}
|
||||
{"instance_id": "pylint-dev__pylint-6506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 104.8, "eval_duration_s": 8.4, "gen_cost_usd": 0.146466}
|
||||
{"instance_id": "pylint-dev__pylint-7080", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 86.0, "eval_duration_s": 14.0, "gen_cost_usd": 0.123609}
|
||||
{"instance_id": "pylint-dev__pylint-7114", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 499.1, "eval_duration_s": 5.5, "gen_cost_usd": 0.63648}
|
||||
{"instance_id": "pylint-dev__pylint-7228", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 226.3, "eval_duration_s": 5.5, "gen_cost_usd": 0.251193}
|
||||
{"instance_id": "pylint-dev__pylint-7993", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 115.0, "eval_duration_s": 5.4, "gen_cost_usd": 0.131088}
|
||||
{"instance_id": "pytest-dev__pytest-11143", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 42.9, "eval_duration_s": 10.9, "gen_cost_usd": 0.046716}
|
||||
{"instance_id": "pytest-dev__pytest-11148", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 294.9, "eval_duration_s": 13.8, "gen_cost_usd": 0.420648}
|
||||
{"instance_id": "pytest-dev__pytest-5103", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 962.4, "eval_duration_s": 12.1, "gen_cost_usd": 1.167606}
|
||||
{"instance_id": "pytest-dev__pytest-5221", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 191.1, "eval_duration_s": 13.2, "gen_cost_usd": 0.187368}
|
||||
{"instance_id": "pytest-dev__pytest-5227", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 18.1, "eval_duration_s": 9.6, "gen_cost_usd": 0.014136}
|
||||
{"instance_id": "pytest-dev__pytest-5413", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.4, "eval_duration_s": 26.6, "gen_cost_usd": 0.055299}
|
||||
{"instance_id": "pytest-dev__pytest-5495", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 95.0, "eval_duration_s": 12.0, "gen_cost_usd": 0.07128}
|
||||
{"instance_id": "pytest-dev__pytest-5692", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 46.3, "eval_duration_s": 24.8, "gen_cost_usd": 0.077898}
|
||||
{"instance_id": "pytest-dev__pytest-6116", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 25.4, "eval_duration_s": 11.6, "gen_cost_usd": 0.02847}
|
||||
{"instance_id": "pytest-dev__pytest-7168", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 48.2, "eval_duration_s": 9.1, "gen_cost_usd": 0.033897}
|
||||
{"instance_id": "pytest-dev__pytest-7220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 239.7, "eval_duration_s": 8.8, "gen_cost_usd": 0.259536}
|
||||
{"instance_id": "pytest-dev__pytest-7373", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.2, "eval_duration_s": 11.1, "gen_cost_usd": 0.057195}
|
||||
{"instance_id": "pytest-dev__pytest-7432", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 62.9, "eval_duration_s": 10.7, "gen_cost_usd": 0.078171}
|
||||
{"instance_id": "pytest-dev__pytest-7490", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 179.9, "eval_duration_s": 12.0, "gen_cost_usd": 0.236649}
|
||||
{"instance_id": "pytest-dev__pytest-8365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 47.4, "eval_duration_s": 28.7, "gen_cost_usd": 0.060828}
|
||||
{"instance_id": "pytest-dev__pytest-8906", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 39.9, "eval_duration_s": 10.6, "gen_cost_usd": 0.047148}
|
||||
{"instance_id": "pytest-dev__pytest-9359", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 232.8, "eval_duration_s": 24.4, "gen_cost_usd": 0.332877}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10297", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 184.2, "eval_duration_s": 136.9, "gen_cost_usd": 0.053574}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10508", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 146.0, "eval_duration_s": 130.5, "gen_cost_usd": 0.016671}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10949", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 222.4, "eval_duration_s": 130.3, "gen_cost_usd": 0.111537}
|
||||
{"instance_id": "scikit-learn__scikit-learn-11040", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 241.5, "eval_duration_s": 164.6, "gen_cost_usd": 0.093969}
|
||||
{"instance_id": "scikit-learn__scikit-learn-11281", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 281.0, "eval_duration_s": 129.5, "gen_cost_usd": 0.129891}
|
||||
{"instance_id": "scikit-learn__scikit-learn-12471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 176.3, "eval_duration_s": 144.1, "gen_cost_usd": 0.048426}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13142", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.6, "eval_duration_s": 156.7, "gen_cost_usd": 0.070095}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13241", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 445.7, "eval_duration_s": 173.5, "gen_cost_usd": 0.374919}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13439", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 216.1, "eval_duration_s": 174.9, "gen_cost_usd": 0.027744}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13496", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 271.6, "eval_duration_s": 178.5, "gen_cost_usd": 0.121494}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13497", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 211.7, "eval_duration_s": 180.1, "gen_cost_usd": 0.050214}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13584", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 277.8, "eval_duration_s": 165.8, "gen_cost_usd": 0.149271}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13779", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 217.3, "eval_duration_s": 213.8, "gen_cost_usd": 0.019875}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 462.8, "eval_duration_s": 210.5, "gen_cost_usd": 0.349221}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14092", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 278.9, "eval_duration_s": 331.6, "gen_cost_usd": 0.118077}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14894", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 230.9, "eval_duration_s": 190.4, "gen_cost_usd": 0.041496}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 334.9, "eval_duration_s": 206.8, "gen_cost_usd": 0.17247}
|
||||
{"instance_id": "scikit-learn__scikit-learn-15512", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 290.9, "eval_duration_s": 208.5, "gen_cost_usd": 0.134169}
|
||||
{"instance_id": "scikit-learn__scikit-learn-15535", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 235.1, "eval_duration_s": 189.3, "gen_cost_usd": 0.053409}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25500", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 347.7, "eval_duration_s": 13.3, "gen_cost_usd": 0.094563}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25570", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 486.2, "eval_duration_s": 14.1, "gen_cost_usd": 0.289725}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25638", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 467.2, "eval_duration_s": 12.8, "gen_cost_usd": 0.281478}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 336.2, "eval_duration_s": 20.0, "gen_cost_usd": 0.092697}
|
||||
{"instance_id": "sphinx-doc__sphinx-10325", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 866.5, "eval_duration_s": 15.6, "gen_cost_usd": 0.661449}
|
||||
{"instance_id": "sphinx-doc__sphinx-10451", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 409.7, "eval_duration_s": 14.8, "gen_cost_usd": 0.550653}
|
||||
{"instance_id": "sphinx-doc__sphinx-11445", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 161.9, "eval_duration_s": 13.4, "gen_cost_usd": 0.1545}
|
||||
{"instance_id": "sphinx-doc__sphinx-7686", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 116.2, "eval_duration_s": 15.3, "gen_cost_usd": 0.138948}
|
||||
{"instance_id": "sphinx-doc__sphinx-7738", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 288.7, "eval_duration_s": 22.8, "gen_cost_usd": 0.315528}
|
||||
{"instance_id": "sphinx-doc__sphinx-7975", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 116.6, "eval_duration_s": 14.5, "gen_cost_usd": 0.154194}
|
||||
{"instance_id": "sphinx-doc__sphinx-8273", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 87.1, "eval_duration_s": 15.2, "gen_cost_usd": 0.113403}
|
||||
{"instance_id": "sphinx-doc__sphinx-8282", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 182.8, "eval_duration_s": 15.1, "gen_cost_usd": 0.239826}
|
||||
{"instance_id": "sphinx-doc__sphinx-8435", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 341.8, "eval_duration_s": 16.3, "gen_cost_usd": 0.44871}
|
||||
{"instance_id": "sphinx-doc__sphinx-8474", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 1124.8, "eval_duration_s": 14.1, "gen_cost_usd": 1.080222}
|
||||
{"instance_id": "sphinx-doc__sphinx-8506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 200.4, "eval_duration_s": 15.0, "gen_cost_usd": 0.244776}
|
||||
{"instance_id": "sphinx-doc__sphinx-8595", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 40.6, "eval_duration_s": 21.9, "gen_cost_usd": 0.046731}
|
||||
{"instance_id": "sphinx-doc__sphinx-8627", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 300.5, "eval_duration_s": 16.7, "gen_cost_usd": 0.427608}
|
||||
{"instance_id": "sphinx-doc__sphinx-8713", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 48.7, "eval_duration_s": 14.5, "gen_cost_usd": 0.052143}
|
||||
{"instance_id": "sphinx-doc__sphinx-8721", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 33.8, "eval_duration_s": 14.9, "gen_cost_usd": 0.045882}
|
||||
{"instance_id": "sphinx-doc__sphinx-8801", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 301.0, "eval_duration_s": 14.5, "gen_cost_usd": 0.365451}
|
||||
{"instance_id": "sympy__sympy-11400", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 91.8, "eval_duration_s": 20.8, "gen_cost_usd": 0.114477}
|
||||
{"instance_id": "sympy__sympy-11870", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 195.0, "eval_duration_s": 516.3, "gen_cost_usd": 0.191145}
|
||||
{"instance_id": "sympy__sympy-11897", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 540.2, "eval_duration_s": 23.1, "gen_cost_usd": 0.632043}
|
||||
{"instance_id": "sympy__sympy-12171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 184.4, "eval_duration_s": 20.7, "gen_cost_usd": 0.159717}
|
||||
{"instance_id": "sympy__sympy-12236", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 475.8, "eval_duration_s": 76.3, "gen_cost_usd": 0.476478}
|
||||
{"instance_id": "sympy__sympy-12419", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 106.1, "eval_duration_s": 21.0, "gen_cost_usd": 0.116844}
|
||||
{"instance_id": "sympy__sympy-12454", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 63.6, "eval_duration_s": 33.2, "gen_cost_usd": 0.049533}
|
||||
{"instance_id": "sympy__sympy-12481", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 135.3, "eval_duration_s": 24.4, "gen_cost_usd": 0.113361}
|
||||
{"instance_id": "sympy__sympy-13031", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 203.7, "eval_duration_s": 23.1, "gen_cost_usd": 0.175527}
|
||||
{"instance_id": "sympy__sympy-13043", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 197.7, "eval_duration_s": 58.3, "gen_cost_usd": 0.090087}
|
||||
{"instance_id": "sympy__sympy-13146", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 1200.3, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "sympy__sympy-13177", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 58.5, "eval_duration_s": 21.1, "gen_cost_usd": 0.047283}
|
||||
{"instance_id": "sympy__sympy-13437", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 79.8, "eval_duration_s": 39.7, "gen_cost_usd": 0.071394}
|
||||
{"instance_id": "sympy__sympy-13471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 96.6, "eval_duration_s": 21.9, "gen_cost_usd": 0.082665}
|
||||
{"instance_id": "sympy__sympy-13480", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 38.9, "eval_duration_s": 25.7, "gen_cost_usd": 0.014853}
|
||||
{"instance_id": "sympy__sympy-13647", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.3, "eval_duration_s": 109.0, "gen_cost_usd": 0.019932}
|
||||
{"instance_id": "sympy__sympy-13773", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 112.9, "eval_duration_s": 21.8, "gen_cost_usd": 0.085689}
|
||||
{"instance_id": "sympy__sympy-13895", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 852.9, "eval_duration_s": 20.5, "gen_cost_usd": 1.095765}
|
||||
{"instance_id": "sympy__sympy-13915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 282.6, "eval_duration_s": 22.6, "gen_cost_usd": 0.279147}
|
||||
{"instance_id": "sympy__sympy-13971", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.0, "eval_duration_s": 27.8, "gen_cost_usd": 0.114513}
|
||||
{"instance_id": "sympy__sympy-14024", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 469.1, "eval_duration_s": 26.8, "gen_cost_usd": 0.62058}
|
||||
{"instance_id": "sympy__sympy-14308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 338.4, "eval_duration_s": 24.9, "gen_cost_usd": 0.414582}
|
||||
{"instance_id": "sympy__sympy-14317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 216.2, "eval_duration_s": 30.4, "gen_cost_usd": 0.247515}
|
||||
{"instance_id": "sympy__sympy-14396", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 93.1, "eval_duration_s": 19.9, "gen_cost_usd": 0.120039}
|
||||
{"instance_id": "sympy__sympy-14774", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 35.4, "eval_duration_s": 27.5, "gen_cost_usd": 0.015927}
|
||||
{"instance_id": "sympy__sympy-14817", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 192.1, "eval_duration_s": 28.3, "gen_cost_usd": 0.187683}
|
||||
{"instance_id": "sympy__sympy-15011", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 101.4, "eval_duration_s": 29.6, "gen_cost_usd": 0.094638}
|
||||
{"instance_id": "sympy__sympy-15308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 62.1, "eval_duration_s": 28.5, "gen_cost_usd": 0.053562}
|
||||
{"instance_id": "sympy__sympy-15345", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 84.8, "eval_duration_s": 22.0, "gen_cost_usd": 0.075042}
|
||||
{"instance_id": "sympy__sympy-15346", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 47.6, "eval_duration_s": 47.3, "gen_cost_usd": 0.036453}
|
||||
{"instance_id": "sympy__sympy-15609", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.2, "eval_duration_s": 24.9, "gen_cost_usd": 0.048666}
|
||||
{"instance_id": "sympy__sympy-15678", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 128.1, "eval_duration_s": 26.2, "gen_cost_usd": 0.13263}
|
||||
{"instance_id": "sympy__sympy-16106", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 146.8, "eval_duration_s": 21.6, "gen_cost_usd": 0.191646}
|
||||
{"instance_id": "sympy__sympy-16281", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 803.3, "eval_duration_s": 27.4, "gen_cost_usd": 0.997038}
|
||||
{"instance_id": "sympy__sympy-16503", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 819.6, "eval_duration_s": 24.5, "gen_cost_usd": 0.995094}
|
||||
{"instance_id": "sympy__sympy-16792", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 162.3, "eval_duration_s": 21.7, "gen_cost_usd": 0.190089}
|
||||
{"instance_id": "sympy__sympy-16988", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 790.2, "eval_duration_s": 89.1, "gen_cost_usd": 0.894474}
|
||||
{"instance_id": "sympy__sympy-17022", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 139.4, "eval_duration_s": 20.2, "gen_cost_usd": 0.159717}
|
||||
{"instance_id": "sympy__sympy-17139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 46.3, "eval_duration_s": 95.9, "gen_cost_usd": 0.029595}
|
||||
{"instance_id": "sympy__sympy-17630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 392.1, "eval_duration_s": 21.3, "gen_cost_usd": 0.487977}
|
||||
{"instance_id": "sympy__sympy-17655", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 112.9, "eval_duration_s": 42.6, "gen_cost_usd": 0.089706}
|
||||
{"instance_id": "sympy__sympy-18057", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.0, "eval_duration_s": 112.7, "gen_cost_usd": 0.085674}
|
||||
{"instance_id": "sympy__sympy-18087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 253.5, "eval_duration_s": 31.4, "gen_cost_usd": 0.253008}
|
||||
{"instance_id": "sympy__sympy-18189", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 32.3, "eval_duration_s": 41.5, "gen_cost_usd": 0.019599}
|
||||
{"instance_id": "sympy__sympy-18199", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 115.3, "eval_duration_s": 109.3, "gen_cost_usd": 0.120789}
|
||||
{"instance_id": "sympy__sympy-18532", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 144.5, "eval_duration_s": 29.5, "gen_cost_usd": 0.124377}
|
||||
{"instance_id": "sympy__sympy-18621", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 43.1, "eval_duration_s": 33.2, "gen_cost_usd": 0.025353}
|
||||
{"instance_id": "sympy__sympy-18698", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 274.1, "eval_duration_s": 29.6, "gen_cost_usd": 0.312117}
|
||||
{"instance_id": "sympy__sympy-18835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 73.0, "eval_duration_s": 21.3, "gen_cost_usd": 0.056202}
|
||||
{"instance_id": "sympy__sympy-19007", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 316.8, "eval_duration_s": 23.6, "gen_cost_usd": 0.380256}
|
||||
{"instance_id": "sympy__sympy-19254", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 468.4, "eval_duration_s": 33.1, "gen_cost_usd": 0.464595}
|
||||
{"instance_id": "sympy__sympy-19487", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.8, "eval_duration_s": 32.8, "gen_cost_usd": 0.057687}
|
||||
{"instance_id": "sympy__sympy-20049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 442.2, "eval_duration_s": 21.2, "gen_cost_usd": 0.588681}
|
||||
{"instance_id": "sympy__sympy-20154", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.6, "eval_duration_s": 21.3, "gen_cost_usd": 0.069927}
|
||||
{"instance_id": "sympy__sympy-20212", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 241.2, "eval_duration_s": 37.5, "gen_cost_usd": 0.13575}
|
||||
{"instance_id": "sympy__sympy-20322", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 71.8, "eval_duration_s": 30.3, "gen_cost_usd": 0.061077}
|
||||
{"instance_id": "sympy__sympy-20442", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 134.0, "eval_duration_s": 27.8, "gen_cost_usd": 0.142527}
|
||||
{"instance_id": "sympy__sympy-20590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 58.7, "eval_duration_s": 32.1, "gen_cost_usd": 0.042108}
|
||||
{"instance_id": "sympy__sympy-20639", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 224.1, "eval_duration_s": 23.1, "gen_cost_usd": 0.245946}
|
||||
{"instance_id": "sympy__sympy-21055", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.4, "eval_duration_s": 46.9, "gen_cost_usd": 0.13524}
|
||||
{"instance_id": "sympy__sympy-21171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 65.1, "eval_duration_s": 27.6, "gen_cost_usd": 0.060528}
|
||||
{"instance_id": "sympy__sympy-21379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.4, "eval_duration_s": 36.9, "gen_cost_usd": 0.0684}
|
||||
{"instance_id": "sympy__sympy-21612", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.1, "eval_duration_s": 47.0, "gen_cost_usd": 0.03927}
|
||||
{"instance_id": "sympy__sympy-21614", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 99.2, "eval_duration_s": 20.3, "gen_cost_usd": 0.091533}
|
||||
{"instance_id": "sympy__sympy-21627", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 60.0, "eval_duration_s": 25.5, "gen_cost_usd": 0.047247}
|
||||
{"instance_id": "sympy__sympy-21847", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 41.6, "eval_duration_s": 19.8, "gen_cost_usd": 0.024429}
|
||||
{"instance_id": "sympy__sympy-22005", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 210.9, "eval_duration_s": 26.3, "gen_cost_usd": 0.26079}
|
||||
{"instance_id": "sympy__sympy-22714", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 144.0, "eval_duration_s": 23.1, "gen_cost_usd": 0.155361}
|
||||
{"instance_id": "sympy__sympy-22840", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 174.0, "eval_duration_s": 29.2, "gen_cost_usd": 0.20598}
|
||||
{"instance_id": "sympy__sympy-23117", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.8, "eval_duration_s": 21.3, "gen_cost_usd": 0.036834}
|
||||
{"instance_id": "sympy__sympy-23191", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 926.9, "eval_duration_s": 21.6, "gen_cost_usd": 1.136349}
|
||||
{"instance_id": "sympy__sympy-23262", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 104.0, "eval_duration_s": 25.1, "gen_cost_usd": 0.087327}
|
||||
{"instance_id": "sympy__sympy-24066", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 140.3, "eval_duration_s": 21.3, "gen_cost_usd": 0.11469}
|
||||
{"instance_id": "sympy__sympy-24102", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 138.3, "eval_duration_s": 21.8, "gen_cost_usd": 0.175524}
|
||||
{"instance_id": "sympy__sympy-24152", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 61.4, "eval_duration_s": 29.1, "gen_cost_usd": 0.042897}
|
||||
{"instance_id": "sympy__sympy-24213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.6, "eval_duration_s": 23.2, "gen_cost_usd": 0.056283}
|
||||
{"instance_id": "sympy__sympy-24909", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 130.1, "eval_duration_s": 22.0, "gen_cost_usd": 0.139866}
|
||||
97
evals/swe-bench/scoreboard/sonnet-20min-20260316/meta.json
Normal file
97
evals/swe-bench/scoreboard/sonnet-20min-20260316/meta.json
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
{
|
||||
"run_name": "sonnet-20min-20260316",
|
||||
"timestamp": "2026-03-16T17:31:20.245784+00:00",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"provider": "anthropic",
|
||||
"fabro_version": "fabro 0.5.0 (4c6ad4d 2026-03-16)",
|
||||
"timeout_s": 1200,
|
||||
"sandbox_cpu": 4,
|
||||
"sandbox_memory_gb": 8,
|
||||
"description": "Sonnet 4.6, 20min timeout, 4 CPU / 8 GB, default prompt",
|
||||
"notes": "",
|
||||
"total_instances": 300,
|
||||
"patched": 297,
|
||||
"patch_pct": 99.0,
|
||||
"resolved": 173,
|
||||
"resolve_pct": 57.7,
|
||||
"total_gen_cost_usd": 55.22,
|
||||
"avg_gen_cost_usd": 0.1841,
|
||||
"gen_wall_time_s": 1466.0,
|
||||
"eval_wall_time_s": 682.3,
|
||||
"avg_gen_duration_s": 211.5,
|
||||
"avg_eval_duration_s": 51.4,
|
||||
"per_repo": {
|
||||
"astropy/astropy": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 1,
|
||||
"resolve_pct": 16.7
|
||||
},
|
||||
"django/django": {
|
||||
"total": 114,
|
||||
"patched": 114,
|
||||
"resolved": 88,
|
||||
"resolve_pct": 77.2
|
||||
},
|
||||
"matplotlib/matplotlib": {
|
||||
"total": 23,
|
||||
"patched": 22,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"mwaskom/seaborn": {
|
||||
"total": 4,
|
||||
"patched": 4,
|
||||
"resolved": 3,
|
||||
"resolve_pct": 75.0
|
||||
},
|
||||
"pallets/flask": {
|
||||
"total": 3,
|
||||
"patched": 3,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"psf/requests": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 5,
|
||||
"resolve_pct": 83.3
|
||||
},
|
||||
"pydata/xarray": {
|
||||
"total": 5,
|
||||
"patched": 4,
|
||||
"resolved": 1,
|
||||
"resolve_pct": 20.0
|
||||
},
|
||||
"pylint-dev/pylint": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"pytest-dev/pytest": {
|
||||
"total": 17,
|
||||
"patched": 17,
|
||||
"resolved": 11,
|
||||
"resolve_pct": 64.7
|
||||
},
|
||||
"scikit-learn/scikit-learn": {
|
||||
"total": 23,
|
||||
"patched": 23,
|
||||
"resolved": 15,
|
||||
"resolve_pct": 65.2
|
||||
},
|
||||
"sphinx-doc/sphinx": {
|
||||
"total": 16,
|
||||
"patched": 16,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"sympy/sympy": {
|
||||
"total": 77,
|
||||
"patched": 76,
|
||||
"resolved": 49,
|
||||
"resolve_pct": 63.6
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue