mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-08-28 05:27:41 +00:00
Record Sonnet 4.6 baseline: 55.7% on SWE-Bench-Lite ($39.78)
Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp
parent
951a37faf8
commit
002468316d
4 changed files with 445 additions and 0 deletions
|
|
@ -1,4 +1,17 @@
|
|||
[
|
||||
{
|
||||
"run_name": "sonnet-baseline-20260316",
|
||||
"date": "2026-03-16",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"provider": "anthropic",
|
||||
"resolved": 167,
|
||||
"total": 300,
|
||||
"resolve_pct": 55.7,
|
||||
"patched": 281,
|
||||
"patch_pct": 93.7,
|
||||
"total_cost_usd": 39.78,
|
||||
"avg_cost_usd": 0.1326
|
||||
},
|
||||
{
|
||||
"run_name": "haiku-baseline-20260316",
|
||||
"date": "2026-03-16",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
# sonnet-baseline-20260316
|
||||
|
||||
**Date:** 2026-03-16
|
||||
**Model:** claude-sonnet-4-6 (anthropic)
|
||||
**Fabro:** fabro 0.5.0 (4c6ad4d 2026-03-16)
|
||||
|
||||
## Description
|
||||
|
||||
Sonnet 4.6 baseline, default prompt, 2 CPU / 4 GB, 10min timeout
|
||||
|
||||
## Results
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Instances | 300 |
|
||||
| Patched | 281 (93.7%) |
|
||||
| **Resolved** | **167 (55.7%)** |
|
||||
| Total gen cost | $39.78 |
|
||||
| Avg gen cost | $0.1326/instance |
|
||||
| Gen wall time | 993.6s |
|
||||
| Eval wall time | 642.3s |
|
||||
|
||||
## Per-repo breakdown
|
||||
|
||||
| Repo | Resolved | Total | Rate |
|
||||
|------|----------|-------|------|
|
||||
| astropy/astropy | 1 | 6 | 16.7% |
|
||||
| django/django | 86 | 114 | 75.4% |
|
||||
| matplotlib/matplotlib | 0 | 23 | 0.0% |
|
||||
| mwaskom/seaborn | 2 | 4 | 50.0% |
|
||||
| pallets/flask | 0 | 3 | 0.0% |
|
||||
| psf/requests | 5 | 6 | 83.3% |
|
||||
| pydata/xarray | 1 | 5 | 20.0% |
|
||||
| pylint-dev/pylint | 0 | 6 | 0.0% |
|
||||
| pytest-dev/pytest | 11 | 17 | 64.7% |
|
||||
| scikit-learn/scikit-learn | 14 | 23 | 60.9% |
|
||||
| sphinx-doc/sphinx | 0 | 16 | 0.0% |
|
||||
| sympy/sympy | 47 | 77 | 61.0% |
|
||||
|
|
@ -0,0 +1,300 @@
|
|||
{"instance_id": "astropy__astropy-12907", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 112.5, "eval_duration_s": 78.5, "gen_cost_usd": 0.072702}
|
||||
{"instance_id": "astropy__astropy-14182", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 196.7, "eval_duration_s": 59.6, "gen_cost_usd": 0.175827}
|
||||
{"instance_id": "astropy__astropy-14365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 117.2, "eval_duration_s": 63.9, "gen_cost_usd": 0.091416}
|
||||
{"instance_id": "astropy__astropy-14995", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 78.2, "eval_duration_s": 67.8, "gen_cost_usd": 0.03861}
|
||||
{"instance_id": "astropy__astropy-6938", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 102.3, "eval_duration_s": 142.3, "gen_cost_usd": 0.013179}
|
||||
{"instance_id": "astropy__astropy-7746", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 125.5, "eval_duration_s": 92.8, "gen_cost_usd": 0.044217}
|
||||
{"instance_id": "django__django-10914", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 203.0, "eval_duration_s": 23.3, "gen_cost_usd": 0.136578}
|
||||
{"instance_id": "django__django-10924", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.1, "eval_duration_s": 24.3, "gen_cost_usd": 0.094581}
|
||||
{"instance_id": "django__django-11001", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 142.5, "eval_duration_s": 51.0, "gen_cost_usd": 0.105576}
|
||||
{"instance_id": "django__django-11019", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 605.7, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "django__django-11039", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 85.5, "eval_duration_s": 34.8, "gen_cost_usd": 0.09552}
|
||||
{"instance_id": "django__django-11049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 45.3, "eval_duration_s": 31.3, "gen_cost_usd": 0.052521}
|
||||
{"instance_id": "django__django-11099", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 38.0, "eval_duration_s": 118.9, "gen_cost_usd": 0.014364}
|
||||
{"instance_id": "django__django-11133", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 42.3, "eval_duration_s": 22.9, "gen_cost_usd": 0.021831}
|
||||
{"instance_id": "django__django-11179", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.3, "eval_duration_s": 31.4, "gen_cost_usd": 0.015576}
|
||||
{"instance_id": "django__django-11283", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 201.0, "eval_duration_s": 25.2, "gen_cost_usd": 0.234933}
|
||||
{"instance_id": "django__django-11422", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.5, "eval_duration_s": 30.9, "gen_cost_usd": 0.09819}
|
||||
{"instance_id": "django__django-11564", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 333.9, "eval_duration_s": 19.2, "gen_cost_usd": 0.413628}
|
||||
{"instance_id": "django__django-11583", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.8, "eval_duration_s": 36.3, "gen_cost_usd": 0.021516}
|
||||
{"instance_id": "django__django-11620", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 74.3, "eval_duration_s": 27.8, "gen_cost_usd": 0.07572}
|
||||
{"instance_id": "django__django-11630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 193.0, "eval_duration_s": 24.0, "gen_cost_usd": 0.1971}
|
||||
{"instance_id": "django__django-11742", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 236.1, "eval_duration_s": 30.8, "gen_cost_usd": 0.20424}
|
||||
{"instance_id": "django__django-11797", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 481.1, "eval_duration_s": 20.1, "gen_cost_usd": 0.446673}
|
||||
{"instance_id": "django__django-11815", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.8, "eval_duration_s": 35.9, "gen_cost_usd": 0.104976}
|
||||
{"instance_id": "django__django-11848", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 84.5, "eval_duration_s": 37.4, "gen_cost_usd": 0.051981}
|
||||
{"instance_id": "django__django-11905", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 93.1, "eval_duration_s": 36.4, "gen_cost_usd": 0.108054}
|
||||
{"instance_id": "django__django-11910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 462.7, "eval_duration_s": 16.1, "gen_cost_usd": 0.55728}
|
||||
{"instance_id": "django__django-11964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 195.6, "eval_duration_s": 20.4, "gen_cost_usd": 0.212775}
|
||||
{"instance_id": "django__django-11999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 68.3, "eval_duration_s": 66.6, "gen_cost_usd": 0.045507}
|
||||
{"instance_id": "django__django-12113", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 165.1, "eval_duration_s": 30.0, "gen_cost_usd": 0.20745}
|
||||
{"instance_id": "django__django-12125", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 120.6, "eval_duration_s": 40.4, "gen_cost_usd": 0.127506}
|
||||
{"instance_id": "django__django-12184", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 183.9, "eval_duration_s": 48.3, "gen_cost_usd": 0.177834}
|
||||
{"instance_id": "django__django-12284", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 75.6, "eval_duration_s": 25.4, "gen_cost_usd": 0.053568}
|
||||
{"instance_id": "django__django-12286", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 66.3, "eval_duration_s": 29.2, "gen_cost_usd": 0.045819}
|
||||
{"instance_id": "django__django-12308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 133.7, "eval_duration_s": 74.7, "gen_cost_usd": 0.166785}
|
||||
{"instance_id": "django__django-12453", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.2, "eval_duration_s": 25.5, "gen_cost_usd": 0.050874}
|
||||
{"instance_id": "django__django-12470", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 513.6, "eval_duration_s": 18.5, "gen_cost_usd": 0.505338}
|
||||
{"instance_id": "django__django-12497", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 90.6, "eval_duration_s": 30.5, "gen_cost_usd": 0.062343}
|
||||
{"instance_id": "django__django-12589", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 251.3, "eval_duration_s": 48.6, "gen_cost_usd": 0.272889}
|
||||
{"instance_id": "django__django-12700", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 63.2, "eval_duration_s": 25.7, "gen_cost_usd": 0.057297}
|
||||
{"instance_id": "django__django-12708", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 142.6, "eval_duration_s": 35.9, "gen_cost_usd": 0.130812}
|
||||
{"instance_id": "django__django-12747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 75.3, "eval_duration_s": 43.9, "gen_cost_usd": 0.049929}
|
||||
{"instance_id": "django__django-12856", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 169.4, "eval_duration_s": 26.9, "gen_cost_usd": 0.169635}
|
||||
{"instance_id": "django__django-12908", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 108.8, "eval_duration_s": 35.1, "gen_cost_usd": 0.104679}
|
||||
{"instance_id": "django__django-12915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 94.6, "eval_duration_s": 32.4, "gen_cost_usd": 0.04815}
|
||||
{"instance_id": "django__django-12983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 96.8, "eval_duration_s": 24.7, "gen_cost_usd": 0.09093}
|
||||
{"instance_id": "django__django-13028", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 30.1, "eval_duration_s": 37.1, "gen_cost_usd": 0.016263}
|
||||
{"instance_id": "django__django-13033", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 29.2, "eval_duration_s": 32.8, "gen_cost_usd": 0.020088}
|
||||
{"instance_id": "django__django-13158", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 124.3, "eval_duration_s": 26.0, "gen_cost_usd": 0.114822}
|
||||
{"instance_id": "django__django-13220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 127.3, "eval_duration_s": 23.6, "gen_cost_usd": 0.124545}
|
||||
{"instance_id": "django__django-13230", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 117.4, "eval_duration_s": 27.9, "gen_cost_usd": 0.13749}
|
||||
{"instance_id": "django__django-13265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 213.4, "eval_duration_s": 21.9, "gen_cost_usd": 0.222177}
|
||||
{"instance_id": "django__django-13315", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 239.5, "eval_duration_s": 19.6, "gen_cost_usd": 0.26547}
|
||||
{"instance_id": "django__django-13321", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 49.8, "eval_duration_s": 25.1, "gen_cost_usd": 0.030936}
|
||||
{"instance_id": "django__django-13401", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.3, "eval_duration_s": 27.9, "gen_cost_usd": 0.074103}
|
||||
{"instance_id": "django__django-13447", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 179.6, "eval_duration_s": 34.9, "gen_cost_usd": 0.133173}
|
||||
{"instance_id": "django__django-13448", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 264.6, "eval_duration_s": 23.4, "gen_cost_usd": 0.309393}
|
||||
{"instance_id": "django__django-13551", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.6, "eval_duration_s": 30.7, "gen_cost_usd": 0.05235}
|
||||
{"instance_id": "django__django-13590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.0, "eval_duration_s": 39.4, "gen_cost_usd": 0.071469}
|
||||
{"instance_id": "django__django-13658", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 38.4, "eval_duration_s": 61.3, "gen_cost_usd": 0.01923}
|
||||
{"instance_id": "django__django-13660", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 64.3, "eval_duration_s": 29.5, "gen_cost_usd": 0.041499}
|
||||
{"instance_id": "django__django-13710", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 139.6, "eval_duration_s": 39.5, "gen_cost_usd": 0.148041}
|
||||
{"instance_id": "django__django-13757", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 134.1, "eval_duration_s": 28.2, "gen_cost_usd": 0.113307}
|
||||
{"instance_id": "django__django-13768", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.3, "eval_duration_s": 37.1, "gen_cost_usd": 0.053163}
|
||||
{"instance_id": "django__django-13925", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 190.4, "eval_duration_s": 22.7, "gen_cost_usd": 0.156345}
|
||||
{"instance_id": "django__django-13933", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 381.2, "eval_duration_s": 17.8, "gen_cost_usd": 0.384363}
|
||||
{"instance_id": "django__django-13964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.8, "eval_duration_s": 45.6, "gen_cost_usd": 0.100275}
|
||||
{"instance_id": "django__django-14016", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 187.3, "eval_duration_s": 28.8, "gen_cost_usd": 0.19587}
|
||||
{"instance_id": "django__django-14017", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 260.1, "eval_duration_s": 25.4, "gen_cost_usd": 0.29916}
|
||||
{"instance_id": "django__django-14155", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 66.9, "eval_duration_s": 39.2, "gen_cost_usd": 0.05961}
|
||||
{"instance_id": "django__django-14238", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.7, "eval_duration_s": 28.3, "gen_cost_usd": 0.041748}
|
||||
{"instance_id": "django__django-14382", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 30.3, "eval_duration_s": 59.9, "gen_cost_usd": 0.013485}
|
||||
{"instance_id": "django__django-14411", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 88.7, "eval_duration_s": 31.9, "gen_cost_usd": 0.076698}
|
||||
{"instance_id": "django__django-14534", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 142.3, "eval_duration_s": 41.9, "gen_cost_usd": 0.140226}
|
||||
{"instance_id": "django__django-14580", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.3, "eval_duration_s": 40.2, "gen_cost_usd": 0.015012}
|
||||
{"instance_id": "django__django-14608", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 173.9, "eval_duration_s": 52.9, "gen_cost_usd": 0.168522}
|
||||
{"instance_id": "django__django-14667", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 256.6, "eval_duration_s": 22.6, "gen_cost_usd": 0.282657}
|
||||
{"instance_id": "django__django-14672", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.0, "eval_duration_s": 52.9, "gen_cost_usd": 0.0327}
|
||||
{"instance_id": "django__django-14730", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 469.6, "eval_duration_s": 48.0, "gen_cost_usd": 0.52137}
|
||||
{"instance_id": "django__django-14752", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.9, "eval_duration_s": 52.4, "gen_cost_usd": 0.057282}
|
||||
{"instance_id": "django__django-14787", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 68.0, "eval_duration_s": 73.4, "gen_cost_usd": 0.04998}
|
||||
{"instance_id": "django__django-14855", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 45.8, "eval_duration_s": 145.2, "gen_cost_usd": 0.023643}
|
||||
{"instance_id": "django__django-14915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.7, "eval_duration_s": 40.4, "gen_cost_usd": 0.017049}
|
||||
{"instance_id": "django__django-14997", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 283.3, "eval_duration_s": 23.1, "gen_cost_usd": 0.312906}
|
||||
{"instance_id": "django__django-14999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 206.2, "eval_duration_s": 26.7, "gen_cost_usd": 0.224217}
|
||||
{"instance_id": "django__django-15061", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 124.0, "eval_duration_s": 101.9, "gen_cost_usd": 0.106311}
|
||||
{"instance_id": "django__django-15202", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 71.0, "eval_duration_s": 37.2, "gen_cost_usd": 0.052821}
|
||||
{"instance_id": "django__django-15213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 330.7, "eval_duration_s": 19.5, "gen_cost_usd": 0.338769}
|
||||
{"instance_id": "django__django-15252", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 133.1, "eval_duration_s": 40.3, "gen_cost_usd": 0.13539}
|
||||
{"instance_id": "django__django-15320", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.0, "eval_duration_s": 38.6, "gen_cost_usd": 0.057657}
|
||||
{"instance_id": "django__django-15347", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 75.6, "eval_duration_s": 29.4, "gen_cost_usd": 0.05088}
|
||||
{"instance_id": "django__django-15388", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 59.0, "eval_duration_s": 26.8, "gen_cost_usd": 0.05031}
|
||||
{"instance_id": "django__django-15400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.2, "eval_duration_s": 37.6, "gen_cost_usd": 0.04971}
|
||||
{"instance_id": "django__django-15498", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.1, "eval_duration_s": 33.4, "gen_cost_usd": 0.114474}
|
||||
{"instance_id": "django__django-15695", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 353.0, "eval_duration_s": 26.3, "gen_cost_usd": 0.380646}
|
||||
{"instance_id": "django__django-15738", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 615.1, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "django__django-15781", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.3, "eval_duration_s": 52.6, "gen_cost_usd": 0.020385}
|
||||
{"instance_id": "django__django-15789", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 76.6, "eval_duration_s": 38.4, "gen_cost_usd": 0.07986}
|
||||
{"instance_id": "django__django-15790", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 85.6, "eval_duration_s": 41.0, "gen_cost_usd": 0.080313}
|
||||
{"instance_id": "django__django-15814", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.3, "eval_duration_s": 32.2, "gen_cost_usd": 0.050913}
|
||||
{"instance_id": "django__django-15819", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 235.7, "eval_duration_s": 43.9, "gen_cost_usd": 0.297603}
|
||||
{"instance_id": "django__django-15851", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.3, "eval_duration_s": 33.4, "gen_cost_usd": 0.054147}
|
||||
{"instance_id": "django__django-15902", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 47.2, "eval_duration_s": 39.2, "gen_cost_usd": 0.036603}
|
||||
{"instance_id": "django__django-15996", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 159.7, "eval_duration_s": 23.8, "gen_cost_usd": 0.129498}
|
||||
{"instance_id": "django__django-16041", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 46.9, "eval_duration_s": 41.7, "gen_cost_usd": 0.038313}
|
||||
{"instance_id": "django__django-16046", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 61.0, "eval_duration_s": 26.4, "gen_cost_usd": 0.049527}
|
||||
{"instance_id": "django__django-16139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 44.6, "eval_duration_s": 27.2, "gen_cost_usd": 0.017859}
|
||||
{"instance_id": "django__django-16229", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 268.5, "eval_duration_s": 20.4, "gen_cost_usd": 0.270381}
|
||||
{"instance_id": "django__django-16255", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.4, "eval_duration_s": 25.5, "gen_cost_usd": 0.012861}
|
||||
{"instance_id": "django__django-16379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 35.5, "eval_duration_s": 113.1, "gen_cost_usd": 0.022248}
|
||||
{"instance_id": "django__django-16400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 87.5, "eval_duration_s": 91.9, "gen_cost_usd": 0.083976}
|
||||
{"instance_id": "django__django-16408", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 408.4, "eval_duration_s": 19.3, "gen_cost_usd": 0.461664}
|
||||
{"instance_id": "django__django-16527", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 25.5, "eval_duration_s": 31.0, "gen_cost_usd": 0.013059}
|
||||
{"instance_id": "django__django-16595", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 58.1, "eval_duration_s": 43.8, "gen_cost_usd": 0.053478}
|
||||
{"instance_id": "django__django-16816", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 434.3, "eval_duration_s": 20.7, "gen_cost_usd": 0.429663}
|
||||
{"instance_id": "django__django-16820", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 601.5, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "django__django-16873", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 74.8, "eval_duration_s": 28.4, "gen_cost_usd": 0.060855}
|
||||
{"instance_id": "django__django-16910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 500.8, "eval_duration_s": 23.7, "gen_cost_usd": 0.607005}
|
||||
{"instance_id": "django__django-17051", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.9, "eval_duration_s": 29.4, "gen_cost_usd": 0.041154}
|
||||
{"instance_id": "django__django-17087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 96.1, "eval_duration_s": 35.6, "gen_cost_usd": 0.064275}
|
||||
{"instance_id": "matplotlib__matplotlib-18869", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 282.7, "eval_duration_s": 75.5, "gen_cost_usd": 0.105255}
|
||||
{"instance_id": "matplotlib__matplotlib-22711", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 226.4, "eval_duration_s": 79.4, "gen_cost_usd": 0.089361}
|
||||
{"instance_id": "matplotlib__matplotlib-22835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 211.4, "eval_duration_s": 103.5, "gen_cost_usd": 0.078297}
|
||||
{"instance_id": "matplotlib__matplotlib-23299", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 534.7, "eval_duration_s": 91.4, "gen_cost_usd": 0.396213}
|
||||
{"instance_id": "matplotlib__matplotlib-23314", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 306.3, "eval_duration_s": 82.1, "gen_cost_usd": 0.093924}
|
||||
{"instance_id": "matplotlib__matplotlib-23476", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 440.0, "eval_duration_s": 94.1, "gen_cost_usd": 0.407541}
|
||||
{"instance_id": "matplotlib__matplotlib-23562", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 332.3, "eval_duration_s": 81.6, "gen_cost_usd": 0.198681}
|
||||
{"instance_id": "matplotlib__matplotlib-23563", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 313.1, "eval_duration_s": 77.5, "gen_cost_usd": 0.309069}
|
||||
{"instance_id": "matplotlib__matplotlib-23913", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 214.3, "eval_duration_s": 83.6, "gen_cost_usd": 0.114642}
|
||||
{"instance_id": "matplotlib__matplotlib-23964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 130.1, "eval_duration_s": 100.8, "gen_cost_usd": 0.017466}
|
||||
{"instance_id": "matplotlib__matplotlib-23987", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 219.7, "eval_duration_s": 195.6, "gen_cost_usd": 0.084459}
|
||||
{"instance_id": "matplotlib__matplotlib-24149", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 169.6, "eval_duration_s": 204.2, "gen_cost_usd": 0.036435}
|
||||
{"instance_id": "matplotlib__matplotlib-24265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 185.0, "eval_duration_s": 89.9, "gen_cost_usd": 0.080229}
|
||||
{"instance_id": "matplotlib__matplotlib-24334", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 215.0, "eval_duration_s": 92.7, "gen_cost_usd": 0.065604}
|
||||
{"instance_id": "matplotlib__matplotlib-24970", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 208.3, "eval_duration_s": 93.4, "gen_cost_usd": 0.0777}
|
||||
{"instance_id": "matplotlib__matplotlib-25079", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 506.8, "eval_duration_s": 91.5, "gen_cost_usd": 0.401136}
|
||||
{"instance_id": "matplotlib__matplotlib-25311", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 601.0, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "matplotlib__matplotlib-25332", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 246.2, "eval_duration_s": 83.1, "gen_cost_usd": 0.089643}
|
||||
{"instance_id": "matplotlib__matplotlib-25433", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.4, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "matplotlib__matplotlib-25442", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.3, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "matplotlib__matplotlib-25498", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 580.8, "eval_duration_s": 90.2, "gen_cost_usd": 0.482124}
|
||||
{"instance_id": "matplotlib__matplotlib-26011", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 239.9, "eval_duration_s": 83.5, "gen_cost_usd": 0.140001}
|
||||
{"instance_id": "matplotlib__matplotlib-26020", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 184.2, "eval_duration_s": 186.1, "gen_cost_usd": 0.066654}
|
||||
{"instance_id": "mwaskom__seaborn-2848", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 508.3, "eval_duration_s": 21.5, "gen_cost_usd": 0.552411}
|
||||
{"instance_id": "mwaskom__seaborn-3010", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 46.9, "eval_duration_s": 33.3, "gen_cost_usd": 0.052899}
|
||||
{"instance_id": "mwaskom__seaborn-3190", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 42.2, "eval_duration_s": 30.9, "gen_cost_usd": 0.04521}
|
||||
{"instance_id": "mwaskom__seaborn-3407", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 222.4, "eval_duration_s": 73.0, "gen_cost_usd": 0.2136}
|
||||
{"instance_id": "pallets__flask-4045", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 53.1, "eval_duration_s": 24.1, "gen_cost_usd": 0.076347}
|
||||
{"instance_id": "pallets__flask-4992", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 39.3, "eval_duration_s": 19.1, "gen_cost_usd": 0.045555}
|
||||
{"instance_id": "pallets__flask-5063", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 127.4, "eval_duration_s": 8.9, "gen_cost_usd": 0.154236}
|
||||
{"instance_id": "psf__requests-1963", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.4, "eval_duration_s": 31.3, "gen_cost_usd": 0.035673}
|
||||
{"instance_id": "psf__requests-2148", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 42.8, "eval_duration_s": 61.7, "gen_cost_usd": 0.057693}
|
||||
{"instance_id": "psf__requests-2317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "timeout", "gen_duration_s": 280.7, "eval_duration_s": 600.2, "gen_cost_usd": 0.07626}
|
||||
{"instance_id": "psf__requests-2674", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 171.8, "eval_duration_s": 82.9, "gen_cost_usd": 0.187542}
|
||||
{"instance_id": "psf__requests-3362", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 62.4, "eval_duration_s": 13.7, "gen_cost_usd": 0.058596}
|
||||
{"instance_id": "psf__requests-863", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 57.5, "eval_duration_s": 36.4, "gen_cost_usd": 0.053295}
|
||||
{"instance_id": "pydata__xarray-3364", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.9, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "pydata__xarray-4094", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 194.7, "eval_duration_s": 16.9, "gen_cost_usd": 0.190782}
|
||||
{"instance_id": "pydata__xarray-4248", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 108.5, "eval_duration_s": 13.8, "gen_cost_usd": 0.130917}
|
||||
{"instance_id": "pydata__xarray-4493", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 86.2, "eval_duration_s": 48.8, "gen_cost_usd": 0.121611}
|
||||
{"instance_id": "pydata__xarray-5131", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 36.7, "eval_duration_s": 42.4, "gen_cost_usd": 0.032739}
|
||||
{"instance_id": "pylint-dev__pylint-5859", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 175.4, "eval_duration_s": 9.1, "gen_cost_usd": 0.236112}
|
||||
{"instance_id": "pylint-dev__pylint-6506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 71.5, "eval_duration_s": 13.6, "gen_cost_usd": 0.098181}
|
||||
{"instance_id": "pylint-dev__pylint-7080", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 136.2, "eval_duration_s": 15.5, "gen_cost_usd": 0.171075}
|
||||
{"instance_id": "pylint-dev__pylint-7114", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 440.6, "eval_duration_s": 5.2, "gen_cost_usd": 0.589275}
|
||||
{"instance_id": "pylint-dev__pylint-7228", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 90.9, "eval_duration_s": 10.0, "gen_cost_usd": 0.109128}
|
||||
{"instance_id": "pylint-dev__pylint-7993", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 77.7, "eval_duration_s": 11.9, "gen_cost_usd": 0.091728}
|
||||
{"instance_id": "pytest-dev__pytest-11143", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 45.9, "eval_duration_s": 24.3, "gen_cost_usd": 0.050079}
|
||||
{"instance_id": "pytest-dev__pytest-11148", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 263.1, "eval_duration_s": 13.2, "gen_cost_usd": 0.329841}
|
||||
{"instance_id": "pytest-dev__pytest-5103", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.9, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "pytest-dev__pytest-5221", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 263.9, "eval_duration_s": 13.5, "gen_cost_usd": 0.255471}
|
||||
{"instance_id": "pytest-dev__pytest-5227", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.3, "eval_duration_s": 34.5, "gen_cost_usd": 0.020928}
|
||||
{"instance_id": "pytest-dev__pytest-5413", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.0, "eval_duration_s": 14.5, "gen_cost_usd": 0.057057}
|
||||
{"instance_id": "pytest-dev__pytest-5495", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 69.3, "eval_duration_s": 15.8, "gen_cost_usd": 0.059319}
|
||||
{"instance_id": "pytest-dev__pytest-5692", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 49.9, "eval_duration_s": 27.0, "gen_cost_usd": 0.072726}
|
||||
{"instance_id": "pytest-dev__pytest-6116", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 30.6, "eval_duration_s": 20.5, "gen_cost_usd": 0.028284}
|
||||
{"instance_id": "pytest-dev__pytest-7168", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 26.7, "eval_duration_s": 25.1, "gen_cost_usd": 0.034452}
|
||||
{"instance_id": "pytest-dev__pytest-7220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 235.4, "eval_duration_s": 9.7, "gen_cost_usd": 0.241527}
|
||||
{"instance_id": "pytest-dev__pytest-7373", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.6, "eval_duration_s": 13.7, "gen_cost_usd": 0.051297}
|
||||
{"instance_id": "pytest-dev__pytest-7432", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.3, "eval_duration_s": 16.1, "gen_cost_usd": 0.063561}
|
||||
{"instance_id": "pytest-dev__pytest-7490", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 100.3, "eval_duration_s": 12.8, "gen_cost_usd": 0.146775}
|
||||
{"instance_id": "pytest-dev__pytest-8365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 68.6, "eval_duration_s": 13.5, "gen_cost_usd": 0.072432}
|
||||
{"instance_id": "pytest-dev__pytest-8906", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 51.3, "eval_duration_s": 16.2, "gen_cost_usd": 0.056994}
|
||||
{"instance_id": "pytest-dev__pytest-9359", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 423.0, "eval_duration_s": 9.3, "gen_cost_usd": 0.513519}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10297", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 179.9, "eval_duration_s": 132.3, "gen_cost_usd": 0.059958}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10508", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 142.8, "eval_duration_s": 133.3, "gen_cost_usd": 0.020919}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10949", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 329.8, "eval_duration_s": 141.7, "gen_cost_usd": 0.267798}
|
||||
{"instance_id": "scikit-learn__scikit-learn-11040", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 213.0, "eval_duration_s": 133.7, "gen_cost_usd": 0.104217}
|
||||
{"instance_id": "scikit-learn__scikit-learn-11281", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 227.0, "eval_duration_s": 136.0, "gen_cost_usd": 0.11433}
|
||||
{"instance_id": "scikit-learn__scikit-learn-12471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 184.9, "eval_duration_s": 151.4, "gen_cost_usd": 0.043431}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13142", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 196.2, "eval_duration_s": 156.3, "gen_cost_usd": 0.034335}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13241", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 451.3, "eval_duration_s": 182.0, "gen_cost_usd": 0.351219}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13439", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 205.7, "eval_duration_s": 174.3, "gen_cost_usd": 0.019278}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13496", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 225.5, "eval_duration_s": 175.9, "gen_cost_usd": 0.092478}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13497", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 243.3, "eval_duration_s": 174.6, "gen_cost_usd": 0.078357}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13584", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 254.5, "eval_duration_s": 179.9, "gen_cost_usd": 0.074706}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13779", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 229.1, "eval_duration_s": 203.7, "gen_cost_usd": 0.020601}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14087", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.8, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14092", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 283.2, "eval_duration_s": 200.9, "gen_cost_usd": 0.110145}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14894", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 228.1, "eval_duration_s": 204.3, "gen_cost_usd": 0.033885}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 339.9, "eval_duration_s": 208.0, "gen_cost_usd": 0.120936}
|
||||
{"instance_id": "scikit-learn__scikit-learn-15512", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 306.4, "eval_duration_s": 194.3, "gen_cost_usd": 0.133101}
|
||||
{"instance_id": "scikit-learn__scikit-learn-15535", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 233.3, "eval_duration_s": 203.0, "gen_cost_usd": 0.047589}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25500", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 451.4, "eval_duration_s": 16.0, "gen_cost_usd": 0.092292}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25570", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.3, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25638", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.2, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 328.1, "eval_duration_s": 10.6, "gen_cost_usd": 0.084288}
|
||||
{"instance_id": "sphinx-doc__sphinx-10325", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 250.0, "eval_duration_s": 14.8, "gen_cost_usd": 0.278871}
|
||||
{"instance_id": "sphinx-doc__sphinx-10451", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 464.4, "eval_duration_s": 59.7, "gen_cost_usd": 0.54561}
|
||||
{"instance_id": "sphinx-doc__sphinx-11445", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 77.7, "eval_duration_s": 21.1, "gen_cost_usd": 0.093483}
|
||||
{"instance_id": "sphinx-doc__sphinx-7686", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 152.1, "eval_duration_s": 15.1, "gen_cost_usd": 0.187743}
|
||||
{"instance_id": "sphinx-doc__sphinx-7738", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 151.5, "eval_duration_s": 15.6, "gen_cost_usd": 0.160485}
|
||||
{"instance_id": "sphinx-doc__sphinx-7975", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 141.0, "eval_duration_s": 14.5, "gen_cost_usd": 0.180909}
|
||||
{"instance_id": "sphinx-doc__sphinx-8273", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 73.2, "eval_duration_s": 16.9, "gen_cost_usd": 0.111054}
|
||||
{"instance_id": "sphinx-doc__sphinx-8282", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 112.6, "eval_duration_s": 17.5, "gen_cost_usd": 0.153642}
|
||||
{"instance_id": "sphinx-doc__sphinx-8435", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 326.2, "eval_duration_s": 13.2, "gen_cost_usd": 0.407775}
|
||||
{"instance_id": "sphinx-doc__sphinx-8474", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.2, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "sphinx-doc__sphinx-8506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 139.2, "eval_duration_s": 14.5, "gen_cost_usd": 0.145218}
|
||||
{"instance_id": "sphinx-doc__sphinx-8595", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 75.1, "eval_duration_s": 17.5, "gen_cost_usd": 0.069192}
|
||||
{"instance_id": "sphinx-doc__sphinx-8627", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 232.6, "eval_duration_s": 16.6, "gen_cost_usd": 0.294498}
|
||||
{"instance_id": "sphinx-doc__sphinx-8713", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 49.8, "eval_duration_s": 17.5, "gen_cost_usd": 0.058677}
|
||||
{"instance_id": "sphinx-doc__sphinx-8721", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 99.5, "eval_duration_s": 15.5, "gen_cost_usd": 0.120267}
|
||||
{"instance_id": "sphinx-doc__sphinx-8801", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 313.3, "eval_duration_s": 13.2, "gen_cost_usd": 0.361779}
|
||||
{"instance_id": "sympy__sympy-11400", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 87.7, "eval_duration_s": 20.3, "gen_cost_usd": 0.085452}
|
||||
{"instance_id": "sympy__sympy-11870", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 120.9, "eval_duration_s": 500.6, "gen_cost_usd": 0.106254}
|
||||
{"instance_id": "sympy__sympy-11897", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.5, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "sympy__sympy-12171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 103.5, "eval_duration_s": 43.1, "gen_cost_usd": 0.093951}
|
||||
{"instance_id": "sympy__sympy-12236", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 361.8, "eval_duration_s": 29.7, "gen_cost_usd": 0.390981}
|
||||
{"instance_id": "sympy__sympy-12419", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 99.3, "eval_duration_s": 59.6, "gen_cost_usd": 0.097242}
|
||||
{"instance_id": "sympy__sympy-12454", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 52.7, "eval_duration_s": 38.3, "gen_cost_usd": 0.03927}
|
||||
{"instance_id": "sympy__sympy-12481", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 151.2, "eval_duration_s": 27.8, "gen_cost_usd": 0.154821}
|
||||
{"instance_id": "sympy__sympy-13031", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 186.4, "eval_duration_s": 28.7, "gen_cost_usd": 0.200958}
|
||||
{"instance_id": "sympy__sympy-13043", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 233.7, "eval_duration_s": 24.5, "gen_cost_usd": 0.234324}
|
||||
{"instance_id": "sympy__sympy-13146", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.1, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "sympy__sympy-13177", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 57.8, "eval_duration_s": 20.8, "gen_cost_usd": 0.041253}
|
||||
{"instance_id": "sympy__sympy-13437", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 108.1, "eval_duration_s": 30.6, "gen_cost_usd": 0.111075}
|
||||
{"instance_id": "sympy__sympy-13471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 88.5, "eval_duration_s": 26.7, "gen_cost_usd": 0.060327}
|
||||
{"instance_id": "sympy__sympy-13480", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 44.3, "eval_duration_s": 28.1, "gen_cost_usd": 0.017181}
|
||||
{"instance_id": "sympy__sympy-13647", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 49.2, "eval_duration_s": 24.7, "gen_cost_usd": 0.03015}
|
||||
{"instance_id": "sympy__sympy-13773", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 123.7, "eval_duration_s": 23.8, "gen_cost_usd": 0.100059}
|
||||
{"instance_id": "sympy__sympy-13895", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.4, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "sympy__sympy-13915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 280.7, "eval_duration_s": 21.3, "gen_cost_usd": 0.258333}
|
||||
{"instance_id": "sympy__sympy-13971", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.9, "eval_duration_s": 21.6, "gen_cost_usd": 0.065253}
|
||||
{"instance_id": "sympy__sympy-14024", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 504.7, "eval_duration_s": 27.6, "gen_cost_usd": 0.63942}
|
||||
{"instance_id": "sympy__sympy-14308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 399.1, "eval_duration_s": 23.8, "gen_cost_usd": 0.501777}
|
||||
{"instance_id": "sympy__sympy-14317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.7, "eval_duration_s": 25.0, "gen_cost_usd": 0.230826}
|
||||
{"instance_id": "sympy__sympy-14396", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.4, "eval_duration_s": 64.7, "gen_cost_usd": 0.073446}
|
||||
{"instance_id": "sympy__sympy-14774", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 28.0, "eval_duration_s": 24.2, "gen_cost_usd": 0.011418}
|
||||
{"instance_id": "sympy__sympy-14817", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 176.8, "eval_duration_s": 24.8, "gen_cost_usd": 0.141678}
|
||||
{"instance_id": "sympy__sympy-15011", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 359.9, "eval_duration_s": 23.6, "gen_cost_usd": 0.382536}
|
||||
{"instance_id": "sympy__sympy-15308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 101.0, "eval_duration_s": 45.5, "gen_cost_usd": 0.080235}
|
||||
{"instance_id": "sympy__sympy-15345", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 90.5, "eval_duration_s": 21.5, "gen_cost_usd": 0.055428}
|
||||
{"instance_id": "sympy__sympy-15346", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 72.0, "eval_duration_s": 42.5, "gen_cost_usd": 0.040305}
|
||||
{"instance_id": "sympy__sympy-15609", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 91.8, "eval_duration_s": 22.8, "gen_cost_usd": 0.044412}
|
||||
{"instance_id": "sympy__sympy-15678", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 265.4, "eval_duration_s": 27.1, "gen_cost_usd": 0.271749}
|
||||
{"instance_id": "sympy__sympy-16106", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 136.7, "eval_duration_s": 23.0, "gen_cost_usd": 0.181446}
|
||||
{"instance_id": "sympy__sympy-16281", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.7, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "sympy__sympy-16503", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.1, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "sympy__sympy-16792", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 205.4, "eval_duration_s": 22.9, "gen_cost_usd": 0.194646}
|
||||
{"instance_id": "sympy__sympy-16988", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.2, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "sympy__sympy-17022", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 243.2, "eval_duration_s": 18.7, "gen_cost_usd": 0.260619}
|
||||
{"instance_id": "sympy__sympy-17139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 91.4, "eval_duration_s": 118.4, "gen_cost_usd": 0.061002}
|
||||
{"instance_id": "sympy__sympy-17630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 508.8, "eval_duration_s": 47.8, "gen_cost_usd": 0.607587}
|
||||
{"instance_id": "sympy__sympy-17655", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 168.3, "eval_duration_s": 34.1, "gen_cost_usd": 0.147522}
|
||||
{"instance_id": "sympy__sympy-18057", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 124.7, "eval_duration_s": 115.2, "gen_cost_usd": 0.085764}
|
||||
{"instance_id": "sympy__sympy-18087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 526.6, "eval_duration_s": 30.7, "gen_cost_usd": 0.547908}
|
||||
{"instance_id": "sympy__sympy-18189", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 61.4, "eval_duration_s": 41.2, "gen_cost_usd": 0.020223}
|
||||
{"instance_id": "sympy__sympy-18199", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 90.8, "eval_duration_s": 90.1, "gen_cost_usd": 0.075705}
|
||||
{"instance_id": "sympy__sympy-18532", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 131.7, "eval_duration_s": 25.8, "gen_cost_usd": 0.089325}
|
||||
{"instance_id": "sympy__sympy-18621", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.9, "eval_duration_s": 24.9, "gen_cost_usd": 0.033663}
|
||||
{"instance_id": "sympy__sympy-18698", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 242.7, "eval_duration_s": 28.2, "gen_cost_usd": 0.215727}
|
||||
{"instance_id": "sympy__sympy-18835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 86.0, "eval_duration_s": 21.0, "gen_cost_usd": 0.061776}
|
||||
{"instance_id": "sympy__sympy-19007", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 226.1, "eval_duration_s": 22.7, "gen_cost_usd": 0.21489}
|
||||
{"instance_id": "sympy__sympy-19254", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 375.9, "eval_duration_s": 54.3, "gen_cost_usd": 0.429435}
|
||||
{"instance_id": "sympy__sympy-19487", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.0, "eval_duration_s": 28.3, "gen_cost_usd": 0.059358}
|
||||
{"instance_id": "sympy__sympy-20049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 276.0, "eval_duration_s": 21.7, "gen_cost_usd": 0.301704}
|
||||
{"instance_id": "sympy__sympy-20154", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.8, "eval_duration_s": 20.9, "gen_cost_usd": 0.05976}
|
||||
{"instance_id": "sympy__sympy-20212", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 181.0, "eval_duration_s": 35.0, "gen_cost_usd": 0.157905}
|
||||
{"instance_id": "sympy__sympy-20322", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 95.4, "eval_duration_s": 34.1, "gen_cost_usd": 0.067455}
|
||||
{"instance_id": "sympy__sympy-20442", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.0, "eval_duration_s": 22.7, "gen_cost_usd": 0.080367}
|
||||
{"instance_id": "sympy__sympy-20590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 52.0, "eval_duration_s": 28.1, "gen_cost_usd": 0.030279}
|
||||
{"instance_id": "sympy__sympy-20639", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 177.2, "eval_duration_s": 35.7, "gen_cost_usd": 0.192159}
|
||||
{"instance_id": "sympy__sympy-21055", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 101.9, "eval_duration_s": 26.8, "gen_cost_usd": 0.095226}
|
||||
{"instance_id": "sympy__sympy-21171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 79.0, "eval_duration_s": 25.2, "gen_cost_usd": 0.057663}
|
||||
{"instance_id": "sympy__sympy-21379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 82.4, "eval_duration_s": 38.3, "gen_cost_usd": 0.064461}
|
||||
{"instance_id": "sympy__sympy-21612", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 72.3, "eval_duration_s": 22.2, "gen_cost_usd": 0.041337}
|
||||
{"instance_id": "sympy__sympy-21614", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 102.2, "eval_duration_s": 23.2, "gen_cost_usd": 0.081867}
|
||||
{"instance_id": "sympy__sympy-21627", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 59.3, "eval_duration_s": 26.2, "gen_cost_usd": 0.041439}
|
||||
{"instance_id": "sympy__sympy-21847", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.1, "eval_duration_s": 22.0, "gen_cost_usd": 0.036519}
|
||||
{"instance_id": "sympy__sympy-22005", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.8, "eval_duration_s": 26.9, "gen_cost_usd": 0.154881}
|
||||
{"instance_id": "sympy__sympy-22714", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 117.6, "eval_duration_s": 24.2, "gen_cost_usd": 0.078186}
|
||||
{"instance_id": "sympy__sympy-22840", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 221.2, "eval_duration_s": 28.3, "gen_cost_usd": 0.27489}
|
||||
{"instance_id": "sympy__sympy-23117", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 57.5, "eval_duration_s": 43.0, "gen_cost_usd": 0.033213}
|
||||
{"instance_id": "sympy__sympy-23191", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.3, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "sympy__sympy-23262", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 124.1, "eval_duration_s": 20.9, "gen_cost_usd": 0.160128}
|
||||
{"instance_id": "sympy__sympy-24066", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 121.7, "eval_duration_s": 24.1, "gen_cost_usd": 0.109968}
|
||||
{"instance_id": "sympy__sympy-24102", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 120.4, "eval_duration_s": 20.7, "gen_cost_usd": 0.151842}
|
||||
{"instance_id": "sympy__sympy-24152", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.5, "eval_duration_s": 21.5, "gen_cost_usd": 0.075042}
|
||||
{"instance_id": "sympy__sympy-24213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 58.7, "eval_duration_s": 23.3, "gen_cost_usd": 0.047781}
|
||||
{"instance_id": "sympy__sympy-24909", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 191.4, "eval_duration_s": 26.7, "gen_cost_usd": 0.200436}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
{
|
||||
"run_name": "sonnet-baseline-20260316",
|
||||
"timestamp": "2026-03-16T15:18:35.681947+00:00",
|
||||
"model": "claude-sonnet-4-6",
|
||||
"provider": "anthropic",
|
||||
"fabro_version": "fabro 0.5.0 (4c6ad4d 2026-03-16)",
|
||||
"description": "Sonnet 4.6 baseline, default prompt, 2 CPU / 4 GB, 10min timeout",
|
||||
"notes": "",
|
||||
"total_instances": 300,
|
||||
"patched": 281,
|
||||
"patch_pct": 93.7,
|
||||
"resolved": 167,
|
||||
"resolve_pct": 55.7,
|
||||
"total_gen_cost_usd": 39.78,
|
||||
"avg_gen_cost_usd": 0.1326,
|
||||
"gen_wall_time_s": 993.6,
|
||||
"eval_wall_time_s": 642.3,
|
||||
"avg_gen_duration_s": 189.7,
|
||||
"avg_eval_duration_s": 47.8,
|
||||
"per_repo": {
|
||||
"astropy/astropy": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 1,
|
||||
"resolve_pct": 16.7
|
||||
},
|
||||
"django/django": {
|
||||
"total": 114,
|
||||
"patched": 111,
|
||||
"resolved": 86,
|
||||
"resolve_pct": 75.4
|
||||
},
|
||||
"matplotlib/matplotlib": {
|
||||
"total": 23,
|
||||
"patched": 20,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"mwaskom/seaborn": {
|
||||
"total": 4,
|
||||
"patched": 4,
|
||||
"resolved": 2,
|
||||
"resolve_pct": 50.0
|
||||
},
|
||||
"pallets/flask": {
|
||||
"total": 3,
|
||||
"patched": 3,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"psf/requests": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 5,
|
||||
"resolve_pct": 83.3
|
||||
},
|
||||
"pydata/xarray": {
|
||||
"total": 5,
|
||||
"patched": 4,
|
||||
"resolved": 1,
|
||||
"resolve_pct": 20.0
|
||||
},
|
||||
"pylint-dev/pylint": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"pytest-dev/pytest": {
|
||||
"total": 17,
|
||||
"patched": 16,
|
||||
"resolved": 11,
|
||||
"resolve_pct": 64.7
|
||||
},
|
||||
"scikit-learn/scikit-learn": {
|
||||
"total": 23,
|
||||
"patched": 20,
|
||||
"resolved": 14,
|
||||
"resolve_pct": 60.9
|
||||
},
|
||||
"sphinx-doc/sphinx": {
|
||||
"total": 16,
|
||||
"patched": 15,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"sympy/sympy": {
|
||||
"total": 77,
|
||||
"patched": 70,
|
||||
"resolved": 47,
|
||||
"resolve_pct": 61.0
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue