diff --git a/evals/swe-bench/scoreboard/gpt54-20min-20260316/README.md b/evals/swe-bench/scoreboard/gpt54-20min-20260316/README.md new file mode 100644 index 000000000..f965a03a6 --- /dev/null +++ b/evals/swe-bench/scoreboard/gpt54-20min-20260316/README.md @@ -0,0 +1,38 @@ +# gpt54-20min-20260316 + +**Date:** 2026-03-17 +**Model:** gpt-5.4 (openai) +**Fabro:** fabro 0.5.0 (b185210 2026-03-16) + +## Description + +GPT-5.4, 20min timeout, 4 CPU / 8 GB, default prompt, 10 concurrency + +## Results + +| Metric | Value | +|--------|-------| +| Instances | 300 | +| Patched | 297 (99.0%) | +| **Resolved** | **197 (65.7%)** | +| Total gen cost | $718.62 | +| Avg gen cost | $2.3954/instance | +| Gen wall time | 1058.9s | +| Eval wall time | 728.0s | + +## Per-repo breakdown + +| Repo | Resolved | Total | Rate | +|------|----------|-------|------| +| astropy/astropy | 1 | 6 | 16.7% | +| django/django | 102 | 114 | 89.5% | +| matplotlib/matplotlib | 1 | 23 | 4.3% | +| mwaskom/seaborn | 3 | 4 | 75.0% | +| pallets/flask | 0 | 3 | 0.0% | +| psf/requests | 4 | 6 | 66.7% | +| pydata/xarray | 3 | 5 | 60.0% | +| pylint-dev/pylint | 0 | 6 | 0.0% | +| pytest-dev/pytest | 15 | 17 | 88.2% | +| scikit-learn/scikit-learn | 18 | 23 | 78.3% | +| sphinx-doc/sphinx | 0 | 16 | 0.0% | +| sympy/sympy | 50 | 77 | 64.9% | diff --git a/evals/swe-bench/scoreboard/gpt54-20min-20260316/instances.jsonl b/evals/swe-bench/scoreboard/gpt54-20min-20260316/instances.jsonl new file mode 100644 index 000000000..7709757f2 --- /dev/null +++ b/evals/swe-bench/scoreboard/gpt54-20min-20260316/instances.jsonl @@ -0,0 +1,300 @@ +{"instance_id": "astropy__astropy-12907", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 156.4, "eval_duration_s": 57.9, "gen_cost_usd": 0.68949} +{"instance_id": "astropy__astropy-14182", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 216.5, "eval_duration_s": 58.3, "gen_cost_usd": 1.02157} +{"instance_id": "astropy__astropy-14365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 111.4, "eval_duration_s": 44.5, "gen_cost_usd": 0.50892} +{"instance_id": "astropy__astropy-14995", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 153.0, "eval_duration_s": 41.9, "gen_cost_usd": 1.359522} +{"instance_id": "astropy__astropy-6938", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 234.2, "eval_duration_s": 86.6, "gen_cost_usd": 1.586958} +{"instance_id": "astropy__astropy-7746", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 252.1, "eval_duration_s": 82.3, "gen_cost_usd": 2.056402} +{"instance_id": "django__django-10914", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 204.1, "eval_duration_s": 23.5, "gen_cost_usd": 1.74649} +{"instance_id": "django__django-10924", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 218.3, "eval_duration_s": 20.6, "gen_cost_usd": 2.066693} +{"instance_id": "django__django-11001", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 204.0, "eval_duration_s": 20.5, "gen_cost_usd": 2.623145} +{"instance_id": "django__django-11019", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 323.9, "eval_duration_s": 26.4, "gen_cost_usd": 3.351723} +{"instance_id": "django__django-11039", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 97.7, "eval_duration_s": 20.3, "gen_cost_usd": 0.493453} +{"instance_id": "django__django-11049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 90.5, "eval_duration_s": 24.1, "gen_cost_usd": 1.003858} +{"instance_id": "django__django-11099", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.6, "eval_duration_s": 24.1, "gen_cost_usd": 0.230558} +{"instance_id": "django__django-11133", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 151.3, "eval_duration_s": 22.6, "gen_cost_usd": 2.045695} +{"instance_id": "django__django-11179", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 87.8, "eval_duration_s": 21.1, "gen_cost_usd": 0.63602} +{"instance_id": "django__django-11283", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 101.0, "eval_duration_s": 20.7, "gen_cost_usd": 0.585305} +{"instance_id": "django__django-11422", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 168.8, "eval_duration_s": 21.1, "gen_cost_usd": 1.267515} +{"instance_id": "django__django-11564", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 242.8, "eval_duration_s": 21.9, "gen_cost_usd": 4.189642} +{"instance_id": "django__django-11583", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 84.5, "eval_duration_s": 21.1, "gen_cost_usd": 0.807007} +{"instance_id": "django__django-11620", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 149.7, "eval_duration_s": 33.3, "gen_cost_usd": 2.0899} +{"instance_id": "django__django-11630", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 174.2, "eval_duration_s": 22.7, "gen_cost_usd": 2.532952} +{"instance_id": "django__django-11742", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 219.4, "eval_duration_s": 24.5, "gen_cost_usd": 3.391935} +{"instance_id": "django__django-11797", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.4, "eval_duration_s": 22.0, "gen_cost_usd": 1.09043} +{"instance_id": "django__django-11815", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 288.2, "eval_duration_s": 19.8, "gen_cost_usd": 3.4485} +{"instance_id": "django__django-11848", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 192.9, "eval_duration_s": 29.9, "gen_cost_usd": 1.8393} +{"instance_id": "django__django-11905", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 191.2, "eval_duration_s": 18.9, "gen_cost_usd": 1.893388} +{"instance_id": "django__django-11910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 342.8, "eval_duration_s": 21.6, "gen_cost_usd": 4.935982} +{"instance_id": "django__django-11964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 117.9, "eval_duration_s": 21.4, "gen_cost_usd": 0.678972} +{"instance_id": "django__django-11999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 227.4, "eval_duration_s": 24.5, "gen_cost_usd": 4.058368} +{"instance_id": "django__django-12113", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 283.4, "eval_duration_s": 24.8, "gen_cost_usd": 2.132203} +{"instance_id": "django__django-12125", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 241.3, "eval_duration_s": 21.0, "gen_cost_usd": 2.988833} +{"instance_id": "django__django-12184", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 172.3, "eval_duration_s": 22.0, "gen_cost_usd": 1.640532} +{"instance_id": "django__django-12284", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 136.7, "eval_duration_s": 24.6, "gen_cost_usd": 1.006025} +{"instance_id": "django__django-12286", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 117.4, "eval_duration_s": 22.0, "gen_cost_usd": 0.7201} +{"instance_id": "django__django-12308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 128.4, "eval_duration_s": 24.0, "gen_cost_usd": 1.079808} +{"instance_id": "django__django-12453", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 142.7, "eval_duration_s": 23.6, "gen_cost_usd": 1.32549} +{"instance_id": "django__django-12470", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 143.2, "eval_duration_s": 24.0, "gen_cost_usd": 1.422133} +{"instance_id": "django__django-12497", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 134.3, "eval_duration_s": 20.4, "gen_cost_usd": 1.311937} +{"instance_id": "django__django-12589", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 383.1, "eval_duration_s": 16.5, "gen_cost_usd": 5.616935} +{"instance_id": "django__django-12700", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 182.1, "eval_duration_s": 26.0, "gen_cost_usd": 0.970707} +{"instance_id": "django__django-12708", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 142.5, "eval_duration_s": 24.7, "gen_cost_usd": 1.459785} +{"instance_id": "django__django-12747", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 249.0, "eval_duration_s": 21.0, "gen_cost_usd": 3.40343} +{"instance_id": "django__django-12856", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 164.2, "eval_duration_s": 21.0, "gen_cost_usd": 3.222548} +{"instance_id": "django__django-12908", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.3, "eval_duration_s": 24.7, "gen_cost_usd": 0.631115} +{"instance_id": "django__django-12915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 203.5, "eval_duration_s": 25.5, "gen_cost_usd": 1.895042} +{"instance_id": "django__django-12983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 174.5, "eval_duration_s": 22.6, "gen_cost_usd": 1.108305} +{"instance_id": "django__django-13028", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 178.7, "eval_duration_s": 24.7, "gen_cost_usd": 1.835417} +{"instance_id": "django__django-13033", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 188.6, "eval_duration_s": 23.6, "gen_cost_usd": 1.128732} +{"instance_id": "django__django-13158", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 288.7, "eval_duration_s": 16.5, "gen_cost_usd": 1.636007} +{"instance_id": "django__django-13220", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 517.7, "eval_duration_s": 15.9, "gen_cost_usd": 3.334192} +{"instance_id": "django__django-13230", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 225.9, "eval_duration_s": 28.3, "gen_cost_usd": 1.709305} +{"instance_id": "django__django-13265", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 307.4, "eval_duration_s": 24.2, "gen_cost_usd": 2.973772} +{"instance_id": "django__django-13315", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 305.4, "eval_duration_s": 21.2, "gen_cost_usd": 2.190392} +{"instance_id": "django__django-13321", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 140.6, "eval_duration_s": 22.7, "gen_cost_usd": 0.715813} +{"instance_id": "django__django-13401", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 238.6, "eval_duration_s": 18.7, "gen_cost_usd": 3.077323} +{"instance_id": "django__django-13447", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 128.3, "eval_duration_s": 22.9, "gen_cost_usd": 1.571862} +{"instance_id": "django__django-13448", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 201.5, "eval_duration_s": 19.8, "gen_cost_usd": 2.18409} +{"instance_id": "django__django-13551", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 185.3, "eval_duration_s": 17.2, "gen_cost_usd": 1.555245} +{"instance_id": "django__django-13590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 118.1, "eval_duration_s": 18.8, "gen_cost_usd": 0.943948} +{"instance_id": "django__django-13658", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 149.7, "eval_duration_s": 46.3, "gen_cost_usd": 1.382618} +{"instance_id": "django__django-13660", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 416.6, "eval_duration_s": 24.6, "gen_cost_usd": 1.582442} +{"instance_id": "django__django-13710", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 184.9, "eval_duration_s": 28.1, "gen_cost_usd": 1.439662} +{"instance_id": "django__django-13757", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 163.9, "eval_duration_s": 23.8, "gen_cost_usd": 1.29925} +{"instance_id": "django__django-13768", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 252.6, "eval_duration_s": 20.8, "gen_cost_usd": 1.206122} +{"instance_id": "django__django-13925", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 371.4, "eval_duration_s": 19.8, "gen_cost_usd": 4.986515} +{"instance_id": "django__django-13933", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 305.2, "eval_duration_s": 25.2, "gen_cost_usd": 1.93721} +{"instance_id": "django__django-13964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 422.9, "eval_duration_s": 18.4, "gen_cost_usd": 2.981083} +{"instance_id": "django__django-14016", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 223.5, "eval_duration_s": 24.0, "gen_cost_usd": 0.757025} +{"instance_id": "django__django-14017", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 173.9, "eval_duration_s": 27.2, "gen_cost_usd": 2.287845} +{"instance_id": "django__django-14155", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 149.6, "eval_duration_s": 24.4, "gen_cost_usd": 2.706917} +{"instance_id": "django__django-14238", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 123.0, "eval_duration_s": 24.0, "gen_cost_usd": 0.999713} +{"instance_id": "django__django-14382", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 117.7, "eval_duration_s": 55.6, "gen_cost_usd": 0.84843} +{"instance_id": "django__django-14411", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 151.9, "eval_duration_s": 29.0, "gen_cost_usd": 1.379345} +{"instance_id": "django__django-14534", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 225.8, "eval_duration_s": 24.6, "gen_cost_usd": 3.558675} +{"instance_id": "django__django-14580", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 209.7, "eval_duration_s": 22.1, "gen_cost_usd": 2.21317} +{"instance_id": "django__django-14608", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 182.1, "eval_duration_s": 41.3, "gen_cost_usd": 1.946575} +{"instance_id": "django__django-14667", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 108.3, "eval_duration_s": 25.2, "gen_cost_usd": 1.036218} +{"instance_id": "django__django-14672", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 204.0, "eval_duration_s": 24.6, "gen_cost_usd": 1.98062} +{"instance_id": "django__django-14730", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 303.7, "eval_duration_s": 28.6, "gen_cost_usd": 7.56565} +{"instance_id": "django__django-14752", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.6, "eval_duration_s": 24.9, "gen_cost_usd": 0.838472} +{"instance_id": "django__django-14787", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 118.2, "eval_duration_s": 21.9, "gen_cost_usd": 0.85976} +{"instance_id": "django__django-14855", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 184.6, "eval_duration_s": 140.6, "gen_cost_usd": 1.72946} +{"instance_id": "django__django-14915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 127.3, "eval_duration_s": 22.2, "gen_cost_usd": 0.865178} +{"instance_id": "django__django-14997", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 186.1, "eval_duration_s": 27.0, "gen_cost_usd": 3.71334} +{"instance_id": "django__django-14999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 248.0, "eval_duration_s": 26.5, "gen_cost_usd": 3.08163} +{"instance_id": "django__django-15061", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 130.1, "eval_duration_s": 22.1, "gen_cost_usd": 1.42632} +{"instance_id": "django__django-15202", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 131.9, "eval_duration_s": 26.5, "gen_cost_usd": 1.068383} +{"instance_id": "django__django-15213", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 249.3, "eval_duration_s": 25.3, "gen_cost_usd": 2.54585} +{"instance_id": "django__django-15252", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 219.4, "eval_duration_s": 26.0, "gen_cost_usd": 3.1585} +{"instance_id": "django__django-15320", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 280.7, "eval_duration_s": 26.7, "gen_cost_usd": 1.483623} +{"instance_id": "django__django-15347", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 111.2, "eval_duration_s": 27.4, "gen_cost_usd": 1.169695} +{"instance_id": "django__django-15388", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 84.6, "eval_duration_s": 26.6, "gen_cost_usd": 0.490283} +{"instance_id": "django__django-15400", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 118.6, "eval_duration_s": 23.9, "gen_cost_usd": 0.956143} +{"instance_id": "django__django-15498", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 74.5, "eval_duration_s": 25.8, "gen_cost_usd": 0.403625} +{"instance_id": "django__django-15695", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 155.5, "eval_duration_s": 26.1, "gen_cost_usd": 1.719585} +{"instance_id": "django__django-15738", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 323.4, "eval_duration_s": 23.5, "gen_cost_usd": 4.19265} +{"instance_id": "django__django-15781", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 118.2, "eval_duration_s": 26.6, "gen_cost_usd": 1.03943} +{"instance_id": "django__django-15789", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 210.8, "eval_duration_s": 25.4, "gen_cost_usd": 1.949002} +{"instance_id": "django__django-15790", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.8, "eval_duration_s": 26.4, "gen_cost_usd": 0.441482} +{"instance_id": "django__django-15814", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 172.0, "eval_duration_s": 23.0, "gen_cost_usd": 1.644645} +{"instance_id": "django__django-15819", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 165.0, "eval_duration_s": 35.9, "gen_cost_usd": 1.612593} +{"instance_id": "django__django-15851", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 106.8, "eval_duration_s": 28.8, "gen_cost_usd": 0.45913} +{"instance_id": "django__django-15902", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 148.8, "eval_duration_s": 26.2, "gen_cost_usd": 1.698745} +{"instance_id": "django__django-15996", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 267.4, "eval_duration_s": 21.9, "gen_cost_usd": 2.409298} +{"instance_id": "django__django-16041", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.6, "eval_duration_s": 26.3, "gen_cost_usd": 0.642478} +{"instance_id": "django__django-16046", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 173.3, "eval_duration_s": 28.5, "gen_cost_usd": 2.171883} +{"instance_id": "django__django-16139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 479.0, "eval_duration_s": 18.9, "gen_cost_usd": 7.642408} +{"instance_id": "django__django-16229", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 166.8, "eval_duration_s": 22.8, "gen_cost_usd": 3.63772} +{"instance_id": "django__django-16255", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 173.1, "eval_duration_s": 30.7, "gen_cost_usd": 2.244095} +{"instance_id": "django__django-16379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 105.4, "eval_duration_s": 95.2, "gen_cost_usd": 0.504205} +{"instance_id": "django__django-16400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 215.1, "eval_duration_s": 25.1, "gen_cost_usd": 2.803183} +{"instance_id": "django__django-16408", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 200.6, "eval_duration_s": 42.8, "gen_cost_usd": 1.453512} +{"instance_id": "django__django-16527", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 162.0, "eval_duration_s": 23.7, "gen_cost_usd": 1.29912} +{"instance_id": "django__django-16595", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 144.6, "eval_duration_s": 23.6, "gen_cost_usd": 1.238158} +{"instance_id": "django__django-16816", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 119.7, "eval_duration_s": 19.6, "gen_cost_usd": 0.859857} +{"instance_id": "django__django-16820", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 250.9, "eval_duration_s": 18.7, "gen_cost_usd": 3.427695} +{"instance_id": "django__django-16873", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 159.4, "eval_duration_s": 20.5, "gen_cost_usd": 0.966745} +{"instance_id": "django__django-16910", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 289.6, "eval_duration_s": 20.2, "gen_cost_usd": 4.12244} +{"instance_id": "django__django-17051", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 250.4, "eval_duration_s": 24.2, "gen_cost_usd": 2.20599} +{"instance_id": "django__django-17087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.0, "eval_duration_s": 18.6, "gen_cost_usd": 0.68301} +{"instance_id": "matplotlib__matplotlib-18869", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 424.3, "eval_duration_s": 153.3, "gen_cost_usd": 2.738025} +{"instance_id": "matplotlib__matplotlib-22711", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 281.5, "eval_duration_s": 107.3, "gen_cost_usd": 1.719018} +{"instance_id": "matplotlib__matplotlib-22835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 340.4, "eval_duration_s": 78.3, "gen_cost_usd": 1.971585} +{"instance_id": "matplotlib__matplotlib-23299", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 511.4, "eval_duration_s": 102.7, "gen_cost_usd": 4.936632} +{"instance_id": "matplotlib__matplotlib-23314", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 407.5, "eval_duration_s": 85.0, "gen_cost_usd": 5.010198} +{"instance_id": "matplotlib__matplotlib-23476", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 308.0, "eval_duration_s": 93.5, "gen_cost_usd": 3.198268} +{"instance_id": "matplotlib__matplotlib-23562", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 501.2, "eval_duration_s": 81.5, "gen_cost_usd": 5.124712} +{"instance_id": "matplotlib__matplotlib-23563", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 515.6, "eval_duration_s": 119.4, "gen_cost_usd": 9.287693} +{"instance_id": "matplotlib__matplotlib-23913", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 284.9, "eval_duration_s": 120.9, "gen_cost_usd": 2.346852} +{"instance_id": "matplotlib__matplotlib-23964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 302.1, "eval_duration_s": 86.5, "gen_cost_usd": 2.06983} +{"instance_id": "matplotlib__matplotlib-23987", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 650.4, "eval_duration_s": 97.4, "gen_cost_usd": 4.2867} +{"instance_id": "matplotlib__matplotlib-24149", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 297.2, "eval_duration_s": 81.7, "gen_cost_usd": 2.349895} +{"instance_id": "matplotlib__matplotlib-24265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 317.5, "eval_duration_s": 89.2, "gen_cost_usd": 1.178287} +{"instance_id": "matplotlib__matplotlib-24334", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 296.0, "eval_duration_s": 110.8, "gen_cost_usd": 1.890707} +{"instance_id": "matplotlib__matplotlib-24970", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 358.1, "eval_duration_s": 86.8, "gen_cost_usd": 1.47285} +{"instance_id": "matplotlib__matplotlib-25079", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 377.0, "eval_duration_s": 101.8, "gen_cost_usd": 2.334758} +{"instance_id": "matplotlib__matplotlib-25311", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 325.6, "eval_duration_s": 85.2, "gen_cost_usd": 2.203288} +{"instance_id": "matplotlib__matplotlib-25332", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 416.2, "eval_duration_s": 87.8, "gen_cost_usd": 3.060293} +{"instance_id": "matplotlib__matplotlib-25433", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 372.8, "eval_duration_s": 78.1, "gen_cost_usd": 2.71822} +{"instance_id": "matplotlib__matplotlib-25442", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 271.0, "eval_duration_s": 114.0, "gen_cost_usd": 1.949798} +{"instance_id": "matplotlib__matplotlib-25498", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 459.4, "eval_duration_s": 109.6, "gen_cost_usd": 4.836943} +{"instance_id": "matplotlib__matplotlib-26011", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 512.5, "eval_duration_s": 81.9, "gen_cost_usd": 0.818145} +{"instance_id": "matplotlib__matplotlib-26020", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 554.8, "eval_duration_s": 94.5, "gen_cost_usd": null} +{"instance_id": "mwaskom__seaborn-2848", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 258.6, "eval_duration_s": 21.4, "gen_cost_usd": 5.149657} +{"instance_id": "mwaskom__seaborn-3010", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 124.3, "eval_duration_s": 13.1, "gen_cost_usd": 0.786167} +{"instance_id": "mwaskom__seaborn-3190", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 143.5, "eval_duration_s": 10.5, "gen_cost_usd": 2.27939} +{"instance_id": "mwaskom__seaborn-3407", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 180.3, "eval_duration_s": 86.4, "gen_cost_usd": null} +{"instance_id": "pallets__flask-4045", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 237.1, "eval_duration_s": 7.2, "gen_cost_usd": null} +{"instance_id": "pallets__flask-4992", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 180.8, "eval_duration_s": 6.2, "gen_cost_usd": null} +{"instance_id": "pallets__flask-5063", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 188.9, "eval_duration_s": 6.6, "gen_cost_usd": 1.964435} +{"instance_id": "psf__requests-1963", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 178.6, "eval_duration_s": 22.8, "gen_cost_usd": 1.876362} +{"instance_id": "psf__requests-2148", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.7, "eval_duration_s": 33.7, "gen_cost_usd": 1.031937} +{"instance_id": "psf__requests-2317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 104.7, "eval_duration_s": 659.3, "gen_cost_usd": 1.229323} +{"instance_id": "psf__requests-2674", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 187.5, "eval_duration_s": 45.3, "gen_cost_usd": 2.603467} +{"instance_id": "psf__requests-3362", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 181.7, "eval_duration_s": 7.2, "gen_cost_usd": 2.909263} +{"instance_id": "psf__requests-863", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 124.0, "eval_duration_s": 29.9, "gen_cost_usd": 1.170713} +{"instance_id": "pydata__xarray-3364", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 177.1, "eval_duration_s": 10.5, "gen_cost_usd": 4.13205} +{"instance_id": "pydata__xarray-4094", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 104.7, "eval_duration_s": 16.3, "gen_cost_usd": 1.052048} +{"instance_id": "pydata__xarray-4248", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 156.0, "eval_duration_s": 10.9, "gen_cost_usd": 1.792585} +{"instance_id": "pydata__xarray-4493", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 228.0, "eval_duration_s": 57.4, "gen_cost_usd": 3.038207} +{"instance_id": "pydata__xarray-5131", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.6, "eval_duration_s": 14.0, "gen_cost_usd": 0.312205} +{"instance_id": "pylint-dev__pylint-5859", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 150.2, "eval_duration_s": 8.1, "gen_cost_usd": null} +{"instance_id": "pylint-dev__pylint-6506", "has_patch": false, "resolved": false, "gen_status": "no_patch", "eval_status": "empty_patch", "gen_duration_s": 20.4, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "pylint-dev__pylint-7080", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 199.7, "eval_duration_s": 8.2, "gen_cost_usd": 2.883985} +{"instance_id": "pylint-dev__pylint-7114", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 362.9, "eval_duration_s": 8.1, "gen_cost_usd": 3.552423} +{"instance_id": "pylint-dev__pylint-7228", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 254.5, "eval_duration_s": 5.8, "gen_cost_usd": 5.078865} +{"instance_id": "pylint-dev__pylint-7993", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 220.8, "eval_duration_s": 12.9, "gen_cost_usd": 2.922473} +{"instance_id": "pytest-dev__pytest-11143", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 143.3, "eval_duration_s": 12.5, "gen_cost_usd": 0.929658} +{"instance_id": "pytest-dev__pytest-11148", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 193.0, "eval_duration_s": 14.7, "gen_cost_usd": 2.057307} +{"instance_id": "pytest-dev__pytest-5103", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 459.6, "eval_duration_s": 13.5, "gen_cost_usd": 7.02103} +{"instance_id": "pytest-dev__pytest-5221", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.6, "eval_duration_s": 13.5, "gen_cost_usd": 1.584342} +{"instance_id": "pytest-dev__pytest-5227", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 146.2, "eval_duration_s": 12.6, "gen_cost_usd": 1.562893} +{"instance_id": "pytest-dev__pytest-5413", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 138.7, "eval_duration_s": 29.3, "gen_cost_usd": 1.43582} +{"instance_id": "pytest-dev__pytest-5495", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 210.6, "eval_duration_s": 12.8, "gen_cost_usd": 1.736195} +{"instance_id": "pytest-dev__pytest-5692", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 189.7, "eval_duration_s": 12.8, "gen_cost_usd": 1.886415} +{"instance_id": "pytest-dev__pytest-6116", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 219.8, "eval_duration_s": 11.9, "gen_cost_usd": 1.237233} +{"instance_id": "pytest-dev__pytest-7168", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 100.1, "eval_duration_s": 11.0, "gen_cost_usd": 0.834305} +{"instance_id": "pytest-dev__pytest-7220", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 210.5, "eval_duration_s": 10.1, "gen_cost_usd": 2.004163} +{"instance_id": "pytest-dev__pytest-7373", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 179.7, "eval_duration_s": 10.6, "gen_cost_usd": 1.10926} +{"instance_id": "pytest-dev__pytest-7432", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 239.1, "eval_duration_s": 11.0, "gen_cost_usd": 2.82791} +{"instance_id": "pytest-dev__pytest-7490", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 251.4, "eval_duration_s": 10.8, "gen_cost_usd": 2.325732} +{"instance_id": "pytest-dev__pytest-8365", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 140.3, "eval_duration_s": 9.7, "gen_cost_usd": 1.597142} +{"instance_id": "pytest-dev__pytest-8906", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 130.9, "eval_duration_s": 10.3, "gen_cost_usd": 0.814678} +{"instance_id": "pytest-dev__pytest-9359", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 198.0, "eval_duration_s": 9.2, "gen_cost_usd": 2.370242} +{"instance_id": "scikit-learn__scikit-learn-10297", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 271.2, "eval_duration_s": 137.7, "gen_cost_usd": 0.924203} +{"instance_id": "scikit-learn__scikit-learn-10508", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 291.1, "eval_duration_s": 135.9, "gen_cost_usd": 2.409785} +{"instance_id": "scikit-learn__scikit-learn-10949", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 293.8, "eval_duration_s": 148.0, "gen_cost_usd": 2.026492} +{"instance_id": "scikit-learn__scikit-learn-11040", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 376.4, "eval_duration_s": 128.1, "gen_cost_usd": 4.27705} +{"instance_id": "scikit-learn__scikit-learn-11281", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 395.5, "eval_duration_s": 156.3, "gen_cost_usd": 2.887047} +{"instance_id": "scikit-learn__scikit-learn-12471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 371.0, "eval_duration_s": 145.0, "gen_cost_usd": 0.883628} +{"instance_id": "scikit-learn__scikit-learn-13142", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 353.0, "eval_duration_s": 156.5, "gen_cost_usd": 1.476478} +{"instance_id": "scikit-learn__scikit-learn-13241", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 436.6, "eval_duration_s": 179.1, "gen_cost_usd": 2.315725} +{"instance_id": "scikit-learn__scikit-learn-13439", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 251.6, "eval_duration_s": 172.6, "gen_cost_usd": 0.720185} +{"instance_id": "scikit-learn__scikit-learn-13496", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 414.1, "eval_duration_s": 215.2, "gen_cost_usd": 3.170315} +{"instance_id": "scikit-learn__scikit-learn-13497", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 257.5, "eval_duration_s": 187.0, "gen_cost_usd": 0.831785} +{"instance_id": "scikit-learn__scikit-learn-13584", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 296.8, "eval_duration_s": 184.2, "gen_cost_usd": 1.356025} +{"instance_id": "scikit-learn__scikit-learn-13779", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 298.7, "eval_duration_s": 188.8, "gen_cost_usd": 1.114115} +{"instance_id": "scikit-learn__scikit-learn-14087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 350.2, "eval_duration_s": 210.8, "gen_cost_usd": 1.70672} +{"instance_id": "scikit-learn__scikit-learn-14092", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 400.8, "eval_duration_s": 211.6, "gen_cost_usd": 1.463583} +{"instance_id": "scikit-learn__scikit-learn-14894", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 328.1, "eval_duration_s": 208.3, "gen_cost_usd": 1.032423} +{"instance_id": "scikit-learn__scikit-learn-14983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 355.3, "eval_duration_s": 190.9, "gen_cost_usd": 1.30809} +{"instance_id": "scikit-learn__scikit-learn-15512", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 381.9, "eval_duration_s": 214.8, "gen_cost_usd": 2.886497} +{"instance_id": "scikit-learn__scikit-learn-15535", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 327.5, "eval_duration_s": 203.9, "gen_cost_usd": 1.558068} +{"instance_id": "scikit-learn__scikit-learn-25500", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 478.1, "eval_duration_s": 12.6, "gen_cost_usd": 3.639213} +{"instance_id": "scikit-learn__scikit-learn-25570", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 489.6, "eval_duration_s": 12.7, "gen_cost_usd": 2.035685} +{"instance_id": "scikit-learn__scikit-learn-25638", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 502.1, "eval_duration_s": 16.0, "gen_cost_usd": 3.511515} +{"instance_id": "scikit-learn__scikit-learn-25747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 641.1, "eval_duration_s": 12.6, "gen_cost_usd": 4.030765} +{"instance_id": "sphinx-doc__sphinx-10325", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 337.3, "eval_duration_s": 17.3, "gen_cost_usd": 2.854847} +{"instance_id": "sphinx-doc__sphinx-10451", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 273.9, "eval_duration_s": 15.8, "gen_cost_usd": 2.903237} +{"instance_id": "sphinx-doc__sphinx-11445", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 167.1, "eval_duration_s": 20.2, "gen_cost_usd": 1.004133} +{"instance_id": "sphinx-doc__sphinx-7686", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 338.1, "eval_duration_s": 13.9, "gen_cost_usd": 4.719258} +{"instance_id": "sphinx-doc__sphinx-7738", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 218.9, "eval_duration_s": 14.9, "gen_cost_usd": 2.844507} +{"instance_id": "sphinx-doc__sphinx-7975", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 215.6, "eval_duration_s": 15.8, "gen_cost_usd": 1.834707} +{"instance_id": "sphinx-doc__sphinx-8273", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 196.4, "eval_duration_s": 22.4, "gen_cost_usd": 2.38874} +{"instance_id": "sphinx-doc__sphinx-8282", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 153.3, "eval_duration_s": 14.7, "gen_cost_usd": 2.27822} +{"instance_id": "sphinx-doc__sphinx-8435", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 435.7, "eval_duration_s": 15.5, "gen_cost_usd": 6.256232} +{"instance_id": "sphinx-doc__sphinx-8474", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 140.9, "eval_duration_s": 17.4, "gen_cost_usd": 1.507655} +{"instance_id": "sphinx-doc__sphinx-8506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 161.6, "eval_duration_s": 19.5, "gen_cost_usd": 2.010185} +{"instance_id": "sphinx-doc__sphinx-8595", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 164.2, "eval_duration_s": 14.8, "gen_cost_usd": 1.74313} +{"instance_id": "sphinx-doc__sphinx-8627", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 345.7, "eval_duration_s": 15.6, "gen_cost_usd": 7.055722} +{"instance_id": "sphinx-doc__sphinx-8713", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 185.5, "eval_duration_s": 16.0, "gen_cost_usd": 2.45189} +{"instance_id": "sphinx-doc__sphinx-8721", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 364.2, "eval_duration_s": 23.9, "gen_cost_usd": 4.867265} +{"instance_id": "sphinx-doc__sphinx-8801", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 197.1, "eval_duration_s": 15.5, "gen_cost_usd": 2.660675} +{"instance_id": "sympy__sympy-11400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 259.1, "eval_duration_s": 24.7, "gen_cost_usd": 4.158325} +{"instance_id": "sympy__sympy-11870", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 433.0, "eval_duration_s": 559.3, "gen_cost_usd": 4.676507} +{"instance_id": "sympy__sympy-11897", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 197.2, "eval_duration_s": 22.9, "gen_cost_usd": 2.664168} +{"instance_id": "sympy__sympy-12171", "has_patch": false, "resolved": false, "gen_status": "no_patch", "eval_status": "empty_patch", "gen_duration_s": 26.2, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sympy__sympy-12236", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 374.0, "eval_duration_s": 35.4, "gen_cost_usd": 5.07947} +{"instance_id": "sympy__sympy-12419", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 204.5, "eval_duration_s": 25.6, "gen_cost_usd": 3.110212} +{"instance_id": "sympy__sympy-12454", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 132.5, "eval_duration_s": 33.9, "gen_cost_usd": 1.126958} +{"instance_id": "sympy__sympy-12481", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 169.6, "eval_duration_s": 19.5, "gen_cost_usd": 1.466727} +{"instance_id": "sympy__sympy-13031", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 229.7, "eval_duration_s": 25.9, "gen_cost_usd": 2.38991} +{"instance_id": "sympy__sympy-13043", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 234.9, "eval_duration_s": 22.1, "gen_cost_usd": 3.109522} +{"instance_id": "sympy__sympy-13146", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 257.5, "eval_duration_s": 32.6, "gen_cost_usd": 3.85205} +{"instance_id": "sympy__sympy-13177", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 234.2, "eval_duration_s": 21.6, "gen_cost_usd": 1.77465} +{"instance_id": "sympy__sympy-13437", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 102.7, "eval_duration_s": 25.9, "gen_cost_usd": 0.99173} +{"instance_id": "sympy__sympy-13471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 213.7, "eval_duration_s": 23.3, "gen_cost_usd": 2.110208} +{"instance_id": "sympy__sympy-13480", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 123.5, "eval_duration_s": 24.7, "gen_cost_usd": 0.964785} +{"instance_id": "sympy__sympy-13647", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 109.5, "eval_duration_s": 25.7, "gen_cost_usd": 1.12644} +{"instance_id": "sympy__sympy-13773", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 284.6, "eval_duration_s": 23.2, "gen_cost_usd": 2.948927} +{"instance_id": "sympy__sympy-13895", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 1215.1, "eval_duration_s": 26.3, "gen_cost_usd": 5.53604} +{"instance_id": "sympy__sympy-13915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 252.7, "eval_duration_s": 22.8, "gen_cost_usd": 6.849138} +{"instance_id": "sympy__sympy-13971", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 160.4, "eval_duration_s": 22.3, "gen_cost_usd": 1.27693} +{"instance_id": "sympy__sympy-14024", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 247.2, "eval_duration_s": 26.8, "gen_cost_usd": 2.324057} +{"instance_id": "sympy__sympy-14308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 941.6, "eval_duration_s": 23.8, "gen_cost_usd": 8.868065} +{"instance_id": "sympy__sympy-14317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 964.7, "eval_duration_s": 22.6, "gen_cost_usd": 2.57058} +{"instance_id": "sympy__sympy-14396", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 855.5, "eval_duration_s": 21.1, "gen_cost_usd": 1.434483} +{"instance_id": "sympy__sympy-14774", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 143.1, "eval_duration_s": 22.9, "gen_cost_usd": 1.07915} +{"instance_id": "sympy__sympy-14817", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 1069.6, "eval_duration_s": 24.4, "gen_cost_usd": 5.317712} +{"instance_id": "sympy__sympy-15011", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 759.3, "eval_duration_s": 22.1, "gen_cost_usd": 3.808568} +{"instance_id": "sympy__sympy-15308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 736.8, "eval_duration_s": 22.6, "gen_cost_usd": 3.068145} +{"instance_id": "sympy__sympy-15345", "has_patch": false, "resolved": false, "gen_status": "no_patch", "eval_status": "empty_patch", "gen_duration_s": 529.4, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sympy__sympy-15346", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 648.9, "eval_duration_s": 43.5, "gen_cost_usd": 2.405345} +{"instance_id": "sympy__sympy-15609", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 618.7, "eval_duration_s": 24.3, "gen_cost_usd": 1.153607} +{"instance_id": "sympy__sympy-15678", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 190.5, "eval_duration_s": 28.3, "gen_cost_usd": 1.684913} +{"instance_id": "sympy__sympy-16106", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 318.5, "eval_duration_s": 21.6, "gen_cost_usd": 6.223677} +{"instance_id": "sympy__sympy-16281", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 500.2, "eval_duration_s": 24.9, "gen_cost_usd": 9.426823} +{"instance_id": "sympy__sympy-16503", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 522.4, "eval_duration_s": 25.5, "gen_cost_usd": 13.68859} +{"instance_id": "sympy__sympy-16792", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 273.7, "eval_duration_s": 23.0, "gen_cost_usd": 5.901588} +{"instance_id": "sympy__sympy-16988", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 547.1, "eval_duration_s": 92.1, "gen_cost_usd": 9.995835} +{"instance_id": "sympy__sympy-17022", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 346.1, "eval_duration_s": 24.3, "gen_cost_usd": 9.072955} +{"instance_id": "sympy__sympy-17139", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 179.4, "eval_duration_s": 95.7, "gen_cost_usd": 2.2365} +{"instance_id": "sympy__sympy-17630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 192.5, "eval_duration_s": 26.3, "gen_cost_usd": 1.824442} +{"instance_id": "sympy__sympy-17655", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 288.8, "eval_duration_s": 30.1, "gen_cost_usd": 2.827482} +{"instance_id": "sympy__sympy-18057", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.5, "eval_duration_s": 107.5, "gen_cost_usd": 1.957335} +{"instance_id": "sympy__sympy-18087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 191.5, "eval_duration_s": 34.7, "gen_cost_usd": 4.332917} +{"instance_id": "sympy__sympy-18189", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.1, "eval_duration_s": 40.6, "gen_cost_usd": 1.321597} +{"instance_id": "sympy__sympy-18199", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 157.1, "eval_duration_s": 88.0, "gen_cost_usd": 1.38589} +{"instance_id": "sympy__sympy-18532", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 294.1, "eval_duration_s": 25.1, "gen_cost_usd": 2.481762} +{"instance_id": "sympy__sympy-18621", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 123.1, "eval_duration_s": 27.0, "gen_cost_usd": 1.304528} +{"instance_id": "sympy__sympy-18698", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 302.8, "eval_duration_s": 30.1, "gen_cost_usd": 2.683502} +{"instance_id": "sympy__sympy-18835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 309.6, "eval_duration_s": 21.5, "gen_cost_usd": 1.633262} +{"instance_id": "sympy__sympy-19007", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 261.4, "eval_duration_s": 21.9, "gen_cost_usd": 2.430398} +{"instance_id": "sympy__sympy-19254", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 357.2, "eval_duration_s": 21.4, "gen_cost_usd": 7.896662} +{"instance_id": "sympy__sympy-19487", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 149.9, "eval_duration_s": 27.9, "gen_cost_usd": 1.622415} +{"instance_id": "sympy__sympy-20049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 305.8, "eval_duration_s": 25.1, "gen_cost_usd": 6.069383} +{"instance_id": "sympy__sympy-20154", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 130.8, "eval_duration_s": 25.0, "gen_cost_usd": 1.295362} +{"instance_id": "sympy__sympy-20212", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 151.6, "eval_duration_s": 38.1, "gen_cost_usd": 1.396877} +{"instance_id": "sympy__sympy-20322", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 222.2, "eval_duration_s": 29.4, "gen_cost_usd": 2.752015} +{"instance_id": "sympy__sympy-20442", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 185.1, "eval_duration_s": 26.6, "gen_cost_usd": 1.86914} +{"instance_id": "sympy__sympy-20590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 108.8, "eval_duration_s": 21.2, "gen_cost_usd": 0.77232} +{"instance_id": "sympy__sympy-20639", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 347.5, "eval_duration_s": 26.4, "gen_cost_usd": 9.299775} +{"instance_id": "sympy__sympy-21055", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 157.2, "eval_duration_s": 28.6, "gen_cost_usd": 2.345425} +{"instance_id": "sympy__sympy-21171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 126.5, "eval_duration_s": 22.6, "gen_cost_usd": 1.164965} +{"instance_id": "sympy__sympy-21379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 198.6, "eval_duration_s": 36.7, "gen_cost_usd": 1.92909} +{"instance_id": "sympy__sympy-21612", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 120.0, "eval_duration_s": 26.8, "gen_cost_usd": 0.82701} +{"instance_id": "sympy__sympy-21614", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 164.5, "eval_duration_s": 21.3, "gen_cost_usd": 2.130688} +{"instance_id": "sympy__sympy-21627", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 152.5, "eval_duration_s": 25.5, "gen_cost_usd": 1.632562} +{"instance_id": "sympy__sympy-21847", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 139.0, "eval_duration_s": 21.6, "gen_cost_usd": 1.073387} +{"instance_id": "sympy__sympy-22005", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 203.9, "eval_duration_s": 25.2, "gen_cost_usd": 2.399485} +{"instance_id": "sympy__sympy-22714", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 249.6, "eval_duration_s": 28.6, "gen_cost_usd": 4.199228} +{"instance_id": "sympy__sympy-22840", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 158.0, "eval_duration_s": 27.7, "gen_cost_usd": 1.713005} +{"instance_id": "sympy__sympy-23117", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 191.9, "eval_duration_s": 22.8, "gen_cost_usd": 2.388033} +{"instance_id": "sympy__sympy-23191", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 345.9, "eval_duration_s": 50.4, "gen_cost_usd": 5.058347} +{"instance_id": "sympy__sympy-23262", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 188.9, "eval_duration_s": 23.2, "gen_cost_usd": 3.485375} +{"instance_id": "sympy__sympy-24066", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 226.3, "eval_duration_s": 25.2, "gen_cost_usd": 1.85473} +{"instance_id": "sympy__sympy-24102", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 219.4, "eval_duration_s": 22.3, "gen_cost_usd": 3.534125} +{"instance_id": "sympy__sympy-24152", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 148.0, "eval_duration_s": 22.7, "gen_cost_usd": 1.488573} +{"instance_id": "sympy__sympy-24213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 170.8, "eval_duration_s": 22.0, "gen_cost_usd": 1.61515} +{"instance_id": "sympy__sympy-24909", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 169.2, "eval_duration_s": 25.6, "gen_cost_usd": 1.626802} diff --git a/evals/swe-bench/scoreboard/gpt54-20min-20260316/meta.json b/evals/swe-bench/scoreboard/gpt54-20min-20260316/meta.json new file mode 100644 index 000000000..ff5314c38 --- /dev/null +++ b/evals/swe-bench/scoreboard/gpt54-20min-20260316/meta.json @@ -0,0 +1,97 @@ +{ + "run_name": "gpt54-20min-20260316", + "timestamp": "2026-03-17T00:15:33.720206+00:00", + "model": "gpt-5.4", + "provider": "openai", + "fabro_version": "fabro 0.5.0 (b185210 2026-03-16)", + "timeout_s": 1200, + "sandbox_cpu": 4, + "sandbox_memory_gb": 8, + "description": "GPT-5.4, 20min timeout, 4 CPU / 8 GB, default prompt, 10 concurrency", + "notes": "", + "total_instances": 300, + "patched": 297, + "patch_pct": 99.0, + "resolved": 197, + "resolve_pct": 65.7, + "total_gen_cost_usd": 718.62, + "avg_gen_cost_usd": 2.3954, + "gen_wall_time_s": 1058.9, + "eval_wall_time_s": 728.0, + "avg_gen_duration_s": 250.7, + "avg_eval_duration_s": 44.6, + "per_repo": { + "astropy/astropy": { + "total": 6, + "patched": 6, + "resolved": 1, + "resolve_pct": 16.7 + }, + "django/django": { + "total": 114, + "patched": 114, + "resolved": 102, + "resolve_pct": 89.5 + }, + "matplotlib/matplotlib": { + "total": 23, + "patched": 23, + "resolved": 1, + "resolve_pct": 4.3 + }, + "mwaskom/seaborn": { + "total": 4, + "patched": 4, + "resolved": 3, + "resolve_pct": 75.0 + }, + "pallets/flask": { + "total": 3, + "patched": 3, + "resolved": 0, + "resolve_pct": 0.0 + }, + "psf/requests": { + "total": 6, + "patched": 6, + "resolved": 4, + "resolve_pct": 66.7 + }, + "pydata/xarray": { + "total": 5, + "patched": 5, + "resolved": 3, + "resolve_pct": 60.0 + }, + "pylint-dev/pylint": { + "total": 6, + "patched": 5, + "resolved": 0, + "resolve_pct": 0.0 + }, + "pytest-dev/pytest": { + "total": 17, + "patched": 17, + "resolved": 15, + "resolve_pct": 88.2 + }, + "scikit-learn/scikit-learn": { + "total": 23, + "patched": 23, + "resolved": 18, + "resolve_pct": 78.3 + }, + "sphinx-doc/sphinx": { + "total": 16, + "patched": 16, + "resolved": 0, + "resolve_pct": 0.0 + }, + "sympy/sympy": { + "total": 77, + "patched": 75, + "resolved": 50, + "resolve_pct": 64.9 + } + } +} diff --git a/evals/swe-bench/scoreboard/leaderboard.json b/evals/swe-bench/scoreboard/leaderboard.json index ec540827e..2c3c231d5 100644 --- a/evals/swe-bench/scoreboard/leaderboard.json +++ b/evals/swe-bench/scoreboard/leaderboard.json @@ -1,4 +1,17 @@ [ + { + "run_name": "gpt54-20min-20260316", + "date": "2026-03-17", + "model": "gpt-5.4", + "provider": "openai", + "resolved": 197, + "total": 300, + "resolve_pct": 65.7, + "patched": 297, + "patch_pct": 99.0, + "total_cost_usd": 718.62, + "avg_cost_usd": 2.3954 + }, { "run_name": "opus-20min-20260316", "date": "2026-03-16",