From 165742d7faa01154ffdd4e4be4d28498986b5b9e Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 16 Mar 2026 13:36:26 -0400 Subject: [PATCH] Record Sonnet 4.6 20min timeout: 57.7% on SWE-Bench-Lite ($55.22) 20min timeout vs 10min: 173 vs 167 resolved (+6), patch rate 99% vs 94%. Also fixes: revert to v4 snapshots, concurrency default to 100, preflight uses actual 4 CPU per sandbox. Co-Authored-By: Claude Opus 4.6 (1M context) --- evals/swe-bench/scoreboard/leaderboard.json | 13 + .../sonnet-20min-20260316/README.md | 38 +++ .../sonnet-20min-20260316/instances.jsonl | 300 ++++++++++++++++++ .../sonnet-20min-20260316/meta.json | 97 ++++++ 4 files changed, 448 insertions(+) create mode 100644 evals/swe-bench/scoreboard/sonnet-20min-20260316/README.md create mode 100644 evals/swe-bench/scoreboard/sonnet-20min-20260316/instances.jsonl create mode 100644 evals/swe-bench/scoreboard/sonnet-20min-20260316/meta.json diff --git a/evals/swe-bench/scoreboard/leaderboard.json b/evals/swe-bench/scoreboard/leaderboard.json index 1b391c7cf..ba3f0205a 100644 --- a/evals/swe-bench/scoreboard/leaderboard.json +++ b/evals/swe-bench/scoreboard/leaderboard.json @@ -1,4 +1,17 @@ [ + { + "run_name": "sonnet-20min-20260316", + "date": "2026-03-16", + "model": "claude-sonnet-4-6", + "provider": "anthropic", + "resolved": 173, + "total": 300, + "resolve_pct": 57.7, + "patched": 297, + "patch_pct": 99.0, + "total_cost_usd": 55.22, + "avg_cost_usd": 0.1841 + }, { "run_name": "sonnet-baseline-20260316", "date": "2026-03-16", diff --git a/evals/swe-bench/scoreboard/sonnet-20min-20260316/README.md b/evals/swe-bench/scoreboard/sonnet-20min-20260316/README.md new file mode 100644 index 000000000..83e07d66c --- /dev/null +++ b/evals/swe-bench/scoreboard/sonnet-20min-20260316/README.md @@ -0,0 +1,38 @@ +# sonnet-20min-20260316 + +**Date:** 2026-03-16 +**Model:** claude-sonnet-4-6 (anthropic) +**Fabro:** fabro 0.5.0 (4c6ad4d 2026-03-16) + +## Description + +Sonnet 4.6, 20min timeout, 4 CPU / 8 GB, default prompt + +## Results + +| Metric | Value | +|--------|-------| +| Instances | 300 | +| Patched | 297 (99.0%) | +| **Resolved** | **173 (57.7%)** | +| Total gen cost | $55.22 | +| Avg gen cost | $0.1841/instance | +| Gen wall time | 1466.0s | +| Eval wall time | 682.3s | + +## Per-repo breakdown + +| Repo | Resolved | Total | Rate | +|------|----------|-------|------| +| astropy/astropy | 1 | 6 | 16.7% | +| django/django | 88 | 114 | 77.2% | +| matplotlib/matplotlib | 0 | 23 | 0.0% | +| mwaskom/seaborn | 3 | 4 | 75.0% | +| pallets/flask | 0 | 3 | 0.0% | +| psf/requests | 5 | 6 | 83.3% | +| pydata/xarray | 1 | 5 | 20.0% | +| pylint-dev/pylint | 0 | 6 | 0.0% | +| pytest-dev/pytest | 11 | 17 | 64.7% | +| scikit-learn/scikit-learn | 15 | 23 | 65.2% | +| sphinx-doc/sphinx | 0 | 16 | 0.0% | +| sympy/sympy | 49 | 77 | 63.6% | diff --git a/evals/swe-bench/scoreboard/sonnet-20min-20260316/instances.jsonl b/evals/swe-bench/scoreboard/sonnet-20min-20260316/instances.jsonl new file mode 100644 index 000000000..e1deaa522 --- /dev/null +++ b/evals/swe-bench/scoreboard/sonnet-20min-20260316/instances.jsonl @@ -0,0 +1,300 @@ +{"instance_id": "astropy__astropy-12907", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 106.8, "eval_duration_s": 56.1, "gen_cost_usd": 0.064464} +{"instance_id": "astropy__astropy-14182", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 164.7, "eval_duration_s": 64.8, "gen_cost_usd": 0.148908} +{"instance_id": "astropy__astropy-14365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 97.7, "eval_duration_s": 42.3, "gen_cost_usd": 0.078402} +{"instance_id": "astropy__astropy-14995", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 71.1, "eval_duration_s": 47.7, "gen_cost_usd": 0.037638} +{"instance_id": "astropy__astropy-6938", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.9, "eval_duration_s": 81.7, "gen_cost_usd": 0.012978} +{"instance_id": "astropy__astropy-7746", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 149.2, "eval_duration_s": 84.5, "gen_cost_usd": 0.058356} +{"instance_id": "django__django-10914", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 136.8, "eval_duration_s": 25.4, "gen_cost_usd": 0.118077} +{"instance_id": "django__django-10924", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.7, "eval_duration_s": 25.3, "gen_cost_usd": 0.134157} +{"instance_id": "django__django-11001", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 161.8, "eval_duration_s": 20.9, "gen_cost_usd": 0.121002} +{"instance_id": "django__django-11019", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 783.1, "eval_duration_s": 607.9, "gen_cost_usd": 0.576756} +{"instance_id": "django__django-11039", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 121.5, "eval_duration_s": 24.9, "gen_cost_usd": 0.141678} +{"instance_id": "django__django-11049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.4, "eval_duration_s": 21.6, "gen_cost_usd": 0.056811} +{"instance_id": "django__django-11099", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 25.6, "eval_duration_s": 104.9, "gen_cost_usd": 0.013665} +{"instance_id": "django__django-11133", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 42.1, "eval_duration_s": 86.4, "gen_cost_usd": 0.018816} +{"instance_id": "django__django-11179", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 29.2, "eval_duration_s": 20.7, "gen_cost_usd": 0.017781} +{"instance_id": "django__django-11283", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 136.9, "eval_duration_s": 21.8, "gen_cost_usd": 0.136041} +{"instance_id": "django__django-11422", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 105.8, "eval_duration_s": 25.0, "gen_cost_usd": 0.121464} +{"instance_id": "django__django-11564", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 266.0, "eval_duration_s": 17.6, "gen_cost_usd": 0.337134} +{"instance_id": "django__django-11583", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 25.1, "eval_duration_s": 22.3, "gen_cost_usd": 0.018714} +{"instance_id": "django__django-11620", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 126.9, "eval_duration_s": 23.1, "gen_cost_usd": 0.126471} +{"instance_id": "django__django-11630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 238.1, "eval_duration_s": 35.7, "gen_cost_usd": 0.259635} +{"instance_id": "django__django-11742", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 232.0, "eval_duration_s": 64.7, "gen_cost_usd": 0.268938} +{"instance_id": "django__django-11797", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 79.5, "eval_duration_s": 34.0, "gen_cost_usd": 0.066111} +{"instance_id": "django__django-11815", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 146.8, "eval_duration_s": 22.0, "gen_cost_usd": 0.134709} +{"instance_id": "django__django-11848", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 83.2, "eval_duration_s": 21.5, "gen_cost_usd": 0.069561} +{"instance_id": "django__django-11905", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 225.0, "eval_duration_s": 20.8, "gen_cost_usd": 0.132522} +{"instance_id": "django__django-11910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 527.1, "eval_duration_s": 22.7, "gen_cost_usd": 0.708078} +{"instance_id": "django__django-11964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 94.7, "eval_duration_s": 21.3, "gen_cost_usd": 0.0783} +{"instance_id": "django__django-11999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 80.2, "eval_duration_s": 33.1, "gen_cost_usd": 0.063777} +{"instance_id": "django__django-12113", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.0, "eval_duration_s": 22.5, "gen_cost_usd": 0.164424} +{"instance_id": "django__django-12125", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.3, "eval_duration_s": 24.6, "gen_cost_usd": 0.104868} +{"instance_id": "django__django-12184", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 145.3, "eval_duration_s": 24.6, "gen_cost_usd": 0.155193} +{"instance_id": "django__django-12284", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.3, "eval_duration_s": 24.0, "gen_cost_usd": 0.106953} +{"instance_id": "django__django-12286", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 60.7, "eval_duration_s": 21.1, "gen_cost_usd": 0.05334} +{"instance_id": "django__django-12308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 305.2, "eval_duration_s": 16.6, "gen_cost_usd": 0.18663} +{"instance_id": "django__django-12453", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.3, "eval_duration_s": 77.9, "gen_cost_usd": 0.114987} +{"instance_id": "django__django-12470", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 588.1, "eval_duration_s": 18.8, "gen_cost_usd": 0.611658} +{"instance_id": "django__django-12497", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.1, "eval_duration_s": 28.6, "gen_cost_usd": 0.044742} +{"instance_id": "django__django-12589", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 898.9, "eval_duration_s": 17.9, "gen_cost_usd": 0.996936} +{"instance_id": "django__django-12700", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 59.6, "eval_duration_s": 22.6, "gen_cost_usd": 0.039462} +{"instance_id": "django__django-12708", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.9, "eval_duration_s": 25.3, "gen_cost_usd": 0.109767} +{"instance_id": "django__django-12747", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 207.4, "eval_duration_s": 17.1, "gen_cost_usd": 0.199233} +{"instance_id": "django__django-12856", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.5, "eval_duration_s": 21.5, "gen_cost_usd": 0.12063} +{"instance_id": "django__django-12908", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 79.3, "eval_duration_s": 55.4, "gen_cost_usd": 0.076269} +{"instance_id": "django__django-12915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 121.1, "eval_duration_s": 38.9, "gen_cost_usd": 0.111195} +{"instance_id": "django__django-12983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.7, "eval_duration_s": 21.8, "gen_cost_usd": 0.062208} +{"instance_id": "django__django-13028", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.8, "eval_duration_s": 23.4, "gen_cost_usd": 0.015219} +{"instance_id": "django__django-13033", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 88.1, "eval_duration_s": 24.9, "gen_cost_usd": 0.050868} +{"instance_id": "django__django-13158", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 175.5, "eval_duration_s": 150.1, "gen_cost_usd": 0.19146} +{"instance_id": "django__django-13220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 174.4, "eval_duration_s": 22.3, "gen_cost_usd": 0.147219} +{"instance_id": "django__django-13230", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 99.7, "eval_duration_s": 61.9, "gen_cost_usd": 0.126726} +{"instance_id": "django__django-13265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 447.1, "eval_duration_s": 16.9, "gen_cost_usd": 0.499551} +{"instance_id": "django__django-13315", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 180.5, "eval_duration_s": 18.1, "gen_cost_usd": 0.182493} +{"instance_id": "django__django-13321", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 43.0, "eval_duration_s": 25.2, "gen_cost_usd": 0.030351} +{"instance_id": "django__django-13401", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.6, "eval_duration_s": 24.9, "gen_cost_usd": 0.065265} +{"instance_id": "django__django-13447", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.5, "eval_duration_s": 27.2, "gen_cost_usd": 0.079938} +{"instance_id": "django__django-13448", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 165.0, "eval_duration_s": 26.1, "gen_cost_usd": 0.155982} +{"instance_id": "django__django-13551", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.0, "eval_duration_s": 21.0, "gen_cost_usd": 0.063585} +{"instance_id": "django__django-13590", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 40.6, "eval_duration_s": 20.7, "gen_cost_usd": 0.019314} +{"instance_id": "django__django-13658", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 36.5, "eval_duration_s": 47.4, "gen_cost_usd": 0.018885} +{"instance_id": "django__django-13660", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 50.2, "eval_duration_s": 23.0, "gen_cost_usd": 0.044526} +{"instance_id": "django__django-13710", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 247.2, "eval_duration_s": 22.2, "gen_cost_usd": 0.280179} +{"instance_id": "django__django-13757", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 125.1, "eval_duration_s": 39.9, "gen_cost_usd": 0.1221} +{"instance_id": "django__django-13768", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 81.1, "eval_duration_s": 21.6, "gen_cost_usd": 0.061737} +{"instance_id": "django__django-13925", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 269.1, "eval_duration_s": 17.8, "gen_cost_usd": 0.279765} +{"instance_id": "django__django-13933", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 249.7, "eval_duration_s": 21.5, "gen_cost_usd": 0.292293} +{"instance_id": "django__django-13964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 41.2, "eval_duration_s": 23.4, "gen_cost_usd": 0.02547} +{"instance_id": "django__django-14016", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 280.0, "eval_duration_s": 20.8, "gen_cost_usd": 0.329361} +{"instance_id": "django__django-14017", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 330.9, "eval_duration_s": 21.4, "gen_cost_usd": 0.397476} +{"instance_id": "django__django-14155", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 88.5, "eval_duration_s": 28.8, "gen_cost_usd": 0.083484} +{"instance_id": "django__django-14238", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.9, "eval_duration_s": 38.7, "gen_cost_usd": 0.044787} +{"instance_id": "django__django-14382", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 129.2, "eval_duration_s": 55.2, "gen_cost_usd": 0.013203} +{"instance_id": "django__django-14411", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 65.1, "eval_duration_s": 25.0, "gen_cost_usd": 0.056901} +{"instance_id": "django__django-14534", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 36.6, "eval_duration_s": 24.3, "gen_cost_usd": 0.017343} +{"instance_id": "django__django-14580", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 31.3, "eval_duration_s": 24.2, "gen_cost_usd": 0.015474} +{"instance_id": "django__django-14608", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 160.7, "eval_duration_s": 46.9, "gen_cost_usd": 0.188115} +{"instance_id": "django__django-14667", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 184.3, "eval_duration_s": 20.8, "gen_cost_usd": 0.19143} +{"instance_id": "django__django-14672", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.7, "eval_duration_s": 28.9, "gen_cost_usd": 0.032706} +{"instance_id": "django__django-14730", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 290.3, "eval_duration_s": 38.3, "gen_cost_usd": 0.306333} +{"instance_id": "django__django-14752", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.3, "eval_duration_s": 26.0, "gen_cost_usd": 0.038664} +{"instance_id": "django__django-14787", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 65.5, "eval_duration_s": 37.9, "gen_cost_usd": 0.049047} +{"instance_id": "django__django-14855", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 32.5, "eval_duration_s": 135.5, "gen_cost_usd": 0.020211} +{"instance_id": "django__django-14915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.9, "eval_duration_s": 24.0, "gen_cost_usd": 0.016164} +{"instance_id": "django__django-14997", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 217.3, "eval_duration_s": 51.7, "gen_cost_usd": 0.24462} +{"instance_id": "django__django-14999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.8, "eval_duration_s": 27.2, "gen_cost_usd": 0.163908} +{"instance_id": "django__django-15061", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 138.0, "eval_duration_s": 26.1, "gen_cost_usd": 0.115611} +{"instance_id": "django__django-15202", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 165.4, "eval_duration_s": 26.6, "gen_cost_usd": 0.163416} +{"instance_id": "django__django-15213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 302.3, "eval_duration_s": 19.6, "gen_cost_usd": 0.33372} +{"instance_id": "django__django-15252", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 1106.4, "eval_duration_s": 19.3, "gen_cost_usd": 1.374828} +{"instance_id": "django__django-15320", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.6, "eval_duration_s": 28.4, "gen_cost_usd": 0.075711} +{"instance_id": "django__django-15347", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 52.4, "eval_duration_s": 27.0, "gen_cost_usd": 0.035637} +{"instance_id": "django__django-15388", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.9, "eval_duration_s": 26.4, "gen_cost_usd": 0.048576} +{"instance_id": "django__django-15400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.0, "eval_duration_s": 23.5, "gen_cost_usd": 0.060801} +{"instance_id": "django__django-15498", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 108.4, "eval_duration_s": 50.8, "gen_cost_usd": 0.098769} +{"instance_id": "django__django-15695", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 429.9, "eval_duration_s": 39.8, "gen_cost_usd": 0.473772} +{"instance_id": "django__django-15738", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 944.5, "eval_duration_s": 19.9, "gen_cost_usd": 1.124073} +{"instance_id": "django__django-15781", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 36.8, "eval_duration_s": 23.4, "gen_cost_usd": 0.021132} +{"instance_id": "django__django-15789", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 43.5, "eval_duration_s": 25.6, "gen_cost_usd": 0.031632} +{"instance_id": "django__django-15790", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.0, "eval_duration_s": 26.0, "gen_cost_usd": 0.115029} +{"instance_id": "django__django-15814", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 74.7, "eval_duration_s": 25.6, "gen_cost_usd": 0.056643} +{"instance_id": "django__django-15819", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 249.1, "eval_duration_s": 19.1, "gen_cost_usd": 0.335946} +{"instance_id": "django__django-15851", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.0, "eval_duration_s": 29.7, "gen_cost_usd": 0.047985} +{"instance_id": "django__django-15902", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 101.8, "eval_duration_s": 27.1, "gen_cost_usd": 0.08235} +{"instance_id": "django__django-15996", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.6, "eval_duration_s": 17.5, "gen_cost_usd": 0.204909} +{"instance_id": "django__django-16041", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 144.1, "eval_duration_s": 25.9, "gen_cost_usd": 0.038502} +{"instance_id": "django__django-16046", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 65.0, "eval_duration_s": 27.8, "gen_cost_usd": 0.05763} +{"instance_id": "django__django-16139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.5, "eval_duration_s": 23.4, "gen_cost_usd": 0.017829} +{"instance_id": "django__django-16229", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 924.9, "eval_duration_s": 18.3, "gen_cost_usd": 1.006125} +{"instance_id": "django__django-16255", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 39.9, "eval_duration_s": 25.6, "gen_cost_usd": 0.013146} +{"instance_id": "django__django-16379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 31.8, "eval_duration_s": 98.6, "gen_cost_usd": 0.026895} +{"instance_id": "django__django-16400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.9, "eval_duration_s": 25.0, "gen_cost_usd": 0.064098} +{"instance_id": "django__django-16408", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 132.7, "eval_duration_s": 45.8, "gen_cost_usd": 0.148632} +{"instance_id": "django__django-16527", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 28.4, "eval_duration_s": 27.3, "gen_cost_usd": 0.013221} +{"instance_id": "django__django-16595", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 63.1, "eval_duration_s": 26.4, "gen_cost_usd": 0.06387} +{"instance_id": "django__django-16816", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 224.9, "eval_duration_s": 23.8, "gen_cost_usd": 0.267636} +{"instance_id": "django__django-16820", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 393.5, "eval_duration_s": 20.5, "gen_cost_usd": 0.541443} +{"instance_id": "django__django-16873", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 58.8, "eval_duration_s": 26.6, "gen_cost_usd": 0.053043} +{"instance_id": "django__django-16910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 495.2, "eval_duration_s": 19.5, "gen_cost_usd": 0.58017} +{"instance_id": "django__django-17051", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 122.6, "eval_duration_s": 24.6, "gen_cost_usd": 0.137709} +{"instance_id": "django__django-17087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 81.8, "eval_duration_s": 28.3, "gen_cost_usd": 0.063789} +{"instance_id": "matplotlib__matplotlib-18869", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 289.3, "eval_duration_s": 89.9, "gen_cost_usd": 0.166473} +{"instance_id": "matplotlib__matplotlib-22711", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 196.1, "eval_duration_s": 107.9, "gen_cost_usd": 0.081615} +{"instance_id": "matplotlib__matplotlib-22835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 401.4, "eval_duration_s": 85.4, "gen_cost_usd": 0.139341} +{"instance_id": "matplotlib__matplotlib-23299", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 405.3, "eval_duration_s": 71.5, "gen_cost_usd": 0.309069} +{"instance_id": "matplotlib__matplotlib-23314", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 112.8, "eval_duration_s": 71.4, "gen_cost_usd": 0.031626} +{"instance_id": "matplotlib__matplotlib-23476", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 400.6, "eval_duration_s": 83.1, "gen_cost_usd": 0.279963} +{"instance_id": "matplotlib__matplotlib-23562", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 233.3, "eval_duration_s": 135.2, "gen_cost_usd": 0.093468} +{"instance_id": "matplotlib__matplotlib-23563", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 328.5, "eval_duration_s": 291.7, "gen_cost_usd": 0.365733} +{"instance_id": "matplotlib__matplotlib-23913", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 218.2, "eval_duration_s": 81.6, "gen_cost_usd": 0.122436} +{"instance_id": "matplotlib__matplotlib-23964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 387.2, "eval_duration_s": 85.9, "gen_cost_usd": 0.02988} +{"instance_id": "matplotlib__matplotlib-23987", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 271.8, "eval_duration_s": 92.2, "gen_cost_usd": 0.089511} +{"instance_id": "matplotlib__matplotlib-24149", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 174.2, "eval_duration_s": 186.8, "gen_cost_usd": 0.037743} +{"instance_id": "matplotlib__matplotlib-24265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 212.6, "eval_duration_s": 83.6, "gen_cost_usd": 0.153486} +{"instance_id": "matplotlib__matplotlib-24334", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 237.7, "eval_duration_s": 85.3, "gen_cost_usd": 0.059715} +{"instance_id": "matplotlib__matplotlib-24970", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 283.9, "eval_duration_s": 268.5, "gen_cost_usd": 0.071205} +{"instance_id": "matplotlib__matplotlib-25079", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 1211.9, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "matplotlib__matplotlib-25311", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 693.0, "eval_duration_s": 109.6, "gen_cost_usd": 0.600099} +{"instance_id": "matplotlib__matplotlib-25332", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 250.7, "eval_duration_s": 373.9, "gen_cost_usd": 0.129576} +{"instance_id": "matplotlib__matplotlib-25433", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 552.5, "eval_duration_s": 83.1, "gen_cost_usd": 0.488568} +{"instance_id": "matplotlib__matplotlib-25442", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 735.9, "eval_duration_s": 87.7, "gen_cost_usd": 0.611346} +{"instance_id": "matplotlib__matplotlib-25498", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 421.1, "eval_duration_s": 90.8, "gen_cost_usd": 0.28884} +{"instance_id": "matplotlib__matplotlib-26011", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 248.2, "eval_duration_s": 193.0, "gen_cost_usd": 0.100854} +{"instance_id": "matplotlib__matplotlib-26020", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 515.1, "eval_duration_s": 77.8, "gen_cost_usd": 0.081144} +{"instance_id": "mwaskom__seaborn-2848", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 443.3, "eval_duration_s": 31.5, "gen_cost_usd": 0.477777} +{"instance_id": "mwaskom__seaborn-3010", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 57.3, "eval_duration_s": 8.9, "gen_cost_usd": 0.06033} +{"instance_id": "mwaskom__seaborn-3190", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 119.0, "eval_duration_s": 10.5, "gen_cost_usd": 0.113079} +{"instance_id": "mwaskom__seaborn-3407", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 273.2, "eval_duration_s": 79.9, "gen_cost_usd": 0.263928} +{"instance_id": "pallets__flask-4045", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 76.0, "eval_duration_s": 9.1, "gen_cost_usd": 0.07935} +{"instance_id": "pallets__flask-4992", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 50.8, "eval_duration_s": 5.3, "gen_cost_usd": 0.049701} +{"instance_id": "pallets__flask-5063", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 116.6, "eval_duration_s": 8.8, "gen_cost_usd": 0.138921} +{"instance_id": "psf__requests-1963", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.3, "eval_duration_s": 40.6, "gen_cost_usd": 0.064146} +{"instance_id": "psf__requests-2148", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 74.6, "eval_duration_s": 20.8, "gen_cost_usd": 0.072666} +{"instance_id": "psf__requests-2317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 259.6, "eval_duration_s": 608.0, "gen_cost_usd": 0.073623} +{"instance_id": "psf__requests-2674", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 128.8, "eval_duration_s": 47.2, "gen_cost_usd": 0.168309} +{"instance_id": "psf__requests-3362", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 48.8, "eval_duration_s": 6.3, "gen_cost_usd": 0.058566} +{"instance_id": "psf__requests-863", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 33.6, "eval_duration_s": 41.4, "gen_cost_usd": 0.04485} +{"instance_id": "pydata__xarray-3364", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 1212.0, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "pydata__xarray-4094", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 106.3, "eval_duration_s": 24.5, "gen_cost_usd": 0.09525} +{"instance_id": "pydata__xarray-4248", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 124.3, "eval_duration_s": 10.9, "gen_cost_usd": 0.169662} +{"instance_id": "pydata__xarray-4493", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 82.4, "eval_duration_s": 58.7, "gen_cost_usd": 0.075021} +{"instance_id": "pydata__xarray-5131", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 34.7, "eval_duration_s": 10.2, "gen_cost_usd": 0.032568} +{"instance_id": "pylint-dev__pylint-5859", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 94.9, "eval_duration_s": 8.9, "gen_cost_usd": 0.136593} +{"instance_id": "pylint-dev__pylint-6506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 104.8, "eval_duration_s": 8.4, "gen_cost_usd": 0.146466} +{"instance_id": "pylint-dev__pylint-7080", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 86.0, "eval_duration_s": 14.0, "gen_cost_usd": 0.123609} +{"instance_id": "pylint-dev__pylint-7114", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 499.1, "eval_duration_s": 5.5, "gen_cost_usd": 0.63648} +{"instance_id": "pylint-dev__pylint-7228", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 226.3, "eval_duration_s": 5.5, "gen_cost_usd": 0.251193} +{"instance_id": "pylint-dev__pylint-7993", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 115.0, "eval_duration_s": 5.4, "gen_cost_usd": 0.131088} +{"instance_id": "pytest-dev__pytest-11143", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 42.9, "eval_duration_s": 10.9, "gen_cost_usd": 0.046716} +{"instance_id": "pytest-dev__pytest-11148", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 294.9, "eval_duration_s": 13.8, "gen_cost_usd": 0.420648} +{"instance_id": "pytest-dev__pytest-5103", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 962.4, "eval_duration_s": 12.1, "gen_cost_usd": 1.167606} +{"instance_id": "pytest-dev__pytest-5221", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 191.1, "eval_duration_s": 13.2, "gen_cost_usd": 0.187368} +{"instance_id": "pytest-dev__pytest-5227", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 18.1, "eval_duration_s": 9.6, "gen_cost_usd": 0.014136} +{"instance_id": "pytest-dev__pytest-5413", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 55.4, "eval_duration_s": 26.6, "gen_cost_usd": 0.055299} +{"instance_id": "pytest-dev__pytest-5495", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 95.0, "eval_duration_s": 12.0, "gen_cost_usd": 0.07128} +{"instance_id": "pytest-dev__pytest-5692", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 46.3, "eval_duration_s": 24.8, "gen_cost_usd": 0.077898} +{"instance_id": "pytest-dev__pytest-6116", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 25.4, "eval_duration_s": 11.6, "gen_cost_usd": 0.02847} +{"instance_id": "pytest-dev__pytest-7168", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 48.2, "eval_duration_s": 9.1, "gen_cost_usd": 0.033897} +{"instance_id": "pytest-dev__pytest-7220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 239.7, "eval_duration_s": 8.8, "gen_cost_usd": 0.259536} +{"instance_id": "pytest-dev__pytest-7373", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 64.2, "eval_duration_s": 11.1, "gen_cost_usd": 0.057195} +{"instance_id": "pytest-dev__pytest-7432", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 62.9, "eval_duration_s": 10.7, "gen_cost_usd": 0.078171} +{"instance_id": "pytest-dev__pytest-7490", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 179.9, "eval_duration_s": 12.0, "gen_cost_usd": 0.236649} +{"instance_id": "pytest-dev__pytest-8365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 47.4, "eval_duration_s": 28.7, "gen_cost_usd": 0.060828} +{"instance_id": "pytest-dev__pytest-8906", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 39.9, "eval_duration_s": 10.6, "gen_cost_usd": 0.047148} +{"instance_id": "pytest-dev__pytest-9359", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 232.8, "eval_duration_s": 24.4, "gen_cost_usd": 0.332877} +{"instance_id": "scikit-learn__scikit-learn-10297", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 184.2, "eval_duration_s": 136.9, "gen_cost_usd": 0.053574} +{"instance_id": "scikit-learn__scikit-learn-10508", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 146.0, "eval_duration_s": 130.5, "gen_cost_usd": 0.016671} +{"instance_id": "scikit-learn__scikit-learn-10949", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 222.4, "eval_duration_s": 130.3, "gen_cost_usd": 0.111537} +{"instance_id": "scikit-learn__scikit-learn-11040", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 241.5, "eval_duration_s": 164.6, "gen_cost_usd": 0.093969} +{"instance_id": "scikit-learn__scikit-learn-11281", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 281.0, "eval_duration_s": 129.5, "gen_cost_usd": 0.129891} +{"instance_id": "scikit-learn__scikit-learn-12471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 176.3, "eval_duration_s": 144.1, "gen_cost_usd": 0.048426} +{"instance_id": "scikit-learn__scikit-learn-13142", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.6, "eval_duration_s": 156.7, "gen_cost_usd": 0.070095} +{"instance_id": "scikit-learn__scikit-learn-13241", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 445.7, "eval_duration_s": 173.5, "gen_cost_usd": 0.374919} +{"instance_id": "scikit-learn__scikit-learn-13439", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 216.1, "eval_duration_s": 174.9, "gen_cost_usd": 0.027744} +{"instance_id": "scikit-learn__scikit-learn-13496", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 271.6, "eval_duration_s": 178.5, "gen_cost_usd": 0.121494} +{"instance_id": "scikit-learn__scikit-learn-13497", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 211.7, "eval_duration_s": 180.1, "gen_cost_usd": 0.050214} +{"instance_id": "scikit-learn__scikit-learn-13584", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 277.8, "eval_duration_s": 165.8, "gen_cost_usd": 0.149271} +{"instance_id": "scikit-learn__scikit-learn-13779", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 217.3, "eval_duration_s": 213.8, "gen_cost_usd": 0.019875} +{"instance_id": "scikit-learn__scikit-learn-14087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 462.8, "eval_duration_s": 210.5, "gen_cost_usd": 0.349221} +{"instance_id": "scikit-learn__scikit-learn-14092", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 278.9, "eval_duration_s": 331.6, "gen_cost_usd": 0.118077} +{"instance_id": "scikit-learn__scikit-learn-14894", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 230.9, "eval_duration_s": 190.4, "gen_cost_usd": 0.041496} +{"instance_id": "scikit-learn__scikit-learn-14983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 334.9, "eval_duration_s": 206.8, "gen_cost_usd": 0.17247} +{"instance_id": "scikit-learn__scikit-learn-15512", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 290.9, "eval_duration_s": 208.5, "gen_cost_usd": 0.134169} +{"instance_id": "scikit-learn__scikit-learn-15535", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 235.1, "eval_duration_s": 189.3, "gen_cost_usd": 0.053409} +{"instance_id": "scikit-learn__scikit-learn-25500", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 347.7, "eval_duration_s": 13.3, "gen_cost_usd": 0.094563} +{"instance_id": "scikit-learn__scikit-learn-25570", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 486.2, "eval_duration_s": 14.1, "gen_cost_usd": 0.289725} +{"instance_id": "scikit-learn__scikit-learn-25638", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 467.2, "eval_duration_s": 12.8, "gen_cost_usd": 0.281478} +{"instance_id": "scikit-learn__scikit-learn-25747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 336.2, "eval_duration_s": 20.0, "gen_cost_usd": 0.092697} +{"instance_id": "sphinx-doc__sphinx-10325", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 866.5, "eval_duration_s": 15.6, "gen_cost_usd": 0.661449} +{"instance_id": "sphinx-doc__sphinx-10451", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 409.7, "eval_duration_s": 14.8, "gen_cost_usd": 0.550653} +{"instance_id": "sphinx-doc__sphinx-11445", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 161.9, "eval_duration_s": 13.4, "gen_cost_usd": 0.1545} +{"instance_id": "sphinx-doc__sphinx-7686", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 116.2, "eval_duration_s": 15.3, "gen_cost_usd": 0.138948} +{"instance_id": "sphinx-doc__sphinx-7738", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 288.7, "eval_duration_s": 22.8, "gen_cost_usd": 0.315528} +{"instance_id": "sphinx-doc__sphinx-7975", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 116.6, "eval_duration_s": 14.5, "gen_cost_usd": 0.154194} +{"instance_id": "sphinx-doc__sphinx-8273", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 87.1, "eval_duration_s": 15.2, "gen_cost_usd": 0.113403} +{"instance_id": "sphinx-doc__sphinx-8282", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 182.8, "eval_duration_s": 15.1, "gen_cost_usd": 0.239826} +{"instance_id": "sphinx-doc__sphinx-8435", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 341.8, "eval_duration_s": 16.3, "gen_cost_usd": 0.44871} +{"instance_id": "sphinx-doc__sphinx-8474", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 1124.8, "eval_duration_s": 14.1, "gen_cost_usd": 1.080222} +{"instance_id": "sphinx-doc__sphinx-8506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 200.4, "eval_duration_s": 15.0, "gen_cost_usd": 0.244776} +{"instance_id": "sphinx-doc__sphinx-8595", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 40.6, "eval_duration_s": 21.9, "gen_cost_usd": 0.046731} +{"instance_id": "sphinx-doc__sphinx-8627", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 300.5, "eval_duration_s": 16.7, "gen_cost_usd": 0.427608} +{"instance_id": "sphinx-doc__sphinx-8713", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 48.7, "eval_duration_s": 14.5, "gen_cost_usd": 0.052143} +{"instance_id": "sphinx-doc__sphinx-8721", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 33.8, "eval_duration_s": 14.9, "gen_cost_usd": 0.045882} +{"instance_id": "sphinx-doc__sphinx-8801", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 301.0, "eval_duration_s": 14.5, "gen_cost_usd": 0.365451} +{"instance_id": "sympy__sympy-11400", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 91.8, "eval_duration_s": 20.8, "gen_cost_usd": 0.114477} +{"instance_id": "sympy__sympy-11870", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 195.0, "eval_duration_s": 516.3, "gen_cost_usd": 0.191145} +{"instance_id": "sympy__sympy-11897", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 540.2, "eval_duration_s": 23.1, "gen_cost_usd": 0.632043} +{"instance_id": "sympy__sympy-12171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 184.4, "eval_duration_s": 20.7, "gen_cost_usd": 0.159717} +{"instance_id": "sympy__sympy-12236", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 475.8, "eval_duration_s": 76.3, "gen_cost_usd": 0.476478} +{"instance_id": "sympy__sympy-12419", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 106.1, "eval_duration_s": 21.0, "gen_cost_usd": 0.116844} +{"instance_id": "sympy__sympy-12454", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 63.6, "eval_duration_s": 33.2, "gen_cost_usd": 0.049533} +{"instance_id": "sympy__sympy-12481", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 135.3, "eval_duration_s": 24.4, "gen_cost_usd": 0.113361} +{"instance_id": "sympy__sympy-13031", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 203.7, "eval_duration_s": 23.1, "gen_cost_usd": 0.175527} +{"instance_id": "sympy__sympy-13043", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 197.7, "eval_duration_s": 58.3, "gen_cost_usd": 0.090087} +{"instance_id": "sympy__sympy-13146", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 1200.3, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sympy__sympy-13177", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 58.5, "eval_duration_s": 21.1, "gen_cost_usd": 0.047283} +{"instance_id": "sympy__sympy-13437", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 79.8, "eval_duration_s": 39.7, "gen_cost_usd": 0.071394} +{"instance_id": "sympy__sympy-13471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 96.6, "eval_duration_s": 21.9, "gen_cost_usd": 0.082665} +{"instance_id": "sympy__sympy-13480", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 38.9, "eval_duration_s": 25.7, "gen_cost_usd": 0.014853} +{"instance_id": "sympy__sympy-13647", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.3, "eval_duration_s": 109.0, "gen_cost_usd": 0.019932} +{"instance_id": "sympy__sympy-13773", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 112.9, "eval_duration_s": 21.8, "gen_cost_usd": 0.085689} +{"instance_id": "sympy__sympy-13895", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 852.9, "eval_duration_s": 20.5, "gen_cost_usd": 1.095765} +{"instance_id": "sympy__sympy-13915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 282.6, "eval_duration_s": 22.6, "gen_cost_usd": 0.279147} +{"instance_id": "sympy__sympy-13971", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.0, "eval_duration_s": 27.8, "gen_cost_usd": 0.114513} +{"instance_id": "sympy__sympy-14024", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 469.1, "eval_duration_s": 26.8, "gen_cost_usd": 0.62058} +{"instance_id": "sympy__sympy-14308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 338.4, "eval_duration_s": 24.9, "gen_cost_usd": 0.414582} +{"instance_id": "sympy__sympy-14317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 216.2, "eval_duration_s": 30.4, "gen_cost_usd": 0.247515} +{"instance_id": "sympy__sympy-14396", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 93.1, "eval_duration_s": 19.9, "gen_cost_usd": 0.120039} +{"instance_id": "sympy__sympy-14774", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 35.4, "eval_duration_s": 27.5, "gen_cost_usd": 0.015927} +{"instance_id": "sympy__sympy-14817", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 192.1, "eval_duration_s": 28.3, "gen_cost_usd": 0.187683} +{"instance_id": "sympy__sympy-15011", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 101.4, "eval_duration_s": 29.6, "gen_cost_usd": 0.094638} +{"instance_id": "sympy__sympy-15308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 62.1, "eval_duration_s": 28.5, "gen_cost_usd": 0.053562} +{"instance_id": "sympy__sympy-15345", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 84.8, "eval_duration_s": 22.0, "gen_cost_usd": 0.075042} +{"instance_id": "sympy__sympy-15346", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 47.6, "eval_duration_s": 47.3, "gen_cost_usd": 0.036453} +{"instance_id": "sympy__sympy-15609", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.2, "eval_duration_s": 24.9, "gen_cost_usd": 0.048666} +{"instance_id": "sympy__sympy-15678", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 128.1, "eval_duration_s": 26.2, "gen_cost_usd": 0.13263} +{"instance_id": "sympy__sympy-16106", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 146.8, "eval_duration_s": 21.6, "gen_cost_usd": 0.191646} +{"instance_id": "sympy__sympy-16281", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 803.3, "eval_duration_s": 27.4, "gen_cost_usd": 0.997038} +{"instance_id": "sympy__sympy-16503", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 819.6, "eval_duration_s": 24.5, "gen_cost_usd": 0.995094} +{"instance_id": "sympy__sympy-16792", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 162.3, "eval_duration_s": 21.7, "gen_cost_usd": 0.190089} +{"instance_id": "sympy__sympy-16988", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 790.2, "eval_duration_s": 89.1, "gen_cost_usd": 0.894474} +{"instance_id": "sympy__sympy-17022", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 139.4, "eval_duration_s": 20.2, "gen_cost_usd": 0.159717} +{"instance_id": "sympy__sympy-17139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 46.3, "eval_duration_s": 95.9, "gen_cost_usd": 0.029595} +{"instance_id": "sympy__sympy-17630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 392.1, "eval_duration_s": 21.3, "gen_cost_usd": 0.487977} +{"instance_id": "sympy__sympy-17655", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 112.9, "eval_duration_s": 42.6, "gen_cost_usd": 0.089706} +{"instance_id": "sympy__sympy-18057", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.0, "eval_duration_s": 112.7, "gen_cost_usd": 0.085674} +{"instance_id": "sympy__sympy-18087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 253.5, "eval_duration_s": 31.4, "gen_cost_usd": 0.253008} +{"instance_id": "sympy__sympy-18189", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 32.3, "eval_duration_s": 41.5, "gen_cost_usd": 0.019599} +{"instance_id": "sympy__sympy-18199", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 115.3, "eval_duration_s": 109.3, "gen_cost_usd": 0.120789} +{"instance_id": "sympy__sympy-18532", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 144.5, "eval_duration_s": 29.5, "gen_cost_usd": 0.124377} +{"instance_id": "sympy__sympy-18621", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 43.1, "eval_duration_s": 33.2, "gen_cost_usd": 0.025353} +{"instance_id": "sympy__sympy-18698", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 274.1, "eval_duration_s": 29.6, "gen_cost_usd": 0.312117} +{"instance_id": "sympy__sympy-18835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 73.0, "eval_duration_s": 21.3, "gen_cost_usd": 0.056202} +{"instance_id": "sympy__sympy-19007", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 316.8, "eval_duration_s": 23.6, "gen_cost_usd": 0.380256} +{"instance_id": "sympy__sympy-19254", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 468.4, "eval_duration_s": 33.1, "gen_cost_usd": 0.464595} +{"instance_id": "sympy__sympy-19487", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 71.8, "eval_duration_s": 32.8, "gen_cost_usd": 0.057687} +{"instance_id": "sympy__sympy-20049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 442.2, "eval_duration_s": 21.2, "gen_cost_usd": 0.588681} +{"instance_id": "sympy__sympy-20154", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.6, "eval_duration_s": 21.3, "gen_cost_usd": 0.069927} +{"instance_id": "sympy__sympy-20212", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 241.2, "eval_duration_s": 37.5, "gen_cost_usd": 0.13575} +{"instance_id": "sympy__sympy-20322", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 71.8, "eval_duration_s": 30.3, "gen_cost_usd": 0.061077} +{"instance_id": "sympy__sympy-20442", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 134.0, "eval_duration_s": 27.8, "gen_cost_usd": 0.142527} +{"instance_id": "sympy__sympy-20590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 58.7, "eval_duration_s": 32.1, "gen_cost_usd": 0.042108} +{"instance_id": "sympy__sympy-20639", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 224.1, "eval_duration_s": 23.1, "gen_cost_usd": 0.245946} +{"instance_id": "sympy__sympy-21055", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.4, "eval_duration_s": 46.9, "gen_cost_usd": 0.13524} +{"instance_id": "sympy__sympy-21171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 65.1, "eval_duration_s": 27.6, "gen_cost_usd": 0.060528} +{"instance_id": "sympy__sympy-21379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.4, "eval_duration_s": 36.9, "gen_cost_usd": 0.0684} +{"instance_id": "sympy__sympy-21612", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 54.1, "eval_duration_s": 47.0, "gen_cost_usd": 0.03927} +{"instance_id": "sympy__sympy-21614", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 99.2, "eval_duration_s": 20.3, "gen_cost_usd": 0.091533} +{"instance_id": "sympy__sympy-21627", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 60.0, "eval_duration_s": 25.5, "gen_cost_usd": 0.047247} +{"instance_id": "sympy__sympy-21847", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 41.6, "eval_duration_s": 19.8, "gen_cost_usd": 0.024429} +{"instance_id": "sympy__sympy-22005", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 210.9, "eval_duration_s": 26.3, "gen_cost_usd": 0.26079} +{"instance_id": "sympy__sympy-22714", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 144.0, "eval_duration_s": 23.1, "gen_cost_usd": 0.155361} +{"instance_id": "sympy__sympy-22840", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 174.0, "eval_duration_s": 29.2, "gen_cost_usd": 0.20598} +{"instance_id": "sympy__sympy-23117", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 56.8, "eval_duration_s": 21.3, "gen_cost_usd": 0.036834} +{"instance_id": "sympy__sympy-23191", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 926.9, "eval_duration_s": 21.6, "gen_cost_usd": 1.136349} +{"instance_id": "sympy__sympy-23262", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 104.0, "eval_duration_s": 25.1, "gen_cost_usd": 0.087327} +{"instance_id": "sympy__sympy-24066", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 140.3, "eval_duration_s": 21.3, "gen_cost_usd": 0.11469} +{"instance_id": "sympy__sympy-24102", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 138.3, "eval_duration_s": 21.8, "gen_cost_usd": 0.175524} +{"instance_id": "sympy__sympy-24152", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 61.4, "eval_duration_s": 29.1, "gen_cost_usd": 0.042897} +{"instance_id": "sympy__sympy-24213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.6, "eval_duration_s": 23.2, "gen_cost_usd": 0.056283} +{"instance_id": "sympy__sympy-24909", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 130.1, "eval_duration_s": 22.0, "gen_cost_usd": 0.139866} diff --git a/evals/swe-bench/scoreboard/sonnet-20min-20260316/meta.json b/evals/swe-bench/scoreboard/sonnet-20min-20260316/meta.json new file mode 100644 index 000000000..f750b5b96 --- /dev/null +++ b/evals/swe-bench/scoreboard/sonnet-20min-20260316/meta.json @@ -0,0 +1,97 @@ +{ + "run_name": "sonnet-20min-20260316", + "timestamp": "2026-03-16T17:31:20.245784+00:00", + "model": "claude-sonnet-4-6", + "provider": "anthropic", + "fabro_version": "fabro 0.5.0 (4c6ad4d 2026-03-16)", + "timeout_s": 1200, + "sandbox_cpu": 4, + "sandbox_memory_gb": 8, + "description": "Sonnet 4.6, 20min timeout, 4 CPU / 8 GB, default prompt", + "notes": "", + "total_instances": 300, + "patched": 297, + "patch_pct": 99.0, + "resolved": 173, + "resolve_pct": 57.7, + "total_gen_cost_usd": 55.22, + "avg_gen_cost_usd": 0.1841, + "gen_wall_time_s": 1466.0, + "eval_wall_time_s": 682.3, + "avg_gen_duration_s": 211.5, + "avg_eval_duration_s": 51.4, + "per_repo": { + "astropy/astropy": { + "total": 6, + "patched": 6, + "resolved": 1, + "resolve_pct": 16.7 + }, + "django/django": { + "total": 114, + "patched": 114, + "resolved": 88, + "resolve_pct": 77.2 + }, + "matplotlib/matplotlib": { + "total": 23, + "patched": 22, + "resolved": 0, + "resolve_pct": 0.0 + }, + "mwaskom/seaborn": { + "total": 4, + "patched": 4, + "resolved": 3, + "resolve_pct": 75.0 + }, + "pallets/flask": { + "total": 3, + "patched": 3, + "resolved": 0, + "resolve_pct": 0.0 + }, + "psf/requests": { + "total": 6, + "patched": 6, + "resolved": 5, + "resolve_pct": 83.3 + }, + "pydata/xarray": { + "total": 5, + "patched": 4, + "resolved": 1, + "resolve_pct": 20.0 + }, + "pylint-dev/pylint": { + "total": 6, + "patched": 6, + "resolved": 0, + "resolve_pct": 0.0 + }, + "pytest-dev/pytest": { + "total": 17, + "patched": 17, + "resolved": 11, + "resolve_pct": 64.7 + }, + "scikit-learn/scikit-learn": { + "total": 23, + "patched": 23, + "resolved": 15, + "resolve_pct": 65.2 + }, + "sphinx-doc/sphinx": { + "total": 16, + "patched": 16, + "resolved": 0, + "resolve_pct": 0.0 + }, + "sympy/sympy": { + "total": 77, + "patched": 76, + "resolved": 49, + "resolve_pct": 63.6 + } + } +}