diff --git a/evals/swe-bench/record_results.py b/evals/swe-bench/record_results.py new file mode 100644 index 000000000..0ffd5db73 --- /dev/null +++ b/evals/swe-bench/record_results.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +"""Record SWE-bench eval results into the scoreboard. + +Reads generation results + eval results and produces a scoreboard entry +with per-instance data, aggregate stats, and run metadata. + +Usage: + python record_results.py \ + --run-name haiku-baseline-20260316 \ + --gen-dir results/haiku-baseline \ + --eval-dir results/haiku-baseline/eval \ + --description "Haiku 4.5 baseline, default prompt, 10min timeout" +""" + +import argparse +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +EVAL_DIR = Path(__file__).parent.resolve() +SCOREBOARD_DIR = EVAL_DIR / "scoreboard" + + +def load_jsonl(path: Path) -> list[dict]: + rows = [] + with open(path) as f: + for line in f: + if line.strip(): + rows.append(json.loads(line)) + return rows + + +def get_cost_from_fabro_run(fabro_run_dir: str | None) -> float | None: + """Extract total_cost from a fabro run's conclusion.json.""" + if not fabro_run_dir: + return None + conclusion = Path(fabro_run_dir) / "conclusion.json" + if not conclusion.exists(): + return None + try: + data = json.loads(conclusion.read_text()) + return data.get("total_cost") + except (json.JSONDecodeError, OSError): + return None + + +def get_fabro_version() -> str: + try: + result = subprocess.run( + ["fabro", "--version"], capture_output=True, text=True, timeout=5 + ) + return result.stdout.strip() + except Exception: + return "unknown" + + +def main(): + parser = argparse.ArgumentParser( + description="Record SWE-bench eval results into the scoreboard" + ) + parser.add_argument( + "--run-name", required=True, + help="Name for this run (e.g. haiku-baseline-20260316)", + ) + parser.add_argument( + "--gen-dir", type=Path, required=True, + help="Generation results directory (contains predictions.jsonl, results.jsonl)", + ) + parser.add_argument( + "--eval-dir", type=Path, required=True, + help="Evaluation results directory (contains eval_results.jsonl)", + ) + parser.add_argument( + "--description", default="", + help="Human-readable description of what was tested", + ) + parser.add_argument( + "--notes", default="", + help="Additional notes or observations", + ) + args = parser.parse_args() + + run_dir = SCOREBOARD_DIR / args.run_name + run_dir.mkdir(parents=True, exist_ok=True) + + # Load generation results + gen_results = {r["instance_id"]: r for r in load_jsonl(args.gen_dir / "results.jsonl")} + gen_summary = json.loads((args.gen_dir / "summary.json").read_text()) + + # Load eval results + eval_results = {r["instance_id"]: r for r in load_jsonl(args.eval_dir / "eval_results.jsonl")} + eval_summary = json.loads((args.eval_dir / "summary.json").read_text()) + + # Build per-instance records + all_instance_ids = sorted(set(gen_results.keys()) | set(eval_results.keys())) + instances = [] + total_gen_cost = 0.0 + for iid in all_instance_ids: + gen = gen_results.get(iid, {}) + evl = eval_results.get(iid, {}) + + has_patch = bool(gen.get("model_patch", "").strip()) + resolved = evl.get("resolved", False) + gen_duration = gen.get("duration_s") + eval_duration = evl.get("duration_s") + gen_status = gen.get("status", "missing") + eval_status = evl.get("status", "not_evaluated") + cost = get_cost_from_fabro_run(gen.get("fabro_run_dir")) + if cost: + total_gen_cost += cost + + instances.append({ + "instance_id": iid, + "has_patch": has_patch, + "resolved": resolved, + "gen_status": gen_status, + "eval_status": eval_status, + "gen_duration_s": gen_duration, + "eval_duration_s": eval_duration, + "gen_cost_usd": round(cost, 6) if cost else None, + }) + + # Write per-instance results + instances_path = run_dir / "instances.jsonl" + with open(instances_path, "w") as f: + for inst in instances: + f.write(json.dumps(inst) + "\n") + + # Compute aggregates + total = len(instances) + patched = sum(1 for i in instances if i["has_patch"]) + resolved = sum(1 for i in instances if i["resolved"]) + resolve_pct = round(100 * resolved / total, 1) if total > 0 else 0 + patch_pct = round(100 * patched / total, 1) if total > 0 else 0 + + gen_durations = [i["gen_duration_s"] for i in instances if i["gen_duration_s"] is not None] + eval_durations = [i["eval_duration_s"] for i in instances if i["eval_duration_s"] is not None] + + # Per-repo breakdown + from collections import Counter + repo_total: Counter[str] = Counter() + repo_resolved: Counter[str] = Counter() + repo_patched: Counter[str] = Counter() + for inst in instances: + parts = inst["instance_id"].split("__") + repo = f"{parts[0]}/{parts[1].rsplit('-', 1)[0]}" if len(parts) >= 2 else inst["instance_id"] + repo_total[repo] += 1 + if inst["has_patch"]: + repo_patched[repo] += 1 + if inst["resolved"]: + repo_resolved[repo] += 1 + + per_repo = {} + for repo in sorted(repo_total): + per_repo[repo] = { + "total": repo_total[repo], + "patched": repo_patched[repo], + "resolved": repo_resolved[repo], + "resolve_pct": round(100 * repo_resolved[repo] / repo_total[repo], 1), + } + + # Write metadata + meta = { + "run_name": args.run_name, + "timestamp": datetime.now(timezone.utc).isoformat(), + "model": gen_summary.get("model", "unknown"), + "provider": gen_summary.get("provider", "unknown"), + "fabro_version": get_fabro_version(), + "description": args.description, + "notes": args.notes, + "total_instances": total, + "patched": patched, + "patch_pct": patch_pct, + "resolved": resolved, + "resolve_pct": resolve_pct, + "total_gen_cost_usd": round(total_gen_cost, 2), + "avg_gen_cost_usd": round(total_gen_cost / total, 4) if total > 0 else 0, + "gen_wall_time_s": gen_summary.get("total_duration_s"), + "eval_wall_time_s": eval_summary.get("wall_duration_s"), + "avg_gen_duration_s": round(sum(gen_durations) / len(gen_durations), 1) if gen_durations else None, + "avg_eval_duration_s": round(sum(eval_durations) / len(eval_durations), 1) if eval_durations else None, + "per_repo": per_repo, + } + (run_dir / "meta.json").write_text(json.dumps(meta, indent=2) + "\n") + + # Write README + readme_lines = [ + f"# {args.run_name}", + "", + f"**Date:** {meta['timestamp'][:10]}", + f"**Model:** {meta['model']} ({meta['provider']})", + f"**Fabro:** {meta['fabro_version']}", + "", + f"## Description", + "", + args.description or "_No description provided._", + "", + f"## Results", + "", + f"| Metric | Value |", + f"|--------|-------|", + f"| Instances | {total} |", + f"| Patched | {patched} ({patch_pct}%) |", + f"| **Resolved** | **{resolved} ({resolve_pct}%)** |", + f"| Total gen cost | ${meta['total_gen_cost_usd']:.2f} |", + f"| Avg gen cost | ${meta['avg_gen_cost_usd']:.4f}/instance |", + f"| Gen wall time | {meta['gen_wall_time_s']}s |", + f"| Eval wall time | {meta['eval_wall_time_s']}s |", + "", + f"## Per-repo breakdown", + "", + f"| Repo | Resolved | Total | Rate |", + f"|------|----------|-------|------|", + ] + for repo, stats in per_repo.items(): + readme_lines.append( + f"| {repo} | {stats['resolved']} | {stats['total']} | {stats['resolve_pct']}% |" + ) + + if args.notes: + readme_lines.extend(["", "## Notes", "", args.notes]) + + (run_dir / "README.md").write_text("\n".join(readme_lines) + "\n") + + # Regenerate leaderboard + regenerate_leaderboard() + + # Print summary + print(f"Recorded results for '{args.run_name}'") + print(f" Resolved: {resolved}/{total} ({resolve_pct}%)") + print(f" Cost: ${meta['total_gen_cost_usd']:.2f} total, ${meta['avg_gen_cost_usd']:.4f}/instance") + print(f" Scoreboard: {run_dir}") + + +def regenerate_leaderboard(): + """Rebuild leaderboard.json from all scoreboard entries.""" + entries = [] + for meta_path in sorted(SCOREBOARD_DIR.glob("*/meta.json")): + meta = json.loads(meta_path.read_text()) + entries.append({ + "run_name": meta["run_name"], + "date": meta["timestamp"][:10], + "model": meta["model"], + "provider": meta["provider"], + "resolved": meta["resolved"], + "total": meta["total_instances"], + "resolve_pct": meta["resolve_pct"], + "patched": meta["patched"], + "patch_pct": meta["patch_pct"], + "total_cost_usd": meta.get("total_gen_cost_usd"), + "avg_cost_usd": meta.get("avg_gen_cost_usd"), + }) + + # Sort by resolve rate descending + entries.sort(key=lambda e: e["resolve_pct"], reverse=True) + (SCOREBOARD_DIR / "leaderboard.json").write_text(json.dumps(entries, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/evals/swe-bench/scoreboard/haiku-baseline-20260316/README.md b/evals/swe-bench/scoreboard/haiku-baseline-20260316/README.md new file mode 100644 index 000000000..94c17fd4b --- /dev/null +++ b/evals/swe-bench/scoreboard/haiku-baseline-20260316/README.md @@ -0,0 +1,38 @@ +# haiku-baseline-20260316 + +**Date:** 2026-03-16 +**Model:** claude-haiku-4-5 (anthropic) +**Fabro:** fabro 0.5.0 (4c6ad4d 2026-03-16) + +## Description + +Haiku 4.5 baseline, default prompt, 2 CPU / 4 GB, 10min timeout + +## Results + +| Metric | Value | +|--------|-------| +| Instances | 300 | +| Patched | 293 (97.7%) | +| **Resolved** | **162 (54.0%)** | +| Total gen cost | $26.13 | +| Avg gen cost | $0.0871/instance | +| Gen wall time | 3559.2s | +| Eval wall time | 625.2s | + +## Per-repo breakdown + +| Repo | Resolved | Total | Rate | +|------|----------|-------|------| +| astropy/astropy | 1 | 6 | 16.7% | +| django/django | 86 | 114 | 75.4% | +| matplotlib/matplotlib | 0 | 23 | 0.0% | +| mwaskom/seaborn | 1 | 4 | 25.0% | +| pallets/flask | 0 | 3 | 0.0% | +| psf/requests | 4 | 6 | 66.7% | +| pydata/xarray | 1 | 5 | 20.0% | +| pylint-dev/pylint | 0 | 6 | 0.0% | +| pytest-dev/pytest | 11 | 17 | 64.7% | +| scikit-learn/scikit-learn | 12 | 23 | 52.2% | +| sphinx-doc/sphinx | 0 | 16 | 0.0% | +| sympy/sympy | 46 | 77 | 59.7% | diff --git a/evals/swe-bench/scoreboard/haiku-baseline-20260316/instances.jsonl b/evals/swe-bench/scoreboard/haiku-baseline-20260316/instances.jsonl new file mode 100644 index 000000000..cb807522a --- /dev/null +++ b/evals/swe-bench/scoreboard/haiku-baseline-20260316/instances.jsonl @@ -0,0 +1,300 @@ +{"instance_id": "astropy__astropy-12907", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 166.0, "eval_duration_s": 63.8, "gen_cost_usd": 0.069072} +{"instance_id": "astropy__astropy-14182", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 189.6, "eval_duration_s": 63.3, "gen_cost_usd": 0.077053} +{"instance_id": "astropy__astropy-14365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 118.7, "eval_duration_s": 44.9, "gen_cost_usd": 0.056338} +{"instance_id": "astropy__astropy-14995", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 162.9, "eval_duration_s": 47.1, "gen_cost_usd": 0.077115} +{"instance_id": "astropy__astropy-6938", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 182.5, "eval_duration_s": 83.1, "gen_cost_usd": 0.076278} +{"instance_id": "astropy__astropy-7746", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 250.5, "eval_duration_s": 83.8, "gen_cost_usd": 0.087522} +{"instance_id": "django__django-10914", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 130.1, "eval_duration_s": 23.5, "gen_cost_usd": 0.073745} +{"instance_id": "django__django-10924", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 134.9, "eval_duration_s": 27.1, "gen_cost_usd": 0.079222} +{"instance_id": "django__django-11001", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 141.9, "eval_duration_s": 24.2, "gen_cost_usd": 0.086682} +{"instance_id": "django__django-11019", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 219.7, "eval_duration_s": 25.3, "gen_cost_usd": 0.132849} +{"instance_id": "django__django-11039", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 120.7, "eval_duration_s": 20.9, "gen_cost_usd": 0.069046} +{"instance_id": "django__django-11049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.7, "eval_duration_s": 17.8, "gen_cost_usd": 0.049394} +{"instance_id": "django__django-11099", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.4, "eval_duration_s": 19.8, "gen_cost_usd": 0.055786} +{"instance_id": "django__django-11133", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 109.4, "eval_duration_s": 22.1, "gen_cost_usd": 0.075125} +{"instance_id": "django__django-11179", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 218.0, "eval_duration_s": 24.1, "gen_cost_usd": 0.097138} +{"instance_id": "django__django-11283", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 201.1, "eval_duration_s": 24.6, "gen_cost_usd": 0.122612} +{"instance_id": "django__django-11422", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 112.7, "eval_duration_s": 25.0, "gen_cost_usd": 0.076847} +{"instance_id": "django__django-11564", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 191.0, "eval_duration_s": 27.7, "gen_cost_usd": 0.11641} +{"instance_id": "django__django-11583", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 100.1, "eval_duration_s": 21.8, "gen_cost_usd": 0.068367} +{"instance_id": "django__django-11620", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 195.3, "eval_duration_s": 22.4, "gen_cost_usd": 0.090314} +{"instance_id": "django__django-11630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 108.3, "eval_duration_s": 25.4, "gen_cost_usd": 0.073506} +{"instance_id": "django__django-11742", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 182.4, "eval_duration_s": 21.6, "gen_cost_usd": 0.101582} +{"instance_id": "django__django-11797", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 190.5, "eval_duration_s": 23.0, "gen_cost_usd": 0.095608} +{"instance_id": "django__django-11815", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 160.9, "eval_duration_s": 19.0, "gen_cost_usd": 0.090641} +{"instance_id": "django__django-11848", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.9, "eval_duration_s": 24.5, "gen_cost_usd": 0.057066} +{"instance_id": "django__django-11905", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 167.0, "eval_duration_s": 21.4, "gen_cost_usd": 0.080023} +{"instance_id": "django__django-11910", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 415.6, "eval_duration_s": 22.1, "gen_cost_usd": 0.257499} +{"instance_id": "django__django-11964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 398.2, "eval_duration_s": 20.6, "gen_cost_usd": 0.111035} +{"instance_id": "django__django-11999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 112.5, "eval_duration_s": 23.7, "gen_cost_usd": 0.058971} +{"instance_id": "django__django-12113", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.6, "eval_duration_s": 19.6, "gen_cost_usd": 0.055079} +{"instance_id": "django__django-12125", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 241.7, "eval_duration_s": 20.0, "gen_cost_usd": 0.127666} +{"instance_id": "django__django-12184", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 141.2, "eval_duration_s": 24.9, "gen_cost_usd": 0.085902} +{"instance_id": "django__django-12284", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.2, "eval_duration_s": 21.7, "gen_cost_usd": 0.071082} +{"instance_id": "django__django-12286", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 104.9, "eval_duration_s": 23.9, "gen_cost_usd": 0.066556} +{"instance_id": "django__django-12308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 153.0, "eval_duration_s": 25.2, "gen_cost_usd": 0.086661} +{"instance_id": "django__django-12453", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 79.7, "eval_duration_s": 29.8, "gen_cost_usd": 0.043009} +{"instance_id": "django__django-12470", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 520.5, "eval_duration_s": 20.3, "gen_cost_usd": 0.251534} +{"instance_id": "django__django-12497", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 82.6, "eval_duration_s": 20.9, "gen_cost_usd": 0.049238} +{"instance_id": "django__django-12589", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 366.7, "eval_duration_s": 21.5, "gen_cost_usd": 0.147055} +{"instance_id": "django__django-12700", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 104.2, "eval_duration_s": 26.4, "gen_cost_usd": 0.054288} +{"instance_id": "django__django-12708", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 217.7, "eval_duration_s": 16.9, "gen_cost_usd": 0.121474} +{"instance_id": "django__django-12747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 269.0, "eval_duration_s": 24.0, "gen_cost_usd": 0.138026} +{"instance_id": "django__django-12856", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 250.0, "eval_duration_s": 24.5, "gen_cost_usd": 0.147799} +{"instance_id": "django__django-12908", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 178.6, "eval_duration_s": 24.7, "gen_cost_usd": 0.079841} +{"instance_id": "django__django-12915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 153.9, "eval_duration_s": 23.2, "gen_cost_usd": 0.089625} +{"instance_id": "django__django-12983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 99.1, "eval_duration_s": 19.7, "gen_cost_usd": 0.057149} +{"instance_id": "django__django-13028", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 94.6, "eval_duration_s": 19.1, "gen_cost_usd": 0.042915} +{"instance_id": "django__django-13033", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 163.0, "eval_duration_s": 23.4, "gen_cost_usd": 0.08646} +{"instance_id": "django__django-13158", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.0, "eval_duration_s": 23.3, "gen_cost_usd": 0.103054} +{"instance_id": "django__django-13220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 133.6, "eval_duration_s": 18.7, "gen_cost_usd": 0.086112} +{"instance_id": "django__django-13230", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 106.7, "eval_duration_s": 21.4, "gen_cost_usd": 0.075269} +{"instance_id": "django__django-13265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 352.9, "eval_duration_s": 23.8, "gen_cost_usd": 0.166218} +{"instance_id": "django__django-13315", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.7, "eval_duration_s": 25.0, "gen_cost_usd": 0.09272} +{"instance_id": "django__django-13321", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 126.4, "eval_duration_s": 27.4, "gen_cost_usd": 0.058362} +{"instance_id": "django__django-13401", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 212.6, "eval_duration_s": 23.3, "gen_cost_usd": 0.107082} +{"instance_id": "django__django-13447", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 141.5, "eval_duration_s": 27.7, "gen_cost_usd": 0.073622} +{"instance_id": "django__django-13448", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 97.1, "eval_duration_s": 21.9, "gen_cost_usd": 0.053798} +{"instance_id": "django__django-13551", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 107.6, "eval_duration_s": 20.9, "gen_cost_usd": 0.06229} +{"instance_id": "django__django-13590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 110.1, "eval_duration_s": 25.4, "gen_cost_usd": 0.054535} +{"instance_id": "django__django-13658", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 130.6, "eval_duration_s": 70.0, "gen_cost_usd": 0.071148} +{"instance_id": "django__django-13660", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 115.1, "eval_duration_s": 26.8, "gen_cost_usd": 0.040614} +{"instance_id": "django__django-13710", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.7, "eval_duration_s": 27.3, "gen_cost_usd": 0.106374} +{"instance_id": "django__django-13757", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.7, "eval_duration_s": 23.7, "gen_cost_usd": 0.085368} +{"instance_id": "django__django-13768", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 144.6, "eval_duration_s": 16.2, "gen_cost_usd": 0.066861} +{"instance_id": "django__django-13925", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 185.2, "eval_duration_s": 24.4, "gen_cost_usd": 0.096661} +{"instance_id": "django__django-13933", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 258.1, "eval_duration_s": 23.3, "gen_cost_usd": 0.130252} +{"instance_id": "django__django-13964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 208.6, "eval_duration_s": 21.7, "gen_cost_usd": 0.094928} +{"instance_id": "django__django-14016", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 100.0, "eval_duration_s": 22.0, "gen_cost_usd": 0.050162} +{"instance_id": "django__django-14017", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 223.9, "eval_duration_s": 27.7, "gen_cost_usd": 0.111232} +{"instance_id": "django__django-14155", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 134.3, "eval_duration_s": 24.8, "gen_cost_usd": 0.07405} +{"instance_id": "django__django-14238", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 101.7, "eval_duration_s": 27.5, "gen_cost_usd": 0.057615} +{"instance_id": "django__django-14382", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 176.8, "eval_duration_s": 46.3, "gen_cost_usd": 0.063674} +{"instance_id": "django__django-14411", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.8, "eval_duration_s": 26.9, "gen_cost_usd": 0.067886} +{"instance_id": "django__django-14534", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 173.5, "eval_duration_s": 27.0, "gen_cost_usd": 0.069069} +{"instance_id": "django__django-14580", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 174.5, "eval_duration_s": 23.6, "gen_cost_usd": 0.084116} +{"instance_id": "django__django-14608", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 153.3, "eval_duration_s": 45.7, "gen_cost_usd": 0.080502} +{"instance_id": "django__django-14667", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 156.5, "eval_duration_s": 27.4, "gen_cost_usd": 0.089558} +{"instance_id": "django__django-14672", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 155.2, "eval_duration_s": 24.9, "gen_cost_usd": 0.070677} +{"instance_id": "django__django-14730", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 221.2, "eval_duration_s": 23.4, "gen_cost_usd": 0.133805} +{"instance_id": "django__django-14752", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.2, "eval_duration_s": 27.0, "gen_cost_usd": 0.050679} +{"instance_id": "django__django-14787", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 103.8, "eval_duration_s": 24.4, "gen_cost_usd": 0.057858} +{"instance_id": "django__django-14855", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 173.9, "eval_duration_s": 134.1, "gen_cost_usd": 0.094746} +{"instance_id": "django__django-14915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 126.9, "eval_duration_s": 27.0, "gen_cost_usd": 0.039618} +{"instance_id": "django__django-14997", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 203.9, "eval_duration_s": 26.8, "gen_cost_usd": 0.093842} +{"instance_id": "django__django-14999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 129.6, "eval_duration_s": 26.8, "gen_cost_usd": 0.073062} +{"instance_id": "django__django-15061", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.0, "eval_duration_s": 20.3, "gen_cost_usd": 0.071132} +{"instance_id": "django__django-15202", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 145.5, "eval_duration_s": 27.4, "gen_cost_usd": 0.074686} +{"instance_id": "django__django-15213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 221.3, "eval_duration_s": 23.2, "gen_cost_usd": 0.124845} +{"instance_id": "django__django-15252", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.0, "eval_duration_s": 28.9, "gen_cost_usd": 0.116092} +{"instance_id": "django__django-15320", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 114.7, "eval_duration_s": 27.3, "gen_cost_usd": 0.054131} +{"instance_id": "django__django-15347", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 132.8, "eval_duration_s": 26.2, "gen_cost_usd": 0.075086} +{"instance_id": "django__django-15388", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.9, "eval_duration_s": 25.1, "gen_cost_usd": 0.03537} +{"instance_id": "django__django-15400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 93.7, "eval_duration_s": 25.1, "gen_cost_usd": 0.061711} +{"instance_id": "django__django-15498", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 139.5, "eval_duration_s": 24.3, "gen_cost_usd": 0.088139} +{"instance_id": "django__django-15695", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 248.9, "eval_duration_s": 23.0, "gen_cost_usd": 0.131051} +{"instance_id": "django__django-15738", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 313.1, "eval_duration_s": 24.7, "gen_cost_usd": 0.149059} +{"instance_id": "django__django-15781", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 204.3, "eval_duration_s": 20.2, "gen_cost_usd": 0.088405} +{"instance_id": "django__django-15789", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 88.1, "eval_duration_s": 29.8, "gen_cost_usd": 0.058314} +{"instance_id": "django__django-15790", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.0, "eval_duration_s": 25.0, "gen_cost_usd": 0.037328} +{"instance_id": "django__django-15814", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 104.8, "eval_duration_s": 24.5, "gen_cost_usd": 0.049743} +{"instance_id": "django__django-15819", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 222.8, "eval_duration_s": 17.3, "gen_cost_usd": 0.139046} +{"instance_id": "django__django-15851", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.0, "eval_duration_s": 22.7, "gen_cost_usd": 0.035849} +{"instance_id": "django__django-15902", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 95.5, "eval_duration_s": 26.4, "gen_cost_usd": 0.047537} +{"instance_id": "django__django-15996", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 143.9, "eval_duration_s": 22.5, "gen_cost_usd": 0.08278} +{"instance_id": "django__django-16041", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 111.8, "eval_duration_s": 19.0, "gen_cost_usd": 0.057034} +{"instance_id": "django__django-16046", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.3, "eval_duration_s": 27.4, "gen_cost_usd": 0.035198} +{"instance_id": "django__django-16139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 94.1, "eval_duration_s": 18.4, "gen_cost_usd": 0.061387} +{"instance_id": "django__django-16229", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "django__django-16255", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 90.6, "eval_duration_s": 19.8, "gen_cost_usd": 0.054666} +{"instance_id": "django__django-16379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.6, "eval_duration_s": 90.6, "gen_cost_usd": 0.042138} +{"instance_id": "django__django-16400", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 121.5, "eval_duration_s": 18.4, "gen_cost_usd": 0.06251} +{"instance_id": "django__django-16408", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 204.8, "eval_duration_s": 24.8, "gen_cost_usd": 0.107816} +{"instance_id": "django__django-16527", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.7, "eval_duration_s": 20.7, "gen_cost_usd": 0.03956} +{"instance_id": "django__django-16595", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.4, "eval_duration_s": 18.5, "gen_cost_usd": 0.069881} +{"instance_id": "django__django-16816", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 549.9, "eval_duration_s": 17.6, "gen_cost_usd": 0.288711} +{"instance_id": "django__django-16820", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.2, "eval_duration_s": 19.9, "gen_cost_usd": 0.124274} +{"instance_id": "django__django-16873", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 110.8, "eval_duration_s": 17.4, "gen_cost_usd": 0.067678} +{"instance_id": "django__django-16910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 481.5, "eval_duration_s": 17.9, "gen_cost_usd": 0.243535} +{"instance_id": "django__django-17051", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 129.3, "eval_duration_s": 17.5, "gen_cost_usd": 0.065104} +{"instance_id": "django__django-17087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 125.9, "eval_duration_s": 19.1, "gen_cost_usd": 0.061734} +{"instance_id": "matplotlib__matplotlib-18869", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 333.7, "eval_duration_s": 70.1, "gen_cost_usd": 0.079759} +{"instance_id": "matplotlib__matplotlib-22711", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 277.9, "eval_duration_s": 83.4, "gen_cost_usd": 0.0866} +{"instance_id": "matplotlib__matplotlib-22835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 298.5, "eval_duration_s": 121.8, "gen_cost_usd": 0.085641} +{"instance_id": "matplotlib__matplotlib-23299", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 403.1, "eval_duration_s": 119.3, "gen_cost_usd": 0.12945} +{"instance_id": "matplotlib__matplotlib-23314", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 212.9, "eval_duration_s": 111.7, "gen_cost_usd": 0.073529} +{"instance_id": "matplotlib__matplotlib-23476", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 338.2, "eval_duration_s": 85.6, "gen_cost_usd": 0.103703} +{"instance_id": "matplotlib__matplotlib-23562", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 265.9, "eval_duration_s": 80.8, "gen_cost_usd": 0.094962} +{"instance_id": "matplotlib__matplotlib-23563", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 341.2, "eval_duration_s": 84.6, "gen_cost_usd": 0.130338} +{"instance_id": "matplotlib__matplotlib-23913", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 312.7, "eval_duration_s": 108.7, "gen_cost_usd": 0.088111} +{"instance_id": "matplotlib__matplotlib-23964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 212.6, "eval_duration_s": 93.4, "gen_cost_usd": 0.042471} +{"instance_id": "matplotlib__matplotlib-23987", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 418.8, "eval_duration_s": 95.7, "gen_cost_usd": 0.093806} +{"instance_id": "matplotlib__matplotlib-24149", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 283.4, "eval_duration_s": 89.9, "gen_cost_usd": 0.05395} +{"instance_id": "matplotlib__matplotlib-24265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 262.4, "eval_duration_s": 85.1, "gen_cost_usd": 0.06309} +{"instance_id": "matplotlib__matplotlib-24334", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 451.6, "eval_duration_s": 122.1, "gen_cost_usd": 0.111089} +{"instance_id": "matplotlib__matplotlib-24970", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 337.0, "eval_duration_s": 80.0, "gen_cost_usd": 0.088502} +{"instance_id": "matplotlib__matplotlib-25079", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 339.2, "eval_duration_s": 87.9, "gen_cost_usd": 0.111273} +{"instance_id": "matplotlib__matplotlib-25311", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 441.9, "eval_duration_s": 87.2, "gen_cost_usd": 0.119274} +{"instance_id": "matplotlib__matplotlib-25332", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 268.9, "eval_duration_s": 82.3, "gen_cost_usd": 0.063431} +{"instance_id": "matplotlib__matplotlib-25433", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 397.0, "eval_duration_s": 81.0, "gen_cost_usd": 0.142743} +{"instance_id": "matplotlib__matplotlib-25442", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 288.0, "eval_duration_s": 82.1, "gen_cost_usd": 0.096203} +{"instance_id": "matplotlib__matplotlib-25498", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 322.1, "eval_duration_s": 128.0, "gen_cost_usd": 0.093743} +{"instance_id": "matplotlib__matplotlib-26011", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 591.4, "eval_duration_s": 91.2, "gen_cost_usd": 0.134968} +{"instance_id": "matplotlib__matplotlib-26020", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 250.7, "eval_duration_s": 112.0, "gen_cost_usd": 0.065568} +{"instance_id": "mwaskom__seaborn-2848", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 393.6, "eval_duration_s": 21.6, "gen_cost_usd": 0.109806} +{"instance_id": "mwaskom__seaborn-3010", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 197.8, "eval_duration_s": 9.1, "gen_cost_usd": 0.041585} +{"instance_id": "mwaskom__seaborn-3190", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 283.8, "eval_duration_s": 12.0, "gen_cost_usd": 0.090697} +{"instance_id": "mwaskom__seaborn-3407", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 386.5, "eval_duration_s": 71.5, "gen_cost_usd": 0.087443} +{"instance_id": "pallets__flask-4045", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 300.3, "eval_duration_s": 4.6, "gen_cost_usd": 0.104249} +{"instance_id": "pallets__flask-4992", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 149.8, "eval_duration_s": 5.2, "gen_cost_usd": 0.040922} +{"instance_id": "pallets__flask-5063", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 158.5, "eval_duration_s": 5.4, "gen_cost_usd": 0.058829} +{"instance_id": "psf__requests-1963", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 288.9, "eval_duration_s": 26.4, "gen_cost_usd": 0.103678} +{"instance_id": "psf__requests-2148", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 189.4, "eval_duration_s": 44.5, "gen_cost_usd": 0.042346} +{"instance_id": "psf__requests-2317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "timeout", "gen_duration_s": 290.4, "eval_duration_s": 600.0, "gen_cost_usd": 0.04981} +{"instance_id": "psf__requests-2674", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 200.2, "eval_duration_s": 37.7, "gen_cost_usd": 0.049318} +{"instance_id": "psf__requests-3362", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 301.1, "eval_duration_s": 5.6, "gen_cost_usd": 0.12329} +{"instance_id": "psf__requests-863", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 197.9, "eval_duration_s": 41.3, "gen_cost_usd": 0.054824} +{"instance_id": "pydata__xarray-3364", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 523.8, "eval_duration_s": 12.1, "gen_cost_usd": 0.232568} +{"instance_id": "pydata__xarray-4094", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 222.8, "eval_duration_s": 16.8, "gen_cost_usd": 0.067573} +{"instance_id": "pydata__xarray-4248", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 312.9, "eval_duration_s": 10.4, "gen_cost_usd": 0.113606} +{"instance_id": "pydata__xarray-4493", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 276.2, "eval_duration_s": 46.2, "gen_cost_usd": 0.091175} +{"instance_id": "pydata__xarray-5131", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 157.6, "eval_duration_s": 9.9, "gen_cost_usd": 0.03347} +{"instance_id": "pylint-dev__pylint-5859", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 103.9, "eval_duration_s": 8.4, "gen_cost_usd": 0.055425} +{"instance_id": "pylint-dev__pylint-6506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 150.7, "eval_duration_s": 8.7, "gen_cost_usd": 0.082501} +{"instance_id": "pylint-dev__pylint-7080", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 252.1, "eval_duration_s": 8.2, "gen_cost_usd": 0.131957} +{"instance_id": "pylint-dev__pylint-7114", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 144.7, "eval_duration_s": 5.6, "gen_cost_usd": 0.064266} +{"instance_id": "pylint-dev__pylint-7228", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 197.8, "eval_duration_s": 6.5, "gen_cost_usd": 0.127746} +{"instance_id": "pylint-dev__pylint-7993", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 106.1, "eval_duration_s": 5.8, "gen_cost_usd": 0.06825} +{"instance_id": "pytest-dev__pytest-11143", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 216.0, "eval_duration_s": 10.5, "gen_cost_usd": 0.069013} +{"instance_id": "pytest-dev__pytest-11148", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 256.1, "eval_duration_s": 12.9, "gen_cost_usd": 0.074746} +{"instance_id": "pytest-dev__pytest-5103", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 462.2, "eval_duration_s": 11.5, "gen_cost_usd": 0.233781} +{"instance_id": "pytest-dev__pytest-5221", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 244.1, "eval_duration_s": 13.8, "gen_cost_usd": 0.073051} +{"instance_id": "pytest-dev__pytest-5227", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 251.4, "eval_duration_s": 11.0, "gen_cost_usd": 0.100449} +{"instance_id": "pytest-dev__pytest-5413", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 207.6, "eval_duration_s": 9.8, "gen_cost_usd": 0.05666} +{"instance_id": "pytest-dev__pytest-5495", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.1, "eval_duration_s": 9.4, "gen_cost_usd": 0.053098} +{"instance_id": "pytest-dev__pytest-5692", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 172.2, "eval_duration_s": 10.9, "gen_cost_usd": 0.061905} +{"instance_id": "pytest-dev__pytest-6116", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 165.2, "eval_duration_s": 14.4, "gen_cost_usd": 0.036604} +{"instance_id": "pytest-dev__pytest-7168", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 191.4, "eval_duration_s": 10.4, "gen_cost_usd": 0.040527} +{"instance_id": "pytest-dev__pytest-7220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 299.9, "eval_duration_s": 11.2, "gen_cost_usd": 0.094994} +{"instance_id": "pytest-dev__pytest-7373", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 177.7, "eval_duration_s": 9.5, "gen_cost_usd": 0.033122} +{"instance_id": "pytest-dev__pytest-7432", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 224.6, "eval_duration_s": 10.6, "gen_cost_usd": 0.064174} +{"instance_id": "pytest-dev__pytest-7490", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 299.9, "eval_duration_s": 11.0, "gen_cost_usd": 0.072562} +{"instance_id": "pytest-dev__pytest-8365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 110.1, "eval_duration_s": 9.8, "gen_cost_usd": 0.065695} +{"instance_id": "pytest-dev__pytest-8906", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 223.0, "eval_duration_s": 11.2, "gen_cost_usd": 0.057594} +{"instance_id": "pytest-dev__pytest-9359", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 290.8, "eval_duration_s": 9.1, "gen_cost_usd": 0.090197} +{"instance_id": "scikit-learn__scikit-learn-10297", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 560.6, "eval_duration_s": 134.5, "gen_cost_usd": 0.072013} +{"instance_id": "scikit-learn__scikit-learn-10508", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 565.0, "eval_duration_s": 125.4, "gen_cost_usd": 0.074143} +{"instance_id": "scikit-learn__scikit-learn-10949", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 569.8, "eval_duration_s": 135.8, "gen_cost_usd": 0.085719} +{"instance_id": "scikit-learn__scikit-learn-11040", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 553.2, "eval_duration_s": 145.5, "gen_cost_usd": 0.08952} +{"instance_id": "scikit-learn__scikit-learn-11281", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "scikit-learn__scikit-learn-12471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 526.0, "eval_duration_s": 144.6, "gen_cost_usd": 0.09117} +{"instance_id": "scikit-learn__scikit-learn-13142", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 537.3, "eval_duration_s": 167.1, "gen_cost_usd": 0.071613} +{"instance_id": "scikit-learn__scikit-learn-13241", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "scikit-learn__scikit-learn-13439", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 503.7, "eval_duration_s": 171.2, "gen_cost_usd": 0.05985} +{"instance_id": "scikit-learn__scikit-learn-13496", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 529.9, "eval_duration_s": 164.4, "gen_cost_usd": 0.075915} +{"instance_id": "scikit-learn__scikit-learn-13497", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 484.9, "eval_duration_s": 174.5, "gen_cost_usd": 0.040341} +{"instance_id": "scikit-learn__scikit-learn-13584", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 482.0, "eval_duration_s": 171.9, "gen_cost_usd": 0.056177} +{"instance_id": "scikit-learn__scikit-learn-13779", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 518.6, "eval_duration_s": 209.8, "gen_cost_usd": 0.066756} +{"instance_id": "scikit-learn__scikit-learn-14087", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "scikit-learn__scikit-learn-14092", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 489.9, "eval_duration_s": 203.1, "gen_cost_usd": 0.088229} +{"instance_id": "scikit-learn__scikit-learn-14894", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 476.1, "eval_duration_s": 202.1, "gen_cost_usd": 0.080505} +{"instance_id": "scikit-learn__scikit-learn-14983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 445.2, "eval_duration_s": 205.2, "gen_cost_usd": 0.080515} +{"instance_id": "scikit-learn__scikit-learn-15512", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 469.5, "eval_duration_s": 205.4, "gen_cost_usd": 0.115147} +{"instance_id": "scikit-learn__scikit-learn-15535", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 334.2, "eval_duration_s": 202.3, "gen_cost_usd": 0.074412} +{"instance_id": "scikit-learn__scikit-learn-25500", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "scikit-learn__scikit-learn-25570", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "scikit-learn__scikit-learn-25638", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 351.4, "eval_duration_s": 13.6, "gen_cost_usd": 0.094304} +{"instance_id": "scikit-learn__scikit-learn-25747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 315.5, "eval_duration_s": 15.0, "gen_cost_usd": 0.114316} +{"instance_id": "sphinx-doc__sphinx-10325", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 242.8, "eval_duration_s": 14.5, "gen_cost_usd": 0.110573} +{"instance_id": "sphinx-doc__sphinx-10451", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 306.1, "eval_duration_s": 13.9, "gen_cost_usd": 0.14587} +{"instance_id": "sphinx-doc__sphinx-11445", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 284.3, "eval_duration_s": 13.9, "gen_cost_usd": 0.11017} +{"instance_id": "sphinx-doc__sphinx-7686", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 403.0, "eval_duration_s": 16.6, "gen_cost_usd": 0.187754} +{"instance_id": "sphinx-doc__sphinx-7738", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 206.5, "eval_duration_s": 16.6, "gen_cost_usd": 0.069386} +{"instance_id": "sphinx-doc__sphinx-7975", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 185.1, "eval_duration_s": 15.6, "gen_cost_usd": 0.128654} +{"instance_id": "sphinx-doc__sphinx-8273", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 126.1, "eval_duration_s": 13.8, "gen_cost_usd": 0.073368} +{"instance_id": "sphinx-doc__sphinx-8282", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 220.1, "eval_duration_s": 15.2, "gen_cost_usd": 0.146973} +{"instance_id": "sphinx-doc__sphinx-8435", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 267.7, "eval_duration_s": 13.5, "gen_cost_usd": 0.14341} +{"instance_id": "sphinx-doc__sphinx-8474", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 365.6, "eval_duration_s": 15.3, "gen_cost_usd": 0.189978} +{"instance_id": "sphinx-doc__sphinx-8506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 205.6, "eval_duration_s": 17.2, "gen_cost_usd": 0.135674} +{"instance_id": "sphinx-doc__sphinx-8595", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 196.5, "eval_duration_s": 15.1, "gen_cost_usd": 0.090744} +{"instance_id": "sphinx-doc__sphinx-8627", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 155.2, "eval_duration_s": 14.4, "gen_cost_usd": 0.108997} +{"instance_id": "sphinx-doc__sphinx-8713", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 116.1, "eval_duration_s": 16.9, "gen_cost_usd": 0.081077} +{"instance_id": "sphinx-doc__sphinx-8721", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 107.8, "eval_duration_s": 11.6, "gen_cost_usd": 0.060054} +{"instance_id": "sphinx-doc__sphinx-8801", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 199.6, "eval_duration_s": 13.7, "gen_cost_usd": 0.134846} +{"instance_id": "sympy__sympy-11400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 303.8, "eval_duration_s": 25.5, "gen_cost_usd": 0.102462} +{"instance_id": "sympy__sympy-11870", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 401.5, "eval_duration_s": 552.7, "gen_cost_usd": 0.194922} +{"instance_id": "sympy__sympy-11897", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 501.3, "eval_duration_s": 27.7, "gen_cost_usd": 0.225207} +{"instance_id": "sympy__sympy-12171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 230.0, "eval_duration_s": 30.6, "gen_cost_usd": 0.097165} +{"instance_id": "sympy__sympy-12236", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 236.9, "eval_duration_s": 36.9, "gen_cost_usd": 0.105261} +{"instance_id": "sympy__sympy-12419", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 166.6, "eval_duration_s": 25.3, "gen_cost_usd": 0.065146} +{"instance_id": "sympy__sympy-12454", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 126.2, "eval_duration_s": 39.8, "gen_cost_usd": 0.054694} +{"instance_id": "sympy__sympy-12481", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 157.6, "eval_duration_s": 20.2, "gen_cost_usd": 0.074173} +{"instance_id": "sympy__sympy-13031", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 168.9, "eval_duration_s": 27.8, "gen_cost_usd": 0.103816} +{"instance_id": "sympy__sympy-13043", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 127.2, "eval_duration_s": 25.0, "gen_cost_usd": 0.07869} +{"instance_id": "sympy__sympy-13146", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 425.4, "eval_duration_s": 34.4, "gen_cost_usd": 0.221145} +{"instance_id": "sympy__sympy-13177", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 78.4, "eval_duration_s": 25.0, "gen_cost_usd": 0.043302} +{"instance_id": "sympy__sympy-13437", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 66.0, "eval_duration_s": 28.8, "gen_cost_usd": 0.055409} +{"instance_id": "sympy__sympy-13471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 114.2, "eval_duration_s": 20.1, "gen_cost_usd": 0.073728} +{"instance_id": "sympy__sympy-13480", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 36.8, "eval_duration_s": 24.4, "gen_cost_usd": 0.010983} +{"instance_id": "sympy__sympy-13647", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.5, "eval_duration_s": 25.3, "gen_cost_usd": 0.053718} +{"instance_id": "sympy__sympy-13773", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 173.9, "eval_duration_s": 24.3, "gen_cost_usd": 0.08569} +{"instance_id": "sympy__sympy-13895", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 518.9, "eval_duration_s": 20.8, "gen_cost_usd": 0.316322} +{"instance_id": "sympy__sympy-13915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 208.7, "eval_duration_s": 22.7, "gen_cost_usd": 0.107024} +{"instance_id": "sympy__sympy-13971", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 86.7, "eval_duration_s": 33.2, "gen_cost_usd": 0.056409} +{"instance_id": "sympy__sympy-14024", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 175.0, "eval_duration_s": 29.6, "gen_cost_usd": 0.11829} +{"instance_id": "sympy__sympy-14308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 262.9, "eval_duration_s": 22.3, "gen_cost_usd": 0.152794} +{"instance_id": "sympy__sympy-14317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 220.1, "eval_duration_s": 22.6, "gen_cost_usd": 0.131946} +{"instance_id": "sympy__sympy-14396", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 85.7, "eval_duration_s": 20.1, "gen_cost_usd": 0.055482} +{"instance_id": "sympy__sympy-14774", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 45.6, "eval_duration_s": 22.6, "gen_cost_usd": 0.020607} +{"instance_id": "sympy__sympy-14817", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 137.1, "eval_duration_s": 28.1, "gen_cost_usd": 0.070551} +{"instance_id": "sympy__sympy-15011", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 315.7, "eval_duration_s": 27.2, "gen_cost_usd": 0.150251} +{"instance_id": "sympy__sympy-15308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 155.2, "eval_duration_s": 25.8, "gen_cost_usd": 0.074156} +{"instance_id": "sympy__sympy-15345", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 177.3, "eval_duration_s": 22.8, "gen_cost_usd": 0.109022} +{"instance_id": "sympy__sympy-15346", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.0, "eval_duration_s": 46.6, "gen_cost_usd": 0.037987} +{"instance_id": "sympy__sympy-15609", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.4, "eval_duration_s": 24.7, "gen_cost_usd": 0.048943} +{"instance_id": "sympy__sympy-15678", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 163.1, "eval_duration_s": 29.8, "gen_cost_usd": 0.095395} +{"instance_id": "sympy__sympy-16106", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 194.4, "eval_duration_s": 31.0, "gen_cost_usd": 0.113395} +{"instance_id": "sympy__sympy-16281", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 111.2, "eval_duration_s": 26.4, "gen_cost_usd": 0.059208} +{"instance_id": "sympy__sympy-16503", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 543.7, "eval_duration_s": 25.5, "gen_cost_usd": 0.273528} +{"instance_id": "sympy__sympy-16792", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 188.8, "eval_duration_s": 25.3, "gen_cost_usd": 0.108599} +{"instance_id": "sympy__sympy-16988", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null} +{"instance_id": "sympy__sympy-17022", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.6, "eval_duration_s": 21.8, "gen_cost_usd": 0.117767} +{"instance_id": "sympy__sympy-17139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 81.3, "eval_duration_s": 98.4, "gen_cost_usd": 0.032179} +{"instance_id": "sympy__sympy-17630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 171.3, "eval_duration_s": 21.6, "gen_cost_usd": 0.09608} +{"instance_id": "sympy__sympy-17655", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 129.4, "eval_duration_s": 28.7, "gen_cost_usd": 0.075998} +{"instance_id": "sympy__sympy-18057", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 138.7, "eval_duration_s": 106.1, "gen_cost_usd": 0.081103} +{"instance_id": "sympy__sympy-18087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 262.5, "eval_duration_s": 31.4, "gen_cost_usd": 0.127353} +{"instance_id": "sympy__sympy-18189", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 99.0, "eval_duration_s": 39.0, "gen_cost_usd": 0.051519} +{"instance_id": "sympy__sympy-18199", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 193.1, "eval_duration_s": 80.8, "gen_cost_usd": 0.101953} +{"instance_id": "sympy__sympy-18532", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 179.0, "eval_duration_s": 25.9, "gen_cost_usd": 0.082584} +{"instance_id": "sympy__sympy-18621", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.4, "eval_duration_s": 20.9, "gen_cost_usd": 0.043878} +{"instance_id": "sympy__sympy-18698", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 303.9, "eval_duration_s": 30.0, "gen_cost_usd": 0.149565} +{"instance_id": "sympy__sympy-18835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 128.1, "eval_duration_s": 22.1, "gen_cost_usd": 0.060965} +{"instance_id": "sympy__sympy-19007", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 322.6, "eval_duration_s": 23.7, "gen_cost_usd": 0.177031} +{"instance_id": "sympy__sympy-19254", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 281.6, "eval_duration_s": 21.8, "gen_cost_usd": 0.168758} +{"instance_id": "sympy__sympy-19487", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 128.3, "eval_duration_s": 28.1, "gen_cost_usd": 0.088216} +{"instance_id": "sympy__sympy-20049", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 256.1, "eval_duration_s": 21.9, "gen_cost_usd": 0.160782} +{"instance_id": "sympy__sympy-20154", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.9, "eval_duration_s": 22.6, "gen_cost_usd": 0.047626} +{"instance_id": "sympy__sympy-20212", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 130.9, "eval_duration_s": 41.6, "gen_cost_usd": 0.082753} +{"instance_id": "sympy__sympy-20322", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 302.6, "eval_duration_s": 34.7, "gen_cost_usd": 0.054426} +{"instance_id": "sympy__sympy-20442", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 152.3, "eval_duration_s": 22.3, "gen_cost_usd": 0.085405} +{"instance_id": "sympy__sympy-20590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 135.6, "eval_duration_s": 20.2, "gen_cost_usd": 0.062288} +{"instance_id": "sympy__sympy-20639", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 173.5, "eval_duration_s": 23.6, "gen_cost_usd": 0.08207} +{"instance_id": "sympy__sympy-21055", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 165.5, "eval_duration_s": 24.8, "gen_cost_usd": 0.088091} +{"instance_id": "sympy__sympy-21171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 115.1, "eval_duration_s": 23.3, "gen_cost_usd": 0.053466} +{"instance_id": "sympy__sympy-21379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 85.8, "eval_duration_s": 32.8, "gen_cost_usd": 0.04783} +{"instance_id": "sympy__sympy-21612", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 84.2, "eval_duration_s": 20.7, "gen_cost_usd": 0.038638} +{"instance_id": "sympy__sympy-21614", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.5, "eval_duration_s": 22.0, "gen_cost_usd": 0.05773} +{"instance_id": "sympy__sympy-21627", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 109.3, "eval_duration_s": 30.4, "gen_cost_usd": 0.071286} +{"instance_id": "sympy__sympy-21847", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 105.2, "eval_duration_s": 20.0, "gen_cost_usd": 0.071166} +{"instance_id": "sympy__sympy-22005", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 99.5, "eval_duration_s": 25.4, "gen_cost_usd": 0.050959} +{"instance_id": "sympy__sympy-22714", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 180.1, "eval_duration_s": 25.2, "gen_cost_usd": 0.041119} +{"instance_id": "sympy__sympy-22840", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 265.2, "eval_duration_s": 34.8, "gen_cost_usd": 0.131769} +{"instance_id": "sympy__sympy-23117", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.6, "eval_duration_s": 23.0, "gen_cost_usd": 0.050324} +{"instance_id": "sympy__sympy-23191", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 185.1, "eval_duration_s": 23.1, "gen_cost_usd": 0.106652} +{"instance_id": "sympy__sympy-23262", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 147.2, "eval_duration_s": 26.8, "gen_cost_usd": 0.10144} +{"instance_id": "sympy__sympy-24066", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 218.3, "eval_duration_s": 23.4, "gen_cost_usd": 0.111426} +{"instance_id": "sympy__sympy-24102", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 133.3, "eval_duration_s": 22.3, "gen_cost_usd": 0.079187} +{"instance_id": "sympy__sympy-24152", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.5, "eval_duration_s": 27.7, "gen_cost_usd": 0.058922} +{"instance_id": "sympy__sympy-24213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.4, "eval_duration_s": 23.4, "gen_cost_usd": 0.042842} +{"instance_id": "sympy__sympy-24909", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 291.3, "eval_duration_s": 22.3, "gen_cost_usd": 0.097123} diff --git a/evals/swe-bench/scoreboard/haiku-baseline-20260316/meta.json b/evals/swe-bench/scoreboard/haiku-baseline-20260316/meta.json new file mode 100644 index 000000000..72426a83d --- /dev/null +++ b/evals/swe-bench/scoreboard/haiku-baseline-20260316/meta.json @@ -0,0 +1,94 @@ +{ + "run_name": "haiku-baseline-20260316", + "timestamp": "2026-03-16T13:53:04.401784+00:00", + "model": "claude-haiku-4-5", + "provider": "anthropic", + "fabro_version": "fabro 0.5.0 (4c6ad4d 2026-03-16)", + "description": "Haiku 4.5 baseline, default prompt, 2 CPU / 4 GB, 10min timeout", + "notes": "", + "total_instances": 300, + "patched": 293, + "patch_pct": 97.7, + "resolved": 162, + "resolve_pct": 54.0, + "total_gen_cost_usd": 26.13, + "avg_gen_cost_usd": 0.0871, + "gen_wall_time_s": 3559.2, + "eval_wall_time_s": 625.2, + "avg_gen_duration_s": 224.3, + "avg_eval_duration_s": 41.5, + "per_repo": { + "astropy/astropy": { + "total": 6, + "patched": 6, + "resolved": 1, + "resolve_pct": 16.7 + }, + "django/django": { + "total": 114, + "patched": 113, + "resolved": 86, + "resolve_pct": 75.4 + }, + "matplotlib/matplotlib": { + "total": 23, + "patched": 23, + "resolved": 0, + "resolve_pct": 0.0 + }, + "mwaskom/seaborn": { + "total": 4, + "patched": 4, + "resolved": 1, + "resolve_pct": 25.0 + }, + "pallets/flask": { + "total": 3, + "patched": 3, + "resolved": 0, + "resolve_pct": 0.0 + }, + "psf/requests": { + "total": 6, + "patched": 6, + "resolved": 4, + "resolve_pct": 66.7 + }, + "pydata/xarray": { + "total": 5, + "patched": 5, + "resolved": 1, + "resolve_pct": 20.0 + }, + "pylint-dev/pylint": { + "total": 6, + "patched": 6, + "resolved": 0, + "resolve_pct": 0.0 + }, + "pytest-dev/pytest": { + "total": 17, + "patched": 17, + "resolved": 11, + "resolve_pct": 64.7 + }, + "scikit-learn/scikit-learn": { + "total": 23, + "patched": 18, + "resolved": 12, + "resolve_pct": 52.2 + }, + "sphinx-doc/sphinx": { + "total": 16, + "patched": 16, + "resolved": 0, + "resolve_pct": 0.0 + }, + "sympy/sympy": { + "total": 77, + "patched": 76, + "resolved": 46, + "resolve_pct": 59.7 + } + } +} diff --git a/evals/swe-bench/scoreboard/leaderboard.json b/evals/swe-bench/scoreboard/leaderboard.json new file mode 100644 index 000000000..83de0fb69 --- /dev/null +++ b/evals/swe-bench/scoreboard/leaderboard.json @@ -0,0 +1,15 @@ +[ + { + "run_name": "haiku-baseline-20260316", + "date": "2026-03-16", + "model": "claude-haiku-4-5", + "provider": "anthropic", + "resolved": 162, + "total": 300, + "resolve_pct": 54.0, + "patched": 293, + "patch_pct": 97.7, + "total_cost_usd": 26.13, + "avg_cost_usd": 0.0871 + } +]