mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-10-08 03:10:26 +00:00
Add scoreboard system and record Haiku 4.5 baseline: 54.0% on SWE-Bench-Lite
record_results.py combines generation + eval results into a git-tracked scoreboard. Per run: README, meta.json, instances.jsonl. Auto-generates leaderboard.json ranked by resolve rate. Haiku 4.5 baseline: 162/300 (54.0%), $26.13 total ($0.087/instance). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp
parent
30049c9ed8
commit
c248aaf92e
5 changed files with 708 additions and 0 deletions
261
evals/swe-bench/record_results.py
Normal file
261
evals/swe-bench/record_results.py
Normal file
|
|
@ -0,0 +1,261 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Record SWE-bench eval results into the scoreboard.
|
||||
|
||||
Reads generation results + eval results and produces a scoreboard entry
|
||||
with per-instance data, aggregate stats, and run metadata.
|
||||
|
||||
Usage:
|
||||
python record_results.py \
|
||||
--run-name haiku-baseline-20260316 \
|
||||
--gen-dir results/haiku-baseline \
|
||||
--eval-dir results/haiku-baseline/eval \
|
||||
--description "Haiku 4.5 baseline, default prompt, 10min timeout"
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
EVAL_DIR = Path(__file__).parent.resolve()
|
||||
SCOREBOARD_DIR = EVAL_DIR / "scoreboard"
|
||||
|
||||
|
||||
def load_jsonl(path: Path) -> list[dict]:
|
||||
rows = []
|
||||
with open(path) as f:
|
||||
for line in f:
|
||||
if line.strip():
|
||||
rows.append(json.loads(line))
|
||||
return rows
|
||||
|
||||
|
||||
def get_cost_from_fabro_run(fabro_run_dir: str | None) -> float | None:
|
||||
"""Extract total_cost from a fabro run's conclusion.json."""
|
||||
if not fabro_run_dir:
|
||||
return None
|
||||
conclusion = Path(fabro_run_dir) / "conclusion.json"
|
||||
if not conclusion.exists():
|
||||
return None
|
||||
try:
|
||||
data = json.loads(conclusion.read_text())
|
||||
return data.get("total_cost")
|
||||
except (json.JSONDecodeError, OSError):
|
||||
return None
|
||||
|
||||
|
||||
def get_fabro_version() -> str:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["fabro", "--version"], capture_output=True, text=True, timeout=5
|
||||
)
|
||||
return result.stdout.strip()
|
||||
except Exception:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Record SWE-bench eval results into the scoreboard"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--run-name", required=True,
|
||||
help="Name for this run (e.g. haiku-baseline-20260316)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--gen-dir", type=Path, required=True,
|
||||
help="Generation results directory (contains predictions.jsonl, results.jsonl)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--eval-dir", type=Path, required=True,
|
||||
help="Evaluation results directory (contains eval_results.jsonl)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--description", default="",
|
||||
help="Human-readable description of what was tested",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--notes", default="",
|
||||
help="Additional notes or observations",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
run_dir = SCOREBOARD_DIR / args.run_name
|
||||
run_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load generation results
|
||||
gen_results = {r["instance_id"]: r for r in load_jsonl(args.gen_dir / "results.jsonl")}
|
||||
gen_summary = json.loads((args.gen_dir / "summary.json").read_text())
|
||||
|
||||
# Load eval results
|
||||
eval_results = {r["instance_id"]: r for r in load_jsonl(args.eval_dir / "eval_results.jsonl")}
|
||||
eval_summary = json.loads((args.eval_dir / "summary.json").read_text())
|
||||
|
||||
# Build per-instance records
|
||||
all_instance_ids = sorted(set(gen_results.keys()) | set(eval_results.keys()))
|
||||
instances = []
|
||||
total_gen_cost = 0.0
|
||||
for iid in all_instance_ids:
|
||||
gen = gen_results.get(iid, {})
|
||||
evl = eval_results.get(iid, {})
|
||||
|
||||
has_patch = bool(gen.get("model_patch", "").strip())
|
||||
resolved = evl.get("resolved", False)
|
||||
gen_duration = gen.get("duration_s")
|
||||
eval_duration = evl.get("duration_s")
|
||||
gen_status = gen.get("status", "missing")
|
||||
eval_status = evl.get("status", "not_evaluated")
|
||||
cost = get_cost_from_fabro_run(gen.get("fabro_run_dir"))
|
||||
if cost:
|
||||
total_gen_cost += cost
|
||||
|
||||
instances.append({
|
||||
"instance_id": iid,
|
||||
"has_patch": has_patch,
|
||||
"resolved": resolved,
|
||||
"gen_status": gen_status,
|
||||
"eval_status": eval_status,
|
||||
"gen_duration_s": gen_duration,
|
||||
"eval_duration_s": eval_duration,
|
||||
"gen_cost_usd": round(cost, 6) if cost else None,
|
||||
})
|
||||
|
||||
# Write per-instance results
|
||||
instances_path = run_dir / "instances.jsonl"
|
||||
with open(instances_path, "w") as f:
|
||||
for inst in instances:
|
||||
f.write(json.dumps(inst) + "\n")
|
||||
|
||||
# Compute aggregates
|
||||
total = len(instances)
|
||||
patched = sum(1 for i in instances if i["has_patch"])
|
||||
resolved = sum(1 for i in instances if i["resolved"])
|
||||
resolve_pct = round(100 * resolved / total, 1) if total > 0 else 0
|
||||
patch_pct = round(100 * patched / total, 1) if total > 0 else 0
|
||||
|
||||
gen_durations = [i["gen_duration_s"] for i in instances if i["gen_duration_s"] is not None]
|
||||
eval_durations = [i["eval_duration_s"] for i in instances if i["eval_duration_s"] is not None]
|
||||
|
||||
# Per-repo breakdown
|
||||
from collections import Counter
|
||||
repo_total: Counter[str] = Counter()
|
||||
repo_resolved: Counter[str] = Counter()
|
||||
repo_patched: Counter[str] = Counter()
|
||||
for inst in instances:
|
||||
parts = inst["instance_id"].split("__")
|
||||
repo = f"{parts[0]}/{parts[1].rsplit('-', 1)[0]}" if len(parts) >= 2 else inst["instance_id"]
|
||||
repo_total[repo] += 1
|
||||
if inst["has_patch"]:
|
||||
repo_patched[repo] += 1
|
||||
if inst["resolved"]:
|
||||
repo_resolved[repo] += 1
|
||||
|
||||
per_repo = {}
|
||||
for repo in sorted(repo_total):
|
||||
per_repo[repo] = {
|
||||
"total": repo_total[repo],
|
||||
"patched": repo_patched[repo],
|
||||
"resolved": repo_resolved[repo],
|
||||
"resolve_pct": round(100 * repo_resolved[repo] / repo_total[repo], 1),
|
||||
}
|
||||
|
||||
# Write metadata
|
||||
meta = {
|
||||
"run_name": args.run_name,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"model": gen_summary.get("model", "unknown"),
|
||||
"provider": gen_summary.get("provider", "unknown"),
|
||||
"fabro_version": get_fabro_version(),
|
||||
"description": args.description,
|
||||
"notes": args.notes,
|
||||
"total_instances": total,
|
||||
"patched": patched,
|
||||
"patch_pct": patch_pct,
|
||||
"resolved": resolved,
|
||||
"resolve_pct": resolve_pct,
|
||||
"total_gen_cost_usd": round(total_gen_cost, 2),
|
||||
"avg_gen_cost_usd": round(total_gen_cost / total, 4) if total > 0 else 0,
|
||||
"gen_wall_time_s": gen_summary.get("total_duration_s"),
|
||||
"eval_wall_time_s": eval_summary.get("wall_duration_s"),
|
||||
"avg_gen_duration_s": round(sum(gen_durations) / len(gen_durations), 1) if gen_durations else None,
|
||||
"avg_eval_duration_s": round(sum(eval_durations) / len(eval_durations), 1) if eval_durations else None,
|
||||
"per_repo": per_repo,
|
||||
}
|
||||
(run_dir / "meta.json").write_text(json.dumps(meta, indent=2) + "\n")
|
||||
|
||||
# Write README
|
||||
readme_lines = [
|
||||
f"# {args.run_name}",
|
||||
"",
|
||||
f"**Date:** {meta['timestamp'][:10]}",
|
||||
f"**Model:** {meta['model']} ({meta['provider']})",
|
||||
f"**Fabro:** {meta['fabro_version']}",
|
||||
"",
|
||||
f"## Description",
|
||||
"",
|
||||
args.description or "_No description provided._",
|
||||
"",
|
||||
f"## Results",
|
||||
"",
|
||||
f"| Metric | Value |",
|
||||
f"|--------|-------|",
|
||||
f"| Instances | {total} |",
|
||||
f"| Patched | {patched} ({patch_pct}%) |",
|
||||
f"| **Resolved** | **{resolved} ({resolve_pct}%)** |",
|
||||
f"| Total gen cost | ${meta['total_gen_cost_usd']:.2f} |",
|
||||
f"| Avg gen cost | ${meta['avg_gen_cost_usd']:.4f}/instance |",
|
||||
f"| Gen wall time | {meta['gen_wall_time_s']}s |",
|
||||
f"| Eval wall time | {meta['eval_wall_time_s']}s |",
|
||||
"",
|
||||
f"## Per-repo breakdown",
|
||||
"",
|
||||
f"| Repo | Resolved | Total | Rate |",
|
||||
f"|------|----------|-------|------|",
|
||||
]
|
||||
for repo, stats in per_repo.items():
|
||||
readme_lines.append(
|
||||
f"| {repo} | {stats['resolved']} | {stats['total']} | {stats['resolve_pct']}% |"
|
||||
)
|
||||
|
||||
if args.notes:
|
||||
readme_lines.extend(["", "## Notes", "", args.notes])
|
||||
|
||||
(run_dir / "README.md").write_text("\n".join(readme_lines) + "\n")
|
||||
|
||||
# Regenerate leaderboard
|
||||
regenerate_leaderboard()
|
||||
|
||||
# Print summary
|
||||
print(f"Recorded results for '{args.run_name}'")
|
||||
print(f" Resolved: {resolved}/{total} ({resolve_pct}%)")
|
||||
print(f" Cost: ${meta['total_gen_cost_usd']:.2f} total, ${meta['avg_gen_cost_usd']:.4f}/instance")
|
||||
print(f" Scoreboard: {run_dir}")
|
||||
|
||||
|
||||
def regenerate_leaderboard():
|
||||
"""Rebuild leaderboard.json from all scoreboard entries."""
|
||||
entries = []
|
||||
for meta_path in sorted(SCOREBOARD_DIR.glob("*/meta.json")):
|
||||
meta = json.loads(meta_path.read_text())
|
||||
entries.append({
|
||||
"run_name": meta["run_name"],
|
||||
"date": meta["timestamp"][:10],
|
||||
"model": meta["model"],
|
||||
"provider": meta["provider"],
|
||||
"resolved": meta["resolved"],
|
||||
"total": meta["total_instances"],
|
||||
"resolve_pct": meta["resolve_pct"],
|
||||
"patched": meta["patched"],
|
||||
"patch_pct": meta["patch_pct"],
|
||||
"total_cost_usd": meta.get("total_gen_cost_usd"),
|
||||
"avg_cost_usd": meta.get("avg_gen_cost_usd"),
|
||||
})
|
||||
|
||||
# Sort by resolve rate descending
|
||||
entries.sort(key=lambda e: e["resolve_pct"], reverse=True)
|
||||
(SCOREBOARD_DIR / "leaderboard.json").write_text(json.dumps(entries, indent=2) + "\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
38
evals/swe-bench/scoreboard/haiku-baseline-20260316/README.md
Normal file
38
evals/swe-bench/scoreboard/haiku-baseline-20260316/README.md
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
# haiku-baseline-20260316
|
||||
|
||||
**Date:** 2026-03-16
|
||||
**Model:** claude-haiku-4-5 (anthropic)
|
||||
**Fabro:** fabro 0.5.0 (4c6ad4d 2026-03-16)
|
||||
|
||||
## Description
|
||||
|
||||
Haiku 4.5 baseline, default prompt, 2 CPU / 4 GB, 10min timeout
|
||||
|
||||
## Results
|
||||
|
||||
| Metric | Value |
|
||||
|--------|-------|
|
||||
| Instances | 300 |
|
||||
| Patched | 293 (97.7%) |
|
||||
| **Resolved** | **162 (54.0%)** |
|
||||
| Total gen cost | $26.13 |
|
||||
| Avg gen cost | $0.0871/instance |
|
||||
| Gen wall time | 3559.2s |
|
||||
| Eval wall time | 625.2s |
|
||||
|
||||
## Per-repo breakdown
|
||||
|
||||
| Repo | Resolved | Total | Rate |
|
||||
|------|----------|-------|------|
|
||||
| astropy/astropy | 1 | 6 | 16.7% |
|
||||
| django/django | 86 | 114 | 75.4% |
|
||||
| matplotlib/matplotlib | 0 | 23 | 0.0% |
|
||||
| mwaskom/seaborn | 1 | 4 | 25.0% |
|
||||
| pallets/flask | 0 | 3 | 0.0% |
|
||||
| psf/requests | 4 | 6 | 66.7% |
|
||||
| pydata/xarray | 1 | 5 | 20.0% |
|
||||
| pylint-dev/pylint | 0 | 6 | 0.0% |
|
||||
| pytest-dev/pytest | 11 | 17 | 64.7% |
|
||||
| scikit-learn/scikit-learn | 12 | 23 | 52.2% |
|
||||
| sphinx-doc/sphinx | 0 | 16 | 0.0% |
|
||||
| sympy/sympy | 46 | 77 | 59.7% |
|
||||
|
|
@ -0,0 +1,300 @@
|
|||
{"instance_id": "astropy__astropy-12907", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 166.0, "eval_duration_s": 63.8, "gen_cost_usd": 0.069072}
|
||||
{"instance_id": "astropy__astropy-14182", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 189.6, "eval_duration_s": 63.3, "gen_cost_usd": 0.077053}
|
||||
{"instance_id": "astropy__astropy-14365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 118.7, "eval_duration_s": 44.9, "gen_cost_usd": 0.056338}
|
||||
{"instance_id": "astropy__astropy-14995", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 162.9, "eval_duration_s": 47.1, "gen_cost_usd": 0.077115}
|
||||
{"instance_id": "astropy__astropy-6938", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 182.5, "eval_duration_s": 83.1, "gen_cost_usd": 0.076278}
|
||||
{"instance_id": "astropy__astropy-7746", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 250.5, "eval_duration_s": 83.8, "gen_cost_usd": 0.087522}
|
||||
{"instance_id": "django__django-10914", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 130.1, "eval_duration_s": 23.5, "gen_cost_usd": 0.073745}
|
||||
{"instance_id": "django__django-10924", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 134.9, "eval_duration_s": 27.1, "gen_cost_usd": 0.079222}
|
||||
{"instance_id": "django__django-11001", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 141.9, "eval_duration_s": 24.2, "gen_cost_usd": 0.086682}
|
||||
{"instance_id": "django__django-11019", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 219.7, "eval_duration_s": 25.3, "gen_cost_usd": 0.132849}
|
||||
{"instance_id": "django__django-11039", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 120.7, "eval_duration_s": 20.9, "gen_cost_usd": 0.069046}
|
||||
{"instance_id": "django__django-11049", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.7, "eval_duration_s": 17.8, "gen_cost_usd": 0.049394}
|
||||
{"instance_id": "django__django-11099", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.4, "eval_duration_s": 19.8, "gen_cost_usd": 0.055786}
|
||||
{"instance_id": "django__django-11133", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 109.4, "eval_duration_s": 22.1, "gen_cost_usd": 0.075125}
|
||||
{"instance_id": "django__django-11179", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 218.0, "eval_duration_s": 24.1, "gen_cost_usd": 0.097138}
|
||||
{"instance_id": "django__django-11283", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 201.1, "eval_duration_s": 24.6, "gen_cost_usd": 0.122612}
|
||||
{"instance_id": "django__django-11422", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 112.7, "eval_duration_s": 25.0, "gen_cost_usd": 0.076847}
|
||||
{"instance_id": "django__django-11564", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 191.0, "eval_duration_s": 27.7, "gen_cost_usd": 0.11641}
|
||||
{"instance_id": "django__django-11583", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 100.1, "eval_duration_s": 21.8, "gen_cost_usd": 0.068367}
|
||||
{"instance_id": "django__django-11620", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 195.3, "eval_duration_s": 22.4, "gen_cost_usd": 0.090314}
|
||||
{"instance_id": "django__django-11630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 108.3, "eval_duration_s": 25.4, "gen_cost_usd": 0.073506}
|
||||
{"instance_id": "django__django-11742", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 182.4, "eval_duration_s": 21.6, "gen_cost_usd": 0.101582}
|
||||
{"instance_id": "django__django-11797", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 190.5, "eval_duration_s": 23.0, "gen_cost_usd": 0.095608}
|
||||
{"instance_id": "django__django-11815", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 160.9, "eval_duration_s": 19.0, "gen_cost_usd": 0.090641}
|
||||
{"instance_id": "django__django-11848", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 89.9, "eval_duration_s": 24.5, "gen_cost_usd": 0.057066}
|
||||
{"instance_id": "django__django-11905", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 167.0, "eval_duration_s": 21.4, "gen_cost_usd": 0.080023}
|
||||
{"instance_id": "django__django-11910", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 415.6, "eval_duration_s": 22.1, "gen_cost_usd": 0.257499}
|
||||
{"instance_id": "django__django-11964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 398.2, "eval_duration_s": 20.6, "gen_cost_usd": 0.111035}
|
||||
{"instance_id": "django__django-11999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 112.5, "eval_duration_s": 23.7, "gen_cost_usd": 0.058971}
|
||||
{"instance_id": "django__django-12113", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.6, "eval_duration_s": 19.6, "gen_cost_usd": 0.055079}
|
||||
{"instance_id": "django__django-12125", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 241.7, "eval_duration_s": 20.0, "gen_cost_usd": 0.127666}
|
||||
{"instance_id": "django__django-12184", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 141.2, "eval_duration_s": 24.9, "gen_cost_usd": 0.085902}
|
||||
{"instance_id": "django__django-12284", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.2, "eval_duration_s": 21.7, "gen_cost_usd": 0.071082}
|
||||
{"instance_id": "django__django-12286", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 104.9, "eval_duration_s": 23.9, "gen_cost_usd": 0.066556}
|
||||
{"instance_id": "django__django-12308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 153.0, "eval_duration_s": 25.2, "gen_cost_usd": 0.086661}
|
||||
{"instance_id": "django__django-12453", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 79.7, "eval_duration_s": 29.8, "gen_cost_usd": 0.043009}
|
||||
{"instance_id": "django__django-12470", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 520.5, "eval_duration_s": 20.3, "gen_cost_usd": 0.251534}
|
||||
{"instance_id": "django__django-12497", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 82.6, "eval_duration_s": 20.9, "gen_cost_usd": 0.049238}
|
||||
{"instance_id": "django__django-12589", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 366.7, "eval_duration_s": 21.5, "gen_cost_usd": 0.147055}
|
||||
{"instance_id": "django__django-12700", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 104.2, "eval_duration_s": 26.4, "gen_cost_usd": 0.054288}
|
||||
{"instance_id": "django__django-12708", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 217.7, "eval_duration_s": 16.9, "gen_cost_usd": 0.121474}
|
||||
{"instance_id": "django__django-12747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 269.0, "eval_duration_s": 24.0, "gen_cost_usd": 0.138026}
|
||||
{"instance_id": "django__django-12856", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 250.0, "eval_duration_s": 24.5, "gen_cost_usd": 0.147799}
|
||||
{"instance_id": "django__django-12908", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 178.6, "eval_duration_s": 24.7, "gen_cost_usd": 0.079841}
|
||||
{"instance_id": "django__django-12915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 153.9, "eval_duration_s": 23.2, "gen_cost_usd": 0.089625}
|
||||
{"instance_id": "django__django-12983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 99.1, "eval_duration_s": 19.7, "gen_cost_usd": 0.057149}
|
||||
{"instance_id": "django__django-13028", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 94.6, "eval_duration_s": 19.1, "gen_cost_usd": 0.042915}
|
||||
{"instance_id": "django__django-13033", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 163.0, "eval_duration_s": 23.4, "gen_cost_usd": 0.08646}
|
||||
{"instance_id": "django__django-13158", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.0, "eval_duration_s": 23.3, "gen_cost_usd": 0.103054}
|
||||
{"instance_id": "django__django-13220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 133.6, "eval_duration_s": 18.7, "gen_cost_usd": 0.086112}
|
||||
{"instance_id": "django__django-13230", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 106.7, "eval_duration_s": 21.4, "gen_cost_usd": 0.075269}
|
||||
{"instance_id": "django__django-13265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 352.9, "eval_duration_s": 23.8, "gen_cost_usd": 0.166218}
|
||||
{"instance_id": "django__django-13315", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.7, "eval_duration_s": 25.0, "gen_cost_usd": 0.09272}
|
||||
{"instance_id": "django__django-13321", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 126.4, "eval_duration_s": 27.4, "gen_cost_usd": 0.058362}
|
||||
{"instance_id": "django__django-13401", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 212.6, "eval_duration_s": 23.3, "gen_cost_usd": 0.107082}
|
||||
{"instance_id": "django__django-13447", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 141.5, "eval_duration_s": 27.7, "gen_cost_usd": 0.073622}
|
||||
{"instance_id": "django__django-13448", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 97.1, "eval_duration_s": 21.9, "gen_cost_usd": 0.053798}
|
||||
{"instance_id": "django__django-13551", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 107.6, "eval_duration_s": 20.9, "gen_cost_usd": 0.06229}
|
||||
{"instance_id": "django__django-13590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 110.1, "eval_duration_s": 25.4, "gen_cost_usd": 0.054535}
|
||||
{"instance_id": "django__django-13658", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 130.6, "eval_duration_s": 70.0, "gen_cost_usd": 0.071148}
|
||||
{"instance_id": "django__django-13660", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 115.1, "eval_duration_s": 26.8, "gen_cost_usd": 0.040614}
|
||||
{"instance_id": "django__django-13710", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.7, "eval_duration_s": 27.3, "gen_cost_usd": 0.106374}
|
||||
{"instance_id": "django__django-13757", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.7, "eval_duration_s": 23.7, "gen_cost_usd": 0.085368}
|
||||
{"instance_id": "django__django-13768", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 144.6, "eval_duration_s": 16.2, "gen_cost_usd": 0.066861}
|
||||
{"instance_id": "django__django-13925", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 185.2, "eval_duration_s": 24.4, "gen_cost_usd": 0.096661}
|
||||
{"instance_id": "django__django-13933", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 258.1, "eval_duration_s": 23.3, "gen_cost_usd": 0.130252}
|
||||
{"instance_id": "django__django-13964", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 208.6, "eval_duration_s": 21.7, "gen_cost_usd": 0.094928}
|
||||
{"instance_id": "django__django-14016", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 100.0, "eval_duration_s": 22.0, "gen_cost_usd": 0.050162}
|
||||
{"instance_id": "django__django-14017", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 223.9, "eval_duration_s": 27.7, "gen_cost_usd": 0.111232}
|
||||
{"instance_id": "django__django-14155", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 134.3, "eval_duration_s": 24.8, "gen_cost_usd": 0.07405}
|
||||
{"instance_id": "django__django-14238", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 101.7, "eval_duration_s": 27.5, "gen_cost_usd": 0.057615}
|
||||
{"instance_id": "django__django-14382", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 176.8, "eval_duration_s": 46.3, "gen_cost_usd": 0.063674}
|
||||
{"instance_id": "django__django-14411", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 133.8, "eval_duration_s": 26.9, "gen_cost_usd": 0.067886}
|
||||
{"instance_id": "django__django-14534", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 173.5, "eval_duration_s": 27.0, "gen_cost_usd": 0.069069}
|
||||
{"instance_id": "django__django-14580", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 174.5, "eval_duration_s": 23.6, "gen_cost_usd": 0.084116}
|
||||
{"instance_id": "django__django-14608", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 153.3, "eval_duration_s": 45.7, "gen_cost_usd": 0.080502}
|
||||
{"instance_id": "django__django-14667", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 156.5, "eval_duration_s": 27.4, "gen_cost_usd": 0.089558}
|
||||
{"instance_id": "django__django-14672", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 155.2, "eval_duration_s": 24.9, "gen_cost_usd": 0.070677}
|
||||
{"instance_id": "django__django-14730", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 221.2, "eval_duration_s": 23.4, "gen_cost_usd": 0.133805}
|
||||
{"instance_id": "django__django-14752", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.2, "eval_duration_s": 27.0, "gen_cost_usd": 0.050679}
|
||||
{"instance_id": "django__django-14787", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 103.8, "eval_duration_s": 24.4, "gen_cost_usd": 0.057858}
|
||||
{"instance_id": "django__django-14855", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 173.9, "eval_duration_s": 134.1, "gen_cost_usd": 0.094746}
|
||||
{"instance_id": "django__django-14915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 126.9, "eval_duration_s": 27.0, "gen_cost_usd": 0.039618}
|
||||
{"instance_id": "django__django-14997", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 203.9, "eval_duration_s": 26.8, "gen_cost_usd": 0.093842}
|
||||
{"instance_id": "django__django-14999", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 129.6, "eval_duration_s": 26.8, "gen_cost_usd": 0.073062}
|
||||
{"instance_id": "django__django-15061", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 116.0, "eval_duration_s": 20.3, "gen_cost_usd": 0.071132}
|
||||
{"instance_id": "django__django-15202", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 145.5, "eval_duration_s": 27.4, "gen_cost_usd": 0.074686}
|
||||
{"instance_id": "django__django-15213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 221.3, "eval_duration_s": 23.2, "gen_cost_usd": 0.124845}
|
||||
{"instance_id": "django__django-15252", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.0, "eval_duration_s": 28.9, "gen_cost_usd": 0.116092}
|
||||
{"instance_id": "django__django-15320", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 114.7, "eval_duration_s": 27.3, "gen_cost_usd": 0.054131}
|
||||
{"instance_id": "django__django-15347", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 132.8, "eval_duration_s": 26.2, "gen_cost_usd": 0.075086}
|
||||
{"instance_id": "django__django-15388", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 69.9, "eval_duration_s": 25.1, "gen_cost_usd": 0.03537}
|
||||
{"instance_id": "django__django-15400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 93.7, "eval_duration_s": 25.1, "gen_cost_usd": 0.061711}
|
||||
{"instance_id": "django__django-15498", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 139.5, "eval_duration_s": 24.3, "gen_cost_usd": 0.088139}
|
||||
{"instance_id": "django__django-15695", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 248.9, "eval_duration_s": 23.0, "gen_cost_usd": 0.131051}
|
||||
{"instance_id": "django__django-15738", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 313.1, "eval_duration_s": 24.7, "gen_cost_usd": 0.149059}
|
||||
{"instance_id": "django__django-15781", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 204.3, "eval_duration_s": 20.2, "gen_cost_usd": 0.088405}
|
||||
{"instance_id": "django__django-15789", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 88.1, "eval_duration_s": 29.8, "gen_cost_usd": 0.058314}
|
||||
{"instance_id": "django__django-15790", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.0, "eval_duration_s": 25.0, "gen_cost_usd": 0.037328}
|
||||
{"instance_id": "django__django-15814", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 104.8, "eval_duration_s": 24.5, "gen_cost_usd": 0.049743}
|
||||
{"instance_id": "django__django-15819", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 222.8, "eval_duration_s": 17.3, "gen_cost_usd": 0.139046}
|
||||
{"instance_id": "django__django-15851", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 67.0, "eval_duration_s": 22.7, "gen_cost_usd": 0.035849}
|
||||
{"instance_id": "django__django-15902", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 95.5, "eval_duration_s": 26.4, "gen_cost_usd": 0.047537}
|
||||
{"instance_id": "django__django-15996", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 143.9, "eval_duration_s": 22.5, "gen_cost_usd": 0.08278}
|
||||
{"instance_id": "django__django-16041", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 111.8, "eval_duration_s": 19.0, "gen_cost_usd": 0.057034}
|
||||
{"instance_id": "django__django-16046", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 70.3, "eval_duration_s": 27.4, "gen_cost_usd": 0.035198}
|
||||
{"instance_id": "django__django-16139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 94.1, "eval_duration_s": 18.4, "gen_cost_usd": 0.061387}
|
||||
{"instance_id": "django__django-16229", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "django__django-16255", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 90.6, "eval_duration_s": 19.8, "gen_cost_usd": 0.054666}
|
||||
{"instance_id": "django__django-16379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 137.6, "eval_duration_s": 90.6, "gen_cost_usd": 0.042138}
|
||||
{"instance_id": "django__django-16400", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 121.5, "eval_duration_s": 18.4, "gen_cost_usd": 0.06251}
|
||||
{"instance_id": "django__django-16408", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 204.8, "eval_duration_s": 24.8, "gen_cost_usd": 0.107816}
|
||||
{"instance_id": "django__django-16527", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.7, "eval_duration_s": 20.7, "gen_cost_usd": 0.03956}
|
||||
{"instance_id": "django__django-16595", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.4, "eval_duration_s": 18.5, "gen_cost_usd": 0.069881}
|
||||
{"instance_id": "django__django-16816", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 549.9, "eval_duration_s": 17.6, "gen_cost_usd": 0.288711}
|
||||
{"instance_id": "django__django-16820", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.2, "eval_duration_s": 19.9, "gen_cost_usd": 0.124274}
|
||||
{"instance_id": "django__django-16873", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 110.8, "eval_duration_s": 17.4, "gen_cost_usd": 0.067678}
|
||||
{"instance_id": "django__django-16910", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 481.5, "eval_duration_s": 17.9, "gen_cost_usd": 0.243535}
|
||||
{"instance_id": "django__django-17051", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 129.3, "eval_duration_s": 17.5, "gen_cost_usd": 0.065104}
|
||||
{"instance_id": "django__django-17087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 125.9, "eval_duration_s": 19.1, "gen_cost_usd": 0.061734}
|
||||
{"instance_id": "matplotlib__matplotlib-18869", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 333.7, "eval_duration_s": 70.1, "gen_cost_usd": 0.079759}
|
||||
{"instance_id": "matplotlib__matplotlib-22711", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 277.9, "eval_duration_s": 83.4, "gen_cost_usd": 0.0866}
|
||||
{"instance_id": "matplotlib__matplotlib-22835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 298.5, "eval_duration_s": 121.8, "gen_cost_usd": 0.085641}
|
||||
{"instance_id": "matplotlib__matplotlib-23299", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 403.1, "eval_duration_s": 119.3, "gen_cost_usd": 0.12945}
|
||||
{"instance_id": "matplotlib__matplotlib-23314", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 212.9, "eval_duration_s": 111.7, "gen_cost_usd": 0.073529}
|
||||
{"instance_id": "matplotlib__matplotlib-23476", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 338.2, "eval_duration_s": 85.6, "gen_cost_usd": 0.103703}
|
||||
{"instance_id": "matplotlib__matplotlib-23562", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 265.9, "eval_duration_s": 80.8, "gen_cost_usd": 0.094962}
|
||||
{"instance_id": "matplotlib__matplotlib-23563", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 341.2, "eval_duration_s": 84.6, "gen_cost_usd": 0.130338}
|
||||
{"instance_id": "matplotlib__matplotlib-23913", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 312.7, "eval_duration_s": 108.7, "gen_cost_usd": 0.088111}
|
||||
{"instance_id": "matplotlib__matplotlib-23964", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 212.6, "eval_duration_s": 93.4, "gen_cost_usd": 0.042471}
|
||||
{"instance_id": "matplotlib__matplotlib-23987", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 418.8, "eval_duration_s": 95.7, "gen_cost_usd": 0.093806}
|
||||
{"instance_id": "matplotlib__matplotlib-24149", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 283.4, "eval_duration_s": 89.9, "gen_cost_usd": 0.05395}
|
||||
{"instance_id": "matplotlib__matplotlib-24265", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 262.4, "eval_duration_s": 85.1, "gen_cost_usd": 0.06309}
|
||||
{"instance_id": "matplotlib__matplotlib-24334", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 451.6, "eval_duration_s": 122.1, "gen_cost_usd": 0.111089}
|
||||
{"instance_id": "matplotlib__matplotlib-24970", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 337.0, "eval_duration_s": 80.0, "gen_cost_usd": 0.088502}
|
||||
{"instance_id": "matplotlib__matplotlib-25079", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 339.2, "eval_duration_s": 87.9, "gen_cost_usd": 0.111273}
|
||||
{"instance_id": "matplotlib__matplotlib-25311", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 441.9, "eval_duration_s": 87.2, "gen_cost_usd": 0.119274}
|
||||
{"instance_id": "matplotlib__matplotlib-25332", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 268.9, "eval_duration_s": 82.3, "gen_cost_usd": 0.063431}
|
||||
{"instance_id": "matplotlib__matplotlib-25433", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 397.0, "eval_duration_s": 81.0, "gen_cost_usd": 0.142743}
|
||||
{"instance_id": "matplotlib__matplotlib-25442", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 288.0, "eval_duration_s": 82.1, "gen_cost_usd": 0.096203}
|
||||
{"instance_id": "matplotlib__matplotlib-25498", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 322.1, "eval_duration_s": 128.0, "gen_cost_usd": 0.093743}
|
||||
{"instance_id": "matplotlib__matplotlib-26011", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 591.4, "eval_duration_s": 91.2, "gen_cost_usd": 0.134968}
|
||||
{"instance_id": "matplotlib__matplotlib-26020", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 250.7, "eval_duration_s": 112.0, "gen_cost_usd": 0.065568}
|
||||
{"instance_id": "mwaskom__seaborn-2848", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 393.6, "eval_duration_s": 21.6, "gen_cost_usd": 0.109806}
|
||||
{"instance_id": "mwaskom__seaborn-3010", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 197.8, "eval_duration_s": 9.1, "gen_cost_usd": 0.041585}
|
||||
{"instance_id": "mwaskom__seaborn-3190", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 283.8, "eval_duration_s": 12.0, "gen_cost_usd": 0.090697}
|
||||
{"instance_id": "mwaskom__seaborn-3407", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 386.5, "eval_duration_s": 71.5, "gen_cost_usd": 0.087443}
|
||||
{"instance_id": "pallets__flask-4045", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 300.3, "eval_duration_s": 4.6, "gen_cost_usd": 0.104249}
|
||||
{"instance_id": "pallets__flask-4992", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 149.8, "eval_duration_s": 5.2, "gen_cost_usd": 0.040922}
|
||||
{"instance_id": "pallets__flask-5063", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 158.5, "eval_duration_s": 5.4, "gen_cost_usd": 0.058829}
|
||||
{"instance_id": "psf__requests-1963", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 288.9, "eval_duration_s": 26.4, "gen_cost_usd": 0.103678}
|
||||
{"instance_id": "psf__requests-2148", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 189.4, "eval_duration_s": 44.5, "gen_cost_usd": 0.042346}
|
||||
{"instance_id": "psf__requests-2317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "timeout", "gen_duration_s": 290.4, "eval_duration_s": 600.0, "gen_cost_usd": 0.04981}
|
||||
{"instance_id": "psf__requests-2674", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 200.2, "eval_duration_s": 37.7, "gen_cost_usd": 0.049318}
|
||||
{"instance_id": "psf__requests-3362", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 301.1, "eval_duration_s": 5.6, "gen_cost_usd": 0.12329}
|
||||
{"instance_id": "psf__requests-863", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 197.9, "eval_duration_s": 41.3, "gen_cost_usd": 0.054824}
|
||||
{"instance_id": "pydata__xarray-3364", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 523.8, "eval_duration_s": 12.1, "gen_cost_usd": 0.232568}
|
||||
{"instance_id": "pydata__xarray-4094", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 222.8, "eval_duration_s": 16.8, "gen_cost_usd": 0.067573}
|
||||
{"instance_id": "pydata__xarray-4248", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 312.9, "eval_duration_s": 10.4, "gen_cost_usd": 0.113606}
|
||||
{"instance_id": "pydata__xarray-4493", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 276.2, "eval_duration_s": 46.2, "gen_cost_usd": 0.091175}
|
||||
{"instance_id": "pydata__xarray-5131", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 157.6, "eval_duration_s": 9.9, "gen_cost_usd": 0.03347}
|
||||
{"instance_id": "pylint-dev__pylint-5859", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 103.9, "eval_duration_s": 8.4, "gen_cost_usd": 0.055425}
|
||||
{"instance_id": "pylint-dev__pylint-6506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 150.7, "eval_duration_s": 8.7, "gen_cost_usd": 0.082501}
|
||||
{"instance_id": "pylint-dev__pylint-7080", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "no_test_results", "gen_duration_s": 252.1, "eval_duration_s": 8.2, "gen_cost_usd": 0.131957}
|
||||
{"instance_id": "pylint-dev__pylint-7114", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 144.7, "eval_duration_s": 5.6, "gen_cost_usd": 0.064266}
|
||||
{"instance_id": "pylint-dev__pylint-7228", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 197.8, "eval_duration_s": 6.5, "gen_cost_usd": 0.127746}
|
||||
{"instance_id": "pylint-dev__pylint-7993", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 106.1, "eval_duration_s": 5.8, "gen_cost_usd": 0.06825}
|
||||
{"instance_id": "pytest-dev__pytest-11143", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 216.0, "eval_duration_s": 10.5, "gen_cost_usd": 0.069013}
|
||||
{"instance_id": "pytest-dev__pytest-11148", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 256.1, "eval_duration_s": 12.9, "gen_cost_usd": 0.074746}
|
||||
{"instance_id": "pytest-dev__pytest-5103", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 462.2, "eval_duration_s": 11.5, "gen_cost_usd": 0.233781}
|
||||
{"instance_id": "pytest-dev__pytest-5221", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 244.1, "eval_duration_s": 13.8, "gen_cost_usd": 0.073051}
|
||||
{"instance_id": "pytest-dev__pytest-5227", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 251.4, "eval_duration_s": 11.0, "gen_cost_usd": 0.100449}
|
||||
{"instance_id": "pytest-dev__pytest-5413", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 207.6, "eval_duration_s": 9.8, "gen_cost_usd": 0.05666}
|
||||
{"instance_id": "pytest-dev__pytest-5495", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 215.1, "eval_duration_s": 9.4, "gen_cost_usd": 0.053098}
|
||||
{"instance_id": "pytest-dev__pytest-5692", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 172.2, "eval_duration_s": 10.9, "gen_cost_usd": 0.061905}
|
||||
{"instance_id": "pytest-dev__pytest-6116", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 165.2, "eval_duration_s": 14.4, "gen_cost_usd": 0.036604}
|
||||
{"instance_id": "pytest-dev__pytest-7168", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 191.4, "eval_duration_s": 10.4, "gen_cost_usd": 0.040527}
|
||||
{"instance_id": "pytest-dev__pytest-7220", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 299.9, "eval_duration_s": 11.2, "gen_cost_usd": 0.094994}
|
||||
{"instance_id": "pytest-dev__pytest-7373", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 177.7, "eval_duration_s": 9.5, "gen_cost_usd": 0.033122}
|
||||
{"instance_id": "pytest-dev__pytest-7432", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 224.6, "eval_duration_s": 10.6, "gen_cost_usd": 0.064174}
|
||||
{"instance_id": "pytest-dev__pytest-7490", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 299.9, "eval_duration_s": 11.0, "gen_cost_usd": 0.072562}
|
||||
{"instance_id": "pytest-dev__pytest-8365", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 110.1, "eval_duration_s": 9.8, "gen_cost_usd": 0.065695}
|
||||
{"instance_id": "pytest-dev__pytest-8906", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 223.0, "eval_duration_s": 11.2, "gen_cost_usd": 0.057594}
|
||||
{"instance_id": "pytest-dev__pytest-9359", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 290.8, "eval_duration_s": 9.1, "gen_cost_usd": 0.090197}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10297", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 560.6, "eval_duration_s": 134.5, "gen_cost_usd": 0.072013}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10508", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 565.0, "eval_duration_s": 125.4, "gen_cost_usd": 0.074143}
|
||||
{"instance_id": "scikit-learn__scikit-learn-10949", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 569.8, "eval_duration_s": 135.8, "gen_cost_usd": 0.085719}
|
||||
{"instance_id": "scikit-learn__scikit-learn-11040", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 553.2, "eval_duration_s": 145.5, "gen_cost_usd": 0.08952}
|
||||
{"instance_id": "scikit-learn__scikit-learn-11281", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "scikit-learn__scikit-learn-12471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 526.0, "eval_duration_s": 144.6, "gen_cost_usd": 0.09117}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13142", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 537.3, "eval_duration_s": 167.1, "gen_cost_usd": 0.071613}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13241", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13439", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 503.7, "eval_duration_s": 171.2, "gen_cost_usd": 0.05985}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13496", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 529.9, "eval_duration_s": 164.4, "gen_cost_usd": 0.075915}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13497", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 484.9, "eval_duration_s": 174.5, "gen_cost_usd": 0.040341}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13584", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 482.0, "eval_duration_s": 171.9, "gen_cost_usd": 0.056177}
|
||||
{"instance_id": "scikit-learn__scikit-learn-13779", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 518.6, "eval_duration_s": 209.8, "gen_cost_usd": 0.066756}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14087", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14092", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 489.9, "eval_duration_s": 203.1, "gen_cost_usd": 0.088229}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14894", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 476.1, "eval_duration_s": 202.1, "gen_cost_usd": 0.080505}
|
||||
{"instance_id": "scikit-learn__scikit-learn-14983", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 445.2, "eval_duration_s": 205.2, "gen_cost_usd": 0.080515}
|
||||
{"instance_id": "scikit-learn__scikit-learn-15512", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 469.5, "eval_duration_s": 205.4, "gen_cost_usd": 0.115147}
|
||||
{"instance_id": "scikit-learn__scikit-learn-15535", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 334.2, "eval_duration_s": 202.3, "gen_cost_usd": 0.074412}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25500", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25570", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25638", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 351.4, "eval_duration_s": 13.6, "gen_cost_usd": 0.094304}
|
||||
{"instance_id": "scikit-learn__scikit-learn-25747", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 315.5, "eval_duration_s": 15.0, "gen_cost_usd": 0.114316}
|
||||
{"instance_id": "sphinx-doc__sphinx-10325", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 242.8, "eval_duration_s": 14.5, "gen_cost_usd": 0.110573}
|
||||
{"instance_id": "sphinx-doc__sphinx-10451", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 306.1, "eval_duration_s": 13.9, "gen_cost_usd": 0.14587}
|
||||
{"instance_id": "sphinx-doc__sphinx-11445", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 284.3, "eval_duration_s": 13.9, "gen_cost_usd": 0.11017}
|
||||
{"instance_id": "sphinx-doc__sphinx-7686", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 403.0, "eval_duration_s": 16.6, "gen_cost_usd": 0.187754}
|
||||
{"instance_id": "sphinx-doc__sphinx-7738", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 206.5, "eval_duration_s": 16.6, "gen_cost_usd": 0.069386}
|
||||
{"instance_id": "sphinx-doc__sphinx-7975", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 185.1, "eval_duration_s": 15.6, "gen_cost_usd": 0.128654}
|
||||
{"instance_id": "sphinx-doc__sphinx-8273", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 126.1, "eval_duration_s": 13.8, "gen_cost_usd": 0.073368}
|
||||
{"instance_id": "sphinx-doc__sphinx-8282", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 220.1, "eval_duration_s": 15.2, "gen_cost_usd": 0.146973}
|
||||
{"instance_id": "sphinx-doc__sphinx-8435", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 267.7, "eval_duration_s": 13.5, "gen_cost_usd": 0.14341}
|
||||
{"instance_id": "sphinx-doc__sphinx-8474", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 365.6, "eval_duration_s": 15.3, "gen_cost_usd": 0.189978}
|
||||
{"instance_id": "sphinx-doc__sphinx-8506", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 205.6, "eval_duration_s": 17.2, "gen_cost_usd": 0.135674}
|
||||
{"instance_id": "sphinx-doc__sphinx-8595", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 196.5, "eval_duration_s": 15.1, "gen_cost_usd": 0.090744}
|
||||
{"instance_id": "sphinx-doc__sphinx-8627", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 155.2, "eval_duration_s": 14.4, "gen_cost_usd": 0.108997}
|
||||
{"instance_id": "sphinx-doc__sphinx-8713", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 116.1, "eval_duration_s": 16.9, "gen_cost_usd": 0.081077}
|
||||
{"instance_id": "sphinx-doc__sphinx-8721", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 107.8, "eval_duration_s": 11.6, "gen_cost_usd": 0.060054}
|
||||
{"instance_id": "sphinx-doc__sphinx-8801", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "patch_failed", "gen_duration_s": 199.6, "eval_duration_s": 13.7, "gen_cost_usd": 0.134846}
|
||||
{"instance_id": "sympy__sympy-11400", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 303.8, "eval_duration_s": 25.5, "gen_cost_usd": 0.102462}
|
||||
{"instance_id": "sympy__sympy-11870", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 401.5, "eval_duration_s": 552.7, "gen_cost_usd": 0.194922}
|
||||
{"instance_id": "sympy__sympy-11897", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 501.3, "eval_duration_s": 27.7, "gen_cost_usd": 0.225207}
|
||||
{"instance_id": "sympy__sympy-12171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 230.0, "eval_duration_s": 30.6, "gen_cost_usd": 0.097165}
|
||||
{"instance_id": "sympy__sympy-12236", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 236.9, "eval_duration_s": 36.9, "gen_cost_usd": 0.105261}
|
||||
{"instance_id": "sympy__sympy-12419", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 166.6, "eval_duration_s": 25.3, "gen_cost_usd": 0.065146}
|
||||
{"instance_id": "sympy__sympy-12454", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 126.2, "eval_duration_s": 39.8, "gen_cost_usd": 0.054694}
|
||||
{"instance_id": "sympy__sympy-12481", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 157.6, "eval_duration_s": 20.2, "gen_cost_usd": 0.074173}
|
||||
{"instance_id": "sympy__sympy-13031", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 168.9, "eval_duration_s": 27.8, "gen_cost_usd": 0.103816}
|
||||
{"instance_id": "sympy__sympy-13043", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 127.2, "eval_duration_s": 25.0, "gen_cost_usd": 0.07869}
|
||||
{"instance_id": "sympy__sympy-13146", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 425.4, "eval_duration_s": 34.4, "gen_cost_usd": 0.221145}
|
||||
{"instance_id": "sympy__sympy-13177", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 78.4, "eval_duration_s": 25.0, "gen_cost_usd": 0.043302}
|
||||
{"instance_id": "sympy__sympy-13437", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 66.0, "eval_duration_s": 28.8, "gen_cost_usd": 0.055409}
|
||||
{"instance_id": "sympy__sympy-13471", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 114.2, "eval_duration_s": 20.1, "gen_cost_usd": 0.073728}
|
||||
{"instance_id": "sympy__sympy-13480", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 36.8, "eval_duration_s": 24.4, "gen_cost_usd": 0.010983}
|
||||
{"instance_id": "sympy__sympy-13647", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.5, "eval_duration_s": 25.3, "gen_cost_usd": 0.053718}
|
||||
{"instance_id": "sympy__sympy-13773", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 173.9, "eval_duration_s": 24.3, "gen_cost_usd": 0.08569}
|
||||
{"instance_id": "sympy__sympy-13895", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 518.9, "eval_duration_s": 20.8, "gen_cost_usd": 0.316322}
|
||||
{"instance_id": "sympy__sympy-13915", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 208.7, "eval_duration_s": 22.7, "gen_cost_usd": 0.107024}
|
||||
{"instance_id": "sympy__sympy-13971", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 86.7, "eval_duration_s": 33.2, "gen_cost_usd": 0.056409}
|
||||
{"instance_id": "sympy__sympy-14024", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 175.0, "eval_duration_s": 29.6, "gen_cost_usd": 0.11829}
|
||||
{"instance_id": "sympy__sympy-14308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 262.9, "eval_duration_s": 22.3, "gen_cost_usd": 0.152794}
|
||||
{"instance_id": "sympy__sympy-14317", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 220.1, "eval_duration_s": 22.6, "gen_cost_usd": 0.131946}
|
||||
{"instance_id": "sympy__sympy-14396", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 85.7, "eval_duration_s": 20.1, "gen_cost_usd": 0.055482}
|
||||
{"instance_id": "sympy__sympy-14774", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 45.6, "eval_duration_s": 22.6, "gen_cost_usd": 0.020607}
|
||||
{"instance_id": "sympy__sympy-14817", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 137.1, "eval_duration_s": 28.1, "gen_cost_usd": 0.070551}
|
||||
{"instance_id": "sympy__sympy-15011", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 315.7, "eval_duration_s": 27.2, "gen_cost_usd": 0.150251}
|
||||
{"instance_id": "sympy__sympy-15308", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 155.2, "eval_duration_s": 25.8, "gen_cost_usd": 0.074156}
|
||||
{"instance_id": "sympy__sympy-15345", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 177.3, "eval_duration_s": 22.8, "gen_cost_usd": 0.109022}
|
||||
{"instance_id": "sympy__sympy-15346", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 76.0, "eval_duration_s": 46.6, "gen_cost_usd": 0.037987}
|
||||
{"instance_id": "sympy__sympy-15609", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.4, "eval_duration_s": 24.7, "gen_cost_usd": 0.048943}
|
||||
{"instance_id": "sympy__sympy-15678", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 163.1, "eval_duration_s": 29.8, "gen_cost_usd": 0.095395}
|
||||
{"instance_id": "sympy__sympy-16106", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 194.4, "eval_duration_s": 31.0, "gen_cost_usd": 0.113395}
|
||||
{"instance_id": "sympy__sympy-16281", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 111.2, "eval_duration_s": 26.4, "gen_cost_usd": 0.059208}
|
||||
{"instance_id": "sympy__sympy-16503", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 543.7, "eval_duration_s": 25.5, "gen_cost_usd": 0.273528}
|
||||
{"instance_id": "sympy__sympy-16792", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 188.8, "eval_duration_s": 25.3, "gen_cost_usd": 0.108599}
|
||||
{"instance_id": "sympy__sympy-16988", "has_patch": false, "resolved": false, "gen_status": "timeout", "eval_status": "empty_patch", "gen_duration_s": 600.0, "eval_duration_s": 0, "gen_cost_usd": null}
|
||||
{"instance_id": "sympy__sympy-17022", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 202.6, "eval_duration_s": 21.8, "gen_cost_usd": 0.117767}
|
||||
{"instance_id": "sympy__sympy-17139", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 81.3, "eval_duration_s": 98.4, "gen_cost_usd": 0.032179}
|
||||
{"instance_id": "sympy__sympy-17630", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 171.3, "eval_duration_s": 21.6, "gen_cost_usd": 0.09608}
|
||||
{"instance_id": "sympy__sympy-17655", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 129.4, "eval_duration_s": 28.7, "gen_cost_usd": 0.075998}
|
||||
{"instance_id": "sympy__sympy-18057", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 138.7, "eval_duration_s": 106.1, "gen_cost_usd": 0.081103}
|
||||
{"instance_id": "sympy__sympy-18087", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 262.5, "eval_duration_s": 31.4, "gen_cost_usd": 0.127353}
|
||||
{"instance_id": "sympy__sympy-18189", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 99.0, "eval_duration_s": 39.0, "gen_cost_usd": 0.051519}
|
||||
{"instance_id": "sympy__sympy-18199", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 193.1, "eval_duration_s": 80.8, "gen_cost_usd": 0.101953}
|
||||
{"instance_id": "sympy__sympy-18532", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 179.0, "eval_duration_s": 25.9, "gen_cost_usd": 0.082584}
|
||||
{"instance_id": "sympy__sympy-18621", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 73.4, "eval_duration_s": 20.9, "gen_cost_usd": 0.043878}
|
||||
{"instance_id": "sympy__sympy-18698", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 303.9, "eval_duration_s": 30.0, "gen_cost_usd": 0.149565}
|
||||
{"instance_id": "sympy__sympy-18835", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 128.1, "eval_duration_s": 22.1, "gen_cost_usd": 0.060965}
|
||||
{"instance_id": "sympy__sympy-19007", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 322.6, "eval_duration_s": 23.7, "gen_cost_usd": 0.177031}
|
||||
{"instance_id": "sympy__sympy-19254", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 281.6, "eval_duration_s": 21.8, "gen_cost_usd": 0.168758}
|
||||
{"instance_id": "sympy__sympy-19487", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 128.3, "eval_duration_s": 28.1, "gen_cost_usd": 0.088216}
|
||||
{"instance_id": "sympy__sympy-20049", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 256.1, "eval_duration_s": 21.9, "gen_cost_usd": 0.160782}
|
||||
{"instance_id": "sympy__sympy-20154", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 77.9, "eval_duration_s": 22.6, "gen_cost_usd": 0.047626}
|
||||
{"instance_id": "sympy__sympy-20212", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 130.9, "eval_duration_s": 41.6, "gen_cost_usd": 0.082753}
|
||||
{"instance_id": "sympy__sympy-20322", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 302.6, "eval_duration_s": 34.7, "gen_cost_usd": 0.054426}
|
||||
{"instance_id": "sympy__sympy-20442", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 152.3, "eval_duration_s": 22.3, "gen_cost_usd": 0.085405}
|
||||
{"instance_id": "sympy__sympy-20590", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 135.6, "eval_duration_s": 20.2, "gen_cost_usd": 0.062288}
|
||||
{"instance_id": "sympy__sympy-20639", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 173.5, "eval_duration_s": 23.6, "gen_cost_usd": 0.08207}
|
||||
{"instance_id": "sympy__sympy-21055", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 165.5, "eval_duration_s": 24.8, "gen_cost_usd": 0.088091}
|
||||
{"instance_id": "sympy__sympy-21171", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 115.1, "eval_duration_s": 23.3, "gen_cost_usd": 0.053466}
|
||||
{"instance_id": "sympy__sympy-21379", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 85.8, "eval_duration_s": 32.8, "gen_cost_usd": 0.04783}
|
||||
{"instance_id": "sympy__sympy-21612", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 84.2, "eval_duration_s": 20.7, "gen_cost_usd": 0.038638}
|
||||
{"instance_id": "sympy__sympy-21614", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 113.5, "eval_duration_s": 22.0, "gen_cost_usd": 0.05773}
|
||||
{"instance_id": "sympy__sympy-21627", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 109.3, "eval_duration_s": 30.4, "gen_cost_usd": 0.071286}
|
||||
{"instance_id": "sympy__sympy-21847", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 105.2, "eval_duration_s": 20.0, "gen_cost_usd": 0.071166}
|
||||
{"instance_id": "sympy__sympy-22005", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 99.5, "eval_duration_s": 25.4, "gen_cost_usd": 0.050959}
|
||||
{"instance_id": "sympy__sympy-22714", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 180.1, "eval_duration_s": 25.2, "gen_cost_usd": 0.041119}
|
||||
{"instance_id": "sympy__sympy-22840", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 265.2, "eval_duration_s": 34.8, "gen_cost_usd": 0.131769}
|
||||
{"instance_id": "sympy__sympy-23117", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 115.6, "eval_duration_s": 23.0, "gen_cost_usd": 0.050324}
|
||||
{"instance_id": "sympy__sympy-23191", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 185.1, "eval_duration_s": 23.1, "gen_cost_usd": 0.106652}
|
||||
{"instance_id": "sympy__sympy-23262", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 147.2, "eval_duration_s": 26.8, "gen_cost_usd": 0.10144}
|
||||
{"instance_id": "sympy__sympy-24066", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 218.3, "eval_duration_s": 23.4, "gen_cost_usd": 0.111426}
|
||||
{"instance_id": "sympy__sympy-24102", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_PARTIAL", "gen_duration_s": 133.3, "eval_duration_s": 22.3, "gen_cost_usd": 0.079187}
|
||||
{"instance_id": "sympy__sympy-24152", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 98.5, "eval_duration_s": 27.7, "gen_cost_usd": 0.058922}
|
||||
{"instance_id": "sympy__sympy-24213", "has_patch": true, "resolved": true, "gen_status": "completed", "eval_status": "RESOLVED_FULL", "gen_duration_s": 92.4, "eval_duration_s": 23.4, "gen_cost_usd": 0.042842}
|
||||
{"instance_id": "sympy__sympy-24909", "has_patch": true, "resolved": false, "gen_status": "completed", "eval_status": "RESOLVED_NO", "gen_duration_s": 291.3, "eval_duration_s": 22.3, "gen_cost_usd": 0.097123}
|
||||
94
evals/swe-bench/scoreboard/haiku-baseline-20260316/meta.json
Normal file
94
evals/swe-bench/scoreboard/haiku-baseline-20260316/meta.json
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
{
|
||||
"run_name": "haiku-baseline-20260316",
|
||||
"timestamp": "2026-03-16T13:53:04.401784+00:00",
|
||||
"model": "claude-haiku-4-5",
|
||||
"provider": "anthropic",
|
||||
"fabro_version": "fabro 0.5.0 (4c6ad4d 2026-03-16)",
|
||||
"description": "Haiku 4.5 baseline, default prompt, 2 CPU / 4 GB, 10min timeout",
|
||||
"notes": "",
|
||||
"total_instances": 300,
|
||||
"patched": 293,
|
||||
"patch_pct": 97.7,
|
||||
"resolved": 162,
|
||||
"resolve_pct": 54.0,
|
||||
"total_gen_cost_usd": 26.13,
|
||||
"avg_gen_cost_usd": 0.0871,
|
||||
"gen_wall_time_s": 3559.2,
|
||||
"eval_wall_time_s": 625.2,
|
||||
"avg_gen_duration_s": 224.3,
|
||||
"avg_eval_duration_s": 41.5,
|
||||
"per_repo": {
|
||||
"astropy/astropy": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 1,
|
||||
"resolve_pct": 16.7
|
||||
},
|
||||
"django/django": {
|
||||
"total": 114,
|
||||
"patched": 113,
|
||||
"resolved": 86,
|
||||
"resolve_pct": 75.4
|
||||
},
|
||||
"matplotlib/matplotlib": {
|
||||
"total": 23,
|
||||
"patched": 23,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"mwaskom/seaborn": {
|
||||
"total": 4,
|
||||
"patched": 4,
|
||||
"resolved": 1,
|
||||
"resolve_pct": 25.0
|
||||
},
|
||||
"pallets/flask": {
|
||||
"total": 3,
|
||||
"patched": 3,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"psf/requests": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 4,
|
||||
"resolve_pct": 66.7
|
||||
},
|
||||
"pydata/xarray": {
|
||||
"total": 5,
|
||||
"patched": 5,
|
||||
"resolved": 1,
|
||||
"resolve_pct": 20.0
|
||||
},
|
||||
"pylint-dev/pylint": {
|
||||
"total": 6,
|
||||
"patched": 6,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"pytest-dev/pytest": {
|
||||
"total": 17,
|
||||
"patched": 17,
|
||||
"resolved": 11,
|
||||
"resolve_pct": 64.7
|
||||
},
|
||||
"scikit-learn/scikit-learn": {
|
||||
"total": 23,
|
||||
"patched": 18,
|
||||
"resolved": 12,
|
||||
"resolve_pct": 52.2
|
||||
},
|
||||
"sphinx-doc/sphinx": {
|
||||
"total": 16,
|
||||
"patched": 16,
|
||||
"resolved": 0,
|
||||
"resolve_pct": 0.0
|
||||
},
|
||||
"sympy/sympy": {
|
||||
"total": 77,
|
||||
"patched": 76,
|
||||
"resolved": 46,
|
||||
"resolve_pct": 59.7
|
||||
}
|
||||
}
|
||||
}
|
||||
15
evals/swe-bench/scoreboard/leaderboard.json
Normal file
15
evals/swe-bench/scoreboard/leaderboard.json
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[
|
||||
{
|
||||
"run_name": "haiku-baseline-20260316",
|
||||
"date": "2026-03-16",
|
||||
"model": "claude-haiku-4-5",
|
||||
"provider": "anthropic",
|
||||
"resolved": 162,
|
||||
"total": 300,
|
||||
"resolve_pct": 54.0,
|
||||
"patched": 293,
|
||||
"patch_pct": 97.7,
|
||||
"total_cost_usd": 26.13,
|
||||
"avg_cost_usd": 0.0871
|
||||
}
|
||||
]
|
||||
Loading…
Add table
Reference in a new issue