mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-08-28 05:27:41 +00:00
5fc9157017
## Summary Removes Fabro's automatic retro generation stage so workflow runs go directly from execution to finalization and optional PR creation. This drops the retro-specific crate, events, projection fields, config/API knobs, and user-facing docs in favor of the existing durable run observability surfaces. ## What Changed - Deleted the `fabro-retro` crate and the workflow `retro` pipeline phase, with finalization now consuming `Executed` state directly. - Removed retro configuration and API surface area, including `--no-retro`, `[run.execution].retros`, manifest `no_retro`, `features.retros`, and run projection `retro*` fields. - Retired typed `retro.*` events while keeping historical event logs readable by deserializing retired retro event names as `Unknown`. - Stopped appending retro sections to generated PR bodies and updated docs, marketing copy, screenshots, and navigation to point users toward observability/event-stream inspection. ## Testing Not run during PR creation; this branch already contained the implementation commit. --- [](https://github.com/EveryInc/compound-engineering-plugin) 🤖 Generated with GPT-5 (unknown context, reasoning unspecified) via [Codex](https://openai.com/codex)
683 lines
22 KiB
Python
683 lines
22 KiB
Python
#!/usr/bin/env python3
|
|
"""Evaluate SWE-bench predictions using Daytona sandboxes.
|
|
|
|
Reuses the same Daytona snapshots from the generation phase. For each
|
|
prediction, creates a sandbox, applies the model patch + test patch,
|
|
runs the test suite, and grades the result using swebench's log parsers.
|
|
|
|
Usage:
|
|
cd evals/swe-bench
|
|
python evaluate_daytona.py \
|
|
--predictions results/haiku-baseline/predictions.jsonl \
|
|
--output-dir results/haiku-baseline/eval \
|
|
2>&1 | tee results/haiku-baseline/eval/console.log
|
|
"""
|
|
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import logging
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
from collections import Counter
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
from pathlib import Path
|
|
|
|
from datasets import load_dataset
|
|
from swebench.harness.constants import MAP_REPO_VERSION_TO_SPECS
|
|
from swebench.harness.grading import (
|
|
get_eval_tests_report,
|
|
get_resolution_status,
|
|
get_logs_eval,
|
|
)
|
|
from swebench.harness.test_spec.test_spec import make_test_spec
|
|
|
|
from gen_dockerfile import generate_dockerfile, repo_version_key
|
|
|
|
EVAL_DIR = Path(__file__).parent.resolve()
|
|
|
|
|
|
def load_completed_ids(output_dir: Path) -> set[str]:
|
|
"""Load instance IDs that have already been evaluated from prior runs."""
|
|
completed = set()
|
|
results_file = output_dir / "eval_results.jsonl"
|
|
if results_file.exists():
|
|
with open(results_file) as f:
|
|
for line in f:
|
|
if line.strip():
|
|
try:
|
|
completed.add(json.loads(line)["instance_id"])
|
|
except (json.JSONDecodeError, KeyError):
|
|
pass
|
|
return completed
|
|
|
|
log = logging.getLogger("swe-eval-grade")
|
|
|
|
HEREDOC_DELIMITER = "EOF_114329324912"
|
|
START_TEST_OUTPUT = ">>>>> Start Test Output"
|
|
END_TEST_OUTPUT = ">>>>> End Test Output"
|
|
APPLY_PATCH_FAIL = ">>>>> Patch Apply Failed"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Logging
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def setup_logging(output_dir: Path):
|
|
log.setLevel(logging.DEBUG)
|
|
fmt = logging.Formatter(
|
|
"%(asctime)s %(levelname)-7s %(message)s", datefmt="%H:%M:%S"
|
|
)
|
|
fh = logging.FileHandler(output_dir / "eval_grade.log")
|
|
fh.setLevel(logging.DEBUG)
|
|
fh.setFormatter(fmt)
|
|
log.addHandler(fh)
|
|
|
|
ch = logging.StreamHandler(sys.stderr)
|
|
ch.setLevel(logging.INFO)
|
|
ch.setFormatter(fmt)
|
|
log.addHandler(ch)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Build eval script
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def get_test_directives(instance: dict) -> list[str]:
|
|
"""Extract test file directives from test_patch."""
|
|
diff_pat = r"diff --git a/.* b/(.*)"
|
|
directives = re.findall(diff_pat, instance["test_patch"])
|
|
non_test_exts = [".txt", ".md", ".rst", ".csv", ".json", ".xml", ".yml", ".yaml"]
|
|
directives = [
|
|
d for d in directives if not any(d.endswith(ext) for ext in non_test_exts)
|
|
]
|
|
if instance["repo"] == "django/django":
|
|
transformed = []
|
|
for d in directives:
|
|
d = d[: -len(".py")] if d.endswith(".py") else d
|
|
d = d[len("tests/"):] if d.startswith("tests/") else d
|
|
d = d.replace("/", ".")
|
|
transformed.append(d)
|
|
directives = transformed
|
|
return directives
|
|
|
|
|
|
def get_modified_files(patch: str) -> list[str]:
|
|
"""Extract modified file paths from a unified diff."""
|
|
return re.findall(r"diff --git a/.* b/(.*)", patch)
|
|
|
|
|
|
def build_eval_script(instance: dict, model_patch: str) -> str:
|
|
"""Build a shell script that applies patches and runs tests.
|
|
|
|
Returns a bash script string that:
|
|
1. Clones the repo and checks out the base commit
|
|
2. Installs the package
|
|
3. Applies the model patch
|
|
4. Resets test files, applies the test patch
|
|
5. Runs the test command with output markers
|
|
"""
|
|
repo = instance["repo"]
|
|
version = instance["version"]
|
|
base_commit = instance["base_commit"]
|
|
test_patch = instance["test_patch"]
|
|
spec = MAP_REPO_VERSION_TO_SPECS.get(repo, {}).get(version, {})
|
|
|
|
install_cmd = spec.get("install", "pip install -e .")
|
|
test_cmd_base = spec.get("test_cmd", "pytest -rA")
|
|
if isinstance(test_cmd_base, list):
|
|
test_cmd_base = test_cmd_base[-1]
|
|
test_directives = get_test_directives(instance)
|
|
test_cmd = " ".join([test_cmd_base] + test_directives)
|
|
|
|
test_files = get_modified_files(test_patch)
|
|
reset_tests = f"git checkout {base_commit} {' '.join(test_files)}"
|
|
apply_test_patch = (
|
|
f"git apply -v - <<'{HEREDOC_DELIMITER}'\n{test_patch}\n{HEREDOC_DELIMITER}"
|
|
)
|
|
|
|
pre_install = spec.get("pre_install", [])
|
|
if isinstance(pre_install, str):
|
|
pre_install = [pre_install]
|
|
|
|
eval_commands = spec.get("eval_commands", [])
|
|
if isinstance(eval_commands, str):
|
|
eval_commands = [eval_commands]
|
|
|
|
lines = [
|
|
"#!/bin/bash",
|
|
"set -e",
|
|
"",
|
|
"# Clone and setup",
|
|
f"git clone https://github.com/{repo}.git .",
|
|
f"git checkout {base_commit}",
|
|
]
|
|
|
|
for cmd in pre_install:
|
|
lines.append(cmd)
|
|
|
|
lines.append(install_cmd)
|
|
|
|
# Eval environment setup (locale, etc.)
|
|
for cmd in eval_commands:
|
|
lines.append(cmd)
|
|
|
|
lines.extend([
|
|
"",
|
|
f"git config --global --add safe.directory /home/daytona/workspace",
|
|
"",
|
|
"# Apply model patch (non-fatal — record failure in output)",
|
|
f"if ! git apply -v - <<'{HEREDOC_DELIMITER}'",
|
|
model_patch,
|
|
HEREDOC_DELIMITER,
|
|
"then",
|
|
f" echo '{APPLY_PATCH_FAIL}'",
|
|
" exit 1",
|
|
"fi",
|
|
"",
|
|
"# Re-install after patching (some repos need this)",
|
|
install_cmd,
|
|
"",
|
|
"# Stop aborting on error — test failures are expected",
|
|
"set +e",
|
|
"",
|
|
"# Reset test files and apply test patch",
|
|
reset_tests,
|
|
apply_test_patch,
|
|
"",
|
|
"# Run tests",
|
|
f"echo '{START_TEST_OUTPUT}'",
|
|
test_cmd,
|
|
f"echo '{END_TEST_OUTPUT}'",
|
|
"",
|
|
"# Clean up test files",
|
|
reset_tests,
|
|
])
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def toml_literal_string(text: str) -> str:
|
|
return f"'''\n{text}'''"
|
|
|
|
|
|
def generate_eval_toml(instance: dict, config_dir: Path) -> str:
|
|
"""Generate a workflow.toml for running the eval script."""
|
|
repo = instance["repo"]
|
|
version = instance["version"]
|
|
snapshot_name = repo_version_key(repo, version)
|
|
dockerfile = generate_dockerfile(repo, version)
|
|
|
|
lines = [
|
|
'version = 1',
|
|
f'graph = "{config_dir / "eval.fabro"}"',
|
|
'',
|
|
'[pull_request]',
|
|
'enabled = false',
|
|
'',
|
|
'[sandbox]',
|
|
'provider = "daytona"',
|
|
'',
|
|
'[sandbox.env]',
|
|
'PATH = "/opt/miniconda3/envs/testbed/bin:/opt/miniconda3/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"',
|
|
'',
|
|
'[sandbox.daytona.snapshot]',
|
|
f'name = "{snapshot_name}"',
|
|
'cpu = 2',
|
|
'memory = 4',
|
|
'disk = 10',
|
|
f'dockerfile = {toml_literal_string(dockerfile)}',
|
|
]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def dot_escape(s: str) -> str:
|
|
return s.replace("\\", "\\\\").replace('"', '\\"').replace("\n", "\\n")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Grade from test output
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def grade_test_output(instance: dict, test_output: str) -> dict:
|
|
"""Grade test output using swebench's log parsers.
|
|
|
|
Returns a dict with 'resolved' (bool), 'status' string, and details.
|
|
"""
|
|
spec = make_test_spec(instance)
|
|
|
|
# Write test output to a temp file for get_logs_eval
|
|
with tempfile.NamedTemporaryFile(mode="w", suffix=".log", delete=False) as f:
|
|
f.write(test_output)
|
|
f.flush()
|
|
log_path = f.name
|
|
|
|
try:
|
|
eval_status_map, patch_applied = get_logs_eval(spec, log_path)
|
|
finally:
|
|
Path(log_path).unlink(missing_ok=True)
|
|
|
|
if not patch_applied:
|
|
return {
|
|
"resolved": False,
|
|
"status": "patch_failed",
|
|
"detail": "Patch did not apply or tests errored",
|
|
}
|
|
|
|
if not eval_status_map:
|
|
return {
|
|
"resolved": False,
|
|
"status": "no_test_results",
|
|
"detail": "Could not parse test results from output",
|
|
}
|
|
|
|
# Build gold results in the format expected by get_eval_tests_report
|
|
gold = {
|
|
"FAIL_TO_PASS": spec.FAIL_TO_PASS,
|
|
"PASS_TO_PASS": spec.PASS_TO_PASS,
|
|
}
|
|
|
|
report = get_eval_tests_report(eval_status_map, gold)
|
|
resolution = get_resolution_status(report)
|
|
|
|
return {
|
|
"resolved": resolution == "RESOLVED_FULL",
|
|
"status": resolution,
|
|
"f2p_total": len(spec.FAIL_TO_PASS),
|
|
"p2p_total": len(spec.PASS_TO_PASS),
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-instance evaluator
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def evaluate_instance(
|
|
instance: dict,
|
|
model_patch: str,
|
|
output_dir: Path,
|
|
timeout: int,
|
|
) -> dict:
|
|
"""Evaluate a single instance by running tests in a Daytona sandbox."""
|
|
instance_id = instance["instance_id"]
|
|
config_dir = output_dir / "configs" / instance_id
|
|
config_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
result = {
|
|
"instance_id": instance_id,
|
|
"resolved": False,
|
|
"status": "error",
|
|
"error": None,
|
|
"duration_s": 0,
|
|
}
|
|
|
|
if not model_patch.strip():
|
|
result["status"] = "empty_patch"
|
|
result["error"] = "No patch to evaluate"
|
|
return result
|
|
|
|
start_time = time.time()
|
|
|
|
try:
|
|
# Build eval script and encode for transport into sandbox
|
|
eval_script = build_eval_script(instance, model_patch)
|
|
(config_dir / "eval.sh").write_text(eval_script)
|
|
b64 = base64.b64encode(eval_script.encode()).decode()
|
|
|
|
# The script attr runs in the sandbox — decode and execute
|
|
run_cmd = f"echo {b64} | base64 -d | bash"
|
|
fabro_content = f'''digraph Eval {{
|
|
rankdir=LR
|
|
start [shape=Mdiamond]
|
|
exit [shape=Msquare]
|
|
run_tests [label="Run Tests", shape=parallelogram, script="{dot_escape(run_cmd)}"]
|
|
start -> run_tests -> exit
|
|
}}
|
|
'''
|
|
(config_dir / "eval.fabro").write_text(fabro_content)
|
|
|
|
toml_content = generate_eval_toml(instance, config_dir)
|
|
toml_file = config_dir / "eval.toml"
|
|
toml_file.write_text(toml_content)
|
|
|
|
cmd = [
|
|
"fabro", "run", str(toml_file),
|
|
"--auto-approve",
|
|
"--label", f"swe-eval={instance_id}",
|
|
]
|
|
|
|
log.debug(f"[{instance_id}] Starting eval")
|
|
proc = subprocess.run(
|
|
cmd,
|
|
cwd="/tmp",
|
|
timeout=timeout,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
# Find fabro run dir from stderr
|
|
fabro_run_dir = None
|
|
for line in proc.stderr.splitlines():
|
|
stripped = line.strip()
|
|
if stripped.startswith("Run:") and "/" in stripped:
|
|
fabro_run_dir = Path(
|
|
stripped.split("Run:", 1)[1].strip().replace("~", str(Path.home()))
|
|
)
|
|
break
|
|
|
|
if proc.returncode != 0:
|
|
(config_dir / "fabro_stderr.log").write_text(proc.stderr)
|
|
log.debug(f"[{instance_id}] fabro exit={proc.returncode}")
|
|
|
|
# Always try to read test output — tests may exit non-zero
|
|
# but stdout.log is still written by fabro
|
|
test_output = ""
|
|
if fabro_run_dir:
|
|
nodes_dir = fabro_run_dir / "nodes"
|
|
if nodes_dir.exists():
|
|
for node_dir in nodes_dir.iterdir():
|
|
if node_dir.name.startswith("run_tests"):
|
|
stdout_log = node_dir / "stdout.log"
|
|
if stdout_log.exists():
|
|
test_output = stdout_log.read_text()
|
|
|
|
if not test_output:
|
|
result["status"] = "no_output"
|
|
result["error"] = f"No test output captured (fabro exit={proc.returncode})"
|
|
else:
|
|
(config_dir / "test_output.log").write_text(test_output)
|
|
try:
|
|
grade = grade_test_output(instance, test_output)
|
|
result["resolved"] = grade["resolved"]
|
|
result["status"] = grade["status"]
|
|
if "detail" in grade:
|
|
result["error"] = grade["detail"]
|
|
except Exception as e:
|
|
result["status"] = "grade_error"
|
|
result["error"] = str(e)
|
|
log.debug(f"[{instance_id}] Grading error: {e}", exc_info=True)
|
|
|
|
except subprocess.TimeoutExpired:
|
|
result["status"] = "timeout"
|
|
result["error"] = f"Timed out after {timeout}s"
|
|
_cleanup_sandbox(instance_id, "swe-eval")
|
|
except Exception as e:
|
|
result["error"] = str(e)
|
|
log.debug(f"[{instance_id}] Exception: {e}")
|
|
|
|
result["duration_s"] = round(time.time() - start_time, 1)
|
|
return result
|
|
|
|
|
|
def _cleanup_sandbox(label_value: str, label_key: str):
|
|
"""Best-effort delete of orphaned Daytona sandbox after timeout."""
|
|
try:
|
|
ps = subprocess.run(
|
|
["fabro", "ps", "--label", f"{label_key}={label_value}", "--json"],
|
|
capture_output=True, text=True, timeout=10,
|
|
)
|
|
runs = json.loads(ps.stdout) if ps.stdout.strip() else []
|
|
for run in runs:
|
|
run_id = run.get("run_id", "")
|
|
if not run_id:
|
|
continue
|
|
sandbox_name = f"fabro-{run_id}"
|
|
subprocess.run(
|
|
["daytona", "sandbox", "delete", sandbox_name],
|
|
capture_output=True, timeout=15,
|
|
)
|
|
log.debug(f"[{label_value}] Deleted sandbox {sandbox_name}")
|
|
except Exception as e:
|
|
log.debug(f"[{label_value}] Sandbox cleanup failed (non-fatal): {e}")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Preflight
|
|
# ---------------------------------------------------------------------------
|
|
|
|
DAYTONA_CPU_LIMIT = 500
|
|
|
|
|
|
def preflight_daytona(max_workers: int, sandbox_cpu: int):
|
|
"""Check that we have enough Daytona CPU headroom before starting."""
|
|
needed = max_workers * sandbox_cpu
|
|
buffer = 1.2
|
|
|
|
used_cpus = 0
|
|
try:
|
|
result = subprocess.run(
|
|
["daytona", "sandbox", "list"],
|
|
capture_output=True, text=True, timeout=10,
|
|
)
|
|
sandbox_count = len(re.findall(
|
|
r'[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}',
|
|
result.stdout,
|
|
))
|
|
used_cpus = sandbox_count * sandbox_cpu
|
|
except Exception:
|
|
pass
|
|
|
|
available = DAYTONA_CPU_LIMIT - used_cpus
|
|
required = int(needed * buffer)
|
|
|
|
if required > available:
|
|
print(f"Preflight FAILED: need {required} CPUs "
|
|
f"({max_workers} workers x {sandbox_cpu} CPU x {buffer} buffer) "
|
|
f"but only {available} available "
|
|
f"({DAYTONA_CPU_LIMIT} limit - {used_cpus} in use)")
|
|
print(f" Reduce --max-workers to {int(available / buffer / sandbox_cpu)} or fewer")
|
|
sys.exit(1)
|
|
|
|
print(f"Preflight OK: {required} CPUs needed, {available} available "
|
|
f"({used_cpus} in use, {DAYTONA_CPU_LIMIT} limit)")
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Evaluate SWE-bench predictions using Daytona sandboxes"
|
|
)
|
|
parser.add_argument(
|
|
"--predictions", type=Path, required=True,
|
|
help="Path to predictions JSONL file",
|
|
)
|
|
parser.add_argument(
|
|
"--output-dir", type=Path, required=True,
|
|
help="Output directory for eval results",
|
|
)
|
|
parser.add_argument(
|
|
"--max-workers", type=int, default=100,
|
|
help="Max concurrent eval sandboxes (default 100)",
|
|
)
|
|
parser.add_argument(
|
|
"--timeout", type=int, default=1200,
|
|
help="Timeout per instance in seconds",
|
|
)
|
|
parser.add_argument(
|
|
"--instance-ids", nargs="+",
|
|
help="Evaluate only these instance IDs",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
args.output_dir = args.output_dir.resolve()
|
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
|
setup_logging(args.output_dir)
|
|
|
|
# --- Preflight: check Daytona capacity --------------------------------
|
|
preflight_daytona(args.max_workers, sandbox_cpu=4)
|
|
|
|
log.info("=" * 64)
|
|
log.info("SWE-bench Evaluation (Daytona)")
|
|
log.info("=" * 64)
|
|
log.info(f" Predictions: {args.predictions}")
|
|
log.info(f" Output: {args.output_dir}")
|
|
log.info(f" Workers: {args.max_workers}")
|
|
log.info(f" Timeout: {args.timeout}s")
|
|
log.info("")
|
|
|
|
# Load predictions
|
|
predictions = {}
|
|
with open(args.predictions) as f:
|
|
for line in f:
|
|
p = json.loads(line)
|
|
predictions[p["instance_id"]] = p["model_patch"]
|
|
log.info(f" {len(predictions)} predictions loaded")
|
|
|
|
# Filter by instance IDs if specified
|
|
if args.instance_ids:
|
|
id_set = set(args.instance_ids)
|
|
predictions = {k: v for k, v in predictions.items() if k in id_set}
|
|
log.info(f" Filtered to {len(predictions)} instances")
|
|
|
|
# Resume: skip already-evaluated instances
|
|
completed_ids = load_completed_ids(args.output_dir)
|
|
if completed_ids:
|
|
before = len(predictions)
|
|
predictions = {k: v for k, v in predictions.items() if k not in completed_ids}
|
|
log.info(f" {len(completed_ids)} already evaluated, {len(predictions)} remaining")
|
|
|
|
# Load dataset instances
|
|
log.info("Loading SWE-bench Lite dataset...")
|
|
dataset = load_dataset("princeton-nlp/SWE-bench_Lite", split="test")
|
|
instances_by_id = {dict(row)["instance_id"]: dict(row) for row in dataset}
|
|
|
|
# Match predictions to instances
|
|
eval_items = []
|
|
for iid, patch in predictions.items():
|
|
if iid in instances_by_id:
|
|
eval_items.append((instances_by_id[iid], patch))
|
|
else:
|
|
log.warning(f" Instance {iid} not found in dataset, skipping")
|
|
|
|
total = len(eval_items)
|
|
log.info(f" {total} instances to evaluate")
|
|
log.info("")
|
|
|
|
# Run evaluations
|
|
lock = threading.Lock()
|
|
counters: Counter[str] = Counter()
|
|
resolved_count = 0
|
|
done_count = 0
|
|
wall_start = time.time()
|
|
|
|
results_file = args.output_dir / "eval_results.jsonl"
|
|
|
|
log.info(f"Evaluating {total} instances (max {args.max_workers} concurrent)...")
|
|
log.info("-" * 64)
|
|
|
|
with ThreadPoolExecutor(max_workers=args.max_workers) as executor:
|
|
futures = {
|
|
executor.submit(
|
|
evaluate_instance, inst, patch, args.output_dir, args.timeout,
|
|
): inst["instance_id"]
|
|
for inst, patch in eval_items
|
|
}
|
|
|
|
with open(results_file, "a") as rf:
|
|
for future in as_completed(futures):
|
|
result = future.result()
|
|
iid = result["instance_id"]
|
|
status = result["status"]
|
|
dur = result["duration_s"]
|
|
resolved = result["resolved"]
|
|
|
|
with lock:
|
|
counters[status] += 1
|
|
if resolved:
|
|
resolved_count += 1
|
|
done_count += 1
|
|
n = done_count
|
|
|
|
rf.write(json.dumps(result) + "\n")
|
|
rf.flush()
|
|
|
|
resolved_mark = "RESOLVED" if resolved else status
|
|
err_info = f" err={result['error'][:80]}" if result.get("error") else ""
|
|
elapsed = round(time.time() - wall_start)
|
|
log.info(
|
|
f"[{n:3d}/{total}] {resolved_mark:<16s} {dur:6.0f}s "
|
|
f"{iid}{err_info}"
|
|
)
|
|
|
|
if n % 10 == 0 or n == total:
|
|
pct = 100 * resolved_count / n if n > 0 else 0
|
|
log.info(
|
|
f" --- progress: {n}/{total} "
|
|
f"resolved={resolved_count} ({pct:.1f}%) "
|
|
f"elapsed={elapsed}s ---"
|
|
)
|
|
|
|
wall_duration = round(time.time() - wall_start, 1)
|
|
|
|
# Recompute summary from the full results file (includes prior runs)
|
|
all_counters: Counter[str] = Counter()
|
|
all_resolved = 0
|
|
all_total = 0
|
|
repo_total: Counter[str] = Counter()
|
|
repo_resolved: Counter[str] = Counter()
|
|
with open(results_file) as f:
|
|
for line in f:
|
|
if not line.strip():
|
|
continue
|
|
r = json.loads(line)
|
|
all_counters[r["status"]] += 1
|
|
all_total += 1
|
|
if r["resolved"]:
|
|
all_resolved += 1
|
|
parts = r["instance_id"].split("__")
|
|
repo = f"{parts[0]}/{parts[1].rsplit('-', 1)[0]}" if len(parts) >= 2 else r["instance_id"]
|
|
repo_total[repo] += 1
|
|
if r["resolved"]:
|
|
repo_resolved[repo] += 1
|
|
|
|
pct = 100 * all_resolved / all_total if all_total > 0 else 0
|
|
|
|
summary = {
|
|
"total": all_total,
|
|
"resolved": all_resolved,
|
|
"resolved_pct": round(pct, 1),
|
|
"status_counts": dict(all_counters),
|
|
"wall_duration_s": wall_duration,
|
|
}
|
|
(args.output_dir / "summary.json").write_text(json.dumps(summary, indent=2))
|
|
|
|
skipped = len(completed_ids)
|
|
log.info("")
|
|
log.info("=" * 64)
|
|
log.info("FINAL RESULTS")
|
|
log.info("=" * 64)
|
|
if skipped:
|
|
log.info(f" Skipped: {skipped} (already evaluated)")
|
|
log.info(f" This run: {total}")
|
|
log.info(f" Total: {all_total}")
|
|
log.info(f" Resolved: {all_resolved} ({pct:.1f}%)")
|
|
log.info(f" Wall time: {wall_duration}s")
|
|
log.info("")
|
|
log.info(f" {'Repo':<35s} {'Resolved':>8s} {'Total':>6s} {'Rate':>6s}")
|
|
log.info(f" {'-'*35} {'-'*8} {'-'*6} {'-'*6}")
|
|
for repo in sorted(repo_total):
|
|
res = repo_resolved[repo]
|
|
tot = repo_total[repo]
|
|
rate = 100 * res / tot if tot > 0 else 0
|
|
log.info(f" {repo:<35s} {res:>8d} {tot:>6d} {rate:>5.1f}%")
|
|
log.info("")
|
|
log.info(f" Status breakdown: {dict(all_counters)}")
|
|
log.info(f" Results: {results_file}")
|
|
log.info(f" Summary: {args.output_dir / 'summary.json'}")
|
|
log.info(f" Full log: {args.output_dir / 'eval_grade.log'}")
|
|
log.info("=" * 64)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|