scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-08-28 05:27:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Add best-effort Daytona sandbox cleanup on timeout and disable PR creation

Browse source 
On timeout, finds the orphaned sandbox via fabro ps --label and deletes
it. Non-fatal if cleanup fails. Also adds [pull_request] enabled=false
to generated workflow.toml configs to prevent eval runs from opening PRs.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-03-16 10:42:47 -04:00
parent cd786bb875
commit e26286d966
2 changed files with 50 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
evals/swe-bench/evaluate_daytona.py
View file

@ -408,6 +408,7 @@ def evaluate_instance(
except subprocess.TimeoutExpired:
result["status"] = "timeout"
result["error"] = f"Timed out after {timeout}s"
_cleanup_sandbox(instance_id, "swe-eval")
except Exception as e:
result["error"] = str(e)
log.debug(f"[{instance_id}] Exception: {e}")
@ -416,6 +417,28 @@ def evaluate_instance(
return result
def _cleanup_sandbox(label_value: str, label_key: str):
"""Best-effort delete of orphaned Daytona sandbox after timeout."""
try:
ps = subprocess.run(
["fabro", "ps", "--label", f"{label_key}={label_value}", "--json"],
capture_output=True, text=True, timeout=10,
)
runs = json.loads(ps.stdout) if ps.stdout.strip() else []
for run in runs:
run_id = run.get("run_id", "")
if not run_id:
continue
sandbox_name = f"fabro-{run_id}"
subprocess.run(
["daytona", "sandbox", "delete", sandbox_name],
capture_output=True, timeout=15,
)
log.debug(f"[{label_value}] Deleted sandbox {sandbox_name}")
except Exception as e:
log.debug(f"[{label_value}] Sandbox cleanup failed (non-fatal): {e}")
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

27
evals/swe-bench/run_eval.py
View file

@ -291,6 +291,7 @@ def run_instance(
except subprocess.TimeoutExpired:
result["status"] = "timeout"
result["error"] = f"Timed out after {timeout}s"
_cleanup_sandbox(instance_id)
except Exception as e:
result["error"] = str(e)
log.debug(f"[{instance_id}] Exception: {e}")
@ -299,6 +300,32 @@ def run_instance(
return result
def _cleanup_sandbox(label_value: str):
"""Best-effort delete of orphaned Daytona sandbox after timeout.
Finds the sandbox via `fabro ps --label --json` to get the run ID,
then deletes any Daytona sandbox whose name contains that run ID.
"""
try:
ps = subprocess.run(
["fabro", "ps", "--label", f"swe-bench={label_value}", "--json"],
capture_output=True, text=True, timeout=10,
)
runs = json.loads(ps.stdout) if ps.stdout.strip() else []
for run in runs:
run_id = run.get("run_id", "")
if not run_id:
continue
sandbox_name = f"fabro-{run_id}"
subprocess.run(
["daytona", "sandbox", "delete", sandbox_name],
capture_output=True, timeout=15,
)
log.debug(f"[{label_value}] Deleted sandbox {sandbox_name}")
except Exception as e:
log.debug(f"[{label_value}] Sandbox cleanup failed (non-fatal): {e}")
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------