From e26286d9662219e7926079f5d0559a1e2edc73fd Mon Sep 17 00:00:00 2001
From: Bryan Helmkamp <bryan@brynary.com>
Date: Mon, 16 Mar 2026 10:42:47 -0400
Subject: [PATCH] Add best-effort Daytona sandbox cleanup on timeout and
 disable PR creation

On timeout, finds the orphaned sandbox via fabro ps --label and deletes
it. Non-fatal if cleanup fails. Also adds [pull_request] enabled=false
to generated workflow.toml configs to prevent eval runs from opening PRs.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 evals/swe-bench/evaluate_daytona.py | 23 +++++++++++++++++++++++
 evals/swe-bench/run_eval.py         | 27 +++++++++++++++++++++++++++
 2 files changed, 50 insertions(+)

diff --git a/evals/swe-bench/evaluate_daytona.py b/evals/swe-bench/evaluate_daytona.py
index c4159229f..7cb1cd7b9 100644
--- a/evals/swe-bench/evaluate_daytona.py
+++ b/evals/swe-bench/evaluate_daytona.py
@@ -408,6 +408,7 @@ def evaluate_instance(
     except subprocess.TimeoutExpired:
         result["status"] = "timeout"
         result["error"] = f"Timed out after {timeout}s"
+        _cleanup_sandbox(instance_id, "swe-eval")
     except Exception as e:
         result["error"] = str(e)
         log.debug(f"[{instance_id}] Exception: {e}")
@@ -416,6 +417,28 @@ def evaluate_instance(
     return result
 
 
+def _cleanup_sandbox(label_value: str, label_key: str):
+    """Best-effort delete of orphaned Daytona sandbox after timeout."""
+    try:
+        ps = subprocess.run(
+            ["fabro", "ps", "--label", f"{label_key}={label_value}", "--json"],
+            capture_output=True, text=True, timeout=10,
+        )
+        runs = json.loads(ps.stdout) if ps.stdout.strip() else []
+        for run in runs:
+            run_id = run.get("run_id", "")
+            if not run_id:
+                continue
+            sandbox_name = f"fabro-{run_id}"
+            subprocess.run(
+                ["daytona", "sandbox", "delete", sandbox_name],
+                capture_output=True, timeout=15,
+            )
+            log.debug(f"[{label_value}] Deleted sandbox {sandbox_name}")
+    except Exception as e:
+        log.debug(f"[{label_value}] Sandbox cleanup failed (non-fatal): {e}")
+
+
 # ---------------------------------------------------------------------------
 # Main
 # ---------------------------------------------------------------------------
diff --git a/evals/swe-bench/run_eval.py b/evals/swe-bench/run_eval.py
index c96244d0a..5e34425bc 100644
--- a/evals/swe-bench/run_eval.py
+++ b/evals/swe-bench/run_eval.py
@@ -291,6 +291,7 @@ def run_instance(
     except subprocess.TimeoutExpired:
         result["status"] = "timeout"
         result["error"] = f"Timed out after {timeout}s"
+        _cleanup_sandbox(instance_id)
     except Exception as e:
         result["error"] = str(e)
         log.debug(f"[{instance_id}] Exception: {e}")
@@ -299,6 +300,32 @@ def run_instance(
     return result
 
 
+def _cleanup_sandbox(label_value: str):
+    """Best-effort delete of orphaned Daytona sandbox after timeout.
+
+    Finds the sandbox via `fabro ps --label --json` to get the run ID,
+    then deletes any Daytona sandbox whose name contains that run ID.
+    """
+    try:
+        ps = subprocess.run(
+            ["fabro", "ps", "--label", f"swe-bench={label_value}", "--json"],
+            capture_output=True, text=True, timeout=10,
+        )
+        runs = json.loads(ps.stdout) if ps.stdout.strip() else []
+        for run in runs:
+            run_id = run.get("run_id", "")
+            if not run_id:
+                continue
+            sandbox_name = f"fabro-{run_id}"
+            subprocess.run(
+                ["daytona", "sandbox", "delete", sandbox_name],
+                capture_output=True, timeout=15,
+            )
+            log.debug(f"[{label_value}] Deleted sandbox {sandbox_name}")
+    except Exception as e:
+        log.debug(f"[{label_value}] Sandbox cleanup failed (non-fatal): {e}")
+
+
 # ---------------------------------------------------------------------------
 # Main
 # ---------------------------------------------------------------------------