From e38d3ea3cecb99b595fcfdba4427781a65b581db Mon Sep 17 00:00:00 2001
From: Bryan Helmkamp <bryan@brynary.com>
Date: Wed, 11 Mar 2026 17:04:17 -0400
Subject: [PATCH] Add tracing to DaytonaSandbox::exec_command to diagnose stall
 watchdog timeouts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

A workflow run hung for 33 minutes in a Daytona shell command despite
a 5-minute client-side timeout in tokio::select!. The root cause is
unknown — tests confirm the select pattern works correctly. Adding
info-level tracing at method entry (with command and timeout_ms),
after process service acquisition, and in each select arm so the
next occurrence reveals exactly where execution stalls.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
---
 .../arc-workflows/src/daytona_sandbox.rs      | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/lib/crates/arc-workflows/src/daytona_sandbox.rs b/lib/crates/arc-workflows/src/daytona_sandbox.rs
index 97f793c44..75882cd66 100644
--- a/lib/crates/arc-workflows/src/daytona_sandbox.rs
+++ b/lib/crates/arc-workflows/src/daytona_sandbox.rs
@@ -1002,6 +1002,8 @@ impl Sandbox for DaytonaSandbox {
         env_vars: Option<&HashMap<String, String>>,
         cancel_token: Option<tokio_util::sync::CancellationToken>,
     ) -> Result<ExecResult, String> {
+        tracing::info!(command, timeout_ms, "exec_command: entered");
+
         let sandbox = self.sandbox()?;
         let start = Instant::now();
 
@@ -1014,6 +1016,11 @@ impl Sandbox for DaytonaSandbox {
             .await
             .map_err(|e| format!("Failed to get process service: {e}"))?;
 
+        tracing::info!(
+            elapsed_ms = start.elapsed().as_millis() as u64,
+            "exec_command: process service acquired, starting select"
+        );
+
         let options = daytona_sdk::ExecuteCommandOptions {
             cwd: Some(cwd),
             env: env_vars.cloned(),
@@ -1048,9 +1055,19 @@ impl Sandbox for DaytonaSandbox {
 
         let result = tokio::select! {
             res = exec_future => {
+                tracing::info!(
+                    elapsed_ms = start.elapsed().as_millis() as u64,
+                    ok = res.is_ok(),
+                    "exec_command: HTTP response received"
+                );
                 res.map_err(|e| format!("Failed to execute command: {e}"))?
             }
             () = tokio::time::sleep(timeout_duration) => {
+                tracing::info!(
+                    elapsed_ms = start.elapsed().as_millis() as u64,
+                    timeout_ms,
+                    "exec_command: client-side timeout fired"
+                );
                 return Ok(ExecResult {
                     stdout: String::new(),
                     stderr: "Command timed out locally".to_string(),
@@ -1060,6 +1077,10 @@ impl Sandbox for DaytonaSandbox {
                 });
             }
             () = token.cancelled() => {
+                tracing::info!(
+                    elapsed_ms = start.elapsed().as_millis() as u64,
+                    "exec_command: cancelled via token"
+                );
                 return Ok(ExecResult {
                     stdout: String::new(),
                     stderr: "Command cancelled".to_string(),