From 552e7d76366f92c3981e429701e4839eef0cc173 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 6 Mar 2026 23:36:00 -0500 Subject: [PATCH] Use setsid to detach CLI process, touch emitter during poll MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Daytona's POST /process/execute blocks until all descendant processes exit. The backgrounded claude process kept the API hanging, causing a 60-second HTTP timeout. Using setsid creates a new session so the child is fully detached — the API now returns in ~200ms. Also touch the event emitter during the poll loop to prevent the stall watchdog from killing the stage while waiting for claude to finish. Falls back gracefully on macOS where setsid isn't available (not needed since the local exec implementation doesn't wait for grandchildren). Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/arc-workflows/src/cli/cli_backend.rs | 10 +- .../tests/daytona_integration.rs | 119 ++++++++++++++++++ 2 files changed, 127 insertions(+), 2 deletions(-) diff --git a/crates/arc-workflows/src/cli/cli_backend.rs b/crates/arc-workflows/src/cli/cli_backend.rs index 2de669957..249beb602 100644 --- a/crates/arc-workflows/src/cli/cli_backend.rs +++ b/crates/arc-workflows/src/cli/cli_backend.rs @@ -285,7 +285,7 @@ impl CodergenBackend for AgentCliBackend { prompt: &str, _context: &Context, _thread_id: Option<&str>, - _emitter: &Arc, + emitter: &Arc, stage_dir: &Path, sandbox: &Arc, ) -> Result { @@ -355,8 +355,13 @@ impl CodergenBackend for AgentCliBackend { } else { format!(". {env_path} && {command}") }; + // Use setsid (if available) to create a new session so the child process is + // fully detached from the shell. Without this, Daytona's POST /process/execute + // blocks until ALL descendant processes exit, causing a 60s HTTP timeout. + // $SID is empty on macOS (where setsid doesn't exist but isn't needed since + // the local exec implementation doesn't wait for grandchildren). let bg_command = format!( - "({inner_command} > {stdout_path} 2>{stderr_path}; echo $? > {exit_code_path}) &\necho $!" + "SID=$(command -v setsid || true)\n$SID sh -c '{inner_command} > {stdout_path} 2>{stderr_path}; echo $? > {exit_code_path}' /dev/null 2>&1 &\necho $!" ); let launch_start = std::time::Instant::now(); let launch_result = sandbox @@ -373,6 +378,7 @@ impl CodergenBackend for AgentCliBackend { let poll_interval = std::time::Duration::from_secs(5); let exit_code: i32 = loop { tokio::time::sleep(poll_interval).await; + emitter.touch(); // keep the stall watchdog alive while polling let poll_result = sandbox .exec_command(&poll_command, 30_000, None, None, None) .await diff --git a/crates/arc-workflows/tests/daytona_integration.rs b/crates/arc-workflows/tests/daytona_integration.rs index 991b61458..6636f3228 100644 --- a/crates/arc-workflows/tests/daytona_integration.rs +++ b/crates/arc-workflows/tests/daytona_integration.rs @@ -1436,3 +1436,122 @@ async fn daytona_git_push_run_branch_to_origin() { env.cleanup().await.unwrap(); } + +/// Diagnose toolbox proxy staleness after idle time. +/// +/// Creates a sandbox, runs a command, sleeps for increasing durations, then +/// retries. If a call fails, makes raw HTTP requests to capture the actual +/// underlying error that the SDK normally swallows. +/// +/// Run: cargo test -p arc-workflows -- --ignored daytona_toolbox_idle_diagnostic --nocapture +#[tokio::test] +#[ignore] +async fn daytona_toolbox_idle_diagnostic() { + let creds = load_github_app_credentials(); + let env = create_env_with_github_app(Some(creds)).await; + env.initialize().await.unwrap(); + + // 1. Verify toolbox works immediately after init + let result = env + .exec_command("echo alive", 30_000, None, None, None) + .await; + eprintln!("[t=0s] exec_command after init: {:?}", result.as_ref().map(|r| r.exit_code)); + assert!(result.is_ok(), "exec_command should work immediately after init"); + + let sandbox_name = env.sandbox_info(); + eprintln!("[t=0s] sandbox: {sandbox_name}"); + + // 2. Sleep for increasing durations and test + for sleep_secs in [30, 60, 90, 120, 180] { + eprintln!("\n--- sleeping {sleep_secs}s ---"); + tokio::time::sleep(std::time::Duration::from_secs(sleep_secs)).await; + + let result = env + .exec_command("echo alive", 30_000, None, None, None) + .await; + + match &result { + Ok(r) => { + eprintln!("[t=+{sleep_secs}s] OK exit_code={} stdout={}", r.exit_code, r.stdout.trim()); + } + Err(e) => { + eprintln!("[t=+{sleep_secs}s] FAILED: {e}"); + + // Diagnose with raw HTTP calls + let api_key = std::env::var("DAYTONA_API_KEY").unwrap_or_default(); + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(15)) + .build() + .unwrap(); + let api_url = std::env::var("DAYTONA_API_URL") + .or_else(|_| std::env::var("DAYTONA_SERVER_URL")) + .unwrap_or_else(|_| "https://app.daytona.io/api".to_string()); + + // Check sandbox state + let state_resp = client + .get(format!("{api_url}/sandbox/{sandbox_name}")) + .bearer_auth(&api_key) + .send() + .await; + match state_resp { + Ok(resp) => { + let body = resp.text().await.unwrap_or_default(); + let state = serde_json::from_str::(&body) + .ok() + .and_then(|v| v.get("state").cloned()); + eprintln!("[diag] sandbox state: {state:?}"); + } + Err(e) => { + eprintln!("[diag] sandbox API failed: {e}"); + } + } + + // Get toolbox proxy URL and try a direct call + let proxy_resp = client + .get(format!("{api_url}/sandbox/{sandbox_name}/toolbox-proxy-url")) + .bearer_auth(&api_key) + .send() + .await; + if let Ok(resp) = proxy_resp { + let body = resp.text().await.unwrap_or_default(); + eprintln!("[diag] proxy URL response: {}", &body[..body.len().min(200)]); + if let Some(url) = serde_json::from_str::(&body) + .ok() + .and_then(|v| v.get("url").and_then(|u| u.as_str()).map(String::from)) + { + let toolbox_url = format!("{url}/{sandbox_name}/process/execute"); + eprintln!("[diag] trying direct POST to {toolbox_url}"); + let direct = client + .post(&toolbox_url) + .bearer_auth(&api_key) + .json(&serde_json::json!({"command": "echo diag", "timeout": 10})) + .send() + .await; + match direct { + Ok(resp) => { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + eprintln!("[diag] direct call: {status} body={}", &body[..body.len().min(300)]); + } + Err(e) => { + // Walk the FULL error source chain + let mut msg = format!("[diag] direct call FAILED: {e}"); + let mut source: Option<&dyn std::error::Error> = std::error::Error::source(&e); + while let Some(cause) = source { + msg.push_str(&format!("\n caused by: {cause}")); + source = cause.source(); + } + eprintln!("{msg}"); + } + } + } + } + + panic!("exec_command failed after {sleep_secs}s idle: {e}"); + } + } + } + + eprintln!("\n=== PASS: all idle durations survived ==="); + env.cleanup().await.unwrap(); +}