From e532842a3bd7b1b144998cf8cf8044557af4efa6 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 6 Mar 2026 19:52:13 -0500 Subject: [PATCH] CLI backend: background+poll to avoid HTTP proxy timeouts on Daytona Daytona's POST /process/execute is synchronous and blocks until the command finishes. Long-running CLI agent sessions (claude, codex, gemini) cause HTTP proxy timeouts. Replace the single blocking exec_command with a background launch + poll pattern: - Generate UUID-based temp file paths to avoid collisions between concurrent CLI nodes - Disable sandbox auto-stop before launching (new Sandbox trait method) - Launch command in background, capture PID - Poll every 5s for exit code file - Read stdout/stderr from temp files after completion - Cleanup temp files Co-Authored-By: Claude Opus 4.6 (1M context) --- crates/arc-agent/src/sandbox.rs | 10 ++ crates/arc-workflows/src/cli/cli_backend.rs | 91 ++++++++++++-- crates/arc-workflows/src/daytona_sandbox.rs | 13 ++ crates/arc-workflows/tests/integration.rs | 129 +++++++++++++++++--- 4 files changed, 212 insertions(+), 31 deletions(-) diff --git a/crates/arc-agent/src/sandbox.rs b/crates/arc-agent/src/sandbox.rs index 23345d90d..8167d510b 100644 --- a/crates/arc-agent/src/sandbox.rs +++ b/crates/arc-agent/src/sandbox.rs @@ -96,6 +96,10 @@ macro_rules! delegate_sandbox { async fn refresh_push_credentials(&self) -> Result<(), String> { self.$field.refresh_push_credentials().await } + + async fn set_autostop_interval(&self, minutes: i32) -> Result<(), String> { + self.$field.set_autostop_interval(minutes).await + } } }; } @@ -339,6 +343,12 @@ pub trait Sandbox: Send + Sync { async fn refresh_push_credentials(&self) -> Result<(), String> { Ok(()) } + + /// Set the auto-stop interval in minutes (0 to disable). + /// Default is a no-op; Daytona overrides to call the Daytona API. + async fn set_autostop_interval(&self, _minutes: i32) -> Result<(), String> { + Ok(()) + } } #[cfg(test)] diff --git a/crates/arc-workflows/src/cli/cli_backend.rs b/crates/arc-workflows/src/cli/cli_backend.rs index fd5159cab..2de669957 100644 --- a/crates/arc-workflows/src/cli/cli_backend.rs +++ b/crates/arc-workflows/src/cli/cli_backend.rs @@ -1,6 +1,7 @@ use std::path::Path; use std::sync::Arc; +use arc_agent::sandbox::ExecResult; use arc_agent::Sandbox; use arc_llm::provider::Provider; use async_trait::async_trait; @@ -291,20 +292,27 @@ impl CodergenBackend for AgentCliBackend { // 1. Snapshot git state before the CLI run let files_before = self.detect_changed_files(sandbox).await; - // 2. Write prompt to temp file - let prompt_path = "/tmp/arc_cli_prompt.txt"; + // 2. Generate unique paths for this run + let run_id = uuid::Uuid::new_v4().to_string(); + let tmp_prefix = format!("/tmp/arc_cli_{run_id}"); + let prompt_path = format!("{tmp_prefix}_prompt.txt"); + let stdout_path = format!("{tmp_prefix}_stdout.log"); + let stderr_path = format!("{tmp_prefix}_stderr.log"); + let exit_code_path = format!("{tmp_prefix}_exit_code"); + let env_path = format!("{tmp_prefix}_env.sh"); + sandbox - .write_file(prompt_path, prompt) + .write_file(&prompt_path, prompt) .await .map_err(|e| ArcError::handler(format!("Failed to write prompt file: {e}")))?; - // 3. Build and execute CLI command + // 3. Build CLI command let model = node.llm_model().unwrap_or(&self.model); let provider = node .llm_provider() .and_then(|s| s.parse::().ok()) .unwrap_or(self.provider); - let command = cli_command_for_provider(provider, model, prompt_path); + let command = cli_command_for_provider(provider, model, &prompt_path); let _ = tokio::fs::create_dir_all(stage_dir).await; let provider_used = serde_json::json!({ @@ -320,7 +328,6 @@ impl CodergenBackend for AgentCliBackend { // Forward provider API key so the CLI tool can authenticate. // Written to a temp file and sourced, since Daytona's exec API doesn't // support env vars and inline export would leak keys in logs. - let env_file = "/tmp/arc_cli_env.sh"; let env_lines: Vec = provider .api_key_env_vars() .iter() @@ -332,20 +339,80 @@ impl CodergenBackend for AgentCliBackend { .collect(); if !env_lines.is_empty() { sandbox - .write_file(env_file, &env_lines.join("\n")) + .write_file(&env_path, &env_lines.join("\n")) .await .map_err(|e| ArcError::handler(format!("Failed to write env file: {e}")))?; } - let full_command = if env_lines.is_empty() { + + // 3a. Disable auto-stop so the sandbox stays alive during long CLI runs + if let Err(e) = sandbox.set_autostop_interval(0).await { + tracing::warn!("Failed to disable sandbox auto-stop: {e}"); + } + + // 3b. Launch CLI command in background + let inner_command = if env_lines.is_empty() { command.clone() } else { - format!(". {env_file} && {command}") + format!(". {env_path} && {command}") + }; + let bg_command = format!( + "({inner_command} > {stdout_path} 2>{stderr_path}; echo $? > {exit_code_path}) &\necho $!" + ); + let launch_start = std::time::Instant::now(); + let launch_result = sandbox + .exec_command(&bg_command, 30_000, None, None, None) + .await + .map_err(|e| ArcError::handler(format!("Failed to launch CLI command: {e}")))?; + let pid = launch_result.stdout.trim(); + tracing::info!(pid, "CLI process launched in background"); + + // 3c. Poll for completion + let poll_command = format!( + "[ -f {exit_code_path} ] && cat {exit_code_path} || echo running" + ); + let poll_interval = std::time::Duration::from_secs(5); + let exit_code: i32 = loop { + tokio::time::sleep(poll_interval).await; + let poll_result = sandbox + .exec_command(&poll_command, 30_000, None, None, None) + .await + .map_err(|e| ArcError::handler(format!("Failed to poll CLI command: {e}")))?; + let status = poll_result.stdout.trim(); + if status != "running" { + break status.parse::().unwrap_or(-1); + } }; - let result = sandbox - .exec_command(&full_command, 600_000, None, None, None) + // 3d. Read results + let duration_ms = + u64::try_from(launch_start.elapsed().as_millis()).unwrap_or(u64::MAX); + let stdout_result = sandbox + .exec_command(&format!("cat {stdout_path}"), 60_000, None, None, None) .await - .map_err(|e| ArcError::handler(format!("CLI command failed: {e}")))?; + .map_err(|e| ArcError::handler(format!("Failed to read stdout: {e}")))?; + let stderr_result = sandbox + .exec_command(&format!("cat {stderr_path}"), 60_000, None, None, None) + .await + .map_err(|e| ArcError::handler(format!("Failed to read stderr: {e}")))?; + + let result = ExecResult { + stdout: stdout_result.stdout, + stderr: stderr_result.stdout, + exit_code, + timed_out: false, + duration_ms, + }; + + // 3e. Cleanup temp files + let _ = sandbox + .exec_command( + &format!("rm -f {tmp_prefix}_*"), + 30_000, + None, + None, + None, + ) + .await; if let Ok(json) = serde_json::to_string_pretty(&serde_json::json!({ "exit_code": result.exit_code, diff --git a/crates/arc-workflows/src/daytona_sandbox.rs b/crates/arc-workflows/src/daytona_sandbox.rs index 0c1146e75..45434808c 100644 --- a/crates/arc-workflows/src/daytona_sandbox.rs +++ b/crates/arc-workflows/src/daytona_sandbox.rs @@ -728,6 +728,19 @@ impl Sandbox for DaytonaSandbox { Ok(()) } + async fn set_autostop_interval(&self, minutes: i32) -> Result<(), String> { + let sandbox_id = self.sandbox()?.id.clone(); + let mut sandbox = self + .client + .get(&sandbox_id) + .await + .map_err(|e| format!("Failed to get sandbox for autostop update: {e}"))?; + sandbox + .set_autostop_interval(minutes) + .await + .map_err(|e| format!("Failed to set autostop interval: {e}")) + } + async fn read_file( &self, path: &str, diff --git a/crates/arc-workflows/tests/integration.rs b/crates/arc-workflows/tests/integration.rs index 97f20e7bf..e5ff77279 100644 --- a/crates/arc-workflows/tests/integration.rs +++ b/crates/arc-workflows/tests/integration.rs @@ -8733,7 +8733,62 @@ impl arc_agent::Sandbox for CliTestEnv { }); } - // CLI command: return configured stdout + // Background launch: return PID + if command.contains("echo $!") { + return Ok(arc_agent::ExecResult { + stdout: "12345\n".into(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 1, + }); + } + + // Poll for completion: return exit code 0 immediately + if command.contains("exit_code") && command.contains("echo running") { + return Ok(arc_agent::ExecResult { + stdout: "0\n".into(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 1, + }); + } + + // Read stdout file + if command.starts_with("cat") && command.contains("stdout.log") { + return Ok(arc_agent::ExecResult { + stdout: self.cli_stdout.clone(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 1, + }); + } + + // Read stderr file + if command.starts_with("cat") && command.contains("stderr.log") { + return Ok(arc_agent::ExecResult { + stdout: String::new(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 1, + }); + } + + // Cleanup temp files + if command.starts_with("rm -f") { + return Ok(arc_agent::ExecResult { + stdout: String::new(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 1, + }); + } + + // Fallback Ok(arc_agent::ExecResult { stdout: self.cli_stdout.clone(), stderr: String::new(), @@ -8810,27 +8865,30 @@ async fn cli_backend_run_writes_prompt_and_calls_exec() { // Verify prompt was written let written = test_env.recorded_written_files(); - assert_eq!( - written.len(), - 1, - "should write exactly one file (the prompt)" + let prompt_file = written + .iter() + .find(|(path, _)| path.contains("_prompt.txt")) + .expect("should write a prompt file"); + assert!( + prompt_file.0.starts_with("/tmp/arc_cli_") && prompt_file.0.ends_with("_prompt.txt"), + "prompt path should use UUID prefix: {}", + prompt_file.0 ); - assert_eq!(written[0].0, "/tmp/arc_cli_prompt.txt"); - assert_eq!(written[0].1, "Fix the authentication bug"); + assert_eq!(prompt_file.1, "Fix the authentication bug"); - // Verify the CLI command was called + // Verify the CLI command was called (now wrapped in background launch) let commands = test_env.recorded_commands(); let cli_cmd = commands .iter() - .find(|c| c.contains("claude")) - .expect("should call claude CLI"); + .find(|c| c.contains("claude") && c.contains("echo $!")) + .expect("should launch claude CLI in background"); assert!(cli_cmd.contains("-p"), "should use pipe mode"); assert!( cli_cmd.contains("claude-opus-4-6"), "should use correct model" ); assert!( - cli_cmd.contains("/tmp/arc_cli_prompt.txt"), + cli_cmd.contains("_prompt.txt"), "should reference prompt file" ); @@ -8909,12 +8967,12 @@ async fn cli_backend_run_with_codex_provider() { .await .expect("CLI backend should succeed"); - // Verify codex command was called + // Verify codex command was called (now wrapped in background launch) let commands = test_env.recorded_commands(); let cli_cmd = commands .iter() - .find(|c| c.contains("codex")) - .expect("should call codex CLI"); + .find(|c| c.contains("codex") && c.contains("echo $!")) + .expect("should launch codex CLI in background"); assert!(cli_cmd.contains("exec --json"), "should use exec mode"); assert!( cli_cmd.contains("gpt-5.3-codex"), @@ -8981,10 +9039,40 @@ async fn cli_backend_run_fails_on_nonzero_exit() { duration_ms: 0, }); } + // Background launch: return PID + if command.contains("echo $!") { + return Ok(arc_agent::ExecResult { + stdout: "12345\n".into(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 0, + }); + } + // Poll: return non-zero exit code + if command.contains("exit_code") && command.contains("echo running") { + return Ok(arc_agent::ExecResult { + stdout: "127\n".into(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 0, + }); + } + // Read stderr file + if command.starts_with("cat") && command.contains("stderr.log") { + return Ok(arc_agent::ExecResult { + stdout: "command not found: claude".into(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 0, + }); + } Ok(arc_agent::ExecResult { stdout: String::new(), - stderr: "command not found: claude".into(), - exit_code: 127, + stderr: String::new(), + exit_code: 0, timed_out: false, duration_ms: 0, }) @@ -9113,7 +9201,10 @@ async fn cli_backend_run_uses_node_model_override() { .expect("should succeed"); let commands = test_env.recorded_commands(); - let cli_cmd = commands.iter().find(|c| c.contains("claude")).unwrap(); + let cli_cmd = commands + .iter() + .find(|c| c.contains("claude") && c.contains("echo $!")) + .unwrap(); assert!( cli_cmd.contains("claude-sonnet-4-5"), "should use node's model override, not default: {cli_cmd}" @@ -9153,8 +9244,8 @@ async fn cli_backend_run_uses_node_provider_override() { let commands = test_env.recorded_commands(); let cli_cmd = commands .iter() - .find(|c| c.contains("codex")) - .expect("should call codex based on provider override"); + .find(|c| c.contains("codex") && c.contains("echo $!")) + .expect("should launch codex based on provider override"); assert!(cli_cmd.contains("gpt-5.3-codex")); }