scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-08-28 05:27:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Add tracing to DaytonaSandbox::exec_command to diagnose stall watchdog timeouts

Browse source 
A workflow run hung for 33 minutes in a Daytona shell command despite
a 5-minute client-side timeout in tokio::select!. The root cause is
unknown — tests confirm the select pattern works correctly. Adding
info-level tracing at method entry (with command and timeout_ms),
after process service acquisition, and in each select arm so the
next occurrence reveals exactly where execution stalls.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-03-11 17:04:17 -04:00
parent 87435aef11
commit e38d3ea3ce
1 changed files with 21 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
lib/crates/arc-workflows/src/daytona_sandbox.rs
View file

@ -1002,6 +1002,8 @@ impl Sandbox for DaytonaSandbox {
env_vars: Option<&HashMap<String, String>>,
cancel_token: Option<tokio_util::sync::CancellationToken>,
) -> Result<ExecResult, String> {
tracing::info!(command, timeout_ms, "exec_command: entered");
let sandbox = self.sandbox()?;
let start = Instant::now();
@ -1014,6 +1016,11 @@ impl Sandbox for DaytonaSandbox {
.await
.map_err(|e| format!("Failed to get process service: {e}"))?;
tracing::info!(
elapsed_ms = start.elapsed().as_millis() as u64,
"exec_command: process service acquired, starting select"
);
let options = daytona_sdk::ExecuteCommandOptions {
cwd: Some(cwd),
env: env_vars.cloned(),
@ -1048,9 +1055,19 @@ impl Sandbox for DaytonaSandbox {
let result = tokio::select! {
res = exec_future => {
tracing::info!(
elapsed_ms = start.elapsed().as_millis() as u64,
ok = res.is_ok(),
"exec_command: HTTP response received"
);
res.map_err(|e| format!("Failed to execute command: {e}"))?
}
() = tokio::time::sleep(timeout_duration) => {
tracing::info!(
elapsed_ms = start.elapsed().as_millis() as u64,
timeout_ms,
"exec_command: client-side timeout fired"
);
return Ok(ExecResult {
stdout: String::new(),
stderr: "Command timed out locally".to_string(),
@ -1060,6 +1077,10 @@ impl Sandbox for DaytonaSandbox {
});
}
() = token.cancelled() => {
tracing::info!(
elapsed_ms = start.elapsed().as_millis() as u64,
"exec_command: cancelled via token"
);
return Ok(ExecResult {
stdout: String::new(),
stderr: "Command cancelled".to_string(),