Merge remote-tracking branch 'origin/main' into feat/git-exec-spans

docs/public/execution/run-configuration.mdx

@@ -321,7 +321,7 @@ memory = "8GB"
 | `network.allow` | CIDRs for `cidr_allow_list`; entries are validated as CIDRs. |
 | `lifecycle.preserve` | Keep the created sandbox after the run finishes. |
 | `lifecycle.stop_on_terminal` | Stop the sandbox when the run reaches a terminal state. |
-| `lifecycle.auto_stop` | Daytona auto-stop duration, such as `"30m"`. |
+| `lifecycle.auto_stop` | Daytona auto-stop duration, such as `"30m"`. Defaults to `"120m"`; `"0s"` disables auto-stop. |
 | `labels` | Provider labels. Merge by key across layers. |
 | `env` | Environment variables passed to command and agent execution. Merge by key across layers. |
 

docs/public/integrations/daytona.mdx

@@ -198,6 +198,10 @@ The `lifecycle.auto_stop` setting tells Daytona to stop the sandbox after a peri
 auto_stop = "30m"
 ```
 
+When `auto_stop` is unset, Fabro applies a default of 120 minutes so a sandbox leaked by an interrupted run is still reclaimed. Set `auto_stop = "0s"` to disable auto-stop and let the sandbox run indefinitely.
+
+Daytona counts inactivity from the last sandbox interaction (a command, file operation, or other API call). Time an agent spends on LLM inference does not touch the sandbox, so intervals shorter than your longest inference call risk stopping the sandbox mid-run.
+
 ## Server defaults
 
 When running via `fabro server start`, the server config at `~/.fabro/settings.toml` can set default Daytona settings for all runs. Run config TOML values override server defaults. Labels are **merged** — run config labels win on key collisions. The `network` setting uses simple override (run config replaces the server default entirely).

lib/components/fabro-agent/src/tools.rs

@@ -226,7 +226,7 @@ pub fn make_edit_file_tool() -> RegisteredTool {
                 };
 
                 ctx.env
-                    .write_file(file_path, &new_content)
+                    .write_existing_file(file_path, &new_content)
                     .await
                     .map_err(|e| e.display_with_causes())?;
                 Ok(format!("Successfully edited {file_path}"))
@@ -1002,6 +1002,7 @@ mod tests {
         )
         .await;
         assert_eq!(result.unwrap(), "Successfully wrote to /out.txt");
+        assert_eq!(env.existing_file_write_count(), 0);
         let written = env.written_files.lock().unwrap();
         assert_eq!(written.len(), 1);
         assert_eq!(written[0].0, "/out.txt");
@@ -1036,6 +1037,7 @@ mod tests {
         )
         .await;
         assert_eq!(result.unwrap(), "Successfully edited /f.txt");
+        assert_eq!(env.existing_file_write_count(), 1);
         let written = env.written_files.lock().unwrap();
         assert_eq!(written.len(), 1);
         assert_eq!(written[0].1, "goodbye world");

lib/components/fabro-llm/src/error.rs

@@ -355,7 +355,14 @@ pub fn error_from_status_code(
     // error types
     let kind = match status_code {
         401 => ProviderErrorKind::Authentication,
-        403 => ProviderErrorKind::AccessDenied,
+        // A 412 is never about the request: no LLM request carries
+        // conditional-request preconditions. Fireworks documents it as
+        // "Account is suspended or there's an issue with account status",
+        // also emitted for a LoRA model that failed to load
+        // (https://docs.fireworks.ai/guides/inference-error-codes). The same
+        // family as `account_deactivated`: deterministic here, but another
+        // provider has independent billing and model inventory.
+        403 | 412 => ProviderErrorKind::AccessDenied,
         404 => ProviderErrorKind::NotFound,
         408 => {
             return Error::RequestTimeout {
@@ -728,6 +735,53 @@ mod tests {
         assert_eq!(err.provider_kind(), Some(ProviderErrorKind::QuotaExceeded));
     }
 
+    /// Fireworks reports an account suspension (spending cap reached or
+    /// unpaid invoices) as HTTP 412 with `code: "PRECONDITION_FAILED"` in
+    /// the body. A chat completion carries no conditional-request
+    /// preconditions, so a 412 is always an account-level lockout, never a
+    /// defect in the request: it must not classify as `InvalidRequest`, and
+    /// a fallback provider with independent billing must stay eligible.
+    #[test]
+    fn account_suspension_412_is_failover_eligible() {
+        let err = error_from_status_code(
+            412,
+            "Account lithoscomputer is suspended, possibly due to reaching \
+             the monthly spending limit or failure to pay past invoices."
+                .into(),
+            "fireworks".into(),
+            // The openai_compatible dialect reads `error.type` as the code,
+            // so the discriminating `PRECONDITION_FAILED` only reaches this
+            // mapping through the status code.
+            Some("error".into()),
+            Some(serde_json::json!({
+                "error": {
+                    "message": "Account lithoscomputer is suspended, possibly due to reaching the monthly spending limit or failure to pay past invoices. Please go to https://fireworks.ai/account/billing for more information.",
+                    "param": null,
+                    "code": "PRECONDITION_FAILED",
+                    "type": "error"
+                },
+                "request_id": "chatcmpl-d9652b89a6604931ac27dddd5ef5bdc0"
+            })),
+            None,
+        );
+
+        assert_eq!(err.provider_kind(), Some(ProviderErrorKind::AccessDenied));
+        assert!(!err.retryable());
+        assert!(err.failover_eligible());
+
+        // A bare 412 with no parseable body classifies the same way.
+        let err = error_from_status_code(
+            412,
+            "Precondition Failed".into(),
+            "fireworks".into(),
+            None,
+            None,
+            None,
+        );
+        assert_eq!(err.provider_kind(), Some(ProviderErrorKind::AccessDenied));
+        assert!(err.failover_eligible());
+    }
+
     #[test]
     fn kind_from_error_code_covers_every_dialect() {
         for (code, expected) in [

lib/components/fabro-sandbox/src/daytona/mod.rs

@@ -70,6 +70,12 @@ const DAYTONA_START_TIMEOUT: Duration = Duration::from_mins(1);
 /// deletion, temporary stdin files) so a stalled REST call cannot block
 /// cancellation/timeout paths indefinitely.
 const DAYTONA_CLEANUP_TIMEOUT: Duration = Duration::from_secs(10);
+/// Auto-stop applied when `lifecycle.auto_stop` is unset. Omitting the field
+/// would inherit Daytona's server-side default of 15 idle minutes, which is
+/// shorter than a single long inference call and stops the sandbox mid-run;
+/// 120 minutes clears any realistic call while still reclaiming sandboxes
+/// leaked by a dead worker. An explicit `0` disables auto-stop entirely.
+const DEFAULT_AUTO_STOP_INTERVAL_MINUTES: i32 = 120;
 
 /// Permissions a Daytona API key needs for Fabro's snapshot and sandbox flow.
 pub const REQUIRED_DAYTONA_PERMISSIONS: &[Permissions] = &[
@@ -520,6 +526,19 @@ impl DaytonaSandbox {
         resolve_path(path, self.working_directory())
     }
 
+    async fn upload_file_content(&self, resolved_path: &str, content: &str) -> crate::Result<()> {
+        let sandbox = self.sandbox()?;
+        let fs_svc = sandbox
+            .fs()
+            .await
+            .map_err(|e| crate::Error::context("Failed to get fs service", e))?;
+
+        fs_svc
+            .upload_file_bytes(resolved_path, content.as_bytes())
+            .await
+            .map_err(|e| crate::Error::context(format!("Failed to write file {resolved_path}"), e))
+    }
+
     /// Verify a Daytona sandbox evaluates commands as non-login Bash.
     ///
     /// Runs on a freshly created sandbox before any Fabro-owned setup, and
@@ -736,7 +755,10 @@ impl DaytonaSandbox {
         daytona_sdk::SandboxBaseParams {
             name: Some(name),
             env_vars: Some(clean_bash_env(None)),
-            auto_stop_interval: self.config.auto_stop_interval,
+            auto_stop_interval: self
+                .config
+                .auto_stop_interval
+                .or(Some(DEFAULT_AUTO_STOP_INTERVAL_MINUTES)),
             labels: Some(managed_labels::merge_for_run(
                 self.config.labels.as_ref(),
                 self.run_id.as_ref(),
@@ -1636,17 +1658,12 @@ impl Sandbox for DaytonaSandbox {
             }
         }
 
-        let fs_svc = sandbox
-            .fs()
-            .await
-            .map_err(|e| crate::Error::context("Failed to get fs service", e))?;
+        self.upload_file_content(&resolved, content).await
+    }
 
-        fs_svc
-            .upload_file_bytes(&resolved, content.as_bytes())
-            .await
-            .map_err(|e| crate::Error::context(format!("Failed to write file {resolved}"), e))?;
-
-        Ok(())
+    async fn write_existing_file(&self, path: &str, content: &str) -> crate::Result<()> {
+        let resolved = self.resolve_path(path);
+        self.upload_file_content(&resolved, content).await
     }
 
     async fn delete_file(&self, path: &str) -> crate::Result<()> {
@@ -2971,6 +2988,10 @@ mod tests {
 
         assert_eq!(params.ephemeral, Some(false));
         assert_eq!(params.auto_delete_interval, Some(-1));
+        assert_eq!(
+            params.auto_stop_interval,
+            Some(DEFAULT_AUTO_STOP_INTERVAL_MINUTES)
+        );
         assert_eq!(
             params.env_vars,
             Some(HashMap::from([(BASH_ENV_VAR.to_string(), String::new())]))
@@ -2984,6 +3005,27 @@ mod tests {
         );
     }
 
+    #[tokio::test]
+    async fn base_params_passes_explicit_auto_stop_through() {
+        for interval in [0, 45] {
+            let sandbox = DaytonaSandbox::new(
+                DaytonaConfig {
+                    auto_stop_interval: Some(interval),
+                    ..DaytonaConfig::default()
+                },
+                None,
+                None,
+                None,
+                None,
+                Some("dtn_test".to_string()),
+            )
+            .await
+            .expect("sandbox config should be valid");
+
+            assert_eq!(sandbox.base_params().auto_stop_interval, Some(interval));
+        }
+    }
+
     #[tokio::test]
     async fn activate_skips_start_when_daytona_reports_started() {
         let server = MockServer::start_async().await;
@@ -3462,6 +3504,62 @@ mod tests {
         delete.assert_async().await;
     }
 
+    #[tokio::test]
+    async fn write_existing_file_skips_parent_directory_creation() {
+        let server = MockServer::start_async().await;
+        let server_url = server.base_url();
+        let sandbox_response = server
+            .mock_async(|when, then| {
+                when.method(GET).path("/sandbox/sandbox-edit");
+                then.status(200)
+                    .header("content-type", "application/json")
+                    .json_body(sandbox_body("sandbox-edit", SandboxState::Started));
+            })
+            .await;
+        let toolbox_response = server
+            .mock_async(|when, then| {
+                when.method(GET)
+                    .path("/sandbox/sandbox-edit/toolbox-proxy-url");
+                then.status(200)
+                    .header("content-type", "application/json")
+                    .json_body(serde_json::json!({"url": server_url}));
+            })
+            .await;
+        let folder = server
+            .mock_async(|when, then| {
+                when.method(POST).path("/sandbox-edit/files/folder");
+                then.status(200);
+            })
+            .await;
+        let upload = server
+            .mock_async(|when, then| {
+                when.method(POST)
+                    .path("/sandbox-edit/files/upload")
+                    .query_param("path", "/home/daytona/workspace/src/lib.rs")
+                    .body_includes("updated contents");
+                then.status(200);
+            })
+            .await;
+
+        let sandbox = mock_daytona_sandbox(&server, "dtn_test", DaytonaConfig::default()).await;
+        let sdk_sandbox = sandbox
+            .client
+            .get("sandbox-edit")
+            .await
+            .expect("get mock sandbox");
+        assert!(sandbox.sandbox.set(sdk_sandbox).is_ok());
+
+        sandbox
+            .write_existing_file("src/lib.rs", "updated contents")
+            .await
+            .expect("write existing file");
+
+        sandbox_response.assert_async().await;
+        toolbox_response.assert_async().await;
+        upload.assert_async().await;
+        folder.assert_calls_async(0).await;
+    }
+
     /// Recover the inner command a wrapper carries, proving it survives the
     /// base64 transport byte-for-byte.
     fn decode_wrapped_command(wrapped: &str) -> String {

lib/components/fabro-sandbox/src/sandbox.rs

@@ -1088,6 +1088,16 @@ pub trait Sandbox: Send + Sync {
     }
 
     async fn write_file(&self, path: &str, content: &str) -> crate::Result<()>;
+
+    /// Write a file that the caller has already confirmed exists.
+    ///
+    /// Providers can override this method to skip setup that is only needed
+    /// when creating a new path. The default preserves the behavior of
+    /// [`Sandbox::write_file`].
+    async fn write_existing_file(&self, path: &str, content: &str) -> crate::Result<()> {
+        self.write_file(path, content).await
+    }
+
     async fn delete_file(&self, path: &str) -> crate::Result<()>;
     async fn file_exists(&self, path: &str) -> crate::Result<bool>;
     async fn list_directory(

lib/components/fabro-sandbox/src/test_support.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 use std::sync::Mutex;
-use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
 use std::time::Duration;
 
 use async_trait::async_trait;
@@ -29,6 +29,8 @@ pub struct MockSandbox {
     pub os_version_str:        String,
     /// Captures (path, content) pairs from `write_file` calls.
     pub written_files:         Mutex<Vec<(String, String)>>,
+    /// Counts calls to `write_existing_file`.
+    pub existing_file_writes:  AtomicUsize,
     /// Captures the `timeout_ms` argument from `exec_command` calls.
     pub captured_timeout:      Mutex<Option<u64>>,
     /// Captures the `command` argument from `exec_command` calls (last only).
@@ -104,6 +106,10 @@ impl MockSandbox {
             .expect("delete_calls lock poisoned")
     }
 
+    pub fn existing_file_write_count(&self) -> usize {
+        self.existing_file_writes.load(Ordering::Relaxed)
+    }
+
     pub fn set_stdio_process(&self, process: MockStdioProcess) {
         *self
             .stdio_process
@@ -156,6 +162,7 @@ impl Default for MockSandbox {
             platform_str:          "darwin",
             os_version_str:        "Darwin 24.0.0".into(),
             written_files:         Mutex::new(Vec::new()),
+            existing_file_writes:  AtomicUsize::new(0),
             captured_timeout:      Mutex::new(None),
             captured_command:      Mutex::new(None),
             captured_commands:     Mutex::new(Vec::new()),
@@ -250,6 +257,11 @@ impl Sandbox for MockSandbox {
         Ok(())
     }
 
+    async fn write_existing_file(&self, path: &str, content: &str) -> crate::Result<()> {
+        self.existing_file_writes.fetch_add(1, Ordering::Relaxed);
+        self.write_file(path, content).await
+    }
+
     async fn delete_file(&self, _path: &str) -> crate::Result<()> {
         Ok(())
     }

lib/components/fabro-store/src/run_state.rs

@@ -1360,8 +1360,7 @@ pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> Run {
         .conclusion
         .as_ref()
         .map(|conclusion| conclusion.timing);
-    let terminal_total = terminal_total_usd_micros(state);
-    let current_total = projected_billing(state).total_usd_micros;
+    let total_usd_micros = projected_billing(state).total_usd_micros;
 
     Run {
         id: *run_id,
@@ -1405,10 +1404,10 @@ pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> Run {
             completed_at,
         },
         timing: run_timing,
-        billing: terminal_total.map(|total_usd_micros| RunBillingSummary {
+        billing: total_usd_micros.map(|total_usd_micros| RunBillingSummary {
             total_usd_micros: Some(total_usd_micros),
         }),
-        size: RunSize::from_total_usd_micros(current_total),
+        size: RunSize::from_total_usd_micros(total_usd_micros),
         ask_fabro: AskFabro::default(),
         diff: diff_summary,
         pull_request: state.pull_request.clone(),
@@ -1421,14 +1420,6 @@ pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> Run {
     }
 }
 
-fn terminal_total_usd_micros(state: &RunProjection) -> Option<i64> {
-    state
-        .conclusion
-        .as_ref()
-        .and_then(|conclusion| conclusion.billing.as_ref())
-        .and_then(|billing| billing.total_usd_micros)
-}
-
 pub(crate) fn projected_billing(state: &RunProjection) -> BilledTokenCounts {
     if let Some(billing) = state
         .conclusion
@@ -1694,11 +1685,12 @@ mod tests {
         CommandTermination, EventBody, FailureCategory, FailureDetail, FailureReason, Graph,
         McpServerStatus, Node, Outcome, ParallelBranchId, PendingReason, PermissionLevel,
         PullRequestCreationStatus, PullRequestLink, QuestionType, ReasoningEffort,
-        RunApprovalState, RunControlAction, RunDiff, RunEvent, RunSize, RunSpec, RunStatus, Speed,
-        StageContextWindowBreakdownItem, StageContextWindowCategory, StageContextWindowCountMethod,
-        StageContextWindowProjection, StageContextWindowStaleness, StageContextWindowWarning,
-        StageHandler, StageModelUsage, StageOutcome, StageState, StageTiming, SubAgentStatus,
-        SuccessReason, WorkflowSettings, first_event_seq, fixtures, test_support,
+        RunApprovalState, RunBillingSummary, RunControlAction, RunDiff, RunEvent, RunSize, RunSpec,
+        RunStatus, Speed, StageContextWindowBreakdownItem, StageContextWindowCategory,
+        StageContextWindowCountMethod, StageContextWindowProjection, StageContextWindowStaleness,
+        StageContextWindowWarning, StageHandler, StageModelUsage, StageOutcome, StageState,
+        StageTiming, SubAgentStatus, SuccessReason, WorkflowSettings, first_event_seq, fixtures,
+        test_support,
     };
     use serde_json::json;
 
@@ -5254,7 +5246,12 @@ mod tests {
 
         let summary = build_summary(&state, &fixtures::RUN_1);
         assert_eq!(summary.size, RunSize::S);
-        assert_eq!(summary.billing, None);
+        assert_eq!(
+            summary.billing,
+            Some(RunBillingSummary {
+                total_usd_micros: Some(20_000_001),
+            })
+        );
     }
 
     #[test]

lib/components/fabro-workflow/src/error.rs

@@ -84,6 +84,8 @@ const TRANSIENT_INFRA_HINTS: &[&str] = &[
     "cross-device link",
     "invalid cross-device link",
     "os error 18",
+    "state change in progress",
+    "sandbox stop still in progress",
 ];
 
 const BUDGET_EXHAUSTED_HINTS: &[&str] = &[
@@ -843,6 +845,18 @@ mod tests {
         assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
     }
 
+    #[test]
+    fn engine_error_with_sandbox_state_change_cause_classifies_transient() {
+        let source = TestOuterError {
+            message: "Failed to start Daytona sandbox",
+            source:  TestCause("Sandbox state change in progress"),
+        };
+        let err = Error::engine_with_source("Pipeline lifecycle operation failed", source);
+
+        assert_eq!(err.failure_category(), FailureCategory::TransientInfra);
+        assert!(err.is_retryable());
+    }
+
     #[test]
     fn handler_error_display() {
         let err = Error::handler("LLM call failed");
@@ -1317,7 +1331,7 @@ mod tests {
 
     #[test]
     fn transient_infra_hints_count() {
-        assert_eq!(TRANSIENT_INFRA_HINTS.len(), 38);
+        assert_eq!(TRANSIENT_INFRA_HINTS.len(), 40);
     }
 
     #[test]
@@ -1486,6 +1500,25 @@ mod tests {
         );
     }
 
+    #[test]
+    fn classify_reason_sandbox_state_change_in_progress() {
+        assert_eq!(
+            classify_failure_reason(
+                "Pipeline lifecycle operation failed: failed to activate sandbox after node \
+                 attempt survey: Failed to start Daytona sandbox: Sandbox state change in progress"
+            ),
+            FailureCategory::TransientInfra
+        );
+    }
+
+    #[test]
+    fn classify_reason_sandbox_stop_still_in_progress() {
+        assert_eq!(
+            classify_failure_reason("Daytona sandbox stop still in progress after 120s"),
+            FailureCategory::TransientInfra
+        );
+    }
+
     #[test]
     fn classify_reason_500() {
         assert_eq!(