From 6dfeeca45e1feb21a96e71174d19656bcc86b795 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 4 Aug 2026 15:01:19 -0400 Subject: [PATCH 1/6] perf(agent): skip Daytona folder request for edits --- lib/components/fabro-agent/src/tools.rs | 4 +- .../fabro-sandbox/src/daytona/mod.rs | 84 ++++++++++++++++--- lib/components/fabro-sandbox/src/sandbox.rs | 10 +++ .../fabro-sandbox/src/test_support.rs | 14 +++- 4 files changed, 100 insertions(+), 12 deletions(-) diff --git a/lib/components/fabro-agent/src/tools.rs b/lib/components/fabro-agent/src/tools.rs index f892c971c..aad8e4e3d 100644 --- a/lib/components/fabro-agent/src/tools.rs +++ b/lib/components/fabro-agent/src/tools.rs @@ -226,7 +226,7 @@ pub fn make_edit_file_tool() -> RegisteredTool { }; ctx.env - .write_file(file_path, &new_content) + .write_existing_file(file_path, &new_content) .await .map_err(|e| e.display_with_causes())?; Ok(format!("Successfully edited {file_path}")) @@ -1002,6 +1002,7 @@ mod tests { ) .await; assert_eq!(result.unwrap(), "Successfully wrote to /out.txt"); + assert_eq!(env.existing_file_write_count(), 0); let written = env.written_files.lock().unwrap(); assert_eq!(written.len(), 1); assert_eq!(written[0].0, "/out.txt"); @@ -1036,6 +1037,7 @@ mod tests { ) .await; assert_eq!(result.unwrap(), "Successfully edited /f.txt"); + assert_eq!(env.existing_file_write_count(), 1); let written = env.written_files.lock().unwrap(); assert_eq!(written.len(), 1); assert_eq!(written[0].1, "goodbye world"); diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index 6371f23d2..72b0d2fff 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -511,6 +511,19 @@ impl DaytonaSandbox { resolve_path(path, self.working_directory()) } + async fn upload_file_content(&self, resolved_path: &str, content: &str) -> crate::Result<()> { + let sandbox = self.sandbox()?; + let fs_svc = sandbox + .fs() + .await + .map_err(|e| crate::Error::context("Failed to get fs service", e))?; + + fs_svc + .upload_file_bytes(resolved_path, content.as_bytes()) + .await + .map_err(|e| crate::Error::context(format!("Failed to write file {resolved_path}"), e)) + } + /// Verify a Daytona sandbox evaluates commands as non-login Bash. /// /// Runs on a freshly created sandbox before any Fabro-owned setup, and @@ -1613,17 +1626,12 @@ impl Sandbox for DaytonaSandbox { } } - let fs_svc = sandbox - .fs() - .await - .map_err(|e| crate::Error::context("Failed to get fs service", e))?; + self.upload_file_content(&resolved, content).await + } - fs_svc - .upload_file_bytes(&resolved, content.as_bytes()) - .await - .map_err(|e| crate::Error::context(format!("Failed to write file {resolved}"), e))?; - - Ok(()) + async fn write_existing_file(&self, path: &str, content: &str) -> crate::Result<()> { + let resolved = self.resolve_path(path); + self.upload_file_content(&resolved, content).await } async fn delete_file(&self, path: &str) -> crate::Result<()> { @@ -3432,6 +3440,62 @@ mod tests { delete.assert_async().await; } + #[tokio::test] + async fn write_existing_file_skips_parent_directory_creation() { + let server = MockServer::start_async().await; + let server_url = server.base_url(); + let sandbox_response = server + .mock_async(|when, then| { + when.method(GET).path("/sandbox/sandbox-edit"); + then.status(200) + .header("content-type", "application/json") + .json_body(sandbox_body("sandbox-edit", SandboxState::Started)); + }) + .await; + let toolbox_response = server + .mock_async(|when, then| { + when.method(GET) + .path("/sandbox/sandbox-edit/toolbox-proxy-url"); + then.status(200) + .header("content-type", "application/json") + .json_body(serde_json::json!({"url": server_url})); + }) + .await; + let folder = server + .mock_async(|when, then| { + when.method(POST).path("/sandbox-edit/files/folder"); + then.status(200); + }) + .await; + let upload = server + .mock_async(|when, then| { + when.method(POST) + .path("/sandbox-edit/files/upload") + .query_param("path", "/home/daytona/workspace/src/lib.rs") + .body_includes("updated contents"); + then.status(200); + }) + .await; + + let sandbox = mock_daytona_sandbox(&server, "dtn_test", DaytonaConfig::default()).await; + let sdk_sandbox = sandbox + .client + .get("sandbox-edit") + .await + .expect("get mock sandbox"); + assert!(sandbox.sandbox.set(sdk_sandbox).is_ok()); + + sandbox + .write_existing_file("src/lib.rs", "updated contents") + .await + .expect("write existing file"); + + sandbox_response.assert_async().await; + toolbox_response.assert_async().await; + upload.assert_async().await; + folder.assert_calls_async(0).await; + } + /// Recover the inner command a wrapper carries, proving it survives the /// base64 transport byte-for-byte. fn decode_wrapped_command(wrapped: &str) -> String { diff --git a/lib/components/fabro-sandbox/src/sandbox.rs b/lib/components/fabro-sandbox/src/sandbox.rs index 6e6e2b336..31a873300 100644 --- a/lib/components/fabro-sandbox/src/sandbox.rs +++ b/lib/components/fabro-sandbox/src/sandbox.rs @@ -1048,6 +1048,16 @@ pub trait Sandbox: Send + Sync { } async fn write_file(&self, path: &str, content: &str) -> crate::Result<()>; + + /// Write a file that the caller has already confirmed exists. + /// + /// Providers can override this method to skip setup that is only needed + /// when creating a new path. The default preserves the behavior of + /// [`Sandbox::write_file`]. + async fn write_existing_file(&self, path: &str, content: &str) -> crate::Result<()> { + self.write_file(path, content).await + } + async fn delete_file(&self, path: &str) -> crate::Result<()>; async fn file_exists(&self, path: &str) -> crate::Result; async fn list_directory( diff --git a/lib/components/fabro-sandbox/src/test_support.rs b/lib/components/fabro-sandbox/src/test_support.rs index 7364dc8b0..d25d9bbf4 100644 --- a/lib/components/fabro-sandbox/src/test_support.rs +++ b/lib/components/fabro-sandbox/src/test_support.rs @@ -1,6 +1,6 @@ use std::collections::HashMap; use std::sync::Mutex; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::time::Duration; use async_trait::async_trait; @@ -29,6 +29,8 @@ pub struct MockSandbox { pub os_version_str: String, /// Captures (path, content) pairs from `write_file` calls. pub written_files: Mutex>, + /// Counts calls to `write_existing_file`. + pub existing_file_writes: AtomicUsize, /// Captures the `timeout_ms` argument from `exec_command` calls. pub captured_timeout: Mutex>, /// Captures the `command` argument from `exec_command` calls (last only). @@ -104,6 +106,10 @@ impl MockSandbox { .expect("delete_calls lock poisoned") } + pub fn existing_file_write_count(&self) -> usize { + self.existing_file_writes.load(Ordering::Relaxed) + } + pub fn set_stdio_process(&self, process: MockStdioProcess) { *self .stdio_process @@ -156,6 +162,7 @@ impl Default for MockSandbox { platform_str: "darwin", os_version_str: "Darwin 24.0.0".into(), written_files: Mutex::new(Vec::new()), + existing_file_writes: AtomicUsize::new(0), captured_timeout: Mutex::new(None), captured_command: Mutex::new(None), captured_commands: Mutex::new(Vec::new()), @@ -250,6 +257,11 @@ impl Sandbox for MockSandbox { Ok(()) } + async fn write_existing_file(&self, path: &str, content: &str) -> crate::Result<()> { + self.existing_file_writes.fetch_add(1, Ordering::Relaxed); + self.write_file(path, content).await + } + async fn delete_file(&self, _path: &str) -> crate::Result<()> { Ok(()) } From a4db43a8892a38b625014fc642d9e9fc79a818f2 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 6 Aug 2026 11:54:32 -0400 Subject: [PATCH 2/6] Report live billing totals for in-progress runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run summaries previously populated billing only from the terminal conclusion event, so the web UI's size chip showed dollar amounts only after a run completed — even though the size letter was already derived from live per-stage usage. Derive billing from the same projected total the size uses. projected_billing already prefers the conclusion's billing once a run concludes, so completed runs still report the authoritative final total. Co-Authored-By: Claude Fable 5 --- lib/components/fabro-store/src/run_state.rs | 26 +++++++++------------ 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/lib/components/fabro-store/src/run_state.rs b/lib/components/fabro-store/src/run_state.rs index 5e4ab5e37..720a76f0e 100644 --- a/lib/components/fabro-store/src/run_state.rs +++ b/lib/components/fabro-store/src/run_state.rs @@ -1360,8 +1360,7 @@ pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> Run { .conclusion .as_ref() .map(|conclusion| conclusion.timing); - let terminal_total = terminal_total_usd_micros(state); - let current_total = projected_billing(state).total_usd_micros; + let total_usd_micros = projected_billing(state).total_usd_micros; Run { id: *run_id, @@ -1405,10 +1404,10 @@ pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> Run { completed_at, }, timing: run_timing, - billing: terminal_total.map(|total_usd_micros| RunBillingSummary { + billing: total_usd_micros.map(|total_usd_micros| RunBillingSummary { total_usd_micros: Some(total_usd_micros), }), - size: RunSize::from_total_usd_micros(current_total), + size: RunSize::from_total_usd_micros(total_usd_micros), ask_fabro: AskFabro::default(), diff: diff_summary, pull_request: state.pull_request.clone(), @@ -1421,14 +1420,6 @@ pub(crate) fn build_summary(state: &RunProjection, run_id: &RunId) -> Run { } } -fn terminal_total_usd_micros(state: &RunProjection) -> Option { - state - .conclusion - .as_ref() - .and_then(|conclusion| conclusion.billing.as_ref()) - .and_then(|billing| billing.total_usd_micros) -} - pub(crate) fn projected_billing(state: &RunProjection) -> BilledTokenCounts { if let Some(billing) = state .conclusion @@ -1693,8 +1684,8 @@ mod tests { BilledTokenCounts, BlockedReason, Checkpoint, CheckpointRecord, CommandTermination, EventBody, FailureCategory, FailureDetail, FailureReason, Graph, McpServerStatus, Node, Outcome, ParallelBranchId, PendingReason, PermissionLevel, PullRequestCreationStatus, - PullRequestLink, QuestionType, ReasoningEffort, RunApprovalState, RunBlobId, - RunControlAction, RunDiff, RunEvent, RunSize, RunSpec, RunStatus, Speed, + PullRequestLink, QuestionType, ReasoningEffort, RunApprovalState, RunBillingSummary, + RunBlobId, RunControlAction, RunDiff, RunEvent, RunSize, RunSpec, RunStatus, Speed, StageContextWindowBreakdownItem, StageContextWindowCategory, StageContextWindowCountMethod, StageContextWindowProjection, StageContextWindowStaleness, StageContextWindowWarning, StageHandler, StageModelUsage, StageOutcome, StageState, StageTiming, SubAgentStatus, @@ -5284,7 +5275,12 @@ mod tests { let summary = build_summary(&state, &fixtures::RUN_1); assert_eq!(summary.size, RunSize::S); - assert_eq!(summary.billing, None); + assert_eq!( + summary.billing, + Some(RunBillingSummary { + total_usd_micros: Some(20_000_001), + }) + ); } #[test] From 18a71ac310f8f429896930500b5d5fb4a6e98682 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 18 Aug 2026 12:08:36 -0400 Subject: [PATCH 3/6] Classify provider 412s as failover-eligible account lockouts Fireworks reports an account suspension (spending cap reached or unpaid invoices) as HTTP 412 with code PRECONDITION_FAILED. The status had no explicit mapping, and the openai_compatible dialect extracts error.type ("error") as the code, so the suspension fell through to InvalidRequest -- a deterministic request defect -- which suppressed both retry and the configured model fallback chain. A live run then died mid-stage with five healthy fallback candidates configured. No LLM request carries conditional-request preconditions, so a 412 is never about the request. Map it to AccessDenied, the same family as the account_deactivated error code: non-retryable on the same provider, eligible for failover to a provider with independent billing. Co-Authored-By: Claude Fable 5 --- lib/components/fabro-llm/src/error.rs | 54 ++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/lib/components/fabro-llm/src/error.rs b/lib/components/fabro-llm/src/error.rs index 6ec6d7886..0409d378e 100644 --- a/lib/components/fabro-llm/src/error.rs +++ b/lib/components/fabro-llm/src/error.rs @@ -355,7 +355,12 @@ pub fn error_from_status_code( // error types let kind = match status_code { 401 => ProviderErrorKind::Authentication, - 403 => ProviderErrorKind::AccessDenied, + // A 412 is never about the request: no LLM request carries + // conditional-request preconditions. Fireworks uses it for + // account-level lockouts (suspension over a spending cap or unpaid + // invoices), the same family as `account_deactivated`: deterministic + // here, but another provider has independent billing. + 403 | 412 => ProviderErrorKind::AccessDenied, 404 => ProviderErrorKind::NotFound, 408 => { return Error::RequestTimeout { @@ -728,6 +733,53 @@ mod tests { assert_eq!(err.provider_kind(), Some(ProviderErrorKind::QuotaExceeded)); } + /// Fireworks reports an account suspension (spending cap reached or + /// unpaid invoices) as HTTP 412 with `code: "PRECONDITION_FAILED"` in + /// the body. A chat completion carries no conditional-request + /// preconditions, so a 412 is always an account-level lockout, never a + /// defect in the request: it must not classify as `InvalidRequest`, and + /// a fallback provider with independent billing must stay eligible. + #[test] + fn account_suspension_412_is_failover_eligible() { + let err = error_from_status_code( + 412, + "Account lithoscomputer is suspended, possibly due to reaching \ + the monthly spending limit or failure to pay past invoices." + .into(), + "fireworks".into(), + // The openai_compatible dialect reads `error.type` as the code, + // so the discriminating `PRECONDITION_FAILED` only reaches this + // mapping through the status code. + Some("error".into()), + Some(serde_json::json!({ + "error": { + "message": "Account lithoscomputer is suspended, possibly due to reaching the monthly spending limit or failure to pay past invoices. Please go to https://fireworks.ai/account/billing for more information.", + "param": null, + "code": "PRECONDITION_FAILED", + "type": "error" + }, + "request_id": "chatcmpl-d9652b89a6604931ac27dddd5ef5bdc0" + })), + None, + ); + + assert_eq!(err.provider_kind(), Some(ProviderErrorKind::AccessDenied)); + assert!(!err.retryable()); + assert!(err.failover_eligible()); + + // A bare 412 with no parseable body classifies the same way. + let err = error_from_status_code( + 412, + "Precondition Failed".into(), + "fireworks".into(), + None, + None, + None, + ); + assert_eq!(err.provider_kind(), Some(ProviderErrorKind::AccessDenied)); + assert!(err.failover_eligible()); + } + #[test] fn kind_from_error_code_covers_every_dialect() { for (code, expected) in [ From 1226ed737776c944fad7921601a31377cd82fb29 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Tue, 18 Aug 2026 12:10:50 -0400 Subject: [PATCH 4/6] Cite Fireworks' documentation for the 412 mapping Co-Authored-By: Claude Fable 5 --- lib/components/fabro-llm/src/error.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/components/fabro-llm/src/error.rs b/lib/components/fabro-llm/src/error.rs index 0409d378e..dd9a98347 100644 --- a/lib/components/fabro-llm/src/error.rs +++ b/lib/components/fabro-llm/src/error.rs @@ -356,10 +356,12 @@ pub fn error_from_status_code( let kind = match status_code { 401 => ProviderErrorKind::Authentication, // A 412 is never about the request: no LLM request carries - // conditional-request preconditions. Fireworks uses it for - // account-level lockouts (suspension over a spending cap or unpaid - // invoices), the same family as `account_deactivated`: deterministic - // here, but another provider has independent billing. + // conditional-request preconditions. Fireworks documents it as + // "Account is suspended or there's an issue with account status", + // also emitted for a LoRA model that failed to load + // (https://docs.fireworks.ai/guides/inference-error-codes). The same + // family as `account_deactivated`: deterministic here, but another + // provider has independent billing and model inventory. 403 | 412 => ProviderErrorKind::AccessDenied, 404 => ProviderErrorKind::NotFound, 408 => { From 0845c331cb38c25db5265d2e1056dc3eb0fee6ec Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 20 Aug 2026 17:26:13 -0400 Subject: [PATCH 5/6] Default Daytona auto-stop to 120 minutes Omitting autoStopInterval from the create-sandbox request inherits Daytona's server-side default of 15 idle minutes. Daytona counts inactivity from the last sandbox interaction, and LLM inference never touches the sandbox, so a single long inference call is enough for the sandbox to auto-stop mid-run: a workflow failed exactly this way, with the sandbox entering its stop transition 15 minutes after the last command while the agent was still thinking. Send an explicit 120-minute default when lifecycle.auto_stop is unset. That clears any realistic inference call while still reclaiming sandboxes leaked by a dead worker. An explicit auto_stop = "0s" still disables auto-stop entirely. Co-Authored-By: Claude Fable 5 --- docs/public/execution/run-configuration.mdx | 2 +- docs/public/integrations/daytona.mdx | 4 +++ .../fabro-sandbox/src/daytona/mod.rs | 36 ++++++++++++++++++- 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/docs/public/execution/run-configuration.mdx b/docs/public/execution/run-configuration.mdx index a8f1bc927..d2ce9bd5e 100644 --- a/docs/public/execution/run-configuration.mdx +++ b/docs/public/execution/run-configuration.mdx @@ -321,7 +321,7 @@ memory = "8GB" | `network.allow` | CIDRs for `cidr_allow_list`; entries are validated as CIDRs. | | `lifecycle.preserve` | Keep the created sandbox after the run finishes. | | `lifecycle.stop_on_terminal` | Stop the sandbox when the run reaches a terminal state. | -| `lifecycle.auto_stop` | Daytona auto-stop duration, such as `"30m"`. | +| `lifecycle.auto_stop` | Daytona auto-stop duration, such as `"30m"`. Defaults to `"120m"`; `"0s"` disables auto-stop. | | `labels` | Provider labels. Merge by key across layers. | | `env` | Environment variables passed to command and agent execution. Merge by key across layers. | diff --git a/docs/public/integrations/daytona.mdx b/docs/public/integrations/daytona.mdx index 6ad19b9b3..1a2f25790 100644 --- a/docs/public/integrations/daytona.mdx +++ b/docs/public/integrations/daytona.mdx @@ -198,6 +198,10 @@ The `lifecycle.auto_stop` setting tells Daytona to stop the sandbox after a peri auto_stop = "30m" ``` +When `auto_stop` is unset, Fabro applies a default of 120 minutes so a sandbox leaked by an interrupted run is still reclaimed. Set `auto_stop = "0s"` to disable auto-stop and let the sandbox run indefinitely. + +Daytona counts inactivity from the last sandbox interaction (a command, file operation, or other API call). Time an agent spends on LLM inference does not touch the sandbox, so intervals shorter than your longest inference call risk stopping the sandbox mid-run. + ## Server defaults When running via `fabro server start`, the server config at `~/.fabro/settings.toml` can set default Daytona settings for all runs. Run config TOML values override server defaults. Labels are **merged** — run config labels win on key collisions. The `network` setting uses simple override (run config replaces the server default entirely). diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index 6371f23d2..91a0cc797 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -68,6 +68,12 @@ const DAYTONA_START_TIMEOUT: Duration = Duration::from_mins(1); /// deletion, temporary stdin files) so a stalled REST call cannot block /// cancellation/timeout paths indefinitely. const DAYTONA_CLEANUP_TIMEOUT: Duration = Duration::from_secs(10); +/// Auto-stop applied when `lifecycle.auto_stop` is unset. Omitting the field +/// would inherit Daytona's server-side default of 15 idle minutes, which is +/// shorter than a single long inference call and stops the sandbox mid-run; +/// 120 minutes clears any realistic call while still reclaiming sandboxes +/// leaked by a dead worker. An explicit `0` disables auto-stop entirely. +const DEFAULT_AUTO_STOP_INTERVAL_MINUTES: i32 = 120; /// Permissions a Daytona API key needs for Fabro's snapshot and sandbox flow. pub const REQUIRED_DAYTONA_PERMISSIONS: &[Permissions] = &[ @@ -727,7 +733,10 @@ impl DaytonaSandbox { daytona_sdk::SandboxBaseParams { name: Some(name), env_vars: Some(clean_bash_env(None)), - auto_stop_interval: self.config.auto_stop_interval, + auto_stop_interval: self + .config + .auto_stop_interval + .or(Some(DEFAULT_AUTO_STOP_INTERVAL_MINUTES)), labels: Some(managed_labels::merge_for_run( self.config.labels.as_ref(), self.run_id.as_ref(), @@ -2950,6 +2959,10 @@ mod tests { assert_eq!(params.ephemeral, Some(false)); assert_eq!(params.auto_delete_interval, Some(-1)); + assert_eq!( + params.auto_stop_interval, + Some(DEFAULT_AUTO_STOP_INTERVAL_MINUTES) + ); assert_eq!( params.env_vars, Some(HashMap::from([(BASH_ENV_VAR.to_string(), String::new())])) @@ -2963,6 +2976,27 @@ mod tests { ); } + #[tokio::test] + async fn base_params_passes_explicit_auto_stop_through() { + for interval in [0, 45] { + let sandbox = DaytonaSandbox::new( + DaytonaConfig { + auto_stop_interval: Some(interval), + ..DaytonaConfig::default() + }, + None, + None, + None, + None, + Some("dtn_test".to_string()), + ) + .await + .expect("sandbox config should be valid"); + + assert_eq!(sandbox.base_params().auto_stop_interval, Some(interval)); + } + } + #[tokio::test] async fn activate_skips_start_when_daytona_reports_started() { let server = MockServer::start_async().await; From f88df59163ad287a093bf26dc600c13be5343daa Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Thu, 20 Aug 2026 17:39:19 -0400 Subject: [PATCH 6/6] Classify sandbox state-change rejections as transient infra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Daytona "Sandbox state change in progress" rejection surfacing through the pipeline lifecycle path ("Pipeline lifecycle operation failed") matched no transient-infra hint, so the run failure was categorized deterministic. The condition is a provider lifecycle transition that finishes on its own — the definition of transient infrastructure — and the deterministic label misinforms retry machinery and anyone reading the failure. Add two transient-infra hints: the provider rejection ("state change in progress") and the bounded-wait timeout an activation reports when a stop transition outlives its budget ("sandbox stop still in progress"). Co-Authored-By: Claude Fable 5 --- lib/components/fabro-workflow/src/error.rs | 35 +++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/components/fabro-workflow/src/error.rs b/lib/components/fabro-workflow/src/error.rs index f8c08ca15..21833a8df 100644 --- a/lib/components/fabro-workflow/src/error.rs +++ b/lib/components/fabro-workflow/src/error.rs @@ -84,6 +84,8 @@ const TRANSIENT_INFRA_HINTS: &[&str] = &[ "cross-device link", "invalid cross-device link", "os error 18", + "state change in progress", + "sandbox stop still in progress", ]; const BUDGET_EXHAUSTED_HINTS: &[&str] = &[ @@ -807,6 +809,18 @@ mod tests { assert_eq!(err.failure_category(), FailureCategory::TransientInfra); } + #[test] + fn engine_error_with_sandbox_state_change_cause_classifies_transient() { + let source = TestOuterError { + message: "Failed to start Daytona sandbox", + source: TestCause("Sandbox state change in progress"), + }; + let err = Error::engine_with_source("Pipeline lifecycle operation failed", source); + + assert_eq!(err.failure_category(), FailureCategory::TransientInfra); + assert!(err.is_retryable()); + } + #[test] fn handler_error_display() { let err = Error::handler("LLM call failed"); @@ -1281,7 +1295,7 @@ mod tests { #[test] fn transient_infra_hints_count() { - assert_eq!(TRANSIENT_INFRA_HINTS.len(), 38); + assert_eq!(TRANSIENT_INFRA_HINTS.len(), 40); } #[test] @@ -1450,6 +1464,25 @@ mod tests { ); } + #[test] + fn classify_reason_sandbox_state_change_in_progress() { + assert_eq!( + classify_failure_reason( + "Pipeline lifecycle operation failed: failed to activate sandbox after node \ + attempt survey: Failed to start Daytona sandbox: Sandbox state change in progress" + ), + FailureCategory::TransientInfra + ); + } + + #[test] + fn classify_reason_sandbox_stop_still_in_progress() { + assert_eq!( + classify_failure_reason("Daytona sandbox stop still in progress after 120s"), + FailureCategory::TransientInfra + ); + } + #[test] fn classify_reason_500() { assert_eq!(