From 8c45b870b49ab92134028af69e589cb212036277 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Fri, 14 Aug 2026 17:10:46 -0400 Subject: [PATCH 01/30] Check out exact sandbox commits --- lib/apps/fabro-server/src/run_manifest.rs | 2 + .../fabro-agent/tests/it/docker_shell.rs | 1 + .../fabro-sandbox/src/clone_source.rs | 351 +++++++++++++++++- .../fabro-sandbox/src/daytona/mod.rs | 303 ++++++++++++++- lib/components/fabro-sandbox/src/docker.rs | 307 ++++++++++++++- .../fabro-sandbox/src/provider/daytona.rs | 1 + .../fabro-sandbox/src/provider/docker.rs | 10 +- .../fabro-sandbox/src/sandbox_spec.rs | 57 ++- .../tests/daytona_streaming_live.rs | 5 + .../fabro-sandbox/tests/docker_streaming.rs | 5 + .../fabro-workflow/src/operations/start.rs | 2 + .../tests/it/daytona_integration.rs | 20 +- .../fabro-workflow/tests/it/integration.rs | 2 +- 13 files changed, 1046 insertions(+), 20 deletions(-) diff --git a/lib/apps/fabro-server/src/run_manifest.rs b/lib/apps/fabro-server/src/run_manifest.rs index 0ee804a3a..fda5f8087 100644 --- a/lib/apps/fabro-server/src/run_manifest.rs +++ b/lib/apps/fabro-server/src/run_manifest.rs @@ -923,6 +923,7 @@ fn preflight_sandbox_spec( run_id: None, clone_origin_url, clone_branch, + clone_commit_sha: None, } } SandboxProviderKind::Daytona => { @@ -934,6 +935,7 @@ fn preflight_sandbox_spec( run_id: None, clone_origin_url, clone_branch, + clone_commit_sha: None, api_key: daytona_api_key, } } diff --git a/lib/components/fabro-agent/tests/it/docker_shell.rs b/lib/components/fabro-agent/tests/it/docker_shell.rs index 48c0d25b3..6f50423ba 100644 --- a/lib/components/fabro-agent/tests/it/docker_shell.rs +++ b/lib/components/fabro-agent/tests/it/docker_shell.rs @@ -28,6 +28,7 @@ async fn shell_reports_real_docker_process_outcome() { None, None, None, + None, ) else { return; }; diff --git a/lib/components/fabro-sandbox/src/clone_source.rs b/lib/components/fabro-sandbox/src/clone_source.rs index 1a90c6b21..53effa04a 100644 --- a/lib/components/fabro-sandbox/src/clone_source.rs +++ b/lib/components/fabro-sandbox/src/clone_source.rs @@ -8,6 +8,7 @@ pub(crate) enum CloneDecision { GitHub { origin_url: String, branch: Option, + commit_sha: Option, }, } @@ -71,6 +72,55 @@ pub(crate) fn repo_symlink_command(layout: &GitHubRepoLayout) -> String { ) } +pub(crate) fn exact_repository_init_command(clone_url: &str, checkout_path: &str) -> String { + format!( + "git -c maintenance.auto=0 -c gc.auto=0 init -- {} && git -C {} remote add origin {}", + sandbox::shell_quote(checkout_path), + sandbox::shell_quote(checkout_path), + sandbox::shell_quote(clone_url), + ) +} + +pub(crate) fn exact_fetch_command( + checkout_path: &str, + fetch_source: &str, + commit_sha: &str, +) -> String { + format!( + "git -C {} -c maintenance.auto=0 -c gc.auto=0 fetch --depth 1 --no-tags {} -- {}", + sandbox::shell_quote(checkout_path), + sandbox::shell_quote(fetch_source), + sandbox::shell_quote(commit_sha), + ) +} + +pub(crate) fn exact_checkout_command(checkout_path: &str) -> String { + format!( + "git -C {} -c advice.detachedHead=false checkout --detach FETCH_HEAD", + sandbox::shell_quote(checkout_path), + ) +} + +pub(crate) fn head_revision_command(checkout_path: &str) -> String { + format!( + "git -C {} rev-parse HEAD", + sandbox::shell_quote(checkout_path), + ) +} + +pub(crate) fn verify_exact_head(output: &str, expected_sha: &str) -> crate::Result<()> { + let actual_sha = output.trim(); + let actual_sha = normalize_exact_commit_sha(actual_sha).map_err(|err| { + crate::Error::context("Exact checkout produced an invalid HEAD commit ID", err) + })?; + if actual_sha != expected_sha { + return Err(crate::Error::message( + "Exact checkout HEAD did not match the requested commit", + )); + } + Ok(()) +} + fn trim_root(root: &str) -> &str { let trimmed = root.trim_end_matches('/'); if trimmed.is_empty() { "/" } else { trimmed } @@ -97,7 +147,25 @@ pub(crate) fn decide_clone( skip_clone: bool, clone_origin_url: Option<&str>, clone_branch: Option<&str>, + clone_commit_sha: Option<&str>, ) -> crate::Result { + let commit_sha = clone_commit_sha + .map(normalize_exact_commit_sha) + .transpose()?; + + if commit_sha.is_some() { + if skip_clone { + return Err(crate::Error::message( + "Exact commit checkout requires cloning to be enabled", + )); + } + if clone_origin_url.is_none_or(|url| url.trim().is_empty()) { + return Err(crate::Error::message( + "Exact commit checkout requires a repository origin", + )); + } + } + if skip_clone { return Ok(CloneDecision::EmptyWorkspace { reason: EmptyWorkspaceReason::SkipClone, @@ -122,9 +190,19 @@ pub(crate) fn decide_clone( branch: clone_branch .filter(|branch| !branch.trim().is_empty()) .map(str::to_string), + commit_sha, }) } +fn normalize_exact_commit_sha(commit_sha: &str) -> crate::Result { + if commit_sha.len() != 40 || !commit_sha.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return Err(crate::Error::message( + "Exact commit SHA must be exactly 40 ASCII hexadecimal characters", + )); + } + Ok(commit_sha.to_ascii_lowercase()) +} + pub(crate) fn clean_clone_origin_for_record(clone_origin_url: Option<&str>) -> Option { clone_origin_url .filter(|url| !url.trim().is_empty()) @@ -136,22 +214,73 @@ pub(crate) fn repo_cloned_for_record( clone_origin_url: Option<&str>, ) -> Option { Some(matches!( - decide_clone(skip_clone, clone_origin_url, None).ok()?, + decide_clone(skip_clone, clone_origin_url, None, None).ok()?, CloneDecision::GitHub { .. } )) } #[cfg(test)] mod tests { + use std::fs; + use std::path::Path; + use std::process::{Command, Output}; + use super::*; + fn isolated_command(command: &mut Command) -> Output { + command + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_AUTHOR_NAME", "Fabro Test") + .env("GIT_AUTHOR_EMAIL", "fabro-test@example.com") + .env("GIT_COMMITTER_NAME", "Fabro Test") + .env("GIT_COMMITTER_EMAIL", "fabro-test@example.com") + .output() + .expect("test command should start") + } + + #[expect( + clippy::disallowed_methods, + reason = "hermetic Git proof intentionally runs the local git executable synchronously" + )] + fn run_git(cwd: &Path, args: &[&str]) -> String { + let output = isolated_command(Command::new("git").current_dir(cwd).args(args)); + assert!( + output.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).expect("git output should be UTF-8") + } + + #[expect( + clippy::disallowed_methods, + reason = "hermetic command-builder proof intentionally runs local Bash synchronously" + )] + fn run_shell(cwd: &Path, command: &str) -> String { + let output = isolated_command(Command::new("/bin/bash").current_dir(cwd).args([ + "--noprofile", + "--norc", + "-c", + command, + ])); + assert!( + output.status.success(), + "command failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8(output.stdout).expect("command output should be UTF-8") + } + #[test] fn skip_clone_overrides_present_origin() { assert_eq!( decide_clone( true, Some("https://gitlab.com/acme/widgets.git"), - Some("main") + Some("main"), + None, ) .unwrap(), CloneDecision::EmptyWorkspace { @@ -163,7 +292,7 @@ mod tests { #[test] fn missing_origin_creates_empty_workspace() { assert_eq!( - decide_clone(false, None, None).unwrap(), + decide_clone(false, None, None, None).unwrap(), CloneDecision::EmptyWorkspace { reason: EmptyWorkspaceReason::MissingOrigin, } @@ -176,23 +305,233 @@ mod tests { decide_clone( false, Some("git@github.com:acme/widgets.git"), - Some("feature/work") + Some("feature/work"), + None, ) .unwrap(), CloneDecision::GitHub { origin_url: "https://github.com/acme/widgets".to_string(), branch: Some("feature/work".to_string()), + commit_sha: None, } ); } #[test] fn non_github_origin_fails_without_skip_clone() { - let error = decide_clone(false, Some("https://gitlab.com/acme/widgets.git"), None) - .expect_err("non-GitHub origins should fail"); + let error = decide_clone( + false, + Some("https://gitlab.com/acme/widgets.git"), + None, + None, + ) + .expect_err("non-GitHub origins should fail"); assert!(error.to_string().contains("GitHub repository origins only")); } + #[test] + fn exact_commit_sha_is_validated_and_normalized() { + let lowercase = "0123456789abcdef0123456789abcdef01234567"; + let uppercase = "ABCDEF0123456789ABCDEF0123456789ABCDEF01"; + + assert_eq!( + decide_clone( + false, + Some("https://github.com/acme/widgets"), + Some("moving-branch"), + Some(lowercase), + ) + .unwrap(), + CloneDecision::GitHub { + origin_url: "https://github.com/acme/widgets".to_string(), + branch: Some("moving-branch".to_string()), + commit_sha: Some(lowercase.to_string()), + } + ); + assert_eq!( + decide_clone( + false, + Some("https://github.com/acme/widgets"), + None, + Some(uppercase), + ) + .unwrap(), + CloneDecision::GitHub { + origin_url: "https://github.com/acme/widgets".to_string(), + branch: None, + commit_sha: Some(uppercase.to_ascii_lowercase()), + } + ); + } + + #[test] + fn exact_commit_sha_rejects_noncanonical_inputs() { + for sha in [ + "", + "0123456789abcdef0123456789abcdef0123456", + "0123456789abcdef0123456789abcdef012345678", + "0123456789abcdef0123456789abcdef0123456g", + " 0123456789abcdef0123456789abcdef01234567", + "0123456789abcdef0123456789abcdef01234567 ", + "0123456789abcdef0123456789abcdef012345é", + ] { + let error = decide_clone( + false, + Some("https://github.com/acme/widgets"), + None, + Some(sha), + ) + .expect_err("invalid exact commit SHA should fail"); + assert!( + error.to_string().contains("40 ASCII hexadecimal"), + "unexpected error for {sha:?}: {error}" + ); + } + } + + #[test] + fn exact_checkout_requires_clone_and_origin() { + let sha = "0123456789abcdef0123456789abcdef01234567"; + let skip_error = decide_clone( + true, + Some("https://github.com/acme/widgets"), + None, + Some(sha), + ) + .expect_err("exact checkout with skip-clone should fail"); + assert!(skip_error.to_string().contains("requires cloning")); + + for origin in [None, Some(""), Some(" ")] { + let error = decide_clone(false, origin, None, Some(sha)) + .expect_err("exact checkout without an origin should fail"); + assert!(error.to_string().contains("requires a repository origin")); + } + } + + #[test] + fn exact_checkout_commands_quote_inputs_and_ignore_branch_metadata() { + let sha = "0123456789abcdef0123456789abcdef01234567"; + let init = exact_repository_init_command( + "https://token@example.com/acme/widgets.git?x=a b", + "/repos/acme's widgets", + ); + let fetch = exact_fetch_command( + "/repos/acme's widgets", + "https://token@example.com/acme/widgets.git?x=a b", + sha, + ); + let checkout = exact_checkout_command("/repos/acme's widgets"); + let verify = head_revision_command("/repos/acme's widgets"); + + assert_eq!( + init, + "git -c maintenance.auto=0 -c gc.auto=0 init -- \"/repos/acme's widgets\" && git -C \"/repos/acme's widgets\" remote add origin 'https://token@example.com/acme/widgets.git?x=a b'" + ); + assert_eq!( + fetch, + "git -C \"/repos/acme's widgets\" -c maintenance.auto=0 -c gc.auto=0 fetch --depth 1 --no-tags 'https://token@example.com/acme/widgets.git?x=a b' -- 0123456789abcdef0123456789abcdef01234567" + ); + assert_eq!( + checkout, + "git -C \"/repos/acme's widgets\" -c advice.detachedHead=false checkout --detach FETCH_HEAD" + ); + assert_eq!(verify, "git -C \"/repos/acme's widgets\" rev-parse HEAD"); + for command in [&init, &fetch, &checkout, &verify] { + assert!(!command.contains("moving-branch")); + } + } + + #[test] + fn exact_checkout_verification_rejects_invalid_or_mismatched_head() { + let expected = "0123456789abcdef0123456789abcdef01234567"; + verify_exact_head("0123456789ABCDEF0123456789ABCDEF01234567\n", expected) + .expect("uppercase command output should normalize"); + + let invalid = verify_exact_head("fatal: not a revision", expected) + .expect_err("non-SHA output should fail verification"); + assert!(invalid.to_string().contains("invalid HEAD commit ID")); + assert!(!invalid.to_string().contains("fatal: not a revision")); + + let mismatched = verify_exact_head("1123456789abcdef0123456789abcdef01234567", expected) + .expect_err("mismatched SHA should fail verification"); + assert!(mismatched.to_string().contains("did not match")); + } + + #[test] + #[expect( + clippy::disallowed_methods, + reason = "hermetic Git proof uses isolated synchronous temp-repository I/O" + )] + fn exact_checkout_fetches_admitted_commit_after_branch_advances() { + let temp = tempfile::tempdir().expect("tempdir"); + let remote = temp.path().join("remote.git"); + let source = temp.path().join("source"); + let checkout = temp.path().join("exact checkout"); + fs::create_dir(&source).expect("source directory"); + + run_git(temp.path(), &[ + "init", + "--bare", + remote.to_str().expect("UTF-8 remote path"), + ]); + run_git(&source, &["init"]); + fs::write(source.join("revision.txt"), "A\n").expect("write commit A"); + run_git(&source, &["add", "revision.txt"]); + run_git(&source, &["commit", "-m", "commit A"]); + run_git(&source, &["branch", "-M", "main"]); + run_git(&source, &[ + "remote", + "add", + "origin", + remote.to_str().expect("UTF-8 remote path"), + ]); + run_git(&source, &["push", "-u", "origin", "main"]); + let admitted_sha = run_git(&source, &["rev-parse", "HEAD"]).trim().to_string(); + + fs::write(source.join("revision.txt"), "B\n").expect("write commit B"); + run_git(&source, &["commit", "-am", "commit B"]); + run_git(&source, &["push", "origin", "main"]); + let advanced_sha = run_git(&source, &["rev-parse", "HEAD"]).trim().to_string(); + assert_ne!(admitted_sha, advanced_sha); + + let remote_path = remote.to_str().expect("UTF-8 remote path"); + let checkout_path = checkout.to_str().expect("UTF-8 checkout path"); + run_shell( + temp.path(), + &exact_repository_init_command(remote_path, checkout_path), + ); + run_shell( + temp.path(), + &exact_fetch_command(checkout_path, remote_path, &admitted_sha), + ); + run_shell(temp.path(), &exact_checkout_command(checkout_path)); + let checked_out_sha = run_shell(temp.path(), &head_revision_command(checkout_path)); + + assert_eq!(checked_out_sha.trim(), admitted_sha); + assert_eq!( + fs::read_to_string(checkout.join("revision.txt")).expect("checked-out contents"), + "A\n" + ); + let symbolic_head = isolated_command(Command::new("git").args([ + "-C", + checkout_path, + "symbolic-ref", + "-q", + "HEAD", + ])); + assert!(!symbolic_head.status.success(), "HEAD should be detached"); + assert_eq!( + run_git(temp.path(), &[ + "--git-dir", + remote_path, + "rev-parse", + "refs/heads/main", + ],) + .trim(), + advanced_sha + ); + } + #[test] fn github_layout_maps_ssh_origin_to_repos_checkout_and_workspace_link() { let layout = github_repo_layout( diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index 6371f23d2..617dc5c15 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -69,6 +69,42 @@ const DAYTONA_START_TIMEOUT: Duration = Duration::from_mins(1); /// cancellation/timeout paths indefinitely. const DAYTONA_CLEANUP_TIMEOUT: Duration = Duration::from_secs(10); +struct DaytonaExactCheckoutFailure { + error: crate::Error, + retry_reason: Option, +} + +fn daytona_clone_branch(requested_branch: Option<&str>, exact_checkout: bool) -> Option { + if exact_checkout { + None + } else { + requested_branch.map(str::to_string) + } +} + +fn daytona_process_exec_result(exit_code: i32, output: String) -> ExecResult { + let (stdout, stderr) = if exit_code == 0 { + (output, String::new()) + } else { + (String::new(), output) + }; + ExecResult { + stdout, + stderr, + exit_code: Some(exit_code), + termination: CommandTermination::Exited, + duration_ms: 0, + } +} + +fn daytona_exact_exec_error( + result: ExecResult, + label: &'static str, + auth_url: Option<&fabro_redact::DisplaySafeUrl>, +) -> crate::Error { + result.into_exec_error_with_redactor(label, |output| redact_auth_url(output, auth_url)) +} + /// Permissions a Daytona API key needs for Fabro's snapshot and sandbox flow. pub const REQUIRED_DAYTONA_PERMISSIONS: &[Permissions] = &[ Permissions::WriteColonSnapshots, @@ -349,6 +385,7 @@ pub struct DaytonaSandbox { /// Explicit branch to clone. When set, overrides the branch detected by /// the submitted run spec. clone_branch: Option, + clone_commit_sha: Option, } impl DaytonaSandbox { @@ -362,8 +399,17 @@ impl DaytonaSandbox { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, api_key: Option, ) -> crate::Result { + if clone_commit_sha.is_some() { + clone_source::decide_clone( + config.skip_clone, + clone_origin_url.as_deref(), + clone_branch.as_deref(), + clone_commit_sha.as_deref(), + )?; + } let api_key = resolve_daytona_api_key(api_key); let client = build_daytona_client(api_key.clone()) .await @@ -383,6 +429,7 @@ impl DaytonaSandbox { run_id, clone_origin_url, clone_branch, + clone_commit_sha, }) } @@ -435,6 +482,7 @@ impl DaytonaSandbox { run_id: None, clone_origin_url, clone_branch, + clone_commit_sha: None, }) } @@ -496,6 +544,57 @@ impl DaytonaSandbox { } } + fn report_clone_failure(&self, origin_url: &str, err: crate::Error) -> crate::Error { + self.emit(SandboxEvent::GitCloneFailed { + url: origin_url.to_string(), + error: err.to_string(), + causes: err.causes(), + }); + err + } + + fn daytona_process_transport_error(label: &'static str, error: &DaytonaError) -> crate::Error { + let error_class = match error { + DaytonaError::RateLimit { .. } => "rate_limited", + DaytonaError::Timeout { .. } => "timeout", + DaytonaError::Api { status_code, .. } if (500..600).contains(status_code) => { + "server_error" + } + DaytonaError::Api { .. } => "api_error", + DaytonaError::NotFound { .. } => "not_found", + DaytonaError::General(_) => "transport_error", + }; + crate::Error::context( + label, + crate::Error::message(format!("Daytona process command failed ({error_class})")), + ) + } + + async fn run_exact_checkout_command( + process_svc: &daytona_sdk::ProcessService, + command: &str, + working_directory: &str, + label: &'static str, + auth_url: Option<&fabro_redact::DisplaySafeUrl>, + ) -> crate::Result { + let response = process_svc + .execute_command( + &wrap_bash_command(command), + daytona_sdk::ExecuteCommandOptions { + cwd: Some(working_directory.to_string()), + ..Default::default() + }, + ) + .await + .map_err(|error| Self::daytona_process_transport_error(label, &error))?; + let result = daytona_process_exec_result(response.exit_code, response.result); + if result.is_success() { + Ok(result) + } else { + Err(daytona_exact_exec_error(result, label, auth_url)) + } + } + fn fail_init(&self, init_start: Instant, err: crate::Error) -> crate::Error { let duration_ms = u64::try_from(init_start.elapsed().as_millis()).unwrap_or(u64::MAX); self.emit(SandboxEvent::InitializeFailed { @@ -1024,6 +1123,7 @@ impl Sandbox for DaytonaSandbox { self.config.skip_clone, self.clone_origin_url.as_deref(), self.clone_branch.as_deref(), + self.clone_commit_sha.as_deref(), ) .map_err(|e| self.fail_init(init_start, e))?; @@ -1048,7 +1148,11 @@ impl Sandbox for DaytonaSandbox { self.set_working_directory(WORKING_DIRECTORY) .map_err(|err| self.fail_init(init_start, err))?; } - CloneDecision::GitHub { origin_url, branch } => { + CloneDecision::GitHub { + origin_url, + branch, + commit_sha, + } => { let layout = clone_source::github_repo_layout(&origin_url, WORKING_DIRECTORY, REPOS_ROOT) .map_err(|err| self.fail_init(init_start, err))?; @@ -1155,7 +1259,7 @@ impl Sandbox for DaytonaSandbox { let origin = origin_url.as_str(); let target = layout.primary_repo_path.as_str(); let options = daytona_sdk::GitCloneOptions { - branch: branch.clone(), + branch: daytona_clone_branch(branch.as_deref(), commit_sha.is_some()), username: username.clone(), password: password.clone(), ..Default::default() @@ -1178,6 +1282,132 @@ impl Sandbox for DaytonaSandbox { }); self.fail_init(init_start, err) })?; + + if let Some(expected_sha) = commit_sha.as_deref() { + let auth_url = match password.as_deref() { + Some(token) => { + match fabro_github::embed_token_in_url(&origin_url, token) { + Ok(url) => Some(url), + Err(error) => { + let error = crate::Error::Context { + message: "Failed to build authenticated URL for \ + Daytona exact checkout" + .to_string(), + source: error.into_boxed_dyn_error(), + }; + let error = + self.report_clone_failure(&origin_url, error); + return Err(self.fail_init(init_start, error)); + } + } + } + None => None, + }; + let fetch_source = auth_url + .as_ref() + .map_or(origin_url.as_str(), |url| url.as_raw_url().as_str()); + let fetch_command = clone_source::exact_fetch_command( + &layout.primary_repo_path, + fetch_source, + expected_sha, + ); + let fetch_result = clone_retry::retry_clone( + SandboxProviderKind::Daytona, + None, + |_attempt| { + let command = fetch_command.as_str(); + let process_svc = &process_svc; + let auth_url = auth_url.as_ref(); + async move { + let response = process_svc + .execute_command( + &wrap_bash_command(command), + daytona_sdk::ExecuteCommandOptions { + cwd: Some("/".to_string()), + ..Default::default() + }, + ) + .await + .map_err(|error| DaytonaExactCheckoutFailure { + retry_reason: classify_clone_failure( + &error, + token_was_freshly_minted, + ), + error: Self::daytona_process_transport_error( + "Daytona exact fetch transport failed", + &error, + ), + })?; + if response.exit_code == 0 { + return Ok(()); + } + let retry_reason = clone_retry::classify_message( + &response.result, + token_was_freshly_minted, + ) + .retry_reason(); + let result = daytona_process_exec_result( + response.exit_code, + response.result, + ); + Err(DaytonaExactCheckoutFailure { + retry_reason, + error: daytona_exact_exec_error( + result, + "git fetch exact commit in Daytona sandbox", + auth_url, + ), + }) + } + }, + |failure: &DaytonaExactCheckoutFailure| failure.retry_reason, + ) + .await; + if let Err(failure) = fetch_result { + let error = self.report_clone_failure(&origin_url, failure.error); + return Err(self.fail_init(init_start, error)); + } + + let checkout_command = + clone_source::exact_checkout_command(&layout.primary_repo_path); + if let Err(error) = Self::run_exact_checkout_command( + &process_svc, + &checkout_command, + "/", + "git checkout exact commit in Daytona sandbox", + auth_url.as_ref(), + ) + .await + { + let error = self.report_clone_failure(&origin_url, error); + return Err(self.fail_init(init_start, error)); + } + + let head_command = + clone_source::head_revision_command(&layout.primary_repo_path); + let head = match Self::run_exact_checkout_command( + &process_svc, + &head_command, + "/", + "verify Daytona exact checkout HEAD", + auth_url.as_ref(), + ) + .await + { + Ok(result) => result, + Err(error) => { + let error = self.report_clone_failure(&origin_url, error); + return Err(self.fail_init(init_start, error)); + } + }; + if let Err(error) = + clone_source::verify_exact_head(&head.stdout, expected_sha) + { + let error = self.report_clone_failure(&origin_url, error); + return Err(self.fail_init(init_start, error)); + } + } + let symlink_cmd = clone_source::repo_symlink_command(&layout); let symlink_result = process_svc .execute_command( @@ -1286,6 +1516,14 @@ impl Sandbox for DaytonaSandbox { } } } + Err(e) if commit_sha.is_some() => { + let err = Self::daytona_process_transport_error( + "Daytona SDK clone failed while preparing exact checkout", + &e, + ); + let err = self.report_clone_failure(&origin_url, err); + return Err(self.fail_init(init_start, err)); + } Err(e) if self.github_app.is_none() => { let err = crate::Error::context( "Git clone failed. If this is a private repository, \ @@ -2691,6 +2929,64 @@ mod tests { use super::*; use crate::sandbox::BASH_PROBE_MARKER; + #[test] + fn exact_checkout_omits_requested_branch_from_daytona_clone() { + let branch = Some("moving-branch".to_string()); + + assert_eq!(daytona_clone_branch(branch.as_deref(), false), branch); + assert_eq!(daytona_clone_branch(branch.as_deref(), true), None); + } + + #[tokio::test] + async fn invalid_exact_sha_fails_before_daytona_client_construction() { + let error = DaytonaSandbox::new( + DaytonaConfig::default(), + None, + None, + Some("https://github.com/acme/widgets".to_string()), + Some("main".to_string()), + Some("not-a-sha".to_string()), + Some("dtn_not_used".to_string()), + ) + .await + .err() + .expect("validation should run before building a Daytona client"); + + assert!(error.to_string().contains("40 ASCII hexadecimal")); + assert!(!error.to_string().contains("Daytona client")); + } + + #[test] + fn exact_checkout_process_failure_redacts_credentials_and_preserves_cause() { + let token = "ghs_daytona_exact_secret"; + let auth_url = fabro_github::embed_token_in_url("https://github.com/acme/widgets", token) + .expect("authenticated URL"); + let result = daytona_process_exec_result( + 128, + format!( + "fatal: unable to access {}: synthetic Daytona failure", + auth_url.as_raw_url() + ), + ); + let error = daytona_exact_exec_error( + result, + "git fetch exact commit in Daytona sandbox", + Some(&auth_url), + ); + + let causes = collect_chain(&error); + assert!( + causes + .iter() + .any(|cause| cause.contains("git fetch exact commit")), + "exec cause should remain structured: {causes:?}" + ); + let rendered = crate::display_for_log(&error); + assert!(!rendered.contains(token)); + assert!(!rendered.contains(auth_url.as_raw_url().as_str())); + assert!(rendered.contains("synthetic Daytona failure")); + } + fn api_key_body(permissions: &[&str]) -> serde_json::Value { serde_json::json!({ "name": "delete-only", @@ -2767,6 +3063,7 @@ mod tests { run_id: None, clone_origin_url: None, clone_branch: None, + clone_commit_sha: None, } } @@ -2941,6 +3238,7 @@ mod tests { None, None, None, + None, Some("dtn_test".to_string()), ) .await @@ -3086,6 +3384,7 @@ mod tests { Some(run_id), None, None, + None, Some("dtn_test".to_string()), ) .await diff --git a/lib/components/fabro-sandbox/src/docker.rs b/lib/components/fabro-sandbox/src/docker.rs index d69dcca4a..5c8c03d97 100644 --- a/lib/components/fabro-sandbox/src/docker.rs +++ b/lib/components/fabro-sandbox/src/docker.rs @@ -135,6 +135,7 @@ pub struct DockerSandbox { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, container_id: OnceCell, repo_cloned: OnceCell, working_directory: OnceCell, @@ -165,7 +166,16 @@ impl DockerSandbox { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, ) -> crate::Result { + if clone_commit_sha.is_some() { + clone_source::decide_clone( + config.skip_clone, + clone_origin_url.as_deref(), + clone_branch.as_deref(), + clone_commit_sha.as_deref(), + )?; + } let docker = Docker::connect_with_local_defaults().map_err(crate::Error::docker_connect)?; Ok(Self::with_docker_client( docker, @@ -174,6 +184,7 @@ impl DockerSandbox { run_id, clone_origin_url, clone_branch, + clone_commit_sha, )) } @@ -184,6 +195,7 @@ impl DockerSandbox { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, ) -> Self { Self { docker, @@ -192,6 +204,7 @@ impl DockerSandbox { run_id, clone_origin_url, clone_branch, + clone_commit_sha, container_id: OnceCell::new(), repo_cloned: OnceCell::new(), working_directory: OnceCell::new(), @@ -217,6 +230,7 @@ impl DockerSandbox { run_id, clone_origin_url.clone(), clone_branch, + None, )?; sandbox.validate_managed_container(container_id).await?; sandbox @@ -734,6 +748,217 @@ impl DockerSandbox { err } + fn exact_checkout_exec_error( + &self, + result: ExecResult, + label: &'static str, + auth_url: Option<&fabro_redact::DisplaySafeUrl>, + ) -> crate::Error { + let source = + result.into_exec_error_with_redactor(label, |output| redact_auth_url(output, auth_url)); + let message = if self.github_app.is_none() { + "Exact Git checkout failed. If this is a private repository, configure a GitHub App \ + with `fabro install` and install it for your organization." + } else { + "Failed to check out exact commit into Docker sandbox" + }; + crate::Error::context(message, source) + } + + async fn run_exact_checkout_command( + &self, + command: &str, + label: &'static str, + auth_url: Option<&fabro_redact::DisplaySafeUrl>, + ) -> crate::Result { + let result = self + .docker_exec_shell(command, 10_000, Some("/"), None, None) + .await + .map_err(|error| crate::Error::context(format!("{label} transport failed"), error))?; + if result.is_success() { + Ok(result) + } else { + Err(self.exact_checkout_exec_error(result, label, auth_url)) + } + } + + async fn checkout_exact_github_commit( + &self, + origin_url: String, + branch: Option, + expected_sha: String, + ) -> crate::Result<()> { + self.verify_git_available().await?; + let layout = clone_source::github_repo_layout(&origin_url, WORKING_DIRECTORY, REPOS_ROOT)?; + let token_was_freshly_minted = self + .github_app + .as_ref() + .is_some_and(GitHubCredentials::mints_installation_token); + + let auth_url = match &self.github_app { + Some(creds) => Some( + fabro_github::resolve_authenticated_url( + &fabro_github::GitHubContext::new(creds, &fabro_github::github_api_base_url()), + &origin_url, + ) + .await + .map_err(|error| crate::Error::Context { + message: "Failed to get GitHub App credentials for exact checkout".to_string(), + source: error.into_boxed_dyn_error(), + })?, + ), + None => None, + }; + let clone_url = auth_url + .as_ref() + .map_or(origin_url.as_str(), |url| url.as_raw_url().as_str()); + + self.emit(SandboxEvent::GitCloneStarted { + url: origin_url.clone(), + branch, + }); + let clone_start = Instant::now(); + + let prepare_command = format!( + "mkdir -p {} {}", + shell_quote(WORKING_DIRECTORY), + shell_quote(&layout.repos_owner_path), + ); + if let Err(error) = self + .run_exact_checkout_command( + &prepare_command, + "prepare Docker exact repository checkout", + auth_url.as_ref(), + ) + .await + { + return Err(self.report_clone_failure(&origin_url, error)); + } + + let init_command = + clone_source::exact_repository_init_command(clone_url, &layout.primary_repo_path); + if let Err(error) = self + .run_exact_checkout_command( + &init_command, + "initialize Docker exact repository checkout", + auth_url.as_ref(), + ) + .await + { + return Err(self.report_clone_failure(&origin_url, error)); + } + + let fetch_command = + clone_source::exact_fetch_command(&layout.primary_repo_path, "origin", &expected_sha); + let clone_deadline = time::Instant::now() + GIT_CLONE_TIMEOUT; + let fetch_result = clone_retry::retry_clone( + SandboxProviderKind::Docker, + Some(clone_deadline), + |_attempt| { + let command = fetch_command.as_str(); + let auth_url = auth_url.as_ref(); + async move { + let remaining = clone_deadline.saturating_duration_since(time::Instant::now()); + let timeout_ms = u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX); + if timeout_ms == 0 { + return Err(DockerCloneFailure { + error: crate::Error::message( + "Docker exact fetch deadline expired before retry", + ), + retry_reason: None, + }); + } + let result = self + .docker_exec_shell_streaming(ExecStreamingRequest { + timeout_ms: Some(timeout_ms), + working_dir: Some("/"), + ..ExecStreamingRequest::new(command) + }) + .await + .map_err(|error| DockerCloneFailure { + error: crate::Error::context( + "Docker exact fetch transport failed", + error, + ), + retry_reason: None, + })? + .result; + if result.is_success() { + return Ok(()); + } + let retry_reason = + classify_docker_clone_result(&result, token_was_freshly_minted); + Err(DockerCloneFailure { + error: self.exact_checkout_exec_error( + result, + "git fetch exact commit", + auth_url, + ), + retry_reason, + }) + } + }, + |failure: &DockerCloneFailure| failure.retry_reason, + ) + .await; + if let Err(failure) = fetch_result { + return Err(self.report_clone_failure(&origin_url, failure.error)); + } + + let checkout_command = clone_source::exact_checkout_command(&layout.primary_repo_path); + if let Err(error) = self + .run_exact_checkout_command( + &checkout_command, + "git checkout exact commit", + auth_url.as_ref(), + ) + .await + { + return Err(self.report_clone_failure(&origin_url, error)); + } + + let head_command = clone_source::head_revision_command(&layout.primary_repo_path); + let head = match self + .run_exact_checkout_command( + &head_command, + "verify Docker exact checkout HEAD", + auth_url.as_ref(), + ) + .await + { + Ok(result) => result, + Err(error) => return Err(self.report_clone_failure(&origin_url, error)), + }; + if let Err(error) = clone_source::verify_exact_head(&head.stdout, &expected_sha) { + return Err(self.report_clone_failure(&origin_url, error)); + } + + let symlink_command = clone_source::repo_symlink_command(&layout); + if let Err(error) = self + .run_exact_checkout_command( + &symlink_command, + "create Docker workspace repo symlink", + auth_url.as_ref(), + ) + .await + { + return Err(self.report_clone_failure(&origin_url, error)); + } + + let _ = self.repo_cloned.set(true); + let _ = self.origin_url.set(origin_url.clone()); + if let Err(error) = self.set_working_directory(layout.execution_directory) { + return Err(self.report_clone_failure(&origin_url, error)); + } + + let clone_duration = u64::try_from(clone_start.elapsed().as_millis()).unwrap_or(u64::MAX); + self.emit(SandboxEvent::GitCloneCompleted { + url: origin_url, + duration_ms: clone_duration, + }); + Ok(()) + } + async fn clone_github_repo( &self, origin_url: String, @@ -1607,6 +1832,7 @@ impl Sandbox for DockerSandbox { self.config.skip_clone, self.clone_origin_url.as_deref(), self.clone_branch.as_deref(), + self.clone_commit_sha.as_deref(), ) .map_err(|e| self.fail_init(init_start, e))?; @@ -1624,8 +1850,18 @@ impl Sandbox for DockerSandbox { } let _ = self.repo_cloned.set(false); } - CloneDecision::GitHub { origin_url, branch } => { - if let Err(e) = self.clone_github_repo(origin_url, branch).await { + CloneDecision::GitHub { + origin_url, + branch, + commit_sha, + } => { + let result = if let Some(commit_sha) = commit_sha { + self.checkout_exact_github_commit(origin_url, branch, commit_sha) + .await + } else { + self.clone_github_repo(origin_url, branch).await + }; + if let Err(e) = result { return Err(self.fail_init(init_start, e)); } } @@ -2420,6 +2656,72 @@ mod tests { ); } + #[test] + fn clone_command_without_branch_retains_legacy_shape() { + let command = git_clone_command( + "https://github.com/fabro-sh/fabro", + None, + "/repos/fabro-sh/fabro", + ); + assert_eq!( + command, + "git -c maintenance.auto=0 -c gc.auto=0 clone --depth 10 --no-tags -- https://github.com/fabro-sh/fabro /repos/fabro-sh/fabro" + ); + } + + #[test] + fn invalid_exact_sha_fails_before_docker_connection() { + let error = DockerSandbox::new( + DockerSandboxOptions::default(), + None, + None, + Some("https://github.com/acme/widgets".to_string()), + Some("main".to_string()), + Some("not-a-sha".to_string()), + ) + .err() + .expect("validation should run before connecting to Docker"); + + assert!(error.to_string().contains("40 ASCII hexadecimal")); + assert!(!error.to_string().contains("Docker daemon")); + } + + #[test] + fn exact_checkout_failure_preserves_safe_source_chain() { + let docker = Docker::connect_with_http("http://127.0.0.1:2375", 5, API_DEFAULT_VERSION) + .expect("mock Docker client should connect"); + let sandbox = test_docker_sandbox(docker, "test-container"); + let token = "ghs_exact_checkout_secret"; + let auth_url = fabro_github::embed_token_in_url("https://github.com/acme/widgets", token) + .expect("authenticated URL"); + let error = sandbox.exact_checkout_exec_error( + ExecResult { + stdout: String::new(), + stderr: format!( + "fatal: unable to access {}: synthetic low-level failure", + auth_url.as_raw_url() + ), + exit_code: Some(128), + termination: CommandTermination::Exited, + duration_ms: 1, + }, + "git fetch exact commit", + Some(&auth_url), + ); + + let causes = error.causes(); + assert!( + causes + .iter() + .any(|cause| cause.contains("git fetch exact commit failed")), + "source chain should retain the exec failure: {causes:?}" + ); + let rendered = crate::display_for_log(&error); + assert!(!rendered.contains(token)); + assert!(!rendered.contains(auth_url.as_raw_url().as_str())); + assert!(rendered.contains("synthetic low-level failure")); + } + #[test] fn clone_result_uses_stderr_before_stdout() { let result = ExecResult { @@ -2716,6 +3018,7 @@ mod tests { None, None, None, + None, ); sandbox .container_id diff --git a/lib/components/fabro-sandbox/src/provider/daytona.rs b/lib/components/fabro-sandbox/src/provider/daytona.rs index 7295afd97..246def6c0 100644 --- a/lib/components/fabro-sandbox/src/provider/daytona.rs +++ b/lib/components/fabro-sandbox/src/provider/daytona.rs @@ -130,6 +130,7 @@ impl SandboxProvider for DaytonaSandboxProvider { run_id, clone_origin_url, clone_branch, + None, Some(api_key), ) .await?; diff --git a/lib/components/fabro-sandbox/src/provider/docker.rs b/lib/components/fabro-sandbox/src/provider/docker.rs index f1f8bb725..75c9068c8 100644 --- a/lib/components/fabro-sandbox/src/provider/docker.rs +++ b/lib/components/fabro-sandbox/src/provider/docker.rs @@ -98,8 +98,14 @@ impl SandboxProvider for DockerSandboxProvider { )); }; - let sandbox = - DockerSandbox::new(config, github_app, run_id, clone_origin_url, clone_branch)?; + let sandbox = DockerSandbox::new( + config, + github_app, + run_id, + clone_origin_url, + clone_branch, + None, + )?; sandbox.initialize().await?; let container_id = sandbox.container_identifier()?.to_string(); self.get(&container_id).await?.ok_or_else(|| { diff --git a/lib/components/fabro-sandbox/src/sandbox_spec.rs b/lib/components/fabro-sandbox/src/sandbox_spec.rs index b6a56bd40..e96f18a93 100644 --- a/lib/components/fabro-sandbox/src/sandbox_spec.rs +++ b/lib/components/fabro-sandbox/src/sandbox_spec.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use std::sync::Arc; -#[cfg(feature = "docker")] +#[cfg(any(feature = "docker", feature = "daytona"))] use anyhow::Context as _; #[cfg(any(feature = "docker", feature = "daytona"))] use fabro_github::GitHubCredentials; @@ -32,6 +32,7 @@ pub enum SandboxSpec { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, }, #[cfg(feature = "daytona")] Daytona { @@ -40,6 +41,7 @@ pub enum SandboxSpec { run_id: Option, clone_origin_url: Option, clone_branch: Option, + clone_commit_sha: Option, api_key: Option, }, } @@ -202,13 +204,24 @@ impl SandboxSpec { run_id, clone_origin_url, clone_branch, + clone_commit_sha, } => { + if clone_commit_sha.is_some() { + clone_source::decide_clone( + config.skip_clone, + clone_origin_url.as_deref(), + clone_branch.as_deref(), + clone_commit_sha.as_deref(), + ) + .context("Invalid Docker exact-checkout request")?; + } let mut sandbox = DockerSandbox::new( config.clone(), github_app.clone(), *run_id, clone_origin_url.clone(), clone_branch.clone(), + clone_commit_sha.clone(), ) .context("Failed to create Docker sandbox")?; if let Some(callback) = event_callback { @@ -223,14 +236,26 @@ impl SandboxSpec { run_id, clone_origin_url, clone_branch, + clone_commit_sha, api_key, } => { + if clone_commit_sha.is_some() { + clone_source::decide_clone( + config.skip_clone, + clone_origin_url.as_deref(), + clone_branch.as_deref(), + clone_commit_sha.as_deref(), + ) + .map_err(anyhow::Error::new) + .context("Invalid Daytona exact-checkout request")?; + } let mut sandbox = DaytonaSandbox::new( config.as_ref().clone(), github_app.clone(), *run_id, clone_origin_url.clone(), clone_branch.clone(), + clone_commit_sha.clone(), api_key.clone(), ) .await @@ -276,6 +301,7 @@ mod tests { run_id: None, clone_origin_url: Some("git@github.com:brynary/rack-test.git".to_string()), clone_branch: Some("main".to_string()), + clone_commit_sha: None, }; let mut sandbox = MockSandbox::linux(); sandbox.working_dir = "/workspace/rack-test"; @@ -300,6 +326,34 @@ mod tests { runtime.primary_repo_link.as_deref(), Some("/workspace/rack-test") ); + let runtime_json = serde_json::to_value(&runtime).expect("runtime should serialize"); + assert!(runtime_json.get("clone_commit_sha").is_none()); + } + + #[cfg(feature = "docker")] + #[tokio::test] + async fn invalid_exact_checkout_spec_fails_before_provider_connection() { + let spec = SandboxSpec::Docker { + config: DockerSandboxOptions::default(), + github_app: None, + run_id: None, + clone_origin_url: Some("https://github.com/acme/widgets".to_string()), + clone_branch: Some("main".to_string()), + clone_commit_sha: Some("not-a-sha".to_string()), + }; + + let error = spec + .build(None) + .await + .err() + .expect("spec validation should run before Docker connection"); + assert!( + error + .to_string() + .contains("Invalid Docker exact-checkout request") + ); + assert!(format!("{error:#}").contains("40 ASCII hexadecimal")); + assert!(!format!("{error:#}").contains("Docker daemon")); } #[cfg(feature = "docker")] @@ -314,6 +368,7 @@ mod tests { run_id: None, clone_origin_url: Some("https://gitlab.com/acme/widgets".to_string()), clone_branch: None, + clone_commit_sha: None, }; let mut sandbox = MockSandbox::linux(); sandbox.working_dir = "/workspace"; diff --git a/lib/components/fabro-sandbox/tests/daytona_streaming_live.rs b/lib/components/fabro-sandbox/tests/daytona_streaming_live.rs index 85e146041..9fdaf8fb8 100644 --- a/lib/components/fabro-sandbox/tests/daytona_streaming_live.rs +++ b/lib/components/fabro-sandbox/tests/daytona_streaming_live.rs @@ -37,6 +37,7 @@ mod daytona_streaming_live { None, None, None, + None, ) .await?, ); @@ -74,6 +75,7 @@ mod daytona_streaming_live { None, None, None, + None, ) .await?; sandbox.initialize().await?; @@ -179,6 +181,7 @@ mod daytona_streaming_live { None, None, None, + None, ) .await?; @@ -228,6 +231,7 @@ mod daytona_streaming_live { Some("https://github.com/brynary/rack-test".to_string()), None, None, + None, ) .await?; @@ -295,6 +299,7 @@ mod daytona_streaming_live { None, None, None, + None, ) .await?; diff --git a/lib/components/fabro-sandbox/tests/docker_streaming.rs b/lib/components/fabro-sandbox/tests/docker_streaming.rs index 6274c638c..c04ccf1a0 100644 --- a/lib/components/fabro-sandbox/tests/docker_streaming.rs +++ b/lib/components/fabro-sandbox/tests/docker_streaming.rs @@ -40,6 +40,7 @@ async fn streaming_timeout_terminates_docker_exec_before_returning() { None, None, None, + None, ) .expect("docker sandbox should construct"); sandbox @@ -112,6 +113,7 @@ async fn streaming_command_receives_exact_stdin_and_eof() { None, None, None, + None, ) .expect("docker sandbox should construct"); sandbox @@ -173,6 +175,7 @@ async fn cloned_docker_sandbox_uses_repos_checkout_and_workspace_symlink() { None, Some("https://github.com/brynary/rack-test".to_string()), None, + None, ) .expect("docker sandbox should construct"); sandbox @@ -239,6 +242,7 @@ async fn docker_runs_clean_bash_through_both_command_paths() { None, None, None, + None, ) .expect("docker sandbox should construct"); sandbox @@ -332,6 +336,7 @@ async fn docker_glob_matches_patterns_containing_a_path_separator() { None, None, None, + None, ) .expect("docker sandbox should construct"); sandbox diff --git a/lib/components/fabro-workflow/src/operations/start.rs b/lib/components/fabro-workflow/src/operations/start.rs index adbaab3f9..dcb352693 100644 --- a/lib/components/fabro-workflow/src/operations/start.rs +++ b/lib/components/fabro-workflow/src/operations/start.rs @@ -430,6 +430,7 @@ impl RunSession { run_id: Some(record.run_id), clone_origin_url: record.repo_origin_url().map(str::to_string), clone_branch: record.base_branch().map(str::to_string), + clone_commit_sha: None, }, SandboxProviderKind::Daytona => { let api_key = vault_guard @@ -441,6 +442,7 @@ impl RunSession { run_id: Some(record.run_id), clone_origin_url: record.repo_origin_url().map(str::to_string), clone_branch: record.base_branch().map(str::to_string), + clone_commit_sha: None, api_key, } } diff --git a/lib/components/fabro-workflow/tests/it/daytona_integration.rs b/lib/components/fabro-workflow/tests/it/daytona_integration.rs index 021eecf11..2a1b6b5ae 100644 --- a/lib/components/fabro-workflow/tests/it/daytona_integration.rs +++ b/lib/components/fabro-workflow/tests/it/daytona_integration.rs @@ -175,9 +175,17 @@ fn test_artifact_store(run_dir: &Path) -> ArtifactStore { async fn create_env_with_github_app( github_app: Option, ) -> DaytonaSandbox { - DaytonaSandbox::new(DaytonaConfig::default(), github_app, None, None, None, None) - .await - .expect("Failed to create Daytona client — is DAYTONA_API_KEY set?") + DaytonaSandbox::new( + DaytonaConfig::default(), + github_app, + None, + None, + None, + None, + None, + ) + .await + .expect("Failed to create Daytona client — is DAYTONA_API_KEY set?") } fn load_github_app_credentials() -> fabro_github::GitHubCredentials { @@ -380,7 +388,7 @@ async fn daytona_snapshot_sandbox() { }; let creds = load_github_app_credentials(); - let env = DaytonaSandbox::new(config, Some(creds), None, None, None, None) + let env = DaytonaSandbox::new(config, Some(creds), None, None, None, None, None) .await .expect("Failed to create Daytona client — is DAYTONA_API_KEY set?"); env.initialize().await.unwrap(); @@ -1580,7 +1588,7 @@ async fn daytona_computer_use_browser_screenshot() { skip_clone: true, ..DaytonaConfig::default() }; - let env = DaytonaSandbox::new(config, None, None, None, None, None) + let env = DaytonaSandbox::new(config, None, None, None, None, None, None) .await .expect("DAYTONA_API_KEY must be set"); env.initialize().await.unwrap(); @@ -1728,7 +1736,7 @@ async fn daytona_playwright_mcp_sandbox_transport() { skip_clone: true, ..DaytonaConfig::default() }; - let sandbox = DaytonaSandbox::new(config, None, None, None, None, None) + let sandbox = DaytonaSandbox::new(config, None, None, None, None, None, None) .await .expect("DAYTONA_API_KEY must be set"); sandbox.initialize().await.unwrap(); diff --git a/lib/components/fabro-workflow/tests/it/integration.rs b/lib/components/fabro-workflow/tests/it/integration.rs index 0206d2c38..0dafedd3f 100644 --- a/lib/components/fabro-workflow/tests/it/integration.rs +++ b/lib/components/fabro-workflow/tests/it/integration.rs @@ -13327,7 +13327,7 @@ async fn asset_collection_docker_sandbox() { ..Default::default() }; let sandbox: Arc = Arc::new( - fabro_agent::DockerSandbox::new(config, None, None, None, None) + fabro_agent::DockerSandbox::new(config, None, None, None, None, None) .expect("Docker not available"), ); sandbox.initialize().await.expect("Docker init failed"); From 65616b455732c2e5321acab1b2a216ca19992e24 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Wed, 19 Aug 2026 15:10:44 -0400 Subject: [PATCH 02/30] Simplify exact-commit checkout across sandbox providers - Fold Docker's exact-checkout path into clone_github_repo so the auth, retry, symlink, and bookkeeping skeleton is shared with branch clones - Skip the Daytona SDK clone for exact checkouts: init and shallow-fetch the admitted commit directly instead of cloning the default branch and discarding it - Combine the detach checkout and HEAD verification into one shell command, saving an exec round trip per init - Drop the spec-level decide_clone pre-checks that duplicated the constructors' fail-fast validation - Share a CloneAttemptFailure struct in clone_retry and the GIT command prefix constant across git command builders Co-Authored-By: Claude Fable 5 --- .../fabro-sandbox/src/clone_retry.rs | 7 + .../fabro-sandbox/src/clone_source.rs | 40 +- .../fabro-sandbox/src/daytona/mod.rs | 572 ++++++++---------- lib/components/fabro-sandbox/src/docker.rs | 394 ++++-------- lib/components/fabro-sandbox/src/sandbox.rs | 2 +- .../fabro-sandbox/src/sandbox_spec.rs | 23 +- 6 files changed, 404 insertions(+), 634 deletions(-) diff --git a/lib/components/fabro-sandbox/src/clone_retry.rs b/lib/components/fabro-sandbox/src/clone_retry.rs index f0f3f0572..75dbb7fec 100644 --- a/lib/components/fabro-sandbox/src/clone_retry.rs +++ b/lib/components/fabro-sandbox/src/clone_retry.rs @@ -50,6 +50,13 @@ impl CloneMessageClass { } } +/// A failed clone or fetch attempt: the terminal error plus whether the +/// failure is worth retrying. +pub(crate) struct CloneAttemptFailure { + pub(crate) error: crate::Error, + pub(crate) retry_reason: Option, +} + /// Message fragments that mean the clone failed on infrastructure. /// /// These are safe to retry whether or not the clone was authenticated. diff --git a/lib/components/fabro-sandbox/src/clone_source.rs b/lib/components/fabro-sandbox/src/clone_source.rs index 53effa04a..15c89e137 100644 --- a/lib/components/fabro-sandbox/src/clone_source.rs +++ b/lib/components/fabro-sandbox/src/clone_source.rs @@ -74,10 +74,10 @@ pub(crate) fn repo_symlink_command(layout: &GitHubRepoLayout) -> String { pub(crate) fn exact_repository_init_command(clone_url: &str, checkout_path: &str) -> String { format!( - "git -c maintenance.auto=0 -c gc.auto=0 init -- {} && git -C {} remote add origin {}", - sandbox::shell_quote(checkout_path), - sandbox::shell_quote(checkout_path), - sandbox::shell_quote(clone_url), + "{git} init -- {path} && git -C {path} remote add origin {origin}", + git = sandbox::GIT, + path = sandbox::shell_quote(checkout_path), + origin = sandbox::shell_quote(clone_url), ) } @@ -87,24 +87,21 @@ pub(crate) fn exact_fetch_command( commit_sha: &str, ) -> String { format!( - "git -C {} -c maintenance.auto=0 -c gc.auto=0 fetch --depth 1 --no-tags {} -- {}", + "{git} -C {} fetch --depth 1 --no-tags {} -- {}", sandbox::shell_quote(checkout_path), sandbox::shell_quote(fetch_source), sandbox::shell_quote(commit_sha), + git = sandbox::GIT, ) } -pub(crate) fn exact_checkout_command(checkout_path: &str) -> String { +/// Detach onto the fetched commit and print the resulting HEAD in one shell +/// command; stdout is the `rev-parse HEAD` output for [`verify_exact_head`]. +pub(crate) fn exact_checkout_verify_command(checkout_path: &str) -> String { format!( - "git -C {} -c advice.detachedHead=false checkout --detach FETCH_HEAD", - sandbox::shell_quote(checkout_path), - ) -} - -pub(crate) fn head_revision_command(checkout_path: &str) -> String { - format!( - "git -C {} rev-parse HEAD", - sandbox::shell_quote(checkout_path), + "git -C {path} -c advice.detachedHead=false checkout --detach FETCH_HEAD && git -C {path} \ + rev-parse HEAD", + path = sandbox::shell_quote(checkout_path), ) } @@ -420,8 +417,7 @@ mod tests { "https://token@example.com/acme/widgets.git?x=a b", sha, ); - let checkout = exact_checkout_command("/repos/acme's widgets"); - let verify = head_revision_command("/repos/acme's widgets"); + let checkout = exact_checkout_verify_command("/repos/acme's widgets"); assert_eq!( init, @@ -429,14 +425,13 @@ mod tests { ); assert_eq!( fetch, - "git -C \"/repos/acme's widgets\" -c maintenance.auto=0 -c gc.auto=0 fetch --depth 1 --no-tags 'https://token@example.com/acme/widgets.git?x=a b' -- 0123456789abcdef0123456789abcdef01234567" + "git -c maintenance.auto=0 -c gc.auto=0 -C \"/repos/acme's widgets\" fetch --depth 1 --no-tags 'https://token@example.com/acme/widgets.git?x=a b' -- 0123456789abcdef0123456789abcdef01234567" ); assert_eq!( checkout, - "git -C \"/repos/acme's widgets\" -c advice.detachedHead=false checkout --detach FETCH_HEAD" + "git -C \"/repos/acme's widgets\" -c advice.detachedHead=false checkout --detach FETCH_HEAD && git -C \"/repos/acme's widgets\" rev-parse HEAD" ); - assert_eq!(verify, "git -C \"/repos/acme's widgets\" rev-parse HEAD"); - for command in [&init, &fetch, &checkout, &verify] { + for command in [&init, &fetch, &checkout] { assert!(!command.contains("moving-branch")); } } @@ -504,8 +499,7 @@ mod tests { temp.path(), &exact_fetch_command(checkout_path, remote_path, &admitted_sha), ); - run_shell(temp.path(), &exact_checkout_command(checkout_path)); - let checked_out_sha = run_shell(temp.path(), &head_revision_command(checkout_path)); + let checked_out_sha = run_shell(temp.path(), &exact_checkout_verify_command(checkout_path)); assert_eq!(checked_out_sha.trim(), admitted_sha); assert_eq!( diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index 617dc5c15..bbc4cfdef 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -69,19 +69,6 @@ const DAYTONA_START_TIMEOUT: Duration = Duration::from_mins(1); /// cancellation/timeout paths indefinitely. const DAYTONA_CLEANUP_TIMEOUT: Duration = Duration::from_secs(10); -struct DaytonaExactCheckoutFailure { - error: crate::Error, - retry_reason: Option, -} - -fn daytona_clone_branch(requested_branch: Option<&str>, exact_checkout: bool) -> Option { - if exact_checkout { - None - } else { - requested_branch.map(str::to_string) - } -} - fn daytona_process_exec_result(exit_code: i32, output: String) -> ExecResult { let (stdout, stderr) = if exit_code == 0 { (output, String::new()) @@ -97,14 +84,6 @@ fn daytona_process_exec_result(exit_code: i32, output: String) -> ExecResult { } } -fn daytona_exact_exec_error( - result: ExecResult, - label: &'static str, - auth_url: Option<&fabro_redact::DisplaySafeUrl>, -) -> crate::Error { - result.into_exec_error_with_redactor(label, |output| redact_auth_url(output, auth_url)) -} - /// Permissions a Daytona API key needs for Fabro's snapshot and sandbox flow. pub const REQUIRED_DAYTONA_PERMISSIONS: &[Permissions] = &[ Permissions::WriteColonSnapshots, @@ -591,10 +570,106 @@ impl DaytonaSandbox { if result.is_success() { Ok(result) } else { - Err(daytona_exact_exec_error(result, label, auth_url)) + Err(result + .into_exec_error_with_redactor(label, |output| redact_auth_url(output, auth_url))) } } + /// Materialize the exact admitted commit through the Daytona process + /// service: init an empty repository, shallow-fetch the commit with clone + /// retry semantics, detach onto it, and verify the resulting HEAD. + async fn checkout_exact_commit( + process_svc: &daytona_sdk::ProcessService, + origin_url: &str, + layout: &clone_source::GitHubRepoLayout, + password: Option<&str>, + expected_sha: &str, + token_was_freshly_minted: bool, + ) -> crate::Result<()> { + let auth_url = match password { + Some(token) => Some(fabro_github::embed_token_in_url(origin_url, token).map_err( + |error| crate::Error::Context { + message: + "Failed to build authenticated URL for Daytona exact checkout".to_string(), + source: error.into_boxed_dyn_error(), + }, + )?), + None => None, + }; + let clone_url = auth_url + .as_ref() + .map_or(origin_url, |url| url.as_raw_url().as_str()); + + let init_command = + clone_source::exact_repository_init_command(clone_url, &layout.primary_repo_path); + Self::run_exact_checkout_command( + process_svc, + &init_command, + "/", + "initialize Daytona exact repository checkout", + auth_url.as_ref(), + ) + .await?; + + let fetch_command = + clone_source::exact_fetch_command(&layout.primary_repo_path, "origin", expected_sha); + clone_retry::retry_clone( + SandboxProviderKind::Daytona, + None, + |_attempt| { + let command = fetch_command.as_str(); + let auth_url = auth_url.as_ref(); + async move { + let response = process_svc + .execute_command( + &wrap_bash_command(command), + daytona_sdk::ExecuteCommandOptions { + cwd: Some("/".to_string()), + ..Default::default() + }, + ) + .await + .map_err(|error| clone_retry::CloneAttemptFailure { + retry_reason: classify_clone_failure(&error, token_was_freshly_minted), + error: Self::daytona_process_transport_error( + "Daytona exact fetch transport failed", + &error, + ), + })?; + if response.exit_code == 0 { + return Ok(()); + } + let retry_reason = + clone_retry::classify_message(&response.result, token_was_freshly_minted) + .retry_reason(); + let result = daytona_process_exec_result(response.exit_code, response.result); + Err(clone_retry::CloneAttemptFailure { + retry_reason, + error: result.into_exec_error_with_redactor( + "git fetch exact commit in Daytona sandbox", + |output| redact_auth_url(output, auth_url), + ), + }) + } + }, + |failure: &clone_retry::CloneAttemptFailure| failure.retry_reason, + ) + .await + .map_err(|failure| failure.error)?; + + let checkout_command = + clone_source::exact_checkout_verify_command(&layout.primary_repo_path); + let head = Self::run_exact_checkout_command( + process_svc, + &checkout_command, + "/", + "git checkout exact commit in Daytona sandbox", + auth_url.as_ref(), + ) + .await?; + clone_source::verify_exact_head(&head.stdout, expected_sha) + } + fn fail_init(&self, init_start: Instant, err: crate::Error) -> crate::Error { let duration_ms = u64::try_from(init_start.elapsed().as_millis()).unwrap_or(u64::MAX); self.emit(SandboxEvent::InitializeFailed { @@ -1241,312 +1316,170 @@ impl Sandbox for DaytonaSandbox { self.fail_init(init_start, err) })?; - let git_svc = sandbox.git().await.map_err(|e| { - let err = crate::Error::context("Failed to get Daytona git service", e); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url.clone(), - error: err.to_string(), - causes: err.causes(), - }); + let process_svc = sandbox.process().await.map_err(|e| { + let err = crate::Error::context("Failed to get Daytona process service", e); + let err = self.report_clone_failure(&origin_url, err); self.fail_init(init_start, err) })?; - let clone_result = clone_retry::retry_clone( - SandboxProviderKind::Daytona, - None, - |_attempt| { - let git_svc = &git_svc; - let origin = origin_url.as_str(); - let target = layout.primary_repo_path.as_str(); - let options = daytona_sdk::GitCloneOptions { - branch: daytona_clone_branch(branch.as_deref(), commit_sha.is_some()), - username: username.clone(), - password: password.clone(), - ..Default::default() - }; - async move { git_svc.clone(origin, target, options).await } - }, - |err: &DaytonaError| classify_clone_failure(err, token_was_freshly_minted), - ) - .await; + if let Some(expected_sha) = commit_sha.as_deref() { + Self::checkout_exact_commit( + &process_svc, + &origin_url, + &layout, + password.as_deref(), + expected_sha, + token_was_freshly_minted, + ) + .await + .map_err(|err| { + let err = self.report_clone_failure(&origin_url, err); + self.fail_init(init_start, err) + })?; + } else { + let git_svc = sandbox.git().await.map_err(|e| { + let err = crate::Error::context("Failed to get Daytona git service", e); + let err = self.report_clone_failure(&origin_url, err); + self.fail_init(init_start, err) + })?; - match clone_result { - Ok(()) => { - let process_svc = sandbox.process().await.map_err(|e| { - let err = - crate::Error::context("Failed to get Daytona process service", e); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url.clone(), - error: err.to_string(), - causes: err.causes(), - }); - self.fail_init(init_start, err) - })?; - - if let Some(expected_sha) = commit_sha.as_deref() { - let auth_url = match password.as_deref() { - Some(token) => { - match fabro_github::embed_token_in_url(&origin_url, token) { - Ok(url) => Some(url), - Err(error) => { - let error = crate::Error::Context { - message: "Failed to build authenticated URL for \ - Daytona exact checkout" - .to_string(), - source: error.into_boxed_dyn_error(), - }; - let error = - self.report_clone_failure(&origin_url, error); - return Err(self.fail_init(init_start, error)); - } - } - } - None => None, + let clone_result = clone_retry::retry_clone( + SandboxProviderKind::Daytona, + None, + |_attempt| { + let git_svc = &git_svc; + let origin = origin_url.as_str(); + let target = layout.primary_repo_path.as_str(); + let options = daytona_sdk::GitCloneOptions { + branch: branch.clone(), + username: username.clone(), + password: password.clone(), + ..Default::default() }; - let fetch_source = auth_url - .as_ref() - .map_or(origin_url.as_str(), |url| url.as_raw_url().as_str()); - let fetch_command = clone_source::exact_fetch_command( - &layout.primary_repo_path, - fetch_source, - expected_sha, + async move { git_svc.clone(origin, target, options).await } + }, + |err: &DaytonaError| classify_clone_failure(err, token_was_freshly_minted), + ) + .await; + + match clone_result { + Ok(()) => {} + Err(e) if self.github_app.is_none() => { + let err = crate::Error::context( + "Git clone failed. If this is a private repository, \ + configure a GitHub App with `fabro install` and install it \ + for your organization.", + e, ); - let fetch_result = clone_retry::retry_clone( - SandboxProviderKind::Daytona, - None, - |_attempt| { - let command = fetch_command.as_str(); - let process_svc = &process_svc; - let auth_url = auth_url.as_ref(); - async move { - let response = process_svc - .execute_command( - &wrap_bash_command(command), - daytona_sdk::ExecuteCommandOptions { - cwd: Some("/".to_string()), - ..Default::default() - }, - ) - .await - .map_err(|error| DaytonaExactCheckoutFailure { - retry_reason: classify_clone_failure( - &error, - token_was_freshly_minted, - ), - error: Self::daytona_process_transport_error( - "Daytona exact fetch transport failed", - &error, - ), - })?; - if response.exit_code == 0 { - return Ok(()); - } - let retry_reason = clone_retry::classify_message( - &response.result, - token_was_freshly_minted, - ) - .retry_reason(); - let result = daytona_process_exec_result( - response.exit_code, - response.result, - ); - Err(DaytonaExactCheckoutFailure { - retry_reason, - error: daytona_exact_exec_error( - result, - "git fetch exact commit in Daytona sandbox", - auth_url, - ), - }) - } - }, - |failure: &DaytonaExactCheckoutFailure| failure.retry_reason, - ) - .await; - if let Err(failure) = fetch_result { - let error = self.report_clone_failure(&origin_url, failure.error); - return Err(self.fail_init(init_start, error)); - } - - let checkout_command = - clone_source::exact_checkout_command(&layout.primary_repo_path); - if let Err(error) = Self::run_exact_checkout_command( - &process_svc, - &checkout_command, - "/", - "git checkout exact commit in Daytona sandbox", - auth_url.as_ref(), - ) - .await - { - let error = self.report_clone_failure(&origin_url, error); - return Err(self.fail_init(init_start, error)); - } - - let head_command = - clone_source::head_revision_command(&layout.primary_repo_path); - let head = match Self::run_exact_checkout_command( - &process_svc, - &head_command, - "/", - "verify Daytona exact checkout HEAD", - auth_url.as_ref(), - ) - .await - { - Ok(result) => result, - Err(error) => { - let error = self.report_clone_failure(&origin_url, error); - return Err(self.fail_init(init_start, error)); - } - }; - if let Err(error) = - clone_source::verify_exact_head(&head.stdout, expected_sha) - { - let error = self.report_clone_failure(&origin_url, error); - return Err(self.fail_init(init_start, error)); - } - } - - let symlink_cmd = clone_source::repo_symlink_command(&layout); - let symlink_result = process_svc - .execute_command( - &wrap_bash_command(&symlink_cmd), - daytona_sdk::ExecuteCommandOptions { - cwd: Some("/".to_string()), - ..Default::default() - }, - ) - .await - .map_err(|e| { - let err = crate::Error::context( - "Failed to create Daytona workspace repo symlink", - e, - ); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url.clone(), - error: err.to_string(), - causes: err.causes(), - }); - self.fail_init(init_start, err) - })?; - if symlink_result.exit_code != 0 { - let err = crate::Error::exec( - "create Daytona workspace repo symlink", - ExecResult { - stdout: symlink_result.result.clone(), - stderr: String::new(), - exit_code: Some(symlink_result.exit_code), - termination: CommandTermination::Exited, - duration_ms: 0, - }, - ); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url.clone(), - error: err.to_string(), - causes: err.causes(), - }); + let err = self.report_clone_failure(&origin_url, err); return Err(self.fail_init(init_start, err)); } + Err(e) => { + let err = crate::Error::context( + "Failed to clone repo into Daytona sandbox", + e, + ); + let err = self.report_clone_failure(&origin_url, err); + return Err(self.fail_init(init_start, err)); + } + } + } - let clone_duration = - u64::try_from(clone_start.elapsed().as_millis()).unwrap_or(u64::MAX); - self.emit(SandboxEvent::GitCloneCompleted { - url: origin_url.clone(), - duration_ms: clone_duration, + let symlink_cmd = clone_source::repo_symlink_command(&layout); + let symlink_result = process_svc + .execute_command( + &wrap_bash_command(&symlink_cmd), + daytona_sdk::ExecuteCommandOptions { + cwd: Some("/".to_string()), + ..Default::default() + }, + ) + .await + .map_err(|e| { + let err = crate::Error::context( + "Failed to create Daytona workspace repo symlink", + e, + ); + let err = self.report_clone_failure(&origin_url, err); + self.fail_init(init_start, err) + })?; + if symlink_result.exit_code != 0 { + let err = + crate::Error::exec("create Daytona workspace repo symlink", ExecResult { + stdout: symlink_result.result.clone(), + stderr: String::new(), + exit_code: Some(symlink_result.exit_code), + termination: CommandTermination::Exited, + duration_ms: 0, }); + let err = self.report_clone_failure(&origin_url, err); + return Err(self.fail_init(init_start, err)); + } - let _ = self.repo_cloned.set(true); - let _ = self.origin_url.set(origin_url.clone()); - self.set_working_directory(layout.execution_directory.clone()) - .map_err(|err| self.fail_init(init_start, err))?; - if let Some(token) = password.as_deref() { - match fabro_github::embed_token_in_url(&origin_url, token) { - Ok(auth_url) => { - let cmd = format!( - "git -c maintenance.auto=0 remote set-url origin {}", - shell_quote(auth_url.as_raw_url().as_str()), + let clone_duration = + u64::try_from(clone_start.elapsed().as_millis()).unwrap_or(u64::MAX); + self.emit(SandboxEvent::GitCloneCompleted { + url: origin_url.clone(), + duration_ms: clone_duration, + }); + + let _ = self.repo_cloned.set(true); + let _ = self.origin_url.set(origin_url.clone()); + self.set_working_directory(layout.execution_directory.clone()) + .map_err(|err| self.fail_init(init_start, err))?; + if let Some(token) = password.as_deref() { + match fabro_github::embed_token_in_url(&origin_url, token) { + Ok(auth_url) => { + let cmd = format!( + "git -c maintenance.auto=0 remote set-url origin {}", + shell_quote(auth_url.as_raw_url().as_str()), + ); + let opts = daytona_sdk::ExecuteCommandOptions { + cwd: Some(layout.execution_directory.clone()), + ..Default::default() + }; + let wrapped = wrap_bash_command(&cmd); + match process_svc.execute_command(&wrapped, opts).await { + Ok(r) if r.exit_code != 0 => { + let err = crate::Error::exec( + "git remote set-url origin (Daytona post-clone)", + ExecResult { + stdout: String::new(), + stderr: redact_auth_url( + &r.result, + Some(&auth_url), + ), + exit_code: Some(r.exit_code), + termination: CommandTermination::Exited, + duration_ms: 0, + }, ); - let opts = daytona_sdk::ExecuteCommandOptions { - cwd: Some(layout.execution_directory.clone()), - ..Default::default() - }; - let wrapped = wrap_bash_command(&cmd); - match process_svc.execute_command(&wrapped, opts).await { - Ok(r) if r.exit_code != 0 => { - let err = crate::Error::exec( - "git remote set-url origin (Daytona post-clone)", - ExecResult { - stdout: String::new(), - stderr: redact_auth_url( - &r.result, - Some(&auth_url), - ), - exit_code: Some(r.exit_code), - termination: CommandTermination::Exited, - duration_ms: 0, - }, - ); - tracing::warn!( - error = %crate::display_for_log(&err), - "Failed to set Daytona sandbox push credentials \ - on origin — subsequent git push from this \ - sandbox will fail" - ); - } - Ok(_) => {} - Err(_) => { - tracing::warn!( - error_class = "daytona_set_url_exec_failed", - "Daytona exec failed while setting push credentials \ - on origin — subsequent git push from this \ - sandbox will fail" - ); - } - } - } - Err(e) => { tracing::warn!( - origin = %origin_url, - error = %e, - "Failed to build authenticated origin URL — \ - subsequent git push from this sandbox will fail" + error = %crate::display_for_log(&err), + "Failed to set Daytona sandbox push credentials \ + on origin — subsequent git push from this \ + sandbox will fail" + ); + } + Ok(_) => {} + Err(_) => { + tracing::warn!( + error_class = "daytona_set_url_exec_failed", + "Daytona exec failed while setting push credentials \ + on origin — subsequent git push from this \ + sandbox will fail" ); } } } - } - Err(e) if commit_sha.is_some() => { - let err = Self::daytona_process_transport_error( - "Daytona SDK clone failed while preparing exact checkout", - &e, - ); - let err = self.report_clone_failure(&origin_url, err); - return Err(self.fail_init(init_start, err)); - } - Err(e) if self.github_app.is_none() => { - let err = crate::Error::context( - "Git clone failed. If this is a private repository, \ - configure a GitHub App with `fabro install` and install it \ - for your organization.", - e, - ); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url, - error: err.to_string(), - causes: err.causes(), - }); - return Err(self.fail_init(init_start, err)); - } - Err(e) => { - let err = - crate::Error::context("Failed to clone repo into Daytona sandbox", e); - self.emit(SandboxEvent::GitCloneFailed { - url: origin_url, - error: err.to_string(), - causes: err.causes(), - }); - return Err(self.fail_init(init_start, err)); + Err(e) => { + tracing::warn!( + origin = %origin_url, + error = %e, + "Failed to build authenticated origin URL — \ + subsequent git push from this sandbox will fail" + ); + } } } } @@ -2929,14 +2862,6 @@ mod tests { use super::*; use crate::sandbox::BASH_PROBE_MARKER; - #[test] - fn exact_checkout_omits_requested_branch_from_daytona_clone() { - let branch = Some("moving-branch".to_string()); - - assert_eq!(daytona_clone_branch(branch.as_deref(), false), branch); - assert_eq!(daytona_clone_branch(branch.as_deref(), true), None); - } - #[tokio::test] async fn invalid_exact_sha_fails_before_daytona_client_construction() { let error = DaytonaSandbox::new( @@ -2968,11 +2893,10 @@ mod tests { auth_url.as_raw_url() ), ); - let error = daytona_exact_exec_error( - result, - "git fetch exact commit in Daytona sandbox", - Some(&auth_url), - ); + let error = result + .into_exec_error_with_redactor("git fetch exact commit in Daytona sandbox", |output| { + redact_auth_url(output, Some(&auth_url)) + }); let causes = collect_chain(&error); assert!( diff --git a/lib/components/fabro-sandbox/src/docker.rs b/lib/components/fabro-sandbox/src/docker.rs index 5c8c03d97..c1acdf1ad 100644 --- a/lib/components/fabro-sandbox/src/docker.rs +++ b/lib/components/fabro-sandbox/src/docker.rs @@ -56,11 +56,6 @@ const EXEC_TERM_GRACE_SECONDS: &str = "0.02"; #[cfg(not(test))] const EXEC_TERM_GRACE_SECONDS: &str = "0.2"; -struct DockerCloneFailure { - error: crate::Error, - retry_reason: Option, -} - fn env_entry_name(entry: &str) -> &str { entry.split_once('=').map_or(entry, |(name, _)| name) } @@ -722,14 +717,15 @@ impl DockerSandbox { Ok(()) } - /// Preserve a failed `git clone` result while masking the auth URL. + /// Preserve a failed git transfer result while masking the auth URL. fn clone_failure_error( &self, result: ExecResult, + label: &'static str, auth_url: Option<&fabro_redact::DisplaySafeUrl>, ) -> crate::Error { - let error = result - .into_exec_error_with_redactor("git clone", |output| redact_auth_url(output, auth_url)); + let error = + result.into_exec_error_with_redactor(label, |output| redact_auth_url(output, auth_url)); let message = if self.github_app.is_none() { "Git clone failed. If this is a private repository, configure a GitHub App with \ `fabro install` and install it for your organization." @@ -748,23 +744,6 @@ impl DockerSandbox { err } - fn exact_checkout_exec_error( - &self, - result: ExecResult, - label: &'static str, - auth_url: Option<&fabro_redact::DisplaySafeUrl>, - ) -> crate::Error { - let source = - result.into_exec_error_with_redactor(label, |output| redact_auth_url(output, auth_url)); - let message = if self.github_app.is_none() { - "Exact Git checkout failed. If this is a private repository, configure a GitHub App \ - with `fabro install` and install it for your organization." - } else { - "Failed to check out exact commit into Docker sandbox" - }; - crate::Error::context(message, source) - } - async fn run_exact_checkout_command( &self, command: &str, @@ -778,15 +757,69 @@ impl DockerSandbox { if result.is_success() { Ok(result) } else { - Err(self.exact_checkout_exec_error(result, label, auth_url)) + Err(self.clone_failure_error(result, label, auth_url)) } } - async fn checkout_exact_github_commit( + /// Run a network git command inside the container with clone retry + /// semantics under the shared clone deadline. + async fn retry_git_transfer( + &self, + command: &str, + label: &'static str, + exec_label: &'static str, + clone_deadline: time::Instant, + token_was_freshly_minted: bool, + auth_url: Option<&fabro_redact::DisplaySafeUrl>, + ) -> Result<(), clone_retry::CloneAttemptFailure> { + clone_retry::retry_clone( + SandboxProviderKind::Docker, + Some(clone_deadline), + |_attempt| async move { + let remaining = clone_deadline.saturating_duration_since(time::Instant::now()); + let timeout_ms = u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX); + if timeout_ms == 0 { + return Err(clone_retry::CloneAttemptFailure { + error: crate::Error::message(format!( + "{label} deadline expired before retry" + )), + retry_reason: None, + }); + } + let result = self + .docker_exec_shell_streaming(ExecStreamingRequest { + timeout_ms: Some(timeout_ms), + working_dir: Some("/"), + ..ExecStreamingRequest::new(command) + }) + .await + .map_err(|error| clone_retry::CloneAttemptFailure { + error: crate::Error::context( + format!("{label} transport failed"), + error, + ), + retry_reason: None, + })? + .result; + if result.is_success() { + return Ok(()); + } + let retry_reason = classify_docker_clone_result(&result, token_was_freshly_minted); + Err(clone_retry::CloneAttemptFailure { + error: self.clone_failure_error(result, exec_label, auth_url), + retry_reason, + }) + }, + |failure: &clone_retry::CloneAttemptFailure| failure.retry_reason, + ) + .await + } + + async fn clone_github_repo( &self, origin_url: String, branch: Option, - expected_sha: String, + commit_sha: Option, ) -> crate::Result<()> { self.verify_git_available().await?; let layout = clone_source::github_repo_layout(&origin_url, WORKING_DIRECTORY, REPOS_ROOT)?; @@ -803,7 +836,7 @@ impl DockerSandbox { ) .await .map_err(|error| crate::Error::Context { - message: "Failed to get GitHub App credentials for exact checkout".to_string(), + message: "Failed to get GitHub App credentials for clone".to_string(), source: error.into_boxed_dyn_error(), })?, ), @@ -813,183 +846,6 @@ impl DockerSandbox { .as_ref() .map_or(origin_url.as_str(), |url| url.as_raw_url().as_str()); - self.emit(SandboxEvent::GitCloneStarted { - url: origin_url.clone(), - branch, - }); - let clone_start = Instant::now(); - - let prepare_command = format!( - "mkdir -p {} {}", - shell_quote(WORKING_DIRECTORY), - shell_quote(&layout.repos_owner_path), - ); - if let Err(error) = self - .run_exact_checkout_command( - &prepare_command, - "prepare Docker exact repository checkout", - auth_url.as_ref(), - ) - .await - { - return Err(self.report_clone_failure(&origin_url, error)); - } - - let init_command = - clone_source::exact_repository_init_command(clone_url, &layout.primary_repo_path); - if let Err(error) = self - .run_exact_checkout_command( - &init_command, - "initialize Docker exact repository checkout", - auth_url.as_ref(), - ) - .await - { - return Err(self.report_clone_failure(&origin_url, error)); - } - - let fetch_command = - clone_source::exact_fetch_command(&layout.primary_repo_path, "origin", &expected_sha); - let clone_deadline = time::Instant::now() + GIT_CLONE_TIMEOUT; - let fetch_result = clone_retry::retry_clone( - SandboxProviderKind::Docker, - Some(clone_deadline), - |_attempt| { - let command = fetch_command.as_str(); - let auth_url = auth_url.as_ref(); - async move { - let remaining = clone_deadline.saturating_duration_since(time::Instant::now()); - let timeout_ms = u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX); - if timeout_ms == 0 { - return Err(DockerCloneFailure { - error: crate::Error::message( - "Docker exact fetch deadline expired before retry", - ), - retry_reason: None, - }); - } - let result = self - .docker_exec_shell_streaming(ExecStreamingRequest { - timeout_ms: Some(timeout_ms), - working_dir: Some("/"), - ..ExecStreamingRequest::new(command) - }) - .await - .map_err(|error| DockerCloneFailure { - error: crate::Error::context( - "Docker exact fetch transport failed", - error, - ), - retry_reason: None, - })? - .result; - if result.is_success() { - return Ok(()); - } - let retry_reason = - classify_docker_clone_result(&result, token_was_freshly_minted); - Err(DockerCloneFailure { - error: self.exact_checkout_exec_error( - result, - "git fetch exact commit", - auth_url, - ), - retry_reason, - }) - } - }, - |failure: &DockerCloneFailure| failure.retry_reason, - ) - .await; - if let Err(failure) = fetch_result { - return Err(self.report_clone_failure(&origin_url, failure.error)); - } - - let checkout_command = clone_source::exact_checkout_command(&layout.primary_repo_path); - if let Err(error) = self - .run_exact_checkout_command( - &checkout_command, - "git checkout exact commit", - auth_url.as_ref(), - ) - .await - { - return Err(self.report_clone_failure(&origin_url, error)); - } - - let head_command = clone_source::head_revision_command(&layout.primary_repo_path); - let head = match self - .run_exact_checkout_command( - &head_command, - "verify Docker exact checkout HEAD", - auth_url.as_ref(), - ) - .await - { - Ok(result) => result, - Err(error) => return Err(self.report_clone_failure(&origin_url, error)), - }; - if let Err(error) = clone_source::verify_exact_head(&head.stdout, &expected_sha) { - return Err(self.report_clone_failure(&origin_url, error)); - } - - let symlink_command = clone_source::repo_symlink_command(&layout); - if let Err(error) = self - .run_exact_checkout_command( - &symlink_command, - "create Docker workspace repo symlink", - auth_url.as_ref(), - ) - .await - { - return Err(self.report_clone_failure(&origin_url, error)); - } - - let _ = self.repo_cloned.set(true); - let _ = self.origin_url.set(origin_url.clone()); - if let Err(error) = self.set_working_directory(layout.execution_directory) { - return Err(self.report_clone_failure(&origin_url, error)); - } - - let clone_duration = u64::try_from(clone_start.elapsed().as_millis()).unwrap_or(u64::MAX); - self.emit(SandboxEvent::GitCloneCompleted { - url: origin_url, - duration_ms: clone_duration, - }); - Ok(()) - } - - async fn clone_github_repo( - &self, - origin_url: String, - branch: Option, - ) -> crate::Result<()> { - self.verify_git_available().await?; - let layout = clone_source::github_repo_layout(&origin_url, WORKING_DIRECTORY, REPOS_ROOT)?; - let token_was_freshly_minted = self - .github_app - .as_ref() - .is_some_and(GitHubCredentials::mints_installation_token); - - let auth_url = match &self.github_app { - Some(creds) => Some( - fabro_github::resolve_authenticated_url( - &fabro_github::GitHubContext::new(creds, &fabro_github::github_api_base_url()), - &origin_url, - ) - .await - .map_err(|e| { - crate::Error::message(format!( - "Failed to get GitHub App credentials for clone: {e}" - )) - })?, - ), - None => None, - }; - let clone_url = auth_url - .as_ref() - .map_or(origin_url.as_str(), |url| url.as_raw_url().as_str()); - self.emit(SandboxEvent::GitCloneStarted { url: origin_url.clone(), branch: branch.clone(), @@ -1015,58 +871,72 @@ impl DockerSandbox { } } - let command = git_clone_command(clone_url, branch.as_deref(), &layout.primary_repo_path); let clone_deadline = time::Instant::now() + GIT_CLONE_TIMEOUT; - let clone_result = clone_retry::retry_clone( - SandboxProviderKind::Docker, - Some(clone_deadline), - |_attempt| { - let command = command.as_str(); - let auth_url = auth_url.as_ref(); - async move { - let remaining = clone_deadline.saturating_duration_since(time::Instant::now()); - let timeout_ms = u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX); - if timeout_ms == 0 { - return Err(DockerCloneFailure { - error: crate::Error::message( - "Docker git clone deadline expired before retry", - ), - retry_reason: None, - }); - } - let result = self - .docker_exec_shell_streaming(ExecStreamingRequest { - timeout_ms: Some(timeout_ms), - working_dir: Some("/"), - ..ExecStreamingRequest::new(command) - }) - .await - .map_err(|error| DockerCloneFailure { - error: crate::Error::context( - "Docker git clone transport failed", - error, - ), - retry_reason: None, - })? - .result; - if result.is_success() { - return Ok(()); - } - let retry_reason = - classify_docker_clone_result(&result, token_was_freshly_minted); - Err(DockerCloneFailure { - error: self.clone_failure_error(result, auth_url), - retry_reason, - }) - } - }, - |failure: &DockerCloneFailure| failure.retry_reason, - ) - .await; + if let Some(expected_sha) = commit_sha.as_deref() { + let init_command = + clone_source::exact_repository_init_command(clone_url, &layout.primary_repo_path); + if let Err(error) = self + .run_exact_checkout_command( + &init_command, + "initialize Docker exact repository checkout", + auth_url.as_ref(), + ) + .await + { + return Err(self.report_clone_failure(&origin_url, error)); + } - if let Err(failure) = clone_result { - let err = failure.error; - return Err(self.report_clone_failure(&origin_url, err)); + let fetch_command = clone_source::exact_fetch_command( + &layout.primary_repo_path, + "origin", + expected_sha, + ); + if let Err(failure) = self + .retry_git_transfer( + &fetch_command, + "Docker exact fetch", + "git fetch exact commit", + clone_deadline, + token_was_freshly_minted, + auth_url.as_ref(), + ) + .await + { + return Err(self.report_clone_failure(&origin_url, failure.error)); + } + + let checkout_command = + clone_source::exact_checkout_verify_command(&layout.primary_repo_path); + let head = match self + .run_exact_checkout_command( + &checkout_command, + "git checkout exact commit", + auth_url.as_ref(), + ) + .await + { + Ok(result) => result, + Err(error) => return Err(self.report_clone_failure(&origin_url, error)), + }; + if let Err(error) = clone_source::verify_exact_head(&head.stdout, expected_sha) { + return Err(self.report_clone_failure(&origin_url, error)); + } + } else { + let command = + git_clone_command(clone_url, branch.as_deref(), &layout.primary_repo_path); + if let Err(failure) = self + .retry_git_transfer( + &command, + "Docker git clone", + "git clone", + clone_deadline, + token_was_freshly_minted, + auth_url.as_ref(), + ) + .await + { + return Err(self.report_clone_failure(&origin_url, failure.error)); + } } let symlink_command = clone_source::repo_symlink_command(&layout); @@ -1584,7 +1454,7 @@ async fn cache_docker_stdio_completion( } fn git_clone_command(clone_url: &str, branch: Option<&str>, checkout_path: &str) -> String { - let mut command = "git -c maintenance.auto=0 -c gc.auto=0 clone".to_string(); + let mut command = format!("{} clone", sandbox::GIT); if let Some(branch) = branch { command.push_str(" --branch "); command.push_str(&shell_quote(branch)); @@ -1855,13 +1725,7 @@ impl Sandbox for DockerSandbox { branch, commit_sha, } => { - let result = if let Some(commit_sha) = commit_sha { - self.checkout_exact_github_commit(origin_url, branch, commit_sha) - .await - } else { - self.clone_github_repo(origin_url, branch).await - }; - if let Err(e) = result { + if let Err(e) = self.clone_github_repo(origin_url, branch, commit_sha).await { return Err(self.fail_init(init_start, e)); } } @@ -2694,7 +2558,7 @@ mod tests { let token = "ghs_exact_checkout_secret"; let auth_url = fabro_github::embed_token_in_url("https://github.com/acme/widgets", token) .expect("authenticated URL"); - let error = sandbox.exact_checkout_exec_error( + let error = sandbox.clone_failure_error( ExecResult { stdout: String::new(), stderr: format!( diff --git a/lib/components/fabro-sandbox/src/sandbox.rs b/lib/components/fabro-sandbox/src/sandbox.rs index 6e6e2b336..94c9a6a54 100644 --- a/lib/components/fabro-sandbox/src/sandbox.rs +++ b/lib/components/fabro-sandbox/src/sandbox.rs @@ -18,7 +18,7 @@ use tokio::time; use tokio_util::sync::CancellationToken; /// Git command prefix that disables background maintenance. -const GIT: &str = "git -c maintenance.auto=0 -c gc.auto=0"; +pub(crate) const GIT: &str = "git -c maintenance.auto=0 -c gc.auto=0"; pub const DEFAULT_EXEC_OUTPUT_TAIL_BYTES: usize = 8 * 1024; diff --git a/lib/components/fabro-sandbox/src/sandbox_spec.rs b/lib/components/fabro-sandbox/src/sandbox_spec.rs index e96f18a93..9550ff949 100644 --- a/lib/components/fabro-sandbox/src/sandbox_spec.rs +++ b/lib/components/fabro-sandbox/src/sandbox_spec.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use std::sync::Arc; -#[cfg(any(feature = "docker", feature = "daytona"))] +#[cfg(feature = "docker")] use anyhow::Context as _; #[cfg(any(feature = "docker", feature = "daytona"))] use fabro_github::GitHubCredentials; @@ -206,15 +206,6 @@ impl SandboxSpec { clone_branch, clone_commit_sha, } => { - if clone_commit_sha.is_some() { - clone_source::decide_clone( - config.skip_clone, - clone_origin_url.as_deref(), - clone_branch.as_deref(), - clone_commit_sha.as_deref(), - ) - .context("Invalid Docker exact-checkout request")?; - } let mut sandbox = DockerSandbox::new( config.clone(), github_app.clone(), @@ -239,16 +230,6 @@ impl SandboxSpec { clone_commit_sha, api_key, } => { - if clone_commit_sha.is_some() { - clone_source::decide_clone( - config.skip_clone, - clone_origin_url.as_deref(), - clone_branch.as_deref(), - clone_commit_sha.as_deref(), - ) - .map_err(anyhow::Error::new) - .context("Invalid Daytona exact-checkout request")?; - } let mut sandbox = DaytonaSandbox::new( config.as_ref().clone(), github_app.clone(), @@ -350,7 +331,7 @@ mod tests { assert!( error .to_string() - .contains("Invalid Docker exact-checkout request") + .contains("Failed to create Docker sandbox") ); assert!(format!("{error:#}").contains("40 ASCII hexadecimal")); assert!(!format!("{error:#}").contains("Docker daemon")); From a35dacd46c4422f028e95fc7c04c74d510d0798e Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Wed, 19 Aug 2026 15:45:41 -0400 Subject: [PATCH 03/30] Import legacy blobs strictly into SQLite --- lib/components/fabro-store/src/blob_store.rs | 7 + .../fabro-store/src/legacy_blob_import.rs | 1376 +++++++++++++++++ lib/components/fabro-store/src/lib.rs | 2 + lib/components/fabro-store/src/slate/mod.rs | 2 +- 4 files changed, 1386 insertions(+), 1 deletion(-) create mode 100644 lib/components/fabro-store/src/legacy_blob_import.rs diff --git a/lib/components/fabro-store/src/blob_store.rs b/lib/components/fabro-store/src/blob_store.rs index 6d19ed978..54b8124d3 100644 --- a/lib/components/fabro-store/src/blob_store.rs +++ b/lib/components/fabro-store/src/blob_store.rs @@ -77,6 +77,13 @@ impl BlobStore { } } + pub(crate) fn sqlite_pool_for_legacy_import(&self) -> Option<&SqlitePool> { + match &self.backend { + BlobBackend::Slate(_) => None, + BlobBackend::Sqlite(pool) => Some(pool), + } + } + pub async fn write(&self, bytes: &[u8]) -> Result { match &self.backend { BlobBackend::Slate(repo) => { diff --git a/lib/components/fabro-store/src/legacy_blob_import.rs b/lib/components/fabro-store/src/legacy_blob_import.rs new file mode 100644 index 000000000..2e1341b14 --- /dev/null +++ b/lib/components/fabro-store/src/legacy_blob_import.rs @@ -0,0 +1,1376 @@ +//! Temporary compatibility importer for the legacy SlateDB blob keyspace. +//! +//! Remove this module with the Slate blob backend after the approved legacy +//! support window ends. + +use std::error::Error as StdError; +use std::fmt; + +use bytes::Bytes; +use fabro_types::BlobHash; +use sqlx::pool::PoolConnection; +use sqlx::{Acquire as _, Sqlite}; +use tracing::debug; + +use crate::keys::SlateKey; +use crate::{BlobStore, Database}; + +const MAX_BATCH_ROWS: usize = 100; +const MAX_BATCH_BYTES: u64 = 1024 * 1024; +const PASSIVE_CHECKPOINT_BYTES: u64 = 8 * 1024 * 1024; + +/// Aggregate progress from one legacy blob import attempt. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct LegacyBlobImportReport { + /// Source rows observed under the exact legacy blob prefix. + pub scanned_rows: u64, + /// Raw source value bytes observed under the exact legacy blob prefix. + pub scanned_bytes: u64, + /// Destination rows newly inserted by committed transactions. + pub imported_rows: u64, + /// Destination bytes newly inserted by committed transactions. + pub imported_bytes: u64, + /// Byte-equal destination rows accepted by committed transactions. + pub existing_rows: u64, + /// Bytes belonging to byte-equal destination rows. + pub existing_bytes: u64, + /// Source rows rejected for malformed keys or digest mismatches. + pub invalid_rows: u64, + /// Destination rows rejected because their bytes differed. + pub conflicting_rows: u64, + /// Successfully committed import transactions. + pub committed_batches: u64, + /// Successful passive WAL checkpoints during the import. + pub passive_checkpoints: u64, +} + +/// A failed legacy blob import and the durable progress completed before it. +pub struct LegacyBlobImportError { + report: LegacyBlobImportReport, + failure: LegacyBlobImportFailure, +} + +impl LegacyBlobImportError { + /// Returns the aggregate durable progress completed before the failure. + #[must_use] + pub fn report(&self) -> &LegacyBlobImportReport { + &self.report + } +} + +impl fmt::Debug for LegacyBlobImportError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("LegacyBlobImportError") + .field("report", &self.report) + .field("failure", &self.failure.kind()) + .finish() + } +} + +impl fmt::Display for LegacyBlobImportError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "legacy blob import failed after scanning {} rows and committing {} batches: {}", + self.report.scanned_rows, self.report.committed_batches, self.failure + ) + } +} + +impl StdError for LegacyBlobImportError { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + Some(&self.failure) + } +} + +#[derive(thiserror::Error)] +enum LegacyBlobImportFailure { + #[error("the legacy blob import target is not backed by SQLite")] + WrongTargetBackend, + + #[error("opening the legacy blob source")] + OpenSource(#[source] crate::Error), + + #[error("opening the legacy blob scan")] + OpenSourceScan(#[source] slatedb::Error), + + #[error("reading the legacy blob scan")] + ReadSourceScan(#[source] slatedb::Error), + + #[error("a legacy blob key is not canonical")] + InvalidSourceKey, + + #[error("legacy blob bytes do not match their key digest")] + SourceDigestMismatch, + + #[error("a legacy blob import counter overflowed")] + CounterOverflow, + + #[error("acquiring the SQLite import connection")] + AcquireConnection(#[source] sqlx::Error), + + #[error("reading the SQLite automatic checkpoint setting")] + ReadAutomaticCheckpoint(#[source] sqlx::Error), + + #[error("disabling SQLite automatic checkpointing")] + DisableAutomaticCheckpoint(#[source] sqlx::Error), + + #[error("starting a SQLite blob import transaction")] + BeginTransaction(#[source] sqlx::Error), + + #[error("inserting a SQLite blob row")] + InsertDestination(#[source] sqlx::Error), + + #[error("reading an existing SQLite blob row")] + ReadDestination(#[source] sqlx::Error), + + #[error("SQLite contains different bytes for a legacy blob hash")] + DestinationConflict, + + #[error("committing a SQLite blob import transaction")] + CommitTransaction(#[source] sqlx::Error), + + #[error("rolling back a failed SQLite blob import transaction")] + RollbackTransaction { + #[source] + source: sqlx::Error, + prior: Box, + }, + + #[error("running a passive SQLite WAL checkpoint")] + PassiveCheckpoint(#[source] sqlx::Error), + + #[error("the passive SQLite WAL checkpoint could not complete")] + PassiveCheckpointBusy, + + #[error("running the final SQLite WAL checkpoint")] + FinalCheckpoint(#[source] sqlx::Error), + + #[error("the final SQLite WAL checkpoint could not complete")] + FinalCheckpointBusy, + + #[error("restoring the SQLite automatic checkpoint setting")] + RestoreAutomaticCheckpoint { + #[source] + source: sqlx::Error, + prior: Option>, + retirement_error: Option, + }, +} + +impl LegacyBlobImportFailure { + fn kind(&self) -> &'static str { + match self { + Self::WrongTargetBackend => "wrong_target_backend", + Self::OpenSource(_) => "open_source", + Self::OpenSourceScan(_) => "open_source_scan", + Self::ReadSourceScan(_) => "read_source_scan", + Self::InvalidSourceKey => "invalid_source_key", + Self::SourceDigestMismatch => "source_digest_mismatch", + Self::CounterOverflow => "counter_overflow", + Self::AcquireConnection(_) => "acquire_connection", + Self::ReadAutomaticCheckpoint(_) => "read_automatic_checkpoint", + Self::DisableAutomaticCheckpoint(_) => "disable_automatic_checkpoint", + Self::BeginTransaction(_) => "begin_transaction", + Self::InsertDestination(_) => "insert_destination", + Self::ReadDestination(_) => "read_destination", + Self::DestinationConflict => "destination_conflict", + Self::CommitTransaction(_) => "commit_transaction", + Self::RollbackTransaction { .. } => "rollback_transaction", + Self::PassiveCheckpoint(_) => "passive_checkpoint", + Self::PassiveCheckpointBusy => "passive_checkpoint_busy", + Self::FinalCheckpoint(_) => "final_checkpoint", + Self::FinalCheckpointBusy => "final_checkpoint_busy", + Self::RestoreAutomaticCheckpoint { .. } => "restore_automatic_checkpoint", + } + } +} + +impl fmt::Debug for LegacyBlobImportFailure { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let mut debug = formatter.debug_struct("LegacyBlobImportFailure"); + debug.field("kind", &self.kind()); + match self { + Self::RollbackTransaction { prior, .. } => { + debug.field("prior_failure", &prior.kind()); + } + Self::RestoreAutomaticCheckpoint { + prior, + retirement_error, + .. + } => { + debug + .field("prior_failure", &prior.as_deref().map(Self::kind)) + .field("retirement_failed", &retirement_error.is_some()); + } + _ => {} + } + debug.finish() + } +} + +#[derive(Default)] +struct ImportControls { + #[cfg(test)] + source_after_rows: Option, + #[cfg(test)] + passive_checkpoint: bool, + #[cfg(test)] + final_checkpoint: bool, + #[cfg(test)] + restore_automatic_checkpoint: bool, +} + +impl ImportControls { + fn source_scan_error(&self, scanned_rows: u64) -> Option { + #[cfg(test)] + if self.source_after_rows == Some(scanned_rows) { + return Some(slatedb::Error::unavailable( + "injected legacy source transport failure".to_owned(), + )); + } + let _ = (self, scanned_rows); + None + } + + fn passive_checkpoint_error(&self) -> Option { + #[cfg(test)] + if self.passive_checkpoint { + return Some(sqlx::Error::Protocol( + "injected passive checkpoint failure".to_owned(), + )); + } + let _ = self; + None + } + + fn final_checkpoint_error(&self) -> Option { + #[cfg(test)] + if self.final_checkpoint { + return Some(sqlx::Error::Protocol( + "injected final checkpoint failure".to_owned(), + )); + } + let _ = self; + None + } + + fn restore_automatic_checkpoint_error(&self) -> Option { + #[cfg(test)] + if self.restore_automatic_checkpoint { + return Some(sqlx::Error::Protocol( + "injected automatic checkpoint restoration failure".to_owned(), + )); + } + let _ = self; + None + } +} + +struct PendingBlob { + hash: BlobHash, + bytes: Bytes, +} + +#[derive(Clone, Copy, Default)] +struct BatchReport { + imported_rows: u64, + imported_bytes: u64, + existing_rows: u64, + existing_bytes: u64, +} + +impl Database { + /// Strictly imports the legacy SlateDB blob keyspace into a SQLite blob + /// store. + /// + /// # Errors + /// + /// Returns a typed error with partial durable progress if source + /// validation, destination persistence, checkpointing, or connection + /// cleanup fails. + pub async fn import_legacy_blobs_into( + &self, + target: &BlobStore, + ) -> std::result::Result { + self.import_legacy_blobs_with_controls(target, &ImportControls::default()) + .await + } + + async fn import_legacy_blobs_with_controls( + &self, + target: &BlobStore, + controls: &ImportControls, + ) -> std::result::Result { + let mut report = LegacyBlobImportReport::default(); + let result = self + .run_legacy_blob_import(target, controls, &mut report) + .await; + + match result { + Ok(()) => { + debug_import_outcome("complete", &report, None); + Ok(report) + } + Err(failure) => { + debug_import_outcome("failed", &report, Some(failure.kind())); + Err(LegacyBlobImportError { report, failure }) + } + } + } + + async fn run_legacy_blob_import( + &self, + target: &BlobStore, + controls: &ImportControls, + report: &mut LegacyBlobImportReport, + ) -> Result<(), LegacyBlobImportFailure> { + let pool = target + .sqlite_pool_for_legacy_import() + .ok_or(LegacyBlobImportFailure::WrongTargetBackend)?; + let mut connection = pool + .acquire() + .await + .map_err(LegacyBlobImportFailure::AcquireConnection)?; + let previous_automatic_checkpoint = sqlx::query_scalar("PRAGMA wal_autocheckpoint") + .fetch_one(&mut *connection) + .await + .map_err(LegacyBlobImportFailure::ReadAutomaticCheckpoint)?; + + let import_result = match set_automatic_checkpoint(&mut connection, 0).await { + Ok(()) => { + self.copy_legacy_blobs(&mut connection, controls, report) + .await + } + Err(source) => Err(LegacyBlobImportFailure::DisableAutomaticCheckpoint(source)), + }; + + let restore_result = if let Some(error) = controls.restore_automatic_checkpoint_error() { + Err(error) + } else { + set_automatic_checkpoint(&mut connection, previous_automatic_checkpoint).await + }; + + if let Err(source) = restore_result { + let retirement_error = connection.close().await.err(); + return Err(LegacyBlobImportFailure::RestoreAutomaticCheckpoint { + source, + prior: import_result.err().map(Box::new), + retirement_error, + }); + } + + import_result + } + + async fn copy_legacy_blobs( + &self, + connection: &mut PoolConnection, + controls: &ImportControls, + report: &mut LegacyBlobImportReport, + ) -> Result<(), LegacyBlobImportFailure> { + let source = self + .open_db() + .await + .map_err(LegacyBlobImportFailure::OpenSource)?; + let prefix = SlateKey::new("blobs").with("sha256").into_prefix(); + let prefix_bytes = prefix.as_ref().to_vec(); + let mut entries = source + .scan_prefix(&prefix_bytes) + .await + .map_err(LegacyBlobImportFailure::OpenSourceScan)?; + let mut pending = Vec::with_capacity(MAX_BATCH_ROWS); + let mut pending_bytes = 0_u64; + let mut bytes_since_checkpoint = 0_u64; + + loop { + if let Some(source) = controls.source_scan_error(report.scanned_rows) { + return Err(LegacyBlobImportFailure::ReadSourceScan(source)); + } + let Some(entry) = entries + .next() + .await + .map_err(LegacyBlobImportFailure::ReadSourceScan)? + else { + break; + }; + checked_add(&mut report.scanned_rows, 1)?; + let value_bytes = usize_to_u64(entry.value.len())?; + checked_add(&mut report.scanned_bytes, value_bytes)?; + + let hash = validate_source_entry(&entry.key, &entry.value, &prefix_bytes, report)?; + + let would_exceed_rows = pending.len() == MAX_BATCH_ROWS; + let would_exceed_bytes = pending_bytes + .checked_add(value_bytes) + .ok_or(LegacyBlobImportFailure::CounterOverflow)? + > MAX_BATCH_BYTES; + if !pending.is_empty() && (would_exceed_rows || would_exceed_bytes) { + commit_pending_batch( + connection, + &mut pending, + &mut pending_bytes, + &mut bytes_since_checkpoint, + controls, + report, + ) + .await?; + } + + pending.push(PendingBlob { + hash, + bytes: entry.value, + }); + checked_add(&mut pending_bytes, value_bytes)?; + + if value_bytes > MAX_BATCH_BYTES { + commit_pending_batch( + connection, + &mut pending, + &mut pending_bytes, + &mut bytes_since_checkpoint, + controls, + report, + ) + .await?; + } + } + + commit_pending_batch( + connection, + &mut pending, + &mut pending_bytes, + &mut bytes_since_checkpoint, + controls, + report, + ) + .await?; + run_checkpoint(connection, CheckpointKind::Final, controls).await + } +} + +fn validate_source_entry( + key: &[u8], + value: &[u8], + prefix: &[u8], + report: &mut LegacyBlobImportReport, +) -> Result { + let Some(suffix) = key.strip_prefix(prefix) else { + return invalid_source_row(report, LegacyBlobImportFailure::InvalidSourceKey); + }; + let canonical = suffix.len() == 64 + && suffix + .iter() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(byte)); + if !canonical { + return invalid_source_row(report, LegacyBlobImportFailure::InvalidSourceKey); + } + + let hash = std::str::from_utf8(suffix) + .ok() + .and_then(|value| value.parse().ok()) + .ok_or(LegacyBlobImportFailure::InvalidSourceKey); + let hash = match hash { + Ok(hash) => hash, + Err(failure) => return invalid_source_row(report, failure), + }; + if BlobHash::new(value) != hash { + return invalid_source_row(report, LegacyBlobImportFailure::SourceDigestMismatch); + } + Ok(hash) +} + +fn invalid_source_row( + report: &mut LegacyBlobImportReport, + failure: LegacyBlobImportFailure, +) -> Result { + checked_add(&mut report.invalid_rows, 1)?; + Err(failure) +} + +async fn commit_pending_batch( + connection: &mut PoolConnection, + pending: &mut Vec, + pending_bytes: &mut u64, + bytes_since_checkpoint: &mut u64, + controls: &ImportControls, + report: &mut LegacyBlobImportReport, +) -> Result<(), LegacyBlobImportFailure> { + if pending.is_empty() { + return Ok(()); + } + + let batch = std::mem::take(pending); + *pending_bytes = 0; + let batch_report = commit_batch(connection, batch, report).await?; + + let mut updated = *report; + checked_add(&mut updated.imported_rows, batch_report.imported_rows)?; + checked_add(&mut updated.imported_bytes, batch_report.imported_bytes)?; + checked_add(&mut updated.existing_rows, batch_report.existing_rows)?; + checked_add(&mut updated.existing_bytes, batch_report.existing_bytes)?; + checked_add(&mut updated.committed_batches, 1)?; + *report = updated; + + checked_add(bytes_since_checkpoint, batch_report.imported_bytes)?; + if *bytes_since_checkpoint >= PASSIVE_CHECKPOINT_BYTES { + run_checkpoint(connection, CheckpointKind::Passive, controls).await?; + checked_add(&mut report.passive_checkpoints, 1)?; + *bytes_since_checkpoint = 0; + } + + Ok(()) +} + +async fn commit_batch( + connection: &mut PoolConnection, + batch: Vec, + report: &mut LegacyBlobImportReport, +) -> Result { + let mut transaction = connection + .begin() + .await + .map_err(LegacyBlobImportFailure::BeginTransaction)?; + let batch_result = async { + let mut batch_report = BatchReport::default(); + for blob in batch { + let value_bytes = usize_to_u64(blob.bytes.len())?; + let result = sqlx::query( + "INSERT INTO blobs (hash, data) VALUES (?, ?) ON CONFLICT(hash) DO NOTHING", + ) + .bind(blob.hash.to_string()) + .bind(blob.bytes.as_ref()) + .execute(&mut *transaction) + .await + .map_err(LegacyBlobImportFailure::InsertDestination)?; + + if result.rows_affected() == 1 { + checked_add(&mut batch_report.imported_rows, 1)?; + checked_add(&mut batch_report.imported_bytes, value_bytes)?; + continue; + } + + let stored: Vec = sqlx::query_scalar("SELECT data FROM blobs WHERE hash = ?") + .bind(blob.hash.to_string()) + .fetch_one(&mut *transaction) + .await + .map_err(LegacyBlobImportFailure::ReadDestination)?; + if stored != blob.bytes { + checked_add(&mut report.conflicting_rows, 1)?; + return Err(LegacyBlobImportFailure::DestinationConflict); + } + checked_add(&mut batch_report.existing_rows, 1)?; + checked_add(&mut batch_report.existing_bytes, value_bytes)?; + } + Ok(batch_report) + } + .await; + + match batch_result { + Ok(batch_report) => { + transaction + .commit() + .await + .map_err(LegacyBlobImportFailure::CommitTransaction)?; + Ok(batch_report) + } + Err(prior) => match transaction.rollback().await { + Ok(()) => Err(prior), + Err(source) => Err(LegacyBlobImportFailure::RollbackTransaction { + source, + prior: Box::new(prior), + }), + }, + } +} + +#[derive(Clone, Copy)] +enum CheckpointKind { + Passive, + Final, +} + +async fn run_checkpoint( + connection: &mut PoolConnection, + kind: CheckpointKind, + controls: &ImportControls, +) -> Result<(), LegacyBlobImportFailure> { + let (statement, injected_error) = match kind { + CheckpointKind::Passive => ( + "PRAGMA wal_checkpoint(PASSIVE)", + controls.passive_checkpoint_error(), + ), + CheckpointKind::Final => ( + "PRAGMA wal_checkpoint(TRUNCATE)", + controls.final_checkpoint_error(), + ), + }; + if let Some(source) = injected_error { + return Err(match kind { + CheckpointKind::Passive => LegacyBlobImportFailure::PassiveCheckpoint(source), + CheckpointKind::Final => LegacyBlobImportFailure::FinalCheckpoint(source), + }); + } + + let result = sqlx::query_as::<_, (i64, i64, i64)>(statement) + .fetch_one(&mut **connection) + .await; + let (busy, _, _) = result.map_err(|source| match kind { + CheckpointKind::Passive => LegacyBlobImportFailure::PassiveCheckpoint(source), + CheckpointKind::Final => LegacyBlobImportFailure::FinalCheckpoint(source), + })?; + if busy != 0 { + return Err(match kind { + CheckpointKind::Passive => LegacyBlobImportFailure::PassiveCheckpointBusy, + CheckpointKind::Final => LegacyBlobImportFailure::FinalCheckpointBusy, + }); + } + Ok(()) +} + +async fn set_automatic_checkpoint( + connection: &mut PoolConnection, + pages: i64, +) -> Result<(), sqlx::Error> { + // `pages` comes directly from SQLite as an integer, so the dynamic PRAGMA + // contains no caller-controlled text and cannot change the SQL shape. + let statement = sqlx::AssertSqlSafe(format!("PRAGMA wal_autocheckpoint = {pages}")); + sqlx::query(statement).execute(&mut **connection).await?; + Ok(()) +} + +fn usize_to_u64(value: usize) -> Result { + u64::try_from(value).map_err(|_| LegacyBlobImportFailure::CounterOverflow) +} + +fn checked_add(value: &mut u64, amount: u64) -> Result<(), LegacyBlobImportFailure> { + *value = value + .checked_add(amount) + .ok_or(LegacyBlobImportFailure::CounterOverflow)?; + Ok(()) +} + +fn debug_import_outcome( + outcome: &'static str, + report: &LegacyBlobImportReport, + failure_kind: Option<&'static str>, +) { + debug!( + outcome, + failure_kind, + scanned_rows = report.scanned_rows, + scanned_bytes = report.scanned_bytes, + imported_rows = report.imported_rows, + imported_bytes = report.imported_bytes, + existing_rows = report.existing_rows, + existing_bytes = report.existing_bytes, + invalid_rows = report.invalid_rows, + conflicting_rows = report.conflicting_rows, + committed_batches = report.committed_batches, + passive_checkpoints = report.passive_checkpoints, + "Legacy blob import finished" + ); +} + +#[cfg(test)] +mod tests { + use std::fmt::{self, Write as _}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::sync::{Arc, Mutex}; + use std::time::Duration; + + use bytes::Bytes; + use fabro_types::BlobHash; + use object_store::memory::InMemory; + use tracing::field::{Field, Visit}; + use tracing::instrument::WithSubscriber as _; + use tracing::span::{Attributes, Id, Record}; + use tracing::{Event, Metadata, Subscriber}; + + use super::{ + ImportControls, LegacyBlobImportFailure, LegacyBlobImportReport, MAX_BATCH_BYTES, + PASSIVE_CHECKPOINT_BYTES, + }; + use crate::keys::SlateKey; + use crate::{BlobStore, Database}; + + type TestResult = std::result::Result>; + + struct TestContext { + _dir: tempfile::TempDir, + source: Database, + source_db: slatedb::Db, + sqlite: fabro_db::Database, + target: BlobStore, + } + + impl TestContext { + async fn new() -> TestResult { + let source = Database::new( + Arc::new(InMemory::new()), + "legacy-blob-import-tests", + Duration::from_millis(1), + None, + ); + let source_db = source.open_db().await?; + let dir = tempfile::tempdir()?; + let sqlite = fabro_db::Database::connect(dir.path().join("fabro.sqlite3")).await?; + sqlite.migrate().await?; + let target = BlobStore::new(sqlite.clone_pool()); + Ok(Self { + _dir: dir, + source, + source_db, + sqlite, + target, + }) + } + + async fn put_blob(&self, bytes: &[u8]) -> TestResult { + let hash = BlobHash::new(bytes); + let key = SlateKey::new("blobs").with("sha256").with(hash); + self.source_db.put(key, bytes).await?; + Ok(hash) + } + + async fn put_raw(&self, key: Vec, bytes: &[u8]) -> TestResult<()> { + self.source_db.put(key, bytes).await?; + Ok(()) + } + + async fn source_entries(&self) -> TestResult, Vec)>> { + let mut entries = self.source_db.scan_prefix(Vec::::new()).await?; + let mut snapshot = Vec::new(); + while let Some(entry) = entries.next().await? { + snapshot.push((entry.key.to_vec(), entry.value.to_vec())); + } + Ok(snapshot) + } + + async fn destination_rows(&self) -> TestResult { + Ok(sqlx::query_scalar("SELECT COUNT(*) FROM blobs") + .fetch_one(self.sqlite.pool()) + .await?) + } + + async fn insert_destination(&self, hash: BlobHash, bytes: &[u8]) -> TestResult<()> { + sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)") + .bind(hash.to_string()) + .bind(bytes) + .execute(self.sqlite.pool()) + .await?; + Ok(()) + } + + async fn delete_destination(&self, hash: BlobHash) -> TestResult<()> { + sqlx::query("DELETE FROM blobs WHERE hash = ?") + .bind(hash.to_string()) + .execute(self.sqlite.pool()) + .await?; + Ok(()) + } + + async fn set_automatic_checkpoint(&self, pages: i64) -> TestResult<()> { + let mut connection = self.sqlite.pool().acquire().await?; + let statement = sqlx::AssertSqlSafe(format!("PRAGMA wal_autocheckpoint = {pages}")); + sqlx::query(statement).execute(&mut *connection).await?; + Ok(()) + } + + async fn automatic_checkpoint(&self) -> TestResult { + let mut connection = self.sqlite.pool().acquire().await?; + Ok(sqlx::query_scalar("PRAGMA wal_autocheckpoint") + .fetch_one(&mut *connection) + .await?) + } + + async fn import(&self) -> TestResult { + Ok(self.source.import_legacy_blobs_into(&self.target).await?) + } + } + + fn legacy_prefix() -> Vec { + SlateKey::new("blobs") + .with("sha256") + .into_prefix() + .as_ref() + .to_vec() + } + + async fn seed_blobs( + context: &TestContext, + count: usize, + ) -> TestResult)>> { + let mut blobs = Vec::with_capacity(count); + for index in 0..count { + let bytes = format!("legacy-blob-{index:04}").into_bytes(); + let hash = context.put_blob(&bytes).await?; + blobs.push((hash, bytes)); + } + blobs.sort_by_key(|(hash, _)| *hash); + Ok(blobs) + } + + #[tokio::test] + async fn imports_valid_raw_blobs_and_reports_aggregate_progress() -> TestResult<()> { + let context = TestContext::new().await?; + let binary = [0_u8, 0xff, 0x80, b'a']; + let binary_hash = context.put_blob(&binary).await?; + let empty_hash = context.put_blob(b"").await?; + let source_before = context.source_entries().await?; + + let report = context.import().await?; + + assert_eq!(report, LegacyBlobImportReport { + scanned_rows: 2, + scanned_bytes: 4, + imported_rows: 2, + imported_bytes: 4, + committed_batches: 1, + ..LegacyBlobImportReport::default() + }); + assert_eq!( + context.target.read(&binary_hash).await?, + Some(Bytes::copy_from_slice(&binary)) + ); + assert_eq!(context.target.read(&empty_hash).await?, Some(Bytes::new())); + assert_eq!(context.source_entries().await?, source_before); + Ok(()) + } + + #[tokio::test] + async fn empty_source_succeeds_without_committing_a_batch() -> TestResult<()> { + let context = TestContext::new().await?; + + let report = context.import().await?; + + assert_eq!(report, LegacyBlobImportReport::default()); + assert_eq!(context.destination_rows().await?, 0); + Ok(()) + } + + #[tokio::test] + async fn scan_is_limited_to_the_exact_legacy_prefix() -> TestResult<()> { + let context = TestContext::new().await?; + context.put_blob(b"included").await?; + context + .source_db + .put( + SlateKey::new("blobs").with("other").with("ignored"), + b"nearby", + ) + .await?; + + let report = context.import().await?; + + assert_eq!(report.scanned_rows, 1); + assert_eq!(report.imported_rows, 1); + assert_eq!(context.destination_rows().await?, 1); + Ok(()) + } + + #[tokio::test] + async fn rejects_every_noncanonical_legacy_key_shape() -> TestResult<()> { + let valid_hash = BlobHash::new(b"valid-shape").to_string(); + let cases = [ + ("empty", Vec::new()), + ("short", vec![b'0'; 63]), + ("long", vec![b'0'; 65]), + ("uppercase", vec![b'A'; 64]), + ("non_hex", vec![b'g'; 64]), + ("non_utf8", vec![0xff; 64]), + ( + "extra_segment", + [valid_hash.as_bytes(), b"\0extra"].concat(), + ), + ]; + + for (case, suffix) in cases { + let context = TestContext::new().await?; + let mut key = legacy_prefix(); + key.extend_from_slice(&suffix); + context.put_raw(key, b"source-bytes").await?; + let source_before = context.source_entries().await?; + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .await + .expect_err("noncanonical key should fail import"); + + assert_eq!(error.report().scanned_rows, 1, "case {case}"); + assert_eq!(error.report().invalid_rows, 1, "case {case}"); + assert_eq!(error.report().committed_batches, 0, "case {case}"); + assert!( + matches!(&error.failure, LegacyBlobImportFailure::InvalidSourceKey), + "case {case}: {error:?}" + ); + assert_eq!(context.destination_rows().await?, 0, "case {case}"); + assert_eq!( + context.source_entries().await?, + source_before, + "case {case}" + ); + } + Ok(()) + } + + #[tokio::test] + async fn rejects_source_bytes_that_do_not_match_the_key_digest() -> TestResult<()> { + let context = TestContext::new().await?; + let mut key = legacy_prefix(); + key.extend_from_slice(BlobHash::new(b"expected").to_string().as_bytes()); + context.put_raw(key, b"different").await?; + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .await + .expect_err("digest mismatch should fail import"); + + assert_eq!(error.report().scanned_rows, 1); + assert_eq!(error.report().scanned_bytes, 9); + assert_eq!(error.report().invalid_rows, 1); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::SourceDigestMismatch + )); + assert_eq!(context.destination_rows().await?, 0); + Ok(()) + } + + #[tokio::test] + async fn invalid_source_row_discards_the_uncommitted_pending_batch() -> TestResult<()> { + let context = TestContext::new().await?; + context.put_blob(b"valid-but-not-yet-committed").await?; + let mut invalid_key = legacy_prefix(); + invalid_key.extend_from_slice(&[b'z'; 64]); + context.put_raw(invalid_key, b"invalid").await?; + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .await + .expect_err("invalid row should stop the import"); + + assert_eq!(error.report().scanned_rows, 2); + assert_eq!(error.report().invalid_rows, 1); + assert_eq!(error.report().imported_rows, 0); + assert_eq!(error.report().committed_batches, 0); + assert_eq!(context.destination_rows().await?, 0); + Ok(()) + } + + #[tokio::test] + async fn equal_destination_rows_are_retry_progress() -> TestResult<()> { + let context = TestContext::new().await?; + context.put_blob(b"first").await?; + context.put_blob(b"second").await?; + + let first = context.import().await?; + let second = context.import().await?; + + assert_eq!(first.imported_rows, 2); + assert_eq!(second.scanned_rows, 2); + assert_eq!(second.imported_rows, 0); + assert_eq!(second.existing_rows, 2); + assert_eq!(second.existing_bytes, 11); + assert_eq!(second.committed_batches, 1); + assert_eq!(context.destination_rows().await?, 2); + Ok(()) + } + + #[tokio::test] + async fn destination_conflict_rolls_back_the_current_batch() -> TestResult<()> { + let context = TestContext::new().await?; + let blobs = seed_blobs(&context, 2).await?; + context + .insert_destination(blobs[1].0, b"conflicting-destination-bytes") + .await?; + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .await + .expect_err("differing destination row should fail import"); + + assert_eq!(error.report().scanned_rows, 2); + assert_eq!(error.report().conflicting_rows, 1); + assert_eq!(error.report().imported_rows, 0); + assert_eq!(error.report().committed_batches, 0); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::DestinationConflict + )); + assert_eq!(context.target.read(&blobs[0].0).await?, None); + assert_eq!(context.destination_rows().await?, 1); + Ok(()) + } + + #[tokio::test] + async fn interrupted_import_preserves_commits_and_retry_converges() -> TestResult<()> { + let context = TestContext::new().await?; + let blobs = seed_blobs(&context, 101).await?; + let conflict = blobs[100].0; + context + .insert_destination(conflict, b"conflicting-destination-bytes") + .await?; + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .await + .expect_err("last-row conflict should interrupt import"); + + assert_eq!(error.report().scanned_rows, 101); + assert_eq!(error.report().imported_rows, 100); + assert_eq!(error.report().conflicting_rows, 1); + assert_eq!(error.report().committed_batches, 1); + assert_eq!(context.destination_rows().await?, 101); + + context.delete_destination(conflict).await?; + let retry = context.import().await?; + assert_eq!(retry.scanned_rows, 101); + assert_eq!(retry.existing_rows, 100); + assert_eq!(retry.imported_rows, 1); + assert_eq!(retry.committed_batches, 2); + assert_eq!(context.destination_rows().await?, 101); + Ok(()) + } + + #[tokio::test] + async fn row_limit_splits_one_hundred_and_one_values() -> TestResult<()> { + let context = TestContext::new().await?; + seed_blobs(&context, 101).await?; + + let report = context.import().await?; + + assert_eq!(report.imported_rows, 101); + assert_eq!(report.committed_batches, 2); + Ok(()) + } + + #[tokio::test] + async fn byte_limit_splits_batches_and_allows_exact_limit() -> TestResult<()> { + let split_context = TestContext::new().await?; + split_context.put_blob(&vec![b'a'; 600 * 1024]).await?; + split_context.put_blob(&vec![b'b'; 600 * 1024]).await?; + let split = split_context.import().await?; + assert_eq!(split.committed_batches, 2); + + let exact_context = TestContext::new().await?; + exact_context + .put_blob(&vec![b'x'; usize::try_from(MAX_BATCH_BYTES)?]) + .await?; + exact_context.put_blob(b"").await?; + let exact = exact_context.import().await?; + assert_eq!(exact.committed_batches, 1); + Ok(()) + } + + #[tokio::test] + async fn oversized_value_is_committed_alone_between_surrounding_batches() -> TestResult<()> { + let context = TestContext::new().await?; + let oversized = vec![b'o'; usize::try_from(MAX_BATCH_BYTES + 1)?]; + let oversized_hash = BlobHash::new(&oversized); + let mut lower = None; + let mut upper = None; + for index in 0_u32..10_000 { + let bytes = format!("surrounding-{index}").into_bytes(); + let hash = BlobHash::new(&bytes); + if hash < oversized_hash && lower.is_none() { + lower = Some(bytes); + } else if hash > oversized_hash && upper.is_none() { + upper = Some(bytes); + } + if lower.is_some() && upper.is_some() { + break; + } + } + let lower = lower.expect("search should find a hash below the oversized value"); + let upper = upper.expect("search should find a hash above the oversized value"); + context.put_blob(&lower).await?; + context.put_blob(&oversized).await?; + context.put_blob(&upper).await?; + + let report = context.import().await?; + + assert_eq!(report.imported_rows, 3); + assert_eq!(report.committed_batches, 3); + Ok(()) + } + + #[tokio::test] + async fn passive_checkpoint_runs_once_after_crossing_each_batch_threshold() -> TestResult<()> { + let crossing_context = TestContext::new().await?; + crossing_context + .put_blob(&vec![b'c'; usize::try_from(PASSIVE_CHECKPOINT_BYTES + 1)?]) + .await?; + let crossing = crossing_context.import().await?; + assert_eq!(crossing.passive_checkpoints, 1); + + let multi_context = TestContext::new().await?; + multi_context + .put_blob(&vec![ + b'm'; + usize::try_from(PASSIVE_CHECKPOINT_BYTES * 2 + 1)? + ]) + .await?; + let multiple = multi_context.import().await?; + assert_eq!(multiple.passive_checkpoints, 1); + Ok(()) + } + + #[tokio::test] + async fn passive_checkpoint_failure_returns_durable_partial_report() -> TestResult<()> { + let context = TestContext::new().await?; + context + .put_blob(&vec![b'p'; usize::try_from(PASSIVE_CHECKPOINT_BYTES + 1)?]) + .await?; + let controls = ImportControls { + passive_checkpoint: true, + ..ImportControls::default() + }; + + let error = context + .source + .import_legacy_blobs_with_controls(&context.target, &controls) + .await + .expect_err("injected passive checkpoint should fail import"); + + assert_eq!(error.report().imported_rows, 1); + assert_eq!(error.report().committed_batches, 1); + assert_eq!(error.report().passive_checkpoints, 0); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::PassiveCheckpoint(sqlx::Error::Protocol(_)) + )); + let failure_source = std::error::Error::source(&error) + .and_then(std::error::Error::source) + .expect("checkpoint failure should preserve its SQL source"); + assert!(failure_source.downcast_ref::().is_some()); + assert_eq!(context.destination_rows().await?, 1); + Ok(()) + } + + #[tokio::test] + async fn final_checkpoint_failure_returns_committed_progress() -> TestResult<()> { + let context = TestContext::new().await?; + context + .put_blob(b"committed-before-final-checkpoint") + .await?; + let controls = ImportControls { + final_checkpoint: true, + ..ImportControls::default() + }; + + let error = context + .source + .import_legacy_blobs_with_controls(&context.target, &controls) + .await + .expect_err("injected final checkpoint should fail import"); + + assert_eq!(error.report().imported_rows, 1); + assert_eq!(error.report().committed_batches, 1); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::FinalCheckpoint(sqlx::Error::Protocol(_)) + )); + assert_eq!(context.destination_rows().await?, 1); + Ok(()) + } + + #[tokio::test] + async fn source_transport_failure_preserves_commits_and_typed_source() -> TestResult<()> { + let context = TestContext::new().await?; + seed_blobs(&context, 101).await?; + let controls = ImportControls { + source_after_rows: Some(101), + ..ImportControls::default() + }; + + let error = context + .source + .import_legacy_blobs_with_controls(&context.target, &controls) + .await + .expect_err("injected source transport failure should stop import"); + + assert_eq!(error.report().scanned_rows, 101); + assert_eq!(error.report().imported_rows, 100); + assert_eq!(error.report().committed_batches, 1); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::ReadSourceScan(_) + )); + let failure_source = std::error::Error::source(&error) + .and_then(std::error::Error::source) + .expect("transport failure should preserve its SlateDB source"); + assert!(failure_source.downcast_ref::().is_some()); + + let retry = context.import().await?; + assert_eq!(retry.existing_rows, 100); + assert_eq!(retry.imported_rows, 1); + assert_eq!(context.destination_rows().await?, 101); + Ok(()) + } + + #[tokio::test] + async fn automatic_checkpoint_setting_is_restored_after_success_and_failure() -> TestResult<()> + { + let success = TestContext::new().await?; + success.set_automatic_checkpoint(37).await?; + success.put_blob(b"success").await?; + success.import().await?; + assert_eq!(success.automatic_checkpoint().await?, 37); + + let failure = TestContext::new().await?; + failure.set_automatic_checkpoint(41).await?; + let mut invalid_key = legacy_prefix(); + invalid_key.extend_from_slice(&[b'z'; 64]); + failure.put_raw(invalid_key, b"invalid").await?; + failure + .source + .import_legacy_blobs_into(&failure.target) + .await + .expect_err("invalid source should fail import"); + assert_eq!(failure.automatic_checkpoint().await?, 41); + Ok(()) + } + + #[tokio::test] + async fn failed_connection_restoration_retires_the_connection() -> TestResult<()> { + let context = TestContext::new().await?; + context.set_automatic_checkpoint(43).await?; + context.put_blob(b"committed-before-restore").await?; + let controls = ImportControls { + restore_automatic_checkpoint: true, + ..ImportControls::default() + }; + + let error = context + .source + .import_legacy_blobs_with_controls(&context.target, &controls) + .await + .expect_err("injected restoration failure should fail import"); + + assert_eq!(error.report().imported_rows, 1); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::RestoreAutomaticCheckpoint { + source: sqlx::Error::Protocol(_), + .. + } + )); + assert_ne!(context.automatic_checkpoint().await?, 0); + Ok(()) + } + + #[tokio::test] + async fn slate_backed_target_is_rejected_before_scanning() -> TestResult<()> { + let context = TestContext::new().await?; + context.put_blob(b"never-scanned").await?; + let slate_target = context.source.blobs().await?; + + let error = context + .source + .import_legacy_blobs_into(&slate_target) + .await + .expect_err("Slate target should be rejected"); + + assert_eq!(*error.report(), LegacyBlobImportReport::default()); + assert!(matches!( + &error.failure, + LegacyBlobImportFailure::WrongTargetBackend + )); + assert_eq!(context.destination_rows().await?, 0); + Ok(()) + } + + #[tokio::test] + async fn logs_and_error_rendering_expose_counts_but_not_row_data() -> TestResult<()> { + let context = TestContext::new().await?; + let sensitive_key_fragment = "sensitive-invalid-legacy-key"; + let sensitive_content = b"sensitive-blob-content"; + let mut key = legacy_prefix(); + key.extend_from_slice(sensitive_key_fragment.as_bytes()); + context.put_raw(key, sensitive_content).await?; + let capture = CapturedEvents::default(); + + let error = context + .source + .import_legacy_blobs_into(&context.target) + .with_subscriber(capture.clone()) + .await + .expect_err("invalid key should fail import"); + + let rendered = format!("{error} {error:?}"); + let events = capture.events().join("\n"); + for output in [&rendered, &events] { + assert!(!output.contains(sensitive_key_fragment)); + assert!(!output.contains(std::str::from_utf8(sensitive_content)?)); + } + assert!(events.contains("scanned_rows=1"), "captured: {events}"); + assert!(events.contains("invalid_rows=1"), "captured: {events}"); + assert!( + events.contains("failure_kind=\"invalid_source_key\""), + "captured: {events}" + ); + assert_eq!(capture.events().len(), 1); + Ok(()) + } + + #[derive(Clone, Default)] + struct CapturedEvents { + events: Arc>>, + next_span_id: Arc, + } + + impl CapturedEvents { + fn events(&self) -> Vec { + self.events + .lock() + .expect("captured tracing events mutex should not be poisoned") + .clone() + } + } + + impl Subscriber for CapturedEvents { + fn enabled(&self, _metadata: &Metadata<'_>) -> bool { + true + } + + fn new_span(&self, _span: &Attributes<'_>) -> Id { + Id::from_u64(self.next_span_id.fetch_add(1, Ordering::Relaxed) + 1) + } + + fn record(&self, _span: &Id, _values: &Record<'_>) {} + + fn record_follows_from(&self, _span: &Id, _follows: &Id) {} + + fn event(&self, event: &Event<'_>) { + let mut visitor = CapturedFields::default(); + event.record(&mut visitor); + self.events + .lock() + .expect("captured tracing events mutex should not be poisoned") + .push(visitor.output); + } + + fn enter(&self, _span: &Id) {} + + fn exit(&self, _span: &Id) {} + } + + #[derive(Default)] + struct CapturedFields { + output: String, + } + + impl Visit for CapturedFields { + fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { + write!(&mut self.output, "{}={value:?};", field.name()) + .expect("writing tracing fields to String cannot fail"); + } + } +} diff --git a/lib/components/fabro-store/src/lib.rs b/lib/components/fabro-store/src/lib.rs index 00a95d4e0..c06c3b852 100644 --- a/lib/components/fabro-store/src/lib.rs +++ b/lib/components/fabro-store/src/lib.rs @@ -5,6 +5,7 @@ mod blob_store; mod error; mod keyed_mutex; mod keys; +mod legacy_blob_import; mod record; mod run_sessions; mod run_state; @@ -25,6 +26,7 @@ pub use fabro_types::{ BlobHash, EventEnvelope, PendingInterviewRecord, Run, RunProjection, StageId, StageProjection, }; pub use keyed_mutex::{KeyedMutex, KeyedMutexGuard}; +pub use legacy_blob_import::{LegacyBlobImportError, LegacyBlobImportReport}; pub use run_sessions::{ ProjectedRunSession, project_run_session, project_run_session_with_context, project_run_sessions, diff --git a/lib/components/fabro-store/src/slate/mod.rs b/lib/components/fabro-store/src/slate/mod.rs index efed28796..780bac427 100644 --- a/lib/components/fabro-store/src/slate/mod.rs +++ b/lib/components/fabro-store/src/slate/mod.rs @@ -100,7 +100,7 @@ impl Database { self.base_prefix.clone() } - async fn open_db(&self) -> Result { + pub(crate) async fn open_db(&self) -> Result { let db = self .db .get_or_try_init(|| async { From e7a32d12d5d4d210192c7b49b362a5b4695a7087 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Thu, 20 Aug 2026 12:16:12 -0400 Subject: [PATCH 04/30] Use native Daytona exact commit checkout --- AGENTS.md | 8 + .../fabro-sandbox/src/clone_source.rs | 31 +- .../fabro-sandbox/src/daytona/mod.rs | 329 +++++------------- lib/components/fabro-sandbox/src/docker.rs | 17 + 4 files changed, 136 insertions(+), 249 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 59712966b..68d143074 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,6 +31,14 @@ macOS note: if `cargo nextest run` fails with `Too many open files (os error 24) - Docker is the default runtime sandbox provider from `defaults.toml`. The Fabro process must have a working Docker client environment (`DOCKER_HOST`, socket access, Docker Desktop behavior, TLS settings, groups/permissions, and any remote daemon policy are operator responsibilities). - The packaged compose service mounts `/var/run/docker.sock` so the server can create sibling run containers on the host daemon. This is host-root-equivalent under Docker's security model; only use it in the trusted, single-tenant deployment model described by the sandbox code/docs. - Docker and Daytona are clone-based providers. When a run manifest has a GitHub origin, they clone it into the provider workspace. Present non-GitHub origins fail unless the provider has `skip_clone = true`; absent origins or `skip_clone = true` create an empty workspace without repository files. +- The sandbox layer also accepts an optional exact commit for future admitted + runs. An exact commit always requires a non-empty branch. Docker initializes + an empty repository, shallow-fetches the SHA, detaches, and verifies HEAD; + Daytona uses its official SDK clone with both `branch` and `commit_id`. Keep + those provider transports distinct, never fall back to a newer branch HEAD, + and do not wire this capability directly from legacy `GitContext.sha`. + Current production callers remain branch-only until the RunIntent admission + cutover supplies a validated branch/SHA pair. ### Release automation - `cargo dev release` — creates the next stable release tag. Use `cargo dev release --nightly` for a nightly prerelease. Use `--dry-run` to print planned commands without mutating git or running Cargo, `--skip-tests` only after running the release-mode smoke yourself, and `--release-date YYYY-MM-DD` or `FABRO_RELEASE_DATE` for deterministic version computation. diff --git a/lib/components/fabro-sandbox/src/clone_source.rs b/lib/components/fabro-sandbox/src/clone_source.rs index 15c89e137..cd6d9113c 100644 --- a/lib/components/fabro-sandbox/src/clone_source.rs +++ b/lib/components/fabro-sandbox/src/clone_source.rs @@ -161,6 +161,11 @@ pub(crate) fn decide_clone( "Exact commit checkout requires a repository origin", )); } + if clone_branch.is_none_or(|branch| branch.trim().is_empty()) { + return Err(crate::Error::message( + "Exact commit checkout requires a repository branch", + )); + } } if skip_clone { @@ -349,13 +354,13 @@ mod tests { decide_clone( false, Some("https://github.com/acme/widgets"), - None, + Some("main"), Some(uppercase), ) .unwrap(), CloneDecision::GitHub { origin_url: "https://github.com/acme/widgets".to_string(), - branch: None, + branch: Some("main".to_string()), commit_sha: Some(uppercase.to_ascii_lowercase()), } ); @@ -387,26 +392,37 @@ mod tests { } #[test] - fn exact_checkout_requires_clone_and_origin() { + fn exact_checkout_requires_clone_origin_and_branch() { let sha = "0123456789abcdef0123456789abcdef01234567"; let skip_error = decide_clone( true, Some("https://github.com/acme/widgets"), - None, + Some("main"), Some(sha), ) .expect_err("exact checkout with skip-clone should fail"); assert!(skip_error.to_string().contains("requires cloning")); for origin in [None, Some(""), Some(" ")] { - let error = decide_clone(false, origin, None, Some(sha)) + let error = decide_clone(false, origin, Some("main"), Some(sha)) .expect_err("exact checkout without an origin should fail"); assert!(error.to_string().contains("requires a repository origin")); } + + for branch in [None, Some(""), Some(" ")] { + let error = decide_clone( + false, + Some("https://github.com/acme/widgets"), + branch, + Some(sha), + ) + .expect_err("exact checkout without a branch should fail"); + assert!(error.to_string().contains("requires a repository branch")); + } } #[test] - fn exact_checkout_commands_quote_inputs_and_ignore_branch_metadata() { + fn docker_exact_checkout_commands_quote_inputs() { let sha = "0123456789abcdef0123456789abcdef01234567"; let init = exact_repository_init_command( "https://token@example.com/acme/widgets.git?x=a b", @@ -431,9 +447,6 @@ mod tests { checkout, "git -C \"/repos/acme's widgets\" -c advice.detachedHead=false checkout --detach FETCH_HEAD && git -C \"/repos/acme's widgets\" rev-parse HEAD" ); - for command in [&init, &fetch, &checkout] { - assert!(!command.contains("moving-branch")); - } } #[test] diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index bbc4cfdef..c03ea734c 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -13,7 +13,7 @@ use daytona_api_client::models::SandboxState; use daytona_api_client::models::api_key_list::Permissions; use daytona_sdk::api_types::SignedPortPreviewUrl; use daytona_sdk::toolbox_types::Command as SessionCommandResult; -use daytona_sdk::{DaytonaError, SessionCommandLogsResult}; +use daytona_sdk::{DaytonaError, GitCloneOptions, SessionCommandLogsResult}; use fabro_github::GitHubCredentials; use fabro_static::EnvVars; use fabro_types::{CommandOutputStream, CommandTermination, RunId, SandboxProviderKind}; @@ -69,18 +69,17 @@ const DAYTONA_START_TIMEOUT: Duration = Duration::from_mins(1); /// cancellation/timeout paths indefinitely. const DAYTONA_CLEANUP_TIMEOUT: Duration = Duration::from_secs(10); -fn daytona_process_exec_result(exit_code: i32, output: String) -> ExecResult { - let (stdout, stderr) = if exit_code == 0 { - (output, String::new()) - } else { - (String::new(), output) - }; - ExecResult { - stdout, - stderr, - exit_code: Some(exit_code), - termination: CommandTermination::Exited, - duration_ms: 0, +fn daytona_git_clone_options( + branch: Option, + commit_id: Option, + username: Option, + password: Option, +) -> GitCloneOptions { + GitCloneOptions { + branch, + commit_id, + username, + password, } } @@ -532,144 +531,6 @@ impl DaytonaSandbox { err } - fn daytona_process_transport_error(label: &'static str, error: &DaytonaError) -> crate::Error { - let error_class = match error { - DaytonaError::RateLimit { .. } => "rate_limited", - DaytonaError::Timeout { .. } => "timeout", - DaytonaError::Api { status_code, .. } if (500..600).contains(status_code) => { - "server_error" - } - DaytonaError::Api { .. } => "api_error", - DaytonaError::NotFound { .. } => "not_found", - DaytonaError::General(_) => "transport_error", - }; - crate::Error::context( - label, - crate::Error::message(format!("Daytona process command failed ({error_class})")), - ) - } - - async fn run_exact_checkout_command( - process_svc: &daytona_sdk::ProcessService, - command: &str, - working_directory: &str, - label: &'static str, - auth_url: Option<&fabro_redact::DisplaySafeUrl>, - ) -> crate::Result { - let response = process_svc - .execute_command( - &wrap_bash_command(command), - daytona_sdk::ExecuteCommandOptions { - cwd: Some(working_directory.to_string()), - ..Default::default() - }, - ) - .await - .map_err(|error| Self::daytona_process_transport_error(label, &error))?; - let result = daytona_process_exec_result(response.exit_code, response.result); - if result.is_success() { - Ok(result) - } else { - Err(result - .into_exec_error_with_redactor(label, |output| redact_auth_url(output, auth_url))) - } - } - - /// Materialize the exact admitted commit through the Daytona process - /// service: init an empty repository, shallow-fetch the commit with clone - /// retry semantics, detach onto it, and verify the resulting HEAD. - async fn checkout_exact_commit( - process_svc: &daytona_sdk::ProcessService, - origin_url: &str, - layout: &clone_source::GitHubRepoLayout, - password: Option<&str>, - expected_sha: &str, - token_was_freshly_minted: bool, - ) -> crate::Result<()> { - let auth_url = match password { - Some(token) => Some(fabro_github::embed_token_in_url(origin_url, token).map_err( - |error| crate::Error::Context { - message: - "Failed to build authenticated URL for Daytona exact checkout".to_string(), - source: error.into_boxed_dyn_error(), - }, - )?), - None => None, - }; - let clone_url = auth_url - .as_ref() - .map_or(origin_url, |url| url.as_raw_url().as_str()); - - let init_command = - clone_source::exact_repository_init_command(clone_url, &layout.primary_repo_path); - Self::run_exact_checkout_command( - process_svc, - &init_command, - "/", - "initialize Daytona exact repository checkout", - auth_url.as_ref(), - ) - .await?; - - let fetch_command = - clone_source::exact_fetch_command(&layout.primary_repo_path, "origin", expected_sha); - clone_retry::retry_clone( - SandboxProviderKind::Daytona, - None, - |_attempt| { - let command = fetch_command.as_str(); - let auth_url = auth_url.as_ref(); - async move { - let response = process_svc - .execute_command( - &wrap_bash_command(command), - daytona_sdk::ExecuteCommandOptions { - cwd: Some("/".to_string()), - ..Default::default() - }, - ) - .await - .map_err(|error| clone_retry::CloneAttemptFailure { - retry_reason: classify_clone_failure(&error, token_was_freshly_minted), - error: Self::daytona_process_transport_error( - "Daytona exact fetch transport failed", - &error, - ), - })?; - if response.exit_code == 0 { - return Ok(()); - } - let retry_reason = - clone_retry::classify_message(&response.result, token_was_freshly_minted) - .retry_reason(); - let result = daytona_process_exec_result(response.exit_code, response.result); - Err(clone_retry::CloneAttemptFailure { - retry_reason, - error: result.into_exec_error_with_redactor( - "git fetch exact commit in Daytona sandbox", - |output| redact_auth_url(output, auth_url), - ), - }) - } - }, - |failure: &clone_retry::CloneAttemptFailure| failure.retry_reason, - ) - .await - .map_err(|failure| failure.error)?; - - let checkout_command = - clone_source::exact_checkout_verify_command(&layout.primary_repo_path); - let head = Self::run_exact_checkout_command( - process_svc, - &checkout_command, - "/", - "git checkout exact commit in Daytona sandbox", - auth_url.as_ref(), - ) - .await?; - clone_source::verify_exact_head(&head.stdout, expected_sha) - } - fn fail_init(&self, init_start: Instant, err: crate::Error) -> crate::Error { let duration_ms = u64::try_from(init_start.elapsed().as_millis()).unwrap_or(u64::MAX); self.emit(SandboxEvent::InitializeFailed { @@ -1316,75 +1177,56 @@ impl Sandbox for DaytonaSandbox { self.fail_init(init_start, err) })?; + let git_svc = sandbox.git().await.map_err(|e| { + let err = crate::Error::context("Failed to get Daytona git service", e); + let err = self.report_clone_failure(&origin_url, err); + self.fail_init(init_start, err) + })?; + + let clone_result = clone_retry::retry_clone( + SandboxProviderKind::Daytona, + None, + |_attempt| { + let git_svc = &git_svc; + let origin = origin_url.as_str(); + let target = layout.primary_repo_path.as_str(); + let options = daytona_git_clone_options( + branch.clone(), + commit_sha.clone(), + username.clone(), + password.clone(), + ); + async move { git_svc.clone(origin, target, options).await } + }, + |err: &DaytonaError| classify_clone_failure(err, token_was_freshly_minted), + ) + .await; + + match clone_result { + Ok(()) => {} + Err(e) if self.github_app.is_none() => { + let err = crate::Error::context( + "Git clone failed. If this is a private repository, configure a \ + GitHub App with `fabro install` and install it for your organization.", + e, + ); + let err = self.report_clone_failure(&origin_url, err); + return Err(self.fail_init(init_start, err)); + } + Err(e) => { + let err = + crate::Error::context("Failed to clone repo into Daytona sandbox", e); + let err = self.report_clone_failure(&origin_url, err); + return Err(self.fail_init(init_start, err)); + } + } + let process_svc = sandbox.process().await.map_err(|e| { let err = crate::Error::context("Failed to get Daytona process service", e); let err = self.report_clone_failure(&origin_url, err); self.fail_init(init_start, err) })?; - if let Some(expected_sha) = commit_sha.as_deref() { - Self::checkout_exact_commit( - &process_svc, - &origin_url, - &layout, - password.as_deref(), - expected_sha, - token_was_freshly_minted, - ) - .await - .map_err(|err| { - let err = self.report_clone_failure(&origin_url, err); - self.fail_init(init_start, err) - })?; - } else { - let git_svc = sandbox.git().await.map_err(|e| { - let err = crate::Error::context("Failed to get Daytona git service", e); - let err = self.report_clone_failure(&origin_url, err); - self.fail_init(init_start, err) - })?; - - let clone_result = clone_retry::retry_clone( - SandboxProviderKind::Daytona, - None, - |_attempt| { - let git_svc = &git_svc; - let origin = origin_url.as_str(); - let target = layout.primary_repo_path.as_str(); - let options = daytona_sdk::GitCloneOptions { - branch: branch.clone(), - username: username.clone(), - password: password.clone(), - ..Default::default() - }; - async move { git_svc.clone(origin, target, options).await } - }, - |err: &DaytonaError| classify_clone_failure(err, token_was_freshly_minted), - ) - .await; - - match clone_result { - Ok(()) => {} - Err(e) if self.github_app.is_none() => { - let err = crate::Error::context( - "Git clone failed. If this is a private repository, \ - configure a GitHub App with `fabro install` and install it \ - for your organization.", - e, - ); - let err = self.report_clone_failure(&origin_url, err); - return Err(self.fail_init(init_start, err)); - } - Err(e) => { - let err = crate::Error::context( - "Failed to clone repo into Daytona sandbox", - e, - ); - let err = self.report_clone_failure(&origin_url, err); - return Err(self.fail_init(init_start, err)); - } - } - } - let symlink_cmd = clone_source::repo_symlink_command(&layout); let symlink_result = process_svc .execute_command( @@ -2881,34 +2723,41 @@ mod tests { assert!(!error.to_string().contains("Daytona client")); } - #[test] - fn exact_checkout_process_failure_redacts_credentials_and_preserves_cause() { - let token = "ghs_daytona_exact_secret"; - let auth_url = fabro_github::embed_token_in_url("https://github.com/acme/widgets", token) - .expect("authenticated URL"); - let result = daytona_process_exec_result( - 128, - format!( - "fatal: unable to access {}: synthetic Daytona failure", - auth_url.as_raw_url() - ), - ); - let error = result - .into_exec_error_with_redactor("git fetch exact commit in Daytona sandbox", |output| { - redact_auth_url(output, Some(&auth_url)) - }); + #[tokio::test] + async fn exact_sha_without_branch_fails_before_daytona_client_construction() { + let error = DaytonaSandbox::new( + DaytonaConfig::default(), + None, + None, + Some("https://github.com/acme/widgets".to_string()), + None, + Some("0123456789abcdef0123456789abcdef01234567".to_string()), + Some("dtn_not_used".to_string()), + ) + .await + .err() + .expect("branch validation should run before building a Daytona client"); - let causes = collect_chain(&error); - assert!( - causes - .iter() - .any(|cause| cause.contains("git fetch exact commit")), - "exec cause should remain structured: {causes:?}" + assert!(error.to_string().contains("requires a repository branch")); + assert!(!error.to_string().contains("Daytona client")); + } + + #[test] + fn exact_checkout_uses_daytona_branch_and_commit_options() { + let options = daytona_git_clone_options( + Some("feature/work".to_string()), + Some("0123456789abcdef0123456789abcdef01234567".to_string()), + Some("x-access-token".to_string()), + Some("secret".to_string()), ); - let rendered = crate::display_for_log(&error); - assert!(!rendered.contains(token)); - assert!(!rendered.contains(auth_url.as_raw_url().as_str())); - assert!(rendered.contains("synthetic Daytona failure")); + + assert_eq!(options.branch.as_deref(), Some("feature/work")); + assert_eq!( + options.commit_id.as_deref(), + Some("0123456789abcdef0123456789abcdef01234567") + ); + assert_eq!(options.username.as_deref(), Some("x-access-token")); + assert_eq!(options.password.as_deref(), Some("secret")); } fn api_key_body(permissions: &[&str]) -> serde_json::Value { diff --git a/lib/components/fabro-sandbox/src/docker.rs b/lib/components/fabro-sandbox/src/docker.rs index c1acdf1ad..891cd6cad 100644 --- a/lib/components/fabro-sandbox/src/docker.rs +++ b/lib/components/fabro-sandbox/src/docker.rs @@ -2550,6 +2550,23 @@ mod tests { assert!(!error.to_string().contains("Docker daemon")); } + #[test] + fn exact_sha_without_branch_fails_before_docker_connection() { + let error = DockerSandbox::new( + DockerSandboxOptions::default(), + None, + None, + Some("https://github.com/acme/widgets".to_string()), + None, + Some("0123456789abcdef0123456789abcdef01234567".to_string()), + ) + .err() + .expect("branch validation should run before connecting to Docker"); + + assert!(error.to_string().contains("requires a repository branch")); + assert!(!error.to_string().contains("Docker daemon")); + } + #[test] fn exact_checkout_failure_preserves_safe_source_chain() { let docker = Docker::connect_with_http("http://127.0.0.1:2375", 5, API_DEFAULT_VERSION) From ebf6f92724f7404fb4def806fea8e09e5695cd1b Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Thu, 20 Aug 2026 13:41:54 -0400 Subject: [PATCH 05/30] Harden exact-commit checkout in clone-based sandboxes Run the local git steps of the Docker exact checkout under the shared clone deadline instead of a fixed 10s timeout, so materializing a large working tree cannot time out and abandon a running checkout in the container. Check the admitted commit out onto the admitted branch rather than detaching. A detached HEAD makes `rev-parse --abbrev-ref HEAD` return "HEAD", which the git setup helper maps to no base branch, silently dropping it for callers that rely on it. Daytona does the same after its native clone and now verifies the resulting HEAD the way Docker does. Fetch the exact commit at the same depth a branch clone uses, so both paths can reach the same number of parent commits, and stop suggesting GitHub App credentials when a purely local git step fails. Document that reachability of the commit from the branch is an admission-time invariant that the sandbox layer does not re-verify. --- AGENTS.md | 16 ++- .../fabro-sandbox/src/clone_source.rs | 86 ++++++++++---- .../fabro-sandbox/src/daytona/mod.rs | 79 +++++++++++++ lib/components/fabro-sandbox/src/docker.rs | 105 +++++++++++++++--- 4 files changed, 247 insertions(+), 39 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 68d143074..acc2f2dec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,12 +33,16 @@ macOS note: if `cargo nextest run` fails with `Too many open files (os error 24) - Docker and Daytona are clone-based providers. When a run manifest has a GitHub origin, they clone it into the provider workspace. Present non-GitHub origins fail unless the provider has `skip_clone = true`; absent origins or `skip_clone = true` create an empty workspace without repository files. - The sandbox layer also accepts an optional exact commit for future admitted runs. An exact commit always requires a non-empty branch. Docker initializes - an empty repository, shallow-fetches the SHA, detaches, and verifies HEAD; - Daytona uses its official SDK clone with both `branch` and `commit_id`. Keep - those provider transports distinct, never fall back to a newer branch HEAD, - and do not wire this capability directly from legacy `GitContext.sha`. - Current production callers remain branch-only until the RunIntent admission - cutover supplies a validated branch/SHA pair. + an empty repository, shallow-fetches the SHA at the same depth as a branch + clone, and checks it out; Daytona uses its official SDK clone with both + `branch` and `commit_id`. Both providers then point the admitted branch at + the commit and verify HEAD, so the workspace still reports the admitted + branch name. Keep those provider transports distinct, never fall back to a + newer branch HEAD, and do not wire this capability directly from legacy + `GitContext.sha`. The sandbox layer does not verify that the commit is + reachable from the branch; admission owns that check. Current production + callers remain branch-only until the RunIntent admission cutover supplies a + validated branch/SHA pair. ### Release automation - `cargo dev release` — creates the next stable release tag. Use `cargo dev release --nightly` for a nightly prerelease. Use `--dry-run` to print planned commands without mutating git or running Cargo, `--skip-tests` only after running the release-mode smoke yourself, and `--release-date YYYY-MM-DD` or `FABRO_RELEASE_DATE` for deterministic version computation. diff --git a/lib/components/fabro-sandbox/src/clone_source.rs b/lib/components/fabro-sandbox/src/clone_source.rs index cd6d9113c..ea343bbf4 100644 --- a/lib/components/fabro-sandbox/src/clone_source.rs +++ b/lib/components/fabro-sandbox/src/clone_source.rs @@ -81,13 +81,20 @@ pub(crate) fn exact_repository_init_command(clone_url: &str, checkout_path: &str ) } +/// Fetch a single admitted commit with the same history depth a branch clone +/// gets, so both paths can reach the same number of parent commits. +/// +/// The fetch names the commit directly rather than the branch: reachability of +/// the commit from the admitted branch is an admission-time invariant, not +/// something this layer re-verifies. pub(crate) fn exact_fetch_command( checkout_path: &str, fetch_source: &str, commit_sha: &str, + depth: usize, ) -> String { format!( - "{git} -C {} fetch --depth 1 --no-tags {} -- {}", + "{git} -C {} fetch --depth {depth} --no-tags {} -- {}", sandbox::shell_quote(checkout_path), sandbox::shell_quote(fetch_source), sandbox::shell_quote(commit_sha), @@ -95,13 +102,45 @@ pub(crate) fn exact_fetch_command( ) } -/// Detach onto the fetched commit and print the resulting HEAD in one shell -/// command; stdout is the `rev-parse HEAD` output for [`verify_exact_head`]. -pub(crate) fn exact_checkout_verify_command(checkout_path: &str) -> String { +/// Point the admitted branch at `revision` and attach HEAD to it. +/// +/// The checkout attaches to a real branch instead of detaching so callers that +/// read the current branch back out of the workspace still see the admitted +/// branch name. +pub(crate) fn exact_branch_checkout_command( + checkout_path: &str, + branch: &str, + revision: &str, +) -> String { format!( - "git -C {path} -c advice.detachedHead=false checkout --detach FETCH_HEAD && git -C {path} \ - rev-parse HEAD", + "{git} -C {path} checkout -B {branch} {revision}", path = sandbox::shell_quote(checkout_path), + branch = sandbox::shell_quote(branch), + revision = sandbox::shell_quote(revision), + git = sandbox::GIT, + ) +} + +/// Print the current HEAD commit and nothing else, for [`verify_exact_head`]. +pub(crate) fn exact_head_revision_command(checkout_path: &str) -> String { + format!( + "{git} -C {path} rev-parse HEAD", + path = sandbox::shell_quote(checkout_path), + git = sandbox::GIT, + ) +} + +/// Check out the admitted branch and print the resulting HEAD in one shell +/// command; stdout is the `rev-parse HEAD` output for [`verify_exact_head`]. +pub(crate) fn exact_checkout_verify_command( + checkout_path: &str, + branch: &str, + revision: &str, +) -> String { + format!( + "{} && {}", + exact_branch_checkout_command(checkout_path, branch, revision), + exact_head_revision_command(checkout_path), ) } @@ -161,6 +200,9 @@ pub(crate) fn decide_clone( "Exact commit checkout requires a repository origin", )); } + // The branch names the checkout the run works on; it is not used to + // constrain which commits may be fetched. Admission is responsible for + // proving the commit belongs to the branch before it reaches here. if clone_branch.is_none_or(|branch| branch.trim().is_empty()) { return Err(crate::Error::message( "Exact commit checkout requires a repository branch", @@ -432,8 +474,10 @@ mod tests { "/repos/acme's widgets", "https://token@example.com/acme/widgets.git?x=a b", sha, + 10, ); - let checkout = exact_checkout_verify_command("/repos/acme's widgets"); + let checkout = + exact_checkout_verify_command("/repos/acme's widgets", "feature/a b", "FETCH_HEAD"); assert_eq!( init, @@ -441,11 +485,11 @@ mod tests { ); assert_eq!( fetch, - "git -c maintenance.auto=0 -c gc.auto=0 -C \"/repos/acme's widgets\" fetch --depth 1 --no-tags 'https://token@example.com/acme/widgets.git?x=a b' -- 0123456789abcdef0123456789abcdef01234567" + "git -c maintenance.auto=0 -c gc.auto=0 -C \"/repos/acme's widgets\" fetch --depth 10 --no-tags 'https://token@example.com/acme/widgets.git?x=a b' -- 0123456789abcdef0123456789abcdef01234567" ); assert_eq!( checkout, - "git -C \"/repos/acme's widgets\" -c advice.detachedHead=false checkout --detach FETCH_HEAD && git -C \"/repos/acme's widgets\" rev-parse HEAD" + "git -c maintenance.auto=0 -c gc.auto=0 -C \"/repos/acme's widgets\" checkout -B 'feature/a b' FETCH_HEAD && git -c maintenance.auto=0 -c gc.auto=0 -C \"/repos/acme's widgets\" rev-parse HEAD" ); } @@ -510,23 +554,27 @@ mod tests { ); run_shell( temp.path(), - &exact_fetch_command(checkout_path, remote_path, &admitted_sha), + &exact_fetch_command(checkout_path, remote_path, &admitted_sha, 10), + ); + let checked_out_sha = run_shell( + temp.path(), + &exact_checkout_verify_command(checkout_path, "main", "FETCH_HEAD"), ); - let checked_out_sha = run_shell(temp.path(), &exact_checkout_verify_command(checkout_path)); assert_eq!(checked_out_sha.trim(), admitted_sha); assert_eq!( fs::read_to_string(checkout.join("revision.txt")).expect("checked-out contents"), "A\n" ); - let symbolic_head = isolated_command(Command::new("git").args([ - "-C", - checkout_path, - "symbolic-ref", - "-q", - "HEAD", - ])); - assert!(!symbolic_head.status.success(), "HEAD should be detached"); + assert_eq!( + run_git(&checkout, &["symbolic-ref", "HEAD"]).trim(), + "refs/heads/main", + "HEAD should stay attached to the admitted branch" + ); + assert_eq!( + run_git(&checkout, &["rev-parse", "--abbrev-ref", "HEAD"]).trim(), + "main" + ); assert_eq!( run_git(temp.path(), &[ "--git-dir", diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index c03ea734c..c18cdad27 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -531,6 +531,63 @@ impl DaytonaSandbox { err } + /// Point the admitted branch at the exact commit and verify the resulting + /// HEAD. + /// + /// Daytona's native clone honors `commit_id`, but leaves the workspace on + /// whatever ref its own checkout produced. Re-pointing the branch keeps the + /// admitted branch name readable back out of the workspace, matching what + /// the Docker provider produces for the same inputs. + async fn attach_exact_commit_branch( + process_svc: &daytona_sdk::ProcessService, + checkout_path: &str, + branch: &str, + expected_sha: &str, + ) -> crate::Result<()> { + Self::run_clone_step( + process_svc, + &clone_source::exact_branch_checkout_command(checkout_path, branch, expected_sha), + "git checkout exact commit", + ) + .await?; + let head = Self::run_clone_step( + process_svc, + &clone_source::exact_head_revision_command(checkout_path), + "git rev-parse HEAD after exact checkout", + ) + .await?; + clone_source::verify_exact_head(&head, expected_sha) + } + + /// Run one local git step of the clone in the sandbox and return its + /// output. + async fn run_clone_step( + process_svc: &daytona_sdk::ProcessService, + command: &str, + label: &'static str, + ) -> crate::Result { + let result = process_svc + .execute_command( + &wrap_bash_command(command), + daytona_sdk::ExecuteCommandOptions { + cwd: Some("/".to_string()), + ..Default::default() + }, + ) + .await + .map_err(|e| crate::Error::context(format!("Failed to run {label}"), e))?; + if result.exit_code != 0 { + return Err(crate::Error::exec(label, ExecResult { + stdout: result.result, + stderr: String::new(), + exit_code: Some(result.exit_code), + termination: CommandTermination::Exited, + duration_ms: 0, + })); + } + Ok(result.result) + } + fn fail_init(&self, init_start: Instant, err: crate::Error) -> crate::Error { let duration_ms = u64::try_from(init_start.elapsed().as_millis()).unwrap_or(u64::MAX); self.emit(SandboxEvent::InitializeFailed { @@ -1227,6 +1284,28 @@ impl Sandbox for DaytonaSandbox { self.fail_init(init_start, err) })?; + if let Some(expected_sha) = commit_sha.as_deref() { + let Some(branch) = branch.as_deref().filter(|branch| !branch.trim().is_empty()) + else { + let err = crate::Error::message( + "Exact commit checkout requires a repository branch", + ); + let err = self.report_clone_failure(&origin_url, err); + return Err(self.fail_init(init_start, err)); + }; + if let Err(err) = Self::attach_exact_commit_branch( + &process_svc, + &layout.primary_repo_path, + branch, + expected_sha, + ) + .await + { + let err = self.report_clone_failure(&origin_url, err); + return Err(self.fail_init(init_start, err)); + } + } + let symlink_cmd = clone_source::repo_symlink_command(&layout); let symlink_result = process_svc .execute_command( diff --git a/lib/components/fabro-sandbox/src/docker.rs b/lib/components/fabro-sandbox/src/docker.rs index 891cd6cad..1ee22ce90 100644 --- a/lib/components/fabro-sandbox/src/docker.rs +++ b/lib/components/fabro-sandbox/src/docker.rs @@ -56,6 +56,14 @@ const EXEC_TERM_GRACE_SECONDS: &str = "0.02"; #[cfg(not(test))] const EXEC_TERM_GRACE_SECONDS: &str = "0.2"; +/// Whether a failing git step talked to the remote. Local steps cannot fail on +/// credentials, so they must not suggest reconfiguring the GitHub App. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum CloneStep { + Network, + Local, +} + fn env_entry_name(entry: &str) -> &str { entry.split_once('=').map_or(entry, |(name, _)| name) } @@ -717,20 +725,23 @@ impl DockerSandbox { Ok(()) } - /// Preserve a failed git transfer result while masking the auth URL. + /// Preserve a failed git step result while masking the auth URL. fn clone_failure_error( &self, result: ExecResult, label: &'static str, auth_url: Option<&fabro_redact::DisplaySafeUrl>, + step: CloneStep, ) -> crate::Error { let error = result.into_exec_error_with_redactor(label, |output| redact_auth_url(output, auth_url)); - let message = if self.github_app.is_none() { - "Git clone failed. If this is a private repository, configure a GitHub App with \ - `fabro install` and install it for your organization." - } else { - "Failed to clone repository into Docker sandbox" + let message = match step { + CloneStep::Network if self.github_app.is_none() => { + "Git clone failed. If this is a private repository, configure a GitHub App with \ + `fabro install` and install it for your organization." + } + CloneStep::Network => "Failed to clone repository into Docker sandbox", + CloneStep::Local => "Failed to prepare the cloned repository in the Docker sandbox", }; crate::Error::context(message, error) } @@ -744,20 +755,34 @@ impl DockerSandbox { err } - async fn run_exact_checkout_command( + /// Run a local (non-network) step of the exact checkout under the shared + /// clone deadline. + /// + /// Materializing a large working tree takes far longer than the short fixed + /// timeout used for trivial container commands, so these steps get the same + /// budget the branch clone path gives its fetch and checkout. + async fn run_exact_local_git_command( &self, command: &str, label: &'static str, + clone_deadline: time::Instant, auth_url: Option<&fabro_redact::DisplaySafeUrl>, ) -> crate::Result { + let remaining = clone_deadline.saturating_duration_since(time::Instant::now()); + let timeout_ms = u64::try_from(remaining.as_millis()).unwrap_or(u64::MAX); + if timeout_ms == 0 { + return Err(crate::Error::message(format!( + "{label} deadline expired before the step could run" + ))); + } let result = self - .docker_exec_shell(command, 10_000, Some("/"), None, None) + .docker_exec_shell(command, timeout_ms, Some("/"), None, None) .await .map_err(|error| crate::Error::context(format!("{label} transport failed"), error))?; if result.is_success() { Ok(result) } else { - Err(self.clone_failure_error(result, label, auth_url)) + Err(self.clone_failure_error(result, label, auth_url, CloneStep::Local)) } } @@ -806,7 +831,12 @@ impl DockerSandbox { } let retry_reason = classify_docker_clone_result(&result, token_was_freshly_minted); Err(clone_retry::CloneAttemptFailure { - error: self.clone_failure_error(result, exec_label, auth_url), + error: self.clone_failure_error( + result, + exec_label, + auth_url, + CloneStep::Network, + ), retry_reason, }) }, @@ -873,12 +903,22 @@ impl DockerSandbox { let clone_deadline = time::Instant::now() + GIT_CLONE_TIMEOUT; if let Some(expected_sha) = commit_sha.as_deref() { + // `decide_clone` already rejects an exact commit without a branch; + // re-check here so the checkout can never silently drop the branch + // name callers read back out of the workspace. + let Some(branch) = branch.as_deref().filter(|branch| !branch.trim().is_empty()) else { + let error = + crate::Error::message("Exact commit checkout requires a repository branch"); + return Err(self.report_clone_failure(&origin_url, error)); + }; + let init_command = clone_source::exact_repository_init_command(clone_url, &layout.primary_repo_path); if let Err(error) = self - .run_exact_checkout_command( + .run_exact_local_git_command( &init_command, "initialize Docker exact repository checkout", + clone_deadline, auth_url.as_ref(), ) .await @@ -890,6 +930,7 @@ impl DockerSandbox { &layout.primary_repo_path, "origin", expected_sha, + GIT_CLONE_DEPTH, ); if let Err(failure) = self .retry_git_transfer( @@ -905,12 +946,16 @@ impl DockerSandbox { return Err(self.report_clone_failure(&origin_url, failure.error)); } - let checkout_command = - clone_source::exact_checkout_verify_command(&layout.primary_repo_path); + let checkout_command = clone_source::exact_checkout_verify_command( + &layout.primary_repo_path, + branch, + "FETCH_HEAD", + ); let head = match self - .run_exact_checkout_command( + .run_exact_local_git_command( &checkout_command, "git checkout exact commit", + clone_deadline, auth_url.as_ref(), ) .await @@ -2588,6 +2633,7 @@ mod tests { }, "git fetch exact commit", Some(&auth_url), + CloneStep::Network, ); let causes = error.causes(); @@ -2603,6 +2649,37 @@ mod tests { assert!(rendered.contains("synthetic low-level failure")); } + #[test] + fn local_checkout_failure_does_not_blame_github_credentials() { + let docker = Docker::connect_with_http("http://127.0.0.1:2375", 5, API_DEFAULT_VERSION) + .expect("mock Docker client should connect"); + let sandbox = test_docker_sandbox(docker, "test-container"); + let failure = ExecResult { + stdout: String::new(), + stderr: "error: pathspec 'FETCH_HEAD' did not match".to_string(), + exit_code: Some(1), + termination: CommandTermination::Exited, + duration_ms: 1, + }; + + let local = crate::display_for_log(&sandbox.clone_failure_error( + failure.clone(), + "git checkout exact commit", + None, + CloneStep::Local, + )); + assert!(!local.contains("fabro install"), "{local}"); + assert!(local.contains("prepare the cloned repository"), "{local}"); + + let network = crate::display_for_log(&sandbox.clone_failure_error( + failure, + "git fetch exact commit", + None, + CloneStep::Network, + )); + assert!(network.contains("fabro install"), "{network}"); + } + #[test] fn clone_result_uses_stderr_before_stdout() { let result = ExecResult { From a611e00fe6b045d0e430ed80414afa428ff47e24 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 07:40:13 -0400 Subject: [PATCH 06/30] Rescue release pushes when origin/main moves mid-release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The release push raced any commit that landed on main while the release smoke ran (~15 minutes): git push was rejected as non-fast-forward and the whole release failed, as seen on the v0.332.0-nightly.1 attempt. Worse, the push was not atomic — if the tag ref had been accepted while the main ref was rejected, the release would have shipped from an orphan commit and main would never have received the version bump. Make the push atomic (both refs or neither) and add a bounded rescue loop: on rejection, drop the bump commit and tag this run created, fast-forward onto the updated origin/main, recompute the version against freshly fetched tags, and rebuild the bump commit on the new tip. The fast-forward uses --ff-only so a genuinely diverged local main (unpushed commits) fails loudly instead of being reset away. The retried tag can include commits the smoke did not test; those commits passed CI to land on main, and the Release workflow re-runs the full test suite on the tagged commit before publishing anything. Co-Authored-By: Claude Fable 5 --- .../fabro-dev/src/commands/release.rs | 430 +++++++++++++++--- 1 file changed, 370 insertions(+), 60 deletions(-) diff --git a/lib/foundation/fabro-dev/src/commands/release.rs b/lib/foundation/fabro-dev/src/commands/release.rs index e088f38d3..b47e9ddb8 100644 --- a/lib/foundation/fabro-dev/src/commands/release.rs +++ b/lib/foundation/fabro-dev/src/commands/release.rs @@ -8,6 +8,7 @@ use super::{PlannedCommand, capture_command, run_command, spa_refresh, workspace const RELEASE_EPOCH: &str = "2026-01-01"; const RELEASE_TEST_SEGMENT_WRITE_KEY: &str = "fake-for-local-smoke"; +const MAX_PUSH_ATTEMPTS: u32 = 4; #[derive(Debug, Args)] pub(crate) struct ReleaseArgs { @@ -36,6 +37,12 @@ struct ReleasePlan { root: PathBuf, } +struct ReleaseVersions { + current: String, + next: String, + tag: String, +} + #[expect( clippy::print_stdout, reason = "dev release command reports progress and dry-run commands directly" @@ -52,64 +59,19 @@ pub(crate) fn release(args: ReleaseArgs) -> Result<()> { }; let cargo_toml = plan.root.join("Cargo.toml"); - let current_version = read_current_version(&cargo_toml)?; - println!("Current version: {current_version}"); - - let base_version = plan.next_base_version()?; - let new_version = plan.compute_release_version(&base_version)?; - let tag = format!("v{new_version}"); - println!("Releasing {new_version} (tag {tag})"); + let versions = plan.compute_versions(&cargo_toml)?; + println!("Current version: {}", versions.current); + println!("Releasing {} (tag {})", versions.next, versions.tag); if plan.dry_run { - plan.print_dry_run(¤t_version, &new_version, &tag); + plan.print_dry_run(&versions); return Ok(()); } plan.ensure_clean_worktree()?; spa_refresh::spa_refresh_root(&plan.root)?; plan.verify_release_tests()?; - update_version(&cargo_toml, ¤t_version, &new_version)?; - println!("Updated {}", cargo_toml.display()); - - run_command( - &plan.root, - &PlannedCommand::new("cargo") - .arg("update") - .arg("--workspace"), - )?; - println!("Updated Cargo.lock"); - - run_command( - &plan.root, - &PlannedCommand::new("git") - .arg("add") - .arg("Cargo.toml") - .arg("Cargo.lock"), - )?; - run_command( - &plan.root, - &PlannedCommand::new("git") - .arg("commit") - .arg("-m") - .arg(format!("Bump version to {new_version}")), - )?; - run_command( - &plan.root, - &PlannedCommand::new("git") - .arg("tag") - .arg("-a") - .arg(&tag) - .arg("-m") - .arg(&tag), - )?; - run_command( - &plan.root, - &PlannedCommand::new("git") - .arg("push") - .arg("origin") - .arg("main") - .arg(&tag), - )?; + let tag = plan.commit_tag_and_push(&cargo_toml, versions)?; println!(); println!("Released {tag}"); @@ -156,6 +118,156 @@ impl ReleasePlan { } } + fn compute_versions(&self, cargo_toml: &Path) -> Result { + let current = read_current_version(cargo_toml)?; + let base_version = self.next_base_version()?; + let next = self.compute_release_version(&base_version)?; + let tag = format!("v{next}"); + Ok(ReleaseVersions { current, next, tag }) + } + + /// Commits the version bump, tags it, and pushes `main` plus the tag + /// atomically. When the push is rejected because origin/main moved while + /// the release ran, rebuilds the bump commit and tag on the fresh tip + /// and retries. + #[expect( + clippy::print_stdout, + reason = "dev release command reports push retry progress directly" + )] + fn commit_tag_and_push( + &self, + cargo_toml: &Path, + mut versions: ReleaseVersions, + ) -> Result { + let mut attempt = 1; + loop { + let start_head = self.head_commit()?; + self.create_bump_commit_and_tag(cargo_toml, &versions)?; + let Err(error) = self.push_main_and_tag(&versions.tag) else { + return Ok(versions.tag); + }; + if attempt == MAX_PUSH_ATTEMPTS { + return Err(error); + } + println!( + "Push failed on attempt {attempt} of {MAX_PUSH_ATTEMPTS}; rebuilding the \ + release on the latest origin/main" + ); + self.resync_with_origin_main(&versions.tag, &start_head)?; + versions = self.compute_versions(cargo_toml)?; + println!("Retrying as {} (tag {})", versions.next, versions.tag); + attempt += 1; + } + } + + #[expect( + clippy::print_stdout, + reason = "dev release command reports version bump progress directly" + )] + fn create_bump_commit_and_tag( + &self, + cargo_toml: &Path, + versions: &ReleaseVersions, + ) -> Result<()> { + update_version(cargo_toml, &versions.current, &versions.next)?; + println!("Updated {}", cargo_toml.display()); + + run_command( + &self.root, + &PlannedCommand::new("cargo") + .arg("update") + .arg("--workspace"), + )?; + println!("Updated Cargo.lock"); + + run_command( + &self.root, + &PlannedCommand::new("git") + .arg("add") + .arg("Cargo.toml") + .arg("Cargo.lock"), + )?; + run_command( + &self.root, + &PlannedCommand::new("git") + .arg("commit") + .arg("-m") + .arg(format!("Bump version to {}", versions.next)), + )?; + run_command( + &self.root, + &PlannedCommand::new("git") + .arg("tag") + .arg("-a") + .arg(&versions.tag) + .arg("-m") + .arg(&versions.tag), + ) + } + + fn push_main_and_tag(&self, tag: &str) -> Result<()> { + run_command(&self.root, &Self::push_command(tag)) + } + + /// Drops the bump commit and tag this run created, then fast-forwards + /// onto the updated origin/main. `--ff-only` refuses to discard commits + /// that did not come from origin, so unpushed local work fails loudly + /// instead of being reset away. + fn resync_with_origin_main(&self, tag: &str, start_head: &str) -> Result<()> { + run_command( + &self.root, + &PlannedCommand::new("git").arg("tag").arg("-d").arg(tag), + )?; + run_command( + &self.root, + &PlannedCommand::new("git") + .arg("reset") + .arg("--hard") + .arg(start_head), + )?; + run_command( + &self.root, + &PlannedCommand::new("git") + .arg("fetch") + .arg("--tags") + .arg("origin") + .arg("main"), + )?; + run_command( + &self.root, + &PlannedCommand::new("git") + .arg("merge") + .arg("--ff-only") + .arg("origin/main"), + ) + .context( + "local main has diverged from origin/main; reconcile manually and rerun the release", + ) + } + + fn head_commit(&self) -> Result { + let output = capture_command( + &self.root, + &PlannedCommand::new("git").arg("rev-parse").arg("HEAD"), + )?; + if !output.status.success() { + bail!( + "failed to resolve HEAD: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) + } + + fn push_command(tag: &str) -> PlannedCommand { + PlannedCommand::new("git") + .arg("push") + .arg("--atomic") + .arg("origin") + .arg("main") + .arg(tag) + } + fn ensure_clean_worktree(&self) -> Result<()> { let output = capture_command( &self.root, @@ -195,7 +307,7 @@ impl ReleasePlan { clippy::print_stdout, reason = "dev release command reports dry-run commands directly" )] - fn print_dry_run(&self, current_version: &str, new_version: &str, tag: &str) { + fn print_dry_run(&self, versions: &ReleaseVersions) { println!("DRY RUN: would refresh SPA assets:"); println!("{}", Self::spa_refresh_command().to_shell_line()); @@ -206,7 +318,10 @@ impl ReleasePlan { println!("{}", Self::release_tests_command().to_shell_line()); } - println!("DRY RUN: would update Cargo.toml version {current_version} -> {new_version}"); + println!( + "DRY RUN: would update Cargo.toml version {} -> {}", + versions.current, versions.next + ); for command in [ PlannedCommand::new("cargo") .arg("update") @@ -218,18 +333,14 @@ impl ReleasePlan { PlannedCommand::new("git") .arg("commit") .arg("-m") - .arg(format!("Bump version to {new_version}")), + .arg(format!("Bump version to {}", versions.next)), PlannedCommand::new("git") .arg("tag") .arg("-a") - .arg(tag) + .arg(&versions.tag) .arg("-m") - .arg(tag), - PlannedCommand::new("git") - .arg("push") - .arg("origin") - .arg("main") - .arg(tag), + .arg(&versions.tag), + Self::push_command(&versions.tag), ] { println!("{}", command.to_shell_line()); } @@ -321,3 +432,202 @@ fn workspace_package_version<'a>( ) }) } + +#[cfg(test)] +mod tests { + use super::*; + + const WORKSPACE_MANIFEST: &str = r#"[workspace] +members = ["app"] + +[workspace.package] +version = "0.1.0" +"#; + + const MEMBER_MANIFEST: &str = r#"[package] +name = "app" +edition = "2021" +version.workspace = true +"#; + + fn git(root: &Path, args: &[&str]) -> String { + let mut command = PlannedCommand::new("git"); + for arg in args { + command = command.arg(*arg); + } + let output = capture_command(root, &command).expect("git should spawn"); + assert!( + output.status.success(), + "git {args:?} failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + String::from_utf8_lossy(&output.stdout).trim().to_string() + } + + fn configure_identity(repo: &Path) { + git(repo, &["config", "user.name", "Release Test"]); + git(repo, &["config", "user.email", "release-test@example.com"]); + } + + /// A fixture with a bare `origin`, a `work` clone releases run from, and + /// an `other` clone that simulates concurrent pushes. + struct RaceFixture { + _dir: tempfile::TempDir, + origin: PathBuf, + work: PathBuf, + other: PathBuf, + } + + #[expect( + clippy::disallowed_methods, + reason = "release tests build git fixture repositories synchronously" + )] + fn race_fixture() -> RaceFixture { + let dir = tempfile::tempdir().expect("creating fixture"); + let origin = dir.path().join("origin.git"); + let work = dir.path().join("work"); + let other = dir.path().join("other"); + + std::fs::create_dir(&origin).expect("creating origin dir"); + git(&origin, &["init", "--bare", "-b", "main"]); + + std::fs::create_dir(&work).expect("creating work dir"); + git(&work, &["init", "-b", "main"]); + configure_identity(&work); + std::fs::write(work.join("Cargo.toml"), WORKSPACE_MANIFEST).expect("writing manifest"); + std::fs::create_dir_all(work.join("app/src")).expect("creating member dirs"); + std::fs::write(work.join("app/Cargo.toml"), MEMBER_MANIFEST) + .expect("writing member manifest"); + std::fs::write(work.join("app/src/lib.rs"), "").expect("writing member lib"); + git(&work, &["add", "."]); + git(&work, &["commit", "-m", "initial"]); + git(&work, &[ + "remote", + "add", + "origin", + origin.to_str().expect("origin path should be utf-8"), + ]); + git(&work, &["push", "-u", "origin", "main"]); + + git(dir.path(), &[ + "clone", + origin.to_str().expect("origin path should be utf-8"), + "other", + ]); + configure_identity(&other); + + RaceFixture { + _dir: dir, + origin, + work, + other, + } + } + + #[expect( + clippy::disallowed_methods, + reason = "release tests write fixture files synchronously" + )] + fn write_file(path: &Path, contents: &str) { + std::fs::write(path, contents).expect("writing fixture file"); + } + + fn nightly_plan(root: &Path) -> ReleasePlan { + ReleasePlan { + nightly: true, + release_date: NaiveDate::from_ymd_opt(2026, 1, 1).expect("valid release date"), + dry_run: false, + skip_tests: true, + root: root.to_path_buf(), + } + } + + #[test] + fn push_rebuilds_bump_commit_when_origin_main_moves() { + let fixture = race_fixture(); + + write_file(&fixture.other.join("README.md"), "concurrent\n"); + git(&fixture.other, &["add", "README.md"]); + git(&fixture.other, &["commit", "-m", "concurrent work"]); + git(&fixture.other, &["push", "origin", "main"]); + + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = plan + .commit_tag_and_push(&cargo_toml, versions) + .expect("push should rescue itself when origin/main moves"); + + assert_eq!(tag, "v0.100.0-nightly.0"); + let subjects = git(&fixture.origin, &["log", "--format=%s", "main"]); + assert_eq!(subjects.lines().collect::>(), [ + "Bump version to 0.100.0-nightly.0", + "concurrent work", + "initial" + ]); + git(&fixture.origin, &[ + "rev-parse", + "--verify", + "refs/tags/v0.100.0-nightly.0", + ]); + } + + #[test] + fn push_recomputes_version_when_tag_is_taken() { + let fixture = race_fixture(); + + git(&fixture.other, &["tag", "v0.100.0-nightly.0"]); + git(&fixture.other, &["push", "origin", "v0.100.0-nightly.0"]); + + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = plan + .commit_tag_and_push(&cargo_toml, versions) + .expect("push should rescue itself when the tag is taken"); + + assert_eq!(tag, "v0.100.0-nightly.1"); + git(&fixture.origin, &[ + "rev-parse", + "--verify", + "refs/tags/v0.100.0-nightly.1", + ]); + let subject = git(&fixture.origin, &["log", "-1", "--format=%s", "main"]); + assert_eq!(subject, "Bump version to 0.100.0-nightly.1"); + } + + #[test] + fn push_preserves_unpushed_local_commits_on_divergence() { + let fixture = race_fixture(); + + write_file(&fixture.work.join("local.txt"), "local\n"); + git(&fixture.work, &["add", "local.txt"]); + git(&fixture.work, &["commit", "-m", "unpushed local work"]); + + write_file(&fixture.other.join("README.md"), "concurrent\n"); + git(&fixture.other, &["add", "README.md"]); + git(&fixture.other, &["commit", "-m", "concurrent work"]); + git(&fixture.other, &["push", "origin", "main"]); + + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let error = plan + .commit_tag_and_push(&cargo_toml, versions) + .expect_err("diverged local main should fail instead of being reset away"); + + assert!( + format!("{error:#}").contains("diverged"), + "error should explain the divergence: {error:#}" + ); + let subject = git(&fixture.work, &["log", "-1", "--format=%s"]); + assert_eq!(subject, "unpushed local work"); + } +} From ded92a215d9530cddf5ca61f5329f722527521da Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 10:37:04 -0400 Subject: [PATCH 07/30] Update Venice model catalog --- docs/public/core-concepts/models.mdx | 6 + .../fabro-llm/src/adapter_registry.rs | 92 +------- .../src/catalog/providers/venice.toml | 214 ++++++++++++++++-- 3 files changed, 206 insertions(+), 106 deletions(-) diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index df78987d7..e3bd5249e 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -59,13 +59,18 @@ Fabro performs this selection once when creating a run and persists the chosen p | `gemini-3.1-flash-lite` | gemini | `gemini-flash-lite`, `gemini-3.1-flash-lite-preview` | 1M | $0.25 / $1.50 | 200 tok/s | | `kimi-k2.5` | moonshot | | 262K | $0.60 / $3.00 | 50 tok/s | | `kimi-k3` | moonshot | `kimi` | 1M | $3.00 / $15.00 | n/a | +| `kimi-k3-fast` | venice | `kimi-fast` | 1M | $4.50 / $22.50 | n/a | | `deepseek-v4-flash` | deepseek | `deepseek`, `deepseek-v4`, `deepseek-flash` | 1,048,576 | $0.14 / $0.28 | n/a | | `deepseek-v4-pro` | deepseek | | 1,048,576 | $0.435 / $0.87 | n/a | +| `grok-4.6` | venice | `grok`, `grok46`, `grok-46` | 500K | $2.27 / $6.80 | n/a | | `laguna-s-2.1` | poolside | `laguna`, `laguna-s` | 1M | $0.10 / $0.20 | n/a | | `laguna-xs-2.1` | poolside | `laguna-xs` | 262K | $0.10 / $0.20 | n/a | | `glm-5.2` | zai | `glm`, `glm5`, `glm52`, `glm5.2` | 1M | $1.40 / $4.40 | n/a | +| `glm-5.3` | venice | `glm`, `glm5`, `glm53`, `glm5.3`, `glm-5-3` | 1M | $1.75 / $5.50 | n/a | | `minimax-m2.5` | minimax | `minimax` | 197K | $0.30 / $1.20 | 45 tok/s | | `mercury-2` | inception | `mercury` | 131K | $0.25 / $0.75 | 1000 tok/s | +| `qwen3.8-max` | venice | `qwen`, `qwen-max`, `qwen3.8`, `qwen-3.8`, `qwen38`, `qwen-3.8-max`, `qwen38-max` | 1M | $2.50 / $7.50 | n/a | +| `qwen3.8-27b` | venice | `qwen-27b`, `qwen-3.8-27b`, `qwen38-27b` | 262K | $0.45 / $3.20 | n/a | Each provider requires its own API key. Server-backed workflows read provider credentials from the server vault (for example `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, `GEMINI_API_KEY`, `DEEPSEEK_API_KEY`, or `POOLSIDE_API_KEY` set with `fabro secret set` or `fabro provider login`). Standalone SDK/CLI flows can opt into env-backed credential sources explicitly. See the [Quick Start](/getting-started/quick-start) for setup. @@ -232,6 +237,7 @@ When no model or provider is specified, Fabro chooses the default offering on th | `moonshot` | `kimi-k3` | | `poolside` | `laguna-s-2.1` | | `zai` | `glm-5.2` | +| `venice` | `deepseek-v4-flash` | | `minimax` | `minimax-m2.5` | | `inception` | `mercury-2` | diff --git a/lib/components/fabro-llm/src/adapter_registry.rs b/lib/components/fabro-llm/src/adapter_registry.rs index a306c846b..60ad8ec42 100644 --- a/lib/components/fabro-llm/src/adapter_registry.rs +++ b/lib/components/fabro-llm/src/adapter_registry.rs @@ -269,91 +269,19 @@ mod tests { .unwrap_or_else(|error| panic!("built-in model '{selector}' should resolve: {error}")) } - /// One row of the route-equivalence table: model id plus the - /// `(deployment_id, transport, codec, billing_policy, agent_profile)` - /// tuple it must resolve to. - type RouteRow = ( - &'static str, - &'static str, - AdapterKind, - CodecKind, - BillingPolicy, - AgentProfileKind, - ); - - /// The compat mapping as an executable table: every built-in catalog - /// model resolves to exactly this tuple. Adding or rerouting a built-in - /// model means updating this table deliberately. #[test] - fn builtin_catalog_route_equivalence_table() { - use AdapterKind as T; - use AgentProfileKind as P; - use BillingPolicy as B; - use CodecKind as C; - - #[rustfmt::skip] - let expected: &[RouteRow] = &[ - // model id deployment_id transport codec billing profile - ("claude-fable-5", "claude-fable-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Claude5), - ("claude-haiku-4-5", "claude-haiku-4-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-opus-4-6", "claude-opus-4-6", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-opus-4-7", "claude-opus-4-7", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-opus-4-8", "claude-opus-4-8", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-opus-5", "claude-opus-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Claude5), - ("claude-sonnet-4-5", "claude-sonnet-4-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-sonnet-4-6", "claude-sonnet-4-6", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Anthropic), - ("claude-sonnet-5", "claude-sonnet-5", T::Anthropic, C::AnthropicMessages, B::Anthropic, P::Claude5), - ("deepseek-v4-flash", "deepseek-v4-flash", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("deepseek-v4-pro", "deepseek-v4-pro", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("gemini-3-flash-preview", "gemini-3-flash-preview", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini), - ("gemini-3.1-flash-lite", "gemini-3.1-flash-lite", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini), - ("gemini-3.1-pro-preview", "gemini-3.1-pro-preview", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini), - ("gemini-3.1-pro-preview-customtools", "gemini-3.1-pro-preview-customtools", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini), - ("gemini-3.5-flash", "gemini-3.5-flash", T::Gemini, C::GeminiGenerate, B::Gemini, P::Gemini), - ("glm-4.7", "glm-4.7", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("glm-5.2", "glm-5.2", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("gpt-5.4", "gpt-5.4", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi), - ("gpt-5.4-mini", "gpt-5.4-mini", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi), - ("gpt-5.4-pro", "gpt-5.4-pro", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi), - ("gpt-5.5", "gpt-5.5", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi), - ("gpt-5.5-pro", "gpt-5.5-pro", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::OpenAi), - ("gpt-5.6-luna", "gpt-5.6-luna", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::Gpt56), - ("gpt-5.6-sol", "gpt-5.6-sol", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::Gpt56), - ("gpt-5.6-terra", "gpt-5.6-terra", T::OpenAi, C::OpenAiResponses, B::OpenAi, P::Gpt56), - ("kimi-k2.5", "kimi-k2.5", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::Kimi), - ("kimi-k3", "kimi-k3", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::Kimi), - ("laguna-s-2.1", "poolside/laguna-s-2.1", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("laguna-xs-2.1", "poolside/laguna-xs-2.1", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("mercury-2", "mercury-2", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("minimax-m2.5", "minimax-m2.5", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("venice-uncensored-1-2", "venice-uncensored-1-2", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ("venice-uncensored-role-play", "venice-uncensored-role-play", T::OpenAiCompatible, C::OpenAiCompatible, B::OpenAi, P::OpenAi), - ]; - + fn every_builtin_catalog_offering_resolves() { let catalog = Catalog::builtin(); - let mut model_ids: Vec<&str> = catalog - .list(None) - .iter() - .map(|model| model.id.as_str()) - .collect(); - model_ids.sort_unstable(); - let mut expected_ids: Vec<&str> = expected.iter().map(|row| row.0).collect(); - expected_ids.sort_unstable(); - assert_eq!( - model_ids, expected_ids, - "route-equivalence table must cover every built-in model row" - ); - - for (model_id, deployment_id, transport, codec, billing_policy, agent_profile) in expected { - let model = select_from_all(catalog, model_id); - let route = resolve_route(catalog, model) - .unwrap_or_else(|| panic!("built-in model '{model_id}' should resolve")); - assert_eq!(route.deployment_id, *deployment_id, "{model_id}"); - assert_eq!(route.transport, *transport, "{model_id}"); - assert_eq!(route.codec, *codec, "{model_id}"); - assert_eq!(route.billing_policy, *billing_policy, "{model_id}"); - assert_eq!(route.agent_profile, *agent_profile, "{model_id}"); + for model in catalog.list(None) { + let route = resolve_route(catalog, model).unwrap_or_else(|| { + panic!( + "built-in offering '{}/{}' should resolve", + model.provider, model.id + ) + }); + assert_eq!(route.provider, model.provider); + assert!(!route.deployment_id.is_empty()); } } diff --git a/lib/foundation/fabro-model/src/catalog/providers/venice.toml b/lib/foundation/fabro-model/src/catalog/providers/venice.toml index dc97c4ab4..9791591e0 100644 --- a/lib/foundation/fabro-model/src/catalog/providers/venice.toml +++ b/lib/foundation/fabro-model/src/catalog/providers/venice.toml @@ -1,46 +1,212 @@ +# Model IDs, capabilities, contexts, and prices are from Venice's published +# model catalog, verified 2026-08-21: +# https://github.com/veniceai/api-docs/blob/59a300b1d036c0c0acc0e5f75c0ab0dd07c40c1c/data/static-models.json + [providers.venice] display_name = "Venice" adapter = "openai_compatible" base_url = "https://api.venice.ai/api/v1" priority = 35 aliases = ["venice-ai"] +billing_policy = "openai" [providers.venice.auth] credentials = ["env:VENICE_API_KEY", "vault:VENICE_API_KEY"] -[providers.venice.models."venice-uncensored-1-2"] -display_name = "Venice Uncensored 1.2" -family = "venice-uncensored" +[providers.venice.models."kimi-k3"] +display_name = "Kimi K3" +family = "kimi-k3" +agent_profile = "kimi" +aliases = ["kimi"] + +[providers.venice.models."kimi-k3".limits] +context_window = 1000000 +max_output = 131072 + +[providers.venice.models."kimi-k3".features] +tools = true +vision = true +reasoning = true +reasoning_by_default = true +prompt_cache = true +sampling_params = false + +[providers.venice.models."kimi-k3".costs] +input_cost_per_mtok = 3.75 +output_cost_per_mtok = 18.75 +cache_input_cost_per_mtok = 0.375 + +[providers.venice.models."kimi-k3-fast"] +api_id = "kimi-k3-fast-api" +display_name = "Kimi K3 Fast" +family = "kimi-k3" +agent_profile = "kimi" +aliases = ["kimi-fast"] + +[providers.venice.models."kimi-k3-fast".limits] +context_window = 1000000 +max_output = 131072 + +[providers.venice.models."kimi-k3-fast".features] +tools = true +vision = true +reasoning = true +reasoning_by_default = true +prompt_cache = true +sampling_params = false + +[providers.venice.models."kimi-k3-fast".costs] +input_cost_per_mtok = 4.5 +output_cost_per_mtok = 22.5 +cache_input_cost_per_mtok = 0.45 + +[providers.venice.models."grok-4.6"] +api_id = "grok-4-6" +display_name = "Grok 4.6" +family = "grok-4" +aliases = ["grok", "grok46", "grok-46"] + +[providers.venice.models."grok-4.6".limits] +context_window = 500000 +max_output = 32000 + +[providers.venice.models."grok-4.6".features] +tools = true +vision = true +reasoning = true +reasoning_effort = "levels" +reasoning_by_default = true +prompt_cache = true + +[providers.venice.models."grok-4.6".controls] +reasoning_effort = ["low", "medium", "high", "xhigh"] + +[providers.venice.models."grok-4.6".costs] +input_cost_per_mtok = 2.27 +output_cost_per_mtok = 6.8 +cache_input_cost_per_mtok = 0.57 + +[providers.venice.models."glm-5.3"] +api_id = "z-ai-glm-5-3" +display_name = "GLM 5.3" +family = "glm-5" +aliases = ["glm", "glm5", "glm53", "glm5.3", "glm-5-3"] + +[providers.venice.models."glm-5.3".limits] +context_window = 1000000 +max_output = 131072 + +[providers.venice.models."glm-5.3".features] +tools = true +vision = false +reasoning = true +reasoning_effort = "levels" +reasoning_by_default = true +prompt_cache = true + +[providers.venice.models."glm-5.3".controls] +reasoning_effort = ["low", "high", "max"] + +[providers.venice.models."glm-5.3".costs] +input_cost_per_mtok = 1.75 +output_cost_per_mtok = 5.5 +cache_input_cost_per_mtok = 0.325 + +[providers.venice.models."deepseek-v4-flash"] +api_id = "deepseek-v4-flash-0731" +display_name = "DeepSeek V4 Flash" +family = "deepseek-v4" +agent_profile = "openai" default = true -aliases = ["venice-uncensored", "vu"] +aliases = ["deepseek-v4", "deepseek", "deepseek-flash"] -[providers.venice.models."venice-uncensored-1-2".limits] -context_window = 128000 -max_output = 8192 +[providers.venice.models."deepseek-v4-flash".limits] +context_window = 1000000 +max_output = 32768 -[providers.venice.models."venice-uncensored-1-2".features] +[providers.venice.models."deepseek-v4-flash".features] +tools = true +vision = false +reasoning = true +reasoning_effort = "levels" +reasoning_by_default = true +prompt_cache = true +sampling_params = false + +[providers.venice.models."deepseek-v4-flash".controls] +reasoning_effort = ["low", "high", "max"] + +[providers.venice.models."deepseek-v4-flash".costs] +input_cost_per_mtok = 0.175 +output_cost_per_mtok = 0.35 +cache_input_cost_per_mtok = 0.035 + +[providers.venice.models."deepseek-v4-pro"] +api_id = "deepseek-v4-pro-0813" +display_name = "DeepSeek V4 Pro" +family = "deepseek-v4" +agent_profile = "openai" +aliases = ["deepseek-pro"] + +[providers.venice.models."deepseek-v4-pro".limits] +context_window = 1000000 +max_output = 32768 + +[providers.venice.models."deepseek-v4-pro".features] +tools = true +vision = false +reasoning = true +reasoning_by_default = true +prompt_cache = true +sampling_params = false + +[providers.venice.models."deepseek-v4-pro".costs] +input_cost_per_mtok = 1.65 +output_cost_per_mtok = 4.95 +cache_input_cost_per_mtok = 0.165 + +[providers.venice.models."qwen3.8-max"] +api_id = "qwen-3-8-max" +display_name = "Qwen 3.8 Max" +family = "qwen3" +aliases = ["qwen", "qwen-max", "qwen3.8", "qwen-3.8", "qwen38", "qwen-3.8-max", "qwen38-max"] + +[providers.venice.models."qwen3.8-max".limits] +context_window = 1000000 +max_output = 131072 + +[providers.venice.models."qwen3.8-max".features] tools = true vision = true -reasoning = false +reasoning = true +reasoning_by_default = true +prompt_cache = true -[providers.venice.models."venice-uncensored-1-2".costs] -input_cost_per_mtok = 0.2 -output_cost_per_mtok = 0.9 +[providers.venice.models."qwen3.8-max".costs] +input_cost_per_mtok = 2.5 +output_cost_per_mtok = 7.5 +cache_input_cost_per_mtok = 0.3125 -[providers.venice.models."venice-uncensored-role-play"] -display_name = "Venice Uncensored Role Play" -family = "venice-uncensored" -aliases = ["venice-roleplay", "vrp"] +[providers.venice.models."qwen3.8-27b"] +api_id = "qwen-3-8-27b" +display_name = "Qwen 3.8 27B" +family = "qwen3.8" +aliases = ["qwen-27b", "qwen-3.8-27b", "qwen38-27b"] -[providers.venice.models."venice-uncensored-role-play".limits] -context_window = 128000 -max_output = 4096 +[providers.venice.models."qwen3.8-27b".limits] +context_window = 262144 +max_output = 131072 -[providers.venice.models."venice-uncensored-role-play".features] +[providers.venice.models."qwen3.8-27b".features] tools = true vision = true -reasoning = false +reasoning = true +reasoning_effort = "levels" +reasoning_by_default = true -[providers.venice.models."venice-uncensored-role-play".costs] -input_cost_per_mtok = 0.5 -output_cost_per_mtok = 2.0 +[providers.venice.models."qwen3.8-27b".controls] +reasoning_effort = ["low", "medium", "xhigh"] + +[providers.venice.models."qwen3.8-27b".costs] +input_cost_per_mtok = 0.45 +output_cost_per_mtok = 3.2 From ff1ca976c3397428b882836b1ec50654761d025a Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 10:48:03 -0400 Subject: [PATCH 08/30] Document Venice model integration --- docs/public/core-concepts/models.mdx | 4 + docs/public/docs.json | 1 + docs/public/integrations/venice.mdx | 125 +++++++++++++++++++++++++++ 3 files changed, 130 insertions(+) create mode 100644 docs/public/integrations/venice.mdx diff --git a/docs/public/core-concepts/models.mdx b/docs/public/core-concepts/models.mdx index e3bd5249e..d471e18fd 100644 --- a/docs/public/core-concepts/models.mdx +++ b/docs/public/core-concepts/models.mdx @@ -174,6 +174,10 @@ Provider `billing_policy` defaults from `adapter` and controls usage-cost estima Provider fields in configuration, APIs, and model routing are provider ID strings. Built-in names like `anthropic`, `openai`, and `gemini` still work, but custom IDs like `proxy` work anywhere a provider ID is accepted. +### Venice + +Fabro ships a built-in [Venice](/integrations/venice) provider with a curated catalog of Venice-hosted Kimi, Grok, GLM, DeepSeek, and Qwen models. Store its API key with `fabro provider login --provider venice`. Pin `provider = "venice"` when a shared model slug must use Venice instead of a higher-priority direct provider. + ### Poolside Fabro ships a built-in [Poolside](/integrations/poolside) provider for Laguna S 2.1 and Laguna XS 2.1 over Poolside's OpenAI-compatible API. Store a direct API key with `fabro provider login --provider poolside`. The same model slugs are also available through the opt-in OpenRouter provider; its vendor-namespaced strings remain provider-only `api_id` values. diff --git a/docs/public/docs.json b/docs/public/docs.json index faf0d0f84..284e5e4cb 100644 --- a/docs/public/docs.json +++ b/docs/public/docs.json @@ -97,6 +97,7 @@ "integrations/litellm", "integrations/bedrock", "integrations/deepseek", + "integrations/venice", "integrations/poolside", "integrations/openrouter", "integrations/modal", diff --git a/docs/public/integrations/venice.mdx b/docs/public/integrations/venice.mdx new file mode 100644 index 000000000..f172ccb34 --- /dev/null +++ b/docs/public/integrations/venice.mdx @@ -0,0 +1,125 @@ +--- +title: "Venice" +description: "Run Kimi, Grok, GLM, DeepSeek, and Qwen models through Venice" +--- + +[Venice](https://venice.ai/) provides an OpenAI-compatible API for hosted text models. Fabro enables the `venice` provider in its built-in catalog and maps stable Fabro model slugs to Venice's API model IDs. + +## Prerequisites + +- A Venice account +- An inference API key from [venice.ai/settings/api](https://venice.ai/settings/api) +- A running Fabro server + +## Configure credentials + +Store the API key in the target Fabro server vault: + +```bash +fabro provider login --provider venice + +# For a non-default remote server: +fabro provider login --server https://your-fabro.example --provider venice + +# Or set the vault token directly: +fabro secret set VENICE_API_KEY +fabro secret --server https://your-fabro.example set VENICE_API_KEY +``` + +Standalone SDK usage outside a Fabro server can use an env-backed credential source explicitly: + +```bash +export VENICE_API_KEY= +``` + +Fabro sends bearer-authenticated Chat Completions requests to `https://api.venice.ai/api/v1`. + +## Included models + +| Fabro model slug | Venice API ID | Context | Max output | Role and aliases | +|---|---|---:|---:|---| +| `kimi-k3` | `kimi-k3` | 1,000,000 | 131,072 | Alias `kimi` | +| `kimi-k3-fast` | `kimi-k3-fast-api` | 1,000,000 | 131,072 | Alias `kimi-fast` | +| `grok-4.6` | `grok-4-6` | 500,000 | 32,000 | Aliases `grok`, `grok46`, `grok-46` | +| `glm-5.3` | `z-ai-glm-5-3` | 1,000,000 | 131,072 | Aliases `glm`, `glm5`, `glm53`, `glm5.3`, `glm-5-3` | +| `deepseek-v4-flash` | `deepseek-v4-flash-0731` | 1,000,000 | 32,768 | Provider default; aliases `deepseek`, `deepseek-v4`, `deepseek-flash` | +| `deepseek-v4-pro` | `deepseek-v4-pro-0813` | 1,000,000 | 32,768 | Alias `deepseek-pro` | +| `qwen3.8-max` | `qwen-3-8-max` | 1,000,000 | 131,072 | Aliases `qwen`, `qwen-max`, `qwen3.8`, `qwen-3.8`, `qwen38`, `qwen-3.8-max`, `qwen38-max` | +| `qwen3.8-27b` | `qwen-3-8-27b` | 262,144 | 131,072 | Aliases `qwen-27b`, `qwen-3.8-27b`, `qwen38-27b` | + +Venice API IDs are also valid provider-scoped selectors. Fabro persists the stable Fabro slug and the selected provider when it creates a run. + +## Select Venice explicitly + +Some Venice models use the same stable slugs as direct providers. An unqualified selector chooses the highest-priority ready provider. For example, `deepseek` can select the direct DeepSeek provider when both API keys are configured. + +Pin Venice when the run must use Venice: + +```bash +fabro model list --provider venice +fabro model test --provider venice --model deepseek-v4-flash --deep +fabro run workflow.fabro --provider venice --model deepseek-v4-flash +``` + +In a workflow stylesheet: + +```dot title="workflow.fabro" +digraph Example { + graph [ + model_stylesheet=" + * { provider: venice; model: deepseek-v4-flash; } + .complex { provider: venice; model: qwen; } + .fast { provider: venice; model: kimi-fast; } + " + ] + + start [shape=Mdiamond, label="Start"] + work [label="Implement", class="complex"] + check [label="Check", class="fast"] + exit [shape=Msquare, label="Exit"] + + start -> work -> check -> exit +} +``` + +The generic Qwen aliases `qwen` and `qwen3.8` select Qwen 3.8 Max. Use a size-specific alias such as `qwen-27b` to select Qwen 3.8 27B. + +## Capabilities and reasoning + +All included models support tool calling and reasoning. Kimi K3, Kimi K3 Fast, Grok 4.6, Qwen 3.8 Max, and Qwen 3.8 27B also accept image input. + +Fabro exposes native reasoning-effort controls only when Venice supports them: + +| Model | Reasoning effort values | +|---|---| +| `grok-4.6` | `low`, `medium`, `high`, `xhigh` | +| `glm-5.3` | `low`, `high`, `max` | +| `deepseek-v4-flash` | `low`, `high`, `max` | +| `qwen3.8-27b` | `low`, `medium`, `xhigh` | + +The other models reason by default but do not expose a Venice reasoning-effort control. Fabro omits sampling parameters for Kimi and DeepSeek because those routes do not use them with their configured reasoning behavior. + +## Pricing and prompt caching + +The built-in catalog uses Venice's published prices per million tokens: + +| Model | Uncached input | Cache hit | Output | +|---|---:|---:|---:| +| `kimi-k3` | $3.75 | $0.375 | $18.75 | +| `kimi-k3-fast` | $4.50 | $0.45 | $22.50 | +| `grok-4.6` | $2.27 | $0.57 | $6.80 | +| `glm-5.3` | $1.75 | $0.325 | $5.50 | +| `deepseek-v4-flash` | $0.175 | $0.035 | $0.35 | +| `deepseek-v4-pro` | $1.65 | $0.165 | $4.95 | +| `qwen3.8-max` | $2.50 | $0.3125 | $7.50 | +| `qwen3.8-27b` | $0.45 | n/a | $3.20 | + +Fabro reports cached input separately when Venice returns cache usage for the selected model. Prices and model availability can change upstream; use `fabro model list --provider venice` to inspect the catalog shipped with your Fabro version and the [Venice model catalog](https://docs.venice.ai/models/overview) for the current upstream service. + +## Troubleshooting + +**"No credential was found for provider 'venice'"** — Store `VENICE_API_KEY` in the server vault with `fabro provider login --provider venice`. Pass `--server` when configuring a remote Fabro server. + +**A shared model used another provider** — Pin Venice with `--provider venice` or `provider: venice` in the workflow stylesheet. Unqualified selectors use provider priority. + +**A Venice API model ID is rejected without a provider** — Use the stable Fabro slug for portable selection, or qualify the API ID with the provider, such as `venice:qwen-3-8-max`. From db1faf02ec345fd690a86457b80812260f0b17ea Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 11:19:28 -0400 Subject: [PATCH 09/30] Fix catalog dispatch invariant for shared models --- lib/components/fabro-llm/src/client.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/components/fabro-llm/src/client.rs b/lib/components/fabro-llm/src/client.rs index fc2802be1..59cfa421d 100644 --- a/lib/components/fabro-llm/src/client.rs +++ b/lib/components/fabro-llm/src/client.rs @@ -2037,17 +2037,20 @@ reasoning = false client } - /// Live-dispatch counterpart of the adapter_registry route-equivalence - /// table: for every built-in model, `resolve_provider` lands on the same - /// provider the resolved route names. + /// For every built-in model selector, live dispatch and catalog selection + /// choose the same provider from the same ready-provider set. #[tokio::test] async fn dispatch_agrees_with_resolve_route_for_every_builtin_model() { let catalog = catalog_with(""); let client = client_with_all_catalog_providers(&catalog).await; + let ready_providers = catalog.all_provider_ids(); for model in catalog.list(None) { - let route = adapter_registry::resolve_route(&catalog, model) - .expect("built-in model should resolve to a route"); + let selected = catalog + .select(model.id.as_str(), None, &ready_providers) + .expect("built-in model should be selectable"); + let route = adapter_registry::resolve_route(&catalog, selected) + .expect("selected built-in model should resolve to a route"); let mut request = test_request(); request.model = model.id.to_string(); From 5104a787ce93e6851ff510d45dd6beccfaad3823 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Fri, 21 Aug 2026 12:08:23 -0400 Subject: [PATCH 10/30] Bound Daytona post-clone setup --- .../fabro-sandbox/src/daytona/mod.rs | 509 +++++++++++++++--- .../fabro-sandbox/src/provider/daytona.rs | 9 +- 2 files changed, 431 insertions(+), 87 deletions(-) diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index c18cdad27..8b14ee462 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -64,6 +64,23 @@ pub(crate) const DAYTONA_DASHBOARD_SANDBOXES_URL: &str = const FABRO_SANDBOX_USER_AGENT: &str = concat!("fabro-sandbox/", env!("CARGO_PKG_VERSION")); const DAYTONA_PROBE_TIMEOUT: Duration = Duration::from_secs(20); const DAYTONA_START_TIMEOUT: Duration = Duration::from_mins(1); +/// Shared budget for required setup after Daytona's native clone returns. +/// +/// The native clone has its own provider lifecycle. Starting this deadline +/// afterward prevents a slow successful clone from consuming the budget for +/// required branch attachment and workspace linking. +const DAYTONA_POST_CLONE_SETUP_TIMEOUT: Duration = Duration::from_mins(5); +/// Best-effort push-credential setup should not consume or extend the required +/// post-clone setup budget. +const DAYTONA_CREDENTIAL_SETUP_TIMEOUT: Duration = Duration::from_secs(10); +/// Grace for the toolbox to return the server-side command timeout response. +/// The SDK truncates the server timeout to whole seconds, so the server can +/// fire up to one second before the shared deadline; this additional grace +/// lets that response reach the client afterward. +const DAYTONA_CLIENT_TIMEOUT_GRACE: Duration = Duration::from_secs(1); +/// The Daytona SDK serializes command timeouts as whole seconds. Do not send a +/// zero-second timeout when the shared deadline is nearly exhausted. +const DAYTONA_MIN_SERVER_TIMEOUT: Duration = Duration::from_secs(1); /// Upper bound on explicit and Drop-triggered Daytona cleanup calls (session /// deletion, temporary stdin files) so a stalled REST call cannot block /// cancellation/timeout paths indefinitely. @@ -83,6 +100,10 @@ fn daytona_git_clone_options( } } +pub(crate) fn daytona_not_found(err: &DaytonaError) -> bool { + matches!(err, DaytonaError::NotFound { .. }) || err.status_code() == Some(404) +} + /// Permissions a Daytona API key needs for Fabro's snapshot and sandbox flow. pub const REQUIRED_DAYTONA_PERMISSIONS: &[Permissions] = &[ Permissions::WriteColonSnapshots, @@ -531,6 +552,20 @@ impl DaytonaSandbox { err } + /// Report a clone/setup failure, clean up the created sandbox, and emit + /// the matching initialization failure. + async fn fail_clone_initialization( + &self, + sandbox: daytona_sdk::Sandbox, + origin_url: &str, + init_start: Instant, + err: crate::Error, + ) -> crate::Error { + let err = self.report_clone_failure(origin_url, err); + let err = self.finish_failed_initialization(sandbox, err).await; + self.fail_init(init_start, err) + } + /// Point the admitted branch at the exact commit and verify the resulting /// HEAD. /// @@ -543,46 +578,85 @@ impl DaytonaSandbox { checkout_path: &str, branch: &str, expected_sha: &str, + deadline: time::Instant, ) -> crate::Result<()> { - Self::run_clone_step( + Self::run_required_post_clone_command( process_svc, &clone_source::exact_branch_checkout_command(checkout_path, branch, expected_sha), + "/", "git checkout exact commit", + deadline, ) .await?; - let head = Self::run_clone_step( + let head = Self::run_required_post_clone_command( process_svc, &clone_source::exact_head_revision_command(checkout_path), + "/", "git rev-parse HEAD after exact checkout", + deadline, ) .await?; clone_source::verify_exact_head(&head, expected_sha) } - /// Run one local git step of the clone in the sandbox and return its - /// output. - async fn run_clone_step( + /// Execute one post-clone command under the shared setup deadline. + /// + /// The SDK timeout asks Daytona to terminate the remote process. The outer + /// timeout is a transport backstop in case the toolbox never returns that + /// result. Callers must not retry after a timeout because the remote + /// process may still be winding down. + async fn execute_post_clone_command( process_svc: &daytona_sdk::ProcessService, command: &str, + cwd: &str, label: &'static str, + deadline: time::Instant, + ) -> crate::Result { + let remaining = deadline.saturating_duration_since(time::Instant::now()); + if remaining < DAYTONA_MIN_SERVER_TIMEOUT { + return Err(crate::Error::message(format!( + "Daytona post-clone setup deadline expired before {label}" + ))); + } + + let wrapped = wrap_bash_command(command); + let options = daytona_sdk::ExecuteCommandOptions { + cwd: Some(cwd.to_string()), + timeout: Some(remaining), + ..Default::default() + }; + let execution = process_svc.execute_command(&wrapped, options); + time::timeout( + remaining.saturating_add(DAYTONA_CLIENT_TIMEOUT_GRACE), + execution, + ) + .await + .map_err(|_| { + crate::Error::message(format!( + "Daytona post-clone setup timed out while running {label}" + )) + })? + .map_err(|e| crate::Error::context(format!("Failed to run {label}"), e)) + } + + /// Run one required local step after the native clone and return stdout. + async fn run_required_post_clone_command( + process_svc: &daytona_sdk::ProcessService, + command: &str, + cwd: &str, + label: &'static str, + deadline: time::Instant, ) -> crate::Result { - let result = process_svc - .execute_command( - &wrap_bash_command(command), - daytona_sdk::ExecuteCommandOptions { - cwd: Some("/".to_string()), - ..Default::default() - }, - ) - .await - .map_err(|e| crate::Error::context(format!("Failed to run {label}"), e))?; + let start = Instant::now(); + let result = + Self::execute_post_clone_command(process_svc, command, cwd, label, deadline).await?; if result.exit_code != 0 { return Err(crate::Error::exec(label, ExecResult { stdout: result.result, stderr: String::new(), exit_code: Some(result.exit_code), termination: CommandTermination::Exited, - duration_ms: 0, + duration_ms: elapsed_ms(start), })); } Ok(result.result) @@ -754,22 +828,60 @@ impl DaytonaSandbox { }) } - /// Discard a sandbox that failed its Bash probe. + /// Discard a sandbox whose initialization failed after creation. /// - /// A failed cleanup is logged rather than returned: the Bash failure is - /// what the operator needs to act on. - async fn delete_unusable_sandbox( - sandbox: &daytona_sdk::Sandbox, - bash_error: crate::Error, - ) -> crate::Error { - if let Err(cleanup_error) = sandbox.delete().await { - tracing::warn!( - error = %cleanup_error, - sandbox = %sandbox.name, - "Failed to delete Daytona sandbox after its Bash check failed" - ); + /// A failed cleanup is logged rather than returned: the initialization + /// failure is what the operator needs to act on. The SDK handle is + /// returned only when the caller should retain it for a lifecycle-level + /// cleanup retry. + async fn cleanup_failed_initialization_sandbox( + sandbox: daytona_sdk::Sandbox, + initialization_error: crate::Error, + ) -> (crate::Error, Option) { + match Self::delete_daytona_sandbox(&sandbox).await { + Ok(()) => (initialization_error, None), + Err(cleanup_error) => { + tracing::warn!( + error = %crate::display_for_log(&cleanup_error), + sandbox = %sandbox.name, + "Failed to delete Daytona sandbox after initialization failed" + ); + (initialization_error, Some(sandbox)) + } } - bash_error + } + + async fn delete_daytona_sandbox(sandbox: &daytona_sdk::Sandbox) -> crate::Result<()> { + match time::timeout(DAYTONA_CLEANUP_TIMEOUT, sandbox.delete()).await { + Ok(Ok(())) => Ok(()), + Ok(Err(err)) if daytona_not_found(&err) => Ok(()), + Ok(Err(err)) => Err(crate::Error::context( + "Failed to delete Daytona sandbox", + err, + )), + Err(_) => Err(crate::Error::message(format!( + "Timed out deleting Daytona sandbox after {}s", + DAYTONA_CLEANUP_TIMEOUT.as_secs() + ))), + } + } + + async fn finish_failed_initialization( + &self, + sandbox: daytona_sdk::Sandbox, + initialization_error: crate::Error, + ) -> crate::Error { + let (initialization_error, retry_sandbox) = + Self::cleanup_failed_initialization_sandbox(sandbox, initialization_error).await; + if let Some(sandbox) = retry_sandbox { + if self.sandbox.set(sandbox).is_err() { + tracing::warn!( + "Failed to retain Daytona sandbox handle after initialization cleanup \ + failed" + ); + } + } + initialization_error } /// Get the sandbox, returning an error if not yet initialized. @@ -1108,7 +1220,7 @@ impl Sandbox for DaytonaSandbox { })?; if let Err(bash_error) = Self::probe_bash(&sandbox).await { - let err = Self::delete_unusable_sandbox(&sandbox, bash_error).await; + let err = self.finish_failed_initialization(sandbox, bash_error).await; return Err(self.fail_init(init_start, err)); } @@ -1267,22 +1379,29 @@ impl Sandbox for DaytonaSandbox { GitHub App with `fabro install` and install it for your organization.", e, ); - let err = self.report_clone_failure(&origin_url, err); - return Err(self.fail_init(init_start, err)); + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); } Err(e) => { let err = crate::Error::context("Failed to clone repo into Daytona sandbox", e); - let err = self.report_clone_failure(&origin_url, err); - return Err(self.fail_init(init_start, err)); + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); } } - let process_svc = sandbox.process().await.map_err(|e| { - let err = crate::Error::context("Failed to get Daytona process service", e); - let err = self.report_clone_failure(&origin_url, err); - self.fail_init(init_start, err) - })?; + let post_clone_deadline = time::Instant::now() + DAYTONA_POST_CLONE_SETUP_TIMEOUT; + let process_svc = match sandbox.process().await { + Ok(process_svc) => process_svc, + Err(e) => { + let err = crate::Error::context("Failed to get Daytona process service", e); + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); + } + }; if let Some(expected_sha) = commit_sha.as_deref() { let Some(branch) = branch.as_deref().filter(|branch| !branch.trim().is_empty()) @@ -1290,51 +1409,38 @@ impl Sandbox for DaytonaSandbox { let err = crate::Error::message( "Exact commit checkout requires a repository branch", ); - let err = self.report_clone_failure(&origin_url, err); - return Err(self.fail_init(init_start, err)); + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); }; if let Err(err) = Self::attach_exact_commit_branch( &process_svc, &layout.primary_repo_path, branch, expected_sha, + post_clone_deadline, ) .await { - let err = self.report_clone_failure(&origin_url, err); - return Err(self.fail_init(init_start, err)); + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); } } let symlink_cmd = clone_source::repo_symlink_command(&layout); - let symlink_result = process_svc - .execute_command( - &wrap_bash_command(&symlink_cmd), - daytona_sdk::ExecuteCommandOptions { - cwd: Some("/".to_string()), - ..Default::default() - }, - ) - .await - .map_err(|e| { - let err = crate::Error::context( - "Failed to create Daytona workspace repo symlink", - e, - ); - let err = self.report_clone_failure(&origin_url, err); - self.fail_init(init_start, err) - })?; - if symlink_result.exit_code != 0 { - let err = - crate::Error::exec("create Daytona workspace repo symlink", ExecResult { - stdout: symlink_result.result.clone(), - stderr: String::new(), - exit_code: Some(symlink_result.exit_code), - termination: CommandTermination::Exited, - duration_ms: 0, - }); - let err = self.report_clone_failure(&origin_url, err); - return Err(self.fail_init(init_start, err)); + if let Err(err) = Self::run_required_post_clone_command( + &process_svc, + &symlink_cmd, + "/", + "create Daytona workspace repo symlink", + post_clone_deadline, + ) + .await + { + return Err(self + .fail_clone_initialization(sandbox, &origin_url, init_start, err) + .await); } let clone_duration = @@ -1351,16 +1457,21 @@ impl Sandbox for DaytonaSandbox { if let Some(token) = password.as_deref() { match fabro_github::embed_token_in_url(&origin_url, token) { Ok(auth_url) => { + let credential_deadline = + time::Instant::now() + DAYTONA_CREDENTIAL_SETUP_TIMEOUT; let cmd = format!( "git -c maintenance.auto=0 remote set-url origin {}", shell_quote(auth_url.as_raw_url().as_str()), ); - let opts = daytona_sdk::ExecuteCommandOptions { - cwd: Some(layout.execution_directory.clone()), - ..Default::default() - }; - let wrapped = wrap_bash_command(&cmd); - match process_svc.execute_command(&wrapped, opts).await { + match Self::execute_post_clone_command( + &process_svc, + &cmd, + &layout.execution_directory, + "git remote set-url origin (Daytona post-clone)", + credential_deadline, + ) + .await + { Ok(r) if r.exit_code != 0 => { let err = crate::Error::exec( "git remote set-url origin (Daytona post-clone)", @@ -1507,8 +1618,7 @@ impl Sandbox for DaytonaSandbox { let start = Instant::now(); if let Some(sandbox) = self.sandbox.get() { tracing::info!("Deleting Daytona sandbox"); - if let Err(e) = sandbox.delete().await { - let err = crate::Error::context("Failed to delete Daytona sandbox", e); + if let Err(err) = Self::delete_daytona_sandbox(sandbox).await { self.emit(SandboxEvent::DeleteFailed { provider: "daytona".into(), error: err.to_string(), @@ -2839,6 +2949,245 @@ mod tests { assert_eq!(options.password.as_deref(), Some("secret")); } + fn mock_sandbox_body(sandbox_id: &str) -> serde_json::Value { + serde_json::json!({ + "id": sandbox_id, + "organizationId": "org-1", + "name": sandbox_id, + "user": "daytona", + "env": {}, + "labels": {}, + "public": false, + "networkBlockAll": false, + "target": "us", + "cpu": 2.0, + "gpu": 0.0, + "memory": 4.0, + "disk": 20.0, + "state": "started" + }) + } + + async fn mock_sandbox_handle(server: &MockServer, sandbox_id: &str) -> daytona_sdk::Sandbox { + let sandbox_response = server + .mock_async(|when, then| { + when.method(GET).path(format!("/sandbox/{sandbox_id}")); + then.status(200) + .header("content-type", "application/json") + .json_body(mock_sandbox_body(sandbox_id)); + }) + .await; + let client = build_daytona_client_with( + Some("dtn_test".to_string()), + Some(server.base_url()), + None, + Some(fabro_test::test_http_client()), + ) + .await + .expect("create Daytona client"); + let sandbox = client.get(sandbox_id).await.expect("get mock sandbox"); + sandbox_response.assert_async().await; + sandbox + } + + async fn mock_process_service( + server: &MockServer, + sandbox_id: &str, + ) -> daytona_sdk::ProcessService { + let server_url = server.base_url(); + let toolbox_response = server + .mock_async(|when, then| { + when.method(GET) + .path(format!("/sandbox/{sandbox_id}/toolbox-proxy-url")); + then.status(200) + .header("content-type", "application/json") + .json_body(serde_json::json!({ "url": server_url })); + }) + .await; + let sandbox = mock_sandbox_handle(server, sandbox_id).await; + let process_svc = sandbox.process().await.expect("get process service"); + toolbox_response.assert_async().await; + process_svc + } + + #[tokio::test] + async fn post_clone_command_sends_server_timeout_and_returns_output() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-post-clone-success"; + let process_svc = mock_process_service(&server, sandbox_id).await; + let execute = server + .mock_async(|when, then| { + when.method(POST) + .path(format!("/{sandbox_id}/process/execute")) + .body_includes(r#""cwd":"/work""#) + .body_matches(r#""timeout":[1-9][0-9]*"#); + then.status(200) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "exitCode": 0, + "result": "expected output" + })); + }) + .await; + + let output = DaytonaSandbox::run_required_post_clone_command( + &process_svc, + "git status --short", + "/work", + "inspect exact checkout", + time::Instant::now() + Duration::from_secs(30), + ) + .await + .expect("post-clone command should succeed"); + + assert_eq!(output, "expected output"); + execute.assert_async().await; + } + + #[tokio::test] + async fn expired_post_clone_deadline_does_not_dispatch_command() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-post-clone-expired"; + let process_svc = mock_process_service(&server, sandbox_id).await; + let execute = server + .mock_async(|when, then| { + when.method(POST) + .path(format!("/{sandbox_id}/process/execute")); + then.status(200) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "exitCode": 0, + "result": "unexpected" + })); + }) + .await; + + let error = DaytonaSandbox::run_required_post_clone_command( + &process_svc, + "git status --short", + "/work", + "inspect exact checkout", + time::Instant::now(), + ) + .await + .expect_err("expired deadline should fail before dispatch"); + + assert!(error.to_string().contains("deadline expired")); + execute.assert_calls_async(0).await; + } + + #[tokio::test] + async fn post_clone_command_has_client_side_timeout_backstop() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-post-clone-stalled"; + let process_svc = mock_process_service(&server, sandbox_id).await; + let execute = server + .mock_async(|when, then| { + when.method(POST) + .path(format!("/{sandbox_id}/process/execute")) + .body_matches(r#""timeout":[1-9][0-9]*"#); + then.status(200) + .delay(Duration::from_secs(10)) + .header("content-type", "application/json") + .json_body(serde_json::json!({ + "exitCode": 0, + "result": "too late" + })); + }) + .await; + + let error = DaytonaSandbox::run_required_post_clone_command( + &process_svc, + "git status --short", + "/work", + "inspect exact checkout", + time::Instant::now() + Duration::from_millis(1_200), + ) + .await + .expect_err("stalled toolbox response should hit the client backstop"); + + assert!(error.to_string().contains("timed out")); + execute.assert_async().await; + } + + #[tokio::test] + async fn failed_initialization_deletes_created_daytona_sandbox() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-failed-initialization"; + let delete = server + .mock_async(|when, then| { + when.method(DELETE).path(format!("/sandbox/{sandbox_id}")); + then.status(200) + .header("content-type", "application/json") + .json_body(mock_sandbox_body(sandbox_id)); + }) + .await; + let sandbox = mock_sandbox_handle(&server, sandbox_id).await; + + let (error, retry_sandbox) = DaytonaSandbox::cleanup_failed_initialization_sandbox( + sandbox, + crate::Error::message("exact checkout failed"), + ) + .await; + + assert_eq!(error.to_string(), "exact checkout failed"); + assert!(retry_sandbox.is_none()); + delete.assert_async().await; + } + + #[tokio::test] + async fn failed_initialization_retains_sandbox_when_delete_fails() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-failed-cleanup"; + let delete = server + .mock_async(|when, then| { + when.method(DELETE).path(format!("/sandbox/{sandbox_id}")); + then.status(500) + .header("content-type", "application/json") + .json_body(serde_json::json!({ "message": "try again" })); + }) + .await; + let sandbox = mock_sandbox_handle(&server, sandbox_id).await; + + let (error, retry_sandbox) = DaytonaSandbox::cleanup_failed_initialization_sandbox( + sandbox, + crate::Error::message("exact checkout failed"), + ) + .await; + + assert_eq!(error.to_string(), "exact checkout failed"); + assert_eq!( + retry_sandbox.as_ref().map(|sandbox| sandbox.id.as_str()), + Some(sandbox_id) + ); + delete.assert_async().await; + } + + #[tokio::test] + async fn failed_initialization_treats_missing_sandbox_as_deleted() { + let server = MockServer::start_async().await; + let sandbox_id = "sandbox-already-deleted"; + let delete = server + .mock_async(|when, then| { + when.method(DELETE).path(format!("/sandbox/{sandbox_id}")); + then.status(404) + .header("content-type", "application/json") + .json_body(serde_json::json!({ "message": "not found" })); + }) + .await; + let sandbox = mock_sandbox_handle(&server, sandbox_id).await; + + let (error, retry_sandbox) = DaytonaSandbox::cleanup_failed_initialization_sandbox( + sandbox, + crate::Error::message("exact checkout failed"), + ) + .await; + + assert_eq!(error.to_string(), "exact checkout failed"); + assert!(retry_sandbox.is_none()); + delete.assert_async().await; + } + fn api_key_body(permissions: &[&str]) -> serde_json::Value { serde_json::json!({ "name": "delete-only", diff --git a/lib/components/fabro-sandbox/src/provider/daytona.rs b/lib/components/fabro-sandbox/src/provider/daytona.rs index 246def6c0..45b212181 100644 --- a/lib/components/fabro-sandbox/src/provider/daytona.rs +++ b/lib/components/fabro-sandbox/src/provider/daytona.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use async_trait::async_trait; -use daytona_sdk::DaytonaError; use fabro_static::EnvVars; use fabro_types::{SandboxInfo, SandboxProviderKind}; @@ -89,7 +88,7 @@ impl SandboxProvider for DaytonaSandboxProvider { let client = self.client().await?; let sandbox = match client.get(id).await { Ok(sandbox) => sandbox, - Err(err) if daytona_not_found(&err) => return Ok(None), + Err(err) if daytona::daytona_not_found(&err) => return Ok(None), Err(err) => { return Err(crate::Error::context( format!("Failed to get Daytona sandbox '{id}'"), @@ -145,7 +144,7 @@ impl SandboxProvider for DaytonaSandboxProvider { let client = self.client().await?; let sandbox = match client.get(id).await { Ok(sandbox) => sandbox, - Err(err) if daytona_not_found(&err) => return Ok(()), + Err(err) if daytona::daytona_not_found(&err) => return Ok(()), Err(err) => { return Err(crate::Error::context( format!("Failed to get Daytona sandbox '{id}' before delete"), @@ -167,7 +166,3 @@ impl SandboxProvider for DaytonaSandboxProvider { fn managed_from_sdk_sandbox(sandbox: &daytona_sdk::Sandbox) -> bool { managed_labels::is_managed(&sandbox.labels) } - -fn daytona_not_found(err: &DaytonaError) -> bool { - matches!(err, DaytonaError::NotFound { .. }) || err.status_code() == Some(404) -} From 27fd48c603c238d48e63954c80b3d94a700d0174 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Fri, 14 Aug 2026 16:34:39 -0400 Subject: [PATCH 11/30] Persist workflow version lineage on runs --- docs/public/api-reference/fabro-api.yaml | 5 + lib/apps/fabro-cli/src/commands/run/attach.rs | 1 + lib/apps/fabro-cli/tests/it/support/mod.rs | 1 + lib/apps/fabro-server/src/run_compiler.rs | 15 +- lib/apps/fabro-server/src/run_files.rs | 29 +- .../fabro-server/src/server/handler/events.rs | 35 +-- .../fabro-server/src/server/handler/pair.rs | 1 + .../fabro-server/src/server/handler/runs.rs | 1 + .../src/server/handler/sessions.rs | 1 + lib/apps/fabro-server/src/server/tests.rs | 8 + .../fabro-server/tests/it/api/run_files.rs | 36 +-- lib/components/fabro-store/src/run_state.rs | 32 +- .../fabro-store/src/run_summary_store.rs | 1 + lib/components/fabro-store/src/slate/mod.rs | 1 + .../fabro-workflow/src/event/convert.rs | 46 ++- .../fabro-workflow/src/event/events.rs | 38 +-- .../fabro-workflow/src/event/sink.rs | 35 +-- lib/components/fabro-workflow/src/git.rs | 35 +-- .../fabro-workflow/src/handler/agent.rs | 35 +-- .../fabro-workflow/src/handler/command.rs | 64 ++-- .../fabro-workflow/src/handler/parallel.rs | 35 +-- .../fabro-workflow/src/handler/prompt.rs | 35 +-- .../fabro-workflow/src/lifecycle/git.rs | 1 + .../fabro-workflow/src/operations/archive.rs | 35 +-- .../fabro-workflow/src/operations/create.rs | 74 ++++- .../fabro-workflow/src/operations/fork.rs | 79 ++--- .../fabro-workflow/src/operations/retry.rs | 13 +- .../fabro-workflow/src/operations/timeline.rs | 29 +- .../src/pipeline/execute/tests.rs | 36 +-- .../fabro-workflow/src/pipeline/finalize.rs | 64 ++-- .../fabro-workflow/src/pipeline/initialize.rs | 36 +-- .../fabro-workflow/src/pipeline/persist.rs | 2 + .../src/pipeline/pull_request.rs | 285 +++++++++--------- .../fabro-workflow/src/run_lookup.rs | 35 +-- .../fabro-workflow/src/run_metadata.rs | 29 +- .../fabro-workflow/src/runtime_store.rs | 35 +-- .../fabro-workflow/src/stage_execution.rs | 29 +- .../fabro-workflow/src/test_support.rs | 35 +-- .../tests/run_projection_round_trip.rs | 9 +- lib/foundation/fabro-types/src/run.rs | 31 +- .../fabro-types/src/run_event/run.rs | 35 ++- .../fabro-types/src/test_support.rs | 29 +- .../fabro-types/tests/run_event_serde.rs | 81 +++-- .../fabro-types/tests/run_spec_serde.rs | 59 +++- .../fabro-api-client/src/models/run-spec.ts | 4 + 45 files changed, 919 insertions(+), 636 deletions(-) diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index 546b9d106..ee2eb143e 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -11434,6 +11434,11 @@ components: type: ["string", "null"] workflow_slug: type: ["string", "null"] + workflow_version_id: + description: Exact immutable root workflow version from which the run was admitted, when applicable. + oneOf: + - $ref: "#/components/schemas/WorkflowVersionId" + - type: "null" automation: oneOf: - $ref: "#/components/schemas/AutomationRef" diff --git a/lib/apps/fabro-cli/src/commands/run/attach.rs b/lib/apps/fabro-cli/src/commands/run/attach.rs index 7c1cf93b7..bc110029d 100644 --- a/lib/apps/fabro-cli/src/commands/run/attach.rs +++ b/lib/apps/fabro-cli/src/commands/run/attach.rs @@ -843,6 +843,7 @@ mod tests { graph: fabro_types::Graph::new("test"), graph_source: None, workflow_slug: None, + workflow_version_id: None, automation: None, source_directory: None, labels: std::collections::HashMap::default(), diff --git a/lib/apps/fabro-cli/tests/it/support/mod.rs b/lib/apps/fabro-cli/tests/it/support/mod.rs index 57e4a98e5..2994b3c0a 100644 --- a/lib/apps/fabro-cli/tests/it/support/mod.rs +++ b/lib/apps/fabro-cli/tests/it/support/mod.rs @@ -47,6 +47,7 @@ pub(crate) fn run_projection_json(run_id: &str, status: &serde_json::Value) -> s graph: Graph::new("Remote Workflow"), graph_source: None, workflow_slug: Some("remote-workflow".to_string()), + workflow_version_id: None, automation: None, source_directory: Some("/srv/repo".to_string()), labels: std::collections::HashMap::default(), diff --git a/lib/apps/fabro-server/src/run_compiler.rs b/lib/apps/fabro-server/src/run_compiler.rs index a018f46d0..291f0bb1d 100644 --- a/lib/apps/fabro-server/src/run_compiler.rs +++ b/lib/apps/fabro-server/src/run_compiler.rs @@ -38,6 +38,7 @@ use fabro_types::settings::interp::{InterpString, ResolveError}; use fabro_types::settings::run::{McpServerSettings, RunGoal}; use fabro_types::{ AutomationRef, GitContext, ManifestPath, RunId, RunProvenance, WorkflowSettings, + WorkflowVersionId, }; use fabro_util::workspace_glob::{WorkspaceGlob, WorkspaceGlobError}; use fabro_workflow::Error as WorkflowError; @@ -91,6 +92,7 @@ pub(crate) struct RawRunCompilerInput { pub(crate) git: Option, pub(crate) storage_root: PathBuf, pub(crate) workflow_slug: Option, + pub(crate) workflow_version_id: Option, pub(crate) provenance: RunProvenance, pub(crate) web_url: Option, pub(crate) submitted_manifest_bytes: Option>, @@ -121,6 +123,7 @@ struct RunMetadata { run_id: Option, storage_root: PathBuf, workflow_slug: Option, + workflow_version_id: Option, submitted_manifest_bytes: Option>, title: Option, automation: Option, @@ -314,6 +317,7 @@ pub(crate) fn normalize_source(input: RawRunCompilerInput) -> Result Result CreateRunPersistenceInput { run_id, storage_root, workflow_slug, + workflow_version_id, submitted_manifest_bytes, title, automation, @@ -545,6 +551,7 @@ pub(crate) fn assemble_run(pinned: PinnedRun) -> CreateRunPersistenceInput { run_id: run_id.expect("run ID should be resolved before compilation"), storage_root, workflow_slug, + workflow_version_id, submitted_manifest_bytes, title, automation, @@ -643,7 +650,9 @@ mod tests { use fabro_model::Catalog; use fabro_types::settings::interp::ResolveCtx; use fabro_types::settings::run::RunGoal; - use fabro_types::{AutomationRef, Principal, RunProvenance, SystemActorKind}; + use fabro_types::{ + AutomationRef, BlobHash, Principal, RunProvenance, SystemActorKind, WorkflowVersionId, + }; use fabro_workflow::workflow_bundle::ParsedWorkflowConfig; use super::*; @@ -711,6 +720,7 @@ mod tests { git: None, storage_root: PathBuf::from("/tmp/fabro-storage"), workflow_slug: None, + workflow_version_id: None, provenance: provenance(), web_url: None, submitted_manifest_bytes: None, @@ -978,11 +988,13 @@ include = ["reports/{{ vars.path }}/*.json"] trigger_id: Some("schedule".to_string()), }; let submitted = b"submitted manifest".to_vec(); + let workflow_version_id = WorkflowVersionId::from(BlobHash::new(b"workflow")); let mut input = raw_input(None, HashMap::new()); input.run_id = Some(run_id); input.parent_id = Some(parent_id); input.title = Some("Compiler boundary".to_string()); input.workflow_slug = Some("compiler-boundary".to_string()); + input.workflow_version_id = Some(workflow_version_id); input.web_url = Some(format!("https://fabro.test/runs/{run_id}")); input.submitted_manifest_bytes = Some(submitted.clone()); input.automation = Some(automation.clone()); @@ -1005,6 +1017,7 @@ include = ["reports/{{ vars.path }}/*.json"] assert_eq!(persistence.run_id(), run_id); assert_eq!(persistence.workflow_slug(), Some("compiler-boundary")); + assert_eq!(persistence.workflow_version_id(), Some(workflow_version_id)); assert_eq!( persistence.submitted_manifest_bytes(), Some(submitted.as_slice()) diff --git a/lib/apps/fabro-server/src/run_files.rs b/lib/apps/fabro-server/src/run_files.rs index 2343933b6..78bc85ff8 100644 --- a/lib/apps/fabro-server/src/run_files.rs +++ b/lib/apps/fabro-server/src/run_files.rs @@ -2376,20 +2376,21 @@ index 1111111..2222222 160000 let mut projection = fabro_store::RunProjection::new( "Test run".to_string(), fabro_types::RunSpec { - run_id: fabro_types::fixtures::RUN_1, - settings: fabro_types::WorkflowSettings::default(), - graph: fabro_types::Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: HashMap::default(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: fabro_types::fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: fabro_types::Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: HashMap::default(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }, chrono::Utc::now(), ); diff --git a/lib/apps/fabro-server/src/server/handler/events.rs b/lib/apps/fabro-server/src/server/handler/events.rs index 5fd655f1d..926fefdca 100644 --- a/lib/apps/fabro-server/src/server/handler/events.rs +++ b/lib/apps/fabro-server/src/server/handler/events.rs @@ -616,23 +616,24 @@ mod stage_events_tests { async fn append_run_created(run_store: &fabro_store::RunDatabase, run_id: &RunId) { workflow_event::append_event(run_store, run_id, &workflow_event::Event::RunCreated { - run_id: *run_id, - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::new(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: *run_id, + title: None, + settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), + graph: serde_json::to_value(Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::new(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .expect("run.created should append"); diff --git a/lib/apps/fabro-server/src/server/handler/pair.rs b/lib/apps/fabro-server/src/server/handler/pair.rs index 6e244bcac..c3c3166ee 100644 --- a/lib/apps/fabro-server/src/server/handler/pair.rs +++ b/lib/apps/fabro-server/src/server/handler/pair.rs @@ -1024,6 +1024,7 @@ mod tests { labels: std::collections::BTreeMap::new(), source_directory: None, workflow_slug: None, + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, diff --git a/lib/apps/fabro-server/src/server/handler/runs.rs b/lib/apps/fabro-server/src/server/handler/runs.rs index 56ae1c754..8a8efc259 100644 --- a/lib/apps/fabro-server/src/server/handler/runs.rs +++ b/lib/apps/fabro-server/src/server/handler/runs.rs @@ -730,6 +730,7 @@ pub(crate) async fn create_run_from_manifest( git: manifest.git.clone(), storage_root: state.server_storage_dir(), workflow_slug: None, + workflow_version_id: None, provenance: run_provenance(&headers, &actor), web_url: None, submitted_manifest_bytes: Some(submitted_manifest_bytes), diff --git a/lib/apps/fabro-server/src/server/handler/sessions.rs b/lib/apps/fabro-server/src/server/handler/sessions.rs index e9bacdfff..835b9ed51 100644 --- a/lib/apps/fabro-server/src/server/handler/sessions.rs +++ b/lib/apps/fabro-server/src/server/handler/sessions.rs @@ -1917,6 +1917,7 @@ reasoning = false graph, graph_source: None, workflow_slug: None, + workflow_version_id: None, automation: None, source_directory: None, labels: HashMap::default(), diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index dc656c32e..4783f3bb2 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -4649,6 +4649,7 @@ async fn append_default_run_created(run_store: &fabro_store::RunDatabase, run_id labels: std::collections::BTreeMap::default(), source_directory: None, workflow_slug: None, + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, @@ -4701,6 +4702,7 @@ async fn create_slack_notification_run( labels: std::collections::BTreeMap::default(), source_directory: None, workflow_slug: workflow_slug.map(str::to_string), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, @@ -5775,6 +5777,7 @@ async fn list_run_stages_distinguishes_visits() { labels: std::collections::BTreeMap::default(), source_directory: None, workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, @@ -5912,6 +5915,7 @@ async fn list_run_stages_exposes_execution_identity_for_resumed_stage() { labels: std::collections::BTreeMap::default(), source_directory: None, workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, @@ -7096,6 +7100,7 @@ async fn create_completed_run_ready_for_pull_request( graph, graph_source: None, workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, source_directory: Some("/tmp/project".to_string()), git: git.clone(), @@ -7117,6 +7122,7 @@ async fn create_completed_run_ready_for_pull_request( labels: run_spec.labels.clone().into_iter().collect(), source_directory: run_spec.source_directory.clone(), workflow_slug: run_spec.workflow_slug.clone(), + workflow_version_id: None, automation: None, provenance: run_spec.provenance.clone(), manifest_blob: None, @@ -14090,6 +14096,7 @@ async fn create_preserved_local_sandbox_run(state: &Arc, run_id: RunId labels: std::collections::BTreeMap::default(), source_directory: Some("/tmp/fabro-run".to_string()), workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, @@ -14840,6 +14847,7 @@ async fn delete_run_retry_after_missing_provider_resource_removes_metadata() { labels: std::collections::BTreeMap::default(), source_directory: Some("/tmp/fabro-run".to_string()), workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, diff --git a/lib/apps/fabro-server/tests/it/api/run_files.rs b/lib/apps/fabro-server/tests/it/api/run_files.rs index 307c4a574..b8aad6410 100644 --- a/lib/apps/fabro-server/tests/it/api/run_files.rs +++ b/lib/apps/fabro-server/tests/it/api/run_files.rs @@ -56,24 +56,26 @@ async fn append_completed_run_with_final_patch( ) { let run_store = store.create_run(run_id).await.expect("create run store"); workflow_event::append_event(&run_store, run_id, &workflow_event::Event::RunCreated { - run_id: *run_id, - title: None, - settings: serde_json::to_value(WorkflowSettings::default()) + run_id: *run_id, + title: None, + settings: serde_json::to_value(WorkflowSettings::default()) .expect("workflow settings should serialize"), - graph: serde_json::to_value(Graph::new("test")).expect("graph should serialize"), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(Graph::new("test")) + .expect("graph should serialize"), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .expect("append RunCreated"); diff --git a/lib/components/fabro-store/src/run_state.rs b/lib/components/fabro-store/src/run_state.rs index 0c59af57a..bcd21a3d2 100644 --- a/lib/components/fabro-store/src/run_state.rs +++ b/lib/components/fabro-store/src/run_state.rs @@ -1040,6 +1040,7 @@ fn projection_from_created(event: &EventEnvelope) -> Result { graph: props.graph.clone(), graph_source: props.workflow_source.clone(), workflow_slug: props.workflow_slug.clone(), + workflow_version_id: props.workflow_version_id, automation: props.automation.clone(), source_directory: props.source_directory.clone(), labels, @@ -1690,8 +1691,8 @@ mod tests { RunStatus, Speed, StageContextWindowBreakdownItem, StageContextWindowCategory, StageContextWindowCountMethod, StageContextWindowProjection, StageContextWindowStaleness, StageContextWindowWarning, StageHandler, StageModelUsage, StageOutcome, StageState, - StageTiming, SubAgentStatus, SuccessReason, WorkflowSettings, first_event_seq, fixtures, - test_support, + StageTiming, SubAgentStatus, SuccessReason, WorkflowSettings, WorkflowVersionId, + first_event_seq, fixtures, test_support, }; use serde_json::json; @@ -2367,12 +2368,39 @@ mod tests { let projection = RunProjection::apply_events(&[event]).unwrap(); assert_eq!(projection.retried_from, None); + assert_eq!(projection.spec.workflow_version_id, None); + let spec_json = serde_json::to_value(&projection.spec).unwrap(); + assert!(spec_json.get("workflow_version_id").is_none()); assert_eq!( build_summary(&projection, &fixtures::RUN_1).retried_from, None ); } + #[test] + fn run_created_projects_workflow_version_id_into_spec() { + let workflow_version_id = WorkflowVersionId::from(BlobHash::new(b"workflow")); + let event = test_raw_event( + 1, + "run.created", + &json!({ + "settings": WorkflowSettings::default(), + "graph": Graph::new("test"), + "workflow_version_id": workflow_version_id, + "labels": {}, + "provenance": test_support::test_run_provenance() + }), + None, + ); + + let projection = RunProjection::apply_events(&[event]).unwrap(); + + assert_eq!( + projection.spec.workflow_version_id, + Some(workflow_version_id) + ); + } + #[test] fn run_created_replay_ignores_unknown_properties() { let provenance = test_support::test_run_provenance(); diff --git a/lib/components/fabro-store/src/run_summary_store.rs b/lib/components/fabro-store/src/run_summary_store.rs index dcbec847e..8017509f3 100644 --- a/lib/components/fabro-store/src/run_summary_store.rs +++ b/lib/components/fabro-store/src/run_summary_store.rs @@ -595,6 +595,7 @@ mod tests { graph: Graph::new("test"), graph_source: None, workflow_slug: Some("test-workflow".to_string()), + workflow_version_id: None, automation: None, source_directory: None, labels: HashMap::new(), diff --git a/lib/components/fabro-store/src/slate/mod.rs b/lib/components/fabro-store/src/slate/mod.rs index 6158e18a4..e8c62402d 100644 --- a/lib/components/fabro-store/src/slate/mod.rs +++ b/lib/components/fabro-store/src/slate/mod.rs @@ -587,6 +587,7 @@ mod tests { graph, graph_source: None, workflow_slug: Some("night-sky".to_string()), + workflow_version_id: None, automation: None, source_directory: Some(format!("/tmp/{label}")), labels: std::collections::HashMap::from([("team".to_string(), "infra".to_string())]), diff --git a/lib/components/fabro-workflow/src/event/convert.rs b/lib/components/fabro-workflow/src/event/convert.rs index 85a02769a..420706919 100644 --- a/lib/components/fabro-workflow/src/event/convert.rs +++ b/lib/components/fabro-workflow/src/event/convert.rs @@ -71,6 +71,7 @@ fn event_body_from_event(event: &Event) -> EventBody { labels, source_directory, workflow_slug, + workflow_version_id, automation, provenance, manifest_blob, @@ -90,6 +91,7 @@ fn event_body_from_event(event: &Event) -> EventBody { labels: labels.clone(), source_directory: source_directory.clone(), workflow_slug: workflow_slug.clone(), + workflow_version_id: *workflow_version_id, automation: automation.clone(), provenance: provenance.clone(), manifest_blob: *manifest_blob, @@ -1443,9 +1445,9 @@ mod tests { use std::collections::BTreeMap; use ::fabro_types::{ - AutomationRef, EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeCode, - RunNoticeLevel, RunProvenance, StageId, SystemActorKind, fixtures, - run_event as fabro_types, + AutomationRef, BlobHash, EventBody, FailureReason, ParallelBranchId, Principal, + RunNoticeCode, RunNoticeLevel, RunProvenance, StageId, SystemActorKind, WorkflowVersionId, + fixtures, run_event as fabro_types, }; use chrono::Utc; use fabro_agent::{ @@ -2828,6 +2830,7 @@ mod tests { name: Some("Nightly".to_string()), trigger_id: Some("schedule_1".to_string()), }; + let workflow_version_id = WorkflowVersionId::from(BlobHash::new(b"workflow")); let stored = to_run_event(&fixtures::RUN_1, &Event::RunCreated { run_id: fixtures::RUN_1, @@ -2838,6 +2841,7 @@ mod tests { labels: BTreeMap::default(), source_directory: Some("/tmp/run".to_string()), workflow_slug: None, + workflow_version_id: Some(workflow_version_id), automation: Some(automation.clone()), provenance, manifest_blob: None, @@ -2854,6 +2858,42 @@ mod tests { panic!("expected run.created body"); }; assert_eq!(props.automation, Some(automation)); + assert_eq!(props.workflow_version_id, Some(workflow_version_id)); + } + + #[test] + fn run_created_omits_absent_workflow_version_id() { + use ::fabro_types::{Graph, WorkflowSettings, fixtures}; + + let stored = to_run_event(&fixtures::RUN_1, &Event::RunCreated { + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), + graph: serde_json::to_value(Graph::new("test")).unwrap(), + workflow_source: None, + labels: BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: RunProvenance { + server: None, + client: None, + subject: user_principal("alice"), + }, + manifest_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, + }); + let EventBody::RunCreated(props) = stored.body else { + panic!("expected run.created body"); + }; + + let json = serde_json::to_value(props).expect("run.created props should serialize"); + assert!(json.get("workflow_version_id").is_none()); } #[test] diff --git a/lib/components/fabro-workflow/src/event/events.rs b/lib/components/fabro-workflow/src/event/events.rs index 03ffe816a..f916f7985 100644 --- a/lib/components/fabro-workflow/src/event/events.rs +++ b/lib/components/fabro-workflow/src/event/events.rs @@ -6,7 +6,7 @@ use ::fabro_types::{ PairTarget, ParallelBranchId, ParallelBranchResult, PendingReason, PermissionLevel, Principal, PullRequestCreationId, PullRequestLink, ReviewTarget, RunFailure, RunId, RunNoticeLevel, RunPairEndedReason, RunPairFailedReason, RunProvenance, RunRunnableSource, RunTiming, - SandboxProviderKind, StageId, StageOutcome, StageTiming, SuccessReason, + SandboxProviderKind, StageId, StageOutcome, StageTiming, SuccessReason, WorkflowVersionId, run_event as fabro_types, }; use fabro_agent::{AgentEvent, SandboxEvent}; @@ -24,34 +24,36 @@ use crate::outcome::{BilledModelUsage, FailureDetail, Outcome}; )] pub enum Event { RunCreated { - run_id: RunId, - title: Option, - settings: serde_json::Value, - graph: serde_json::Value, + run_id: RunId, + title: Option, + settings: serde_json::Value, + graph: serde_json::Value, #[serde(default, skip_serializing_if = "Option::is_none")] - workflow_source: Option, - labels: BTreeMap, + workflow_source: Option, + labels: BTreeMap, #[serde(default, skip_serializing_if = "Option::is_none")] - source_directory: Option, + source_directory: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - workflow_slug: Option, + workflow_slug: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - automation: Option, - provenance: RunProvenance, + workflow_version_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - manifest_blob: Option, + automation: Option, + provenance: RunProvenance, #[serde(default, skip_serializing_if = "Option::is_none")] - spec_blob: Option, + manifest_blob: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - git: Option, + spec_blob: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - fork_source_ref: Option, + git: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - retried_from: Option, + fork_source_ref: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - parent_id: Option, + retried_from: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - web_url: Option, + parent_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + web_url: Option, }, WorkflowRunStarted { name: String, diff --git a/lib/components/fabro-workflow/src/event/sink.rs b/lib/components/fabro-workflow/src/event/sink.rs index 150c69f72..9588a3ded 100644 --- a/lib/components/fabro-workflow/src/event/sink.rs +++ b/lib/components/fabro-workflow/src/event/sink.rs @@ -279,23 +279,24 @@ mod tests { ); let run_store = store.create_run(&fixtures::RUN_7).await.unwrap(); append_event(&run_store, &fixtures::RUN_7, &Event::RunCreated { - run_id: fixtures::RUN_7, - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::new(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_7, + title: None, + settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), + graph: serde_json::to_value(Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::new(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/git.rs b/lib/components/fabro-workflow/src/git.rs index e7ca4feda..5d72aa43d 100644 --- a/lib/components/fabro-workflow/src/git.rs +++ b/lib/components/fabro-workflow/src/git.rs @@ -352,24 +352,25 @@ mod tests { let store = test_store(); let run = store.create_run(&fixtures::RUN_1).await.unwrap(); append_event(&run, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) .unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/handler/agent.rs b/lib/components/fabro-workflow/src/handler/agent.rs index 99d20431b..b3c67612b 100644 --- a/lib/components/fabro-workflow/src/handler/agent.rs +++ b/lib/components/fabro-workflow/src/handler/agent.rs @@ -489,24 +489,25 @@ mod tests { run_store, &fixtures::RUN_1, &crate::event::Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) .unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }, ) .await diff --git a/lib/components/fabro-workflow/src/handler/command.rs b/lib/components/fabro-workflow/src/handler/command.rs index db9682a1c..c24353ccd 100644 --- a/lib/components/fabro-workflow/src/handler/command.rs +++ b/lib/components/fabro-workflow/src/handler/command.rs @@ -363,20 +363,21 @@ mod tests { Ok(RunProjection::new( "Test run".to_string(), RunSpec { - run_id: fixtures::RUN_1, - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: std::collections::HashMap::default(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: std::collections::HashMap::default(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }, chrono::Utc::now(), )) @@ -464,23 +465,24 @@ mod tests { run_store, &fixtures::RUN_1, &crate::event::Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), + graph: serde_json::to_value(Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }, ) .await diff --git a/lib/components/fabro-workflow/src/handler/parallel.rs b/lib/components/fabro-workflow/src/handler/parallel.rs index 82773470b..1d245659f 100644 --- a/lib/components/fabro-workflow/src/handler/parallel.rs +++ b/lib/components/fabro-workflow/src/handler/parallel.rs @@ -944,24 +944,25 @@ mod tests { run_store, &fixtures::RUN_1, &crate::event::Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) .unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), - workflow_source: None, - labels: BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), + workflow_source: None, + labels: BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }, ) .await diff --git a/lib/components/fabro-workflow/src/handler/prompt.rs b/lib/components/fabro-workflow/src/handler/prompt.rs index 630332878..e61a2809c 100644 --- a/lib/components/fabro-workflow/src/handler/prompt.rs +++ b/lib/components/fabro-workflow/src/handler/prompt.rs @@ -267,24 +267,25 @@ mod tests { run_store, &fixtures::RUN_1, &crate::event::Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) .unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }, ) .await diff --git a/lib/components/fabro-workflow/src/lifecycle/git.rs b/lib/components/fabro-workflow/src/lifecycle/git.rs index b8e06088e..691690b21 100644 --- a/lib/components/fabro-workflow/src/lifecycle/git.rs +++ b/lib/components/fabro-workflow/src/lifecycle/git.rs @@ -782,6 +782,7 @@ mod tests { labels: BTreeMap::new(), source_directory: Some("/tmp/project".to_string()), workflow_slug: Some("metadata".to_string()), + workflow_version_id: None, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, diff --git a/lib/components/fabro-workflow/src/operations/archive.rs b/lib/components/fabro-workflow/src/operations/archive.rs index 92d9124cf..28e28ae77 100644 --- a/lib/components/fabro-workflow/src/operations/archive.rs +++ b/lib/components/fabro-workflow/src/operations/archive.rs @@ -215,24 +215,25 @@ mod tests { async fn seed_created(run_store: &fabro_store::RunDatabase, run_id: &RunId) { event::append_event(run_store, run_id, &Event::RunCreated { - run_id: *run_id, - title: None, - settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) + run_id: *run_id, + title: None, + settings: serde_json::to_value(fabro_types::WorkflowSettings::default()) .unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::default(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::default(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/operations/create.rs b/lib/components/fabro-workflow/src/operations/create.rs index 7783a9b59..9647a7179 100644 --- a/lib/components/fabro-workflow/src/operations/create.rs +++ b/lib/components/fabro-workflow/src/operations/create.rs @@ -17,7 +17,7 @@ use fabro_store::{Database, RunDatabase}; use fabro_template::TemplateContext; use fabro_types::{ AutomationRef, BlobHash, ForkSourceRef, GitContext, ManifestPath, RunId, RunProvenance, - WorkflowSettings, + WorkflowSettings, WorkflowVersionId, }; use fabro_util::json::normalize_json_value; use tokio::task::spawn_blocking; @@ -100,6 +100,7 @@ impl CreateRunInput { run_id, storage_root, workflow_slug, + workflow_version_id: None, submitted_manifest_bytes, title, automation, @@ -133,6 +134,7 @@ pub struct CreateRunPersistenceMetadata { pub run_id: RunId, pub storage_root: PathBuf, pub workflow_slug: Option, + pub workflow_version_id: Option, pub submitted_manifest_bytes: Option>, pub title: Option, pub automation: Option, @@ -202,6 +204,7 @@ pub struct CreateRunPersistenceInput { run_id: RunId, run_dir: PathBuf, workflow_slug: Option, + workflow_version_id: Option, submitted_manifest_bytes: Option>, title: Option, automation: Option, @@ -229,6 +232,10 @@ impl CreateRunPersistenceInput { self.workflow_slug.as_deref() } + pub fn workflow_version_id(&self) -> Option { + self.workflow_version_id + } + pub fn submitted_manifest_bytes(&self) -> Option<&[u8]> { self.submitted_manifest_bytes.as_deref() } @@ -397,6 +404,7 @@ pub fn assemble_create_run_persistence_input( run_id, storage_root, workflow_slug, + workflow_version_id, submitted_manifest_bytes, title, automation, @@ -417,6 +425,7 @@ pub fn assemble_create_run_persistence_input( run_id, run_dir, workflow_slug, + workflow_version_id, submitted_manifest_bytes, title, automation, @@ -438,6 +447,7 @@ pub async fn persist_create_run( run_id, run_dir, workflow_slug, + workflow_version_id, submitted_manifest_bytes, title, automation, @@ -465,6 +475,7 @@ pub async fn persist_create_run( graph: validated.graph().clone(), graph_source: Some(validated.source().to_string()), workflow_slug, + workflow_version_id, automation, source_directory: Some(source_directory), labels, @@ -552,6 +563,7 @@ async fn persist_created_run( .collect::>(), source_directory: record.source_directory.clone(), workflow_slug: record.workflow_slug.clone(), + workflow_version_id: record.workflow_version_id, automation: record.automation.clone(), provenance: record.provenance.clone(), manifest_blob, @@ -677,7 +689,9 @@ mod tests { use fabro_store::Database; use fabro_types::settings::InterpString; use fabro_types::settings::run::RunMode; - use fabro_types::{EventBody, WorkflowSettings, fixtures, test_support}; + use fabro_types::{ + BlobHash, EventBody, WorkflowSettings, WorkflowVersionId, fixtures, test_support, + }; use fabro_util::error::collect_chain; use fabro_validate::Severity; use object_store::local::LocalFileSystem; @@ -1709,6 +1723,7 @@ reasoning = false run_id: fixtures::RUN_1, storage_root: PathBuf::from("/tmp/storage"), workflow_slug: None, + workflow_version_id: None, submitted_manifest_bytes: None, title: None, automation: None, @@ -1761,7 +1776,9 @@ reasoning = false std::fs::write(&dot_path, "this is no longer a graph").unwrap(); let materialized = materialize_create_run(compiled, catalog.as_ref()).unwrap(); - let metadata = persistence_metadata(&request, fixtures::RUN_2, &storage_root); + let workflow_version_id = WorkflowVersionId::from(BlobHash::new(b"workflow")); + let mut metadata = persistence_metadata(&request, fixtures::RUN_2, &storage_root); + metadata.workflow_version_id = Some(workflow_version_id); let input = assemble_create_run_persistence_input(materialized, metadata); let store = memory_store(); let created = persist_create_run(store.as_ref(), input).await.unwrap(); @@ -1775,6 +1792,7 @@ reasoning = false let state = run_store.state().await.unwrap(); assert_eq!(state.spec.graph.goal(), "Compiled goal"); assert_eq!(state.spec.automation, Some(automation)); + assert_eq!(state.spec.workflow_version_id, Some(workflow_version_id)); let events = run_store.list_events().await.unwrap(); assert_eq!( events @@ -1790,6 +1808,7 @@ reasoning = false created.workflow_source.as_deref(), Some(compiled_source.as_str()) ); + assert_eq!(created.workflow_version_id, Some(workflow_version_id)); let manifest_blob = created .manifest_blob .as_ref() @@ -1805,6 +1824,55 @@ reasoning = false ); } + #[tokio::test] + async fn legacy_create_input_persists_without_workflow_version_id() { + let dir = tempfile::tempdir().unwrap(); + let store = memory_store(); + let run_id = fixtures::RUN_64; + let request = CreateRunInput { + workflow: WorkflowInput::DotSource { + source: MINIMAL_DOT.to_string(), + base_dir: None, + }, + settings: test_default_settings(), + vars: HashMap::new(), + cwd: dir.path().to_path_buf(), + workflow_slug: Some("legacy-create".to_string()), + workflow_path: None, + workflow_bundle: None, + submitted_manifest_bytes: None, + run_id: Some(run_id), + title: None, + automation: None, + git: None, + fork_source_ref: None, + parent_id: None, + provenance: test_support::test_run_provenance(), + configured_providers: test_provider_ids(), + web_url: None, + }; + + create( + store.as_ref(), + request, + dir.path().join("storage"), + test_catalog(), + ) + .await + .unwrap(); + + let run_store = store.open_run_reader(&run_id).await.unwrap(); + let state = run_store.state().await.unwrap(); + assert_eq!(state.spec.workflow_version_id, None); + let events = run_store.list_events().await.unwrap(); + let EventBody::RunCreated(created) = &events[0].event.body else { + panic!("first durable event should be run.created"); + }; + assert_eq!(created.workflow_version_id, None); + let json = serde_json::to_value(created).unwrap(); + assert!(json.get("workflow_version_id").is_none()); + } + #[tokio::test] async fn create_returns_validation_failed_with_diagnostics() { let dot = r#"digraph Test { diff --git a/lib/components/fabro-workflow/src/operations/fork.rs b/lib/components/fabro-workflow/src/operations/fork.rs index 3f728211d..857e02e47 100644 --- a/lib/components/fabro-workflow/src/operations/fork.rs +++ b/lib/components/fabro-workflow/src/operations/fork.rs @@ -149,27 +149,28 @@ async fn persist_forked_run( .map_err(|err| Error::engine(err.to_string()))?; event::append_event(&run_store, &spec.run_id, &Event::RunCreated { - run_id: spec.run_id, - title: None, - settings: serde_json::to_value(&spec.settings) + run_id: spec.run_id, + title: None, + settings: serde_json::to_value(&spec.settings) .map_err(|err| Error::engine(err.to_string()))?, - graph: serde_json::to_value(&spec.graph) + graph: serde_json::to_value(&spec.graph) .map_err(|err| Error::engine(err.to_string()))?, - workflow_source: projection.spec.graph_source.clone(), - labels: spec.labels.clone().into_iter().collect(), - source_directory: spec.source_directory.clone(), - workflow_slug: spec.workflow_slug.clone(), - automation: spec.automation.clone(), - provenance: spec.provenance.clone(), - manifest_blob: spec.manifest_blob, + workflow_source: projection.spec.graph_source.clone(), + labels: spec.labels.clone().into_iter().collect(), + source_directory: spec.source_directory.clone(), + workflow_slug: spec.workflow_slug.clone(), + workflow_version_id: spec.workflow_version_id, + automation: spec.automation.clone(), + provenance: spec.provenance.clone(), + manifest_blob: spec.manifest_blob, // Content-addressed, so the forked run reads the source run's // unredacted spec bytes through the same id. - spec_blob: spec.spec_blob, - git: spec.git.clone(), - fork_source_ref: spec.fork_source_ref.clone(), - retried_from: None, - parent_id: None, - web_url: None, + spec_blob: spec.spec_blob, + git: spec.git.clone(), + fork_source_ref: spec.fork_source_ref.clone(), + retried_from: None, + parent_id: None, + web_url: None, }) .await .map_err(|err| Error::engine(err.to_string()))?; @@ -286,7 +287,9 @@ mod tests { use fabro_graphviz::graph::Graph; use fabro_store::{Database, RunProjectionReducer}; - use fabro_types::{StageId, WorkflowSettings, fixtures, test_support}; + use fabro_types::{ + BlobHash, StageId, WorkflowSettings, WorkflowVersionId, fixtures, test_support, + }; use object_store::memory::InMemory; use super::*; @@ -371,30 +374,32 @@ mod tests { let source = store.create_run(&source_run_id).await.unwrap(); let graph = Graph::new("fork-source"); let settings = WorkflowSettings::default(); + let workflow_version_id = WorkflowVersionId::from(BlobHash::new(b"workflow")); event::append_event(&source, &source_run_id, &Event::RunCreated { - run_id: source_run_id, - title: None, - settings: serde_json::to_value(&settings).unwrap(), - graph: serde_json::to_value(&graph).unwrap(), - workflow_source: Some("digraph fork_source {}".to_string()), - labels: BTreeMap::new(), - source_directory: Some("/client/source".to_string()), - workflow_slug: Some("fork-source".to_string()), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: Some(fabro_types::GitContext { + run_id: source_run_id, + title: None, + settings: serde_json::to_value(&settings).unwrap(), + graph: serde_json::to_value(&graph).unwrap(), + workflow_source: Some("digraph fork_source {}".to_string()), + labels: BTreeMap::new(), + source_directory: Some("/client/source".to_string()), + workflow_slug: Some("fork-source".to_string()), + workflow_version_id: Some(workflow_version_id), + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: Some(fabro_types::GitContext { origin_url: "https://github.com/example/repo.git".to_string(), branch: "main".to_string(), sha: None, dirty: fabro_types::DirtyStatus::Clean, }), - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -463,6 +468,10 @@ mod tests { assert_eq!(node.response.as_deref(), Some("historical response")); assert_eq!(forked_state.checkpoints.len(), 1); + assert_eq!( + forked_state.spec.workflow_version_id, + Some(workflow_version_id) + ); assert_eq!( forked_state.spec.fork_source_ref.unwrap().source_run_id, source_run_id diff --git a/lib/components/fabro-workflow/src/operations/retry.rs b/lib/components/fabro-workflow/src/operations/retry.rs index 010c08f3f..b2cfa3022 100644 --- a/lib/components/fabro-workflow/src/operations/retry.rs +++ b/lib/components/fabro-workflow/src/operations/retry.rs @@ -48,6 +48,7 @@ pub async fn retry_run( graph, graph_source, workflow_slug, + workflow_version_id, automation, source_directory, labels, @@ -76,6 +77,7 @@ pub async fn retry_run( labels: labels.into_iter().collect::>(), source_directory, workflow_slug, + workflow_version_id, automation, provenance: input.provenance.clone(), manifest_blob, @@ -123,7 +125,7 @@ mod tests { use fabro_types::{ AuthMethod, BlobHash, DirtyStatus, FailureReason, ForkSourceRef, GitContext, Graph, IdpIdentity, Principal, PullRequestLink, RunRunnableSource, RunServerProvenance, RunTiming, - WorkflowSettings, fixtures, + WorkflowSettings, WorkflowVersionId, fixtures, }; use object_store::memory::InMemory; @@ -165,6 +167,10 @@ mod tests { } } + fn workflow_version_id() -> WorkflowVersionId { + WorkflowVersionId::from(BlobHash::new(b"workflow")) + } + async fn append_created( store: &fabro_store::RunDatabase, run_id: RunId, @@ -186,6 +192,7 @@ mod tests { labels: labels.into_iter().collect(), source_directory: Some("/workspace/source".to_string()), workflow_slug: Some("retry-source".to_string()), + workflow_version_id: Some(workflow_version_id()), automation: None, provenance: provenance("source-user"), manifest_blob, @@ -393,6 +400,10 @@ mod tests { Some(&"test".to_string()) ); assert_eq!(retry_state.spec.graph.name, "retry_source"); + assert_eq!( + retry_state.spec.workflow_version_id, + Some(workflow_version_id()) + ); assert_eq!( retry_state.spec.graph_source.as_deref(), Some("digraph retry_source { start -> exit }") diff --git a/lib/components/fabro-workflow/src/operations/timeline.rs b/lib/components/fabro-workflow/src/operations/timeline.rs index 83319745e..b3f51350e 100644 --- a/lib/components/fabro-workflow/src/operations/timeline.rs +++ b/lib/components/fabro-workflow/src/operations/timeline.rs @@ -241,20 +241,21 @@ mod tests { RunProjection::new( "Test run".to_string(), RunSpec { - run_id: fixtures::RUN_1, - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }, Utc::now(), ) diff --git a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs index 7bc1d6dff..8056192bd 100644 --- a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs @@ -156,6 +156,7 @@ fn persisted_workflow(graph: Graph, source: String, run_dir: &Path, run_id: RunI graph, graph_source: None, workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, source_directory: Some( std::env::current_dir() @@ -208,23 +209,24 @@ async fn seed_created_and_starting( graph: &Graph, ) { append_event(run_store, &run_options.run_id, &Event::RunCreated { - run_id: run_options.run_id, - title: None, - settings: serde_json::to_value(&run_options.settings).unwrap(), - graph: serde_json::to_value(graph).unwrap(), - workflow_source: None, - labels: run_options.labels.clone().into_iter().collect(), - source_directory: Some(std::env::current_dir().unwrap().display().to_string()), - workflow_slug: run_options.workflow_slug.clone(), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: run_options.pre_run_git.clone(), - fork_source_ref: run_options.fork_source_ref.clone(), - retried_from: None, - parent_id: None, - web_url: None, + run_id: run_options.run_id, + title: None, + settings: serde_json::to_value(&run_options.settings).unwrap(), + graph: serde_json::to_value(graph).unwrap(), + workflow_source: None, + labels: run_options.labels.clone().into_iter().collect(), + source_directory: Some(std::env::current_dir().unwrap().display().to_string()), + workflow_slug: run_options.workflow_slug.clone(), + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: run_options.pre_run_git.clone(), + fork_source_ref: run_options.fork_source_ref.clone(), + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/pipeline/finalize.rs b/lib/components/fabro-workflow/src/pipeline/finalize.rs index 9bdb18c39..18b7ea15b 100644 --- a/lib/components/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/components/fabro-workflow/src/pipeline/finalize.rs @@ -787,23 +787,24 @@ mod tests { async fn seeded_run_store() -> RunDatabase { let run_store = test_store().create_run(&test_run_id()).await.unwrap(); append_event(&run_store, &test_run_id(), &Event::RunCreated { - run_id: test_run_id(), - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(fabro_types::Graph::new("metadata")).unwrap(), - workflow_source: None, - labels: std::collections::BTreeMap::new(), - source_directory: Some("/tmp/project".to_string()), - workflow_slug: Some("metadata".to_string()), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: test_run_id(), + title: None, + settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), + graph: serde_json::to_value(fabro_types::Graph::new("metadata")).unwrap(), + workflow_source: None, + labels: std::collections::BTreeMap::new(), + source_directory: Some("/tmp/project".to_string()), + workflow_slug: Some("metadata".to_string()), + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -906,20 +907,21 @@ mod tests { RunProjection::new( "Test run".to_string(), RunSpec { - run_id: test_run_id(), - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: test_run_id(), + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }, chrono::Utc::now(), ) diff --git a/lib/components/fabro-workflow/src/pipeline/initialize.rs b/lib/components/fabro-workflow/src/pipeline/initialize.rs index 1c4711b7f..a91929c7b 100644 --- a/lib/components/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/components/fabro-workflow/src/pipeline/initialize.rs @@ -854,6 +854,7 @@ mod tests { graph, graph_source: None, workflow_slug: Some("test".to_string()), + workflow_version_id: None, automation: None, source_directory: Some(std::env::current_dir().unwrap().display().to_string()), git: Some(fabro_types::GitContext { @@ -1039,23 +1040,24 @@ mod tests { run_options.settings = settings; run_options.fork_source_ref = fork_source_ref; crate::event::append_event(&run_store, &test_run_id(), &Event::RunCreated { - run_id: test_run_id(), - title: None, - settings: serde_json::to_value(&run_options.settings).unwrap(), - graph: serde_json::to_value(&graph).unwrap(), - workflow_source: None, - labels: BTreeMap::new(), - source_directory: Some(workspace.display().to_string()), - workflow_slug: Some("test".to_string()), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: run_options.fork_source_ref.clone(), - retried_from: None, - parent_id: None, - web_url: None, + run_id: test_run_id(), + title: None, + settings: serde_json::to_value(&run_options.settings).unwrap(), + graph: serde_json::to_value(&graph).unwrap(), + workflow_source: None, + labels: BTreeMap::new(), + source_directory: Some(workspace.display().to_string()), + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: run_options.fork_source_ref.clone(), + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/pipeline/persist.rs b/lib/components/fabro-workflow/src/pipeline/persist.rs index 303241ff5..95cab3c32 100644 --- a/lib/components/fabro-workflow/src/pipeline/persist.rs +++ b/lib/components/fabro-workflow/src/pipeline/persist.rs @@ -171,6 +171,7 @@ mod tests { graph, graph_source: None, workflow_slug: Some("ship".to_string()), + workflow_version_id: None, automation: None, source_directory: Some("/tmp/project".to_string()), git: Some(fabro_types::GitContext { @@ -220,6 +221,7 @@ mod tests { labels: record.labels.clone().into_iter().collect(), source_directory: record.source_directory.clone(), workflow_slug: record.workflow_slug.clone(), + workflow_version_id: None, automation: record.automation.clone(), provenance: record.provenance.clone(), manifest_blob: None, diff --git a/lib/components/fabro-workflow/src/pipeline/pull_request.rs b/lib/components/fabro-workflow/src/pipeline/pull_request.rs index f3f7e4a78..152769978 100644 --- a/lib/components/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/components/fabro-workflow/src/pipeline/pull_request.rs @@ -819,20 +819,21 @@ mod tests { RunProjection::new( "Test run".to_string(), RunSpec { - run_id: fixtures::RUN_1, - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }, Utc::now(), ) @@ -1096,44 +1097,46 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let run_spec = RunSpec { - run_id: fixtures::RUN_1, - settings: fabro_types::WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: Some("test".to_string()), - automation: None, - source_directory: Some("/tmp/project".to_string()), - git: Some(fabro_types::GitContext { + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + source_directory: Some("/tmp/project".to_string()), + git: Some(fabro_types::GitContext { origin_url: String::new(), branch: "main".to_string(), sha: None, dirty: fabro_types::DirtyStatus::Clean, }), - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - fork_source_ref: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + fork_source_ref: None, }; append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&run_spec.settings).unwrap(), - graph: serde_json::to_value(&run_spec.graph).unwrap(), - workflow_source: Some("digraph test { plan -> code }".to_string()), - labels: run_spec.labels.clone().into_iter().collect(), - source_directory: run_spec.source_directory.clone(), - workflow_slug: run_spec.workflow_slug.clone(), - automation: None, - provenance: run_spec.provenance.clone(), - manifest_blob: None, - spec_blob: None, - git: run_spec.git.clone(), - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: Some("digraph test { plan -> code }".to_string()), + labels: run_spec.labels.clone().into_iter().collect(), + source_directory: run_spec.source_directory.clone(), + workflow_slug: run_spec.workflow_slug.clone(), + workflow_version_id: None, + automation: None, + provenance: run_spec.provenance.clone(), + manifest_blob: None, + spec_blob: None, + git: run_spec.git.clone(), + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -1165,44 +1168,46 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let run_spec = RunSpec { - run_id: fixtures::RUN_1, - settings: fabro_types::WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: Some("test".to_string()), - automation: None, - source_directory: Some("/tmp/project".to_string()), - git: Some(fabro_types::GitContext { + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + source_directory: Some("/tmp/project".to_string()), + git: Some(fabro_types::GitContext { origin_url: String::new(), branch: "main".to_string(), sha: None, dirty: fabro_types::DirtyStatus::Clean, }), - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - fork_source_ref: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + fork_source_ref: None, }; append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&run_spec.settings).unwrap(), - graph: serde_json::to_value(&run_spec.graph).unwrap(), - workflow_source: Some("digraph test { plan -> code }".to_string()), - labels: run_spec.labels.clone().into_iter().collect(), - source_directory: run_spec.source_directory.clone(), - workflow_slug: run_spec.workflow_slug.clone(), - automation: None, - provenance: run_spec.provenance.clone(), - manifest_blob: None, - spec_blob: None, - git: run_spec.git.clone(), - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: Some("digraph test { plan -> code }".to_string()), + labels: run_spec.labels.clone().into_iter().collect(), + source_directory: run_spec.source_directory.clone(), + workflow_slug: run_spec.workflow_slug.clone(), + workflow_version_id: None, + automation: None, + provenance: run_spec.provenance.clone(), + manifest_blob: None, + spec_blob: None, + git: run_spec.git.clone(), + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -1589,39 +1594,41 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); let run_spec = RunSpec { - run_id: fixtures::RUN_1, - settings: fabro_types::WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: Some("test".to_string()), - automation: None, - source_directory: Some("/tmp/project".to_string()), - git: None, - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + source_directory: Some("/tmp/project".to_string()), + git: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + fork_source_ref: None, }; append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&run_spec.settings).unwrap(), - graph: serde_json::to_value(&run_spec.graph).unwrap(), - workflow_source: Some("digraph test { plan -> code }".to_string()), - labels: run_spec.labels.clone().into_iter().collect(), - source_directory: run_spec.source_directory.clone(), - workflow_slug: run_spec.workflow_slug.clone(), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: Some("digraph test { plan -> code }".to_string()), + labels: run_spec.labels.clone().into_iter().collect(), + source_directory: run_spec.source_directory.clone(), + workflow_slug: run_spec.workflow_slug.clone(), + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); @@ -1808,39 +1815,41 @@ mod tests { let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); // Seed a completed run so the PR body can include run details. let run_spec = RunSpec { - run_id: fixtures::RUN_1, - settings: fabro_types::WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - git: None, - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: fabro_types::WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + git: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + fork_source_ref: None, }; append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&run_spec.settings).unwrap(), - graph: serde_json::to_value(&run_spec.graph).unwrap(), - workflow_source: None, - labels: run_spec.labels.clone().into_iter().collect(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: None, + labels: run_spec.labels.clone().into_iter().collect(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/run_lookup.rs b/lib/components/fabro-workflow/src/run_lookup.rs index 8a66dbf91..2b37861cf 100644 --- a/lib/components/fabro-workflow/src/run_lookup.rs +++ b/lib/components/fabro-workflow/src/run_lookup.rs @@ -490,23 +490,24 @@ mod tests { let run_spec = sample_run_spec(); let run_store = store.create_run(&fixtures::RUN_1).await.unwrap(); append_event(&run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&run_spec.settings).unwrap(), - graph: serde_json::to_value(&run_spec.graph).unwrap(), - workflow_source: None, - labels: run_spec.labels.clone().into_iter().collect(), - source_directory: run_spec.source_directory.clone(), - workflow_slug: run_spec.workflow_slug.clone(), - automation: None, - provenance: run_spec.provenance.clone(), - manifest_blob: None, - spec_blob: None, - git: run_spec.git.clone(), - fork_source_ref: run_spec.fork_source_ref.clone(), - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&run_spec.settings).unwrap(), + graph: serde_json::to_value(&run_spec.graph).unwrap(), + workflow_source: None, + labels: run_spec.labels.clone().into_iter().collect(), + source_directory: run_spec.source_directory.clone(), + workflow_slug: run_spec.workflow_slug.clone(), + workflow_version_id: None, + automation: None, + provenance: run_spec.provenance.clone(), + manifest_blob: None, + spec_blob: None, + git: run_spec.git.clone(), + fork_source_ref: run_spec.fork_source_ref.clone(), + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/run_metadata.rs b/lib/components/fabro-workflow/src/run_metadata.rs index 942b662f1..c4ae9951a 100644 --- a/lib/components/fabro-workflow/src/run_metadata.rs +++ b/lib/components/fabro-workflow/src/run_metadata.rs @@ -680,25 +680,26 @@ mod tests { let projection = RunProjection::new( "Metadata".to_string(), RunSpec { - run_id: fabro_types::fixtures::RUN_1, - settings: WorkflowSettings::default(), - graph: fabro_types::Graph::new("metadata"), - graph_source: None, - workflow_slug: Some("metadata".to_string()), - automation: None, - source_directory: Some("/Users/client/project".to_string()), - git: Some(GitContext { + run_id: fabro_types::fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph: fabro_types::Graph::new("metadata"), + graph_source: None, + workflow_slug: Some("metadata".to_string()), + workflow_version_id: None, + automation: None, + source_directory: Some("/Users/client/project".to_string()), + git: Some(GitContext { origin_url: "https://github.com/fabro-sh/fabro.git".to_string(), branch: "main".to_string(), sha: None, dirty: DirtyStatus::Clean, }), - labels: HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - fork_source_ref: None, + labels: HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + fork_source_ref: None, }, chrono::Utc::now(), ); diff --git a/lib/components/fabro-workflow/src/runtime_store.rs b/lib/components/fabro-workflow/src/runtime_store.rs index 2e506400c..b68caa27a 100644 --- a/lib/components/fabro-workflow/src/runtime_store.rs +++ b/lib/components/fabro-workflow/src/runtime_store.rs @@ -146,23 +146,24 @@ mod tests { async fn append_created_event(run_store: &fabro_store::RunDatabase) { let record = test_run_spec(); append_event(run_store, &fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(&record.settings).unwrap(), - graph: serde_json::to_value(&record.graph).unwrap(), - workflow_source: Some("digraph test {}".to_string()), - labels: std::collections::BTreeMap::new(), - source_directory: Some("/tmp/test".to_string()), - workflow_slug: Some("test".to_string()), - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + run_id: fixtures::RUN_1, + title: None, + settings: serde_json::to_value(&record.settings).unwrap(), + graph: serde_json::to_value(&record.graph).unwrap(), + workflow_source: Some("digraph test {}".to_string()), + labels: std::collections::BTreeMap::new(), + source_directory: Some("/tmp/test".to_string()), + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }) .await .unwrap(); diff --git a/lib/components/fabro-workflow/src/stage_execution.rs b/lib/components/fabro-workflow/src/stage_execution.rs index ec755127f..dfdf8a3ba 100644 --- a/lib/components/fabro-workflow/src/stage_execution.rs +++ b/lib/components/fabro-workflow/src/stage_execution.rs @@ -198,20 +198,21 @@ mod tests { fn projection_with_stages(stages: &[(&str, u32, u32)]) -> RunProjection { let spec = RunSpec { - run_id: RunId::new(), - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: std::collections::HashMap::new(), - provenance: test_support::test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: RunId::new(), + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: std::collections::HashMap::new(), + provenance: test_support::test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, }; let mut projection = RunProjection::new(String::new(), spec, Utc::now()); for (node_id, visit, seq) in stages { diff --git a/lib/components/fabro-workflow/src/test_support.rs b/lib/components/fabro-workflow/src/test_support.rs index c7407b7b9..55f7f9cd4 100644 --- a/lib/components/fabro-workflow/src/test_support.rs +++ b/lib/components/fabro-workflow/src/test_support.rs @@ -182,34 +182,35 @@ async fn initialized( .expect("failed to create slate-backed test run store"); let run_store = inner_store; append_event(&run_store, &run_options.run_id, &Event::RunCreated { - run_id: run_options.run_id, - title: None, - settings: serde_json::to_value(&run_options.settings) + run_id: run_options.run_id, + title: None, + settings: serde_json::to_value(&run_options.settings) .expect("failed to serialize settings"), - graph: serde_json::to_value(graph).expect("failed to serialize graph"), - workflow_source: None, - labels: run_options + graph: serde_json::to_value(graph).expect("failed to serialize graph"), + workflow_source: None, + labels: run_options .labels .clone() .into_iter() .collect::>(), - source_directory: Some(sandbox.working_directory().to_string()), - workflow_slug: run_options.workflow_slug.clone(), - automation: None, - provenance: fabro_types::RunProvenance { + source_directory: Some(sandbox.working_directory().to_string()), + workflow_slug: run_options.workflow_slug.clone(), + workflow_version_id: None, + automation: None, + provenance: fabro_types::RunProvenance { server: None, client: None, subject: fabro_types::Principal::System { system_kind: fabro_types::SystemActorKind::Engine, }, }, - manifest_blob: None, - spec_blob: None, - git: run_options.pre_run_git.clone(), - fork_source_ref: run_options.fork_source_ref.clone(), - retried_from: None, - parent_id: None, - web_url: None, + manifest_blob: None, + spec_blob: None, + git: run_options.pre_run_git.clone(), + fork_source_ref: run_options.fork_source_ref.clone(), + retried_from: None, + parent_id: None, + web_url: None, }) .await .expect("failed to seed run.created event in run store"); diff --git a/lib/foundation/fabro-api/tests/run_projection_round_trip.rs b/lib/foundation/fabro-api/tests/run_projection_round_trip.rs index f1a00a5a5..c2b0b43d8 100644 --- a/lib/foundation/fabro-api/tests/run_projection_round_trip.rs +++ b/lib/foundation/fabro-api/tests/run_projection_round_trip.rs @@ -1,7 +1,7 @@ use std::any::{TypeId, type_name}; use fabro_api::types::RunProjection as ApiRunProjection; -use fabro_types::{RunProjection, RunSpec, test_support}; +use fabro_types::{BlobHash, RunProjection, RunSpec, WorkflowVersionId, test_support}; use serde_json::json; #[test] fn run_projection_reuses_canonical_type() { @@ -12,7 +12,7 @@ fn run_projection_reuses_canonical_type() { fn run_projection_round_trips_populated_projection() { let value = json!({ "title": "Test run", - "spec": run_spec_json(), + "spec": run_spec_json(Some(WorkflowVersionId::from(BlobHash::new(b"workflow")))), "start": null, "status": { "kind": "submitted" }, "status_updated_at": "2026-04-29T12:34:00Z", @@ -108,7 +108,7 @@ fn run_projection_round_trips_populated_projection() { fn run_projection_round_trips_with_pending_control_unset() { let value = json!({ "title": "Test run", - "spec": run_spec_json(), + "spec": run_spec_json(None), "start": null, "status": { "kind": "submitted" }, "status_updated_at": "2026-04-29T12:34:00Z", @@ -127,9 +127,10 @@ fn run_projection_round_trips_with_pending_control_unset() { assert_eq!(serde_json::to_value(projection).unwrap(), value); } -fn run_spec_json() -> serde_json::Value { +fn run_spec_json(workflow_version_id: Option) -> serde_json::Value { serde_json::to_value(RunSpec { graph_source: Some("digraph test {}".to_string()), + workflow_version_id, ..test_support::test_run_spec() }) .unwrap() diff --git a/lib/foundation/fabro-types/src/run.rs b/lib/foundation/fabro-types/src/run.rs index 86f1910d7..34e56ffaf 100644 --- a/lib/foundation/fabro-types/src/run.rs +++ b/lib/foundation/fabro-types/src/run.rs @@ -8,6 +8,7 @@ use crate::graph::Graph; use crate::principal::Principal; use crate::run_id::RunId; use crate::run_summary::AutomationRef; +use crate::workflow_version_id::WorkflowVersionId; #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] pub struct RunServerProvenance { @@ -58,33 +59,35 @@ pub struct ForkSourceRef { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct RunSpec { - pub run_id: RunId, - pub settings: WorkflowSettings, - pub graph: Graph, + pub run_id: RunId, + pub settings: WorkflowSettings, + pub graph: Graph, #[serde(default, skip_serializing_if = "Option::is_none")] - pub graph_source: Option, + pub graph_source: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub workflow_slug: Option, + pub workflow_slug: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub automation: Option, + pub workflow_version_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub source_directory: Option, + pub automation: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_directory: Option, #[serde(default, skip_serializing_if = "HashMap::is_empty")] - pub labels: HashMap, - pub provenance: RunProvenance, + pub labels: HashMap, + pub provenance: RunProvenance, #[serde(default, skip_serializing_if = "Option::is_none")] - pub manifest_blob: Option, + pub manifest_blob: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub definition_blob: Option, + pub definition_blob: Option, /// Unredacted copy of this spec in the blob store. Stored events pass /// through secret redaction, so the spec folded from them is display /// data; execution must load the spec from this blob. #[serde(default, skip_serializing_if = "Option::is_none")] - pub spec_blob: Option, + pub spec_blob: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub git: Option, + pub git: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub fork_source_ref: Option, + pub fork_source_ref: Option, } impl RunSpec { diff --git a/lib/foundation/fabro-types/src/run_event/run.rs b/lib/foundation/fabro-types/src/run_event/run.rs index c7a17d990..8ef5a361e 100644 --- a/lib/foundation/fabro-types/src/run_event/run.rs +++ b/lib/foundation/fabro-types/src/run_event/run.rs @@ -7,42 +7,45 @@ use crate::status::{BlockedReason, PendingReason, SuccessReason}; use crate::{ AutomationRef, BlobHash, DiffSummary, ForkSourceRef, GitContext, Graph, PairId, PairTarget, RunControlAction, RunFailure, RunId, RunProvenance, RunTiming, WorkflowSettings, + WorkflowVersionId, }; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RunCreatedProps { #[serde(default, skip_serializing_if = "Option::is_none")] - pub title: Option, - pub settings: WorkflowSettings, - pub graph: Graph, + pub title: Option, + pub settings: WorkflowSettings, + pub graph: Graph, #[serde(default, skip_serializing_if = "Option::is_none")] - pub workflow_source: Option, + pub workflow_source: Option, #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub labels: BTreeMap, + pub labels: BTreeMap, #[serde(default, skip_serializing_if = "Option::is_none")] - pub source_directory: Option, + pub source_directory: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub workflow_slug: Option, + pub workflow_slug: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub automation: Option, - pub provenance: RunProvenance, + pub workflow_version_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub manifest_blob: Option, + pub automation: Option, + pub provenance: RunProvenance, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub manifest_blob: Option, /// Unredacted copy of the run spec in the blob store. The settings and /// graph on this event are redacted at the sink; execution loads the /// spec from this blob instead. #[serde(default, skip_serializing_if = "Option::is_none")] - pub spec_blob: Option, + pub spec_blob: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub git: Option, + pub git: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub fork_source_ref: Option, + pub fork_source_ref: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub retried_from: Option, + pub retried_from: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub parent_id: Option, + pub parent_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub web_url: Option, + pub web_url: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] diff --git a/lib/foundation/fabro-types/src/test_support.rs b/lib/foundation/fabro-types/src/test_support.rs index 41af963f1..96e507315 100644 --- a/lib/foundation/fabro-types/src/test_support.rs +++ b/lib/foundation/fabro-types/src/test_support.rs @@ -38,19 +38,20 @@ pub fn test_run_provenance() -> RunProvenance { #[must_use] pub fn test_run_spec() -> RunSpec { RunSpec { - run_id: fixtures::RUN_1, - settings: WorkflowSettings::default(), - graph: Graph::new("test"), - graph_source: None, - workflow_slug: None, - automation: None, - source_directory: None, - labels: HashMap::new(), - provenance: test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph: Graph::new("test"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: HashMap::new(), + provenance: test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, } } diff --git a/lib/foundation/fabro-types/tests/run_event_serde.rs b/lib/foundation/fabro-types/tests/run_event_serde.rs index f287a6acc..c5a84fe78 100644 --- a/lib/foundation/fabro-types/tests/run_event_serde.rs +++ b/lib/foundation/fabro-types/tests/run_event_serde.rs @@ -7,7 +7,9 @@ use fabro_types::run_event::{RunSessionTurnFailedCode, RunSessionTurnFailedProps use fabro_types::settings::InterpString; use fabro_types::settings::run::RunGoal; use fabro_types::test_support::test_run_provenance; -use fabro_types::{AutomationRef, EventBody, TurnId, WorkflowSettings, fixtures}; +use fabro_types::{ + AutomationRef, BlobHash, EventBody, TurnId, WorkflowSettings, WorkflowVersionId, fixtures, +}; fn templated_settings() -> WorkflowSettings { let mut settings = WorkflowSettings::default(); @@ -18,34 +20,37 @@ fn templated_settings() -> WorkflowSettings { #[test] fn run_created_props_round_trip_templated_settings() { let props = RunCreatedProps { - title: Some("Ship task".to_string()), - settings: templated_settings(), - graph: Graph::new("ship"), - workflow_source: Some("digraph Ship { start -> exit }".to_string()), - labels: BTreeMap::from([("team".to_string(), "platform".to_string())]), - source_directory: Some("/Users/client/project".to_string()), - workflow_slug: Some("demo".to_string()), - automation: Some(AutomationRef { + title: Some("Ship task".to_string()), + settings: templated_settings(), + graph: Graph::new("ship"), + workflow_source: Some("digraph Ship { start -> exit }".to_string()), + labels: BTreeMap::from([("team".to_string(), "platform".to_string())]), + source_directory: Some("/Users/client/project".to_string()), + workflow_slug: Some("demo".to_string()), + workflow_version_id: Some(WorkflowVersionId::from(BlobHash::new(b"workflow"))), + automation: Some(AutomationRef { id: "nightly".to_string(), name: Some("Nightly".to_string()), trigger_id: Some("schedule_1".to_string()), }), - provenance: test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: Some(GitContext { + provenance: test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: Some(GitContext { origin_url: "https://github.com/fabro-sh/fabro.git".to_string(), branch: "main".to_string(), sha: None, dirty: DirtyStatus::Unknown, }), - fork_source_ref: Some(ForkSourceRef { + fork_source_ref: Some(ForkSourceRef { source_run_id: fixtures::RUN_2, checkpoint_sha: "def456".to_string(), }), - retried_from: Some(fixtures::RUN_1), - parent_id: Some(fixtures::RUN_2), - web_url: Some("http://localhost:3000/runs/01JNQVR7M0EJ5GKAT2SC4ERS1Z".to_string()), + retried_from: Some(fixtures::RUN_1), + parent_id: Some(fixtures::RUN_2), + web_url: Some( + "http://localhost:3000/runs/01JNQVR7M0EJ5GKAT2SC4ERS1Z".to_string(), + ), }; let json = serde_json::to_value(&props).expect("props should serialize"); @@ -67,6 +72,10 @@ fn run_created_props_round_trip_templated_settings() { assert_eq!(json["parent_id"], fixtures::RUN_2.to_string()); assert_eq!(json["automation"]["id"], "nightly"); assert_eq!(json["automation"]["trigger_id"], "schedule_1"); + assert_eq!( + json["workflow_version_id"], + BlobHash::new(b"workflow").to_string() + ); let round_trip: RunCreatedProps = serde_json::from_value(json.clone()).expect("props should deserialize"); @@ -84,22 +93,23 @@ fn run_created_props_round_trip_templated_settings() { #[test] fn run_created_props_omits_web_url_when_absent() { let props = RunCreatedProps { - title: None, - settings: WorkflowSettings::default(), - graph: Graph::new("ship"), - workflow_source: None, - labels: BTreeMap::new(), - source_directory: None, - workflow_slug: None, - automation: None, - provenance: test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, + title: None, + settings: WorkflowSettings::default(), + graph: Graph::new("ship"), + workflow_source: None, + labels: BTreeMap::new(), + source_directory: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + provenance: test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, }; let json = serde_json::to_value(&props).expect("props should serialize"); @@ -115,6 +125,10 @@ fn run_created_props_omits_web_url_when_absent() { json.get("retried_from").is_none(), "retried_from must be omitted when None, got {json}" ); + assert!( + json.get("workflow_version_id").is_none(), + "workflow_version_id must be omitted when None, got {json}" + ); let round_trip: RunCreatedProps = serde_json::from_value(json.clone()).expect("props should deserialize"); @@ -137,6 +151,7 @@ fn run_created_props_defaults_additive_fields_for_legacy_events() { serde_json::from_value(json).expect("legacy props should deserialize"); assert_eq!(props.retried_from, None); assert_eq!(props.automation, None); + assert_eq!(props.workflow_version_id, None); } #[test] diff --git a/lib/foundation/fabro-types/tests/run_spec_serde.rs b/lib/foundation/fabro-types/tests/run_spec_serde.rs index ec7a7ef40..06d953c69 100644 --- a/lib/foundation/fabro-types/tests/run_spec_serde.rs +++ b/lib/foundation/fabro-types/tests/run_spec_serde.rs @@ -5,7 +5,7 @@ use fabro_types::run::{DirtyStatus, ForkSourceRef, GitContext, RunSpec}; use fabro_types::settings::InterpString; use fabro_types::settings::run::RunGoal; use fabro_types::test_support::test_run_provenance; -use fabro_types::{AutomationRef, WorkflowSettings, fixtures}; +use fabro_types::{AutomationRef, BlobHash, WorkflowSettings, WorkflowVersionId, fixtures}; fn templated_settings() -> WorkflowSettings { let mut settings = WorkflowSettings::default(); @@ -16,29 +16,30 @@ fn templated_settings() -> WorkflowSettings { #[test] fn run_spec_round_trips_templated_settings() { let record = RunSpec { - run_id: fixtures::RUN_1, - settings: templated_settings(), - graph: Graph::new("ship"), - graph_source: None, - workflow_slug: Some("demo".to_string()), - automation: Some(AutomationRef { + run_id: fixtures::RUN_1, + settings: templated_settings(), + graph: Graph::new("ship"), + graph_source: None, + workflow_slug: Some("demo".to_string()), + workflow_version_id: Some(WorkflowVersionId::from(BlobHash::new(b"workflow"))), + automation: Some(AutomationRef { id: "nightly".to_string(), name: Some("Nightly".to_string()), trigger_id: Some("schedule_1".to_string()), }), - source_directory: Some("/Users/client/project".to_string()), - labels: HashMap::from([("team".to_string(), "platform".to_string())]), - provenance: test_run_provenance(), - manifest_blob: None, - definition_blob: None, - spec_blob: None, - git: Some(GitContext { + source_directory: Some("/Users/client/project".to_string()), + labels: HashMap::from([("team".to_string(), "platform".to_string())]), + provenance: test_run_provenance(), + manifest_blob: None, + definition_blob: None, + spec_blob: None, + git: Some(GitContext { origin_url: "https://github.com/fabro-sh/fabro.git".to_string(), branch: "main".to_string(), sha: Some("abc123".to_string()), dirty: DirtyStatus::Clean, }), - fork_source_ref: Some(ForkSourceRef { + fork_source_ref: Some(ForkSourceRef { source_run_id: fixtures::RUN_2, checkpoint_sha: "def456".to_string(), }), @@ -59,6 +60,10 @@ fn run_spec_round_trips_templated_settings() { assert_eq!(json["fork_source_ref"]["checkpoint_sha"], "def456"); assert_eq!(json["automation"]["id"], "nightly"); assert_eq!(json["automation"]["trigger_id"], "schedule_1"); + assert_eq!( + json["workflow_version_id"], + BlobHash::new(b"workflow").to_string() + ); let round_trip: RunSpec = serde_json::from_value(json.clone()).expect("record should deserialize"); @@ -85,4 +90,28 @@ fn run_spec_defaults_automation_for_legacy_specs() { let record: RunSpec = serde_json::from_value(json).expect("legacy spec should deserialize"); assert_eq!(record.automation, None); + assert_eq!(record.workflow_version_id, None); +} + +#[test] +fn run_spec_omits_absent_workflow_version_id() { + let record = RunSpec { + run_id: fixtures::RUN_1, + settings: WorkflowSettings::default(), + graph: Graph::new("ship"), + graph_source: None, + workflow_slug: None, + workflow_version_id: None, + automation: None, + source_directory: None, + labels: HashMap::new(), + provenance: test_run_provenance(), + manifest_blob: None, + definition_blob: None, + git: None, + fork_source_ref: None, + }; + + let json = serde_json::to_value(record).expect("record should serialize"); + assert!(json.get("workflow_version_id").is_none()); } diff --git a/lib/packages/fabro-api-client/src/models/run-spec.ts b/lib/packages/fabro-api-client/src/models/run-spec.ts index e83be609b..a9ed38078 100644 --- a/lib/packages/fabro-api-client/src/models/run-spec.ts +++ b/lib/packages/fabro-api-client/src/models/run-spec.ts @@ -38,6 +38,10 @@ export interface RunSpec { 'graph': { [key: string]: any; }; 'graph_source'?: string | null; 'workflow_slug'?: string | null; + /** + * SHA-256 identity of validated canonical workflow-version bytes. + */ + 'workflow_version_id'?: string | null; 'automation'?: AutomationRef | null; 'source_directory'?: string | null; 'labels'?: { [key: string]: string; }; From 9d3aa7a4d444ba5b000198929d4c78600a94ee69 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Wed, 19 Aug 2026 15:58:16 -0400 Subject: [PATCH 12/30] Consolidate workflow-version lineage test coverage The lineage field's `skip_serializing_if` behavior was asserted five times across three crates. Keep the two assertions in fabro-types, which owns the attribute, and drop the duplicates: - Delete `run_created_omits_absent_workflow_version_id` from event/convert.rs, a copy of the test above it that re-checked another crate's serde attribute. convert.rs's own responsibility is covered by the existing field assertion. - Delete `legacy_create_input_persists_without_workflow_version_id`, which ran the full create() pipeline to prove a hardcoded `None` literal is `None`. `CreateRunInput` has no such field, so no input could change the result. - Fold `run_spec_omits_absent_workflow_version_id` into the adjacent legacy-spec test, which already holds an all-`None` record. - Drop the off-topic spec re-serialization from run_state.rs's retried_from test. Add `test_support::test_workflow_version_id()` alongside `test_run_provenance()` and use it everywhere, replacing eight copies of the same magic seed across five crates plus two assertion sites that recomputed the hash inline. This also subsumes retry.rs's private helper of the same shape. Revert the `run_spec_json` parameterization in the projection round-trip test: `RunProjection` is a `with_replacement` alias for the canonical type, so the `Some` and `None` call sites exercise identical code. Have the two run.created literals that mirror a `RunSpec` read the spec's lineage field instead of hardcoding `None`, so the mirrors stay accurate once a producer populates it. Co-Authored-By: Claude Opus 5 (1M context) --- lib/apps/fabro-server/src/run_compiler.rs | 6 +- lib/apps/fabro-server/src/server/tests.rs | 2 +- lib/components/fabro-store/src/run_state.rs | 8 +-- .../fabro-workflow/src/event/convert.rs | 43 ++------------- .../fabro-workflow/src/operations/create.rs | 55 +------------------ .../fabro-workflow/src/operations/fork.rs | 6 +- .../fabro-workflow/src/operations/retry.rs | 10 +--- .../src/pipeline/pull_request.rs | 6 +- .../tests/run_projection_round_trip.rs | 10 ++-- .../fabro-types/src/test_support.rs | 8 ++- .../fabro-types/tests/run_event_serde.rs | 10 ++-- .../fabro-types/tests/run_spec_serde.rs | 32 ++--------- 12 files changed, 42 insertions(+), 154 deletions(-) diff --git a/lib/apps/fabro-server/src/run_compiler.rs b/lib/apps/fabro-server/src/run_compiler.rs index 291f0bb1d..b5358f1a3 100644 --- a/lib/apps/fabro-server/src/run_compiler.rs +++ b/lib/apps/fabro-server/src/run_compiler.rs @@ -650,9 +650,7 @@ mod tests { use fabro_model::Catalog; use fabro_types::settings::interp::ResolveCtx; use fabro_types::settings::run::RunGoal; - use fabro_types::{ - AutomationRef, BlobHash, Principal, RunProvenance, SystemActorKind, WorkflowVersionId, - }; + use fabro_types::{AutomationRef, Principal, RunProvenance, SystemActorKind}; use fabro_workflow::workflow_bundle::ParsedWorkflowConfig; use super::*; @@ -988,7 +986,7 @@ include = ["reports/{{ vars.path }}/*.json"] trigger_id: Some("schedule".to_string()), }; let submitted = b"submitted manifest".to_vec(); - let workflow_version_id = WorkflowVersionId::from(BlobHash::new(b"workflow")); + let workflow_version_id = fabro_types::test_support::test_workflow_version_id(); let mut input = raw_input(None, HashMap::new()); input.run_id = Some(run_id); input.parent_id = Some(parent_id); diff --git a/lib/apps/fabro-server/src/server/tests.rs b/lib/apps/fabro-server/src/server/tests.rs index 4783f3bb2..dc21a12e6 100644 --- a/lib/apps/fabro-server/src/server/tests.rs +++ b/lib/apps/fabro-server/src/server/tests.rs @@ -7122,7 +7122,7 @@ async fn create_completed_run_ready_for_pull_request( labels: run_spec.labels.clone().into_iter().collect(), source_directory: run_spec.source_directory.clone(), workflow_slug: run_spec.workflow_slug.clone(), - workflow_version_id: None, + workflow_version_id: run_spec.workflow_version_id, automation: None, provenance: run_spec.provenance.clone(), manifest_blob: None, diff --git a/lib/components/fabro-store/src/run_state.rs b/lib/components/fabro-store/src/run_state.rs index bcd21a3d2..da54a2d29 100644 --- a/lib/components/fabro-store/src/run_state.rs +++ b/lib/components/fabro-store/src/run_state.rs @@ -1691,8 +1691,8 @@ mod tests { RunStatus, Speed, StageContextWindowBreakdownItem, StageContextWindowCategory, StageContextWindowCountMethod, StageContextWindowProjection, StageContextWindowStaleness, StageContextWindowWarning, StageHandler, StageModelUsage, StageOutcome, StageState, - StageTiming, SubAgentStatus, SuccessReason, WorkflowSettings, WorkflowVersionId, - first_event_seq, fixtures, test_support, + StageTiming, SubAgentStatus, SuccessReason, WorkflowSettings, first_event_seq, fixtures, + test_support, }; use serde_json::json; @@ -2369,8 +2369,6 @@ mod tests { let projection = RunProjection::apply_events(&[event]).unwrap(); assert_eq!(projection.retried_from, None); assert_eq!(projection.spec.workflow_version_id, None); - let spec_json = serde_json::to_value(&projection.spec).unwrap(); - assert!(spec_json.get("workflow_version_id").is_none()); assert_eq!( build_summary(&projection, &fixtures::RUN_1).retried_from, None @@ -2379,7 +2377,7 @@ mod tests { #[test] fn run_created_projects_workflow_version_id_into_spec() { - let workflow_version_id = WorkflowVersionId::from(BlobHash::new(b"workflow")); + let workflow_version_id = test_support::test_workflow_version_id(); let event = test_raw_event( 1, "run.created", diff --git a/lib/components/fabro-workflow/src/event/convert.rs b/lib/components/fabro-workflow/src/event/convert.rs index 420706919..7d9af8f79 100644 --- a/lib/components/fabro-workflow/src/event/convert.rs +++ b/lib/components/fabro-workflow/src/event/convert.rs @@ -1445,9 +1445,9 @@ mod tests { use std::collections::BTreeMap; use ::fabro_types::{ - AutomationRef, BlobHash, EventBody, FailureReason, ParallelBranchId, Principal, - RunNoticeCode, RunNoticeLevel, RunProvenance, StageId, SystemActorKind, WorkflowVersionId, - fixtures, run_event as fabro_types, + AutomationRef, EventBody, FailureReason, ParallelBranchId, Principal, RunNoticeCode, + RunNoticeLevel, RunProvenance, StageId, SystemActorKind, fixtures, + run_event as fabro_types, test_support, }; use chrono::Utc; use fabro_agent::{ @@ -2830,7 +2830,7 @@ mod tests { name: Some("Nightly".to_string()), trigger_id: Some("schedule_1".to_string()), }; - let workflow_version_id = WorkflowVersionId::from(BlobHash::new(b"workflow")); + let workflow_version_id = test_support::test_workflow_version_id(); let stored = to_run_event(&fixtures::RUN_1, &Event::RunCreated { run_id: fixtures::RUN_1, @@ -2861,41 +2861,6 @@ mod tests { assert_eq!(props.workflow_version_id, Some(workflow_version_id)); } - #[test] - fn run_created_omits_absent_workflow_version_id() { - use ::fabro_types::{Graph, WorkflowSettings, fixtures}; - - let stored = to_run_event(&fixtures::RUN_1, &Event::RunCreated { - run_id: fixtures::RUN_1, - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(Graph::new("test")).unwrap(), - workflow_source: None, - labels: BTreeMap::default(), - source_directory: None, - workflow_slug: None, - workflow_version_id: None, - automation: None, - provenance: RunProvenance { - server: None, - client: None, - subject: user_principal("alice"), - }, - manifest_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, - }); - let EventBody::RunCreated(props) = stored.body else { - panic!("expected run.created body"); - }; - - let json = serde_json::to_value(props).expect("run.created props should serialize"); - assert!(json.get("workflow_version_id").is_none()); - } - #[test] fn agent_memory_loaded_maps_to_typed_event_body() { let stored = to_run_event(&fixtures::RUN_1, &Event::Agent { diff --git a/lib/components/fabro-workflow/src/operations/create.rs b/lib/components/fabro-workflow/src/operations/create.rs index 9647a7179..54f095bd2 100644 --- a/lib/components/fabro-workflow/src/operations/create.rs +++ b/lib/components/fabro-workflow/src/operations/create.rs @@ -689,9 +689,7 @@ mod tests { use fabro_store::Database; use fabro_types::settings::InterpString; use fabro_types::settings::run::RunMode; - use fabro_types::{ - BlobHash, EventBody, WorkflowSettings, WorkflowVersionId, fixtures, test_support, - }; + use fabro_types::{EventBody, WorkflowSettings, fixtures, test_support}; use fabro_util::error::collect_chain; use fabro_validate::Severity; use object_store::local::LocalFileSystem; @@ -1776,7 +1774,7 @@ reasoning = false std::fs::write(&dot_path, "this is no longer a graph").unwrap(); let materialized = materialize_create_run(compiled, catalog.as_ref()).unwrap(); - let workflow_version_id = WorkflowVersionId::from(BlobHash::new(b"workflow")); + let workflow_version_id = test_support::test_workflow_version_id(); let mut metadata = persistence_metadata(&request, fixtures::RUN_2, &storage_root); metadata.workflow_version_id = Some(workflow_version_id); let input = assemble_create_run_persistence_input(materialized, metadata); @@ -1824,55 +1822,6 @@ reasoning = false ); } - #[tokio::test] - async fn legacy_create_input_persists_without_workflow_version_id() { - let dir = tempfile::tempdir().unwrap(); - let store = memory_store(); - let run_id = fixtures::RUN_64; - let request = CreateRunInput { - workflow: WorkflowInput::DotSource { - source: MINIMAL_DOT.to_string(), - base_dir: None, - }, - settings: test_default_settings(), - vars: HashMap::new(), - cwd: dir.path().to_path_buf(), - workflow_slug: Some("legacy-create".to_string()), - workflow_path: None, - workflow_bundle: None, - submitted_manifest_bytes: None, - run_id: Some(run_id), - title: None, - automation: None, - git: None, - fork_source_ref: None, - parent_id: None, - provenance: test_support::test_run_provenance(), - configured_providers: test_provider_ids(), - web_url: None, - }; - - create( - store.as_ref(), - request, - dir.path().join("storage"), - test_catalog(), - ) - .await - .unwrap(); - - let run_store = store.open_run_reader(&run_id).await.unwrap(); - let state = run_store.state().await.unwrap(); - assert_eq!(state.spec.workflow_version_id, None); - let events = run_store.list_events().await.unwrap(); - let EventBody::RunCreated(created) = &events[0].event.body else { - panic!("first durable event should be run.created"); - }; - assert_eq!(created.workflow_version_id, None); - let json = serde_json::to_value(created).unwrap(); - assert!(json.get("workflow_version_id").is_none()); - } - #[tokio::test] async fn create_returns_validation_failed_with_diagnostics() { let dot = r#"digraph Test { diff --git a/lib/components/fabro-workflow/src/operations/fork.rs b/lib/components/fabro-workflow/src/operations/fork.rs index 857e02e47..1439ab471 100644 --- a/lib/components/fabro-workflow/src/operations/fork.rs +++ b/lib/components/fabro-workflow/src/operations/fork.rs @@ -287,9 +287,7 @@ mod tests { use fabro_graphviz::graph::Graph; use fabro_store::{Database, RunProjectionReducer}; - use fabro_types::{ - BlobHash, StageId, WorkflowSettings, WorkflowVersionId, fixtures, test_support, - }; + use fabro_types::{StageId, WorkflowSettings, fixtures, test_support}; use object_store::memory::InMemory; use super::*; @@ -374,7 +372,7 @@ mod tests { let source = store.create_run(&source_run_id).await.unwrap(); let graph = Graph::new("fork-source"); let settings = WorkflowSettings::default(); - let workflow_version_id = WorkflowVersionId::from(BlobHash::new(b"workflow")); + let workflow_version_id = test_support::test_workflow_version_id(); event::append_event(&source, &source_run_id, &Event::RunCreated { run_id: source_run_id, diff --git a/lib/components/fabro-workflow/src/operations/retry.rs b/lib/components/fabro-workflow/src/operations/retry.rs index b2cfa3022..8acde0e43 100644 --- a/lib/components/fabro-workflow/src/operations/retry.rs +++ b/lib/components/fabro-workflow/src/operations/retry.rs @@ -125,7 +125,7 @@ mod tests { use fabro_types::{ AuthMethod, BlobHash, DirtyStatus, FailureReason, ForkSourceRef, GitContext, Graph, IdpIdentity, Principal, PullRequestLink, RunRunnableSource, RunServerProvenance, RunTiming, - WorkflowSettings, WorkflowVersionId, fixtures, + WorkflowSettings, fixtures, test_support, }; use object_store::memory::InMemory; @@ -167,10 +167,6 @@ mod tests { } } - fn workflow_version_id() -> WorkflowVersionId { - WorkflowVersionId::from(BlobHash::new(b"workflow")) - } - async fn append_created( store: &fabro_store::RunDatabase, run_id: RunId, @@ -192,7 +188,7 @@ mod tests { labels: labels.into_iter().collect(), source_directory: Some("/workspace/source".to_string()), workflow_slug: Some("retry-source".to_string()), - workflow_version_id: Some(workflow_version_id()), + workflow_version_id: Some(test_support::test_workflow_version_id()), automation: None, provenance: provenance("source-user"), manifest_blob, @@ -402,7 +398,7 @@ mod tests { assert_eq!(retry_state.spec.graph.name, "retry_source"); assert_eq!( retry_state.spec.workflow_version_id, - Some(workflow_version_id()) + Some(test_support::test_workflow_version_id()) ); assert_eq!( retry_state.spec.graph_source.as_deref(), diff --git a/lib/components/fabro-workflow/src/pipeline/pull_request.rs b/lib/components/fabro-workflow/src/pipeline/pull_request.rs index 152769978..c0e35acfc 100644 --- a/lib/components/fabro-workflow/src/pipeline/pull_request.rs +++ b/lib/components/fabro-workflow/src/pipeline/pull_request.rs @@ -1127,7 +1127,7 @@ mod tests { labels: run_spec.labels.clone().into_iter().collect(), source_directory: run_spec.source_directory.clone(), workflow_slug: run_spec.workflow_slug.clone(), - workflow_version_id: None, + workflow_version_id: run_spec.workflow_version_id, automation: None, provenance: run_spec.provenance.clone(), manifest_blob: None, @@ -1198,7 +1198,7 @@ mod tests { labels: run_spec.labels.clone().into_iter().collect(), source_directory: run_spec.source_directory.clone(), workflow_slug: run_spec.workflow_slug.clone(), - workflow_version_id: None, + workflow_version_id: run_spec.workflow_version_id, automation: None, provenance: run_spec.provenance.clone(), manifest_blob: None, @@ -1619,7 +1619,7 @@ mod tests { labels: run_spec.labels.clone().into_iter().collect(), source_directory: run_spec.source_directory.clone(), workflow_slug: run_spec.workflow_slug.clone(), - workflow_version_id: None, + workflow_version_id: run_spec.workflow_version_id, automation: None, provenance: test_support::test_run_provenance(), manifest_blob: None, diff --git a/lib/foundation/fabro-api/tests/run_projection_round_trip.rs b/lib/foundation/fabro-api/tests/run_projection_round_trip.rs index c2b0b43d8..e09a867a6 100644 --- a/lib/foundation/fabro-api/tests/run_projection_round_trip.rs +++ b/lib/foundation/fabro-api/tests/run_projection_round_trip.rs @@ -1,7 +1,7 @@ use std::any::{TypeId, type_name}; use fabro_api::types::RunProjection as ApiRunProjection; -use fabro_types::{BlobHash, RunProjection, RunSpec, WorkflowVersionId, test_support}; +use fabro_types::{RunProjection, RunSpec, test_support}; use serde_json::json; #[test] fn run_projection_reuses_canonical_type() { @@ -12,7 +12,7 @@ fn run_projection_reuses_canonical_type() { fn run_projection_round_trips_populated_projection() { let value = json!({ "title": "Test run", - "spec": run_spec_json(Some(WorkflowVersionId::from(BlobHash::new(b"workflow")))), + "spec": run_spec_json(), "start": null, "status": { "kind": "submitted" }, "status_updated_at": "2026-04-29T12:34:00Z", @@ -108,7 +108,7 @@ fn run_projection_round_trips_populated_projection() { fn run_projection_round_trips_with_pending_control_unset() { let value = json!({ "title": "Test run", - "spec": run_spec_json(None), + "spec": run_spec_json(), "start": null, "status": { "kind": "submitted" }, "status_updated_at": "2026-04-29T12:34:00Z", @@ -127,10 +127,10 @@ fn run_projection_round_trips_with_pending_control_unset() { assert_eq!(serde_json::to_value(projection).unwrap(), value); } -fn run_spec_json(workflow_version_id: Option) -> serde_json::Value { +fn run_spec_json() -> serde_json::Value { serde_json::to_value(RunSpec { graph_source: Some("digraph test {}".to_string()), - workflow_version_id, + workflow_version_id: Some(test_support::test_workflow_version_id()), ..test_support::test_run_spec() }) .unwrap() diff --git a/lib/foundation/fabro-types/src/test_support.rs b/lib/foundation/fabro-types/src/test_support.rs index 96e507315..c5065cb76 100644 --- a/lib/foundation/fabro-types/src/test_support.rs +++ b/lib/foundation/fabro-types/src/test_support.rs @@ -1,7 +1,8 @@ use std::collections::HashMap; use crate::{ - AuthMethod, Graph, IdpIdentity, Principal, RunProvenance, RunSpec, WorkflowSettings, fixtures, + AuthMethod, BlobHash, Graph, IdpIdentity, Principal, RunProvenance, RunSpec, WorkflowSettings, + WorkflowVersionId, fixtures, }; #[must_use] @@ -55,3 +56,8 @@ pub fn test_run_spec() -> RunSpec { fork_source_ref: None, } } + +#[must_use] +pub fn test_workflow_version_id() -> WorkflowVersionId { + WorkflowVersionId::from(BlobHash::new(b"workflow")) +} diff --git a/lib/foundation/fabro-types/tests/run_event_serde.rs b/lib/foundation/fabro-types/tests/run_event_serde.rs index c5a84fe78..2645b9ca4 100644 --- a/lib/foundation/fabro-types/tests/run_event_serde.rs +++ b/lib/foundation/fabro-types/tests/run_event_serde.rs @@ -6,10 +6,8 @@ use fabro_types::run_event::run::{RunCreatedProps, RunParentLinkedProps, RunPare use fabro_types::run_event::{RunSessionTurnFailedCode, RunSessionTurnFailedProps}; use fabro_types::settings::InterpString; use fabro_types::settings::run::RunGoal; -use fabro_types::test_support::test_run_provenance; -use fabro_types::{ - AutomationRef, BlobHash, EventBody, TurnId, WorkflowSettings, WorkflowVersionId, fixtures, -}; +use fabro_types::test_support::{test_run_provenance, test_workflow_version_id}; +use fabro_types::{AutomationRef, EventBody, TurnId, WorkflowSettings, fixtures}; fn templated_settings() -> WorkflowSettings { let mut settings = WorkflowSettings::default(); @@ -27,7 +25,7 @@ fn run_created_props_round_trip_templated_settings() { labels: BTreeMap::from([("team".to_string(), "platform".to_string())]), source_directory: Some("/Users/client/project".to_string()), workflow_slug: Some("demo".to_string()), - workflow_version_id: Some(WorkflowVersionId::from(BlobHash::new(b"workflow"))), + workflow_version_id: Some(test_workflow_version_id()), automation: Some(AutomationRef { id: "nightly".to_string(), name: Some("Nightly".to_string()), @@ -74,7 +72,7 @@ fn run_created_props_round_trip_templated_settings() { assert_eq!(json["automation"]["trigger_id"], "schedule_1"); assert_eq!( json["workflow_version_id"], - BlobHash::new(b"workflow").to_string() + test_workflow_version_id().to_string() ); let round_trip: RunCreatedProps = diff --git a/lib/foundation/fabro-types/tests/run_spec_serde.rs b/lib/foundation/fabro-types/tests/run_spec_serde.rs index 06d953c69..49432940c 100644 --- a/lib/foundation/fabro-types/tests/run_spec_serde.rs +++ b/lib/foundation/fabro-types/tests/run_spec_serde.rs @@ -4,8 +4,8 @@ use fabro_types::graph::Graph; use fabro_types::run::{DirtyStatus, ForkSourceRef, GitContext, RunSpec}; use fabro_types::settings::InterpString; use fabro_types::settings::run::RunGoal; -use fabro_types::test_support::test_run_provenance; -use fabro_types::{AutomationRef, BlobHash, WorkflowSettings, WorkflowVersionId, fixtures}; +use fabro_types::test_support::{test_run_provenance, test_workflow_version_id}; +use fabro_types::{AutomationRef, WorkflowSettings, fixtures}; fn templated_settings() -> WorkflowSettings { let mut settings = WorkflowSettings::default(); @@ -21,7 +21,7 @@ fn run_spec_round_trips_templated_settings() { graph: Graph::new("ship"), graph_source: None, workflow_slug: Some("demo".to_string()), - workflow_version_id: Some(WorkflowVersionId::from(BlobHash::new(b"workflow"))), + workflow_version_id: Some(test_workflow_version_id()), automation: Some(AutomationRef { id: "nightly".to_string(), name: Some("Nightly".to_string()), @@ -62,7 +62,7 @@ fn run_spec_round_trips_templated_settings() { assert_eq!(json["automation"]["trigger_id"], "schedule_1"); assert_eq!( json["workflow_version_id"], - BlobHash::new(b"workflow").to_string() + test_workflow_version_id().to_string() ); let round_trip: RunSpec = serde_json::from_value(json.clone()).expect("record should deserialize"); @@ -91,27 +91,7 @@ fn run_spec_defaults_automation_for_legacy_specs() { assert_eq!(record.automation, None); assert_eq!(record.workflow_version_id, None); -} -#[test] -fn run_spec_omits_absent_workflow_version_id() { - let record = RunSpec { - run_id: fixtures::RUN_1, - settings: WorkflowSettings::default(), - graph: Graph::new("ship"), - graph_source: None, - workflow_slug: None, - workflow_version_id: None, - automation: None, - source_directory: None, - labels: HashMap::new(), - provenance: test_run_provenance(), - manifest_blob: None, - definition_blob: None, - git: None, - fork_source_ref: None, - }; - - let json = serde_json::to_value(record).expect("record should serialize"); - assert!(json.get("workflow_version_id").is_none()); + let round_trip = serde_json::to_value(&record).expect("record should serialize"); + assert!(round_trip.get("workflow_version_id").is_none()); } From b964602b0bffc2771ab093c60cf80f52bc4451e5 Mon Sep 17 00:00:00 2001 From: Scott Werner Date: Fri, 21 Aug 2026 13:58:04 -0400 Subject: [PATCH 13/30] Harden legacy blob import cleanup --- .../fabro-store/src/legacy_blob_import.rs | 242 +++++++++++++++--- 1 file changed, 210 insertions(+), 32 deletions(-) diff --git a/lib/components/fabro-store/src/legacy_blob_import.rs b/lib/components/fabro-store/src/legacy_blob_import.rs index 2e1341b14..cf7f4390b 100644 --- a/lib/components/fabro-store/src/legacy_blob_import.rs +++ b/lib/components/fabro-store/src/legacy_blob_import.rs @@ -10,6 +10,8 @@ use bytes::Bytes; use fabro_types::BlobHash; use sqlx::pool::PoolConnection; use sqlx::{Acquire as _, Sqlite}; +#[cfg(test)] +use tokio::sync::Barrier; use tracing::debug; use crate::keys::SlateKey; @@ -56,6 +58,18 @@ impl LegacyBlobImportError { pub fn report(&self) -> &LegacyBlobImportReport { &self.report } + + /// Returns secondary errors encountered while cleaning up the failed + /// import. + /// + /// The standard error source chain preserves the failure that interrupted + /// the import. Because that chain is linear, rollback, connection-setting + /// restoration, and connection-retirement errors are exposed separately. + pub fn cleanup_errors(&self) -> impl Iterator { + let mut errors = Vec::new(); + self.failure.collect_cleanup_errors(&mut errors); + errors.into_iter() + } } impl fmt::Debug for LegacyBlobImportError { @@ -80,11 +94,12 @@ impl fmt::Display for LegacyBlobImportError { impl StdError for LegacyBlobImportError { fn source(&self) -> Option<&(dyn StdError + 'static)> { - Some(&self.failure) + Some(self.failure.primary_failure()) } } -#[derive(thiserror::Error)] +#[derive(strum::IntoStaticStr, thiserror::Error)] +#[strum(serialize_all = "snake_case")] enum LegacyBlobImportFailure { #[error("the legacy blob import target is not backed by SQLite")] WrongTargetBackend, @@ -161,28 +176,39 @@ enum LegacyBlobImportFailure { impl LegacyBlobImportFailure { fn kind(&self) -> &'static str { + self.into() + } + + fn primary_failure(&self) -> &Self { match self { - Self::WrongTargetBackend => "wrong_target_backend", - Self::OpenSource(_) => "open_source", - Self::OpenSourceScan(_) => "open_source_scan", - Self::ReadSourceScan(_) => "read_source_scan", - Self::InvalidSourceKey => "invalid_source_key", - Self::SourceDigestMismatch => "source_digest_mismatch", - Self::CounterOverflow => "counter_overflow", - Self::AcquireConnection(_) => "acquire_connection", - Self::ReadAutomaticCheckpoint(_) => "read_automatic_checkpoint", - Self::DisableAutomaticCheckpoint(_) => "disable_automatic_checkpoint", - Self::BeginTransaction(_) => "begin_transaction", - Self::InsertDestination(_) => "insert_destination", - Self::ReadDestination(_) => "read_destination", - Self::DestinationConflict => "destination_conflict", - Self::CommitTransaction(_) => "commit_transaction", - Self::RollbackTransaction { .. } => "rollback_transaction", - Self::PassiveCheckpoint(_) => "passive_checkpoint", - Self::PassiveCheckpointBusy => "passive_checkpoint_busy", - Self::FinalCheckpoint(_) => "final_checkpoint", - Self::FinalCheckpointBusy => "final_checkpoint_busy", - Self::RestoreAutomaticCheckpoint { .. } => "restore_automatic_checkpoint", + Self::RollbackTransaction { prior, .. } + | Self::RestoreAutomaticCheckpoint { + prior: Some(prior), .. + } => prior.primary_failure(), + _ => self, + } + } + + fn collect_cleanup_errors<'a>(&'a self, errors: &mut Vec<&'a (dyn StdError + 'static)>) { + match self { + Self::RollbackTransaction { source, prior } => { + errors.push(source); + prior.collect_cleanup_errors(errors); + } + Self::RestoreAutomaticCheckpoint { + source, + prior, + retirement_error, + } => { + errors.push(source); + if let Some(retirement_error) = retirement_error { + errors.push(retirement_error); + } + if let Some(prior) = prior { + prior.collect_cleanup_errors(errors); + } + } + _ => {} } } } @@ -213,16 +239,26 @@ impl fmt::Debug for LegacyBlobImportFailure { #[derive(Default)] struct ImportControls { #[cfg(test)] - source_after_rows: Option, + after_automatic_checkpoint_disabled: Option>, #[cfg(test)] - passive_checkpoint: bool, + source_after_rows: Option, #[cfg(test)] - final_checkpoint: bool, + passive_checkpoint: bool, #[cfg(test)] - restore_automatic_checkpoint: bool, + final_checkpoint: bool, + #[cfg(test)] + restore_automatic_checkpoint: bool, } impl ImportControls { + #[cfg(test)] + async fn after_automatic_checkpoint_disabled(&self) { + if let Some(barrier) = &self.after_automatic_checkpoint_disabled { + barrier.wait().await; + barrier.wait().await; + } + } + fn source_scan_error(&self, scanned_rows: u64) -> Option { #[cfg(test)] if self.source_after_rows == Some(scanned_rows) { @@ -268,6 +304,52 @@ impl ImportControls { } } +struct ImportConnection { + connection: Option>, + retire_on_drop: bool, +} + +impl ImportConnection { + fn new(connection: PoolConnection) -> Self { + Self { + connection: Some(connection), + retire_on_drop: false, + } + } + + fn get_mut(&mut self) -> &mut PoolConnection { + self.connection + .as_mut() + .expect("import connection exists until explicit retirement") + } + + fn retire_if_dropped(&mut self) { + self.retire_on_drop = true; + } + + fn checkpoint_setting_restored(&mut self) { + self.retire_on_drop = false; + } + + async fn retire(mut self) -> Result<(), sqlx::Error> { + let Some(mut connection) = self.connection.take() else { + return Ok(()); + }; + connection.close_on_drop(); + connection.close().await + } +} + +impl Drop for ImportConnection { + fn drop(&mut self) { + if self.retire_on_drop { + if let Some(connection) = &mut self.connection { + connection.close_on_drop(); + } + } + } +} + struct PendingBlob { hash: BlobHash, bytes: Bytes, @@ -337,10 +419,17 @@ impl Database { .fetch_one(&mut *connection) .await .map_err(LegacyBlobImportFailure::ReadAutomaticCheckpoint)?; + let mut connection = ImportConnection::new(connection); + // A cancelled PRAGMA future may already have changed connection-local + // state. Arm retirement before the first mutating await so an altered + // connection can never return to the pool without restoration. + connection.retire_if_dropped(); - let import_result = match set_automatic_checkpoint(&mut connection, 0).await { + let import_result = match set_automatic_checkpoint(connection.get_mut(), 0).await { Ok(()) => { - self.copy_legacy_blobs(&mut connection, controls, report) + #[cfg(test)] + controls.after_automatic_checkpoint_disabled().await; + self.copy_legacy_blobs(connection.get_mut(), controls, report) .await } Err(source) => Err(LegacyBlobImportFailure::DisableAutomaticCheckpoint(source)), @@ -349,11 +438,11 @@ impl Database { let restore_result = if let Some(error) = controls.restore_automatic_checkpoint_error() { Err(error) } else { - set_automatic_checkpoint(&mut connection, previous_automatic_checkpoint).await + set_automatic_checkpoint(connection.get_mut(), previous_automatic_checkpoint).await }; if let Err(source) = restore_result { - let retirement_error = connection.close().await.err(); + let retirement_error = connection.retire().await.err(); return Err(LegacyBlobImportFailure::RestoreAutomaticCheckpoint { source, prior: import_result.err().map(Box::new), @@ -361,6 +450,7 @@ impl Database { }); } + connection.checkpoint_setting_restored(); import_result } @@ -683,6 +773,8 @@ mod tests { use bytes::Bytes; use fabro_types::BlobHash; use object_store::memory::InMemory; + use sqlx::sqlite::{SqliteConnectOptions, SqliteJournalMode, SqlitePoolOptions}; + use tokio::sync::Barrier; use tracing::field::{Field, Visit}; use tracing::instrument::WithSubscriber as _; use tracing::span::{Attributes, Id, Record}; @@ -690,7 +782,7 @@ mod tests { use super::{ ImportControls, LegacyBlobImportFailure, LegacyBlobImportReport, MAX_BATCH_BYTES, - PASSIVE_CHECKPOINT_BYTES, + PASSIVE_CHECKPOINT_BYTES, set_automatic_checkpoint, }; use crate::keys::SlateKey; use crate::{BlobStore, Database}; @@ -1215,6 +1307,37 @@ mod tests { Ok(()) } + #[tokio::test] + async fn restoration_failure_preserves_the_prior_source_chain() -> TestResult<()> { + let context = TestContext::new().await?; + let controls = ImportControls { + source_after_rows: Some(0), + restore_automatic_checkpoint: true, + ..ImportControls::default() + }; + + let error = context + .source + .import_legacy_blobs_with_controls(&context.target, &controls) + .await + .expect_err("both injected failures should fail import"); + + let mut current: Option<&(dyn std::error::Error + 'static)> = Some(&error); + let mut saw_slate_source = false; + while let Some(source) = current { + saw_slate_source |= source.downcast_ref::().is_some(); + current = source.source(); + } + assert!(saw_slate_source, "prior source was absent from the chain"); + assert!( + error + .cleanup_errors() + .any(|source| source.downcast_ref::().is_some()), + "restoration source was absent from cleanup errors" + ); + Ok(()) + } + #[tokio::test] async fn automatic_checkpoint_setting_is_restored_after_success_and_failure() -> TestResult<()> { @@ -1238,6 +1361,61 @@ mod tests { Ok(()) } + #[tokio::test] + async fn cancellation_retires_a_connection_with_disabled_checkpointing() -> TestResult<()> { + let source = Database::new( + Arc::new(InMemory::new()), + "legacy-blob-import-cancellation-test", + Duration::from_millis(1), + None, + ); + let dir = tempfile::tempdir()?; + let options = SqliteConnectOptions::new() + .filename(dir.path().join("fabro.sqlite3")) + .create_if_missing(true) + .journal_mode(SqliteJournalMode::Wal); + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await?; + sqlx::query("CREATE TABLE blobs (hash TEXT PRIMARY KEY NOT NULL, data BLOB NOT NULL)") + .execute(&pool) + .await?; + let target = Arc::new(BlobStore::new(pool.clone())); + + let mut connection = pool.acquire().await?; + set_automatic_checkpoint(&mut connection, 73).await?; + drop(connection); + + let barrier = Arc::new(Barrier::new(2)); + let controls = ImportControls { + after_automatic_checkpoint_disabled: Some(Arc::clone(&barrier)), + ..ImportControls::default() + }; + let task = tokio::spawn({ + let source = source.clone(); + let target = Arc::clone(&target); + async move { + let mut report = LegacyBlobImportReport::default(); + source + .run_legacy_blob_import(&target, &controls, &mut report) + .await + } + }); + + barrier.wait().await; + task.abort(); + let join_error = task.await.expect_err("aborted import should be cancelled"); + assert!(join_error.is_cancelled()); + + let mut connection = pool.acquire().await?; + let observed: i64 = sqlx::query_scalar("PRAGMA wal_autocheckpoint") + .fetch_one(&mut *connection) + .await?; + assert_ne!(observed, 0); + Ok(()) + } + #[tokio::test] async fn failed_connection_restoration_retires_the_connection() -> TestResult<()> { let context = TestContext::new().await?; From 7de3b409ed67a6b0d65dca65762089fe3a0d2d7c Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 14:12:32 -0400 Subject: [PATCH 14/30] Request Packages read permission in the GitHub App manifest Fabro can mint a scoped sandbox GITHUB_TOKEN via [run.integrations.github.permissions], but apps registered through the manifest flow could not grant packages = "read" because the manifest never requested it. Add Packages (read-only) so freshly registered apps can download private GitHub Packages (for example npm registry dependencies) inside sandboxes, mirroring how GitHub Actions workflows use their built-in GITHUB_TOKEN for registry reads. Existing apps still need the permission added manually in the app's settings, as the docs already describe. Co-Authored-By: Claude Fable 5 --- docs/public/integrations/github.mdx | 1 + lib/apps/fabro-cli/src/commands/install.rs | 7 ++++++- lib/apps/fabro-server/src/install.rs | 18 +++++++++++++++++- 3 files changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/public/integrations/github.mdx b/docs/public/integrations/github.mdx index a08185561..faa251dc9 100644 --- a/docs/public/integrations/github.mdx +++ b/docs/public/integrations/github.mdx @@ -62,6 +62,7 @@ When you choose the GitHub App strategy, the CLI opens GitHub with a pre-filled | Emails | Read | Read verified email for OAuth login | | Dependabot alerts | Write | Read and manage repository vulnerability alerts | | Organization projects | Write | Read and update organization Projects V2 | + | Packages | Read | Download private GitHub Packages (e.g. npm registry) with the sandbox `GITHUB_TOKEN` | These permissions are included when Fabro registers a new app. For an existing GitHub App, add the missing permissions in the app's settings, then approve the permission update on each installation before workflows can use them. diff --git a/lib/apps/fabro-cli/src/commands/install.rs b/lib/apps/fabro-cli/src/commands/install.rs index 973524899..2a3f401b4 100644 --- a/lib/apps/fabro-cli/src/commands/install.rs +++ b/lib/apps/fabro-cli/src/commands/install.rs @@ -950,7 +950,8 @@ fn build_github_app_manifest(app_name: &str, port: u16, web_url: &str) -> serde_ "issues": "write", "emails": "read", "vulnerability_alerts": "write", - "organization_projects": "write" + "organization_projects": "write", + "packages": "read" }, "default_events": [] }) @@ -2682,6 +2683,10 @@ client_id = "client-id" manifest["default_permissions"]["organization_projects"], serde_json::json!("write"), ); + assert_eq!( + manifest["default_permissions"]["packages"], + serde_json::json!("read"), + ); } #[test] diff --git a/lib/apps/fabro-server/src/install.rs b/lib/apps/fabro-server/src/install.rs index 5f7d9b05a..0fae34bbb 100644 --- a/lib/apps/fabro-server/src/install.rs +++ b/lib/apps/fabro-server/src/install.rs @@ -2059,7 +2059,8 @@ fn build_github_app_manifest( "issues": "write", "emails": "read", "vulnerability_alerts": "write", - "organization_projects": "write" + "organization_projects": "write", + "packages": "read" }, "default_events": [] }) @@ -2375,6 +2376,21 @@ mod tests { ); } + #[test] + fn github_app_manifest_includes_packages_read_permission() { + let manifest = build_github_app_manifest( + "Fabro Test", + "https://fabro.example/setup", + "https://fabro.example/auth/callback/github", + "https://fabro.example/setup", + ); + + assert_eq!( + manifest["default_permissions"]["packages"], + serde_json::json!("read"), + ); + } + #[test] fn token_validation_accepts_any_matching_source() { let state = InstallAppState::for_test("expected"); From f2047ad9a98291ab0446c1da12e363d0c6d3eb40 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 14:15:46 -0400 Subject: [PATCH 15/30] feat(config): add validated additional github repositories Add `additional_repositories` to `[run.integrations.github]`: a list of full `owner/repository` slugs, beyond the implicit run origin, that the minted GITHUB_TOKEN must cover. - `GitHubRepositorySlug` gains FromStr, Display, string serde, and case-insensitive Eq/Ord/Hash identity while preserving the submitted spelling for display and serialization. - The config layer keeps raw strings; the higher-precedence list replaces the lower one wholesale, with `[]` as an explicit clear, resolving independently from the `permissions` map. - Resolution validates each entry with indexed error paths: slug grammar, case-insensitive duplicates, one shared owner, the 499-repository cap, and a required `contents = "read"|"write"` permission (templated values are re-checked at the runtime boundary). - `RunIntegrationsGithubSettings` resolves permissions and repositories together through `resolve_integration()` so consumers cannot pick up one without the other; the field is omitted from serialization when empty, keeping single-repository settings byte-identical. Co-Authored-By: Claude Fable 5 --- lib/apps/fabro-cli/src/commands/run/runner.rs | 5 +- lib/foundation/fabro-config/src/layers/run.rs | 14 +- .../fabro-config/src/resolve/run.rs | 143 +++++- .../fabro-config/src/tests/resolve_run.rs | 441 ++++++++++++++++++ lib/foundation/fabro-types/src/repository.rs | 158 ++++++- .../fabro-types/src/settings/run.rs | 105 ++++- 6 files changed, 846 insertions(+), 20 deletions(-) diff --git a/lib/apps/fabro-cli/src/commands/run/runner.rs b/lib/apps/fabro-cli/src/commands/run/runner.rs index 713838555..d3f21e78a 100644 --- a/lib/apps/fabro-cli/src/commands/run/runner.rs +++ b/lib/apps/fabro-cli/src/commands/run/runner.rs @@ -1762,7 +1762,10 @@ mod tests { .parse::() .expect("test provider should parse"); run.integrations = RunIntegrationsSettings { - github: RunIntegrationsGithubSettings { permissions }, + github: RunIntegrationsGithubSettings { + permissions, + ..RunIntegrationsGithubSettings::default() + }, }; run } diff --git a/lib/foundation/fabro-config/src/layers/run.rs b/lib/foundation/fabro-config/src/layers/run.rs index 8b020666f..7b6f9d57d 100644 --- a/lib/foundation/fabro-config/src/layers/run.rs +++ b/lib/foundation/fabro-config/src/layers/run.rs @@ -88,13 +88,23 @@ pub struct RunIntegrationsLayer { #[serde(deny_unknown_fields)] pub struct RunIntegrationsGithubLayer { #[serde(default, skip_serializing_if = "Option::is_none")] - pub permissions: Option>, + pub permissions: Option>, + /// Extra `owner/repository` slugs the minted `GITHUB_TOKEN` must cover in + /// addition to the implicit run origin. Kept as raw strings in this + /// sparse layer; slug validation happens at resolve time so diagnostics + /// can carry indexed paths. The higher-precedence list replaces the lower + /// one wholesale (`Some(vec![])` is an explicit clear); no `...` splice. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub additional_repositories: Option>, } impl Combine for RunIntegrationsGithubLayer { fn combine(self, other: Self) -> Self { Self { - permissions: self.permissions.or(other.permissions), + permissions: self.permissions.or(other.permissions), + additional_repositories: self + .additional_repositories + .or(other.additional_repositories), } } } diff --git a/lib/foundation/fabro-config/src/resolve/run.rs b/lib/foundation/fabro-config/src/resolve/run.rs index 9010f25f8..3fea93ab4 100644 --- a/lib/foundation/fabro-config/src/resolve/run.rs +++ b/lib/foundation/fabro-config/src/resolve/run.rs @@ -1,6 +1,8 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use fabro_types::GitHubRepositorySlug; use fabro_types::settings::InterpString; +use fabro_types::settings::interp::ResolveCtx; use fabro_types::settings::run::{ ArtifactsSettings, GitAuthorSettings, HookDefinition, HookType, InterviewProviderSettings, McpServerSettings, McpTransport, MergeStrategy, NotificationProviderSettings, @@ -17,9 +19,9 @@ use crate::{ EnvironmentLayer, HookAgentMarker, HookEntry, HookTlsMode, InterviewProviderLayer, InterviewsLayer, McpEntryLayer, MergeMap, ModelRefOrSplice, NotificationProviderLayer, NotificationRouteLayer, RunAgentLayer, RunArtifactsLayer, RunCheckpointLayer, RunCloneLayer, - RunExecutionLayer, RunGitLayer, RunGoalLayer, RunIntegrationsLayer, RunLayer, - RunMetaBranchLayer, RunModelLayer, RunPrepareLayer, RunPullRequestLayer, RunRunBranchLayer, - RunScmLayer, StickyMap, StringOrSplice, + RunExecutionLayer, RunGitLayer, RunGoalLayer, RunIntegrationsGithubLayer, RunIntegrationsLayer, + RunLayer, RunMetaBranchLayer, RunModelLayer, RunPrepareLayer, RunPullRequestLayer, + RunRunBranchLayer, RunScmLayer, StickyMap, StringOrSplice, }; pub fn resolve_run( @@ -85,23 +87,140 @@ pub fn resolve_run( scm: resolve_scm(layer.scm.as_ref()), pull_request, artifacts: resolve_artifacts(layer.artifacts.as_ref(), errors), - integrations: resolve_integrations(layer.integrations.as_ref()), + integrations: resolve_integrations(layer.integrations.as_ref(), errors), } } -fn resolve_integrations(layer: Option<&RunIntegrationsLayer>) -> RunIntegrationsSettings { +fn resolve_integrations( + layer: Option<&RunIntegrationsLayer>, + errors: &mut Vec, +) -> RunIntegrationsSettings { let github = layer .and_then(|integrations| integrations.github.as_ref()) - .map(|github| RunIntegrationsGithubSettings { - // Collapse `Option>` -> `HashMap<...>`: both `None` - // and `Some({})` resolve to an empty map (no token requested). - // The presence distinction is only meaningful at merge time. - permissions: github.permissions.clone().unwrap_or_default(), - }) + .map(|github| resolve_integrations_github(github, errors)) .unwrap_or_default(); RunIntegrationsSettings { github } } +/// GitHub caps one installation token at 500 repositories; the implicit run +/// origin takes one slot. +const MAX_ADDITIONAL_REPOSITORIES: usize = 499; + +fn resolve_integrations_github( + github: &RunIntegrationsGithubLayer, + errors: &mut Vec, +) -> RunIntegrationsGithubSettings { + // Collapse `Option>` -> `HashMap<...>`: both `None` + // and `Some({})` resolve to an empty map (no token requested). + // The presence distinction is only meaningful at merge time. The same + // collapse applies to `additional_repositories` (`Some(vec![])` is an + // explicit clear that resolves to the empty set). + let permissions = github.permissions.clone().unwrap_or_default(); + let raw_repositories = github + .additional_repositories + .as_deref() + .unwrap_or_default(); + + if raw_repositories.len() > MAX_ADDITIONAL_REPOSITORIES { + errors.push(ResolveError::Invalid { + path: "run.integrations.github.additional_repositories".to_string(), + reason: format!( + "at most {MAX_ADDITIONAL_REPOSITORIES} additional repositories are supported (the \ + run origin takes the remaining slot of GitHub's 500-repository token limit), got \ + {}", + raw_repositories.len() + ), + }); + } + + let mut additional_repositories: BTreeSet = BTreeSet::new(); + for (index, value) in raw_repositories.iter().enumerate() { + let path = format!("run.integrations.github.additional_repositories[{index}]"); + let Ok(slug) = value.parse::() else { + errors.push(ResolveError::Invalid { + path, + reason: format!( + "`{value}` is not a full GitHub `owner/repository` slug (no scheme, host, \ + ref, or extra path component)" + ), + }); + continue; + }; + if let Some(existing) = additional_repositories.get(&slug) { + errors.push(ResolveError::Invalid { + path, + reason: format!( + "`{value}` duplicates `{existing}` (repository identity is case-insensitive)" + ), + }); + continue; + } + if let Some(first) = additional_repositories.first() { + if !first.same_owner(&slug) { + errors.push(ResolveError::Invalid { + path, + reason: format!( + "`{value}` has owner `{}` but `{first}` has owner `{}`; all repositories \ + must share one owner because one GitHub App installation covers one \ + account", + slug.owner(), + first.owner() + ), + }); + continue; + } + } + additional_repositories.insert(slug); + } + + if !additional_repositories.is_empty() { + validate_additional_repository_permissions(&permissions, errors); + } + + RunIntegrationsGithubSettings { + permissions, + additional_repositories, + } +} + +/// A non-empty additional-repository set needs a token that can reach +/// repository contents. Only a literal `contents` value is checked here; a +/// templated value is re-checked after interpolation at the runtime boundary. +fn validate_additional_repository_permissions( + permissions: &HashMap, + errors: &mut Vec, +) { + if permissions.is_empty() { + errors.push(ResolveError::Invalid { + path: "run.integrations.github.additional_repositories".to_string(), + reason: "additional repositories require [run.integrations.github.permissions] with \ + a `contents` permission; a higher layer may have cleared the permissions" + .to_string(), + }); + return; + } + let Some(contents) = permissions.get("contents") else { + errors.push(ResolveError::Invalid { + path: "run.integrations.github.permissions".to_string(), + reason: "additional repositories require the `contents` permission (`read` or \ + `write`)" + .to_string(), + }); + return; + }; + if let Ok(literal) = contents.resolve_with(&mut ResolveCtx::new()) { + if literal != "read" && literal != "write" { + errors.push(ResolveError::Invalid { + path: "run.integrations.github.permissions.contents".to_string(), + reason: format!( + "additional repositories require `contents = \"read\"` or `contents = \ + \"write\"`, got `{literal}`" + ), + }); + } + } +} + fn resolve_goal(goal: Option<&RunGoalLayer>) -> Option { match goal? { RunGoalLayer::Inline(value) => Some(RunGoal::Inline(value.clone())), diff --git a/lib/foundation/fabro-config/src/tests/resolve_run.rs b/lib/foundation/fabro-config/src/tests/resolve_run.rs index d416eb651..2867ca011 100644 --- a/lib/foundation/fabro-config/src/tests/resolve_run.rs +++ b/lib/foundation/fabro-config/src/tests/resolve_run.rs @@ -879,6 +879,447 @@ issues = "{{ env.GH_PERM_LEVEL }}" } } +mod run_integrations_github_additional_repositories { + //! Layer + resolver tests for + //! `[run.integrations.github].additional_repositories`. + //! + //! The list replaces wholesale across layers (`[]` is an explicit clear), + //! resolves independently from `permissions`, and validates each entry as + //! a full `owner/repository` slug with indexed error paths. + + use crate::SettingsLayer; + use crate::layers::Combine; + + fn parse_settings(source: &str) -> SettingsLayer { + source + .parse::() + .expect("fixture should parse via SettingsLayer") + } + + fn invalid_paths_and_reasons(error: crate::Error) -> Vec<(String, String)> { + let errors = match error { + crate::Error::Resolve { errors, .. } => errors, + other => panic!("expected structured resolve errors, got {other:#}"), + }; + errors + .into_iter() + .map(|error| match error { + crate::ResolveError::Invalid { path, reason } => (path, reason), + other => panic!("expected invalid-value error, got {other}"), + }) + .collect() + } + + fn resolved_repositories(settings: &fabro_types::WorkflowSettings) -> Vec { + settings + .run + .integrations + .github + .additional_repositories + .iter() + .map(ToString::to_string) + .collect() + } + + #[test] + fn resolves_one_and_multiple_repositories() { + let one = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +"#, + ) + .expect("one additional repository should resolve"); + assert_eq!(resolved_repositories(&one), vec!["fabro-sh/keystone"]); + assert!(one.run.integrations.github.has_additional_repositories()); + + let many = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone", "fabro-sh/arc"] +permissions = { contents = "write" } +"#, + ) + .expect("multiple additional repositories should resolve"); + assert_eq!(resolved_repositories(&many), vec![ + "fabro-sh/arc", + "fabro-sh/keystone", + ]); + } + + #[test] + fn rejects_malformed_slugs_with_indexed_paths() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = [ + "fabro-sh/keystone", + "https://github.com/fabro-sh/arc", + "git@github.com:fabro-sh/arc.git", + "fabro-sh/arc@main", + "not-a-slug", +] +permissions = { contents = "read" } +"#, + ) + .expect_err("malformed slugs should not resolve"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec![ + "run.integrations.github.additional_repositories[1]", + "run.integrations.github.additional_repositories[2]", + "run.integrations.github.additional_repositories[3]", + "run.integrations.github.additional_repositories[4]", + ] + ); + assert!( + invalid[0].1.contains("owner/repository"), + "reason should explain the slug grammar: {}", + invalid[0].1 + ); + } + + #[test] + fn rejects_duplicate_and_case_variant_duplicate_slugs() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone", "Fabro-SH/Keystone"] +permissions = { contents = "read" } +"#, + ) + .expect_err("case-variant duplicate slugs should not resolve"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec!["run.integrations.github.additional_repositories[1]"] + ); + assert!( + invalid[0].1.contains("case-insensitive"), + "reason should mention case-insensitive identity: {}", + invalid[0].1 + ); + } + + #[test] + fn rejects_cross_owner_additional_repositories() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone", "lithoscomputer/conveyor"] +permissions = { contents = "read" } +"#, + ) + .expect_err("cross-owner additional repositories should not resolve"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec!["run.integrations.github.additional_repositories[1]"] + ); + assert!( + invalid[0].1.contains("share one owner"), + "reason should explain the single-owner requirement: {}", + invalid[0].1 + ); + } + + #[test] + fn rejects_more_than_the_installation_token_repository_limit() { + let repositories = (0..500) + .map(|index| format!("\"owner/repo-{index}\"")) + .collect::>() + .join(", "); + let error = super::workflow_settings_from_toml(&format!( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = [{repositories}] +permissions = {{ contents = "read" }} +"#, + )) + .expect_err("500 additional repositories should not resolve"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid[0].0, + "run.integrations.github.additional_repositories" + ); + assert!(invalid[0].1.contains("499"), "{}", invalid[0].1); + } + + #[test] + fn rejects_additional_repositories_without_permissions() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +"#, + ) + .expect_err("additional repositories without permissions should not resolve"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec!["run.integrations.github.additional_repositories"] + ); + } + + #[test] + fn rejects_additional_repositories_without_the_contents_permission() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { issues = "read" } +"#, + ) + .expect_err("additional repositories require the contents permission"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec!["run.integrations.github.permissions"] + ); + } + + #[test] + fn rejects_a_literal_contents_permission_that_is_not_read_or_write() { + let error = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "admin" } +"#, + ) + .expect_err("literal contents permission must be read or write"); + + let invalid = invalid_paths_and_reasons(error); + assert_eq!( + invalid + .iter() + .map(|(path, _)| path.as_str()) + .collect::>(), + vec!["run.integrations.github.permissions.contents"] + ); + } + + #[test] + fn defers_a_templated_contents_permission_to_the_runtime_boundary() { + let settings = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "{{ vars.GH_CONTENTS }}" } +"#, + ) + .expect("templated contents permission resolves; the value is re-checked at runtime"); + + assert_eq!(resolved_repositories(&settings), vec!["fabro-sh/keystone"]); + } + + #[test] + fn higher_layer_replaces_the_repository_list_wholesale() { + let workflow = parse_settings( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +"#, + ); + let user = parse_settings( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/arc", "fabro-sh/widgets"] +permissions = { contents = "read" } +"#, + ); + let merged = workflow.combine(user); + + let resolved = + super::workflow_settings_from_layer(merged).expect("merged settings should resolve"); + + // The lists never union: the higher layer's single entry wins, while + // the permission map inherits independently from the lower layer. + assert_eq!(resolved_repositories(&resolved), vec!["fabro-sh/keystone"]); + assert_eq!( + resolved.run.integrations.github.permissions.len(), + 1, + "permissions should inherit from the lower layer" + ); + } + + #[test] + fn absent_higher_layer_inherits_the_lower_repository_list() { + let workflow = parse_settings("_version = 1\n"); + let user = parse_settings( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +"#, + ); + let merged = workflow.combine(user); + + let resolved = + super::workflow_settings_from_layer(merged).expect("merged settings should resolve"); + assert_eq!(resolved_repositories(&resolved), vec!["fabro-sh/keystone"]); + } + + #[test] + fn empty_higher_layer_list_clears_inherited_repositories() { + let workflow = parse_settings( + r" +_version = 1 + +[run.integrations.github] +additional_repositories = [] +", + ); + let user = parse_settings( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +"#, + ); + let merged = workflow.combine(user); + + let resolved = + super::workflow_settings_from_layer(merged).expect("merged settings should resolve"); + + assert!( + resolved + .run + .integrations + .github + .additional_repositories + .is_empty(), + "explicit [] should clear the inherited repository list" + ); + // Permissions survive the repository clear: each field resolves + // independently. + assert!(resolved.run.integrations.github.is_token_requested()); + } + + #[test] + fn rejects_repositories_that_survive_a_cross_layer_permission_clear() { + let workflow = parse_settings( + r" +_version = 1 + +[run.integrations.github] +permissions = {} +", + ); + let user = parse_settings( + r#" +_version = 1 + +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +"#, + ); + let merged = workflow.combine(user); + + let error = super::workflow_settings_from_layer(merged) + .map(|_| ()) + .expect_err("repositories with cleared permissions should not resolve"); + + let message = error.to_string(); + assert!( + message.contains("additional_repositories"), + "error should name the invalid combination: {message}" + ); + } + + #[test] + fn permissions_only_and_fully_empty_shapes_are_preserved() { + let permissions_only = super::workflow_settings_from_toml( + r#" +_version = 1 + +[run.integrations.github.permissions] +issues = "read" +"#, + ) + .expect("permissions-only settings should resolve"); + assert!( + permissions_only + .run + .integrations + .github + .additional_repositories + .is_empty() + ); + assert!( + permissions_only + .run + .integrations + .github + .is_token_requested() + ); + + let empty = super::workflow_settings_from_toml("_version = 1\n") + .expect("empty settings should resolve"); + assert!( + empty + .run + .integrations + .github + .additional_repositories + .is_empty() + ); + assert!(!empty.run.integrations.github.is_token_requested()); + } +} + mod run_agent { use crate::SettingsLayer; use crate::layers::Combine; diff --git a/lib/foundation/fabro-types/src/repository.rs b/lib/foundation/fabro-types/src/repository.rs index 778b3ebf0..9c98d1d88 100644 --- a/lib/foundation/fabro-types/src/repository.rs +++ b/lib/foundation/fabro-types/src/repository.rs @@ -1,3 +1,8 @@ +use std::cmp::Ordering; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::str::FromStr; + use serde::{Deserialize, Serialize}; #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -25,8 +30,10 @@ impl RepositoryRef { /// /// Construction enforces GitHub's owner and repository name syntax on the /// exact submitted bytes; no trimming, case folding, or other normalization -/// is performed. -#[derive(Debug, Clone, PartialEq, Eq)] +/// is performed. The original spelling is preserved for `Display` and +/// serialization, while identity (`Eq`, `Ord`, `Hash`) is case-insensitive +/// to match GitHub's treatment of owner and repository names. +#[derive(Debug, Clone)] pub struct GitHubRepositorySlug { owner: String, repo: String, @@ -57,6 +64,90 @@ impl GitHubRepositorySlug { pub fn repo(&self) -> &str { &self.repo } + + /// Whether `other` names the same repository owner, ignoring ASCII case. + /// Owner and repository names are validated ASCII, so ASCII folding is + /// exact. + #[must_use] + pub fn same_owner(&self, other: &Self) -> bool { + self.owner.eq_ignore_ascii_case(&other.owner) + } + + fn canonical_key(&self) -> (String, String) { + ( + self.owner.to_ascii_lowercase(), + self.repo.to_ascii_lowercase(), + ) + } +} + +impl PartialEq for GitHubRepositorySlug { + fn eq(&self, other: &Self) -> bool { + self.owner.eq_ignore_ascii_case(&other.owner) && self.repo.eq_ignore_ascii_case(&other.repo) + } +} + +impl Eq for GitHubRepositorySlug {} + +impl PartialOrd for GitHubRepositorySlug { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for GitHubRepositorySlug { + fn cmp(&self, other: &Self) -> Ordering { + self.canonical_key().cmp(&other.canonical_key()) + } +} + +impl Hash for GitHubRepositorySlug { + fn hash(&self, state: &mut H) { + self.canonical_key().hash(state); + } +} + +impl fmt::Display for GitHubRepositorySlug { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}/{}", self.owner, self.repo) + } +} + +/// Parse failure for [`GitHubRepositorySlug`]. The offending input is not +/// echoed back because config surfaces already attach the value and its path. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error( + "expected a GitHub `owner/repository` slug with no scheme, host, ref, or extra path component" +)] +pub struct GitHubRepositorySlugError; + +impl FromStr for GitHubRepositorySlug { + type Err = GitHubRepositorySlugError; + + fn from_str(value: &str) -> Result { + Self::try_new(value).ok_or(GitHubRepositorySlugError) + } +} + +impl Serialize for GitHubRepositorySlug { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + serializer.collect_str(self) + } +} + +impl<'de> Deserialize<'de> for GitHubRepositorySlug { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + use serde::de::Error as _; + + let value = String::deserialize(deserializer)?; + value.parse().map_err(D::Error::custom) + } } fn valid_github_owner(value: &str) -> bool { @@ -234,6 +325,69 @@ mod tests { assert!(GitHubRepositorySlug::try_new(&over_repo).is_none()); } + #[test] + fn slug_identity_is_case_insensitive_but_display_preserves_case() { + let mixed: GitHubRepositorySlug = "Fabro-SH/Keystone".parse().unwrap(); + let lower: GitHubRepositorySlug = "fabro-sh/keystone".parse().unwrap(); + + assert_eq!(mixed, lower); + assert_eq!(mixed.cmp(&lower), std::cmp::Ordering::Equal); + assert!(mixed.same_owner(&lower)); + assert_eq!(mixed.to_string(), "Fabro-SH/Keystone"); + + let mut hashes = std::collections::HashSet::new(); + hashes.insert(mixed.clone()); + assert!( + !hashes.insert(lower.clone()), + "case variants share identity" + ); + + let mut ordered = std::collections::BTreeSet::new(); + ordered.insert(mixed); + assert!(!ordered.insert(lower), "case variants share ordering"); + } + + #[test] + fn slug_ordering_sorts_by_canonical_form() { + let mut slugs: Vec = ["owner/Zeta", "Owner/alpha", "owner/Beta"] + .iter() + .map(|value| value.parse().unwrap()) + .collect(); + slugs.sort(); + let rendered: Vec = slugs.iter().map(ToString::to_string).collect(); + assert_eq!(rendered, ["Owner/alpha", "owner/Beta", "owner/Zeta"]); + } + + #[test] + fn slug_from_str_rejects_urls_and_hosts() { + let cases = [ + "https://github.com/owner/repo", + "git@github.com:owner/repo.git", + "ssh://git@github.com/owner/repo", + "github.com/owner/repo", + "owner/repo@main", + "owner/repo#ref", + " owner/repo", + "owner/repo ", + ]; + for input in cases { + assert!(input.parse::().is_err(), "{input}"); + } + } + + #[test] + fn slug_serde_round_trips_as_a_string() { + let slug: GitHubRepositorySlug = "Fabro-SH/Keystone".parse().unwrap(); + let json = serde_json::to_string(&slug).unwrap(); + assert_eq!(json, "\"Fabro-SH/Keystone\""); + + let parsed: GitHubRepositorySlug = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, slug); + + let err = serde_json::from_str::("\"not a slug\"").unwrap_err(); + assert!(err.to_string().contains("owner/repository"), "{err}"); + } + #[test] fn valid_ref_selectors_are_accepted() { let max = "a".repeat(255); diff --git a/lib/foundation/fabro-types/src/settings/run.rs b/lib/foundation/fabro-types/src/settings/run.rs index 8e589a57a..12cd04fce 100644 --- a/lib/foundation/fabro-types/src/settings/run.rs +++ b/lib/foundation/fabro-types/src/settings/run.rs @@ -6,7 +6,7 @@ //! notifications, interviews, agent knobs, hooks, SCM targeting, pull-request //! behavior, and artifact collection. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::path::PathBuf; use std::time::Duration as StdDuration; @@ -603,9 +603,19 @@ pub struct RunIntegrationsSettings { /// presence-vs-clear distinction is only meaningful at the layer-merge /// stage; the resolved form collapses both `None` and `Some({})` into an /// empty map. +/// +/// `additional_repositories` lists repositories, beyond the implicit run +/// origin, that the minted `GITHUB_TOKEN` must cover. Configuration +/// resolution guarantees a non-empty set comes with a non-empty permission +/// map that includes `contents`; runs persisted before the field existed +/// deserialize to an empty set via the serde default. #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] pub struct RunIntegrationsGithubSettings { - pub permissions: HashMap, + pub permissions: HashMap, + /// Omitted when empty so settings serialized by this release stay + /// byte-identical to earlier releases for single-repository runs. + #[serde(default, skip_serializing_if = "BTreeSet::is_empty")] + pub additional_repositories: BTreeSet, } impl RunIntegrationsGithubSettings { @@ -616,6 +626,11 @@ impl RunIntegrationsGithubSettings { !self.permissions.is_empty() } + /// Whether the run declares additional repositories beyond the origin. + pub fn has_additional_repositories(&self) -> bool { + !self.additional_repositories.is_empty() + } + /// Resolve every `permissions` value. `{{ vars.* }}` is substituted /// server-side at run creation, so values are literal by this point; a /// still-unresolved token fails closed rather than reaching the GitHub API @@ -627,6 +642,42 @@ impl RunIntegrationsGithubSettings { .map(|(name, value)| Ok((name.clone(), value.resolve_with(&mut ctx)?))) .collect() } + + /// Resolve the whole runtime integration request: interpolated + /// permissions plus the declared additional repositories, produced + /// together so consumers cannot pick up one without the other. + pub fn resolve_integration(&self) -> Result { + Ok(ResolvedGithubIntegration { + permissions: self.resolve_permissions()?, + additional_repositories: self.additional_repositories.clone(), + }) + } +} + +/// The resolved runtime GitHub integration request for one run: interpolated +/// permission values plus the declared additional repositories. +/// +/// This is the single value carried from run materialization into workflow +/// startup, replacing parallel permission/repository collections that could +/// drift apart. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct ResolvedGithubIntegration { + pub permissions: HashMap, + pub additional_repositories: BTreeSet, +} + +impl ResolvedGithubIntegration { + /// Mirrors [`RunIntegrationsGithubSettings::is_token_requested`] for the + /// resolved form. + #[must_use] + pub fn is_token_requested(&self) -> bool { + !self.permissions.is_empty() + } + + #[must_use] + pub fn has_additional_repositories(&self) -> bool { + !self.additional_repositories.is_empty() + } } #[cfg(test)] @@ -635,10 +686,11 @@ mod run_integrations_github_tests { fn settings(permissions: &[(&str, &str)]) -> RunIntegrationsGithubSettings { RunIntegrationsGithubSettings { - permissions: permissions + permissions: permissions .iter() .map(|(k, v)| ((*k).to_string(), InterpString::parse(v))) .collect(), + additional_repositories: std::collections::BTreeSet::new(), } } @@ -670,6 +722,53 @@ mod run_integrations_github_tests { fn resolve_permissions_is_empty_for_empty_settings() { assert!(settings(&[]).resolve_permissions().unwrap().is_empty()); } + + #[test] + fn settings_without_additional_repositories_field_deserialize_to_empty_set() { + // Persisted run.created events from releases before + // `additional_repositories` existed omit the field entirely. + let parsed: RunIntegrationsGithubSettings = serde_json::from_value(serde_json::json!({ + "permissions": { "contents": "read" } + })) + .expect("legacy settings should deserialize"); + + assert!(parsed.additional_repositories.is_empty()); + assert!(!parsed.has_additional_repositories()); + } + + #[test] + fn resolve_integration_carries_permissions_and_repositories_together() { + let mut s = settings(&[("contents", "read")]); + s.additional_repositories + .insert("fabro-sh/keystone".parse().unwrap()); + + let resolved = s.resolve_integration().unwrap(); + + assert!(resolved.is_token_requested()); + assert!(resolved.has_additional_repositories()); + assert_eq!( + resolved.permissions.get("contents"), + Some(&"read".to_string()) + ); + assert_eq!( + resolved + .additional_repositories + .iter() + .map(ToString::to_string) + .collect::>(), + vec!["fabro-sh/keystone"] + ); + } + + #[test] + fn resolve_integration_fails_on_an_unresolved_permission_token() { + let mut s = settings(&[("contents", "{{ env.GH_PERM_LEVEL }}")]); + s.additional_repositories + .insert("fabro-sh/keystone".parse().unwrap()); + + let err = s.resolve_integration().unwrap_err(); + assert_eq!(err.namespace, Namespace::Env); + } } /// The resolved source of a run goal. From 7bfed23153b8fd8ccf40ef2c4d1d5df72515db22 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 14:24:24 -0400 Subject: [PATCH 16/30] feat(github): mint one installation token for the effective repository set MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `GitHubRepositoryAccess`, the secret-free validated value describing a run's effective GitHub repository set: the primary origin repository plus the declared additional repositories with the shared permission map. - The constructor normalizes HTTPS and both SSH origin spellings to one primary slug, rejects a missing or non-GitHub origin when additional repositories are declared, rejects primary duplication and cross-owner additional repositories, and re-checks that interpolated permissions carry `contents = "read"|"write"` — exposing targets in deterministic primary-first order. - `resolve_shared_installation` resolves every target's App installation with the App JWT and requires one shared installation ID, naming the repository the App cannot see before any mint. - The installation-token mint now accepts a repository-name list; the single-repository entry points delegate to it, and the request body lists every projected name with the shared permissions. - `InstallationTokenSource::for_access` builds a source over the access value; caching, refresh margin, and single-flight are unchanged. - The scripted `MockHttpClient` and test RSA key move to a shared crate-internal `tests_mock` module. Co-Authored-By: Claude Fable 5 --- lib/components/fabro-github/src/access.rs | 505 ++++++++++++++++++ lib/components/fabro-github/src/lib.rs | 206 +++---- lib/components/fabro-github/src/tests_mock.rs | 93 ++++ .../fabro-github/src/token_source.rs | 72 ++- 4 files changed, 775 insertions(+), 101 deletions(-) create mode 100644 lib/components/fabro-github/src/access.rs create mode 100644 lib/components/fabro-github/src/tests_mock.rs diff --git a/lib/components/fabro-github/src/access.rs b/lib/components/fabro-github/src/access.rs new file mode 100644 index 000000000..6ca1eb2aa --- /dev/null +++ b/lib/components/fabro-github/src/access.rs @@ -0,0 +1,505 @@ +//! The validated effective repository set for one run's GitHub access. +//! +//! [`GitHubRepositoryAccess`] is the single value both server preflight and +//! workflow initialization construct from the run origin, the declared +//! additional repositories, and the resolved shared permissions — so the two +//! paths cannot disagree about which repositories a run's `GITHUB_TOKEN` +//! covers. It carries no token or key material. + +use std::collections::{BTreeSet, HashMap}; + +use anyhow::{Context as _, bail}; +use fabro_types::GitHubRepositorySlug; + +use crate::{GitHubAppCredentials, HttpClient, HttpMethod}; + +/// The validated effective repository set for a run: the primary origin +/// repository plus zero or more distinct additional repositories, all with +/// one shared owner, and the shared permission map that scopes the token. +/// +/// Secret-free by construction: `Debug` may render everywhere the run +/// pipeline logs. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GitHubRepositoryAccess { + primary: GitHubRepositorySlug, + /// Sorted, deduplicated, primary excluded. + additional: Vec, + permissions: HashMap, +} + +impl GitHubRepositoryAccess { + /// Build the effective access request. + /// + /// Returns `Ok(None)` when no origin URL is available and no additional + /// repositories are declared — the legacy "nothing to scope" state whose + /// handling stays with the caller. Every declared-additional invariant is + /// enforced here: + /// + /// - a declared additional set requires a GitHub origin, + /// - the additional set cannot contain the primary repository, + /// - every additional repository shares the primary's owner + /// (case-insensitive) because one App installation covers one account, + /// - a declared additional set requires interpolated permissions with + /// `contents = "read"` or `contents = "write"`. + pub fn new( + origin_url: Option<&str>, + additional_repositories: &BTreeSet, + permissions: HashMap, + ) -> anyhow::Result> { + let origin_url = origin_url.map(str::trim).filter(|url| !url.is_empty()); + let Some(origin_url) = origin_url else { + if additional_repositories.is_empty() { + return Ok(None); + } + bail!( + "run.integrations.github.additional_repositories requires a GitHub run origin; \ + this run has no repository origin URL" + ); + }; + + let normalized = crate::normalize_repo_origin_url(origin_url); + let (owner, repo) = crate::parse_github_owner_repo(&normalized) + .context("parsing GitHub origin for repository access")?; + let Some(primary) = GitHubRepositorySlug::try_new(&format!("{owner}/{repo}")) else { + bail!("run origin does not name a valid GitHub `owner/repository`: {owner}/{repo}"); + }; + + if !additional_repositories.is_empty() { + validate_additional_permissions(&permissions)?; + } + + let mut additional = Vec::with_capacity(additional_repositories.len()); + for slug in additional_repositories { + if *slug == primary { + bail!( + "run.integrations.github.additional_repositories must not repeat the run \ + origin repository `{primary}` — the origin is always included" + ); + } + if !slug.same_owner(&primary) { + bail!( + "additional repository `{slug}` has owner `{}` but the run origin `{primary}` \ + has owner `{}`; all repositories must share one owner because one GitHub App \ + installation covers one account", + slug.owner(), + primary.owner() + ); + } + additional.push(slug.clone()); + } + + Ok(Some(Self { + primary, + additional, + permissions, + })) + } + + #[must_use] + pub fn primary(&self) -> &GitHubRepositorySlug { + &self.primary + } + + /// Every repository in the effective set, primary first, then the + /// additional repositories in their deterministic sorted order. + #[must_use] + pub fn targets(&self) -> Vec<&GitHubRepositorySlug> { + std::iter::once(&self.primary) + .chain(self.additional.iter()) + .collect() + } + + /// Project each validated slug to its repository-name component for the + /// installation-token mint request, which accepts names within the + /// selected installation. Every target shares the primary's owner, so the + /// projection loses nothing. + #[must_use] + pub fn repository_names(&self) -> Vec { + self.targets() + .into_iter() + .map(|slug| slug.repo().to_string()) + .collect() + } + + #[must_use] + pub fn owner(&self) -> &str { + self.primary.owner() + } + + #[must_use] + pub fn permissions(&self) -> &HashMap { + &self.permissions + } + + pub fn permissions_json(&self) -> anyhow::Result { + serde_json::to_value(&self.permissions).context("serializing GitHub permissions") + } + + #[must_use] + pub fn has_additional_repositories(&self) -> bool { + !self.additional.is_empty() + } + + /// Resolve every target's App installation and require one shared + /// installation ID, so a repository the App cannot see — or one that + /// resolves to a different installation — is named before any token is + /// minted. Targets are checked in deterministic primary-first order. + pub async fn resolve_shared_installation( + &self, + creds: &GitHubAppCredentials, + client: &impl HttpClient, + base_url: &str, + ) -> anyhow::Result { + #[derive(serde::Deserialize)] + struct Installation { + id: u64, + } + + let jwt = crate::sign_app_jwt(&creds.app_id, &creds.private_key_pem)?; + let auth = format!("Bearer {jwt}"); + let mut shared: Option<(u64, &GitHubRepositorySlug)> = None; + for slug in self.targets() { + let endpoint = format!( + "{base_url}/repos/{}/{}/installation", + slug.owner(), + slug.repo() + ); + let response = client + .request( + HttpMethod::Get, + &endpoint, + &crate::github_headers(&auth), + None, + ) + .await + .with_context(|| format!("looking up the GitHub App installation for {slug}"))?; + match response.status { + 200 => {} + 404 => bail!( + "the GitHub App installation cannot see repository {slug}; add it to the \ + installation's repository access" + ), + status => bail!( + "unexpected status {status} looking up the GitHub App installation for {slug}" + ), + } + let installation: Installation = response + .json() + .with_context(|| format!("parsing the installation response for {slug}"))?; + match shared { + None => shared = Some((installation.id, slug)), + Some((id, first)) if id != installation.id => bail!( + "repository {slug} belongs to GitHub App installation {} but {first} belongs \ + to installation {id}; all repositories must share one installation", + installation.id + ), + Some(_) => {} + } + } + let (id, _) = shared.expect("the effective repository set always contains the primary"); + Ok(id) + } +} + +/// A non-empty additional set needs a token that can reach repository +/// contents. Configuration resolution already checked literal values; this +/// is the runtime re-check after `{{ vars.* }}` interpolation. +fn validate_additional_permissions(permissions: &HashMap) -> anyhow::Result<()> { + let Some(contents) = permissions.get("contents") else { + bail!( + "run.integrations.github.additional_repositories requires the `contents` permission \ + (`read` or `write`)" + ); + }; + if contents != "read" && contents != "write" { + bail!( + "run.integrations.github.additional_repositories requires `contents = \"read\"` or \ + `contents = \"write\"`, got `{contents}`" + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn slugs(values: &[&str]) -> BTreeSet { + values + .iter() + .map(|value| value.parse().expect("test slug should parse")) + .collect() + } + + fn contents_read() -> HashMap { + HashMap::from([("contents".to_string(), "read".to_string())]) + } + + fn access( + origin: &str, + additional: &[&str], + permissions: HashMap, + ) -> anyhow::Result> { + GitHubRepositoryAccess::new(Some(origin), &slugs(additional), permissions) + } + + #[test] + fn https_and_both_ssh_origin_forms_normalize_to_the_same_primary() { + let origins = [ + "https://github.com/fabro-sh/fabro.git", + "git@github.com:fabro-sh/fabro.git", + "ssh://git@github.com/fabro-sh/fabro.git", + "https://github.com/fabro-sh/fabro", + ]; + for origin in origins { + let access = access(origin, &[], HashMap::new()) + .expect(origin) + .expect("origin should produce an access value"); + assert_eq!(access.primary().to_string(), "fabro-sh/fabro", "{origin}"); + } + } + + #[test] + fn no_origin_and_no_additional_repositories_is_none() { + let access = GitHubRepositoryAccess::new(None, &BTreeSet::new(), HashMap::new()).unwrap(); + assert!(access.is_none()); + + let blank = + GitHubRepositoryAccess::new(Some(" "), &BTreeSet::new(), HashMap::new()).unwrap(); + assert!(blank.is_none()); + } + + #[test] + fn additional_repositories_require_an_origin() { + let err = + GitHubRepositoryAccess::new(None, &slugs(&["fabro-sh/keystone"]), contents_read()) + .unwrap_err(); + assert!( + err.to_string().contains("requires a GitHub run origin"), + "{err:#}" + ); + } + + #[test] + fn additional_repositories_require_a_github_origin() { + let err = access( + "https://gitlab.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + contents_read(), + ) + .unwrap_err(); + assert!(err.to_string().contains("repository access"), "{err:#}"); + } + + #[test] + fn rejects_primary_duplication_regardless_of_url_spelling_or_case() { + let origins = [ + "https://github.com/Fabro-SH/Fabro.git", + "git@github.com:fabro-sh/fabro.git", + "ssh://git@github.com/fabro-sh/fabro", + ]; + for origin in origins { + let err = access(origin, &["fabro-sh/FABRO"], contents_read()).unwrap_err(); + assert!( + err.to_string().contains("must not repeat the run origin"), + "{origin}: {err:#}" + ); + } + } + + #[test] + fn rejects_an_additional_repository_with_a_different_owner() { + let err = access( + "https://github.com/fabro-sh/fabro", + &["lithoscomputer/conveyor"], + contents_read(), + ) + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains("lithoscomputer/conveyor"), "{message}"); + assert!(message.contains("share one owner"), "{message}"); + } + + #[test] + fn rejects_a_declared_set_without_a_contents_permission() { + let missing = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + HashMap::new(), + ) + .unwrap_err(); + assert!( + missing.to_string().contains("`contents` permission"), + "{missing:#}" + ); + + let wrong_level = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + HashMap::from([("contents".to_string(), "admin".to_string())]), + ) + .unwrap_err(); + assert!( + wrong_level.to_string().contains("got `admin`"), + "{wrong_level:#}" + ); + } + + #[test] + fn targets_retain_every_full_slug_exactly_once_primary_first() { + let access = access( + "git@github.com:fabro-sh/fabro.git", + &["fabro-sh/keystone", "fabro-sh/arc"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let targets: Vec = access.targets().iter().map(ToString::to_string).collect(); + assert_eq!(targets, vec![ + "fabro-sh/fabro", + "fabro-sh/arc", + "fabro-sh/keystone", + ]); + assert_eq!(access.repository_names(), vec!["fabro", "arc", "keystone"]); + assert_eq!(access.owner(), "fabro-sh"); + assert!(access.has_additional_repositories()); + } + + #[test] + fn debug_output_contains_only_repositories_and_permissions() { + let access = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/arc"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let rendered = format!("{access:?}"); + assert!(rendered.contains("fabro-sh"), "{rendered}"); + assert!(rendered.contains("contents"), "{rendered}"); + // The value carries no token or key material by construction; its + // fields are exactly the repository slugs and the permission map. + assert!(!rendered.to_lowercase().contains("token"), "{rendered}"); + assert!(!rendered.to_lowercase().contains("key"), "{rendered}"); + } + + #[tokio::test] + async fn resolve_shared_installation_names_the_invisible_repository() { + use crate::HttpMethod; + use crate::tests_mock::{MockHttpClient, test_rsa_key}; + + let mock = MockHttpClient::new() + .on( + HttpMethod::Get, + "/repos/fabro-sh/fabro/installation", + 200, + r#"{"id": 7}"#, + ) + .on( + HttpMethod::Get, + "/repos/fabro-sh/keystone/installation", + 404, + "{}", + ); + let creds = GitHubAppCredentials { + app_id: "test".to_string(), + private_key_pem: test_rsa_key().to_string(), + slug: None, + }; + let access = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let err = access + .resolve_shared_installation(&creds, &mock, "") + .await + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains("fabro-sh/keystone"), "{message}"); + assert!(message.contains("cannot see"), "{message}"); + } + + #[tokio::test] + async fn resolve_shared_installation_requires_one_installation_id() { + use crate::HttpMethod; + use crate::tests_mock::{MockHttpClient, test_rsa_key}; + + let mock = MockHttpClient::new() + .on( + HttpMethod::Get, + "/repos/fabro-sh/fabro/installation", + 200, + r#"{"id": 7}"#, + ) + .on( + HttpMethod::Get, + "/repos/fabro-sh/keystone/installation", + 200, + r#"{"id": 8}"#, + ); + let creds = GitHubAppCredentials { + app_id: "test".to_string(), + private_key_pem: test_rsa_key().to_string(), + slug: None, + }; + let access = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let err = access + .resolve_shared_installation(&creds, &mock, "") + .await + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains("installation 8"), "{message}"); + assert!(message.contains("fabro-sh/keystone"), "{message}"); + } + + #[tokio::test] + async fn resolve_shared_installation_returns_the_shared_id() { + use crate::HttpMethod; + use crate::tests_mock::{MockHttpClient, test_rsa_key}; + + let mock = MockHttpClient::new() + .on( + HttpMethod::Get, + "/repos/fabro-sh/fabro/installation", + 200, + r#"{"id": 7}"#, + ) + .on( + HttpMethod::Get, + "/repos/fabro-sh/keystone/installation", + 200, + r#"{"id": 7}"#, + ); + let creds = GitHubAppCredentials { + app_id: "test".to_string(), + private_key_pem: test_rsa_key().to_string(), + slug: None, + }; + let access = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let id = access + .resolve_shared_installation(&creds, &mock, "") + .await + .unwrap(); + assert_eq!(id, 7); + } +} diff --git a/lib/components/fabro-github/src/lib.rs b/lib/components/fabro-github/src/lib.rs index a38c6c8ce..71d586213 100644 --- a/lib/components/fabro-github/src/lib.rs +++ b/lib/components/fabro-github/src/lib.rs @@ -9,10 +9,15 @@ use fabro_types::settings::run::MergeStrategy; use serde::Deserialize; use tokio::process::Command; +pub mod access; pub mod token_source; #[cfg(any(test, feature = "test-support"))] pub mod test_support; +#[cfg(test)] +pub(crate) mod tests_mock; + +pub use access::GitHubRepositoryAccess; pub const GITHUB_API_BASE_URL: &str = "https://api.github.com"; @@ -151,6 +156,29 @@ impl GitHubAppCredentials { base_url: &str, permissions: serde_json::Value, install_url: Option<&str>, + ) -> anyhow::Result { + self.mint_installation_token_for_repositories( + client, + owner, + &[repo.to_string()], + base_url, + permissions, + install_url, + ) + .await + } + + /// Mint one installation token scoped to every repository in + /// `repository_names` (names within `owner`'s installation, primary + /// first) with the shared `permissions`. + pub async fn mint_installation_token_for_repositories( + &self, + client: &impl HttpClient, + owner: &str, + repository_names: &[String], + base_url: &str, + permissions: serde_json::Value, + install_url: Option<&str>, ) -> anyhow::Result { let jwt = sign_app_jwt(&self.app_id, &self.private_key_pem)?; let default_install_url = self.installation_url(owner); @@ -159,7 +187,7 @@ impl GitHubAppCredentials { client, &jwt, owner, - repo, + repository_names, base_url, permissions, install_url, @@ -475,16 +503,24 @@ pub async fn create_installation_access_token_with_permissions_and_install_url( permissions: serde_json::Value, install_url: Option<&str>, ) -> anyhow::Result { - mint_installation_token_with_jwt(client, jwt, owner, repo, base_url, permissions, install_url) - .await - .map(|token| token.token) + mint_installation_token_with_jwt( + client, + jwt, + owner, + &[repo.to_string()], + base_url, + permissions, + install_url, + ) + .await + .map(|token| token.token) } async fn mint_installation_token_with_jwt( client: &impl HttpClient, jwt: &str, owner: &str, - repo: &str, + repos: &[String], base_url: &str, permissions: serde_json::Value, install_url: Option<&str>, @@ -500,8 +536,15 @@ async fn mint_installation_token_with_jwt( expires_at: DateTime, } - // Step 1: Find the installation for this repo - let installation_endpoint = format!("{base_url}/repos/{owner}/{repo}/installation"); + let Some(primary_repo) = repos.first() else { + bail!("installation token mint requires at least one repository"); + }; + + // Step 1: Find the installation via the primary repository. Multi- + // repository callers resolve every repository's installation up front + // (`GitHubRepositoryAccess::resolve_shared_installation`), so the + // primary stands for the whole set here. + let installation_endpoint = format!("{base_url}/repos/{owner}/{primary_repo}/installation"); let auth = format!("Bearer {jwt}"); let resp = client .request( @@ -555,7 +598,7 @@ async fn mint_installation_token_with_jwt( installation.id ); let body = serde_json::json!({ - "repositories": [repo], + "repositories": repos, "permissions": permissions, }); @@ -573,8 +616,9 @@ async fn mint_installation_token_with_jwt( 201 => {} 422 => { bail!( - "GitHub App does not have access to repository {repo}. \ - Update the installation's repository permissions to include it." + "GitHub App does not have access to every requested repository ({}). \ + Update the installation's repository permissions to include them.", + repos.join(", ") ); } 401 => { @@ -1715,7 +1759,7 @@ mod tests { // ----------------------------------------------------------------------- fn test_rsa_key() -> &'static str { - include_str!("testdata/rsa_private.pem") + tests_mock::test_rsa_key() } #[test] @@ -1769,89 +1813,7 @@ mod tests { // MockHttpClient // ----------------------------------------------------------------------- - struct MockRoute { - method: HttpMethod, - path: String, - status: u16, - response_body: String, - assert_header: Option<(String, MockHeaderCheck)>, - assert_body_json: Option, - } - - enum MockHeaderCheck { - Equals(String), - } - - struct MockHttpClient { - routes: Vec, - } - - impl MockHttpClient { - fn new() -> Self { - Self { routes: vec![] } - } - - fn on(mut self, method: HttpMethod, path: &str, status: u16, body: &str) -> Self { - self.routes.push(MockRoute { - method, - path: path.to_string(), - status, - response_body: body.to_string(), - assert_header: None, - assert_body_json: None, - }); - self - } - - fn with_req_header(mut self, name: &str, value: &str) -> Self { - self.routes.last_mut().unwrap().assert_header = - Some((name.to_string(), MockHeaderCheck::Equals(value.to_string()))); - self - } - - fn with_req_body(mut self, json_str: &str) -> Self { - self.routes.last_mut().unwrap().assert_body_json = - Some(serde_json::from_str(json_str).unwrap()); - self - } - } - - impl HttpClient for MockHttpClient { - async fn request( - &self, - method: HttpMethod, - url: &str, - headers: &[(&str, &str)], - body: Option<&serde_json::Value>, - ) -> anyhow::Result { - for route in &self.routes { - if method == route.method && url.ends_with(&route.path) { - if let Some((name, MockHeaderCheck::Equals(expected))) = &route.assert_header { - let (_, v) = headers - .iter() - .find(|(k, _)| *k == name.as_str()) - .unwrap_or_else(|| { - panic!("Expected header '{name}' not found in request to {url}") - }); - assert_eq!(*v, expected.as_str(), "Header '{name}' mismatch for {url}"); - } - if let Some(expected_body) = &route.assert_body_json { - let actual = body.expect("Expected request body"); - assert_eq!(actual, expected_body, "Request body mismatch for {url}"); - } - return Ok(HttpResponse::new(route.status, route.response_body.clone())); - } - } - panic!( - "No mock route for {:?} {url}\nRegistered routes: {:?}", - method, - self.routes - .iter() - .map(|r| format!("{:?} {}", r.method, r.path)) - .collect::>() - ); - } - } + use crate::tests_mock::{self, MockHttpClient}; // ----------------------------------------------------------------------- // create_installation_access_token — success @@ -1901,6 +1863,62 @@ mod tests { ); } + /// The multi-repository mint sends one request listing every projected + /// repository name exactly once, primary first, with the shared + /// permissions; the installation lookup uses the primary repository. + #[tokio::test] + async fn multi_repository_mint_lists_every_repository_name_once() { + let access = GitHubRepositoryAccess::new( + Some("git@github.com:owner/repo.git"), + &[ + "owner/keystone".parse().unwrap(), + "owner/arc".parse().unwrap(), + ] + .into_iter() + .collect(), + std::collections::HashMap::from([("contents".to_string(), "read".to_string())]), + ) + .unwrap() + .expect("origin should produce an access value"); + + let mock = MockHttpClient::new() + .on( + HttpMethod::Get, + "/repos/owner/repo/installation", + 200, + r#"{"id": 123}"#, + ) + .on( + HttpMethod::Post, + "/app/installations/123/access_tokens", + 201, + r#"{"token": "ghs_multi", "expires_at": "2026-01-01T12:00:00Z"}"#, + ) + .with_req_body( + r#"{"permissions":{"contents":"read"},"repositories":["repo","arc","keystone"]}"#, + ); + + let creds = GitHubAppCredentials { + app_id: "test".to_string(), + private_key_pem: test_rsa_key().to_string(), + slug: None, + }; + + let token = creds + .mint_installation_token_for_repositories( + &mock, + access.owner(), + &access.repository_names(), + "", + access.permissions_json().unwrap(), + None, + ) + .await + .unwrap(); + + assert_eq!(token.token, "ghs_multi"); + } + #[tokio::test] async fn create_iat_requests_only_contents_write() { let mock = MockHttpClient::new() diff --git a/lib/components/fabro-github/src/tests_mock.rs b/lib/components/fabro-github/src/tests_mock.rs new file mode 100644 index 000000000..da6d7920e --- /dev/null +++ b/lib/components/fabro-github/src/tests_mock.rs @@ -0,0 +1,93 @@ +//! Crate-internal test doubles shared by the `lib.rs` and `access` test +//! modules: a scripted [`HttpClient`] and a throwaway RSA key for JWT +//! signing. + +use crate::{HttpClient, HttpMethod, HttpResponse}; + +pub(crate) fn test_rsa_key() -> &'static str { + include_str!("testdata/rsa_private.pem") +} + +pub(crate) struct MockRoute { + method: HttpMethod, + path: String, + status: u16, + response_body: String, + assert_header: Option<(String, MockHeaderCheck)>, + assert_body_json: Option, +} + +pub(crate) enum MockHeaderCheck { + Equals(String), +} + +pub(crate) struct MockHttpClient { + routes: Vec, +} + +impl MockHttpClient { + pub(crate) fn new() -> Self { + Self { routes: vec![] } + } + + pub(crate) fn on(mut self, method: HttpMethod, path: &str, status: u16, body: &str) -> Self { + self.routes.push(MockRoute { + method, + path: path.to_string(), + status, + response_body: body.to_string(), + assert_header: None, + assert_body_json: None, + }); + self + } + + pub(crate) fn with_req_header(mut self, name: &str, value: &str) -> Self { + self.routes.last_mut().unwrap().assert_header = + Some((name.to_string(), MockHeaderCheck::Equals(value.to_string()))); + self + } + + pub(crate) fn with_req_body(mut self, json_str: &str) -> Self { + self.routes.last_mut().unwrap().assert_body_json = + Some(serde_json::from_str(json_str).unwrap()); + self + } +} + +impl HttpClient for MockHttpClient { + async fn request( + &self, + method: HttpMethod, + url: &str, + headers: &[(&str, &str)], + body: Option<&serde_json::Value>, + ) -> anyhow::Result { + for route in &self.routes { + if method == route.method && url.ends_with(&route.path) { + if let Some((name, MockHeaderCheck::Equals(expected))) = &route.assert_header { + let (_, v) = headers + .iter() + .find(|(k, _)| *k == name.as_str()) + .unwrap_or_else(|| { + panic!("Expected header '{name}' not found in request to {url}") + }); + assert_eq!(*v, expected.as_str(), "Header '{name}' mismatch for {url}"); + } + if let Some(expected_body) = &route.assert_body_json { + let actual = body.expect("Expected request body"); + assert_eq!(actual, expected_body, "Request body mismatch for {url}"); + } + return Ok(HttpResponse::new(route.status, route.response_body.clone())); + } + } + panic!( + "No mock route for {:?} {url}\nRegistered routes: {:?}", + method, + self.routes + .iter() + .map(|r| format!("{:?} {}", r.method, r.path)) + .collect::>() + ); + } +} diff --git a/lib/components/fabro-github/src/token_source.rs b/lib/components/fabro-github/src/token_source.rs index 276c5f9b8..d8c16228c 100644 --- a/lib/components/fabro-github/src/token_source.rs +++ b/lib/components/fabro-github/src/token_source.rs @@ -15,7 +15,7 @@ use std::fmt; use std::sync::Arc; use std::time::Duration; -use anyhow::Context as _; +use anyhow::{Context as _, bail}; use chrono::{DateTime, Utc}; use tokio::sync::Mutex; @@ -146,12 +146,14 @@ pub(crate) trait InstallationTokenMinter: Send + Sync { async fn mint(&self) -> anyhow::Result; } -/// Real minter backed by GitHub App credentials. +/// Real minter backed by GitHub App credentials. `repos` lists repository +/// names within the owner's installation, primary first; the minted token is +/// scoped to exactly that set. struct AppTokenMinter { creds: GitHubAppCredentials, http: fabro_http::HttpClient, owner: String, - repo: String, + repos: Vec, base_url: String, permissions: serde_json::Value, } @@ -160,10 +162,10 @@ struct AppTokenMinter { impl InstallationTokenMinter for AppTokenMinter { async fn mint(&self) -> anyhow::Result { self.creds - .mint_installation_token( + .mint_installation_token_for_repositories( &self.http, &self.owner, - &self.repo, + &self.repos, &self.base_url, self.permissions.clone(), None, @@ -243,7 +245,38 @@ impl InstallationTokenSource { repo: String, permissions: serde_json::Value, ) -> anyhow::Result> { - let repo_display = format!("{owner}/{repo}"); + Self::for_repositories(creds, owner, vec![repo], permissions) + } + + /// Build a source for a validated effective repository set. Minted + /// tokens are scoped to every repository in the set with the shared + /// permissions; caching, refresh margin, and single-flight behavior are + /// identical to the single-repository source. + pub fn for_access( + creds: &GitHubCredentials, + access: &crate::GitHubRepositoryAccess, + ) -> anyhow::Result> { + Self::for_repositories( + creds, + access.owner().to_string(), + access.repository_names(), + access.permissions_json()?, + ) + } + + fn for_repositories( + creds: &GitHubCredentials, + owner: String, + repos: Vec, + permissions: serde_json::Value, + ) -> anyhow::Result> { + let repo_display = match repos.as_slice() { + [primary] => format!("{owner}/{primary}"), + [primary, additional @ ..] => { + format!("{owner}/{primary} (+{} additional)", additional.len()) + } + [] => bail!("token source requires at least one repository"), + }; let state = match creds { GitHubCredentials::Pat(token) => SourceState::Pat(SecretString::new(token.clone())), GitHubCredentials::Installation(token) => SourceState::Installation(token.clone()), @@ -256,7 +289,7 @@ impl InstallationTokenSource { creds: app.clone(), http, owner, - repo, + repos, base_url: crate::github_api_base_url(), permissions, }), @@ -502,6 +535,31 @@ mod tests { assert!(!source.mints_installation_tokens()); } + /// A source built from a validated multi-repository access value uses the + /// same state machine as the single-repository constructor: static + /// credentials pass through, and App credentials share the cache + /// machinery exercised by the `with_minter` tests below. + #[tokio::test] + async fn for_access_source_resolves_like_the_single_repository_source() { + let access = crate::GitHubRepositoryAccess::new( + Some("https://github.com/owner/repo.git"), + &["owner/keystone".parse().unwrap()].into_iter().collect(), + std::collections::HashMap::from([("contents".to_string(), "read".to_string())]), + ) + .unwrap() + .expect("origin should produce an access value"); + + let source = InstallationTokenSource::for_access( + &GitHubCredentials::Pat("ghp_pat".to_string()), + &access, + ) + .unwrap(); + + let resolved = source.resolve().await.unwrap(); + assert_eq!(resolved.token.expose(), "ghp_pat"); + assert!(resolved.snapshot.is_static()); + } + #[tokio::test] async fn static_installation_token_resolves_until_expiry() { let valid = InstallationTokenSource::for_origin( From d8edd410f38af3e2f052cb7cb945a6b08027c4d6 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 14:47:31 -0400 Subject: [PATCH 17/30] feat(workflow): bridge git and gh to the shared token Carry the resolved GitHub integration (permissions plus declared additional repositories) as one value from run materialization into workflow startup, and make the sandbox environment reach every declared repository through the single managed GITHUB_TOKEN. - `StartServices.github_permissions` becomes `github_integration: ResolvedGithubIntegration`; CLI and server workers build it with `resolve_integration()` after interpolation and pass it through `SandboxEnvSpec` as one unit. - `build_sandbox_env` constructs the validated `GitHubRepositoryAccess` and scopes the App token source to the whole effective set. Missing credentials or a missing origin are hard initialization errors when additional repositories are declared; legacy permissions-only configuration keeps its best-effort behavior. - When additional repositories are declared, initialization eagerly resolves each repository's App installation (naming any repository the App cannot see) and the token itself, so an inaccessible declared repository fails before the first workflow stage. - A new `git_bridge` module injects secret-free `GIT_CONFIG_*` entries into the stage environment: a github.com credential helper that reads `$GITHUB_TOKEN` at invocation time, per-repository SSH-to-HTTPS `insteadOf` rewrites, and `GIT_TERMINAL_PROMPT=0`. Entries append after a valid user-provided Git config overlay and fail clearly on a malformed one. Contract tests drive the installed git binary against local fixtures for the rewrite, credential, prefix-collision, and overlay-preservation behaviors. - The long-running ACP notice now says all declared repository access expires together. Co-Authored-By: Claude Fable 5 --- lib/apps/fabro-cli/src/commands/run/runner.rs | 6 +- lib/apps/fabro-server/src/server.rs | 8 +- lib/components/fabro-github/src/access.rs | 13 + .../fabro-workflow/src/git_bridge.rs | 448 ++++++++++++++++++ .../fabro-workflow/src/handler/llm/acp.rs | 5 +- lib/components/fabro-workflow/src/lib.rs | 1 + .../fabro-workflow/src/operations/start.rs | 21 +- .../src/pipeline/execute/tests.rs | 8 +- .../fabro-workflow/src/pipeline/initialize.rs | 322 +++++++++++-- .../fabro-workflow/src/pipeline/types.rs | 9 +- 10 files changed, 785 insertions(+), 56 deletions(-) create mode 100644 lib/components/fabro-workflow/src/git_bridge.rs diff --git a/lib/apps/fabro-cli/src/commands/run/runner.rs b/lib/apps/fabro-cli/src/commands/run/runner.rs index d3f21e78a..9d5bb5388 100644 --- a/lib/apps/fabro-cli/src/commands/run/runner.rs +++ b/lib/apps/fabro-cli/src/commands/run/runner.rs @@ -161,13 +161,13 @@ pub(crate) async fn execute( artifact_sink, run_control: Some(run_control), github_app, - github_permissions: run_spec + github_integration: run_spec .settings .run .integrations .github - .resolve_permissions() - .context("failed to resolve github permissions")?, + .resolve_integration() + .context("failed to resolve github integration")?, vault, catalog, on_node: None, diff --git a/lib/apps/fabro-server/src/server.rs b/lib/apps/fabro-server/src/server.rs index 8c7d5514d..3c6935f51 100644 --- a/lib/apps/fabro-server/src/server.rs +++ b/lib/apps/fabro-server/src/server.rs @@ -4081,15 +4081,15 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { return; } }; - let github_permissions = match persisted + let github_integration = match persisted .run_spec() .settings .run .integrations .github - .resolve_permissions() + .resolve_integration() { - Ok(permissions) => permissions, + Ok(integration) => integration, Err(err) => { tracing::error!( run_id = %run_id, @@ -4131,7 +4131,7 @@ async fn execute_run_in_process(state: Arc, run_id: RunId) { artifact_sink: Some(ArtifactSink::Store(state.artifact_store.clone())), run_control: None, github_app, - github_permissions, + github_integration, vault: Arc::new(AsyncRwLock::new(vault.into_vault())), catalog: state.catalog(), on_node: None, diff --git a/lib/components/fabro-github/src/access.rs b/lib/components/fabro-github/src/access.rs index 6ca1eb2aa..b71fd0c64 100644 --- a/lib/components/fabro-github/src/access.rs +++ b/lib/components/fabro-github/src/access.rs @@ -140,6 +140,19 @@ impl GitHubRepositoryAccess { !self.additional.is_empty() } + /// [`Self::resolve_shared_installation`] against the production GitHub + /// API with a fresh HTTP client. + pub async fn resolve_shared_installation_via_api( + &self, + creds: &GitHubAppCredentials, + ) -> anyhow::Result { + let client = fabro_http::http_client() + .map_err(anyhow::Error::new) + .context("building HTTP client for installation resolution")?; + self.resolve_shared_installation(creds, &client, &crate::github_api_base_url()) + .await + } + /// Resolve every target's App installation and require one shared /// installation ID, so a repository the App cannot see — or one that /// resolves to a different installation — is named before any token is diff --git a/lib/components/fabro-workflow/src/git_bridge.rs b/lib/components/fabro-workflow/src/git_bridge.rs new file mode 100644 index 000000000..df9c7e300 --- /dev/null +++ b/lib/components/fabro-workflow/src/git_bridge.rs @@ -0,0 +1,448 @@ +//! Secret-free Git bridging environment for additional-repository access. +//! +//! When a run declares additional GitHub repositories, every resolved +//! command/tool/ACP environment receives `GIT_CONFIG_COUNT` / +//! `GIT_CONFIG_KEY_n` / `GIT_CONFIG_VALUE_n` entries that make plain Git +//! commands work against the declared set through the managed +//! `GITHUB_TOKEN`: +//! +//! - a credential helper for `https://github.com` that reads `$GITHUB_TOKEN` +//! from the invoking Git process's environment at invocation time, so token +//! refresh flows through per-stage environment resolution with no bridging +//! update; +//! - per-repository `url..insteadOf` rewrites for the +//! `git@github.com:owner/repo[.git]` and +//! `ssh://git@github.com/owner/repo[.git]` SSH spellings of each effective +//! repository. +//! +//! None of the values contain a secret; the token lives only in +//! `GITHUB_TOKEN`. +//! +//! The credential helper is host-scoped to `https://github.com`, not +//! path-scoped. This is safe because the token is scoped server-side to the +//! declared repository set and is only ever offered to github.com. It does +//! change one failure mode for *undeclared* repositories: public HTTPS +//! clones are unaffected (Git tries unauthenticated first), while private +//! undeclared HTTPS repositories fail with a GitHub authorization error +//! instead of a missing-credential error. Both fail; only the diagnostic +//! differs. +//! +//! `insteadOf` matches by string prefix, not exactly: a rule for +//! `owner/repo` also matches `owner/repo-other`. An undeclared repository +//! that shares a declared prefix is therefore rewritten to HTTPS; the scoped +//! token is invalid for it at GitHub, so authority is unchanged, but its Git +//! transport changes from SSH to HTTPS. + +use std::collections::HashMap; + +use fabro_types::GitHubRepositorySlug; + +use crate::error::Error; + +/// Section base for the effective repositories' HTTPS routes. +const GITHUB_HTTPS_BASE: &str = "https://github.com/"; + +const CREDENTIAL_HELPER_KEY: &str = "credential.https://github.com.helper"; +/// Reads the invoking process's `$GITHUB_TOKEN` at invocation time; contains +/// no secret itself. Non-`get` operations (`store`, `erase`) are ignored. +const CREDENTIAL_HELPER: &str = r#"!f() { if [ "$1" = get ]; then echo username=x-access-token; echo "password=$GITHUB_TOKEN"; fi; }; f"#; + +/// Merge the bridging entries into `env` for the effective repository set +/// (primary first). Appends after any valid user-provided `GIT_CONFIG_COUNT` +/// overlay without overwriting it, and fails with a configuration error when +/// the user overlay is malformed rather than silently replacing it. +pub(crate) fn merge_git_bridge_env( + env: &mut HashMap, + targets: &[&GitHubRepositorySlug], +) -> Result<(), Error> { + let start = user_git_config_count(env)?; + for (offset, (key, value)) in bridge_entries(targets, GITHUB_HTTPS_BASE) + .into_iter() + .enumerate() + { + let index = start + offset; + env.insert(format!("GIT_CONFIG_KEY_{index}"), key); + env.insert(format!("GIT_CONFIG_VALUE_{index}"), value); + } + let total = start + bridge_entry_count(targets); + env.insert("GIT_CONFIG_COUNT".to_string(), total.to_string()); + // Fail instead of hanging when access is missing or invalid; a user who + // explicitly configured prompting keeps their value. + env.entry("GIT_TERMINAL_PROMPT".to_string()) + .or_insert_with(|| "0".to_string()); + Ok(()) +} + +fn bridge_entry_count(targets: &[&GitHubRepositorySlug]) -> usize { + 1 + targets.len() * 2 +} + +/// The bridge's Git config entries in order: the credential helper, then two +/// SSH-to-HTTPS rewrites per repository. `https_base` is +/// [`GITHUB_HTTPS_BASE`] in production; contract tests substitute a local +/// `file://` root to prove real Git applies the generated entries without +/// touching the network. +fn bridge_entries(targets: &[&GitHubRepositorySlug], https_base: &str) -> Vec<(String, String)> { + let mut entries = Vec::with_capacity(bridge_entry_count(targets)); + entries.push(( + CREDENTIAL_HELPER_KEY.to_string(), + CREDENTIAL_HELPER.to_string(), + )); + for slug in targets { + let owner = slug.owner(); + let repo = slug.repo(); + let https = format!("{https_base}{owner}/{repo}"); + // One prefix rule per SSH spelling covers both the bare and `.git` + // suffixed forms. + entries.push(( + format!("url.{https}.insteadOf"), + format!("git@github.com:{owner}/{repo}"), + )); + entries.push(( + format!("url.{https}.insteadOf"), + format!("ssh://git@github.com/{owner}/{repo}"), + )); + } + entries +} + +/// Validate and measure a user-provided `GIT_CONFIG_COUNT` overlay so the +/// bridge appends after it. Orphaned `GIT_CONFIG_KEY_n` entries without a +/// count are inert to Git and are treated as absent. +fn user_git_config_count(env: &HashMap) -> Result { + let Some(raw) = env.get("GIT_CONFIG_COUNT") else { + return Ok(0); + }; + let count: usize = raw.trim().parse().map_err(|_| { + Error::Precondition(format!( + "environment variable GIT_CONFIG_COUNT must be a non-negative integer to combine \ + with Fabro's Git bridging entries, got `{raw}`" + )) + })?; + for index in 0..count { + let key = format!("GIT_CONFIG_KEY_{index}"); + let value = format!("GIT_CONFIG_VALUE_{index}"); + if !env.contains_key(&key) || !env.contains_key(&value) { + return Err(Error::Precondition(format!( + "GIT_CONFIG_COUNT is {count} but {key} or {value} is missing; fix the indexed \ + Git config overlay so Fabro can append its bridging entries after it" + ))); + } + } + Ok(count) +} + +#[cfg(test)] +#[expect( + clippy::disallowed_methods, + clippy::disallowed_types, + reason = "contract tests drive the installed git binary synchronously in non-async tests" +)] +mod tests { + use std::path::Path; + use std::process::Command; + + use super::*; + + fn slug(value: &str) -> GitHubRepositorySlug { + value.parse().expect("test slug should parse") + } + + fn bridged_env( + base_env: HashMap, + targets: &[&GitHubRepositorySlug], + ) -> HashMap { + let mut env = base_env; + merge_git_bridge_env(&mut env, targets).expect("bridge entries should merge"); + env + } + + /// Run `git` with ONLY the bridge-relevant environment: the inherited + /// user/system/global Git config is disabled so assertions observe just + /// the generated entries. + fn git(args: &[&str], env: &HashMap, cwd: &Path) -> std::process::Output { + let mut command = Command::new("git"); + command + .args(args) + .current_dir(cwd) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_ASKPASS", "true"); + for (key, value) in env { + command.env(key, value); + } + command.output().expect("git should run") + } + + /// Create a bare fixture answering both the bare and `.git`-suffixed + /// routes, the way GitHub serves both HTTPS spellings. + fn init_bare_fixture(root: &Path, owner_repo: &str) -> String { + let fixture = root.join(format!("{owner_repo}.git")); + std::fs::create_dir_all(&fixture).unwrap(); + let init = Command::new("git") + .args(["init", "--bare", "--initial-branch=main"]) + .arg(&fixture) + .output() + .expect("git init should run"); + assert!(init.status.success(), "{init:?}"); + #[cfg(unix)] + std::os::unix::fs::symlink(&fixture, root.join(owner_repo)).unwrap(); + format!("file://{}/", root.display()) + } + + #[test] + fn no_targets_means_no_bridge_call_and_empty_env_stays_empty() { + // The caller only bridges when the additional set is non-empty; the + // pure entry builder is still total for the primary-only case. + assert_eq!(bridge_entry_count(&[]), 1); + let env: HashMap = HashMap::new(); + assert!(!env.contains_key("GIT_CONFIG_COUNT")); + } + + #[test] + fn merges_helper_rewrites_count_and_terminal_prompt() { + let keystone = slug("fabro-sh/keystone"); + let fabro = slug("fabro-sh/fabro"); + let env = bridged_env(HashMap::new(), &[&fabro, &keystone]); + + assert_eq!(env.get("GIT_CONFIG_COUNT").map(String::as_str), Some("5")); + assert_eq!( + env.get("GIT_CONFIG_KEY_0").map(String::as_str), + Some("credential.https://github.com.helper") + ); + assert_eq!( + env.get("GIT_CONFIG_KEY_1").map(String::as_str), + Some("url.https://github.com/fabro-sh/fabro.insteadOf") + ); + assert_eq!( + env.get("GIT_CONFIG_VALUE_1").map(String::as_str), + Some("git@github.com:fabro-sh/fabro") + ); + assert_eq!( + env.get("GIT_CONFIG_VALUE_2").map(String::as_str), + Some("ssh://git@github.com/fabro-sh/fabro") + ); + assert_eq!( + env.get("GIT_TERMINAL_PROMPT").map(String::as_str), + Some("0") + ); + + // No secrets anywhere in the generated values. + for (key, value) in &env { + assert!(!value.contains("ghs_"), "{key}={value}"); + } + } + + #[test] + fn respects_an_explicit_user_terminal_prompt() { + let keystone = slug("fabro-sh/keystone"); + let env = bridged_env( + HashMap::from([("GIT_TERMINAL_PROMPT".to_string(), "1".to_string())]), + &[&keystone], + ); + assert_eq!( + env.get("GIT_TERMINAL_PROMPT").map(String::as_str), + Some("1") + ); + } + + #[test] + fn appends_after_a_valid_user_git_config_overlay() { + let keystone = slug("fabro-sh/keystone"); + let env = bridged_env( + HashMap::from([ + ("GIT_CONFIG_COUNT".to_string(), "1".to_string()), + ("GIT_CONFIG_KEY_0".to_string(), "user.name".to_string()), + ("GIT_CONFIG_VALUE_0".to_string(), "Overlay User".to_string()), + ]), + &[&keystone], + ); + + assert_eq!(env.get("GIT_CONFIG_COUNT").map(String::as_str), Some("4")); + assert_eq!( + env.get("GIT_CONFIG_KEY_0").map(String::as_str), + Some("user.name"), + "user entry must survive at its original index" + ); + assert_eq!( + env.get("GIT_CONFIG_KEY_1").map(String::as_str), + Some("credential.https://github.com.helper") + ); + + // Real Git sees both the user's entry and the appended bridge entry. + let dir = tempfile::tempdir().unwrap(); + let output = git(&["config", "--list"], &env, dir.path()); + assert!(output.status.success(), "{output:?}"); + let listed = String::from_utf8_lossy(&output.stdout); + assert!(listed.contains("user.name=Overlay User"), "{listed}"); + assert!( + listed.contains("credential.https://github.com.helper"), + "{listed}" + ); + } + + #[test] + fn rejects_a_malformed_user_git_config_overlay() { + let keystone = slug("fabro-sh/keystone"); + + let mut non_numeric = HashMap::from([("GIT_CONFIG_COUNT".to_string(), "two".to_string())]); + let err = merge_git_bridge_env(&mut non_numeric, &[&keystone]).unwrap_err(); + assert!(err.to_string().contains("GIT_CONFIG_COUNT"), "{err}"); + + let mut missing_index = HashMap::from([ + ("GIT_CONFIG_COUNT".to_string(), "2".to_string()), + ("GIT_CONFIG_KEY_0".to_string(), "user.name".to_string()), + ("GIT_CONFIG_VALUE_0".to_string(), "Overlay".to_string()), + ]); + let err = merge_git_bridge_env(&mut missing_index, &[&keystone]).unwrap_err(); + assert!(err.to_string().contains("GIT_CONFIG_KEY_1"), "{err}"); + } + + /// With the bridge active, `git credential fill` for github.com resolves + /// through the generated helper and reads `$GITHUB_TOKEN` from the + /// invoking process environment at invocation time. + #[test] + fn credential_helper_reads_github_token_at_invocation_time() { + use std::io::Write as _; + + let keystone = slug("fabro-sh/keystone"); + let mut env = bridged_env(HashMap::new(), &[&keystone]); + env.insert("GITHUB_TOKEN".to_string(), "test-token-value".to_string()); + + let dir = tempfile::tempdir().unwrap(); + let mut command = Command::new("git"); + command + .args(["credential", "fill"]) + .current_dir(dir.path()) + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()); + for (key, value) in &env { + command.env(key, value); + } + let mut child = command.spawn().expect("git credential fill should spawn"); + child + .stdin + .as_mut() + .unwrap() + .write_all(b"protocol=https\nhost=github.com\npath=fabro-sh/keystone\n\n") + .unwrap(); + let output = child.wait_with_output().unwrap(); + + assert!(output.status.success(), "{output:?}"); + let filled = String::from_utf8_lossy(&output.stdout); + assert!(filled.contains("username=x-access-token"), "{filled}"); + assert!(filled.contains("password=test-token-value"), "{filled}"); + } + + /// Real Git applies the generated `insteadOf` rewrites: the exact SSH + /// spellings of a declared repository resolve to their HTTPS-analog + /// route (a local `file://` fixture here, so no network is involved), + /// while `GIT_SSH_COMMAND=false` proves SSH is never attempted. + #[test] + fn declared_ssh_urls_rewrite_to_the_https_route() { + let root = tempfile::tempdir().unwrap(); + let base = init_bare_fixture(root.path(), "fabro-sh/keystone"); + let keystone = slug("fabro-sh/keystone"); + + let mut env: HashMap = HashMap::new(); + for (offset, (key, value)) in bridge_entries(&[&keystone], &base).into_iter().enumerate() { + env.insert(format!("GIT_CONFIG_KEY_{offset}"), key); + env.insert(format!("GIT_CONFIG_VALUE_{offset}"), value); + } + env.insert("GIT_CONFIG_COUNT".to_string(), "3".to_string()); + env.insert("GIT_SSH_COMMAND".to_string(), "false".to_string()); + + for url in [ + "ssh://git@github.com/fabro-sh/keystone.git", + "ssh://git@github.com/fabro-sh/keystone", + "git@github.com:fabro-sh/keystone.git", + "git@github.com:fabro-sh/keystone", + ] { + let output = git(&["ls-remote", url], &env, root.path()); + assert!( + output.status.success(), + "{url} should rewrite to the fixture route: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + } + + /// An undeclared SSH URL that shares no declared prefix is not + /// rewritten: Git still routes it to SSH, where the scripted + /// `GIT_SSH_COMMAND=false` fails immediately without network access. + #[test] + fn undeclared_ssh_urls_are_not_rewritten() { + let root = tempfile::tempdir().unwrap(); + let base = init_bare_fixture(root.path(), "fabro-sh/keystone"); + let keystone = slug("fabro-sh/keystone"); + + let mut env: HashMap = HashMap::new(); + for (offset, (key, value)) in bridge_entries(&[&keystone], &base).into_iter().enumerate() { + env.insert(format!("GIT_CONFIG_KEY_{offset}"), key); + env.insert(format!("GIT_CONFIG_VALUE_{offset}"), value); + } + env.insert("GIT_CONFIG_COUNT".to_string(), "3".to_string()); + env.insert("GIT_SSH_COMMAND".to_string(), "false".to_string()); + env.insert("GITHUB_TOKEN".to_string(), "test-token-value".to_string()); + + let output = git( + &["ls-remote", "git@github.com:fabro-sh/undeclared"], + &env, + root.path(), + ); + assert!(!output.status.success(), "{output:?}"); + let stderr = String::from_utf8_lossy(&output.stderr); + // Not rewritten: the failure never mentions the local HTTPS-analog + // fixture route, so Git still chose the SSH transport. + assert!( + !stderr.contains(&root.path().display().to_string()), + "undeclared URL must not be rewritten to the fixture route: {stderr}" + ); + assert!(!stderr.contains("test-token-value"), "{stderr}"); + } + + /// Prefix collision: with `fabro-sh/keystone` declared, both SSH + /// spellings of `fabro-sh/keystone-other` are rewritten to the HTTPS + /// route (prefix match), where access fails — at GitHub this is an + /// authorization error for the scoped token — and no token leaks into + /// the output. + #[test] + fn prefix_colliding_undeclared_repositories_rewrite_and_fail_without_token_leak() { + let root = tempfile::tempdir().unwrap(); + let base = init_bare_fixture(root.path(), "fabro-sh/keystone"); + let keystone = slug("fabro-sh/keystone"); + + let mut env: HashMap = HashMap::new(); + for (offset, (key, value)) in bridge_entries(&[&keystone], &base).into_iter().enumerate() { + env.insert(format!("GIT_CONFIG_KEY_{offset}"), key); + env.insert(format!("GIT_CONFIG_VALUE_{offset}"), value); + } + env.insert("GIT_CONFIG_COUNT".to_string(), "3".to_string()); + env.insert("GIT_SSH_COMMAND".to_string(), "false".to_string()); + env.insert("GITHUB_TOKEN".to_string(), "test-token-value".to_string()); + + for url in [ + "git@github.com:fabro-sh/keystone-other", + "ssh://git@github.com/fabro-sh/keystone-other.git", + ] { + let output = git(&["ls-remote", url], &env, root.path()); + assert!(!output.status.success(), "{url}: {output:?}"); + let stderr = String::from_utf8_lossy(&output.stderr); + // The failure names the (missing) HTTPS-analog fixture route, + // proving the prefix rule rewrote the URL away from SSH. + assert!( + stderr.contains("keystone-other"), + "{url} must be rewritten away from SSH, got: {stderr}" + ); + assert!( + stderr.contains(&root.path().display().to_string()), + "{url} must land on the rewritten route, got: {stderr}" + ); + assert!(!stderr.contains("test-token-value"), "{stderr}"); + } + } +} diff --git a/lib/components/fabro-workflow/src/handler/llm/acp.rs b/lib/components/fabro-workflow/src/handler/llm/acp.rs index cfd498beb..9e9f30c39 100644 --- a/lib/components/fabro-workflow/src/handler/llm/acp.rs +++ b/lib/components/fabro-workflow/src/handler/llm/acp.rs @@ -473,8 +473,9 @@ impl AgentAcpBackend { emitter.notice( RunNoticeLevel::Info, RunNoticeCode::GithubTokenRefreshLimited, - "ACP agent stages receive workflow env at process launch; stages running beyond \ - token expiry may need to be retried.", + "ACP agent stages receive workflow env at process launch; GITHUB_TOKEN access to \ + every declared repository expires together, so stages running beyond token \ + expiry may need to be retried.", ); } provider diff --git a/lib/components/fabro-workflow/src/lib.rs b/lib/components/fabro-workflow/src/lib.rs index 3178542d7..29cd629ec 100644 --- a/lib/components/fabro-workflow/src/lib.rs +++ b/lib/components/fabro-workflow/src/lib.rs @@ -293,6 +293,7 @@ pub mod error; pub mod event; pub mod file_resolver; pub mod git; +pub(crate) mod git_bridge; pub(crate) mod graph; pub mod handler; mod hook_context; diff --git a/lib/components/fabro-workflow/src/operations/start.rs b/lib/components/fabro-workflow/src/operations/start.rs index 3ba470151..3acf6255b 100644 --- a/lib/components/fabro-workflow/src/operations/start.rs +++ b/lib/components/fabro-workflow/src/operations/start.rs @@ -1,4 +1,4 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::path::Path; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; @@ -17,7 +17,7 @@ use fabro_sandbox::{DockerSandboxOptions, SandboxSpec}; use fabro_static::EnvVars; use fabro_types::settings::run::{ ApprovalMode, McpServerSettings as ResolvedMcpServerSettings, PullRequestSettings, - ResolvedMcpEntry, RunMode, RunNamespace as ResolvedRunSettings, + ResolvedGithubIntegration, ResolvedMcpEntry, RunMode, RunNamespace as ResolvedRunSettings, RunPrepareSettings as ResolvedRunPrepareSettings, }; use fabro_types::{ManifestPath, RunId, RunRunnableSource, SandboxProviderKind}; @@ -104,9 +104,10 @@ pub struct StartServices { pub artifact_sink: Option, pub run_control: Option>, pub github_app: Option, - /// Server-resolved GitHub integration permissions to inject into the - /// sandbox env. Empty when github integration has no permissions. - pub github_permissions: HashMap, + /// The resolved GitHub integration request (interpolated permissions + /// plus declared additional repositories) to inject into the sandbox + /// env. Empty when the github integration requests no token. + pub github_integration: ResolvedGithubIntegration, pub vault: Arc>, pub catalog: Arc, pub on_node: crate::OnNodeCallback, @@ -452,11 +453,13 @@ impl RunSession { .environment .resolve_env(secret_lookup) .map_err(|err| Error::engine_with_source("failed to resolve run environment", err))?; - let github_permissions: Option> = - (!services.github_permissions.is_empty()).then(|| services.github_permissions.clone()); + let github_integration = services + .github_integration + .is_token_requested() + .then(|| services.github_integration.clone()); let sandbox_env = SandboxEnvSpec { toml_env, - github_permissions, + github_integration, origin_url: record.repo_origin_url().map(str::to_string), }; @@ -1725,7 +1728,7 @@ reasoning = false artifact_sink: None, run_control: None, github_app: None, - github_permissions: HashMap::new(), + github_integration: ResolvedGithubIntegration::default(), vault: Arc::new(AsyncRwLock::new(start_vault(&[]))), catalog: test_catalog(), on_node: None, diff --git a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs index 8056192bd..af83d7cfe 100644 --- a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs @@ -287,7 +287,7 @@ async fn execute_test_run_with_options( hooks: HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -349,7 +349,7 @@ async fn execute_runs_start_to_exit_and_returns_final_context() { hooks: HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -488,7 +488,7 @@ async fn resumed_in_flight_node_starts_a_new_stage_execution() { hooks: HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -599,7 +599,7 @@ async fn run_with_lifecycle( hooks: HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), diff --git a/lib/components/fabro-workflow/src/pipeline/initialize.rs b/lib/components/fabro-workflow/src/pipeline/initialize.rs index a91929c7b..cde82cacf 100644 --- a/lib/components/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/components/fabro-workflow/src/pipeline/initialize.rs @@ -24,6 +24,7 @@ use super::types::{InitOptions, Initialized, LlmSpec, Persisted, SandboxEnvSpec} use crate::error::Error; use crate::event::{Event, RunNoticeCode, RunNoticeLevel}; use crate::git::GitAuthor; +use crate::git_bridge; use crate::handler::llm::{AgentAcpBackend, AgentApiBackend, BackendRouter, routing}; use crate::handler::{HandlerRegistry, default_registry}; #[cfg(test)] @@ -37,10 +38,14 @@ use crate::services::{ use crate::stage_execution::{StageExecutionSeed, StageExecutionTracker}; use crate::steering_hub::SteeringHub; -type BuiltSandboxEnv = ( - HashMap, - Option>, -); +struct BuiltSandboxEnv { + env: HashMap, + github_token: Option>, + /// The validated effective repository set behind `github_token`. + /// Present only in App mode or when additional repositories are + /// declared; drives the eager access validation at initialization. + github_access: Option, +} async fn run_hooks( hook_runner: Option<&HookRunner>, @@ -91,41 +96,120 @@ fn build_sandbox_env( spec: &SandboxEnvSpec, github_app: Option<&fabro_github::GitHubCredentials>, ) -> Result { - let env = spec.toml_env.clone(); + let mut env = spec.toml_env.clone(); - let Some(permissions) = spec.github_permissions.as_ref().filter(|p| !p.is_empty()) else { - return Ok((env, None)); + let no_token = |env| BuiltSandboxEnv { + env, + github_token: None, + github_access: None, }; + let Some(integration) = spec + .github_integration + .as_ref() + .filter(|integration| integration.is_token_requested()) + else { + return Ok(no_token(env)); + }; + let declares_additional = integration.has_additional_repositories(); let Some(creds) = github_app else { - return Ok((env, None)); + if declares_additional { + // Legacy permissions-only configuration stays best-effort, but a + // declared additional set is an explicit access requirement. + return Err(Error::Precondition( + "run.integrations.github.additional_repositories requires GitHub credentials, \ + but none are configured" + .to_string(), + )); + } + return Ok(no_token(env)); }; - let source = match creds { + // Validate the effective repository set whenever it matters: App mode + // scopes the mint to it, and any declared additional set must hold its + // invariants regardless of credential kind. Legacy PAT/static + // permissions-only runs skip it to preserve their origin-agnostic + // behavior. + let github_access = + if declares_additional || matches!(creds, fabro_github::GitHubCredentials::App(_)) { + fabro_github::GitHubRepositoryAccess::new( + spec.origin_url.as_deref(), + &integration.additional_repositories, + integration.permissions.clone(), + ) + .map_err(|err| { + Error::engine_with_anyhow("Failed to validate GitHub repository access", err) + })? + } else { + None + }; + + let github_token = match creds { fabro_github::GitHubCredentials::Pat(token) => { Some(InstallationTokenSource::pat(token.clone())) } fabro_github::GitHubCredentials::Installation(token) => { Some(InstallationTokenSource::installation(token.clone())) } - fabro_github::GitHubCredentials::App(_) => { - let Some(origin_url) = spec.origin_url.as_deref() else { - return Ok((env, None)); - }; - let https_url = fabro_github::ssh_url_to_https(origin_url); - let (owner, repo) = fabro_github::parse_github_owner_repo(&https_url) - .map_err(|err| Error::engine_with_anyhow("Failed to parse GitHub origin", err))?; - let permissions = serde_json::to_value(permissions).map_err(|err| { - Error::engine_with_source("Failed to serialize GitHub permissions", err) - })?; - Some( - InstallationTokenSource::for_repository(creds, owner, repo, permissions).map_err( - |err| Error::engine_with_anyhow("Failed to build GitHub token source", err), - )?, - ) - } + fabro_github::GitHubCredentials::App(_) => match github_access.as_ref() { + Some(access) => Some(InstallationTokenSource::for_access(creds, access).map_err( + |err| Error::engine_with_anyhow("Failed to build GitHub token source", err), + )?), + // No origin URL and nothing declared: keep the legacy + // best-effort skip. + None => None, + }, }; - Ok((env, source)) + if declares_additional { + let access = github_access + .as_ref() + .expect("access is always constructed when additional repositories are declared"); + git_bridge::merge_git_bridge_env(&mut env, &access.targets())?; + } + + Ok(BuiltSandboxEnv { + env, + github_token, + github_access, + }) +} + +/// When additional repositories are declared, prove the whole effective set +/// is reachable before the first workflow stage: resolve every repository's +/// App installation (naming any repository the App cannot see), then resolve +/// the token once eagerly. Legacy permissions-only runs skip this and keep +/// their best-effort behavior. +async fn validate_declared_repository_access( + built: &BuiltSandboxEnv, + github_app: Option<&fabro_github::GitHubCredentials>, +) -> Result<(), Error> { + let Some(access) = built + .github_access + .as_ref() + .filter(|access| access.has_additional_repositories()) + else { + return Ok(()); + }; + if let Some(fabro_github::GitHubCredentials::App(app)) = github_app { + access + .resolve_shared_installation_via_api(app) + .await + .map_err(|err| { + Error::engine_with_anyhow( + "Declared additional GitHub repository is not accessible", + err, + ) + })?; + } + if let Some(source) = built.github_token.as_ref() { + source.resolve().await.map_err(|err| { + Error::engine_with_anyhow( + "Failed to resolve GitHub access for the declared repository set", + err, + ) + })?; + } + Ok(()) } async fn build_registry( @@ -443,10 +527,17 @@ pub async fn initialize( }); } - let (base_env, github_token) = build_sandbox_env( + let built_env = build_sandbox_env( &options.sandbox_env, options.run_options.github_app.as_ref(), )?; + validate_declared_repository_access(&built_env, options.run_options.github_app.as_ref()) + .await?; + let BuiltSandboxEnv { + env: base_env, + github_token, + github_access: _, + } = built_env; let tool_env_provider = Arc::new(WorkflowToolEnvProvider { base_env: base_env.clone(), github_token: github_token.clone(), @@ -818,7 +909,7 @@ mod tests { hooks: fabro_hooks::HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -948,7 +1039,7 @@ mod tests { let initialized = initialize(persisted, InitOptions { sandbox_env: SandboxEnvSpec { toml_env: HashMap::from([("TEST_KEY".to_string(), "value".to_string())]), - github_permissions: None, + github_integration: None, origin_url: None, }, ..test_init_options( @@ -1271,7 +1362,7 @@ mod tests { hooks: fabro_hooks::HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault, @@ -1366,7 +1457,7 @@ mod tests { hooks: fabro_hooks::HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -1508,7 +1599,7 @@ mod tests { hooks: fabro_hooks::HookSettings { hooks: vec![] }, sandbox_env: SandboxEnvSpec { toml_env: HashMap::new(), - github_permissions: None, + github_integration: None, origin_url: None, }, vault: auth_test_support::empty_vault(), @@ -1524,4 +1615,171 @@ mod tests { assert!(matches!(result, Err(Error::Cancelled))); } + + mod github_integration_env { + //! Focused tests for `build_sandbox_env` / + //! `validate_declared_repository_access` around declared additional + //! repositories. Installation-resolution failure naming is covered + //! by `fabro_github::access` tests; these prove the initialization + //! wiring: hard errors for declared sets, best-effort behavior for + //! legacy permissions-only configuration. + + use fabro_github::test_support::{InstallationTokenMinter, installation_token_source}; + use fabro_github::{GitHubAppCredentials, GitHubCredentials, InstallationToken}; + use fabro_types::settings::run::ResolvedGithubIntegration; + + use super::*; + + fn integration(additional: &[&str]) -> ResolvedGithubIntegration { + ResolvedGithubIntegration { + permissions: HashMap::from([( + "contents".to_string(), + "read".to_string(), + )]), + additional_repositories: additional + .iter() + .map(|value| value.parse().expect("test slug should parse")) + .collect(), + } + } + + fn spec( + origin: Option<&str>, + github_integration: Option, + ) -> SandboxEnvSpec { + SandboxEnvSpec { + toml_env: HashMap::new(), + github_integration, + origin_url: origin.map(str::to_string), + } + } + + #[test] + fn declared_additional_repositories_require_credentials() { + let spec = spec( + Some("https://github.com/fabro-sh/fabro"), + Some(integration(&["fabro-sh/keystone"])), + ); + let Err(err) = build_sandbox_env(&spec, None) else { + panic!("declared additional repositories without credentials must fail"); + }; + assert!( + err.to_string().contains("requires GitHub credentials"), + "{err}" + ); + } + + #[test] + fn declared_additional_repositories_require_an_origin() { + let spec = spec(None, Some(integration(&["fabro-sh/keystone"]))); + let creds = GitHubCredentials::Pat("ghp_x".to_string()); + let Err(err) = build_sandbox_env(&spec, Some(&creds)) else { + panic!("declared additional repositories without an origin must fail"); + }; + assert!( + err.to_string().contains("GitHub repository access"), + "{err}" + ); + } + + #[test] + fn declared_repositories_inject_bridge_entries_and_keep_the_pat_source() { + let spec = spec( + Some("https://github.com/fabro-sh/fabro"), + Some(integration(&["fabro-sh/keystone"])), + ); + let creds = GitHubCredentials::Pat("ghp_x".to_string()); + let built = build_sandbox_env(&spec, Some(&creds)).unwrap(); + + assert!(built.github_token.is_some()); + let access = built.github_access.expect("access should be constructed"); + assert!(access.has_additional_repositories()); + // Helper entry plus two SSH rewrites for each of the two + // effective repositories (origin + declared additional). + assert_eq!( + built.env.get("GIT_CONFIG_COUNT").map(String::as_str), + Some("5") + ); + assert_eq!( + built.env.get("GIT_CONFIG_KEY_0").map(String::as_str), + Some("credential.https://github.com.helper") + ); + assert_eq!( + built.env.get("GIT_TERMINAL_PROMPT").map(String::as_str), + Some("0") + ); + } + + #[test] + fn legacy_permissions_only_configuration_stays_best_effort() { + // No credentials: no error, no token source, no bridge entries. + let no_creds = spec( + Some("https://github.com/fabro-sh/fabro"), + Some(integration(&[])), + ); + let built = build_sandbox_env(&no_creds, None).unwrap(); + assert!(built.github_token.is_none()); + assert!(!built.env.contains_key("GIT_CONFIG_COUNT")); + + // App credentials without an origin: legacy best-effort skip. + let creds = GitHubCredentials::App(GitHubAppCredentials { + app_id: "1".to_string(), + private_key_pem: "unused".to_string(), + slug: None, + }); + let no_origin = spec(None, Some(integration(&[]))); + let built = build_sandbox_env(&no_origin, Some(&creds)).unwrap(); + assert!(built.github_token.is_none()); + assert!(built.github_access.is_none()); + } + + struct FailingMinter; + + #[async_trait::async_trait] + impl InstallationTokenMinter for FailingMinter { + async fn mint(&self) -> anyhow::Result { + Err(anyhow::anyhow!("scripted mint failure")) + } + } + + #[tokio::test] + async fn eager_validation_fails_when_the_declared_token_cannot_resolve() { + let access = fabro_github::GitHubRepositoryAccess::new( + Some("https://github.com/fabro-sh/fabro"), + &["fabro-sh/keystone".parse().unwrap()].into_iter().collect(), + HashMap::from([("contents".to_string(), "read".to_string())]), + ) + .unwrap(); + let built = BuiltSandboxEnv { + env: HashMap::new(), + github_token: Some(installation_token_source( + "fabro-sh/fabro (+1 additional)", + Arc::new(FailingMinter), + )), + github_access: access, + }; + + let err = validate_declared_repository_access(&built, None) + .await + .unwrap_err(); + let message = err.to_string(); + assert!(message.contains("declared repository set"), "{message}"); + } + + #[tokio::test] + async fn eager_validation_skips_legacy_permissions_only_runs() { + let built = BuiltSandboxEnv { + env: HashMap::new(), + github_token: Some(installation_token_source( + "fabro-sh/fabro", + Arc::new(FailingMinter), + )), + github_access: None, + }; + + validate_declared_repository_access(&built, None) + .await + .expect("legacy permissions-only runs must not resolve eagerly"); + } + } } diff --git a/lib/components/fabro-workflow/src/pipeline/types.rs b/lib/components/fabro-workflow/src/pipeline/types.rs index ebbe3f4f8..c65f1fdf6 100644 --- a/lib/components/fabro-workflow/src/pipeline/types.rs +++ b/lib/components/fabro-workflow/src/pipeline/types.rs @@ -8,7 +8,9 @@ use fabro_mcp::config::McpServerSettings; use fabro_model::{Catalog, ProviderId}; use fabro_sandbox::SandboxSpec; use fabro_template::TemplateContext; -use fabro_types::settings::run::{PullRequestSettings, RunModelControls}; +use fabro_types::settings::run::{ + PullRequestSettings, ResolvedGithubIntegration, RunModelControls, +}; use fabro_types::{ManifestPath, RunId, RunProjection}; use fabro_validate::{Diagnostic, Severity}; use fabro_vault::Vault; @@ -246,7 +248,10 @@ pub struct LlmSpec { #[derive(Clone)] pub struct SandboxEnvSpec { pub toml_env: HashMap, - pub github_permissions: Option>, + /// The resolved GitHub integration request (interpolated permissions + /// plus declared additional repositories). `None` when the run requests + /// no `GITHUB_TOKEN`. + pub github_integration: Option, pub origin_url: Option, } From d95b6cace1d2953be32397ed71dfd1882523d462 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 14:59:45 -0400 Subject: [PATCH 18/30] feat(server): preflight additional github repository access When a run declares additional repositories, preflight now proves the whole effective set works instead of treating a minted token as proof: - It constructs the same validated `GitHubRepositoryAccess` used by runtime initialization, so the two paths cannot disagree. - In App mode it first resolves every repository's installation with the App JWT and requires one shared installation ID, naming any repository the App cannot see before the mint; then it mints the one scoped token, failing with the raw error on rejection. - Every effective repository gets a non-interactive `git ls-remote HEAD` probe through a shared helper that keeps the token out of the URL, argv, and errors (a credential helper reads GITHUB_TOKEN from the child environment), retries auth-shaped failures with the same token to cover replication lag (classified via fabro_sandbox::classify_failure), and reports one check per repository in deterministic primary-first order under bounded concurrency. - A resolved run environment that defines GH_TOKEN produces a warning (gh prefers it over the managed token) without failing preflight. - With no additional repositories declared, the primary-only mint check is byte-for-byte unchanged. Co-Authored-By: Claude Fable 5 --- lib/apps/fabro-server/src/run_manifest.rs | 639 +++++++++++++++++++++- 1 file changed, 633 insertions(+), 6 deletions(-) diff --git a/lib/apps/fabro-server/src/run_manifest.rs b/lib/apps/fabro-server/src/run_manifest.rs index fda5f8087..44ec4fd0c 100644 --- a/lib/apps/fabro-server/src/run_manifest.rs +++ b/lib/apps/fabro-server/src/run_manifest.rs @@ -12,6 +12,7 @@ use fabro_config::{ CliLayer, CliOutputLayer, EnvironmentLayer, MergeMap, RunLayer, SettingsLayer, WorkflowSettingsBuilder, parse_input_overrides, parse_labels, project, }; +use fabro_github::token_source::{InstallationTokenSource, ResolvedToken, TokenSnapshot}; use fabro_graphviz::graph::{Graph, is_llm_handler_type}; use fabro_graphviz::render::apply_direction; use fabro_llm::model_test::{ModelTestStatus, run_basic_model_probe}; @@ -1200,14 +1201,40 @@ async fn run_github_token_check( resolved_run: &RunNamespace, github_app: Option, ) -> bool { - if !resolved_run.integrations.github.is_token_requested() { + run_github_token_check_with( + checks, + prepared, + resolved_run, + github_app, + mint_scoped_github_token, + probe_github_repository, + ) + .await +} + +async fn run_github_token_check_with( + checks: &mut Vec, + prepared: &PreparedManifest, + resolved_run: &RunNamespace, + github_app: Option, + mint_scoped_token: M, + probe_repository: P, +) -> bool +where + M: FnOnce(fabro_github::GitHubRepositoryAccess, fabro_github::GitHubCredentials) -> MFut, + MFut: Future>, + P: Fn(fabro_types::GitHubRepositorySlug, ResolvedToken) -> PFut, + PFut: Future>, +{ + let github = &resolved_run.integrations.github; + if !github.is_token_requested() { return true; } // Resolve InterpString permission values eagerly for token minting and // for display in the preflight report. - let github_permissions = match resolved_run.integrations.github.resolve_permissions() { - Ok(permissions) => permissions, + let integration = match github.resolve_integration() { + Ok(integration) => integration, Err(err) => { checks.push(CheckResult { name: "GitHub Token".into(), @@ -1219,13 +1246,191 @@ async fn run_github_token_check( return false; } }; - - let perm_details = github_permissions + let perm_details = integration + .permissions .iter() .map(|(key, value)| CheckDetail::new(format!("{key}: {value}"))) .collect::>(); + + if !integration.has_additional_repositories() { + // Primary-only behavior is unchanged: a mint check when credentials + // and an origin exist, a warning otherwise, and no Git-content probe + // (permissions-only workflows may request non-contents permissions). + return check_primary_only_github_token( + checks, + prepared, + github_app, + &integration.permissions, + perm_details, + ) + .await; + } + + // `gh` checks GH_TOKEN before GITHUB_TOKEN, so a user-defined GH_TOKEN + // bypasses the managed scoped token for gh commands. Warn without + // failing; the value is the workflow author's responsibility. + if resolved_run.environment.env.contains_key("GH_TOKEN") { + checks.push(CheckResult { + name: "GH_TOKEN Override".into(), + status: CheckStatus::Warning, + summary: "gh will not use the managed token".into(), + details: vec![], + remediation: Some( + "The resolved run environment defines GH_TOKEN, which the gh CLI prefers over \ + the managed GITHUB_TOKEN; gh commands will not use the token scoped to the \ + declared repositories." + .to_string(), + ), + }); + } + + let Some(git) = prepared.git.as_ref() else { + checks.push(CheckResult { + name: "GitHub Token".into(), + status: CheckStatus::Error, + summary: "missing origin".into(), + details: perm_details, + remediation: Some( + "run.integrations.github.additional_repositories requires a GitHub run origin, \ + but this run has no repository origin URL" + .to_string(), + ), + }); + return false; + }; + let Some(creds) = github_app else { + checks.push(CheckResult { + name: "GitHub Token".into(), + status: CheckStatus::Error, + summary: "missing credentials".into(), + details: perm_details, + remediation: Some( + "run.integrations.github.additional_repositories requires GitHub credentials, \ + but none are configured on the server" + .to_string(), + ), + }); + return false; + }; + // The same validated access value runtime initialization constructs, so + // preflight and runtime cannot disagree about the effective set. + let access = match fabro_github::GitHubRepositoryAccess::new( + Some(&git.origin_url), + &integration.additional_repositories, + integration.permissions.clone(), + ) { + Ok(Some(access)) => access, + Ok(None) => { + checks.push(CheckResult { + name: "GitHub Token".into(), + status: CheckStatus::Error, + summary: "missing origin".into(), + details: perm_details, + remediation: Some( + "run.integrations.github.additional_repositories requires a GitHub run \ + origin, but the origin URL is empty" + .to_string(), + ), + }); + return false; + } + Err(err) => { + checks.push(CheckResult { + name: "GitHub Token".into(), + status: CheckStatus::Error, + summary: "invalid repository set".into(), + details: perm_details, + remediation: Some(format!("{err:#}")), + }); + return false; + } + }; + + // One mint scoped to the whole effective set. In App mode the minter + // first resolves every repository's installation so a failure names the + // repository the App cannot see. + let token = match mint_scoped_token(access.clone(), creds).await { + Ok(token) => token, + Err(err) => { + checks.push(CheckResult { + name: "GitHub Token".into(), + status: CheckStatus::Error, + summary: "failed".into(), + details: perm_details, + remediation: Some(err), + }); + return false; + } + }; + checks.push(CheckResult { + name: "GitHub Token".into(), + status: CheckStatus::Pass, + summary: "minted".into(), + details: perm_details.clone(), + remediation: None, + }); + + // Probe every effective repository with bounded concurrency, then report + // in deterministic primary-first order. Possession of a scoped token is + // not proof of access; the probe also verifies PAT/static credentials. + let targets: Vec = + access.targets().into_iter().cloned().collect(); + let probe_repository = &probe_repository; + let mut results: Vec<(usize, CheckResult)> = + stream::iter(targets.into_iter().enumerate().map(|(index, slug)| { + let token = token.clone(); + let perm_details = perm_details.clone(); + async move { + let check = match probe_repository(slug.clone(), token).await { + Ok(()) => CheckResult { + name: format!("GitHub Repository ({slug})"), + status: CheckStatus::Pass, + summary: "reachable".into(), + details: perm_details, + remediation: None, + }, + Err(err) => CheckResult { + name: format!("GitHub Repository ({slug})"), + status: CheckStatus::Error, + summary: "failed".into(), + details: perm_details, + remediation: Some(format!("Failed to verify repository access: {err}")), + }, + }; + (index, check) + } + })) + .buffer_unordered(REPOSITORY_PROBE_CONCURRENCY) + .collect() + .await; + results.sort_by_key(|(index, _)| *index); + + let mut ok = true; + for (_, check) in results { + if check.status != CheckStatus::Pass { + ok = false; + } + checks.push(check); + } + ok +} + +/// Bounded concurrency for per-repository `git ls-remote` probes. +const REPOSITORY_PROBE_CONCURRENCY: usize = 4; + +/// Total probe attempts per repository when failures classify as retryable +/// (token replication lag or transient infrastructure). +const REPOSITORY_PROBE_ATTEMPTS: u64 = 3; + +async fn check_primary_only_github_token( + checks: &mut Vec, + prepared: &PreparedManifest, + github_app: Option, + permissions: &HashMap, + perm_details: Vec, +) -> bool { if let (Some(creds), Some(git)) = (&github_app, prepared.git.as_ref()) { - match mint_github_token(creds, &git.origin_url, &github_permissions).await { + match mint_github_token(creds, &git.origin_url, permissions).await { Ok(_) => { checks.push(CheckResult { name: "GitHub Token".into(), @@ -1259,6 +1464,106 @@ async fn run_github_token_check( } } +/// Production minter for the multi-repository path: resolve every +/// repository's installation in App mode (naming any repository the App +/// cannot see, or one on a different installation), then mint the single +/// scoped token. A mint rejection after a clean resolution check surfaces +/// the raw error. +async fn mint_scoped_github_token( + access: fabro_github::GitHubRepositoryAccess, + creds: fabro_github::GitHubCredentials, +) -> std::result::Result { + if let fabro_github::GitHubCredentials::App(app) = &creds { + access + .resolve_shared_installation_via_api(app) + .await + .map_err(|err| format!("{err:#}"))?; + } + let source = + InstallationTokenSource::for_access(&creds, &access).map_err(|err| format!("{err:#}"))?; + source + .resolve() + .await + .map_err(|err| format!("Failed to mint GitHub token: {err:#}")) +} + +/// Production per-repository probe: a non-interactive +/// `git ls-remote HEAD` authenticated through a credential +/// helper that reads `GITHUB_TOKEN` from the child process environment, so +/// the token never appears in the URL, argv, or rendered errors. Mirrors the +/// runtime `git_bridge` credential helper in `fabro-workflow`. +async fn probe_github_repository( + slug: fabro_types::GitHubRepositorySlug, + token: ResolvedToken, +) -> std::result::Result<(), String> { + let url = format!("https://github.com/{}/{}", slug.owner(), slug.repo()); + probe_with_replication_retry(token.snapshot, || run_probe_ls_remote(&url, &token)).await +} + +/// Retry auth-shaped failures with the SAME token: replication of a given +/// token only makes progress, while re-minting would restart the replication +/// clock. Classification matches the sandbox git retry policy +/// (`fabro_sandbox::classify_failure`). +async fn probe_with_replication_retry( + snapshot: TokenSnapshot, + run: F, +) -> std::result::Result<(), String> +where + F: Fn() -> Fut, + Fut: Future>, +{ + let credential_context = fabro_sandbox::CredentialContext::from_snapshot(Some(&snapshot)); + let mut attempt = 0u64; + loop { + attempt += 1; + let Err(message) = run().await else { + return Ok(()); + }; + if attempt >= REPOSITORY_PROBE_ATTEMPTS + || fabro_sandbox::classify_failure(&message, credential_context).is_none() + { + return Err(message); + } + time::sleep(Duration::from_secs(attempt)).await; + } +} + +async fn run_probe_ls_remote(url: &str, token: &ResolvedToken) -> std::result::Result<(), String> { + let mut command = Command::new("git"); + command + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GITHUB_TOKEN", token.token.expose()) + .env("GIT_CONFIG_COUNT", "1") + .env("GIT_CONFIG_KEY_0", "credential.https://github.com.helper") + .env( + "GIT_CONFIG_VALUE_0", + r#"!f() { if [ "$1" = get ]; then echo username=x-access-token; echo "password=$GITHUB_TOKEN"; fi; }; f"#, + ) + .args(["ls-remote", url, "HEAD"]); + + let output = time::timeout(Duration::from_secs(10), command.output()) + .await + .map_err(|_| "git ls-remote timed out after 10s".to_string())? + .map_err(|err| format!("Failed to run git ls-remote: {err}"))?; + if output.status.success() { + return Ok(()); + } + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if !stderr.is_empty() { + Err(stderr) + } else if !stdout.is_empty() { + Err(stdout) + } else { + Err(format!( + "git ls-remote exited with status {}", + output.status + )) + } +} + async fn mint_github_token( creds: &fabro_github::GitHubCredentials, origin_url: &str, @@ -2974,4 +3279,326 @@ dockerfile = { path = "Dockerfile" } ); } } + + mod github_additional_repository_checks { + //! Seam-injected tests for the declared-additional-repositories + //! preflight path: one scoped mint, per-repository probes with + //! deterministic primary-first reporting, GH_TOKEN warning, and the + //! replication-lag retry policy. + + use std::sync::Mutex as StdMutex; + use std::sync::atomic::{AtomicU64, Ordering}; + + use fabro_github::token_source::{ + ResolvedToken, SecretString, TokenProvenance, TokenSnapshot, + }; + use fabro_types::settings::run::RunIntegrationsGithubSettings; + + use super::*; + + fn static_token(secret: &str) -> ResolvedToken { + ResolvedToken { + token: SecretString::new(secret.to_string()), + snapshot: TokenSnapshot { + generation: 0, + provenance: TokenProvenance::Static, + }, + refresh_failed: false, + } + } + + fn fresh_minted_snapshot() -> TokenSnapshot { + let now = chrono::Utc::now(); + TokenSnapshot { + generation: 1, + provenance: TokenProvenance::Minted { + minted_at: now, + expires_at: now + chrono::Duration::minutes(60), + }, + } + } + + fn declared(origin: &str, additional: &[&str]) -> (PreparedManifest, RunNamespace) { + let (prepared, mut resolved) = prepared_and_resolved_for_sandbox( + SandboxProviderKind::Local, + true, + Some(git_context(origin, "main")), + ); + resolved.integrations.github = RunIntegrationsGithubSettings { + permissions: HashMap::from([( + "contents".to_string(), + InterpString::parse("read"), + )]), + additional_repositories: additional + .iter() + .map(|value| value.parse().expect("test slug should parse")) + .collect(), + }; + (prepared, resolved) + } + + fn pat_creds() -> fabro_github::GitHubCredentials { + fabro_github::GitHubCredentials::Pat("ghp_test".to_string()) + } + + #[tokio::test(start_paused = true)] + async fn reports_each_repository_primary_first_despite_probe_completion_order() { + let (prepared, resolved) = declared("https://github.com/acme/widgets", &[ + "acme/zeta", + "acme/alpha", + ]); + let minted = Arc::new(StdMutex::new(Vec::new())); + let minted_for_seam = Arc::clone(&minted); + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + Some(pat_creds()), + move |access, _creds| { + minted_for_seam.lock().unwrap().push(access); + async { Ok(static_token("scoped-token")) } + }, + |slug, _token| async move { + // Invert completion order: the primary finishes last. + let delay = match slug.repo() { + "widgets" => 30, + "alpha" => 20, + _ => 10, + }; + time::sleep(Duration::from_millis(delay)).await; + Ok(()) + }, + ) + .await; + + assert!(ok); + // One mint listing every repository with the shared permissions. + let minted = minted.lock().unwrap(); + assert_eq!(minted.len(), 1); + assert_eq!(minted[0].repository_names(), vec![ + "widgets", "alpha", "zeta" + ]); + assert_eq!( + minted[0].permissions().get("contents").map(String::as_str), + Some("read") + ); + + let names: Vec<&str> = checks.iter().map(|check| check.name.as_str()).collect(); + assert_eq!(names, vec![ + "GitHub Token", + "GitHub Repository (acme/widgets)", + "GitHub Repository (acme/alpha)", + "GitHub Repository (acme/zeta)", + ]); + assert!(checks.iter().all(|check| check.status == CheckStatus::Pass)); + // The token never reaches check output. + for check in &checks { + let rendered = format!("{check:?}"); + assert!(!rendered.contains("scoped-token"), "{rendered}"); + } + } + + #[tokio::test] + async fn installation_resolution_failure_names_only_the_inaccessible_repository() { + let (prepared, resolved) = + declared("https://github.com/acme/widgets", &["acme/keystone"]); + let probes = Arc::new(AtomicU64::new(0)); + let probes_for_seam = Arc::clone(&probes); + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + Some(pat_creds()), + |_access, _creds| async { + Err( + "the GitHub App installation cannot see repository acme/keystone; add \ + it to the installation's repository access" + .to_string(), + ) + }, + move |_slug, _token| { + probes_for_seam.fetch_add(1, Ordering::SeqCst); + async { Ok(()) } + }, + ) + .await; + + assert!(!ok); + assert_eq!( + probes.load(Ordering::SeqCst), + 0, + "no probes after a failed mint" + ); + assert_eq!(checks.last().unwrap().name, "GitHub Token"); + assert_eq!(checks.last().unwrap().status, CheckStatus::Error); + let remediation = checks.last().unwrap().remediation.as_deref().unwrap(); + assert!(remediation.contains("acme/keystone"), "{remediation}"); + assert!(!remediation.contains("acme/widgets"), "{remediation}"); + } + + #[tokio::test] + async fn successful_mint_with_failed_probe_still_fails() { + let (prepared, resolved) = + declared("https://github.com/acme/widgets", &["acme/keystone"]); + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + Some(pat_creds()), + |_access, _creds| async { Ok(static_token("scoped-token")) }, + |slug, _token| async move { + if slug.repo() == "keystone" { + Err("remote: Repository not found.".to_string()) + } else { + Ok(()) + } + }, + ) + .await; + + assert!(!ok); + let keystone = checks + .iter() + .find(|check| check.name == "GitHub Repository (acme/keystone)") + .expect("keystone probe result should be reported"); + assert_eq!(keystone.status, CheckStatus::Error); + assert!( + !keystone + .remediation + .as_deref() + .unwrap_or_default() + .contains("scoped-token") + ); + let widgets = checks + .iter() + .find(|check| check.name == "GitHub Repository (acme/widgets)") + .expect("primary probe result should be reported"); + assert_eq!(widgets.status, CheckStatus::Pass); + } + + #[tokio::test] + async fn resolved_gh_token_warns_without_failing() { + let (prepared, mut resolved) = + declared("https://github.com/acme/widgets", &["acme/keystone"]); + resolved + .environment + .env + .insert("GH_TOKEN".to_string(), InterpString::parse("user-token")); + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + Some(pat_creds()), + |_access, _creds| async { Ok(static_token("scoped-token")) }, + |_slug, _token| async { Ok(()) }, + ) + .await; + + assert!(ok, "a GH_TOKEN override warns but does not fail preflight"); + let warning = checks + .iter() + .find(|check| check.name == "GH_TOKEN Override") + .expect("GH_TOKEN warning should be reported"); + assert_eq!(warning.status, CheckStatus::Warning); + } + + #[tokio::test] + async fn missing_origin_fails_for_declared_repositories() { + let (prepared, resolved) = { + let (mut prepared, resolved) = + declared("https://github.com/acme/widgets", &["acme/keystone"]); + prepared.git = None; + (prepared, resolved) + }; + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + Some(pat_creds()), + |_access, _creds| async { Ok(static_token("scoped-token")) }, + |_slug, _token| async { Ok(()) }, + ) + .await; + + assert!(!ok); + assert_eq!(checks.last().unwrap().summary, "missing origin"); + } + + #[tokio::test] + async fn missing_credentials_fail_for_declared_repositories() { + let (prepared, resolved) = + declared("https://github.com/acme/widgets", &["acme/keystone"]); + let mut checks = Vec::new(); + + let ok = run_github_token_check_with( + &mut checks, + &prepared, + &resolved, + None, + |_access, _creds| async { Ok(static_token("scoped-token")) }, + |_slug, _token| async { Ok(()) }, + ) + .await; + + assert!(!ok); + assert_eq!(checks.last().unwrap().summary, "missing credentials"); + } + + #[tokio::test(start_paused = true)] + async fn replication_lag_failure_retries_with_the_same_token_and_succeeds() { + let attempts = Arc::new(AtomicU64::new(0)); + let attempts_for_run = Arc::clone(&attempts); + + let result = probe_with_replication_retry(fresh_minted_snapshot(), move || { + let attempts = Arc::clone(&attempts_for_run); + async move { + if attempts.fetch_add(1, Ordering::SeqCst) == 0 { + Err("remote: Repository not found.".to_string()) + } else { + Ok(()) + } + } + }) + .await; + + assert!(result.is_ok()); + assert_eq!(attempts.load(Ordering::SeqCst), 2); + } + + #[tokio::test(start_paused = true)] + async fn static_credential_auth_failures_do_not_retry() { + let attempts = Arc::new(AtomicU64::new(0)); + let attempts_for_run = Arc::clone(&attempts); + let static_snapshot = TokenSnapshot { + generation: 0, + provenance: TokenProvenance::Static, + }; + + let result = probe_with_replication_retry(static_snapshot, move || { + let attempts = Arc::clone(&attempts_for_run); + async move { + attempts.fetch_add(1, Ordering::SeqCst); + Err("remote: Repository not found.".to_string()) + } + }) + .await; + + assert!(result.is_err()); + assert_eq!( + attempts.load(Ordering::SeqCst), + 1, + "a 404 with a static credential cannot become valid by waiting" + ); + } + } } From 84b75f29f102ff7d2d0090563bb4959f3e77f75c Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 15:12:23 -0400 Subject: [PATCH 19/30] docs(api): document additional github repository access - Add `additional_repositories` to the RunIntegrationsGithubSettings OpenAPI schema and reuse the canonical Rust settings types through `with_replacement`, with type-identity witnesses and JSON parity tests for populated and empty repository sets. - Regenerate the TypeScript API client. - Document the feature in the GitHub integration and run-configuration guides: exact layer replacement rules, single-token scope, gh/API support, App-versus-PAT scope, the same-owner/same-installation requirement, validation errors, supported Git URL forms, hard-failure semantics for declared repositories, GH_TOKEN precedence, and the security boundary (no second server-side repository intersection; contents = "write" lets any stage push to any declared repository). Correct the earlier claim that injecting GITHUB_TOKEN alone makes arbitrary additional private clones work. - Add a dated changelog entry and an opt-in live GitHub App e2e test that verifies a scoped multi-repository token reads every declared repository (and that a primary-only token cannot), with repositories supplied through the test environment. Co-Authored-By: Claude Fable 5 --- docs/public/api-reference/fabro-api.yaml | 11 ++ docs/public/changelog/2026-08-21.mdx | 20 +++ docs/public/docs.json | 7 + docs/public/execution/run-configuration.mdx | 18 +++ docs/public/integrations/github.mdx | 42 ++++- docs/public/reference/user-configuration.mdx | 2 +- .../fabro-github/tests/live_access.rs | 146 ++++++++++++++++++ lib/foundation/fabro-api/build.rs | 13 ++ lib/foundation/fabro-api/src/lib.rs | 5 +- .../tests/run_integrations_round_trip.rs | 63 +++++++- .../run-integrations-github-settings.ts | 4 + .../fabro-api-client/src/models/run-spec.ts | 2 +- 12 files changed, 325 insertions(+), 8 deletions(-) create mode 100644 docs/public/changelog/2026-08-21.mdx create mode 100644 lib/components/fabro-github/tests/live_access.rs diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index ee2eb143e..3cc616b16 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -14447,6 +14447,17 @@ components: type: object additionalProperties: type: string + additional_repositories: + type: array + description: | + Additional GitHub repositories, beyond the implicit run origin, + that the minted GITHUB_TOKEN must cover. Each entry is a full + `owner/repository` slug; every repository must share one owner + with the run origin. Omitted when empty; settings persisted + before this field existed deserialize to an empty set. + items: + type: string + uniqueItems: true RunGoal: oneOf: diff --git a/docs/public/changelog/2026-08-21.mdx b/docs/public/changelog/2026-08-21.mdx new file mode 100644 index 000000000..7dac75da1 --- /dev/null +++ b/docs/public/changelog/2026-08-21.mdx @@ -0,0 +1,20 @@ +--- +title: "Additional GitHub repositories" +date: "2026-08-21" +--- + +## One token for the whole repository set + +A run can now declare additional GitHub repositories that its stages may access through the managed `GITHUB_TOKEN`: + +```toml +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +``` + +The run origin stays implicit, and Fabro mints one installation token scoped to the origin plus every declared repository with the shared permission map. Inside stages, `gh` commands, raw GitHub API calls, plain Git over HTTPS, and the common SSH URL spellings (`git@github.com:owner/repo` and `ssh://git@github.com/owner/repo`) all work against the declared set — the SSH forms are transparently rewritten to authenticated HTTPS with no secret placed in Git configuration. + +Every repository must share one owner and be reachable by the origin's GitHub App installation. Preflight resolves each repository's installation, mints the scoped token once, and probes every repository with `git ls-remote`, naming the exact repository when something is not accessible; run initialization enforces the same checks. A declared-but-inaccessible repository fails the run before its first stage. + +Declaring additional repositories requires `contents = "read"` or `contents = "write"`. With `contents = "write"`, any stage can push to any declared repository — declare the smallest set and weakest permissions that work. See [Additional repositories](/integrations/github#additional-repositories) for details, including layering rules and `GH_TOKEN` precedence. diff --git a/docs/public/docs.json b/docs/public/docs.json index 284e5e4cb..efc9025ce 100644 --- a/docs/public/docs.json +++ b/docs/public/docs.json @@ -294,6 +294,13 @@ "tab": "Changelog", "icon": "clock-rotate-left", "groups": [ + { + "group": "August 2026", + "icon": "clock-rotate-left", + "pages": [ + "changelog/2026-08-21" + ] + }, { "group": "July 2026", "icon": "clock-rotate-left", diff --git a/docs/public/execution/run-configuration.mdx b/docs/public/execution/run-configuration.mdx index d2ce9bd5e..4e09b13f8 100644 --- a/docs/public/execution/run-configuration.mdx +++ b/docs/public/execution/run-configuration.mdx @@ -363,6 +363,24 @@ Only requested permissions are included. The upper bound is the permission set g This table follows the normal settings precedence order. A higher-precedence layer can set `permissions = {}` to clear inherited permissions and run without a GitHub token. +### `[run.integrations.github].additional_repositories` + +Declare extra GitHub repositories, beyond the implicit run origin, that the minted `GITHUB_TOKEN` must cover. The one `permissions` map applies to the origin and every declared repository. + +```toml title="run.toml" +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +``` + +Each entry is a full `owner/repository` slug. Every repository in the effective set must share one owner and be reachable by the origin repository's GitHub App installation. A non-empty list requires `contents = "read"` or `contents = "write"`. Malformed slugs, case-insensitive duplicates, cross-owner sets, and sets larger than 499 entries fail configuration validation with indexed error paths such as `run.integrations.github.additional_repositories[1]`. + +Unlike permissions-only configuration, declared additional repositories are a hard requirement: missing credentials, a missing origin, or an inaccessible declared repository fails preflight and run initialization with the repository named. + +The higher-precedence list replaces the lower one wholesale — no union and no `...` splice — and `additional_repositories = []` explicitly clears an inherited list. `additional_repositories` and `permissions` resolve independently; if layering leaves repositories declared while permissions were cleared, resolution reports the invalid combination instead of dropping either field. + +See [Additional repositories](/integrations/github#additional-repositories) for what works inside stages (`gh`, GitHub API, plain Git over HTTPS and the common SSH spellings) and for the security boundary. + ### `[run.notifications]` Define named notification routes for run events. Slack lifecycle notifications are configured here, not in server config. diff --git a/docs/public/integrations/github.mdx b/docs/public/integrations/github.mdx index a08185561..4beab0b18 100644 --- a/docs/public/integrations/github.mdx +++ b/docs/public/integrations/github.mdx @@ -228,7 +228,7 @@ For public repositories, the clone works without credentials. The token is still ### GITHUB_TOKEN injection -When any settings layer declares `[run.integrations.github.permissions]`, Fabro prepares a scoped GitHub App token source and exposes it as the `GITHUB_TOKEN` environment variable in sandbox command and agent execution. Agents running inside the sandbox can use this token for GitHub API calls, cloning additional private repos, or pushing to branches. The GitHub CLI (`gh`) reads `GITHUB_TOKEN` automatically, so command stages can run `gh pr list`, `gh issue create`, and similar commands without an explicit `gh auth login`. +When any settings layer declares `[run.integrations.github.permissions]`, Fabro prepares a scoped GitHub App token source and exposes it as the `GITHUB_TOKEN` environment variable in sandbox command and agent execution. Agents running inside the sandbox can use this token for GitHub API calls and pushes within the granted permissions. The GitHub CLI (`gh`) reads `GITHUB_TOKEN` automatically, so command stages can run `gh pr list`, `gh issue create`, and similar commands without an explicit `gh auth login`. ```toml title="workflow.toml" [run.integrations.github.permissions] @@ -238,6 +238,46 @@ pull_requests = "write" Only the listed permissions are requested — the token is scoped to the minimum access needed. If the GitHub App isn't configured or the repository lacks an installation, the run logs a warning and continues without the token. +In App mode, the token covers only the run's origin repository unless the run declares [additional repositories](#additional-repositories). Injecting `GITHUB_TOKEN` alone does not make other private repositories reachable. + +### Additional repositories + +A run can declare extra GitHub repositories that its stages may access through the same `GITHUB_TOKEN`: + +```toml title="workflow.toml" +[run.integrations.github] +additional_repositories = ["fabro-sh/keystone"] +permissions = { contents = "read" } +``` + +The run origin stays implicit — never list it. Each entry is a full `owner/repository` slug (no scheme, host, ref, or extra path component). Fabro mints **one** installation token scoped to the origin plus every declared repository, with the one shared `permissions` map applying to all of them. + +What works against every declared repository, within the granted permissions: + +- **`gh` CLI and raw GitHub API calls** through `GITHUB_TOKEN`. +- **Plain Git over HTTPS** (`git clone https://github.com/owner/repo`), through a secret-free credential helper that reads `$GITHUB_TOKEN` at invocation time. +- **The common SSH spellings** `git@github.com:owner/repo[.git]` and `ssh://git@github.com/owner/repo[.git]`, through per-repository SSH-to-HTTPS rewrites injected into the stage environment. + +Fabro does not clone additional repositories for you; a workflow that needs one on disk adds its own clone step (`git clone https://github.com/owner/repo` or `gh repo clone owner/repo`). + +Requirements and validation: + +- Every repository in the effective set must share **one owner** and be reachable by the origin repository's GitHub App installation, because one App installation covers one account. Cross-owner declarations fail configuration validation; a same-owner repository outside the installation fails preflight and run initialization with the repository named. +- A non-empty `additional_repositories` requires `contents = "read"` or `contents = "write"` in the permission map. +- Malformed slugs, duplicates (repository identity is case-insensitive), and sets larger than 499 entries fail configuration validation with indexed error paths. +- Unlike permissions-only configuration, declared additional repositories are a hard requirement: missing GitHub credentials, a missing origin, or an inaccessible declared repository fails preflight and run initialization instead of continuing without the token. +- Layering: the higher-precedence `additional_repositories` list replaces the lower one wholesale (no union, no `...` splice), and `additional_repositories = []` explicitly clears an inherited list. `permissions` keeps its existing whole-map replacement behavior. If layering leaves repositories declared with permissions cleared, configuration resolution reports the invalid combination. + +Behavior notes: + +- **Token strategy (PAT):** the configured PAT is used as-is. The repository list drives validation and preflight probes, but it cannot narrow the PAT's inherent GitHub scope — App mode remains the least-authority option. +- **`GH_TOKEN` precedence:** `gh` checks `GH_TOKEN` before `GITHUB_TOKEN`. If the resolved run environment defines `GH_TOKEN`, `gh` uses it instead of the managed token; Fabro never sets or removes `GH_TOKEN`, and preflight warns when additional repositories are declared alongside one. +- **SSH rewrites match by prefix.** With `owner/repo` declared, the SSH spelling of `owner/repo-other` is also rewritten to HTTPS. The scoped token is invalid for undeclared repositories at GitHub, so authority is unchanged — but a private undeclared repository fails with a GitHub authorization error instead of a missing-credential or SSH error. + +#### Security boundary + +Workflow authors may name any repository reachable by the server's GitHub App installation; Fabro applies no second server-side repository intersection. The token is scoped server-side to exactly the declared set — a request to an undeclared repository fails at GitHub, and Fabro never mints an unscoped installation-wide token. With `contents = "write"`, **any stage can push to any declared repository**. Declare the smallest repository set and the weakest permissions that work. + Installation Access Tokens are short-lived. Fabro refreshes its own credentials before checkpoint pushes. For ACP/CLI agent turns launched with GitHub App push credentials, Fabro also re-mints the token and rewrites the sandbox's `origin` URL before the ACP process starts, then every 45 minutes for the lifetime of that turn. Refresh failures are logged and do not fail the stage. `FABRO_PUSH_CRED_REFRESH_AHEAD` defaults to enabled; set it to `0`, `false`, `off`, `no`, or an empty value to disable both turn-entry and background refresh. `FABRO_PUSH_CRED_REFRESH_INTERVAL_SECONDS` overrides the background interval, and `0` disables only the background loop. This refresh loop is ACP-specific; command and native/API agent stages do not run it. Reconnected sandboxes for resumed or parked runs currently lack the App credentials needed for ACP refresh, so the refresh is skipped there. diff --git a/docs/public/reference/user-configuration.mdx b/docs/public/reference/user-configuration.mdx index cf60ea2a4..e68f36896 100644 --- a/docs/public/reference/user-configuration.mdx +++ b/docs/public/reference/user-configuration.mdx @@ -34,7 +34,7 @@ Files that omit `_version` are treated as version `1`. The legacy top-level `ver | Scope | Examples | |---|---| | CLI-only | `[cli.target]`, `[cli.auth]`, `[cli.exec]`, `[cli.output]`, `[cli.updates]`, `[cli.logging]` | -| Shared run defaults | `[run.model]`, `[run.environment]`, `[environments.]`, `[run.checkpoint]`, `[run.inputs]`, `[run.prepare]`, `[run.pull_request]`, `[run.integrations.github.permissions]`, `[run.hooks]`, `[run.agent.mcps]` | +| Shared run defaults | `[run.model]`, `[run.environment]`, `[environments.]`, `[run.checkpoint]`, `[run.inputs]`, `[run.prepare]`, `[run.pull_request]`, `[run.integrations.github]`, `[run.hooks]`, `[run.agent.mcps]` | | Shared LLM catalog | `[llm.providers.]`, provider-scoped `[llm.providers..models.]` offerings, limits, features, controls, and costs | | Server-only | `[server.listen]`, `[server.api]`, `[server.web]`, `[server.auth]`, `[server.storage]`, `[server.artifacts]`, `[server.slatedb]`, `[server.scheduler]`, `[server.logging]`, `[server.integrations]` | diff --git a/lib/components/fabro-github/tests/live_access.rs b/lib/components/fabro-github/tests/live_access.rs new file mode 100644 index 000000000..2440b6547 --- /dev/null +++ b/lib/components/fabro-github/tests/live_access.rs @@ -0,0 +1,146 @@ +//! Opt-in live GitHub App test for additional-repository access. +//! +//! Verifies against the real GitHub API that one installation token scoped +//! to the primary repository plus one declared additional repository grants +//! Git read access to both. Runs only in live mode with these variables set +//! (it skips clearly otherwise): +//! +//! - `FABRO_TEST_GITHUB_APP_ID` — GitHub App id +//! - `GITHUB_APP_PRIVATE_KEY` — App private key (PEM, or base64-encoded PEM) +//! - `FABRO_TEST_GITHUB_ORIGIN` — HTTPS origin URL of the primary repository +//! - `FABRO_TEST_GITHUB_ADDITIONAL_REPO` — an `owner/repository` slug the +//! installation can see, ideally private, sharing the origin's owner +//! +//! The repositories come from the environment so no private slug is baked +//! into durable test output, and the minted token is only ever passed to +//! `git` through the child process environment. + +use std::collections::BTreeSet; +use std::process::Stdio; +use std::time::Duration; + +use base64::engine::general_purpose::STANDARD; +use fabro_github::token_source::InstallationTokenSource; +use fabro_github::{GitHubAppCredentials, GitHubCredentials, GitHubRepositoryAccess}; +use fabro_types::GitHubRepositorySlug; +use tokio::process::Command; +use tokio::time::sleep; + +fn env_var(name: &str) -> String { + #[expect( + clippy::disallowed_methods, + reason = "live e2e configuration comes from the process environment by design" + )] + std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set for this live test")) +} + +fn private_key_pem() -> String { + let raw = env_var("GITHUB_APP_PRIVATE_KEY"); + if raw.starts_with("-----") { + return raw; + } + let bytes = base64::Engine::decode(&STANDARD, &raw) + .expect("GITHUB_APP_PRIVATE_KEY is not valid base64"); + String::from_utf8(bytes).expect("GITHUB_APP_PRIVATE_KEY decoded to invalid UTF-8") +} + +async fn ls_remote_with_token(slug: &GitHubRepositorySlug, token: &str) -> bool { + let url = format!("https://github.com/{}/{}", slug.owner(), slug.repo()); + let output = Command::new("git") + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env("GITHUB_TOKEN", token) + .env("GIT_CONFIG_COUNT", "1") + .env("GIT_CONFIG_KEY_0", "credential.https://github.com.helper") + .env( + "GIT_CONFIG_VALUE_0", + r#"!f() { if [ "$1" = get ]; then echo username=x-access-token; echo "password=$GITHUB_TOKEN"; fi; }; f"#, + ) + .args(["ls-remote", &url, "HEAD"]) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .output() + .await + .expect("git should run"); + output.status.success() +} + +#[fabro_macros::e2e_test( + live("FABRO_TEST_GITHUB_APP_ID"), + live("GITHUB_APP_PRIVATE_KEY"), + live("FABRO_TEST_GITHUB_ORIGIN"), + live("FABRO_TEST_GITHUB_ADDITIONAL_REPO") +)] +async fn scoped_token_reaches_the_declared_additional_repository() { + let app = GitHubAppCredentials { + app_id: env_var("FABRO_TEST_GITHUB_APP_ID"), + private_key_pem: private_key_pem(), + slug: None, + }; + let origin = env_var("FABRO_TEST_GITHUB_ORIGIN"); + let additional: GitHubRepositorySlug = env_var("FABRO_TEST_GITHUB_ADDITIONAL_REPO") + .parse() + .expect("FABRO_TEST_GITHUB_ADDITIONAL_REPO must be an owner/repository slug"); + let additional_set: BTreeSet = [additional.clone()].into_iter().collect(); + + let access = GitHubRepositoryAccess::new( + Some(&origin), + &additional_set, + std::collections::HashMap::from([("contents".to_string(), "read".to_string())]), + ) + .expect("access request should validate") + .expect("origin should produce an access value"); + + // Every target resolves to one shared installation. + access + .resolve_shared_installation_via_api(&app) + .await + .expect("all repositories should share one App installation"); + + // One mint scoped to the whole effective set. + let source = InstallationTokenSource::for_access(&GitHubCredentials::App(app.clone()), &access) + .expect("token source should build"); + let resolved = source.resolve().await.expect("scoped mint should succeed"); + let token = resolved.token.expose(); + + // The one token reads both the primary and the additional repository. + // A freshly minted token can hit GitHub's replication lag, so retry a + // few times with the same token before failing. + for slug in access.targets() { + let mut reachable = false; + for _ in 0..3 { + if ls_remote_with_token(slug, token).await { + reachable = true; + break; + } + sleep(Duration::from_secs(2)).await; + } + assert!( + reachable, + "scoped token should read every declared repository" + ); + } + + // Negative scope check: a token minted for the primary alone must not + // read the additional repository (proves server-side scoping, not just + // possession of a token). + let primary_only = GitHubRepositoryAccess::new( + Some(&origin), + &BTreeSet::new(), + std::collections::HashMap::from([("contents".to_string(), "read".to_string())]), + ) + .expect("primary-only access should validate") + .expect("origin should produce an access value"); + let narrow_source = + InstallationTokenSource::for_access(&GitHubCredentials::App(app), &primary_only) + .expect("primary-only token source should build"); + let narrow = narrow_source + .resolve() + .await + .expect("primary-only mint should succeed"); + assert!( + !ls_remote_with_token(&additional, narrow.token.expose()).await, + "a primary-only token must not read the additional repository" + ); +} diff --git a/lib/foundation/fabro-api/build.rs b/lib/foundation/fabro-api/build.rs index 0fb3cccd0..73f0bfd19 100644 --- a/lib/foundation/fabro-api/build.rs +++ b/lib/foundation/fabro-api/build.rs @@ -208,6 +208,19 @@ fn main() { ("DiffSummary", "fabro_types::DiffSummary", &[]), ("RepositoryRef", "fabro_types::RepositoryRef", &[]), ("WorkflowSettings", "fabro_types::WorkflowSettings", &[]), + // Run-level GitHub integration settings reuse the canonical resolved + // types instead of generating parallel API DTOs; the wire shape is + // identical (InterpString serializes as its source string). + ( + "RunIntegrationsSettings", + "fabro_types::settings::run::RunIntegrationsSettings", + &[], + ), + ( + "RunIntegrationsGithubSettings", + "fabro_types::settings::run::RunIntegrationsGithubSettings", + &[], + ), ("ServerSettings", "fabro_types::ServerSettings", &[]), ( "ServerNamespace", diff --git a/lib/foundation/fabro-api/src/lib.rs b/lib/foundation/fabro-api/src/lib.rs index c53e682c7..a845a8252 100644 --- a/lib/foundation/fabro-api/src/lib.rs +++ b/lib/foundation/fabro-api/src/lib.rs @@ -25,7 +25,10 @@ pub mod types { ReasoningEffortFeature, Speed as BillingSpeed, TokenCounts as CompletionUsage, }; pub use fabro_types::run_event::AgentSessionActivatedProps; - pub use fabro_types::settings::run::{McpHttpProtocol, RunModelControls, RunModelSettings}; + pub use fabro_types::settings::run::{ + McpHttpProtocol, RunIntegrationsGithubSettings, RunIntegrationsSettings, RunModelControls, + RunModelSettings, + }; pub use fabro_types::settings::server::{ GithubIntegrationSettings, GithubIntegrationStrategy, IntegrationWebhooksSettings, LogDestination, ObjectStoreSettings, ServerApiSettings, ServerArtifactsSettings, diff --git a/lib/foundation/fabro-api/tests/run_integrations_round_trip.rs b/lib/foundation/fabro-api/tests/run_integrations_round_trip.rs index 243268f1c..bd5af9330 100644 --- a/lib/foundation/fabro-api/tests/run_integrations_round_trip.rs +++ b/lib/foundation/fabro-api/tests/run_integrations_round_trip.rs @@ -1,8 +1,10 @@ -//! JSON parity test for `RunIntegrationsGithubSettings`. +//! JSON parity and type-identity tests for `RunIntegrationsGithubSettings`. //! -//! Asserts that the API-side generated `RunIntegrationsGithubSettings` and -//! the canonical Rust resolved type round-trip through the same JSON shape. -//! Covers both the populated and empty-permissions cases. +//! Asserts that the API-side `RunIntegrationsGithubSettings` is the +//! canonical Rust resolved type (via `with_replacement` in `build.rs`) and +//! that both names round-trip through the same JSON shape. Covers the +//! populated and empty permissions cases as well as populated and empty +//! `additional_repositories` sets. use fabro_api::types::{ RunIntegrationsGithubSettings as ApiRunIntegrationsGithubSettings, @@ -11,6 +13,22 @@ use fabro_api::types::{ use fabro_types::settings::run::{RunIntegrationsGithubSettings, RunIntegrationsSettings}; use serde_json::json; +/// Type-identity witnesses: the generated API names are the canonical Rust +/// types, not parallel DTOs. Compiles only when they are the same type. +#[expect(dead_code, reason = "compile-time type-identity witness")] +fn github_settings_type_identity( + value: ApiRunIntegrationsGithubSettings, +) -> RunIntegrationsGithubSettings { + value +} + +#[expect(dead_code, reason = "compile-time type-identity witness")] +fn integrations_settings_type_identity( + value: ApiRunIntegrationsSettings, +) -> RunIntegrationsSettings { + value +} + #[test] fn run_integrations_github_settings_round_trips_with_permissions() { let json_value = json!({ @@ -44,6 +62,43 @@ fn run_integrations_github_settings_round_trips_empty_permissions() { assert_eq!(serde_json::to_value(&canonical).unwrap(), json_value); } +#[test] +fn run_integrations_github_settings_round_trips_additional_repositories() { + let json_value = json!({ + "permissions": { "contents": "read" }, + "additional_repositories": ["fabro-sh/arc", "fabro-sh/keystone"], + }); + + let api: ApiRunIntegrationsGithubSettings = + serde_json::from_value(json_value.clone()).expect("api type should parse repositories"); + let canonical: RunIntegrationsGithubSettings = + serde_json::from_value(json_value.clone()).expect("canonical type should parse"); + + assert_eq!(serde_json::to_value(&api).unwrap(), json_value); + assert_eq!(serde_json::to_value(&canonical).unwrap(), json_value); +} + +#[test] +fn run_integrations_github_settings_omits_an_empty_repository_set() { + // An absent field and an explicit empty array both deserialize to the + // empty set, and the empty set serializes back with the field omitted — + // keeping single-repository settings byte-identical to older releases. + let empty_array = json!({ + "permissions": {}, + "additional_repositories": [], + }); + let omitted = json!({ "permissions": {} }); + + let api: ApiRunIntegrationsGithubSettings = + serde_json::from_value(empty_array).expect("api type should parse an empty array"); + let canonical: RunIntegrationsGithubSettings = + serde_json::from_value(omitted.clone()).expect("canonical type should parse"); + + assert!(api.additional_repositories.is_empty()); + assert_eq!(serde_json::to_value(&api).unwrap(), omitted); + assert_eq!(serde_json::to_value(&canonical).unwrap(), omitted); +} + #[test] fn run_integrations_settings_round_trips() { let json_value = json!({ diff --git a/lib/packages/fabro-api-client/src/models/run-integrations-github-settings.ts b/lib/packages/fabro-api-client/src/models/run-integrations-github-settings.ts index a28ae5b4b..df9e432ba 100644 --- a/lib/packages/fabro-api-client/src/models/run-integrations-github-settings.ts +++ b/lib/packages/fabro-api-client/src/models/run-integrations-github-settings.ts @@ -16,4 +16,8 @@ export interface RunIntegrationsGithubSettings { 'permissions': { [key: string]: string; }; + /** + * Additional GitHub repositories, beyond the implicit run origin, that the minted GITHUB_TOKEN must cover. Each entry is a full `owner/repository` slug; every repository must share one owner with the run origin. Omitted when empty; settings persisted before this field existed deserialize to an empty set. + */ + 'additional_repositories'?: Array; } diff --git a/lib/packages/fabro-api-client/src/models/run-spec.ts b/lib/packages/fabro-api-client/src/models/run-spec.ts index a9ed38078..a9e41bec6 100644 --- a/lib/packages/fabro-api-client/src/models/run-spec.ts +++ b/lib/packages/fabro-api-client/src/models/run-spec.ts @@ -39,7 +39,7 @@ export interface RunSpec { 'graph_source'?: string | null; 'workflow_slug'?: string | null; /** - * SHA-256 identity of validated canonical workflow-version bytes. + * SHA-256 identity of validated canonical workflow-version bytes. Hex input is case-insensitive; Fabro emits the canonical lowercase form. */ 'workflow_version_id'?: string | null; 'automation'?: AutomationRef | null; From 68e3cb84197e0d67eff7f1be7bb48291bd891d62 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 15:55:45 -0400 Subject: [PATCH 20/30] fix(llm): capture Venice top-level costs --- .../src/codec/openai_compatible/response.rs | 4 +- .../src/codec/openai_compatible/stream.rs | 12 ++-- .../src/codec/openai_compatible/wire.rs | 7 +++ .../tests/it/wire/openai_compatible.rs | 39 ++++++++++++ ...e__decode_usage_venice_top_level_cost.snap | 55 +++++++++++++++++ ...e__stream_usage_venice_top_level_cost.snap | 60 +++++++++++++++++++ 6 files changed, 172 insertions(+), 5 deletions(-) create mode 100644 lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_usage_venice_top_level_cost.snap create mode 100644 lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_usage_venice_top_level_cost.snap diff --git a/lib/components/fabro-llm/src/codec/openai_compatible/response.rs b/lib/components/fabro-llm/src/codec/openai_compatible/response.rs index f7df78729..dc2779bf1 100644 --- a/lib/components/fabro-llm/src/codec/openai_compatible/response.rs +++ b/lib/components/fabro-llm/src/codec/openai_compatible/response.rs @@ -63,7 +63,9 @@ pub(super) fn decode_response( let wire_usage = api_resp.usage.as_ref(); let usage = wire_usage.map_or_else(TokenCounts::default, ApiUsage::token_counts); - let cost_usd = wire_usage.and_then(|u| u.cost); + let cost_usd = wire_usage + .and_then(|usage| usage.cost) + .or_else(|| api_resp.cost.as_ref().and_then(|cost| cost.usd)); let cost_source = translate::authoritative_cost_source(cost_usd); Ok(Response { diff --git a/lib/components/fabro-llm/src/codec/openai_compatible/stream.rs b/lib/components/fabro-llm/src/codec/openai_compatible/stream.rs index f389ee086..bb2633b91 100644 --- a/lib/components/fabro-llm/src/codec/openai_compatible/stream.rs +++ b/lib/components/fabro-llm/src/codec/openai_compatible/stream.rs @@ -29,8 +29,8 @@ pub(super) struct StreamState { /// True after `finish_events()` has run (guards against duplicates). finished: bool, rate_limit: Option, - /// In-band USD cost from the usage chunk (OpenRouter), surfaced as - /// authoritative on the final response. + /// In-band USD cost from the response, surfaced as authoritative on the + /// final response. cost_usd: Option, } @@ -72,9 +72,13 @@ impl StreamState { // Capture usage if present (often in a dedicated chunk). if let Some(usage) = &chunk.usage { self.usage = usage.token_counts(); - // Keep a previously seen cost when a later usage chunk omits it. - self.cost_usd = usage.cost.or(self.cost_usd); } + let cost_usd = chunk + .usage + .as_ref() + .and_then(|usage| usage.cost) + .or_else(|| chunk.cost.as_ref().and_then(|cost| cost.usd)); + self.cost_usd = cost_usd.or(self.cost_usd); let choices = chunk.choices.as_mut()?; let choice = choices.first_mut()?; diff --git a/lib/components/fabro-llm/src/codec/openai_compatible/wire.rs b/lib/components/fabro-llm/src/codec/openai_compatible/wire.rs index 62c6ede6b..83823d5f7 100644 --- a/lib/components/fabro-llm/src/codec/openai_compatible/wire.rs +++ b/lib/components/fabro-llm/src/codec/openai_compatible/wire.rs @@ -125,6 +125,12 @@ pub(super) struct ApiResponse { pub model: String, pub choices: Vec, pub usage: Option, + pub cost: Option, +} + +#[derive(serde::Deserialize)] +pub(super) struct ApiCost { + pub usd: Option, } #[derive(serde::Deserialize)] @@ -377,6 +383,7 @@ pub(super) struct StreamChunk { pub model: Option, pub choices: Option>, pub usage: Option, + pub cost: Option, } #[derive(serde::Deserialize)] diff --git a/lib/components/fabro-llm/tests/it/wire/openai_compatible.rs b/lib/components/fabro-llm/tests/it/wire/openai_compatible.rs index 0ff337cc5..22615dcd6 100644 --- a/lib/components/fabro-llm/tests/it/wire/openai_compatible.rs +++ b/lib/components/fabro-llm/tests/it/wire/openai_compatible.rs @@ -795,6 +795,31 @@ async fn decode_usage_openrouter_cost_and_cache_write() { fabro_test::fabro_json_snapshot!(response); } +/// Venice reports authoritative USD cost in a top-level object rather than +/// the OpenRouter `usage.cost` field. +#[tokio::test] +async fn decode_usage_venice_top_level_cost() { + let response = decode_response(serde_json::json!({ + "id": "chatcmpl_venice_test", + "object": "chat.completion", + "created": CREATED_TS, + "model": MODEL, + "choices": [{ + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop" + }], + "cost": {"usd": 0.00042, "diem": 0.0}, + "usage": { + "prompt_tokens": 12, + "completion_tokens": 2, + "total_tokens": 14 + } + })) + .await; + fabro_test::fabro_json_snapshot!(response); +} + // --------------------------------------------------------------------------- // Stream // --------------------------------------------------------------------------- @@ -841,6 +866,20 @@ async fn stream_usage_openrouter_cost() { fabro_test::fabro_json_snapshot!(events); } +/// Venice streams authoritative USD cost in a top-level object on the usage +/// chunk. +#[tokio::test] +async fn stream_usage_venice_top_level_cost() { + let sse = support::sse_data_transcript(&[ + r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{"role":"assistant","content":"Hi"},"finish_reason":null}]}"#, + r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}"#, + r#"{"id":"chatcmpl_venice_stream","object":"chat.completion.chunk","created":1700000000,"model":"test-model","choices":[],"cost":{"usd":0.00031,"diem":0.0},"usage":{"prompt_tokens":12,"completion_tokens":2,"total_tokens":14}}"#, + "[DONE]", + ]); + let (_capture, events) = stream_capture(&base_request(MODEL), &sse).await; + fabro_test::fabro_json_snapshot!(events); +} + #[tokio::test] async fn stream_tool_call_deltas() { let sse = support::sse_data_transcript(&[ diff --git a/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_usage_venice_top_level_cost.snap b/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_usage_venice_top_level_cost.snap new file mode 100644 index 000000000..7236fc7c8 --- /dev/null +++ b/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__decode_usage_venice_top_level_cost.snap @@ -0,0 +1,55 @@ +--- +source: lib/components/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +{ + "id": "chatcmpl_venice_test", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "ok" + } + ] + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 12, + "output_tokens": 2, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": { + "id": "chatcmpl_venice_test", + "object": "chat.completion", + "created": 1700000000, + "model": "test-model", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "ok" + }, + "finish_reason": "stop" + } + ], + "cost": { + "usd": 0.00042, + "diem": 0.0 + }, + "usage": { + "prompt_tokens": 12, + "completion_tokens": 2, + "total_tokens": 14 + } + }, + "warnings": [], + "rate_limit": null, + "cost_usd": 0.00042, + "cost_source": "authoritative" +} diff --git a/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_usage_venice_top_level_cost.snap b/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_usage_venice_top_level_cost.snap new file mode 100644 index 000000000..b2ac8b212 --- /dev/null +++ b/lib/components/fabro-llm/tests/it/wire/snapshots/it__wire__openai_compatible__stream_usage_venice_top_level_cost.snap @@ -0,0 +1,60 @@ +--- +source: lib/components/fabro-llm/tests/it/wire/openai_compatible.rs +expression: rendered +--- +[ + { + "type": "stream_start" + }, + { + "type": "text_start", + "text_id": null + }, + { + "type": "text_delta", + "delta": "Hi", + "text_id": null + }, + { + "type": "text_end", + "text_id": null + }, + { + "type": "finish", + "finish_reason": "stop", + "usage": { + "input_tokens": 12, + "output_tokens": 2, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "response": { + "id": "chatcmpl_venice_stream", + "model": "test-model", + "provider": "openai-compatible", + "message": { + "role": "assistant", + "content": [ + { + "kind": "text", + "data": "Hi" + } + ] + }, + "finish_reason": "stop", + "usage": { + "input_tokens": 12, + "output_tokens": 2, + "reasoning_tokens": 0, + "cache_read_tokens": 0, + "cache_write_tokens": 0 + }, + "raw": null, + "warnings": [], + "rate_limit": null, + "cost_usd": 0.00031, + "cost_source": "authoritative" + } + } +] From 47954f731e3fc55dd638b451cef7cbbb9e7c8ac1 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 16:31:48 -0400 Subject: [PATCH 21/30] refactor(github): deduplicate additional-repository access plumbing Consolidate the copies that review found across the feature: - One GITHUB_CREDENTIAL_HELPER / GITHUB_CREDENTIAL_HELPER_KEY pair in fabro-github, with apply_probe_git_env() for probe commands; the runtime git bridge, server preflight probe, and live contract test all consume it so the probes exercise exactly what the bridge configures. - GitHubRepositoryAccess::resolve_verified_token() owns the resolve-installations-then-mint choreography shared by server preflight, workflow initialization, and the live test. - A shared lookup_installation() helper backs both the shared-installation resolution and the mint's installation lookup. - The contents = read|write rule lives once as RunIntegrationsGithubSettings::contents_permission_allows_repository_access. - The preflight probe paces retries with fabro-sandbox's exported replication_backoff() (3s/9s) instead of a contradicting 1s/2s loop, and shares one run_ls_remote() runner with the existing remote-ref check. Also: collapse the dead Ok(None) arm and repeated error blocks in the preflight token check, drop the derivable bridge_entry_count(), privatize resolve_permissions() behind resolve_integration(), make GitHubRepositorySlug ordering/hashing allocation-free, and use EnvVars constants for env names. Co-Authored-By: Claude Fable 5 --- lib/apps/fabro-server/src/run_manifest.rs | 220 ++++++++---------- lib/components/fabro-github/src/access.rs | 76 +++--- lib/components/fabro-github/src/lib.rs | 120 +++++++--- .../fabro-github/tests/live_access.rs | 42 ++-- lib/components/fabro-sandbox/src/git_retry.rs | 13 +- lib/components/fabro-sandbox/src/lib.rs | 4 +- .../fabro-workflow/src/git_bridge.rs | 26 +-- .../fabro-workflow/src/pipeline/initialize.rs | 40 ++-- .../fabro-config/src/resolve/run.rs | 2 +- lib/foundation/fabro-types/src/repository.rs | 25 +- .../fabro-types/src/settings/run.rs | 12 +- 11 files changed, 304 insertions(+), 276 deletions(-) diff --git a/lib/apps/fabro-server/src/run_manifest.rs b/lib/apps/fabro-server/src/run_manifest.rs index 44ec4fd0c..7bc97f646 100644 --- a/lib/apps/fabro-server/src/run_manifest.rs +++ b/lib/apps/fabro-server/src/run_manifest.rs @@ -873,6 +873,15 @@ async fn check_git_remote_ref( command.arg(branch); } + run_ls_remote(command) + .await + .map_err(|message| redact_auth_url(&message, auth_url.as_ref())) +} + +/// Run a prepared `git ls-remote` invocation with a 10s timeout, reducing a +/// failure to its most useful message: stderr, then stdout, then the exit +/// status. +async fn run_ls_remote(mut command: Command) -> std::result::Result<(), String> { let output = time::timeout(Duration::from_secs(10), command.output()) .await .map_err(|_| "git ls-remote timed out after 10s".to_string())? @@ -884,14 +893,13 @@ async fn check_git_remote_ref( let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); - let message = if !stderr.is_empty() { + Err(if !stderr.is_empty() { stderr } else if !stdout.is_empty() { stdout } else { format!("git ls-remote exited with status {}", output.status) - }; - Err(redact_auth_url(&message, auth_url.as_ref())) + }) } fn preflight_sandbox_spec( @@ -1236,14 +1244,12 @@ where let integration = match github.resolve_integration() { Ok(integration) => integration, Err(err) => { - checks.push(CheckResult { - name: "GitHub Token".into(), - status: CheckStatus::Error, - summary: "invalid permissions".into(), - details: vec![], - remediation: Some(format!("Failed to resolve GitHub permissions: {err}")), - }); - return false; + return fail_github_token_check( + checks, + &[], + "invalid permissions", + format!("Failed to resolve GitHub permissions: {err}"), + ); } }; let perm_details = integration @@ -1269,7 +1275,7 @@ where // `gh` checks GH_TOKEN before GITHUB_TOKEN, so a user-defined GH_TOKEN // bypasses the managed scoped token for gh commands. Warn without // failing; the value is the workflow author's responsibility. - if resolved_run.environment.env.contains_key("GH_TOKEN") { + if resolved_run.environment.env.contains_key(EnvVars::GH_TOKEN) { checks.push(CheckResult { name: "GH_TOKEN Override".into(), status: CheckStatus::Warning, @@ -1284,65 +1290,44 @@ where }); } - let Some(git) = prepared.git.as_ref() else { - checks.push(CheckResult { - name: "GitHub Token".into(), - status: CheckStatus::Error, - summary: "missing origin".into(), - details: perm_details, - remediation: Some( - "run.integrations.github.additional_repositories requires a GitHub run origin, \ - but this run has no repository origin URL" - .to_string(), - ), - }); - return false; - }; let Some(creds) = github_app else { - checks.push(CheckResult { - name: "GitHub Token".into(), - status: CheckStatus::Error, - summary: "missing credentials".into(), - details: perm_details, - remediation: Some( - "run.integrations.github.additional_repositories requires GitHub credentials, \ - but none are configured on the server" - .to_string(), - ), - }); - return false; + return fail_github_token_check( + checks, + &perm_details, + "missing credentials", + "run.integrations.github.additional_repositories requires GitHub credentials, but \ + none are configured on the server" + .to_string(), + ); }; // The same validated access value runtime initialization constructs, so - // preflight and runtime cannot disagree about the effective set. + // preflight and runtime cannot disagree about the effective set. A + // missing origin fails inside `new` with the canonical remediation. let access = match fabro_github::GitHubRepositoryAccess::new( - Some(&git.origin_url), + prepared.git.as_ref().map(|git| git.origin_url.as_str()), &integration.additional_repositories, integration.permissions.clone(), ) { Ok(Some(access)) => access, + // `new` returns `Ok(None)` only when nothing is declared, and the + // declared set is non-empty here. Fail closed instead of panicking. Ok(None) => { - checks.push(CheckResult { - name: "GitHub Token".into(), - status: CheckStatus::Error, - summary: "missing origin".into(), - details: perm_details, - remediation: Some( - "run.integrations.github.additional_repositories requires a GitHub run \ - origin, but the origin URL is empty" - .to_string(), - ), - }); - return false; + return fail_github_token_check( + checks, + &perm_details, + "missing origin", + "run.integrations.github.additional_repositories requires a GitHub run origin, \ + but this run has no repository origin URL" + .to_string(), + ); } Err(err) => { - checks.push(CheckResult { - name: "GitHub Token".into(), - status: CheckStatus::Error, - summary: "invalid repository set".into(), - details: perm_details, - remediation: Some(format!("{err:#}")), - }); - return false; + return fail_github_token_check( + checks, + &perm_details, + "invalid repository set", + format!("{err:#}"), + ); } }; @@ -1352,14 +1337,7 @@ where let token = match mint_scoped_token(access.clone(), creds).await { Ok(token) => token, Err(err) => { - checks.push(CheckResult { - name: "GitHub Token".into(), - status: CheckStatus::Error, - summary: "failed".into(), - details: perm_details, - remediation: Some(err), - }); - return false; + return fail_github_token_check(checks, &perm_details, "failed", err); } }; checks.push(CheckResult { @@ -1415,12 +1393,29 @@ where ok } +/// Report one "GitHub Token" preflight failure and fail the check. +fn fail_github_token_check( + checks: &mut Vec, + perm_details: &[CheckDetail], + summary: &str, + remediation: String, +) -> bool { + checks.push(CheckResult { + name: "GitHub Token".into(), + status: CheckStatus::Error, + summary: summary.into(), + details: perm_details.to_vec(), + remediation: Some(remediation), + }); + false +} + /// Bounded concurrency for per-repository `git ls-remote` probes. const REPOSITORY_PROBE_CONCURRENCY: usize = 4; /// Total probe attempts per repository when failures classify as retryable /// (token replication lag or transient infrastructure). -const REPOSITORY_PROBE_ATTEMPTS: u64 = 3; +const REPOSITORY_PROBE_ATTEMPTS: u32 = 3; async fn check_primary_only_github_token( checks: &mut Vec, @@ -1464,34 +1459,29 @@ async fn check_primary_only_github_token( } } -/// Production minter for the multi-repository path: resolve every -/// repository's installation in App mode (naming any repository the App -/// cannot see, or one on a different installation), then mint the single -/// scoped token. A mint rejection after a clean resolution check surfaces -/// the raw error. +/// Production minter for the multi-repository path: +/// [`fabro_github::GitHubRepositoryAccess::resolve_verified_token`] over a +/// source scoped to the effective set. In App mode that resolves every +/// repository's installation first, so a failure names the repository the +/// App cannot see (or one on a different installation). async fn mint_scoped_github_token( access: fabro_github::GitHubRepositoryAccess, creds: fabro_github::GitHubCredentials, ) -> std::result::Result { - if let fabro_github::GitHubCredentials::App(app) = &creds { - access - .resolve_shared_installation_via_api(app) - .await - .map_err(|err| format!("{err:#}"))?; - } let source = InstallationTokenSource::for_access(&creds, &access).map_err(|err| format!("{err:#}"))?; - source - .resolve() + access + .resolve_verified_token(&creds, &source) .await - .map_err(|err| format!("Failed to mint GitHub token: {err:#}")) + .map_err(|err| format!("{err:#}")) } /// Production per-repository probe: a non-interactive -/// `git ls-remote HEAD` authenticated through a credential -/// helper that reads `GITHUB_TOKEN` from the child process environment, so -/// the token never appears in the URL, argv, or rendered errors. Mirrors the -/// runtime `git_bridge` credential helper in `fabro-workflow`. +/// `git ls-remote HEAD` authenticated through +/// [`fabro_github::GITHUB_CREDENTIAL_HELPER`] reading `GITHUB_TOKEN` from +/// the child process environment, so the token never appears in the URL, +/// argv, or rendered errors — exactly what the runtime `git_bridge` +/// configures in `fabro-workflow`. async fn probe_github_repository( slug: fabro_types::GitHubRepositorySlug, token: ResolvedToken, @@ -1502,8 +1492,8 @@ async fn probe_github_repository( /// Retry auth-shaped failures with the SAME token: replication of a given /// token only makes progress, while re-minting would restart the replication -/// clock. Classification matches the sandbox git retry policy -/// (`fabro_sandbox::classify_failure`). +/// clock. Classification and pacing match the sandbox git retry policy +/// (`fabro_sandbox::classify_failure` / `fabro_sandbox::replication_backoff`). async fn probe_with_replication_retry( snapshot: TokenSnapshot, run: F, @@ -1513,7 +1503,8 @@ where Fut: Future>, { let credential_context = fabro_sandbox::CredentialContext::from_snapshot(Some(&snapshot)); - let mut attempt = 0u64; + let backoff = fabro_sandbox::replication_backoff(); + let mut attempt = 0u32; loop { attempt += 1; let Err(message) = run().await else { @@ -1524,44 +1515,15 @@ where { return Err(message); } - time::sleep(Duration::from_secs(attempt)).await; + time::sleep(backoff.delay_for_attempt(attempt)).await; } } async fn run_probe_ls_remote(url: &str, token: &ResolvedToken) -> std::result::Result<(), String> { let mut command = Command::new("git"); - command - .env("GIT_TERMINAL_PROMPT", "0") - .env("GIT_CONFIG_NOSYSTEM", "1") - .env("GIT_CONFIG_GLOBAL", "/dev/null") - .env("GITHUB_TOKEN", token.token.expose()) - .env("GIT_CONFIG_COUNT", "1") - .env("GIT_CONFIG_KEY_0", "credential.https://github.com.helper") - .env( - "GIT_CONFIG_VALUE_0", - r#"!f() { if [ "$1" = get ]; then echo username=x-access-token; echo "password=$GITHUB_TOKEN"; fi; }; f"#, - ) - .args(["ls-remote", url, "HEAD"]); - - let output = time::timeout(Duration::from_secs(10), command.output()) - .await - .map_err(|_| "git ls-remote timed out after 10s".to_string())? - .map_err(|err| format!("Failed to run git ls-remote: {err}"))?; - if output.status.success() { - return Ok(()); - } - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_string(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string(); - if !stderr.is_empty() { - Err(stderr) - } else if !stdout.is_empty() { - Err(stdout) - } else { - Err(format!( - "git ls-remote exited with status {}", - output.status - )) - } + fabro_github::apply_probe_git_env(&mut command, token.token.expose()); + command.args(["ls-remote", url, "HEAD"]); + run_ls_remote(command).await } async fn mint_github_token( @@ -3531,7 +3493,17 @@ dockerfile = { path = "Dockerfile" } .await; assert!(!ok); - assert_eq!(checks.last().unwrap().summary, "missing origin"); + let check = checks.last().unwrap(); + assert_eq!(check.summary, "invalid repository set"); + assert!( + check + .remediation + .as_deref() + .unwrap_or_default() + .contains("requires a GitHub run origin"), + "{:?}", + check.remediation + ); } #[tokio::test] diff --git a/lib/components/fabro-github/src/access.rs b/lib/components/fabro-github/src/access.rs index b71fd0c64..6f3c2d02e 100644 --- a/lib/components/fabro-github/src/access.rs +++ b/lib/components/fabro-github/src/access.rs @@ -10,8 +10,10 @@ use std::collections::{BTreeSet, HashMap}; use anyhow::{Context as _, bail}; use fabro_types::GitHubRepositorySlug; +use fabro_types::settings::run::RunIntegrationsGithubSettings; -use crate::{GitHubAppCredentials, HttpClient, HttpMethod}; +use crate::token_source::{InstallationTokenSource, ResolvedToken}; +use crate::{GitHubAppCredentials, GitHubCredentials, HttpClient, InstallationLookup}; /// The validated effective repository set for a run: the primary origin /// repository plus zero or more distinct additional repositories, all with @@ -163,48 +165,31 @@ impl GitHubRepositoryAccess { client: &impl HttpClient, base_url: &str, ) -> anyhow::Result { - #[derive(serde::Deserialize)] - struct Installation { - id: u64, - } - let jwt = crate::sign_app_jwt(&creds.app_id, &creds.private_key_pem)?; - let auth = format!("Bearer {jwt}"); let mut shared: Option<(u64, &GitHubRepositorySlug)> = None; for slug in self.targets() { - let endpoint = format!( - "{base_url}/repos/{}/{}/installation", - slug.owner(), - slug.repo() - ); - let response = client - .request( - HttpMethod::Get, - &endpoint, - &crate::github_headers(&auth), - None, - ) - .await - .with_context(|| format!("looking up the GitHub App installation for {slug}"))?; - match response.status { - 200 => {} - 404 => bail!( + let lookup = + crate::lookup_installation(client, &jwt, base_url, slug.owner(), slug.repo()) + .await + .with_context(|| { + format!("looking up the GitHub App installation for {slug}") + })?; + let id = match lookup { + InstallationLookup::Found(id) => id, + InstallationLookup::NotFound => bail!( "the GitHub App installation cannot see repository {slug}; add it to the \ installation's repository access" ), - status => bail!( + InstallationLookup::Failed(status) => bail!( "unexpected status {status} looking up the GitHub App installation for {slug}" ), - } - let installation: Installation = response - .json() - .with_context(|| format!("parsing the installation response for {slug}"))?; + }; match shared { - None => shared = Some((installation.id, slug)), - Some((id, first)) if id != installation.id => bail!( - "repository {slug} belongs to GitHub App installation {} but {first} belongs \ - to installation {id}; all repositories must share one installation", - installation.id + None => shared = Some((id, slug)), + Some((shared_id, first)) if shared_id != id => bail!( + "repository {slug} belongs to GitHub App installation {id} but {first} \ + belongs to installation {shared_id}; all repositories must share one \ + installation" ), Some(_) => {} } @@ -212,6 +197,27 @@ impl GitHubRepositoryAccess { let (id, _) = shared.expect("the effective repository set always contains the primary"); Ok(id) } + + /// Prove this access value is usable and produce its token: in App mode, + /// first resolve every target's installation + /// ([`Self::resolve_shared_installation_via_api`]) so a failure names the + /// repository the App cannot see, then resolve `source` once — for App + /// credentials that eagerly mints the token scoped to the whole effective + /// set. The one choreography server preflight and workflow + /// initialization share. + pub async fn resolve_verified_token( + &self, + creds: &GitHubCredentials, + source: &InstallationTokenSource, + ) -> anyhow::Result { + if let GitHubCredentials::App(app) = creds { + self.resolve_shared_installation_via_api(app).await?; + } + source + .resolve() + .await + .context("Failed to resolve the GitHub token for the effective repository set") + } } /// A non-empty additional set needs a token that can reach repository @@ -224,7 +230,7 @@ fn validate_additional_permissions(permissions: &HashMap) -> any (`read` or `write`)" ); }; - if contents != "read" && contents != "write" { + if !RunIntegrationsGithubSettings::contents_permission_allows_repository_access(contents) { bail!( "run.integrations.github.additional_repositories requires `contents = \"read\"` or \ `contents = \"write\"`, got `{contents}`" diff --git a/lib/components/fabro-github/src/lib.rs b/lib/components/fabro-github/src/lib.rs index 71d586213..94ba085a7 100644 --- a/lib/components/fabro-github/src/lib.rs +++ b/lib/components/fabro-github/src/lib.rs @@ -21,6 +21,33 @@ pub use access::GitHubRepositoryAccess; pub const GITHUB_API_BASE_URL: &str = "https://api.github.com"; +/// Git config key that routes github.com HTTPS credentials through +/// [`GITHUB_CREDENTIAL_HELPER`]. +pub const GITHUB_CREDENTIAL_HELPER_KEY: &str = "credential.https://github.com.helper"; + +/// Secret-free git credential helper: reads `$GITHUB_TOKEN` from the +/// invoking git process's environment at invocation time, so the token never +/// lands in git configuration, argv, or rendered errors. Non-`get` +/// operations (`store`, `erase`) are ignored. The one definition shared by +/// the runtime git bridge, server preflight probes, and the live contract +/// test, so the probes always exercise exactly what the bridge configures. +pub const GITHUB_CREDENTIAL_HELPER: &str = r#"!f() { if [ "$1" = get ]; then echo username=x-access-token; echo "password=$GITHUB_TOKEN"; fi; }; f"#; + +/// Configure a `git` invocation to authenticate to github.com through +/// [`GITHUB_CREDENTIAL_HELPER`] with `token`, isolated from user/system git +/// configuration and terminal prompts. The token is passed only through the +/// child process environment. +pub fn apply_probe_git_env(command: &mut Command, token: &str) { + command + .env("GIT_TERMINAL_PROMPT", "0") + .env("GIT_CONFIG_NOSYSTEM", "1") + .env("GIT_CONFIG_GLOBAL", "/dev/null") + .env(EnvVars::GITHUB_TOKEN, token) + .env("GIT_CONFIG_COUNT", "1") + .env("GIT_CONFIG_KEY_0", GITHUB_CREDENTIAL_HELPER_KEY) + .env("GIT_CONFIG_VALUE_0", GITHUB_CREDENTIAL_HELPER); +} + /// Returns the GitHub API base URL, allowing override via `GITHUB_BASE_URL` env /// var. #[expect( @@ -516,6 +543,52 @@ pub async fn create_installation_access_token_with_permissions_and_install_url( .map(|token| token.token) } +/// Outcome of one GitHub App installation lookup +/// (`GET /repos/{owner}/{repo}/installation`). +/// +/// Status interpretation stays with callers because their user guidance +/// differs: the mint path speaks about the owner's App installation, the +/// multi-repository resolution names the specific repository. +pub(crate) enum InstallationLookup { + Found(u64), + /// 404: the App cannot see the repository (or is not installed at all). + NotFound, + /// Any other non-200 status. + Failed(u16), +} + +/// Look up the App installation covering `owner/repo` and parse its id. +/// Transport and parse failures carry no caller-specific context; callers +/// attach their own. +pub(crate) async fn lookup_installation( + client: &impl HttpClient, + jwt: &str, + base_url: &str, + owner: &str, + repo: &str, +) -> anyhow::Result { + #[derive(Deserialize)] + struct Installation { + id: u64, + } + + let endpoint = format!("{base_url}/repos/{owner}/{repo}/installation"); + let auth = format!("Bearer {jwt}"); + let resp = client + .request(HttpMethod::Get, &endpoint, &github_headers(&auth), None) + .await?; + match resp.status { + 200 => { + let installation: Installation = resp + .json() + .context("Failed to parse installation response")?; + Ok(InstallationLookup::Found(installation.id)) + } + 404 => Ok(InstallationLookup::NotFound), + status => Ok(InstallationLookup::Failed(status)), + } +} + async fn mint_installation_token_with_jwt( client: &impl HttpClient, jwt: &str, @@ -525,11 +598,6 @@ async fn mint_installation_token_with_jwt( permissions: serde_json::Value, install_url: Option<&str>, ) -> anyhow::Result { - #[derive(Deserialize)] - struct Installation { - id: u64, - } - #[derive(Deserialize)] struct AccessToken { token: String, @@ -544,21 +612,12 @@ async fn mint_installation_token_with_jwt( // repository callers resolve every repository's installation up front // (`GitHubRepositoryAccess::resolve_shared_installation`), so the // primary stands for the whole set here. - let installation_endpoint = format!("{base_url}/repos/{owner}/{primary_repo}/installation"); - let auth = format!("Bearer {jwt}"); - let resp = client - .request( - HttpMethod::Get, - &installation_endpoint, - &github_headers(&auth), - None, - ) + let installation_id = match lookup_installation(client, jwt, base_url, owner, primary_repo) .await - .context("Failed to look up GitHub App installation")?; - - match resp.status { - 200 => {} - 404 => { + .context("Failed to look up GitHub App installation")? + { + InstallationLookup::Found(id) => id, + InstallationLookup::NotFound => { let install_url = install_url.map_or_else( || format!("https://github.com/organizations/{owner}/settings/installations"), str::to_string, @@ -568,35 +627,26 @@ async fn mint_installation_token_with_jwt( Install it at {install_url}" ); } - 403 => { + InstallationLookup::Failed(403) => { bail!( "GitHub App installation is suspended. \ Re-enable it in your organization's GitHub App settings." ); } - 401 => { + InstallationLookup::Failed(401) => { bail!( "GitHub App authentication failed. \ Check that app_id and GITHUB_APP_PRIVATE_KEY are correct." ); } - _ => { - bail!( - "Unexpected status {} looking up GitHub App installation", - resp.status - ); + InstallationLookup::Failed(status) => { + bail!("Unexpected status {status} looking up GitHub App installation"); } - } - - let installation: Installation = resp - .json() - .context("Failed to parse installation response")?; + }; // Step 2: Create a scoped access token - let token_url = format!( - "{base_url}/app/installations/{}/access_tokens", - installation.id - ); + let auth = format!("Bearer {jwt}"); + let token_url = format!("{base_url}/app/installations/{installation_id}/access_tokens"); let body = serde_json::json!({ "repositories": repos, "permissions": permissions, diff --git a/lib/components/fabro-github/tests/live_access.rs b/lib/components/fabro-github/tests/live_access.rs index 2440b6547..d31dae934 100644 --- a/lib/components/fabro-github/tests/live_access.rs +++ b/lib/components/fabro-github/tests/live_access.rs @@ -19,7 +19,6 @@ use std::collections::BTreeSet; use std::process::Stdio; use std::time::Duration; -use base64::engine::general_purpose::STANDARD; use fabro_github::token_source::InstallationTokenSource; use fabro_github::{GitHubAppCredentials, GitHubCredentials, GitHubRepositoryAccess}; use fabro_types::GitHubRepositorySlug; @@ -35,28 +34,16 @@ fn env_var(name: &str) -> String { } fn private_key_pem() -> String { - let raw = env_var("GITHUB_APP_PRIVATE_KEY"); - if raw.starts_with("-----") { - return raw; - } - let bytes = base64::Engine::decode(&STANDARD, &raw) - .expect("GITHUB_APP_PRIVATE_KEY is not valid base64"); - String::from_utf8(bytes).expect("GITHUB_APP_PRIVATE_KEY decoded to invalid UTF-8") + GitHubAppCredentials::private_key_from_env() + .expect("GITHUB_APP_PRIVATE_KEY should decode as PEM or base64 PEM") + .expect("GITHUB_APP_PRIVATE_KEY must be set for this live test") } async fn ls_remote_with_token(slug: &GitHubRepositorySlug, token: &str) -> bool { let url = format!("https://github.com/{}/{}", slug.owner(), slug.repo()); - let output = Command::new("git") - .env("GIT_TERMINAL_PROMPT", "0") - .env("GIT_CONFIG_NOSYSTEM", "1") - .env("GIT_CONFIG_GLOBAL", "/dev/null") - .env("GITHUB_TOKEN", token) - .env("GIT_CONFIG_COUNT", "1") - .env("GIT_CONFIG_KEY_0", "credential.https://github.com.helper") - .env( - "GIT_CONFIG_VALUE_0", - r#"!f() { if [ "$1" = get ]; then echo username=x-access-token; echo "password=$GITHUB_TOKEN"; fi; }; f"#, - ) + let mut command = Command::new("git"); + fabro_github::apply_probe_git_env(&mut command, token); + let output = command .args(["ls-remote", &url, "HEAD"]) .stdout(Stdio::null()) .stderr(Stdio::null()) @@ -92,16 +79,15 @@ async fn scoped_token_reaches_the_declared_additional_repository() { .expect("access request should validate") .expect("origin should produce an access value"); - // Every target resolves to one shared installation. - access - .resolve_shared_installation_via_api(&app) + // The production choreography: every target resolves to one shared + // installation, then one mint scoped to the whole effective set. + let creds = GitHubCredentials::App(app.clone()); + let source = + InstallationTokenSource::for_access(&creds, &access).expect("token source should build"); + let resolved = access + .resolve_verified_token(&creds, &source) .await - .expect("all repositories should share one App installation"); - - // One mint scoped to the whole effective set. - let source = InstallationTokenSource::for_access(&GitHubCredentials::App(app.clone()), &access) - .expect("token source should build"); - let resolved = source.resolve().await.expect("scoped mint should succeed"); + .expect("scoped mint should succeed"); let token = resolved.token.expose(); // The one token reads both the primary and the additional repository. diff --git a/lib/components/fabro-sandbox/src/git_retry.rs b/lib/components/fabro-sandbox/src/git_retry.rs index 7b16cc256..c0954bd6b 100644 --- a/lib/components/fabro-sandbox/src/git_retry.rs +++ b/lib/components/fabro-sandbox/src/git_retry.rs @@ -198,8 +198,11 @@ pub fn classify_failure(message: &str, cred: CredentialContext) -> Option BackoffPolicy { +/// window and spend an attempt for nothing. Exported so other retried git +/// operations against a freshly minted token (e.g. server preflight probes) +/// pace themselves by the same policy. +#[must_use] +pub fn replication_backoff() -> BackoffPolicy { BackoffPolicy { initial_delay: Duration::from_secs(3), factor: 3.0, @@ -236,7 +239,7 @@ impl RetryPlan { pub fn clone_default(outer_deadline: Option) -> Self { Self { max_attempts: 3, - backoff: clone_backoff(), + backoff: replication_backoff(), max_elapsed: None, per_attempt_timeout: None, outer_deadline, @@ -249,7 +252,7 @@ impl RetryPlan { pub fn checkpoint_push() -> Self { Self { max_attempts: 3, - backoff: clone_backoff(), + backoff: replication_backoff(), max_elapsed: Some(Duration::from_secs(90)), per_attempt_timeout: Some(Duration::from_mins(1)), outer_deadline: None, @@ -736,7 +739,7 @@ mod tests { let attempts = Attempts::default(); let plan = RetryPlan { max_attempts: 5, - backoff: clone_backoff(), + backoff: replication_backoff(), max_elapsed: Some(Duration::from_secs(4)), per_attempt_timeout: None, outer_deadline: None, diff --git a/lib/components/fabro-sandbox/src/lib.rs b/lib/components/fabro-sandbox/src/lib.rs index fb442e9fb..4803c3fe8 100644 --- a/lib/components/fabro-sandbox/src/lib.rs +++ b/lib/components/fabro-sandbox/src/lib.rs @@ -43,7 +43,9 @@ pub use fabro_github::token_source::{ InstallationTokenSource, ResolvedToken, TokenProvenance, TokenSnapshot, }; pub use fabro_types::{RunSandboxInstance, SandboxProviderKind}; -pub use git_retry::{CredentialContext, GitRetryReason, RetryPlan, classify_failure}; +pub use git_retry::{ + CredentialContext, GitRetryReason, RetryPlan, classify_failure, replication_backoff, +}; pub use local::LocalSandbox; #[cfg(feature = "daytona")] pub use provider::daytona::DaytonaSandboxProvider; diff --git a/lib/components/fabro-workflow/src/git_bridge.rs b/lib/components/fabro-workflow/src/git_bridge.rs index df9c7e300..4f822811a 100644 --- a/lib/components/fabro-workflow/src/git_bridge.rs +++ b/lib/components/fabro-workflow/src/git_bridge.rs @@ -35,6 +35,7 @@ use std::collections::HashMap; +use fabro_github::{GITHUB_CREDENTIAL_HELPER, GITHUB_CREDENTIAL_HELPER_KEY}; use fabro_types::GitHubRepositorySlug; use crate::error::Error; @@ -42,11 +43,6 @@ use crate::error::Error; /// Section base for the effective repositories' HTTPS routes. const GITHUB_HTTPS_BASE: &str = "https://github.com/"; -const CREDENTIAL_HELPER_KEY: &str = "credential.https://github.com.helper"; -/// Reads the invoking process's `$GITHUB_TOKEN` at invocation time; contains -/// no secret itself. Non-`get` operations (`store`, `erase`) are ignored. -const CREDENTIAL_HELPER: &str = r#"!f() { if [ "$1" = get ]; then echo username=x-access-token; echo "password=$GITHUB_TOKEN"; fi; }; f"#; - /// Merge the bridging entries into `env` for the effective repository set /// (primary first). Appends after any valid user-provided `GIT_CONFIG_COUNT` /// overlay without overwriting it, and fails with a configuration error when @@ -56,15 +52,13 @@ pub(crate) fn merge_git_bridge_env( targets: &[&GitHubRepositorySlug], ) -> Result<(), Error> { let start = user_git_config_count(env)?; - for (offset, (key, value)) in bridge_entries(targets, GITHUB_HTTPS_BASE) - .into_iter() - .enumerate() - { + let entries = bridge_entries(targets, GITHUB_HTTPS_BASE); + let total = start + entries.len(); + for (offset, (key, value)) in entries.into_iter().enumerate() { let index = start + offset; env.insert(format!("GIT_CONFIG_KEY_{index}"), key); env.insert(format!("GIT_CONFIG_VALUE_{index}"), value); } - let total = start + bridge_entry_count(targets); env.insert("GIT_CONFIG_COUNT".to_string(), total.to_string()); // Fail instead of hanging when access is missing or invalid; a user who // explicitly configured prompting keeps their value. @@ -73,20 +67,16 @@ pub(crate) fn merge_git_bridge_env( Ok(()) } -fn bridge_entry_count(targets: &[&GitHubRepositorySlug]) -> usize { - 1 + targets.len() * 2 -} - /// The bridge's Git config entries in order: the credential helper, then two /// SSH-to-HTTPS rewrites per repository. `https_base` is /// [`GITHUB_HTTPS_BASE`] in production; contract tests substitute a local /// `file://` root to prove real Git applies the generated entries without /// touching the network. fn bridge_entries(targets: &[&GitHubRepositorySlug], https_base: &str) -> Vec<(String, String)> { - let mut entries = Vec::with_capacity(bridge_entry_count(targets)); + let mut entries = Vec::with_capacity(1 + targets.len() * 2); entries.push(( - CREDENTIAL_HELPER_KEY.to_string(), - CREDENTIAL_HELPER.to_string(), + GITHUB_CREDENTIAL_HELPER_KEY.to_string(), + GITHUB_CREDENTIAL_HELPER.to_string(), )); for slug in targets { let owner = slug.owner(); @@ -195,7 +185,7 @@ mod tests { fn no_targets_means_no_bridge_call_and_empty_env_stays_empty() { // The caller only bridges when the additional set is non-empty; the // pure entry builder is still total for the primary-only case. - assert_eq!(bridge_entry_count(&[]), 1); + assert_eq!(bridge_entries(&[], GITHUB_HTTPS_BASE).len(), 1); let env: HashMap = HashMap::new(); assert!(!env.contains_key("GIT_CONFIG_COUNT")); } diff --git a/lib/components/fabro-workflow/src/pipeline/initialize.rs b/lib/components/fabro-workflow/src/pipeline/initialize.rs index cde82cacf..809eccdd8 100644 --- a/lib/components/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/components/fabro-workflow/src/pipeline/initialize.rs @@ -175,10 +175,10 @@ fn build_sandbox_env( } /// When additional repositories are declared, prove the whole effective set -/// is reachable before the first workflow stage: resolve every repository's -/// App installation (naming any repository the App cannot see), then resolve -/// the token once eagerly. Legacy permissions-only runs skip this and keep -/// their best-effort behavior. +/// is reachable before the first workflow stage +/// ([`fabro_github::GitHubRepositoryAccess::resolve_verified_token`]). +/// Legacy permissions-only runs skip this and keep their best-effort +/// behavior. async fn validate_declared_repository_access( built: &BuiltSandboxEnv, github_app: Option<&fabro_github::GitHubCredentials>, @@ -190,25 +190,24 @@ async fn validate_declared_repository_access( else { return Ok(()); }; - if let Some(fabro_github::GitHubCredentials::App(app)) = github_app { - access - .resolve_shared_installation_via_api(app) - .await - .map_err(|err| { - Error::engine_with_anyhow( - "Declared additional GitHub repository is not accessible", - err, - ) - })?; - } - if let Some(source) = built.github_token.as_ref() { - source.resolve().await.map_err(|err| { + // `build_sandbox_env` guarantees credentials and a token source whenever + // additional repositories are declared; fail closed if that ever breaks. + let (Some(creds), Some(source)) = (github_app, built.github_token.as_ref()) else { + return Err(Error::Precondition( + "run.integrations.github.additional_repositories requires GitHub credentials, but \ + none are configured" + .to_string(), + )); + }; + access + .resolve_verified_token(creds, source) + .await + .map_err(|err| { Error::engine_with_anyhow( - "Failed to resolve GitHub access for the declared repository set", + "Failed to verify GitHub access for the declared repository set", err, ) })?; - } Ok(()) } @@ -1759,7 +1758,8 @@ mod tests { github_access: access, }; - let err = validate_declared_repository_access(&built, None) + let creds = GitHubCredentials::Pat("ghp_x".to_string()); + let err = validate_declared_repository_access(&built, Some(&creds)) .await .unwrap_err(); let message = err.to_string(); diff --git a/lib/foundation/fabro-config/src/resolve/run.rs b/lib/foundation/fabro-config/src/resolve/run.rs index 3fea93ab4..6ec601af7 100644 --- a/lib/foundation/fabro-config/src/resolve/run.rs +++ b/lib/foundation/fabro-config/src/resolve/run.rs @@ -209,7 +209,7 @@ fn validate_additional_repository_permissions( return; }; if let Ok(literal) = contents.resolve_with(&mut ResolveCtx::new()) { - if literal != "read" && literal != "write" { + if !RunIntegrationsGithubSettings::contents_permission_allows_repository_access(&literal) { errors.push(ResolveError::Invalid { path: "run.integrations.github.permissions.contents".to_string(), reason: format!( diff --git a/lib/foundation/fabro-types/src/repository.rs b/lib/foundation/fabro-types/src/repository.rs index 9c98d1d88..e7b29cef2 100644 --- a/lib/foundation/fabro-types/src/repository.rs +++ b/lib/foundation/fabro-types/src/repository.rs @@ -72,13 +72,12 @@ impl GitHubRepositorySlug { pub fn same_owner(&self, other: &Self) -> bool { self.owner.eq_ignore_ascii_case(&other.owner) } +} - fn canonical_key(&self) -> (String, String) { - ( - self.owner.to_ascii_lowercase(), - self.repo.to_ascii_lowercase(), - ) - } +/// Case-folded bytes for identity comparisons without allocating; owner and +/// repository names are validated ASCII, so ASCII folding is exact. +fn folded_bytes(value: &str) -> impl Iterator + '_ { + value.bytes().map(|byte| byte.to_ascii_lowercase()) } impl PartialEq for GitHubRepositorySlug { @@ -97,13 +96,23 @@ impl PartialOrd for GitHubRepositorySlug { impl Ord for GitHubRepositorySlug { fn cmp(&self, other: &Self) -> Ordering { - self.canonical_key().cmp(&other.canonical_key()) + folded_bytes(&self.owner) + .cmp(folded_bytes(&other.owner)) + .then_with(|| folded_bytes(&self.repo).cmp(folded_bytes(&other.repo))) } } impl Hash for GitHubRepositorySlug { fn hash(&self, state: &mut H) { - self.canonical_key().hash(state); + for byte in folded_bytes(&self.owner) { + state.write_u8(byte); + } + // `/` cannot appear in a validated owner, so the folded + // `owner/repo` encoding stays unambiguous. + state.write_u8(b'/'); + for byte in folded_bytes(&self.repo) { + state.write_u8(byte); + } } } diff --git a/lib/foundation/fabro-types/src/settings/run.rs b/lib/foundation/fabro-types/src/settings/run.rs index 12cd04fce..a763b7c4e 100644 --- a/lib/foundation/fabro-types/src/settings/run.rs +++ b/lib/foundation/fabro-types/src/settings/run.rs @@ -631,11 +631,21 @@ impl RunIntegrationsGithubSettings { !self.additional_repositories.is_empty() } + /// Whether a resolved `contents` permission level lets the token reach + /// repository contents — the level a declared `additional_repositories` + /// set requires. The one definition shared by config-time validation and + /// the runtime re-check after interpolation, so the accepted levels + /// cannot drift between the two layers. + #[must_use] + pub fn contents_permission_allows_repository_access(value: &str) -> bool { + value == "read" || value == "write" + } + /// Resolve every `permissions` value. `{{ vars.* }}` is substituted /// server-side at run creation, so values are literal by this point; a /// still-unresolved token fails closed rather than reaching the GitHub API /// as literal text. - pub fn resolve_permissions(&self) -> Result, ResolveError> { + fn resolve_permissions(&self) -> Result, ResolveError> { let mut ctx = ResolveCtx::new(); self.permissions .iter() From 438bab29f08f8d15120ee5eea7a99b5eabf1f323 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 17:11:24 -0400 Subject: [PATCH 22/30] feat: support shallow sandbox clones --- Cargo.lock | 76 +++++++++++++++---- Cargo.toml | 4 +- docs/public/api-reference/fabro-api.yaml | 4 + docs/public/execution/environments.mdx | 2 +- docs/public/execution/run-configuration.mdx | 6 ++ docs/public/integrations/daytona.mdx | 9 +++ lib/apps/fabro-server/src/run_manifest.rs | 4 +- lib/components/fabro-sandbox/src/config.rs | 1 + .../fabro-sandbox/src/daytona/mod.rs | 32 +++++--- lib/components/fabro-sandbox/src/details.rs | 30 ++++++-- lib/components/fabro-sandbox/src/docker.rs | 31 ++++++-- .../fabro-sandbox/src/from_environment.rs | 42 +++++----- .../fabro-workflow/src/operations/start.rs | 12 ++- lib/foundation/fabro-config/src/layers/run.rs | 2 + .../fabro-config/src/resolve/run.rs | 20 ++++- .../fabro-config/src/tests/resolve_run.rs | 26 +++++++ .../fabro-types/src/settings/run.rs | 7 +- .../src/models/run-clone-settings.ts | 1 + 18 files changed, 241 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 800846e58..4a8172d93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1870,7 +1870,7 @@ checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" [[package]] name = "daytona-api-client" version = "0.1.0" -source = "git+https://github.com/brynary/daytona-sdk-rust?rev=73c9c458dd1a1d096afd3521175637af82afd8d8#73c9c458dd1a1d096afd3521175637af82afd8d8" +source = "git+https://github.com/brynary/daytona-sdk-rust?rev=be2c7b7272740d47c023cac8abc9f63c1a51a511#be2c7b7272740d47c023cac8abc9f63c1a51a511" dependencies = [ "reqwest 0.13.2", "reqwest-middleware", @@ -1884,7 +1884,7 @@ dependencies = [ [[package]] name = "daytona-sdk" version = "0.1.0" -source = "git+https://github.com/brynary/daytona-sdk-rust?rev=73c9c458dd1a1d096afd3521175637af82afd8d8#73c9c458dd1a1d096afd3521175637af82afd8d8" +source = "git+https://github.com/brynary/daytona-sdk-rust?rev=be2c7b7272740d47c023cac8abc9f63c1a51a511#be2c7b7272740d47c023cac8abc9f63c1a51a511" dependencies = [ "daytona-api-client", "daytona-toolbox-client", @@ -1904,13 +1904,15 @@ dependencies = [ [[package]] name = "daytona-toolbox-client" version = "0.1.0" -source = "git+https://github.com/brynary/daytona-sdk-rust?rev=73c9c458dd1a1d096afd3521175637af82afd8d8#73c9c458dd1a1d096afd3521175637af82afd8d8" +source = "git+https://github.com/brynary/daytona-sdk-rust?rev=be2c7b7272740d47c023cac8abc9f63c1a51a511#be2c7b7272740d47c023cac8abc9f63c1a51a511" dependencies = [ "reqwest 0.13.2", "reqwest-middleware", "serde", "serde_json", "serde_repr", + "tokio", + "tokio-util", "url", ] @@ -2072,7 +2074,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -2199,7 +2201,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4388,6 +4390,22 @@ dependencies = [ "webpki-roots 1.0.6", ] +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" @@ -4440,7 +4458,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.62.2", + "windows-core 0.61.2", ] [[package]] @@ -5315,6 +5333,23 @@ dependencies = [ "getrandom 0.2.17", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe 0.2.1", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -5389,7 +5424,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6386,7 +6421,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -6671,11 +6706,13 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", + "hyper-tls", "hyper-util", "js-sys", "log", "mime", "mime_guess", + "native-tls", "percent-encoding", "pin-project-lite", "quinn", @@ -6687,6 +6724,7 @@ dependencies = [ "serde_urlencoded", "sync_wrapper", "tokio", + "tokio-native-tls", "tokio-rustls", "tokio-util", "tower", @@ -6860,7 +6898,7 @@ dependencies = [ "errno 0.3.14", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6919,7 +6957,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7443,7 +7481,7 @@ version = "1.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" dependencies = [ - "errno 0.3.14", + "errno 0.2.8", "libc", ] @@ -8033,7 +8071,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8079,7 +8117,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230a1b821ccbd75b185820a1f1ff7b14d21da1e442e22c0863ea5f08771a8874" dependencies = [ "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8231,6 +8269,16 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.4" @@ -9132,7 +9180,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4a6437546..ebf4764e2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -97,8 +97,8 @@ twin-openai = { path = "test/twin/openai" } twin-github = { path = "test/twin/github" } tokio-tungstenite = { version = "0.26", features = ["rustls-tls-webpki-roots"] } futures-util = "0.3" -daytona-sdk = { git = "https://github.com/brynary/daytona-sdk-rust", rev = "73c9c458dd1a1d096afd3521175637af82afd8d8", package = "daytona-sdk" } -daytona-api-client = { git = "https://github.com/brynary/daytona-sdk-rust", rev = "73c9c458dd1a1d096afd3521175637af82afd8d8", package = "daytona-api-client" } +daytona-sdk = { git = "https://github.com/brynary/daytona-sdk-rust", rev = "be2c7b7272740d47c023cac8abc9f63c1a51a511", package = "daytona-sdk" } +daytona-api-client = { git = "https://github.com/brynary/daytona-sdk-rust", rev = "be2c7b7272740d47c023cac8abc9f63c1a51a511", package = "daytona-api-client" } sentry = { version = "0.35", default-features = false, features = ["backtrace", "contexts", "ureq", "rustls"] } fork = "0.2" exec = "0.3" diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index ee2eb143e..dafddf42f 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -14633,6 +14633,10 @@ components: properties: enabled: type: boolean + depth: + type: integer + format: int32 + minimum: 1 RunBranchSettings: type: object diff --git a/docs/public/execution/environments.mdx b/docs/public/execution/environments.mdx index 82a72d5ed..c58e602b9 100644 --- a/docs/public/execution/environments.mdx +++ b/docs/public/execution/environments.mdx @@ -258,7 +258,7 @@ memory = "4GB" mode = "block" ``` -Docker and Daytona are clone-based providers. When a run has a GitHub origin, Fabro clones it into the provider workspace. Set `[run.clone] enabled = false` to start with an empty workspace. Docker and Daytona ignore `cwd`; use the provider-owned workspace layout and `run.working_dir` for repository-relative commands. +Docker and Daytona are clone-based providers. When a run has a GitHub origin, Fabro clones it into the provider workspace. Set `[run.clone] enabled = false` to start with an empty workspace, or set a positive `[run.clone] depth` to limit the downloaded Git history. Docker and Daytona ignore `cwd`; use the provider-owned workspace layout and `run.working_dir` for repository-relative commands. The image must provide `/bin/bash`; Fabro evaluates every sandbox command with it and has no `sh` fallback. Commands run in a **non-login** shell, so login profiles (`/etc/profile.d/*.sh`, `~/.bash_profile`, and `nvm`/`rbenv`/`sdkman` initializers) are not sourced — put anything they set into the Dockerfile's `ENV` instead. Fabro verifies Bash during initialization and again on resume, and fails with remediation rather than reporting the sandbox ready. diff --git a/docs/public/execution/run-configuration.mdx b/docs/public/execution/run-configuration.mdx index d2ce9bd5e..14c6c1ff7 100644 --- a/docs/public/execution/run-configuration.mdx +++ b/docs/public/execution/run-configuration.mdx @@ -241,10 +241,16 @@ Configure whether clone-based sandboxes clone the run's GitHub origin before exe ```toml title="run.toml" [run.clone] enabled = true +depth = 1 ``` Set `enabled = false` to start Docker and Daytona runs with an empty provider workspace. Use [prepare steps](#runprepare) to clone or create any files the workflow needs. +| Field | Description | +|---|---| +| `enabled` | When `false`, Fabro skips the repository clone. Defaults to `true`. | +| `depth` | Optional positive Git history depth. Applies to Docker and Daytona. If omitted, Daytona clones full history and Docker uses its default depth of 10. | + ### `[run.run_branch]` Configure Fabro's managed `fabro/run/` checkpoint branch. diff --git a/docs/public/integrations/daytona.mdx b/docs/public/integrations/daytona.mdx index 1a2f25790..05fa190e9 100644 --- a/docs/public/integrations/daytona.mdx +++ b/docs/public/integrations/daytona.mdx @@ -141,6 +141,15 @@ provider = "daytona" enabled = false ``` +For a faster clone that keeps only the newest commit, set a clone depth: + +```toml title="run.toml" +[run.clone] +depth = 1 +``` + +If `depth` is omitted, Daytona clones the full repository history. + If the clone fails without GitHub access configured, Fabro suggests running the setup flow: ``` diff --git a/lib/apps/fabro-server/src/run_manifest.rs b/lib/apps/fabro-server/src/run_manifest.rs index fda5f8087..6a5045e81 100644 --- a/lib/apps/fabro-server/src/run_manifest.rs +++ b/lib/apps/fabro-server/src/run_manifest.rs @@ -669,11 +669,11 @@ pub(crate) fn effective_sandbox_provider(settings: &RunNamespace) -> SandboxProv } fn resolve_daytona_config(settings: &RunNamespace) -> DaytonaConfig { - daytona_config_from_environment(&settings.environment, !settings.clone.enabled) + daytona_config_from_environment(&settings.environment, &settings.clone) } fn resolve_docker_config(settings: &RunNamespace) -> DockerSandboxOptions { - docker_config_from_environment(&settings.environment, !settings.clone.enabled) + docker_config_from_environment(&settings.environment, &settings.clone) } #[derive(Clone, Debug, PartialEq, Eq)] diff --git a/lib/components/fabro-sandbox/src/config.rs b/lib/components/fabro-sandbox/src/config.rs index c1304db5a..66f55354f 100644 --- a/lib/components/fabro-sandbox/src/config.rs +++ b/lib/components/fabro-sandbox/src/config.rs @@ -18,6 +18,7 @@ pub struct DaytonaSettings { pub labels: Option>, pub snapshot: Option, pub network: Option, + pub clone_depth: Option, #[serde(default)] pub skip_clone: bool, } diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index a932ae826..725f110c8 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -106,12 +106,15 @@ fn daytona_git_clone_options( commit_id: Option, username: Option, password: Option, + depth: Option, ) -> GitCloneOptions { GitCloneOptions { branch, commit_id, username, password, + depth, + ..GitCloneOptions::default() } } @@ -121,10 +124,10 @@ pub(crate) fn daytona_not_found(err: &DaytonaError) -> bool { /// Permissions a Daytona API key needs for Fabro's snapshot and sandbox flow. pub const REQUIRED_DAYTONA_PERMISSIONS: &[Permissions] = &[ - Permissions::WriteColonSnapshots, - Permissions::DeleteColonSnapshots, - Permissions::WriteColonSandboxes, - Permissions::DeleteColonSandboxes, + Permissions::WRITE_SNAPSHOTS, + Permissions::DELETE_SNAPSHOTS, + Permissions::WRITE_SANDBOXES, + Permissions::DELETE_SANDBOXES, ]; pub use crate::config::{ @@ -258,10 +261,10 @@ fn join_perms(perms: &[Permissions]) -> String { fn perm_wire_str(permission: Permissions) -> &'static str { match permission { - Permissions::WriteColonSnapshots => "write:snapshots", - Permissions::DeleteColonSnapshots => "delete:snapshots", - Permissions::WriteColonSandboxes => "write:sandboxes", - Permissions::DeleteColonSandboxes => "delete:sandboxes", + Permissions::WRITE_SNAPSHOTS => "write:snapshots", + Permissions::DELETE_SNAPSHOTS => "delete:snapshots", + Permissions::WRITE_SANDBOXES => "write:sandboxes", + Permissions::DELETE_SANDBOXES => "delete:sandboxes", _ => "unknown", } } @@ -1625,6 +1628,7 @@ impl Sandbox for DaytonaSandbox { commit_sha.clone(), username.clone(), password.clone(), + self.config.clone_depth, ); async move { git_svc.clone(origin, target, options).await } }, @@ -3145,6 +3149,7 @@ mod tests { Some("0123456789abcdef0123456789abcdef01234567".to_string()), Some("x-access-token".to_string()), Some("secret".to_string()), + Some(1), ); assert_eq!(options.branch.as_deref(), Some("feature/work")); @@ -3154,6 +3159,7 @@ mod tests { ); assert_eq!(options.username.as_deref(), Some("x-access-token")); assert_eq!(options.password.as_deref(), Some("secret")); + assert_eq!(options.depth, Some(1)); } fn mock_sandbox_body(sandbox_id: &str) -> serde_json::Value { @@ -3171,6 +3177,7 @@ mod tests { "gpu": 0.0, "memory": 4.0, "disk": 20.0, + "toolboxProxyUrl": "https://proxy.example.com/toolbox", "state": "started" }) } @@ -3488,6 +3495,7 @@ mod tests { "size": null, "entrypoint": null, "errorReason": null, + "sourceSandboxId": null, "lastUsedAt": null, "createdAt": "2026-05-01T00:00:00Z", "updatedAt": "2026-05-01T00:00:00Z" @@ -3509,6 +3517,7 @@ mod tests { "gpu": 0.0, "memory": 4.0, "disk": 20.0, + "toolboxProxyUrl": "https://proxy.example.com/toolbox", "state": state.to_string() }) } @@ -3519,6 +3528,7 @@ mod tests { assert!(config.snapshot.is_none()); assert!(config.auto_stop_interval.is_none()); assert!(config.labels.is_none()); + assert!(config.clone_depth.is_none()); } #[test] @@ -4195,10 +4205,7 @@ mod tests { fn missing_display_uses_daytona_wire_scope_names() { let check = DaytonaKeyCheck { key_name: "delete-only".to_string(), - missing: vec![ - Permissions::WriteColonSnapshots, - Permissions::WriteColonSandboxes, - ], + missing: vec![Permissions::WRITE_SNAPSHOTS, Permissions::WRITE_SANDBOXES], }; assert_eq!(check.missing_display(), "write:snapshots, write:sandboxes"); @@ -4337,6 +4344,7 @@ mod tests { "gpu": 0.0, "memory": 4.0, "disk": 20.0, + "toolboxProxyUrl": "https://proxy.example.com/toolbox", "state": "started" })); }) diff --git a/lib/components/fabro-sandbox/src/details.rs b/lib/components/fabro-sandbox/src/details.rs index 2ada66652..0d7fe6390 100644 --- a/lib/components/fabro-sandbox/src/details.rs +++ b/lib/components/fabro-sandbox/src/details.rs @@ -672,18 +672,22 @@ pub(crate) mod daytona { DaytonaState::Creating | DaytonaState::PendingBuild | DaytonaState::BuildingSnapshot - | DaytonaState::PullingSnapshot => SandboxState::Provisioning, - DaytonaState::Starting => SandboxState::Starting, - DaytonaState::Started => SandboxState::Running, - DaytonaState::Stopping | DaytonaState::Archiving => SandboxState::Stopping, + | DaytonaState::PullingSnapshot + | DaytonaState::Forking => SandboxState::Provisioning, + DaytonaState::Starting | DaytonaState::Resuming => SandboxState::Starting, + DaytonaState::Started | DaytonaState::Snapshotting => SandboxState::Running, + DaytonaState::Stopping | DaytonaState::Archiving | DaytonaState::Pausing => { + SandboxState::Stopping + } DaytonaState::Stopped => SandboxState::Stopped, + DaytonaState::Paused => SandboxState::Paused, DaytonaState::Restoring => SandboxState::Restoring, DaytonaState::Resizing => SandboxState::Resizing, DaytonaState::Archived => SandboxState::Archived, DaytonaState::Destroying => SandboxState::Deleting, DaytonaState::Destroyed => SandboxState::Deleted, DaytonaState::Error | DaytonaState::BuildFailed => SandboxState::Error, - DaytonaState::Unknown => SandboxState::Unknown, + DaytonaState::Unknown | DaytonaState::UnknownDefaultOpenApi => SandboxState::Unknown, } } @@ -755,6 +759,22 @@ pub(crate) mod daytona { ); } + #[test] + fn pause_states_normalize_to_fabro_states() { + assert_eq!( + normalize_daytona_state(DaytonaState::Pausing), + SandboxState::Stopping + ); + assert_eq!( + normalize_daytona_state(DaytonaState::Paused), + SandboxState::Paused + ); + assert_eq!( + normalize_daytona_state(DaytonaState::Resuming), + SandboxState::Starting + ); + } + #[test] fn gibibytes_to_bytes_converts_positive_values() { assert_eq!(gibibytes_to_bytes(2.0), Some(2 * 1024 * 1024 * 1024)); diff --git a/lib/components/fabro-sandbox/src/docker.rs b/lib/components/fabro-sandbox/src/docker.rs index c3842ef9a..4bc512d04 100644 --- a/lib/components/fabro-sandbox/src/docker.rs +++ b/lib/components/fabro-sandbox/src/docker.rs @@ -48,7 +48,7 @@ const DOCKER_BASH_REQUIREMENT: &str = "Docker sandboxes require /bin/bash for ev pub(crate) const WORKING_DIRECTORY: &str = "/workspace"; pub(crate) const REPOS_ROOT: &str = "/repos"; -const GIT_CLONE_DEPTH: usize = 10; +const DEFAULT_GIT_CLONE_DEPTH: usize = 10; const GIT_CLONE_TIMEOUT: Duration = Duration::from_mins(5); #[cfg(test)] const EXEC_STOP_POLL_SLEEP_SECONDS: &str = "0.005"; @@ -121,6 +121,8 @@ pub struct DockerSandboxOptions { pub auto_pull: bool, /// Additional `KEY=VALUE` environment variables for the container. pub env_vars: Vec, + /// Maximum Git history depth fetched during clone. + pub clone_depth: usize, /// Create an empty workspace instead of cloning even when an origin exists. pub skip_clone: bool, } @@ -134,6 +136,7 @@ impl Default for DockerSandboxOptions { cpu_quota: None, auto_pull: true, env_vars: Vec::new(), + clone_depth: DEFAULT_GIT_CLONE_DEPTH, skip_clone: false, } } @@ -954,7 +957,7 @@ impl DockerSandbox { &layout.primary_repo_path, "origin", expected_sha, - GIT_CLONE_DEPTH, + self.config.clone_depth, ); if let Err(failure) = self .retry_git_transfer( @@ -992,8 +995,12 @@ impl DockerSandbox { return Err(self.report_clone_failure(&origin_url, error)); } } else { - let command = - git_clone_command(clone_url, branch.as_deref(), &layout.primary_repo_path); + let command = git_clone_command( + clone_url, + branch.as_deref(), + &layout.primary_repo_path, + self.config.clone_depth, + ); if let Err(failure) = self .retry_git_transfer( &command, @@ -1529,7 +1536,12 @@ async fn cache_docker_stdio_completion( } } -fn git_clone_command(clone_url: &str, branch: Option<&str>, checkout_path: &str) -> String { +fn git_clone_command( + clone_url: &str, + branch: Option<&str>, + checkout_path: &str, + depth: usize, +) -> String { let mut command = format!("{} clone", sandbox::GIT); if let Some(branch) = branch { command.push_str(" --branch "); @@ -1537,7 +1549,7 @@ fn git_clone_command(clone_url: &str, branch: Option<&str>, checkout_path: &str) command.push_str(" --single-branch"); } command.push_str(" --depth "); - command.push_str(&GIT_CLONE_DEPTH.to_string()); + command.push_str(&depth.to_string()); command.push_str(" --no-tags"); command.push_str(" -- "); command.push_str(&shell_quote(clone_url)); @@ -2556,19 +2568,21 @@ mod tests { let options = DockerSandboxOptions::default(); assert_eq!(options.image, "buildpack-deps:noble"); assert_eq!(options.network_mode.as_deref(), Some("bridge")); + assert_eq!(options.clone_depth, DEFAULT_GIT_CLONE_DEPTH); assert!(!options.skip_clone); } #[test] - fn clone_command_uses_depth_ten_without_tags_for_branch_clone() { + fn clone_command_uses_configured_depth_without_tags_for_branch_clone() { let command = git_clone_command( "https://github.com/fabro-sh/fabro", Some("main"), "/repos/fabro-sh/fabro", + 1, ); assert_eq!( command, - "git -c maintenance.auto=0 -c gc.auto=0 clone --branch main --single-branch --depth 10 --no-tags -- https://github.com/fabro-sh/fabro /repos/fabro-sh/fabro" + "git -c maintenance.auto=0 -c gc.auto=0 clone --branch main --single-branch --depth 1 --no-tags -- https://github.com/fabro-sh/fabro /repos/fabro-sh/fabro" ); } @@ -2578,6 +2592,7 @@ mod tests { "https://github.com/fabro-sh/fabro", None, "/repos/fabro-sh/fabro", + DEFAULT_GIT_CLONE_DEPTH, ); assert_eq!( command, diff --git a/lib/components/fabro-sandbox/src/from_environment.rs b/lib/components/fabro-sandbox/src/from_environment.rs index 5399927ea..c73023354 100644 --- a/lib/components/fabro-sandbox/src/from_environment.rs +++ b/lib/components/fabro-sandbox/src/from_environment.rs @@ -8,7 +8,9 @@ use std::path::{Path, PathBuf}; use fabro_types::settings::ResolveError; #[cfg(feature = "daytona")] use fabro_types::settings::run::DockerfileSource as ResolvedDockerfileSource; -use fabro_types::settings::run::{EnvironmentNetworkMode, RunEnvironmentSettings}; +use fabro_types::settings::run::{ + EnvironmentNetworkMode, RunCloneSettings, RunEnvironmentSettings, +}; #[cfg(feature = "daytona")] use crate::config::{ @@ -23,19 +25,16 @@ use crate::docker::DockerSandboxOptions; #[must_use] pub fn daytona_config_from_environment( settings: &RunEnvironmentSettings, - skip_clone: bool, + clone: &RunCloneSettings, ) -> DaytonaConfig { DaytonaConfig { auto_stop_interval: settings .lifecycle .auto_stop .map(|duration| duration_to_minutes_i32(duration.as_std())), - labels: (!settings.labels.is_empty()).then(|| settings.labels.clone()), - snapshot: settings - .image - .dockerfile - .as_ref() - .map(|dockerfile| DaytonaSnapshotSettings { + labels: (!settings.labels.is_empty()).then(|| settings.labels.clone()), + snapshot: settings.image.dockerfile.as_ref().map(|dockerfile| { + DaytonaSnapshotSettings { cpu: settings.resources.cpu, memory: settings .resources @@ -53,15 +52,17 @@ pub fn daytona_config_from_environment( SandboxDockerfileSource::Path { path: path.clone() } } }), - }), - network: Some(match settings.network.mode { + } + }), + network: Some(match settings.network.mode { EnvironmentNetworkMode::Block => DaytonaNetwork::Block, EnvironmentNetworkMode::AllowAll => DaytonaNetwork::AllowAll, EnvironmentNetworkMode::CidrAllowList => { DaytonaNetwork::AllowList(settings.network.allow.clone()) } }), - skip_clone, + clone_depth: clone.depth, + skip_clone: !clone.enabled, } } @@ -69,7 +70,7 @@ pub fn daytona_config_from_environment( #[must_use] pub fn docker_config_from_environment( settings: &RunEnvironmentSettings, - skip_clone: bool, + clone: &RunCloneSettings, ) -> DockerSandboxOptions { // No vault is available on this path (server preflight / manifest), so a // `{{ secrets.* }}` value keeps its source form. Nothing else is left to @@ -84,25 +85,23 @@ pub fn docker_config_from_environment( .iter() .map(|(key, value)| (key.clone(), value.as_source())) .collect(); - docker_config_from_environment_env(settings, skip_clone, env) + docker_config_from_environment_env(settings, clone, env) } #[cfg(feature = "docker")] pub fn docker_config_from_environment_with_secrets( settings: &RunEnvironmentSettings, - skip_clone: bool, + clone: &RunCloneSettings, secrets_lookup: impl FnMut(&str) -> Option, ) -> Result { let env = settings.resolve_env(secrets_lookup)?; - Ok(docker_config_from_environment_env( - settings, skip_clone, env, - )) + Ok(docker_config_from_environment_env(settings, clone, env)) } #[cfg(feature = "docker")] fn docker_config_from_environment_env( settings: &RunEnvironmentSettings, - skip_clone: bool, + clone: &RunCloneSettings, env: std::collections::HashMap, ) -> DockerSandboxOptions { let mut env_vars = env @@ -133,7 +132,12 @@ fn docker_config_from_environment_env( .cpu .map(|cpu| i64::from(cpu).saturating_mul(100_000)), env_vars, - skip_clone, + clone_depth: clone + .depth + .and_then(|depth| usize::try_from(depth).ok()) + .filter(|depth| *depth > 0) + .unwrap_or(default_options.clone_depth), + skip_clone: !clone.enabled, ..DockerSandboxOptions::default() } } diff --git a/lib/components/fabro-workflow/src/operations/start.rs b/lib/components/fabro-workflow/src/operations/start.rs index 3ba470151..54ace936c 100644 --- a/lib/components/fabro-workflow/src/operations/start.rs +++ b/lib/components/fabro-workflow/src/operations/start.rs @@ -591,7 +591,7 @@ fn resolve_sandbox_provider(settings: &ResolvedRunSettings) -> SandboxProviderKi } fn resolve_daytona_config(settings: &ResolvedRunSettings) -> DaytonaConfig { - daytona_config_from_environment(&settings.environment, !settings.clone.enabled) + daytona_config_from_environment(&settings.environment, &settings.clone) } fn resolve_docker_config( @@ -600,7 +600,7 @@ fn resolve_docker_config( ) -> Result { docker_config_from_environment_with_secrets( &settings.environment, - !settings.clone.enabled, + &settings.clone, secrets_lookup, ) .map_err(|err| Error::engine_with_source("failed to resolve Docker environment config", err)) @@ -1285,6 +1285,7 @@ reasoning = false let settings = settings_from_run_layer(RunLayer { clone: Some(RunCloneLayer { enabled: Some(false), + depth: Some(1), }), ..RunLayer::default() }); @@ -1295,6 +1296,13 @@ reasoning = false .skip_clone ); assert!(resolve_daytona_config(&settings.run).skip_clone); + assert_eq!(resolve_daytona_config(&settings.run).clone_depth, Some(1)); + assert_eq!( + resolve_docker_config(&settings.run, |_| None) + .unwrap() + .clone_depth, + 1 + ); } #[test] diff --git a/lib/foundation/fabro-config/src/layers/run.rs b/lib/foundation/fabro-config/src/layers/run.rs index 8b020666f..b8c265b8d 100644 --- a/lib/foundation/fabro-config/src/layers/run.rs +++ b/lib/foundation/fabro-config/src/layers/run.rs @@ -303,6 +303,8 @@ pub struct RunCheckpointLayer { pub struct RunCloneLayer { #[serde(default, skip_serializing_if = "Option::is_none")] pub enabled: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub depth: Option, } /// `[run.run_branch]` — Fabro-managed checkpoint branch policy. diff --git a/lib/foundation/fabro-config/src/resolve/run.rs b/lib/foundation/fabro-config/src/resolve/run.rs index 9010f25f8..b078c8dad 100644 --- a/lib/foundation/fabro-config/src/resolve/run.rs +++ b/lib/foundation/fabro-config/src/resolve/run.rs @@ -28,7 +28,7 @@ pub fn resolve_run( mcp_server_catalog: &HashMap, errors: &mut Vec, ) -> RunNamespace { - let clone = resolve_clone(layer.clone.as_ref()); + let clone = resolve_clone(layer.clone.as_ref(), errors); let run_branch = resolve_run_branch(layer.run_branch.as_ref()); let mut meta_branch = resolve_meta_branch(layer.meta_branch.as_ref()); if !run_branch.enabled { @@ -244,9 +244,25 @@ fn resolve_checkpoint(checkpoint: Option<&RunCheckpointLayer>) -> RunCheckpointS } } -fn resolve_clone(clone: Option<&RunCloneLayer>) -> RunCloneSettings { +fn resolve_clone( + clone: Option<&RunCloneLayer>, + errors: &mut Vec, +) -> RunCloneSettings { + let depth = clone.and_then(|clone| clone.depth).and_then(|depth| { + if depth < 1 { + errors.push(ResolveError::Invalid { + path: "run.clone.depth".to_string(), + reason: "depth must be at least 1".to_string(), + }); + None + } else { + Some(depth) + } + }); + RunCloneSettings { enabled: clone.and_then(|clone| clone.enabled).unwrap_or(true), + depth, } } diff --git a/lib/foundation/fabro-config/src/tests/resolve_run.rs b/lib/foundation/fabro-config/src/tests/resolve_run.rs index d416eb651..890dd2028 100644 --- a/lib/foundation/fabro-config/src/tests/resolve_run.rs +++ b/lib/foundation/fabro-config/src/tests/resolve_run.rs @@ -126,6 +126,7 @@ fn resolves_run_defaults_from_empty_settings() { assert!(!settings.environment.lifecycle.preserve); assert!(settings.environment.lifecycle.stop_on_terminal); assert!(settings.clone.enabled); + assert_eq!(settings.clone.depth, None); assert!(settings.run_branch.enabled); assert!(settings.run_branch.push); assert!(settings.meta_branch.enabled); @@ -296,6 +297,7 @@ _version = 1 [run.clone] enabled = false +depth = 1 [run.run_branch] enabled = true @@ -310,12 +312,36 @@ push = false .run; assert!(!settings.clone.enabled); + assert_eq!(settings.clone.depth, Some(1)); assert!(settings.run_branch.enabled); assert!(!settings.run_branch.push); assert!(settings.meta_branch.enabled); assert!(!settings.meta_branch.push); } +#[test] +fn rejects_non_positive_clone_depth() { + let error = super::workflow_settings_from_toml( + r" +_version = 1 + +[run.clone] +depth = 0 +", + ) + .expect_err("zero clone depth should not resolve"); + + let message = error.to_string(); + assert!( + message.contains("run.clone.depth"), + "unexpected error: {message}" + ); + assert!( + message.contains("at least 1"), + "unexpected error: {message}" + ); +} + #[test] fn disabling_run_branch_forces_meta_branch_off() { let settings = super::workflow_settings_from_toml( diff --git a/lib/foundation/fabro-types/src/settings/run.rs b/lib/foundation/fabro-types/src/settings/run.rs index 8e589a57a..5d96e9ce2 100644 --- a/lib/foundation/fabro-types/src/settings/run.rs +++ b/lib/foundation/fabro-types/src/settings/run.rs @@ -961,11 +961,16 @@ impl Default for RunCheckpointSettings { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RunCloneSettings { pub enabled: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub depth: Option, } impl Default for RunCloneSettings { fn default() -> Self { - Self { enabled: true } + Self { + enabled: true, + depth: None, + } } } diff --git a/lib/packages/fabro-api-client/src/models/run-clone-settings.ts b/lib/packages/fabro-api-client/src/models/run-clone-settings.ts index 7258d9d8d..48c5959df 100644 --- a/lib/packages/fabro-api-client/src/models/run-clone-settings.ts +++ b/lib/packages/fabro-api-client/src/models/run-clone-settings.ts @@ -16,4 +16,5 @@ export interface RunCloneSettings { 'enabled': boolean; + 'depth'?: number; } From 4c467cd6ba8c6a8b818d59cac3c4ff71ab3d2a10 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 17:12:27 -0400 Subject: [PATCH 23/30] refactor(github): simplify repository access checks --- Cargo.lock | 1 + .../src/automation_materializer.rs | 5 +- lib/apps/fabro-server/src/git_checkout.rs | 18 +- lib/apps/fabro-server/src/run_manifest.rs | 88 +++++----- lib/components/fabro-github/Cargo.toml | 1 + lib/components/fabro-github/src/access.rs | 155 ++++++++++++------ lib/components/fabro-github/src/lib.rs | 53 ++++-- lib/components/fabro-github/src/tests_mock.rs | 15 +- .../fabro-github/src/token_source.rs | 111 ++++++++----- .../fabro-github/tests/live_access.rs | 22 +-- .../fabro-sandbox/src/daytona/mod.rs | 2 +- lib/components/fabro-sandbox/src/docker.rs | 2 +- lib/components/fabro-sandbox/src/git_retry.rs | 32 ++-- lib/components/fabro-sandbox/src/lib.rs | 2 +- .../fabro-workflow/src/pipeline/initialize.rs | 76 ++++----- lib/foundation/fabro-types/src/repository.rs | 14 ++ 16 files changed, 355 insertions(+), 242 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 800846e58..4138e679d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2663,6 +2663,7 @@ dependencies = [ "fabro-static", "fabro-test", "fabro-types", + "futures", "jsonwebtoken", "serde", "serde_json", diff --git a/lib/apps/fabro-server/src/automation_materializer.rs b/lib/apps/fabro-server/src/automation_materializer.rs index b57d53b22..b2f1751b7 100644 --- a/lib/apps/fabro-server/src/automation_materializer.rs +++ b/lib/apps/fabro-server/src/automation_materializer.rs @@ -12,8 +12,7 @@ use fabro_util::error::collect_chain; use tokio::{fs, task}; use crate::git_checkout::{ - GitCheckoutError, GitRepoCache, WorktreePrepareInput, github_metadata_url, - resolve_git_auth_config, + GitCheckoutError, GitRepoCache, WorktreePrepareInput, resolve_git_auth_config, }; #[derive(Debug, Clone, PartialEq, Eq)] @@ -199,7 +198,7 @@ fn build_manifest_from_checkout( let mut manifest = built.manifest; manifest.git = Some(GitContext { - origin_url: github_metadata_url(&git_context.repo), + origin_url: git_context.repo.https_url(), branch: git_context.ref_selector, sha: Some(git_context.checked_out_sha), dirty: DirtyStatus::Clean, diff --git a/lib/apps/fabro-server/src/git_checkout.rs b/lib/apps/fabro-server/src/git_checkout.rs index 8bdb4f869..3648c6b4c 100644 --- a/lib/apps/fabro-server/src/git_checkout.rs +++ b/lib/apps/fabro-server/src/git_checkout.rs @@ -165,11 +165,9 @@ async fn bare_clone_may_be_corrupt(bare_dir: &Path) -> bool { } fn github_clone_url(repo: &GitHubRepositorySlug) -> String { - format!("https://github.com/{}/{}.git", repo.owner(), repo.repo()) -} - -pub(crate) fn github_metadata_url(repo: &GitHubRepositorySlug) -> String { - format!("https://github.com/{}/{}", repo.owner(), repo.repo()) + let mut url = repo.https_url(); + url.push_str(".git"); + url } #[derive(Clone, Debug, PartialEq, Eq)] @@ -478,10 +476,7 @@ mod tests { github_clone_url(&repo), "https://github.com/fabro-sh/fabro.git" ); - assert_eq!( - github_metadata_url(&repo), - "https://github.com/fabro-sh/fabro" - ); + assert_eq!(repo.https_url(), "https://github.com/fabro-sh/fabro"); assert!(!github_clone_url(&repo).contains('@')); } @@ -491,10 +486,7 @@ mod tests { assert_eq!(repo.owner(), "owner"); assert_eq!(repo.repo(), ".github"); - assert_eq!( - github_metadata_url(&repo), - "https://github.com/owner/.github" - ); + assert_eq!(repo.https_url(), "https://github.com/owner/.github"); } #[test] diff --git a/lib/apps/fabro-server/src/run_manifest.rs b/lib/apps/fabro-server/src/run_manifest.rs index 7bc97f646..941cadf3f 100644 --- a/lib/apps/fabro-server/src/run_manifest.rs +++ b/lib/apps/fabro-server/src/run_manifest.rs @@ -882,6 +882,9 @@ async fn check_git_remote_ref( /// failure to its most useful message: stderr, then stdout, then the exit /// status. async fn run_ls_remote(mut command: Command) -> std::result::Result<(), String> { + // Dropping a timed-out `Command::output` future does not stop the child + // unless kill-on-drop is enabled. + command.kill_on_drop(true); let output = time::timeout(Duration::from_secs(10), command.output()) .await .map_err(|_| "git ls-remote timed out after 10s".to_string())? @@ -1246,7 +1249,7 @@ where Err(err) => { return fail_github_token_check( checks, - &[], + Vec::new(), "invalid permissions", format!("Failed to resolve GitHub permissions: {err}"), ); @@ -1290,10 +1293,25 @@ where }); } + let Some(origin_url) = prepared + .git + .as_ref() + .map(|git| git.origin_url.trim()) + .filter(|url| !url.is_empty()) + else { + return fail_github_token_check( + checks, + perm_details, + "missing origin", + "run.integrations.github.additional_repositories requires a GitHub run origin, but \ + this run has no repository origin URL" + .to_string(), + ); + }; let Some(creds) = github_app else { return fail_github_token_check( checks, - &perm_details, + perm_details, "missing credentials", "run.integrations.github.additional_repositories requires GitHub credentials, but \ none are configured on the server" @@ -1301,10 +1319,9 @@ where ); }; // The same validated access value runtime initialization constructs, so - // preflight and runtime cannot disagree about the effective set. A - // missing origin fails inside `new` with the canonical remediation. + // preflight and runtime cannot disagree about the effective set. let access = match fabro_github::GitHubRepositoryAccess::new( - prepared.git.as_ref().map(|git| git.origin_url.as_str()), + Some(origin_url), &integration.additional_repositories, integration.permissions.clone(), ) { @@ -1314,7 +1331,7 @@ where Ok(None) => { return fail_github_token_check( checks, - &perm_details, + perm_details, "missing origin", "run.integrations.github.additional_repositories requires a GitHub run origin, \ but this run has no repository origin URL" @@ -1324,7 +1341,7 @@ where Err(err) => { return fail_github_token_check( checks, - &perm_details, + perm_details, "invalid repository set", format!("{err:#}"), ); @@ -1337,7 +1354,7 @@ where let token = match mint_scoped_token(access.clone(), creds).await { Ok(token) => token, Err(err) => { - return fail_github_token_check(checks, &perm_details, "failed", err); + return fail_github_token_check(checks, perm_details, "failed", err); } }; checks.push(CheckResult { @@ -1396,7 +1413,7 @@ where /// Report one "GitHub Token" preflight failure and fail the check. fn fail_github_token_check( checks: &mut Vec, - perm_details: &[CheckDetail], + perm_details: Vec, summary: &str, remediation: String, ) -> bool { @@ -1404,7 +1421,7 @@ fn fail_github_token_check( name: "GitHub Token".into(), status: CheckStatus::Error, summary: summary.into(), - details: perm_details.to_vec(), + details: perm_details, remediation: Some(remediation), }); false @@ -1413,10 +1430,6 @@ fn fail_github_token_check( /// Bounded concurrency for per-repository `git ls-remote` probes. const REPOSITORY_PROBE_CONCURRENCY: usize = 4; -/// Total probe attempts per repository when failures classify as retryable -/// (token replication lag or transient infrastructure). -const REPOSITORY_PROBE_ATTEMPTS: u32 = 3; - async fn check_primary_only_github_token( checks: &mut Vec, prepared: &PreparedManifest, @@ -1459,21 +1472,16 @@ async fn check_primary_only_github_token( } } -/// Production minter for the multi-repository path: -/// [`fabro_github::GitHubRepositoryAccess::resolve_verified_token`] over a -/// source scoped to the effective set. In App mode that resolves every -/// repository's installation first, so a failure names the repository the -/// App cannot see (or one on a different installation). +/// Production minter for the multi-repository path. The source owns the +/// effective set. In App mode its first resolve checks every repository's +/// installation before minting the scoped token. async fn mint_scoped_github_token( access: fabro_github::GitHubRepositoryAccess, creds: fabro_github::GitHubCredentials, ) -> std::result::Result { let source = InstallationTokenSource::for_access(&creds, &access).map_err(|err| format!("{err:#}"))?; - access - .resolve_verified_token(&creds, &source) - .await - .map_err(|err| format!("{err:#}")) + source.resolve().await.map_err(|err| format!("{err:#}")) } /// Production per-repository probe: a non-interactive @@ -1486,37 +1494,31 @@ async fn probe_github_repository( slug: fabro_types::GitHubRepositorySlug, token: ResolvedToken, ) -> std::result::Result<(), String> { - let url = format!("https://github.com/{}/{}", slug.owner(), slug.repo()); + let url = slug.https_url(); probe_with_replication_retry(token.snapshot, || run_probe_ls_remote(&url, &token)).await } /// Retry auth-shaped failures with the SAME token: replication of a given /// token only makes progress, while re-minting would restart the replication -/// clock. Classification and pacing match the sandbox git retry policy -/// (`fabro_sandbox::classify_failure` / `fabro_sandbox::replication_backoff`). +/// clock. The sandbox git retry executor owns attempt limits, +/// classification, and pacing. async fn probe_with_replication_retry( snapshot: TokenSnapshot, - run: F, + mut run: F, ) -> std::result::Result<(), String> where - F: Fn() -> Fut, + F: FnMut() -> Fut, Fut: Future>, { let credential_context = fabro_sandbox::CredentialContext::from_snapshot(Some(&snapshot)); - let backoff = fabro_sandbox::replication_backoff(); - let mut attempt = 0u32; - loop { - attempt += 1; - let Err(message) = run().await else { - return Ok(()); - }; - if attempt >= REPOSITORY_PROBE_ATTEMPTS - || fabro_sandbox::classify_failure(&message, credential_context).is_none() - { - return Err(message); - } - time::sleep(backoff.delay_for_attempt(attempt)).await; - } + fabro_sandbox::retry_git_operation( + SandboxProviderKind::Local, + "repository probe", + &fabro_sandbox::RetryPlan::repository_probe(), + |_attempt| run(), + |message| fabro_sandbox::classify_failure(message, credential_context), + ) + .await } async fn run_probe_ls_remote(url: &str, token: &ResolvedToken) -> std::result::Result<(), String> { @@ -3494,7 +3496,7 @@ dockerfile = { path = "Dockerfile" } assert!(!ok); let check = checks.last().unwrap(); - assert_eq!(check.summary, "invalid repository set"); + assert_eq!(check.summary, "missing origin"); assert!( check .remediation diff --git a/lib/components/fabro-github/Cargo.toml b/lib/components/fabro-github/Cargo.toml index 8f755dd79..a2d6d7abd 100644 --- a/lib/components/fabro-github/Cargo.toml +++ b/lib/components/fabro-github/Cargo.toml @@ -25,6 +25,7 @@ fabro-http.workspace = true fabro-redact.workspace = true fabro-static.workspace = true fabro-types = { path = "../../foundation/fabro-types" } +futures.workspace = true jsonwebtoken.workspace = true chrono.workspace = true tracing.workspace = true diff --git a/lib/components/fabro-github/src/access.rs b/lib/components/fabro-github/src/access.rs index 6f3c2d02e..774ecc445 100644 --- a/lib/components/fabro-github/src/access.rs +++ b/lib/components/fabro-github/src/access.rs @@ -11,9 +11,13 @@ use std::collections::{BTreeSet, HashMap}; use anyhow::{Context as _, bail}; use fabro_types::GitHubRepositorySlug; use fabro_types::settings::run::RunIntegrationsGithubSettings; +use futures::stream::{self, StreamExt as _}; -use crate::token_source::{InstallationTokenSource, ResolvedToken}; -use crate::{GitHubAppCredentials, GitHubCredentials, HttpClient, InstallationLookup}; +use crate::{GitHubAppCredentials, HttpClient, InstallationLookup, InstallationToken}; + +/// Keep GitHub installation lookups bounded while avoiding one network round +/// trip at a time for large declared repository sets. +const INSTALLATION_LOOKUP_CONCURRENCY: usize = 4; /// The validated effective repository set for a run: the primary origin /// repository plus zero or more distinct additional repositories, all with @@ -142,38 +146,29 @@ impl GitHubRepositoryAccess { !self.additional.is_empty() } - /// [`Self::resolve_shared_installation`] against the production GitHub - /// API with a fresh HTTP client. - pub async fn resolve_shared_installation_via_api( - &self, - creds: &GitHubAppCredentials, - ) -> anyhow::Result { - let client = fabro_http::http_client() - .map_err(anyhow::Error::new) - .context("building HTTP client for installation resolution")?; - self.resolve_shared_installation(creds, &client, &crate::github_api_base_url()) - .await - } - /// Resolve every target's App installation and require one shared /// installation ID, so a repository the App cannot see — or one that /// resolves to a different installation — is named before any token is /// minted. Targets are checked in deterministic primary-first order. - pub async fn resolve_shared_installation( + async fn resolve_shared_installation_with_jwt( &self, - creds: &GitHubAppCredentials, client: &impl HttpClient, + jwt: &str, base_url: &str, ) -> anyhow::Result { - let jwt = crate::sign_app_jwt(&creds.app_id, &creds.private_key_pem)?; - let mut shared: Option<(u64, &GitHubRepositorySlug)> = None; - for slug in self.targets() { + let targets: Vec = self.targets().into_iter().cloned().collect(); + let mut lookups = stream::iter(targets.into_iter().map(|slug| async move { let lookup = - crate::lookup_installation(client, &jwt, base_url, slug.owner(), slug.repo()) + crate::lookup_installation(client, jwt, base_url, slug.owner(), slug.repo()) .await - .with_context(|| { - format!("looking up the GitHub App installation for {slug}") - })?; + .with_context(|| format!("looking up the GitHub App installation for {slug}")); + (slug, lookup) + })) + .buffered(INSTALLATION_LOOKUP_CONCURRENCY); + + let mut shared: Option<(u64, GitHubRepositorySlug)> = None; + while let Some((slug, lookup)) = lookups.next().await { + let lookup = lookup?; let id = match lookup { InstallationLookup::Found(id) => id, InstallationLookup::NotFound => bail!( @@ -184,9 +179,9 @@ impl GitHubRepositoryAccess { "unexpected status {status} looking up the GitHub App installation for {slug}" ), }; - match shared { + match &shared { None => shared = Some((id, slug)), - Some((shared_id, first)) if shared_id != id => bail!( + Some((shared_id, first)) if *shared_id != id => bail!( "repository {slug} belongs to GitHub App installation {id} but {first} \ belongs to installation {shared_id}; all repositories must share one \ installation" @@ -198,25 +193,28 @@ impl GitHubRepositoryAccess { Ok(id) } - /// Prove this access value is usable and produce its token: in App mode, - /// first resolve every target's installation - /// ([`Self::resolve_shared_installation_via_api`]) so a failure names the - /// repository the App cannot see, then resolve `source` once — for App - /// credentials that eagerly mints the token scoped to the whole effective - /// set. The one choreography server preflight and workflow - /// initialization share. - pub async fn resolve_verified_token( + /// Resolve every target to one installation, then mint one token scoped + /// to this exact repository set without looking up the primary twice. + pub(crate) async fn mint_installation_token( &self, - creds: &GitHubCredentials, - source: &InstallationTokenSource, - ) -> anyhow::Result { - if let GitHubCredentials::App(app) = creds { - self.resolve_shared_installation_via_api(app).await?; - } - source - .resolve() + creds: &GitHubAppCredentials, + client: &impl HttpClient, + base_url: &str, + ) -> anyhow::Result { + let jwt = crate::sign_app_jwt(&creds.app_id, &creds.private_key_pem)?; + let installation_id = self + .resolve_shared_installation_with_jwt(client, &jwt, base_url) .await - .context("Failed to resolve the GitHub token for the effective repository set") + .context("resolving the shared GitHub App installation")?; + crate::mint_installation_token_for_id_with_jwt( + client, + &jwt, + installation_id, + &self.repository_names(), + base_url, + self.permissions_json()?, + ) + .await } } @@ -436,7 +434,11 @@ mod tests { .unwrap(); let err = access - .resolve_shared_installation(&creds, &mock, "") + .resolve_shared_installation_with_jwt( + &mock, + &crate::sign_app_jwt(&creds.app_id, &creds.private_key_pem).unwrap(), + "", + ) .await .unwrap_err(); let message = err.to_string(); @@ -476,7 +478,11 @@ mod tests { .unwrap(); let err = access - .resolve_shared_installation(&creds, &mock, "") + .resolve_shared_installation_with_jwt( + &mock, + &crate::sign_app_jwt(&creds.app_id, &creds.private_key_pem).unwrap(), + "", + ) .await .unwrap_err(); let message = err.to_string(); @@ -516,9 +522,66 @@ mod tests { .unwrap(); let id = access - .resolve_shared_installation(&creds, &mock, "") + .resolve_shared_installation_with_jwt( + &mock, + &crate::sign_app_jwt(&creds.app_id, &creds.private_key_pem).unwrap(), + "", + ) .await .unwrap(); assert_eq!(id, 7); } + + #[tokio::test] + async fn access_mint_reuses_the_resolved_installation_id() { + use crate::HttpMethod; + use crate::tests_mock::{MockHttpClient, test_rsa_key}; + + let mock = MockHttpClient::new() + .on( + HttpMethod::Get, + "/repos/fabro-sh/fabro/installation", + 200, + r#"{"id": 7}"#, + ) + .on( + HttpMethod::Get, + "/repos/fabro-sh/keystone/installation", + 200, + r#"{"id": 7}"#, + ) + .on( + HttpMethod::Post, + "/app/installations/7/access_tokens", + 201, + r#"{"token":"scoped","expires_at":"2099-01-01T00:00:00Z"}"#, + ) + .with_req_body( + r#"{"repositories":["fabro","keystone"],"permissions":{"contents":"read"}}"#, + ); + let creds = GitHubAppCredentials { + app_id: "test".to_string(), + private_key_pem: test_rsa_key().to_string(), + slug: None, + }; + let access = access( + "https://github.com/fabro-sh/fabro", + &["fabro-sh/keystone"], + contents_read(), + ) + .unwrap() + .unwrap(); + + let token = access + .mint_installation_token(&creds, &mock, "") + .await + .unwrap(); + + assert_eq!(token.token, "scoped"); + assert_eq!( + mock.request_count(), + 3, + "each repository should be looked up once before the mint" + ); + } } diff --git a/lib/components/fabro-github/src/lib.rs b/lib/components/fabro-github/src/lib.rs index 94ba085a7..87990e757 100644 --- a/lib/components/fabro-github/src/lib.rs +++ b/lib/components/fabro-github/src/lib.rs @@ -598,12 +598,6 @@ async fn mint_installation_token_with_jwt( permissions: serde_json::Value, install_url: Option<&str>, ) -> anyhow::Result { - #[derive(Deserialize)] - struct AccessToken { - token: String, - expires_at: DateTime, - } - let Some(primary_repo) = repos.first() else { bail!("installation token mint requires at least one repository"); }; @@ -644,7 +638,33 @@ async fn mint_installation_token_with_jwt( } }; - // Step 2: Create a scoped access token + mint_installation_token_for_id_with_jwt( + client, + jwt, + installation_id, + repos, + base_url, + permissions, + ) + .await +} + +/// Create a repository-scoped token for an installation already resolved by +/// the caller. +pub(crate) async fn mint_installation_token_for_id_with_jwt( + client: &impl HttpClient, + jwt: &str, + installation_id: u64, + repos: &[String], + base_url: &str, + permissions: serde_json::Value, +) -> anyhow::Result { + #[derive(Deserialize)] + struct AccessToken { + token: String, + expires_at: DateTime, + } + let auth = format!("Bearer {jwt}"); let token_url = format!("{base_url}/app/installations/{installation_id}/access_tokens"); let body = serde_json::json!({ @@ -1159,25 +1179,24 @@ pub async fn check_app_installed( repo: &str, base_url: &str, ) -> anyhow::Result { - let url = format!("{base_url}/repos/{owner}/{repo}/installation"); - let auth = format!("Bearer {jwt}"); - let resp = client - .request(HttpMethod::Get, &url, &github_headers(&auth), None) + let lookup = lookup_installation(client, jwt, base_url, owner, repo) .await .context("Failed to check GitHub App installation")?; - match resp.status { - 200 => Ok(true), - 404 => Ok(false), - 401 => bail!( + match lookup { + InstallationLookup::Found(_) => Ok(true), + InstallationLookup::NotFound => Ok(false), + InstallationLookup::Failed(401) => bail!( "GitHub App authentication failed. \ Check that app_id and GITHUB_APP_PRIVATE_KEY are correct." ), - 403 => bail!( + InstallationLookup::Failed(403) => bail!( "GitHub App installation is suspended. \ Re-enable it in your organization's GitHub App settings." ), - status => bail!("Unexpected status {status} checking GitHub App installation"), + InstallationLookup::Failed(status) => { + bail!("Unexpected status {status} checking GitHub App installation") + } } } diff --git a/lib/components/fabro-github/src/tests_mock.rs b/lib/components/fabro-github/src/tests_mock.rs index da6d7920e..d0f344de3 100644 --- a/lib/components/fabro-github/src/tests_mock.rs +++ b/lib/components/fabro-github/src/tests_mock.rs @@ -2,6 +2,8 @@ //! modules: a scripted [`HttpClient`] and a throwaway RSA key for JWT //! signing. +use std::sync::atomic::{AtomicUsize, Ordering}; + use crate::{HttpClient, HttpMethod, HttpResponse}; pub(crate) fn test_rsa_key() -> &'static str { @@ -22,12 +24,16 @@ pub(crate) enum MockHeaderCheck { } pub(crate) struct MockHttpClient { - routes: Vec, + routes: Vec, + request_count: AtomicUsize, } impl MockHttpClient { pub(crate) fn new() -> Self { - Self { routes: vec![] } + Self { + routes: vec![], + request_count: AtomicUsize::new(0), + } } pub(crate) fn on(mut self, method: HttpMethod, path: &str, status: u16, body: &str) -> Self { @@ -53,6 +59,10 @@ impl MockHttpClient { Some(serde_json::from_str(json_str).unwrap()); self } + + pub(crate) fn request_count(&self) -> usize { + self.request_count.load(Ordering::SeqCst) + } } impl HttpClient for MockHttpClient { @@ -63,6 +73,7 @@ impl HttpClient for MockHttpClient { headers: &[(&str, &str)], body: Option<&serde_json::Value>, ) -> anyhow::Result { + self.request_count.fetch_add(1, Ordering::SeqCst); for route in &self.routes { if method == route.method && url.ends_with(&route.path) { if let Some((name, MockHeaderCheck::Equals(expected))) = &route.assert_header { diff --git a/lib/components/fabro-github/src/token_source.rs b/lib/components/fabro-github/src/token_source.rs index d8c16228c..6e67f3423 100644 --- a/lib/components/fabro-github/src/token_source.rs +++ b/lib/components/fabro-github/src/token_source.rs @@ -146,31 +146,54 @@ pub(crate) trait InstallationTokenMinter: Send + Sync { async fn mint(&self) -> anyhow::Result; } -/// Real minter backed by GitHub App credentials. `repos` lists repository -/// names within the owner's installation, primary first; the minted token is -/// scoped to exactly that set. +/// Repository scope an App-backed source owns. +enum AppTokenScope { + /// A single-repository source that resolves its installation during each + /// mint. + Repository { + owner: String, + repo: String, + permissions: serde_json::Value, + }, + /// A validated declared set. Each mint resolves every target to one App + /// installation before creating the token. + Access(crate::GitHubRepositoryAccess), +} + +/// Real minter backed by GitHub App credentials. struct AppTokenMinter { - creds: GitHubAppCredentials, - http: fabro_http::HttpClient, - owner: String, - repos: Vec, - base_url: String, - permissions: serde_json::Value, + creds: GitHubAppCredentials, + http: fabro_http::HttpClient, + base_url: String, + scope: AppTokenScope, } #[async_trait::async_trait] impl InstallationTokenMinter for AppTokenMinter { async fn mint(&self) -> anyhow::Result { - self.creds - .mint_installation_token_for_repositories( - &self.http, - &self.owner, - &self.repos, - &self.base_url, - self.permissions.clone(), - None, - ) - .await + match &self.scope { + AppTokenScope::Repository { + owner, + repo, + permissions, + } => { + self.creds + .mint_installation_token( + &self.http, + owner, + repo, + &self.base_url, + permissions.clone(), + None, + ) + .await + } + AppTokenScope::Access(access) => { + access + .mint_installation_token(&self.creds, &self.http, &self.base_url) + .await + } + } } } @@ -222,6 +245,17 @@ pub struct InstallationTokenSource { state: SourceState, } +fn repository_set_display(owner: &str, repos: &[String]) -> anyhow::Result { + match repos { + [primary] => Ok(format!("{owner}/{primary}")), + [primary, additional @ ..] => Ok(format!( + "{owner}/{primary} (+{} additional)", + additional.len() + )), + [] => bail!("token source requires at least one repository"), + } +} + impl InstallationTokenSource { /// Build a source for `creds` against the repository in `origin_url`. /// @@ -245,38 +279,33 @@ impl InstallationTokenSource { repo: String, permissions: serde_json::Value, ) -> anyhow::Result> { - Self::for_repositories(creds, owner, vec![repo], permissions) + let repo_display = format!("{owner}/{repo}"); + Self::with_app_scope(creds, repo_display, AppTokenScope::Repository { + owner, + repo, + permissions, + }) } /// Build a source for a validated effective repository set. Minted /// tokens are scoped to every repository in the set with the shared - /// permissions; caching, refresh margin, and single-flight behavior are - /// identical to the single-repository source. + /// permissions. App-backed sources also resolve every repository to one + /// shared installation before each mint. Caching, refresh margin, and + /// single-flight behavior are identical to the single-repository source. pub fn for_access( creds: &GitHubCredentials, access: &crate::GitHubRepositoryAccess, ) -> anyhow::Result> { - Self::for_repositories( - creds, - access.owner().to_string(), - access.repository_names(), - access.permissions_json()?, - ) + let repository_names = access.repository_names(); + let repo_display = repository_set_display(access.owner(), &repository_names)?; + Self::with_app_scope(creds, repo_display, AppTokenScope::Access(access.clone())) } - fn for_repositories( + fn with_app_scope( creds: &GitHubCredentials, - owner: String, - repos: Vec, - permissions: serde_json::Value, + repo_display: String, + scope: AppTokenScope, ) -> anyhow::Result> { - let repo_display = match repos.as_slice() { - [primary] => format!("{owner}/{primary}"), - [primary, additional @ ..] => { - format!("{owner}/{primary} (+{} additional)", additional.len()) - } - [] => bail!("token source requires at least one repository"), - }; let state = match creds { GitHubCredentials::Pat(token) => SourceState::Pat(SecretString::new(token.clone())), GitHubCredentials::Installation(token) => SourceState::Installation(token.clone()), @@ -288,10 +317,8 @@ impl InstallationTokenSource { minter: Box::new(AppTokenMinter { creds: app.clone(), http, - owner, - repos, base_url: crate::github_api_base_url(), - permissions, + scope, }), cache: Mutex::new(None), } diff --git a/lib/components/fabro-github/tests/live_access.rs b/lib/components/fabro-github/tests/live_access.rs index d31dae934..0b13a582c 100644 --- a/lib/components/fabro-github/tests/live_access.rs +++ b/lib/components/fabro-github/tests/live_access.rs @@ -33,14 +33,8 @@ fn env_var(name: &str) -> String { std::env::var(name).unwrap_or_else(|_| panic!("{name} must be set for this live test")) } -fn private_key_pem() -> String { - GitHubAppCredentials::private_key_from_env() - .expect("GITHUB_APP_PRIVATE_KEY should decode as PEM or base64 PEM") - .expect("GITHUB_APP_PRIVATE_KEY must be set for this live test") -} - async fn ls_remote_with_token(slug: &GitHubRepositorySlug, token: &str) -> bool { - let url = format!("https://github.com/{}/{}", slug.owner(), slug.repo()); + let url = slug.https_url(); let mut command = Command::new("git"); fabro_github::apply_probe_git_env(&mut command, token); let output = command @@ -60,11 +54,10 @@ async fn ls_remote_with_token(slug: &GitHubRepositorySlug, token: &str) -> bool live("FABRO_TEST_GITHUB_ADDITIONAL_REPO") )] async fn scoped_token_reaches_the_declared_additional_repository() { - let app = GitHubAppCredentials { - app_id: env_var("FABRO_TEST_GITHUB_APP_ID"), - private_key_pem: private_key_pem(), - slug: None, - }; + let app_id = env_var("FABRO_TEST_GITHUB_APP_ID"); + let app = GitHubAppCredentials::from_env(Some(&app_id)) + .expect("GITHUB_APP_PRIVATE_KEY should decode as PEM or base64 PEM") + .expect("GITHUB_APP_PRIVATE_KEY must be set for this live test"); let origin = env_var("FABRO_TEST_GITHUB_ORIGIN"); let additional: GitHubRepositorySlug = env_var("FABRO_TEST_GITHUB_ADDITIONAL_REPO") .parse() @@ -84,10 +77,7 @@ async fn scoped_token_reaches_the_declared_additional_repository() { let creds = GitHubCredentials::App(app.clone()); let source = InstallationTokenSource::for_access(&creds, &access).expect("token source should build"); - let resolved = access - .resolve_verified_token(&creds, &source) - .await - .expect("scoped mint should succeed"); + let resolved = source.resolve().await.expect("scoped mint should succeed"); let token = resolved.token.expose(); // The one token reads both the primary and the additional repository. diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index a932ae826..7f4b6c5d3 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -1612,7 +1612,7 @@ impl Sandbox for DaytonaSandbox { })?; let clone_plan = git_retry::RetryPlan::clone_default(None); - let clone_result = git_retry::retry_clone( + let clone_result = git_retry::retry_git_operation( SandboxProviderKind::Daytona, "clone", &clone_plan, diff --git a/lib/components/fabro-sandbox/src/docker.rs b/lib/components/fabro-sandbox/src/docker.rs index c3842ef9a..55e60b295 100644 --- a/lib/components/fabro-sandbox/src/docker.rs +++ b/lib/components/fabro-sandbox/src/docker.rs @@ -811,7 +811,7 @@ impl DockerSandbox { auth_url: Option<&fabro_redact::DisplaySafeUrl>, ) -> Result<(), DockerCloneFailure> { let plan = git_retry::RetryPlan::clone_default(Some(clone_deadline)); - git_retry::retry_clone( + git_retry::retry_git_operation( SandboxProviderKind::Docker, op, &plan, diff --git a/lib/components/fabro-sandbox/src/git_retry.rs b/lib/components/fabro-sandbox/src/git_retry.rs index c0954bd6b..93e64973f 100644 --- a/lib/components/fabro-sandbox/src/git_retry.rs +++ b/lib/components/fabro-sandbox/src/git_retry.rs @@ -198,11 +198,8 @@ pub fn classify_failure(message: &str, cred: CredentialContext) -> Option BackoffPolicy { +/// window and spend an attempt for nothing. +fn replication_backoff() -> BackoffPolicy { BackoffPolicy { initial_delay: Duration::from_secs(3), factor: 3.0, @@ -232,6 +229,13 @@ pub struct RetryPlan { } impl RetryPlan { + /// Host-side repository probes use the same attempt count and pacing as + /// clone operations against a freshly minted token. + #[must_use] + pub fn repository_probe() -> Self { + Self::clone_default(None) + } + /// The clone policy both providers already trust: 3 attempts, 3s/9s /// backoff, no plan-level bounds. Docker supplies its existing absolute /// five-minute deadline through `outer_deadline`; Daytona supplies none. @@ -319,13 +323,13 @@ impl RetryPlan { } } -/// Run a clone operation, repeating it while the failure looks transient. +/// Run a git operation, repeating it while the failure looks transient. /// /// `attempt` receives the 1-based attempt number. `classify` decides whether /// an error is worth repeating; `None` returns it to the caller untouched. /// A retry starts only when its backoff fits before the plan's effective /// deadline. The final error is returned as-is. -pub(crate) async fn retry_clone( +pub async fn retry_git_operation( provider: SandboxProviderKind, op: &str, plan: &RetryPlan, @@ -598,7 +602,7 @@ mod tests { async fn first_success_runs_one_attempt() { let attempts = Attempts::default(); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "clone", &RetryPlan::clone_default(None), @@ -618,7 +622,7 @@ mod tests { async fn retries_until_a_later_attempt_succeeds() { let attempts = Attempts::default(); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "clone", &RetryPlan::clone_default(None), @@ -644,7 +648,7 @@ mod tests { async fn exhausted_attempts_return_the_final_error() { let attempts = Attempts::default(); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "clone", &RetryPlan::clone_default(None), @@ -668,7 +672,7 @@ mod tests { async fn unretryable_failure_stops_immediately() { let attempts = Attempts::default(); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "clone", &RetryPlan::clone_default(None), @@ -695,7 +699,7 @@ mod tests { let attempts = Attempts::default(); let deadline = time::Instant::now() + Duration::from_secs(2); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "clone", &RetryPlan::clone_default(Some(deadline)), @@ -718,7 +722,7 @@ mod tests { async fn unbounded_plan_runs_all_attempts() { let attempts = Attempts::default(); - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Daytona, "clone", &RetryPlan::clone_default(None), @@ -745,7 +749,7 @@ mod tests { outer_deadline: None, }; - let result = retry_clone( + let result = retry_git_operation( SandboxProviderKind::Docker, "push", &plan, diff --git a/lib/components/fabro-sandbox/src/lib.rs b/lib/components/fabro-sandbox/src/lib.rs index 4803c3fe8..3594aa0c0 100644 --- a/lib/components/fabro-sandbox/src/lib.rs +++ b/lib/components/fabro-sandbox/src/lib.rs @@ -44,7 +44,7 @@ pub use fabro_github::token_source::{ }; pub use fabro_types::{RunSandboxInstance, SandboxProviderKind}; pub use git_retry::{ - CredentialContext, GitRetryReason, RetryPlan, classify_failure, replication_backoff, + CredentialContext, GitRetryReason, RetryPlan, classify_failure, retry_git_operation, }; pub use local::LocalSandbox; #[cfg(feature = "daytona")] diff --git a/lib/components/fabro-workflow/src/pipeline/initialize.rs b/lib/components/fabro-workflow/src/pipeline/initialize.rs index 809eccdd8..286e8bb77 100644 --- a/lib/components/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/components/fabro-workflow/src/pipeline/initialize.rs @@ -143,20 +143,20 @@ fn build_sandbox_env( None }; - let github_token = match creds { - fabro_github::GitHubCredentials::Pat(token) => { - Some(InstallationTokenSource::pat(token.clone())) - } - fabro_github::GitHubCredentials::Installation(token) => { - Some(InstallationTokenSource::installation(token.clone())) - } - fabro_github::GitHubCredentials::App(_) => match github_access.as_ref() { - Some(access) => Some(InstallationTokenSource::for_access(creds, access).map_err( - |err| Error::engine_with_anyhow("Failed to build GitHub token source", err), - )?), - // No origin URL and nothing declared: keep the legacy + let github_token = match github_access.as_ref() { + Some(access) => Some(InstallationTokenSource::for_access(creds, access).map_err( + |err| Error::engine_with_anyhow("Failed to build GitHub token source", err), + )?), + None => match creds { + fabro_github::GitHubCredentials::Pat(token) => { + Some(InstallationTokenSource::pat(token.clone())) + } + fabro_github::GitHubCredentials::Installation(token) => { + Some(InstallationTokenSource::installation(token.clone())) + } + // No origin URL and nothing declared: keep the legacy App-mode // best-effort skip. - None => None, + fabro_github::GitHubCredentials::App(_) => None, }, }; @@ -174,40 +174,34 @@ fn build_sandbox_env( }) } -/// When additional repositories are declared, prove the whole effective set -/// is reachable before the first workflow stage -/// ([`fabro_github::GitHubRepositoryAccess::resolve_verified_token`]). -/// Legacy permissions-only runs skip this and keep their best-effort -/// behavior. -async fn validate_declared_repository_access( - built: &BuiltSandboxEnv, - github_app: Option<&fabro_github::GitHubCredentials>, -) -> Result<(), Error> { - let Some(access) = built +/// When additional repositories are declared, resolve their token before the +/// first workflow stage. App-backed sources first check that every target is +/// on one installation. Static credentials resolve locally; the first Git +/// operation remains their access check. Legacy permissions-only runs skip +/// eager resolution. +async fn resolve_declared_repository_token(built: &BuiltSandboxEnv) -> Result<(), Error> { + let Some(_) = built .github_access .as_ref() .filter(|access| access.has_additional_repositories()) else { return Ok(()); }; - // `build_sandbox_env` guarantees credentials and a token source whenever - // additional repositories are declared; fail closed if that ever breaks. - let (Some(creds), Some(source)) = (github_app, built.github_token.as_ref()) else { + // `build_sandbox_env` guarantees a token source whenever additional + // repositories are declared; fail closed if that ever breaks. + let Some(source) = built.github_token.as_ref() else { return Err(Error::Precondition( "run.integrations.github.additional_repositories requires GitHub credentials, but \ none are configured" .to_string(), )); }; - access - .resolve_verified_token(creds, source) - .await - .map_err(|err| { - Error::engine_with_anyhow( - "Failed to verify GitHub access for the declared repository set", - err, - ) - })?; + source.resolve().await.map_err(|err| { + Error::engine_with_anyhow( + "Failed to resolve the GitHub token for the declared repository set", + err, + ) + })?; Ok(()) } @@ -530,8 +524,7 @@ pub async fn initialize( &options.sandbox_env, options.run_options.github_app.as_ref(), )?; - validate_declared_repository_access(&built_env, options.run_options.github_app.as_ref()) - .await?; + resolve_declared_repository_token(&built_env).await?; let BuiltSandboxEnv { env: base_env, github_token, @@ -1617,7 +1610,7 @@ mod tests { mod github_integration_env { //! Focused tests for `build_sandbox_env` / - //! `validate_declared_repository_access` around declared additional + //! `resolve_declared_repository_token` around declared additional //! repositories. Installation-resolution failure naming is covered //! by `fabro_github::access` tests; these prove the initialization //! wiring: hard errors for declared sets, best-effort behavior for @@ -1758,10 +1751,7 @@ mod tests { github_access: access, }; - let creds = GitHubCredentials::Pat("ghp_x".to_string()); - let err = validate_declared_repository_access(&built, Some(&creds)) - .await - .unwrap_err(); + let err = resolve_declared_repository_token(&built).await.unwrap_err(); let message = err.to_string(); assert!(message.contains("declared repository set"), "{message}"); } @@ -1777,7 +1767,7 @@ mod tests { github_access: None, }; - validate_declared_repository_access(&built, None) + resolve_declared_repository_token(&built) .await .expect("legacy permissions-only runs must not resolve eagerly"); } diff --git a/lib/foundation/fabro-types/src/repository.rs b/lib/foundation/fabro-types/src/repository.rs index e7b29cef2..fcddc9bea 100644 --- a/lib/foundation/fabro-types/src/repository.rs +++ b/lib/foundation/fabro-types/src/repository.rs @@ -65,6 +65,12 @@ impl GitHubRepositorySlug { &self.repo } + /// Canonical credential-free HTTPS URL for this GitHub repository. + #[must_use] + pub fn https_url(&self) -> String { + format!("https://github.com/{self}") + } + /// Whether `other` names the same repository owner, ignoring ASCII case. /// Owner and repository names are validated ASCII, so ASCII folding is /// exact. @@ -356,6 +362,14 @@ mod tests { assert!(!ordered.insert(lower), "case variants share ordering"); } + #[test] + fn slug_https_url_preserves_spelling_and_has_no_credentials() { + let slug: GitHubRepositorySlug = "Fabro-SH/Keystone".parse().unwrap(); + + assert_eq!(slug.https_url(), "https://github.com/Fabro-SH/Keystone"); + assert!(!slug.https_url().contains('@')); + } + #[test] fn slug_ordering_sorts_by_canonical_form() { let mut slugs: Vec = ["owner/Zeta", "Owner/alpha", "owner/Beta"] From 6179470eb2c69ae9cd9f33c9070bf365afeec52d Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 17:25:58 -0400 Subject: [PATCH 24/30] feat: default sandbox clone depth to 100 --- docs/public/api-reference/fabro-api.yaml | 4 ++- docs/public/execution/environments.mdx | 2 +- docs/public/execution/run-configuration.mdx | 4 +-- docs/public/integrations/daytona.mdx | 4 +-- .../fabro-sandbox/src/clone_source.rs | 20 +++++++++++- lib/components/fabro-sandbox/src/config.rs | 15 ++++++++- .../fabro-sandbox/src/daytona/mod.rs | 2 +- lib/components/fabro-sandbox/src/docker.rs | 27 +++++++++++++--- .../fabro-sandbox/src/from_environment.rs | 3 +- .../fabro-workflow/src/operations/start.rs | 32 +++++++++++++++++++ .../fabro-config/src/resolve/run.rs | 25 ++++++++------- .../fabro-config/src/tests/resolve_run.rs | 26 ++++++++++++--- .../fabro-types/src/settings/run.rs | 19 +++++++++-- .../src/models/run-clone-settings.ts | 3 ++ 14 files changed, 152 insertions(+), 34 deletions(-) diff --git a/docs/public/api-reference/fabro-api.yaml b/docs/public/api-reference/fabro-api.yaml index dafddf42f..9e0655ff7 100644 --- a/docs/public/api-reference/fabro-api.yaml +++ b/docs/public/api-reference/fabro-api.yaml @@ -14636,7 +14636,9 @@ components: depth: type: integer format: int32 - minimum: 1 + minimum: 0 + default: 100 + description: Git history depth. Set to 0 to clone full history. RunBranchSettings: type: object diff --git a/docs/public/execution/environments.mdx b/docs/public/execution/environments.mdx index c58e602b9..a97509bd9 100644 --- a/docs/public/execution/environments.mdx +++ b/docs/public/execution/environments.mdx @@ -258,7 +258,7 @@ memory = "4GB" mode = "block" ``` -Docker and Daytona are clone-based providers. When a run has a GitHub origin, Fabro clones it into the provider workspace. Set `[run.clone] enabled = false` to start with an empty workspace, or set a positive `[run.clone] depth` to limit the downloaded Git history. Docker and Daytona ignore `cwd`; use the provider-owned workspace layout and `run.working_dir` for repository-relative commands. +Docker and Daytona are clone-based providers. When a run has a GitHub origin, Fabro clones it into the provider workspace with a history depth of 100. Set `[run.clone] enabled = false` to start with an empty workspace. Set `[run.clone] depth = 0` to clone full history. Docker and Daytona ignore `cwd`; use the provider-owned workspace layout and `run.working_dir` for repository-relative commands. The image must provide `/bin/bash`; Fabro evaluates every sandbox command with it and has no `sh` fallback. Commands run in a **non-login** shell, so login profiles (`/etc/profile.d/*.sh`, `~/.bash_profile`, and `nvm`/`rbenv`/`sdkman` initializers) are not sourced — put anything they set into the Dockerfile's `ENV` instead. Fabro verifies Bash during initialization and again on resume, and fails with remediation rather than reporting the sandbox ready. diff --git a/docs/public/execution/run-configuration.mdx b/docs/public/execution/run-configuration.mdx index 14c6c1ff7..f77137ffc 100644 --- a/docs/public/execution/run-configuration.mdx +++ b/docs/public/execution/run-configuration.mdx @@ -241,7 +241,7 @@ Configure whether clone-based sandboxes clone the run's GitHub origin before exe ```toml title="run.toml" [run.clone] enabled = true -depth = 1 +depth = 100 ``` Set `enabled = false` to start Docker and Daytona runs with an empty provider workspace. Use [prepare steps](#runprepare) to clone or create any files the workflow needs. @@ -249,7 +249,7 @@ Set `enabled = false` to start Docker and Daytona runs with an empty provider wo | Field | Description | |---|---| | `enabled` | When `false`, Fabro skips the repository clone. Defaults to `true`. | -| `depth` | Optional positive Git history depth. Applies to Docker and Daytona. If omitted, Daytona clones full history and Docker uses its default depth of 10. | +| `depth` | Git history depth for Docker and Daytona. Defaults to `100`. Set it to `0` to clone full history. | ### `[run.run_branch]` diff --git a/docs/public/integrations/daytona.mdx b/docs/public/integrations/daytona.mdx index 05fa190e9..828debc0d 100644 --- a/docs/public/integrations/daytona.mdx +++ b/docs/public/integrations/daytona.mdx @@ -141,14 +141,14 @@ provider = "daytona" enabled = false ``` -For a faster clone that keeps only the newest commit, set a clone depth: +Daytona clones 100 commits by default. To keep only the newest commit, set a smaller clone depth: ```toml title="run.toml" [run.clone] depth = 1 ``` -If `depth` is omitted, Daytona clones the full repository history. +Set `depth = 0` to clone the full repository history. If the clone fails without GitHub access configured, Fabro suggests running the setup flow: diff --git a/lib/components/fabro-sandbox/src/clone_source.rs b/lib/components/fabro-sandbox/src/clone_source.rs index ea343bbf4..964084eb2 100644 --- a/lib/components/fabro-sandbox/src/clone_source.rs +++ b/lib/components/fabro-sandbox/src/clone_source.rs @@ -93,8 +93,13 @@ pub(crate) fn exact_fetch_command( commit_sha: &str, depth: usize, ) -> String { + let depth_arg = if depth == 0 { + String::new() + } else { + format!(" --depth {depth}") + }; format!( - "{git} -C {} fetch --depth {depth} --no-tags {} -- {}", + "{git} -C {} fetch{depth_arg} --no-tags {} -- {}", sandbox::shell_quote(checkout_path), sandbox::shell_quote(fetch_source), sandbox::shell_quote(commit_sha), @@ -493,6 +498,19 @@ mod tests { ); } + #[test] + fn exact_fetch_omits_depth_for_full_history() { + assert_eq!( + exact_fetch_command( + "/repos/acme/widgets", + "origin", + "0123456789abcdef0123456789abcdef01234567", + 0, + ), + "git -c maintenance.auto=0 -c gc.auto=0 -C /repos/acme/widgets fetch --no-tags origin -- 0123456789abcdef0123456789abcdef01234567" + ); + } + #[test] fn exact_checkout_verification_rejects_invalid_or_mismatched_head() { let expected = "0123456789abcdef0123456789abcdef01234567"; diff --git a/lib/components/fabro-sandbox/src/config.rs b/lib/components/fabro-sandbox/src/config.rs index 66f55354f..bddc7c369 100644 --- a/lib/components/fabro-sandbox/src/config.rs +++ b/lib/components/fabro-sandbox/src/config.rs @@ -12,7 +12,7 @@ use std::collections::HashMap; use serde::de::{self, MapAccess, Visitor}; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] pub struct DaytonaSettings { pub auto_stop_interval: Option, pub labels: Option>, @@ -23,6 +23,19 @@ pub struct DaytonaSettings { pub skip_clone: bool, } +impl Default for DaytonaSettings { + fn default() -> Self { + Self { + auto_stop_interval: None, + labels: None, + snapshot: None, + network: None, + clone_depth: Some(100), + skip_clone: false, + } + } +} + #[derive(Clone, Debug, PartialEq)] pub enum DaytonaNetwork { Block, diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index 725f110c8..1d1d8da23 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -3528,7 +3528,7 @@ mod tests { assert!(config.snapshot.is_none()); assert!(config.auto_stop_interval.is_none()); assert!(config.labels.is_none()); - assert!(config.clone_depth.is_none()); + assert_eq!(config.clone_depth, Some(100)); } #[test] diff --git a/lib/components/fabro-sandbox/src/docker.rs b/lib/components/fabro-sandbox/src/docker.rs index 4bc512d04..97e965ed8 100644 --- a/lib/components/fabro-sandbox/src/docker.rs +++ b/lib/components/fabro-sandbox/src/docker.rs @@ -48,7 +48,7 @@ const DOCKER_BASH_REQUIREMENT: &str = "Docker sandboxes require /bin/bash for ev pub(crate) const WORKING_DIRECTORY: &str = "/workspace"; pub(crate) const REPOS_ROOT: &str = "/repos"; -const DEFAULT_GIT_CLONE_DEPTH: usize = 10; +const DEFAULT_GIT_CLONE_DEPTH: usize = 100; const GIT_CLONE_TIMEOUT: Duration = Duration::from_mins(5); #[cfg(test)] const EXEC_STOP_POLL_SLEEP_SECONDS: &str = "0.005"; @@ -121,7 +121,8 @@ pub struct DockerSandboxOptions { pub auto_pull: bool, /// Additional `KEY=VALUE` environment variables for the container. pub env_vars: Vec, - /// Maximum Git history depth fetched during clone. + /// Maximum Git history depth fetched during clone. Zero fetches full + /// history. pub clone_depth: usize, /// Create an empty workspace instead of cloning even when an origin exists. pub skip_clone: bool, @@ -1548,8 +1549,10 @@ fn git_clone_command( command.push_str(&shell_quote(branch)); command.push_str(" --single-branch"); } - command.push_str(" --depth "); - command.push_str(&depth.to_string()); + if depth > 0 { + command.push_str(" --depth "); + command.push_str(&depth.to_string()); + } command.push_str(" --no-tags"); command.push_str(" -- "); command.push_str(&shell_quote(clone_url)); @@ -2596,7 +2599,21 @@ mod tests { ); assert_eq!( command, - "git -c maintenance.auto=0 -c gc.auto=0 clone --depth 10 --no-tags -- https://github.com/fabro-sh/fabro /repos/fabro-sh/fabro" + "git -c maintenance.auto=0 -c gc.auto=0 clone --depth 100 --no-tags -- https://github.com/fabro-sh/fabro /repos/fabro-sh/fabro" + ); + } + + #[test] + fn clone_command_omits_depth_for_full_clone() { + let command = git_clone_command( + "https://github.com/fabro-sh/fabro", + Some("main"), + "/repos/fabro-sh/fabro", + 0, + ); + assert_eq!( + command, + "git -c maintenance.auto=0 -c gc.auto=0 clone --branch main --single-branch --no-tags -- https://github.com/fabro-sh/fabro /repos/fabro-sh/fabro" ); } diff --git a/lib/components/fabro-sandbox/src/from_environment.rs b/lib/components/fabro-sandbox/src/from_environment.rs index c73023354..4bf76696e 100644 --- a/lib/components/fabro-sandbox/src/from_environment.rs +++ b/lib/components/fabro-sandbox/src/from_environment.rs @@ -61,7 +61,7 @@ pub fn daytona_config_from_environment( DaytonaNetwork::AllowList(settings.network.allow.clone()) } }), - clone_depth: clone.depth, + clone_depth: clone.depth.filter(|depth| *depth > 0), skip_clone: !clone.enabled, } } @@ -135,7 +135,6 @@ fn docker_config_from_environment_env( clone_depth: clone .depth .and_then(|depth| usize::try_from(depth).ok()) - .filter(|depth| *depth > 0) .unwrap_or(default_options.clone_depth), skip_clone: !clone.enabled, ..DockerSandboxOptions::default() diff --git a/lib/components/fabro-workflow/src/operations/start.rs b/lib/components/fabro-workflow/src/operations/start.rs index 54ace936c..9955f1a8d 100644 --- a/lib/components/fabro-workflow/src/operations/start.rs +++ b/lib/components/fabro-workflow/src/operations/start.rs @@ -1305,6 +1305,38 @@ reasoning = false ); } + #[test] + fn zero_clone_depth_requests_full_history_from_clone_providers() { + let settings = settings_from_run_layer(RunLayer { + clone: Some(RunCloneLayer { + enabled: None, + depth: Some(0), + }), + ..RunLayer::default() + }); + + assert_eq!(resolve_daytona_config(&settings.run).clone_depth, None); + assert_eq!( + resolve_docker_config(&settings.run, |_| None) + .unwrap() + .clone_depth, + 0 + ); + } + + #[test] + fn clone_providers_default_to_depth_100() { + let settings = settings_from_run_layer(RunLayer::default()); + + assert_eq!(resolve_daytona_config(&settings.run).clone_depth, Some(100)); + assert_eq!( + resolve_docker_config(&settings.run, |_| None) + .unwrap() + .clone_depth, + 100 + ); + } + #[test] fn runtime_mcp_server_wraps_resolve_error_source() { let settings = ResolvedMcpServerSettings { diff --git a/lib/foundation/fabro-config/src/resolve/run.rs b/lib/foundation/fabro-config/src/resolve/run.rs index b078c8dad..ba64fa67d 100644 --- a/lib/foundation/fabro-config/src/resolve/run.rs +++ b/lib/foundation/fabro-config/src/resolve/run.rs @@ -248,21 +248,24 @@ fn resolve_clone( clone: Option<&RunCloneLayer>, errors: &mut Vec, ) -> RunCloneSettings { - let depth = clone.and_then(|clone| clone.depth).and_then(|depth| { - if depth < 1 { - errors.push(ResolveError::Invalid { - path: "run.clone.depth".to_string(), - reason: "depth must be at least 1".to_string(), + let depth = + clone + .and_then(|clone| clone.depth) + .map_or(RunCloneSettings::DEFAULT_DEPTH, |depth| { + if depth < 0 { + errors.push(ResolveError::Invalid { + path: "run.clone.depth".to_string(), + reason: "depth must be at least 0".to_string(), + }); + RunCloneSettings::DEFAULT_DEPTH + } else { + depth + } }); - None - } else { - Some(depth) - } - }); RunCloneSettings { enabled: clone.and_then(|clone| clone.enabled).unwrap_or(true), - depth, + depth: Some(depth), } } diff --git a/lib/foundation/fabro-config/src/tests/resolve_run.rs b/lib/foundation/fabro-config/src/tests/resolve_run.rs index 890dd2028..d66b075ea 100644 --- a/lib/foundation/fabro-config/src/tests/resolve_run.rs +++ b/lib/foundation/fabro-config/src/tests/resolve_run.rs @@ -126,7 +126,7 @@ fn resolves_run_defaults_from_empty_settings() { assert!(!settings.environment.lifecycle.preserve); assert!(settings.environment.lifecycle.stop_on_terminal); assert!(settings.clone.enabled); - assert_eq!(settings.clone.depth, None); + assert_eq!(settings.clone.depth, Some(100)); assert!(settings.run_branch.enabled); assert!(settings.run_branch.push); assert!(settings.meta_branch.enabled); @@ -320,8 +320,8 @@ push = false } #[test] -fn rejects_non_positive_clone_depth() { - let error = super::workflow_settings_from_toml( +fn zero_clone_depth_requests_full_history() { + let settings = super::workflow_settings_from_toml( r" _version = 1 @@ -329,7 +329,23 @@ _version = 1 depth = 0 ", ) - .expect_err("zero clone depth should not resolve"); + .expect("zero clone depth should resolve") + .run; + + assert_eq!(settings.clone.depth, Some(0)); +} + +#[test] +fn rejects_negative_clone_depth() { + let error = super::workflow_settings_from_toml( + r" +_version = 1 + +[run.clone] +depth = -1 +", + ) + .expect_err("negative clone depth should not resolve"); let message = error.to_string(); assert!( @@ -337,7 +353,7 @@ depth = 0 "unexpected error: {message}" ); assert!( - message.contains("at least 1"), + message.contains("at least 0"), "unexpected error: {message}" ); } diff --git a/lib/foundation/fabro-types/src/settings/run.rs b/lib/foundation/fabro-types/src/settings/run.rs index 5d96e9ce2..a47cb48fb 100644 --- a/lib/foundation/fabro-types/src/settings/run.rs +++ b/lib/foundation/fabro-types/src/settings/run.rs @@ -961,19 +961,34 @@ impl Default for RunCheckpointSettings { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RunCloneSettings { pub enabled: bool, - #[serde(default, skip_serializing_if = "Option::is_none")] + #[serde( + default = "default_clone_depth", + skip_serializing_if = "Option::is_none" + )] pub depth: Option, } +impl RunCloneSettings { + pub const DEFAULT_DEPTH: i32 = 100; +} + impl Default for RunCloneSettings { fn default() -> Self { Self { enabled: true, - depth: None, + depth: Some(Self::DEFAULT_DEPTH), } } } +#[expect( + clippy::unnecessary_wraps, + reason = "serde default provider must return the field's Option type" +)] +fn default_clone_depth() -> Option { + Some(RunCloneSettings::DEFAULT_DEPTH) +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RunBranchSettings { pub enabled: bool, diff --git a/lib/packages/fabro-api-client/src/models/run-clone-settings.ts b/lib/packages/fabro-api-client/src/models/run-clone-settings.ts index 48c5959df..79791f502 100644 --- a/lib/packages/fabro-api-client/src/models/run-clone-settings.ts +++ b/lib/packages/fabro-api-client/src/models/run-clone-settings.ts @@ -16,5 +16,8 @@ export interface RunCloneSettings { 'enabled': boolean; + /** + * Git history depth. Set to 0 to clone full history. + */ 'depth'?: number; } From 1669791956882590636089c244b84fdb3e61ceff Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 17:47:02 -0400 Subject: [PATCH 25/30] test: update clone depth snapshots --- lib/apps/fabro-cli/tests/it/cmd/attach.rs | 1 + lib/apps/fabro-cli/tests/it/cmd/inspect.rs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/apps/fabro-cli/tests/it/cmd/attach.rs b/lib/apps/fabro-cli/tests/it/cmd/attach.rs index 9f388c14b..93cd58c83 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/attach.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/attach.rs @@ -924,6 +924,7 @@ fn attach_json_errors_without_prompting_for_human_input() { "skip_git_hooks": false }, "clone": { + "depth": 100, "enabled": true }, "environment": { diff --git a/lib/apps/fabro-cli/tests/it/cmd/inspect.rs b/lib/apps/fabro-cli/tests/it/cmd/inspect.rs index d01e477e1..56821fe45 100644 --- a/lib/apps/fabro-cli/tests/it/cmd/inspect.rs +++ b/lib/apps/fabro-cli/tests/it/cmd/inspect.rs @@ -152,7 +152,8 @@ fn inspect_resolves_selector_via_server_endpoint() { "commit_timeout_ms": 30000 }, "clone": { - "enabled": true + "enabled": true, + "depth": 100 }, "run_branch": { "enabled": true, From fe1d9dc69138079fdcb4fbc04bcdb80b924ae7db Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 17:57:22 -0400 Subject: [PATCH 26/30] test: isolate SQLite checkpoint restoration --- .../fabro-store/src/legacy_blob_import.rs | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/lib/components/fabro-store/src/legacy_blob_import.rs b/lib/components/fabro-store/src/legacy_blob_import.rs index cf7f4390b..06199edc2 100644 --- a/lib/components/fabro-store/src/legacy_blob_import.rs +++ b/lib/components/fabro-store/src/legacy_blob_import.rs @@ -790,10 +790,10 @@ mod tests { type TestResult = std::result::Result>; struct TestContext { - _dir: tempfile::TempDir, + dir: tempfile::TempDir, source: Database, source_db: slatedb::Db, - sqlite: fabro_db::Database, + sqlite: sqlx::SqlitePool, target: BlobStore, } @@ -809,9 +809,10 @@ mod tests { let dir = tempfile::tempdir()?; let sqlite = fabro_db::Database::connect(dir.path().join("fabro.sqlite3")).await?; sqlite.migrate().await?; - let target = BlobStore::new(sqlite.clone_pool()); + let sqlite = sqlite.clone_pool(); + let target = BlobStore::new(sqlite.clone()); Ok(Self { - _dir: dir, + dir, source, source_db, sqlite, @@ -819,6 +820,22 @@ mod tests { }) } + async fn new_with_single_sqlite_connection() -> TestResult { + let mut context = Self::new().await?; + context.sqlite.close().await; + let options = SqliteConnectOptions::new() + .filename(context.dir.path().join("fabro.sqlite3")) + .foreign_keys(true) + .journal_mode(SqliteJournalMode::Wal); + let sqlite = SqlitePoolOptions::new() + .max_connections(1) + .connect_with(options) + .await?; + context.target = BlobStore::new(sqlite.clone()); + context.sqlite = sqlite; + Ok(context) + } + async fn put_blob(&self, bytes: &[u8]) -> TestResult { let hash = BlobHash::new(bytes); let key = SlateKey::new("blobs").with("sha256").with(hash); @@ -842,7 +859,7 @@ mod tests { async fn destination_rows(&self) -> TestResult { Ok(sqlx::query_scalar("SELECT COUNT(*) FROM blobs") - .fetch_one(self.sqlite.pool()) + .fetch_one(&self.sqlite) .await?) } @@ -850,7 +867,7 @@ mod tests { sqlx::query("INSERT INTO blobs (hash, data) VALUES (?, ?)") .bind(hash.to_string()) .bind(bytes) - .execute(self.sqlite.pool()) + .execute(&self.sqlite) .await?; Ok(()) } @@ -858,20 +875,20 @@ mod tests { async fn delete_destination(&self, hash: BlobHash) -> TestResult<()> { sqlx::query("DELETE FROM blobs WHERE hash = ?") .bind(hash.to_string()) - .execute(self.sqlite.pool()) + .execute(&self.sqlite) .await?; Ok(()) } async fn set_automatic_checkpoint(&self, pages: i64) -> TestResult<()> { - let mut connection = self.sqlite.pool().acquire().await?; + let mut connection = self.sqlite.acquire().await?; let statement = sqlx::AssertSqlSafe(format!("PRAGMA wal_autocheckpoint = {pages}")); sqlx::query(statement).execute(&mut *connection).await?; Ok(()) } async fn automatic_checkpoint(&self) -> TestResult { - let mut connection = self.sqlite.pool().acquire().await?; + let mut connection = self.sqlite.acquire().await?; Ok(sqlx::query_scalar("PRAGMA wal_autocheckpoint") .fetch_one(&mut *connection) .await?) @@ -1341,13 +1358,13 @@ mod tests { #[tokio::test] async fn automatic_checkpoint_setting_is_restored_after_success_and_failure() -> TestResult<()> { - let success = TestContext::new().await?; + let success = TestContext::new_with_single_sqlite_connection().await?; success.set_automatic_checkpoint(37).await?; success.put_blob(b"success").await?; success.import().await?; assert_eq!(success.automatic_checkpoint().await?, 37); - let failure = TestContext::new().await?; + let failure = TestContext::new_with_single_sqlite_connection().await?; failure.set_automatic_checkpoint(41).await?; let mut invalid_key = legacy_prefix(); invalid_key.extend_from_slice(&[b'z'; 64]); From 129fa0ea0c8e080bf498c8559b70989c672f87da Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 18:31:22 -0400 Subject: [PATCH 27/30] fix: fail runs when event persistence is lost --- docs/internal/events-strategy.md | 4 + lib/components/fabro-workflow/src/event.rs | 4 +- .../fabro-workflow/src/event/sink.rs | 146 ++++++++---- .../fabro-workflow/src/handler/agent.rs | 8 +- .../fabro-workflow/src/handler/command.rs | 16 +- .../fabro-workflow/src/handler/parallel.rs | 2 +- .../fabro-workflow/src/handler/prompt.rs | 2 +- .../fabro-workflow/src/operations/resume.rs | 2 +- .../fabro-workflow/src/operations/start.rs | 216 ++++++++++++++++-- .../src/pipeline/execute/tests.rs | 2 +- .../fabro-workflow/src/pipeline/finalize.rs | 6 +- .../fabro-workflow/src/pipeline/initialize.rs | 26 ++- .../fabro-workflow/src/test_support.rs | 18 +- 13 files changed, 364 insertions(+), 88 deletions(-) diff --git a/docs/internal/events-strategy.md b/docs/internal/events-strategy.md index 8059687c3..0d62bcaf6 100644 --- a/docs/internal/events-strategy.md +++ b/docs/internal/events-strategy.md @@ -186,4 +186,8 @@ Do not rebuild or mutate the `RunEvent` in downstream listeners. Any JSONL sink, the run store, and SSE should reflect the same canonical envelope bytes after redaction. +An active workflow treats any run-event sink write failure as fatal. It cancels execution and +attempts to persist `run.failed` through the direct sink path. Persistence-error logs must include +the full source chain so an HTTP status or transport failure remains visible. + `status.json` remains the authoritative completion signal for detached runs. Terminal run status should only be written after all post-run work is finished. diff --git a/lib/components/fabro-workflow/src/event.rs b/lib/components/fabro-workflow/src/event.rs index a5c1f583e..656b69922 100644 --- a/lib/components/fabro-workflow/src/event.rs +++ b/lib/components/fabro-workflow/src/event.rs @@ -18,7 +18,7 @@ pub use self::redaction::{ build_redacted_event_payload, event_payload_from_redacted_json, redacted_event_json, }; pub use self::sink::{ - RunEventLogger, RunEventSink, StoreProgressLogger, append_event, append_event_if, - append_event_to_sink, + RunEventLogger, RunEventPersistenceError, RunEventSink, StoreProgressLogger, append_event, + append_event_if, append_event_to_sink, }; pub use crate::stage_scope::StageScope; diff --git a/lib/components/fabro-workflow/src/event/sink.rs b/lib/components/fabro-workflow/src/event/sink.rs index 9588a3ded..e27a584b0 100644 --- a/lib/components/fabro-workflow/src/event/sink.rs +++ b/lib/components/fabro-workflow/src/event/sink.rs @@ -5,8 +5,9 @@ use std::sync::Arc; use ::fabro_types::{RunEvent, RunId, RunProjection}; use anyhow::Result; use fabro_store::RunDatabase; +use fabro_util::error::{SharedError, collect_chain}; use tokio::io::{AsyncWrite, AsyncWriteExt}; -use tokio::sync::{Mutex as AsyncMutex, mpsc, oneshot}; +use tokio::sync::{Mutex as AsyncMutex, mpsc, oneshot, watch}; use super::emitter::Emitter; use super::redaction::{build_redacted_event_payload, redacted_event_json}; @@ -149,86 +150,102 @@ impl RunEventSink { )] enum RunEventCommand { Event(RunEvent), - Flush(oneshot::Sender<()>), + Flush(oneshot::Sender>), +} + +#[derive(Clone, Debug, thiserror::Error)] +pub enum RunEventPersistenceError { + #[error("failed to persist run event {event} for run {run_id}")] + Write { + run_id: RunId, + event: String, + #[source] + source: SharedError, + }, + #[error("run event persistence task stopped")] + TaskStopped, } #[derive(Clone)] pub struct RunEventLogger { - tx: mpsc::UnboundedSender, + tx: mpsc::UnboundedSender, + failure_rx: watch::Receiver>, } impl RunEventLogger { #[must_use] pub fn new(sink: RunEventSink) -> Self { let (tx, mut rx) = mpsc::unbounded_channel(); + let (failure_tx, failure_rx) = watch::channel(None); tokio::spawn(async move { - // A dropped run event is unrecoverable history loss, so the first - // one is an ERROR worth investigating. A broken sink fails for - // every event that follows, so report the rest as a count at flush - // instead of one ERROR per event. Flush runs per stage and per - // agent turn, so only losses since the last summary are reported. - let mut write_failures: u64 = 0; - let mut summarized_failures: u64 = 0; + let mut persistence_failure = None; while let Some(command) = rx.recv().await { match command { RunEventCommand::Event(event) => { + if persistence_failure.is_some() { + continue; + } if let Err(err) = sink.write_run_event(&event).await { - write_failures += 1; - if write_failures == 1 { - tracing::error!( - run_id = %event.run_id, - event = %event.body.event_name(), - error = %err, - "Failed to write run event", - ); - } else { - tracing::debug!( - run_id = %event.run_id, - event = %event.body.event_name(), - failures = write_failures, - error = %err, - "Failed to write run event", - ); - } + let rendered_error = collect_chain(err.as_ref()).join(": "); + tracing::error!( + run_id = %event.run_id, + event = %event.body.event_name(), + error = %rendered_error, + "Failed to persist run event; stopping workflow", + ); + let failure = RunEventPersistenceError::Write { + run_id: event.run_id, + event: event.body.event_name().to_string(), + source: SharedError::new(err), + }; + persistence_failure = Some(failure.clone()); + failure_tx.send_replace(Some(failure)); } } RunEventCommand::Flush(tx) => { - if write_failures > summarized_failures { - tracing::error!( - lost = write_failures - summarized_failures, - total = write_failures, - "Run events were lost to write failures", - ); - summarized_failures = write_failures; - } - let _ = tx.send(()); + let result = persistence_failure.clone().map_or(Ok(()), Err); + let _ = tx.send(result); } } } }); - Self { tx } + Self { tx, failure_rx } } pub fn register(&self, emitter: &Emitter) { let tx = self.tx.clone(); emitter.on_event(move |event| { if tx.send(RunEventCommand::Event(event.clone())).is_err() { - tracing::warn!("Run event logger channel closed while forwarding event"); + tracing::error!( + run_id = %event.run_id, + event = %event.body.event_name(), + "Run event persistence task stopped while forwarding event", + ); } }); } - pub async fn flush(&self) { + pub async fn wait_for_failure(&self) -> RunEventPersistenceError { + let mut failure_rx = self.failure_rx.clone(); + loop { + if let Some(failure) = failure_rx.borrow_and_update().clone() { + return failure; + } + if failure_rx.changed().await.is_err() { + return RunEventPersistenceError::TaskStopped; + } + } + } + + pub async fn flush(&self) -> Result<(), RunEventPersistenceError> { let (tx, rx) = oneshot::channel(); if self.tx.send(RunEventCommand::Flush(tx)).is_err() { - tracing::warn!("Run event logger channel closed before flush"); - return; - } - if rx.await.is_err() { - tracing::warn!("Run event logger flush dropped before completion"); + return Err(RunEventPersistenceError::TaskStopped); } + rx.await + .map_err(|_| RunEventPersistenceError::TaskStopped)? } } @@ -249,14 +266,15 @@ impl StoreProgressLogger { self.inner.register(emitter); } - pub async fn flush(&self) { - self.inner.flush().await; + pub async fn flush(&self) -> Result<(), RunEventPersistenceError> { + self.inner.flush().await } } #[cfg(test)] mod tests { use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; use ::fabro_types::{Graph, RunNoticeLevel, WorkflowSettings, fixtures}; use fabro_types::test_support; @@ -439,7 +457,7 @@ mod tests { logger.register(&emitter); emitter.emit(&Event::RunPaused); - logger.flush().await; + logger.flush().await.unwrap(); let mut reader = BufReader::new(reader); let mut line = String::new(); @@ -448,4 +466,38 @@ mod tests { let payload = event_payload_from_redacted_json(line.trim_end(), &fixtures::RUN_8).unwrap(); assert_eq!(payload.as_value()["event"], "run.paused"); } + + #[tokio::test] + async fn run_event_logger_latches_write_failure_and_preserves_cause_chain() { + let writes = Arc::new(AtomicUsize::new(0)); + let writes_for_sink = Arc::clone(&writes); + let sink = RunEventSink::callback(move |_| { + writes_for_sink.fetch_add(1, Ordering::SeqCst); + async { + Err( + anyhow::anyhow!("request failed with status 413 Payload Too Large") + .context("worker lost canonical run store during append run event"), + ) + } + }); + let logger = RunEventLogger::new(sink); + let emitter = Emitter::new(fixtures::RUN_8); + logger.register(&emitter); + + emitter.emit(&Event::RunPaused); + + let failure = logger.wait_for_failure().await; + let rendered = collect_chain(&failure).join(": "); + assert!(rendered.contains("run.paused"), "{rendered}"); + assert!( + rendered.contains("worker lost canonical run store"), + "{rendered}" + ); + assert!(rendered.contains("413 Payload Too Large"), "{rendered}"); + + emitter.emit(&Event::RunUnpaused); + let flush_failure = logger.flush().await.unwrap_err(); + assert_eq!(collect_chain(&flush_failure), collect_chain(&failure)); + assert_eq!(writes.load(Ordering::SeqCst), 1); + } } diff --git a/lib/components/fabro-workflow/src/handler/agent.rs b/lib/components/fabro-workflow/src/handler/agent.rs index b3c67612b..b262cb976 100644 --- a/lib/components/fabro-workflow/src/handler/agent.rs +++ b/lib/components/fabro-workflow/src/handler/agent.rs @@ -630,7 +630,7 @@ mod tests { .execute(&node, &context, &graph, tmp.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let state = run_store.state().await.unwrap(); let node_state = state.stage(&StageId::new("plan", 1)).unwrap(); @@ -657,7 +657,7 @@ mod tests { .execute(&node, &context, &graph, tmp.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let state = run_store.state().await.unwrap(); let node_state = state.stage(&StageId::new("work", 1)).unwrap(); @@ -1151,7 +1151,7 @@ All checks passed. .execute(&node, &context, &graph, tmp.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let state = run_store.state().await.unwrap(); let node_state = state.stage(&StageId::new("step", 1)).unwrap(); @@ -1555,7 +1555,7 @@ Some text in between. .execute(&node, &context, &graph, tmp.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let state = run_store.state().await.unwrap(); let node_state = state.stage(&StageId::new("report", 1)).unwrap(); diff --git a/lib/components/fabro-workflow/src/handler/command.rs b/lib/components/fabro-workflow/src/handler/command.rs index c24353ccd..49c43230c 100644 --- a/lib/components/fabro-workflow/src/handler/command.rs +++ b/lib/components/fabro-workflow/src/handler/command.rs @@ -1077,7 +1077,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1108,7 +1108,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1135,7 +1135,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1162,7 +1162,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1187,7 +1187,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1212,7 +1212,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1242,7 +1242,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap_err(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node_state = snapshot.stage(&StageId::new("script_node", 1)).unwrap(); @@ -1269,7 +1269,7 @@ mod tests { .execute(&node, &context, &graph, run_dir.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let snapshot = run_store.state().await.unwrap(); let node = snapshot diff --git a/lib/components/fabro-workflow/src/handler/parallel.rs b/lib/components/fabro-workflow/src/handler/parallel.rs index 1d245659f..fed0fd03e 100644 --- a/lib/components/fabro-workflow/src/handler/parallel.rs +++ b/lib/components/fabro-workflow/src/handler/parallel.rs @@ -1405,7 +1405,7 @@ mod tests { .execute(&node, &context, &graph, Path::new("/tmp/test"), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); assert_eq!(outcome.status, StageOutcome::Succeeded); let results: Vec = diff --git a/lib/components/fabro-workflow/src/handler/prompt.rs b/lib/components/fabro-workflow/src/handler/prompt.rs index e61a2809c..f2d3df535 100644 --- a/lib/components/fabro-workflow/src/handler/prompt.rs +++ b/lib/components/fabro-workflow/src/handler/prompt.rs @@ -570,7 +570,7 @@ mod tests { .execute(&node, &context, &graph, tmp.path(), &services) .await .unwrap(); - logger.flush().await; + logger.flush().await.unwrap(); let state = run_store.state().await.unwrap(); let node_state = state.stage(&StageId::new("classify", 1)).unwrap(); diff --git a/lib/components/fabro-workflow/src/operations/resume.rs b/lib/components/fabro-workflow/src/operations/resume.rs index 433947716..1911c4a66 100644 --- a/lib/components/fabro-workflow/src/operations/resume.rs +++ b/lib/components/fabro-workflow/src/operations/resume.rs @@ -44,7 +44,7 @@ pub async fn resume(run_dir: &Path, services: StartServices) -> Result Result Result Error { + cancel_token.cancel(); + Error::engine_with_source("run event persistence failed", error) +} + impl RunSession { async fn new(persisted: &Persisted, services: StartServices) -> Result { let record = persisted.run_spec(); @@ -691,6 +709,7 @@ impl RunSession { resume: Option, ) -> Result { let on_node = self.on_node.clone(); + let run_cancel_token = self.cancel_token.clone(); let record = persisted.run_spec(); let run_options = RunOptions { @@ -779,7 +798,28 @@ impl RunSession { seed_context: self.seed_context, fabro_run_tools: self.fabro_run_tools, }; - let mut initialized = Box::pin(pipeline::initialize(persisted, init_options)).await?; + let mut initializing = Box::pin(pipeline::initialize(persisted, init_options)); + let initialized = tokio::select! { + result = &mut initializing => result, + failure = store_progress_logger.wait_for_failure() => { + return Err(stop_for_run_event_persistence_failure( + &run_cancel_token, + failure, + )); + } + }; + let mut initialized = match initialized { + Ok(initialized) => initialized, + Err(err) => { + if let Err(failure) = store_progress_logger.flush().await { + return Err(stop_for_run_event_persistence_failure( + &run_cancel_token, + failure, + )); + } + return Err(err); + } + }; initialized.on_node = on_node; let sandbox_for_cleanup = Arc::clone(&initialized.engine.run.sandbox); @@ -803,8 +843,23 @@ impl RunSession { steering_hub_for_drain.drain_pending_at_run_end(); }); - let executed = pipeline::execute(initialized).await; - store_progress_logger.flush().await; + store_progress_logger.flush().await.map_err(|failure| { + stop_for_run_event_persistence_failure(&run_cancel_token, failure) + })?; + + let mut executing = Box::pin(pipeline::execute(initialized)); + let executed = tokio::select! { + executed = &mut executing => executed, + failure = store_progress_logger.wait_for_failure() => { + return Err(stop_for_run_event_persistence_failure( + &run_cancel_token, + failure, + )); + } + }; + store_progress_logger.flush().await.map_err(|failure| { + stop_for_run_event_persistence_failure(&run_cancel_token, failure) + })?; let final_context = Some(executed.final_context.clone()); let finalize_opts = FinalizeOptions { @@ -824,16 +879,30 @@ impl RunSession { model: self.pr_model, }; - let concluding = async { + let mut concluding = Box::pin(async { let concluded = Box::pin(pipeline::conclude(executed, &finalize_opts)).await?; let published = Box::pin(pipeline::publish(concluded, &publish_opts)).await; Box::pin(pipeline::finalize(published, &finalize_opts)).await + }); + let concluding = tokio::select! { + result = &mut concluding => result, + failure = store_progress_logger.wait_for_failure() => { + return Err(stop_for_run_event_persistence_failure( + &run_cancel_token, + failure, + )); + } }; - let finalized = match concluding.await { + let finalized = match concluding { Ok(finalized) => finalized, Err(err) => { self.steering_hub.drain_pending_at_run_end(); - store_progress_logger.flush().await; + if let Err(failure) = store_progress_logger.flush().await { + return Err(stop_for_run_event_persistence_failure( + &run_cancel_token, + failure, + )); + } return Err(err); } }; @@ -842,7 +911,9 @@ impl RunSession { // scopeguard above re-runs as a no-op (drain is idempotent on an // already-empty buffer) on the way out of scope. self.steering_hub.drain_pending_at_run_end(); - store_progress_logger.flush().await; + store_progress_logger.flush().await.map_err(|failure| { + stop_for_run_event_persistence_failure(&run_cancel_token, failure) + })?; scopeguard::ScopeGuard::into_inner(cleanup_guard); @@ -1002,13 +1073,20 @@ impl Drop for DetachedRunCompletionGuard { 0, ) .await; - let _ = append_event_to_sink(&event_sink, &run_id, &Event::RunNotice { + if let Err(err) = append_event_to_sink(&event_sink, &run_id, &Event::RunNotice { level: RunNoticeLevel::Error, code: code.to_string(), message: message.to_string(), exec_output_tail: None, }) - .await; + .await + { + let rendered_error = collect_chain(err.as_ref()).join(": "); + tracing::warn!( + error = %rendered_error, + "Failed to append detached completion notice", + ); + } }); } } @@ -1032,7 +1110,11 @@ async fn persist_detached_failure( exec_output_tail: None, }; if let Err(err) = append_event_to_sink(event_sink, &run_id, &event).await { - tracing::warn!(error = %err, "Failed to append detached failure notice"); + let rendered_error = collect_chain(err.as_ref()).join(": "); + tracing::warn!( + error = %rendered_error, + "Failed to append detached failure notice", + ); } Ok(()) @@ -1091,8 +1173,19 @@ mod tests { work -> exit }"#; + const BLOCKING_DOT: &str = r#"digraph Test { + graph [goal="Wait forever"] + start [shape=Mdiamond] + block [type="blocking"] + exit [shape=Msquare] + start -> block + block -> exit + }"#; + struct TimedOutcomeHandler; + struct BlockingHandler; + fn timed_success_outcome() -> Outcome { let mut outcome = Outcome::success(); outcome.timing = Some(StageTiming::new(0, 100, 50)); @@ -1124,6 +1217,31 @@ mod tests { } } + #[async_trait::async_trait] + impl Handler for BlockingHandler { + async fn execute( + &self, + _node: &fabro_graphviz::graph::Node, + _context: &Context, + _graph: &fabro_graphviz::graph::Graph, + _run_dir: &Path, + _services: &EngineServices, + ) -> Result { + std::future::pending().await + } + + async fn simulate( + &self, + _node: &fabro_graphviz::graph::Node, + _context: &Context, + _graph: &fabro_graphviz::graph::Graph, + _run_dir: &Path, + _services: &EngineServices, + ) -> Result { + std::future::pending().await + } + } + fn memory_store() -> Arc { Arc::new(Database::new( Arc::new(InMemory::new()), @@ -2087,6 +2205,74 @@ reasoning = false assert!(run_store.state().await.unwrap().conclusion.is_some()); } + #[tokio::test] + async fn event_persistence_failure_stops_execution_and_fails_run() { + let temp = tempfile::tempdir().unwrap(); + let (storage_root, run_dir) = storage_root_and_run_dir(&temp); + let emitter = Arc::new(Emitter::new(fixtures::RUN_1)); + let mut registry = test_registry(); + registry.register("blocking", Box::new(BlockingHandler)); + let (_persisted, store) = persisted_workflow(BLOCKING_DOT, &storage_root).await; + let run_store = store.open_run(&fixtures::RUN_1).await.unwrap(); + let canonical_sink = RunEventSink::store(run_store.clone()); + let mut services = test_start_services(&store, &run_dir, emitter, Arc::new(registry)).await; + let cancel_token = services.cancel_token.clone(); + services.event_sink = RunEventSink::callback(move |event| { + let canonical_sink = canonical_sink.clone(); + async move { + if matches!(&event.body, EventBody::StageStarted(_)) + && event.node_id.as_deref() == Some("block") + { + return Err(anyhow::anyhow!( + "request failed with status 413 Payload Too Large" + ) + .context("worker lost canonical run store during append run event")); + } + canonical_sink.write_run_event(&event).await + } + }); + + let result = tokio::time::timeout(Duration::from_secs(2), start(&run_dir, services)) + .await + .expect("event persistence failure should stop the blocking stage"); + let Err(error) = result else { + panic!("event persistence failure should fail the run"); + }; + + assert!(cancel_token.is_cancelled()); + let rendered = error.display_with_causes(); + assert!( + rendered.contains("run event persistence failed"), + "{rendered}" + ); + assert!(rendered.contains("stage.started"), "{rendered}"); + assert!(rendered.contains("413 Payload Too Large"), "{rendered}"); + + let projection = run_store.state().await.unwrap(); + assert!(matches!(projection.status, RunStatus::Failed { .. })); + let events = run_store.list_events().await.unwrap(); + let run_failed = events + .iter() + .find_map(|event| match &event.event.body { + EventBody::RunFailed(properties) => Some(properties), + _ => None, + }) + .expect("persistence failure should emit run.failed"); + assert!( + run_failed + .failure + .detail + .causes + .iter() + .any(|cause| cause.contains("413 Payload Too Large")) + ); + assert!( + events + .iter() + .all(|event| !matches!(&event.event.body, EventBody::RunCompleted(_))) + ); + } + #[tokio::test] async fn start_can_run_bundle_backed_child_workflow_without_workflow_bundle_json() { let temp = tempfile::tempdir().unwrap(); diff --git a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs index 8056192bd..de726ea78 100644 --- a/lib/components/fabro-workflow/src/pipeline/execute/tests.rs +++ b/lib/components/fabro-workflow/src/pipeline/execute/tests.rs @@ -304,7 +304,7 @@ async fn execute_test_run_with_options( .unwrap(); let executed = execute(initialized).await; - store_logger.flush().await; + store_logger.flush().await.unwrap(); executed } diff --git a/lib/components/fabro-workflow/src/pipeline/finalize.rs b/lib/components/fabro-workflow/src/pipeline/finalize.rs index 18b7ea15b..76dde359f 100644 --- a/lib/components/fabro-workflow/src/pipeline/finalize.rs +++ b/lib/components/fabro-workflow/src/pipeline/finalize.rs @@ -1114,8 +1114,8 @@ mod tests { let temp = tempfile::tempdir().unwrap(); let run_dir = temp.path().join("run"); std::fs::create_dir_all(&run_dir).unwrap(); - let inner_store = test_store().create_run(&test_run_id()).await.unwrap(); - let run_store = inner_store; + let run_store = seeded_run_store().await; + crate::test_support::mark_run_running(&run_store, &test_run_id()).await; let emitter = Arc::new(Emitter::new(test_run_id())); let store_logger = StoreProgressLogger::new(run_store.clone()); store_logger.register(&emitter); @@ -1158,7 +1158,7 @@ mod tests { }) .await .unwrap(); - store_logger.flush().await; + store_logger.flush().await.unwrap(); assert_eq!(concluded.conclusion.status, StageOutcome::Succeeded); } diff --git a/lib/components/fabro-workflow/src/pipeline/initialize.rs b/lib/components/fabro-workflow/src/pipeline/initialize.rs index a91929c7b..7706ae411 100644 --- a/lib/components/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/components/fabro-workflow/src/pipeline/initialize.rs @@ -1326,10 +1326,32 @@ mod tests { let run_dir = temp.path().join("run"); std::fs::create_dir_all(&run_dir).unwrap(); let (graph, source) = simple_graph(); - let persisted = test_persisted(graph, source, &run_dir); + let persisted = test_persisted(graph.clone(), source, &run_dir); let emitter = Arc::new(crate::event::Emitter::new(test_run_id())); let store = memory_store(); let run_store = store.create_run(&test_run_id()).await.unwrap(); + crate::event::append_event(&run_store, &test_run_id(), &Event::RunCreated { + run_id: test_run_id(), + title: None, + settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), + graph: serde_json::to_value(graph).unwrap(), + workflow_source: None, + labels: BTreeMap::new(), + source_directory: None, + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref: None, + retried_from: None, + parent_id: None, + web_url: None, + }) + .await + .unwrap(); let store_logger = StoreProgressLogger::new(run_store.clone()); let seen = Arc::new(std::sync::Mutex::new(Vec::new())); emitter.on_event({ @@ -1380,7 +1402,7 @@ mod tests { }) .await .unwrap(); - store_logger.flush().await; + store_logger.flush().await.unwrap(); assert_eq!(initialized.run_options.run_dir, run_dir); assert!( diff --git a/lib/components/fabro-workflow/src/test_support.rs b/lib/components/fabro-workflow/src/test_support.rs index 55f7f9cd4..6c9f55b99 100644 --- a/lib/components/fabro-workflow/src/test_support.rs +++ b/lib/components/fabro-workflow/src/test_support.rs @@ -52,7 +52,11 @@ pub(crate) fn test_configured_provider_ids( /// persisted before tests reopen the run store. async fn execute_and_emit_terminal(initialized: InitializedState) -> Executed { let executed = Box::pin(pipeline::execute(initialized.initialized)).await; - initialized.store_logger.flush().await; + initialized + .store_logger + .flush() + .await + .expect("test run events should persist"); let state = executed.engine.run.run_store.state().await.ok(); let billing = state.as_ref().and_then(billing_from_projection); let event = build_terminal_event( @@ -65,7 +69,11 @@ async fn execute_and_emit_terminal(initialized: InitializedState) -> Executed { billing, ); executed.engine.run.emitter.emit(&event); - initialized.store_logger.flush().await; + initialized + .store_logger + .flush() + .await + .expect("test run events should persist"); executed } @@ -481,7 +489,11 @@ pub async fn run_graph_with_state_and_llm_source( ) .await; let executed = pipeline::execute(initialized.initialized).await; - initialized.store_logger.flush().await; + initialized + .store_logger + .flush() + .await + .expect("test run events should persist"); let outcome = executed.outcome?; let state = executed .engine From 09f5bb0f84f11a3174edd7ba8eca436d676e3e5a Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 18:37:39 -0400 Subject: [PATCH 28/30] Simplify clone depth plumbing Make RunCloneSettings::DEFAULT_DEPTH the single owner of the default depth, and interpret the "0 = full history" sentinel in one place via RunCloneSettings::depth_limit(). Docker's clone_depth becomes Option to match Daytona's encoding, with a shared depth_argument() helper for both git command builders. Drop the unreachable Option on the resolved depth field, the hand-written DaytonaSettings::Default, and the pure-forwarding daytona_git_clone_options helper. The blob-import test helper reuses the pool's own connect options instead of rebuilding a partial copy. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019bgXj5J218RXfiT72qhbLV --- .../fabro-sandbox/src/clone_source.rs | 20 +++---- lib/components/fabro-sandbox/src/config.rs | 17 ++---- .../fabro-sandbox/src/daytona/mod.rs | 54 ++++--------------- lib/components/fabro-sandbox/src/docker.rs | 24 ++++----- .../fabro-sandbox/src/from_environment.rs | 7 ++- .../fabro-store/src/legacy_blob_import.rs | 9 ++-- .../fabro-workflow/src/operations/start.rs | 6 +-- .../fabro-config/src/resolve/run.rs | 26 ++++----- .../fabro-config/src/tests/resolve_run.rs | 6 +-- .../fabro-types/src/settings/run.rs | 22 ++++---- 10 files changed, 67 insertions(+), 124 deletions(-) diff --git a/lib/components/fabro-sandbox/src/clone_source.rs b/lib/components/fabro-sandbox/src/clone_source.rs index 964084eb2..9f03a93e3 100644 --- a/lib/components/fabro-sandbox/src/clone_source.rs +++ b/lib/components/fabro-sandbox/src/clone_source.rs @@ -91,13 +91,9 @@ pub(crate) fn exact_fetch_command( checkout_path: &str, fetch_source: &str, commit_sha: &str, - depth: usize, + depth: Option, ) -> String { - let depth_arg = if depth == 0 { - String::new() - } else { - format!(" --depth {depth}") - }; + let depth_arg = depth_argument(depth); format!( "{git} -C {} fetch{depth_arg} --no-tags {} -- {}", sandbox::shell_quote(checkout_path), @@ -107,6 +103,12 @@ pub(crate) fn exact_fetch_command( ) } +/// Leading-space ` --depth N` fragment for a Git command, or empty when +/// `depth` is `None` to fetch full history. +pub(crate) fn depth_argument(depth: Option) -> String { + depth.map_or_else(String::new, |depth| format!(" --depth {depth}")) +} + /// Point the admitted branch at `revision` and attach HEAD to it. /// /// The checkout attaches to a real branch instead of detaching so callers that @@ -479,7 +481,7 @@ mod tests { "/repos/acme's widgets", "https://token@example.com/acme/widgets.git?x=a b", sha, - 10, + Some(10), ); let checkout = exact_checkout_verify_command("/repos/acme's widgets", "feature/a b", "FETCH_HEAD"); @@ -505,7 +507,7 @@ mod tests { "/repos/acme/widgets", "origin", "0123456789abcdef0123456789abcdef01234567", - 0, + None, ), "git -c maintenance.auto=0 -c gc.auto=0 -C /repos/acme/widgets fetch --no-tags origin -- 0123456789abcdef0123456789abcdef01234567" ); @@ -572,7 +574,7 @@ mod tests { ); run_shell( temp.path(), - &exact_fetch_command(checkout_path, remote_path, &admitted_sha, 10), + &exact_fetch_command(checkout_path, remote_path, &admitted_sha, Some(10)), ); let checked_out_sha = run_shell( temp.path(), diff --git a/lib/components/fabro-sandbox/src/config.rs b/lib/components/fabro-sandbox/src/config.rs index bddc7c369..208278d18 100644 --- a/lib/components/fabro-sandbox/src/config.rs +++ b/lib/components/fabro-sandbox/src/config.rs @@ -12,30 +12,19 @@ use std::collections::HashMap; use serde::de::{self, MapAccess, Visitor}; use serde::{Deserialize, Serialize}; -#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)] +#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)] pub struct DaytonaSettings { pub auto_stop_interval: Option, pub labels: Option>, pub snapshot: Option, pub network: Option, + /// Git history depth for the repository clone; `None` clones full + /// history. pub clone_depth: Option, #[serde(default)] pub skip_clone: bool, } -impl Default for DaytonaSettings { - fn default() -> Self { - Self { - auto_stop_interval: None, - labels: None, - snapshot: None, - network: None, - clone_depth: Some(100), - skip_clone: false, - } - } -} - #[derive(Clone, Debug, PartialEq)] pub enum DaytonaNetwork { Block, diff --git a/lib/components/fabro-sandbox/src/daytona/mod.rs b/lib/components/fabro-sandbox/src/daytona/mod.rs index 1d1d8da23..0081a05ce 100644 --- a/lib/components/fabro-sandbox/src/daytona/mod.rs +++ b/lib/components/fabro-sandbox/src/daytona/mod.rs @@ -101,23 +101,6 @@ const DAYTONA_STATE_CHANGE_POLL_INTERVAL: Duration = Duration::from_secs(1); /// leaked by a dead worker. An explicit `0` disables auto-stop entirely. const DEFAULT_AUTO_STOP_INTERVAL_MINUTES: i32 = 120; -fn daytona_git_clone_options( - branch: Option, - commit_id: Option, - username: Option, - password: Option, - depth: Option, -) -> GitCloneOptions { - GitCloneOptions { - branch, - commit_id, - username, - password, - depth, - ..GitCloneOptions::default() - } -} - pub(crate) fn daytona_not_found(err: &DaytonaError) -> bool { matches!(err, DaytonaError::NotFound { .. }) || err.status_code() == Some(404) } @@ -1623,13 +1606,14 @@ impl Sandbox for DaytonaSandbox { let git_svc = &git_svc; let origin = origin_url.as_str(); let target = layout.primary_repo_path.as_str(); - let options = daytona_git_clone_options( - branch.clone(), - commit_sha.clone(), - username.clone(), - password.clone(), - self.config.clone_depth, - ); + let options = GitCloneOptions { + branch: branch.clone(), + commit_id: commit_sha.clone(), + username: username.clone(), + password: password.clone(), + depth: self.config.clone_depth, + ..GitCloneOptions::default() + }; async move { git_svc.clone(origin, target, options).await } }, |err: &DaytonaError| classify_clone_failure(err, clone_credential_context), @@ -3142,26 +3126,6 @@ mod tests { assert!(!error.to_string().contains("Daytona client")); } - #[test] - fn exact_checkout_uses_daytona_branch_and_commit_options() { - let options = daytona_git_clone_options( - Some("feature/work".to_string()), - Some("0123456789abcdef0123456789abcdef01234567".to_string()), - Some("x-access-token".to_string()), - Some("secret".to_string()), - Some(1), - ); - - assert_eq!(options.branch.as_deref(), Some("feature/work")); - assert_eq!( - options.commit_id.as_deref(), - Some("0123456789abcdef0123456789abcdef01234567") - ); - assert_eq!(options.username.as_deref(), Some("x-access-token")); - assert_eq!(options.password.as_deref(), Some("secret")); - assert_eq!(options.depth, Some(1)); - } - fn mock_sandbox_body(sandbox_id: &str) -> serde_json::Value { serde_json::json!({ "id": sandbox_id, @@ -3528,7 +3492,7 @@ mod tests { assert!(config.snapshot.is_none()); assert!(config.auto_stop_interval.is_none()); assert!(config.labels.is_none()); - assert_eq!(config.clone_depth, Some(100)); + assert!(config.clone_depth.is_none()); } #[test] diff --git a/lib/components/fabro-sandbox/src/docker.rs b/lib/components/fabro-sandbox/src/docker.rs index 97e965ed8..0585ecbdf 100644 --- a/lib/components/fabro-sandbox/src/docker.rs +++ b/lib/components/fabro-sandbox/src/docker.rs @@ -18,6 +18,7 @@ use bollard::image::CreateImageOptions; use bollard::models::{ContainerInspectResponse, HostConfig}; use fabro_github::GitHubCredentials; use fabro_github::token_source::InstallationTokenSource; +use fabro_types::settings::run::RunCloneSettings; use fabro_types::{CommandOutputStream, CommandTermination, RunId, SandboxProviderKind}; use fabro_util::time::elapsed_ms; use futures::StreamExt; @@ -48,7 +49,7 @@ const DOCKER_BASH_REQUIREMENT: &str = "Docker sandboxes require /bin/bash for ev pub(crate) const WORKING_DIRECTORY: &str = "/workspace"; pub(crate) const REPOS_ROOT: &str = "/repos"; -const DEFAULT_GIT_CLONE_DEPTH: usize = 100; +const DEFAULT_GIT_CLONE_DEPTH: usize = RunCloneSettings::DEFAULT_DEPTH.unsigned_abs() as usize; const GIT_CLONE_TIMEOUT: Duration = Duration::from_mins(5); #[cfg(test)] const EXEC_STOP_POLL_SLEEP_SECONDS: &str = "0.005"; @@ -121,9 +122,9 @@ pub struct DockerSandboxOptions { pub auto_pull: bool, /// Additional `KEY=VALUE` environment variables for the container. pub env_vars: Vec, - /// Maximum Git history depth fetched during clone. Zero fetches full + /// Maximum Git history depth fetched during clone; `None` fetches full /// history. - pub clone_depth: usize, + pub clone_depth: Option, /// Create an empty workspace instead of cloning even when an origin exists. pub skip_clone: bool, } @@ -137,7 +138,7 @@ impl Default for DockerSandboxOptions { cpu_quota: None, auto_pull: true, env_vars: Vec::new(), - clone_depth: DEFAULT_GIT_CLONE_DEPTH, + clone_depth: Some(DEFAULT_GIT_CLONE_DEPTH), skip_clone: false, } } @@ -1541,7 +1542,7 @@ fn git_clone_command( clone_url: &str, branch: Option<&str>, checkout_path: &str, - depth: usize, + depth: Option, ) -> String { let mut command = format!("{} clone", sandbox::GIT); if let Some(branch) = branch { @@ -1549,10 +1550,7 @@ fn git_clone_command( command.push_str(&shell_quote(branch)); command.push_str(" --single-branch"); } - if depth > 0 { - command.push_str(" --depth "); - command.push_str(&depth.to_string()); - } + command.push_str(&clone_source::depth_argument(depth)); command.push_str(" --no-tags"); command.push_str(" -- "); command.push_str(&shell_quote(clone_url)); @@ -2571,7 +2569,7 @@ mod tests { let options = DockerSandboxOptions::default(); assert_eq!(options.image, "buildpack-deps:noble"); assert_eq!(options.network_mode.as_deref(), Some("bridge")); - assert_eq!(options.clone_depth, DEFAULT_GIT_CLONE_DEPTH); + assert_eq!(options.clone_depth, Some(DEFAULT_GIT_CLONE_DEPTH)); assert!(!options.skip_clone); } @@ -2581,7 +2579,7 @@ mod tests { "https://github.com/fabro-sh/fabro", Some("main"), "/repos/fabro-sh/fabro", - 1, + Some(1), ); assert_eq!( command, @@ -2595,7 +2593,7 @@ mod tests { "https://github.com/fabro-sh/fabro", None, "/repos/fabro-sh/fabro", - DEFAULT_GIT_CLONE_DEPTH, + Some(DEFAULT_GIT_CLONE_DEPTH), ); assert_eq!( command, @@ -2609,7 +2607,7 @@ mod tests { "https://github.com/fabro-sh/fabro", Some("main"), "/repos/fabro-sh/fabro", - 0, + None, ); assert_eq!( command, diff --git a/lib/components/fabro-sandbox/src/from_environment.rs b/lib/components/fabro-sandbox/src/from_environment.rs index 4bf76696e..657b4c179 100644 --- a/lib/components/fabro-sandbox/src/from_environment.rs +++ b/lib/components/fabro-sandbox/src/from_environment.rs @@ -61,7 +61,7 @@ pub fn daytona_config_from_environment( DaytonaNetwork::AllowList(settings.network.allow.clone()) } }), - clone_depth: clone.depth.filter(|depth| *depth > 0), + clone_depth: clone.depth_limit(), skip_clone: !clone.enabled, } } @@ -133,9 +133,8 @@ fn docker_config_from_environment_env( .map(|cpu| i64::from(cpu).saturating_mul(100_000)), env_vars, clone_depth: clone - .depth - .and_then(|depth| usize::try_from(depth).ok()) - .unwrap_or(default_options.clone_depth), + .depth_limit() + .and_then(|depth| usize::try_from(depth).ok()), skip_clone: !clone.enabled, ..DockerSandboxOptions::default() } diff --git a/lib/components/fabro-store/src/legacy_blob_import.rs b/lib/components/fabro-store/src/legacy_blob_import.rs index 06199edc2..d9ea4ba93 100644 --- a/lib/components/fabro-store/src/legacy_blob_import.rs +++ b/lib/components/fabro-store/src/legacy_blob_import.rs @@ -790,7 +790,7 @@ mod tests { type TestResult = std::result::Result>; struct TestContext { - dir: tempfile::TempDir, + _dir: tempfile::TempDir, source: Database, source_db: slatedb::Db, sqlite: sqlx::SqlitePool, @@ -812,7 +812,7 @@ mod tests { let sqlite = sqlite.clone_pool(); let target = BlobStore::new(sqlite.clone()); Ok(Self { - dir, + _dir: dir, source, source_db, sqlite, @@ -822,11 +822,8 @@ mod tests { async fn new_with_single_sqlite_connection() -> TestResult { let mut context = Self::new().await?; + let options = context.sqlite.connect_options().as_ref().clone(); context.sqlite.close().await; - let options = SqliteConnectOptions::new() - .filename(context.dir.path().join("fabro.sqlite3")) - .foreign_keys(true) - .journal_mode(SqliteJournalMode::Wal); let sqlite = SqlitePoolOptions::new() .max_connections(1) .connect_with(options) diff --git a/lib/components/fabro-workflow/src/operations/start.rs b/lib/components/fabro-workflow/src/operations/start.rs index 9955f1a8d..053548123 100644 --- a/lib/components/fabro-workflow/src/operations/start.rs +++ b/lib/components/fabro-workflow/src/operations/start.rs @@ -1301,7 +1301,7 @@ reasoning = false resolve_docker_config(&settings.run, |_| None) .unwrap() .clone_depth, - 1 + Some(1) ); } @@ -1320,7 +1320,7 @@ reasoning = false resolve_docker_config(&settings.run, |_| None) .unwrap() .clone_depth, - 0 + None ); } @@ -1333,7 +1333,7 @@ reasoning = false resolve_docker_config(&settings.run, |_| None) .unwrap() .clone_depth, - 100 + Some(100) ); } diff --git a/lib/foundation/fabro-config/src/resolve/run.rs b/lib/foundation/fabro-config/src/resolve/run.rs index ba64fa67d..a60bb9b9f 100644 --- a/lib/foundation/fabro-config/src/resolve/run.rs +++ b/lib/foundation/fabro-config/src/resolve/run.rs @@ -248,24 +248,20 @@ fn resolve_clone( clone: Option<&RunCloneLayer>, errors: &mut Vec, ) -> RunCloneSettings { - let depth = - clone - .and_then(|clone| clone.depth) - .map_or(RunCloneSettings::DEFAULT_DEPTH, |depth| { - if depth < 0 { - errors.push(ResolveError::Invalid { - path: "run.clone.depth".to_string(), - reason: "depth must be at least 0".to_string(), - }); - RunCloneSettings::DEFAULT_DEPTH - } else { - depth - } - }); + let mut depth = clone + .and_then(|clone| clone.depth) + .unwrap_or(RunCloneSettings::DEFAULT_DEPTH); + if depth < 0 { + errors.push(ResolveError::Invalid { + path: "run.clone.depth".to_string(), + reason: "depth must be at least 0".to_string(), + }); + depth = RunCloneSettings::DEFAULT_DEPTH; + } RunCloneSettings { enabled: clone.and_then(|clone| clone.enabled).unwrap_or(true), - depth: Some(depth), + depth, } } diff --git a/lib/foundation/fabro-config/src/tests/resolve_run.rs b/lib/foundation/fabro-config/src/tests/resolve_run.rs index d66b075ea..c738869e9 100644 --- a/lib/foundation/fabro-config/src/tests/resolve_run.rs +++ b/lib/foundation/fabro-config/src/tests/resolve_run.rs @@ -126,7 +126,7 @@ fn resolves_run_defaults_from_empty_settings() { assert!(!settings.environment.lifecycle.preserve); assert!(settings.environment.lifecycle.stop_on_terminal); assert!(settings.clone.enabled); - assert_eq!(settings.clone.depth, Some(100)); + assert_eq!(settings.clone.depth, 100); assert!(settings.run_branch.enabled); assert!(settings.run_branch.push); assert!(settings.meta_branch.enabled); @@ -312,7 +312,7 @@ push = false .run; assert!(!settings.clone.enabled); - assert_eq!(settings.clone.depth, Some(1)); + assert_eq!(settings.clone.depth, 1); assert!(settings.run_branch.enabled); assert!(!settings.run_branch.push); assert!(settings.meta_branch.enabled); @@ -332,7 +332,7 @@ depth = 0 .expect("zero clone depth should resolve") .run; - assert_eq!(settings.clone.depth, Some(0)); + assert_eq!(settings.clone.depth, 0); } #[test] diff --git a/lib/foundation/fabro-types/src/settings/run.rs b/lib/foundation/fabro-types/src/settings/run.rs index a47cb48fb..7854ee6d3 100644 --- a/lib/foundation/fabro-types/src/settings/run.rs +++ b/lib/foundation/fabro-types/src/settings/run.rs @@ -961,32 +961,30 @@ impl Default for RunCheckpointSettings { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct RunCloneSettings { pub enabled: bool, - #[serde( - default = "default_clone_depth", - skip_serializing_if = "Option::is_none" - )] - pub depth: Option, + #[serde(default = "default_clone_depth")] + pub depth: i32, } impl RunCloneSettings { pub const DEFAULT_DEPTH: i32 = 100; + + /// Git history depth to fetch, or `None` to fetch full history. + pub fn depth_limit(&self) -> Option { + (self.depth > 0).then_some(self.depth) + } } impl Default for RunCloneSettings { fn default() -> Self { Self { enabled: true, - depth: Some(Self::DEFAULT_DEPTH), + depth: Self::DEFAULT_DEPTH, } } } -#[expect( - clippy::unnecessary_wraps, - reason = "serde default provider must return the field's Option type" -)] -fn default_clone_depth() -> Option { - Some(RunCloneSettings::DEFAULT_DEPTH) +fn default_clone_depth() -> i32 { + RunCloneSettings::DEFAULT_DEPTH } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] From 0d028f9b1e7e3a476faade69366ab14ab41026ae Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 19:28:55 -0400 Subject: [PATCH 29/30] Harden release push recovery --- .../fabro-dev/src/commands/bench_tests.rs | 19 +- lib/foundation/fabro-dev/src/commands/mod.rs | 18 + .../fabro-dev/src/commands/release.rs | 599 +++++++++++++++--- lib/foundation/fabro-dev/tests/it/release.rs | 6 +- 4 files changed, 533 insertions(+), 109 deletions(-) diff --git a/lib/foundation/fabro-dev/src/commands/bench_tests.rs b/lib/foundation/fabro-dev/src/commands/bench_tests.rs index 39616c4b8..81264fb9d 100644 --- a/lib/foundation/fabro-dev/src/commands/bench_tests.rs +++ b/lib/foundation/fabro-dev/src/commands/bench_tests.rs @@ -8,7 +8,7 @@ use clap::Args; use quick_xml::events::{BytesStart, Event}; use quick_xml::reader::Reader; -use super::{PlannedCommand, capture_command, command, workspace_root}; +use super::{PlannedCommand, command, workspace_root}; const TOOL_CONFIG_RELATIVE: &str = "target/bench-tests/nextest-tool.toml"; const TOOL_CONFIG_BODY: &str = "\ @@ -67,7 +67,7 @@ pub(crate) fn bench_tests(args: BenchTestsArgs) -> Result<()> { } let root = workspace_root(); - let git_sha = resolve_head_sha(&root)?; + let git_sha = super::resolve_git_revision(&root, "HEAD")?; let tool_config = ensure_tool_config(&root)?; let junit_path = root .join("target") @@ -180,21 +180,6 @@ fn ensure_tool_config(root: &Path) -> Result { Ok(path) } -fn resolve_head_sha(root: &Path) -> Result { - let cmd = PlannedCommand::new("git").arg("rev-parse").arg("HEAD"); - let output = capture_command(root, &cmd)?; - if !output.status.success() { - bail!( - "git rev-parse HEAD failed: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - Ok(String::from_utf8(output.stdout) - .context("git rev-parse HEAD returned non-UTF-8")? - .trim() - .to_string()) -} - #[expect( clippy::disallowed_methods, reason = "fabro-dev opens the CSV file synchronously; bench-tests is a CLI tool, not a Tokio runtime" diff --git a/lib/foundation/fabro-dev/src/commands/mod.rs b/lib/foundation/fabro-dev/src/commands/mod.rs index a59142898..cb2d9c8cf 100644 --- a/lib/foundation/fabro-dev/src/commands/mod.rs +++ b/lib/foundation/fabro-dev/src/commands/mod.rs @@ -148,6 +148,24 @@ pub(crate) fn capture_command(cwd: &Path, planned: &PlannedCommand) -> Result Result { + let command = PlannedCommand::new("git") + .arg("rev-parse") + .arg("--verify") + .arg(revision); + let output = capture_command(root, &command)?; + if !output.status.success() { + anyhow::bail!( + "git rev-parse --verify {revision} failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + } + Ok(String::from_utf8(output.stdout) + .with_context(|| format!("git rev-parse --verify {revision} returned non-UTF-8"))? + .trim() + .to_string()) +} + #[expect( clippy::disallowed_methods, reason = "dev CLI sanitizes inherited Cargo build-script env before spawning nested cargo" diff --git a/lib/foundation/fabro-dev/src/commands/release.rs b/lib/foundation/fabro-dev/src/commands/release.rs index b47e9ddb8..9883f91af 100644 --- a/lib/foundation/fabro-dev/src/commands/release.rs +++ b/lib/foundation/fabro-dev/src/commands/release.rs @@ -9,6 +9,8 @@ use super::{PlannedCommand, capture_command, run_command, spa_refresh, workspace const RELEASE_EPOCH: &str = "2026-01-01"; const RELEASE_TEST_SEGMENT_WRITE_KEY: &str = "fake-for-local-smoke"; const MAX_PUSH_ATTEMPTS: u32 = 4; +const RELEASE_BRANCH: &str = "main"; +const RELEASE_REMOTE: &str = "origin"; #[derive(Debug, Args)] pub(crate) struct ReleaseArgs { @@ -40,7 +42,50 @@ struct ReleasePlan { struct ReleaseVersions { current: String, next: String, - tag: String, +} + +impl ReleaseVersions { + fn tag(&self) -> String { + format!("v{}", self.next) + } +} + +struct RemoteReleaseState { + main: String, + tag: Option, +} + +#[derive(Debug, PartialEq, Eq)] +enum PushFailureDisposition { + Published, + Retryable, + Unchanged, + Inconsistent, +} + +impl RemoteReleaseState { + fn classify_push_failure( + &self, + remote_main_before: &str, + release_head: &str, + local_contains_remote_before: bool, + ) -> PushFailureDisposition { + let main_is_release = self.main == release_head; + let tag_is_release = self.tag.as_deref() == Some(release_head); + + if main_is_release && tag_is_release { + PushFailureDisposition::Published + } else if main_is_release || tag_is_release { + PushFailureDisposition::Inconsistent + } else if !local_contains_remote_before + || self.main != remote_main_before + || self.tag.is_some() + { + PushFailureDisposition::Retryable + } else { + PushFailureDisposition::Unchanged + } + } } #[expect( @@ -60,14 +105,16 @@ pub(crate) fn release(args: ReleaseArgs) -> Result<()> { let cargo_toml = plan.root.join("Cargo.toml"); let versions = plan.compute_versions(&cargo_toml)?; + let tag = versions.tag(); println!("Current version: {}", versions.current); - println!("Releasing {} (tag {})", versions.next, versions.tag); + println!("Releasing {} (tag {tag})", versions.next); if plan.dry_run { plan.print_dry_run(&versions); return Ok(()); } + plan.ensure_main_branch()?; plan.ensure_clean_worktree()?; spa_refresh::spa_refresh_root(&plan.root)?; plan.verify_release_tests()?; @@ -122,8 +169,7 @@ impl ReleasePlan { let current = read_current_version(cargo_toml)?; let base_version = self.next_base_version()?; let next = self.compute_release_version(&base_version)?; - let tag = format!("v{next}"); - Ok(ReleaseVersions { current, next, tag }) + Ok(ReleaseVersions { current, next }) } /// Commits the version bump, tags it, and pushes `main` plus the tag @@ -141,21 +187,66 @@ impl ReleasePlan { ) -> Result { let mut attempt = 1; loop { - let start_head = self.head_commit()?; + self.ensure_main_branch()?; + self.ensure_clean_worktree() + .context("working tree changed while the release was running")?; + + let start_head = super::resolve_git_revision(&self.root, "HEAD")?; + let remote_tracking_ref = Self::remote_tracking_ref(); + let remote_main_before = + super::resolve_git_revision(&self.root, &remote_tracking_ref).with_context(|| { + format!( + "release requires a local {} tracking ref; fetch {RELEASE_REMOTE} and retry", + Self::remote_branch() + ) + })?; + let local_contains_remote_before = + self.commit_is_ancestor(&remote_main_before, &start_head)?; + let tag = versions.tag(); self.create_bump_commit_and_tag(cargo_toml, &versions)?; - let Err(error) = self.push_main_and_tag(&versions.tag) else { - return Ok(versions.tag); + let release_head = super::resolve_git_revision(&self.root, "HEAD")?; + let Err(push_error) = self.push_main_and_tag(&tag) else { + return Ok(tag); }; - if attempt == MAX_PUSH_ATTEMPTS { - return Err(error); + + let disposition = self + .push_failure_disposition( + &tag, + &remote_main_before, + &release_head, + local_contains_remote_before, + ) + .context("push failed and origin could not be verified")?; + match disposition { + PushFailureDisposition::Published => { + println!( + "Push reported an error, but {RELEASE_REMOTE} contains {RELEASE_BRANCH} \ + and {tag} at {release_head}" + ); + return Ok(tag); + } + PushFailureDisposition::Unchanged => return Err(push_error), + PushFailureDisposition::Inconsistent => { + return Err(push_error.context( + "origin contains only part of the release; inspect the remote refs before \ + retrying", + )); + } + PushFailureDisposition::Retryable => { + if attempt == MAX_PUSH_ATTEMPTS { + return Err(push_error); + } + } } + + let remote_branch = Self::remote_branch(); println!( - "Push failed on attempt {attempt} of {MAX_PUSH_ATTEMPTS}; rebuilding the \ - release on the latest origin/main" + "Origin changed during push attempt {attempt} of {MAX_PUSH_ATTEMPTS}; rebuilding \ + the release on the latest {remote_branch}" ); - self.resync_with_origin_main(&versions.tag, &start_head)?; + self.resync_with_origin_main(&tag, &start_head, &release_head)?; versions = self.compute_versions(cargo_toml)?; - println!("Retrying as {} (tag {})", versions.next, versions.tag); + println!("Retrying as {} (tag {})", versions.next, versions.tag()); attempt += 1; } } @@ -172,102 +263,249 @@ impl ReleasePlan { update_version(cargo_toml, &versions.current, &versions.next)?; println!("Updated {}", cargo_toml.display()); - run_command( - &self.root, - &PlannedCommand::new("cargo") - .arg("update") - .arg("--workspace"), - )?; + let [cargo_update, git_add, git_commit, git_tag] = Self::bump_commands(versions); + run_command(&self.root, &cargo_update)?; println!("Updated Cargo.lock"); - run_command( - &self.root, - &PlannedCommand::new("git") + for command in [git_add, git_commit, git_tag] { + run_command(&self.root, &command)?; + } + Ok(()) + } + + fn bump_commands(versions: &ReleaseVersions) -> [PlannedCommand; 4] { + let tag = versions.tag(); + [ + PlannedCommand::new("cargo") + .arg("update") + .arg("--workspace"), + PlannedCommand::new("git") .arg("add") .arg("Cargo.toml") .arg("Cargo.lock"), - )?; - run_command( - &self.root, - &PlannedCommand::new("git") + PlannedCommand::new("git") .arg("commit") .arg("-m") .arg(format!("Bump version to {}", versions.next)), - )?; - run_command( - &self.root, - &PlannedCommand::new("git") + PlannedCommand::new("git") .arg("tag") .arg("-a") - .arg(&versions.tag) + .arg(&tag) .arg("-m") - .arg(&versions.tag), - ) + .arg(tag), + ] } fn push_main_and_tag(&self, tag: &str) -> Result<()> { run_command(&self.root, &Self::push_command(tag)) } - /// Drops the bump commit and tag this run created, then fast-forwards - /// onto the updated origin/main. `--ff-only` refuses to discard commits - /// that did not come from origin, so unpushed local work fails loudly - /// instead of being reset away. - fn resync_with_origin_main(&self, tag: &str, start_head: &str) -> Result<()> { - run_command( + fn push_failure_disposition( + &self, + tag: &str, + remote_main_before: &str, + release_head: &str, + local_contains_remote_before: bool, + ) -> Result { + Ok(self.remote_release_state(tag)?.classify_push_failure( + remote_main_before, + release_head, + local_contains_remote_before, + )) + } + + fn commit_is_ancestor(&self, ancestor: &str, descendant: &str) -> Result { + let output = capture_command( &self.root, - &PlannedCommand::new("git").arg("tag").arg("-d").arg(tag), + &PlannedCommand::new("git") + .arg("merge-base") + .arg("--is-ancestor") + .arg(ancestor) + .arg(descendant), )?; + if output.status.success() { + return Ok(true); + } + if output.status.code() == Some(1) { + return Ok(false); + } + bail!( + "failed to compare git commits {ancestor} and {descendant}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ) + } + + fn remote_release_state(&self, tag: &str) -> Result { + let branch_ref = Self::branch_ref(); + let tag_ref = Self::tag_ref(tag); + let peeled_tag_ref = format!("{tag_ref}^{{}}"); + let output = capture_command( + &self.root, + &PlannedCommand::new("git") + .arg("ls-remote") + .arg(RELEASE_REMOTE) + .arg(&branch_ref) + .arg(&tag_ref) + .arg(&peeled_tag_ref), + )?; + if !output.status.success() { + bail!( + "failed to inspect release refs on {RELEASE_REMOTE}: {}", + String::from_utf8_lossy(&output.stderr).trim() + ); + } + + let stdout = + String::from_utf8(output.stdout).context("git ls-remote returned non-UTF-8 output")?; + let mut main = None; + let mut tag_object = None; + let mut peeled_tag = None; + for line in stdout.lines() { + let (object_id, reference) = line + .split_once('\t') + .with_context(|| format!("unexpected git ls-remote output: {line}"))?; + if reference == branch_ref { + main = Some(object_id.to_string()); + } else if reference == tag_ref { + tag_object = Some(object_id.to_string()); + } else if reference == peeled_tag_ref { + peeled_tag = Some(object_id.to_string()); + } + } + + let remote_branch = Self::remote_branch(); + let main = main + .with_context(|| format!("{remote_branch} was not present in git ls-remote output"))?; + Ok(RemoteReleaseState { + main, + tag: peeled_tag.or(tag_object), + }) + } + + /// Drops the bump commit and tag this run created, then fast-forwards + /// onto the updated origin/main. `--keep` preserves worktree edits, and + /// `--ff-only` refuses to discard commits that did not come from origin. + fn resync_with_origin_main( + &self, + tag: &str, + start_head: &str, + release_head: &str, + ) -> Result<()> { + self.ensure_main_branch()?; + let current_head = super::resolve_git_revision(&self.root, "HEAD")?; + if current_head != release_head { + bail!( + "local {RELEASE_BRANCH} changed while the release was running; refusing to move \ + it from {current_head} back to {start_head}" + ); + } + + let tag_ref = Self::tag_ref(tag); + let tag_object = super::resolve_git_revision(&self.root, &tag_ref) + .with_context(|| format!("local release tag {tag} changed while the release ran"))?; + let tag_commit = super::resolve_git_revision(&self.root, &format!("{tag_ref}^{{commit}}")) + .with_context(|| format!("local release tag {tag} changed while the release ran"))?; + if tag_commit != release_head { + bail!( + "local release tag {tag} changed while the release was running; refusing to \ + delete it" + ); + } + run_command( &self.root, &PlannedCommand::new("git") .arg("reset") - .arg("--hard") + .arg("--keep") .arg(start_head), + ) + .context( + "working tree changed while the release was running; local edits were preserved", )?; + run_command( + &self.root, + &PlannedCommand::new("git") + .arg("update-ref") + .arg("-d") + .arg(&tag_ref) + .arg(tag_object), + ) + .with_context(|| format!("local release tag {tag} changed while the release ran"))?; + self.ensure_clean_worktree() + .context("working tree changed while the release was running")?; run_command( &self.root, &PlannedCommand::new("git") .arg("fetch") .arg("--tags") - .arg("origin") - .arg("main"), + .arg(RELEASE_REMOTE) + .arg(RELEASE_BRANCH), )?; + let remote_branch = Self::remote_branch(); run_command( &self.root, &PlannedCommand::new("git") .arg("merge") .arg("--ff-only") - .arg("origin/main"), - ) - .context( - "local main has diverged from origin/main; reconcile manually and rerun the release", + .arg(&remote_branch), ) + .with_context(|| { + format!( + "local {RELEASE_BRANCH} has diverged from {remote_branch}; reconcile manually and \ + rerun the release" + ) + })?; + self.ensure_clean_worktree() + .context("working tree changed while the release was running") } - fn head_commit(&self) -> Result { - let output = capture_command( - &self.root, - &PlannedCommand::new("git").arg("rev-parse").arg("HEAD"), - )?; - if !output.status.success() { - bail!( - "failed to resolve HEAD: {}", - String::from_utf8_lossy(&output.stderr) - ); - } - Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) + fn branch_ref() -> String { + format!("refs/heads/{RELEASE_BRANCH}") + } + + fn remote_branch() -> String { + format!("{RELEASE_REMOTE}/{RELEASE_BRANCH}") + } + + fn remote_tracking_ref() -> String { + format!("refs/remotes/{RELEASE_REMOTE}/{RELEASE_BRANCH}") + } + + fn tag_ref(tag: &str) -> String { + format!("refs/tags/{tag}") } fn push_command(tag: &str) -> PlannedCommand { PlannedCommand::new("git") .arg("push") .arg("--atomic") - .arg("origin") - .arg("main") + .arg(RELEASE_REMOTE) + .arg(RELEASE_BRANCH) .arg(tag) } + fn ensure_main_branch(&self) -> Result<()> { + let output = capture_command( + &self.root, + &PlannedCommand::new("git") + .arg("symbolic-ref") + .arg("--quiet") + .arg("HEAD"), + )?; + if !output.status.success() { + bail!("release must run from {RELEASE_BRANCH}; HEAD is not a local branch"); + } + + let head_ref = String::from_utf8(output.stdout) + .context("git symbolic-ref returned non-UTF-8 output")?; + let head_ref = head_ref.trim(); + if head_ref != Self::branch_ref() { + let branch = head_ref.strip_prefix("refs/heads/").unwrap_or(head_ref); + bail!("release must run from {RELEASE_BRANCH}; current branch is {branch}"); + } + Ok(()) + } + fn ensure_clean_worktree(&self) -> Result<()> { let output = capture_command( &self.root, @@ -322,26 +560,11 @@ impl ReleasePlan { "DRY RUN: would update Cargo.toml version {} -> {}", versions.current, versions.next ); - for command in [ - PlannedCommand::new("cargo") - .arg("update") - .arg("--workspace"), - PlannedCommand::new("git") - .arg("add") - .arg("Cargo.toml") - .arg("Cargo.lock"), - PlannedCommand::new("git") - .arg("commit") - .arg("-m") - .arg(format!("Bump version to {}", versions.next)), - PlannedCommand::new("git") - .arg("tag") - .arg("-a") - .arg(&versions.tag) - .arg("-m") - .arg(&versions.tag), - Self::push_command(&versions.tag), - ] { + let tag = versions.tag(); + for command in Self::bump_commands(versions) + .into_iter() + .chain(std::iter::once(Self::push_command(&tag))) + { println!("{}", command.to_shell_line()); } } @@ -543,14 +766,181 @@ version.workspace = true } } - #[test] - fn push_rebuilds_bump_commit_when_origin_main_moves() { - let fixture = race_fixture(); - + fn push_concurrent_commit(fixture: &RaceFixture) { write_file(&fixture.other.join("README.md"), "concurrent\n"); git(&fixture.other, &["add", "README.md"]); git(&fixture.other, &["commit", "-m", "concurrent work"]); - git(&fixture.other, &["push", "origin", "main"]); + git(&fixture.other, &["push", RELEASE_REMOTE, RELEASE_BRANCH]); + } + + #[test] + fn failed_push_check_recognizes_a_published_release() { + let fixture = race_fixture(); + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = versions.tag(); + let start_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving initial HEAD"); + plan.create_bump_commit_and_tag(&cargo_toml, &versions) + .expect("creating release commit and tag"); + let release_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving release HEAD"); + git(&fixture.work, &[ + "push", + "--atomic", + RELEASE_REMOTE, + RELEASE_BRANCH, + tag.as_str(), + ]); + + assert_eq!( + plan.push_failure_disposition(&tag, &start_head, &release_head, true) + .expect("inspecting published refs"), + PushFailureDisposition::Published + ); + } + + #[test] + fn push_failure_classification_rejects_an_unchanged_remote() { + let state = RemoteReleaseState { + main: "start".to_string(), + tag: None, + }; + + assert_eq!( + state.classify_push_failure("start", "release", true), + PushFailureDisposition::Unchanged + ); + } + + #[test] + fn release_requires_the_main_branch() { + let fixture = race_fixture(); + git(&fixture.work, &["switch", "-c", "feature"]); + + let error = nightly_plan(&fixture.work) + .ensure_main_branch() + .expect_err("a release from another branch should fail"); + + assert!( + format!("{error:#}").contains("release must run from main; current branch is feature"), + "error should identify the required and current branches: {error:#}" + ); + } + + #[test] + fn resync_preserves_worktree_edits() { + let fixture = race_fixture(); + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = versions.tag(); + let start_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving initial HEAD"); + plan.create_bump_commit_and_tag(&cargo_toml, &versions) + .expect("creating release commit and tag"); + let release_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving release HEAD"); + write_file(&fixture.work.join("app/src/lib.rs"), "user edit\n"); + + let error = plan + .resync_with_origin_main(&tag, &start_head, &release_head) + .expect_err("resync should stop when the worktree changes"); + + assert!( + format!("{error:#}").contains("working tree changed while the release was running"), + "error should explain why resync stopped: {error:#}" + ); + let diff = git(&fixture.work, &["diff", "--", "app/src/lib.rs"]); + assert!( + diff.contains("+user edit"), + "resync should preserve the worktree edit:\n{diff}" + ); + } + + #[test] + fn resync_preserves_a_commit_created_after_the_release_commit() { + let fixture = race_fixture(); + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = versions.tag(); + let start_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving initial HEAD"); + plan.create_bump_commit_and_tag(&cargo_toml, &versions) + .expect("creating release commit and tag"); + let release_head = super::super::resolve_git_revision(&fixture.work, "HEAD") + .expect("resolving release HEAD"); + write_file(&fixture.work.join("local.txt"), "concurrent local work\n"); + git(&fixture.work, &["add", "local.txt"]); + git(&fixture.work, &["commit", "-m", "concurrent local work"]); + + let error = plan + .resync_with_origin_main(&tag, &start_head, &release_head) + .expect_err("resync should stop when local main changes"); + + assert!( + format!("{error:#}").contains("local main changed while the release was running"), + "error should explain why resync stopped: {error:#}" + ); + let subject = git(&fixture.work, &["log", "-1", "--format=%s"]); + assert_eq!(subject, "concurrent local work"); + git(&fixture.work, &[ + "rev-parse", + "--verify", + &ReleasePlan::tag_ref(&tag), + ]); + } + + #[test] + fn push_does_not_retry_when_remote_refs_are_unchanged() { + let fixture = race_fixture(); + write_file(&fixture.work.join("local.txt"), "unpushed local work\n"); + git(&fixture.work, &["add", "local.txt"]); + git(&fixture.work, &["commit", "-m", "unpushed local work"]); + + let missing_push_remote = fixture + .origin + .parent() + .expect("origin should have a parent") + .join("missing.git"); + git(&fixture.work, &[ + "remote", + "set-url", + "--push", + RELEASE_REMOTE, + missing_push_remote + .to_str() + .expect("push remote path should be utf-8"), + ]); + + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + plan.commit_tag_and_push(&cargo_toml, versions) + .expect_err("an unrelated push failure should be returned"); + + let subjects = git(&fixture.work, &["log", "--format=%s"]); + assert_eq!(subjects.lines().collect::>(), [ + "Bump version to 0.100.0-nightly.0", + "unpushed local work", + "initial" + ]); + } + + #[test] + fn push_rebuilds_bump_commit_when_origin_main_moves() { + let fixture = race_fixture(); + push_concurrent_commit(&fixture); let plan = nightly_plan(&fixture.work); let cargo_toml = fixture.work.join("Cargo.toml"); @@ -575,11 +965,41 @@ version.workspace = true ]); } + #[test] + fn push_rebuilds_when_origin_main_was_fetched_but_not_merged() { + let fixture = race_fixture(); + push_concurrent_commit(&fixture); + git(&fixture.work, &["fetch", RELEASE_REMOTE, RELEASE_BRANCH]); + + let plan = nightly_plan(&fixture.work); + let cargo_toml = fixture.work.join("Cargo.toml"); + let versions = plan + .compute_versions(&cargo_toml) + .expect("computing versions"); + let tag = plan + .commit_tag_and_push(&cargo_toml, versions) + .expect("push should rescue a local main behind its tracking branch"); + + assert_eq!(tag, "v0.100.0-nightly.0"); + let subjects = git(&fixture.origin, &["log", "--format=%s", RELEASE_BRANCH]); + assert_eq!(subjects.lines().collect::>(), [ + "Bump version to 0.100.0-nightly.0", + "concurrent work", + "initial" + ]); + } + #[test] fn push_recomputes_version_when_tag_is_taken() { let fixture = race_fixture(); - git(&fixture.other, &["tag", "v0.100.0-nightly.0"]); + git(&fixture.other, &[ + "tag", + "-a", + "v0.100.0-nightly.0", + "-m", + "v0.100.0-nightly.0", + ]); git(&fixture.other, &["push", "origin", "v0.100.0-nightly.0"]); let plan = nightly_plan(&fixture.work); @@ -609,10 +1029,7 @@ version.workspace = true git(&fixture.work, &["add", "local.txt"]); git(&fixture.work, &["commit", "-m", "unpushed local work"]); - write_file(&fixture.other.join("README.md"), "concurrent\n"); - git(&fixture.other, &["add", "README.md"]); - git(&fixture.other, &["commit", "-m", "concurrent work"]); - git(&fixture.other, &["push", "origin", "main"]); + push_concurrent_commit(&fixture); let plan = nightly_plan(&fixture.work); let cargo_toml = fixture.work.join("Cargo.toml"); diff --git a/lib/foundation/fabro-dev/tests/it/release.rs b/lib/foundation/fabro-dev/tests/it/release.rs index 28fb4cf94..a548e25ed 100644 --- a/lib/foundation/fabro-dev/tests/it/release.rs +++ b/lib/foundation/fabro-dev/tests/it/release.rs @@ -34,7 +34,7 @@ members = [] version = "0.1.0" "#, ); - git(fixture.path(), &["init"]); + git(fixture.path(), &["init", "-b", "main"]); git(fixture.path(), &["config", "user.name", "Release Test"]); git(fixture.path(), &[ "config", @@ -107,6 +107,10 @@ fn dry_run_computes_stable_version_from_date() { stdout.contains("git tag -a v0.100.0 -m v0.100.0"), "dry-run should print release tag command:\n{stdout}" ); + assert!( + stdout.contains("git push --atomic origin main v0.100.0"), + "dry-run should print the atomic push command:\n{stdout}" + ); assert!( stdout.contains( "unset GH_TOKEN GITHUB_TOKEN && SEGMENT_WRITE_KEY=fake-for-local-smoke cargo nextest run --locked" From 61394ba2f1dcfba9700f044b156563b1769b1fd9 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Fri, 21 Aug 2026 19:29:14 -0400 Subject: [PATCH 30/30] Simplify run-event persistence failure plumbing - Make the failure watch channel the single record of the latched failure; drop the worker task's mirrored local state. - Replace the hand-rolled wait loop with watch::Receiver::wait_for. - Extract race_persistence/flush_or_stop helpers so the select!/flush scaffolding in RunSession::run exists once instead of three times. - Return RunEventPersistenceError from append_event_to_sink and add a From impl on Error, replacing four hand-written per-event message strings with the event name derived from the event itself. - Dedupe the RunCreated test seed literal in initialize.rs and drop the dead BlockingHandler::simulate override. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Ryyhtbc1eNtCLw8GjrFQXZ --- lib/components/fabro-workflow/src/error.rs | 7 + .../fabro-workflow/src/event/sink.rs | 38 ++--- .../fabro-workflow/src/operations/resume.rs | 3 +- .../fabro-workflow/src/operations/start.rs | 140 ++++++++---------- .../fabro-workflow/src/pipeline/initialize.rs | 93 ++++++------ 5 files changed, 139 insertions(+), 142 deletions(-) diff --git a/lib/components/fabro-workflow/src/error.rs b/lib/components/fabro-workflow/src/error.rs index 290aacf91..7bedbda70 100644 --- a/lib/components/fabro-workflow/src/error.rs +++ b/lib/components/fabro-workflow/src/error.rs @@ -14,6 +14,7 @@ use fabro_validate::Diagnostic; use regex::Regex; use thiserror::Error as ThisError; +use crate::event::RunEventPersistenceError; use crate::outcome::{FailureDetail, Outcome, StageOutcome}; /// Classify an LLM error into a `FailureCategory` based on its structure. @@ -721,6 +722,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: RunEventPersistenceError) -> Self { + Self::engine_with_source("run event persistence failed", err) + } +} + impl From for Error { fn from(err: fabro_checkpoint::MetadataError) -> Self { match err { diff --git a/lib/components/fabro-workflow/src/event/sink.rs b/lib/components/fabro-workflow/src/event/sink.rs index e27a584b0..a82f2e20f 100644 --- a/lib/components/fabro-workflow/src/event/sink.rs +++ b/lib/components/fabro-workflow/src/event/sink.rs @@ -43,9 +43,15 @@ pub async fn append_event_to_sink( sink: &RunEventSink, run_id: &RunId, event: &Event, -) -> Result<()> { +) -> Result<(), RunEventPersistenceError> { let stored = to_run_event(run_id, event); - sink.write_run_event(&stored).await + sink.write_run_event(&stored) + .await + .map_err(|err| RunEventPersistenceError::Write { + run_id: *run_id, + event: stored.body.event_name().to_string(), + source: SharedError::new(err), + }) } #[derive(Clone)] @@ -179,11 +185,12 @@ impl RunEventLogger { let (failure_tx, failure_rx) = watch::channel(None); tokio::spawn(async move { - let mut persistence_failure = None; + // The watch channel is the single record of the latched failure: + // the worker is its only writer, so borrowing it here cannot race. while let Some(command) = rx.recv().await { match command { RunEventCommand::Event(event) => { - if persistence_failure.is_some() { + if failure_tx.borrow().is_some() { continue; } if let Err(err) = sink.write_run_event(&event).await { @@ -194,17 +201,15 @@ impl RunEventLogger { error = %rendered_error, "Failed to persist run event; stopping workflow", ); - let failure = RunEventPersistenceError::Write { + failure_tx.send_replace(Some(RunEventPersistenceError::Write { run_id: event.run_id, event: event.body.event_name().to_string(), source: SharedError::new(err), - }; - persistence_failure = Some(failure.clone()); - failure_tx.send_replace(Some(failure)); + })); } } RunEventCommand::Flush(tx) => { - let result = persistence_failure.clone().map_or(Ok(()), Err); + let result = failure_tx.borrow().clone().map_or(Ok(()), Err); let _ = tx.send(result); } } @@ -229,13 +234,12 @@ impl RunEventLogger { pub async fn wait_for_failure(&self) -> RunEventPersistenceError { let mut failure_rx = self.failure_rx.clone(); - loop { - if let Some(failure) = failure_rx.borrow_and_update().clone() { - return failure; - } - if failure_rx.changed().await.is_err() { - return RunEventPersistenceError::TaskStopped; - } + let failure = failure_rx.wait_for(Option::is_some).await; + match failure { + Ok(failure) => failure + .clone() + .expect("wait_for only returns values matching the predicate"), + Err(_) => RunEventPersistenceError::TaskStopped, } } @@ -245,7 +249,7 @@ impl RunEventLogger { return Err(RunEventPersistenceError::TaskStopped); } rx.await - .map_err(|_| RunEventPersistenceError::TaskStopped)? + .unwrap_or(Err(RunEventPersistenceError::TaskStopped)) } } diff --git a/lib/components/fabro-workflow/src/operations/resume.rs b/lib/components/fabro-workflow/src/operations/resume.rs index 1911c4a66..466ae5e0c 100644 --- a/lib/components/fabro-workflow/src/operations/resume.rs +++ b/lib/components/fabro-workflow/src/operations/resume.rs @@ -43,8 +43,7 @@ pub async fn resume(run_dir: &Path, services: StartServices) -> Result Result Result Error { cancel_token.cancel(); - Error::engine_with_source("run event persistence failed", error) + error.into() +} + +/// Race a pipeline step against the first latched run-event persistence +/// failure. When the failure wins, the step future is dropped mid-flight and +/// the run token is cancelled. +async fn race_persistence( + logger: &RunEventLogger, + cancel_token: &CancellationToken, + step: impl Future, +) -> Result { + tokio::select! { + result = step => Ok(result), + failure = logger.wait_for_failure() => { + Err(stop_for_run_event_persistence_failure(cancel_token, failure)) + } + } +} + +async fn flush_or_stop( + logger: &RunEventLogger, + cancel_token: &CancellationToken, +) -> Result<(), Error> { + logger + .flush() + .await + .map_err(|failure| stop_for_run_event_persistence_failure(cancel_token, failure)) } impl RunSession { @@ -798,25 +821,16 @@ impl RunSession { seed_context: self.seed_context, fabro_run_tools: self.fabro_run_tools, }; - let mut initializing = Box::pin(pipeline::initialize(persisted, init_options)); - let initialized = tokio::select! { - result = &mut initializing => result, - failure = store_progress_logger.wait_for_failure() => { - return Err(stop_for_run_event_persistence_failure( - &run_cancel_token, - failure, - )); - } - }; - let mut initialized = match initialized { + let mut initialized = match race_persistence( + &store_progress_logger, + &run_cancel_token, + Box::pin(pipeline::initialize(persisted, init_options)), + ) + .await? + { Ok(initialized) => initialized, Err(err) => { - if let Err(failure) = store_progress_logger.flush().await { - return Err(stop_for_run_event_persistence_failure( - &run_cancel_token, - failure, - )); - } + flush_or_stop(&store_progress_logger, &run_cancel_token).await?; return Err(err); } }; @@ -843,23 +857,15 @@ impl RunSession { steering_hub_for_drain.drain_pending_at_run_end(); }); - store_progress_logger.flush().await.map_err(|failure| { - stop_for_run_event_persistence_failure(&run_cancel_token, failure) - })?; + flush_or_stop(&store_progress_logger, &run_cancel_token).await?; - let mut executing = Box::pin(pipeline::execute(initialized)); - let executed = tokio::select! { - executed = &mut executing => executed, - failure = store_progress_logger.wait_for_failure() => { - return Err(stop_for_run_event_persistence_failure( - &run_cancel_token, - failure, - )); - } - }; - store_progress_logger.flush().await.map_err(|failure| { - stop_for_run_event_persistence_failure(&run_cancel_token, failure) - })?; + let executed = race_persistence( + &store_progress_logger, + &run_cancel_token, + Box::pin(pipeline::execute(initialized)), + ) + .await?; + flush_or_stop(&store_progress_logger, &run_cancel_token).await?; let final_context = Some(executed.final_context.clone()); let finalize_opts = FinalizeOptions { @@ -879,30 +885,21 @@ impl RunSession { model: self.pr_model, }; - let mut concluding = Box::pin(async { - let concluded = Box::pin(pipeline::conclude(executed, &finalize_opts)).await?; - let published = Box::pin(pipeline::publish(concluded, &publish_opts)).await; - Box::pin(pipeline::finalize(published, &finalize_opts)).await - }); - let concluding = tokio::select! { - result = &mut concluding => result, - failure = store_progress_logger.wait_for_failure() => { - return Err(stop_for_run_event_persistence_failure( - &run_cancel_token, - failure, - )); - } - }; + let concluding = race_persistence( + &store_progress_logger, + &run_cancel_token, + Box::pin(async { + let concluded = Box::pin(pipeline::conclude(executed, &finalize_opts)).await?; + let published = Box::pin(pipeline::publish(concluded, &publish_opts)).await; + Box::pin(pipeline::finalize(published, &finalize_opts)).await + }), + ) + .await?; let finalized = match concluding { Ok(finalized) => finalized, Err(err) => { self.steering_hub.drain_pending_at_run_end(); - if let Err(failure) = store_progress_logger.flush().await { - return Err(stop_for_run_event_persistence_failure( - &run_cancel_token, - failure, - )); - } + flush_or_stop(&store_progress_logger, &run_cancel_token).await?; return Err(err); } }; @@ -911,9 +908,7 @@ impl RunSession { // scopeguard above re-runs as a no-op (drain is idempotent on an // already-empty buffer) on the way out of scope. self.steering_hub.drain_pending_at_run_end(); - store_progress_logger.flush().await.map_err(|failure| { - stop_for_run_event_persistence_failure(&run_cancel_token, failure) - })?; + flush_or_stop(&store_progress_logger, &run_cancel_token).await?; scopeguard::ScopeGuard::into_inner(cleanup_guard); @@ -1081,7 +1076,7 @@ impl Drop for DetachedRunCompletionGuard { }) .await { - let rendered_error = collect_chain(err.as_ref()).join(": "); + let rendered_error = collect_chain(&err).join(": "); tracing::warn!( error = %rendered_error, "Failed to append detached completion notice", @@ -1110,7 +1105,7 @@ async fn persist_detached_failure( exec_output_tail: None, }; if let Err(err) = append_event_to_sink(event_sink, &run_id, &event).await { - let rendered_error = collect_chain(err.as_ref()).join(": "); + let rendered_error = collect_chain(&err).join(": "); tracing::warn!( error = %rendered_error, "Failed to append detached failure notice", @@ -1229,17 +1224,6 @@ mod tests { ) -> Result { std::future::pending().await } - - async fn simulate( - &self, - _node: &fabro_graphviz::graph::Node, - _context: &Context, - _graph: &fabro_graphviz::graph::Graph, - _run_dir: &Path, - _services: &EngineServices, - ) -> Result { - std::future::pending().await - } } fn memory_store() -> Arc { diff --git a/lib/components/fabro-workflow/src/pipeline/initialize.rs b/lib/components/fabro-workflow/src/pipeline/initialize.rs index 7706ae411..d7f8a5930 100644 --- a/lib/components/fabro-workflow/src/pipeline/initialize.rs +++ b/lib/components/fabro-workflow/src/pipeline/initialize.rs @@ -668,7 +668,7 @@ mod tests { use fabro_graphviz::graph::{AttrValue, Edge, Graph, Node}; use fabro_interview::AutoApproveInterviewer; use fabro_sandbox::SandboxSpec; - use fabro_store::Database; + use fabro_store::{Database, RunDatabase}; use fabro_types::settings::run::RunModelControls; use fabro_types::{ EventBody, ForkSourceRef, RunEvent, RunId, WorkflowSettings, fixtures, test_support, @@ -713,6 +713,37 @@ mod tests { )) } + async fn seed_run_created( + run_store: &RunDatabase, + settings: serde_json::Value, + graph: serde_json::Value, + source_directory: Option, + fork_source_ref: Option, + ) { + crate::event::append_event(run_store, &test_run_id(), &Event::RunCreated { + run_id: test_run_id(), + title: None, + settings, + graph, + workflow_source: None, + labels: BTreeMap::new(), + source_directory, + workflow_slug: Some("test".to_string()), + workflow_version_id: None, + automation: None, + provenance: test_support::test_run_provenance(), + manifest_blob: None, + spec_blob: None, + git: None, + fork_source_ref, + retried_from: None, + parent_id: None, + web_url: None, + }) + .await + .unwrap(); + } + fn simple_graph() -> (Graph, String) { let source = r"digraph test { start [shape=Mdiamond]; @@ -1039,28 +1070,14 @@ mod tests { let mut run_options = test_settings(&run_dir); run_options.settings = settings; run_options.fork_source_ref = fork_source_ref; - crate::event::append_event(&run_store, &test_run_id(), &Event::RunCreated { - run_id: test_run_id(), - title: None, - settings: serde_json::to_value(&run_options.settings).unwrap(), - graph: serde_json::to_value(&graph).unwrap(), - workflow_source: None, - labels: BTreeMap::new(), - source_directory: Some(workspace.display().to_string()), - workflow_slug: Some("test".to_string()), - workflow_version_id: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: run_options.fork_source_ref.clone(), - retried_from: None, - parent_id: None, - web_url: None, - }) - .await - .unwrap(); + seed_run_created( + &run_store, + serde_json::to_value(&run_options.settings).unwrap(), + serde_json::to_value(&graph).unwrap(), + Some(workspace.display().to_string()), + run_options.fork_source_ref.clone(), + ) + .await; initialize(persisted, InitOptions { resume: Some(ResumeState::for_test( @@ -1330,28 +1347,14 @@ mod tests { let emitter = Arc::new(crate::event::Emitter::new(test_run_id())); let store = memory_store(); let run_store = store.create_run(&test_run_id()).await.unwrap(); - crate::event::append_event(&run_store, &test_run_id(), &Event::RunCreated { - run_id: test_run_id(), - title: None, - settings: serde_json::to_value(WorkflowSettings::default()).unwrap(), - graph: serde_json::to_value(graph).unwrap(), - workflow_source: None, - labels: BTreeMap::new(), - source_directory: None, - workflow_slug: Some("test".to_string()), - workflow_version_id: None, - automation: None, - provenance: test_support::test_run_provenance(), - manifest_blob: None, - spec_blob: None, - git: None, - fork_source_ref: None, - retried_from: None, - parent_id: None, - web_url: None, - }) - .await - .unwrap(); + seed_run_created( + &run_store, + serde_json::to_value(WorkflowSettings::default()).unwrap(), + serde_json::to_value(graph).unwrap(), + None, + None, + ) + .await; let store_logger = StoreProgressLogger::new(run_store.clone()); let seen = Arc::new(std::sync::Mutex::new(Vec::new())); emitter.on_event({