mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-08-28 05:27:41 +00:00
Add exe.dev cancellation, git cloning, and checkpointing support
- Wire up cancel_token in ExeSandbox::exec_command via tokio::select! - Clone the local git repo into exe.dev VMs after initialization - Add GitCredentialSandbox decorator for push credential refresh - Enable GitCheckpointMode::Remote for exe.dev (rename daytona_* → remote_*) - Rename setup_daytona_git → setup_remote_git (already sandbox-agnostic) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
e60c431a6b
commit
fc7ecde756
4 changed files with 414 additions and 30 deletions
|
|
@ -246,7 +246,7 @@ impl Sandbox for ExeSandbox {
|
|||
timeout_ms: u64,
|
||||
working_dir: Option<&str>,
|
||||
env_vars: Option<&HashMap<String, String>>,
|
||||
_cancel_token: Option<CancellationToken>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
) -> Result<ExecResult, String> {
|
||||
let ssh = self.data_ssh()?;
|
||||
let start = Instant::now();
|
||||
|
|
@ -273,7 +273,20 @@ impl Sandbox for ExeSandbox {
|
|||
let full_cmd = parts.join(" ");
|
||||
|
||||
let timeout = std::time::Duration::from_millis(timeout_ms);
|
||||
let output = ssh.run_command_with_timeout(&full_cmd, timeout).await;
|
||||
let token = cancel_token.unwrap_or_default();
|
||||
let output = tokio::select! {
|
||||
res = ssh.run_command_with_timeout(&full_cmd, timeout) => res,
|
||||
() = token.cancelled() => {
|
||||
let duration_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
return Ok(ExecResult {
|
||||
stdout: String::new(),
|
||||
stderr: "Command cancelled".to_string(),
|
||||
exit_code: -1,
|
||||
timed_out: true,
|
||||
duration_ms,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
let duration_ms = u64::try_from(start.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
|
||||
|
|
@ -684,7 +697,7 @@ mod tests {
|
|||
}
|
||||
|
||||
/// Helper: create an ExeSandbox with mock data SSH already initialized (skipping lifecycle).
|
||||
fn sandbox_with_mock_data(data_ssh: MockSshRunner) -> ExeSandbox {
|
||||
fn sandbox_with_mock_data(data_ssh: impl SshRunner + 'static) -> ExeSandbox {
|
||||
let mgmt = MockSshRunner::new();
|
||||
let sandbox = ExeSandbox::new(Box::new(mgmt));
|
||||
let _ = sandbox.vm_name.set("test-vm".to_string());
|
||||
|
|
@ -796,6 +809,53 @@ mod tests {
|
|||
assert_eq!(result.exit_code, -1);
|
||||
}
|
||||
|
||||
/// SSH runner that never completes — blocks forever on a Notify.
|
||||
struct HangingSshRunner;
|
||||
|
||||
#[async_trait]
|
||||
impl SshRunner for HangingSshRunner {
|
||||
async fn run_command(&self, _command: &str) -> Result<SshOutput, String> {
|
||||
std::future::pending().await
|
||||
}
|
||||
|
||||
async fn run_command_with_timeout(
|
||||
&self,
|
||||
_command: &str,
|
||||
_timeout: std::time::Duration,
|
||||
) -> Result<SshOutput, String> {
|
||||
std::future::pending().await
|
||||
}
|
||||
|
||||
async fn upload_file(&self, _path: &str, _content: &[u8]) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn download_file(&self, _path: &str) -> Result<Vec<u8>, String> {
|
||||
Ok(Vec::new())
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn exec_command_cancelled() {
|
||||
let sandbox = sandbox_with_mock_data(HangingSshRunner);
|
||||
|
||||
let token = CancellationToken::new();
|
||||
let token_clone = token.clone();
|
||||
|
||||
// Cancel immediately so the select! picks it up
|
||||
token_clone.cancel();
|
||||
|
||||
let result = sandbox
|
||||
.exec_command("sleep 999", 60_000, None, None, Some(token))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(result.timed_out);
|
||||
assert_eq!(result.exit_code, -1);
|
||||
assert_eq!(result.stderr, "Command cancelled");
|
||||
assert!(result.stdout.is_empty());
|
||||
}
|
||||
|
||||
// ---- Step 3: read_file ----
|
||||
|
||||
#[tokio::test]
|
||||
|
|
|
|||
|
|
@ -526,7 +526,7 @@ pub async fn run_command(
|
|||
env.set_event_callback(Arc::new(move |event| {
|
||||
emitter_cb.emit(&crate::event::WorkflowRunEvent::Sandbox { event });
|
||||
}));
|
||||
Arc::new(env)
|
||||
Arc::new(env) as Arc<dyn Sandbox>
|
||||
}
|
||||
SandboxProvider::Local => {
|
||||
let mut env = LocalSandbox::new(cwd);
|
||||
|
|
@ -538,15 +538,33 @@ pub async fn run_command(
|
|||
}
|
||||
};
|
||||
|
||||
// Wrap with ReadBeforeWriteSandbox to enforce read-before-write guard
|
||||
let sandbox: Arc<dyn Sandbox> = Arc::new(arc_agent::ReadBeforeWriteSandbox::new(sandbox));
|
||||
|
||||
// Initialize sandbox (creates sandbox/container once for the whole run)
|
||||
sandbox
|
||||
.initialize()
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to initialize sandbox: {e}"))?;
|
||||
|
||||
// Clone repo into exe.dev VM after initialization
|
||||
let exe_origin_url = if sandbox_provider == SandboxProvider::Exe {
|
||||
clone_repo_into_exe(&*sandbox, &emitter, &original_cwd, &github_app).await?
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Wrap exe.dev sandbox with GitCredentialSandbox for push credential refresh
|
||||
let sandbox: Arc<dyn Sandbox> = if sandbox_provider == SandboxProvider::Exe {
|
||||
Arc::new(crate::git_credential_sandbox::GitCredentialSandbox::new(
|
||||
sandbox,
|
||||
exe_origin_url,
|
||||
github_app.clone(),
|
||||
))
|
||||
} else {
|
||||
sandbox
|
||||
};
|
||||
|
||||
// Wrap with ReadBeforeWriteSandbox to enforce read-before-write guard
|
||||
let sandbox: Arc<dyn Sandbox> = Arc::new(arc_agent::ReadBeforeWriteSandbox::new(sandbox));
|
||||
|
||||
// Safety net: if we panic or return early, best-effort cleanup via spawn.
|
||||
let sandbox_for_cleanup = Arc::clone(&sandbox);
|
||||
let cleanup_guard = scopeguard::guard((), move |()| {
|
||||
|
|
@ -561,22 +579,24 @@ pub async fn run_command(
|
|||
}
|
||||
});
|
||||
|
||||
// Set up git inside Daytona sandbox (if applicable)
|
||||
let (daytona_base_sha, daytona_branch, daytona_base_branch) =
|
||||
if sandbox_provider == SandboxProvider::Daytona {
|
||||
match setup_daytona_git(&*sandbox, &run_id).await {
|
||||
Ok((base, branch, base_br)) => (Some(base), Some(branch), base_br),
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"{} Daytona git setup failed ({e}), running without git checkpoints.",
|
||||
styles.yellow.apply_to("Warning:"),
|
||||
);
|
||||
(None, None, None)
|
||||
}
|
||||
// Set up git inside remote sandbox (Daytona or exe.dev) for checkpoint commits
|
||||
let (remote_base_sha, remote_branch, remote_base_branch) = if matches!(
|
||||
sandbox_provider,
|
||||
SandboxProvider::Daytona | SandboxProvider::Exe
|
||||
) {
|
||||
match setup_remote_git(&*sandbox, &run_id).await {
|
||||
Ok((base, branch, base_br)) => (Some(base), Some(branch), base_br),
|
||||
Err(e) => {
|
||||
eprintln!(
|
||||
"{} Remote git setup failed ({e}), running without git checkpoints.",
|
||||
styles.yellow.apply_to("Warning:"),
|
||||
);
|
||||
(None, None, None)
|
||||
}
|
||||
} else {
|
||||
(None, None, None)
|
||||
};
|
||||
}
|
||||
} else {
|
||||
(None, None, None)
|
||||
};
|
||||
|
||||
// Create SSH access if requested
|
||||
if args.ssh {
|
||||
|
|
@ -730,7 +750,7 @@ pub async fn run_command(
|
|||
|
||||
// 7. Execute
|
||||
// Set up metadata branch for git checkpointing (host or remote)
|
||||
let meta_branch = if worktree_work_dir.is_some() || daytona_base_sha.is_some() {
|
||||
let meta_branch = if worktree_work_dir.is_some() || remote_base_sha.is_some() {
|
||||
Some(crate::git::MetadataStore::branch_name(&run_id))
|
||||
} else {
|
||||
None
|
||||
|
|
@ -748,13 +768,12 @@ pub async fn run_command(
|
|||
SandboxProvider::Local | SandboxProvider::Docker => {
|
||||
worktree_work_dir.map(GitCheckpointMode::Host)
|
||||
}
|
||||
SandboxProvider::Daytona => daytona_base_sha
|
||||
SandboxProvider::Daytona | SandboxProvider::Exe => remote_base_sha
|
||||
.as_ref()
|
||||
.map(|_| GitCheckpointMode::Remote(original_cwd.clone())),
|
||||
SandboxProvider::Exe => None,
|
||||
},
|
||||
base_sha: worktree_base_sha.or(daytona_base_sha),
|
||||
run_branch: worktree_branch.or(daytona_branch),
|
||||
base_sha: worktree_base_sha.or(remote_base_sha),
|
||||
run_branch: worktree_branch.or(remote_branch),
|
||||
meta_branch,
|
||||
labels: args
|
||||
.label
|
||||
|
|
@ -765,7 +784,7 @@ pub async fn run_command(
|
|||
checkpoint_exclude_globs,
|
||||
github_app: github_app.clone(),
|
||||
git_author,
|
||||
base_branch: detected_base_branch.or(daytona_base_branch),
|
||||
base_branch: detected_base_branch.or(remote_base_branch),
|
||||
pull_request_enabled: run_cfg
|
||||
.as_ref()
|
||||
.and_then(|c| c.pull_request.as_ref())
|
||||
|
|
@ -1003,9 +1022,150 @@ fn setup_worktree(
|
|||
Ok((worktree_path.clone(), worktree_path, branch_name, base_sha))
|
||||
}
|
||||
|
||||
/// Set up git inside a Daytona sandbox for checkpoint commits.
|
||||
/// Clone the local git repo into an exe.dev VM.
|
||||
/// Returns the HTTPS origin URL on success, or None if no git repo was detected.
|
||||
async fn clone_repo_into_exe(
|
||||
sandbox: &dyn arc_agent::Sandbox,
|
||||
emitter: &std::sync::Arc<crate::event::EventEmitter>,
|
||||
cwd: &std::path::Path,
|
||||
github_app: &Option<crate::github_app::GitHubAppCredentials>,
|
||||
) -> anyhow::Result<Option<String>> {
|
||||
use arc_agent::sandbox::SandboxEvent;
|
||||
|
||||
let (detected_url, branch) = match crate::daytona_sandbox::detect_repo_info(cwd) {
|
||||
Ok(info) => info,
|
||||
Err(_) => return Ok(None), // no git repo — continue without cloning
|
||||
};
|
||||
|
||||
let url = crate::github_app::ssh_url_to_https(&detected_url);
|
||||
emitter.emit(&crate::event::WorkflowRunEvent::Sandbox {
|
||||
event: SandboxEvent::GitCloneStarted {
|
||||
url: url.clone(),
|
||||
branch: branch.clone(),
|
||||
},
|
||||
});
|
||||
let clone_start = std::time::Instant::now();
|
||||
|
||||
// Resolve clone credentials via GitHub App or fall back to no auth
|
||||
let token = match github_app {
|
||||
Some(creds) => {
|
||||
let (owner, repo) = crate::github_app::parse_github_owner_repo(&url).map_err(|e| {
|
||||
emitter.emit(&crate::event::WorkflowRunEvent::Sandbox {
|
||||
event: SandboxEvent::GitCloneFailed {
|
||||
url: url.clone(),
|
||||
error: e.clone(),
|
||||
},
|
||||
});
|
||||
anyhow::anyhow!("Failed to parse GitHub URL for clone: {e}")
|
||||
})?;
|
||||
let (_username, password) =
|
||||
crate::github_app::resolve_clone_credentials(creds, &owner, &repo)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
emitter.emit(&crate::event::WorkflowRunEvent::Sandbox {
|
||||
event: SandboxEvent::GitCloneFailed {
|
||||
url: url.clone(),
|
||||
error: e.clone(),
|
||||
},
|
||||
});
|
||||
anyhow::anyhow!("Failed to get GitHub App credentials for clone: {e}")
|
||||
})?;
|
||||
password
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let auth_url = match &token {
|
||||
Some(t) => url.replacen("https://", &format!("https://x-access-token:{t}@"), 1),
|
||||
None => url.clone(),
|
||||
};
|
||||
|
||||
let working_dir = sandbox.working_directory();
|
||||
let branch_flag = branch
|
||||
.as_deref()
|
||||
.map(|b| format!(" --branch '{}'", b.replace('\'', "'\\''")))
|
||||
.unwrap_or_default();
|
||||
|
||||
// Try cloning directly into the working directory
|
||||
let clone_cmd = format!(
|
||||
"git clone{branch_flag} '{}' '{working_dir}'",
|
||||
auth_url.replace('\'', "'\\''"),
|
||||
);
|
||||
let clone_result = sandbox
|
||||
.exec_command(&clone_cmd, 300_000, Some("/tmp"), None, None)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("git clone failed: {e}"))?;
|
||||
|
||||
if clone_result.exit_code != 0 {
|
||||
// Fall back to init + fetch + checkout if directory is not empty
|
||||
if clone_result.stderr.contains("not an empty directory")
|
||||
|| clone_result
|
||||
.stderr
|
||||
.contains("already exists and is not an empty")
|
||||
{
|
||||
let init_cmds = [
|
||||
"git init",
|
||||
&format!(
|
||||
"git remote add origin '{}'",
|
||||
auth_url.replace('\'', "'\\''")
|
||||
),
|
||||
"git fetch origin",
|
||||
&format!("git checkout {}", branch.as_deref().unwrap_or("main")),
|
||||
];
|
||||
for cmd in &init_cmds {
|
||||
let r = sandbox
|
||||
.exec_command(cmd, 300_000, None, None, None)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!("git fallback command failed: {e}"))?;
|
||||
if r.exit_code != 0 {
|
||||
let err = format!("git fallback failed on '{cmd}': {}", r.stderr);
|
||||
emitter.emit(&crate::event::WorkflowRunEvent::Sandbox {
|
||||
event: SandboxEvent::GitCloneFailed {
|
||||
url: url.clone(),
|
||||
error: err.clone(),
|
||||
},
|
||||
});
|
||||
anyhow::bail!("{err}");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let err = format!(
|
||||
"git clone failed (exit {}): {}",
|
||||
clone_result.exit_code, clone_result.stderr
|
||||
);
|
||||
emitter.emit(&crate::event::WorkflowRunEvent::Sandbox {
|
||||
event: SandboxEvent::GitCloneFailed {
|
||||
url: url.clone(),
|
||||
error: err.clone(),
|
||||
},
|
||||
});
|
||||
anyhow::bail!("{err}");
|
||||
}
|
||||
}
|
||||
|
||||
// Set clean push credentials on the remote
|
||||
let set_url_cmd = format!(
|
||||
"git remote set-url origin '{}'",
|
||||
auth_url.replace('\'', "'\\''"),
|
||||
);
|
||||
let _ = sandbox
|
||||
.exec_command(&set_url_cmd, 10_000, None, None, None)
|
||||
.await;
|
||||
|
||||
let duration_ms = u64::try_from(clone_start.elapsed().as_millis()).unwrap_or(u64::MAX);
|
||||
emitter.emit(&crate::event::WorkflowRunEvent::Sandbox {
|
||||
event: SandboxEvent::GitCloneCompleted {
|
||||
url: url.clone(),
|
||||
duration_ms,
|
||||
},
|
||||
});
|
||||
|
||||
Ok(Some(url))
|
||||
}
|
||||
|
||||
/// Set up git inside a remote sandbox (Daytona or exe.dev) for checkpoint commits.
|
||||
/// Returns (base_sha, branch_name, base_branch) on success.
|
||||
async fn setup_daytona_git(
|
||||
async fn setup_remote_git(
|
||||
sandbox: &dyn arc_agent::Sandbox,
|
||||
run_id: &str,
|
||||
) -> anyhow::Result<(String, String, Option<String>)> {
|
||||
|
|
|
|||
163
crates/arc-workflows/src/git_credential_sandbox.rs
Normal file
163
crates/arc-workflows/src/git_credential_sandbox.rs
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_agent::sandbox::{DirEntry, ExecResult, GrepOptions, Sandbox};
|
||||
use async_trait::async_trait;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::github_app::GitHubAppCredentials;
|
||||
|
||||
/// Sandbox decorator that overrides `refresh_push_credentials` to rotate
|
||||
/// GitHub App tokens on the git remote URL.
|
||||
///
|
||||
/// This lets `ExeSandbox` (in `arc-exe`) get credential refresh behaviour
|
||||
/// without depending on `github_app` (which lives in `arc-workflows`).
|
||||
pub struct GitCredentialSandbox {
|
||||
inner: Arc<dyn Sandbox>,
|
||||
origin_url: Option<String>,
|
||||
github_app: Option<GitHubAppCredentials>,
|
||||
}
|
||||
|
||||
impl GitCredentialSandbox {
|
||||
pub fn new(
|
||||
inner: Arc<dyn Sandbox>,
|
||||
origin_url: Option<String>,
|
||||
github_app: Option<GitHubAppCredentials>,
|
||||
) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
origin_url,
|
||||
github_app,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Sandbox for GitCredentialSandbox {
|
||||
async fn initialize(&self) -> Result<(), String> {
|
||||
self.inner.initialize().await
|
||||
}
|
||||
|
||||
async fn cleanup(&self) -> Result<(), String> {
|
||||
self.inner.cleanup().await
|
||||
}
|
||||
|
||||
async fn exec_command(
|
||||
&self,
|
||||
command: &str,
|
||||
timeout_ms: u64,
|
||||
working_dir: Option<&str>,
|
||||
env_vars: Option<&HashMap<String, String>>,
|
||||
cancel_token: Option<CancellationToken>,
|
||||
) -> Result<ExecResult, String> {
|
||||
self.inner
|
||||
.exec_command(command, timeout_ms, working_dir, env_vars, cancel_token)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn read_file(
|
||||
&self,
|
||||
path: &str,
|
||||
offset: Option<usize>,
|
||||
limit: Option<usize>,
|
||||
) -> Result<String, String> {
|
||||
self.inner.read_file(path, offset, limit).await
|
||||
}
|
||||
|
||||
async fn write_file(&self, path: &str, content: &str) -> Result<(), String> {
|
||||
self.inner.write_file(path, content).await
|
||||
}
|
||||
|
||||
async fn delete_file(&self, path: &str) -> Result<(), String> {
|
||||
self.inner.delete_file(path).await
|
||||
}
|
||||
|
||||
async fn file_exists(&self, path: &str) -> Result<bool, String> {
|
||||
self.inner.file_exists(path).await
|
||||
}
|
||||
|
||||
async fn list_directory(
|
||||
&self,
|
||||
path: &str,
|
||||
depth: Option<usize>,
|
||||
) -> Result<Vec<DirEntry>, String> {
|
||||
self.inner.list_directory(path, depth).await
|
||||
}
|
||||
|
||||
async fn grep(
|
||||
&self,
|
||||
pattern: &str,
|
||||
path: &str,
|
||||
options: &GrepOptions,
|
||||
) -> Result<Vec<String>, String> {
|
||||
self.inner.grep(pattern, path, options).await
|
||||
}
|
||||
|
||||
async fn glob(&self, pattern: &str, path: Option<&str>) -> Result<Vec<String>, String> {
|
||||
self.inner.glob(pattern, path).await
|
||||
}
|
||||
|
||||
async fn download_file_to_local(
|
||||
&self,
|
||||
remote_path: &str,
|
||||
local_path: &std::path::Path,
|
||||
) -> Result<(), String> {
|
||||
self.inner
|
||||
.download_file_to_local(remote_path, local_path)
|
||||
.await
|
||||
}
|
||||
|
||||
fn working_directory(&self) -> &str {
|
||||
self.inner.working_directory()
|
||||
}
|
||||
|
||||
fn platform(&self) -> &str {
|
||||
self.inner.platform()
|
||||
}
|
||||
|
||||
fn os_version(&self) -> String {
|
||||
self.inner.os_version()
|
||||
}
|
||||
|
||||
fn sandbox_info(&self) -> String {
|
||||
self.inner.sandbox_info()
|
||||
}
|
||||
|
||||
async fn refresh_push_credentials(&self) -> Result<(), String> {
|
||||
let origin_url = match &self.origin_url {
|
||||
Some(url) => url,
|
||||
None => return Ok(()),
|
||||
};
|
||||
let creds = match &self.github_app {
|
||||
Some(c) => c,
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
let (owner, repo) = crate::github_app::parse_github_owner_repo(origin_url)
|
||||
.map_err(|e| format!("Failed to parse origin URL for credential refresh: {e}"))?;
|
||||
|
||||
let (_username, password) =
|
||||
crate::github_app::resolve_clone_credentials(creds, &owner, &repo)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to refresh GitHub App token: {e}"))?;
|
||||
|
||||
if let Some(token) = password {
|
||||
let auth_url =
|
||||
origin_url.replacen("https://", &format!("https://x-access-token:{token}@"), 1);
|
||||
let cmd = format!(
|
||||
"git -c maintenance.auto=0 remote set-url origin '{}'",
|
||||
auth_url.replace('\'', "'\\''"),
|
||||
);
|
||||
self.inner
|
||||
.exec_command(&cmd, 10_000, None, None, None)
|
||||
.await
|
||||
.map_err(|e| format!("Failed to set refreshed push credentials: {e}"))?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn set_autostop_interval(&self, minutes: i32) -> Result<(), String> {
|
||||
self.inner.set_autostop_interval(minutes).await
|
||||
}
|
||||
}
|
||||
|
|
@ -37,6 +37,7 @@ pub mod engine;
|
|||
pub mod error;
|
||||
pub mod event;
|
||||
pub mod git;
|
||||
pub(crate) mod git_credential_sandbox;
|
||||
pub mod github_app;
|
||||
pub mod graph;
|
||||
pub mod handler;
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue