Add DaytonaSandbox::computer_use() and browser screenshot integration test

Expose the Daytona SDK's ComputerUseService through DaytonaSandbox so
callers can start the virtual desktop, take screenshots, and interact
with mouse/keyboard programmatically.

The new integration test validates the full flow: start the desktop
environment, launch Chromium on DISPLAY=:0, and capture a screenshot
of example.com via the Computer Use API.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-03-10 01:54:14 -04:00
parent fef76409e5
commit 2e1015241e
2 changed files with 175 additions and 0 deletions

View file

@ -204,6 +204,17 @@ impl DaytonaSandbox {
self.event_callback = Some(cb);
}
/// Get the `ComputerUseService` for this sandbox.
///
/// Requires the sandbox to be initialized first.
pub async fn computer_use(&self) -> Result<daytona_sdk::ComputerUseService, String> {
let sandbox = self.sandbox()?;
sandbox
.computer_use()
.await
.map_err(|e| format!("Failed to get computer use service: {e}"))
}
/// Create SSH access and return the connection command string.
pub async fn create_ssh_access(&self) -> Result<String, String> {
let sandbox = self.sandbox()?;

View file

@ -31,6 +31,9 @@ async fn create_env_with_github_app(
github_app: Option<arc_github::GitHubAppCredentials>,
) -> DaytonaSandbox {
dotenvy::dotenv().ok();
if let Some(home) = dirs::home_dir() {
dotenvy::from_path(home.join(".arc/.env")).ok();
}
let client = daytona_sdk::Client::new()
.await
.expect("Failed to create Daytona client — is DAYTONA_API_KEY set?");
@ -1848,3 +1851,164 @@ async fn daytona_cp_upload_download_round_trip() {
// 9. Cleanup
env.cleanup().await.unwrap();
}
#[tokio::test]
#[ignore]
async fn daytona_computer_use_browser_screenshot() {
use base64::Engine;
// Run from a temp dir so detect_repo_info() finds no git repo and skips cloning.
let tmp = tempfile::tempdir().unwrap();
std::env::set_current_dir(tmp.path()).unwrap();
dotenvy::dotenv().ok();
if let Some(home) = dirs::home_dir() {
dotenvy::from_path(home.join(".arc/.env")).ok();
}
let client = daytona_sdk::Client::new()
.await
.expect("DAYTONA_API_KEY must be set");
let config = DaytonaConfig {
snapshot: Some(DaytonaSnapshotConfig {
name: "daytona-medium".into(),
cpu: None,
memory: None,
disk: None,
dockerfile: None,
}),
..DaytonaConfig::default()
};
let env = DaytonaSandbox::new(client, config, None, None);
env.initialize().await.unwrap();
// 1. Start the computer use desktop environment (Xvfb, xfce4, etc.)
let cu = env.computer_use().await.unwrap();
let start_resp = cu.start().await.expect("computer_use.start() failed");
eprintln!("Computer use started: {:?}", start_resp.message);
// 2. Find or install a browser
let check = env
.exec_command(
"which chromium || which chromium-browser || which google-chrome || echo NONE",
30_000,
None,
None,
None,
)
.await
.unwrap();
eprintln!("Browser check: {}", check.stdout.trim());
if check.stdout.trim() == "NONE" {
let install_result = env
.exec_command(
"apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y -qq chromium 2>&1",
180_000, None, None, None,
)
.await
.unwrap();
eprintln!(
"Browser install exit_code={}, last_line={}",
install_result.exit_code,
install_result.stdout.lines().last().unwrap_or("")
);
assert_eq!(install_result.exit_code, 0, "Chromium install failed");
}
let browser_bin = env
.exec_command(
"which chromium || which chromium-browser || which google-chrome",
10_000,
None,
None,
None,
)
.await
.unwrap();
let browser = browser_bin.stdout.trim().to_string();
eprintln!("Using browser: {browser}");
// 3. Detect the DISPLAY that computer use started
let display_check = env
.exec_command(
"ps aux | grep Xvfb | grep -v grep | head -1",
10_000,
None,
None,
None,
)
.await
.unwrap();
eprintln!("Xvfb process: {}", display_check.stdout.trim());
// 4. Launch browser with setsid to fully detach, and log stderr
let launch_cmd = format!(
"DISPLAY=:0 setsid {browser} --no-sandbox --disable-gpu \
--window-size=1024,768 --window-position=0,0 \
https://example.com > /tmp/chrome_stdout.log 2>/tmp/chrome_stderr.log &\n\
sleep 2 && echo launched"
);
let launch_result = env
.exec_command(&launch_cmd, 30_000, None, None, None)
.await
.unwrap();
eprintln!("Browser launch exit_code={}", launch_result.exit_code);
// 5. Wait for the page to load, then check if browser is running
tokio::time::sleep(std::time::Duration::from_secs(8)).await;
let ps_check = env
.exec_command(
"ps aux | grep -i chrom | grep -v grep",
10_000,
None,
None,
None,
)
.await
.unwrap();
eprintln!("Chrome processes:\n{}", ps_check.stdout);
let stderr_check = env
.exec_command(
"cat /tmp/chrome_stderr.log 2>/dev/null | tail -20",
10_000,
None,
None,
None,
)
.await
.unwrap();
eprintln!("Chrome stderr:\n{}", stderr_check.stdout);
// 5. Take a screenshot via the Computer Use API
let screenshot = cu
.screenshot()
.take_full_screen()
.await
.expect("screenshot failed");
let b64_data = screenshot
.screenshot
.expect("screenshot response had no data");
eprintln!(
"Screenshot captured: {} bytes base64 ({} bytes decoded approx)",
b64_data.len(),
b64_data.len() * 3 / 4
);
assert!(!b64_data.is_empty(), "screenshot should not be empty");
// 6. Decode and save to /tmp for manual inspection
let png_bytes = base64::engine::general_purpose::STANDARD
.decode(&b64_data)
.expect("base64 decode failed");
let output_path = "/tmp/daytona_browser_screenshot.png";
std::fs::write(output_path, &png_bytes).expect("failed to write screenshot");
eprintln!(
"Screenshot saved to {output_path} ({} bytes)",
png_bytes.len()
);
// 7. Cleanup
cu.stop().await.ok();
env.cleanup().await.unwrap();
}