diff --git a/crates/attractor/src/cli/cli_backend.rs b/crates/attractor/src/cli/cli_backend.rs new file mode 100644 index 000000000..03c0d5124 --- /dev/null +++ b/crates/attractor/src/cli/cli_backend.rs @@ -0,0 +1,652 @@ +use std::path::Path; +use std::sync::Arc; + +use agent::ExecutionEnvironment; +use async_trait::async_trait; + +use crate::context::Context; +use crate::error::AttractorError; +use crate::event::EventEmitter; +use crate::graph::Node; +use crate::handler::codergen::{CodergenBackend, CodergenResult}; +use crate::outcome::StageUsage; + +/// Models that are only available through CLI tools (not via API). +const CLI_ONLY_MODELS: &[&str] = &["gpt-5.3-codex-spark"]; + +/// Returns true if the given model is only available through a CLI tool. +#[must_use] +pub fn is_cli_only_model(model: &str) -> bool { + CLI_ONLY_MODELS.contains(&model) +} + +/// Build the CLI command string for a given provider. +/// +/// The `prompt_file` is the path to a file containing the prompt text, which +/// will be shell-redirected into the command's stdin. +#[must_use] +pub fn cli_command_for_provider(provider: &str, model: &str, prompt_file: &str) -> String { + let model_flag = if model.is_empty() { + String::new() + } else { + match provider { + "openai" => format!(" -m {model}"), + "gemini" => format!(" -m {model}"), + _ => format!(" --model {model}"), + } + }; + match provider { + // --full-auto: sandboxed auto-execution, escalates on request + "openai" => format!("codex exec --json --full-auto{model_flag} < {prompt_file}"), + // --yolo: auto-approve all tool calls + "gemini" => format!("gemini -o json --yolo{model_flag} < {prompt_file}"), + // --dangerously-skip-permissions: bypass all permission checks (required for non-interactive use). + // CLAUDECODE= unset to allow running inside a Claude Code session. + _ => format!("CLAUDECODE= claude -p --output-format stream-json --dangerously-skip-permissions{model_flag} < {prompt_file}"), + } +} + +/// Parsed response from a CLI tool invocation. +#[derive(Debug)] +pub struct CliResponse { + pub text: String, + pub input_tokens: i64, + pub output_tokens: i64, +} + +/// Parse NDJSON output from Claude CLI (`--output-format stream-json`). +/// +/// Looks for the last `{"type":"result",...}` line, extracts `result` text and `usage`. +fn parse_claude_ndjson(output: &str) -> Option { + let mut last_result: Option = None; + + for line in output.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + if let Ok(value) = serde_json::from_str::(line) { + if value.get("type").and_then(|t| t.as_str()) == Some("result") { + last_result = Some(value); + } + } + } + + let result = last_result?; + let text = result.get("result").and_then(|v| v.as_str()).unwrap_or("").to_string(); + let input_tokens = result + .pointer("/usage/input_tokens") + .and_then(|v| v.as_i64()) + .unwrap_or(0); + let output_tokens = result + .pointer("/usage/output_tokens") + .and_then(|v| v.as_i64()) + .unwrap_or(0); + + Some(CliResponse { + text, + input_tokens, + output_tokens, + }) +} + +/// Parse NDJSON output from Codex CLI (`codex exec --json`). +/// +/// Codex emits NDJSON lines. Text comes from `item.completed` events where +/// `item.type == "agent_message"`. Usage comes from the `turn.completed` event. +fn parse_codex_ndjson(output: &str) -> Option { + let mut last_message_text = String::new(); + let mut input_tokens: i64 = 0; + let mut output_tokens: i64 = 0; + let mut found_anything = false; + + for line in output.lines() { + let line = line.trim(); + if line.is_empty() { + continue; + } + let value: serde_json::Value = match serde_json::from_str(line) { + Ok(v) => v, + Err(_) => continue, + }; + + let event_type = value.get("type").and_then(|t| t.as_str()).unwrap_or(""); + + match event_type { + "item.completed" => { + let item_type = value.pointer("/item/type").and_then(|t| t.as_str()).unwrap_or(""); + if item_type == "agent_message" { + if let Some(text) = value.pointer("/item/text").and_then(|t| t.as_str()) { + last_message_text = text.to_string(); + found_anything = true; + } + } + } + "turn.completed" => { + input_tokens = value.pointer("/usage/input_tokens").and_then(|v| v.as_i64()).unwrap_or(0); + output_tokens = value.pointer("/usage/output_tokens").and_then(|v| v.as_i64()).unwrap_or(0); + found_anything = true; + } + _ => {} + } + } + + if !found_anything { + return None; + } + + Some(CliResponse { + text: last_message_text, + input_tokens, + output_tokens, + }) +} + +/// Parse JSON output from Gemini CLI (`-o json`). +/// +/// Gemini outputs a single JSON object with `response` for text and +/// `stats.models..tokens` for usage. +fn parse_gemini_json(output: &str) -> Option { + let value: serde_json::Value = serde_json::from_str(output.trim()).ok()?; + let text = value.get("response").and_then(|v| v.as_str()).unwrap_or("").to_string(); + + // Extract tokens from the first model in stats.models + let (input_tokens, output_tokens) = value + .pointer("/stats/models") + .and_then(|m| m.as_object()) + .and_then(|models| models.values().next()) + .map(|model_stats| { + let input = model_stats.pointer("/tokens/input").and_then(|v| v.as_i64()).unwrap_or(0); + let output = model_stats.pointer("/tokens/candidates").and_then(|v| v.as_i64()).unwrap_or(0); + (input, output) + }) + .unwrap_or((0, 0)); + + Some(CliResponse { + text, + input_tokens, + output_tokens, + }) +} + +/// Parse CLI output, choosing the right parser based on provider. +pub fn parse_cli_response(provider: &str, output: &str) -> Option { + match provider { + "openai" => parse_codex_ndjson(output), + "gemini" => parse_gemini_json(output), + _ => parse_claude_ndjson(output), + } +} + +/// CLI backend that invokes external CLI tools (claude, codex, gemini) via `exec_command()`. +pub struct CliBackend { + model: String, + provider: String, +} + +impl CliBackend { + #[must_use] + pub fn new(model: String, provider: String) -> Self { + Self { model, provider } + } + + /// Detect changed files by comparing git state before and after the CLI run. + async fn detect_changed_files( + &self, + execution_env: &Arc, + ) -> Vec { + // Get unstaged changes + let diff_result = execution_env + .exec_command("git diff --name-only", 30_000, None, None, None) + .await; + + // Get untracked files + let untracked_result = execution_env + .exec_command( + "git ls-files --others --exclude-standard", + 30_000, + None, + None, + None, + ) + .await; + + let mut files: Vec = Vec::new(); + + if let Ok(result) = diff_result { + if result.exit_code == 0 { + files.extend( + result + .stdout + .lines() + .filter(|l| !l.trim().is_empty()) + .map(String::from), + ); + } + } + + if let Ok(result) = untracked_result { + if result.exit_code == 0 { + files.extend( + result + .stdout + .lines() + .filter(|l| !l.trim().is_empty()) + .map(String::from), + ); + } + } + + files.sort(); + files.dedup(); + files + } +} + +#[async_trait] +impl CodergenBackend for CliBackend { + async fn run( + &self, + node: &Node, + prompt: &str, + _context: &Context, + _thread_id: Option<&str>, + _emitter: &Arc, + stage_dir: &Path, + execution_env: &Arc, + ) -> Result { + // 1. Snapshot git state before the CLI run + let files_before = self.detect_changed_files(execution_env).await; + + // 2. Write prompt to temp file + let prompt_path = "/tmp/attractor_cli_prompt.txt"; + execution_env + .write_file(prompt_path, prompt) + .await + .map_err(|e| AttractorError::Handler(format!("Failed to write prompt file: {e}")))?; + + // 3. Build and execute CLI command + let model = node.llm_model().unwrap_or(&self.model); + let provider = node.llm_provider().unwrap_or(&self.provider); + let command = cli_command_for_provider(provider, model, prompt_path); + + let _ = tokio::fs::create_dir_all(stage_dir).await; + let provider_used = serde_json::json!({ + "mode": "cli", + "provider": provider, + "model": model, + "command": &command, + }); + if let Ok(json) = serde_json::to_string_pretty(&provider_used) { + let _ = tokio::fs::write(stage_dir.join("provider_used.json"), json).await; + } + + let result = execution_env + .exec_command(&command, 600_000, None, None, None) + .await + .map_err(|e| AttractorError::Handler(format!("CLI command failed: {e}")))?; + + if let Ok(json) = serde_json::to_string_pretty(&serde_json::json!({ + "exit_code": result.exit_code, + "stdout_len": result.stdout.len(), + "stderr_len": result.stderr.len(), + "duration_ms": result.duration_ms, + })) { + let _ = tokio::fs::write(stage_dir.join("cli_result_meta.json"), json).await; + } + + if result.exit_code != 0 { + return Err(AttractorError::Handler(format!( + "CLI command exited with code {}: {}", + result.exit_code, + result.stderr.chars().take(500).collect::() + ))); + } + + // 4. Parse the CLI output + let parsed = parse_cli_response(provider, &result.stdout).ok_or_else(|| { + AttractorError::Handler("Failed to parse CLI output".to_string()) + })?; + + // 5. Detect changed files + let files_after = self.detect_changed_files(execution_env).await; + let files_touched: Vec = files_after + .into_iter() + .filter(|f| !files_before.contains(f)) + .collect(); + + let mut stage_usage = StageUsage { + model: model.to_string(), + input_tokens: parsed.input_tokens, + output_tokens: parsed.output_tokens, + cache_read_tokens: None, + cache_write_tokens: None, + reasoning_tokens: None, + cost: None, + }; + stage_usage.cost = super::compute_stage_cost(&stage_usage); + + Ok(CodergenResult::Text { + text: parsed.text, + usage: Some(stage_usage), + files_touched, + }) + } +} + +/// Routes codergen invocations to either the API backend or CLI backend +/// based on node attributes and model type. +pub struct BackendRouter { + api_backend: Box, + cli_backend: CliBackend, +} + +impl BackendRouter { + #[must_use] + pub fn new(api_backend: Box, cli_backend: CliBackend) -> Self { + Self { + api_backend, + cli_backend, + } + } + + fn should_use_cli(&self, node: &Node) -> bool { + // Explicit backend="cli" attribute on the node + if node.backend() == Some("cli") { + return true; + } + + // CLI-only model on the node + if let Some(model) = node.llm_model() { + if is_cli_only_model(model) { + return true; + } + } + + false + } +} + +#[async_trait] +impl CodergenBackend for BackendRouter { + async fn run( + &self, + node: &Node, + prompt: &str, + context: &Context, + thread_id: Option<&str>, + emitter: &Arc, + stage_dir: &Path, + execution_env: &Arc, + ) -> Result { + if self.should_use_cli(node) { + self.cli_backend + .run(node, prompt, context, thread_id, emitter, stage_dir, execution_env) + .await + } else { + self.api_backend + .run(node, prompt, context, thread_id, emitter, stage_dir, execution_env) + .await + } + } + + async fn one_shot( + &self, + node: &Node, + prompt: &str, + stage_dir: &Path, + ) -> Result { + // CLI backend doesn't support one_shot, always route to API + self.api_backend.one_shot(node, prompt, stage_dir).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::graph::AttrValue; + + // -- Cycle 1: cli_command_for_provider -- + + #[test] + fn cli_command_for_codex() { + let cmd = cli_command_for_provider("openai", "gpt-5.3-codex-spark", "/tmp/prompt.txt"); + assert!(cmd.starts_with("codex exec --json --full-auto")); + assert!(cmd.contains("-m gpt-5.3-codex-spark")); + assert!(cmd.ends_with("< /tmp/prompt.txt")); + } + + #[test] + fn cli_command_for_claude() { + let cmd = cli_command_for_provider("anthropic", "claude-opus-4-6", "/tmp/prompt.txt"); + assert!(cmd.contains("claude -p")); + assert!(cmd.contains("--dangerously-skip-permissions")); + assert!(cmd.contains("--output-format stream-json")); + assert!(cmd.contains("--model claude-opus-4-6")); + } + + #[test] + fn cli_command_for_gemini() { + let cmd = cli_command_for_provider("gemini", "gemini-3.1-pro", "/tmp/prompt.txt"); + assert!(cmd.starts_with("gemini -o json --yolo")); + assert!(cmd.contains("-m gemini-3.1-pro")); + } + + #[test] + fn cli_command_defaults_to_claude() { + let cmd = cli_command_for_provider("unknown_provider", "some-model", "/tmp/prompt.txt"); + assert!(cmd.contains("claude ")); + assert!(cmd.contains("--dangerously-skip-permissions")); + } + + #[test] + fn cli_command_omits_model_when_empty() { + let cmd = cli_command_for_provider("openai", "", "/tmp/prompt.txt"); + assert!(cmd.starts_with("codex exec --json --full-auto")); + assert!(!cmd.contains("-m ")); + let cmd = cli_command_for_provider("anthropic", "", "/tmp/prompt.txt"); + assert!(cmd.contains("--dangerously-skip-permissions")); + assert!(!cmd.contains("--model ")); + let cmd = cli_command_for_provider("gemini", "", "/tmp/prompt.txt"); + assert!(cmd.contains("--yolo")); + assert!(!cmd.contains("-m ")); + } + + // -- Cycle 2: is_cli_only_model -- + + #[test] + fn codex_spark_is_cli_only() { + assert!(is_cli_only_model("gpt-5.3-codex-spark")); + } + + #[test] + fn claude_opus_is_not_cli_only() { + assert!(!is_cli_only_model("claude-opus-4-6")); + } + + #[test] + fn gemini_is_not_cli_only() { + assert!(!is_cli_only_model("gemini-3.1-pro-preview")); + } + + // -- Cycle 3: parse_cli_response — Claude/Gemini NDJSON -- + + #[test] + fn parse_claude_ndjson_extracts_text_and_usage() { + let output = r#"{"type":"system","message":"Claude CLI v1.0"} +{"type":"assistant","message":{"content":"thinking..."}} +{"type":"result","result":"Here is the implementation.","usage":{"input_tokens":100,"output_tokens":50}}"#; + let response = parse_cli_response("anthropic", output).unwrap(); + assert_eq!(response.text, "Here is the implementation."); + assert_eq!(response.input_tokens, 100); + assert_eq!(response.output_tokens, 50); + } + + #[test] + fn parse_claude_ndjson_uses_last_result() { + let output = r#"{"type":"result","result":"first","usage":{"input_tokens":10,"output_tokens":5}} +{"type":"result","result":"second","usage":{"input_tokens":20,"output_tokens":10}}"#; + let response = parse_cli_response("anthropic", output).unwrap(); + assert_eq!(response.text, "second"); + assert_eq!(response.input_tokens, 20); + } + + #[test] + fn parse_claude_ndjson_returns_none_for_no_result() { + let output = r#"{"type":"system","message":"hello"} +{"type":"assistant","message":{"content":"no result line"}}"#; + assert!(parse_cli_response("anthropic", output).is_none()); + } + + #[test] + fn parse_gemini_json_extracts_text_and_usage() { + let output = r#"{"session_id":"abc","response":"Gemini says hello","stats":{"models":{"gemini-2.5-flash":{"tokens":{"input":200,"candidates":80,"total":280}}}}}"#; + let response = parse_cli_response("gemini", output).unwrap(); + assert_eq!(response.text, "Gemini says hello"); + assert_eq!(response.input_tokens, 200); + assert_eq!(response.output_tokens, 80); + } + + #[test] + fn parse_gemini_json_handles_missing_stats() { + let output = r#"{"response":"hello"}"#; + let response = parse_cli_response("gemini", output).unwrap(); + assert_eq!(response.text, "hello"); + assert_eq!(response.input_tokens, 0); + assert_eq!(response.output_tokens, 0); + } + + #[test] + fn parse_gemini_json_returns_none_for_invalid_json() { + assert!(parse_cli_response("gemini", "not json").is_none()); + } + + // -- Cycle 4: parse_cli_response — Codex NDJSON -- + + #[test] + fn parse_codex_ndjson_extracts_text_and_usage() { + let output = r#"{"type":"thread.started","thread_id":"abc"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"thinking..."}} +{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"Fixed the bug."}} +{"type":"turn.completed","usage":{"input_tokens":300,"output_tokens":150}}"#; + let response = parse_cli_response("openai", output).unwrap(); + assert_eq!(response.text, "Fixed the bug."); + assert_eq!(response.input_tokens, 300); + assert_eq!(response.output_tokens, 150); + } + + #[test] + fn parse_codex_ndjson_handles_no_message() { + let output = r#"{"type":"turn.completed","usage":{"input_tokens":10,"output_tokens":5}}"#; + let response = parse_cli_response("openai", output).unwrap(); + assert_eq!(response.text, ""); + assert_eq!(response.input_tokens, 10); + } + + #[test] + fn parse_codex_ndjson_returns_none_for_no_events() { + assert!(parse_cli_response("openai", "not json at all").is_none()); + } + + // -- Cycle 5: Node::backend() accessor (tested here since the accessor is simple) -- + + #[test] + fn node_backend_returns_none_by_default() { + let node = Node::new("test"); + assert_eq!(node.backend(), None); + } + + #[test] + fn node_backend_returns_cli_when_set() { + let mut node = Node::new("test"); + node.attrs + .insert("backend".to_string(), AttrValue::String("cli".to_string())); + assert_eq!(node.backend(), Some("cli")); + } + + // -- Cycle 6: backend in stylesheet (tested in stylesheet.rs) -- + + // -- Cycle 7: BackendRouter routing logic -- + + #[test] + fn router_uses_cli_for_backend_attr() { + let mut node = Node::new("test"); + node.attrs + .insert("backend".to_string(), AttrValue::String("cli".to_string())); + + let cli_backend = CliBackend::new("model".into(), "anthropic".into()); + let router = BackendRouter::new( + Box::new(StubBackend), + cli_backend, + ); + assert!(router.should_use_cli(&node)); + } + + #[test] + fn router_uses_cli_for_cli_only_model() { + let mut node = Node::new("test"); + node.attrs.insert( + "llm_model".to_string(), + AttrValue::String("gpt-5.3-codex-spark".to_string()), + ); + + let cli_backend = CliBackend::new("model".into(), "openai".into()); + let router = BackendRouter::new( + Box::new(StubBackend), + cli_backend, + ); + assert!(router.should_use_cli(&node)); + } + + #[test] + fn router_uses_api_by_default() { + let node = Node::new("test"); + + let cli_backend = CliBackend::new("model".into(), "anthropic".into()); + let router = BackendRouter::new( + Box::new(StubBackend), + cli_backend, + ); + assert!(!router.should_use_cli(&node)); + } + + #[test] + fn router_uses_api_for_non_cli_model() { + let mut node = Node::new("test"); + node.attrs.insert( + "llm_model".to_string(), + AttrValue::String("claude-opus-4-6".to_string()), + ); + + let cli_backend = CliBackend::new("model".into(), "anthropic".into()); + let router = BackendRouter::new( + Box::new(StubBackend), + cli_backend, + ); + assert!(!router.should_use_cli(&node)); + } + + /// Minimal stub backend for testing routing logic. + struct StubBackend; + + #[async_trait] + impl CodergenBackend for StubBackend { + async fn run( + &self, + _node: &Node, + _prompt: &str, + _context: &Context, + _thread_id: Option<&str>, + _emitter: &Arc, + _stage_dir: &Path, + _execution_env: &Arc, + ) -> Result { + Ok(CodergenResult::Text { + text: "stub".to_string(), + usage: None, + files_touched: Vec::new(), + }) + } + } +} diff --git a/crates/attractor/src/cli/mod.rs b/crates/attractor/src/cli/mod.rs index 58b62e576..620556f46 100644 --- a/crates/attractor/src/cli/mod.rs +++ b/crates/attractor/src/cli/mod.rs @@ -1,4 +1,5 @@ pub mod backend; +pub mod cli_backend; pub mod run; #[cfg(feature = "server")] pub mod serve; diff --git a/crates/attractor/src/cli/run.rs b/crates/attractor/src/cli/run.rs index c372c1412..b7933f2aa 100644 --- a/crates/attractor/src/cli/run.rs +++ b/crates/attractor/src/cli/run.rs @@ -19,6 +19,7 @@ use crate::pipeline::PipelineBuilder; use crate::validation::Severity; use super::backend::AgentBackend; +use super::cli_backend::{BackendRouter, CliBackend}; use super::task_config; use super::{compute_stage_cost, format_cost, format_duration_human, format_event_detail, format_event_summary, format_tokens_human, print_diagnostics, read_dot_file, ExecutionEnvKind, RunArgs}; @@ -403,12 +404,17 @@ pub async fn run_command(args: RunArgs, styles: &'static Styles) -> anyhow::Resu if dry_run_mode { None } else { - Some(Box::new(AgentBackend::new( + let api = AgentBackend::new( model.clone(), provider.clone(), args.verbose, styles, - ))) + ); + let cli = CliBackend::new( + model.clone(), + provider.clone().unwrap_or_else(|| "anthropic".to_string()), + ); + Some(Box::new(BackendRouter::new(Box::new(api), cli))) } }); let engine = PipelineEngine::with_interviewer(registry, Arc::clone(&emitter), interviewer, Arc::clone(&execution_env)); diff --git a/crates/attractor/src/graph/types.rs b/crates/attractor/src/graph/types.rs index af4017e47..7a3863354 100644 --- a/crates/attractor/src/graph/types.rs +++ b/crates/attractor/src/graph/types.rs @@ -225,6 +225,11 @@ impl Node { self.str_attr("retry_policy") } + #[must_use] + pub fn backend(&self) -> Option<&str> { + self.str_attr("backend") + } + /// Returns the codergen mode for this node. Defaults to `AgentLoop` when absent. pub fn codergen_mode(&self) -> Result { match self.str_attr("codergen_mode") { diff --git a/crates/attractor/src/stylesheet.rs b/crates/attractor/src/stylesheet.rs index b97423008..313ca707e 100644 --- a/crates/attractor/src/stylesheet.rs +++ b/crates/attractor/src/stylesheet.rs @@ -179,7 +179,7 @@ fn parse_declarations(remaining: &mut &str) -> Result, Attracto } /// Recognized stylesheet properties. -const STYLESHEET_PROPERTIES: &[&str] = &["llm_model", "llm_provider", "reasoning_effort"]; +const STYLESHEET_PROPERTIES: &[&str] = &["llm_model", "llm_provider", "reasoning_effort", "backend"]; /// Apply a stylesheet to a graph. Rules are applied by specificity order; /// higher specificity wins. Explicit node attributes are never overridden. @@ -530,4 +530,33 @@ mod tests { Some(&AttrValue::String("sonnet".into())) ); } + + #[test] + fn apply_backend_property_via_stylesheet() { + let ss = parse_stylesheet("* { backend: cli; }").unwrap(); + let mut graph = Graph::new("test"); + graph.nodes.insert("a".into(), Node::new("a")); + apply_stylesheet(&ss, &mut graph); + + assert_eq!( + graph.nodes["a"].attrs.get("backend"), + Some(&AttrValue::String("cli".into())) + ); + } + + #[test] + fn backend_property_not_overridden_by_stylesheet() { + let ss = parse_stylesheet("* { backend: cli; }").unwrap(); + let mut graph = Graph::new("test"); + let mut node = Node::new("a"); + node.attrs + .insert("backend".into(), AttrValue::String("api".into())); + graph.nodes.insert("a".into(), node); + apply_stylesheet(&ss, &mut graph); + + assert_eq!( + graph.nodes["a"].attrs.get("backend"), + Some(&AttrValue::String("api".into())) + ); + } } \ No newline at end of file diff --git a/crates/attractor/tests/daytona_integration.rs b/crates/attractor/tests/daytona_integration.rs index 94a9e8c47..024b9051f 100644 --- a/crates/attractor/tests/daytona_integration.rs +++ b/crates/attractor/tests/daytona_integration.rs @@ -307,3 +307,116 @@ async fn daytona_pipeline_artifact_offload_and_sync() { env.cleanup().await.unwrap(); } + +// --------------------------------------------------------------------------- +// CLI Backend on Daytona — real CLI tools via exec_command +// --------------------------------------------------------------------------- + +use attractor::cli::cli_backend::CliBackend; +use attractor::handler::codergen::{CodergenBackend, CodergenResult}; + +/// Helper: run a real CLI backend test on Daytona. +/// +/// Installs the CLI tool in the sandbox, then runs the CliBackend against it. +async fn run_daytona_cli_test( + provider: &str, + model: &str, + install_command: &str, +) { + let env = create_env().await; + env.initialize().await.unwrap(); + let env: Arc = Arc::new(env); + + // Install the CLI tool inside the Daytona sandbox + let install_result = env + .exec_command(install_command, 120_000, None, None, None) + .await + .expect("install command should not error"); + assert_eq!( + install_result.exit_code, 0, + "install command failed (exit {}): {}", + install_result.exit_code, install_result.stdout + ); + + let backend = CliBackend::new(model.to_string(), provider.to_string()); + let node = Node::new("daytona_cli_test"); + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + let result = backend + .run( + &node, + "What is 2+2? Reply with just the number.", + &context, + None, + &emitter, + dir.path(), + &env, + ) + .await; + + match result { + Ok(CodergenResult::Text { text, usage, .. }) => { + assert!( + text.contains('4'), + "{provider}/{model} on Daytona: expected '4', got: {text}" + ); + if let Some(u) = usage { + assert!( + u.input_tokens > 0, + "{provider}/{model}: input_tokens should be > 0" + ); + } + } + Ok(CodergenResult::Full(_)) => panic!("expected Text result"), + Err(e) => panic!("{provider}/{model} on Daytona failed: {e}"), + } + + // Verify log files + let provider_path = dir.path().join("provider_used.json"); + assert!( + provider_path.exists(), + "{provider}/{model}: provider_used.json should exist" + ); + let provider_json: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(&provider_path).unwrap(), + ) + .unwrap(); + assert_eq!(provider_json["mode"], "cli"); + + env.cleanup().await.unwrap(); +} + +#[tokio::test] +#[ignore] // requires DAYTONA_API_KEY + Claude CLI auth +async fn daytona_cli_claude() { + run_daytona_cli_test( + "anthropic", + "haiku", + "curl -fsSL https://claude.ai/install.sh | sh", + ) + .await; +} + +#[tokio::test] +#[ignore] // requires DAYTONA_API_KEY + OpenAI/Codex auth +async fn daytona_cli_codex() { + run_daytona_cli_test( + "openai", + "o4-mini", + "npm install -g @openai/codex", + ) + .await; +} + +#[tokio::test] +#[ignore] // requires DAYTONA_API_KEY + Gemini auth +async fn daytona_cli_gemini() { + run_daytona_cli_test( + "gemini", + "gemini-2.5-flash", + "npm install -g @google/gemini-cli", + ) + .await; +} diff --git a/crates/attractor/tests/integration.rs b/crates/attractor/tests/integration.rs index 242ce7ef3..ab7d97322 100644 --- a/crates/attractor/tests/integration.rs +++ b/crates/attractor/tests/integration.rs @@ -7185,4 +7185,739 @@ async fn node_dir_uses_visit_count_on_revisit() { ).unwrap(); assert_eq!(first_json["status"], "fail"); assert_eq!(second_json["status"], "success"); +} + +// --------------------------------------------------------------------------- +// CLI Backend end-to-end tests +// --------------------------------------------------------------------------- + +use attractor::cli::cli_backend::{BackendRouter, CliBackend}; + +/// A mock execution environment for CLI backend e2e tests. +/// Records all exec_command and write_file calls, and returns configurable +/// responses based on command content. +struct CliTestEnv { + /// All commands passed to exec_command, in order. + commands: std::sync::Mutex>, + /// All (path, content) pairs from write_file. + written_files: std::sync::Mutex>, + /// The stdout to return when the CLI command (not git) is executed. + cli_stdout: String, + /// Files returned by "git diff --name-only" AFTER the CLI runs. + /// First call returns empty (before), second returns these (after). + git_diff_call_count: std::sync::atomic::AtomicU32, + git_diff_after: String, +} + +impl CliTestEnv { + fn new(cli_stdout: &str) -> Self { + Self { + commands: std::sync::Mutex::new(Vec::new()), + written_files: std::sync::Mutex::new(Vec::new()), + cli_stdout: cli_stdout.to_string(), + git_diff_call_count: std::sync::atomic::AtomicU32::new(0), + git_diff_after: String::new(), + } + } + + fn with_git_diff_after(mut self, files: &str) -> Self { + self.git_diff_after = files.to_string(); + self + } + + fn recorded_commands(&self) -> Vec { + self.commands.lock().unwrap().clone() + } + + fn recorded_written_files(&self) -> Vec<(String, String)> { + self.written_files.lock().unwrap().clone() + } +} + +#[async_trait::async_trait] +impl agent::ExecutionEnvironment for CliTestEnv { + async fn read_file(&self, _path: &str, _offset: Option, _limit: Option) -> Result { + Ok(String::new()) + } + + async fn write_file(&self, path: &str, content: &str) -> Result<(), String> { + self.written_files.lock().unwrap().push((path.to_string(), content.to_string())); + Ok(()) + } + + async fn delete_file(&self, _path: &str) -> Result<(), String> { + Ok(()) + } + + async fn file_exists(&self, _path: &str) -> Result { + Ok(false) + } + + async fn list_directory(&self, _path: &str, _depth: Option) -> Result, String> { + Ok(vec![]) + } + + async fn exec_command( + &self, + command: &str, + _timeout_ms: u64, + _working_dir: Option<&str>, + _env_vars: Option<&std::collections::HashMap>, + _cancel_token: Option, + ) -> Result { + self.commands.lock().unwrap().push(command.to_string()); + + // git diff calls: first pair returns empty (before), second pair returns configured files + if command.starts_with("git diff") || command.starts_with("git ls-files") { + let call_num = self.git_diff_call_count.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + // Calls 0,1 = before snapshot (empty), calls 2,3 = after snapshot + let stdout = if call_num >= 2 && command.starts_with("git diff") { + self.git_diff_after.clone() + } else { + String::new() + }; + return Ok(agent::ExecResult { + stdout, + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 5, + }); + } + + // CLI command: return configured stdout + Ok(agent::ExecResult { + stdout: self.cli_stdout.clone(), + stderr: String::new(), + exit_code: 0, + timed_out: false, + duration_ms: 100, + }) + } + + async fn grep(&self, _pattern: &str, _path: &str, _options: &agent::GrepOptions) -> Result, String> { + Ok(vec![]) + } + + async fn glob(&self, _pattern: &str, _path: Option<&str>) -> Result, String> { + Ok(vec![]) + } + + async fn initialize(&self) -> Result<(), String> { + Ok(()) + } + + async fn cleanup(&self) -> Result<(), String> { + Ok(()) + } + + fn working_directory(&self) -> &str { + "/tmp/test" + } + + fn platform(&self) -> &str { + "darwin" + } + + fn os_version(&self) -> String { + "Darwin 24.0.0".into() + } +} + +// -- Cycle 8: CliBackend::run() e2e via mock ExecutionEnvironment -- + +#[tokio::test] +async fn cli_backend_run_writes_prompt_and_calls_exec() { + let claude_output = r#"{"type":"result","result":"I fixed the bug.","usage":{"input_tokens":500,"output_tokens":200}}"#; + let test_env = Arc::new(CliTestEnv::new(claude_output)); + let env: Arc = test_env.clone(); + let backend = CliBackend::new("claude-opus-4-6".into(), "anthropic".into()); + + let node = Node::new("fix_code"); + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + let result = backend + .run(&node, "Fix the authentication bug", &context, None, &emitter, dir.path(), &env) + .await + .expect("CLI backend should succeed"); + + // Verify prompt was written + let written = test_env.recorded_written_files(); + assert_eq!(written.len(), 1, "should write exactly one file (the prompt)"); + assert_eq!(written[0].0, "/tmp/attractor_cli_prompt.txt"); + assert_eq!(written[0].1, "Fix the authentication bug"); + + // Verify the CLI command was called + let commands = test_env.recorded_commands(); + let cli_cmd = commands.iter().find(|c| c.contains("claude")).expect("should call claude CLI"); + assert!(cli_cmd.contains("-p"), "should use pipe mode"); + assert!(cli_cmd.contains("claude-opus-4-6"), "should use correct model"); + assert!(cli_cmd.contains("/tmp/attractor_cli_prompt.txt"), "should reference prompt file"); + + // Verify parsed response + match result { + CodergenResult::Text { text, usage, files_touched } => { + assert_eq!(text, "I fixed the bug."); + let usage = usage.expect("should have usage"); + assert_eq!(usage.input_tokens, 500); + assert_eq!(usage.output_tokens, 200); + assert!(files_touched.is_empty(), "no files changed before/after"); + } + CodergenResult::Full(_) => panic!("expected Text result, got Full"), + } +} + +#[tokio::test] +async fn cli_backend_run_detects_changed_files() { + let claude_output = r#"{"type":"result","result":"Created new file.","usage":{"input_tokens":100,"output_tokens":50}}"#; + let env: Arc = Arc::new( + CliTestEnv::new(claude_output) + .with_git_diff_after("src/main.rs\nsrc/lib.rs\n"), + ); + let backend = CliBackend::new("claude-opus-4-6".into(), "anthropic".into()); + + let node = Node::new("implement"); + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + let result = backend + .run(&node, "Add a new feature", &context, None, &emitter, dir.path(), &env) + .await + .expect("CLI backend should succeed"); + + match result { + CodergenResult::Text { files_touched, .. } => { + assert_eq!(files_touched, vec!["src/lib.rs", "src/main.rs"]); + } + CodergenResult::Full(_) => panic!("expected Text result"), + } +} + +#[tokio::test] +async fn cli_backend_run_with_codex_provider() { + let codex_output = "{\"type\":\"item.completed\",\"item\":{\"id\":\"item_0\",\"type\":\"agent_message\",\"text\":\"Implemented the feature.\"}}\n{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":300,\"output_tokens\":150}}"; + let test_env = Arc::new(CliTestEnv::new(codex_output)); + let env: Arc = test_env.clone(); + let backend = CliBackend::new("gpt-5.3-codex-spark".into(), "openai".into()); + + let node = Node::new("implement"); + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + let result = backend + .run(&node, "Build the API", &context, None, &emitter, dir.path(), &env) + .await + .expect("CLI backend should succeed"); + + // Verify codex command was called + let commands = test_env.recorded_commands(); + let cli_cmd = commands.iter().find(|c| c.contains("codex")).expect("should call codex CLI"); + assert!(cli_cmd.contains("exec --json"), "should use exec mode"); + assert!(cli_cmd.contains("gpt-5.3-codex-spark"), "should use correct model"); + + match result { + CodergenResult::Text { text, usage, .. } => { + assert_eq!(text, "Implemented the feature."); + let usage = usage.expect("should have usage"); + assert_eq!(usage.input_tokens, 300); + assert_eq!(usage.output_tokens, 150); + } + CodergenResult::Full(_) => panic!("expected Text result"), + } +} + +#[tokio::test] +async fn cli_backend_run_fails_on_nonzero_exit() { + let env = Arc::new(CliTestEnv::new("")); + + // Override exec_command to return non-zero for the CLI call + struct FailingCliEnv; + #[async_trait::async_trait] + impl agent::ExecutionEnvironment for FailingCliEnv { + async fn read_file(&self, _: &str, _: Option, _: Option) -> Result { Ok(String::new()) } + async fn write_file(&self, _: &str, _: &str) -> Result<(), String> { Ok(()) } + async fn delete_file(&self, _: &str) -> Result<(), String> { Ok(()) } + async fn file_exists(&self, _: &str) -> Result { Ok(false) } + async fn list_directory(&self, _: &str, _: Option) -> Result, String> { Ok(vec![]) } + async fn exec_command(&self, command: &str, _: u64, _: Option<&str>, _: Option<&std::collections::HashMap>, _: Option) -> Result { + if command.starts_with("git") { + return Ok(agent::ExecResult { stdout: String::new(), stderr: String::new(), exit_code: 0, timed_out: false, duration_ms: 0 }); + } + Ok(agent::ExecResult { stdout: String::new(), stderr: "command not found: claude".into(), exit_code: 127, timed_out: false, duration_ms: 0 }) + } + async fn grep(&self, _: &str, _: &str, _: &agent::GrepOptions) -> Result, String> { Ok(vec![]) } + async fn glob(&self, _: &str, _: Option<&str>) -> Result, String> { Ok(vec![]) } + async fn initialize(&self) -> Result<(), String> { Ok(()) } + async fn cleanup(&self) -> Result<(), String> { Ok(()) } + fn working_directory(&self) -> &str { "/tmp" } + fn platform(&self) -> &str { "darwin" } + fn os_version(&self) -> String { "Darwin 24.0.0".into() } + } + + let failing_env: Arc = Arc::new(FailingCliEnv); + let backend = CliBackend::new("claude-opus-4-6".into(), "anthropic".into()); + let node = Node::new("step"); + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + let _ = env; // unused, just for the above struct + + let result = backend + .run(&node, "do something", &context, None, &emitter, dir.path(), &failing_env) + .await; + + let err = match result { + Err(e) => e, + Ok(_) => panic!("should fail on non-zero exit"), + }; + + assert!(err.to_string().contains("exited with code 127"), "error: {err}"); + assert!(err.to_string().contains("command not found"), "error: {err}"); +} + +#[tokio::test] +async fn cli_backend_run_fails_on_unparseable_output() { + let env: Arc = Arc::new(CliTestEnv::new("this is not json at all")); + let backend = CliBackend::new("claude-opus-4-6".into(), "anthropic".into()); + + let node = Node::new("step"); + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + let result = backend + .run(&node, "do something", &context, None, &emitter, dir.path(), &env) + .await; + + let err = match result { + Err(e) => e, + Ok(_) => panic!("should fail on unparseable output"), + }; + + assert!(err.to_string().contains("Failed to parse CLI output"), "error: {err}"); +} + +#[tokio::test] +async fn cli_backend_run_uses_node_model_override() { + let claude_output = r#"{"type":"result","result":"ok","usage":{"input_tokens":10,"output_tokens":5}}"#; + let test_env = Arc::new(CliTestEnv::new(claude_output)); + let env: Arc = test_env.clone(); + let backend = CliBackend::new("default-model".into(), "anthropic".into()); + + let mut node = Node::new("step"); + node.attrs.insert("llm_model".to_string(), AttrValue::String("claude-sonnet-4-5".to_string())); + + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + backend + .run(&node, "test", &context, None, &emitter, dir.path(), &env) + .await + .expect("should succeed"); + + let commands = test_env.recorded_commands(); + let cli_cmd = commands.iter().find(|c| c.contains("claude")).unwrap(); + assert!(cli_cmd.contains("claude-sonnet-4-5"), "should use node's model override, not default: {cli_cmd}"); + assert!(!cli_cmd.contains("default-model"), "should NOT use default model: {cli_cmd}"); +} + +#[tokio::test] +async fn cli_backend_run_uses_node_provider_override() { + let codex_output = "{\"type\":\"item.completed\",\"item\":{\"id\":\"item_0\",\"type\":\"agent_message\",\"text\":\"ok\"}}\n{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":5}}"; + let test_env = Arc::new(CliTestEnv::new(codex_output)); + let env: Arc = test_env.clone(); + let backend = CliBackend::new("default-model".into(), "anthropic".into()); + + let mut node = Node::new("step"); + node.attrs.insert("llm_provider".to_string(), AttrValue::String("openai".to_string())); + node.attrs.insert("llm_model".to_string(), AttrValue::String("gpt-5.3-codex-spark".to_string())); + + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + backend + .run(&node, "test", &context, None, &emitter, dir.path(), &env) + .await + .expect("should succeed"); + + let commands = test_env.recorded_commands(); + let cli_cmd = commands.iter().find(|c| c.contains("codex")).expect("should call codex based on provider override"); + assert!(cli_cmd.contains("gpt-5.3-codex-spark")); +} + +#[tokio::test] +async fn cli_backend_run_writes_provider_used_json() { + let claude_output = r#"{"type":"result","result":"done","usage":{"input_tokens":10,"output_tokens":5}}"#; + let env: Arc = Arc::new(CliTestEnv::new(claude_output)); + let backend = CliBackend::new("claude-opus-4-6".into(), "anthropic".into()); + + let node = Node::new("step"); + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + backend + .run(&node, "test", &context, None, &emitter, dir.path(), &env) + .await + .expect("should succeed"); + + let provider_path = dir.path().join("provider_used.json"); + assert!(provider_path.exists(), "should write provider_used.json"); + let provider_json: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(&provider_path).unwrap() + ).unwrap(); + assert_eq!(provider_json["mode"], "cli"); + assert_eq!(provider_json["provider"], "anthropic"); + assert_eq!(provider_json["model"], "claude-opus-4-6"); + assert!(provider_json["command"].as_str().unwrap().contains("claude")); +} + +// -- BackendRouter e2e: delegates to correct backend -- + +#[tokio::test] +async fn backend_router_delegates_to_cli_for_cli_node() { + let claude_output = r#"{"type":"result","result":"CLI response","usage":{"input_tokens":10,"output_tokens":5}}"#; + let env: Arc = Arc::new(CliTestEnv::new(claude_output)); + + let api_backend = Box::new(MockCodergenBackend); // would return "Response for ..." + let cli = CliBackend::new("claude-opus-4-6".into(), "anthropic".into()); + let router = BackendRouter::new(api_backend, cli); + + let mut node = Node::new("cli_step"); + node.attrs.insert("backend".to_string(), AttrValue::String("cli".to_string())); + node.attrs.insert("prompt".to_string(), AttrValue::String("Fix the bug".to_string())); + + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + let result = router + .run(&node, "Fix the bug", &context, None, &emitter, dir.path(), &env) + .await + .expect("router should succeed"); + + match result { + CodergenResult::Text { text, .. } => { + assert_eq!(text, "CLI response", "should use CLI backend response, not mock API"); + } + CodergenResult::Full(_) => panic!("expected Text result"), + } +} + +#[tokio::test] +async fn backend_router_delegates_to_api_for_normal_node() { + let env = local_env(); + + let api_backend = Box::new(MockCodergenBackend); + let cli = CliBackend::new("claude-opus-4-6".into(), "anthropic".into()); + let router = BackendRouter::new(api_backend, cli); + + let mut node = Node::new("api_step"); + node.attrs.insert("prompt".to_string(), AttrValue::String("Plan the work".to_string())); + + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + let result = router + .run(&node, "Plan the work", &context, None, &emitter, dir.path(), &env) + .await + .expect("router should succeed"); + + match result { + CodergenResult::Text { text, .. } => { + assert!(text.starts_with("Response for api_step"), "should use API mock response: {text}"); + } + CodergenResult::Full(_) => panic!("expected Text result"), + } +} + +#[tokio::test] +async fn backend_router_delegates_to_cli_for_cli_only_model() { + let codex_output = "{\"type\":\"item.completed\",\"item\":{\"id\":\"item_0\",\"type\":\"agent_message\",\"text\":\"Codex did it\"}}\n{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":5}}"; + let env: Arc = Arc::new(CliTestEnv::new(codex_output)); + + let api_backend = Box::new(MockCodergenBackend); + let cli = CliBackend::new("gpt-5.3-codex-spark".into(), "openai".into()); + let router = BackendRouter::new(api_backend, cli); + + let mut node = Node::new("codex_step"); + node.attrs.insert("llm_model".to_string(), AttrValue::String("gpt-5.3-codex-spark".to_string())); + node.attrs.insert("llm_provider".to_string(), AttrValue::String("openai".to_string())); + + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + let result = router + .run(&node, "Build it", &context, None, &emitter, dir.path(), &env) + .await + .expect("router should succeed"); + + match result { + CodergenResult::Text { text, .. } => { + assert_eq!(text, "Codex did it", "should route to CLI backend for CLI-only model"); + } + CodergenResult::Full(_) => panic!("expected Text result"), + } +} + +// -- Full pipeline e2e with BackendRouter -- + +#[tokio::test] +async fn full_pipeline_with_cli_backend_node() { + // Pipeline: start -> api_work -> cli_work -> exit + // api_work uses MockCodergenBackend (API), cli_work has backend="cli" + let claude_output = r#"{"type":"result","result":"CLI completed the task.","usage":{"input_tokens":100,"output_tokens":50}}"#; + let env: Arc = Arc::new(CliTestEnv::new(claude_output)); + + let mut graph = Graph::new("CliPipelineTest"); + + let mut start = Node::new("start"); + start.attrs.insert("shape".to_string(), AttrValue::String("Mdiamond".to_string())); + graph.nodes.insert("start".to_string(), start); + + let mut exit = Node::new("exit"); + exit.attrs.insert("shape".to_string(), AttrValue::String("Msquare".to_string())); + graph.nodes.insert("exit".to_string(), exit); + + let mut api_work = Node::new("api_work"); + api_work.attrs.insert("shape".to_string(), AttrValue::String("box".to_string())); + api_work.attrs.insert("prompt".to_string(), AttrValue::String("Plan the work".to_string())); + graph.nodes.insert("api_work".to_string(), api_work); + + let mut cli_work = Node::new("cli_work"); + cli_work.attrs.insert("shape".to_string(), AttrValue::String("box".to_string())); + cli_work.attrs.insert("prompt".to_string(), AttrValue::String("Implement via CLI".to_string())); + cli_work.attrs.insert("backend".to_string(), AttrValue::String("cli".to_string())); + graph.nodes.insert("cli_work".to_string(), cli_work); + + graph.edges.push(Edge::new("start", "api_work")); + graph.edges.push(Edge::new("api_work", "cli_work")); + graph.edges.push(Edge::new("cli_work", "exit")); + + // Build engine with BackendRouter + let api = MockCodergenBackend; + let cli = CliBackend::new("claude-opus-4-6".into(), "anthropic".into()); + let router = BackendRouter::new(Box::new(api), cli); + let codergen_handler = CodergenHandler::new(Some(Box::new(router))); + + let mut registry = HandlerRegistry::new(Box::new(codergen_handler)); + registry.register("start", Box::new(StartHandler)); + registry.register("exit", Box::new(ExitHandler)); + registry.register("codergen", Box::new(CodergenHandler::new(Some(Box::new({ + // Second BackendRouter for the "codergen" handler + let api2 = MockCodergenBackend; + let cli2 = CliBackend::new("claude-opus-4-6".into(), "anthropic".into()); + BackendRouter::new(Box::new(api2), cli2) + }))))); + + let dir = tempfile::tempdir().unwrap(); + let engine = PipelineEngine::new(registry, Arc::new(EventEmitter::new()), env); + let config = RunConfig { + logs_root: dir.path().to_path_buf(), + cancel_token: None, + dry_run: false, + }; + + let outcome = engine.run(&graph, &config).await.expect("pipeline should succeed"); + assert_eq!(outcome.status, StageStatus::Success); + + // Verify api_work used mock (its response.md should contain "Response for") + let api_response = std::fs::read_to_string( + dir.path().join("nodes").join("api_work").join("response.md") + ).unwrap(); + assert!(api_response.starts_with("Response for api_work"), "API node should use mock: {api_response}"); + + // Verify cli_work used CLI backend (its response.md should contain CLI response) + let cli_response = std::fs::read_to_string( + dir.path().join("nodes").join("cli_work").join("response.md") + ).unwrap(); + assert_eq!(cli_response, "CLI completed the task.", "CLI node should use CLI backend: {cli_response}"); + + // Verify cli_work wrote provider_used.json with mode=cli + let provider_json: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(dir.path().join("nodes").join("cli_work").join("provider_used.json")).unwrap() + ).unwrap(); + assert_eq!(provider_json["mode"], "cli"); +} + +// -- Stylesheet applies backend property to nodes in a full pipeline -- + +#[tokio::test] +async fn stylesheet_backend_property_routes_to_cli() { + let claude_output = r#"{"type":"result","result":"Styled CLI response.","usage":{"input_tokens":10,"output_tokens":5}}"#; + let env: Arc = Arc::new(CliTestEnv::new(claude_output)); + + let mut graph = Graph::new("StylesheetTest"); + graph.attrs.insert( + "model_stylesheet".to_string(), + AttrValue::String(".cli-node { backend: cli; }".to_string()), + ); + + let mut start = Node::new("start"); + start.attrs.insert("shape".to_string(), AttrValue::String("Mdiamond".to_string())); + graph.nodes.insert("start".to_string(), start); + + let mut exit = Node::new("exit"); + exit.attrs.insert("shape".to_string(), AttrValue::String("Msquare".to_string())); + graph.nodes.insert("exit".to_string(), exit); + + let mut work = Node::new("work"); + work.attrs.insert("shape".to_string(), AttrValue::String("box".to_string())); + work.attrs.insert("prompt".to_string(), AttrValue::String("Do work".to_string())); + work.classes.push("cli-node".to_string()); + graph.nodes.insert("work".to_string(), work); + + graph.edges.push(Edge::new("start", "work")); + graph.edges.push(Edge::new("work", "exit")); + + // Apply stylesheet + let ss = parse_stylesheet(graph.model_stylesheet()).unwrap(); + apply_stylesheet(&ss, &mut graph); + + // Verify the stylesheet applied the backend property + assert_eq!( + graph.nodes["work"].backend(), + Some("cli"), + "stylesheet should set backend=cli on .cli-node" + ); + + // Run the pipeline + let api = MockCodergenBackend; + let cli = CliBackend::new("claude-opus-4-6".into(), "anthropic".into()); + let router = BackendRouter::new(Box::new(api), cli); + + let mut registry = HandlerRegistry::new(Box::new(CodergenHandler::new(Some(Box::new(router))))); + registry.register("start", Box::new(StartHandler)); + registry.register("exit", Box::new(ExitHandler)); + let api2 = MockCodergenBackend; + let cli2 = CliBackend::new("claude-opus-4-6".into(), "anthropic".into()); + let router2 = BackendRouter::new(Box::new(api2), cli2); + registry.register("codergen", Box::new(CodergenHandler::new(Some(Box::new(router2))))); + + let dir = tempfile::tempdir().unwrap(); + let engine = PipelineEngine::new(registry, Arc::new(EventEmitter::new()), env); + let config = RunConfig { + logs_root: dir.path().to_path_buf(), + cancel_token: None, + dry_run: false, + }; + + let outcome = engine.run(&graph, &config).await.expect("pipeline should succeed"); + assert_eq!(outcome.status, StageStatus::Success); + + let response = std::fs::read_to_string( + dir.path().join("nodes").join("work").join("response.md") + ).unwrap(); + assert_eq!(response, "Styled CLI response.", "stylesheet-driven node should use CLI backend"); +} + +// --------------------------------------------------------------------------- +// Real CLI backend e2e tests (require actual CLI tools installed) +// --------------------------------------------------------------------------- + +use attractor::cli::cli_backend::parse_cli_response; + +/// Run a real CLI tool via LocalExecutionEnvironment and verify the full flow. +async fn run_real_cli_test(provider: &str, model: &str) { + let env = local_env(); + let backend = CliBackend::new(model.to_string(), provider.to_string()); + + let mut node = Node::new("real_cli_test"); + node.attrs.insert("prompt".to_string(), AttrValue::String("What is 2+2? Reply with just the number.".to_string())); + + let context = Context::new(); + let emitter = Arc::new(EventEmitter::new()); + let dir = tempfile::tempdir().unwrap(); + + let result = backend + .run(&node, "What is 2+2? Reply with just the number.", &context, None, &emitter, dir.path(), &env) + .await + .expect(&format!("CLI backend ({provider}/{model}) should succeed")); + + match result { + CodergenResult::Text { text, usage, .. } => { + assert!( + text.contains('4'), + "{provider}/{model}: expected response to contain '4', got: {text}" + ); + let usage = usage.expect(&format!("{provider}/{model}: should have usage")); + assert!(usage.input_tokens > 0, "{provider}/{model}: input_tokens should be > 0, got {}", usage.input_tokens); + } + CodergenResult::Full(_) => panic!("expected Text result from {provider}/{model}"), + } + + // Verify log files were written + let provider_path = dir.path().join("provider_used.json"); + assert!(provider_path.exists(), "{provider}/{model}: provider_used.json should exist"); + let provider_json: serde_json::Value = serde_json::from_str( + &std::fs::read_to_string(&provider_path).unwrap() + ).unwrap(); + assert_eq!(provider_json["mode"], "cli"); + assert_eq!(provider_json["provider"], provider); +} + +#[tokio::test] +#[ignore] // requires `claude` CLI installed +async fn real_cli_claude() { + run_real_cli_test("anthropic", "haiku").await; +} + +#[tokio::test] +#[ignore] // requires `codex` CLI installed and OpenAI auth +async fn real_cli_codex() { + run_real_cli_test("openai", "").await; +} + +#[tokio::test] +#[ignore] // requires `gemini` CLI installed and Google auth +async fn real_cli_gemini() { + run_real_cli_test("gemini", "gemini-2.5-flash").await; +} + +/// Verify parse_cli_response works against real Claude CLI output captured from stream-json. +#[test] +fn parse_real_claude_stream_json() { + // Real output captured from: claude -p --output-format stream-json --model haiku "What is 2+2?" + let output = r#"{"type":"system","subtype":"init","cwd":"/tmp","session_id":"abc"} +{"type":"assistant","message":{"content":[{"type":"text","text":"4"}]}} +{"type":"result","subtype":"success","is_error":false,"duration_ms":2000,"num_turns":1,"result":"4","usage":{"input_tokens":9,"output_tokens":5}}"#; + let response = parse_cli_response("anthropic", output).unwrap(); + assert_eq!(response.text, "4"); + assert_eq!(response.input_tokens, 9); + assert_eq!(response.output_tokens, 5); +} + +/// Verify parse_cli_response works against real Codex CLI output. +#[test] +fn parse_real_codex_ndjson() { + // Real output captured from: echo "What is 2+2?" | codex exec --json + let output = r#"{"type":"thread.started","thread_id":"019ca1ec-1e86-79b2-b2b2-b1d963f1aea2"} +{"type":"turn.started"} +{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"**Confirming simple numeric reply**"}} +{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"4"}} +{"type":"turn.completed","usage":{"input_tokens":7999,"cached_input_tokens":7040,"output_tokens":33}}"#; + let response = parse_cli_response("openai", output).unwrap(); + assert_eq!(response.text, "4"); + assert_eq!(response.input_tokens, 7999); + assert_eq!(response.output_tokens, 33); +} + +/// Verify parse_cli_response works against real Gemini CLI output. +#[test] +fn parse_real_gemini_json() { + // Real output captured from: gemini "What is 2+2?" -m gemini-2.5-flash --sandbox -o json + let output = r#"{"session_id":"abc","response":"4","stats":{"models":{"gemini-2.5-flash":{"api":{"totalRequests":1,"totalErrors":0,"totalLatencyMs":618},"tokens":{"input":123,"prompt":8911,"candidates":1,"total":8912,"cached":8788,"thoughts":0,"tool":0}}},"tools":{"totalCalls":0},"files":{"totalLinesAdded":0,"totalLinesRemoved":0}}}"#; + let response = parse_cli_response("gemini", output).unwrap(); + assert_eq!(response.text, "4"); + assert_eq!(response.input_tokens, 123); + assert_eq!(response.output_tokens, 1); } \ No newline at end of file