mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-07 08:27:12 +00:00
Add CLI backend support for Claude, Codex, and Gemini CLIs
Route codergen nodes to external CLI tools (claude, codex, gemini) via exec_command() based on node backend="cli" attribute, stylesheet rules, or CLI-only model detection. Key components: - CliBackend: writes prompt to temp file, shells out to CLI tool, parses NDJSON/JSON response, detects file changes via git diff - BackendRouter: wraps AgentBackend + CliBackend, routes per-node - Parsers matched to real CLI output formats (Claude stream-json, Codex NDJSON, Gemini JSON) - Permission flags for non-interactive use (--dangerously-skip-permissions, --full-auto, --yolo) - Node::backend() accessor and "backend" stylesheet property Tested against real Claude, Codex, and Gemini CLIs locally. Daytona integration tests added (ignored, require DAYTONA_API_KEY). Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp
parent
8c90945ab6
commit
bb5f9899ed
7 changed files with 1544 additions and 3 deletions
652
crates/attractor/src/cli/cli_backend.rs
Normal file
652
crates/attractor/src/cli/cli_backend.rs
Normal file
|
|
@ -0,0 +1,652 @@
|
|||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
use agent::ExecutionEnvironment;
|
||||
use async_trait::async_trait;
|
||||
|
||||
use crate::context::Context;
|
||||
use crate::error::AttractorError;
|
||||
use crate::event::EventEmitter;
|
||||
use crate::graph::Node;
|
||||
use crate::handler::codergen::{CodergenBackend, CodergenResult};
|
||||
use crate::outcome::StageUsage;
|
||||
|
||||
/// Models that are only available through CLI tools (not via API).
|
||||
const CLI_ONLY_MODELS: &[&str] = &["gpt-5.3-codex-spark"];
|
||||
|
||||
/// Returns true if the given model is only available through a CLI tool.
|
||||
#[must_use]
|
||||
pub fn is_cli_only_model(model: &str) -> bool {
|
||||
CLI_ONLY_MODELS.contains(&model)
|
||||
}
|
||||
|
||||
/// Build the CLI command string for a given provider.
|
||||
///
|
||||
/// The `prompt_file` is the path to a file containing the prompt text, which
|
||||
/// will be shell-redirected into the command's stdin.
|
||||
#[must_use]
|
||||
pub fn cli_command_for_provider(provider: &str, model: &str, prompt_file: &str) -> String {
|
||||
let model_flag = if model.is_empty() {
|
||||
String::new()
|
||||
} else {
|
||||
match provider {
|
||||
"openai" => format!(" -m {model}"),
|
||||
"gemini" => format!(" -m {model}"),
|
||||
_ => format!(" --model {model}"),
|
||||
}
|
||||
};
|
||||
match provider {
|
||||
// --full-auto: sandboxed auto-execution, escalates on request
|
||||
"openai" => format!("codex exec --json --full-auto{model_flag} < {prompt_file}"),
|
||||
// --yolo: auto-approve all tool calls
|
||||
"gemini" => format!("gemini -o json --yolo{model_flag} < {prompt_file}"),
|
||||
// --dangerously-skip-permissions: bypass all permission checks (required for non-interactive use).
|
||||
// CLAUDECODE= unset to allow running inside a Claude Code session.
|
||||
_ => format!("CLAUDECODE= claude -p --output-format stream-json --dangerously-skip-permissions{model_flag} < {prompt_file}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Parsed response from a CLI tool invocation.
|
||||
#[derive(Debug)]
|
||||
pub struct CliResponse {
|
||||
pub text: String,
|
||||
pub input_tokens: i64,
|
||||
pub output_tokens: i64,
|
||||
}
|
||||
|
||||
/// Parse NDJSON output from Claude CLI (`--output-format stream-json`).
|
||||
///
|
||||
/// Looks for the last `{"type":"result",...}` line, extracts `result` text and `usage`.
|
||||
fn parse_claude_ndjson(output: &str) -> Option<CliResponse> {
|
||||
let mut last_result: Option<serde_json::Value> = None;
|
||||
|
||||
for line in output.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if let Ok(value) = serde_json::from_str::<serde_json::Value>(line) {
|
||||
if value.get("type").and_then(|t| t.as_str()) == Some("result") {
|
||||
last_result = Some(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let result = last_result?;
|
||||
let text = result.get("result").and_then(|v| v.as_str()).unwrap_or("").to_string();
|
||||
let input_tokens = result
|
||||
.pointer("/usage/input_tokens")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(0);
|
||||
let output_tokens = result
|
||||
.pointer("/usage/output_tokens")
|
||||
.and_then(|v| v.as_i64())
|
||||
.unwrap_or(0);
|
||||
|
||||
Some(CliResponse {
|
||||
text,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse NDJSON output from Codex CLI (`codex exec --json`).
|
||||
///
|
||||
/// Codex emits NDJSON lines. Text comes from `item.completed` events where
|
||||
/// `item.type == "agent_message"`. Usage comes from the `turn.completed` event.
|
||||
fn parse_codex_ndjson(output: &str) -> Option<CliResponse> {
|
||||
let mut last_message_text = String::new();
|
||||
let mut input_tokens: i64 = 0;
|
||||
let mut output_tokens: i64 = 0;
|
||||
let mut found_anything = false;
|
||||
|
||||
for line in output.lines() {
|
||||
let line = line.trim();
|
||||
if line.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let value: serde_json::Value = match serde_json::from_str(line) {
|
||||
Ok(v) => v,
|
||||
Err(_) => continue,
|
||||
};
|
||||
|
||||
let event_type = value.get("type").and_then(|t| t.as_str()).unwrap_or("");
|
||||
|
||||
match event_type {
|
||||
"item.completed" => {
|
||||
let item_type = value.pointer("/item/type").and_then(|t| t.as_str()).unwrap_or("");
|
||||
if item_type == "agent_message" {
|
||||
if let Some(text) = value.pointer("/item/text").and_then(|t| t.as_str()) {
|
||||
last_message_text = text.to_string();
|
||||
found_anything = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
"turn.completed" => {
|
||||
input_tokens = value.pointer("/usage/input_tokens").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
output_tokens = value.pointer("/usage/output_tokens").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
found_anything = true;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if !found_anything {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(CliResponse {
|
||||
text: last_message_text,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse JSON output from Gemini CLI (`-o json`).
|
||||
///
|
||||
/// Gemini outputs a single JSON object with `response` for text and
|
||||
/// `stats.models.<model>.tokens` for usage.
|
||||
fn parse_gemini_json(output: &str) -> Option<CliResponse> {
|
||||
let value: serde_json::Value = serde_json::from_str(output.trim()).ok()?;
|
||||
let text = value.get("response").and_then(|v| v.as_str()).unwrap_or("").to_string();
|
||||
|
||||
// Extract tokens from the first model in stats.models
|
||||
let (input_tokens, output_tokens) = value
|
||||
.pointer("/stats/models")
|
||||
.and_then(|m| m.as_object())
|
||||
.and_then(|models| models.values().next())
|
||||
.map(|model_stats| {
|
||||
let input = model_stats.pointer("/tokens/input").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
let output = model_stats.pointer("/tokens/candidates").and_then(|v| v.as_i64()).unwrap_or(0);
|
||||
(input, output)
|
||||
})
|
||||
.unwrap_or((0, 0));
|
||||
|
||||
Some(CliResponse {
|
||||
text,
|
||||
input_tokens,
|
||||
output_tokens,
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse CLI output, choosing the right parser based on provider.
|
||||
pub fn parse_cli_response(provider: &str, output: &str) -> Option<CliResponse> {
|
||||
match provider {
|
||||
"openai" => parse_codex_ndjson(output),
|
||||
"gemini" => parse_gemini_json(output),
|
||||
_ => parse_claude_ndjson(output),
|
||||
}
|
||||
}
|
||||
|
||||
/// CLI backend that invokes external CLI tools (claude, codex, gemini) via `exec_command()`.
|
||||
pub struct CliBackend {
|
||||
model: String,
|
||||
provider: String,
|
||||
}
|
||||
|
||||
impl CliBackend {
|
||||
#[must_use]
|
||||
pub fn new(model: String, provider: String) -> Self {
|
||||
Self { model, provider }
|
||||
}
|
||||
|
||||
/// Detect changed files by comparing git state before and after the CLI run.
|
||||
async fn detect_changed_files(
|
||||
&self,
|
||||
execution_env: &Arc<dyn ExecutionEnvironment>,
|
||||
) -> Vec<String> {
|
||||
// Get unstaged changes
|
||||
let diff_result = execution_env
|
||||
.exec_command("git diff --name-only", 30_000, None, None, None)
|
||||
.await;
|
||||
|
||||
// Get untracked files
|
||||
let untracked_result = execution_env
|
||||
.exec_command(
|
||||
"git ls-files --others --exclude-standard",
|
||||
30_000,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
let mut files: Vec<String> = Vec::new();
|
||||
|
||||
if let Ok(result) = diff_result {
|
||||
if result.exit_code == 0 {
|
||||
files.extend(
|
||||
result
|
||||
.stdout
|
||||
.lines()
|
||||
.filter(|l| !l.trim().is_empty())
|
||||
.map(String::from),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(result) = untracked_result {
|
||||
if result.exit_code == 0 {
|
||||
files.extend(
|
||||
result
|
||||
.stdout
|
||||
.lines()
|
||||
.filter(|l| !l.trim().is_empty())
|
||||
.map(String::from),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
files.sort();
|
||||
files.dedup();
|
||||
files
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl CodergenBackend for CliBackend {
|
||||
async fn run(
|
||||
&self,
|
||||
node: &Node,
|
||||
prompt: &str,
|
||||
_context: &Context,
|
||||
_thread_id: Option<&str>,
|
||||
_emitter: &Arc<EventEmitter>,
|
||||
stage_dir: &Path,
|
||||
execution_env: &Arc<dyn ExecutionEnvironment>,
|
||||
) -> Result<CodergenResult, AttractorError> {
|
||||
// 1. Snapshot git state before the CLI run
|
||||
let files_before = self.detect_changed_files(execution_env).await;
|
||||
|
||||
// 2. Write prompt to temp file
|
||||
let prompt_path = "/tmp/attractor_cli_prompt.txt";
|
||||
execution_env
|
||||
.write_file(prompt_path, prompt)
|
||||
.await
|
||||
.map_err(|e| AttractorError::Handler(format!("Failed to write prompt file: {e}")))?;
|
||||
|
||||
// 3. Build and execute CLI command
|
||||
let model = node.llm_model().unwrap_or(&self.model);
|
||||
let provider = node.llm_provider().unwrap_or(&self.provider);
|
||||
let command = cli_command_for_provider(provider, model, prompt_path);
|
||||
|
||||
let _ = tokio::fs::create_dir_all(stage_dir).await;
|
||||
let provider_used = serde_json::json!({
|
||||
"mode": "cli",
|
||||
"provider": provider,
|
||||
"model": model,
|
||||
"command": &command,
|
||||
});
|
||||
if let Ok(json) = serde_json::to_string_pretty(&provider_used) {
|
||||
let _ = tokio::fs::write(stage_dir.join("provider_used.json"), json).await;
|
||||
}
|
||||
|
||||
let result = execution_env
|
||||
.exec_command(&command, 600_000, None, None, None)
|
||||
.await
|
||||
.map_err(|e| AttractorError::Handler(format!("CLI command failed: {e}")))?;
|
||||
|
||||
if let Ok(json) = serde_json::to_string_pretty(&serde_json::json!({
|
||||
"exit_code": result.exit_code,
|
||||
"stdout_len": result.stdout.len(),
|
||||
"stderr_len": result.stderr.len(),
|
||||
"duration_ms": result.duration_ms,
|
||||
})) {
|
||||
let _ = tokio::fs::write(stage_dir.join("cli_result_meta.json"), json).await;
|
||||
}
|
||||
|
||||
if result.exit_code != 0 {
|
||||
return Err(AttractorError::Handler(format!(
|
||||
"CLI command exited with code {}: {}",
|
||||
result.exit_code,
|
||||
result.stderr.chars().take(500).collect::<String>()
|
||||
)));
|
||||
}
|
||||
|
||||
// 4. Parse the CLI output
|
||||
let parsed = parse_cli_response(provider, &result.stdout).ok_or_else(|| {
|
||||
AttractorError::Handler("Failed to parse CLI output".to_string())
|
||||
})?;
|
||||
|
||||
// 5. Detect changed files
|
||||
let files_after = self.detect_changed_files(execution_env).await;
|
||||
let files_touched: Vec<String> = files_after
|
||||
.into_iter()
|
||||
.filter(|f| !files_before.contains(f))
|
||||
.collect();
|
||||
|
||||
let mut stage_usage = StageUsage {
|
||||
model: model.to_string(),
|
||||
input_tokens: parsed.input_tokens,
|
||||
output_tokens: parsed.output_tokens,
|
||||
cache_read_tokens: None,
|
||||
cache_write_tokens: None,
|
||||
reasoning_tokens: None,
|
||||
cost: None,
|
||||
};
|
||||
stage_usage.cost = super::compute_stage_cost(&stage_usage);
|
||||
|
||||
Ok(CodergenResult::Text {
|
||||
text: parsed.text,
|
||||
usage: Some(stage_usage),
|
||||
files_touched,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Routes codergen invocations to either the API backend or CLI backend
|
||||
/// based on node attributes and model type.
|
||||
pub struct BackendRouter {
|
||||
api_backend: Box<dyn CodergenBackend>,
|
||||
cli_backend: CliBackend,
|
||||
}
|
||||
|
||||
impl BackendRouter {
|
||||
#[must_use]
|
||||
pub fn new(api_backend: Box<dyn CodergenBackend>, cli_backend: CliBackend) -> Self {
|
||||
Self {
|
||||
api_backend,
|
||||
cli_backend,
|
||||
}
|
||||
}
|
||||
|
||||
fn should_use_cli(&self, node: &Node) -> bool {
|
||||
// Explicit backend="cli" attribute on the node
|
||||
if node.backend() == Some("cli") {
|
||||
return true;
|
||||
}
|
||||
|
||||
// CLI-only model on the node
|
||||
if let Some(model) = node.llm_model() {
|
||||
if is_cli_only_model(model) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl CodergenBackend for BackendRouter {
|
||||
async fn run(
|
||||
&self,
|
||||
node: &Node,
|
||||
prompt: &str,
|
||||
context: &Context,
|
||||
thread_id: Option<&str>,
|
||||
emitter: &Arc<EventEmitter>,
|
||||
stage_dir: &Path,
|
||||
execution_env: &Arc<dyn ExecutionEnvironment>,
|
||||
) -> Result<CodergenResult, AttractorError> {
|
||||
if self.should_use_cli(node) {
|
||||
self.cli_backend
|
||||
.run(node, prompt, context, thread_id, emitter, stage_dir, execution_env)
|
||||
.await
|
||||
} else {
|
||||
self.api_backend
|
||||
.run(node, prompt, context, thread_id, emitter, stage_dir, execution_env)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
async fn one_shot(
|
||||
&self,
|
||||
node: &Node,
|
||||
prompt: &str,
|
||||
stage_dir: &Path,
|
||||
) -> Result<CodergenResult, AttractorError> {
|
||||
// CLI backend doesn't support one_shot, always route to API
|
||||
self.api_backend.one_shot(node, prompt, stage_dir).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::graph::AttrValue;
|
||||
|
||||
// -- Cycle 1: cli_command_for_provider --
|
||||
|
||||
#[test]
|
||||
fn cli_command_for_codex() {
|
||||
let cmd = cli_command_for_provider("openai", "gpt-5.3-codex-spark", "/tmp/prompt.txt");
|
||||
assert!(cmd.starts_with("codex exec --json --full-auto"));
|
||||
assert!(cmd.contains("-m gpt-5.3-codex-spark"));
|
||||
assert!(cmd.ends_with("< /tmp/prompt.txt"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_command_for_claude() {
|
||||
let cmd = cli_command_for_provider("anthropic", "claude-opus-4-6", "/tmp/prompt.txt");
|
||||
assert!(cmd.contains("claude -p"));
|
||||
assert!(cmd.contains("--dangerously-skip-permissions"));
|
||||
assert!(cmd.contains("--output-format stream-json"));
|
||||
assert!(cmd.contains("--model claude-opus-4-6"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_command_for_gemini() {
|
||||
let cmd = cli_command_for_provider("gemini", "gemini-3.1-pro", "/tmp/prompt.txt");
|
||||
assert!(cmd.starts_with("gemini -o json --yolo"));
|
||||
assert!(cmd.contains("-m gemini-3.1-pro"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_command_defaults_to_claude() {
|
||||
let cmd = cli_command_for_provider("unknown_provider", "some-model", "/tmp/prompt.txt");
|
||||
assert!(cmd.contains("claude "));
|
||||
assert!(cmd.contains("--dangerously-skip-permissions"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cli_command_omits_model_when_empty() {
|
||||
let cmd = cli_command_for_provider("openai", "", "/tmp/prompt.txt");
|
||||
assert!(cmd.starts_with("codex exec --json --full-auto"));
|
||||
assert!(!cmd.contains("-m "));
|
||||
let cmd = cli_command_for_provider("anthropic", "", "/tmp/prompt.txt");
|
||||
assert!(cmd.contains("--dangerously-skip-permissions"));
|
||||
assert!(!cmd.contains("--model "));
|
||||
let cmd = cli_command_for_provider("gemini", "", "/tmp/prompt.txt");
|
||||
assert!(cmd.contains("--yolo"));
|
||||
assert!(!cmd.contains("-m "));
|
||||
}
|
||||
|
||||
// -- Cycle 2: is_cli_only_model --
|
||||
|
||||
#[test]
|
||||
fn codex_spark_is_cli_only() {
|
||||
assert!(is_cli_only_model("gpt-5.3-codex-spark"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn claude_opus_is_not_cli_only() {
|
||||
assert!(!is_cli_only_model("claude-opus-4-6"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gemini_is_not_cli_only() {
|
||||
assert!(!is_cli_only_model("gemini-3.1-pro-preview"));
|
||||
}
|
||||
|
||||
// -- Cycle 3: parse_cli_response — Claude/Gemini NDJSON --
|
||||
|
||||
#[test]
|
||||
fn parse_claude_ndjson_extracts_text_and_usage() {
|
||||
let output = r#"{"type":"system","message":"Claude CLI v1.0"}
|
||||
{"type":"assistant","message":{"content":"thinking..."}}
|
||||
{"type":"result","result":"Here is the implementation.","usage":{"input_tokens":100,"output_tokens":50}}"#;
|
||||
let response = parse_cli_response("anthropic", output).unwrap();
|
||||
assert_eq!(response.text, "Here is the implementation.");
|
||||
assert_eq!(response.input_tokens, 100);
|
||||
assert_eq!(response.output_tokens, 50);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_claude_ndjson_uses_last_result() {
|
||||
let output = r#"{"type":"result","result":"first","usage":{"input_tokens":10,"output_tokens":5}}
|
||||
{"type":"result","result":"second","usage":{"input_tokens":20,"output_tokens":10}}"#;
|
||||
let response = parse_cli_response("anthropic", output).unwrap();
|
||||
assert_eq!(response.text, "second");
|
||||
assert_eq!(response.input_tokens, 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_claude_ndjson_returns_none_for_no_result() {
|
||||
let output = r#"{"type":"system","message":"hello"}
|
||||
{"type":"assistant","message":{"content":"no result line"}}"#;
|
||||
assert!(parse_cli_response("anthropic", output).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_gemini_json_extracts_text_and_usage() {
|
||||
let output = r#"{"session_id":"abc","response":"Gemini says hello","stats":{"models":{"gemini-2.5-flash":{"tokens":{"input":200,"candidates":80,"total":280}}}}}"#;
|
||||
let response = parse_cli_response("gemini", output).unwrap();
|
||||
assert_eq!(response.text, "Gemini says hello");
|
||||
assert_eq!(response.input_tokens, 200);
|
||||
assert_eq!(response.output_tokens, 80);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_gemini_json_handles_missing_stats() {
|
||||
let output = r#"{"response":"hello"}"#;
|
||||
let response = parse_cli_response("gemini", output).unwrap();
|
||||
assert_eq!(response.text, "hello");
|
||||
assert_eq!(response.input_tokens, 0);
|
||||
assert_eq!(response.output_tokens, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_gemini_json_returns_none_for_invalid_json() {
|
||||
assert!(parse_cli_response("gemini", "not json").is_none());
|
||||
}
|
||||
|
||||
// -- Cycle 4: parse_cli_response — Codex NDJSON --
|
||||
|
||||
#[test]
|
||||
fn parse_codex_ndjson_extracts_text_and_usage() {
|
||||
let output = r#"{"type":"thread.started","thread_id":"abc"}
|
||||
{"type":"turn.started"}
|
||||
{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"thinking..."}}
|
||||
{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"Fixed the bug."}}
|
||||
{"type":"turn.completed","usage":{"input_tokens":300,"output_tokens":150}}"#;
|
||||
let response = parse_cli_response("openai", output).unwrap();
|
||||
assert_eq!(response.text, "Fixed the bug.");
|
||||
assert_eq!(response.input_tokens, 300);
|
||||
assert_eq!(response.output_tokens, 150);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_codex_ndjson_handles_no_message() {
|
||||
let output = r#"{"type":"turn.completed","usage":{"input_tokens":10,"output_tokens":5}}"#;
|
||||
let response = parse_cli_response("openai", output).unwrap();
|
||||
assert_eq!(response.text, "");
|
||||
assert_eq!(response.input_tokens, 10);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_codex_ndjson_returns_none_for_no_events() {
|
||||
assert!(parse_cli_response("openai", "not json at all").is_none());
|
||||
}
|
||||
|
||||
// -- Cycle 5: Node::backend() accessor (tested here since the accessor is simple) --
|
||||
|
||||
#[test]
|
||||
fn node_backend_returns_none_by_default() {
|
||||
let node = Node::new("test");
|
||||
assert_eq!(node.backend(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn node_backend_returns_cli_when_set() {
|
||||
let mut node = Node::new("test");
|
||||
node.attrs
|
||||
.insert("backend".to_string(), AttrValue::String("cli".to_string()));
|
||||
assert_eq!(node.backend(), Some("cli"));
|
||||
}
|
||||
|
||||
// -- Cycle 6: backend in stylesheet (tested in stylesheet.rs) --
|
||||
|
||||
// -- Cycle 7: BackendRouter routing logic --
|
||||
|
||||
#[test]
|
||||
fn router_uses_cli_for_backend_attr() {
|
||||
let mut node = Node::new("test");
|
||||
node.attrs
|
||||
.insert("backend".to_string(), AttrValue::String("cli".to_string()));
|
||||
|
||||
let cli_backend = CliBackend::new("model".into(), "anthropic".into());
|
||||
let router = BackendRouter::new(
|
||||
Box::new(StubBackend),
|
||||
cli_backend,
|
||||
);
|
||||
assert!(router.should_use_cli(&node));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn router_uses_cli_for_cli_only_model() {
|
||||
let mut node = Node::new("test");
|
||||
node.attrs.insert(
|
||||
"llm_model".to_string(),
|
||||
AttrValue::String("gpt-5.3-codex-spark".to_string()),
|
||||
);
|
||||
|
||||
let cli_backend = CliBackend::new("model".into(), "openai".into());
|
||||
let router = BackendRouter::new(
|
||||
Box::new(StubBackend),
|
||||
cli_backend,
|
||||
);
|
||||
assert!(router.should_use_cli(&node));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn router_uses_api_by_default() {
|
||||
let node = Node::new("test");
|
||||
|
||||
let cli_backend = CliBackend::new("model".into(), "anthropic".into());
|
||||
let router = BackendRouter::new(
|
||||
Box::new(StubBackend),
|
||||
cli_backend,
|
||||
);
|
||||
assert!(!router.should_use_cli(&node));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn router_uses_api_for_non_cli_model() {
|
||||
let mut node = Node::new("test");
|
||||
node.attrs.insert(
|
||||
"llm_model".to_string(),
|
||||
AttrValue::String("claude-opus-4-6".to_string()),
|
||||
);
|
||||
|
||||
let cli_backend = CliBackend::new("model".into(), "anthropic".into());
|
||||
let router = BackendRouter::new(
|
||||
Box::new(StubBackend),
|
||||
cli_backend,
|
||||
);
|
||||
assert!(!router.should_use_cli(&node));
|
||||
}
|
||||
|
||||
/// Minimal stub backend for testing routing logic.
|
||||
struct StubBackend;
|
||||
|
||||
#[async_trait]
|
||||
impl CodergenBackend for StubBackend {
|
||||
async fn run(
|
||||
&self,
|
||||
_node: &Node,
|
||||
_prompt: &str,
|
||||
_context: &Context,
|
||||
_thread_id: Option<&str>,
|
||||
_emitter: &Arc<EventEmitter>,
|
||||
_stage_dir: &Path,
|
||||
_execution_env: &Arc<dyn ExecutionEnvironment>,
|
||||
) -> Result<CodergenResult, AttractorError> {
|
||||
Ok(CodergenResult::Text {
|
||||
text: "stub".to_string(),
|
||||
usage: None,
|
||||
files_touched: Vec::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,4 +1,5 @@
|
|||
pub mod backend;
|
||||
pub mod cli_backend;
|
||||
pub mod run;
|
||||
#[cfg(feature = "server")]
|
||||
pub mod serve;
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ use crate::pipeline::PipelineBuilder;
|
|||
use crate::validation::Severity;
|
||||
|
||||
use super::backend::AgentBackend;
|
||||
use super::cli_backend::{BackendRouter, CliBackend};
|
||||
use super::task_config;
|
||||
use super::{compute_stage_cost, format_cost, format_duration_human, format_event_detail, format_event_summary, format_tokens_human, print_diagnostics, read_dot_file, ExecutionEnvKind, RunArgs};
|
||||
|
||||
|
|
@ -403,12 +404,17 @@ pub async fn run_command(args: RunArgs, styles: &'static Styles) -> anyhow::Resu
|
|||
if dry_run_mode {
|
||||
None
|
||||
} else {
|
||||
Some(Box::new(AgentBackend::new(
|
||||
let api = AgentBackend::new(
|
||||
model.clone(),
|
||||
provider.clone(),
|
||||
args.verbose,
|
||||
styles,
|
||||
)))
|
||||
);
|
||||
let cli = CliBackend::new(
|
||||
model.clone(),
|
||||
provider.clone().unwrap_or_else(|| "anthropic".to_string()),
|
||||
);
|
||||
Some(Box::new(BackendRouter::new(Box::new(api), cli)))
|
||||
}
|
||||
});
|
||||
let engine = PipelineEngine::with_interviewer(registry, Arc::clone(&emitter), interviewer, Arc::clone(&execution_env));
|
||||
|
|
|
|||
|
|
@ -225,6 +225,11 @@ impl Node {
|
|||
self.str_attr("retry_policy")
|
||||
}
|
||||
|
||||
#[must_use]
|
||||
pub fn backend(&self) -> Option<&str> {
|
||||
self.str_attr("backend")
|
||||
}
|
||||
|
||||
/// Returns the codergen mode for this node. Defaults to `AgentLoop` when absent.
|
||||
pub fn codergen_mode(&self) -> Result<CodergenMode, AttractorError> {
|
||||
match self.str_attr("codergen_mode") {
|
||||
|
|
|
|||
|
|
@ -179,7 +179,7 @@ fn parse_declarations(remaining: &mut &str) -> Result<Vec<Declaration>, Attracto
|
|||
}
|
||||
|
||||
/// Recognized stylesheet properties.
|
||||
const STYLESHEET_PROPERTIES: &[&str] = &["llm_model", "llm_provider", "reasoning_effort"];
|
||||
const STYLESHEET_PROPERTIES: &[&str] = &["llm_model", "llm_provider", "reasoning_effort", "backend"];
|
||||
|
||||
/// Apply a stylesheet to a graph. Rules are applied by specificity order;
|
||||
/// higher specificity wins. Explicit node attributes are never overridden.
|
||||
|
|
@ -530,4 +530,33 @@ mod tests {
|
|||
Some(&AttrValue::String("sonnet".into()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn apply_backend_property_via_stylesheet() {
|
||||
let ss = parse_stylesheet("* { backend: cli; }").unwrap();
|
||||
let mut graph = Graph::new("test");
|
||||
graph.nodes.insert("a".into(), Node::new("a"));
|
||||
apply_stylesheet(&ss, &mut graph);
|
||||
|
||||
assert_eq!(
|
||||
graph.nodes["a"].attrs.get("backend"),
|
||||
Some(&AttrValue::String("cli".into()))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backend_property_not_overridden_by_stylesheet() {
|
||||
let ss = parse_stylesheet("* { backend: cli; }").unwrap();
|
||||
let mut graph = Graph::new("test");
|
||||
let mut node = Node::new("a");
|
||||
node.attrs
|
||||
.insert("backend".into(), AttrValue::String("api".into()));
|
||||
graph.nodes.insert("a".into(), node);
|
||||
apply_stylesheet(&ss, &mut graph);
|
||||
|
||||
assert_eq!(
|
||||
graph.nodes["a"].attrs.get("backend"),
|
||||
Some(&AttrValue::String("api".into()))
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -307,3 +307,116 @@ async fn daytona_pipeline_artifact_offload_and_sync() {
|
|||
|
||||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI Backend on Daytona — real CLI tools via exec_command
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
use attractor::cli::cli_backend::CliBackend;
|
||||
use attractor::handler::codergen::{CodergenBackend, CodergenResult};
|
||||
|
||||
/// Helper: run a real CLI backend test on Daytona.
|
||||
///
|
||||
/// Installs the CLI tool in the sandbox, then runs the CliBackend against it.
|
||||
async fn run_daytona_cli_test(
|
||||
provider: &str,
|
||||
model: &str,
|
||||
install_command: &str,
|
||||
) {
|
||||
let env = create_env().await;
|
||||
env.initialize().await.unwrap();
|
||||
let env: Arc<dyn ExecutionEnvironment> = Arc::new(env);
|
||||
|
||||
// Install the CLI tool inside the Daytona sandbox
|
||||
let install_result = env
|
||||
.exec_command(install_command, 120_000, None, None, None)
|
||||
.await
|
||||
.expect("install command should not error");
|
||||
assert_eq!(
|
||||
install_result.exit_code, 0,
|
||||
"install command failed (exit {}): {}",
|
||||
install_result.exit_code, install_result.stdout
|
||||
);
|
||||
|
||||
let backend = CliBackend::new(model.to_string(), provider.to_string());
|
||||
let node = Node::new("daytona_cli_test");
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = backend
|
||||
.run(
|
||||
&node,
|
||||
"What is 2+2? Reply with just the number.",
|
||||
&context,
|
||||
None,
|
||||
&emitter,
|
||||
dir.path(),
|
||||
&env,
|
||||
)
|
||||
.await;
|
||||
|
||||
match result {
|
||||
Ok(CodergenResult::Text { text, usage, .. }) => {
|
||||
assert!(
|
||||
text.contains('4'),
|
||||
"{provider}/{model} on Daytona: expected '4', got: {text}"
|
||||
);
|
||||
if let Some(u) = usage {
|
||||
assert!(
|
||||
u.input_tokens > 0,
|
||||
"{provider}/{model}: input_tokens should be > 0"
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(CodergenResult::Full(_)) => panic!("expected Text result"),
|
||||
Err(e) => panic!("{provider}/{model} on Daytona failed: {e}"),
|
||||
}
|
||||
|
||||
// Verify log files
|
||||
let provider_path = dir.path().join("provider_used.json");
|
||||
assert!(
|
||||
provider_path.exists(),
|
||||
"{provider}/{model}: provider_used.json should exist"
|
||||
);
|
||||
let provider_json: serde_json::Value = serde_json::from_str(
|
||||
&std::fs::read_to_string(&provider_path).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(provider_json["mode"], "cli");
|
||||
|
||||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires DAYTONA_API_KEY + Claude CLI auth
|
||||
async fn daytona_cli_claude() {
|
||||
run_daytona_cli_test(
|
||||
"anthropic",
|
||||
"haiku",
|
||||
"curl -fsSL https://claude.ai/install.sh | sh",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires DAYTONA_API_KEY + OpenAI/Codex auth
|
||||
async fn daytona_cli_codex() {
|
||||
run_daytona_cli_test(
|
||||
"openai",
|
||||
"o4-mini",
|
||||
"npm install -g @openai/codex",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires DAYTONA_API_KEY + Gemini auth
|
||||
async fn daytona_cli_gemini() {
|
||||
run_daytona_cli_test(
|
||||
"gemini",
|
||||
"gemini-2.5-flash",
|
||||
"npm install -g @google/gemini-cli",
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7185,4 +7185,739 @@ async fn node_dir_uses_visit_count_on_revisit() {
|
|||
).unwrap();
|
||||
assert_eq!(first_json["status"], "fail");
|
||||
assert_eq!(second_json["status"], "success");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// CLI Backend end-to-end tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
use attractor::cli::cli_backend::{BackendRouter, CliBackend};
|
||||
|
||||
/// A mock execution environment for CLI backend e2e tests.
|
||||
/// Records all exec_command and write_file calls, and returns configurable
|
||||
/// responses based on command content.
|
||||
struct CliTestEnv {
|
||||
/// All commands passed to exec_command, in order.
|
||||
commands: std::sync::Mutex<Vec<String>>,
|
||||
/// All (path, content) pairs from write_file.
|
||||
written_files: std::sync::Mutex<Vec<(String, String)>>,
|
||||
/// The stdout to return when the CLI command (not git) is executed.
|
||||
cli_stdout: String,
|
||||
/// Files returned by "git diff --name-only" AFTER the CLI runs.
|
||||
/// First call returns empty (before), second returns these (after).
|
||||
git_diff_call_count: std::sync::atomic::AtomicU32,
|
||||
git_diff_after: String,
|
||||
}
|
||||
|
||||
impl CliTestEnv {
|
||||
fn new(cli_stdout: &str) -> Self {
|
||||
Self {
|
||||
commands: std::sync::Mutex::new(Vec::new()),
|
||||
written_files: std::sync::Mutex::new(Vec::new()),
|
||||
cli_stdout: cli_stdout.to_string(),
|
||||
git_diff_call_count: std::sync::atomic::AtomicU32::new(0),
|
||||
git_diff_after: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_git_diff_after(mut self, files: &str) -> Self {
|
||||
self.git_diff_after = files.to_string();
|
||||
self
|
||||
}
|
||||
|
||||
fn recorded_commands(&self) -> Vec<String> {
|
||||
self.commands.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
fn recorded_written_files(&self) -> Vec<(String, String)> {
|
||||
self.written_files.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl agent::ExecutionEnvironment for CliTestEnv {
|
||||
async fn read_file(&self, _path: &str, _offset: Option<usize>, _limit: Option<usize>) -> Result<String, String> {
|
||||
Ok(String::new())
|
||||
}
|
||||
|
||||
async fn write_file(&self, path: &str, content: &str) -> Result<(), String> {
|
||||
self.written_files.lock().unwrap().push((path.to_string(), content.to_string()));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_file(&self, _path: &str) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn file_exists(&self, _path: &str) -> Result<bool, String> {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
async fn list_directory(&self, _path: &str, _depth: Option<usize>) -> Result<Vec<agent::DirEntry>, String> {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
async fn exec_command(
|
||||
&self,
|
||||
command: &str,
|
||||
_timeout_ms: u64,
|
||||
_working_dir: Option<&str>,
|
||||
_env_vars: Option<&std::collections::HashMap<String, String>>,
|
||||
_cancel_token: Option<tokio_util::sync::CancellationToken>,
|
||||
) -> Result<agent::ExecResult, String> {
|
||||
self.commands.lock().unwrap().push(command.to_string());
|
||||
|
||||
// git diff calls: first pair returns empty (before), second pair returns configured files
|
||||
if command.starts_with("git diff") || command.starts_with("git ls-files") {
|
||||
let call_num = self.git_diff_call_count.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
// Calls 0,1 = before snapshot (empty), calls 2,3 = after snapshot
|
||||
let stdout = if call_num >= 2 && command.starts_with("git diff") {
|
||||
self.git_diff_after.clone()
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
return Ok(agent::ExecResult {
|
||||
stdout,
|
||||
stderr: String::new(),
|
||||
exit_code: 0,
|
||||
timed_out: false,
|
||||
duration_ms: 5,
|
||||
});
|
||||
}
|
||||
|
||||
// CLI command: return configured stdout
|
||||
Ok(agent::ExecResult {
|
||||
stdout: self.cli_stdout.clone(),
|
||||
stderr: String::new(),
|
||||
exit_code: 0,
|
||||
timed_out: false,
|
||||
duration_ms: 100,
|
||||
})
|
||||
}
|
||||
|
||||
async fn grep(&self, _pattern: &str, _path: &str, _options: &agent::GrepOptions) -> Result<Vec<String>, String> {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
async fn glob(&self, _pattern: &str, _path: Option<&str>) -> Result<Vec<String>, String> {
|
||||
Ok(vec![])
|
||||
}
|
||||
|
||||
async fn initialize(&self) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn cleanup(&self) -> Result<(), String> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn working_directory(&self) -> &str {
|
||||
"/tmp/test"
|
||||
}
|
||||
|
||||
fn platform(&self) -> &str {
|
||||
"darwin"
|
||||
}
|
||||
|
||||
fn os_version(&self) -> String {
|
||||
"Darwin 24.0.0".into()
|
||||
}
|
||||
}
|
||||
|
||||
// -- Cycle 8: CliBackend::run() e2e via mock ExecutionEnvironment --
|
||||
|
||||
#[tokio::test]
|
||||
async fn cli_backend_run_writes_prompt_and_calls_exec() {
|
||||
let claude_output = r#"{"type":"result","result":"I fixed the bug.","usage":{"input_tokens":500,"output_tokens":200}}"#;
|
||||
let test_env = Arc::new(CliTestEnv::new(claude_output));
|
||||
let env: Arc<dyn agent::ExecutionEnvironment> = test_env.clone();
|
||||
let backend = CliBackend::new("claude-opus-4-6".into(), "anthropic".into());
|
||||
|
||||
let node = Node::new("fix_code");
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = backend
|
||||
.run(&node, "Fix the authentication bug", &context, None, &emitter, dir.path(), &env)
|
||||
.await
|
||||
.expect("CLI backend should succeed");
|
||||
|
||||
// Verify prompt was written
|
||||
let written = test_env.recorded_written_files();
|
||||
assert_eq!(written.len(), 1, "should write exactly one file (the prompt)");
|
||||
assert_eq!(written[0].0, "/tmp/attractor_cli_prompt.txt");
|
||||
assert_eq!(written[0].1, "Fix the authentication bug");
|
||||
|
||||
// Verify the CLI command was called
|
||||
let commands = test_env.recorded_commands();
|
||||
let cli_cmd = commands.iter().find(|c| c.contains("claude")).expect("should call claude CLI");
|
||||
assert!(cli_cmd.contains("-p"), "should use pipe mode");
|
||||
assert!(cli_cmd.contains("claude-opus-4-6"), "should use correct model");
|
||||
assert!(cli_cmd.contains("/tmp/attractor_cli_prompt.txt"), "should reference prompt file");
|
||||
|
||||
// Verify parsed response
|
||||
match result {
|
||||
CodergenResult::Text { text, usage, files_touched } => {
|
||||
assert_eq!(text, "I fixed the bug.");
|
||||
let usage = usage.expect("should have usage");
|
||||
assert_eq!(usage.input_tokens, 500);
|
||||
assert_eq!(usage.output_tokens, 200);
|
||||
assert!(files_touched.is_empty(), "no files changed before/after");
|
||||
}
|
||||
CodergenResult::Full(_) => panic!("expected Text result, got Full"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cli_backend_run_detects_changed_files() {
|
||||
let claude_output = r#"{"type":"result","result":"Created new file.","usage":{"input_tokens":100,"output_tokens":50}}"#;
|
||||
let env: Arc<dyn agent::ExecutionEnvironment> = Arc::new(
|
||||
CliTestEnv::new(claude_output)
|
||||
.with_git_diff_after("src/main.rs\nsrc/lib.rs\n"),
|
||||
);
|
||||
let backend = CliBackend::new("claude-opus-4-6".into(), "anthropic".into());
|
||||
|
||||
let node = Node::new("implement");
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = backend
|
||||
.run(&node, "Add a new feature", &context, None, &emitter, dir.path(), &env)
|
||||
.await
|
||||
.expect("CLI backend should succeed");
|
||||
|
||||
match result {
|
||||
CodergenResult::Text { files_touched, .. } => {
|
||||
assert_eq!(files_touched, vec!["src/lib.rs", "src/main.rs"]);
|
||||
}
|
||||
CodergenResult::Full(_) => panic!("expected Text result"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cli_backend_run_with_codex_provider() {
|
||||
let codex_output = "{\"type\":\"item.completed\",\"item\":{\"id\":\"item_0\",\"type\":\"agent_message\",\"text\":\"Implemented the feature.\"}}\n{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":300,\"output_tokens\":150}}";
|
||||
let test_env = Arc::new(CliTestEnv::new(codex_output));
|
||||
let env: Arc<dyn agent::ExecutionEnvironment> = test_env.clone();
|
||||
let backend = CliBackend::new("gpt-5.3-codex-spark".into(), "openai".into());
|
||||
|
||||
let node = Node::new("implement");
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = backend
|
||||
.run(&node, "Build the API", &context, None, &emitter, dir.path(), &env)
|
||||
.await
|
||||
.expect("CLI backend should succeed");
|
||||
|
||||
// Verify codex command was called
|
||||
let commands = test_env.recorded_commands();
|
||||
let cli_cmd = commands.iter().find(|c| c.contains("codex")).expect("should call codex CLI");
|
||||
assert!(cli_cmd.contains("exec --json"), "should use exec mode");
|
||||
assert!(cli_cmd.contains("gpt-5.3-codex-spark"), "should use correct model");
|
||||
|
||||
match result {
|
||||
CodergenResult::Text { text, usage, .. } => {
|
||||
assert_eq!(text, "Implemented the feature.");
|
||||
let usage = usage.expect("should have usage");
|
||||
assert_eq!(usage.input_tokens, 300);
|
||||
assert_eq!(usage.output_tokens, 150);
|
||||
}
|
||||
CodergenResult::Full(_) => panic!("expected Text result"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cli_backend_run_fails_on_nonzero_exit() {
|
||||
let env = Arc::new(CliTestEnv::new(""));
|
||||
|
||||
// Override exec_command to return non-zero for the CLI call
|
||||
struct FailingCliEnv;
|
||||
#[async_trait::async_trait]
|
||||
impl agent::ExecutionEnvironment for FailingCliEnv {
|
||||
async fn read_file(&self, _: &str, _: Option<usize>, _: Option<usize>) -> Result<String, String> { Ok(String::new()) }
|
||||
async fn write_file(&self, _: &str, _: &str) -> Result<(), String> { Ok(()) }
|
||||
async fn delete_file(&self, _: &str) -> Result<(), String> { Ok(()) }
|
||||
async fn file_exists(&self, _: &str) -> Result<bool, String> { Ok(false) }
|
||||
async fn list_directory(&self, _: &str, _: Option<usize>) -> Result<Vec<agent::DirEntry>, String> { Ok(vec![]) }
|
||||
async fn exec_command(&self, command: &str, _: u64, _: Option<&str>, _: Option<&std::collections::HashMap<String, String>>, _: Option<tokio_util::sync::CancellationToken>) -> Result<agent::ExecResult, String> {
|
||||
if command.starts_with("git") {
|
||||
return Ok(agent::ExecResult { stdout: String::new(), stderr: String::new(), exit_code: 0, timed_out: false, duration_ms: 0 });
|
||||
}
|
||||
Ok(agent::ExecResult { stdout: String::new(), stderr: "command not found: claude".into(), exit_code: 127, timed_out: false, duration_ms: 0 })
|
||||
}
|
||||
async fn grep(&self, _: &str, _: &str, _: &agent::GrepOptions) -> Result<Vec<String>, String> { Ok(vec![]) }
|
||||
async fn glob(&self, _: &str, _: Option<&str>) -> Result<Vec<String>, String> { Ok(vec![]) }
|
||||
async fn initialize(&self) -> Result<(), String> { Ok(()) }
|
||||
async fn cleanup(&self) -> Result<(), String> { Ok(()) }
|
||||
fn working_directory(&self) -> &str { "/tmp" }
|
||||
fn platform(&self) -> &str { "darwin" }
|
||||
fn os_version(&self) -> String { "Darwin 24.0.0".into() }
|
||||
}
|
||||
|
||||
let failing_env: Arc<dyn agent::ExecutionEnvironment> = Arc::new(FailingCliEnv);
|
||||
let backend = CliBackend::new("claude-opus-4-6".into(), "anthropic".into());
|
||||
let node = Node::new("step");
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let _ = env; // unused, just for the above struct
|
||||
|
||||
let result = backend
|
||||
.run(&node, "do something", &context, None, &emitter, dir.path(), &failing_env)
|
||||
.await;
|
||||
|
||||
let err = match result {
|
||||
Err(e) => e,
|
||||
Ok(_) => panic!("should fail on non-zero exit"),
|
||||
};
|
||||
|
||||
assert!(err.to_string().contains("exited with code 127"), "error: {err}");
|
||||
assert!(err.to_string().contains("command not found"), "error: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cli_backend_run_fails_on_unparseable_output() {
|
||||
let env: Arc<dyn agent::ExecutionEnvironment> = Arc::new(CliTestEnv::new("this is not json at all"));
|
||||
let backend = CliBackend::new("claude-opus-4-6".into(), "anthropic".into());
|
||||
|
||||
let node = Node::new("step");
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = backend
|
||||
.run(&node, "do something", &context, None, &emitter, dir.path(), &env)
|
||||
.await;
|
||||
|
||||
let err = match result {
|
||||
Err(e) => e,
|
||||
Ok(_) => panic!("should fail on unparseable output"),
|
||||
};
|
||||
|
||||
assert!(err.to_string().contains("Failed to parse CLI output"), "error: {err}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cli_backend_run_uses_node_model_override() {
|
||||
let claude_output = r#"{"type":"result","result":"ok","usage":{"input_tokens":10,"output_tokens":5}}"#;
|
||||
let test_env = Arc::new(CliTestEnv::new(claude_output));
|
||||
let env: Arc<dyn agent::ExecutionEnvironment> = test_env.clone();
|
||||
let backend = CliBackend::new("default-model".into(), "anthropic".into());
|
||||
|
||||
let mut node = Node::new("step");
|
||||
node.attrs.insert("llm_model".to_string(), AttrValue::String("claude-sonnet-4-5".to_string()));
|
||||
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
backend
|
||||
.run(&node, "test", &context, None, &emitter, dir.path(), &env)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
|
||||
let commands = test_env.recorded_commands();
|
||||
let cli_cmd = commands.iter().find(|c| c.contains("claude")).unwrap();
|
||||
assert!(cli_cmd.contains("claude-sonnet-4-5"), "should use node's model override, not default: {cli_cmd}");
|
||||
assert!(!cli_cmd.contains("default-model"), "should NOT use default model: {cli_cmd}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cli_backend_run_uses_node_provider_override() {
|
||||
let codex_output = "{\"type\":\"item.completed\",\"item\":{\"id\":\"item_0\",\"type\":\"agent_message\",\"text\":\"ok\"}}\n{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":5}}";
|
||||
let test_env = Arc::new(CliTestEnv::new(codex_output));
|
||||
let env: Arc<dyn agent::ExecutionEnvironment> = test_env.clone();
|
||||
let backend = CliBackend::new("default-model".into(), "anthropic".into());
|
||||
|
||||
let mut node = Node::new("step");
|
||||
node.attrs.insert("llm_provider".to_string(), AttrValue::String("openai".to_string()));
|
||||
node.attrs.insert("llm_model".to_string(), AttrValue::String("gpt-5.3-codex-spark".to_string()));
|
||||
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
backend
|
||||
.run(&node, "test", &context, None, &emitter, dir.path(), &env)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
|
||||
let commands = test_env.recorded_commands();
|
||||
let cli_cmd = commands.iter().find(|c| c.contains("codex")).expect("should call codex based on provider override");
|
||||
assert!(cli_cmd.contains("gpt-5.3-codex-spark"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cli_backend_run_writes_provider_used_json() {
|
||||
let claude_output = r#"{"type":"result","result":"done","usage":{"input_tokens":10,"output_tokens":5}}"#;
|
||||
let env: Arc<dyn agent::ExecutionEnvironment> = Arc::new(CliTestEnv::new(claude_output));
|
||||
let backend = CliBackend::new("claude-opus-4-6".into(), "anthropic".into());
|
||||
|
||||
let node = Node::new("step");
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
backend
|
||||
.run(&node, "test", &context, None, &emitter, dir.path(), &env)
|
||||
.await
|
||||
.expect("should succeed");
|
||||
|
||||
let provider_path = dir.path().join("provider_used.json");
|
||||
assert!(provider_path.exists(), "should write provider_used.json");
|
||||
let provider_json: serde_json::Value = serde_json::from_str(
|
||||
&std::fs::read_to_string(&provider_path).unwrap()
|
||||
).unwrap();
|
||||
assert_eq!(provider_json["mode"], "cli");
|
||||
assert_eq!(provider_json["provider"], "anthropic");
|
||||
assert_eq!(provider_json["model"], "claude-opus-4-6");
|
||||
assert!(provider_json["command"].as_str().unwrap().contains("claude"));
|
||||
}
|
||||
|
||||
// -- BackendRouter e2e: delegates to correct backend --
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_router_delegates_to_cli_for_cli_node() {
|
||||
let claude_output = r#"{"type":"result","result":"CLI response","usage":{"input_tokens":10,"output_tokens":5}}"#;
|
||||
let env: Arc<dyn agent::ExecutionEnvironment> = Arc::new(CliTestEnv::new(claude_output));
|
||||
|
||||
let api_backend = Box::new(MockCodergenBackend); // would return "Response for ..."
|
||||
let cli = CliBackend::new("claude-opus-4-6".into(), "anthropic".into());
|
||||
let router = BackendRouter::new(api_backend, cli);
|
||||
|
||||
let mut node = Node::new("cli_step");
|
||||
node.attrs.insert("backend".to_string(), AttrValue::String("cli".to_string()));
|
||||
node.attrs.insert("prompt".to_string(), AttrValue::String("Fix the bug".to_string()));
|
||||
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = router
|
||||
.run(&node, "Fix the bug", &context, None, &emitter, dir.path(), &env)
|
||||
.await
|
||||
.expect("router should succeed");
|
||||
|
||||
match result {
|
||||
CodergenResult::Text { text, .. } => {
|
||||
assert_eq!(text, "CLI response", "should use CLI backend response, not mock API");
|
||||
}
|
||||
CodergenResult::Full(_) => panic!("expected Text result"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_router_delegates_to_api_for_normal_node() {
|
||||
let env = local_env();
|
||||
|
||||
let api_backend = Box::new(MockCodergenBackend);
|
||||
let cli = CliBackend::new("claude-opus-4-6".into(), "anthropic".into());
|
||||
let router = BackendRouter::new(api_backend, cli);
|
||||
|
||||
let mut node = Node::new("api_step");
|
||||
node.attrs.insert("prompt".to_string(), AttrValue::String("Plan the work".to_string()));
|
||||
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = router
|
||||
.run(&node, "Plan the work", &context, None, &emitter, dir.path(), &env)
|
||||
.await
|
||||
.expect("router should succeed");
|
||||
|
||||
match result {
|
||||
CodergenResult::Text { text, .. } => {
|
||||
assert!(text.starts_with("Response for api_step"), "should use API mock response: {text}");
|
||||
}
|
||||
CodergenResult::Full(_) => panic!("expected Text result"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backend_router_delegates_to_cli_for_cli_only_model() {
|
||||
let codex_output = "{\"type\":\"item.completed\",\"item\":{\"id\":\"item_0\",\"type\":\"agent_message\",\"text\":\"Codex did it\"}}\n{\"type\":\"turn.completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":5}}";
|
||||
let env: Arc<dyn agent::ExecutionEnvironment> = Arc::new(CliTestEnv::new(codex_output));
|
||||
|
||||
let api_backend = Box::new(MockCodergenBackend);
|
||||
let cli = CliBackend::new("gpt-5.3-codex-spark".into(), "openai".into());
|
||||
let router = BackendRouter::new(api_backend, cli);
|
||||
|
||||
let mut node = Node::new("codex_step");
|
||||
node.attrs.insert("llm_model".to_string(), AttrValue::String("gpt-5.3-codex-spark".to_string()));
|
||||
node.attrs.insert("llm_provider".to_string(), AttrValue::String("openai".to_string()));
|
||||
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = router
|
||||
.run(&node, "Build it", &context, None, &emitter, dir.path(), &env)
|
||||
.await
|
||||
.expect("router should succeed");
|
||||
|
||||
match result {
|
||||
CodergenResult::Text { text, .. } => {
|
||||
assert_eq!(text, "Codex did it", "should route to CLI backend for CLI-only model");
|
||||
}
|
||||
CodergenResult::Full(_) => panic!("expected Text result"),
|
||||
}
|
||||
}
|
||||
|
||||
// -- Full pipeline e2e with BackendRouter --
|
||||
|
||||
#[tokio::test]
|
||||
async fn full_pipeline_with_cli_backend_node() {
|
||||
// Pipeline: start -> api_work -> cli_work -> exit
|
||||
// api_work uses MockCodergenBackend (API), cli_work has backend="cli"
|
||||
let claude_output = r#"{"type":"result","result":"CLI completed the task.","usage":{"input_tokens":100,"output_tokens":50}}"#;
|
||||
let env: Arc<dyn agent::ExecutionEnvironment> = Arc::new(CliTestEnv::new(claude_output));
|
||||
|
||||
let mut graph = Graph::new("CliPipelineTest");
|
||||
|
||||
let mut start = Node::new("start");
|
||||
start.attrs.insert("shape".to_string(), AttrValue::String("Mdiamond".to_string()));
|
||||
graph.nodes.insert("start".to_string(), start);
|
||||
|
||||
let mut exit = Node::new("exit");
|
||||
exit.attrs.insert("shape".to_string(), AttrValue::String("Msquare".to_string()));
|
||||
graph.nodes.insert("exit".to_string(), exit);
|
||||
|
||||
let mut api_work = Node::new("api_work");
|
||||
api_work.attrs.insert("shape".to_string(), AttrValue::String("box".to_string()));
|
||||
api_work.attrs.insert("prompt".to_string(), AttrValue::String("Plan the work".to_string()));
|
||||
graph.nodes.insert("api_work".to_string(), api_work);
|
||||
|
||||
let mut cli_work = Node::new("cli_work");
|
||||
cli_work.attrs.insert("shape".to_string(), AttrValue::String("box".to_string()));
|
||||
cli_work.attrs.insert("prompt".to_string(), AttrValue::String("Implement via CLI".to_string()));
|
||||
cli_work.attrs.insert("backend".to_string(), AttrValue::String("cli".to_string()));
|
||||
graph.nodes.insert("cli_work".to_string(), cli_work);
|
||||
|
||||
graph.edges.push(Edge::new("start", "api_work"));
|
||||
graph.edges.push(Edge::new("api_work", "cli_work"));
|
||||
graph.edges.push(Edge::new("cli_work", "exit"));
|
||||
|
||||
// Build engine with BackendRouter
|
||||
let api = MockCodergenBackend;
|
||||
let cli = CliBackend::new("claude-opus-4-6".into(), "anthropic".into());
|
||||
let router = BackendRouter::new(Box::new(api), cli);
|
||||
let codergen_handler = CodergenHandler::new(Some(Box::new(router)));
|
||||
|
||||
let mut registry = HandlerRegistry::new(Box::new(codergen_handler));
|
||||
registry.register("start", Box::new(StartHandler));
|
||||
registry.register("exit", Box::new(ExitHandler));
|
||||
registry.register("codergen", Box::new(CodergenHandler::new(Some(Box::new({
|
||||
// Second BackendRouter for the "codergen" handler
|
||||
let api2 = MockCodergenBackend;
|
||||
let cli2 = CliBackend::new("claude-opus-4-6".into(), "anthropic".into());
|
||||
BackendRouter::new(Box::new(api2), cli2)
|
||||
})))));
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let engine = PipelineEngine::new(registry, Arc::new(EventEmitter::new()), env);
|
||||
let config = RunConfig {
|
||||
logs_root: dir.path().to_path_buf(),
|
||||
cancel_token: None,
|
||||
dry_run: false,
|
||||
};
|
||||
|
||||
let outcome = engine.run(&graph, &config).await.expect("pipeline should succeed");
|
||||
assert_eq!(outcome.status, StageStatus::Success);
|
||||
|
||||
// Verify api_work used mock (its response.md should contain "Response for")
|
||||
let api_response = std::fs::read_to_string(
|
||||
dir.path().join("nodes").join("api_work").join("response.md")
|
||||
).unwrap();
|
||||
assert!(api_response.starts_with("Response for api_work"), "API node should use mock: {api_response}");
|
||||
|
||||
// Verify cli_work used CLI backend (its response.md should contain CLI response)
|
||||
let cli_response = std::fs::read_to_string(
|
||||
dir.path().join("nodes").join("cli_work").join("response.md")
|
||||
).unwrap();
|
||||
assert_eq!(cli_response, "CLI completed the task.", "CLI node should use CLI backend: {cli_response}");
|
||||
|
||||
// Verify cli_work wrote provider_used.json with mode=cli
|
||||
let provider_json: serde_json::Value = serde_json::from_str(
|
||||
&std::fs::read_to_string(dir.path().join("nodes").join("cli_work").join("provider_used.json")).unwrap()
|
||||
).unwrap();
|
||||
assert_eq!(provider_json["mode"], "cli");
|
||||
}
|
||||
|
||||
// -- Stylesheet applies backend property to nodes in a full pipeline --
|
||||
|
||||
#[tokio::test]
|
||||
async fn stylesheet_backend_property_routes_to_cli() {
|
||||
let claude_output = r#"{"type":"result","result":"Styled CLI response.","usage":{"input_tokens":10,"output_tokens":5}}"#;
|
||||
let env: Arc<dyn agent::ExecutionEnvironment> = Arc::new(CliTestEnv::new(claude_output));
|
||||
|
||||
let mut graph = Graph::new("StylesheetTest");
|
||||
graph.attrs.insert(
|
||||
"model_stylesheet".to_string(),
|
||||
AttrValue::String(".cli-node { backend: cli; }".to_string()),
|
||||
);
|
||||
|
||||
let mut start = Node::new("start");
|
||||
start.attrs.insert("shape".to_string(), AttrValue::String("Mdiamond".to_string()));
|
||||
graph.nodes.insert("start".to_string(), start);
|
||||
|
||||
let mut exit = Node::new("exit");
|
||||
exit.attrs.insert("shape".to_string(), AttrValue::String("Msquare".to_string()));
|
||||
graph.nodes.insert("exit".to_string(), exit);
|
||||
|
||||
let mut work = Node::new("work");
|
||||
work.attrs.insert("shape".to_string(), AttrValue::String("box".to_string()));
|
||||
work.attrs.insert("prompt".to_string(), AttrValue::String("Do work".to_string()));
|
||||
work.classes.push("cli-node".to_string());
|
||||
graph.nodes.insert("work".to_string(), work);
|
||||
|
||||
graph.edges.push(Edge::new("start", "work"));
|
||||
graph.edges.push(Edge::new("work", "exit"));
|
||||
|
||||
// Apply stylesheet
|
||||
let ss = parse_stylesheet(graph.model_stylesheet()).unwrap();
|
||||
apply_stylesheet(&ss, &mut graph);
|
||||
|
||||
// Verify the stylesheet applied the backend property
|
||||
assert_eq!(
|
||||
graph.nodes["work"].backend(),
|
||||
Some("cli"),
|
||||
"stylesheet should set backend=cli on .cli-node"
|
||||
);
|
||||
|
||||
// Run the pipeline
|
||||
let api = MockCodergenBackend;
|
||||
let cli = CliBackend::new("claude-opus-4-6".into(), "anthropic".into());
|
||||
let router = BackendRouter::new(Box::new(api), cli);
|
||||
|
||||
let mut registry = HandlerRegistry::new(Box::new(CodergenHandler::new(Some(Box::new(router)))));
|
||||
registry.register("start", Box::new(StartHandler));
|
||||
registry.register("exit", Box::new(ExitHandler));
|
||||
let api2 = MockCodergenBackend;
|
||||
let cli2 = CliBackend::new("claude-opus-4-6".into(), "anthropic".into());
|
||||
let router2 = BackendRouter::new(Box::new(api2), cli2);
|
||||
registry.register("codergen", Box::new(CodergenHandler::new(Some(Box::new(router2)))));
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let engine = PipelineEngine::new(registry, Arc::new(EventEmitter::new()), env);
|
||||
let config = RunConfig {
|
||||
logs_root: dir.path().to_path_buf(),
|
||||
cancel_token: None,
|
||||
dry_run: false,
|
||||
};
|
||||
|
||||
let outcome = engine.run(&graph, &config).await.expect("pipeline should succeed");
|
||||
assert_eq!(outcome.status, StageStatus::Success);
|
||||
|
||||
let response = std::fs::read_to_string(
|
||||
dir.path().join("nodes").join("work").join("response.md")
|
||||
).unwrap();
|
||||
assert_eq!(response, "Styled CLI response.", "stylesheet-driven node should use CLI backend");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Real CLI backend e2e tests (require actual CLI tools installed)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
use attractor::cli::cli_backend::parse_cli_response;
|
||||
|
||||
/// Run a real CLI tool via LocalExecutionEnvironment and verify the full flow.
|
||||
async fn run_real_cli_test(provider: &str, model: &str) {
|
||||
let env = local_env();
|
||||
let backend = CliBackend::new(model.to_string(), provider.to_string());
|
||||
|
||||
let mut node = Node::new("real_cli_test");
|
||||
node.attrs.insert("prompt".to_string(), AttrValue::String("What is 2+2? Reply with just the number.".to_string()));
|
||||
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::new());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = backend
|
||||
.run(&node, "What is 2+2? Reply with just the number.", &context, None, &emitter, dir.path(), &env)
|
||||
.await
|
||||
.expect(&format!("CLI backend ({provider}/{model}) should succeed"));
|
||||
|
||||
match result {
|
||||
CodergenResult::Text { text, usage, .. } => {
|
||||
assert!(
|
||||
text.contains('4'),
|
||||
"{provider}/{model}: expected response to contain '4', got: {text}"
|
||||
);
|
||||
let usage = usage.expect(&format!("{provider}/{model}: should have usage"));
|
||||
assert!(usage.input_tokens > 0, "{provider}/{model}: input_tokens should be > 0, got {}", usage.input_tokens);
|
||||
}
|
||||
CodergenResult::Full(_) => panic!("expected Text result from {provider}/{model}"),
|
||||
}
|
||||
|
||||
// Verify log files were written
|
||||
let provider_path = dir.path().join("provider_used.json");
|
||||
assert!(provider_path.exists(), "{provider}/{model}: provider_used.json should exist");
|
||||
let provider_json: serde_json::Value = serde_json::from_str(
|
||||
&std::fs::read_to_string(&provider_path).unwrap()
|
||||
).unwrap();
|
||||
assert_eq!(provider_json["mode"], "cli");
|
||||
assert_eq!(provider_json["provider"], provider);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires `claude` CLI installed
|
||||
async fn real_cli_claude() {
|
||||
run_real_cli_test("anthropic", "haiku").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires `codex` CLI installed and OpenAI auth
|
||||
async fn real_cli_codex() {
|
||||
run_real_cli_test("openai", "").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires `gemini` CLI installed and Google auth
|
||||
async fn real_cli_gemini() {
|
||||
run_real_cli_test("gemini", "gemini-2.5-flash").await;
|
||||
}
|
||||
|
||||
/// Verify parse_cli_response works against real Claude CLI output captured from stream-json.
|
||||
#[test]
|
||||
fn parse_real_claude_stream_json() {
|
||||
// Real output captured from: claude -p --output-format stream-json --model haiku "What is 2+2?"
|
||||
let output = r#"{"type":"system","subtype":"init","cwd":"/tmp","session_id":"abc"}
|
||||
{"type":"assistant","message":{"content":[{"type":"text","text":"4"}]}}
|
||||
{"type":"result","subtype":"success","is_error":false,"duration_ms":2000,"num_turns":1,"result":"4","usage":{"input_tokens":9,"output_tokens":5}}"#;
|
||||
let response = parse_cli_response("anthropic", output).unwrap();
|
||||
assert_eq!(response.text, "4");
|
||||
assert_eq!(response.input_tokens, 9);
|
||||
assert_eq!(response.output_tokens, 5);
|
||||
}
|
||||
|
||||
/// Verify parse_cli_response works against real Codex CLI output.
|
||||
#[test]
|
||||
fn parse_real_codex_ndjson() {
|
||||
// Real output captured from: echo "What is 2+2?" | codex exec --json
|
||||
let output = r#"{"type":"thread.started","thread_id":"019ca1ec-1e86-79b2-b2b2-b1d963f1aea2"}
|
||||
{"type":"turn.started"}
|
||||
{"type":"item.completed","item":{"id":"item_0","type":"reasoning","text":"**Confirming simple numeric reply**"}}
|
||||
{"type":"item.completed","item":{"id":"item_1","type":"agent_message","text":"4"}}
|
||||
{"type":"turn.completed","usage":{"input_tokens":7999,"cached_input_tokens":7040,"output_tokens":33}}"#;
|
||||
let response = parse_cli_response("openai", output).unwrap();
|
||||
assert_eq!(response.text, "4");
|
||||
assert_eq!(response.input_tokens, 7999);
|
||||
assert_eq!(response.output_tokens, 33);
|
||||
}
|
||||
|
||||
/// Verify parse_cli_response works against real Gemini CLI output.
|
||||
#[test]
|
||||
fn parse_real_gemini_json() {
|
||||
// Real output captured from: gemini "What is 2+2?" -m gemini-2.5-flash --sandbox -o json
|
||||
let output = r#"{"session_id":"abc","response":"4","stats":{"models":{"gemini-2.5-flash":{"api":{"totalRequests":1,"totalErrors":0,"totalLatencyMs":618},"tokens":{"input":123,"prompt":8911,"candidates":1,"total":8912,"cached":8788,"thoughts":0,"tool":0}}},"tools":{"totalCalls":0},"files":{"totalLinesAdded":0,"totalLinesRemoved":0}}}"#;
|
||||
let response = parse_cli_response("gemini", output).unwrap();
|
||||
assert_eq!(response.text, "4");
|
||||
assert_eq!(response.input_tokens, 123);
|
||||
assert_eq!(response.output_tokens, 1);
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue