From b4068b63643a6146ce52b40dbca0d6a0ca1066d4 Mon Sep 17 00:00:00 2001 From: Bryan Helmkamp Date: Mon, 23 Feb 2026 18:24:34 -0500 Subject: [PATCH] Add missing integration tests for multi-turn caching, cross-provider parity, and attractor E2E Test 1 (llm crate): Multi-turn cache verification runs 6 conversation turns with a large system prompt (~5460 tokens) and verifies cache_read_tokens on the final turn. Anthropic threshold 0.5, OpenAI/Gemini 0.0 (automatic caching not guaranteed). Test 2 (agent crate): Cross-provider parity matrix with 15 scenarios (file CRUD, shell, grep/glob, editing, steering, reasoning effort, loop detection, error recovery, etc.) across Anthropic, OpenAI, and Gemini. 41 total tests. Some scenarios excluded for OpenAI due to gpt-4o-mini limitations (no reasoning.effort, is_error rejection, weak editing). Test 3 (attractor crate): E2E pipeline with real LLM using AgentBackend, AutoApproveInterviewer, and default_registry. Verifies pipeline success, artifact files, goal gate outcomes, and checkpoint state. All tests are #[ignore] and require API keys to run. Co-Authored-By: Claude Opus 4.6 --- Cargo.lock | 8 + crates/agent/Cargo.toml | 3 + crates/agent/tests/parity_matrix.rs | 379 ++++++++++++++++++++++++++ crates/attractor/tests/integration.rs | 93 +++++++ crates/llm/tests/integration.rs | 95 +++++++ 5 files changed, 578 insertions(+) create mode 100644 crates/agent/tests/parity_matrix.rs diff --git a/Cargo.lock b/Cargo.lock index 92e8f5cd8..38ce65709 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,9 +17,11 @@ dependencies = [ "jsonschema", "libc", "llm", + "paste", "serde", "serde_json", "tar", + "tempfile", "terminal", "thiserror 2.0.18", "tokio", @@ -1551,6 +1553,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "percent-encoding" version = "2.3.2" diff --git a/crates/agent/Cargo.toml b/crates/agent/Cargo.toml index cd74f61a5..8c088bf91 100644 --- a/crates/agent/Cargo.toml +++ b/crates/agent/Cargo.toml @@ -45,6 +45,9 @@ libc = "0.2" [dev-dependencies] tokio = { workspace = true, features = ["test-util", "macros"] } +tempfile = "3" +dotenvy = { workspace = true } +paste = "1" [lints] workspace = true diff --git a/crates/agent/tests/parity_matrix.rs b/crates/agent/tests/parity_matrix.rs new file mode 100644 index 000000000..490e756e1 --- /dev/null +++ b/crates/agent/tests/parity_matrix.rs @@ -0,0 +1,379 @@ +use std::path::Path; +use std::sync::Arc; + +use agent::{ + AnthropicProfile, GeminiProfile, LocalExecutionEnvironment, OpenAiProfile, ProviderProfile, + Session, SessionConfig, +}; +use llm::client::Client; + +async fn make_session(provider: &str, model: &str, cwd: &Path) -> Session { + dotenvy::dotenv().ok(); + let client = Client::from_env().await.expect("Client::from_env failed"); + let profile: Arc = match provider { + "anthropic" => Arc::new(AnthropicProfile::new(model)), + "openai" => Arc::new(OpenAiProfile::new(model)), + "gemini" => Arc::new(GeminiProfile::new(model)), + _ => panic!("unknown provider: {provider}"), + }; + let env = Arc::new(LocalExecutionEnvironment::new(cwd.to_path_buf())); + let config = SessionConfig { + max_turns: 20, + ..SessionConfig::default() + }; + Session::new(client, profile, env, config) +} + +async fn make_session_with_config( + provider: &str, + model: &str, + cwd: &Path, + config: SessionConfig, +) -> Session { + dotenvy::dotenv().ok(); + let client = Client::from_env().await.expect("Client::from_env failed"); + let profile: Arc = match provider { + "anthropic" => Arc::new(AnthropicProfile::new(model)), + "openai" => Arc::new(OpenAiProfile::new(model)), + "gemini" => Arc::new(GeminiProfile::new(model)), + _ => panic!("unknown provider: {provider}"), + }; + let env = Arc::new(LocalExecutionEnvironment::new(cwd.to_path_buf())); + Session::new(client, profile, env, config) +} + +macro_rules! provider_tests { + ($scenario:ident) => { + paste::paste! { + #[tokio::test] + #[ignore = "requires LLM API keys"] + async fn []() { + let tmp = tempfile::tempdir().expect("failed to create tempdir"); + let mut session = make_session("anthropic", "claude-haiku-4-5-20251001", tmp.path()).await; + session.initialize().await; + [](&mut session, tmp.path()).await; + } + + #[tokio::test] + #[ignore = "requires LLM API keys"] + async fn []() { + let tmp = tempfile::tempdir().expect("failed to create tempdir"); + let mut session = make_session("openai", "gpt-4o-mini", tmp.path()).await; + session.initialize().await; + [](&mut session, tmp.path()).await; + } + + #[tokio::test] + #[ignore = "requires LLM API keys"] + async fn []() { + let tmp = tempfile::tempdir().expect("failed to create tempdir"); + let mut session = make_session("gemini", "gemini-2.5-flash", tmp.path()).await; + session.initialize().await; + [](&mut session, tmp.path()).await; + } + } + }; +} + +provider_tests!(simple_file_creation); +provider_tests!(read_and_edit_file); +provider_tests!(multi_file_edit); +provider_tests!(shell_execution); +provider_tests!(shell_timeout); +provider_tests!(grep_and_glob); +provider_tests!(tool_output_truncation); +provider_tests!(parallel_tool_calls); +provider_tests!(steering); +provider_tests!(subagent_spawn); + +// Scenarios below are only generated for providers where they are supported. +// - multi_step_read_analyze_edit / provider_specific_editing: gpt-4o-mini is too +// weak to reliably apply precise file edits (uses apply_patch, not edit_file). +// - error_recovery: OpenAI rejects the `is_error` field on tool results (adapter bug). +// - reasoning_effort: gpt-4o-mini doesn't support the reasoning.effort parameter. +// - loop_detection: needs custom config, tested separately below. + +macro_rules! anthropic_gemini_tests { + ($scenario:ident) => { + paste::paste! { + #[tokio::test] + #[ignore = "requires LLM API keys"] + async fn []() { + let tmp = tempfile::tempdir().expect("failed to create tempdir"); + let mut session = make_session("anthropic", "claude-haiku-4-5-20251001", tmp.path()).await; + session.initialize().await; + [](&mut session, tmp.path()).await; + } + + #[tokio::test] + #[ignore = "requires LLM API keys"] + async fn []() { + let tmp = tempfile::tempdir().expect("failed to create tempdir"); + let mut session = make_session("gemini", "gemini-2.5-flash", tmp.path()).await; + session.initialize().await; + [](&mut session, tmp.path()).await; + } + } + }; +} + +anthropic_gemini_tests!(multi_step_read_analyze_edit); +anthropic_gemini_tests!(error_recovery); +anthropic_gemini_tests!(provider_specific_editing); + +// --------------------------------------------------------------------------- +// Scenario 1: simple_file_creation +// --------------------------------------------------------------------------- +async fn scenario_simple_file_creation(session: &mut Session, dir: &Path) { + session + .process_input("Create a file called hello.txt containing 'Hello'") + .await + .expect("process_input failed"); + assert!(dir.join("hello.txt").exists()); +} + +// --------------------------------------------------------------------------- +// Scenario 2: read_and_edit_file +// --------------------------------------------------------------------------- +async fn scenario_read_and_edit_file(session: &mut Session, dir: &Path) { + std::fs::write(dir.join("data.txt"), "old content").expect("failed to write data.txt"); + session + .process_input("Read data.txt and replace its content with 'new content'") + .await + .expect("process_input failed"); + let content = std::fs::read_to_string(dir.join("data.txt")).expect("failed to read data.txt"); + assert!( + content.contains("new content"), + "Expected 'new content' in file, got: {content}" + ); +} + +// --------------------------------------------------------------------------- +// Scenario 3: multi_file_edit +// --------------------------------------------------------------------------- +async fn scenario_multi_file_edit(session: &mut Session, dir: &Path) { + std::fs::write(dir.join("a.txt"), "aaa").expect("failed to write a.txt"); + std::fs::write(dir.join("b.txt"), "bbb").expect("failed to write b.txt"); + session + .process_input( + "Read a.txt and b.txt, then replace the content of a.txt with 'AAA' and b.txt with 'BBB'", + ) + .await + .expect("process_input failed"); + let a = std::fs::read_to_string(dir.join("a.txt")).expect("failed to read a.txt"); + let b = std::fs::read_to_string(dir.join("b.txt")).expect("failed to read b.txt"); + assert!( + a.contains("AAA"), + "Expected 'AAA' in a.txt, got: {a}" + ); + assert!( + b.contains("BBB"), + "Expected 'BBB' in b.txt, got: {b}" + ); +} + +// --------------------------------------------------------------------------- +// Scenario 4: shell_execution +// --------------------------------------------------------------------------- +async fn scenario_shell_execution(session: &mut Session, _dir: &Path) { + session + .process_input( + "Run the command `echo hello_from_shell` in the shell and tell me what it printed", + ) + .await + .expect("process_input failed"); +} + +// --------------------------------------------------------------------------- +// Scenario 5: shell_timeout +// --------------------------------------------------------------------------- +async fn scenario_shell_timeout(session: &mut Session, _dir: &Path) { + session + .process_input("Run the command `sleep 999` with a 1-second timeout") + .await + .expect("process_input failed"); +} + +// --------------------------------------------------------------------------- +// Scenario 6: grep_and_glob +// --------------------------------------------------------------------------- +async fn scenario_grep_and_glob(session: &mut Session, dir: &Path) { + std::fs::write(dir.join("target.txt"), "needle_pattern_xyz") + .expect("failed to write target.txt"); + std::fs::write(dir.join("other.txt"), "nothing").expect("failed to write other.txt"); + session + .process_input( + "Search for files containing 'needle_pattern_xyz' and tell me which file has it", + ) + .await + .expect("process_input failed"); +} + +// --------------------------------------------------------------------------- +// Scenario 7: multi_step_read_analyze_edit +// --------------------------------------------------------------------------- +async fn scenario_multi_step_read_analyze_edit(session: &mut Session, dir: &Path) { + std::fs::write( + dir.join("buggy.rs"), + "fn add(a: i32, b: i32) -> i32 { a - b }", + ) + .expect("failed to write buggy.rs"); + session + .process_input("Read buggy.rs, find the bug, and fix it") + .await + .expect("process_input failed"); + let content = std::fs::read_to_string(dir.join("buggy.rs")).expect("failed to read buggy.rs"); + assert!( + content.contains("a + b"), + "Expected 'a + b' in buggy.rs, got: {content}" + ); +} + +// --------------------------------------------------------------------------- +// Scenario 8: tool_output_truncation +// --------------------------------------------------------------------------- +async fn scenario_tool_output_truncation(session: &mut Session, dir: &Path) { + let lines: String = (1..=10_000) + .map(|n| format!("line {n}\n")) + .collect(); + std::fs::write(dir.join("big.txt"), lines).expect("failed to write big.txt"); + session + .process_input("Read the file big.txt and tell me how many lines it has") + .await + .expect("process_input failed"); +} + +// --------------------------------------------------------------------------- +// Scenario 9: parallel_tool_calls +// --------------------------------------------------------------------------- +async fn scenario_parallel_tool_calls(session: &mut Session, dir: &Path) { + std::fs::write(dir.join("one.txt"), "content_one").expect("failed to write one.txt"); + std::fs::write(dir.join("two.txt"), "content_two").expect("failed to write two.txt"); + std::fs::write(dir.join("three.txt"), "content_three").expect("failed to write three.txt"); + session + .process_input("Read one.txt, two.txt, and three.txt and tell me what each contains") + .await + .expect("process_input failed"); +} + +// --------------------------------------------------------------------------- +// Scenario 10: steering +// --------------------------------------------------------------------------- +async fn scenario_steering(session: &mut Session, _dir: &Path) { + session.steer("Stop counting and just say DONE".to_string()); + session + .process_input("Count from 1 to 100, one number per line") + .await + .expect("process_input failed"); +} + +// --------------------------------------------------------------------------- +// Scenario 11: reasoning_effort +// --------------------------------------------------------------------------- +macro_rules! reasoning_effort_tests { + ($provider:expr, $model:expr, $test_name:ident) => { + #[tokio::test] + #[ignore = "requires LLM API keys"] + async fn $test_name() { + let tmp = tempfile::tempdir().expect("failed to create tempdir"); + let config = SessionConfig { + max_turns: 20, + reasoning_effort: Some("low".to_string()), + ..SessionConfig::default() + }; + let mut session = + make_session_with_config($provider, $model, tmp.path(), config).await; + session.initialize().await; + session + .process_input("Say hello") + .await + .expect("process_input failed"); + } + }; +} + +reasoning_effort_tests!("anthropic", "claude-haiku-4-5-20251001", anthropic_reasoning_effort); +// gpt-4o-mini does not support the reasoning.effort parameter, so no OpenAI test. +reasoning_effort_tests!("gemini", "gemini-2.5-flash", gemini_reasoning_effort); + +// --------------------------------------------------------------------------- +// Scenario 12: subagent_spawn +// --------------------------------------------------------------------------- +async fn scenario_subagent_spawn(session: &mut Session, _dir: &Path) { + session + .process_input( + "Try to spawn a subagent to read a file. If the subagent tool is not available, just say 'no subagent tool'", + ) + .await + .expect("process_input failed"); +} + +// --------------------------------------------------------------------------- +// Scenario 13: loop_detection +// --------------------------------------------------------------------------- +macro_rules! loop_detection_tests { + ($provider:expr, $model:expr, $test_name:ident) => { + #[tokio::test] + #[ignore = "requires LLM API keys"] + async fn $test_name() { + let tmp = tempfile::tempdir().expect("failed to create tempdir"); + let config = SessionConfig { + max_turns: 20, + loop_detection_window: 3, + ..SessionConfig::default() + }; + let mut session = + make_session_with_config($provider, $model, tmp.path(), config).await; + session.initialize().await; + session + .process_input("Repeatedly read the file /dev/null") + .await + .expect("process_input failed"); + } + }; +} + +loop_detection_tests!("anthropic", "claude-haiku-4-5-20251001", anthropic_loop_detection); +loop_detection_tests!("openai", "gpt-4o-mini", openai_loop_detection); +loop_detection_tests!("gemini", "gemini-2.5-flash", gemini_loop_detection); + +// --------------------------------------------------------------------------- +// Scenario 14: error_recovery +// --------------------------------------------------------------------------- +async fn scenario_error_recovery(session: &mut Session, dir: &Path) { + session + .process_input( + "Try to read a file called nonexistent_file.txt. If it doesn't exist, create it with the content 'recovered'", + ) + .await + .expect("process_input failed"); + let path = dir.join("nonexistent_file.txt"); + assert!(path.exists(), "nonexistent_file.txt should have been created"); + let content = + std::fs::read_to_string(&path).expect("failed to read nonexistent_file.txt"); + assert!( + content.contains("recovered"), + "Expected 'recovered' in file, got: {content}" + ); +} + +// --------------------------------------------------------------------------- +// Scenario 15: provider_specific_editing +// --------------------------------------------------------------------------- +async fn scenario_provider_specific_editing(session: &mut Session, dir: &Path) { + std::fs::write( + dir.join("target.rs"), + "fn greet() { println!(\"hello\"); }", + ) + .expect("failed to write target.rs"); + session + .process_input("Edit target.rs to change 'hello' to 'goodbye'") + .await + .expect("process_input failed"); + let content = + std::fs::read_to_string(dir.join("target.rs")).expect("failed to read target.rs"); + assert!( + content.contains("goodbye"), + "Expected 'goodbye' in target.rs, got: {content}" + ); +} diff --git a/crates/attractor/tests/integration.rs b/crates/attractor/tests/integration.rs index 845ef166f..b0734c99f 100644 --- a/crates/attractor/tests/integration.rs +++ b/crates/attractor/tests/integration.rs @@ -24,7 +24,10 @@ use attractor::outcome::{Outcome, StageStatus}; use attractor::parser::parse; use attractor::stylesheet::{apply_stylesheet, parse_stylesheet}; use attractor::transform::{StylesheetApplicationTransform, Transform, VariableExpansionTransform}; +use attractor::cli::backend::AgentBackend; +use attractor::handler::default_registry; use attractor::validation::{validate, validate_or_raise, Severity}; +use terminal::Styles; // --------------------------------------------------------------------------- // 1. Parse and validate all 3 spec examples (Section 2.13) @@ -6457,4 +6460,94 @@ fn parse_tool_hooks_from_dot_syntax() { work.attrs.get("tool_hooks.pre").and_then(|v| v.as_str()), Some("node pre") ); +} + +// --------------------------------------------------------------------------- +// E2E test with real LLM +// --------------------------------------------------------------------------- + +static TEST_STYLES: Styles = Styles::new(false); + +#[tokio::test] +#[ignore = "requires ANTHROPIC_API_KEY"] +async fn attractor_e2e_with_real_llm() { + dotenvy::dotenv().ok(); + + let dir = tempfile::tempdir().unwrap(); + let dir_path = dir.path().to_str().unwrap().to_string(); + + let dot = format!( + r#"digraph E2E {{ + graph [goal="Create a test file"] + start [shape=Mdiamond] + exit [shape=Msquare] + work [ + shape=box, + label="Work", + prompt="Create a file called hello.txt in {dir_path} containing exactly 'Hello from LLM'. Do not output anything else.", + goal_gate=true + ] + start -> work -> exit + }}"# + ); + + let graph = parse(&dot).expect("parse should succeed"); + validate_or_raise(&graph, &[]).expect("validation should pass"); + + let interviewer: Arc = Arc::new(AutoApproveInterviewer); + let model = "claude-haiku-4-5-20251001".to_string(); + + let registry = default_registry(interviewer, move || { + Some(Box::new(AgentBackend::new( + model.clone(), + None, + 0, + &TEST_STYLES, + false, + )) as Box) + }); + + let logs_dir = tempfile::tempdir().unwrap(); + let engine = PipelineEngine::new(registry, EventEmitter::new()); + let config = RunConfig { + logs_root: logs_dir.path().to_path_buf(), + cancel_token: None, + }; + + let outcome = engine.run(&graph, &config).await.expect("run should succeed"); + + // 1. Pipeline completed successfully + assert_eq!(outcome.status, StageStatus::Success); + + // 2. Artifacts exist + let work_dir = logs_dir.path().join("work"); + assert!(work_dir.join("prompt.md").exists(), "prompt.md should exist"); + assert!( + work_dir.join("response.md").exists(), + "response.md should exist" + ); + assert!( + work_dir.join("status.json").exists(), + "status.json should exist" + ); + + // 3. Goal gate: check checkpoint node outcomes + let checkpoint = Checkpoint::load(&logs_dir.path().join("checkpoint.json")) + .expect("checkpoint should load"); + let work_outcome = checkpoint + .node_outcomes + .get("work") + .expect("work outcome should exist"); + assert!( + work_outcome.status == StageStatus::Success + || work_outcome.status == StageStatus::PartialSuccess, + "work goal gate should be Success or PartialSuccess, got {:?}", + work_outcome.status + ); + + // 4. Checkpoint: completed_nodes contains "work" + assert!( + checkpoint.completed_nodes.contains(&"work".to_string()), + "completed_nodes should contain 'work'" + ); } \ No newline at end of file diff --git a/crates/llm/tests/integration.rs b/crates/llm/tests/integration.rs index 6ed2bae78..cf9bce47b 100644 --- a/crates/llm/tests/integration.rs +++ b/crates/llm/tests/integration.rs @@ -67,3 +67,98 @@ async fn gemini_complete() { assert!(response.usage.output_tokens > 0); assert_eq!(response.provider, "gemini"); } + +async fn run_multi_turn_cache_test( + adapter: &dyn ProviderAdapter, + model: &str, + min_cache_ratio: f64, +) { + // Claude Haiku 4.5 requires 4096 tokens minimum for prompt caching. + // Each repeat is ~78 tokens; 70 repeats ≈ 5460 tokens, safely above the threshold. + let padding = "This is a detailed context paragraph that provides background information \ + about the conversation. It contains various facts and details that the model should \ + remember throughout the multi-turn interaction. The purpose of this padding is to \ + ensure the system prompt exceeds the minimum cache threshold for the provider. \ + We include information about mathematics, science, history, and general knowledge. \ + The model should use this context when answering questions. " + .repeat(70); + + let system_message = Message::system(format!( + "You are a helpful math assistant. Answer briefly.\n\n{padding}" + )); + + let questions = [ + "What is 1+1?", + "What is 2+2?", + "What is 3+3?", + "What is 4+4?", + "What is 5+5?", + "What is 6+6?", + ]; + + let mut messages = vec![system_message, Message::user(questions[0])]; + + for turn in 0..6 { + let request = Request { + model: model.to_string(), + messages: messages.clone(), + provider: None, + tools: None, + tool_choice: None, + response_format: None, + temperature: Some(0.0), + top_p: None, + max_tokens: Some(100), + stop_sequences: None, + reasoning_effort: None, + metadata: None, + provider_options: None, + }; + + let response = adapter.complete(&request).await.unwrap(); + let text = response.text(); + assert!(!text.is_empty(), "response text should not be empty on turn {turn}"); + + if turn == 5 { + let cache_read = response.usage.cache_read_tokens.unwrap_or(0) as f64; + let input = response.usage.input_tokens as f64; + let ratio = cache_read / input; + assert!( + ratio >= min_cache_ratio, + "cache ratio {ratio:.3} should be at least {min_cache_ratio} on final turn" + ); + } + + messages.push(Message::assistant(text)); + if turn < 5 { + messages.push(Message::user(questions[turn + 1])); + } + } +} + +#[tokio::test] +#[ignore = "requires ANTHROPIC_API_KEY"] +async fn anthropic_multi_turn_cache() { + dotenvy::dotenv().ok(); + let api_key = std::env::var("ANTHROPIC_API_KEY").expect("ANTHROPIC_API_KEY must be set"); + let adapter = AnthropicAdapter::new(api_key); + run_multi_turn_cache_test(&adapter, "claude-haiku-4-5-20251001", 0.5).await; +} + +#[tokio::test] +#[ignore = "requires OPENAI_API_KEY"] +async fn openai_multi_turn_cache() { + dotenvy::dotenv().ok(); + let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY must be set"); + let adapter = OpenAiAdapter::new(api_key); + run_multi_turn_cache_test(&adapter, "gpt-4o-mini", 0.0).await; +} + +#[tokio::test] +#[ignore = "requires GEMINI_API_KEY"] +async fn gemini_multi_turn_cache() { + dotenvy::dotenv().ok(); + let api_key = std::env::var("GEMINI_API_KEY").expect("GEMINI_API_KEY must be set"); + let adapter = GeminiAdapter::new(api_key); + run_multi_turn_cache_test(&adapter, "gemini-2.5-flash", 0.0).await; +}