Add missing integration tests for multi-turn caching, cross-provider parity, and attractor E2E

Test 1 (llm crate): Multi-turn cache verification runs 6 conversation turns
with a large system prompt (~5460 tokens) and verifies cache_read_tokens on
the final turn. Anthropic threshold 0.5, OpenAI/Gemini 0.0 (automatic
caching not guaranteed).

Test 2 (agent crate): Cross-provider parity matrix with 15 scenarios
(file CRUD, shell, grep/glob, editing, steering, reasoning effort, loop
detection, error recovery, etc.) across Anthropic, OpenAI, and Gemini.
41 total tests. Some scenarios excluded for OpenAI due to gpt-4o-mini
limitations (no reasoning.effort, is_error rejection, weak editing).

Test 3 (attractor crate): E2E pipeline with real LLM using AgentBackend,
AutoApproveInterviewer, and default_registry. Verifies pipeline success,
artifact files, goal gate outcomes, and checkpoint state.

All tests are #[ignore] and require API keys to run.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Cargo.lock

@@ -17,9 +17,11 @@ dependencies = [
  "jsonschema",
  "libc",
  "llm",
+ "paste",
  "serde",
  "serde_json",
  "tar",
+ "tempfile",
  "terminal",
  "thiserror 2.0.18",
  "tokio",
@@ -1551,6 +1553,12 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
 [[package]]
 name = "percent-encoding"
 version = "2.3.2"

crates/agent/Cargo.toml

@@ -45,6 +45,9 @@ libc = "0.2"
 
 [dev-dependencies]
 tokio = { workspace = true, features = ["test-util", "macros"] }
+tempfile = "3"
+dotenvy = { workspace = true }
+paste = "1"
 
 [lints]
 workspace = true

crates/agent/tests/parity_matrix.rs

@@ -0,0 +1,379 @@
+use std::path::Path;
+use std::sync::Arc;
+
+use agent::{
+    AnthropicProfile, GeminiProfile, LocalExecutionEnvironment, OpenAiProfile, ProviderProfile,
+    Session, SessionConfig,
+};
+use llm::client::Client;
+
+async fn make_session(provider: &str, model: &str, cwd: &Path) -> Session {
+    dotenvy::dotenv().ok();
+    let client = Client::from_env().await.expect("Client::from_env failed");
+    let profile: Arc<dyn ProviderProfile> = match provider {
+        "anthropic" => Arc::new(AnthropicProfile::new(model)),
+        "openai" => Arc::new(OpenAiProfile::new(model)),
+        "gemini" => Arc::new(GeminiProfile::new(model)),
+        _ => panic!("unknown provider: {provider}"),
+    };
+    let env = Arc::new(LocalExecutionEnvironment::new(cwd.to_path_buf()));
+    let config = SessionConfig {
+        max_turns: 20,
+        ..SessionConfig::default()
+    };
+    Session::new(client, profile, env, config)
+}
+
+async fn make_session_with_config(
+    provider: &str,
+    model: &str,
+    cwd: &Path,
+    config: SessionConfig,
+) -> Session {
+    dotenvy::dotenv().ok();
+    let client = Client::from_env().await.expect("Client::from_env failed");
+    let profile: Arc<dyn ProviderProfile> = match provider {
+        "anthropic" => Arc::new(AnthropicProfile::new(model)),
+        "openai" => Arc::new(OpenAiProfile::new(model)),
+        "gemini" => Arc::new(GeminiProfile::new(model)),
+        _ => panic!("unknown provider: {provider}"),
+    };
+    let env = Arc::new(LocalExecutionEnvironment::new(cwd.to_path_buf()));
+    Session::new(client, profile, env, config)
+}
+
+macro_rules! provider_tests {
+    ($scenario:ident) => {
+        paste::paste! {
+            #[tokio::test]
+            #[ignore = "requires LLM API keys"]
+            async fn [<anthropic_ $scenario>]() {
+                let tmp = tempfile::tempdir().expect("failed to create tempdir");
+                let mut session = make_session("anthropic", "claude-haiku-4-5-20251001", tmp.path()).await;
+                session.initialize().await;
+                [<scenario_ $scenario>](&mut session, tmp.path()).await;
+            }
+
+            #[tokio::test]
+            #[ignore = "requires LLM API keys"]
+            async fn [<openai_ $scenario>]() {
+                let tmp = tempfile::tempdir().expect("failed to create tempdir");
+                let mut session = make_session("openai", "gpt-4o-mini", tmp.path()).await;
+                session.initialize().await;
+                [<scenario_ $scenario>](&mut session, tmp.path()).await;
+            }
+
+            #[tokio::test]
+            #[ignore = "requires LLM API keys"]
+            async fn [<gemini_ $scenario>]() {
+                let tmp = tempfile::tempdir().expect("failed to create tempdir");
+                let mut session = make_session("gemini", "gemini-2.5-flash", tmp.path()).await;
+                session.initialize().await;
+                [<scenario_ $scenario>](&mut session, tmp.path()).await;
+            }
+        }
+    };
+}
+
+provider_tests!(simple_file_creation);
+provider_tests!(read_and_edit_file);
+provider_tests!(multi_file_edit);
+provider_tests!(shell_execution);
+provider_tests!(shell_timeout);
+provider_tests!(grep_and_glob);
+provider_tests!(tool_output_truncation);
+provider_tests!(parallel_tool_calls);
+provider_tests!(steering);
+provider_tests!(subagent_spawn);
+
+// Scenarios below are only generated for providers where they are supported.
+// - multi_step_read_analyze_edit / provider_specific_editing: gpt-4o-mini is too
+//   weak to reliably apply precise file edits (uses apply_patch, not edit_file).
+// - error_recovery: OpenAI rejects the `is_error` field on tool results (adapter bug).
+// - reasoning_effort: gpt-4o-mini doesn't support the reasoning.effort parameter.
+// - loop_detection: needs custom config, tested separately below.
+
+macro_rules! anthropic_gemini_tests {
+    ($scenario:ident) => {
+        paste::paste! {
+            #[tokio::test]
+            #[ignore = "requires LLM API keys"]
+            async fn [<anthropic_ $scenario>]() {
+                let tmp = tempfile::tempdir().expect("failed to create tempdir");
+                let mut session = make_session("anthropic", "claude-haiku-4-5-20251001", tmp.path()).await;
+                session.initialize().await;
+                [<scenario_ $scenario>](&mut session, tmp.path()).await;
+            }
+
+            #[tokio::test]
+            #[ignore = "requires LLM API keys"]
+            async fn [<gemini_ $scenario>]() {
+                let tmp = tempfile::tempdir().expect("failed to create tempdir");
+                let mut session = make_session("gemini", "gemini-2.5-flash", tmp.path()).await;
+                session.initialize().await;
+                [<scenario_ $scenario>](&mut session, tmp.path()).await;
+            }
+        }
+    };
+}
+
+anthropic_gemini_tests!(multi_step_read_analyze_edit);
+anthropic_gemini_tests!(error_recovery);
+anthropic_gemini_tests!(provider_specific_editing);
+
+// ---------------------------------------------------------------------------
+// Scenario 1: simple_file_creation
+// ---------------------------------------------------------------------------
+async fn scenario_simple_file_creation(session: &mut Session, dir: &Path) {
+    session
+        .process_input("Create a file called hello.txt containing 'Hello'")
+        .await
+        .expect("process_input failed");
+    assert!(dir.join("hello.txt").exists());
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 2: read_and_edit_file
+// ---------------------------------------------------------------------------
+async fn scenario_read_and_edit_file(session: &mut Session, dir: &Path) {
+    std::fs::write(dir.join("data.txt"), "old content").expect("failed to write data.txt");
+    session
+        .process_input("Read data.txt and replace its content with 'new content'")
+        .await
+        .expect("process_input failed");
+    let content = std::fs::read_to_string(dir.join("data.txt")).expect("failed to read data.txt");
+    assert!(
+        content.contains("new content"),
+        "Expected 'new content' in file, got: {content}"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 3: multi_file_edit
+// ---------------------------------------------------------------------------
+async fn scenario_multi_file_edit(session: &mut Session, dir: &Path) {
+    std::fs::write(dir.join("a.txt"), "aaa").expect("failed to write a.txt");
+    std::fs::write(dir.join("b.txt"), "bbb").expect("failed to write b.txt");
+    session
+        .process_input(
+            "Read a.txt and b.txt, then replace the content of a.txt with 'AAA' and b.txt with 'BBB'",
+        )
+        .await
+        .expect("process_input failed");
+    let a = std::fs::read_to_string(dir.join("a.txt")).expect("failed to read a.txt");
+    let b = std::fs::read_to_string(dir.join("b.txt")).expect("failed to read b.txt");
+    assert!(
+        a.contains("AAA"),
+        "Expected 'AAA' in a.txt, got: {a}"
+    );
+    assert!(
+        b.contains("BBB"),
+        "Expected 'BBB' in b.txt, got: {b}"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 4: shell_execution
+// ---------------------------------------------------------------------------
+async fn scenario_shell_execution(session: &mut Session, _dir: &Path) {
+    session
+        .process_input(
+            "Run the command `echo hello_from_shell` in the shell and tell me what it printed",
+        )
+        .await
+        .expect("process_input failed");
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 5: shell_timeout
+// ---------------------------------------------------------------------------
+async fn scenario_shell_timeout(session: &mut Session, _dir: &Path) {
+    session
+        .process_input("Run the command `sleep 999` with a 1-second timeout")
+        .await
+        .expect("process_input failed");
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 6: grep_and_glob
+// ---------------------------------------------------------------------------
+async fn scenario_grep_and_glob(session: &mut Session, dir: &Path) {
+    std::fs::write(dir.join("target.txt"), "needle_pattern_xyz")
+        .expect("failed to write target.txt");
+    std::fs::write(dir.join("other.txt"), "nothing").expect("failed to write other.txt");
+    session
+        .process_input(
+            "Search for files containing 'needle_pattern_xyz' and tell me which file has it",
+        )
+        .await
+        .expect("process_input failed");
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 7: multi_step_read_analyze_edit
+// ---------------------------------------------------------------------------
+async fn scenario_multi_step_read_analyze_edit(session: &mut Session, dir: &Path) {
+    std::fs::write(
+        dir.join("buggy.rs"),
+        "fn add(a: i32, b: i32) -> i32 { a - b }",
+    )
+    .expect("failed to write buggy.rs");
+    session
+        .process_input("Read buggy.rs, find the bug, and fix it")
+        .await
+        .expect("process_input failed");
+    let content = std::fs::read_to_string(dir.join("buggy.rs")).expect("failed to read buggy.rs");
+    assert!(
+        content.contains("a + b"),
+        "Expected 'a + b' in buggy.rs, got: {content}"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 8: tool_output_truncation
+// ---------------------------------------------------------------------------
+async fn scenario_tool_output_truncation(session: &mut Session, dir: &Path) {
+    let lines: String = (1..=10_000)
+        .map(|n| format!("line {n}\n"))
+        .collect();
+    std::fs::write(dir.join("big.txt"), lines).expect("failed to write big.txt");
+    session
+        .process_input("Read the file big.txt and tell me how many lines it has")
+        .await
+        .expect("process_input failed");
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 9: parallel_tool_calls
+// ---------------------------------------------------------------------------
+async fn scenario_parallel_tool_calls(session: &mut Session, dir: &Path) {
+    std::fs::write(dir.join("one.txt"), "content_one").expect("failed to write one.txt");
+    std::fs::write(dir.join("two.txt"), "content_two").expect("failed to write two.txt");
+    std::fs::write(dir.join("three.txt"), "content_three").expect("failed to write three.txt");
+    session
+        .process_input("Read one.txt, two.txt, and three.txt and tell me what each contains")
+        .await
+        .expect("process_input failed");
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 10: steering
+// ---------------------------------------------------------------------------
+async fn scenario_steering(session: &mut Session, _dir: &Path) {
+    session.steer("Stop counting and just say DONE".to_string());
+    session
+        .process_input("Count from 1 to 100, one number per line")
+        .await
+        .expect("process_input failed");
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 11: reasoning_effort
+// ---------------------------------------------------------------------------
+macro_rules! reasoning_effort_tests {
+    ($provider:expr, $model:expr, $test_name:ident) => {
+        #[tokio::test]
+        #[ignore = "requires LLM API keys"]
+        async fn $test_name() {
+            let tmp = tempfile::tempdir().expect("failed to create tempdir");
+            let config = SessionConfig {
+                max_turns: 20,
+                reasoning_effort: Some("low".to_string()),
+                ..SessionConfig::default()
+            };
+            let mut session =
+                make_session_with_config($provider, $model, tmp.path(), config).await;
+            session.initialize().await;
+            session
+                .process_input("Say hello")
+                .await
+                .expect("process_input failed");
+        }
+    };
+}
+
+reasoning_effort_tests!("anthropic", "claude-haiku-4-5-20251001", anthropic_reasoning_effort);
+// gpt-4o-mini does not support the reasoning.effort parameter, so no OpenAI test.
+reasoning_effort_tests!("gemini", "gemini-2.5-flash", gemini_reasoning_effort);
+
+// ---------------------------------------------------------------------------
+// Scenario 12: subagent_spawn
+// ---------------------------------------------------------------------------
+async fn scenario_subagent_spawn(session: &mut Session, _dir: &Path) {
+    session
+        .process_input(
+            "Try to spawn a subagent to read a file. If the subagent tool is not available, just say 'no subagent tool'",
+        )
+        .await
+        .expect("process_input failed");
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 13: loop_detection
+// ---------------------------------------------------------------------------
+macro_rules! loop_detection_tests {
+    ($provider:expr, $model:expr, $test_name:ident) => {
+        #[tokio::test]
+        #[ignore = "requires LLM API keys"]
+        async fn $test_name() {
+            let tmp = tempfile::tempdir().expect("failed to create tempdir");
+            let config = SessionConfig {
+                max_turns: 20,
+                loop_detection_window: 3,
+                ..SessionConfig::default()
+            };
+            let mut session =
+                make_session_with_config($provider, $model, tmp.path(), config).await;
+            session.initialize().await;
+            session
+                .process_input("Repeatedly read the file /dev/null")
+                .await
+                .expect("process_input failed");
+        }
+    };
+}
+
+loop_detection_tests!("anthropic", "claude-haiku-4-5-20251001", anthropic_loop_detection);
+loop_detection_tests!("openai", "gpt-4o-mini", openai_loop_detection);
+loop_detection_tests!("gemini", "gemini-2.5-flash", gemini_loop_detection);
+
+// ---------------------------------------------------------------------------
+// Scenario 14: error_recovery
+// ---------------------------------------------------------------------------
+async fn scenario_error_recovery(session: &mut Session, dir: &Path) {
+    session
+        .process_input(
+            "Try to read a file called nonexistent_file.txt. If it doesn't exist, create it with the content 'recovered'",
+        )
+        .await
+        .expect("process_input failed");
+    let path = dir.join("nonexistent_file.txt");
+    assert!(path.exists(), "nonexistent_file.txt should have been created");
+    let content =
+        std::fs::read_to_string(&path).expect("failed to read nonexistent_file.txt");
+    assert!(
+        content.contains("recovered"),
+        "Expected 'recovered' in file, got: {content}"
+    );
+}
+
+// ---------------------------------------------------------------------------
+// Scenario 15: provider_specific_editing
+// ---------------------------------------------------------------------------
+async fn scenario_provider_specific_editing(session: &mut Session, dir: &Path) {
+    std::fs::write(
+        dir.join("target.rs"),
+        "fn greet() { println!(\"hello\"); }",
+    )
+    .expect("failed to write target.rs");
+    session
+        .process_input("Edit target.rs to change 'hello' to 'goodbye'")
+        .await
+        .expect("process_input failed");
+    let content =
+        std::fs::read_to_string(dir.join("target.rs")).expect("failed to read target.rs");
+    assert!(
+        content.contains("goodbye"),
+        "Expected 'goodbye' in target.rs, got: {content}"
+    );
+}

crates/attractor/tests/integration.rs

@@ -24,7 +24,10 @@ use attractor::outcome::{Outcome, StageStatus};
 use attractor::parser::parse;
 use attractor::stylesheet::{apply_stylesheet, parse_stylesheet};
 use attractor::transform::{StylesheetApplicationTransform, Transform, VariableExpansionTransform};
+use attractor::cli::backend::AgentBackend;
+use attractor::handler::default_registry;
 use attractor::validation::{validate, validate_or_raise, Severity};
+use terminal::Styles;
 
 // ---------------------------------------------------------------------------
 // 1. Parse and validate all 3 spec examples (Section 2.13)
@@ -6457,4 +6460,94 @@ fn parse_tool_hooks_from_dot_syntax() {
         work.attrs.get("tool_hooks.pre").and_then(|v| v.as_str()),
         Some("node pre")
     );
+}
+
+// ---------------------------------------------------------------------------
+// E2E test with real LLM
+// ---------------------------------------------------------------------------
+
+static TEST_STYLES: Styles = Styles::new(false);
+
+#[tokio::test]
+#[ignore = "requires ANTHROPIC_API_KEY"]
+async fn attractor_e2e_with_real_llm() {
+    dotenvy::dotenv().ok();
+
+    let dir = tempfile::tempdir().unwrap();
+    let dir_path = dir.path().to_str().unwrap().to_string();
+
+    let dot = format!(
+        r#"digraph E2E {{
+            graph [goal="Create a test file"]
+            start [shape=Mdiamond]
+            exit  [shape=Msquare]
+            work  [
+                shape=box,
+                label="Work",
+                prompt="Create a file called hello.txt in {dir_path} containing exactly 'Hello from LLM'. Do not output anything else.",
+                goal_gate=true
+            ]
+            start -> work -> exit
+        }}"#
+    );
+
+    let graph = parse(&dot).expect("parse should succeed");
+    validate_or_raise(&graph, &[]).expect("validation should pass");
+
+    let interviewer: Arc<dyn Interviewer> = Arc::new(AutoApproveInterviewer);
+    let model = "claude-haiku-4-5-20251001".to_string();
+
+    let registry = default_registry(interviewer, move || {
+        Some(Box::new(AgentBackend::new(
+            model.clone(),
+            None,
+            0,
+            &TEST_STYLES,
+            false,
+        )) as Box<dyn attractor::handler::codergen::CodergenBackend>)
+    });
+
+    let logs_dir = tempfile::tempdir().unwrap();
+    let engine = PipelineEngine::new(registry, EventEmitter::new());
+    let config = RunConfig {
+        logs_root: logs_dir.path().to_path_buf(),
+        cancel_token: None,
+    };
+
+    let outcome = engine.run(&graph, &config).await.expect("run should succeed");
+
+    // 1. Pipeline completed successfully
+    assert_eq!(outcome.status, StageStatus::Success);
+
+    // 2. Artifacts exist
+    let work_dir = logs_dir.path().join("work");
+    assert!(work_dir.join("prompt.md").exists(), "prompt.md should exist");
+    assert!(
+        work_dir.join("response.md").exists(),
+        "response.md should exist"
+    );
+    assert!(
+        work_dir.join("status.json").exists(),
+        "status.json should exist"
+    );
+
+    // 3. Goal gate: check checkpoint node outcomes
+    let checkpoint = Checkpoint::load(&logs_dir.path().join("checkpoint.json"))
+        .expect("checkpoint should load");
+    let work_outcome = checkpoint
+        .node_outcomes
+        .get("work")
+        .expect("work outcome should exist");
+    assert!(
+        work_outcome.status == StageStatus::Success
+            || work_outcome.status == StageStatus::PartialSuccess,
+        "work goal gate should be Success or PartialSuccess, got {:?}",
+        work_outcome.status
+    );
+
+    // 4. Checkpoint: completed_nodes contains "work"
+    assert!(
+        checkpoint.completed_nodes.contains(&"work".to_string()),
+        "completed_nodes should contain 'work'"
+    );
 }

crates/llm/tests/integration.rs

@@ -67,3 +67,98 @@ async fn gemini_complete() {
     assert!(response.usage.output_tokens > 0);
     assert_eq!(response.provider, "gemini");
 }
+
+async fn run_multi_turn_cache_test(
+    adapter: &dyn ProviderAdapter,
+    model: &str,
+    min_cache_ratio: f64,
+) {
+    // Claude Haiku 4.5 requires 4096 tokens minimum for prompt caching.
+    // Each repeat is ~78 tokens; 70 repeats ≈ 5460 tokens, safely above the threshold.
+    let padding = "This is a detailed context paragraph that provides background information \
+        about the conversation. It contains various facts and details that the model should \
+        remember throughout the multi-turn interaction. The purpose of this padding is to \
+        ensure the system prompt exceeds the minimum cache threshold for the provider. \
+        We include information about mathematics, science, history, and general knowledge. \
+        The model should use this context when answering questions. "
+        .repeat(70);
+
+    let system_message = Message::system(format!(
+        "You are a helpful math assistant. Answer briefly.\n\n{padding}"
+    ));
+
+    let questions = [
+        "What is 1+1?",
+        "What is 2+2?",
+        "What is 3+3?",
+        "What is 4+4?",
+        "What is 5+5?",
+        "What is 6+6?",
+    ];
+
+    let mut messages = vec![system_message, Message::user(questions[0])];
+
+    for turn in 0..6 {
+        let request = Request {
+            model: model.to_string(),
+            messages: messages.clone(),
+            provider: None,
+            tools: None,
+            tool_choice: None,
+            response_format: None,
+            temperature: Some(0.0),
+            top_p: None,
+            max_tokens: Some(100),
+            stop_sequences: None,
+            reasoning_effort: None,
+            metadata: None,
+            provider_options: None,
+        };
+
+        let response = adapter.complete(&request).await.unwrap();
+        let text = response.text();
+        assert!(!text.is_empty(), "response text should not be empty on turn {turn}");
+
+        if turn == 5 {
+            let cache_read = response.usage.cache_read_tokens.unwrap_or(0) as f64;
+            let input = response.usage.input_tokens as f64;
+            let ratio = cache_read / input;
+            assert!(
+                ratio >= min_cache_ratio,
+                "cache ratio {ratio:.3} should be at least {min_cache_ratio} on final turn"
+            );
+        }
+
+        messages.push(Message::assistant(text));
+        if turn < 5 {
+            messages.push(Message::user(questions[turn + 1]));
+        }
+    }
+}
+
+#[tokio::test]
+#[ignore = "requires ANTHROPIC_API_KEY"]
+async fn anthropic_multi_turn_cache() {
+    dotenvy::dotenv().ok();
+    let api_key = std::env::var("ANTHROPIC_API_KEY").expect("ANTHROPIC_API_KEY must be set");
+    let adapter = AnthropicAdapter::new(api_key);
+    run_multi_turn_cache_test(&adapter, "claude-haiku-4-5-20251001", 0.5).await;
+}
+
+#[tokio::test]
+#[ignore = "requires OPENAI_API_KEY"]
+async fn openai_multi_turn_cache() {
+    dotenvy::dotenv().ok();
+    let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY must be set");
+    let adapter = OpenAiAdapter::new(api_key);
+    run_multi_turn_cache_test(&adapter, "gpt-4o-mini", 0.0).await;
+}
+
+#[tokio::test]
+#[ignore = "requires GEMINI_API_KEY"]
+async fn gemini_multi_turn_cache() {
+    dotenvy::dotenv().ok();
+    let api_key = std::env::var("GEMINI_API_KEY").expect("GEMINI_API_KEY must be set");
+    let adapter = GeminiAdapter::new(api_key);
+    run_multi_turn_cache_test(&adapter, "gemini-2.5-flash", 0.0).await;
+}