mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-10-07 03:00:29 +00:00
ca56c15f2f
Let callers decide whether to wrap in Arc. Also consolidates the two state() fetches in build_pr_body into one. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1025 lines
37 KiB
Rust
1025 lines
37 KiB
Rust
#![expect(
|
|
clippy::disallowed_methods,
|
|
reason = "agent parity test harness: sync std::fs for staging fixture trees and reading captured outputs"
|
|
)]
|
|
|
|
use std::collections::HashMap;
|
|
use std::fmt::Write as _;
|
|
use std::path::Path;
|
|
use std::sync::Arc;
|
|
|
|
use fabro_agent::subagent::SessionFactory;
|
|
use fabro_agent::{
|
|
AgentProfile, AnthropicProfile, GeminiProfile, LocalSandbox, OpenAiProfile, Session,
|
|
SessionOptions, SubAgentManager, WebFetchSummarizer,
|
|
};
|
|
use fabro_auth::EnvCredentialSource;
|
|
use fabro_llm::client::Client;
|
|
use fabro_llm::provider::{Provider, ProviderAdapter};
|
|
use fabro_llm::providers::OpenAiAdapter;
|
|
use fabro_model::ModelHandle;
|
|
use fabro_test::{TwinScenario, TwinScenarios, TwinToolCall, twin_openai};
|
|
use tokio::sync::Mutex as AsyncMutex;
|
|
|
|
#[derive(Clone)]
|
|
struct OpenAiTwinOptions {
|
|
base_url: String,
|
|
api_key: String,
|
|
}
|
|
|
|
fn summarizer_model_id(provider: Provider) -> ModelHandle {
|
|
match provider {
|
|
Provider::OpenAi
|
|
| Provider::Kimi
|
|
| Provider::Zai
|
|
| Provider::Minimax
|
|
| Provider::Inception
|
|
| Provider::OpenAiCompatible => ModelHandle::ByName {
|
|
provider: Provider::OpenAi,
|
|
model: "gpt-5.4-mini".to_string(),
|
|
},
|
|
Provider::Gemini => ModelHandle::ByName {
|
|
provider: Provider::Gemini,
|
|
model: "gemini-3-flash-preview".to_string(),
|
|
},
|
|
Provider::Anthropic => ModelHandle::ByName {
|
|
provider: Provider::Anthropic,
|
|
model: "claude-haiku-4-5".to_string(),
|
|
},
|
|
}
|
|
}
|
|
|
|
fn build_summarizer(provider: Provider, client: &Client) -> WebFetchSummarizer {
|
|
WebFetchSummarizer {
|
|
client: client.clone(),
|
|
model_id: summarizer_model_id(provider),
|
|
}
|
|
}
|
|
|
|
fn build_profile(provider: Provider, model: &str, client: &Client) -> Box<dyn AgentProfile> {
|
|
let summarizer = Some(build_summarizer(provider, client));
|
|
match provider {
|
|
Provider::Anthropic => Box::new(AnthropicProfile::with_summarizer(model, summarizer)),
|
|
Provider::OpenAi => Box::new(OpenAiProfile::with_summarizer(model, summarizer)),
|
|
Provider::Kimi
|
|
| Provider::Zai
|
|
| Provider::Minimax
|
|
| Provider::Inception
|
|
| Provider::OpenAiCompatible => {
|
|
Box::new(OpenAiProfile::with_summarizer(model, summarizer).with_provider(provider))
|
|
}
|
|
Provider::Gemini => Box::new(GeminiProfile::with_summarizer(model, summarizer)),
|
|
}
|
|
}
|
|
|
|
async fn make_session(
|
|
provider: Provider,
|
|
model: &str,
|
|
cwd: &Path,
|
|
twin: Option<OpenAiTwinOptions>,
|
|
) -> Session {
|
|
let client = make_client(provider, twin.as_ref()).await;
|
|
let mut profile = build_profile(provider, model, &client);
|
|
let env = Arc::new(LocalSandbox::new(cwd.to_path_buf()));
|
|
|
|
// Register subagent tools so spawn_agent / wait / send_input / close_agent are
|
|
// available
|
|
let manager = Arc::new(AsyncMutex::new(SubAgentManager::new(3)));
|
|
let factory_client = client.clone();
|
|
let factory_model: String = model.to_string();
|
|
let factory_cwd = cwd.to_path_buf();
|
|
let factory: SessionFactory = Arc::new(move || {
|
|
let sub_profile: Arc<dyn AgentProfile> = {
|
|
let summarizer = Some(build_summarizer(provider, &factory_client));
|
|
match provider {
|
|
Provider::Anthropic => Arc::new(AnthropicProfile::with_summarizer(
|
|
&factory_model,
|
|
summarizer,
|
|
)),
|
|
Provider::OpenAi => {
|
|
Arc::new(OpenAiProfile::with_summarizer(&factory_model, summarizer))
|
|
}
|
|
Provider::Kimi
|
|
| Provider::Zai
|
|
| Provider::Minimax
|
|
| Provider::Inception
|
|
| Provider::OpenAiCompatible => Arc::new(
|
|
OpenAiProfile::with_summarizer(&factory_model, summarizer)
|
|
.with_provider(provider),
|
|
),
|
|
Provider::Gemini => {
|
|
Arc::new(GeminiProfile::with_summarizer(&factory_model, summarizer))
|
|
}
|
|
}
|
|
};
|
|
let sub_env = Arc::new(LocalSandbox::new(factory_cwd.clone()));
|
|
Session::new(
|
|
factory_client.clone(),
|
|
sub_profile,
|
|
sub_env,
|
|
SessionOptions::default(),
|
|
None,
|
|
)
|
|
});
|
|
profile.register_subagent_tools(manager, factory, 0);
|
|
|
|
let profile: Arc<dyn AgentProfile> = Arc::from(profile);
|
|
let config = SessionOptions {
|
|
max_turns: 20,
|
|
..SessionOptions::default()
|
|
};
|
|
Session::new(client, profile, env, config, None)
|
|
}
|
|
|
|
async fn make_session_with_config(
|
|
provider: Provider,
|
|
model: &str,
|
|
cwd: &Path,
|
|
config: SessionOptions,
|
|
twin: Option<OpenAiTwinOptions>,
|
|
) -> Session {
|
|
let client = make_client(provider, twin.as_ref()).await;
|
|
let profile: Arc<dyn AgentProfile> = Arc::from(build_profile(provider, model, &client));
|
|
let env = Arc::new(LocalSandbox::new(cwd.to_path_buf()));
|
|
Session::new(client, profile, env, config, None)
|
|
}
|
|
|
|
async fn make_client(provider: Provider, twin: Option<&OpenAiTwinOptions>) -> Client {
|
|
if provider == Provider::OpenAi && fabro_test::TestMode::from_env().is_twin() {
|
|
return make_twin_client(twin.expect("openai twin config should be provided"));
|
|
}
|
|
|
|
let source = EnvCredentialSource::new();
|
|
Client::from_source(&source)
|
|
.await
|
|
.expect("Client::from_source failed")
|
|
}
|
|
|
|
fn make_twin_client(twin: &OpenAiTwinOptions) -> Client {
|
|
let adapter: Arc<dyn ProviderAdapter> =
|
|
Arc::new(OpenAiAdapter::new(twin.api_key.clone()).with_base_url(twin.base_url.clone()));
|
|
let mut providers: HashMap<String, Arc<dyn ProviderAdapter>> = HashMap::new();
|
|
providers.insert("openai".to_string(), adapter);
|
|
Client::new(providers, Some("openai".to_string()), Vec::new())
|
|
}
|
|
|
|
macro_rules! provider_test {
|
|
($scenario:ident, $provider:expr, $model:expr, $prefix:ident, keys = [$($key:expr),+ $(,)?]) => {
|
|
paste::paste! {
|
|
#[fabro_macros::e2e_test($(live($key)),+)]
|
|
async fn [<$prefix _ $scenario>]() {
|
|
let tmp = tempfile::tempdir().expect("failed to create tempdir");
|
|
let mut session = make_session($provider, $model, tmp.path(), None).await;
|
|
session.initialize().await;
|
|
[<scenario_ $scenario>](&mut session, tmp.path()).await;
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
macro_rules! openai_twin_provider_test {
|
|
($scenario:ident) => {
|
|
paste::paste! {
|
|
#[fabro_macros::e2e_test(twin, live("OPENAI_API_KEY"))]
|
|
async fn [<openai_twin_ $scenario>]() {
|
|
let tmp = tempfile::tempdir().expect("failed to create tempdir");
|
|
let (base_url, api_key) = fabro_test::e2e_openai!();
|
|
let twin = OpenAiTwinOptions { base_url, api_key };
|
|
if fabro_test::TestMode::from_env().is_twin() {
|
|
load_openai_twin_scenario(stringify!($scenario), &twin.api_key, tmp.path())
|
|
.await;
|
|
}
|
|
let mut session = make_session(
|
|
Provider::OpenAi,
|
|
"gpt-5.4-mini",
|
|
tmp.path(),
|
|
Some(twin),
|
|
).await;
|
|
session.initialize().await;
|
|
[<scenario_ $scenario>](&mut session, tmp.path()).await;
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
macro_rules! provider_tests {
|
|
($scenario:ident) => {
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Anthropic,
|
|
"claude-haiku-4-5",
|
|
anthropic,
|
|
keys = ["ANTHROPIC_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Gemini,
|
|
"gemini-3-flash-preview",
|
|
gemini,
|
|
keys = ["GEMINI_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Kimi,
|
|
"kimi-k2.5",
|
|
kimi,
|
|
keys = ["KIMI_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Zai,
|
|
"glm-4.7",
|
|
zai,
|
|
keys = ["ZAI_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Minimax,
|
|
"minimax-m2.5",
|
|
minimax,
|
|
keys = ["MINIMAX_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Inception,
|
|
"mercury-2",
|
|
inception,
|
|
keys = ["INCEPTION_API_KEY"]
|
|
);
|
|
};
|
|
}
|
|
|
|
provider_tests!(simple_file_creation);
|
|
openai_twin_provider_test!(simple_file_creation);
|
|
provider_tests!(read_and_edit_file);
|
|
openai_twin_provider_test!(read_and_edit_file);
|
|
provider_tests!(multi_file_edit);
|
|
openai_twin_provider_test!(multi_file_edit);
|
|
provider_tests!(shell_execution);
|
|
openai_twin_provider_test!(shell_execution);
|
|
provider_tests!(shell_timeout);
|
|
openai_twin_provider_test!(shell_timeout);
|
|
provider_tests!(grep_and_glob);
|
|
openai_twin_provider_test!(grep_and_glob);
|
|
provider_tests!(tool_output_truncation);
|
|
openai_twin_provider_test!(tool_output_truncation);
|
|
provider_tests!(parallel_tool_calls);
|
|
openai_twin_provider_test!(parallel_tool_calls);
|
|
provider_tests!(steering_before_input);
|
|
openai_twin_provider_test!(steering_before_input);
|
|
provider_tests!(steering_mid_task);
|
|
provider_tests!(follow_up);
|
|
openai_twin_provider_test!(follow_up);
|
|
provider_tests!(subagent_spawn);
|
|
|
|
provider_test!(
|
|
web_fetch,
|
|
Provider::Anthropic,
|
|
"claude-haiku-4-5",
|
|
anthropic,
|
|
keys = ["ANTHROPIC_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
web_fetch,
|
|
Provider::OpenAi,
|
|
"gpt-5.4-mini",
|
|
openai,
|
|
keys = ["OPENAI_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
web_fetch,
|
|
Provider::Gemini,
|
|
"gemini-3-flash-preview",
|
|
gemini,
|
|
keys = ["GEMINI_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
web_fetch,
|
|
Provider::Kimi,
|
|
"kimi-k2.5",
|
|
kimi,
|
|
keys = ["KIMI_API_KEY", "OPENAI_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
provider_test!(
|
|
web_fetch,
|
|
Provider::Zai,
|
|
"glm-4.7",
|
|
zai,
|
|
keys = ["ZAI_API_KEY", "OPENAI_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
web_fetch,
|
|
Provider::Minimax,
|
|
"minimax-m2.5",
|
|
minimax,
|
|
keys = ["MINIMAX_API_KEY", "OPENAI_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
provider_test!(
|
|
web_fetch,
|
|
Provider::Inception,
|
|
"mercury-2",
|
|
inception,
|
|
keys = ["INCEPTION_API_KEY", "OPENAI_API_KEY"]
|
|
);
|
|
|
|
provider_test!(
|
|
web_search,
|
|
Provider::Anthropic,
|
|
"claude-haiku-4-5",
|
|
anthropic,
|
|
keys = ["ANTHROPIC_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
web_search,
|
|
Provider::OpenAi,
|
|
"gpt-5.4-mini",
|
|
openai,
|
|
keys = ["OPENAI_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
web_search,
|
|
Provider::Gemini,
|
|
"gemini-3-flash-preview",
|
|
gemini,
|
|
keys = ["GEMINI_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
web_search,
|
|
Provider::Kimi,
|
|
"kimi-k2.5",
|
|
kimi,
|
|
keys = ["KIMI_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
provider_test!(
|
|
web_search,
|
|
Provider::Zai,
|
|
"glm-4.7",
|
|
zai,
|
|
keys = ["ZAI_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
web_search,
|
|
Provider::Minimax,
|
|
"minimax-m2.5",
|
|
minimax,
|
|
keys = ["MINIMAX_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
provider_test!(
|
|
web_search,
|
|
Provider::Inception,
|
|
"mercury-2",
|
|
inception,
|
|
keys = ["INCEPTION_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
|
);
|
|
|
|
// Scenarios below are only generated for providers where they are supported.
|
|
// - multi_step_read_analyze_edit / provider_specific_editing: gpt-4o-mini is
|
|
// too weak to reliably apply precise file edits (uses apply_patch, not
|
|
// edit_file).
|
|
// - reasoning_effort: gpt-4o-mini doesn't support the reasoning.effort
|
|
// parameter.
|
|
// - loop_detection: needs custom config, tested separately below.
|
|
|
|
provider_tests!(error_recovery);
|
|
openai_twin_provider_test!(error_recovery);
|
|
|
|
// gpt-5-mini is too weak to reliably apply precise file edits (uses
|
|
// apply_patch, not edit_file).
|
|
macro_rules! non_openai_provider_tests {
|
|
($scenario:ident) => {
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Anthropic,
|
|
"claude-haiku-4-5",
|
|
anthropic,
|
|
keys = ["ANTHROPIC_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Gemini,
|
|
"gemini-3-flash-preview",
|
|
gemini,
|
|
keys = ["GEMINI_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Kimi,
|
|
"kimi-k2.5",
|
|
kimi,
|
|
keys = ["KIMI_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Zai,
|
|
"glm-4.7",
|
|
zai,
|
|
keys = ["ZAI_API_KEY"]
|
|
);
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Minimax,
|
|
"minimax-m2.5",
|
|
minimax,
|
|
keys = ["MINIMAX_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
provider_test!(
|
|
$scenario,
|
|
Provider::Inception,
|
|
"mercury-2",
|
|
inception,
|
|
keys = ["INCEPTION_API_KEY"]
|
|
);
|
|
};
|
|
}
|
|
|
|
non_openai_provider_tests!(multi_step_read_analyze_edit);
|
|
non_openai_provider_tests!(provider_specific_editing);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 1: simple_file_creation
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_simple_file_creation(session: &mut Session, dir: &Path) {
|
|
session
|
|
.process_input("Create a file called hello.txt containing 'Hello'")
|
|
.await
|
|
.expect("process_input failed");
|
|
assert!(dir.join("hello.txt").exists());
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 2: read_and_edit_file
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_read_and_edit_file(session: &mut Session, dir: &Path) {
|
|
std::fs::write(dir.join("data.txt"), "old content").expect("failed to write data.txt");
|
|
session
|
|
.process_input("Read data.txt and replace its content with 'new content'")
|
|
.await
|
|
.expect("process_input failed");
|
|
let content = std::fs::read_to_string(dir.join("data.txt")).expect("failed to read data.txt");
|
|
assert!(
|
|
content.contains("new content"),
|
|
"Expected 'new content' in file, got: {content}"
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 3: multi_file_edit
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_multi_file_edit(session: &mut Session, dir: &Path) {
|
|
std::fs::write(dir.join("a.txt"), "aaa").expect("failed to write a.txt");
|
|
std::fs::write(dir.join("b.txt"), "bbb").expect("failed to write b.txt");
|
|
session
|
|
.process_input(
|
|
"Read a.txt and b.txt, then replace the content of a.txt with 'AAA' and b.txt with 'BBB'",
|
|
)
|
|
.await
|
|
.expect("process_input failed");
|
|
let a = std::fs::read_to_string(dir.join("a.txt")).expect("failed to read a.txt");
|
|
let b = std::fs::read_to_string(dir.join("b.txt")).expect("failed to read b.txt");
|
|
assert!(a.contains("AAA"), "Expected 'AAA' in a.txt, got: {a}");
|
|
assert!(b.contains("BBB"), "Expected 'BBB' in b.txt, got: {b}");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 4: shell_execution
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_shell_execution(session: &mut Session, _dir: &Path) {
|
|
session
|
|
.process_input(
|
|
"Run the command `echo hello_from_shell` in the shell and tell me what it printed",
|
|
)
|
|
.await
|
|
.expect("process_input failed");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 5: shell_timeout
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_shell_timeout(session: &mut Session, _dir: &Path) {
|
|
session
|
|
.process_input("Run the command `sleep 999` with a 1-second timeout")
|
|
.await
|
|
.expect("process_input failed");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 6: grep_and_glob
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_grep_and_glob(session: &mut Session, dir: &Path) {
|
|
std::fs::write(dir.join("target.txt"), "needle_pattern_xyz")
|
|
.expect("failed to write target.txt");
|
|
std::fs::write(dir.join("other.txt"), "nothing").expect("failed to write other.txt");
|
|
session
|
|
.process_input(
|
|
"Search for files containing 'needle_pattern_xyz' and tell me which file has it",
|
|
)
|
|
.await
|
|
.expect("process_input failed");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 7: multi_step_read_analyze_edit
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_multi_step_read_analyze_edit(session: &mut Session, dir: &Path) {
|
|
std::fs::write(
|
|
dir.join("buggy.rs"),
|
|
"fn add(a: i32, b: i32) -> i32 { a - b }",
|
|
)
|
|
.expect("failed to write buggy.rs");
|
|
session
|
|
.process_input("Read buggy.rs, find the bug, and fix it")
|
|
.await
|
|
.expect("process_input failed");
|
|
let content = std::fs::read_to_string(dir.join("buggy.rs")).expect("failed to read buggy.rs");
|
|
assert!(
|
|
content.contains("a + b"),
|
|
"Expected 'a + b' in buggy.rs, got: {content}"
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 8: tool_output_truncation
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_tool_output_truncation(session: &mut Session, dir: &Path) {
|
|
let lines = (1..=10_000).fold(String::new(), |mut acc, n| {
|
|
let _ = writeln!(acc, "line {n}");
|
|
acc
|
|
});
|
|
std::fs::write(dir.join("big.txt"), lines).expect("failed to write big.txt");
|
|
session
|
|
.process_input("Read the file big.txt and tell me how many lines it has")
|
|
.await
|
|
.expect("process_input failed");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 9: parallel_tool_calls
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_parallel_tool_calls(session: &mut Session, dir: &Path) {
|
|
std::fs::write(dir.join("one.txt"), "content_one").expect("failed to write one.txt");
|
|
std::fs::write(dir.join("two.txt"), "content_two").expect("failed to write two.txt");
|
|
std::fs::write(dir.join("three.txt"), "content_three").expect("failed to write three.txt");
|
|
session
|
|
.process_input("Read one.txt, two.txt, and three.txt and tell me what each contains")
|
|
.await
|
|
.expect("process_input failed");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 10a: steering_before_input
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_steering_before_input(session: &mut Session, _dir: &Path) {
|
|
session.steer("Stop counting and just say DONE".to_string());
|
|
session
|
|
.process_input("Count from 1 to 100, one number per line")
|
|
.await
|
|
.expect("process_input failed");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 10b: steering_mid_task
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_steering_mid_task(session: &mut Session, dir: &Path) {
|
|
// Setup: create a file the LLM will read (triggering a tool call)
|
|
std::fs::write(dir.join("task.txt"), "read me first").expect("write task.txt");
|
|
|
|
// Grab handles before process_input borrows &mut self
|
|
let steering_queue = session.steering_queue_handle();
|
|
let mut rx = session.subscribe();
|
|
|
|
// Spawn a task that waits for the first tool call, then injects steering
|
|
let steer_task = tokio::spawn(async move {
|
|
while let Ok(event) = rx.recv().await {
|
|
if matches!(
|
|
event.event,
|
|
fabro_agent::AgentEvent::ToolCallCompleted { .. }
|
|
) {
|
|
steering_queue
|
|
.lock()
|
|
.expect("steering queue lock")
|
|
.push_back(
|
|
"Stop what you are doing. Create a file called steered.txt containing 'steered' and do nothing else.".to_string(),
|
|
);
|
|
break;
|
|
}
|
|
}
|
|
});
|
|
|
|
session
|
|
.process_input(
|
|
"Read task.txt, then create files a.txt, b.txt, c.txt, d.txt, e.txt each containing their letter",
|
|
)
|
|
.await
|
|
.expect("process_input failed");
|
|
|
|
steer_task.await.expect("steer task panicked");
|
|
|
|
// The steering message should have redirected the LLM to create steered.txt
|
|
assert!(
|
|
dir.join("steered.txt").exists(),
|
|
"steered.txt should exist — steering mid-task should have redirected the LLM"
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 10c: follow_up
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_follow_up(session: &mut Session, dir: &Path) {
|
|
session.follow_up("Create a file called second.txt containing 'second'".to_string());
|
|
session
|
|
.process_input("Create a file called first.txt containing 'first'")
|
|
.await
|
|
.expect("process_input failed");
|
|
|
|
let first = dir.join("first.txt");
|
|
let second = dir.join("second.txt");
|
|
assert!(first.exists(), "first.txt should exist");
|
|
assert!(second.exists(), "second.txt should exist");
|
|
let first_content = std::fs::read_to_string(&first).expect("read first.txt");
|
|
let second_content = std::fs::read_to_string(&second).expect("read second.txt");
|
|
assert!(
|
|
first_content.contains("first"),
|
|
"first.txt should contain 'first', got: {first_content}"
|
|
);
|
|
assert!(
|
|
second_content.contains("second"),
|
|
"second.txt should contain 'second', got: {second_content}"
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 11: reasoning_effort
|
|
// ---------------------------------------------------------------------------
|
|
macro_rules! reasoning_effort_tests {
|
|
($provider:expr, $model:expr, $test_name:ident, keys = [$($key:expr),+ $(,)?]) => {
|
|
#[fabro_macros::e2e_test($(live($key)),+)]
|
|
async fn $test_name() {
|
|
let tmp = tempfile::tempdir().expect("failed to create tempdir");
|
|
let config = SessionOptions {
|
|
max_turns: 20,
|
|
reasoning_effort: Some(fabro_llm::types::ReasoningEffort::Low),
|
|
..SessionOptions::default()
|
|
};
|
|
let mut session =
|
|
make_session_with_config($provider, $model, tmp.path(), config, None).await;
|
|
session.initialize().await;
|
|
session
|
|
.process_input("Say hello")
|
|
.await
|
|
.expect("process_input failed");
|
|
}
|
|
};
|
|
}
|
|
|
|
reasoning_effort_tests!(
|
|
Provider::Anthropic,
|
|
"claude-haiku-4-5",
|
|
anthropic_reasoning_effort,
|
|
keys = ["ANTHROPIC_API_KEY"]
|
|
);
|
|
// gpt-5-mini does not support the reasoning.effort parameter, so no OpenAI
|
|
// test.
|
|
reasoning_effort_tests!(
|
|
Provider::Gemini,
|
|
"gemini-3-flash-preview",
|
|
gemini_reasoning_effort,
|
|
keys = ["GEMINI_API_KEY"]
|
|
);
|
|
reasoning_effort_tests!(
|
|
Provider::Kimi,
|
|
"kimi-k2.5",
|
|
kimi_reasoning_effort,
|
|
keys = ["KIMI_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
reasoning_effort_tests!(
|
|
Provider::Zai,
|
|
"glm-4.7",
|
|
zai_reasoning_effort,
|
|
keys = ["ZAI_API_KEY"]
|
|
);
|
|
reasoning_effort_tests!(
|
|
Provider::Minimax,
|
|
"minimax-m2.5",
|
|
minimax_reasoning_effort,
|
|
keys = ["MINIMAX_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
reasoning_effort_tests!(
|
|
Provider::Inception,
|
|
"mercury-2",
|
|
inception_reasoning_effort,
|
|
keys = ["INCEPTION_API_KEY"]
|
|
);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 12: subagent_spawn
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_subagent_spawn(session: &mut Session, dir: &Path) {
|
|
std::fs::write(dir.join("secret.txt"), "the_secret_value").expect("failed to write secret.txt");
|
|
session
|
|
.process_input(
|
|
"Spawn a subagent to read the file secret.txt and report its contents. \
|
|
Wait for the subagent to finish, then tell me what it found.",
|
|
)
|
|
.await
|
|
.expect("process_input failed");
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 13: loop_detection
|
|
// ---------------------------------------------------------------------------
|
|
macro_rules! loop_detection_tests {
|
|
($provider:expr, $model:expr, $test_name:ident, keys = [$($key:expr),+ $(,)?]) => {
|
|
#[fabro_macros::e2e_test($(live($key)),+)]
|
|
async fn $test_name() {
|
|
let tmp = tempfile::tempdir().expect("failed to create tempdir");
|
|
let config = SessionOptions {
|
|
max_turns: 20,
|
|
loop_detection_window: 3,
|
|
..SessionOptions::default()
|
|
};
|
|
let mut session =
|
|
make_session_with_config($provider, $model, tmp.path(), config, None).await;
|
|
session.initialize().await;
|
|
session
|
|
.process_input("Repeatedly read the file /dev/null")
|
|
.await
|
|
.expect("process_input failed");
|
|
}
|
|
};
|
|
}
|
|
|
|
loop_detection_tests!(
|
|
Provider::Anthropic,
|
|
"claude-haiku-4-5",
|
|
anthropic_loop_detection,
|
|
keys = ["ANTHROPIC_API_KEY"]
|
|
);
|
|
loop_detection_tests!(
|
|
Provider::OpenAi,
|
|
"gpt-5.4-mini",
|
|
openai_loop_detection,
|
|
keys = ["OPENAI_API_KEY"]
|
|
);
|
|
loop_detection_tests!(
|
|
Provider::Gemini,
|
|
"gemini-3-flash-preview",
|
|
gemini_loop_detection,
|
|
keys = ["GEMINI_API_KEY"]
|
|
);
|
|
loop_detection_tests!(
|
|
Provider::Kimi,
|
|
"kimi-k2.5",
|
|
kimi_loop_detection,
|
|
keys = ["KIMI_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
loop_detection_tests!(
|
|
Provider::Zai,
|
|
"glm-4.7",
|
|
zai_loop_detection,
|
|
keys = ["ZAI_API_KEY"]
|
|
);
|
|
loop_detection_tests!(
|
|
Provider::Minimax,
|
|
"minimax-m2.5",
|
|
minimax_loop_detection,
|
|
keys = ["MINIMAX_API_KEY"]
|
|
);
|
|
#[cfg(feature = "quarantine")]
|
|
loop_detection_tests!(
|
|
Provider::Inception,
|
|
"mercury-2",
|
|
inception_loop_detection,
|
|
keys = ["INCEPTION_API_KEY"]
|
|
);
|
|
|
|
async fn load_openai_twin_scenario(name: &str, namespace: &str, cwd: &Path) {
|
|
let scenarios = match name {
|
|
"simple_file_creation" => TwinScenarios::new(namespace.to_string()).scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains("Create a file called hello.txt containing 'Hello'")
|
|
.tool_call(TwinToolCall::write_file("hello.txt", "Hello"))
|
|
.text("Done."),
|
|
),
|
|
"read_and_edit_file" => TwinScenarios::new(namespace.to_string())
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains("Read data.txt and replace its content with 'new content'")
|
|
.tool_call(TwinToolCall::read_file("data.txt")),
|
|
)
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.tool_call(TwinToolCall::write_file("data.txt", "new content"))
|
|
.text("Done."),
|
|
),
|
|
"multi_file_edit" => TwinScenarios::new(namespace.to_string())
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains(
|
|
"Read a.txt and b.txt, then replace the content of a.txt with 'AAA' and b.txt with 'BBB'",
|
|
)
|
|
.tool_calls(vec![
|
|
TwinToolCall::read_file("a.txt"),
|
|
TwinToolCall::read_file("b.txt"),
|
|
]),
|
|
)
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.tool_calls(vec![
|
|
TwinToolCall::write_file("a.txt", "AAA"),
|
|
TwinToolCall::write_file("b.txt", "BBB"),
|
|
])
|
|
.text("Done."),
|
|
),
|
|
"shell_execution" => TwinScenarios::new(namespace.to_string()).scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains(
|
|
"Run the command `echo hello_from_shell` in the shell and tell me what it printed",
|
|
)
|
|
.tool_call(TwinToolCall::shell("echo hello_from_shell"))
|
|
.text("It printed hello_from_shell."),
|
|
),
|
|
"shell_timeout" => TwinScenarios::new(namespace.to_string()).scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains("Run the command `sleep 999` with a 1-second timeout")
|
|
.tool_call(TwinToolCall::shell_with_timeout("sleep 999", 1000))
|
|
.text("The command timed out."),
|
|
),
|
|
"grep_and_glob" => TwinScenarios::new(namespace.to_string())
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains(
|
|
"Search for files containing 'needle_pattern_xyz' and tell me which file has it",
|
|
)
|
|
.tool_calls(vec![
|
|
TwinToolCall::glob_pattern("*.txt", cwd.display().to_string()),
|
|
TwinToolCall::grep_pattern("needle_pattern_xyz", "."),
|
|
]),
|
|
)
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.text("target.txt contains needle_pattern_xyz."),
|
|
),
|
|
"tool_output_truncation" => TwinScenarios::new(namespace.to_string()).scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains("Read the file big.txt and tell me how many lines it has")
|
|
.tool_call(TwinToolCall::read_file("big.txt"))
|
|
.text("The file has 10000 lines."),
|
|
),
|
|
"parallel_tool_calls" => TwinScenarios::new(namespace.to_string()).scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains("Read one.txt, two.txt, and three.txt and tell me what each contains")
|
|
.tool_calls(vec![
|
|
TwinToolCall::read_file("one.txt"),
|
|
TwinToolCall::read_file("two.txt"),
|
|
TwinToolCall::read_file("three.txt"),
|
|
])
|
|
.text("one: content_one, two: content_two, three: content_three"),
|
|
),
|
|
"steering_before_input" => TwinScenarios::new(namespace.to_string()).scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains("Count from 1 to 100, one number per line")
|
|
.text("DONE"),
|
|
),
|
|
"follow_up" => TwinScenarios::new(namespace.to_string())
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains("Create a file called first.txt containing 'first'")
|
|
.tool_call(TwinToolCall::write_file("first.txt", "first"))
|
|
.text("Created first.txt."),
|
|
)
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains("Create a file called second.txt containing 'second'")
|
|
.tool_call(TwinToolCall::write_file("second.txt", "second"))
|
|
.text("Created second.txt."),
|
|
),
|
|
"error_recovery" => TwinScenarios::new(namespace.to_string())
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains(
|
|
"Try to read a file called nonexistent_file.txt. If it doesn't exist, create it with the content 'recovered'",
|
|
)
|
|
.tool_call(TwinToolCall::read_file("nonexistent_file.txt")),
|
|
)
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.tool_call(TwinToolCall::write_file("nonexistent_file.txt", "recovered"))
|
|
.text("Created the file."),
|
|
),
|
|
other => panic!("missing openai twin scenario for {other}"),
|
|
};
|
|
|
|
scenarios.load(twin_openai().await).await;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 14: error_recovery
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_error_recovery(session: &mut Session, dir: &Path) {
|
|
session
|
|
.process_input(
|
|
"Try to read a file called nonexistent_file.txt. If it doesn't exist, create it with the content 'recovered'",
|
|
)
|
|
.await
|
|
.expect("process_input failed");
|
|
let path = dir.join("nonexistent_file.txt");
|
|
assert!(
|
|
path.exists(),
|
|
"nonexistent_file.txt should have been created"
|
|
);
|
|
let content = std::fs::read_to_string(&path).expect("failed to read nonexistent_file.txt");
|
|
assert!(
|
|
content.contains("recovered"),
|
|
"Expected 'recovered' in file, got: {content}"
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 15: web_fetch
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_web_fetch(session: &mut Session, dir: &Path) {
|
|
// Test basic fetch (HTML-to-markdown conversion)
|
|
session
|
|
.process_input(
|
|
"Use the web_fetch tool to fetch https://example.com and write its content to a file called fetched.txt",
|
|
)
|
|
.await
|
|
.expect("process_input failed");
|
|
let path = dir.join("fetched.txt");
|
|
assert!(path.exists(), "fetched.txt should have been created");
|
|
let content = std::fs::read_to_string(&path).expect("failed to read fetched.txt");
|
|
let lower = content.to_lowercase();
|
|
assert!(
|
|
lower.contains("example domain")
|
|
|| lower.contains("example.com")
|
|
|| lower.contains("example")
|
|
&& (lower.contains("documentation")
|
|
|| lower.contains("iana")
|
|
|| lower.contains("illustrative")),
|
|
"Expected content related to example.com, got first 200 chars: {}",
|
|
&content[..content.len().min(200)]
|
|
);
|
|
|
|
// Test fetch with prompt parameter (LLM summarization)
|
|
session
|
|
.process_input(
|
|
"Use the web_fetch tool with the prompt parameter to fetch https://example.com and answer: 'What is the title heading on this page?' Write only the answer to a file called answer.txt",
|
|
)
|
|
.await
|
|
.expect("process_input failed for prompt test");
|
|
let answer_path = dir.join("answer.txt");
|
|
assert!(answer_path.exists(), "answer.txt should have been created");
|
|
let answer = std::fs::read_to_string(&answer_path).expect("failed to read answer.txt");
|
|
assert!(
|
|
answer.to_lowercase().contains("example domain")
|
|
|| answer.to_lowercase().contains("example"),
|
|
"Expected answer to mention 'example domain' or 'example', got: {answer}"
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 16: web_search
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_web_search(session: &mut Session, dir: &Path) {
|
|
session
|
|
.process_input(
|
|
"Use the web_search tool to search for 'Rust programming language' and write the first result's title and URL to a file called search_results.txt",
|
|
)
|
|
.await
|
|
.expect("process_input failed");
|
|
let path = dir.join("search_results.txt");
|
|
assert!(path.exists(), "search_results.txt should have been created");
|
|
let content = std::fs::read_to_string(&path).expect("failed to read search_results.txt");
|
|
assert!(
|
|
!content.is_empty(),
|
|
"search_results.txt should not be empty"
|
|
);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Scenario 17: provider_specific_editing
|
|
// ---------------------------------------------------------------------------
|
|
async fn scenario_provider_specific_editing(session: &mut Session, dir: &Path) {
|
|
std::fs::write(dir.join("target.rs"), "fn greet() { println!(\"hello\"); }")
|
|
.expect("failed to write target.rs");
|
|
session
|
|
.process_input("Edit target.rs to change 'hello' to 'goodbye'")
|
|
.await
|
|
.expect("process_input failed");
|
|
let content = std::fs::read_to_string(dir.join("target.rs")).expect("failed to read target.rs");
|
|
assert!(
|
|
content.contains("goodbye"),
|
|
"Expected 'goodbye' in target.rs, got: {content}"
|
|
);
|
|
}
|