mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-06 08:18:58 +00:00
Optimize slow Docker and real CLI tests
- Move real_cli_claude/codex/gemini tests from fabro-workflow to fabro-cli, which has a 20s nextest timeout (vs 6s default), and add poll_interval(10ms) - Reduce DockerSandbox stop_container grace period from 5s to 1s - Reduce timeout_handling test sleep from 60s to 2s Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
ddd7550408
commit
f0ebf341db
4 changed files with 101 additions and 94 deletions
|
|
@ -5,6 +5,7 @@ mod conditional_branching;
|
|||
mod dry_run_examples;
|
||||
mod full_stack;
|
||||
mod human_gate;
|
||||
mod real_cli;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
|
|
|||
97
lib/crates/fabro-cli/tests/it/workflow/real_cli.rs
Normal file
97
lib/crates/fabro-cli/tests/it/workflow/real_cli.rs
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use fabro_graphviz::graph::{AttrValue, Node};
|
||||
use fabro_llm::provider::Provider;
|
||||
use fabro_workflow::context::Context;
|
||||
use fabro_workflow::event::EventEmitter;
|
||||
use fabro_workflow::handler::agent::{CodergenBackend, CodergenResult};
|
||||
use fabro_workflow::handler::llm::cli::AgentCliBackend;
|
||||
|
||||
/// Run a real CLI tool via LocalSandbox and verify the full flow.
|
||||
async fn run_real_cli_test(provider: Provider, model: &str) {
|
||||
let workspace = tempfile::tempdir().unwrap();
|
||||
let env: Arc<dyn fabro_agent::Sandbox> = Arc::new(fabro_agent::LocalSandbox::new(
|
||||
workspace.path().to_path_buf(),
|
||||
));
|
||||
let backend = AgentCliBackend::new(model.to_string(), provider)
|
||||
.with_poll_interval(Duration::from_millis(10));
|
||||
|
||||
let mut node = Node::new("real_cli_test");
|
||||
node.attrs.insert(
|
||||
"prompt".to_string(),
|
||||
AttrValue::String("What is 2+2? Reply with just the number.".to_string()),
|
||||
);
|
||||
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::default());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = backend
|
||||
.run(
|
||||
&node,
|
||||
"What is 2+2? Reply with just the number.",
|
||||
&context,
|
||||
None,
|
||||
&emitter,
|
||||
dir.path(),
|
||||
&env,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("CLI backend ({provider}/{model}) should succeed"));
|
||||
|
||||
match result {
|
||||
CodergenResult::Text { text, usage, .. } => {
|
||||
assert!(
|
||||
text.contains('4'),
|
||||
"{provider}/{model}: expected response to contain '4', got: {text}"
|
||||
);
|
||||
let usage = usage.unwrap_or_else(|| panic!("{provider}/{model}: should have usage"));
|
||||
assert!(
|
||||
usage.input_tokens > 0,
|
||||
"{provider}/{model}: input_tokens should be > 0, got {}",
|
||||
usage.input_tokens
|
||||
);
|
||||
}
|
||||
CodergenResult::Full(_) => panic!("expected Text result from {provider}/{model}"),
|
||||
}
|
||||
|
||||
// Verify log files were written
|
||||
let provider_path = dir.path().join("provider_used.json");
|
||||
assert!(
|
||||
provider_path.exists(),
|
||||
"{provider}/{model}: provider_used.json should exist"
|
||||
);
|
||||
let provider_json: serde_json::Value =
|
||||
serde_json::from_str(&std::fs::read_to_string(&provider_path).unwrap()).unwrap();
|
||||
assert_eq!(provider_json["mode"], "cli");
|
||||
assert_eq!(provider_json["provider"], provider.as_str());
|
||||
|
||||
// Verify CLI output was streamed to stage_dir during poll
|
||||
let stdout_log = dir.path().join("cli_stdout.log");
|
||||
assert!(
|
||||
stdout_log.exists(),
|
||||
"{provider}/{model}: cli_stdout.log should be written during poll"
|
||||
);
|
||||
let stdout_content = std::fs::read_to_string(&stdout_log).unwrap();
|
||||
assert!(
|
||||
!stdout_content.is_empty(),
|
||||
"{provider}/{model}: cli_stdout.log should not be empty"
|
||||
);
|
||||
}
|
||||
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn real_cli_claude() {
|
||||
run_real_cli_test(Provider::Anthropic, "haiku").await;
|
||||
}
|
||||
|
||||
#[fabro_macros::e2e_test(live("OPENAI_API_KEY"))]
|
||||
async fn real_cli_codex() {
|
||||
run_real_cli_test(Provider::OpenAi, "").await;
|
||||
}
|
||||
|
||||
#[fabro_macros::e2e_test(live("GEMINI_API_KEY"))]
|
||||
async fn real_cli_gemini() {
|
||||
run_real_cli_test(Provider::Gemini, "gemini-2.5-flash").await;
|
||||
}
|
||||
|
|
@ -455,8 +455,8 @@ impl Sandbox for DockerSandbox {
|
|||
return Ok(());
|
||||
};
|
||||
|
||||
// Stop with 5-second grace period; ignore "not running" errors
|
||||
let stop_opts = StopContainerOptions { t: 5 };
|
||||
// Stop with 1-second grace period; ignore "not running" errors
|
||||
let stop_opts = StopContainerOptions { t: 1 };
|
||||
let _ = self
|
||||
.docker
|
||||
.stop_container(&container_id, Some(stop_opts))
|
||||
|
|
@ -877,7 +877,7 @@ mod tests {
|
|||
env.initialize().await.unwrap();
|
||||
|
||||
let result = env
|
||||
.exec_command("sleep 60", 1000, None, None, None)
|
||||
.exec_command("sleep 2", 1000, None, None, None)
|
||||
.await
|
||||
.unwrap();
|
||||
assert!(result.timed_out);
|
||||
|
|
|
|||
|
|
@ -10067,97 +10067,6 @@ async fn stylesheet_backend_property_routes_to_cli() {
|
|||
);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Real CLI backend e2e tests (require actual CLI tools installed)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Run a real CLI tool via LocalSandbox and verify the full flow.
|
||||
async fn run_real_cli_test(provider: Provider, model: &str) {
|
||||
let env = local_env();
|
||||
let backend = AgentCliBackend::new(model.to_string(), provider);
|
||||
|
||||
let mut node = Node::new("real_cli_test");
|
||||
node.attrs.insert(
|
||||
"prompt".to_string(),
|
||||
AttrValue::String("What is 2+2? Reply with just the number.".to_string()),
|
||||
);
|
||||
|
||||
let context = Context::new();
|
||||
let emitter = Arc::new(EventEmitter::default());
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
|
||||
let result = backend
|
||||
.run(
|
||||
&node,
|
||||
"What is 2+2? Reply with just the number.",
|
||||
&context,
|
||||
None,
|
||||
&emitter,
|
||||
dir.path(),
|
||||
&env,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap_or_else(|_| panic!("CLI backend ({provider}/{model}) should succeed"));
|
||||
|
||||
match result {
|
||||
CodergenResult::Text { text, usage, .. } => {
|
||||
assert!(
|
||||
text.contains('4'),
|
||||
"{provider}/{model}: expected response to contain '4', got: {text}"
|
||||
);
|
||||
let usage = usage.unwrap_or_else(|| panic!("{provider}/{model}: should have usage"));
|
||||
assert!(
|
||||
usage.input_tokens > 0,
|
||||
"{provider}/{model}: input_tokens should be > 0, got {}",
|
||||
usage.input_tokens
|
||||
);
|
||||
}
|
||||
CodergenResult::Full(_) => panic!("expected Text result from {provider}/{model}"),
|
||||
}
|
||||
|
||||
// Verify log files were written
|
||||
let provider_path = dir.path().join("provider_used.json");
|
||||
assert!(
|
||||
provider_path.exists(),
|
||||
"{provider}/{model}: provider_used.json should exist"
|
||||
);
|
||||
let provider_json: serde_json::Value =
|
||||
serde_json::from_str(&std::fs::read_to_string(&provider_path).unwrap()).unwrap();
|
||||
assert_eq!(provider_json["mode"], "cli");
|
||||
assert_eq!(provider_json["provider"], provider.as_str());
|
||||
|
||||
// Verify CLI output was streamed to stage_dir during poll
|
||||
let stdout_log = dir.path().join("cli_stdout.log");
|
||||
assert!(
|
||||
stdout_log.exists(),
|
||||
"{provider}/{model}: cli_stdout.log should be written during poll"
|
||||
);
|
||||
let stdout_content = std::fs::read_to_string(&stdout_log).unwrap();
|
||||
assert!(
|
||||
!stdout_content.is_empty(),
|
||||
"{provider}/{model}: cli_stdout.log should not be empty"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires `claude` CLI installed
|
||||
async fn real_cli_claude() {
|
||||
run_real_cli_test(Provider::Anthropic, "haiku").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires `codex` CLI installed and OpenAI auth
|
||||
async fn real_cli_codex() {
|
||||
run_real_cli_test(Provider::OpenAi, "").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires `gemini` CLI installed and Google auth
|
||||
async fn real_cli_gemini() {
|
||||
run_real_cli_test(Provider::Gemini, "gemini-2.5-flash").await;
|
||||
}
|
||||
|
||||
/// Verify parse_cli_response works against real Claude CLI output captured from stream-json.
|
||||
#[test]
|
||||
fn parse_real_claude_stream_json() {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue