mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-10-08 03:10:26 +00:00
Centralize E2E test env var handling
This commit is contained in:
parent
7f2d840007
commit
fb6f0eae1e
27 changed files with 480 additions and 255 deletions
|
|
@ -107,3 +107,4 @@ Never run `cargo insta accept` without first checking what's pending — it acce
|
|||
|
||||
- `fabro run <name>` — run a workflow by name (resolves `fabro/workflows/<name>/workflow.toml`), e.g. `fabro run repl`
|
||||
- Use `--no-retro` to skip the retro step and finish faster
|
||||
- `#[e2e_test(live("VAR"))]` — use for tests requiring secrets; it generates `#[ignore]` plus env gating. Use `#[e2e_test()]` for sandbox-only E2E tests with no secret deps. Behavior is controlled by `FABRO_TEST_MODE` (`off`, `live`, `strict`), and `cargo nextest run --profile e2e ...` implies `strict`.
|
||||
|
|
|
|||
10
Cargo.lock
generated
10
Cargo.lock
generated
|
|
@ -1406,12 +1406,13 @@ dependencies = [
|
|||
"chrono",
|
||||
"clap",
|
||||
"dirs",
|
||||
"dotenvy",
|
||||
"fabro-config",
|
||||
"fabro-llm",
|
||||
"fabro-macros",
|
||||
"fabro-mcp",
|
||||
"fabro-model",
|
||||
"fabro-sandbox",
|
||||
"fabro-test",
|
||||
"fabro-util",
|
||||
"futures",
|
||||
"glob",
|
||||
|
|
@ -1488,6 +1489,7 @@ dependencies = [
|
|||
"fabro-hooks",
|
||||
"fabro-interview",
|
||||
"fabro-llm",
|
||||
"fabro-macros",
|
||||
"fabro-mcp",
|
||||
"fabro-model",
|
||||
"fabro-openai-oauth",
|
||||
|
|
@ -1668,8 +1670,9 @@ dependencies = [
|
|||
"clap",
|
||||
"cli-table",
|
||||
"dialoguer",
|
||||
"dotenvy",
|
||||
"fabro-macros",
|
||||
"fabro-model",
|
||||
"fabro-test",
|
||||
"fabro-util",
|
||||
"futures",
|
||||
"http",
|
||||
|
|
@ -1994,7 +1997,6 @@ dependencies = [
|
|||
"base64",
|
||||
"chrono",
|
||||
"dirs",
|
||||
"dotenvy",
|
||||
"fabro-agent",
|
||||
"fabro-checkpoint",
|
||||
"fabro-config",
|
||||
|
|
@ -2005,11 +2007,13 @@ dependencies = [
|
|||
"fabro-hooks",
|
||||
"fabro-interview",
|
||||
"fabro-llm",
|
||||
"fabro-macros",
|
||||
"fabro-mcp",
|
||||
"fabro-model",
|
||||
"fabro-retro",
|
||||
"fabro-sandbox",
|
||||
"fabro-store",
|
||||
"fabro-test",
|
||||
"fabro-types",
|
||||
"fabro-util",
|
||||
"fabro-validate",
|
||||
|
|
|
|||
|
|
@ -258,6 +258,19 @@ Tests should be stable on any developer machine.
|
|||
- Mark tests that require real providers, real sandboxes, or external services with `#[ignore]` and a clear reason.
|
||||
- Filter or normalize machine-specific output in snapshots.
|
||||
|
||||
## E2E env vars
|
||||
|
||||
Secret-backed E2E tests should use the shared test helpers instead of ad hoc env handling.
|
||||
|
||||
- Use `#[e2e_test(live("VAR"))]` for tests that require one or more env vars.
|
||||
- Use `#[e2e_test()]` for sandbox-only E2E tests that do not require secrets.
|
||||
- `FABRO_TEST_MODE=off` skips all `#[e2e_test]` tests.
|
||||
- `FABRO_TEST_MODE=live` runs `#[e2e_test]` tests and skips any missing `live("VAR")` dependencies.
|
||||
- `FABRO_TEST_MODE=strict` runs `#[e2e_test]` tests and fails immediately if any required env var is missing.
|
||||
- `cargo nextest run --profile e2e ...` also implies strict mode through nextest's runtime `NEXTEST_PROFILE=e2e` environment variable.
|
||||
- Use `fabro_test::require_env()` inside helper functions that need to read E2E env vars after the test-level guard has run.
|
||||
- Do not call `dotenvy` from tests. E2E env vars must come from the shell or the test runner profile.
|
||||
|
||||
If a test depends on `.env` or real credentials, it must be clearly marked and opt-in.
|
||||
|
||||
## Naming rules
|
||||
|
|
|
|||
|
|
@ -24,7 +24,6 @@ workspace = true
|
|||
[dependencies]
|
||||
clap.workspace = true
|
||||
anyhow.workspace = true
|
||||
dotenvy.workspace = true
|
||||
fabro-config = { path = "../fabro-config", features = ["clap"] }
|
||||
fabro-llm = { path = "../fabro-llm" }
|
||||
fabro-model = { path = "../fabro-model" }
|
||||
|
|
@ -54,6 +53,7 @@ libc = "0.2"
|
|||
[dev-dependencies]
|
||||
tokio = { workspace = true, features = ["test-util", "macros"] }
|
||||
tempfile = "3"
|
||||
dotenvy = { workspace = true }
|
||||
paste = "1"
|
||||
fabro-sandbox = { path = "../fabro-sandbox", features = ["test-support"] }
|
||||
fabro-macros = { path = "../fabro-macros" }
|
||||
fabro-test = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -1414,8 +1414,7 @@ mod tests {
|
|||
assert_eq!(html_to_markdown(plain), plain);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires BRAVE_SEARCH_API_KEY env var"]
|
||||
#[fabro_macros::e2e_test(live("BRAVE_SEARCH_API_KEY"))]
|
||||
async fn web_search_returns_results() {
|
||||
let api_key = std::env::var("BRAVE_SEARCH_API_KEY")
|
||||
.expect("BRAVE_SEARCH_API_KEY must be set to run this test");
|
||||
|
|
|
|||
|
|
@ -58,7 +58,6 @@ fn build_profile(provider: Provider, model: &str, client: &Client) -> Box<dyn Ag
|
|||
}
|
||||
|
||||
async fn make_session(provider: Provider, model: &str, cwd: &Path) -> Session {
|
||||
dotenvy::dotenv().ok();
|
||||
let client = Client::from_env().await.expect("Client::from_env failed");
|
||||
let mut profile = build_profile(provider, model, &client);
|
||||
let env = Arc::new(LocalSandbox::new(cwd.to_path_buf()));
|
||||
|
|
@ -117,7 +116,6 @@ async fn make_session_with_config(
|
|||
cwd: &Path,
|
||||
config: SessionConfig,
|
||||
) -> Session {
|
||||
dotenvy::dotenv().ok();
|
||||
let client = Client::from_env().await.expect("Client::from_env failed");
|
||||
let profile: Arc<dyn AgentProfile> = Arc::from(build_profile(provider, model, &client));
|
||||
let env = Arc::new(LocalSandbox::new(cwd.to_path_buf()));
|
||||
|
|
@ -125,10 +123,9 @@ async fn make_session_with_config(
|
|||
}
|
||||
|
||||
macro_rules! provider_test {
|
||||
($scenario:ident, $provider:expr, $model:expr, $prefix:ident) => {
|
||||
($scenario:ident, $provider:expr, $model:expr, $prefix:ident, keys = [$($key:expr),+ $(,)?]) => {
|
||||
paste::paste! {
|
||||
#[tokio::test]
|
||||
#[ignore = "requires LLM API keys"]
|
||||
#[fabro_macros::e2e_test($(live($key)),+)]
|
||||
async fn [<$prefix _ $scenario>]() {
|
||||
let tmp = tempfile::tempdir().expect("failed to create tempdir");
|
||||
let mut session = make_session($provider, $model, tmp.path()).await;
|
||||
|
|
@ -145,21 +142,53 @@ macro_rules! provider_tests {
|
|||
$scenario,
|
||||
Provider::Anthropic,
|
||||
"claude-haiku-4-5",
|
||||
anthropic
|
||||
anthropic,
|
||||
keys = ["ANTHROPIC_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
$scenario,
|
||||
Provider::OpenAi,
|
||||
"gpt-5.4-mini",
|
||||
openai,
|
||||
keys = ["OPENAI_API_KEY"]
|
||||
);
|
||||
provider_test!($scenario, Provider::OpenAi, "gpt-5.4-mini", openai);
|
||||
provider_test!(
|
||||
$scenario,
|
||||
Provider::Gemini,
|
||||
"gemini-3-flash-preview",
|
||||
gemini
|
||||
gemini,
|
||||
keys = ["GEMINI_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
$scenario,
|
||||
Provider::Kimi,
|
||||
"kimi-k2.5",
|
||||
kimi,
|
||||
keys = ["KIMI_API_KEY"]
|
||||
);
|
||||
provider_test!($scenario, Provider::Kimi, "kimi-k2.5", kimi);
|
||||
#[cfg(feature = "quarantine")]
|
||||
provider_test!($scenario, Provider::Zai, "glm-4.7", zai);
|
||||
provider_test!($scenario, Provider::Minimax, "minimax-m2.5", minimax);
|
||||
provider_test!(
|
||||
$scenario,
|
||||
Provider::Zai,
|
||||
"glm-4.7",
|
||||
zai,
|
||||
keys = ["ZAI_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
$scenario,
|
||||
Provider::Minimax,
|
||||
"minimax-m2.5",
|
||||
minimax,
|
||||
keys = ["MINIMAX_API_KEY"]
|
||||
);
|
||||
#[cfg(feature = "quarantine")]
|
||||
provider_test!($scenario, Provider::Inception, "mercury-2", inception);
|
||||
provider_test!(
|
||||
$scenario,
|
||||
Provider::Inception,
|
||||
"mercury-2",
|
||||
inception,
|
||||
keys = ["INCEPTION_API_KEY"]
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -176,8 +205,109 @@ provider_tests!(steering_mid_task);
|
|||
provider_tests!(follow_up);
|
||||
provider_tests!(subagent_spawn);
|
||||
|
||||
provider_tests!(web_fetch);
|
||||
provider_tests!(web_search);
|
||||
provider_test!(
|
||||
web_fetch,
|
||||
Provider::Anthropic,
|
||||
"claude-haiku-4-5",
|
||||
anthropic,
|
||||
keys = ["ANTHROPIC_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
web_fetch,
|
||||
Provider::OpenAi,
|
||||
"gpt-5.4-mini",
|
||||
openai,
|
||||
keys = ["OPENAI_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
web_fetch,
|
||||
Provider::Gemini,
|
||||
"gemini-3-flash-preview",
|
||||
gemini,
|
||||
keys = ["GEMINI_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
web_fetch,
|
||||
Provider::Kimi,
|
||||
"kimi-k2.5",
|
||||
kimi,
|
||||
keys = ["KIMI_API_KEY", "OPENAI_API_KEY"]
|
||||
);
|
||||
#[cfg(feature = "quarantine")]
|
||||
provider_test!(
|
||||
web_fetch,
|
||||
Provider::Zai,
|
||||
"glm-4.7",
|
||||
zai,
|
||||
keys = ["ZAI_API_KEY", "OPENAI_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
web_fetch,
|
||||
Provider::Minimax,
|
||||
"minimax-m2.5",
|
||||
minimax,
|
||||
keys = ["MINIMAX_API_KEY", "OPENAI_API_KEY"]
|
||||
);
|
||||
#[cfg(feature = "quarantine")]
|
||||
provider_test!(
|
||||
web_fetch,
|
||||
Provider::Inception,
|
||||
"mercury-2",
|
||||
inception,
|
||||
keys = ["INCEPTION_API_KEY", "OPENAI_API_KEY"]
|
||||
);
|
||||
|
||||
provider_test!(
|
||||
web_search,
|
||||
Provider::Anthropic,
|
||||
"claude-haiku-4-5",
|
||||
anthropic,
|
||||
keys = ["ANTHROPIC_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
web_search,
|
||||
Provider::OpenAi,
|
||||
"gpt-5.4-mini",
|
||||
openai,
|
||||
keys = ["OPENAI_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
web_search,
|
||||
Provider::Gemini,
|
||||
"gemini-3-flash-preview",
|
||||
gemini,
|
||||
keys = ["GEMINI_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
web_search,
|
||||
Provider::Kimi,
|
||||
"kimi-k2.5",
|
||||
kimi,
|
||||
keys = ["KIMI_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
||||
);
|
||||
#[cfg(feature = "quarantine")]
|
||||
provider_test!(
|
||||
web_search,
|
||||
Provider::Zai,
|
||||
"glm-4.7",
|
||||
zai,
|
||||
keys = ["ZAI_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
web_search,
|
||||
Provider::Minimax,
|
||||
"minimax-m2.5",
|
||||
minimax,
|
||||
keys = ["MINIMAX_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
||||
);
|
||||
#[cfg(feature = "quarantine")]
|
||||
provider_test!(
|
||||
web_search,
|
||||
Provider::Inception,
|
||||
"mercury-2",
|
||||
inception,
|
||||
keys = ["INCEPTION_API_KEY", "BRAVE_SEARCH_API_KEY"]
|
||||
);
|
||||
|
||||
// Scenarios below are only generated for providers where they are supported.
|
||||
// - multi_step_read_analyze_edit / provider_specific_editing: gpt-4o-mini is too
|
||||
|
|
@ -194,20 +324,46 @@ macro_rules! non_openai_provider_tests {
|
|||
$scenario,
|
||||
Provider::Anthropic,
|
||||
"claude-haiku-4-5",
|
||||
anthropic
|
||||
anthropic,
|
||||
keys = ["ANTHROPIC_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
$scenario,
|
||||
Provider::Gemini,
|
||||
"gemini-3-flash-preview",
|
||||
gemini
|
||||
gemini,
|
||||
keys = ["GEMINI_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
$scenario,
|
||||
Provider::Kimi,
|
||||
"kimi-k2.5",
|
||||
kimi,
|
||||
keys = ["KIMI_API_KEY"]
|
||||
);
|
||||
provider_test!($scenario, Provider::Kimi, "kimi-k2.5", kimi);
|
||||
#[cfg(feature = "quarantine")]
|
||||
provider_test!($scenario, Provider::Zai, "glm-4.7", zai);
|
||||
provider_test!($scenario, Provider::Minimax, "minimax-m2.5", minimax);
|
||||
provider_test!(
|
||||
$scenario,
|
||||
Provider::Zai,
|
||||
"glm-4.7",
|
||||
zai,
|
||||
keys = ["ZAI_API_KEY"]
|
||||
);
|
||||
provider_test!(
|
||||
$scenario,
|
||||
Provider::Minimax,
|
||||
"minimax-m2.5",
|
||||
minimax,
|
||||
keys = ["MINIMAX_API_KEY"]
|
||||
);
|
||||
#[cfg(feature = "quarantine")]
|
||||
provider_test!($scenario, Provider::Inception, "mercury-2", inception);
|
||||
provider_test!(
|
||||
$scenario,
|
||||
Provider::Inception,
|
||||
"mercury-2",
|
||||
inception,
|
||||
keys = ["INCEPTION_API_KEY"]
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -430,9 +586,8 @@ async fn scenario_follow_up(session: &mut Session, dir: &Path) {
|
|||
// Scenario 11: reasoning_effort
|
||||
// ---------------------------------------------------------------------------
|
||||
macro_rules! reasoning_effort_tests {
|
||||
($provider:expr, $model:expr, $test_name:ident) => {
|
||||
#[tokio::test]
|
||||
#[ignore = "requires LLM API keys"]
|
||||
($provider:expr, $model:expr, $test_name:ident, keys = [$($key:expr),+ $(,)?]) => {
|
||||
#[fabro_macros::e2e_test($(live($key)),+)]
|
||||
async fn $test_name() {
|
||||
let tmp = tempfile::tempdir().expect("failed to create tempdir");
|
||||
let config = SessionConfig {
|
||||
|
|
@ -453,20 +608,42 @@ macro_rules! reasoning_effort_tests {
|
|||
reasoning_effort_tests!(
|
||||
Provider::Anthropic,
|
||||
"claude-haiku-4-5",
|
||||
anthropic_reasoning_effort
|
||||
anthropic_reasoning_effort,
|
||||
keys = ["ANTHROPIC_API_KEY"]
|
||||
);
|
||||
// gpt-5-mini does not support the reasoning.effort parameter, so no OpenAI test.
|
||||
reasoning_effort_tests!(
|
||||
Provider::Gemini,
|
||||
"gemini-3-flash-preview",
|
||||
gemini_reasoning_effort
|
||||
gemini_reasoning_effort,
|
||||
keys = ["GEMINI_API_KEY"]
|
||||
);
|
||||
reasoning_effort_tests!(
|
||||
Provider::Kimi,
|
||||
"kimi-k2.5",
|
||||
kimi_reasoning_effort,
|
||||
keys = ["KIMI_API_KEY"]
|
||||
);
|
||||
reasoning_effort_tests!(Provider::Kimi, "kimi-k2.5", kimi_reasoning_effort);
|
||||
#[cfg(feature = "quarantine")]
|
||||
reasoning_effort_tests!(Provider::Zai, "glm-4.7", zai_reasoning_effort);
|
||||
reasoning_effort_tests!(Provider::Minimax, "minimax-m2.5", minimax_reasoning_effort);
|
||||
reasoning_effort_tests!(
|
||||
Provider::Zai,
|
||||
"glm-4.7",
|
||||
zai_reasoning_effort,
|
||||
keys = ["ZAI_API_KEY"]
|
||||
);
|
||||
reasoning_effort_tests!(
|
||||
Provider::Minimax,
|
||||
"minimax-m2.5",
|
||||
minimax_reasoning_effort,
|
||||
keys = ["MINIMAX_API_KEY"]
|
||||
);
|
||||
#[cfg(feature = "quarantine")]
|
||||
reasoning_effort_tests!(Provider::Inception, "mercury-2", inception_reasoning_effort);
|
||||
reasoning_effort_tests!(
|
||||
Provider::Inception,
|
||||
"mercury-2",
|
||||
inception_reasoning_effort,
|
||||
keys = ["INCEPTION_API_KEY"]
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 12: subagent_spawn
|
||||
|
|
@ -486,9 +663,8 @@ async fn scenario_subagent_spawn(session: &mut Session, dir: &Path) {
|
|||
// Scenario 13: loop_detection
|
||||
// ---------------------------------------------------------------------------
|
||||
macro_rules! loop_detection_tests {
|
||||
($provider:expr, $model:expr, $test_name:ident) => {
|
||||
#[tokio::test]
|
||||
#[ignore = "requires LLM API keys"]
|
||||
($provider:expr, $model:expr, $test_name:ident, keys = [$($key:expr),+ $(,)?]) => {
|
||||
#[fabro_macros::e2e_test($(live($key)),+)]
|
||||
async fn $test_name() {
|
||||
let tmp = tempfile::tempdir().expect("failed to create tempdir");
|
||||
let config = SessionConfig {
|
||||
|
|
@ -509,20 +685,47 @@ macro_rules! loop_detection_tests {
|
|||
loop_detection_tests!(
|
||||
Provider::Anthropic,
|
||||
"claude-haiku-4-5",
|
||||
anthropic_loop_detection
|
||||
anthropic_loop_detection,
|
||||
keys = ["ANTHROPIC_API_KEY"]
|
||||
);
|
||||
loop_detection_tests!(
|
||||
Provider::OpenAi,
|
||||
"gpt-5.4-mini",
|
||||
openai_loop_detection,
|
||||
keys = ["OPENAI_API_KEY"]
|
||||
);
|
||||
loop_detection_tests!(Provider::OpenAi, "gpt-5.4-mini", openai_loop_detection);
|
||||
loop_detection_tests!(
|
||||
Provider::Gemini,
|
||||
"gemini-3-flash-preview",
|
||||
gemini_loop_detection
|
||||
gemini_loop_detection,
|
||||
keys = ["GEMINI_API_KEY"]
|
||||
);
|
||||
loop_detection_tests!(
|
||||
Provider::Kimi,
|
||||
"kimi-k2.5",
|
||||
kimi_loop_detection,
|
||||
keys = ["KIMI_API_KEY"]
|
||||
);
|
||||
loop_detection_tests!(Provider::Kimi, "kimi-k2.5", kimi_loop_detection);
|
||||
#[cfg(feature = "quarantine")]
|
||||
loop_detection_tests!(Provider::Zai, "glm-4.7", zai_loop_detection);
|
||||
loop_detection_tests!(Provider::Minimax, "minimax-m2.5", minimax_loop_detection);
|
||||
loop_detection_tests!(
|
||||
Provider::Zai,
|
||||
"glm-4.7",
|
||||
zai_loop_detection,
|
||||
keys = ["ZAI_API_KEY"]
|
||||
);
|
||||
loop_detection_tests!(
|
||||
Provider::Minimax,
|
||||
"minimax-m2.5",
|
||||
minimax_loop_detection,
|
||||
keys = ["MINIMAX_API_KEY"]
|
||||
);
|
||||
#[cfg(feature = "quarantine")]
|
||||
loop_detection_tests!(Provider::Inception, "mercury-2", inception_loop_detection);
|
||||
loop_detection_tests!(
|
||||
Provider::Inception,
|
||||
"mercury-2",
|
||||
inception_loop_detection,
|
||||
keys = ["INCEPTION_API_KEY"]
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Scenario 14: error_recovery
|
||||
|
|
|
|||
|
|
@ -100,3 +100,4 @@ serde_json.workspace = true
|
|||
tempfile = "3"
|
||||
httpmock = "0.8"
|
||||
fabro-test = { workspace = true }
|
||||
fabro-macros = { path = "../fabro-macros" }
|
||||
|
|
|
|||
|
|
@ -63,10 +63,8 @@ fn dry_run_flag() {
|
|||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "scenario: requires ANTHROPIC_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn live_doctor() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
context.doctor().assert().success();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -110,10 +110,8 @@ fn exec_uses_user_config_defaults() {
|
|||
");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires API key"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn exec_creates_file() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
context
|
||||
.exec_cmd()
|
||||
|
|
@ -140,10 +138,8 @@ fn exec_creates_file() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires API key"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn exec_shell_command() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
context
|
||||
.exec_cmd()
|
||||
|
|
@ -163,10 +159,8 @@ fn exec_shell_command() {
|
|||
.success();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires API key"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn exec_read_only_blocks_write() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
context
|
||||
.exec_cmd()
|
||||
|
|
@ -190,10 +184,8 @@ fn exec_read_only_blocks_write() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires API key"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn exec_json_output_format() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
let output = context
|
||||
.exec_cmd()
|
||||
|
|
@ -229,10 +221,8 @@ fn exec_json_output_format() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires API key"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn exec_read_and_edit() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
context.write_temp("data.txt", "old content");
|
||||
context
|
||||
|
|
|
|||
|
|
@ -88,10 +88,8 @@ fn prompt_concatenates_stdin_and_arg() {
|
|||
result.stderr(predicate::str::contains("no prompt provided").not());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires API key"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn prompt_no_stream_generates_response() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
context
|
||||
.llm()
|
||||
|
|
@ -107,10 +105,8 @@ fn prompt_no_stream_generates_response() {
|
|||
.stdout(predicate::str::is_empty().not());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires API key"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn prompt_stream_generates_response() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
context
|
||||
.llm()
|
||||
|
|
@ -125,10 +121,8 @@ fn prompt_stream_generates_response() {
|
|||
.stdout(predicate::str::is_empty().not());
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires API key"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn prompt_usage_shows_tokens() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
context
|
||||
.llm()
|
||||
|
|
@ -145,10 +139,8 @@ fn prompt_usage_shows_tokens() {
|
|||
.stderr(predicate::str::contains("Tokens:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires API key"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn prompt_schema_no_stream_generates_json() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
let assert = context
|
||||
.llm()
|
||||
|
|
@ -169,10 +161,8 @@ fn prompt_schema_no_stream_generates_json() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires API key"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn prompt_schema_stream_generates_json() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
let assert = context
|
||||
.llm()
|
||||
|
|
@ -193,10 +183,8 @@ fn prompt_schema_stream_generates_json() {
|
|||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "requires API key"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn chat_multi_turn_with_system_prompt() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
let assert = context
|
||||
.command()
|
||||
|
|
|
|||
|
|
@ -2,10 +2,8 @@ use std::time::Duration;
|
|||
|
||||
use fabro_test::test_context;
|
||||
|
||||
#[test]
|
||||
#[ignore = "scenario: requires ANTHROPIC_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
fn test_exec_creates_file() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
|
||||
let mut cmd = context.exec_cmd();
|
||||
|
|
|
|||
|
|
@ -4,10 +4,8 @@ use serde_json::Value;
|
|||
use super::{fixture, read_json, timeout_for};
|
||||
use crate::support::fabro_json_snapshot;
|
||||
|
||||
#[test]
|
||||
#[ignore = "scenario: requires local sandbox"]
|
||||
#[fabro_macros::e2e_test()]
|
||||
fn local_run_lifecycle() {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
|
||||
let cmd = |args: &[&str]| -> assert_cmd::assert::Assert {
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@ use fabro_test::test_context;
|
|||
|
||||
use super::{completed_nodes, find_run_dir, fixture, read_conclusion, sandbox_tests, timeout_for};
|
||||
|
||||
sandbox_tests!(agent_linear);
|
||||
sandbox_tests!(agent_linear, keys = ["ANTHROPIC_API_KEY"]);
|
||||
|
||||
fn scenario_agent_linear(sandbox: &str) {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
|
||||
context
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@ use fabro_test::test_context;
|
|||
|
||||
use super::{completed_nodes, find_run_dir, fixture, read_conclusion, sandbox_tests, timeout_for};
|
||||
|
||||
sandbox_tests!(command_agent_mixed);
|
||||
sandbox_tests!(command_agent_mixed, keys = ["ANTHROPIC_API_KEY"]);
|
||||
|
||||
fn scenario_command_agent_mixed(sandbox: &str) {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
|
||||
context
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ use super::{completed_nodes, find_run_dir, fixture, read_conclusion, sandbox_tes
|
|||
sandbox_tests!(command_pipeline);
|
||||
|
||||
fn scenario_command_pipeline(sandbox: &str) {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
|
||||
context
|
||||
|
|
|
|||
|
|
@ -5,7 +5,6 @@ use super::{completed_nodes, find_run_dir, fixture, read_conclusion, sandbox_tes
|
|||
sandbox_tests!(conditional_branching);
|
||||
|
||||
fn scenario_conditional_branching(sandbox: &str) {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
|
||||
context
|
||||
|
|
|
|||
|
|
@ -5,10 +5,9 @@ use super::{
|
|||
timeout_for,
|
||||
};
|
||||
|
||||
sandbox_tests!(full_stack);
|
||||
sandbox_tests!(full_stack, keys = ["ANTHROPIC_API_KEY"]);
|
||||
|
||||
fn scenario_full_stack(sandbox: &str) {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
|
||||
context
|
||||
|
|
|
|||
|
|
@ -2,10 +2,9 @@ use fabro_test::test_context;
|
|||
|
||||
use super::{completed_nodes, find_run_dir, fixture, read_conclusion, sandbox_tests, timeout_for};
|
||||
|
||||
sandbox_tests!(human_gate);
|
||||
sandbox_tests!(human_gate, keys = ["ANTHROPIC_API_KEY"]);
|
||||
|
||||
fn scenario_human_gate(sandbox: &str) {
|
||||
dotenvy::dotenv().ok();
|
||||
let context = test_context!();
|
||||
|
||||
context
|
||||
|
|
|
|||
|
|
@ -70,15 +70,16 @@ pub(super) fn find_run_dir(storage_dir: &Path) -> PathBuf {
|
|||
|
||||
macro_rules! sandbox_tests {
|
||||
($name:ident) => {
|
||||
sandbox_tests!($name, keys = []);
|
||||
};
|
||||
($name:ident, keys = [$($key:expr),* $(,)?]) => {
|
||||
paste::paste! {
|
||||
#[test]
|
||||
#[ignore = "scenario: requires local sandbox"]
|
||||
#[fabro_macros::e2e_test($(live($key)),*)]
|
||||
fn [<local_ $name>]() {
|
||||
[<scenario_ $name>]("local");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "scenario: requires DAYTONA_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY") $(, live($key))*)]
|
||||
fn [<daytona_ $name>]() {
|
||||
[<scenario_ $name>]("daytona");
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ fabro-util = { path = "../fabro-util" }
|
|||
http = "1"
|
||||
insta = { workspace = true }
|
||||
tokio = { workspace = true, features = ["test-util", "macros"] }
|
||||
dotenvy.workspace = true
|
||||
httpmock = "0.8"
|
||||
serde_json.workspace = true
|
||||
serde_json.workspace = true
|
||||
fabro-macros = { path = "../fabro-macros" }
|
||||
fabro-test = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -1809,7 +1809,7 @@ mod tests {
|
|||
option: vec![],
|
||||
};
|
||||
|
||||
let result = run_prompt_via_server(args, &server).await;
|
||||
let result = run_prompt_via_server(args, &server, false).await;
|
||||
assert!(result.is_ok());
|
||||
mock.assert_async().await;
|
||||
}
|
||||
|
|
@ -1849,7 +1849,7 @@ data: {\"type\":\"finish\",\"finish_reason\":\"stop\",\"usage\":{\"input_tokens\
|
|||
option: vec![],
|
||||
};
|
||||
|
||||
let result = run_prompt_via_server(args, &server).await;
|
||||
let result = run_prompt_via_server(args, &server, false).await;
|
||||
assert!(result.is_ok());
|
||||
mock.assert_async().await;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,10 +21,8 @@ fn make_request(model: &str) -> Request {
|
|||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires ANTHROPIC_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn anthropic_complete() {
|
||||
dotenvy::dotenv().ok();
|
||||
let api_key = std::env::var("ANTHROPIC_API_KEY").expect("ANTHROPIC_API_KEY must be set");
|
||||
let adapter = AnthropicAdapter::new(api_key);
|
||||
let request = make_request("claude-haiku-4-5");
|
||||
|
|
@ -40,10 +38,8 @@ async fn anthropic_complete() {
|
|||
assert_eq!(response.provider, "anthropic");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires OPENAI_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("OPENAI_API_KEY"))]
|
||||
async fn openai_complete() {
|
||||
dotenvy::dotenv().ok();
|
||||
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY must be set");
|
||||
let adapter = OpenAiAdapter::new(api_key);
|
||||
let request = make_request("gpt-4o-mini");
|
||||
|
|
@ -59,8 +55,7 @@ async fn openai_complete() {
|
|||
assert_eq!(response.provider, "openai");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires OPENAI_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("OPENAI_API_KEY"))]
|
||||
async fn openai_gpt_5_3_codex_complete() {
|
||||
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY must be set");
|
||||
let adapter = OpenAiAdapter::new(api_key);
|
||||
|
|
@ -76,10 +71,8 @@ async fn openai_gpt_5_3_codex_complete() {
|
|||
assert_eq!(response.provider, "openai");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires GEMINI_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("GEMINI_API_KEY"))]
|
||||
async fn gemini_complete() {
|
||||
dotenvy::dotenv().ok();
|
||||
let api_key = std::env::var("GEMINI_API_KEY").expect("GEMINI_API_KEY must be set");
|
||||
let adapter = GeminiAdapter::new(api_key);
|
||||
let request = make_request("gemini-2.5-flash");
|
||||
|
|
@ -168,28 +161,22 @@ async fn run_multi_turn_cache_test(
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires ANTHROPIC_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn anthropic_multi_turn_cache() {
|
||||
dotenvy::dotenv().ok();
|
||||
let api_key = std::env::var("ANTHROPIC_API_KEY").expect("ANTHROPIC_API_KEY must be set");
|
||||
let adapter = AnthropicAdapter::new(api_key);
|
||||
run_multi_turn_cache_test(&adapter, "claude-haiku-4-5", 0.5).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires OPENAI_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("OPENAI_API_KEY"))]
|
||||
async fn openai_multi_turn_cache() {
|
||||
dotenvy::dotenv().ok();
|
||||
let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY must be set");
|
||||
let adapter = OpenAiAdapter::new(api_key);
|
||||
run_multi_turn_cache_test(&adapter, "gpt-4o-mini", 0.5).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires GEMINI_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("GEMINI_API_KEY"))]
|
||||
async fn gemini_multi_turn_cache() {
|
||||
dotenvy::dotenv().ok();
|
||||
let api_key = std::env::var("GEMINI_API_KEY").expect("GEMINI_API_KEY must be set");
|
||||
let adapter = GeminiAdapter::new(api_key);
|
||||
run_multi_turn_cache_test(&adapter, "gemini-2.5-flash", 0.5).await;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,34 @@
|
|||
use proc_macro::TokenStream;
|
||||
use quote::quote;
|
||||
use syn::{Data, DeriveInput, Fields, parse_macro_input};
|
||||
use syn::parse::{Parse, ParseStream};
|
||||
use syn::punctuated::Punctuated;
|
||||
use syn::{
|
||||
Data, DeriveInput, Fields, Ident, ItemFn, LitStr, Token, parenthesized, parse_macro_input,
|
||||
};
|
||||
|
||||
enum E2eRequirement {
|
||||
Live(LitStr),
|
||||
}
|
||||
|
||||
impl Parse for E2eRequirement {
|
||||
fn parse(input: ParseStream<'_>) -> syn::Result<Self> {
|
||||
let ident = input.parse::<Ident>()?;
|
||||
if ident != "live" {
|
||||
return Err(syn::Error::new(
|
||||
ident.span(),
|
||||
"expected `live(\"ENV_VAR\")`",
|
||||
));
|
||||
}
|
||||
|
||||
let content;
|
||||
parenthesized!(content in input);
|
||||
let env_var = content.parse::<LitStr>()?;
|
||||
if !content.is_empty() {
|
||||
return Err(content.error("expected a single string literal"));
|
||||
}
|
||||
Ok(Self::Live(env_var))
|
||||
}
|
||||
}
|
||||
|
||||
#[proc_macro_derive(Combine)]
|
||||
pub fn derive_combine(input: TokenStream) -> TokenStream {
|
||||
|
|
@ -50,3 +78,73 @@ pub fn derive_combine(input: TokenStream) -> TokenStream {
|
|||
}
|
||||
.into()
|
||||
}
|
||||
|
||||
#[proc_macro_attribute]
|
||||
pub fn e2e_test(attr: TokenStream, item: TokenStream) -> TokenStream {
|
||||
let requirements =
|
||||
parse_macro_input!(attr with Punctuated::<E2eRequirement, Token![,]>::parse_terminated);
|
||||
let input = parse_macro_input!(item as ItemFn);
|
||||
|
||||
let attrs = input.attrs;
|
||||
let vis = input.vis;
|
||||
let sig = input.sig;
|
||||
let block = input.block;
|
||||
|
||||
let env_vars: Vec<_> = requirements
|
||||
.into_iter()
|
||||
.map(|requirement| match requirement {
|
||||
E2eRequirement::Live(env_var) => env_var,
|
||||
})
|
||||
.collect();
|
||||
|
||||
let ignore_reason = if env_vars.is_empty() {
|
||||
"e2e".to_string()
|
||||
} else {
|
||||
let joined = env_vars
|
||||
.iter()
|
||||
.map(LitStr::value)
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
format!("e2e: {joined}")
|
||||
};
|
||||
|
||||
let test_attr = if sig.asyncness.is_some() {
|
||||
quote!(#[tokio::test])
|
||||
} else {
|
||||
quote!(#[test])
|
||||
};
|
||||
|
||||
let env_guards = env_vars.iter().map(|env_var| {
|
||||
let env_name = env_var.value();
|
||||
let strict_message = format!("{env_name} not set (FABRO_TEST_MODE=strict)");
|
||||
let skip_message = format!("skipping: {env_name} not set");
|
||||
quote! {
|
||||
if ::std::env::var(#env_var).is_err() {
|
||||
if __mode == ::fabro_test::TestMode::Strict {
|
||||
panic!(#strict_message);
|
||||
}
|
||||
eprintln!(#skip_message);
|
||||
return;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
quote! {
|
||||
#(#attrs)*
|
||||
#test_attr
|
||||
#[ignore = #ignore_reason]
|
||||
#vis #sig {
|
||||
let __mode = ::fabro_test::TestMode::from_env();
|
||||
|
||||
if __mode == ::fabro_test::TestMode::Off {
|
||||
eprintln!("skipping: FABRO_TEST_MODE is off");
|
||||
return;
|
||||
}
|
||||
|
||||
#(#env_guards)*
|
||||
|
||||
#block
|
||||
}
|
||||
}
|
||||
.into()
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,6 +22,43 @@ static INSTA_FILTERS: &[(&str, &str)] = &[
|
|||
(r"\\([\w\d])", "/$1"),
|
||||
];
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
|
||||
pub enum TestMode {
|
||||
#[default]
|
||||
Off,
|
||||
Live,
|
||||
Strict,
|
||||
}
|
||||
|
||||
impl TestMode {
|
||||
#[must_use]
|
||||
pub fn from_env() -> Self {
|
||||
match std::env::var("FABRO_TEST_MODE").as_deref() {
|
||||
Ok("live") => Self::Live,
|
||||
Ok("strict") => Self::Strict,
|
||||
_ => match std::env::var("NEXTEST_PROFILE").as_deref() {
|
||||
Ok("e2e") => Self::Strict,
|
||||
_ => Self::Off,
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read an env var required by an E2E test, with mode-aware skip/strict behavior.
|
||||
#[must_use]
|
||||
pub fn require_env(name: &str) -> Option<String> {
|
||||
match std::env::var(name) {
|
||||
Ok(value) => Some(value),
|
||||
Err(_) => {
|
||||
if TestMode::from_env() == TestMode::Strict {
|
||||
panic!("{name} not set (FABRO_TEST_MODE=strict)");
|
||||
}
|
||||
eprintln!("skipping: {name} not set");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// An isolated test context for running fabro CLI commands.
|
||||
///
|
||||
/// Creates temporary directories for home, storage, and working directory,
|
||||
|
|
|
|||
|
|
@ -18,7 +18,6 @@ workspace = true
|
|||
|
||||
[dependencies]
|
||||
anyhow.workspace = true
|
||||
dotenvy.workspace = true
|
||||
fabro-agent = { path = "../fabro-agent" }
|
||||
fabro-config = { path = "../fabro-config" }
|
||||
fabro-graphviz = { path = "../fabro-graphviz" }
|
||||
|
|
@ -66,6 +65,7 @@ toml.workspace = true
|
|||
fabro-mcp = { path = "../fabro-mcp" }
|
||||
tokio = { workspace = true, features = ["test-util", "macros"] }
|
||||
tempfile = "3"
|
||||
dotenvy.workspace = true
|
||||
assert_cmd = "2"
|
||||
predicates = "3"
|
||||
fabro-macros = { path = "../fabro-macros" }
|
||||
fabro-test = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -52,18 +52,12 @@ async fn create_env() -> DaytonaSandbox {
|
|||
async fn create_env_with_github_app(
|
||||
github_app: Option<fabro_github::GitHubAppCredentials>,
|
||||
) -> DaytonaSandbox {
|
||||
dotenvy::dotenv().ok();
|
||||
if let Some(home) = dirs::home_dir() {
|
||||
dotenvy::from_path(home.join(".fabro/.env")).ok();
|
||||
}
|
||||
DaytonaSandbox::new(DaytonaConfig::default(), github_app, None, None)
|
||||
.await
|
||||
.expect("Failed to create Daytona client — is DAYTONA_API_KEY set?")
|
||||
}
|
||||
|
||||
fn load_github_app_credentials() -> fabro_github::GitHubAppCredentials {
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
// Read app_id from ~/.fabro/server.toml
|
||||
let home = dirs::home_dir().expect("No home directory");
|
||||
let config_path = home.join(".fabro/server.toml");
|
||||
|
|
@ -100,8 +94,7 @@ fn load_github_app_credentials() -> fabro_github::GitHubAppCredentials {
|
|||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_exec_command() {
|
||||
let creds = load_github_app_credentials();
|
||||
let env = create_env_with_github_app(Some(creds)).await;
|
||||
|
|
@ -117,8 +110,7 @@ async fn daytona_exec_command() {
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_exec_command_with_pipe() {
|
||||
let creds = load_github_app_credentials();
|
||||
let env = create_env_with_github_app(Some(creds)).await;
|
||||
|
|
@ -134,8 +126,7 @@ async fn daytona_exec_command_with_pipe() {
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_exec_command_cancelled() {
|
||||
let creds = load_github_app_credentials();
|
||||
let env = create_env_with_github_app(Some(creds)).await;
|
||||
|
|
@ -163,8 +154,7 @@ async fn daytona_exec_command_cancelled() {
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_exec_command_local_timeout() {
|
||||
let creds = load_github_app_credentials();
|
||||
let env = create_env_with_github_app(Some(creds)).await;
|
||||
|
|
@ -197,8 +187,7 @@ async fn daytona_exec_command_local_timeout() {
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_file_round_trip() {
|
||||
let env = create_env().await;
|
||||
env.initialize().await.unwrap();
|
||||
|
|
@ -223,8 +212,7 @@ async fn daytona_file_round_trip() {
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_full_lifecycle() {
|
||||
let env = create_env().await;
|
||||
|
||||
|
|
@ -249,13 +237,10 @@ async fn daytona_full_lifecycle() {
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_snapshot_sandbox() {
|
||||
use fabro_sandbox::daytona::DaytonaSnapshotConfig;
|
||||
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
let config = DaytonaConfig {
|
||||
auto_stop_interval: Some(60),
|
||||
snapshot: Some(DaytonaSnapshotConfig {
|
||||
|
|
@ -287,8 +272,7 @@ async fn daytona_snapshot_sandbox() {
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_artifact_sync_uploads_and_rewrites_pointer() {
|
||||
let env = create_env().await;
|
||||
env.initialize().await.unwrap();
|
||||
|
|
@ -364,8 +348,7 @@ impl Handler for LargeOutputHandler {
|
|||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_pipeline_artifact_offload_and_sync() {
|
||||
let env = create_env().await;
|
||||
env.initialize().await.unwrap();
|
||||
|
|
@ -521,8 +504,7 @@ async fn setup_daytona_git(sandbox: &dyn Sandbox) -> (RunId, String, String) {
|
|||
(run_id, base_sha, branch_name)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_git_checkpoint_remote_emits_events() {
|
||||
let env = create_env().await;
|
||||
env.initialize().await.unwrap();
|
||||
|
|
@ -682,8 +664,7 @@ use fabro_workflow::handler::parallel::ParallelHandler;
|
|||
|
||||
/// End-to-end: parallel branches get isolated worktrees in Daytona sandbox,
|
||||
/// fan-in fast-forwards to winner.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_parallel_git_branching_e2e() {
|
||||
let env = create_env().await;
|
||||
env.initialize().await.unwrap();
|
||||
|
|
@ -909,7 +890,6 @@ use fabro_workflow::handler::llm::AgentCliBackend;
|
|||
/// Installs the CLI tool in the sandbox, then runs the AgentCliBackend against it.
|
||||
async fn run_daytona_cli_test(provider: Provider, model: &str, install_command: &str) {
|
||||
let creds = load_github_app_credentials();
|
||||
dotenvy::dotenv().ok();
|
||||
let config = DaytonaConfig {
|
||||
snapshot: Some(DaytonaSnapshotConfig {
|
||||
name: "daytona-medium".into(),
|
||||
|
|
@ -1016,8 +996,7 @@ async fn run_daytona_cli_test(provider: Provider, model: &str, install_command:
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires DAYTONA_API_KEY + Claude CLI auth
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_cli_claude() {
|
||||
run_daytona_cli_test(
|
||||
Provider::Anthropic,
|
||||
|
|
@ -1027,14 +1006,12 @@ async fn daytona_cli_claude() {
|
|||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires DAYTONA_API_KEY + OpenAI/Codex auth
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_cli_codex() {
|
||||
run_daytona_cli_test(Provider::OpenAi, "o4-mini", "npm install -g @openai/codex").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore] // requires DAYTONA_API_KEY + Gemini auth
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_cli_gemini() {
|
||||
run_daytona_cli_test(
|
||||
Provider::Gemini,
|
||||
|
|
@ -1052,8 +1029,7 @@ use fabro_workflow::git::MetadataStore;
|
|||
|
||||
/// End-to-end test: pipeline with git checkpointing enabled + `meta_branch`
|
||||
/// writes shadow branch on the host repo and includes `Fabro-Checkpoint` trailer in sandbox commits.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_git_checkpoint_with_shadow_branch() {
|
||||
let env = create_env().await;
|
||||
env.initialize().await.unwrap();
|
||||
|
|
@ -1237,8 +1213,7 @@ impl Handler for AssetCreatorHandler {
|
|||
|
||||
/// Daytona sandbox: asset collection discovers files on the remote sandbox and
|
||||
/// downloads them to the local logs directory.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_asset_collection() {
|
||||
let env = create_env().await;
|
||||
env.initialize().await.unwrap();
|
||||
|
|
@ -1325,8 +1300,7 @@ async fn daytona_asset_collection() {
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_ssh_access() {
|
||||
let env = create_env().await;
|
||||
env.initialize().await.unwrap();
|
||||
|
|
@ -1341,8 +1315,7 @@ async fn daytona_ssh_access() {
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_ssh_access_before_init_fails() {
|
||||
let env = create_env().await;
|
||||
|
||||
|
|
@ -1360,8 +1333,7 @@ async fn daytona_ssh_access_before_init_fails() {
|
|||
|
||||
/// E2E: Clone the current (private) repo using GitHub App IAT credentials.
|
||||
/// Verifies the full flow: JWT signing, installation lookup, token creation, clone.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_clone_private_repo_with_github_app_iat() {
|
||||
let creds = load_github_app_credentials();
|
||||
let env = create_env_with_github_app(Some(creds)).await;
|
||||
|
|
@ -1418,8 +1390,7 @@ async fn daytona_clone_private_repo_with_github_app_iat() {
|
|||
}
|
||||
|
||||
/// E2E: Verify that repos in an installed org get credentials (needed for pushing).
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_clone_public_repo_gets_credentials() {
|
||||
let creds = load_github_app_credentials();
|
||||
|
||||
|
|
@ -1441,8 +1412,7 @@ async fn daytona_clone_public_repo_gets_credentials() {
|
|||
|
||||
/// E2E: Verify that requesting an IAT for a repo the app isn't installed on
|
||||
/// gives a clear error message.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_iat_not_installed_gives_clear_error() {
|
||||
let creds = load_github_app_credentials();
|
||||
|
||||
|
|
@ -1465,8 +1435,7 @@ async fn daytona_iat_not_installed_gives_clear_error() {
|
|||
|
||||
/// E2E: After each remote checkpoint, the run branch is pushed to origin.
|
||||
/// Verifies the branch appears on the remote via `git ls-remote`.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_git_push_run_branch_to_origin() {
|
||||
let creds = load_github_app_credentials();
|
||||
let env = create_env_with_github_app(Some(creds)).await;
|
||||
|
|
@ -1598,8 +1567,7 @@ async fn daytona_git_push_run_branch_to_origin() {
|
|||
/// underlying error that the SDK normally swallows.
|
||||
///
|
||||
/// Run: cargo test -p arc-workflows -- --ignored daytona_toolbox_idle_diagnostic --nocapture
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_toolbox_idle_diagnostic() {
|
||||
let creds = load_github_app_credentials();
|
||||
let env = create_env_with_github_app(Some(creds)).await;
|
||||
|
|
@ -1733,8 +1701,7 @@ async fn daytona_toolbox_idle_diagnostic() {
|
|||
///
|
||||
/// Creates a sandbox, saves a SandboxRecord, reconnects via `cp::reconnect`,
|
||||
/// uploads a file, downloads it back, and verifies the round-trip.
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))]
|
||||
async fn daytona_cp_upload_download_round_trip() {
|
||||
use fabro_sandbox::SandboxRecord;
|
||||
use fabro_sandbox::reconnect::reconnect;
|
||||
|
|
@ -1829,15 +1796,9 @@ async fn daytona_cp_upload_download_round_trip() {
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"))]
|
||||
async fn daytona_computer_use_browser_screenshot() {
|
||||
use base64::Engine;
|
||||
|
||||
dotenvy::dotenv().ok();
|
||||
if let Some(home) = dirs::home_dir() {
|
||||
dotenvy::from_path(home.join(".fabro/.env")).ok();
|
||||
}
|
||||
let config = DaytonaConfig {
|
||||
snapshot: Some(DaytonaSnapshotConfig {
|
||||
name: "daytona-medium".into(),
|
||||
|
|
@ -1987,16 +1948,11 @@ async fn daytona_computer_use_browser_screenshot() {
|
|||
env.cleanup().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"))]
|
||||
async fn daytona_playwright_mcp_sandbox_transport() {
|
||||
use fabro_agent::Sandbox;
|
||||
|
||||
// Create sandbox from daytona-medium (has Node.js + Chromium)
|
||||
dotenvy::dotenv().ok();
|
||||
if let Some(home) = dirs::home_dir() {
|
||||
dotenvy::from_path(home.join(".fabro/.env")).ok();
|
||||
}
|
||||
let config = DaytonaConfig {
|
||||
snapshot: Some(DaytonaSnapshotConfig {
|
||||
name: "daytona-medium".into(),
|
||||
|
|
|
|||
|
|
@ -5895,10 +5895,7 @@ mod real_llm {
|
|||
}
|
||||
|
||||
async fn make_llm_client() -> Option<Arc<Client>> {
|
||||
let _ = dotenvy::dotenv();
|
||||
if std::env::var("ANTHROPIC_API_KEY").is_err() {
|
||||
return None;
|
||||
}
|
||||
fabro_test::require_env("ANTHROPIC_API_KEY")?;
|
||||
let client = Client::from_env()
|
||||
.await
|
||||
.expect("unified-llm client should initialize from env");
|
||||
|
|
@ -5925,15 +5922,9 @@ mod real_llm {
|
|||
use fabro_workflow::run_options::RunOptions;
|
||||
use fabro_workflow::test_support::WorkflowRunner;
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn real_llm_linear_pipeline() {
|
||||
let client = if let Some(c) = make_llm_client().await {
|
||||
c
|
||||
} else {
|
||||
eprintln!("Skipping: ANTHROPIC_API_KEY not set");
|
||||
return;
|
||||
};
|
||||
let client = make_llm_client().await.unwrap();
|
||||
|
||||
let mut graph = Graph::new("RealLLMLinear");
|
||||
graph.attrs.insert(
|
||||
|
|
@ -6040,15 +6031,9 @@ mod real_llm {
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn real_llm_two_stage_pipeline() {
|
||||
let client = if let Some(c) = make_llm_client().await {
|
||||
c
|
||||
} else {
|
||||
eprintln!("Skipping: ANTHROPIC_API_KEY not set");
|
||||
return;
|
||||
};
|
||||
let client = make_llm_client().await.unwrap();
|
||||
|
||||
let mut graph = Graph::new("RealLLMTwoStage");
|
||||
graph.attrs.insert(
|
||||
|
|
@ -6137,15 +6122,9 @@ mod real_llm {
|
|||
assert_eq!(last_stage, Some("review"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn real_llm_human_gate_auto_approve() {
|
||||
let client = if let Some(c) = make_llm_client().await {
|
||||
c
|
||||
} else {
|
||||
eprintln!("Skipping: ANTHROPIC_API_KEY not set");
|
||||
return;
|
||||
};
|
||||
let client = make_llm_client().await.unwrap();
|
||||
|
||||
let mut graph = Graph::new("RealLLMGate");
|
||||
graph.attrs.insert(
|
||||
|
|
@ -6286,15 +6265,9 @@ mod real_llm {
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn real_llm_one_shot_pipeline() {
|
||||
let client = if let Some(c) = make_llm_client().await {
|
||||
c
|
||||
} else {
|
||||
eprintln!("Skipping: ANTHROPIC_API_KEY not set");
|
||||
return;
|
||||
};
|
||||
let client = make_llm_client().await.unwrap();
|
||||
|
||||
let mut graph = Graph::new("RealLLMOneShot");
|
||||
graph.attrs.insert(
|
||||
|
|
@ -8105,11 +8078,8 @@ timeout_ms = 120000
|
|||
|
||||
// --- Prompt/Agent hook E2E with real LLM ---
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires ANTHROPIC_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn hook_prompt_proceed_allows_run() {
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
let hooks = vec![fabro_hooks::HookDefinition {
|
||||
name: Some("prompt-proceed".into()),
|
||||
event: fabro_hooks::HookEvent::RunStart,
|
||||
|
|
@ -8132,11 +8102,8 @@ async fn hook_prompt_proceed_allows_run() {
|
|||
assert_eq!(outcome.status, StageStatus::Success);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires ANTHROPIC_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn hook_prompt_block_prevents_run() {
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
// Use a factual question that evaluates to false: "Is 2+2=5?"
|
||||
let hooks = vec![fabro_hooks::HookDefinition {
|
||||
name: Some("prompt-block".into()),
|
||||
|
|
@ -8163,11 +8130,8 @@ async fn hook_prompt_block_prevents_run() {
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires ANTHROPIC_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn hook_agent_proceed_allows_run() {
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
let hooks = vec![fabro_hooks::HookDefinition {
|
||||
name: Some("agent-proceed".into()),
|
||||
event: fabro_hooks::HookEvent::RunStart,
|
||||
|
|
@ -8191,11 +8155,8 @@ async fn hook_agent_proceed_allows_run() {
|
|||
assert_eq!(outcome.status, StageStatus::Success);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires ANTHROPIC_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn hook_agent_with_tool_use() {
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let marker = dir.path().join("hook_check.txt");
|
||||
std::fs::write(&marker, "READY").unwrap();
|
||||
|
|
@ -8267,11 +8228,8 @@ async fn hooks_do_not_duplicate_workflow_events() {
|
|||
// E2E test with real LLM
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "requires ANTHROPIC_API_KEY"]
|
||||
#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))]
|
||||
async fn arc_e2e_with_real_llm() {
|
||||
dotenvy::dotenv().ok();
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dir_path = dir.path().to_str().unwrap().to_string();
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue