diff --git a/AGENTS.md b/AGENTS.md index 9539a80c3..493968fda 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -107,3 +107,4 @@ Never run `cargo insta accept` without first checking what's pending — it acce - `fabro run ` — run a workflow by name (resolves `fabro/workflows//workflow.toml`), e.g. `fabro run repl` - Use `--no-retro` to skip the retro step and finish faster +- `#[e2e_test(live("VAR"))]` — use for tests requiring secrets; it generates `#[ignore]` plus env gating. Use `#[e2e_test()]` for sandbox-only E2E tests with no secret deps. Behavior is controlled by `FABRO_TEST_MODE` (`off`, `live`, `strict`), and `cargo nextest run --profile e2e ...` implies `strict`. diff --git a/Cargo.lock b/Cargo.lock index 4309f9aa5..e44583067 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1406,12 +1406,13 @@ dependencies = [ "chrono", "clap", "dirs", - "dotenvy", "fabro-config", "fabro-llm", + "fabro-macros", "fabro-mcp", "fabro-model", "fabro-sandbox", + "fabro-test", "fabro-util", "futures", "glob", @@ -1488,6 +1489,7 @@ dependencies = [ "fabro-hooks", "fabro-interview", "fabro-llm", + "fabro-macros", "fabro-mcp", "fabro-model", "fabro-openai-oauth", @@ -1668,8 +1670,9 @@ dependencies = [ "clap", "cli-table", "dialoguer", - "dotenvy", + "fabro-macros", "fabro-model", + "fabro-test", "fabro-util", "futures", "http", @@ -1994,7 +1997,6 @@ dependencies = [ "base64", "chrono", "dirs", - "dotenvy", "fabro-agent", "fabro-checkpoint", "fabro-config", @@ -2005,11 +2007,13 @@ dependencies = [ "fabro-hooks", "fabro-interview", "fabro-llm", + "fabro-macros", "fabro-mcp", "fabro-model", "fabro-retro", "fabro-sandbox", "fabro-store", + "fabro-test", "fabro-types", "fabro-util", "fabro-validate", diff --git a/files-internal/testing-strategy.md b/files-internal/testing-strategy.md index cd565ec80..03731233c 100644 --- a/files-internal/testing-strategy.md +++ b/files-internal/testing-strategy.md @@ -258,6 +258,19 @@ Tests should be stable on any developer machine. - Mark tests that require real providers, real sandboxes, or external services with `#[ignore]` and a clear reason. - Filter or normalize machine-specific output in snapshots. +## E2E env vars + +Secret-backed E2E tests should use the shared test helpers instead of ad hoc env handling. + +- Use `#[e2e_test(live("VAR"))]` for tests that require one or more env vars. +- Use `#[e2e_test()]` for sandbox-only E2E tests that do not require secrets. +- `FABRO_TEST_MODE=off` skips all `#[e2e_test]` tests. +- `FABRO_TEST_MODE=live` runs `#[e2e_test]` tests and skips any missing `live("VAR")` dependencies. +- `FABRO_TEST_MODE=strict` runs `#[e2e_test]` tests and fails immediately if any required env var is missing. +- `cargo nextest run --profile e2e ...` also implies strict mode through nextest's runtime `NEXTEST_PROFILE=e2e` environment variable. +- Use `fabro_test::require_env()` inside helper functions that need to read E2E env vars after the test-level guard has run. +- Do not call `dotenvy` from tests. E2E env vars must come from the shell or the test runner profile. + If a test depends on `.env` or real credentials, it must be clearly marked and opt-in. ## Naming rules diff --git a/lib/crates/fabro-agent/Cargo.toml b/lib/crates/fabro-agent/Cargo.toml index 91a24d70d..2657bd3f3 100644 --- a/lib/crates/fabro-agent/Cargo.toml +++ b/lib/crates/fabro-agent/Cargo.toml @@ -24,7 +24,6 @@ workspace = true [dependencies] clap.workspace = true anyhow.workspace = true -dotenvy.workspace = true fabro-config = { path = "../fabro-config", features = ["clap"] } fabro-llm = { path = "../fabro-llm" } fabro-model = { path = "../fabro-model" } @@ -54,6 +53,7 @@ libc = "0.2" [dev-dependencies] tokio = { workspace = true, features = ["test-util", "macros"] } tempfile = "3" -dotenvy = { workspace = true } paste = "1" fabro-sandbox = { path = "../fabro-sandbox", features = ["test-support"] } +fabro-macros = { path = "../fabro-macros" } +fabro-test = { workspace = true } diff --git a/lib/crates/fabro-agent/src/tools.rs b/lib/crates/fabro-agent/src/tools.rs index acc885ea2..a78c92bf3 100644 --- a/lib/crates/fabro-agent/src/tools.rs +++ b/lib/crates/fabro-agent/src/tools.rs @@ -1414,8 +1414,7 @@ mod tests { assert_eq!(html_to_markdown(plain), plain); } - #[tokio::test] - #[ignore = "requires BRAVE_SEARCH_API_KEY env var"] + #[fabro_macros::e2e_test(live("BRAVE_SEARCH_API_KEY"))] async fn web_search_returns_results() { let api_key = std::env::var("BRAVE_SEARCH_API_KEY") .expect("BRAVE_SEARCH_API_KEY must be set to run this test"); diff --git a/lib/crates/fabro-agent/tests/it/parity_matrix.rs b/lib/crates/fabro-agent/tests/it/parity_matrix.rs index 06ee6aca7..a0d709258 100644 --- a/lib/crates/fabro-agent/tests/it/parity_matrix.rs +++ b/lib/crates/fabro-agent/tests/it/parity_matrix.rs @@ -58,7 +58,6 @@ fn build_profile(provider: Provider, model: &str, client: &Client) -> Box Session { - dotenvy::dotenv().ok(); let client = Client::from_env().await.expect("Client::from_env failed"); let mut profile = build_profile(provider, model, &client); let env = Arc::new(LocalSandbox::new(cwd.to_path_buf())); @@ -117,7 +116,6 @@ async fn make_session_with_config( cwd: &Path, config: SessionConfig, ) -> Session { - dotenvy::dotenv().ok(); let client = Client::from_env().await.expect("Client::from_env failed"); let profile: Arc = Arc::from(build_profile(provider, model, &client)); let env = Arc::new(LocalSandbox::new(cwd.to_path_buf())); @@ -125,10 +123,9 @@ async fn make_session_with_config( } macro_rules! provider_test { - ($scenario:ident, $provider:expr, $model:expr, $prefix:ident) => { + ($scenario:ident, $provider:expr, $model:expr, $prefix:ident, keys = [$($key:expr),+ $(,)?]) => { paste::paste! { - #[tokio::test] - #[ignore = "requires LLM API keys"] + #[fabro_macros::e2e_test($(live($key)),+)] async fn [<$prefix _ $scenario>]() { let tmp = tempfile::tempdir().expect("failed to create tempdir"); let mut session = make_session($provider, $model, tmp.path()).await; @@ -145,21 +142,53 @@ macro_rules! provider_tests { $scenario, Provider::Anthropic, "claude-haiku-4-5", - anthropic + anthropic, + keys = ["ANTHROPIC_API_KEY"] + ); + provider_test!( + $scenario, + Provider::OpenAi, + "gpt-5.4-mini", + openai, + keys = ["OPENAI_API_KEY"] ); - provider_test!($scenario, Provider::OpenAi, "gpt-5.4-mini", openai); provider_test!( $scenario, Provider::Gemini, "gemini-3-flash-preview", - gemini + gemini, + keys = ["GEMINI_API_KEY"] + ); + provider_test!( + $scenario, + Provider::Kimi, + "kimi-k2.5", + kimi, + keys = ["KIMI_API_KEY"] ); - provider_test!($scenario, Provider::Kimi, "kimi-k2.5", kimi); #[cfg(feature = "quarantine")] - provider_test!($scenario, Provider::Zai, "glm-4.7", zai); - provider_test!($scenario, Provider::Minimax, "minimax-m2.5", minimax); + provider_test!( + $scenario, + Provider::Zai, + "glm-4.7", + zai, + keys = ["ZAI_API_KEY"] + ); + provider_test!( + $scenario, + Provider::Minimax, + "minimax-m2.5", + minimax, + keys = ["MINIMAX_API_KEY"] + ); #[cfg(feature = "quarantine")] - provider_test!($scenario, Provider::Inception, "mercury-2", inception); + provider_test!( + $scenario, + Provider::Inception, + "mercury-2", + inception, + keys = ["INCEPTION_API_KEY"] + ); }; } @@ -176,8 +205,109 @@ provider_tests!(steering_mid_task); provider_tests!(follow_up); provider_tests!(subagent_spawn); -provider_tests!(web_fetch); -provider_tests!(web_search); +provider_test!( + web_fetch, + Provider::Anthropic, + "claude-haiku-4-5", + anthropic, + keys = ["ANTHROPIC_API_KEY"] +); +provider_test!( + web_fetch, + Provider::OpenAi, + "gpt-5.4-mini", + openai, + keys = ["OPENAI_API_KEY"] +); +provider_test!( + web_fetch, + Provider::Gemini, + "gemini-3-flash-preview", + gemini, + keys = ["GEMINI_API_KEY"] +); +provider_test!( + web_fetch, + Provider::Kimi, + "kimi-k2.5", + kimi, + keys = ["KIMI_API_KEY", "OPENAI_API_KEY"] +); +#[cfg(feature = "quarantine")] +provider_test!( + web_fetch, + Provider::Zai, + "glm-4.7", + zai, + keys = ["ZAI_API_KEY", "OPENAI_API_KEY"] +); +provider_test!( + web_fetch, + Provider::Minimax, + "minimax-m2.5", + minimax, + keys = ["MINIMAX_API_KEY", "OPENAI_API_KEY"] +); +#[cfg(feature = "quarantine")] +provider_test!( + web_fetch, + Provider::Inception, + "mercury-2", + inception, + keys = ["INCEPTION_API_KEY", "OPENAI_API_KEY"] +); + +provider_test!( + web_search, + Provider::Anthropic, + "claude-haiku-4-5", + anthropic, + keys = ["ANTHROPIC_API_KEY", "BRAVE_SEARCH_API_KEY"] +); +provider_test!( + web_search, + Provider::OpenAi, + "gpt-5.4-mini", + openai, + keys = ["OPENAI_API_KEY", "BRAVE_SEARCH_API_KEY"] +); +provider_test!( + web_search, + Provider::Gemini, + "gemini-3-flash-preview", + gemini, + keys = ["GEMINI_API_KEY", "BRAVE_SEARCH_API_KEY"] +); +provider_test!( + web_search, + Provider::Kimi, + "kimi-k2.5", + kimi, + keys = ["KIMI_API_KEY", "BRAVE_SEARCH_API_KEY"] +); +#[cfg(feature = "quarantine")] +provider_test!( + web_search, + Provider::Zai, + "glm-4.7", + zai, + keys = ["ZAI_API_KEY", "BRAVE_SEARCH_API_KEY"] +); +provider_test!( + web_search, + Provider::Minimax, + "minimax-m2.5", + minimax, + keys = ["MINIMAX_API_KEY", "BRAVE_SEARCH_API_KEY"] +); +#[cfg(feature = "quarantine")] +provider_test!( + web_search, + Provider::Inception, + "mercury-2", + inception, + keys = ["INCEPTION_API_KEY", "BRAVE_SEARCH_API_KEY"] +); // Scenarios below are only generated for providers where they are supported. // - multi_step_read_analyze_edit / provider_specific_editing: gpt-4o-mini is too @@ -194,20 +324,46 @@ macro_rules! non_openai_provider_tests { $scenario, Provider::Anthropic, "claude-haiku-4-5", - anthropic + anthropic, + keys = ["ANTHROPIC_API_KEY"] ); provider_test!( $scenario, Provider::Gemini, "gemini-3-flash-preview", - gemini + gemini, + keys = ["GEMINI_API_KEY"] + ); + provider_test!( + $scenario, + Provider::Kimi, + "kimi-k2.5", + kimi, + keys = ["KIMI_API_KEY"] ); - provider_test!($scenario, Provider::Kimi, "kimi-k2.5", kimi); #[cfg(feature = "quarantine")] - provider_test!($scenario, Provider::Zai, "glm-4.7", zai); - provider_test!($scenario, Provider::Minimax, "minimax-m2.5", minimax); + provider_test!( + $scenario, + Provider::Zai, + "glm-4.7", + zai, + keys = ["ZAI_API_KEY"] + ); + provider_test!( + $scenario, + Provider::Minimax, + "minimax-m2.5", + minimax, + keys = ["MINIMAX_API_KEY"] + ); #[cfg(feature = "quarantine")] - provider_test!($scenario, Provider::Inception, "mercury-2", inception); + provider_test!( + $scenario, + Provider::Inception, + "mercury-2", + inception, + keys = ["INCEPTION_API_KEY"] + ); }; } @@ -430,9 +586,8 @@ async fn scenario_follow_up(session: &mut Session, dir: &Path) { // Scenario 11: reasoning_effort // --------------------------------------------------------------------------- macro_rules! reasoning_effort_tests { - ($provider:expr, $model:expr, $test_name:ident) => { - #[tokio::test] - #[ignore = "requires LLM API keys"] + ($provider:expr, $model:expr, $test_name:ident, keys = [$($key:expr),+ $(,)?]) => { + #[fabro_macros::e2e_test($(live($key)),+)] async fn $test_name() { let tmp = tempfile::tempdir().expect("failed to create tempdir"); let config = SessionConfig { @@ -453,20 +608,42 @@ macro_rules! reasoning_effort_tests { reasoning_effort_tests!( Provider::Anthropic, "claude-haiku-4-5", - anthropic_reasoning_effort + anthropic_reasoning_effort, + keys = ["ANTHROPIC_API_KEY"] ); // gpt-5-mini does not support the reasoning.effort parameter, so no OpenAI test. reasoning_effort_tests!( Provider::Gemini, "gemini-3-flash-preview", - gemini_reasoning_effort + gemini_reasoning_effort, + keys = ["GEMINI_API_KEY"] +); +reasoning_effort_tests!( + Provider::Kimi, + "kimi-k2.5", + kimi_reasoning_effort, + keys = ["KIMI_API_KEY"] ); -reasoning_effort_tests!(Provider::Kimi, "kimi-k2.5", kimi_reasoning_effort); #[cfg(feature = "quarantine")] -reasoning_effort_tests!(Provider::Zai, "glm-4.7", zai_reasoning_effort); -reasoning_effort_tests!(Provider::Minimax, "minimax-m2.5", minimax_reasoning_effort); +reasoning_effort_tests!( + Provider::Zai, + "glm-4.7", + zai_reasoning_effort, + keys = ["ZAI_API_KEY"] +); +reasoning_effort_tests!( + Provider::Minimax, + "minimax-m2.5", + minimax_reasoning_effort, + keys = ["MINIMAX_API_KEY"] +); #[cfg(feature = "quarantine")] -reasoning_effort_tests!(Provider::Inception, "mercury-2", inception_reasoning_effort); +reasoning_effort_tests!( + Provider::Inception, + "mercury-2", + inception_reasoning_effort, + keys = ["INCEPTION_API_KEY"] +); // --------------------------------------------------------------------------- // Scenario 12: subagent_spawn @@ -486,9 +663,8 @@ async fn scenario_subagent_spawn(session: &mut Session, dir: &Path) { // Scenario 13: loop_detection // --------------------------------------------------------------------------- macro_rules! loop_detection_tests { - ($provider:expr, $model:expr, $test_name:ident) => { - #[tokio::test] - #[ignore = "requires LLM API keys"] + ($provider:expr, $model:expr, $test_name:ident, keys = [$($key:expr),+ $(,)?]) => { + #[fabro_macros::e2e_test($(live($key)),+)] async fn $test_name() { let tmp = tempfile::tempdir().expect("failed to create tempdir"); let config = SessionConfig { @@ -509,20 +685,47 @@ macro_rules! loop_detection_tests { loop_detection_tests!( Provider::Anthropic, "claude-haiku-4-5", - anthropic_loop_detection + anthropic_loop_detection, + keys = ["ANTHROPIC_API_KEY"] +); +loop_detection_tests!( + Provider::OpenAi, + "gpt-5.4-mini", + openai_loop_detection, + keys = ["OPENAI_API_KEY"] ); -loop_detection_tests!(Provider::OpenAi, "gpt-5.4-mini", openai_loop_detection); loop_detection_tests!( Provider::Gemini, "gemini-3-flash-preview", - gemini_loop_detection + gemini_loop_detection, + keys = ["GEMINI_API_KEY"] +); +loop_detection_tests!( + Provider::Kimi, + "kimi-k2.5", + kimi_loop_detection, + keys = ["KIMI_API_KEY"] ); -loop_detection_tests!(Provider::Kimi, "kimi-k2.5", kimi_loop_detection); #[cfg(feature = "quarantine")] -loop_detection_tests!(Provider::Zai, "glm-4.7", zai_loop_detection); -loop_detection_tests!(Provider::Minimax, "minimax-m2.5", minimax_loop_detection); +loop_detection_tests!( + Provider::Zai, + "glm-4.7", + zai_loop_detection, + keys = ["ZAI_API_KEY"] +); +loop_detection_tests!( + Provider::Minimax, + "minimax-m2.5", + minimax_loop_detection, + keys = ["MINIMAX_API_KEY"] +); #[cfg(feature = "quarantine")] -loop_detection_tests!(Provider::Inception, "mercury-2", inception_loop_detection); +loop_detection_tests!( + Provider::Inception, + "mercury-2", + inception_loop_detection, + keys = ["INCEPTION_API_KEY"] +); // --------------------------------------------------------------------------- // Scenario 14: error_recovery diff --git a/lib/crates/fabro-cli/Cargo.toml b/lib/crates/fabro-cli/Cargo.toml index dc375eede..7f3d138f3 100644 --- a/lib/crates/fabro-cli/Cargo.toml +++ b/lib/crates/fabro-cli/Cargo.toml @@ -100,3 +100,4 @@ serde_json.workspace = true tempfile = "3" httpmock = "0.8" fabro-test = { workspace = true } +fabro-macros = { path = "../fabro-macros" } diff --git a/lib/crates/fabro-cli/tests/it/cmd/doctor.rs b/lib/crates/fabro-cli/tests/it/cmd/doctor.rs index 90880b824..31e26163f 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/doctor.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/doctor.rs @@ -63,10 +63,8 @@ fn dry_run_flag() { "); } -#[test] -#[ignore = "scenario: requires ANTHROPIC_API_KEY"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn live_doctor() { - dotenvy::dotenv().ok(); let context = test_context!(); context.doctor().assert().success(); } diff --git a/lib/crates/fabro-cli/tests/it/cmd/exec.rs b/lib/crates/fabro-cli/tests/it/cmd/exec.rs index 8d8166ccd..67404a174 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/exec.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/exec.rs @@ -110,10 +110,8 @@ fn exec_uses_user_config_defaults() { "); } -#[test] -#[ignore = "requires API key"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn exec_creates_file() { - dotenvy::dotenv().ok(); let context = test_context!(); context .exec_cmd() @@ -140,10 +138,8 @@ fn exec_creates_file() { ); } -#[test] -#[ignore = "requires API key"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn exec_shell_command() { - dotenvy::dotenv().ok(); let context = test_context!(); context .exec_cmd() @@ -163,10 +159,8 @@ fn exec_shell_command() { .success(); } -#[test] -#[ignore = "requires API key"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn exec_read_only_blocks_write() { - dotenvy::dotenv().ok(); let context = test_context!(); context .exec_cmd() @@ -190,10 +184,8 @@ fn exec_read_only_blocks_write() { ); } -#[test] -#[ignore = "requires API key"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn exec_json_output_format() { - dotenvy::dotenv().ok(); let context = test_context!(); let output = context .exec_cmd() @@ -229,10 +221,8 @@ fn exec_json_output_format() { ); } -#[test] -#[ignore = "requires API key"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn exec_read_and_edit() { - dotenvy::dotenv().ok(); let context = test_context!(); context.write_temp("data.txt", "old content"); context diff --git a/lib/crates/fabro-cli/tests/it/cmd/llm.rs b/lib/crates/fabro-cli/tests/it/cmd/llm.rs index d95f762df..a5faf364e 100644 --- a/lib/crates/fabro-cli/tests/it/cmd/llm.rs +++ b/lib/crates/fabro-cli/tests/it/cmd/llm.rs @@ -88,10 +88,8 @@ fn prompt_concatenates_stdin_and_arg() { result.stderr(predicate::str::contains("no prompt provided").not()); } -#[test] -#[ignore = "requires API key"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn prompt_no_stream_generates_response() { - dotenvy::dotenv().ok(); let context = test_context!(); context .llm() @@ -107,10 +105,8 @@ fn prompt_no_stream_generates_response() { .stdout(predicate::str::is_empty().not()); } -#[test] -#[ignore = "requires API key"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn prompt_stream_generates_response() { - dotenvy::dotenv().ok(); let context = test_context!(); context .llm() @@ -125,10 +121,8 @@ fn prompt_stream_generates_response() { .stdout(predicate::str::is_empty().not()); } -#[test] -#[ignore = "requires API key"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn prompt_usage_shows_tokens() { - dotenvy::dotenv().ok(); let context = test_context!(); context .llm() @@ -145,10 +139,8 @@ fn prompt_usage_shows_tokens() { .stderr(predicate::str::contains("Tokens:")); } -#[test] -#[ignore = "requires API key"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn prompt_schema_no_stream_generates_json() { - dotenvy::dotenv().ok(); let context = test_context!(); let assert = context .llm() @@ -169,10 +161,8 @@ fn prompt_schema_no_stream_generates_json() { ); } -#[test] -#[ignore = "requires API key"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn prompt_schema_stream_generates_json() { - dotenvy::dotenv().ok(); let context = test_context!(); let assert = context .llm() @@ -193,10 +183,8 @@ fn prompt_schema_stream_generates_json() { ); } -#[test] -#[ignore = "requires API key"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn chat_multi_turn_with_system_prompt() { - dotenvy::dotenv().ok(); let context = test_context!(); let assert = context .command() diff --git a/lib/crates/fabro-cli/tests/it/scenario/exec.rs b/lib/crates/fabro-cli/tests/it/scenario/exec.rs index df3c4c6e1..d12e43ae0 100644 --- a/lib/crates/fabro-cli/tests/it/scenario/exec.rs +++ b/lib/crates/fabro-cli/tests/it/scenario/exec.rs @@ -2,10 +2,8 @@ use std::time::Duration; use fabro_test::test_context; -#[test] -#[ignore = "scenario: requires ANTHROPIC_API_KEY"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] fn test_exec_creates_file() { - dotenvy::dotenv().ok(); let context = test_context!(); let mut cmd = context.exec_cmd(); diff --git a/lib/crates/fabro-cli/tests/it/scenario/lifecycle.rs b/lib/crates/fabro-cli/tests/it/scenario/lifecycle.rs index 4bcf7684d..17a11aac6 100644 --- a/lib/crates/fabro-cli/tests/it/scenario/lifecycle.rs +++ b/lib/crates/fabro-cli/tests/it/scenario/lifecycle.rs @@ -4,10 +4,8 @@ use serde_json::Value; use super::{fixture, read_json, timeout_for}; use crate::support::fabro_json_snapshot; -#[test] -#[ignore = "scenario: requires local sandbox"] +#[fabro_macros::e2e_test()] fn local_run_lifecycle() { - dotenvy::dotenv().ok(); let context = test_context!(); let cmd = |args: &[&str]| -> assert_cmd::assert::Assert { diff --git a/lib/crates/fabro-cli/tests/it/workflow/agent_linear.rs b/lib/crates/fabro-cli/tests/it/workflow/agent_linear.rs index dc6b07450..b41d29595 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/agent_linear.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/agent_linear.rs @@ -2,10 +2,9 @@ use fabro_test::test_context; use super::{completed_nodes, find_run_dir, fixture, read_conclusion, sandbox_tests, timeout_for}; -sandbox_tests!(agent_linear); +sandbox_tests!(agent_linear, keys = ["ANTHROPIC_API_KEY"]); fn scenario_agent_linear(sandbox: &str) { - dotenvy::dotenv().ok(); let context = test_context!(); context diff --git a/lib/crates/fabro-cli/tests/it/workflow/command_agent_mixed.rs b/lib/crates/fabro-cli/tests/it/workflow/command_agent_mixed.rs index be92514b3..78c834073 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/command_agent_mixed.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/command_agent_mixed.rs @@ -2,10 +2,9 @@ use fabro_test::test_context; use super::{completed_nodes, find_run_dir, fixture, read_conclusion, sandbox_tests, timeout_for}; -sandbox_tests!(command_agent_mixed); +sandbox_tests!(command_agent_mixed, keys = ["ANTHROPIC_API_KEY"]); fn scenario_command_agent_mixed(sandbox: &str) { - dotenvy::dotenv().ok(); let context = test_context!(); context diff --git a/lib/crates/fabro-cli/tests/it/workflow/command_pipeline.rs b/lib/crates/fabro-cli/tests/it/workflow/command_pipeline.rs index 515a6ea24..56c85b034 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/command_pipeline.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/command_pipeline.rs @@ -5,7 +5,6 @@ use super::{completed_nodes, find_run_dir, fixture, read_conclusion, sandbox_tes sandbox_tests!(command_pipeline); fn scenario_command_pipeline(sandbox: &str) { - dotenvy::dotenv().ok(); let context = test_context!(); context diff --git a/lib/crates/fabro-cli/tests/it/workflow/conditional_branching.rs b/lib/crates/fabro-cli/tests/it/workflow/conditional_branching.rs index 678d86396..0b8e13373 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/conditional_branching.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/conditional_branching.rs @@ -5,7 +5,6 @@ use super::{completed_nodes, find_run_dir, fixture, read_conclusion, sandbox_tes sandbox_tests!(conditional_branching); fn scenario_conditional_branching(sandbox: &str) { - dotenvy::dotenv().ok(); let context = test_context!(); context diff --git a/lib/crates/fabro-cli/tests/it/workflow/full_stack.rs b/lib/crates/fabro-cli/tests/it/workflow/full_stack.rs index 50fae6256..c55c5acf4 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/full_stack.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/full_stack.rs @@ -5,10 +5,9 @@ use super::{ timeout_for, }; -sandbox_tests!(full_stack); +sandbox_tests!(full_stack, keys = ["ANTHROPIC_API_KEY"]); fn scenario_full_stack(sandbox: &str) { - dotenvy::dotenv().ok(); let context = test_context!(); context diff --git a/lib/crates/fabro-cli/tests/it/workflow/human_gate.rs b/lib/crates/fabro-cli/tests/it/workflow/human_gate.rs index 9207937a0..fa429f4b8 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/human_gate.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/human_gate.rs @@ -2,10 +2,9 @@ use fabro_test::test_context; use super::{completed_nodes, find_run_dir, fixture, read_conclusion, sandbox_tests, timeout_for}; -sandbox_tests!(human_gate); +sandbox_tests!(human_gate, keys = ["ANTHROPIC_API_KEY"]); fn scenario_human_gate(sandbox: &str) { - dotenvy::dotenv().ok(); let context = test_context!(); context diff --git a/lib/crates/fabro-cli/tests/it/workflow/mod.rs b/lib/crates/fabro-cli/tests/it/workflow/mod.rs index 3304b3105..42a1b8bb9 100644 --- a/lib/crates/fabro-cli/tests/it/workflow/mod.rs +++ b/lib/crates/fabro-cli/tests/it/workflow/mod.rs @@ -70,15 +70,16 @@ pub(super) fn find_run_dir(storage_dir: &Path) -> PathBuf { macro_rules! sandbox_tests { ($name:ident) => { + sandbox_tests!($name, keys = []); + }; + ($name:ident, keys = [$($key:expr),* $(,)?]) => { paste::paste! { - #[test] - #[ignore = "scenario: requires local sandbox"] + #[fabro_macros::e2e_test($(live($key)),*)] fn []() { []("local"); } - #[test] - #[ignore = "scenario: requires DAYTONA_API_KEY"] + #[fabro_macros::e2e_test(live("DAYTONA_API_KEY") $(, live($key))*)] fn []() { []("daytona"); } diff --git a/lib/crates/fabro-llm/Cargo.toml b/lib/crates/fabro-llm/Cargo.toml index c0f5f4259..636879597 100644 --- a/lib/crates/fabro-llm/Cargo.toml +++ b/lib/crates/fabro-llm/Cargo.toml @@ -43,6 +43,7 @@ fabro-util = { path = "../fabro-util" } http = "1" insta = { workspace = true } tokio = { workspace = true, features = ["test-util", "macros"] } -dotenvy.workspace = true httpmock = "0.8" -serde_json.workspace = true \ No newline at end of file +serde_json.workspace = true +fabro-macros = { path = "../fabro-macros" } +fabro-test = { workspace = true } diff --git a/lib/crates/fabro-llm/src/cli.rs b/lib/crates/fabro-llm/src/cli.rs index a8a810053..28e1c4cbc 100644 --- a/lib/crates/fabro-llm/src/cli.rs +++ b/lib/crates/fabro-llm/src/cli.rs @@ -1809,7 +1809,7 @@ mod tests { option: vec![], }; - let result = run_prompt_via_server(args, &server).await; + let result = run_prompt_via_server(args, &server, false).await; assert!(result.is_ok()); mock.assert_async().await; } @@ -1849,7 +1849,7 @@ data: {\"type\":\"finish\",\"finish_reason\":\"stop\",\"usage\":{\"input_tokens\ option: vec![], }; - let result = run_prompt_via_server(args, &server).await; + let result = run_prompt_via_server(args, &server, false).await; assert!(result.is_ok()); mock.assert_async().await; } diff --git a/lib/crates/fabro-llm/tests/integration.rs b/lib/crates/fabro-llm/tests/integration.rs index 64de86961..66c5ac24c 100644 --- a/lib/crates/fabro-llm/tests/integration.rs +++ b/lib/crates/fabro-llm/tests/integration.rs @@ -21,10 +21,8 @@ fn make_request(model: &str) -> Request { } } -#[tokio::test] -#[ignore = "requires ANTHROPIC_API_KEY"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn anthropic_complete() { - dotenvy::dotenv().ok(); let api_key = std::env::var("ANTHROPIC_API_KEY").expect("ANTHROPIC_API_KEY must be set"); let adapter = AnthropicAdapter::new(api_key); let request = make_request("claude-haiku-4-5"); @@ -40,10 +38,8 @@ async fn anthropic_complete() { assert_eq!(response.provider, "anthropic"); } -#[tokio::test] -#[ignore = "requires OPENAI_API_KEY"] +#[fabro_macros::e2e_test(live("OPENAI_API_KEY"))] async fn openai_complete() { - dotenvy::dotenv().ok(); let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY must be set"); let adapter = OpenAiAdapter::new(api_key); let request = make_request("gpt-4o-mini"); @@ -59,8 +55,7 @@ async fn openai_complete() { assert_eq!(response.provider, "openai"); } -#[tokio::test] -#[ignore = "requires OPENAI_API_KEY"] +#[fabro_macros::e2e_test(live("OPENAI_API_KEY"))] async fn openai_gpt_5_3_codex_complete() { let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY must be set"); let adapter = OpenAiAdapter::new(api_key); @@ -76,10 +71,8 @@ async fn openai_gpt_5_3_codex_complete() { assert_eq!(response.provider, "openai"); } -#[tokio::test] -#[ignore = "requires GEMINI_API_KEY"] +#[fabro_macros::e2e_test(live("GEMINI_API_KEY"))] async fn gemini_complete() { - dotenvy::dotenv().ok(); let api_key = std::env::var("GEMINI_API_KEY").expect("GEMINI_API_KEY must be set"); let adapter = GeminiAdapter::new(api_key); let request = make_request("gemini-2.5-flash"); @@ -168,28 +161,22 @@ async fn run_multi_turn_cache_test( ); } -#[tokio::test] -#[ignore = "requires ANTHROPIC_API_KEY"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn anthropic_multi_turn_cache() { - dotenvy::dotenv().ok(); let api_key = std::env::var("ANTHROPIC_API_KEY").expect("ANTHROPIC_API_KEY must be set"); let adapter = AnthropicAdapter::new(api_key); run_multi_turn_cache_test(&adapter, "claude-haiku-4-5", 0.5).await; } -#[tokio::test] -#[ignore = "requires OPENAI_API_KEY"] +#[fabro_macros::e2e_test(live("OPENAI_API_KEY"))] async fn openai_multi_turn_cache() { - dotenvy::dotenv().ok(); let api_key = std::env::var("OPENAI_API_KEY").expect("OPENAI_API_KEY must be set"); let adapter = OpenAiAdapter::new(api_key); run_multi_turn_cache_test(&adapter, "gpt-4o-mini", 0.5).await; } -#[tokio::test] -#[ignore = "requires GEMINI_API_KEY"] +#[fabro_macros::e2e_test(live("GEMINI_API_KEY"))] async fn gemini_multi_turn_cache() { - dotenvy::dotenv().ok(); let api_key = std::env::var("GEMINI_API_KEY").expect("GEMINI_API_KEY must be set"); let adapter = GeminiAdapter::new(api_key); run_multi_turn_cache_test(&adapter, "gemini-2.5-flash", 0.5).await; diff --git a/lib/crates/fabro-macros/src/lib.rs b/lib/crates/fabro-macros/src/lib.rs index 0a212437f..369ccd66f 100644 --- a/lib/crates/fabro-macros/src/lib.rs +++ b/lib/crates/fabro-macros/src/lib.rs @@ -1,6 +1,34 @@ use proc_macro::TokenStream; use quote::quote; -use syn::{Data, DeriveInput, Fields, parse_macro_input}; +use syn::parse::{Parse, ParseStream}; +use syn::punctuated::Punctuated; +use syn::{ + Data, DeriveInput, Fields, Ident, ItemFn, LitStr, Token, parenthesized, parse_macro_input, +}; + +enum E2eRequirement { + Live(LitStr), +} + +impl Parse for E2eRequirement { + fn parse(input: ParseStream<'_>) -> syn::Result { + let ident = input.parse::()?; + if ident != "live" { + return Err(syn::Error::new( + ident.span(), + "expected `live(\"ENV_VAR\")`", + )); + } + + let content; + parenthesized!(content in input); + let env_var = content.parse::()?; + if !content.is_empty() { + return Err(content.error("expected a single string literal")); + } + Ok(Self::Live(env_var)) + } +} #[proc_macro_derive(Combine)] pub fn derive_combine(input: TokenStream) -> TokenStream { @@ -50,3 +78,73 @@ pub fn derive_combine(input: TokenStream) -> TokenStream { } .into() } + +#[proc_macro_attribute] +pub fn e2e_test(attr: TokenStream, item: TokenStream) -> TokenStream { + let requirements = + parse_macro_input!(attr with Punctuated::::parse_terminated); + let input = parse_macro_input!(item as ItemFn); + + let attrs = input.attrs; + let vis = input.vis; + let sig = input.sig; + let block = input.block; + + let env_vars: Vec<_> = requirements + .into_iter() + .map(|requirement| match requirement { + E2eRequirement::Live(env_var) => env_var, + }) + .collect(); + + let ignore_reason = if env_vars.is_empty() { + "e2e".to_string() + } else { + let joined = env_vars + .iter() + .map(LitStr::value) + .collect::>() + .join(", "); + format!("e2e: {joined}") + }; + + let test_attr = if sig.asyncness.is_some() { + quote!(#[tokio::test]) + } else { + quote!(#[test]) + }; + + let env_guards = env_vars.iter().map(|env_var| { + let env_name = env_var.value(); + let strict_message = format!("{env_name} not set (FABRO_TEST_MODE=strict)"); + let skip_message = format!("skipping: {env_name} not set"); + quote! { + if ::std::env::var(#env_var).is_err() { + if __mode == ::fabro_test::TestMode::Strict { + panic!(#strict_message); + } + eprintln!(#skip_message); + return; + } + } + }); + + quote! { + #(#attrs)* + #test_attr + #[ignore = #ignore_reason] + #vis #sig { + let __mode = ::fabro_test::TestMode::from_env(); + + if __mode == ::fabro_test::TestMode::Off { + eprintln!("skipping: FABRO_TEST_MODE is off"); + return; + } + + #(#env_guards)* + + #block + } + } + .into() +} diff --git a/lib/crates/fabro-test/src/lib.rs b/lib/crates/fabro-test/src/lib.rs index b62bdd056..88a226b77 100644 --- a/lib/crates/fabro-test/src/lib.rs +++ b/lib/crates/fabro-test/src/lib.rs @@ -22,6 +22,43 @@ static INSTA_FILTERS: &[(&str, &str)] = &[ (r"\\([\w\d])", "/$1"), ]; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum TestMode { + #[default] + Off, + Live, + Strict, +} + +impl TestMode { + #[must_use] + pub fn from_env() -> Self { + match std::env::var("FABRO_TEST_MODE").as_deref() { + Ok("live") => Self::Live, + Ok("strict") => Self::Strict, + _ => match std::env::var("NEXTEST_PROFILE").as_deref() { + Ok("e2e") => Self::Strict, + _ => Self::Off, + }, + } + } +} + +/// Read an env var required by an E2E test, with mode-aware skip/strict behavior. +#[must_use] +pub fn require_env(name: &str) -> Option { + match std::env::var(name) { + Ok(value) => Some(value), + Err(_) => { + if TestMode::from_env() == TestMode::Strict { + panic!("{name} not set (FABRO_TEST_MODE=strict)"); + } + eprintln!("skipping: {name} not set"); + None + } + } +} + /// An isolated test context for running fabro CLI commands. /// /// Creates temporary directories for home, storage, and working directory, diff --git a/lib/crates/fabro-workflow/Cargo.toml b/lib/crates/fabro-workflow/Cargo.toml index f70831eb9..25ea58832 100644 --- a/lib/crates/fabro-workflow/Cargo.toml +++ b/lib/crates/fabro-workflow/Cargo.toml @@ -18,7 +18,6 @@ workspace = true [dependencies] anyhow.workspace = true -dotenvy.workspace = true fabro-agent = { path = "../fabro-agent" } fabro-config = { path = "../fabro-config" } fabro-graphviz = { path = "../fabro-graphviz" } @@ -66,6 +65,7 @@ toml.workspace = true fabro-mcp = { path = "../fabro-mcp" } tokio = { workspace = true, features = ["test-util", "macros"] } tempfile = "3" -dotenvy.workspace = true assert_cmd = "2" predicates = "3" +fabro-macros = { path = "../fabro-macros" } +fabro-test = { workspace = true } diff --git a/lib/crates/fabro-workflow/tests/it/daytona_integration.rs b/lib/crates/fabro-workflow/tests/it/daytona_integration.rs index a4988a26c..59101951d 100644 --- a/lib/crates/fabro-workflow/tests/it/daytona_integration.rs +++ b/lib/crates/fabro-workflow/tests/it/daytona_integration.rs @@ -52,18 +52,12 @@ async fn create_env() -> DaytonaSandbox { async fn create_env_with_github_app( github_app: Option, ) -> DaytonaSandbox { - dotenvy::dotenv().ok(); - if let Some(home) = dirs::home_dir() { - dotenvy::from_path(home.join(".fabro/.env")).ok(); - } DaytonaSandbox::new(DaytonaConfig::default(), github_app, None, None) .await .expect("Failed to create Daytona client — is DAYTONA_API_KEY set?") } fn load_github_app_credentials() -> fabro_github::GitHubAppCredentials { - dotenvy::dotenv().ok(); - // Read app_id from ~/.fabro/server.toml let home = dirs::home_dir().expect("No home directory"); let config_path = home.join(".fabro/server.toml"); @@ -100,8 +94,7 @@ fn load_github_app_credentials() -> fabro_github::GitHubAppCredentials { } } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_exec_command() { let creds = load_github_app_credentials(); let env = create_env_with_github_app(Some(creds)).await; @@ -117,8 +110,7 @@ async fn daytona_exec_command() { env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_exec_command_with_pipe() { let creds = load_github_app_credentials(); let env = create_env_with_github_app(Some(creds)).await; @@ -134,8 +126,7 @@ async fn daytona_exec_command_with_pipe() { env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_exec_command_cancelled() { let creds = load_github_app_credentials(); let env = create_env_with_github_app(Some(creds)).await; @@ -163,8 +154,7 @@ async fn daytona_exec_command_cancelled() { env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_exec_command_local_timeout() { let creds = load_github_app_credentials(); let env = create_env_with_github_app(Some(creds)).await; @@ -197,8 +187,7 @@ async fn daytona_exec_command_local_timeout() { env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_file_round_trip() { let env = create_env().await; env.initialize().await.unwrap(); @@ -223,8 +212,7 @@ async fn daytona_file_round_trip() { env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_full_lifecycle() { let env = create_env().await; @@ -249,13 +237,10 @@ async fn daytona_full_lifecycle() { env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_snapshot_sandbox() { use fabro_sandbox::daytona::DaytonaSnapshotConfig; - dotenvy::dotenv().ok(); - let config = DaytonaConfig { auto_stop_interval: Some(60), snapshot: Some(DaytonaSnapshotConfig { @@ -287,8 +272,7 @@ async fn daytona_snapshot_sandbox() { env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_artifact_sync_uploads_and_rewrites_pointer() { let env = create_env().await; env.initialize().await.unwrap(); @@ -364,8 +348,7 @@ impl Handler for LargeOutputHandler { } } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_pipeline_artifact_offload_and_sync() { let env = create_env().await; env.initialize().await.unwrap(); @@ -521,8 +504,7 @@ async fn setup_daytona_git(sandbox: &dyn Sandbox) -> (RunId, String, String) { (run_id, base_sha, branch_name) } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_git_checkpoint_remote_emits_events() { let env = create_env().await; env.initialize().await.unwrap(); @@ -682,8 +664,7 @@ use fabro_workflow::handler::parallel::ParallelHandler; /// End-to-end: parallel branches get isolated worktrees in Daytona sandbox, /// fan-in fast-forwards to winner. -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_parallel_git_branching_e2e() { let env = create_env().await; env.initialize().await.unwrap(); @@ -909,7 +890,6 @@ use fabro_workflow::handler::llm::AgentCliBackend; /// Installs the CLI tool in the sandbox, then runs the AgentCliBackend against it. async fn run_daytona_cli_test(provider: Provider, model: &str, install_command: &str) { let creds = load_github_app_credentials(); - dotenvy::dotenv().ok(); let config = DaytonaConfig { snapshot: Some(DaytonaSnapshotConfig { name: "daytona-medium".into(), @@ -1016,8 +996,7 @@ async fn run_daytona_cli_test(provider: Provider, model: &str, install_command: env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] // requires DAYTONA_API_KEY + Claude CLI auth +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_cli_claude() { run_daytona_cli_test( Provider::Anthropic, @@ -1027,14 +1006,12 @@ async fn daytona_cli_claude() { .await; } -#[tokio::test] -#[ignore] // requires DAYTONA_API_KEY + OpenAI/Codex auth +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_cli_codex() { run_daytona_cli_test(Provider::OpenAi, "o4-mini", "npm install -g @openai/codex").await; } -#[tokio::test] -#[ignore] // requires DAYTONA_API_KEY + Gemini auth +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_cli_gemini() { run_daytona_cli_test( Provider::Gemini, @@ -1052,8 +1029,7 @@ use fabro_workflow::git::MetadataStore; /// End-to-end test: pipeline with git checkpointing enabled + `meta_branch` /// writes shadow branch on the host repo and includes `Fabro-Checkpoint` trailer in sandbox commits. -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_git_checkpoint_with_shadow_branch() { let env = create_env().await; env.initialize().await.unwrap(); @@ -1237,8 +1213,7 @@ impl Handler for AssetCreatorHandler { /// Daytona sandbox: asset collection discovers files on the remote sandbox and /// downloads them to the local logs directory. -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_asset_collection() { let env = create_env().await; env.initialize().await.unwrap(); @@ -1325,8 +1300,7 @@ async fn daytona_asset_collection() { env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_ssh_access() { let env = create_env().await; env.initialize().await.unwrap(); @@ -1341,8 +1315,7 @@ async fn daytona_ssh_access() { env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_ssh_access_before_init_fails() { let env = create_env().await; @@ -1360,8 +1333,7 @@ async fn daytona_ssh_access_before_init_fails() { /// E2E: Clone the current (private) repo using GitHub App IAT credentials. /// Verifies the full flow: JWT signing, installation lookup, token creation, clone. -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_clone_private_repo_with_github_app_iat() { let creds = load_github_app_credentials(); let env = create_env_with_github_app(Some(creds)).await; @@ -1418,8 +1390,7 @@ async fn daytona_clone_private_repo_with_github_app_iat() { } /// E2E: Verify that repos in an installed org get credentials (needed for pushing). -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_clone_public_repo_gets_credentials() { let creds = load_github_app_credentials(); @@ -1441,8 +1412,7 @@ async fn daytona_clone_public_repo_gets_credentials() { /// E2E: Verify that requesting an IAT for a repo the app isn't installed on /// gives a clear error message. -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_iat_not_installed_gives_clear_error() { let creds = load_github_app_credentials(); @@ -1465,8 +1435,7 @@ async fn daytona_iat_not_installed_gives_clear_error() { /// E2E: After each remote checkpoint, the run branch is pushed to origin. /// Verifies the branch appears on the remote via `git ls-remote`. -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_git_push_run_branch_to_origin() { let creds = load_github_app_credentials(); let env = create_env_with_github_app(Some(creds)).await; @@ -1598,8 +1567,7 @@ async fn daytona_git_push_run_branch_to_origin() { /// underlying error that the SDK normally swallows. /// /// Run: cargo test -p arc-workflows -- --ignored daytona_toolbox_idle_diagnostic --nocapture -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_toolbox_idle_diagnostic() { let creds = load_github_app_credentials(); let env = create_env_with_github_app(Some(creds)).await; @@ -1733,8 +1701,7 @@ async fn daytona_toolbox_idle_diagnostic() { /// /// Creates a sandbox, saves a SandboxRecord, reconnects via `cp::reconnect`, /// uploads a file, downloads it back, and verifies the round-trip. -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"), live("GITHUB_APP_PRIVATE_KEY"))] async fn daytona_cp_upload_download_round_trip() { use fabro_sandbox::SandboxRecord; use fabro_sandbox::reconnect::reconnect; @@ -1829,15 +1796,9 @@ async fn daytona_cp_upload_download_round_trip() { env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"))] async fn daytona_computer_use_browser_screenshot() { use base64::Engine; - - dotenvy::dotenv().ok(); - if let Some(home) = dirs::home_dir() { - dotenvy::from_path(home.join(".fabro/.env")).ok(); - } let config = DaytonaConfig { snapshot: Some(DaytonaSnapshotConfig { name: "daytona-medium".into(), @@ -1987,16 +1948,11 @@ async fn daytona_computer_use_browser_screenshot() { env.cleanup().await.unwrap(); } -#[tokio::test] -#[ignore] +#[fabro_macros::e2e_test(live("DAYTONA_API_KEY"))] async fn daytona_playwright_mcp_sandbox_transport() { use fabro_agent::Sandbox; // Create sandbox from daytona-medium (has Node.js + Chromium) - dotenvy::dotenv().ok(); - if let Some(home) = dirs::home_dir() { - dotenvy::from_path(home.join(".fabro/.env")).ok(); - } let config = DaytonaConfig { snapshot: Some(DaytonaSnapshotConfig { name: "daytona-medium".into(), diff --git a/lib/crates/fabro-workflow/tests/it/integration.rs b/lib/crates/fabro-workflow/tests/it/integration.rs index 16b221d56..cf2b9c642 100644 --- a/lib/crates/fabro-workflow/tests/it/integration.rs +++ b/lib/crates/fabro-workflow/tests/it/integration.rs @@ -5895,10 +5895,7 @@ mod real_llm { } async fn make_llm_client() -> Option> { - let _ = dotenvy::dotenv(); - if std::env::var("ANTHROPIC_API_KEY").is_err() { - return None; - } + fabro_test::require_env("ANTHROPIC_API_KEY")?; let client = Client::from_env() .await .expect("unified-llm client should initialize from env"); @@ -5925,15 +5922,9 @@ mod real_llm { use fabro_workflow::run_options::RunOptions; use fabro_workflow::test_support::WorkflowRunner; - #[tokio::test] - #[ignore] + #[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn real_llm_linear_pipeline() { - let client = if let Some(c) = make_llm_client().await { - c - } else { - eprintln!("Skipping: ANTHROPIC_API_KEY not set"); - return; - }; + let client = make_llm_client().await.unwrap(); let mut graph = Graph::new("RealLLMLinear"); graph.attrs.insert( @@ -6040,15 +6031,9 @@ mod real_llm { ); } - #[tokio::test] - #[ignore] + #[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn real_llm_two_stage_pipeline() { - let client = if let Some(c) = make_llm_client().await { - c - } else { - eprintln!("Skipping: ANTHROPIC_API_KEY not set"); - return; - }; + let client = make_llm_client().await.unwrap(); let mut graph = Graph::new("RealLLMTwoStage"); graph.attrs.insert( @@ -6137,15 +6122,9 @@ mod real_llm { assert_eq!(last_stage, Some("review")); } - #[tokio::test] - #[ignore] + #[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn real_llm_human_gate_auto_approve() { - let client = if let Some(c) = make_llm_client().await { - c - } else { - eprintln!("Skipping: ANTHROPIC_API_KEY not set"); - return; - }; + let client = make_llm_client().await.unwrap(); let mut graph = Graph::new("RealLLMGate"); graph.attrs.insert( @@ -6286,15 +6265,9 @@ mod real_llm { ); } - #[tokio::test] - #[ignore] + #[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn real_llm_one_shot_pipeline() { - let client = if let Some(c) = make_llm_client().await { - c - } else { - eprintln!("Skipping: ANTHROPIC_API_KEY not set"); - return; - }; + let client = make_llm_client().await.unwrap(); let mut graph = Graph::new("RealLLMOneShot"); graph.attrs.insert( @@ -8105,11 +8078,8 @@ timeout_ms = 120000 // --- Prompt/Agent hook E2E with real LLM --- -#[tokio::test] -#[ignore = "requires ANTHROPIC_API_KEY"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn hook_prompt_proceed_allows_run() { - dotenvy::dotenv().ok(); - let hooks = vec![fabro_hooks::HookDefinition { name: Some("prompt-proceed".into()), event: fabro_hooks::HookEvent::RunStart, @@ -8132,11 +8102,8 @@ async fn hook_prompt_proceed_allows_run() { assert_eq!(outcome.status, StageStatus::Success); } -#[tokio::test] -#[ignore = "requires ANTHROPIC_API_KEY"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn hook_prompt_block_prevents_run() { - dotenvy::dotenv().ok(); - // Use a factual question that evaluates to false: "Is 2+2=5?" let hooks = vec![fabro_hooks::HookDefinition { name: Some("prompt-block".into()), @@ -8163,11 +8130,8 @@ async fn hook_prompt_block_prevents_run() { ); } -#[tokio::test] -#[ignore = "requires ANTHROPIC_API_KEY"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn hook_agent_proceed_allows_run() { - dotenvy::dotenv().ok(); - let hooks = vec![fabro_hooks::HookDefinition { name: Some("agent-proceed".into()), event: fabro_hooks::HookEvent::RunStart, @@ -8191,11 +8155,8 @@ async fn hook_agent_proceed_allows_run() { assert_eq!(outcome.status, StageStatus::Success); } -#[tokio::test] -#[ignore = "requires ANTHROPIC_API_KEY"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn hook_agent_with_tool_use() { - dotenvy::dotenv().ok(); - let dir = tempfile::tempdir().unwrap(); let marker = dir.path().join("hook_check.txt"); std::fs::write(&marker, "READY").unwrap(); @@ -8267,11 +8228,8 @@ async fn hooks_do_not_duplicate_workflow_events() { // E2E test with real LLM // --------------------------------------------------------------------------- -#[tokio::test] -#[ignore = "requires ANTHROPIC_API_KEY"] +#[fabro_macros::e2e_test(live("ANTHROPIC_API_KEY"))] async fn arc_e2e_with_real_llm() { - dotenvy::dotenv().ok(); - let dir = tempfile::tempdir().unwrap(); let dir_path = dir.path().to_str().unwrap().to_string();