mirror of
https://github.com/fabro-sh/fabro.git
synced 2026-09-17 23:52:34 +00:00
a9086f8442
The hook tests wrote their `[[run.hooks]]` entries into the user's settings file. `fabro run` no longer transmits `run` settings from there — it warns and points at `workflow.toml` — so no hook ran in any of these tests. The three that expect the run to proceed kept passing for the wrong reason. Each hooked test now writes a workflow config that names its graph and carries the hooks, and runs that config. The twin-mode server settings stay in the settings file, which is where they belong. The hooks reach the run now, but the twin-mode tests still cannot pass on this branch: the run executes in the isolated server, which never learns the twin's base URL and so calls the real OpenAI API with the namespace as a key. That plumbing belongs with the lithos credential resolution on main, not here. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
426 lines
13 KiB
Rust
426 lines
13 KiB
Rust
#![allow(
|
|
clippy::absolute_paths,
|
|
clippy::needless_borrow,
|
|
clippy::needless_borrows_for_generic_args,
|
|
reason = "These workflow-hook tests value explicit fixtures over pedantic style lints."
|
|
)]
|
|
#![expect(
|
|
clippy::disallowed_methods,
|
|
reason = "integration tests stage fixtures with sync std::fs; test infrastructure, not Tokio-hot path"
|
|
)]
|
|
|
|
use std::process::Output;
|
|
|
|
use fabro_config::Storage;
|
|
use fabro_test::{
|
|
TestMode, TwinOpenAi, TwinScenario, TwinScenarios, TwinToolCall, test_context, twin_openai,
|
|
};
|
|
use fabro_vault::{SecretType, Vault};
|
|
|
|
use super::read_conclusion;
|
|
|
|
async fn run_success_output(mut cmd: assert_cmd::Command) -> Output {
|
|
tokio::task::spawn_blocking(move || cmd.assert().success().get_output().clone())
|
|
.await
|
|
.expect("blocking command task should complete")
|
|
}
|
|
|
|
async fn run_failure_output(mut cmd: assert_cmd::Command) -> Output {
|
|
tokio::task::spawn_blocking(move || cmd.assert().failure().get_output().clone())
|
|
.await
|
|
.expect("blocking command task should complete")
|
|
}
|
|
|
|
fn hook_model() -> &'static str {
|
|
if TestMode::from_env().is_twin() {
|
|
"gpt-5.4-mini"
|
|
} else {
|
|
"haiku"
|
|
}
|
|
}
|
|
|
|
fn stage_model() -> &'static str {
|
|
if TestMode::from_env().is_twin() {
|
|
"gpt-5.4-mini"
|
|
} else {
|
|
"claude-haiku-4-5"
|
|
}
|
|
}
|
|
|
|
fn stage_provider() -> &'static str {
|
|
if TestMode::from_env().is_twin() {
|
|
"openai"
|
|
} else {
|
|
"anthropic"
|
|
}
|
|
}
|
|
|
|
fn toml_path(path: &std::path::Path) -> String {
|
|
path.display()
|
|
.to_string()
|
|
.replace('\\', "\\\\")
|
|
.replace('"', "\\\"")
|
|
}
|
|
|
|
fn twin_server_storage_dir(context: &fabro_test::TestContext) -> std::path::PathBuf {
|
|
context.temp_dir.join("hook-server-storage")
|
|
}
|
|
|
|
/// The twin-mode server settings: a private storage root and dev-token auth.
|
|
/// Live mode runs against the developer's own settings.
|
|
fn write_server_settings(context: &fabro_test::TestContext) {
|
|
if !TestMode::from_env().is_twin() {
|
|
return;
|
|
}
|
|
context.write_home(
|
|
".fabro/settings.toml",
|
|
format!(
|
|
r#"[server.storage]
|
|
root = "{}"
|
|
|
|
[server.auth]
|
|
methods = ["dev-token"]
|
|
"#,
|
|
toml_path(&twin_server_storage_dir(context)),
|
|
),
|
|
);
|
|
}
|
|
|
|
fn seed_openai_vault(storage_dir: &std::path::Path, api_key: &str) {
|
|
let mut vault =
|
|
Vault::load(Storage::new(storage_dir).secrets_path()).expect("test vault should load");
|
|
vault
|
|
.set("OPENAI_API_KEY", api_key, SecretType::Token, None)
|
|
.expect("OpenAI credential should store in test vault");
|
|
}
|
|
|
|
fn configure_twin_server(
|
|
context: &mut fabro_test::TestContext,
|
|
_twin: &TwinOpenAi,
|
|
namespace: &str,
|
|
) {
|
|
seed_openai_vault(&twin_server_storage_dir(context), namespace);
|
|
context.isolated_server();
|
|
}
|
|
|
|
fn write_workflow(context: &fabro_test::TestContext, name: &str, dot: &str) -> std::path::PathBuf {
|
|
context.write_temp(name, dot);
|
|
context.temp_dir.join(name)
|
|
}
|
|
|
|
/// A workflow config that bundles the graph `<name>.fabro` with `hooks`,
|
|
/// which is where run hooks live: `fabro run` does not transmit `run`
|
|
/// settings from the user's settings file. Returns the config path to run.
|
|
fn write_hooked_workflow(
|
|
context: &fabro_test::TestContext,
|
|
name: &str,
|
|
dot: &str,
|
|
hooks: &str,
|
|
) -> std::path::PathBuf {
|
|
let graph = format!("{name}.fabro");
|
|
context.write_temp(&graph, dot);
|
|
let config = format!("{name}.toml");
|
|
context.write_temp(
|
|
&config,
|
|
format!(
|
|
r#"_version = 1
|
|
|
|
[workflow]
|
|
graph = "{graph}"
|
|
{hooks}"#
|
|
),
|
|
);
|
|
context.temp_dir.join(config)
|
|
}
|
|
|
|
fn configure_hook_env(cmd: &mut assert_cmd::Command, hook_model: &str) {
|
|
cmd.env_remove("CHATGPT_ACCOUNT_ID");
|
|
cmd.env_remove("OPENAI_ORG_ID");
|
|
cmd.env_remove("OPENAI_PROJECT_ID");
|
|
if TestMode::from_env().is_twin() {
|
|
cmd.env_remove("ANTHROPIC_API_KEY");
|
|
}
|
|
cmd.arg("--environment").arg("local");
|
|
cmd.arg("--auto-approve");
|
|
cmd.arg("--provider").arg(stage_provider());
|
|
cmd.arg("--model").arg(hook_model);
|
|
}
|
|
|
|
async fn conclusion_status(context: &fabro_test::TestContext) -> String {
|
|
let run_dir = context.single_run_dir();
|
|
tokio::task::spawn_blocking(move || {
|
|
read_conclusion(&run_dir)["status"]
|
|
.as_str()
|
|
.expect("conclusion should include a string status")
|
|
.to_string()
|
|
})
|
|
.await
|
|
.expect("conclusion status task should complete")
|
|
}
|
|
|
|
#[fabro_macros::e2e_test(twin, live("ANTHROPIC_API_KEY"))]
|
|
async fn hook_prompt_proceed_allows_run() {
|
|
let mut context = test_context!();
|
|
write_server_settings(&context);
|
|
let workflow = write_hooked_workflow(
|
|
&context,
|
|
"hook_prompt_proceed",
|
|
r"digraph HookTest {
|
|
start [shape=Mdiamond]
|
|
exit [shape=Msquare]
|
|
start -> exit
|
|
}",
|
|
&format!(
|
|
r#"
|
|
[[run.hooks]]
|
|
name = "prompt-proceed"
|
|
event = "run_start"
|
|
prompt = "A workflow is starting. Always approve. Respond with {{\"ok\": true}}."
|
|
model = "{model}"
|
|
"#,
|
|
model = hook_model()
|
|
),
|
|
);
|
|
|
|
if TestMode::from_env().is_twin() {
|
|
let twin = twin_openai().await;
|
|
let namespace = format!("{}::{}", module_path!(), line!());
|
|
TwinScenarios::new(namespace.clone())
|
|
.scenario(TwinScenario::responses("gpt-5.4-mini").text(r#"{"ok":true}"#))
|
|
.load(twin)
|
|
.await;
|
|
configure_twin_server(&mut context, twin, &namespace);
|
|
let mut cmd = context.run_cmd();
|
|
configure_hook_env(&mut cmd, stage_model());
|
|
twin.configure_command(&mut cmd, &namespace);
|
|
cmd.arg(&workflow);
|
|
run_success_output(cmd).await;
|
|
} else {
|
|
let mut cmd = context.run_cmd();
|
|
configure_hook_env(&mut cmd, stage_model());
|
|
cmd.arg(&workflow);
|
|
run_success_output(cmd).await;
|
|
}
|
|
|
|
assert_eq!(conclusion_status(&context).await, "succeeded");
|
|
}
|
|
|
|
#[fabro_macros::e2e_test(twin, live("ANTHROPIC_API_KEY"))]
|
|
async fn hook_prompt_block_prevents_run() {
|
|
let mut context = test_context!();
|
|
write_server_settings(&context);
|
|
let workflow = write_hooked_workflow(
|
|
&context,
|
|
"hook_prompt_block",
|
|
r"digraph HookTest {
|
|
start [shape=Mdiamond]
|
|
exit [shape=Msquare]
|
|
start -> exit
|
|
}",
|
|
&format!(
|
|
r#"
|
|
[[run.hooks]]
|
|
name = "prompt-block"
|
|
event = "run_start"
|
|
prompt = "Check: is 2+2 equal to 5? If the statement is true, respond {{\"ok\": true}}. If false, respond {{\"ok\": false, \"reason\": \"math check failed\"}}."
|
|
model = "{model}"
|
|
"#,
|
|
model = hook_model()
|
|
),
|
|
);
|
|
|
|
let output = if TestMode::from_env().is_twin() {
|
|
let twin = twin_openai().await;
|
|
let namespace = format!("{}::{}", module_path!(), line!());
|
|
TwinScenarios::new(namespace.clone())
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.text(r#"{"ok":false,"reason":"math check failed"}"#),
|
|
)
|
|
.load(twin)
|
|
.await;
|
|
configure_twin_server(&mut context, twin, &namespace);
|
|
let mut cmd = context.run_cmd();
|
|
configure_hook_env(&mut cmd, stage_model());
|
|
twin.configure_command(&mut cmd, &namespace);
|
|
cmd.arg(&workflow);
|
|
run_failure_output(cmd).await
|
|
} else {
|
|
let mut cmd = context.run_cmd();
|
|
configure_hook_env(&mut cmd, stage_model());
|
|
cmd.arg(&workflow);
|
|
run_failure_output(cmd).await
|
|
};
|
|
|
|
let stderr = String::from_utf8(output.stderr).unwrap();
|
|
assert!(
|
|
stderr.contains("math check failed"),
|
|
"stderr should include hook block reason, got: {stderr}"
|
|
);
|
|
}
|
|
|
|
#[fabro_macros::e2e_test(twin, live("ANTHROPIC_API_KEY"))]
|
|
async fn hook_agent_proceed_allows_run() {
|
|
let mut context = test_context!();
|
|
write_server_settings(&context);
|
|
let workflow = write_hooked_workflow(
|
|
&context,
|
|
"hook_agent_proceed",
|
|
r"digraph HookTest {
|
|
start [shape=Mdiamond]
|
|
exit [shape=Msquare]
|
|
start -> exit
|
|
}",
|
|
&format!(
|
|
r#"
|
|
[[run.hooks]]
|
|
name = "agent-proceed"
|
|
event = "run_start"
|
|
prompt = "A workflow is starting. Always approve. Respond with {{\"ok\": true}}. Do not use any tools."
|
|
model = "{model}"
|
|
max_tool_rounds = 1
|
|
agent = "enabled"
|
|
"#,
|
|
model = hook_model()
|
|
),
|
|
);
|
|
|
|
if TestMode::from_env().is_twin() {
|
|
let twin = twin_openai().await;
|
|
let namespace = format!("{}::{}", module_path!(), line!());
|
|
TwinScenarios::new(namespace.clone())
|
|
.scenario(TwinScenario::responses("gpt-5.4-mini").text(r#"{"ok":true}"#))
|
|
.load(twin)
|
|
.await;
|
|
configure_twin_server(&mut context, twin, &namespace);
|
|
let mut cmd = context.run_cmd();
|
|
configure_hook_env(&mut cmd, stage_model());
|
|
twin.configure_command(&mut cmd, &namespace);
|
|
cmd.arg(&workflow);
|
|
run_success_output(cmd).await;
|
|
} else {
|
|
let mut cmd = context.run_cmd();
|
|
configure_hook_env(&mut cmd, stage_model());
|
|
cmd.arg(&workflow);
|
|
run_success_output(cmd).await;
|
|
}
|
|
|
|
assert_eq!(conclusion_status(&context).await, "succeeded");
|
|
}
|
|
|
|
#[fabro_macros::e2e_test(twin, live("ANTHROPIC_API_KEY"))]
|
|
async fn hook_agent_with_tool_use() {
|
|
let mut context = test_context!();
|
|
let marker = context.temp_dir.join("hook_check.txt");
|
|
std::fs::write(&marker, "READY").unwrap();
|
|
write_server_settings(&context);
|
|
let workflow = write_hooked_workflow(
|
|
&context,
|
|
"hook_agent_tools",
|
|
r"digraph HookTest {
|
|
start [shape=Mdiamond]
|
|
exit [shape=Msquare]
|
|
start -> exit
|
|
}",
|
|
&format!(
|
|
r#"
|
|
[[run.hooks]]
|
|
name = "agent-tools"
|
|
event = "run_start"
|
|
prompt = "Read the file at {path} using the read_file tool. If it contains 'READY', respond with {{\"ok\": true}}. Otherwise respond with {{\"ok\": false, \"reason\": \"not ready\"}}."
|
|
model = "{model}"
|
|
max_tool_rounds = 5
|
|
agent = "enabled"
|
|
"#,
|
|
path = marker.display(),
|
|
model = hook_model()
|
|
),
|
|
);
|
|
|
|
if TestMode::from_env().is_twin() {
|
|
let twin = twin_openai().await;
|
|
let namespace = format!("{}::{}", module_path!(), line!());
|
|
TwinScenarios::new(namespace.clone())
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.tool_call(TwinToolCall::read_file(marker.display().to_string())),
|
|
)
|
|
.scenario(TwinScenario::responses("gpt-5.4-mini").text(r#"{"ok":true}"#))
|
|
.load(twin)
|
|
.await;
|
|
configure_twin_server(&mut context, twin, &namespace);
|
|
let mut cmd = context.run_cmd();
|
|
configure_hook_env(&mut cmd, stage_model());
|
|
twin.configure_command(&mut cmd, &namespace);
|
|
cmd.arg(&workflow);
|
|
run_success_output(cmd).await;
|
|
} else {
|
|
let mut cmd = context.run_cmd();
|
|
configure_hook_env(&mut cmd, stage_model());
|
|
cmd.arg(&workflow);
|
|
run_success_output(cmd).await;
|
|
}
|
|
|
|
assert_eq!(conclusion_status(&context).await, "succeeded");
|
|
}
|
|
|
|
#[fabro_macros::e2e_test(twin, live("ANTHROPIC_API_KEY"))]
|
|
async fn arc_e2e_with_real_llm() {
|
|
let mut context = test_context!();
|
|
write_server_settings(&context);
|
|
let hello = context.temp_dir.join("hello.txt");
|
|
let workflow = write_workflow(
|
|
&context,
|
|
"arc_e2e_real_llm.fabro",
|
|
&format!(
|
|
r#"digraph E2E {{
|
|
graph [goal="Create a test file"]
|
|
start [shape=Mdiamond]
|
|
exit [shape=Msquare]
|
|
work [
|
|
shape=box,
|
|
label="Work",
|
|
prompt="Create a file called hello.txt in {} containing exactly 'Hello from LLM'. Do not output anything else.",
|
|
goal_gate=true
|
|
]
|
|
start -> work -> exit
|
|
}}"#,
|
|
context.temp_dir.display()
|
|
),
|
|
);
|
|
|
|
if TestMode::from_env().is_twin() {
|
|
let twin = twin_openai().await;
|
|
let namespace = format!("{}::{}", module_path!(), line!());
|
|
TwinScenarios::new(namespace.clone())
|
|
.scenario(
|
|
TwinScenario::responses("gpt-5.4-mini")
|
|
.input_contains("Create a file called hello.txt")
|
|
.tool_call(TwinToolCall::write_file(
|
|
hello.display().to_string(),
|
|
"Hello from LLM",
|
|
))
|
|
.text("Done."),
|
|
)
|
|
.load(twin)
|
|
.await;
|
|
configure_twin_server(&mut context, twin, &namespace);
|
|
let mut cmd = context.run_cmd();
|
|
configure_hook_env(&mut cmd, stage_model());
|
|
twin.configure_command(&mut cmd, &namespace);
|
|
cmd.arg(&workflow);
|
|
run_success_output(cmd).await;
|
|
} else {
|
|
let mut cmd = context.run_cmd();
|
|
configure_hook_env(&mut cmd, stage_model());
|
|
cmd.arg(&workflow);
|
|
run_success_output(cmd).await;
|
|
}
|
|
|
|
assert_eq!(
|
|
std::fs::read_to_string(&hello).unwrap(),
|
|
"Hello from LLM",
|
|
"workflow should create the expected file"
|
|
);
|
|
assert_eq!(conclusion_status(&context).await, "succeeded");
|
|
}
|