fabro/lib/components/fabro-workflow/src/handler/agent.rs
2026-09-13 08:49:50 -06:00

1605 lines
56 KiB
Rust

use std::path::Path;
use std::sync::Arc;
use async_trait::async_trait;
use fabro_graphviz::graph::{Graph, Node};
use fabro_sandbox::RunSandbox;
use fabro_types::{StageModelUsage, StageTiming};
use pebble_agent::ToolMiddleware;
use pebble_coding_agent::extensions::HumanInputProvider;
pub(crate) use structured_output::extract_status_fields;
use tokio_util::sync::CancellationToken;
use super::llm::EffectiveRequestControls;
use super::structured_output::{
self, OutputSchemaKind, StructuredOutputError, ValidatedStructuredOutput,
};
use super::{EngineServices, Handler, NodeTimeoutPolicy};
use crate::context::{Context, WorkflowContext, keys};
use crate::error::Error;
use crate::event::{Emitter, Event, StageScope};
use crate::interview_runtime::WorkflowHumanInput;
use crate::outcome::{BilledModelUsage, Outcome, OutcomeExt};
const LAST_FILE_ROUTING_EXTENSIONS: &[&str] = &["json", "md"];
/// Result from a `CodergenBackend` invocation.
#[allow(
clippy::large_enum_variant,
reason = "Text payload is the common case; Full(Box<Outcome>) is the rare alternative."
)]
pub enum CodergenResult {
Text {
text: String,
/// The stage's billing: for an agent, the whole session tree's
/// tokens under the root's route.
usage: Option<BilledModelUsage>,
/// `usage` split by model, when the backend billed subagents at
/// their own models. Empty when `usage` is the one row.
usage_by_model: Vec<BilledModelUsage>,
files_touched: Vec<String>,
last_file_touched: Option<String>,
/// Active timing observed by the backend. The wall field is ignored by
/// the executor on this hop; executor wall time remains authoritative.
timing: StageTiming,
},
Full(Box<Outcome>),
}
pub struct CodergenRunRequest<'a> {
pub node: &'a Node,
pub prompt: &'a str,
pub context: &'a Context,
pub thread_id: Option<&'a str>,
pub emitter: &'a Arc<Emitter>,
pub sandbox: &'a Arc<RunSandbox>,
/// Tool hooks the stage's agent (and its subagents) run under.
pub tool_middleware: Option<Arc<dyn ToolMiddleware>>,
pub cancel_token: CancellationToken,
/// Where the agent's `ask_user` questions go.
pub human_input: Option<Arc<dyn HumanInputProvider>>,
}
pub struct OneShotRequest<'a> {
pub node: &'a Node,
pub prompt: &'a str,
pub system_prompt: Option<&'a str>,
pub emitter: &'a Arc<Emitter>,
pub stage_scope: &'a StageScope,
pub sandbox: &'a Arc<RunSandbox>,
pub cancel_token: CancellationToken,
}
/// Emit the canonical `Event::Prompt` for a stage prompt and return the
/// resolved [`StageScope`] so the caller can keep building events scoped to
/// the same stage.
///
/// Both `AgentHandler` and `PromptHandler` build the same payload, so the
/// per-emit fallback rules — node-provided
/// `provider`/`model` overrides over run-level defaults, and the backend's
/// `EffectiveRequestControls` (or `Default::default()` when no backend is
/// attached) — live in one place.
pub(crate) fn emit_stage_prompt(
services: &EngineServices,
context: &Context,
node: &Node,
prompt: &str,
mode: &str,
backend: Option<&dyn CodergenBackend>,
) -> Result<StageScope, Error> {
let prompt_provider = node
.provider()
.map(String::from)
.or_else(|| Some(services.run.provider_id.to_string()));
let prompt_model = node
.model()
.map(String::from)
.or_else(|| Some(services.run.model.clone()));
let stage_scope = StageScope::for_handler(context, &node.id);
let request_controls = backend
.map(|b| b.effective_request_controls(node))
.transpose()?
.unwrap_or_default();
services.run.emitter.emit_scoped(
&Event::Prompt {
stage: node.id.clone(),
visit: stage_scope.visit,
text: prompt.to_string(),
mode: Some(mode.to_string()),
provider: prompt_provider,
model: prompt_model,
reasoning_effort: request_controls.reasoning_effort,
speed: request_controls.speed,
},
&stage_scope,
);
Ok(stage_scope)
}
/// Backend interface for LLM execution in codergen nodes.
#[async_trait]
pub trait CodergenBackend: Send + Sync {
/// Run a multi-turn agent loop (the default codergen mode).
async fn run(&self, request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error>;
/// Run a single LLM call with no tools (one_shot mode).
async fn one_shot(&self, _request: OneShotRequest<'_>) -> Result<CodergenResult, Error> {
Err(Error::Validation(
"one_shot mode not supported by this backend".into(),
))
}
async fn shutdown(&self, _emitter: &Arc<Emitter>) {}
fn effective_request_controls(&self, _node: &Node) -> Result<EffectiveRequestControls, Error> {
Ok(EffectiveRequestControls::default())
}
fn node_timeout_policy(&self, _node: &Node) -> NodeTimeoutPolicy {
NodeTimeoutPolicy::ExecutorEnforced
}
}
/// The default handler for LLM task nodes.
pub struct AgentHandler {
backend: Option<Box<dyn CodergenBackend>>,
}
impl AgentHandler {
#[must_use]
pub fn new(backend: Option<Box<dyn CodergenBackend>>) -> Self {
Self { backend }
}
}
pub(crate) async fn validate_agent_output_sources(
schema: &OutputSchemaKind,
response_text: &str,
sandbox: &Arc<RunSandbox>,
last_file_touched: Option<&str>,
) -> Result<ValidatedStructuredOutput, StructuredOutputError> {
if !matches!(schema, OutputSchemaKind::Routing) {
return structured_output::validate_response_text(schema, response_text);
}
let initial_error = match structured_output::validate_response_text(schema, response_text) {
Ok(validated) => return Ok(validated),
Err(error) if error.allows_routing_fallback() => error,
Err(error) => return Err(error),
};
let mut fallback_error = initial_error;
if let Some(status_json) = read_sandbox_file(sandbox, "status.json").await {
match structured_output::validate_response_text(schema, &status_json) {
Ok(validated) => return Ok(validated),
Err(error) if error.allows_routing_fallback() => {
fallback_error = error;
}
Err(error) => return Err(error),
}
}
if let Some(path) = last_file_touched {
if let Some(routing_json) = read_last_file_routing_json(sandbox, path).await {
return structured_output::validate_response_text(schema, &routing_json);
}
}
Err(fallback_error)
}
async fn read_sandbox_file(sandbox: &Arc<RunSandbox>, path: &str) -> Option<String> {
sandbox.read_file_text(path).await.ok()
}
/// Extract the terminal JSON object from the last-touched file when it has an
/// eligible extension. Does not check that the object contains routing fields;
/// callers validate that.
async fn read_last_file_routing_json(sandbox: &Arc<RunSandbox>, path: &str) -> Option<String> {
let extension = Path::new(path).extension()?.to_str()?;
if !LAST_FILE_ROUTING_EXTENSIONS
.iter()
.any(|allowed| extension.eq_ignore_ascii_case(allowed))
{
return None;
}
let contents = read_sandbox_file(sandbox, path).await?;
structured_output::terminal_json_object(&contents).map(str::to_owned)
}
/// Truncate a string to at most `max_chars` characters (char-boundary safe).
pub(crate) fn truncate(s: &str, max_chars: usize) -> &str {
if s.len() <= max_chars {
s
} else {
&s[..s.floor_char_boundary(max_chars)]
}
}
/// Shared simulate implementation for LLM-backed handlers (agent & prompt).
/// Produces a simulated outcome with standard context updates.
pub(crate) fn simulate_llm_handler(node: &Node) -> Outcome {
let simulated_text = format!("[Simulated] Response for stage: {}", node.id);
let mut outcome = Outcome::simulated(&node.id);
outcome
.context_updates
.insert(keys::LAST_STAGE.to_string(), serde_json::json!(node.id));
outcome.context_updates.insert(
keys::LAST_RESPONSE.to_string(),
serde_json::json!(truncate(&simulated_text, 200)),
);
outcome.context_updates.insert(
keys::response_key(&node.id),
serde_json::json!(&simulated_text),
);
outcome
}
#[async_trait]
impl Handler for AgentHandler {
async fn shutdown(&self, emitter: &Arc<Emitter>) {
if let Some(backend) = self.backend.as_ref() {
backend.shutdown(emitter).await;
}
}
async fn simulate(
&self,
node: &Node,
_context: &Context,
_graph: &Graph,
_run_dir: &Path,
_services: &EngineServices,
) -> Result<Outcome, Error> {
Ok(simulate_llm_handler(node))
}
async fn execute(
&self,
node: &Node,
context: &Context,
graph: &Graph,
_run_dir: &Path,
services: &EngineServices,
) -> Result<Outcome, Error> {
// 1. Build prompt (prepend fidelity preamble if present)
let raw_prompt = node.prompt_or_label();
let preamble = context.preamble();
let prompt = if preamble.is_empty() {
raw_prompt.to_string()
} else {
format!("{preamble}\n\n{raw_prompt}")
};
let output_schema = structured_output::parse_node_output_schema(node)?;
let prompt = match output_schema.as_ref() {
Some(schema) => schema.agent_prompt(&prompt),
None => prompt,
};
let stage_scope = emit_stage_prompt(
services,
context,
node,
&prompt,
StageModelUsage::MODE_AGENT,
self.backend.as_deref(),
)?;
let human_input: Arc<dyn HumanInputProvider> = Arc::new(WorkflowHumanInput::new(
Arc::clone(&services.interviewer),
Arc::clone(&services.run.emitter),
stage_scope.clone(),
node.id.clone(),
Arc::clone(&services.run.interview_blocker),
));
// 3. Call LLM backend (agent loop)
let thread_id = context.thread_id();
let run_id = context.parsed_run_id()?;
let tool_middleware: Option<Arc<dyn ToolMiddleware>> =
services.run.hook_runner.as_ref().map(|hr| {
Arc::new(fabro_hooks::WorkflowToolHookCallback {
hook_runner: Arc::clone(hr),
sandbox: Arc::clone(&services.run.sandbox),
run_id,
workflow_name: graph.name.clone(),
hook_execution_context: services.run.locations.hook_execution_context(),
node_id: node.id.clone(),
}) as Arc<dyn ToolMiddleware>
});
let (
response_text,
stage_usage,
stage_usage_by_model,
backend_files_touched,
last_file_touched,
timing,
) = if let Some(backend) = &self.backend {
let result = backend
.run(CodergenRunRequest {
node,
prompt: &prompt,
context,
thread_id: thread_id.as_deref(),
emitter: &services.run.emitter,
sandbox: &services.run.sandbox,
tool_middleware,
cancel_token: services.run.cancel_token(),
human_input: Some(human_input),
})
.await;
match result {
Ok(CodergenResult::Full(outcome)) => return Ok(*outcome),
Ok(CodergenResult::Text {
text,
usage,
usage_by_model,
files_touched,
last_file_touched,
timing,
}) => (
text,
usage,
usage_by_model,
files_touched,
last_file_touched,
timing,
),
Err(Error::Cancelled) => return Err(Error::Cancelled),
Err(e) if e.is_retryable() => {
return Err(e);
}
Err(e) => {
return Ok(e.to_fail_outcome());
}
}
} else {
(
format!("[Simulated] Response for stage: {}", node.id),
None,
Vec::new(),
Vec::new(),
None,
StageTiming::default(),
)
};
let response_model = stage_usage
.as_ref()
.map(|usage| usage.model_id().to_string())
.or_else(|| node.model().map(String::from))
.unwrap_or_default();
let response_provider = node
.provider()
.map(String::from)
.or_else(|| Some(services.run.provider_id.to_string()))
.unwrap_or_default();
services.run.emitter.emit_scoped(
&Event::PromptCompleted {
node_id: node.id.clone(),
response: response_text.clone(),
model: response_model,
provider: response_provider,
billing: stage_usage.clone(),
},
&stage_scope,
);
// Build and write status
let mut outcome = Outcome::success();
outcome.notes = Some(format!("Stage completed: {}", node.id));
outcome
.context_updates
.insert(keys::LAST_STAGE.to_string(), serde_json::json!(node.id));
outcome.context_updates.insert(
keys::LAST_RESPONSE.to_string(),
serde_json::json!(truncate(&response_text, 200)),
);
outcome.context_updates.insert(
keys::response_key(&node.id),
serde_json::json!(&response_text),
);
if let Some(schema) = output_schema.as_ref() {
if let Ok(validated) = validate_agent_output_sources(
schema,
&response_text,
&services.run.sandbox,
last_file_touched.as_deref(),
)
.await
{
structured_output::apply_validated_output(node, schema, &validated, &mut outcome);
} else {
let mut failed =
structured_output::exhausted_failure_outcome(node.output_retries());
failed.timing = Some(timing);
failed.usage = stage_usage;
failed.usage_by_model = stage_usage_by_model;
failed.files_touched = backend_files_touched;
return Ok(failed);
}
} else {
// 7b. Parse routing directives from response text, falling back to
// status.json written by the agent into the sandbox CWD, then to
// a terminal JSON object in an eligible last-written file.
let found_in_response = extract_status_fields(&response_text, &mut outcome);
if !found_in_response {
let mut found_in_status_json = false;
if let Some(status_json) =
read_sandbox_file(&services.run.sandbox, "status.json").await
{
found_in_status_json = extract_status_fields(&status_json, &mut outcome);
}
if !found_in_status_json {
if let Some(ref path) = last_file_touched {
if let Some(routing_json) =
read_last_file_routing_json(&services.run.sandbox, path).await
{
extract_status_fields(&routing_json, &mut outcome);
}
}
}
}
}
outcome.usage = stage_usage;
outcome.usage_by_model = stage_usage_by_model;
outcome.files_touched = backend_files_touched;
outcome.timing = Some(timing);
Ok(outcome)
}
fn node_timeout_policy(&self, node: &Node) -> NodeTimeoutPolicy {
self.backend
.as_ref()
.map_or(NodeTimeoutPolicy::ExecutorEnforced, |backend| {
backend.node_timeout_policy(node)
})
}
}
#[cfg(test)]
#[expect(
clippy::disallowed_methods,
reason = "tests persist per-iteration state fixtures"
)]
mod tests {
use std::sync::Arc;
use std::time::Duration;
use fabro_graphviz::graph::AttrValue;
use fabro_store::{Database, RunDatabase, StageId};
use fabro_types::{fixtures, test_support};
use lithos_llm::types::{ReasoningEffort, Speed};
use object_store::memory::InMemory;
use tempfile::TempDir;
use super::*;
fn make_services() -> EngineServices {
EngineServices::test_default()
}
fn test_store() -> Arc<Database> {
Arc::new(fabro_store::test_support::test_database(
Arc::new(InMemory::new()),
"",
Duration::from_millis(1),
None,
))
}
async fn make_services_with_run_store() -> (
EngineServices,
RunDatabase,
crate::event::StoreProgressLogger,
) {
let store = test_store();
let run_store = store.create_run(&fixtures::RUN_1).await.unwrap();
seed_created(&run_store).await;
let mut services = EngineServices::test_default();
services.run = services
.run
.with_emitter(Arc::new(crate::event::Emitter::new(fixtures::RUN_1)))
.with_run_store(run_store.clone().into());
let logger = crate::event::StoreProgressLogger::new(run_store.clone());
logger.register(services.run.emitter.as_ref());
(services, run_store, logger)
}
async fn seed_created(run_store: &RunDatabase) {
crate::event::append_event(
run_store,
&fixtures::RUN_1,
&crate::event::Event::RunCreated {
run_id: fixtures::RUN_1,
title: None,
settings: serde_json::to_value(fabro_types::WorkflowSettings::default())
.unwrap(),
graph: serde_json::to_value(fabro_types::Graph::new("test")).unwrap(),
workflow_source: None,
labels: std::collections::BTreeMap::default(),
source_directory: None,
workflow_slug: None,
workflow_version_id: None,
target: None,
automation: None,
provenance: test_support::test_run_provenance(),
spec_blob: None,
git: None,
fork_source_ref: None,
retried_from: None,
parent_id: None,
web_url: None,
},
)
.await
.unwrap();
}
fn test_context() -> Context {
let context = Context::new();
context.set(
crate::context::keys::INTERNAL_RUN_ID,
serde_json::json!(fixtures::RUN_1.to_string()),
);
context
}
struct LastFileBackend {
path: String,
}
#[async_trait]
impl CodergenBackend for LastFileBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: "Done writing results.".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: vec![self.path.clone()],
last_file_touched: Some(self.path.clone()),
timing: StageTiming::default(),
})
}
}
async fn sandbox_with_file(path: &str, contents: &str) -> (TempDir, Arc<RunSandbox>) {
let sandbox_dir = TempDir::new().unwrap();
std::fs::write(sandbox_dir.path().join(path), contents).unwrap();
let sandbox: Arc<RunSandbox> = Arc::new(
fabro_sandbox::local_sandbox(sandbox_dir.path().to_path_buf())
.await
.unwrap(),
);
(sandbox_dir, sandbox)
}
async fn execute_with_last_file(path: &str, contents: &str) -> Outcome {
let (_sandbox_dir, sandbox) = sandbox_with_file(path, contents).await;
let handler = AgentHandler::new(Some(Box::new(LastFileBackend {
path: path.to_string(),
})));
let node = Node::new("step");
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let mut services = EngineServices::test_default();
services.run = services.run.with_sandbox(sandbox);
handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap()
}
async fn validate_routing_with_last_file(
path: &str,
contents: &str,
) -> Result<ValidatedStructuredOutput, StructuredOutputError> {
let (_sandbox_dir, sandbox) = sandbox_with_file(path, contents).await;
validate_agent_output_sources(
&OutputSchemaKind::Routing,
"Done writing results.",
&sandbox,
Some(path),
)
.await
}
#[tokio::test]
async fn codergen_handler_simulate() {
let handler = AgentHandler::new(None);
let node = Node::new("plan");
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let outcome = handler
.simulate(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
assert_eq!(outcome.notes.as_deref(), Some("[Simulated] plan"));
assert_eq!(
outcome.context_updates.get(keys::LAST_STAGE),
Some(&serde_json::json!("plan"))
);
assert!(outcome.context_updates.contains_key(keys::LAST_RESPONSE));
assert_eq!(
outcome.context_updates.get(&keys::response_key("plan")),
Some(&serde_json::json!("[Simulated] Response for stage: plan"))
);
}
#[tokio::test]
async fn codergen_handler_uses_already_rendered_prompt() {
let handler = AgentHandler::new(None);
let mut node = Node::new("plan");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Achieve: Build a feature".to_string()),
);
let context = test_context();
let mut graph = Graph::new("test");
graph.attrs.insert(
"goal".to_string(),
AttrValue::String("Build a feature".to_string()),
);
let tmp = TempDir::new().unwrap();
let (services, run_store, logger) = make_services_with_run_store().await;
handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap();
logger.flush().await.unwrap();
let state = run_store.state().await.unwrap();
let node_state = state.stage(&StageId::new("plan", 1)).unwrap();
assert_eq!(
node_state.prompt.as_deref(),
Some("Achieve: Build a feature")
);
}
#[tokio::test]
async fn codergen_handler_falls_back_to_label() {
let handler = AgentHandler::new(None);
let mut node = Node::new("work");
node.attrs.insert(
"label".to_string(),
AttrValue::String("Do work".to_string()),
);
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let (services, run_store, logger) = make_services_with_run_store().await;
handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap();
logger.flush().await.unwrap();
let state = run_store.state().await.unwrap();
let node_state = state.stage(&StageId::new("work", 1)).unwrap();
assert_eq!(node_state.prompt.as_deref(), Some("Do work"));
}
#[tokio::test]
async fn codergen_handler_context_updates() {
let handler = AgentHandler::new(None);
let node = Node::new("step");
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
assert_eq!(
outcome.context_updates.get(keys::LAST_STAGE),
Some(&serde_json::json!("step"))
);
assert!(outcome.context_updates.contains_key(keys::LAST_RESPONSE));
assert_eq!(
outcome.context_updates.get(&keys::response_key("step")),
Some(&serde_json::json!("[Simulated] Response for stage: step"))
);
}
#[tokio::test]
async fn codergen_handler_falls_back_to_status_json_in_sandbox() {
// Simulation mode returns text with no JSON directives, so the
// handler should fall back to reading status.json from the sandbox CWD.
let sandbox_dir = TempDir::new().unwrap();
std::fs::write(
sandbox_dir.path().join("status.json"),
r#"{"outcome": "failed", "failure_reason": "tests failed"}"#,
)
.unwrap();
let handler = AgentHandler::new(None);
let node = Node::new("step");
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let mut services = EngineServices::test_default();
services.run = services.run.with_sandbox(std::sync::Arc::new(
fabro_sandbox::local_sandbox(sandbox_dir.path().to_path_buf())
.await
.unwrap(),
));
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Failed {
retry_requested: false,
});
assert_eq!(outcome.failure_reason(), Some("tests failed"));
}
#[tokio::test]
async fn codergen_handler_prefers_response_text_over_status_json() {
// Backend returns response text with routing directives — status.json
// in the sandbox should be ignored.
struct DirectiveBackend;
#[async_trait]
impl CodergenBackend for DirectiveBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text:
r#"Done. {"outcome": "succeeded", "preferred_next_label": "approve"}"#
.to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let sandbox_dir = TempDir::new().unwrap();
std::fs::write(
sandbox_dir.path().join("status.json"),
r#"{"outcome": "failed", "failure_reason": "should be ignored"}"#,
)
.unwrap();
let handler = AgentHandler::new(Some(Box::new(DirectiveBackend)));
let node = Node::new("step");
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let mut services = EngineServices::test_default();
services.run = services.run.with_sandbox(std::sync::Arc::new(
fabro_sandbox::local_sandbox(sandbox_dir.path().to_path_buf())
.await
.unwrap(),
));
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
assert_eq!(outcome.preferred_label.as_deref(), Some("approve"));
assert!(outcome.failure.is_none());
}
#[tokio::test]
async fn codergen_handler_copies_backend_timing_to_outcome() {
struct TimingBackend;
#[async_trait]
impl CodergenBackend for TimingBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: "done".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::new(0, 200, 300),
})
}
}
let handler = AgentHandler::new(Some(Box::new(TimingBackend)));
let node = Node::new("step");
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
assert_eq!(outcome.timing, Some(StageTiming::new(0, 200, 300)));
}
#[tokio::test]
async fn codergen_handler_extracts_status_from_last_file_touched() {
let outcome = execute_with_last_file(
"results.md",
r#"# Results
{"context_updates": {"verified": "true"}}
"#,
)
.await;
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
assert_eq!(
outcome.context_updates.get("verified"),
Some(&serde_json::json!("true")),
);
}
#[tokio::test]
async fn codergen_handler_ignores_nonterminal_status_in_last_markdown_file() {
let outcome = execute_with_last_file(
"results.md",
r#"{"outcome":"failed","failure_reason":"tests failed"}
All checks passed.
"#,
)
.await;
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
assert!(outcome.failure.is_none());
}
#[tokio::test]
async fn codergen_handler_ignores_terminal_status_in_disallowed_last_file() {
let outcome = execute_with_last_file(
"command.rs",
r#"{"outcome":"failed","failure_reason":"tests failed"}"#,
)
.await;
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
assert!(outcome.failure.is_none());
}
#[tokio::test]
async fn codergen_handler_output_schema_routing_uses_status_json_fallback_when_response_has_no_json()
{
let sandbox_dir = TempDir::new().unwrap();
std::fs::write(
sandbox_dir.path().join("status.json"),
r#"{"preferred_next_label": "review"}"#,
)
.unwrap();
let handler = AgentHandler::new(None);
let mut node = Node::new("step");
node.attrs.insert(
"output_schema".to_string(),
AttrValue::String("routing".to_string()),
);
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let mut services = EngineServices::test_default();
services.run = services.run.with_sandbox(std::sync::Arc::new(
fabro_sandbox::local_sandbox(sandbox_dir.path().to_path_buf())
.await
.unwrap(),
));
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
assert_eq!(outcome.preferred_label.as_deref(), Some("review"));
}
#[tokio::test]
async fn validated_routing_accepts_terminal_status_in_json_file_case_insensitively() {
let validated = validate_routing_with_last_file(
"results.JSON",
"# Results\n\n{\"preferred_next_label\":\"review\"}\n",
)
.await
.unwrap();
assert_eq!(
validated.value,
serde_json::json!({"preferred_next_label": "review"}),
);
}
#[tokio::test]
async fn validated_routing_ignores_nonterminal_status_in_last_markdown_file() {
let error = validate_routing_with_last_file(
"results.md",
"{\"outcome\":\"failed\",\"failure_reason\":\"tests failed\"}\nAll checks passed.",
)
.await
.unwrap_err();
assert_eq!(
error.kind(),
structured_output::StructuredOutputErrorKind::NoJsonObject,
);
}
#[tokio::test]
async fn validated_routing_ignores_terminal_status_in_disallowed_last_file() {
let error = validate_routing_with_last_file(
"command.rs",
r#"{"outcome":"failed","failure_reason":"tests failed"}"#,
)
.await
.unwrap_err();
assert_eq!(
error.kind(),
structured_output::StructuredOutputErrorKind::NoJsonObject,
);
}
#[tokio::test]
async fn codergen_handler_output_schema_routing_rejects_malformed_response_before_status_json_fallback()
{
struct BadRoutingBackend;
#[async_trait]
impl CodergenBackend for BadRoutingBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: r#"{"suggested_next_ids": [1]}"#.to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let sandbox_dir = TempDir::new().unwrap();
std::fs::write(
sandbox_dir.path().join("status.json"),
r#"{"preferred_next_label": "should_not_use"}"#,
)
.unwrap();
let handler = AgentHandler::new(Some(Box::new(BadRoutingBackend)));
let mut node = Node::new("step");
node.attrs.insert(
"output_schema".to_string(),
AttrValue::String("routing".to_string()),
);
node.attrs
.insert("output_retries".to_string(), AttrValue::Integer(0));
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let mut services = EngineServices::test_default();
services.run = services.run.with_sandbox(std::sync::Arc::new(
fabro_sandbox::local_sandbox(sandbox_dir.path().to_path_buf())
.await
.unwrap(),
));
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Failed {
retry_requested: false,
});
assert_eq!(
outcome.failure_reason(),
Some("output schema validation failed after 0 repair attempt(s)")
);
assert!(outcome.preferred_label.is_none());
}
#[tokio::test]
async fn codergen_handler_custom_output_schema_updates_output_context_key() {
struct CustomOutputBackend;
#[async_trait]
impl CodergenBackend for CustomOutputBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
Ok(CodergenResult::Text {
text: r#"{"passed": true}"#.to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let handler = AgentHandler::new(Some(Box::new(CustomOutputBackend)));
let mut node = Node::new("audit");
node.attrs.insert(
"output_schema".to_string(),
AttrValue::String(
r#"{"type":"object","required":["passed"],"properties":{"passed":{"type":"boolean"}}}"#
.to_string(),
),
);
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
assert_eq!(
outcome.context_updates.get("output.audit"),
Some(&serde_json::json!({"passed": true})),
);
}
#[tokio::test]
async fn codergen_handler_appends_output_schema_contract_to_prompt() {
use std::sync::{Arc, Mutex};
struct PromptCapturingBackend {
captured_prompt: Arc<Mutex<Option<String>>>,
}
#[async_trait]
impl CodergenBackend for PromptCapturingBackend {
async fn run(&self, request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
*self.captured_prompt.lock().unwrap() = Some(request.prompt.to_string());
Ok(CodergenResult::Text {
text: r#"{"passed": true}"#.to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let captured = Arc::new(Mutex::new(None));
let handler = AgentHandler::new(Some(Box::new(PromptCapturingBackend {
captured_prompt: captured.clone(),
})));
let mut node = Node::new("audit");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Audit the result".to_string()),
);
node.attrs.insert(
"output_schema".to_string(),
AttrValue::String(
r#"{"type":"object","required":["passed"],"properties":{"passed":{"type":"boolean"}}}"#
.to_string(),
),
);
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
let prompt = captured.lock().unwrap().clone().unwrap();
assert!(
prompt.starts_with("Audit the result\n\n"),
"task prompt should come first, got: {prompt}"
);
assert!(
prompt.contains("Fabro final-output contract"),
"contract heading missing, got: {prompt}"
);
assert!(
prompt.contains(
"It applies only to your final response, not to intermediate tool calls."
),
"contract should scope itself to the final response, got: {prompt}"
);
assert!(
prompt.contains(r#""required":["passed"]"#),
"contract should embed the resolved schema, got: {prompt}"
);
assert!(
prompt.ends_with("Do not ask the user to provide or choose the output shape."),
"contract should close the prompt, got: {prompt}"
);
}
#[tokio::test]
async fn codergen_handler_projects_provider_used_from_agent_session_events() {
struct ProviderEventBackend;
#[async_trait]
impl CodergenBackend for ProviderEventBackend {
async fn run(&self, request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
let scope = StageScope::for_handler(request.context, &request.node.id);
request.emitter.emit_scoped(
&crate::event::Event::AgentSessionActivated {
node_id: request.node.id.clone(),
visit: scope.visit,
session_id: "session_123".to_string(),
thread_id: None,
provider: Some("openai".to_string()),
model: Some("gpt-5.4".to_string()),
reasoning_effort: Some(ReasoningEffort::High),
speed: Some(Speed::Fast),
permission_level: None,
capabilities: vec![fabro_types::SessionCapability::Steer],
},
&scope,
);
Ok(CodergenResult::Text {
text: "done".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let handler = AgentHandler::new(Some(Box::new(ProviderEventBackend)));
let node = Node::new("step");
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let (services, run_store, logger) = make_services_with_run_store().await;
handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap();
logger.flush().await.unwrap();
let state = run_store.state().await.unwrap();
let node_state = state.stage(&StageId::new("step", 1)).unwrap();
let provider_used = node_state.provider_used.as_ref().unwrap();
assert_eq!(provider_used.provider.as_deref(), Some("openai"));
assert_eq!(provider_used.reasoning_effort, Some(ReasoningEffort::High));
assert_eq!(provider_used.speed, Some(Speed::Fast));
}
#[test]
fn truncate_short_string() {
assert_eq!(truncate("hello", 200), "hello");
}
#[test]
fn truncate_long_string() {
let long = "a".repeat(300);
assert_eq!(truncate(&long, 200).len(), 200);
}
#[tokio::test]
async fn codergen_handler_passes_thread_id_to_backend() {
use std::sync::{Arc, Mutex};
struct ThreadCapturingBackend {
captured_thread_id: Arc<Mutex<Option<Option<String>>>>,
}
#[async_trait]
impl CodergenBackend for ThreadCapturingBackend {
async fn run(&self, request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
*self.captured_thread_id.lock().unwrap() =
Some(request.thread_id.map(String::from));
Ok(CodergenResult::Text {
text: "ok".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let captured = Arc::new(Mutex::new(None));
let backend = ThreadCapturingBackend {
captured_thread_id: captured.clone(),
};
let handler = AgentHandler::new(Some(Box::new(backend)));
let node = Node::new("work");
let context = test_context();
// Simulate what the engine stores in internal.thread_id
context.set(keys::INTERNAL_THREAD_ID, serde_json::json!("main"));
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
let result = captured.lock().unwrap().clone();
assert_eq!(result, Some(Some("main".to_string())));
}
#[tokio::test]
async fn codergen_handler_passes_none_thread_id_when_absent() {
use std::sync::{Arc, Mutex};
struct ThreadCapturingBackend {
captured_thread_id: Arc<Mutex<Option<Option<String>>>>,
}
#[async_trait]
impl CodergenBackend for ThreadCapturingBackend {
async fn run(&self, request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
*self.captured_thread_id.lock().unwrap() =
Some(request.thread_id.map(String::from));
Ok(CodergenResult::Text {
text: "ok".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let captured = Arc::new(Mutex::new(None));
let backend = ThreadCapturingBackend {
captured_thread_id: captured.clone(),
};
let handler = AgentHandler::new(Some(Box::new(backend)));
let node = Node::new("work");
let context = test_context();
// No thread context set
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
let result = captured.lock().unwrap().clone();
assert_eq!(result, Some(None));
}
#[tokio::test]
async fn codergen_handler_propagates_retryable_backend_error() {
struct FailingBackend;
#[async_trait]
impl CodergenBackend for FailingBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
Err(Error::handler("Request timed out".to_string()))
}
}
let handler = AgentHandler::new(Some(Box::new(FailingBackend)));
let node = Node::new("step");
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let result = handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await;
let err = result.unwrap_err();
assert!(err.is_retryable());
assert!(err.to_string().contains("Request timed out"));
}
#[test]
fn extract_status_fields_from_fenced_code_block() {
let text = r#"Here is my analysis of the code.
```json
{"preferred_next_label": "fix", "outcome": "succeeded"}
```
That's it."#;
let mut outcome = Outcome::success();
extract_status_fields(text, &mut outcome);
assert_eq!(outcome.preferred_label.as_deref(), Some("fix"));
}
#[test]
fn extract_status_fields_from_bare_json() {
let text = r#"I recommend routing to fix.
{"preferred_next_label": "fix_batch"}"#;
let mut outcome = Outcome::success();
extract_status_fields(text, &mut outcome);
assert_eq!(outcome.preferred_label.as_deref(), Some("fix_batch"));
}
#[test]
fn extract_status_fields_no_json() {
let text = "Just some plain text response with no JSON at all.";
let mut outcome = Outcome::success();
extract_status_fields(text, &mut outcome);
assert!(outcome.preferred_label.is_none());
assert!(outcome.suggested_next_ids.is_empty());
}
#[test]
fn extract_status_fields_json_without_status_fields() {
let text = r#"Here is some data: {"name": "test", "count": 42}"#;
let mut outcome = Outcome::success();
extract_status_fields(text, &mut outcome);
assert!(outcome.preferred_label.is_none());
assert!(outcome.suggested_next_ids.is_empty());
}
#[test]
fn extract_status_fields_context_updates_and_suggested_ids() {
let text = r#"```json
{
"preferred_next_label": "review",
"suggested_next_ids": ["node_a", "node_b"],
"context_updates": {"fix.files_changed": 3, "fix.summary": "patched"}
}
```"#;
let mut outcome = Outcome::success();
outcome
.context_updates
.insert("existing_key".to_string(), serde_json::json!("keep"));
extract_status_fields(text, &mut outcome);
assert_eq!(outcome.preferred_label.as_deref(), Some("review"));
assert_eq!(outcome.suggested_next_ids, vec!["node_a", "node_b"]);
assert_eq!(
outcome.context_updates.get("fix.files_changed"),
Some(&serde_json::json!(3))
);
assert_eq!(
outcome.context_updates.get("fix.summary"),
Some(&serde_json::json!("patched"))
);
// Existing keys preserved
assert_eq!(
outcome.context_updates.get("existing_key"),
Some(&serde_json::json!("keep"))
);
}
#[test]
fn extract_status_fields_outcome_fail_with_reason() {
let text = r#"{"outcome": "failed", "failure_reason": "tests failed"}"#;
let mut outcome = Outcome::success();
extract_status_fields(text, &mut outcome);
assert_eq!(outcome.status, crate::outcome::StageOutcome::Failed {
retry_requested: false,
});
assert_eq!(outcome.failure_reason(), Some("tests failed"));
}
#[test]
fn extract_status_fields_outcome_success() {
let text = r#"{"outcome": "succeeded"}"#;
let mut outcome = Outcome::success();
extract_status_fields(text, &mut outcome);
assert_eq!(outcome.status, crate::outcome::StageOutcome::Succeeded);
assert!(outcome.failure.is_none());
}
#[test]
fn extract_status_fields_outcome_fail_without_reason() {
let text = r#"{"outcome": "failed"}"#;
let mut outcome = Outcome::success();
extract_status_fields(text, &mut outcome);
assert_eq!(outcome.status, crate::outcome::StageOutcome::Failed {
retry_requested: false,
});
assert!(outcome.failure.is_none());
}
#[test]
fn extract_status_fields_uses_last_match() {
let text = r#"{"preferred_next_label": "first"}
Some text in between.
{"preferred_next_label": "second"}"#;
let mut outcome = Outcome::success();
extract_status_fields(text, &mut outcome);
assert_eq!(outcome.preferred_label.as_deref(), Some("second"));
}
#[tokio::test]
async fn codergen_handler_returns_fail_outcome_for_non_retryable_backend_error() {
struct ValidationFailBackend;
#[async_trait]
impl CodergenBackend for ValidationFailBackend {
async fn run(&self, _request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
Err(Error::Validation("bad config".to_string()))
}
}
let handler = AgentHandler::new(Some(Box::new(ValidationFailBackend)));
let node = Node::new("step");
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageOutcome::Failed {
retry_requested: false,
});
assert!(outcome.failure_reason().unwrap().contains("bad config"));
}
#[tokio::test]
async fn codergen_handler_prepends_preamble_to_prompt() {
use std::sync::{Arc, Mutex};
struct PromptCapturingBackend {
captured_prompt: Arc<Mutex<Option<String>>>,
}
#[async_trait]
impl CodergenBackend for PromptCapturingBackend {
async fn run(&self, request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
*self.captured_prompt.lock().unwrap() = Some(request.prompt.to_string());
Ok(CodergenResult::Text {
text: "ok".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let captured = Arc::new(Mutex::new(None));
let backend = PromptCapturingBackend {
captured_prompt: captured.clone(),
};
let handler = AgentHandler::new(Some(Box::new(backend)));
let mut node = Node::new("report");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Summarize the results".to_string()),
);
let context = test_context();
context.set(
keys::CURRENT_PREAMBLE,
serde_json::json!("## Test Output\n10 passed, 0 failed"),
);
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
let prompt = captured.lock().unwrap().clone().unwrap();
assert!(
prompt.starts_with("## Test Output\n10 passed, 0 failed"),
"prompt should start with preamble, got: {prompt}"
);
assert!(
prompt.ends_with("Summarize the results"),
"prompt should end with original prompt, got: {prompt}"
);
assert!(
prompt.contains("\n\nSummarize"),
"preamble and prompt should be separated by blank line"
);
}
#[tokio::test]
async fn codergen_handler_no_preamble_when_empty() {
use std::sync::{Arc, Mutex};
struct PromptCapturingBackend {
captured_prompt: Arc<Mutex<Option<String>>>,
}
#[async_trait]
impl CodergenBackend for PromptCapturingBackend {
async fn run(&self, request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
*self.captured_prompt.lock().unwrap() = Some(request.prompt.to_string());
Ok(CodergenResult::Text {
text: "ok".to_string(),
usage_by_model: Vec::new(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let captured = Arc::new(Mutex::new(None));
let backend = PromptCapturingBackend {
captured_prompt: captured.clone(),
};
let handler = AgentHandler::new(Some(Box::new(backend)));
let mut node = Node::new("report");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Summarize the results".to_string()),
);
let context = test_context();
// No preamble set -- context.get_string returns ""
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
let prompt = captured.lock().unwrap().clone().unwrap();
assert_eq!(prompt, "Summarize the results");
}
#[tokio::test]
async fn codergen_handler_preamble_written_to_prompt_md() {
let handler = AgentHandler::new(None);
let mut node = Node::new("report");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Summarize".to_string()),
);
let context = test_context();
context.set(
keys::CURRENT_PREAMBLE,
serde_json::json!("## Script Output\nAll tests passed"),
);
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let (services, run_store, logger) = make_services_with_run_store().await;
handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap();
logger.flush().await.unwrap();
let state = run_store.state().await.unwrap();
let node_state = state.stage(&StageId::new("report", 1)).unwrap();
let prompt_content = node_state.prompt.as_deref().unwrap();
assert!(
prompt_content.contains("## Script Output\nAll tests passed"),
"prompt.md should contain preamble"
);
assert!(
prompt_content.contains("Summarize"),
"prompt.md should contain original prompt"
);
}
}