scriptoriumadmin/fabro
1
0
Fork 0
mirror of https://github.com/fabro-sh/fabro.git synced 2026-10-07 03:00:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 4

Extract outcome and failure_reason from LLM routing directives

Browse source 
The extract_status_fields function recognized "outcome" as a field for
detecting status JSON objects but never read its value — LLM responses
like {"outcome": "fail", "failure_reason": "tests failed"} were silently
ignored and the outcome was always Success.

Now extract_status_fields reads the outcome field to set the node status
and failure_reason to populate the failure detail. Also adds a fallback:
if no routing directives are found in the response text, the handler
reads status.json from the sandbox CWD (written by agents that prefer
file output over inline JSON). Response text always takes priority.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Bryan Helmkamp 2026-03-07 09:38:16 -05:00
parent 5df7e178ac
commit ce80cb6101
2 changed files with 173 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

164
crates/arc-workflows/src/handler/agent.rs
View file

@ -79,6 +79,7 @@ pub(crate) fn expand_variables(text: &str, graph: &Graph) -> Result<String, ArcE
const STATUS_FIELDS: &[&str] = &[
"preferred_next_label",
"outcome",
"failure_reason",
"suggested_next_ids",
"context_updates",
];
@ -127,7 +128,7 @@ fn find_json_objects(text: &str) -> Vec<&str> {
/// Searches for the last JSON object in the response that contains at least
/// one status field (`preferred_next_label`, `outcome`, `suggested_next_ids`,
/// `context_updates`). Merges extracted fields into the outcome.
pub(crate) fn extract_status_fields(text: &str, outcome: &mut Outcome) {
pub(crate) fn extract_status_fields(text: &str, outcome: &mut Outcome) -> bool {
let candidates = find_json_objects(text);
let parsed = candidates.iter().rev().find_map(|candidate| {
@ -140,8 +141,8 @@ pub(crate) fn extract_status_fields(text: &str, outcome: &mut Outcome) {
None
});
let Some(value) = parsed else { return };
let Some(obj) = value.as_object() else { return };
let Some(value) = parsed else { return false };
let Some(obj) = value.as_object() else { return false };
if let Some(label) = obj.get("preferred_next_label").and_then(|v| v.as_str()) {
outcome.preferred_label = Some(label.to_string());
@ -157,11 +158,27 @@ pub(crate) fn extract_status_fields(text: &str, outcome: &mut Outcome) {
}
}
if let Some(status_str) = obj.get("outcome").and_then(|v| v.as_str()) {
if let Ok(status) = status_str.parse::<crate::outcome::StageStatus>() {
outcome.status = status;
if outcome.status == crate::outcome::StageStatus::Fail {
if let Some(reason) = obj.get("failure_reason").and_then(|v| v.as_str()) {
outcome.failure = Some(crate::outcome::FailureDetail::new(
reason,
crate::error::FailureClass::Deterministic,
));
}
}
}
}
if let Some(updates) = obj.get("context_updates").and_then(|v| v.as_object()) {
for (key, val) in updates {
outcome.context_updates.insert(key.clone(), val.clone());
}
}
true
}
/// Truncate a string to at most `max_chars` characters (char-boundary safe).
@ -262,8 +279,20 @@ impl Handler for AgentHandler {
serde_json::json!(&response_text),
);
// 7b. Parse routing directives from response text
extract_status_fields(&response_text, &mut outcome);
// 7b. Parse routing directives from response text, falling back to
// status.json written by the agent into the sandbox CWD
let found_in_response = extract_status_fields(&response_text, &mut outcome);
if !found_in_response {
if let Ok(result) = services
.sandbox
.exec_command("cat status.json", 5_000, None, None, None)
.await
{
if result.exit_code == 0 {
extract_status_fields(&result.stdout, &mut outcome);
}
}
}
outcome.usage = stage_usage;
outcome.files_touched = backend_files_touched;
@ -404,6 +433,104 @@ mod tests {
);
}
#[tokio::test]
async fn codergen_handler_falls_back_to_status_json_in_sandbox() {
// Simulation mode returns text with no JSON directives, so the
// handler should fall back to reading status.json from the sandbox CWD.
let sandbox_dir = TempDir::new().unwrap();
std::fs::write(
sandbox_dir.path().join("status.json"),
r#"{"outcome": "fail", "failure_reason": "tests failed"}"#,
)
.unwrap();
let handler = AgentHandler::new(None);
let node = Node::new("step");
let context = Context::new();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let services = EngineServices {
registry: std::sync::Arc::new(HandlerRegistry::new(Box::new(StartHandler))),
emitter: std::sync::Arc::new(EventEmitter::new()),
sandbox: std::sync::Arc::new(arc_agent::LocalSandbox::new(
sandbox_dir.path().to_path_buf(),
)),
git_state: std::sync::RwLock::new(None),
hook_runner: None,
};
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageStatus::Fail);
assert_eq!(outcome.failure_reason(), Some("tests failed"));
}
#[tokio::test]
async fn codergen_handler_prefers_response_text_over_status_json() {
use std::sync::Arc;
// Backend returns response text with routing directives — status.json
// in the sandbox should be ignored.
struct DirectiveBackend;
#[async_trait]
impl CodergenBackend for DirectiveBackend {
async fn run(
&self,
_node: &Node,
_prompt: &str,
_context: &Context,
_thread_id: Option<&str>,
_emitter: &Arc<EventEmitter>,
_stage_dir: &Path,
_sandbox: &Arc<dyn arc_agent::Sandbox>,
) -> Result<CodergenResult, ArcError> {
Ok(CodergenResult::Text {
text: r#"Done. {"outcome": "success", "preferred_next_label": "approve"}"#
.to_string(),
usage: None,
files_touched: Vec::new(),
})
}
}
let sandbox_dir = TempDir::new().unwrap();
std::fs::write(
sandbox_dir.path().join("status.json"),
r#"{"outcome": "fail", "failure_reason": "should be ignored"}"#,
)
.unwrap();
let handler = AgentHandler::new(Some(Box::new(DirectiveBackend)));
let node = Node::new("step");
let context = Context::new();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
let services = EngineServices {
registry: std::sync::Arc::new(HandlerRegistry::new(Box::new(StartHandler))),
emitter: std::sync::Arc::new(EventEmitter::new()),
sandbox: std::sync::Arc::new(arc_agent::LocalSandbox::new(
sandbox_dir.path().to_path_buf(),
)),
git_state: std::sync::RwLock::new(None),
hook_runner: None,
};
let outcome = handler
.execute(&node, &context, &graph, tmp.path(), &services)
.await
.unwrap();
assert_eq!(outcome.status, crate::outcome::StageStatus::Success);
assert_eq!(outcome.preferred_label.as_deref(), Some("approve"));
assert!(outcome.failure.is_none());
}
#[test]
fn expand_variables_replaces_goal() {
let mut graph = Graph::new("test");
@ -657,6 +784,33 @@ That's it."#;
);
}
#[test]
fn extract_status_fields_outcome_fail_with_reason() {
let text = r#"{"outcome": "fail", "failure_reason": "tests failed"}"#;
let mut outcome = Outcome::success();
extract_status_fields(text, &mut outcome);
assert_eq!(outcome.status, crate::outcome::StageStatus::Fail);
assert_eq!(outcome.failure_reason(), Some("tests failed"));
}
#[test]
fn extract_status_fields_outcome_success() {
let text = r#"{"outcome": "success"}"#;
let mut outcome = Outcome::success();
extract_status_fields(text, &mut outcome);
assert_eq!(outcome.status, crate::outcome::StageStatus::Success);
assert!(outcome.failure.is_none());
}
#[test]
fn extract_status_fields_outcome_fail_without_reason() {
let text = r#"{"outcome": "fail"}"#;
let mut outcome = Outcome::success();
extract_status_fields(text, &mut outcome);
assert_eq!(outcome.status, crate::outcome::StageStatus::Fail);
assert!(outcome.failure.is_none());
}
#[test]
fn extract_status_fields_uses_last_match() {
let text = r#"{"preferred_next_label": "first"}

16
docs/agents/outputs.mdx
View file

@ -35,6 +35,8 @@ Agent and prompt nodes can influence which edge is taken after they complete by
```json
{
"outcome": "fail",
"failure_reason": "tests failed",
"preferred_next_label": "fix",
"suggested_next_ids": ["implement", "review"],
"context_updates": { "tests_passed": true, "coverage": 85 }
@ -43,13 +45,15 @@ Agent and prompt nodes can influence which edge is taken after they complete by
| Field | Effect |
|---|---|
| `outcome` | Sets the node status: `success`, `fail`, `partial_success`, `retry`, or `skipped` |
| `failure_reason` | When `outcome` is `fail`, provides a structured failure message |
| `preferred_next_label` | Matched against edge labels to select the next node |
| `suggested_next_ids` | Ordered list of preferred target node IDs |
| `context_updates` | Key-value pairs merged into the run context |
### How extraction works
Arc finds all balanced `{...}` JSON objects in the response text, parses each one, and uses the **last** object that contains at least one recognized field (`preferred_next_label`, `outcome`, `suggested_next_ids`, `context_updates`). The JSON can appear anywhere in the response -- inside a fenced code block, inline with natural language, or at the end.
Arc finds all balanced `{...}` JSON objects in the response text, parses each one, and uses the **last** object that contains at least one recognized field (`preferred_next_label`, `outcome`, `failure_reason`, `suggested_next_ids`, `context_updates`). The JSON can appear anywhere in the response -- inside a fenced code block, inline with natural language, or at the end.
```markdown
I've reviewed the code and found several issues that need fixing.
@ -58,7 +62,15 @@ The test coverage is below the threshold.
{"preferred_next_label": "fix", "context_updates": {"coverage": 72}}
```
JSON objects without recognized fields are ignored. If no routing directive is found, the transition falls through to condition matching, unconditional edges, or weight-based tiebreaking as described in [Transitions](/workflows/transitions).
JSON objects without recognized fields are ignored.
### Fallback: status.json file
If no routing directives are found in the response text, Arc checks whether the agent wrote a `status.json` file into the sandbox working directory. If the file exists, Arc extracts routing directives from it using the same logic. This is useful for agents that write structured output to files rather than including JSON in their response text.
Response text directives always take priority -- `status.json` is only read as a fallback when the response contains no recognized routing fields.
If neither source provides routing directives, the transition falls through to condition matching, unconditional edges, or weight-based tiebreaking as described in [Transitions](/workflows/transitions).
### Instructing the agent