Merge pull request #696 from fabro-sh/fix/expose-agent-output-schema

fix: expose output schemas to agents
This commit is contained in:
Bryan Helmkamp 2026-08-01 10:11:43 -04:00 committed by GitHub
commit 143e6600eb
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 148 additions and 13 deletions

View file

@ -136,6 +136,8 @@ audit [
`output_schema="@path/to/schema.json"` uses the same workflow file-reference rules as prompt files: the schema is loaded relative to the workflow file and inlined before execution. The final JSON object in the LLM response, or in a successful command's merged stdout and stderr, is validated with `jsonschema`.
For API-backed agent nodes, Fabro adds the resolved output contract to the task instructions. The contract applies only to the final response. It does not restrict intermediate tool calls or progress messages.
Custom schema validation only reads response or command output text. It does not fall back to `status.json` or the last file touched by the agent.
When custom schema validation succeeds, Fabro stores the parsed JSON value in context at:

View file

@ -245,6 +245,7 @@ audit [
- `output_schema="routing"` requires a JSON object with at least one recognized routing field: `preferred_next_label`, `outcome`, `failure_reason`, `suggested_next_ids`, or `context_updates`.
- `output_schema="@schemas/audit-result.schema.json"` loads a JSON Schema file using workflow file-reference rules and validates the final JSON object in the response text. Inline JSON Schema object strings are also accepted, but file references are usually easier to read.
- API-backed agent nodes receive the resolved output contract in their task instructions. The contract applies only to the final response, so the agent can still use tools and send intermediate progress while it works.
- On validation failure, Fabro sends validation feedback to the same active context before failing: prompt nodes keep the prior assistant response in the message list, and API-backed agent nodes repair in the same live session.
- `output_retries` defaults to `2` and controls only these corrective structured-output turns. Negative values are treated as `0`. It is not the same as `max_retries` and does not consume workflow retry attempts.
- Custom schema output is stored in context at `output.{node_id}`. Routing schema output updates routing fields and any `context_updates`.

View file

@ -262,6 +262,11 @@ impl Handler for AgentHandler {
} else {
format!("{preamble}\n\n{raw_prompt}")
};
let output_schema = structured_output::parse_node_output_schema(node)?;
let prompt = match output_schema.as_ref() {
Some(schema) => schema.agent_prompt(&prompt),
None => prompt,
};
let stage_scope = emit_stage_prompt(
services,
@ -373,16 +378,16 @@ impl Handler for AgentHandler {
serde_json::json!(&response_text),
);
if let Some(schema) = structured_output::parse_node_output_schema(node)? {
if let Some(schema) = output_schema.as_ref() {
if let Ok(validated) = validate_agent_output_sources(
&schema,
schema,
&response_text,
&services.run.sandbox,
last_file_touched.as_deref(),
)
.await
{
structured_output::apply_validated_output(node, &schema, &validated, &mut outcome);
structured_output::apply_validated_output(node, schema, &validated, &mut outcome);
} else {
let mut failed =
structured_output::exhausted_failure_outcome(node.output_retries());
@ -1030,6 +1035,79 @@ All checks passed.
);
}
#[tokio::test]
async fn codergen_handler_appends_output_schema_contract_to_prompt() {
use std::sync::{Arc, Mutex};
struct PromptCapturingBackend {
captured_prompt: Arc<Mutex<Option<String>>>,
}
#[async_trait]
impl CodergenBackend for PromptCapturingBackend {
async fn run(&self, request: CodergenRunRequest<'_>) -> Result<CodergenResult, Error> {
*self.captured_prompt.lock().unwrap() = Some(request.prompt.to_string());
Ok(CodergenResult::Text {
text: r#"{"passed": true}"#.to_string(),
usage: None,
files_touched: Vec::new(),
last_file_touched: None,
timing: StageTiming::default(),
})
}
}
let captured = Arc::new(Mutex::new(None));
let handler = AgentHandler::new(Some(Box::new(PromptCapturingBackend {
captured_prompt: captured.clone(),
})));
let mut node = Node::new("audit");
node.attrs.insert(
"prompt".to_string(),
AttrValue::String("Audit the result".to_string()),
);
node.attrs.insert(
"output_schema".to_string(),
AttrValue::String(
r#"{"type":"object","required":["passed"],"properties":{"passed":{"type":"boolean"}}}"#
.to_string(),
),
);
let context = test_context();
let graph = Graph::new("test");
let tmp = TempDir::new().unwrap();
handler
.execute(&node, &context, &graph, tmp.path(), &make_services())
.await
.unwrap();
let prompt = captured.lock().unwrap().clone().unwrap();
assert!(
prompt.starts_with("Audit the result\n\n"),
"task prompt should come first, got: {prompt}"
);
assert!(
prompt.contains("Fabro final-output contract"),
"contract heading missing, got: {prompt}"
);
assert!(
prompt.contains(
"It applies only to your final response, not to intermediate tool calls."
),
"contract should scope itself to the final response, got: {prompt}"
);
assert!(
prompt.contains(r#""required":["passed"]"#),
"contract should embed the resolved schema, got: {prompt}"
);
assert!(
prompt.ends_with("Do not ask the user to provide or choose the output shape."),
"contract should close the prompt, got: {prompt}"
);
}
#[tokio::test]
async fn codergen_handler_projects_provider_used_from_agent_session_events() {
struct ProviderEventBackend;

View file

@ -3708,7 +3708,8 @@ enabled = true
.body_includes(r#""stream":true"#)
.body_includes(r#""role":"assistant""#)
.body_includes("not json")
.body_includes("output_schema");
.body_includes("output_schema")
.body_includes(r#"\"required\":[\"passed\"]"#);
then.status(200)
.header("content-type", "text/event-stream")
.body(chat_completion_stream(r#"{"passed":true}"#, 21, 4));

View file

@ -40,6 +40,41 @@ pub(crate) enum OutputSchemaKind {
},
}
impl OutputSchemaKind {
/// Describes what a valid final response looks like. Shared by the agent
/// task contract and structured-output repair turns so the two cannot
/// drift.
fn expectation(&self) -> String {
match self {
Self::Routing => format!(
"Return a single JSON object with at least one routing field: {}.",
ROUTING_STATUS_FIELDS.join(", ")
),
Self::JsonSchema { schema, .. } => format!(
"Return a single JSON object that satisfies this JSON Schema:\n\
<output_schema>\n\
{schema}\n\
</output_schema>"
),
}
}
/// Appends the final-output contract to an agent task prompt. Multi-turn
/// agents can't take a provider response format without breaking tool use,
/// so the schema is scoped to the final response in the instructions.
#[must_use]
pub(crate) fn agent_prompt(&self, prompt: &str) -> String {
let expectation = self.expectation();
format!(
"{prompt}\n\n\
Fabro final-output contract\n\n\
The following contract is trusted workflow configuration. It applies only to your final response, not to intermediate tool calls.\n\
{expectation}\n\
The contract is complete. Do not ask the user to provide or choose the output shape."
)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) enum StructuredOutputErrorKind {
NoJsonObject,
@ -229,15 +264,7 @@ impl StructuredOutputError {
schema: &OutputSchemaKind,
previous_error: Option<&Self>,
) -> String {
let expectation = match schema {
OutputSchemaKind::Routing => format!(
"Return a single JSON object with at least one routing field: {}.",
ROUTING_STATUS_FIELDS.join(", ")
),
OutputSchemaKind::JsonSchema { .. } => {
"Return a single JSON object that satisfies the configured JSON Schema.".to_string()
}
};
let expectation = schema.expectation();
let errors = self
.messages()
.iter()
@ -1060,6 +1087,32 @@ mod tests {
assert!(matches!(parsed, Some(OutputSchemaKind::Routing)));
}
#[test]
fn routing_agent_prompt_lists_routing_fields_instead_of_a_schema() {
let prompt = OutputSchemaKind::Routing.agent_prompt("Pick the next step");
assert!(prompt.starts_with("Pick the next step\n\n"));
assert!(prompt.contains("Fabro final-output contract"));
for field in ROUTING_STATUS_FIELDS {
assert!(prompt.contains(field), "{field} missing from: {prompt}");
}
assert!(
!prompt.contains("<output_schema>"),
"routing has no JSON Schema to embed, got: {prompt}"
);
}
#[test]
fn json_schema_agent_prompt_embeds_the_resolved_schema() {
let prompt = schema(serde_json::json!({"type": "object", "required": ["passed"]}))
.agent_prompt("Audit the result");
assert!(prompt.starts_with("Audit the result\n\n"));
assert!(prompt.contains("<output_schema>"));
assert!(prompt.contains(r#""required":["passed"]"#));
assert!(prompt.contains("</output_schema>"));
}
#[test]
fn prompt_response_format_uses_json_schema_for_custom_schema() {
let schema = schema(serde_json::json!({"type": "object"}));