diff --git a/plugins/home/README.md b/plugins/home/README.md
index 8353db29..c98c9a87 100644
--- a/plugins/home/README.md
+++ b/plugins/home/README.md
@@ -919,6 +919,14 @@ are deleted when review finishes, fails or is cancelled. Reports retain commit a
source-evaluation provenance; `source_path` records the temporary location used
during review and is no longer available afterward. Model or malformed-report failures are
recorded and force a full rebuild; sandbox setup failure stops evaluation.
+Both construction-equivalence and candidate-admission reports accept JSON wrapped
+in prose or a code fence, ignore think blocks, and repair trailing commas outside
+strings. If parsing or schema validation still fails, the reviewer makes one
+format-only structured-output call with thinking disabled and no tools. This call
+retains the system prompt, compression summary and investigation history, including
+the invalid final reply. Successful corrections are saved in the session and their
+token usage is included in the report. If correction fails, the existing fallback
+applies: construction is rebuilt or candidate admission allows evaluation.
Reviewer shell commands receive an explicit minimal environment instead of
inheriting host model credentials or other environment values.
Cancellation propagates, and a branch/commit change during review fails before
diff --git a/plugins/home/src/reme_home/agent/candidate_gate.py b/plugins/home/src/reme_home/agent/candidate_gate.py
index e99836ec..4951f243 100644
--- a/plugins/home/src/reme_home/agent/candidate_gate.py
+++ b/plugins/home/src/reme_home/agent/candidate_gate.py
@@ -124,6 +124,7 @@ async def _review_candidate(wrapper, home, context, budget, scope, memory_limit_
session_id=None,
fork_session=False,
output_schema=None,
+ _structured_report_model=CandidateGateReport,
permission_mode="bypass",
)
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})
diff --git a/plugins/home/src/reme_home/agent/construction_equivalence.py b/plugins/home/src/reme_home/agent/construction_equivalence.py
index 58b6d903..a2b20610 100644
--- a/plugins/home/src/reme_home/agent/construction_equivalence.py
+++ b/plugins/home/src/reme_home/agent/construction_equivalence.py
@@ -100,6 +100,7 @@ async def _review_construction(wrapper, home, context, scope, memory_limit_mib):
session_id=None,
fork_session=False,
output_schema=None,
+ _structured_report_model=ConstructionEquivalenceReport,
permission_mode="bypass",
)
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})
diff --git a/plugins/home/src/reme_home/agent/structured_report.py b/plugins/home/src/reme_home/agent/structured_report.py
new file mode 100644
index 00000000..5cba8838
--- /dev/null
+++ b/plugins/home/src/reme_home/agent/structured_report.py
@@ -0,0 +1,177 @@
+"""Shared recovery for schema-constrained final reports from ReAct agents."""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+from typing import Any, TypeVar
+
+from agentscope.message import AssistantMsg, Msg, Usage, UserMsg
+from pydantic import BaseModel
+
+from .bounded import BoundedReActAgent
+
+ReportT = TypeVar("ReportT", bound=BaseModel)
+
+logger = logging.getLogger(__name__)
+
+
+class StructuredReportAgent(BoundedReActAgent):
+ """Validate review reports and make at most one context-preserving correction."""
+
+ def __init__(self, *args, report_model: type[ReportT], **kwargs):
+ super().__init__(*args, **kwargs)
+ self.report_model = report_model
+
+ async def reply(self, *args, **kwargs) -> Msg:
+ """Keep normal wrapper persistence and accounting around report recovery."""
+ reply = await super().reply(*args, **kwargs)
+ try:
+ report = self.report_model.model_validate(parse_json_object(reply, report_name=self.name))
+ except ValueError as exc:
+ logger.warning(
+ "%s returned invalid structured output; requesting one format-only correction: %s",
+ self.name,
+ exc,
+ )
+ model_input = await self._prepare_model_input()
+ correction = UserMsg(
+ name="home_report_recovery",
+ content=(
+ "Your previous report did not conform to the required JSON schema. Using only the "
+ "analysis and evidence already present in this conversation, "
+ "immediately return the final decision. "
+ "Do not inspect more files, call tools, add Markdown, or include any other commentary."
+ ),
+ )
+ # Do not forward tools: this call only reformats the existing conclusion.
+ corrected = await self.model.generate_structured_output(
+ messages=[*model_input["messages"], correction],
+ structured_model=self.report_model,
+ **_thinking_disabled_kwargs(self.model),
+ )
+ report = self.report_model.model_validate(corrected.content)
+ usage = None
+ if reply.usage is not None and corrected.usage is not None:
+ usage = Usage(
+ **{key: getattr(reply.usage, key) + getattr(corrected.usage, key, 0) for key in Usage.model_fields},
+ )
+ reply = AssistantMsg(name=self.name, content=report.model_dump_json(), usage=usage)
+ self.state.context.extend([correction, reply])
+ return reply
+ return reply.model_copy(
+ update={"content": AssistantMsg(name=self.name, content=report.model_dump_json()).content},
+ )
+
+
+def parse_json_object(message: Msg, *, report_name: str) -> dict[str, Any]:
+ """Parse an object with conservative repairs for common model formatting errors."""
+
+ text = message.get_text_content() or ""
+ candidates = [text.strip()]
+ without_thinking = re.sub(r".*?", "", text, flags=re.DOTALL | re.IGNORECASE).strip()
+ if without_thinking and without_thinking != candidates[0]:
+ candidates.append(without_thinking)
+ fenced_candidates = [
+ match.group(1).strip()
+ for match in re.finditer(r"```(?:json)?\s*(.*?)```", without_thinking, flags=re.DOTALL | re.IGNORECASE)
+ ]
+ if len(fenced_candidates) == 1:
+ candidates.extend(fenced_candidates)
+ object_candidates = _extract_json_objects(without_thinking)
+ if len(object_candidates) == 1:
+ candidates.extend(object_candidates)
+
+ saw_non_object = False
+ seen: set[str] = set()
+ for candidate in candidates:
+ if not candidate or candidate in seen:
+ continue
+ seen.add(candidate)
+ repaired = _remove_trailing_commas(candidate)
+ for attempt in (candidate,) if repaired == candidate else (candidate, repaired):
+ try:
+ payload = json.loads(attempt)
+ except json.JSONDecodeError:
+ continue
+ if isinstance(payload, dict):
+ return payload
+ saw_non_object = True
+ if saw_non_object:
+ raise ValueError(f"{report_name} report must be a JSON object")
+ raise ValueError(f"{report_name} did not return a JSON object") from None
+
+
+def _extract_json_objects(text: str) -> list[str]:
+ """Extract balanced JSON-looking objects without interpreting braces inside strings."""
+
+ objects: list[str] = []
+ start: int | None = None
+ depth = 0
+ in_string = False
+ escaped = False
+ for index, character in enumerate(text):
+ if in_string:
+ if escaped:
+ escaped = False
+ elif character == "\\":
+ escaped = True
+ elif character == '"':
+ in_string = False
+ continue
+ if character == '"':
+ in_string = True
+ elif character == "{":
+ if depth == 0:
+ start = index
+ depth += 1
+ elif character == "}" and depth:
+ depth -= 1
+ if depth == 0 and start is not None:
+ objects.append(text[start : index + 1])
+ start = None
+ return objects
+
+
+def _remove_trailing_commas(text: str) -> str:
+ """Remove commas immediately before closing JSON containers, outside strings."""
+
+ output: list[str] = []
+ in_string = False
+ escaped = False
+ index = 0
+ while index < len(text):
+ character = text[index]
+ if in_string:
+ output.append(character)
+ if escaped:
+ escaped = False
+ elif character == "\\":
+ escaped = True
+ elif character == '"':
+ in_string = False
+ index += 1
+ continue
+ if character == '"':
+ in_string = True
+ if character == ",":
+ lookahead = index + 1
+ while lookahead < len(text) and text[lookahead].isspace():
+ lookahead += 1
+ if lookahead < len(text) and text[lookahead] in "}]":
+ index += 1
+ continue
+ output.append(character)
+ index += 1
+ return "".join(output)
+
+
+def _thinking_disabled_kwargs(model: Any) -> dict[str, Any]:
+ """Build per-request settings for a deterministic format-only correction."""
+
+ extra_body = dict(getattr(model, "extra_body", None) or {})
+ extra_body.pop("enable_thinking", None)
+ extra_body.pop("thinking_budget", None)
+ extra_body.update({"reasoning_effort": "none", "preserve_thinking": False})
+ return {"extra_body": extra_body}
diff --git a/plugins/home/src/reme_home/agent/wrapper.py b/plugins/home/src/reme_home/agent/wrapper.py
index efb814c2..3a5091ac 100644
--- a/plugins/home/src/reme_home/agent/wrapper.py
+++ b/plugins/home/src/reme_home/agent/wrapper.py
@@ -11,6 +11,7 @@ from reme.components.as_llm import BaseAsLLM
from ..judge import HomeJudge
from .bounded import BoundedReActAgent, RunTimeBudgetTimer
+from .structured_report import StructuredReportAgent
class BoundedAgentWrapper(AsAgentWrapper):
@@ -24,6 +25,9 @@ class BoundedAgentWrapper(AsAgentWrapper):
def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> BoundedReActAgent:
budget = options.get("time_budget_minutes")
timer = None if budget is None else RunTimeBudgetTimer(budget, agent_label=self.name)
+ report_model = options.get("_structured_report_model")
+ if report_model is not None:
+ return StructuredReportAgent(report_model=report_model, run_time_budget_timer=timer, **kwargs)
return BoundedReActAgent(run_time_budget_timer=timer, **kwargs)
diff --git a/plugins/home/tests/test_candidate_gate.py b/plugins/home/tests/test_candidate_gate.py
index d4a92e14..bde00438 100644
--- a/plugins/home/tests/test_candidate_gate.py
+++ b/plugins/home/tests/test_candidate_gate.py
@@ -6,8 +6,10 @@ import asyncio
from pathlib import Path
import sys
from types import SimpleNamespace
+from unittest.mock import AsyncMock
import pytest
+from agentscope.model import ChatUsage, StructuredResponse
from agentscope.tool import LocalBackend
from reme_home.agent import candidate_gate, workflow_common
from reme_home.evaluation.artifacts import read_json
@@ -20,6 +22,8 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
prepared_home,
+ rebuild_review,
+ os_sandbox_available,
)
from test_workflows import ScriptedModel, set_model
@@ -99,6 +103,7 @@ async def test_gate_before_sampling_preserves_budget_and_pins_code(app, prepared
@pytest.mark.asyncio
+@pytest.mark.usefixtures("rebuild_review")
async def test_baseline_debug_unbudgeted_and_late_phase_skip_gate(app, prepared, monkeypatch):
async def unexpected(*args, **kwargs):
raise AssertionError("Gate must be skipped")
@@ -161,6 +166,29 @@ async def test_model_failure_allows_validation_with_saved_diagnostics(app, prepa
assert not list((prepared / "tmp").glob("candidate-gate-*"))
+@pytest.mark.asyncio
+async def test_corrected_gate_rejects_before_spending_budget(app, prepared, sandbox):
+ commit_candidate(prepared)
+ model = ScriptedModel(final="Reject this candidate: only prompt tuning.")
+ model.generate_structured_output = AsyncMock(
+ return_value=StructuredResponse(
+ content={"decision": "reject", "reason": "Only prompt tuning"},
+ usage=ChatUsage(input_tokens=7, output_tokens=11, time=0),
+ ),
+ )
+ set_model(app, "home_candidate_gate", model)
+ response = await app.run_job("home_evaluate", max_validations=10)
+ assert not response.success
+ assert response.answer["error_code"] == "CANDIDATE_GATE_REJECTED"
+ report = read_json(Path(response.answer["candidate_gate"]["review_path"]))
+ assert report["decision"] == "reject" and "review_failed" not in report
+ assert report["usage"]["input_tokens"] == 9 and report["usage"]["output_tokens"] == 14
+ assert report["session_id"] and not Path(report["source_path"]).exists()
+ assert validation_budget_status(prepared, 10)["used_validations"] == 0
+ assert not FakeFactory.instances
+ model.generate_structured_output.assert_awaited_once()
+
+
@pytest.mark.asyncio
async def test_sandbox_setup_failure_never_calls_model_or_evaluates(app, prepared, monkeypatch):
commit_candidate(prepared)
@@ -214,6 +242,7 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared,
@pytest.mark.asyncio
+@pytest.mark.usefixtures("os_sandbox_available")
async def test_real_gate_sandbox_denies_evidence_live_code_writes_and_network(app, prepared, tmp_path):
commit_candidate(prepared)
context = candidate_gate.admission_context(prepared)
diff --git a/plugins/home/tests/test_construction_equivalence.py b/plugins/home/tests/test_construction_equivalence.py
index 41f91ab7..f42636ff 100644
--- a/plugins/home/tests/test_construction_equivalence.py
+++ b/plugins/home/tests/test_construction_equivalence.py
@@ -11,6 +11,7 @@ from types import MethodType, SimpleNamespace
from unittest.mock import AsyncMock
import pytest
+from agentscope.model import ChatUsage, StructuredResponse
from agentscope.tool import LocalBackend
from reme_home.agent import candidate_gate, construction_equivalence, optimizer, workflow_common
from reme_home.code.snapshot import read_code_file
@@ -26,6 +27,7 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
prepared_home,
+ os_sandbox_available,
)
from test_workflows import ScriptedModel, final_report, set_model
@@ -56,6 +58,30 @@ def force_draw(monkeypatch, draw):
monkeypatch.setattr(reuse.random, "SystemRandom", lambda: SimpleNamespace(random=lambda: draw))
+@pytest.mark.asyncio
+async def test_corrected_equivalence_authorizes_replay(app, prepared, reviewer, sandbox, monkeypatch):
+ assert (await app.run_job("home_evaluate")).success
+ commit_candidate(prepared)
+ model = ScriptedModel(final="Construction is unchanged: only query behavior changes.")
+ model.generate_structured_output = AsyncMock(
+ return_value=StructuredResponse(
+ content={"decision": "construction_unchanged", "reason": "Only query behavior changes"},
+ usage=ChatUsage(input_tokens=7, output_tokens=11, time=0),
+ ),
+ )
+ set_model(app, "home_construction_equivalence", model)
+ force_draw(monkeypatch, 0.1)
+ response = await app.run_job("home_evaluate", sampling=True)
+ assert response.success, response.answer
+ assert response.answer["execution_mode"] == "query_only_replay"
+ report = response.answer["automatic_construction_reuse"]["equivalence_report"]
+ assert report["decision"] == "construction_unchanged" and "review_failed" not in report
+ assert report["usage"]["input_tokens"] == 9 and report["usage"]["output_tokens"] == 14
+ assert read_json(Path(report["review_path"]))["session_id"]
+ assert not Path(report["source_path"]).exists()
+ model.generate_structured_output.assert_awaited_once()
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("changed", [False, True])
async def test_large_resources_export_for_both_reviewers(app, prepared, reviewer, sandbox, monkeypatch, changed):
@@ -509,6 +535,7 @@ async def test_optimizer_final_selection_accepts_automatic_replay(app, prepared,
@pytest.mark.asyncio
+@pytest.mark.usefixtures("os_sandbox_available")
async def test_real_equivalence_sandbox_allows_only_copied_construction_and_source(
app,
prepared,
diff --git a/plugins/home/tests/test_evaluation.py b/plugins/home/tests/test_evaluation.py
index 0d65396c..0462f827 100644
--- a/plugins/home/tests/test_evaluation.py
+++ b/plugins/home/tests/test_evaluation.py
@@ -6,6 +6,8 @@ import asyncio
import io
import json
from pathlib import Path
+import shutil
+import sys
import tarfile
from unittest.mock import AsyncMock
@@ -23,6 +25,31 @@ from test_preparation import ( # noqa: F401 # pylint: disable=unused-import
)
+@pytest.fixture
+def rebuild_review(monkeypatch):
+ """Keep evaluation bookkeeping tests independent of review models and OS sandboxes."""
+ from reme_home.agent import construction_equivalence
+
+ async def review(wrapper, home, context, spec, cases, fingerprints, *, memory_limit_mib=None):
+ return {**context, "decision": "construction_changed", "reason": "Fixture rebuild"}
+
+ monkeypatch.setattr(construction_equivalence, "review_construction", review)
+
+
+@pytest.fixture
+def os_sandbox_available():
+ """Skip real boundary tests only when the platform or executable is unsupported."""
+ if sys.platform.startswith("linux"):
+ executable = "bwrap"
+ elif sys.platform == "darwin":
+ executable = "sandbox-exec"
+ else:
+ pytest.skip(f"No file sandbox backend for {sys.platform}")
+ return
+ if shutil.which(executable) is None:
+ pytest.skip(f"Sandbox boundary test requires {executable} on PATH")
+
+
def archive(path, files):
path.parent.mkdir(parents=True, exist_ok=True)
with tarfile.open(path, "w:gz") as stream:
diff --git a/plugins/home/tests/test_evaluation_history.py b/plugins/home/tests/test_evaluation_history.py
index 885f81c2..7fb2aaa0 100644
--- a/plugins/home/tests/test_evaluation_history.py
+++ b/plugins/home/tests/test_evaluation_history.py
@@ -6,6 +6,7 @@ import json
from pathlib import Path
import pytest
+from agentscope.message import ToolResultState
from reme_home.evaluation.history import HomeEvaluationHistory, EvaluationRecord, read_state
from reme_home.sampling import SamplingPolicy, create_sampling_plan
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
@@ -14,10 +15,13 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
prepared_home,
+ rebuild_review,
)
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
+pytestmark = pytest.mark.usefixtures("rebuild_review")
+
def history(app):
return app.context.components["home.evaluation_history"]["default"]
@@ -176,7 +180,8 @@ async def test_job_defaults_and_hidden_agent_injection_use_one_job(app, prepared
)
assert "sampling" not in tool.input_schema["properties"]
commit_candidate(prepared)
- await tool.call()
+ result = await tool.call()
+ assert result.state == ToolResultState.SUCCESS, result
records = history(app).list_results()
assert records[-1].case_count == records[-1].scored_query_count == 1
assert len(records) == 3
diff --git a/plugins/home/tests/test_sampling.py b/plugins/home/tests/test_sampling.py
index 6ab4dac8..ef0c826d 100644
--- a/plugins/home/tests/test_sampling.py
+++ b/plugins/home/tests/test_sampling.py
@@ -18,8 +18,11 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
prepared_home,
+ rebuild_review,
)
+pytestmark = pytest.mark.usefixtures("rebuild_review")
+
def cases():
return [
@@ -277,6 +280,7 @@ async def test_replay_sampling_is_limited_to_donor_cases(app, prepared):
# The next full rebuild must see the entire training set again.
commit_candidate(prepared)
response = await app.run_job("home_evaluate", sampling=True)
+ assert response.success, response.answer
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
assert set(plan["case_draw_weights"]) == {"1", "2"} and plan["eligibility"] is None
diff --git a/plugins/home/tests/test_structured_report.py b/plugins/home/tests/test_structured_report.py
new file mode 100644
index 00000000..ef76f65e
--- /dev/null
+++ b/plugins/home/tests/test_structured_report.py
@@ -0,0 +1,130 @@
+"""Review report recovery without network or provider dependencies."""
+
+# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access,unused-argument
+
+import asyncio
+from unittest.mock import AsyncMock
+
+import pytest
+from agentscope.agent import ReActConfig
+from agentscope.message import AssistantMsg, UserMsg
+from agentscope.model import ChatUsage, StructuredResponse
+from reme_home.agent.candidate_gate import CandidateGateReport
+from reme_home.agent.structured_report import StructuredReportAgent, parse_json_object
+
+from test_workflows import ScriptedModel
+
+
+@pytest.mark.parametrize(
+ "text",
+ [
+ '{"decision":"reject","reason":"Mechanism unchanged"}',
+ 'private analysis\n{"decision":"reject","reason":"Mechanism unchanged"}',
+ 'Conclusion:\n```json\n{"decision":"reject","reason":"Mechanism unchanged",}\n```',
+ 'Conclusion: {"decision":"reject","reason":"Mechanism unchanged",} Done.',
+ ],
+)
+def test_parse_review_formats(text):
+ assert parse_json_object(AssistantMsg(name="review", content=text), report_name="review") == {
+ "decision": "reject",
+ "reason": "Mechanism unchanged",
+ }
+
+
+@pytest.mark.parametrize("text", ["invalid", "[]", '{"decision":"accept"} {"decision":"reject"}'])
+def test_invalid_or_ambiguous_reports_require_correction(text):
+ with pytest.raises(ValueError):
+ parse_json_object(AssistantMsg(name="review", content=text), report_name="review")
+
+
+def test_repairs_preserve_strings_and_nested_json():
+ text = 'Report: {"reason":"literal ,} and escaped \\"quote\\" {", "items":[{"value":1,},],}'
+ assert parse_json_object(AssistantMsg(name="review", content=text), report_name="review") == {
+ "reason": 'literal ,} and escaped "quote" {',
+ "items": [{"value": 1}],
+ }
+
+
+def make_agent(model):
+ return StructuredReportAgent(
+ name="review",
+ system_prompt="Review committed code.",
+ model=model,
+ report_model=CandidateGateReport,
+ react_config=ReActConfig(max_iters=3),
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("missing_usage", [False, True])
+@pytest.mark.parametrize(
+ "initial",
+ ["Reject: mechanism unchanged.", '{"decision":"reject","reason":"x","extra":true}'],
+)
+async def test_one_correction_retains_history_disables_tools_and_accounts_usage(missing_usage, initial):
+ model = ScriptedModel(final=initial)
+ model.extra_body = {"enable_thinking": True, "thinking_budget": 100, "host_option": "retained"}
+ model.generate_structured_output = AsyncMock(
+ return_value=StructuredResponse(
+ content={"decision": "reject", "reason": "Mechanism unchanged"},
+ usage=None if missing_usage else ChatUsage(input_tokens=7, output_tokens=11, time=0),
+ ),
+ )
+ agent = make_agent(model)
+ agent.state.summary = "Earlier investigation summary."
+ reply = await agent.reply(UserMsg(name="host", content="Review this candidate."))
+ assert CandidateGateReport.model_validate_json(reply.get_text_content()).decision == "reject"
+ assert len(model.calls) == 1
+ model.generate_structured_output.assert_awaited_once()
+ options = model.generate_structured_output.call_args.kwargs
+ assert set(options) == {"messages", "structured_model", "extra_body"}
+ assert options["structured_model"] is CandidateGateReport
+ assert options["extra_body"] == {
+ "reasoning_effort": "none",
+ "preserve_thinking": False,
+ "host_option": "retained",
+ }
+ text = "\n".join(message.get_text_content() or "" for message in options["messages"])
+ for expected in (
+ "Review committed code.",
+ "Earlier investigation summary.",
+ "Review this candidate.",
+ initial,
+ "Do not inspect more files, call tools",
+ ):
+ assert expected in text
+ assert model.extra_body["enable_thinking"] is True
+ assert agent.state.context[-1] is reply
+ if missing_usage:
+ assert reply.usage is None
+ else:
+ assert reply.usage.input_tokens == 9 and reply.usage.output_tokens == 14
+
+
+@pytest.mark.asyncio
+async def test_tolerant_parse_avoids_model_correction():
+ model = ScriptedModel(final='Decision: {"decision":"accept","reason":"Mechanism changes",}')
+ model.generate_structured_output = AsyncMock()
+ reply = await make_agent(model).reply(UserMsg(name="host", content="Review."))
+ assert CandidateGateReport.model_validate_json(reply.get_text_content()).decision == "accept"
+ model.generate_structured_output.assert_not_awaited()
+ assert reply.usage.input_tokens == 2
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("failure", ["invalid", "provider", "cancel"])
+async def test_correction_is_not_retried_and_cancellation_propagates(failure):
+ model = ScriptedModel(final="invalid")
+ model.generate_structured_output = AsyncMock(
+ return_value=StructuredResponse(content={"decision": "unknown", "reason": "Invalid"}),
+ )
+ expected = ValueError
+ if failure == "provider":
+ model.generate_structured_output.side_effect = RuntimeError("offline")
+ expected = RuntimeError
+ elif failure == "cancel":
+ model.generate_structured_output.side_effect = asyncio.CancelledError()
+ expected = asyncio.CancelledError
+ with pytest.raises(expected):
+ await make_agent(model).reply(UserMsg(name="host", content="Review."))
+ model.generate_structured_output.assert_awaited_once()
diff --git a/plugins/home/tests/test_validation_budget.py b/plugins/home/tests/test_validation_budget.py
index ef222cc8..52822f56 100644
--- a/plugins/home/tests/test_validation_budget.py
+++ b/plugins/home/tests/test_validation_budget.py
@@ -16,8 +16,11 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
prepared_home,
+ rebuild_review,
)
+pytestmark = pytest.mark.usefixtures("rebuild_review")
+
def seed_history(home, keys):
write_json(