feat(home): add structured-report agent and gate/equivalence test coverage

Add StructuredReportAgent wrapper selectable via the
_structured_report_model option in BoundedAgentWrapper, plus its
dedicated unit tests. Extend candidate gate, construction equivalence,
evaluation history, sampling, and validation budget tests, and document
the new behavior in the plugin README.
This commit is contained in:
酱牛肉 2026-11-02 01:44:31 +08:00
parent 97e42e0caf
commit abf36eceec
12 changed files with 417 additions and 1 deletions

View file

@ -919,6 +919,14 @@ are deleted when review finishes, fails or is cancelled. Reports retain commit a
source-evaluation provenance; `source_path` records the temporary location used
during review and is no longer available afterward. Model or malformed-report failures are
recorded and force a full rebuild; sandbox setup failure stops evaluation.
Both construction-equivalence and candidate-admission reports accept JSON wrapped
in prose or a code fence, ignore think blocks, and repair trailing commas outside
strings. If parsing or schema validation still fails, the reviewer makes one
format-only structured-output call with thinking disabled and no tools. This call
retains the system prompt, compression summary and investigation history, including
the invalid final reply. Successful corrections are saved in the session and their
token usage is included in the report. If correction fails, the existing fallback
applies: construction is rebuilt or candidate admission allows evaluation.
Reviewer shell commands receive an explicit minimal environment instead of
inheriting host model credentials or other environment values.
Cancellation propagates, and a branch/commit change during review fails before

View file

@ -124,6 +124,7 @@ async def _review_candidate(wrapper, home, context, budget, scope, memory_limit_
session_id=None,
fork_session=False,
output_schema=None,
_structured_report_model=CandidateGateReport,
permission_mode="bypass",
)
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})

View file

@ -100,6 +100,7 @@ async def _review_construction(wrapper, home, context, scope, memory_limit_mib):
session_id=None,
fork_session=False,
output_schema=None,
_structured_report_model=ConstructionEquivalenceReport,
permission_mode="bypass",
)
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})

View file

@ -0,0 +1,177 @@
"""Shared recovery for schema-constrained final reports from ReAct agents."""
from __future__ import annotations
import json
import logging
import re
from typing import Any, TypeVar
from agentscope.message import AssistantMsg, Msg, Usage, UserMsg
from pydantic import BaseModel
from .bounded import BoundedReActAgent
ReportT = TypeVar("ReportT", bound=BaseModel)
logger = logging.getLogger(__name__)
class StructuredReportAgent(BoundedReActAgent):
"""Validate review reports and make at most one context-preserving correction."""
def __init__(self, *args, report_model: type[ReportT], **kwargs):
super().__init__(*args, **kwargs)
self.report_model = report_model
async def reply(self, *args, **kwargs) -> Msg:
"""Keep normal wrapper persistence and accounting around report recovery."""
reply = await super().reply(*args, **kwargs)
try:
report = self.report_model.model_validate(parse_json_object(reply, report_name=self.name))
except ValueError as exc:
logger.warning(
"%s returned invalid structured output; requesting one format-only correction: %s",
self.name,
exc,
)
model_input = await self._prepare_model_input()
correction = UserMsg(
name="home_report_recovery",
content=(
"Your previous report did not conform to the required JSON schema. Using only the "
"analysis and evidence already present in this conversation, "
"immediately return the final decision. "
"Do not inspect more files, call tools, add Markdown, or include any other commentary."
),
)
# Do not forward tools: this call only reformats the existing conclusion.
corrected = await self.model.generate_structured_output(
messages=[*model_input["messages"], correction],
structured_model=self.report_model,
**_thinking_disabled_kwargs(self.model),
)
report = self.report_model.model_validate(corrected.content)
usage = None
if reply.usage is not None and corrected.usage is not None:
usage = Usage(
**{key: getattr(reply.usage, key) + getattr(corrected.usage, key, 0) for key in Usage.model_fields},
)
reply = AssistantMsg(name=self.name, content=report.model_dump_json(), usage=usage)
self.state.context.extend([correction, reply])
return reply
return reply.model_copy(
update={"content": AssistantMsg(name=self.name, content=report.model_dump_json()).content},
)
def parse_json_object(message: Msg, *, report_name: str) -> dict[str, Any]:
"""Parse an object with conservative repairs for common model formatting errors."""
text = message.get_text_content() or ""
candidates = [text.strip()]
without_thinking = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL | re.IGNORECASE).strip()
if without_thinking and without_thinking != candidates[0]:
candidates.append(without_thinking)
fenced_candidates = [
match.group(1).strip()
for match in re.finditer(r"```(?:json)?\s*(.*?)```", without_thinking, flags=re.DOTALL | re.IGNORECASE)
]
if len(fenced_candidates) == 1:
candidates.extend(fenced_candidates)
object_candidates = _extract_json_objects(without_thinking)
if len(object_candidates) == 1:
candidates.extend(object_candidates)
saw_non_object = False
seen: set[str] = set()
for candidate in candidates:
if not candidate or candidate in seen:
continue
seen.add(candidate)
repaired = _remove_trailing_commas(candidate)
for attempt in (candidate,) if repaired == candidate else (candidate, repaired):
try:
payload = json.loads(attempt)
except json.JSONDecodeError:
continue
if isinstance(payload, dict):
return payload
saw_non_object = True
if saw_non_object:
raise ValueError(f"{report_name} report must be a JSON object")
raise ValueError(f"{report_name} did not return a JSON object") from None
def _extract_json_objects(text: str) -> list[str]:
"""Extract balanced JSON-looking objects without interpreting braces inside strings."""
objects: list[str] = []
start: int | None = None
depth = 0
in_string = False
escaped = False
for index, character in enumerate(text):
if in_string:
if escaped:
escaped = False
elif character == "\\":
escaped = True
elif character == '"':
in_string = False
continue
if character == '"':
in_string = True
elif character == "{":
if depth == 0:
start = index
depth += 1
elif character == "}" and depth:
depth -= 1
if depth == 0 and start is not None:
objects.append(text[start : index + 1])
start = None
return objects
def _remove_trailing_commas(text: str) -> str:
"""Remove commas immediately before closing JSON containers, outside strings."""
output: list[str] = []
in_string = False
escaped = False
index = 0
while index < len(text):
character = text[index]
if in_string:
output.append(character)
if escaped:
escaped = False
elif character == "\\":
escaped = True
elif character == '"':
in_string = False
index += 1
continue
if character == '"':
in_string = True
if character == ",":
lookahead = index + 1
while lookahead < len(text) and text[lookahead].isspace():
lookahead += 1
if lookahead < len(text) and text[lookahead] in "}]":
index += 1
continue
output.append(character)
index += 1
return "".join(output)
def _thinking_disabled_kwargs(model: Any) -> dict[str, Any]:
"""Build per-request settings for a deterministic format-only correction."""
extra_body = dict(getattr(model, "extra_body", None) or {})
extra_body.pop("enable_thinking", None)
extra_body.pop("thinking_budget", None)
extra_body.update({"reasoning_effort": "none", "preserve_thinking": False})
return {"extra_body": extra_body}

View file

@ -11,6 +11,7 @@ from reme.components.as_llm import BaseAsLLM
from ..judge import HomeJudge
from .bounded import BoundedReActAgent, RunTimeBudgetTimer
from .structured_report import StructuredReportAgent
class BoundedAgentWrapper(AsAgentWrapper):
@ -24,6 +25,9 @@ class BoundedAgentWrapper(AsAgentWrapper):
def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> BoundedReActAgent:
budget = options.get("time_budget_minutes")
timer = None if budget is None else RunTimeBudgetTimer(budget, agent_label=self.name)
report_model = options.get("_structured_report_model")
if report_model is not None:
return StructuredReportAgent(report_model=report_model, run_time_budget_timer=timer, **kwargs)
return BoundedReActAgent(run_time_budget_timer=timer, **kwargs)

View file

@ -6,8 +6,10 @@ import asyncio
from pathlib import Path
import sys
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from agentscope.model import ChatUsage, StructuredResponse
from agentscope.tool import LocalBackend
from reme_home.agent import candidate_gate, workflow_common
from reme_home.evaluation.artifacts import read_json
@ -20,6 +22,8 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
prepared_home,
rebuild_review,
os_sandbox_available,
)
from test_workflows import ScriptedModel, set_model
@ -99,6 +103,7 @@ async def test_gate_before_sampling_preserves_budget_and_pins_code(app, prepared
@pytest.mark.asyncio
@pytest.mark.usefixtures("rebuild_review")
async def test_baseline_debug_unbudgeted_and_late_phase_skip_gate(app, prepared, monkeypatch):
async def unexpected(*args, **kwargs):
raise AssertionError("Gate must be skipped")
@ -161,6 +166,29 @@ async def test_model_failure_allows_validation_with_saved_diagnostics(app, prepa
assert not list((prepared / "tmp").glob("candidate-gate-*"))
@pytest.mark.asyncio
async def test_corrected_gate_rejects_before_spending_budget(app, prepared, sandbox):
commit_candidate(prepared)
model = ScriptedModel(final="Reject this candidate: only prompt tuning.")
model.generate_structured_output = AsyncMock(
return_value=StructuredResponse(
content={"decision": "reject", "reason": "Only prompt tuning"},
usage=ChatUsage(input_tokens=7, output_tokens=11, time=0),
),
)
set_model(app, "home_candidate_gate", model)
response = await app.run_job("home_evaluate", max_validations=10)
assert not response.success
assert response.answer["error_code"] == "CANDIDATE_GATE_REJECTED"
report = read_json(Path(response.answer["candidate_gate"]["review_path"]))
assert report["decision"] == "reject" and "review_failed" not in report
assert report["usage"]["input_tokens"] == 9 and report["usage"]["output_tokens"] == 14
assert report["session_id"] and not Path(report["source_path"]).exists()
assert validation_budget_status(prepared, 10)["used_validations"] == 0
assert not FakeFactory.instances
model.generate_structured_output.assert_awaited_once()
@pytest.mark.asyncio
async def test_sandbox_setup_failure_never_calls_model_or_evaluates(app, prepared, monkeypatch):
commit_candidate(prepared)
@ -214,6 +242,7 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared,
@pytest.mark.asyncio
@pytest.mark.usefixtures("os_sandbox_available")
async def test_real_gate_sandbox_denies_evidence_live_code_writes_and_network(app, prepared, tmp_path):
commit_candidate(prepared)
context = candidate_gate.admission_context(prepared)

View file

@ -11,6 +11,7 @@ from types import MethodType, SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from agentscope.model import ChatUsage, StructuredResponse
from agentscope.tool import LocalBackend
from reme_home.agent import candidate_gate, construction_equivalence, optimizer, workflow_common
from reme_home.code.snapshot import read_code_file
@ -26,6 +27,7 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
prepared_home,
os_sandbox_available,
)
from test_workflows import ScriptedModel, final_report, set_model
@ -56,6 +58,30 @@ def force_draw(monkeypatch, draw):
monkeypatch.setattr(reuse.random, "SystemRandom", lambda: SimpleNamespace(random=lambda: draw))
@pytest.mark.asyncio
async def test_corrected_equivalence_authorizes_replay(app, prepared, reviewer, sandbox, monkeypatch):
assert (await app.run_job("home_evaluate")).success
commit_candidate(prepared)
model = ScriptedModel(final="Construction is unchanged: only query behavior changes.")
model.generate_structured_output = AsyncMock(
return_value=StructuredResponse(
content={"decision": "construction_unchanged", "reason": "Only query behavior changes"},
usage=ChatUsage(input_tokens=7, output_tokens=11, time=0),
),
)
set_model(app, "home_construction_equivalence", model)
force_draw(monkeypatch, 0.1)
response = await app.run_job("home_evaluate", sampling=True)
assert response.success, response.answer
assert response.answer["execution_mode"] == "query_only_replay"
report = response.answer["automatic_construction_reuse"]["equivalence_report"]
assert report["decision"] == "construction_unchanged" and "review_failed" not in report
assert report["usage"]["input_tokens"] == 9 and report["usage"]["output_tokens"] == 14
assert read_json(Path(report["review_path"]))["session_id"]
assert not Path(report["source_path"]).exists()
model.generate_structured_output.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize("changed", [False, True])
async def test_large_resources_export_for_both_reviewers(app, prepared, reviewer, sandbox, monkeypatch, changed):
@ -509,6 +535,7 @@ async def test_optimizer_final_selection_accepts_automatic_replay(app, prepared,
@pytest.mark.asyncio
@pytest.mark.usefixtures("os_sandbox_available")
async def test_real_equivalence_sandbox_allows_only_copied_construction_and_source(
app,
prepared,

View file

@ -6,6 +6,8 @@ import asyncio
import io
import json
from pathlib import Path
import shutil
import sys
import tarfile
from unittest.mock import AsyncMock
@ -23,6 +25,31 @@ from test_preparation import ( # noqa: F401 # pylint: disable=unused-import
)
@pytest.fixture
def rebuild_review(monkeypatch):
"""Keep evaluation bookkeeping tests independent of review models and OS sandboxes."""
from reme_home.agent import construction_equivalence
async def review(wrapper, home, context, spec, cases, fingerprints, *, memory_limit_mib=None):
return {**context, "decision": "construction_changed", "reason": "Fixture rebuild"}
monkeypatch.setattr(construction_equivalence, "review_construction", review)
@pytest.fixture
def os_sandbox_available():
"""Skip real boundary tests only when the platform or executable is unsupported."""
if sys.platform.startswith("linux"):
executable = "bwrap"
elif sys.platform == "darwin":
executable = "sandbox-exec"
else:
pytest.skip(f"No file sandbox backend for {sys.platform}")
return
if shutil.which(executable) is None:
pytest.skip(f"Sandbox boundary test requires {executable} on PATH")
def archive(path, files):
path.parent.mkdir(parents=True, exist_ok=True)
with tarfile.open(path, "w:gz") as stream:

View file

@ -6,6 +6,7 @@ import json
from pathlib import Path
import pytest
from agentscope.message import ToolResultState
from reme_home.evaluation.history import HomeEvaluationHistory, EvaluationRecord, read_state
from reme_home.sampling import SamplingPolicy, create_sampling_plan
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
@ -14,10 +15,13 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
prepared_home,
rebuild_review,
)
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
pytestmark = pytest.mark.usefixtures("rebuild_review")
def history(app):
return app.context.components["home.evaluation_history"]["default"]
@ -176,7 +180,8 @@ async def test_job_defaults_and_hidden_agent_injection_use_one_job(app, prepared
)
assert "sampling" not in tool.input_schema["properties"]
commit_candidate(prepared)
await tool.call()
result = await tool.call()
assert result.state == ToolResultState.SUCCESS, result
records = history(app).list_results()
assert records[-1].case_count == records[-1].scored_query_count == 1
assert len(records) == 3

View file

@ -18,8 +18,11 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
prepared_home,
rebuild_review,
)
pytestmark = pytest.mark.usefixtures("rebuild_review")
def cases():
return [
@ -277,6 +280,7 @@ async def test_replay_sampling_is_limited_to_donor_cases(app, prepared):
# The next full rebuild must see the entire training set again.
commit_candidate(prepared)
response = await app.run_job("home_evaluate", sampling=True)
assert response.success, response.answer
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
assert set(plan["case_draw_weights"]) == {"1", "2"} and plan["eligibility"] is None

View file

@ -0,0 +1,130 @@
"""Review report recovery without network or provider dependencies."""
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access,unused-argument
import asyncio
from unittest.mock import AsyncMock
import pytest
from agentscope.agent import ReActConfig
from agentscope.message import AssistantMsg, UserMsg
from agentscope.model import ChatUsage, StructuredResponse
from reme_home.agent.candidate_gate import CandidateGateReport
from reme_home.agent.structured_report import StructuredReportAgent, parse_json_object
from test_workflows import ScriptedModel
@pytest.mark.parametrize(
"text",
[
'{"decision":"reject","reason":"Mechanism unchanged"}',
'<think>private analysis</think>\n{"decision":"reject","reason":"Mechanism unchanged"}',
'Conclusion:\n```json\n{"decision":"reject","reason":"Mechanism unchanged",}\n```',
'Conclusion: {"decision":"reject","reason":"Mechanism unchanged",} Done.',
],
)
def test_parse_review_formats(text):
assert parse_json_object(AssistantMsg(name="review", content=text), report_name="review") == {
"decision": "reject",
"reason": "Mechanism unchanged",
}
@pytest.mark.parametrize("text", ["invalid", "[]", '{"decision":"accept"} {"decision":"reject"}'])
def test_invalid_or_ambiguous_reports_require_correction(text):
with pytest.raises(ValueError):
parse_json_object(AssistantMsg(name="review", content=text), report_name="review")
def test_repairs_preserve_strings_and_nested_json():
text = 'Report: {"reason":"literal ,} and escaped \\"quote\\" {", "items":[{"value":1,},],}'
assert parse_json_object(AssistantMsg(name="review", content=text), report_name="review") == {
"reason": 'literal ,} and escaped "quote" {',
"items": [{"value": 1}],
}
def make_agent(model):
return StructuredReportAgent(
name="review",
system_prompt="Review committed code.",
model=model,
report_model=CandidateGateReport,
react_config=ReActConfig(max_iters=3),
)
@pytest.mark.asyncio
@pytest.mark.parametrize("missing_usage", [False, True])
@pytest.mark.parametrize(
"initial",
["Reject: mechanism unchanged.", '{"decision":"reject","reason":"x","extra":true}'],
)
async def test_one_correction_retains_history_disables_tools_and_accounts_usage(missing_usage, initial):
model = ScriptedModel(final=initial)
model.extra_body = {"enable_thinking": True, "thinking_budget": 100, "host_option": "retained"}
model.generate_structured_output = AsyncMock(
return_value=StructuredResponse(
content={"decision": "reject", "reason": "Mechanism unchanged"},
usage=None if missing_usage else ChatUsage(input_tokens=7, output_tokens=11, time=0),
),
)
agent = make_agent(model)
agent.state.summary = "Earlier investigation summary."
reply = await agent.reply(UserMsg(name="host", content="Review this candidate."))
assert CandidateGateReport.model_validate_json(reply.get_text_content()).decision == "reject"
assert len(model.calls) == 1
model.generate_structured_output.assert_awaited_once()
options = model.generate_structured_output.call_args.kwargs
assert set(options) == {"messages", "structured_model", "extra_body"}
assert options["structured_model"] is CandidateGateReport
assert options["extra_body"] == {
"reasoning_effort": "none",
"preserve_thinking": False,
"host_option": "retained",
}
text = "\n".join(message.get_text_content() or "" for message in options["messages"])
for expected in (
"Review committed code.",
"Earlier investigation summary.",
"Review this candidate.",
initial,
"Do not inspect more files, call tools",
):
assert expected in text
assert model.extra_body["enable_thinking"] is True
assert agent.state.context[-1] is reply
if missing_usage:
assert reply.usage is None
else:
assert reply.usage.input_tokens == 9 and reply.usage.output_tokens == 14
@pytest.mark.asyncio
async def test_tolerant_parse_avoids_model_correction():
model = ScriptedModel(final='Decision: {"decision":"accept","reason":"Mechanism changes",}')
model.generate_structured_output = AsyncMock()
reply = await make_agent(model).reply(UserMsg(name="host", content="Review."))
assert CandidateGateReport.model_validate_json(reply.get_text_content()).decision == "accept"
model.generate_structured_output.assert_not_awaited()
assert reply.usage.input_tokens == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["invalid", "provider", "cancel"])
async def test_correction_is_not_retried_and_cancellation_propagates(failure):
model = ScriptedModel(final="invalid")
model.generate_structured_output = AsyncMock(
return_value=StructuredResponse(content={"decision": "unknown", "reason": "Invalid"}),
)
expected = ValueError
if failure == "provider":
model.generate_structured_output.side_effect = RuntimeError("offline")
expected = RuntimeError
elif failure == "cancel":
model.generate_structured_output.side_effect = asyncio.CancelledError()
expected = asyncio.CancelledError
with pytest.raises(expected):
await make_agent(model).reply(UserMsg(name="host", content="Review."))
model.generate_structured_output.assert_awaited_once()

View file

@ -16,8 +16,11 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
home_application,
preparation_inputs,
prepared_home,
rebuild_review,
)
pytestmark = pytest.mark.usefixtures("rebuild_review")
def seed_history(home, keys):
write_json(