diff --git a/plugins/home/README.md b/plugins/home/README.md index 8353db29..c98c9a87 100644 --- a/plugins/home/README.md +++ b/plugins/home/README.md @@ -919,6 +919,14 @@ are deleted when review finishes, fails or is cancelled. Reports retain commit a source-evaluation provenance; `source_path` records the temporary location used during review and is no longer available afterward. Model or malformed-report failures are recorded and force a full rebuild; sandbox setup failure stops evaluation. +Both construction-equivalence and candidate-admission reports accept JSON wrapped +in prose or a code fence, ignore think blocks, and repair trailing commas outside +strings. If parsing or schema validation still fails, the reviewer makes one +format-only structured-output call with thinking disabled and no tools. This call +retains the system prompt, compression summary and investigation history, including +the invalid final reply. Successful corrections are saved in the session and their +token usage is included in the report. If correction fails, the existing fallback +applies: construction is rebuilt or candidate admission allows evaluation. Reviewer shell commands receive an explicit minimal environment instead of inheriting host model credentials or other environment values. Cancellation propagates, and a branch/commit change during review fails before diff --git a/plugins/home/src/reme_home/agent/candidate_gate.py b/plugins/home/src/reme_home/agent/candidate_gate.py index e99836ec..4951f243 100644 --- a/plugins/home/src/reme_home/agent/candidate_gate.py +++ b/plugins/home/src/reme_home/agent/candidate_gate.py @@ -124,6 +124,7 @@ async def _review_candidate(wrapper, home, context, budget, scope, memory_limit_ session_id=None, fork_session=False, output_schema=None, + _structured_report_model=CandidateGateReport, permission_mode="bypass", ) report.update({key: reply.get(key) for key in ("session_id", "usage", "result")}) diff --git a/plugins/home/src/reme_home/agent/construction_equivalence.py b/plugins/home/src/reme_home/agent/construction_equivalence.py index 58b6d903..a2b20610 100644 --- a/plugins/home/src/reme_home/agent/construction_equivalence.py +++ b/plugins/home/src/reme_home/agent/construction_equivalence.py @@ -100,6 +100,7 @@ async def _review_construction(wrapper, home, context, scope, memory_limit_mib): session_id=None, fork_session=False, output_schema=None, + _structured_report_model=ConstructionEquivalenceReport, permission_mode="bypass", ) report.update({key: reply.get(key) for key in ("session_id", "usage", "result")}) diff --git a/plugins/home/src/reme_home/agent/structured_report.py b/plugins/home/src/reme_home/agent/structured_report.py new file mode 100644 index 00000000..5cba8838 --- /dev/null +++ b/plugins/home/src/reme_home/agent/structured_report.py @@ -0,0 +1,177 @@ +"""Shared recovery for schema-constrained final reports from ReAct agents.""" + +from __future__ import annotations + +import json +import logging +import re +from typing import Any, TypeVar + +from agentscope.message import AssistantMsg, Msg, Usage, UserMsg +from pydantic import BaseModel + +from .bounded import BoundedReActAgent + +ReportT = TypeVar("ReportT", bound=BaseModel) + +logger = logging.getLogger(__name__) + + +class StructuredReportAgent(BoundedReActAgent): + """Validate review reports and make at most one context-preserving correction.""" + + def __init__(self, *args, report_model: type[ReportT], **kwargs): + super().__init__(*args, **kwargs) + self.report_model = report_model + + async def reply(self, *args, **kwargs) -> Msg: + """Keep normal wrapper persistence and accounting around report recovery.""" + reply = await super().reply(*args, **kwargs) + try: + report = self.report_model.model_validate(parse_json_object(reply, report_name=self.name)) + except ValueError as exc: + logger.warning( + "%s returned invalid structured output; requesting one format-only correction: %s", + self.name, + exc, + ) + model_input = await self._prepare_model_input() + correction = UserMsg( + name="home_report_recovery", + content=( + "Your previous report did not conform to the required JSON schema. Using only the " + "analysis and evidence already present in this conversation, " + "immediately return the final decision. " + "Do not inspect more files, call tools, add Markdown, or include any other commentary." + ), + ) + # Do not forward tools: this call only reformats the existing conclusion. + corrected = await self.model.generate_structured_output( + messages=[*model_input["messages"], correction], + structured_model=self.report_model, + **_thinking_disabled_kwargs(self.model), + ) + report = self.report_model.model_validate(corrected.content) + usage = None + if reply.usage is not None and corrected.usage is not None: + usage = Usage( + **{key: getattr(reply.usage, key) + getattr(corrected.usage, key, 0) for key in Usage.model_fields}, + ) + reply = AssistantMsg(name=self.name, content=report.model_dump_json(), usage=usage) + self.state.context.extend([correction, reply]) + return reply + return reply.model_copy( + update={"content": AssistantMsg(name=self.name, content=report.model_dump_json()).content}, + ) + + +def parse_json_object(message: Msg, *, report_name: str) -> dict[str, Any]: + """Parse an object with conservative repairs for common model formatting errors.""" + + text = message.get_text_content() or "" + candidates = [text.strip()] + without_thinking = re.sub(r".*?", "", text, flags=re.DOTALL | re.IGNORECASE).strip() + if without_thinking and without_thinking != candidates[0]: + candidates.append(without_thinking) + fenced_candidates = [ + match.group(1).strip() + for match in re.finditer(r"```(?:json)?\s*(.*?)```", without_thinking, flags=re.DOTALL | re.IGNORECASE) + ] + if len(fenced_candidates) == 1: + candidates.extend(fenced_candidates) + object_candidates = _extract_json_objects(without_thinking) + if len(object_candidates) == 1: + candidates.extend(object_candidates) + + saw_non_object = False + seen: set[str] = set() + for candidate in candidates: + if not candidate or candidate in seen: + continue + seen.add(candidate) + repaired = _remove_trailing_commas(candidate) + for attempt in (candidate,) if repaired == candidate else (candidate, repaired): + try: + payload = json.loads(attempt) + except json.JSONDecodeError: + continue + if isinstance(payload, dict): + return payload + saw_non_object = True + if saw_non_object: + raise ValueError(f"{report_name} report must be a JSON object") + raise ValueError(f"{report_name} did not return a JSON object") from None + + +def _extract_json_objects(text: str) -> list[str]: + """Extract balanced JSON-looking objects without interpreting braces inside strings.""" + + objects: list[str] = [] + start: int | None = None + depth = 0 + in_string = False + escaped = False + for index, character in enumerate(text): + if in_string: + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == '"': + in_string = False + continue + if character == '"': + in_string = True + elif character == "{": + if depth == 0: + start = index + depth += 1 + elif character == "}" and depth: + depth -= 1 + if depth == 0 and start is not None: + objects.append(text[start : index + 1]) + start = None + return objects + + +def _remove_trailing_commas(text: str) -> str: + """Remove commas immediately before closing JSON containers, outside strings.""" + + output: list[str] = [] + in_string = False + escaped = False + index = 0 + while index < len(text): + character = text[index] + if in_string: + output.append(character) + if escaped: + escaped = False + elif character == "\\": + escaped = True + elif character == '"': + in_string = False + index += 1 + continue + if character == '"': + in_string = True + if character == ",": + lookahead = index + 1 + while lookahead < len(text) and text[lookahead].isspace(): + lookahead += 1 + if lookahead < len(text) and text[lookahead] in "}]": + index += 1 + continue + output.append(character) + index += 1 + return "".join(output) + + +def _thinking_disabled_kwargs(model: Any) -> dict[str, Any]: + """Build per-request settings for a deterministic format-only correction.""" + + extra_body = dict(getattr(model, "extra_body", None) or {}) + extra_body.pop("enable_thinking", None) + extra_body.pop("thinking_budget", None) + extra_body.update({"reasoning_effort": "none", "preserve_thinking": False}) + return {"extra_body": extra_body} diff --git a/plugins/home/src/reme_home/agent/wrapper.py b/plugins/home/src/reme_home/agent/wrapper.py index efb814c2..3a5091ac 100644 --- a/plugins/home/src/reme_home/agent/wrapper.py +++ b/plugins/home/src/reme_home/agent/wrapper.py @@ -11,6 +11,7 @@ from reme.components.as_llm import BaseAsLLM from ..judge import HomeJudge from .bounded import BoundedReActAgent, RunTimeBudgetTimer +from .structured_report import StructuredReportAgent class BoundedAgentWrapper(AsAgentWrapper): @@ -24,6 +25,9 @@ class BoundedAgentWrapper(AsAgentWrapper): def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> BoundedReActAgent: budget = options.get("time_budget_minutes") timer = None if budget is None else RunTimeBudgetTimer(budget, agent_label=self.name) + report_model = options.get("_structured_report_model") + if report_model is not None: + return StructuredReportAgent(report_model=report_model, run_time_budget_timer=timer, **kwargs) return BoundedReActAgent(run_time_budget_timer=timer, **kwargs) diff --git a/plugins/home/tests/test_candidate_gate.py b/plugins/home/tests/test_candidate_gate.py index d4a92e14..bde00438 100644 --- a/plugins/home/tests/test_candidate_gate.py +++ b/plugins/home/tests/test_candidate_gate.py @@ -6,8 +6,10 @@ import asyncio from pathlib import Path import sys from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest +from agentscope.model import ChatUsage, StructuredResponse from agentscope.tool import LocalBackend from reme_home.agent import candidate_gate, workflow_common from reme_home.evaluation.artifacts import read_json @@ -20,6 +22,8 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import home_application, preparation_inputs, prepared_home, + rebuild_review, + os_sandbox_available, ) from test_workflows import ScriptedModel, set_model @@ -99,6 +103,7 @@ async def test_gate_before_sampling_preserves_budget_and_pins_code(app, prepared @pytest.mark.asyncio +@pytest.mark.usefixtures("rebuild_review") async def test_baseline_debug_unbudgeted_and_late_phase_skip_gate(app, prepared, monkeypatch): async def unexpected(*args, **kwargs): raise AssertionError("Gate must be skipped") @@ -161,6 +166,29 @@ async def test_model_failure_allows_validation_with_saved_diagnostics(app, prepa assert not list((prepared / "tmp").glob("candidate-gate-*")) +@pytest.mark.asyncio +async def test_corrected_gate_rejects_before_spending_budget(app, prepared, sandbox): + commit_candidate(prepared) + model = ScriptedModel(final="Reject this candidate: only prompt tuning.") + model.generate_structured_output = AsyncMock( + return_value=StructuredResponse( + content={"decision": "reject", "reason": "Only prompt tuning"}, + usage=ChatUsage(input_tokens=7, output_tokens=11, time=0), + ), + ) + set_model(app, "home_candidate_gate", model) + response = await app.run_job("home_evaluate", max_validations=10) + assert not response.success + assert response.answer["error_code"] == "CANDIDATE_GATE_REJECTED" + report = read_json(Path(response.answer["candidate_gate"]["review_path"])) + assert report["decision"] == "reject" and "review_failed" not in report + assert report["usage"]["input_tokens"] == 9 and report["usage"]["output_tokens"] == 14 + assert report["session_id"] and not Path(report["source_path"]).exists() + assert validation_budget_status(prepared, 10)["used_validations"] == 0 + assert not FakeFactory.instances + model.generate_structured_output.assert_awaited_once() + + @pytest.mark.asyncio async def test_sandbox_setup_failure_never_calls_model_or_evaluates(app, prepared, monkeypatch): commit_candidate(prepared) @@ -214,6 +242,7 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared, @pytest.mark.asyncio +@pytest.mark.usefixtures("os_sandbox_available") async def test_real_gate_sandbox_denies_evidence_live_code_writes_and_network(app, prepared, tmp_path): commit_candidate(prepared) context = candidate_gate.admission_context(prepared) diff --git a/plugins/home/tests/test_construction_equivalence.py b/plugins/home/tests/test_construction_equivalence.py index 41f91ab7..f42636ff 100644 --- a/plugins/home/tests/test_construction_equivalence.py +++ b/plugins/home/tests/test_construction_equivalence.py @@ -11,6 +11,7 @@ from types import MethodType, SimpleNamespace from unittest.mock import AsyncMock import pytest +from agentscope.model import ChatUsage, StructuredResponse from agentscope.tool import LocalBackend from reme_home.agent import candidate_gate, construction_equivalence, optimizer, workflow_common from reme_home.code.snapshot import read_code_file @@ -26,6 +27,7 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import home_application, preparation_inputs, prepared_home, + os_sandbox_available, ) from test_workflows import ScriptedModel, final_report, set_model @@ -56,6 +58,30 @@ def force_draw(monkeypatch, draw): monkeypatch.setattr(reuse.random, "SystemRandom", lambda: SimpleNamespace(random=lambda: draw)) +@pytest.mark.asyncio +async def test_corrected_equivalence_authorizes_replay(app, prepared, reviewer, sandbox, monkeypatch): + assert (await app.run_job("home_evaluate")).success + commit_candidate(prepared) + model = ScriptedModel(final="Construction is unchanged: only query behavior changes.") + model.generate_structured_output = AsyncMock( + return_value=StructuredResponse( + content={"decision": "construction_unchanged", "reason": "Only query behavior changes"}, + usage=ChatUsage(input_tokens=7, output_tokens=11, time=0), + ), + ) + set_model(app, "home_construction_equivalence", model) + force_draw(monkeypatch, 0.1) + response = await app.run_job("home_evaluate", sampling=True) + assert response.success, response.answer + assert response.answer["execution_mode"] == "query_only_replay" + report = response.answer["automatic_construction_reuse"]["equivalence_report"] + assert report["decision"] == "construction_unchanged" and "review_failed" not in report + assert report["usage"]["input_tokens"] == 9 and report["usage"]["output_tokens"] == 14 + assert read_json(Path(report["review_path"]))["session_id"] + assert not Path(report["source_path"]).exists() + model.generate_structured_output.assert_awaited_once() + + @pytest.mark.asyncio @pytest.mark.parametrize("changed", [False, True]) async def test_large_resources_export_for_both_reviewers(app, prepared, reviewer, sandbox, monkeypatch, changed): @@ -509,6 +535,7 @@ async def test_optimizer_final_selection_accepts_automatic_replay(app, prepared, @pytest.mark.asyncio +@pytest.mark.usefixtures("os_sandbox_available") async def test_real_equivalence_sandbox_allows_only_copied_construction_and_source( app, prepared, diff --git a/plugins/home/tests/test_evaluation.py b/plugins/home/tests/test_evaluation.py index 0d65396c..0462f827 100644 --- a/plugins/home/tests/test_evaluation.py +++ b/plugins/home/tests/test_evaluation.py @@ -6,6 +6,8 @@ import asyncio import io import json from pathlib import Path +import shutil +import sys import tarfile from unittest.mock import AsyncMock @@ -23,6 +25,31 @@ from test_preparation import ( # noqa: F401 # pylint: disable=unused-import ) +@pytest.fixture +def rebuild_review(monkeypatch): + """Keep evaluation bookkeeping tests independent of review models and OS sandboxes.""" + from reme_home.agent import construction_equivalence + + async def review(wrapper, home, context, spec, cases, fingerprints, *, memory_limit_mib=None): + return {**context, "decision": "construction_changed", "reason": "Fixture rebuild"} + + monkeypatch.setattr(construction_equivalence, "review_construction", review) + + +@pytest.fixture +def os_sandbox_available(): + """Skip real boundary tests only when the platform or executable is unsupported.""" + if sys.platform.startswith("linux"): + executable = "bwrap" + elif sys.platform == "darwin": + executable = "sandbox-exec" + else: + pytest.skip(f"No file sandbox backend for {sys.platform}") + return + if shutil.which(executable) is None: + pytest.skip(f"Sandbox boundary test requires {executable} on PATH") + + def archive(path, files): path.parent.mkdir(parents=True, exist_ok=True) with tarfile.open(path, "w:gz") as stream: diff --git a/plugins/home/tests/test_evaluation_history.py b/plugins/home/tests/test_evaluation_history.py index 885f81c2..7fb2aaa0 100644 --- a/plugins/home/tests/test_evaluation_history.py +++ b/plugins/home/tests/test_evaluation_history.py @@ -6,6 +6,7 @@ import json from pathlib import Path import pytest +from agentscope.message import ToolResultState from reme_home.evaluation.history import HomeEvaluationHistory, EvaluationRecord, read_state from reme_home.sampling import SamplingPolicy, create_sampling_plan from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import @@ -14,10 +15,13 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import home_application, preparation_inputs, prepared_home, + rebuild_review, ) from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper +pytestmark = pytest.mark.usefixtures("rebuild_review") + def history(app): return app.context.components["home.evaluation_history"]["default"] @@ -176,7 +180,8 @@ async def test_job_defaults_and_hidden_agent_injection_use_one_job(app, prepared ) assert "sampling" not in tool.input_schema["properties"] commit_candidate(prepared) - await tool.call() + result = await tool.call() + assert result.state == ToolResultState.SUCCESS, result records = history(app).list_results() assert records[-1].case_count == records[-1].scored_query_count == 1 assert len(records) == 3 diff --git a/plugins/home/tests/test_sampling.py b/plugins/home/tests/test_sampling.py index 6ab4dac8..ef0c826d 100644 --- a/plugins/home/tests/test_sampling.py +++ b/plugins/home/tests/test_sampling.py @@ -18,8 +18,11 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import home_application, preparation_inputs, prepared_home, + rebuild_review, ) +pytestmark = pytest.mark.usefixtures("rebuild_review") + def cases(): return [ @@ -277,6 +280,7 @@ async def test_replay_sampling_is_limited_to_donor_cases(app, prepared): # The next full rebuild must see the entire training set again. commit_candidate(prepared) response = await app.run_job("home_evaluate", sampling=True) + assert response.success, response.answer manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text()) plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text()) assert set(plan["case_draw_weights"]) == {"1", "2"} and plan["eligibility"] is None diff --git a/plugins/home/tests/test_structured_report.py b/plugins/home/tests/test_structured_report.py new file mode 100644 index 00000000..ef76f65e --- /dev/null +++ b/plugins/home/tests/test_structured_report.py @@ -0,0 +1,130 @@ +"""Review report recovery without network or provider dependencies.""" + +# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access,unused-argument + +import asyncio +from unittest.mock import AsyncMock + +import pytest +from agentscope.agent import ReActConfig +from agentscope.message import AssistantMsg, UserMsg +from agentscope.model import ChatUsage, StructuredResponse +from reme_home.agent.candidate_gate import CandidateGateReport +from reme_home.agent.structured_report import StructuredReportAgent, parse_json_object + +from test_workflows import ScriptedModel + + +@pytest.mark.parametrize( + "text", + [ + '{"decision":"reject","reason":"Mechanism unchanged"}', + 'private analysis\n{"decision":"reject","reason":"Mechanism unchanged"}', + 'Conclusion:\n```json\n{"decision":"reject","reason":"Mechanism unchanged",}\n```', + 'Conclusion: {"decision":"reject","reason":"Mechanism unchanged",} Done.', + ], +) +def test_parse_review_formats(text): + assert parse_json_object(AssistantMsg(name="review", content=text), report_name="review") == { + "decision": "reject", + "reason": "Mechanism unchanged", + } + + +@pytest.mark.parametrize("text", ["invalid", "[]", '{"decision":"accept"} {"decision":"reject"}']) +def test_invalid_or_ambiguous_reports_require_correction(text): + with pytest.raises(ValueError): + parse_json_object(AssistantMsg(name="review", content=text), report_name="review") + + +def test_repairs_preserve_strings_and_nested_json(): + text = 'Report: {"reason":"literal ,} and escaped \\"quote\\" {", "items":[{"value":1,},],}' + assert parse_json_object(AssistantMsg(name="review", content=text), report_name="review") == { + "reason": 'literal ,} and escaped "quote" {', + "items": [{"value": 1}], + } + + +def make_agent(model): + return StructuredReportAgent( + name="review", + system_prompt="Review committed code.", + model=model, + report_model=CandidateGateReport, + react_config=ReActConfig(max_iters=3), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("missing_usage", [False, True]) +@pytest.mark.parametrize( + "initial", + ["Reject: mechanism unchanged.", '{"decision":"reject","reason":"x","extra":true}'], +) +async def test_one_correction_retains_history_disables_tools_and_accounts_usage(missing_usage, initial): + model = ScriptedModel(final=initial) + model.extra_body = {"enable_thinking": True, "thinking_budget": 100, "host_option": "retained"} + model.generate_structured_output = AsyncMock( + return_value=StructuredResponse( + content={"decision": "reject", "reason": "Mechanism unchanged"}, + usage=None if missing_usage else ChatUsage(input_tokens=7, output_tokens=11, time=0), + ), + ) + agent = make_agent(model) + agent.state.summary = "Earlier investigation summary." + reply = await agent.reply(UserMsg(name="host", content="Review this candidate.")) + assert CandidateGateReport.model_validate_json(reply.get_text_content()).decision == "reject" + assert len(model.calls) == 1 + model.generate_structured_output.assert_awaited_once() + options = model.generate_structured_output.call_args.kwargs + assert set(options) == {"messages", "structured_model", "extra_body"} + assert options["structured_model"] is CandidateGateReport + assert options["extra_body"] == { + "reasoning_effort": "none", + "preserve_thinking": False, + "host_option": "retained", + } + text = "\n".join(message.get_text_content() or "" for message in options["messages"]) + for expected in ( + "Review committed code.", + "Earlier investigation summary.", + "Review this candidate.", + initial, + "Do not inspect more files, call tools", + ): + assert expected in text + assert model.extra_body["enable_thinking"] is True + assert agent.state.context[-1] is reply + if missing_usage: + assert reply.usage is None + else: + assert reply.usage.input_tokens == 9 and reply.usage.output_tokens == 14 + + +@pytest.mark.asyncio +async def test_tolerant_parse_avoids_model_correction(): + model = ScriptedModel(final='Decision: {"decision":"accept","reason":"Mechanism changes",}') + model.generate_structured_output = AsyncMock() + reply = await make_agent(model).reply(UserMsg(name="host", content="Review.")) + assert CandidateGateReport.model_validate_json(reply.get_text_content()).decision == "accept" + model.generate_structured_output.assert_not_awaited() + assert reply.usage.input_tokens == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", ["invalid", "provider", "cancel"]) +async def test_correction_is_not_retried_and_cancellation_propagates(failure): + model = ScriptedModel(final="invalid") + model.generate_structured_output = AsyncMock( + return_value=StructuredResponse(content={"decision": "unknown", "reason": "Invalid"}), + ) + expected = ValueError + if failure == "provider": + model.generate_structured_output.side_effect = RuntimeError("offline") + expected = RuntimeError + elif failure == "cancel": + model.generate_structured_output.side_effect = asyncio.CancelledError() + expected = asyncio.CancelledError + with pytest.raises(expected): + await make_agent(model).reply(UserMsg(name="host", content="Review.")) + model.generate_structured_output.assert_awaited_once() diff --git a/plugins/home/tests/test_validation_budget.py b/plugins/home/tests/test_validation_budget.py index ef222cc8..52822f56 100644 --- a/plugins/home/tests/test_validation_budget.py +++ b/plugins/home/tests/test_validation_budget.py @@ -16,8 +16,11 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import home_application, preparation_inputs, prepared_home, + rebuild_review, ) +pytestmark = pytest.mark.usefixtures("rebuild_review") + def seed_history(home, keys): write_json(