mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-15 09:11:24 +00:00
feat(home): add structured-report agent and gate/equivalence test coverage
Add StructuredReportAgent wrapper selectable via the _structured_report_model option in BoundedAgentWrapper, plus its dedicated unit tests. Extend candidate gate, construction equivalence, evaluation history, sampling, and validation budget tests, and document the new behavior in the plugin README.
This commit is contained in:
parent
97e42e0caf
commit
abf36eceec
12 changed files with 417 additions and 1 deletions
|
|
@ -919,6 +919,14 @@ are deleted when review finishes, fails or is cancelled. Reports retain commit a
|
|||
source-evaluation provenance; `source_path` records the temporary location used
|
||||
during review and is no longer available afterward. Model or malformed-report failures are
|
||||
recorded and force a full rebuild; sandbox setup failure stops evaluation.
|
||||
Both construction-equivalence and candidate-admission reports accept JSON wrapped
|
||||
in prose or a code fence, ignore think blocks, and repair trailing commas outside
|
||||
strings. If parsing or schema validation still fails, the reviewer makes one
|
||||
format-only structured-output call with thinking disabled and no tools. This call
|
||||
retains the system prompt, compression summary and investigation history, including
|
||||
the invalid final reply. Successful corrections are saved in the session and their
|
||||
token usage is included in the report. If correction fails, the existing fallback
|
||||
applies: construction is rebuilt or candidate admission allows evaluation.
|
||||
Reviewer shell commands receive an explicit minimal environment instead of
|
||||
inheriting host model credentials or other environment values.
|
||||
Cancellation propagates, and a branch/commit change during review fails before
|
||||
|
|
|
|||
|
|
@ -124,6 +124,7 @@ async def _review_candidate(wrapper, home, context, budget, scope, memory_limit_
|
|||
session_id=None,
|
||||
fork_session=False,
|
||||
output_schema=None,
|
||||
_structured_report_model=CandidateGateReport,
|
||||
permission_mode="bypass",
|
||||
)
|
||||
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})
|
||||
|
|
|
|||
|
|
@ -100,6 +100,7 @@ async def _review_construction(wrapper, home, context, scope, memory_limit_mib):
|
|||
session_id=None,
|
||||
fork_session=False,
|
||||
output_schema=None,
|
||||
_structured_report_model=ConstructionEquivalenceReport,
|
||||
permission_mode="bypass",
|
||||
)
|
||||
report.update({key: reply.get(key) for key in ("session_id", "usage", "result")})
|
||||
|
|
|
|||
177
plugins/home/src/reme_home/agent/structured_report.py
Normal file
177
plugins/home/src/reme_home/agent/structured_report.py
Normal file
|
|
@ -0,0 +1,177 @@
|
|||
"""Shared recovery for schema-constrained final reports from ReAct agents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from agentscope.message import AssistantMsg, Msg, Usage, UserMsg
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .bounded import BoundedReActAgent
|
||||
|
||||
ReportT = TypeVar("ReportT", bound=BaseModel)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class StructuredReportAgent(BoundedReActAgent):
|
||||
"""Validate review reports and make at most one context-preserving correction."""
|
||||
|
||||
def __init__(self, *args, report_model: type[ReportT], **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.report_model = report_model
|
||||
|
||||
async def reply(self, *args, **kwargs) -> Msg:
|
||||
"""Keep normal wrapper persistence and accounting around report recovery."""
|
||||
reply = await super().reply(*args, **kwargs)
|
||||
try:
|
||||
report = self.report_model.model_validate(parse_json_object(reply, report_name=self.name))
|
||||
except ValueError as exc:
|
||||
logger.warning(
|
||||
"%s returned invalid structured output; requesting one format-only correction: %s",
|
||||
self.name,
|
||||
exc,
|
||||
)
|
||||
model_input = await self._prepare_model_input()
|
||||
correction = UserMsg(
|
||||
name="home_report_recovery",
|
||||
content=(
|
||||
"Your previous report did not conform to the required JSON schema. Using only the "
|
||||
"analysis and evidence already present in this conversation, "
|
||||
"immediately return the final decision. "
|
||||
"Do not inspect more files, call tools, add Markdown, or include any other commentary."
|
||||
),
|
||||
)
|
||||
# Do not forward tools: this call only reformats the existing conclusion.
|
||||
corrected = await self.model.generate_structured_output(
|
||||
messages=[*model_input["messages"], correction],
|
||||
structured_model=self.report_model,
|
||||
**_thinking_disabled_kwargs(self.model),
|
||||
)
|
||||
report = self.report_model.model_validate(corrected.content)
|
||||
usage = None
|
||||
if reply.usage is not None and corrected.usage is not None:
|
||||
usage = Usage(
|
||||
**{key: getattr(reply.usage, key) + getattr(corrected.usage, key, 0) for key in Usage.model_fields},
|
||||
)
|
||||
reply = AssistantMsg(name=self.name, content=report.model_dump_json(), usage=usage)
|
||||
self.state.context.extend([correction, reply])
|
||||
return reply
|
||||
return reply.model_copy(
|
||||
update={"content": AssistantMsg(name=self.name, content=report.model_dump_json()).content},
|
||||
)
|
||||
|
||||
|
||||
def parse_json_object(message: Msg, *, report_name: str) -> dict[str, Any]:
|
||||
"""Parse an object with conservative repairs for common model formatting errors."""
|
||||
|
||||
text = message.get_text_content() or ""
|
||||
candidates = [text.strip()]
|
||||
without_thinking = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL | re.IGNORECASE).strip()
|
||||
if without_thinking and without_thinking != candidates[0]:
|
||||
candidates.append(without_thinking)
|
||||
fenced_candidates = [
|
||||
match.group(1).strip()
|
||||
for match in re.finditer(r"```(?:json)?\s*(.*?)```", without_thinking, flags=re.DOTALL | re.IGNORECASE)
|
||||
]
|
||||
if len(fenced_candidates) == 1:
|
||||
candidates.extend(fenced_candidates)
|
||||
object_candidates = _extract_json_objects(without_thinking)
|
||||
if len(object_candidates) == 1:
|
||||
candidates.extend(object_candidates)
|
||||
|
||||
saw_non_object = False
|
||||
seen: set[str] = set()
|
||||
for candidate in candidates:
|
||||
if not candidate or candidate in seen:
|
||||
continue
|
||||
seen.add(candidate)
|
||||
repaired = _remove_trailing_commas(candidate)
|
||||
for attempt in (candidate,) if repaired == candidate else (candidate, repaired):
|
||||
try:
|
||||
payload = json.loads(attempt)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
if isinstance(payload, dict):
|
||||
return payload
|
||||
saw_non_object = True
|
||||
if saw_non_object:
|
||||
raise ValueError(f"{report_name} report must be a JSON object")
|
||||
raise ValueError(f"{report_name} did not return a JSON object") from None
|
||||
|
||||
|
||||
def _extract_json_objects(text: str) -> list[str]:
|
||||
"""Extract balanced JSON-looking objects without interpreting braces inside strings."""
|
||||
|
||||
objects: list[str] = []
|
||||
start: int | None = None
|
||||
depth = 0
|
||||
in_string = False
|
||||
escaped = False
|
||||
for index, character in enumerate(text):
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif character == "\\":
|
||||
escaped = True
|
||||
elif character == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if character == '"':
|
||||
in_string = True
|
||||
elif character == "{":
|
||||
if depth == 0:
|
||||
start = index
|
||||
depth += 1
|
||||
elif character == "}" and depth:
|
||||
depth -= 1
|
||||
if depth == 0 and start is not None:
|
||||
objects.append(text[start : index + 1])
|
||||
start = None
|
||||
return objects
|
||||
|
||||
|
||||
def _remove_trailing_commas(text: str) -> str:
|
||||
"""Remove commas immediately before closing JSON containers, outside strings."""
|
||||
|
||||
output: list[str] = []
|
||||
in_string = False
|
||||
escaped = False
|
||||
index = 0
|
||||
while index < len(text):
|
||||
character = text[index]
|
||||
if in_string:
|
||||
output.append(character)
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif character == "\\":
|
||||
escaped = True
|
||||
elif character == '"':
|
||||
in_string = False
|
||||
index += 1
|
||||
continue
|
||||
if character == '"':
|
||||
in_string = True
|
||||
if character == ",":
|
||||
lookahead = index + 1
|
||||
while lookahead < len(text) and text[lookahead].isspace():
|
||||
lookahead += 1
|
||||
if lookahead < len(text) and text[lookahead] in "}]":
|
||||
index += 1
|
||||
continue
|
||||
output.append(character)
|
||||
index += 1
|
||||
return "".join(output)
|
||||
|
||||
|
||||
def _thinking_disabled_kwargs(model: Any) -> dict[str, Any]:
|
||||
"""Build per-request settings for a deterministic format-only correction."""
|
||||
|
||||
extra_body = dict(getattr(model, "extra_body", None) or {})
|
||||
extra_body.pop("enable_thinking", None)
|
||||
extra_body.pop("thinking_budget", None)
|
||||
extra_body.update({"reasoning_effort": "none", "preserve_thinking": False})
|
||||
return {"extra_body": extra_body}
|
||||
|
|
@ -11,6 +11,7 @@ from reme.components.as_llm import BaseAsLLM
|
|||
|
||||
from ..judge import HomeJudge
|
||||
from .bounded import BoundedReActAgent, RunTimeBudgetTimer
|
||||
from .structured_report import StructuredReportAgent
|
||||
|
||||
|
||||
class BoundedAgentWrapper(AsAgentWrapper):
|
||||
|
|
@ -24,6 +25,9 @@ class BoundedAgentWrapper(AsAgentWrapper):
|
|||
def _create_agent(self, *, options: dict[str, Any], **kwargs: Any) -> BoundedReActAgent:
|
||||
budget = options.get("time_budget_minutes")
|
||||
timer = None if budget is None else RunTimeBudgetTimer(budget, agent_label=self.name)
|
||||
report_model = options.get("_structured_report_model")
|
||||
if report_model is not None:
|
||||
return StructuredReportAgent(report_model=report_model, run_time_budget_timer=timer, **kwargs)
|
||||
return BoundedReActAgent(run_time_budget_timer=timer, **kwargs)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -6,8 +6,10 @@ import asyncio
|
|||
from pathlib import Path
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from agentscope.model import ChatUsage, StructuredResponse
|
||||
from agentscope.tool import LocalBackend
|
||||
from reme_home.agent import candidate_gate, workflow_common
|
||||
from reme_home.evaluation.artifacts import read_json
|
||||
|
|
@ -20,6 +22,8 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
|||
home_application,
|
||||
preparation_inputs,
|
||||
prepared_home,
|
||||
rebuild_review,
|
||||
os_sandbox_available,
|
||||
)
|
||||
from test_workflows import ScriptedModel, set_model
|
||||
|
||||
|
|
@ -99,6 +103,7 @@ async def test_gate_before_sampling_preserves_budget_and_pins_code(app, prepared
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("rebuild_review")
|
||||
async def test_baseline_debug_unbudgeted_and_late_phase_skip_gate(app, prepared, monkeypatch):
|
||||
async def unexpected(*args, **kwargs):
|
||||
raise AssertionError("Gate must be skipped")
|
||||
|
|
@ -161,6 +166,29 @@ async def test_model_failure_allows_validation_with_saved_diagnostics(app, prepa
|
|||
assert not list((prepared / "tmp").glob("candidate-gate-*"))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_corrected_gate_rejects_before_spending_budget(app, prepared, sandbox):
|
||||
commit_candidate(prepared)
|
||||
model = ScriptedModel(final="Reject this candidate: only prompt tuning.")
|
||||
model.generate_structured_output = AsyncMock(
|
||||
return_value=StructuredResponse(
|
||||
content={"decision": "reject", "reason": "Only prompt tuning"},
|
||||
usage=ChatUsage(input_tokens=7, output_tokens=11, time=0),
|
||||
),
|
||||
)
|
||||
set_model(app, "home_candidate_gate", model)
|
||||
response = await app.run_job("home_evaluate", max_validations=10)
|
||||
assert not response.success
|
||||
assert response.answer["error_code"] == "CANDIDATE_GATE_REJECTED"
|
||||
report = read_json(Path(response.answer["candidate_gate"]["review_path"]))
|
||||
assert report["decision"] == "reject" and "review_failed" not in report
|
||||
assert report["usage"]["input_tokens"] == 9 and report["usage"]["output_tokens"] == 14
|
||||
assert report["session_id"] and not Path(report["source_path"]).exists()
|
||||
assert validation_budget_status(prepared, 10)["used_validations"] == 0
|
||||
assert not FakeFactory.instances
|
||||
model.generate_structured_output.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sandbox_setup_failure_never_calls_model_or_evaluates(app, prepared, monkeypatch):
|
||||
commit_candidate(prepared)
|
||||
|
|
@ -214,6 +242,7 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared,
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("os_sandbox_available")
|
||||
async def test_real_gate_sandbox_denies_evidence_live_code_writes_and_network(app, prepared, tmp_path):
|
||||
commit_candidate(prepared)
|
||||
context = candidate_gate.admission_context(prepared)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from types import MethodType, SimpleNamespace
|
|||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from agentscope.model import ChatUsage, StructuredResponse
|
||||
from agentscope.tool import LocalBackend
|
||||
from reme_home.agent import candidate_gate, construction_equivalence, optimizer, workflow_common
|
||||
from reme_home.code.snapshot import read_code_file
|
||||
|
|
@ -26,6 +27,7 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
|||
home_application,
|
||||
preparation_inputs,
|
||||
prepared_home,
|
||||
os_sandbox_available,
|
||||
)
|
||||
from test_workflows import ScriptedModel, final_report, set_model
|
||||
|
||||
|
|
@ -56,6 +58,30 @@ def force_draw(monkeypatch, draw):
|
|||
monkeypatch.setattr(reuse.random, "SystemRandom", lambda: SimpleNamespace(random=lambda: draw))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_corrected_equivalence_authorizes_replay(app, prepared, reviewer, sandbox, monkeypatch):
|
||||
assert (await app.run_job("home_evaluate")).success
|
||||
commit_candidate(prepared)
|
||||
model = ScriptedModel(final="Construction is unchanged: only query behavior changes.")
|
||||
model.generate_structured_output = AsyncMock(
|
||||
return_value=StructuredResponse(
|
||||
content={"decision": "construction_unchanged", "reason": "Only query behavior changes"},
|
||||
usage=ChatUsage(input_tokens=7, output_tokens=11, time=0),
|
||||
),
|
||||
)
|
||||
set_model(app, "home_construction_equivalence", model)
|
||||
force_draw(monkeypatch, 0.1)
|
||||
response = await app.run_job("home_evaluate", sampling=True)
|
||||
assert response.success, response.answer
|
||||
assert response.answer["execution_mode"] == "query_only_replay"
|
||||
report = response.answer["automatic_construction_reuse"]["equivalence_report"]
|
||||
assert report["decision"] == "construction_unchanged" and "review_failed" not in report
|
||||
assert report["usage"]["input_tokens"] == 9 and report["usage"]["output_tokens"] == 14
|
||||
assert read_json(Path(report["review_path"]))["session_id"]
|
||||
assert not Path(report["source_path"]).exists()
|
||||
model.generate_structured_output.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("changed", [False, True])
|
||||
async def test_large_resources_export_for_both_reviewers(app, prepared, reviewer, sandbox, monkeypatch, changed):
|
||||
|
|
@ -509,6 +535,7 @@ async def test_optimizer_final_selection_accepts_automatic_replay(app, prepared,
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("os_sandbox_available")
|
||||
async def test_real_equivalence_sandbox_allows_only_copied_construction_and_source(
|
||||
app,
|
||||
prepared,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ import asyncio
|
|||
import io
|
||||
import json
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
import tarfile
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
|
|
@ -23,6 +25,31 @@ from test_preparation import ( # noqa: F401 # pylint: disable=unused-import
|
|||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def rebuild_review(monkeypatch):
|
||||
"""Keep evaluation bookkeeping tests independent of review models and OS sandboxes."""
|
||||
from reme_home.agent import construction_equivalence
|
||||
|
||||
async def review(wrapper, home, context, spec, cases, fingerprints, *, memory_limit_mib=None):
|
||||
return {**context, "decision": "construction_changed", "reason": "Fixture rebuild"}
|
||||
|
||||
monkeypatch.setattr(construction_equivalence, "review_construction", review)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def os_sandbox_available():
|
||||
"""Skip real boundary tests only when the platform or executable is unsupported."""
|
||||
if sys.platform.startswith("linux"):
|
||||
executable = "bwrap"
|
||||
elif sys.platform == "darwin":
|
||||
executable = "sandbox-exec"
|
||||
else:
|
||||
pytest.skip(f"No file sandbox backend for {sys.platform}")
|
||||
return
|
||||
if shutil.which(executable) is None:
|
||||
pytest.skip(f"Sandbox boundary test requires {executable} on PATH")
|
||||
|
||||
|
||||
def archive(path, files):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tarfile.open(path, "w:gz") as stream:
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import json
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from agentscope.message import ToolResultState
|
||||
from reme_home.evaluation.history import HomeEvaluationHistory, EvaluationRecord, read_state
|
||||
from reme_home.sampling import SamplingPolicy, create_sampling_plan
|
||||
from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
||||
|
|
@ -14,10 +15,13 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
|||
home_application,
|
||||
preparation_inputs,
|
||||
prepared_home,
|
||||
rebuild_review,
|
||||
)
|
||||
|
||||
from reme.components.agent_wrapper.as_agent_wrapper import AsAgentWrapper
|
||||
|
||||
pytestmark = pytest.mark.usefixtures("rebuild_review")
|
||||
|
||||
|
||||
def history(app):
|
||||
return app.context.components["home.evaluation_history"]["default"]
|
||||
|
|
@ -176,7 +180,8 @@ async def test_job_defaults_and_hidden_agent_injection_use_one_job(app, prepared
|
|||
)
|
||||
assert "sampling" not in tool.input_schema["properties"]
|
||||
commit_candidate(prepared)
|
||||
await tool.call()
|
||||
result = await tool.call()
|
||||
assert result.state == ToolResultState.SUCCESS, result
|
||||
records = history(app).list_results()
|
||||
assert records[-1].case_count == records[-1].scored_query_count == 1
|
||||
assert len(records) == 3
|
||||
|
|
|
|||
|
|
@ -18,8 +18,11 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
|||
home_application,
|
||||
preparation_inputs,
|
||||
prepared_home,
|
||||
rebuild_review,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.usefixtures("rebuild_review")
|
||||
|
||||
|
||||
def cases():
|
||||
return [
|
||||
|
|
@ -277,6 +280,7 @@ async def test_replay_sampling_is_limited_to_donor_cases(app, prepared):
|
|||
# The next full rebuild must see the entire training set again.
|
||||
commit_candidate(prepared)
|
||||
response = await app.run_job("home_evaluate", sampling=True)
|
||||
assert response.success, response.answer
|
||||
manifest = json.loads((Path(response.answer["result_dir"]) / "manifest.json").read_text())
|
||||
plan = json.loads((prepared / manifest["sampling"]["plan_path"]).read_text())
|
||||
assert set(plan["case_draw_weights"]) == {"1", "2"} and plan["eligibility"] is None
|
||||
|
|
|
|||
130
plugins/home/tests/test_structured_report.py
Normal file
130
plugins/home/tests/test_structured_report.py
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
"""Review report recovery without network or provider dependencies."""
|
||||
|
||||
# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access,unused-argument
|
||||
|
||||
import asyncio
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from agentscope.agent import ReActConfig
|
||||
from agentscope.message import AssistantMsg, UserMsg
|
||||
from agentscope.model import ChatUsage, StructuredResponse
|
||||
from reme_home.agent.candidate_gate import CandidateGateReport
|
||||
from reme_home.agent.structured_report import StructuredReportAgent, parse_json_object
|
||||
|
||||
from test_workflows import ScriptedModel
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
[
|
||||
'{"decision":"reject","reason":"Mechanism unchanged"}',
|
||||
'<think>private analysis</think>\n{"decision":"reject","reason":"Mechanism unchanged"}',
|
||||
'Conclusion:\n```json\n{"decision":"reject","reason":"Mechanism unchanged",}\n```',
|
||||
'Conclusion: {"decision":"reject","reason":"Mechanism unchanged",} Done.',
|
||||
],
|
||||
)
|
||||
def test_parse_review_formats(text):
|
||||
assert parse_json_object(AssistantMsg(name="review", content=text), report_name="review") == {
|
||||
"decision": "reject",
|
||||
"reason": "Mechanism unchanged",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("text", ["invalid", "[]", '{"decision":"accept"} {"decision":"reject"}'])
|
||||
def test_invalid_or_ambiguous_reports_require_correction(text):
|
||||
with pytest.raises(ValueError):
|
||||
parse_json_object(AssistantMsg(name="review", content=text), report_name="review")
|
||||
|
||||
|
||||
def test_repairs_preserve_strings_and_nested_json():
|
||||
text = 'Report: {"reason":"literal ,} and escaped \\"quote\\" {", "items":[{"value":1,},],}'
|
||||
assert parse_json_object(AssistantMsg(name="review", content=text), report_name="review") == {
|
||||
"reason": 'literal ,} and escaped "quote" {',
|
||||
"items": [{"value": 1}],
|
||||
}
|
||||
|
||||
|
||||
def make_agent(model):
|
||||
return StructuredReportAgent(
|
||||
name="review",
|
||||
system_prompt="Review committed code.",
|
||||
model=model,
|
||||
report_model=CandidateGateReport,
|
||||
react_config=ReActConfig(max_iters=3),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("missing_usage", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"initial",
|
||||
["Reject: mechanism unchanged.", '{"decision":"reject","reason":"x","extra":true}'],
|
||||
)
|
||||
async def test_one_correction_retains_history_disables_tools_and_accounts_usage(missing_usage, initial):
|
||||
model = ScriptedModel(final=initial)
|
||||
model.extra_body = {"enable_thinking": True, "thinking_budget": 100, "host_option": "retained"}
|
||||
model.generate_structured_output = AsyncMock(
|
||||
return_value=StructuredResponse(
|
||||
content={"decision": "reject", "reason": "Mechanism unchanged"},
|
||||
usage=None if missing_usage else ChatUsage(input_tokens=7, output_tokens=11, time=0),
|
||||
),
|
||||
)
|
||||
agent = make_agent(model)
|
||||
agent.state.summary = "Earlier investigation summary."
|
||||
reply = await agent.reply(UserMsg(name="host", content="Review this candidate."))
|
||||
assert CandidateGateReport.model_validate_json(reply.get_text_content()).decision == "reject"
|
||||
assert len(model.calls) == 1
|
||||
model.generate_structured_output.assert_awaited_once()
|
||||
options = model.generate_structured_output.call_args.kwargs
|
||||
assert set(options) == {"messages", "structured_model", "extra_body"}
|
||||
assert options["structured_model"] is CandidateGateReport
|
||||
assert options["extra_body"] == {
|
||||
"reasoning_effort": "none",
|
||||
"preserve_thinking": False,
|
||||
"host_option": "retained",
|
||||
}
|
||||
text = "\n".join(message.get_text_content() or "" for message in options["messages"])
|
||||
for expected in (
|
||||
"Review committed code.",
|
||||
"Earlier investigation summary.",
|
||||
"Review this candidate.",
|
||||
initial,
|
||||
"Do not inspect more files, call tools",
|
||||
):
|
||||
assert expected in text
|
||||
assert model.extra_body["enable_thinking"] is True
|
||||
assert agent.state.context[-1] is reply
|
||||
if missing_usage:
|
||||
assert reply.usage is None
|
||||
else:
|
||||
assert reply.usage.input_tokens == 9 and reply.usage.output_tokens == 14
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tolerant_parse_avoids_model_correction():
|
||||
model = ScriptedModel(final='Decision: {"decision":"accept","reason":"Mechanism changes",}')
|
||||
model.generate_structured_output = AsyncMock()
|
||||
reply = await make_agent(model).reply(UserMsg(name="host", content="Review."))
|
||||
assert CandidateGateReport.model_validate_json(reply.get_text_content()).decision == "accept"
|
||||
model.generate_structured_output.assert_not_awaited()
|
||||
assert reply.usage.input_tokens == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", ["invalid", "provider", "cancel"])
|
||||
async def test_correction_is_not_retried_and_cancellation_propagates(failure):
|
||||
model = ScriptedModel(final="invalid")
|
||||
model.generate_structured_output = AsyncMock(
|
||||
return_value=StructuredResponse(content={"decision": "unknown", "reason": "Invalid"}),
|
||||
)
|
||||
expected = ValueError
|
||||
if failure == "provider":
|
||||
model.generate_structured_output.side_effect = RuntimeError("offline")
|
||||
expected = RuntimeError
|
||||
elif failure == "cancel":
|
||||
model.generate_structured_output.side_effect = asyncio.CancelledError()
|
||||
expected = asyncio.CancelledError
|
||||
with pytest.raises(expected):
|
||||
await make_agent(model).reply(UserMsg(name="host", content="Review."))
|
||||
model.generate_structured_output.assert_awaited_once()
|
||||
|
|
@ -16,8 +16,11 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
|
|||
home_application,
|
||||
preparation_inputs,
|
||||
prepared_home,
|
||||
rebuild_review,
|
||||
)
|
||||
|
||||
pytestmark = pytest.mark.usefixtures("rebuild_review")
|
||||
|
||||
|
||||
def seed_history(home, keys):
|
||||
write_json(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue