diff --git a/plugins/home/src/reme_home/agent/optimizer.py b/plugins/home/src/reme_home/agent/optimizer.py index 7ab902b5..5a29286a 100644 --- a/plugins/home/src/reme_home/agent/optimizer.py +++ b/plugins/home/src/reme_home/agent/optimizer.py @@ -2,6 +2,7 @@ import asyncio import json +import re import time from pathlib import Path from typing import Callable @@ -35,6 +36,14 @@ MUTATION_JOBS = ( "home_commit_candidate", ) PENDING_CYCLE_TRANSITIONS = "home.optimizer.pending_cycle_transitions" +VALIDATION_BUDGET_EXHAUSTED_HINT = ( + "The formal validation budget is exhausted. All previous validation-cycle timer reminders " + "are expired. Select the best candidate using existing evaluation evidence; do not develop new candidates " + "or run further evaluations. Complete any necessary evidence and repository checks, and leave the selected " + "evaluated branch and full commit checked out with no uncommitted changes. Return exactly one JSON object " + "with branch_name, commit_sha, evaluation_path and report. Put all summary and explanation in report; " + "do not include Markdown fences or text outside the JSON object." +) class ValidationCycleTimer(RunTimeBudgetTimer): @@ -197,11 +206,23 @@ class OptimizationResult(SpecModel): def parse_result(text): - """Accept a JSON object, optionally enclosed in one Markdown JSON fence.""" + """Accept JSON, including a single JSON fence surrounded by commentary.""" text = text.strip() + original = text if text.startswith("```") and text.endswith("```"): text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip() - return OptimizationResult.model_validate(json.loads(text)) + try: + payload = json.loads(text) + except json.JSONDecodeError: + fences = re.findall( + r"^[ \t]*```json[ \t]*\r?\n(.*?)^[ \t]*```[ \t]*\r?$", + original, + flags=re.MULTILINE | re.DOTALL | re.IGNORECASE, + ) + if len(fences) != 1 or len(re.findall(r"^[ \t]*```", original, flags=re.MULTILINE)) != 2: + raise + payload = json.loads(fences[0]) + return OptimizationResult.model_validate(payload) EVALUATION_OPTIONS = { @@ -352,7 +373,7 @@ def validation_budget_tool(home, max_evaluations): ) -def evaluation_tool(wrapper, options, *, home, timer=None, max_evaluations=10): +def evaluation_tool(wrapper, options, *, home, timer=None, max_evaluations=10, budget_exhausted=False): """Only debug mode is model-selected; the host decides construction replay.""" job = required_job(wrapper, "home_evaluate") validate_max_evaluations(max_evaluations) @@ -363,6 +384,7 @@ def evaluation_tool(wrapper, options, *, home, timer=None, max_evaluations=10): Debug checks execution, not answer quality. Inspect infrastructure failures separately; retries keep the existing commit and selection recovery rules. """ + nonlocal budget_exhausted if not isinstance(debug, bool): raise ValueError("debug must be boolean") if set(kwargs) - {"_agent_state"}: @@ -385,13 +407,26 @@ def evaluation_tool(wrapper, options, *, home, timer=None, max_evaluations=10): budget = await blocking(validation_budget_status, home, max_evaluations) if isinstance(response.answer, dict): response.answer = {**response.answer, "validation_budget": budget} - completed = await blocking(completed_validation, home, response.answer, started_before) - if completed is not None: - manifest, validation_number = completed + if budget["remaining_validations"] == 0: if timer is not None: - timer.reset() - transition = validation_cycle_transition(manifest, validation_number, budget["remaining_validations"]) - state.middle_context.setdefault(PENDING_CYCLE_TRANSITIONS, []).append(transition.hint) + timer.stop() + if not budget_exhausted: + state.middle_context.setdefault(PENDING_CYCLE_TRANSITIONS, []).append( + VALIDATION_BUDGET_EXHAUSTED_HINT, + ) + budget_exhausted = True + else: + completed = await blocking(completed_validation, home, response.answer, started_before) + if completed is not None: + manifest, validation_number = completed + if timer is not None: + timer.reset() + transition = validation_cycle_transition( + manifest, + validation_number, + budget["remaining_validations"], + ) + state.middle_context.setdefault(PENDING_CYCLE_TRANSITIONS, []).append(transition.hint) return {"success": response.success, "answer": response.answer, "metadata": response.metadata} tool = FunctionTool(evaluate, name="home_evaluate", is_concurrency_safe=False, is_state_injected=True) @@ -454,6 +489,9 @@ class HomeOptimizeStep(BaseStep): timer = self.agent_wrapper.create_cycle_timer() max_evaluations = self.agent_wrapper.validation_limit() budget = await blocking(validation_budget_status, home, max_evaluations) + budget_exhausted = budget["remaining_validations"] == 0 + if budget_exhausted and timer is not None: + timer.stop() reply = await workflow_reply( self.agent_wrapper, home, @@ -461,7 +499,9 @@ class HomeOptimizeStep(BaseStep): f"Begin optimization.\n{spec.instruction}\n\n" f"Completed full initial baseline: {baseline['evaluation_path']}\n" f"Formal validation budget: {json.dumps(budget)}\n" + f"{VALIDATION_BUDGET_EXHAUSTED_HINT if budget_exhausted else ''}\n" "Return exactly one JSON object with branch_name, commit_sha (full SHA), evaluation_path and report. " + "Do not include Markdown fences or text outside the JSON object; put your summary in report. " "Leave that evaluated branch and commit checked out with no uncommitted changes.", tool_factory=lambda backend: [ *create_draft_tools()[0], @@ -474,6 +514,7 @@ class HomeOptimizeStep(BaseStep): home=home, timer=timer, max_evaluations=max_evaluations, + budget_exhausted=budget_exhausted, ), ], job_names=MUTATION_JOBS, diff --git a/plugins/home/src/reme_home/agent/optimizer.yaml b/plugins/home/src/reme_home/agent/optimizer.yaml index be0fa473..630957d5 100644 --- a/plugins/home/src/reme_home/agent/optimizer.yaml +++ b/plugins/home/src/reme_home/agent/optimizer.yaml @@ -64,6 +64,7 @@ system_prompt: |- Finish with a clean checkout of the best evaluated branch and full commit. Choosing the baseline is valid if improvements are unsupported. Return exactly one JSON object with branch_name, commit_sha, evaluation_path, - and report. report must explain the chosen evidence, comparisons, regressions, costs and limitations. The host + and report. Do not include Markdown fences or text outside the JSON object. Put all summary and explanation + in report, which must explain the chosen evidence, comparisons, regressions, costs and limitations. The host will reject a mismatched checkout or missing, incomplete, debug or infrastructure-failed evidence. Query-only evidence is valid only when the host selected it after a successful construction-equivalence review. diff --git a/plugins/home/tests/test_workflows.py b/plugins/home/tests/test_workflows.py index 72c9b85d..1a746556 100644 --- a/plugins/home/tests/test_workflows.py +++ b/plugins/home/tests/test_workflows.py @@ -10,12 +10,14 @@ import sys from importlib import import_module from pathlib import Path from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest import yaml from agentscope.formatter import OpenAIChatFormatter from agentscope.message import HintBlock, TextBlock, ToolCallBlock from agentscope.model import ChatResponse, ChatUsage +from agentscope.state import AgentState from agentscope.tool import LocalBackend from jsonschema import Draft202012Validator from reme_home.agent import optimizer as optimizer_module, diagnostic as diagnostic_module, workflow_common, case_study @@ -32,6 +34,8 @@ from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import prepared_home, ) +from reme.schema import Response + class ScriptedModel(RecordingModel): def __init__(self, calls=(), final="Supported diagnostic report."): @@ -94,6 +98,45 @@ def final_report(home, branch="init", path=None): ) +@pytest.mark.parametrize( + "template", + [ + "{}", + "```\n{}\n```", + "```json\n{}\n```", + "Completed.\n```json\n{}\n```\nSee report.", + "Completed.\r\n```JSON\r\n{}\r\n```", + ], +) +def test_parse_optimizer_result_formats(template): + claim = {"branch_name": "candidate", "commit_sha": "a" * 40, "evaluation_path": "run", "report": "Evidence"} + assert optimizer_module.parse_result(template.format(json.dumps(claim))).model_dump() == claim + + +@pytest.mark.parametrize( + "text", + [ + "No result", + 'Completed. {"report":"Evidence"}', + "Completed.\n```json\n{}", + "Completed.\n```python\n{}\n```", + "Completed.\n```json\n{}\n```\n```json\n{}\n```", + "Completed.\n```text\nexample\n```\n```json\n{}\n```", + 'Completed.\n```json\n{"report":"Evidence",}\n```', + ], +) +def test_parse_optimizer_result_rejects_invalid_or_multiple_fences(text): + with pytest.raises(json.JSONDecodeError): + optimizer_module.parse_result(text) + + +@pytest.mark.parametrize("changes", [{"commit_sha": "short"}, {"report": ""}, {"unexpected": True}]) +def test_mixed_optimizer_result_keeps_schema_validation(changes): + claim = {"branch_name": "candidate", "commit_sha": "a" * 40, "evaluation_path": "run", "report": "Evidence"} + with pytest.raises(ValueError): + optimizer_module.parse_result(f"Completed.\n```json\n{json.dumps({**claim, **changes})}\n```") + + @pytest.mark.asyncio async def test_draft_scope_across_optimizer_diagnoses_and_invocations(app, prepared, sandbox): path = await baseline(app) @@ -161,7 +204,14 @@ async def test_draft_scope_across_optimizer_diagnoses_and_invocations(app, prepa @pytest.mark.asyncio -async def test_real_optimization_diagnoses_edits_debugs_and_evaluates(app, prepared, sandbox, monkeypatch): +@pytest.mark.parametrize("max_evaluations", [10, 1]) +async def test_real_optimization_diagnoses_edits_debugs_and_evaluates( + app, + prepared, + sandbox, + monkeypatch, + max_evaluations, +): home = prepared now, timers = [0.0], [] timer_class = optimizer_module.ValidationCycleTimer @@ -210,7 +260,8 @@ async def test_real_optimization_diagnoses_edits_debugs_and_evaluates(app, prepa ], final=lambda: final_report(home, "mechanism"), ) - set_model(app, "home_optimizer", optimizer) + wrapper = set_model(app, "home_optimizer", optimizer) + monkeypatch.setattr(wrapper, "validation_limit", lambda: max_evaluations) response = await app.run_job("home_optimize", instruction="Improve general retrieval") assert response.success, response.answer assert target.read_text() == "value = 2\n" @@ -223,10 +274,16 @@ async def test_real_optimization_diagnoses_edits_debugs_and_evaluates(app, prepa if isinstance(block, HintBlock) and block.source == "meta_reme_validation_cycle_transition" ] assert len(transitions) == 1 - assert "Formal validation 1 completed successfully" in transitions[0] - assert "Remaining formal validations: 9. Start cycle 2." in transitions[0] assert len(timers) == 1 - assert timers[0]._started_at == now[0] - 60 * 60 # Reset before the final reasoning call. + if max_evaluations == 1: + assert transitions[0] == optimizer_module.VALIDATION_BUDGET_EXHAUSTED_HINT + assert "Start cycle" not in transitions[0] + assert not timers[0].active + assert timers[0].next_reminder() is None + else: + assert "Formal validation 1 completed successfully" in transitions[0] + assert "Remaining formal validations: 9. Start cycle 2." in transitions[0] + assert timers[0]._started_at == now[0] - 60 * 60 # Reset before the final reasoning call. assert any( isinstance(block, HintBlock) and block.source == "meta_reme_validation_timer" for message in optimizer.calls[-1]["messages"] @@ -237,8 +294,11 @@ async def test_real_optimization_diagnoses_edits_debugs_and_evaluates(app, prepa assert "sampled coverage" in response.answer["report"] assert read_json(Path(response.metadata["optimization_report_path"]))["status"] == "completed" assert len(diagnostic.calls) == 2 - assert len(gate.calls) == 1 - assert read_json(next((home / "logs/candidate-gates").glob("*.json")))["decision"] == "accept" + if max_evaluations == 1: + assert not gate.calls # A one-validation budget starts in refinement, without admission review. + else: + assert len(gate.calls) == 1 + assert read_json(next((home / "logs/candidate-gates").glob("*.json")))["decision"] == "accept" source = next((home / "logs/diagnostic-trajectories").glob("*.json")) trace = read_json(source) assert trace["session_id"] and any("Improve general retrieval" in str(msg) for msg in trace["messages"]) @@ -261,6 +321,47 @@ async def test_real_optimization_diagnoses_edits_debugs_and_evaluates(app, prepa assert all(str(home / "code/repo") not in policy["writable_paths"] for policy in sandbox) +@pytest.mark.asyncio +@pytest.mark.parametrize("success", [True, False]) +@pytest.mark.parametrize("with_timer", [True, False]) +async def test_exhausted_evaluation_hint_is_once_even_after_failure(monkeypatch, success, with_timer): + job = AsyncMock(return_value=Response(success=success, answer={"status": "completed" if success else "failed"})) + monkeypatch.setattr(optimizer_module, "required_job", lambda *_args: job) + monkeypatch.setattr(optimizer_module, "read_state", lambda _home: {"started_validations": []}) + monkeypatch.setattr(optimizer_module, "validation_budget_status", lambda *_args: {"remaining_validations": 0}) + timer = optimizer_module.ValidationCycleTimer(30) if with_timer else None + state = AgentState() + tool = optimizer_module.evaluation_tool(None, {}, home=None, timer=timer) + await tool.call(_agent_state=state) + assert state.middle_context.pop(optimizer_module.PENDING_CYCLE_TRANSITIONS) == [ + optimizer_module.VALIDATION_BUDGET_EXHAUSTED_HINT, + ] + await tool.call(_agent_state=state) + assert optimizer_module.PENDING_CYCLE_TRANSITIONS not in state.middle_context + if timer is not None: + assert not timer.active + assert job.await_count == 2 + + +@pytest.mark.asyncio +async def test_optimizer_starts_with_exhausted_budget(app, prepared, sandbox, monkeypatch): + path = await baseline(app) + timer = optimizer_module.ValidationCycleTimer(30) + model = ScriptedModel(final=lambda: final_report(prepared, path=path)) + wrapper = set_model(app, "home_optimizer", model) + monkeypatch.setattr(wrapper, "create_cycle_timer", lambda: timer) + monkeypatch.setattr(optimizer_module, "validation_budget_status", lambda *_args: {"remaining_validations": 0}) + response = await app.run_job("home_optimize") + assert response.success, response.answer + assert not timer.active + task = "\n".join(msg.get_text_content() or "" for msg in model.calls[0]["messages"]) + assert task.count(optimizer_module.VALIDATION_BUDGET_EXHAUSTED_HINT) == 1 + assert len(FakeFactory.instances) == 1 # Only the baseline ran. + schemas = {item["function"]["name"] for item in model.calls[0]["tools"]} + assert set(optimizer_module.MUTATION_JOBS) <= schemas + assert "home_evaluate" in schemas + + @pytest.mark.asyncio @pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) async def test_benchmark_optimizer_applies_budgets_to_real_workflow( @@ -322,8 +423,9 @@ async def test_benchmark_optimizer_applies_budgets_to_real_workflow( @pytest.mark.asyncio -async def test_optimizer_creates_baseline_once_and_verifies_baseline_fallback(app, prepared, sandbox): - model = ScriptedModel(final=lambda: final_report(prepared)) +@pytest.mark.parametrize("template", ["{}", "```json\n{}\n```", "Completed.\n```json\n{}\n```\nSee report."]) +async def test_optimizer_creates_baseline_once_and_verifies_baseline_fallback(app, prepared, sandbox, template): + model = ScriptedModel(final=lambda: template.format(final_report(prepared))) set_model(app, "home_optimizer", model) response = await app.run_job("home_optimize") assert response.success, response.answer @@ -390,7 +492,10 @@ async def test_bad_baseline_rejected_before_optimizer(app, prepared, sandbox, ch @pytest.mark.asyncio -@pytest.mark.parametrize("change", ["dirty", "branch", "commit", "path", "infra", "debug", "replay", "invalid_json"]) +@pytest.mark.parametrize( + "change", + ["dirty", "branch", "mixed_branch", "commit", "path", "infra", "debug", "replay", "invalid_json"], +) async def test_final_claim_rejected_without_discarding_changes(app, prepared, sandbox, change): path = Path(await baseline(app)) target = prepared / "code/repo/plugins/beam/src/reme_beam/__init__.py" @@ -399,7 +504,7 @@ async def test_final_claim_rejected_without_discarding_changes(app, prepared, sa claim = json.loads(final_report(prepared, path=path)) if change == "dirty": target.write_text("preserve unfinished changes\n") - elif change == "branch": + elif change in {"branch", "mixed_branch"}: claim["branch_name"] = "invented" elif change == "commit": claim["commit_sha"] = "0" * 40 @@ -417,7 +522,8 @@ async def test_final_claim_rejected_without_discarding_changes(app, prepared, sa write_json(path / "manifest.json", manifest) else: return "not a structured final claim" - return json.dumps(claim) + result = json.dumps(claim) + return f"Completed.\n```json\n{result}\n```" if change == "mixed_branch" else result set_model(app, "home_optimizer", ScriptedModel(final=finish)) response = await app.run_job("home_optimize")