mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-15 09:11:24 +00:00
refactor(home): rename max_validations to max_evaluations
This commit is contained in:
parent
eaf38c5da3
commit
75ca3e2011
22 changed files with 120 additions and 85 deletions
|
|
@ -168,7 +168,7 @@ The top-level `validation_cycle_time_budget_minutes` configures the host optimiz
|
|||
advisory time between completed formal evaluations. It defaults to 150 minutes;
|
||||
set a positive number to change it or null to disable timed reminders.
|
||||
|
||||
The top-level `max_validations` sets the optimizer's formal-evaluation budget
|
||||
The top-level `max_evaluations` sets the optimizer's formal-evaluation budget
|
||||
(positive integer, default 10). A slot is consumed at first sandbox startup,
|
||||
including later failures or cancellations; initial baseline and debug are excluded.
|
||||
|
||||
|
|
|
|||
|
|
@ -157,7 +157,7 @@ Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。
|
|||
顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的
|
||||
建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。
|
||||
|
||||
顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
|
||||
顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
|
||||
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
|
||||
|
||||
准备完成后,用同一份配置进行本地一次性优化:
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
# Paths are relative to the ReMe repository root, not the cwd.
|
||||
workspace_dir: benchmark/beam/workspaces/home-100K
|
||||
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
|
||||
max_validations: 10
|
||||
max_evaluations: 10
|
||||
# Target minutes between completed formal optimizer evaluations; null disables timed reminders.
|
||||
validation_cycle_time_budget_minutes: 150
|
||||
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ class RunHomeConfig(BaseModel):
|
|||
preparation: dict
|
||||
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
|
||||
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
|
||||
max_validations: int = Field(default=10, ge=1)
|
||||
max_evaluations: int = Field(default=10, ge=1)
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
|
|
@ -75,7 +75,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
|
|||
"agent_wrapper": {
|
||||
"home_optimizer": {
|
||||
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
|
||||
"max_validations": config.max_validations,
|
||||
"max_evaluations": config.max_evaluations,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ class TestRunConfig(SpecModel):
|
|||
options.debug,
|
||||
options.sampling,
|
||||
options.record_history,
|
||||
options.max_validations is not None,
|
||||
options.max_evaluations is not None,
|
||||
options.sampling_policy is not None,
|
||||
options.case_ids is not None,
|
||||
options.query_ids is not None,
|
||||
|
|
|
|||
|
|
@ -141,7 +141,7 @@ The top-level `validation_cycle_time_budget_minutes` configures the host optimiz
|
|||
advisory time between completed formal evaluations. It defaults to 150 minutes;
|
||||
set a positive number to change it or null to disable timed reminders.
|
||||
|
||||
The top-level `max_validations` sets the optimizer's formal-evaluation budget
|
||||
The top-level `max_evaluations` sets the optimizer's formal-evaluation budget
|
||||
(positive integer, default 10). A slot is consumed at first sandbox startup,
|
||||
including later failures or cancellations; initial baseline and debug are excluded.
|
||||
|
||||
|
|
|
|||
|
|
@ -128,7 +128,7 @@ Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=.
|
|||
顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的
|
||||
建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。
|
||||
|
||||
顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
|
||||
顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
|
||||
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
|
||||
|
||||
准备完成后,用同一份配置进行本地一次性优化:
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@
|
|||
# Preserve dev_meta's fixed training split; groups 6-10 remain held out.
|
||||
workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5
|
||||
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
|
||||
max_validations: 10
|
||||
max_evaluations: 10
|
||||
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
|
||||
sampling_policy:
|
||||
max_cases: 5
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ class RunHomeConfig(BaseModel):
|
|||
preparation: dict
|
||||
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
|
||||
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
|
||||
max_validations: int = Field(default=10, ge=1)
|
||||
max_evaluations: int = Field(default=10, ge=1)
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
|
|
@ -75,7 +75,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
|
|||
"agent_wrapper": {
|
||||
"home_optimizer": {
|
||||
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
|
||||
"max_validations": config.max_validations,
|
||||
"max_evaluations": config.max_evaluations,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
|
|
|||
|
|
@ -942,7 +942,7 @@ context tokens) and
|
|||
`components.home.evaluation.default.construction_equivalence_memory_limit_mib`
|
||||
uses the existing OS sandbox memory-limit contract.
|
||||
|
||||
During the first `max_validations // 2` non-init formal starts, a fresh
|
||||
During the first `max_evaluations // 2` non-init formal starts, a fresh
|
||||
`home_candidate_gate` agent reviews each new candidate before sampling or starting
|
||||
an evaluation sandbox. It compares the candidate with its nearest previously
|
||||
evaluated ancestor (falling back to `init`). It accepts runtime mechanism changes
|
||||
|
|
@ -950,7 +950,7 @@ and concrete execution, safety or contract fixes; it rejects prompt/parameter-on
|
|||
tuning, cosmetic changes and dead code during this early exploration phase.
|
||||
Ambiguous evidence is accepted. Baselines, debug, infrastructure retries and later
|
||||
refinement bypass admission. Ordinary unbudgeted `home_evaluate` calls remain
|
||||
ungated; supplying `max_validations` enables the same admission policy.
|
||||
ungated; supplying `max_evaluations` enables the same admission policy.
|
||||
|
||||
The reviewer receives only read-only source exports of the two exact commits and
|
||||
their complete diff under `home/tmp/candidate-gate-*/`. Its Read, Glob, Grep and
|
||||
|
|
@ -1025,8 +1025,11 @@ Evaluation tools queue transitions in request state; the optimizer injects them
|
|||
before the next reasoning call, after all tool results have been written. Thus a
|
||||
transition never separates an API tool call from its matching result.
|
||||
|
||||
`components.agent_wrapper.home_optimizer.max_validations` defaults to 10, matching
|
||||
dev_meta. `get_validation_budget` reports consumed and remaining slots, the search
|
||||
`components.agent_wrapper.home_optimizer.max_evaluations` defaults to 10, matching
|
||||
dev_meta. This setting and the `home_evaluate` argument were renamed from
|
||||
`max_validations`; update existing configuration and callers to `max_evaluations`.
|
||||
The old name is rejected rather than silently falling back to the default budget.
|
||||
`get_validation_budget` reports consumed and remaining slots, the search
|
||||
phase and next-candidate guidance. A non-init formal run consumes one slot when
|
||||
its first sandbox initializes successfully, including runs that later fail or are
|
||||
cancelled. Initial baselines, debug, rejected calls and pre-start failures are free.
|
||||
|
|
@ -1034,10 +1037,10 @@ Accounting reuses persistent started-run evidence and survives optimizer restart
|
|||
removing the rebuildable sampling cache does not restore the budget. The evaluator
|
||||
checks the limit while holding the formal-validation lock, before starting another
|
||||
run. At exhaustion it returns `VALIDATION_BUDGET_EXHAUSTED`; the optimizer selects
|
||||
from existing evidence. The first `max_validations // 2` starts favor structural
|
||||
from existing evidence. The first `max_evaluations // 2` starts favor structural
|
||||
exploration; later starts favor evidence-driven refinement. This guidance is advisory.
|
||||
Ordinary `home_evaluate` calls remain unbounded unless the host supplies
|
||||
`max_validations`; all recorded non-init formal starts in that workspace count.
|
||||
`max_evaluations`; all recorded non-init formal starts in that workspace count.
|
||||
|
||||
Sessions and model usage follow ReMe's wrapper conventions; each nested call reports its own usage.
|
||||
The optimizer follows dev_meta's compression recovery policy. Recoverable model
|
||||
|
|
@ -1070,7 +1073,7 @@ components:
|
|||
react_config:
|
||||
max_iters: 3000
|
||||
validation_cycle_time_budget_minutes: 150
|
||||
max_validations: 10
|
||||
max_evaluations: 10
|
||||
```
|
||||
|
||||
`evaluation_options` also accepts `base_image`, `fail_fast` and
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ from ..evaluation.artifacts import contained, read_json, write_json
|
|||
from ..evaluation.component import blocking
|
||||
from ..evaluation.inputs import EvaluationSpec, load_inputs
|
||||
from ..evaluation.history import read_state
|
||||
from ..evaluation.budget import validate_max_validations, validation_budget_status
|
||||
from ..evaluation.budget import validate_max_evaluations, validation_budget_status
|
||||
from ..preparation.spec import SpecModel
|
||||
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
|
||||
from .diagnostic import diagnostic_tool, evaluation_paths
|
||||
|
|
@ -165,7 +165,10 @@ class OptimizerAgentWrapper(CaseStudyAgentWrapper):
|
|||
|
||||
def validation_limit(self):
|
||||
"""Keep the formal count limit host-owned and independent of elapsed time."""
|
||||
return validate_max_validations(self._merged_kwargs({}).get("max_validations", 10))
|
||||
options = self._merged_kwargs({})
|
||||
if "max_validations" in options:
|
||||
raise ValueError("Use max_evaluations instead of max_validations")
|
||||
return validate_max_evaluations(options.get("max_evaluations", 10))
|
||||
|
||||
def _create_agent(self, *, options, **kwargs):
|
||||
timer = (
|
||||
|
|
@ -367,13 +370,13 @@ def verify_final(home, result):
|
|||
return evidence["evaluation_path"]
|
||||
|
||||
|
||||
def validation_budget_tool(home, max_validations):
|
||||
def validation_budget_tool(home, max_evaluations):
|
||||
"""Expose dev_meta's read-only count, phase and next-candidate guidance."""
|
||||
validate_max_validations(max_validations)
|
||||
validate_max_evaluations(max_evaluations)
|
||||
|
||||
async def get_validation_budget():
|
||||
"""Return started formal validations, remaining count and optimization guidance."""
|
||||
return await blocking(validation_budget_status, home, max_validations)
|
||||
return await blocking(validation_budget_status, home, max_evaluations)
|
||||
|
||||
return FunctionTool(
|
||||
get_validation_budget,
|
||||
|
|
@ -387,10 +390,10 @@ def validation_budget_tool(home, max_validations):
|
|||
)
|
||||
|
||||
|
||||
def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10):
|
||||
def evaluation_tool(wrapper, options, *, home, timer=None, max_evaluations=10):
|
||||
"""Only debug mode is model-selected; the host decides construction replay."""
|
||||
job = required_job(wrapper, "home_evaluate")
|
||||
validate_max_validations(max_validations)
|
||||
validate_max_evaluations(max_evaluations)
|
||||
|
||||
async def evaluate(debug: bool = False, **kwargs):
|
||||
"""Run debug after committing; after debug passes, run a sampled formal evaluation.
|
||||
|
|
@ -408,7 +411,7 @@ def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10):
|
|||
started_before = await blocking(lambda: set(read_state(home)["started_validations"])) if not debug else set()
|
||||
response = await job(
|
||||
**options,
|
||||
**({"max_validations": max_validations} if not debug else {}),
|
||||
**({"max_evaluations": max_evaluations} if not debug else {}),
|
||||
debug=debug,
|
||||
sampling=not debug,
|
||||
record_history=True,
|
||||
|
|
@ -417,7 +420,7 @@ def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10):
|
|||
reuse_construction_from=None,
|
||||
)
|
||||
if not debug:
|
||||
budget = await blocking(validation_budget_status, home, max_validations)
|
||||
budget = await blocking(validation_budget_status, home, max_evaluations)
|
||||
if isinstance(response.answer, dict):
|
||||
response.answer = {**response.answer, "validation_budget": budget}
|
||||
completed = await blocking(completed_validation, home, response.answer, started_before)
|
||||
|
|
@ -487,8 +490,8 @@ class HomeOptimizeStep(BaseStep):
|
|||
self.context.response.metadata["optimization_report_path"] = report_path
|
||||
try:
|
||||
timer = self.agent_wrapper.create_cycle_timer()
|
||||
max_validations = self.agent_wrapper.validation_limit()
|
||||
budget = await blocking(validation_budget_status, home, max_validations)
|
||||
max_evaluations = self.agent_wrapper.validation_limit()
|
||||
budget = await blocking(validation_budget_status, home, max_evaluations)
|
||||
reply = await workflow_reply(
|
||||
self.agent_wrapper,
|
||||
home,
|
||||
|
|
@ -501,13 +504,13 @@ class HomeOptimizeStep(BaseStep):
|
|||
tool_factory=lambda backend: [
|
||||
Edit(backend=backend),
|
||||
diagnostic_tool(self.agent_wrapper),
|
||||
validation_budget_tool(home, max_validations),
|
||||
validation_budget_tool(home, max_evaluations),
|
||||
evaluation_tool(
|
||||
self.agent_wrapper,
|
||||
options,
|
||||
home=home,
|
||||
timer=timer,
|
||||
max_validations=max_validations,
|
||||
max_evaluations=max_evaluations,
|
||||
),
|
||||
],
|
||||
job_names=MUTATION_JOBS,
|
||||
|
|
|
|||
|
|
@ -3,19 +3,19 @@
|
|||
from .history import read_state
|
||||
|
||||
|
||||
def validate_max_validations(value):
|
||||
def validate_max_evaluations(value):
|
||||
"""Accept only a positive host-supplied integer, matching dev_meta."""
|
||||
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
||||
raise ValueError("max_validations must be a positive integer")
|
||||
raise ValueError("max_evaluations must be a positive integer")
|
||||
return value
|
||||
|
||||
|
||||
def validation_budget_status(home, max_validations):
|
||||
def validation_budget_status(home, max_evaluations):
|
||||
"""Report started formal runs, independently of completion, score or sampler success."""
|
||||
validate_max_validations(max_validations)
|
||||
validate_max_evaluations(max_evaluations)
|
||||
used = sum(not key.startswith("init/") for key in read_state(home)["started_validations"])
|
||||
remaining = max(0, max_validations - used)
|
||||
gated = max_validations // 2
|
||||
remaining = max(0, max_evaluations - used)
|
||||
gated = max_evaluations // 2
|
||||
if not remaining:
|
||||
phase = "exhausted"
|
||||
guidance = (
|
||||
|
|
@ -35,7 +35,7 @@ def validation_budget_status(home, max_validations):
|
|||
)
|
||||
return {
|
||||
"used_validations": used,
|
||||
"max_validations": max_validations,
|
||||
"max_evaluations": max_evaluations,
|
||||
"remaining_validations": remaining,
|
||||
"gated_validation_count": gated,
|
||||
"optimization_phase": phase,
|
||||
|
|
|
|||
|
|
@ -91,9 +91,9 @@ class HomeEvaluation(BaseComponent):
|
|||
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
|
||||
lock = nullcontext() if spec.debug else _initialization_lock(self.workspace.state_path / "validation.lock")
|
||||
with lock:
|
||||
budgeted = spec.max_validations is not None and not spec.debug
|
||||
budgeted = spec.max_evaluations is not None and not spec.debug
|
||||
if budgeted:
|
||||
budget = await blocking(validation_budget_status, self.workspace.home_path, spec.max_validations)
|
||||
budget = await blocking(validation_budget_status, self.workspace.home_path, spec.max_evaluations)
|
||||
if budget["remaining_validations"] < 1:
|
||||
return {
|
||||
"status": "rejected",
|
||||
|
|
@ -141,7 +141,7 @@ class HomeEvaluation(BaseComponent):
|
|||
result["validation_budget"] = await blocking(
|
||||
validation_budget_status,
|
||||
self.workspace.home_path,
|
||||
spec.max_validations,
|
||||
spec.max_evaluations,
|
||||
)
|
||||
return result
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ class EvaluationSpec(SpecModel):
|
|||
debug: bool = False
|
||||
sampling: bool = False
|
||||
record_history: bool = True
|
||||
max_validations: int | None = Field(default=None, ge=1)
|
||||
max_evaluations: int | None = Field(default=None, ge=1)
|
||||
sampling_policy: SamplingPolicy | None = None
|
||||
concurrency: int = Field(default=4, ge=1)
|
||||
timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False)
|
||||
|
|
@ -68,7 +68,7 @@ class EvaluationSpec(SpecModel):
|
|||
@model_validator(mode="after")
|
||||
def resolve_mode(self):
|
||||
"""Debug stops on failure by default and never reuses scored evidence."""
|
||||
if self.max_validations is not None and not self.debug and not self.record_history:
|
||||
if self.max_evaluations is not None and not self.debug and not self.record_history:
|
||||
raise ValueError("Budgeted formal evaluations require record_history=true")
|
||||
if self.debug and self.reuse_construction_from:
|
||||
raise ValueError("Construction reuse requires training-set")
|
||||
|
|
|
|||
|
|
@ -17,6 +17,8 @@ class HomeEvaluateStep(BaseStep):
|
|||
if self.context.get("_allowed_paths") is not None:
|
||||
raise ValueError("Home evaluation Jobs require unrestricted access to their workspace output directories")
|
||||
arguments = {name: self.context.get(name) for name in EvaluationSpec.model_fields if name in self.context}
|
||||
if "max_validations" in self.context:
|
||||
raise ValueError("Use max_evaluations instead of max_validations")
|
||||
# Do not silently accept a removed toggle that conflicts with reuse paths.
|
||||
if "eval_only" in self.context:
|
||||
raise ValueError("Use reuse_construction_from instead of eval_only")
|
||||
|
|
|
|||
|
|
@ -97,7 +97,7 @@ application_defaults:
|
|||
max_iters: 60
|
||||
home_optimizer:
|
||||
backend: home_optimizer_agent
|
||||
max_validations: 10
|
||||
max_evaluations: 10
|
||||
as_llm: home_optimizer
|
||||
validation_cycle_time_budget_minutes: 150
|
||||
context_config:
|
||||
|
|
@ -340,7 +340,7 @@ application_defaults:
|
|||
description: "Stop on the first failure; defaults to debug (false for scored runs, true for debug)"
|
||||
max_construction_failure_rate: {type: number, minimum: 0, maximum: 1}
|
||||
max_artifact_bytes: {type: integer, minimum: 1}
|
||||
max_validations:
|
||||
max_evaluations:
|
||||
type: [integer, "null"]
|
||||
minimum: 1
|
||||
description: "Host formal-evaluation budget; early new candidates pass admission review; optimizer injects its limit"
|
||||
|
|
|
|||
|
|
@ -89,7 +89,7 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
|
|||
}
|
||||
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
|
||||
"validation_cycle_time_budget_minutes": 150,
|
||||
"max_validations": 10,
|
||||
"max_evaluations": 10,
|
||||
}
|
||||
application.start.assert_awaited_once()
|
||||
application.close.assert_awaited_once()
|
||||
|
|
@ -156,7 +156,7 @@ async def test_runner_passes_cycle_budget_to_optimizer(tmp_path, monkeypatch, be
|
|||
expected = 150 if budget == "omitted" else budget
|
||||
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
|
||||
"validation_cycle_time_budget_minutes": expected,
|
||||
"max_validations": 10,
|
||||
"max_evaluations": 10,
|
||||
}
|
||||
assert "validation_cycle_time_budget_minutes" not in application.run_job.call_args.kwargs["spec"]
|
||||
application.close.assert_awaited_once()
|
||||
|
|
@ -204,9 +204,9 @@ async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchma
|
|||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
|
||||
if limit == "omitted":
|
||||
config.pop("max_validations", None)
|
||||
config.pop("max_evaluations", None)
|
||||
else:
|
||||
config["max_validations"] = limit
|
||||
config["max_evaluations"] = limit
|
||||
path = tmp_path / "home.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
application = AsyncMock()
|
||||
|
|
@ -220,16 +220,25 @@ async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchma
|
|||
monkeypatch.setattr(module, "Application", factory)
|
||||
if limit in ("omitted", 3):
|
||||
await module.prepare(path)
|
||||
assert observed["components"]["agent_wrapper"]["home_optimizer"]["max_validations"] == (
|
||||
assert observed["components"]["agent_wrapper"]["home_optimizer"]["max_evaluations"] == (
|
||||
10 if limit == "omitted" else limit
|
||||
)
|
||||
assert "max_validations" not in application.run_job.call_args.kwargs["spec"]
|
||||
assert "max_evaluations" not in application.run_job.call_args.kwargs["spec"]
|
||||
else:
|
||||
with pytest.raises(ValueError):
|
||||
await module.prepare(path)
|
||||
assert not observed
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [beam_runner, lme_runner])
|
||||
def test_runner_rejects_old_budget_name(module):
|
||||
"""Reject obsolete config keys instead of using the default budget."""
|
||||
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
|
||||
config["max_validations"] = config.pop("max_evaluations")
|
||||
with pytest.raises(ValueError):
|
||||
module.RunHomeConfig.model_validate(config)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
@pytest.mark.parametrize("minutes", [30, None])
|
||||
|
|
@ -237,7 +246,7 @@ async def test_optimizer_lifecycle_uses_same_config_and_does_not_prepare(tmp_pat
|
|||
"""A separate optimization launch consumes the same count, timer and sampling settings."""
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
|
||||
config.update(max_validations=3, validation_cycle_time_budget_minutes=minutes)
|
||||
config.update(max_evaluations=3, validation_cycle_time_budget_minutes=minutes)
|
||||
path = tmp_path / "home.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
application = AsyncMock()
|
||||
|
|
@ -252,7 +261,7 @@ async def test_optimizer_lifecycle_uses_same_config_and_does_not_prepare(tmp_pat
|
|||
result = await module.optimize(path, workspace_dir=str(tmp_path / "existing"), instruction="Improve retrieval")
|
||||
assert result["status"] == "completed"
|
||||
assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
|
||||
"max_validations": 3,
|
||||
"max_evaluations": 3,
|
||||
"validation_cycle_time_budget_minutes": minutes,
|
||||
}
|
||||
application.run_job.assert_awaited_once_with("home_optimize", instruction="Improve retrieval")
|
||||
|
|
|
|||
|
|
@ -75,7 +75,7 @@ async def test_gate_before_sampling_preserves_budget_and_pins_code(app, prepared
|
|||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(component, "create_sampling_plan", draw)
|
||||
response = await app.run_job("home_evaluate", max_validations=10, sampling=True)
|
||||
response = await app.run_job("home_evaluate", max_evaluations=10, sampling=True)
|
||||
report = response.answer["candidate_gate"]
|
||||
assert report["base_commit"] == initial and report["candidate_commit"] == commit
|
||||
saved = read_json(Path(report["review_path"]))
|
||||
|
|
@ -109,16 +109,16 @@ async def test_baseline_debug_unbudgeted_and_late_phase_skip_gate(app, prepared,
|
|||
raise AssertionError("Gate must be skipped")
|
||||
|
||||
monkeypatch.setattr(candidate_gate, "review_candidate", unexpected)
|
||||
assert (await app.run_job("home_evaluate", max_validations=10)).success # Baseline.
|
||||
assert (await app.run_job("home_evaluate", max_evaluations=10)).success # Baseline.
|
||||
commit_candidate(prepared)
|
||||
assert (await app.run_job("home_evaluate", debug=True, max_validations=10)).success
|
||||
assert (await app.run_job("home_evaluate", debug=True, max_evaluations=10)).success
|
||||
assert (await app.run_job("home_evaluate")).success # Ordinary explicit evaluations remain ungated.
|
||||
commit_candidate(prepared)
|
||||
result = await app.run_job("home_evaluate", max_validations=2) # One previous non-init start: second half.
|
||||
result = await app.run_job("home_evaluate", max_evaluations=2) # One previous non-init start: second half.
|
||||
assert result.success, result.answer
|
||||
assert "candidate_gate" not in result.answer
|
||||
commit_candidate(prepared)
|
||||
result = await app.run_job("home_evaluate", max_validations=2)
|
||||
result = await app.run_job("home_evaluate", max_evaluations=2)
|
||||
assert result.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED"
|
||||
|
||||
|
||||
|
|
@ -132,7 +132,7 @@ async def test_nearest_ancestor_and_infrastructure_retry(app, prepared, sandbox,
|
|||
commit_candidate(prepared)
|
||||
set_model(app, "home_candidate_gate", ScriptedModel(final='{"decision":"accept","reason":"New mechanism"}'))
|
||||
monkeypatch.setattr(FakeFactory, "bad_export", True)
|
||||
failed = await app.run_job("home_evaluate", max_validations=10, sampling=True)
|
||||
failed = await app.run_job("home_evaluate", max_evaluations=10, sampling=True)
|
||||
assert not failed.success
|
||||
assert failed.answer["candidate_gate"]["base_commit"] == base
|
||||
|
||||
|
|
@ -141,9 +141,9 @@ async def test_nearest_ancestor_and_infrastructure_retry(app, prepared, sandbox,
|
|||
|
||||
monkeypatch.setattr(candidate_gate, "review_candidate", unexpected)
|
||||
monkeypatch.setattr(FakeFactory, "bad_export", False)
|
||||
recovered = await app.run_job("home_evaluate", max_validations=10, sampling=True)
|
||||
recovered = await app.run_job("home_evaluate", max_evaluations=10, sampling=True)
|
||||
assert recovered.success, recovered.answer
|
||||
repeat = await app.run_job("home_evaluate", max_validations=10)
|
||||
repeat = await app.run_job("home_evaluate", max_evaluations=10)
|
||||
assert repeat.answer["error_code"] == "UNCHANGED_CANDIDATE"
|
||||
|
||||
|
||||
|
|
@ -158,7 +158,7 @@ async def test_model_failure_allows_validation_with_saved_diagnostics(app, prepa
|
|||
raise RuntimeError("model unavailable")
|
||||
|
||||
monkeypatch.setattr(wrapper, "reply", broken)
|
||||
response = await app.run_job("home_evaluate", max_validations=10)
|
||||
response = await app.run_job("home_evaluate", max_evaluations=10)
|
||||
assert response.success, response.answer
|
||||
report = response.answer["candidate_gate"]
|
||||
assert report["decision"] == "accept" and report["review_failed"]
|
||||
|
|
@ -177,7 +177,7 @@ async def test_corrected_gate_rejects_before_spending_budget(app, prepared, sand
|
|||
),
|
||||
)
|
||||
set_model(app, "home_candidate_gate", model)
|
||||
response = await app.run_job("home_evaluate", max_validations=10)
|
||||
response = await app.run_job("home_evaluate", max_evaluations=10)
|
||||
assert not response.success
|
||||
assert response.answer["error_code"] == "CANDIDATE_GATE_REJECTED"
|
||||
report = read_json(Path(response.answer["candidate_gate"]["review_path"]))
|
||||
|
|
@ -199,7 +199,7 @@ async def test_sandbox_setup_failure_never_calls_model_or_evaluates(app, prepare
|
|||
raise RuntimeError("sandbox unavailable")
|
||||
|
||||
monkeypatch.setattr(workflow_common, "create_case_study_sandbox", unavailable)
|
||||
result = await app.run_job("home_evaluate", max_validations=10)
|
||||
result = await app.run_job("home_evaluate", max_evaluations=10)
|
||||
assert not result.success and "sandbox unavailable" in str(result.answer)
|
||||
assert not model.calls and not FakeFactory.instances
|
||||
assert validation_budget_status(prepared, 10)["used_validations"] == 0
|
||||
|
|
@ -218,9 +218,9 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared,
|
|||
return {"result": '{"decision":"accept","reason":"Mechanism change"}'}
|
||||
|
||||
monkeypatch.setattr(wrapper, "reply", paused)
|
||||
task = asyncio.create_task(app.run_job("home_evaluate", max_validations=10))
|
||||
task = asyncio.create_task(app.run_job("home_evaluate", max_evaluations=10))
|
||||
await asyncio.wait_for(entered.wait(), timeout=10)
|
||||
concurrent = await app.run_job("home_evaluate", max_validations=10)
|
||||
concurrent = await app.run_job("home_evaluate", max_evaluations=10)
|
||||
assert not concurrent.success # Formal lock spans review and evaluation.
|
||||
commit_candidate(prepared)
|
||||
release.set()
|
||||
|
|
@ -229,7 +229,7 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared,
|
|||
assert not FakeFactory.instances
|
||||
entered.clear()
|
||||
release.clear()
|
||||
task = asyncio.create_task(app.run_job("home_evaluate", max_validations=10))
|
||||
task = asyncio.create_task(app.run_job("home_evaluate", max_evaluations=10))
|
||||
await asyncio.wait_for(entered.wait(), timeout=10)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
|
|
@ -237,7 +237,7 @@ async def test_review_change_race_and_cancellation_release_locks(app, prepared,
|
|||
assert not list((prepared / "tmp").glob("candidate-gate-*"))
|
||||
assert validation_budget_status(prepared, 10)["used_validations"] == 0
|
||||
release.set()
|
||||
result = await app.run_job("home_evaluate", max_validations=10)
|
||||
result = await app.run_job("home_evaluate", max_evaluations=10)
|
||||
assert result.success, result.answer
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -129,7 +129,7 @@ async def test_large_resources_export_for_both_reviewers(app, prepared, reviewer
|
|||
ScriptedModel(final=finish),
|
||||
)
|
||||
force_draw(monkeypatch, 0.1)
|
||||
response = await app.run_job("home_evaluate", sampling=True, max_validations=10)
|
||||
response = await app.run_job("home_evaluate", sampling=True, max_evaluations=10)
|
||||
assert response.success, response.answer
|
||||
assert response.answer["execution_mode"] == "query_only_replay"
|
||||
for report in (
|
||||
|
|
@ -172,7 +172,7 @@ async def test_equivalent_candidate_uses_host_probability(
|
|||
)
|
||||
set_model(app, "home_construction_equivalence", model)
|
||||
force_draw(monkeypatch, draw)
|
||||
result = await app.run_job("home_evaluate", sampling=True, max_validations=10)
|
||||
result = await app.run_job("home_evaluate", sampling=True, max_evaluations=10)
|
||||
assert result.success, result.answer
|
||||
decision = result.answer["automatic_construction_reuse"]
|
||||
assert decision["selected"] == selected and decision["draw"] == draw and decision["probability"] == 0.6
|
||||
|
|
@ -346,13 +346,13 @@ async def test_infrastructure_retry_preserves_replay_and_selection(app, prepared
|
|||
set_model(app, "home_construction_equivalence", model)
|
||||
force_draw(monkeypatch, 0.1)
|
||||
monkeypatch.setattr(FakeFactory, "bad_export", True)
|
||||
failed = await app.run_job("home_evaluate", sampling=True, max_validations=10)
|
||||
failed = await app.run_job("home_evaluate", sampling=True, max_evaluations=10)
|
||||
assert not failed.success
|
||||
failed_manifest = read_json(Path(failed.answer["result_dir"]) / "manifest.json")
|
||||
assert (await app.run_job("home_create_candidate_branch", branch_name="retry-branch")).success
|
||||
monkeypatch.setattr(FakeFactory, "bad_export", False)
|
||||
monkeypatch.setattr(reuse, "select_query_only", lambda: pytest.fail("Recovery must not redraw"))
|
||||
recovered = await app.run_job("home_evaluate", sampling=True, max_validations=10)
|
||||
recovered = await app.run_job("home_evaluate", sampling=True, max_evaluations=10)
|
||||
assert recovered.success, recovered.answer
|
||||
manifest = read_json(Path(recovered.answer["result_dir"]) / "manifest.json")
|
||||
assert manifest["automatic_construction_reuse"] == {
|
||||
|
|
@ -411,7 +411,7 @@ async def test_sandbox_failure_stops_before_model_sampling_or_budget(app, prepar
|
|||
raise RuntimeError("sandbox unavailable")
|
||||
|
||||
monkeypatch.setattr(workflow_common, "create_case_study_sandbox", unavailable)
|
||||
response = await app.run_job("home_evaluate", sampling=True, max_validations=10)
|
||||
response = await app.run_job("home_evaluate", sampling=True, max_evaluations=10)
|
||||
assert not response.success and "sandbox unavailable" in str(response.answer)
|
||||
assert not model.calls and len(FakeFactory.instances) == 1
|
||||
assert not list((prepared / "tmp").glob("construction-equivalence-*"))
|
||||
|
|
@ -459,7 +459,7 @@ async def test_partial_export_and_report_failures_clean_review_files(
|
|||
response = await app.run_job(
|
||||
"home_evaluate",
|
||||
sampling=True,
|
||||
**({"max_validations": 10} if kind == "candidate-gate" else {}),
|
||||
**({"max_evaluations": 10} if kind == "candidate-gate" else {}),
|
||||
)
|
||||
assert not response.success and "failed" in str(response.answer)
|
||||
assert not list((prepared / "tmp").glob(f"{kind}-*"))
|
||||
|
|
|
|||
|
|
@ -269,7 +269,7 @@ async def test_benchmark_entry_points_select_correct_defaults(benchmark, monkeyp
|
|||
runner.assert_awaited_once_with(module.DEFAULT_CONFIG, expected_benchmark=benchmark, commit="abcdef0")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_validations": 1}])
|
||||
@pytest.mark.parametrize("options", [{"sampling": True}, {"record_history": True}, {"max_evaluations": 1}])
|
||||
def test_test_runner_rejects_optimizer_modes(options):
|
||||
with pytest.raises(ValueError):
|
||||
home_eval.TestRunConfig.model_validate(
|
||||
|
|
|
|||
|
|
@ -60,21 +60,39 @@ def test_dev_meta_phase_boundaries(tmp_path, used, phase, remaining):
|
|||
|
||||
@pytest.mark.parametrize("invalid", [True, False, 0, -1, 1.5, "10", None])
|
||||
def test_invalid_optimizer_budgets_rejected(invalid):
|
||||
wrapper = OptimizerAgentWrapper(name="optimizer", max_validations=invalid)
|
||||
wrapper = OptimizerAgentWrapper(name="optimizer", max_evaluations=invalid)
|
||||
with pytest.raises(ValueError, match="positive integer"):
|
||||
wrapper.validation_limit()
|
||||
if invalid is not None:
|
||||
with pytest.raises(ValueError):
|
||||
EvaluationSpec(max_validations=invalid)
|
||||
EvaluationSpec(max_evaluations=invalid)
|
||||
|
||||
|
||||
def test_budgeted_evaluation_cannot_disable_started_history():
|
||||
with pytest.raises(ValueError, match="record_history"):
|
||||
EvaluationSpec(max_validations=10, record_history=False)
|
||||
assert EvaluationSpec(debug=True, max_validations=1, record_history=False).debug
|
||||
EvaluationSpec(max_evaluations=10, record_history=False)
|
||||
assert EvaluationSpec(debug=True, max_evaluations=1, record_history=False).debug
|
||||
assert OptimizerAgentWrapper(name="optimizer").validation_limit() == 10
|
||||
|
||||
|
||||
def test_old_budget_name_is_rejected():
|
||||
with pytest.raises(ValueError):
|
||||
EvaluationSpec(max_validations=3)
|
||||
# Application defaults can supply the new name alongside an old override.
|
||||
wrapper = OptimizerAgentWrapper(name="optimizer", max_evaluations=10, max_validations=3)
|
||||
with pytest.raises(ValueError, match="Use max_evaluations"):
|
||||
wrapper.validation_limit()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_job_rejects_old_budget_before_starting_sandbox(app):
|
||||
count = len(FakeFactory.instances)
|
||||
response = await app.run_job("home_evaluate", max_validations=3)
|
||||
assert not response.success
|
||||
assert "Use max_evaluations" in str(response.answer)
|
||||
assert len(FakeFactory.instances) == count
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_budget_tool_reports_persisted_count_without_mutation(tmp_path):
|
||||
seed_history(tmp_path, ["init/base/1", "candidate/hash/1"])
|
||||
|
|
@ -93,7 +111,7 @@ async def test_exhaustion_blocks_before_pool_and_survives_cache_rebuild(app, pre
|
|||
assert baseline.success
|
||||
assert validation_budget_status(prepared, 1)["used_validations"] == 0
|
||||
commit_candidate(prepared)
|
||||
result = await app.run_job("home_evaluate", max_validations=1)
|
||||
result = await app.run_job("home_evaluate", max_evaluations=1)
|
||||
assert result.success, result.answer
|
||||
assert result.answer["validation_budget"]["remaining_validations"] == 0
|
||||
count = len(FakeFactory.instances)
|
||||
|
|
@ -101,12 +119,12 @@ async def test_exhaustion_blocks_before_pool_and_survives_cache_rebuild(app, pre
|
|||
(prepared / "sampling/state.json").unlink()
|
||||
assert validation_budget_status(prepared, 1)["used_validations"] == 1
|
||||
commit_candidate(prepared)
|
||||
rejected = await app.run_job("home_evaluate", max_validations=1)
|
||||
rejected = await app.run_job("home_evaluate", max_evaluations=1)
|
||||
assert not rejected.success
|
||||
assert rejected.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED"
|
||||
assert rejected.answer["result_dir"] is None
|
||||
assert len(FakeFactory.instances) == count
|
||||
debug = await app.run_job("home_evaluate", debug=True, max_validations=1)
|
||||
debug = await app.run_job("home_evaluate", debug=True, max_evaluations=1)
|
||||
assert debug.success, debug.answer
|
||||
assert "validation_budget" not in debug.answer
|
||||
assert validation_budget_status(prepared, 1)["used_validations"] == 1
|
||||
|
|
@ -129,7 +147,7 @@ async def test_charge_depends_on_first_sandbox_start_not_result(app, prepared, m
|
|||
raise RuntimeError("sandbox initialization failed")
|
||||
|
||||
monkeypatch.setattr(evaluation, "DockerReMeSandboxFactory", PrestartFailureFactory)
|
||||
task = asyncio.create_task(app.run_job("home_evaluate", max_validations=1))
|
||||
task = asyncio.create_task(app.run_job("home_evaluate", max_evaluations=1))
|
||||
if outcome == "cancelled":
|
||||
while not FakeFactory.instances:
|
||||
await asyncio.sleep(0)
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ async def test_benchmark_optimizer_applies_budgets_to_real_workflow(
|
|||
{
|
||||
"workspace_dir": str(prepared.parent),
|
||||
"preparation": inputs[0],
|
||||
"max_validations": 3,
|
||||
"max_evaluations": 3,
|
||||
"validation_cycle_time_budget_minutes": 30,
|
||||
},
|
||||
),
|
||||
|
|
@ -250,7 +250,7 @@ async def test_benchmark_optimizer_applies_budgets_to_real_workflow(
|
|||
assert len(agents) == 1
|
||||
assert agents[0].validation_cycle_timer.time_budget_minutes == 30
|
||||
task = "\n".join(msg.get_text_content() or "" for msg in model.calls[0]["messages"])
|
||||
assert '"max_validations": 3' in task
|
||||
assert '"max_evaluations": 3' in task
|
||||
assert '"remaining_validations": 3' in task
|
||||
assert len(FakeFactory.instances) == 1 # Only the earlier baseline ran.
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue