mirror of
https://github.com/agentscope-ai/ReMe.git
synced 2026-11-15 09:11:24 +00:00
fix(benchmark): propagate Home sandbox memory limits
This commit is contained in:
parent
93898e314b
commit
6cd9c9c6b2
9 changed files with 122 additions and 2 deletions
|
|
@ -172,6 +172,11 @@ The top-level `max_evaluations` sets the optimizer's formal-evaluation budget
|
|||
(positive integer, default 10). A slot is consumed at first sandbox startup,
|
||||
including later failures or cancellations; initial baseline and debug are excluded.
|
||||
|
||||
The top-level `memory_limit_mib` sets the host agent sandbox memory limit for
|
||||
optimization, diagnosis, case studies and evaluation reviews. Set `memory_limit_mib: 0`
|
||||
in the runner YAML to disable the Linux cgroup limit. Null or omission keeps platform
|
||||
defaults (4096 MiB on Linux); macOS does not enforce memory limits.
|
||||
|
||||
After preparation, run local one-shot optimization with the same configuration:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -160,6 +160,10 @@ Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。
|
|||
顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
|
||||
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
|
||||
|
||||
顶层 `memory_limit_mib` 配置优化、诊断、case study 和评估审查的宿主 agent 沙箱内存限制。
|
||||
在 runner YAML 中设置 `memory_limit_mib: 0` 可关闭 Linux cgroup 内存限制;
|
||||
省略或设为 null 时保留平台默认值(Linux 为 4096 MiB),macOS 不执行内存限制。
|
||||
|
||||
准备完成后,用同一份配置进行本地一次性优化:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -3,6 +3,8 @@
|
|||
workspace_dir: benchmark/beam/workspaces/home-100K
|
||||
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
|
||||
max_evaluations: 10
|
||||
# Host agent sandbox memory limit in MiB; null keeps platform defaults, 0 disables the Linux cgroup limit.
|
||||
memory_limit_mib: 0
|
||||
# Target minutes between completed formal optimizer evaluations; null disables timed reminders.
|
||||
validation_cycle_time_budget_minutes: 150
|
||||
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ class RunHomeConfig(BaseModel):
|
|||
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
|
||||
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
|
||||
max_evaluations: int = Field(default=10, ge=1)
|
||||
memory_limit_mib: int | None = Field(default=None, ge=0)
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
|
|
@ -69,11 +70,35 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
|
|||
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
|
||||
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
|
||||
load_env(PROJECT_ROOT / ".env", override=False)
|
||||
memory_options = {}
|
||||
jobs = {}
|
||||
if config.memory_limit_mib is not None:
|
||||
memory_options = {
|
||||
"candidate_gate_memory_limit_mib": config.memory_limit_mib,
|
||||
"construction_equivalence_memory_limit_mib": config.memory_limit_mib,
|
||||
}
|
||||
for name, backend, wrapper in (
|
||||
("home_optimize", "home_optimize_step", "home_optimizer"),
|
||||
("home_diagnose", "home_diagnose_step", "home_diagnostic"),
|
||||
("home_case_study", "home_case_study_step", "home_case_study"),
|
||||
):
|
||||
jobs[name] = {
|
||||
"steps": [
|
||||
{
|
||||
"backend": backend,
|
||||
"agent_wrapper": wrapper,
|
||||
"memory_limit_mib": config.memory_limit_mib,
|
||||
**({"concurrency": 4, "output_token_limit": 2500} if name == "home_case_study" else {}),
|
||||
},
|
||||
],
|
||||
}
|
||||
app = Application(
|
||||
workspace_dir=str(workspace),
|
||||
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
|
||||
components={
|
||||
"home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}},
|
||||
"home.evaluation": {
|
||||
"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json"), **memory_options},
|
||||
},
|
||||
"agent_wrapper": {
|
||||
"home_optimizer": {
|
||||
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
|
||||
|
|
@ -81,6 +106,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
|
|||
},
|
||||
},
|
||||
},
|
||||
jobs=jobs,
|
||||
service={"backend": "cli"},
|
||||
enable_logo=False,
|
||||
log_to_file=False,
|
||||
|
|
|
|||
|
|
@ -145,6 +145,11 @@ The top-level `max_evaluations` sets the optimizer's formal-evaluation budget
|
|||
(positive integer, default 10). A slot is consumed at first sandbox startup,
|
||||
including later failures or cancellations; initial baseline and debug are excluded.
|
||||
|
||||
The top-level `memory_limit_mib` sets the host agent sandbox memory limit for
|
||||
optimization, diagnosis, case studies and evaluation reviews. Set `memory_limit_mib: 0`
|
||||
in the runner YAML to disable the Linux cgroup limit. Null or omission keeps platform
|
||||
defaults (4096 MiB on Linux); macOS does not enforce memory limits.
|
||||
|
||||
After preparation, run local one-shot optimization with the same configuration:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -131,6 +131,10 @@ Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=.
|
|||
顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
|
||||
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
|
||||
|
||||
顶层 `memory_limit_mib` 配置优化、诊断、case study 和评估审查的宿主 agent 沙箱内存限制。
|
||||
在 runner YAML 中设置 `memory_limit_mib: 0` 可关闭 Linux cgroup 内存限制;
|
||||
省略或设为 null 时保留平台默认值(Linux 为 4096 MiB),macOS 不执行内存限制。
|
||||
|
||||
准备完成后,用同一份配置进行本地一次性优化:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@
|
|||
workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5
|
||||
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
|
||||
max_evaluations: 10
|
||||
# Host agent sandbox memory limit in MiB; null keeps platform defaults, 0 disables the Linux cgroup limit.
|
||||
memory_limit_mib: 0
|
||||
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
|
||||
sampling_policy:
|
||||
max_cases: 5
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ class RunHomeConfig(BaseModel):
|
|||
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
|
||||
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
|
||||
max_evaluations: int = Field(default=10, ge=1)
|
||||
memory_limit_mib: int | None = Field(default=None, ge=0)
|
||||
|
||||
|
||||
def project_path(value: str, project_root: Path) -> Path:
|
||||
|
|
@ -69,11 +70,35 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
|
|||
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
|
||||
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
|
||||
load_env(PROJECT_ROOT / ".env", override=False)
|
||||
memory_options = {}
|
||||
jobs = {}
|
||||
if config.memory_limit_mib is not None:
|
||||
memory_options = {
|
||||
"candidate_gate_memory_limit_mib": config.memory_limit_mib,
|
||||
"construction_equivalence_memory_limit_mib": config.memory_limit_mib,
|
||||
}
|
||||
for name, backend, wrapper in (
|
||||
("home_optimize", "home_optimize_step", "home_optimizer"),
|
||||
("home_diagnose", "home_diagnose_step", "home_diagnostic"),
|
||||
("home_case_study", "home_case_study_step", "home_case_study"),
|
||||
):
|
||||
jobs[name] = {
|
||||
"steps": [
|
||||
{
|
||||
"backend": backend,
|
||||
"agent_wrapper": wrapper,
|
||||
"memory_limit_mib": config.memory_limit_mib,
|
||||
**({"concurrency": 4, "output_token_limit": 2500} if name == "home_case_study" else {}),
|
||||
},
|
||||
],
|
||||
}
|
||||
app = Application(
|
||||
workspace_dir=str(workspace),
|
||||
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
|
||||
components={
|
||||
"home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}},
|
||||
"home.evaluation": {
|
||||
"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json"), **memory_options},
|
||||
},
|
||||
"agent_wrapper": {
|
||||
"home_optimizer": {
|
||||
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
|
||||
|
|
@ -81,6 +106,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
|
|||
},
|
||||
},
|
||||
},
|
||||
jobs=jobs,
|
||||
service={"backend": "cli"},
|
||||
enable_logo=False,
|
||||
log_to_file=False,
|
||||
|
|
|
|||
|
|
@ -272,6 +272,52 @@ async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchma
|
|||
assert not observed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
|
||||
@pytest.mark.parametrize("limit", ["omitted", None, 0, 1024, -1, True, "0", 1.5])
|
||||
async def test_runner_sandbox_memory_policy(tmp_path, monkeypatch, benchmark, limit):
|
||||
"""Keep zero explicit across every host sandbox and preserve platform defaults when unset."""
|
||||
module = import_module(f"benchmark.{benchmark}.run_home")
|
||||
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
|
||||
if limit == "omitted":
|
||||
config.pop("memory_limit_mib", None)
|
||||
else:
|
||||
config["memory_limit_mib"] = limit
|
||||
path = tmp_path / "home.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
application = AsyncMock()
|
||||
application.run_job.return_value = Response(answer={})
|
||||
observed = {}
|
||||
|
||||
def factory(**kwargs):
|
||||
observed.update(kwargs)
|
||||
return application
|
||||
|
||||
monkeypatch.setattr(module, "Application", factory)
|
||||
if isinstance(limit, bool) or limit in (-1, "0", 1.5):
|
||||
with pytest.raises(ValueError):
|
||||
await module.optimize(path)
|
||||
assert not observed
|
||||
return
|
||||
await module.optimize(path)
|
||||
evaluation = observed["components"]["home.evaluation"]["default"]
|
||||
if limit in ("omitted", None):
|
||||
assert observed["jobs"] == {}
|
||||
assert "candidate_gate_memory_limit_mib" not in evaluation
|
||||
assert "construction_equivalence_memory_limit_mib" not in evaluation
|
||||
else:
|
||||
assert evaluation["candidate_gate_memory_limit_mib"] == limit
|
||||
assert evaluation["construction_equivalence_memory_limit_mib"] == limit
|
||||
defaults = yaml.safe_load((ROOT / "plugins/home/src/reme_home/plugin.yaml").read_text())[
|
||||
"application_defaults"
|
||||
]["jobs"]
|
||||
assert set(observed["jobs"]) == {"home_optimize", "home_diagnose", "home_case_study"}
|
||||
for name, job in observed["jobs"].items():
|
||||
assert job["steps"] == [{**defaults[name]["steps"][0], "memory_limit_mib": limit}]
|
||||
application.run_job.assert_awaited_once_with("home_optimize")
|
||||
application.close.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("module", [beam_runner, lme_runner])
|
||||
def test_runner_rejects_old_budget_name(module):
|
||||
"""Reject obsolete config keys instead of using the default budget."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue