fix(benchmark): propagate Home sandbox memory limits

This commit is contained in:
酱牛肉 2026-11-03 11:44:32 +08:00
parent 93898e314b
commit 6cd9c9c6b2
9 changed files with 122 additions and 2 deletions

View file

@ -172,6 +172,11 @@ The top-level `max_evaluations` sets the optimizer's formal-evaluation budget
(positive integer, default 10). A slot is consumed at first sandbox startup,
including later failures or cancellations; initial baseline and debug are excluded.
The top-level `memory_limit_mib` sets the host agent sandbox memory limit for
optimization, diagnosis, case studies and evaluation reviews. Set `memory_limit_mib: 0`
in the runner YAML to disable the Linux cgroup limit. Null or omission keeps platform
defaults (4096 MiB on Linux); macOS does not enforce memory limits.
After preparation, run local one-shot optimization with the same configuration:
```bash

View file

@ -160,6 +160,10 @@ Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。
顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
顶层 `memory_limit_mib` 配置优化、诊断、case study 和评估审查的宿主 agent 沙箱内存限制。
在 runner YAML 中设置 `memory_limit_mib: 0` 可关闭 Linux cgroup 内存限制;
省略或设为 null 时保留平台默认值(Linux 为 4096 MiB),macOS 不执行内存限制。
准备完成后,用同一份配置进行本地一次性优化:
```bash

View file

@ -3,6 +3,8 @@
workspace_dir: benchmark/beam/workspaces/home-100K
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
max_evaluations: 10
# Host agent sandbox memory limit in MiB; null keeps platform defaults, 0 disables the Linux cgroup limit.
memory_limit_mib: 0
# Target minutes between completed formal optimizer evaluations; null disables timed reminders.
validation_cycle_time_budget_minutes: 150
# Policy defaults for Home evaluation; each call still controls sampling=true/false.

View file

@ -36,6 +36,7 @@ class RunHomeConfig(BaseModel):
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
max_evaluations: int = Field(default=10, ge=1)
memory_limit_mib: int | None = Field(default=None, ge=0)
def project_path(value: str, project_root: Path) -> Path:
@ -69,11 +70,35 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
load_env(PROJECT_ROOT / ".env", override=False)
memory_options = {}
jobs = {}
if config.memory_limit_mib is not None:
memory_options = {
"candidate_gate_memory_limit_mib": config.memory_limit_mib,
"construction_equivalence_memory_limit_mib": config.memory_limit_mib,
}
for name, backend, wrapper in (
("home_optimize", "home_optimize_step", "home_optimizer"),
("home_diagnose", "home_diagnose_step", "home_diagnostic"),
("home_case_study", "home_case_study_step", "home_case_study"),
):
jobs[name] = {
"steps": [
{
"backend": backend,
"agent_wrapper": wrapper,
"memory_limit_mib": config.memory_limit_mib,
**({"concurrency": 4, "output_token_limit": 2500} if name == "home_case_study" else {}),
},
],
}
app = Application(
workspace_dir=str(workspace),
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
components={
"home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}},
"home.evaluation": {
"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json"), **memory_options},
},
"agent_wrapper": {
"home_optimizer": {
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
@ -81,6 +106,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
},
},
},
jobs=jobs,
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,

View file

@ -145,6 +145,11 @@ The top-level `max_evaluations` sets the optimizer's formal-evaluation budget
(positive integer, default 10). A slot is consumed at first sandbox startup,
including later failures or cancellations; initial baseline and debug are excluded.
The top-level `memory_limit_mib` sets the host agent sandbox memory limit for
optimization, diagnosis, case studies and evaluation reviews. Set `memory_limit_mib: 0`
in the runner YAML to disable the Linux cgroup limit. Null or omission keeps platform
defaults (4096 MiB on Linux); macOS does not enforce memory limits.
After preparation, run local one-shot optimization with the same configuration:
```bash

View file

@ -131,6 +131,10 @@ Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=.
顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
顶层 `memory_limit_mib` 配置优化、诊断、case study 和评估审查的宿主 agent 沙箱内存限制。
在 runner YAML 中设置 `memory_limit_mib: 0` 可关闭 Linux cgroup 内存限制;
省略或设为 null 时保留平台默认值(Linux 为 4096 MiB),macOS 不执行内存限制。
准备完成后,用同一份配置进行本地一次性优化:
```bash

View file

@ -4,6 +4,8 @@
workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5
# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
max_evaluations: 10
# Host agent sandbox memory limit in MiB; null keeps platform defaults, 0 disables the Linux cgroup limit.
memory_limit_mib: 0
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
sampling_policy:
max_cases: 5

View file

@ -36,6 +36,7 @@ class RunHomeConfig(BaseModel):
sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
max_evaluations: int = Field(default=10, ge=1)
memory_limit_mib: int | None = Field(default=None, ge=0)
def project_path(value: str, project_root: Path) -> Path:
@ -69,11 +70,35 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
"""Apply the same runner configuration to each preparation or optimization lifecycle."""
workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
load_env(PROJECT_ROOT / ".env", override=False)
memory_options = {}
jobs = {}
if config.memory_limit_mib is not None:
memory_options = {
"candidate_gate_memory_limit_mib": config.memory_limit_mib,
"construction_equivalence_memory_limit_mib": config.memory_limit_mib,
}
for name, backend, wrapper in (
("home_optimize", "home_optimize_step", "home_optimizer"),
("home_diagnose", "home_diagnose_step", "home_diagnostic"),
("home_case_study", "home_case_study_step", "home_case_study"),
):
jobs[name] = {
"steps": [
{
"backend": backend,
"agent_wrapper": wrapper,
"memory_limit_mib": config.memory_limit_mib,
**({"concurrency": 4, "output_token_limit": 2500} if name == "home_case_study" else {}),
},
],
}
app = Application(
workspace_dir=str(workspace),
plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
components={
"home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}},
"home.evaluation": {
"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json"), **memory_options},
},
"agent_wrapper": {
"home_optimizer": {
"validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
@ -81,6 +106,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins
},
},
},
jobs=jobs,
service={"backend": "cli"},
enable_logo=False,
log_to_file=False,

View file

@ -272,6 +272,52 @@ async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchma
assert not observed
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
@pytest.mark.parametrize("limit", ["omitted", None, 0, 1024, -1, True, "0", 1.5])
async def test_runner_sandbox_memory_policy(tmp_path, monkeypatch, benchmark, limit):
"""Keep zero explicit across every host sandbox and preserve platform defaults when unset."""
module = import_module(f"benchmark.{benchmark}.run_home")
config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
if limit == "omitted":
config.pop("memory_limit_mib", None)
else:
config["memory_limit_mib"] = limit
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
application = AsyncMock()
application.run_job.return_value = Response(answer={})
observed = {}
def factory(**kwargs):
observed.update(kwargs)
return application
monkeypatch.setattr(module, "Application", factory)
if isinstance(limit, bool) or limit in (-1, "0", 1.5):
with pytest.raises(ValueError):
await module.optimize(path)
assert not observed
return
await module.optimize(path)
evaluation = observed["components"]["home.evaluation"]["default"]
if limit in ("omitted", None):
assert observed["jobs"] == {}
assert "candidate_gate_memory_limit_mib" not in evaluation
assert "construction_equivalence_memory_limit_mib" not in evaluation
else:
assert evaluation["candidate_gate_memory_limit_mib"] == limit
assert evaluation["construction_equivalence_memory_limit_mib"] == limit
defaults = yaml.safe_load((ROOT / "plugins/home/src/reme_home/plugin.yaml").read_text())[
"application_defaults"
]["jobs"]
assert set(observed["jobs"]) == {"home_optimize", "home_diagnose", "home_case_study"}
for name, job in observed["jobs"].items():
assert job["steps"] == [{**defaults[name]["steps"][0], "memory_limit_mib": limit}]
application.run_job.assert_awaited_once_with("home_optimize")
application.close.assert_awaited_once()
@pytest.mark.parametrize("module", [beam_runner, lme_runner])
def test_runner_rejects_old_budget_name(module):
"""Reject obsolete config keys instead of using the default budget."""