From 6cd9c9c6b2b08ae01b1c2afa55745ec1505ff5a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=B1=E7=89=9B=E8=82=89?= Date: Tue, 3 Nov 2026 11:44:32 +0800 Subject: [PATCH] fix(benchmark): propagate Home sandbox memory limits --- benchmark/beam/README.md | 5 +++ benchmark/beam/README_ZH.md | 4 ++ benchmark/beam/config_home.yaml | 2 + benchmark/beam/run_home.py | 28 ++++++++++++- benchmark/longmemeval/README.md | 5 +++ benchmark/longmemeval/README_ZH.md | 4 ++ benchmark/longmemeval/config_home.yaml | 2 + benchmark/longmemeval/run_home.py | 28 ++++++++++++- plugins/home/tests/test_benchmark_runner.py | 46 +++++++++++++++++++++ 9 files changed, 122 insertions(+), 2 deletions(-) diff --git a/benchmark/beam/README.md b/benchmark/beam/README.md index 798eaa98..60d04ee0 100644 --- a/benchmark/beam/README.md +++ b/benchmark/beam/README.md @@ -172,6 +172,11 @@ The top-level `max_evaluations` sets the optimizer's formal-evaluation budget (positive integer, default 10). A slot is consumed at first sandbox startup, including later failures or cancellations; initial baseline and debug are excluded. +The top-level `memory_limit_mib` sets the host agent sandbox memory limit for +optimization, diagnosis, case studies and evaluation reviews. Set `memory_limit_mib: 0` +in the runner YAML to disable the Linux cgroup limit. Null or omission keeps platform +defaults (4096 MiB on Linux); macOS does not enforce memory limits. + After preparation, run local one-shot optimization with the same configuration: ```bash diff --git a/benchmark/beam/README_ZH.md b/benchmark/beam/README_ZH.md index 9647fc37..f78d33a1 100644 --- a/benchmark/beam/README_ZH.md +++ b/benchmark/beam/README_ZH.md @@ -160,6 +160,10 @@ Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。 顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。 首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。 +顶层 `memory_limit_mib` 配置优化、诊断、case study 和评估审查的宿主 agent 沙箱内存限制。 +在 runner YAML 中设置 `memory_limit_mib: 0` 可关闭 Linux cgroup 内存限制; +省略或设为 null 时保留平台默认值(Linux 为 4096 MiB),macOS 不执行内存限制。 + 准备完成后,用同一份配置进行本地一次性优化: ```bash diff --git a/benchmark/beam/config_home.yaml b/benchmark/beam/config_home.yaml index fa3fb901..10163110 100644 --- a/benchmark/beam/config_home.yaml +++ b/benchmark/beam/config_home.yaml @@ -3,6 +3,8 @@ workspace_dir: benchmark/beam/workspaces/home-100K # Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug. max_evaluations: 10 +# Host agent sandbox memory limit in MiB; null keeps platform defaults, 0 disables the Linux cgroup limit. +memory_limit_mib: 0 # Target minutes between completed formal optimizer evaluations; null disables timed reminders. validation_cycle_time_budget_minutes: 150 # Policy defaults for Home evaluation; each call still controls sampling=true/false. diff --git a/benchmark/beam/run_home.py b/benchmark/beam/run_home.py index e70c4359..8a3700d6 100644 --- a/benchmark/beam/run_home.py +++ b/benchmark/beam/run_home.py @@ -36,6 +36,7 @@ class RunHomeConfig(BaseModel): sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy) validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False) max_evaluations: int = Field(default=10, ge=1) + memory_limit_mib: int | None = Field(default=None, ge=0) def project_path(value: str, project_root: Path) -> Path: @@ -69,11 +70,35 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins """Apply the same runner configuration to each preparation or optimization lifecycle.""" workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir) load_env(PROJECT_ROOT / ".env", override=False) + memory_options = {} + jobs = {} + if config.memory_limit_mib is not None: + memory_options = { + "candidate_gate_memory_limit_mib": config.memory_limit_mib, + "construction_equivalence_memory_limit_mib": config.memory_limit_mib, + } + for name, backend, wrapper in ( + ("home_optimize", "home_optimize_step", "home_optimizer"), + ("home_diagnose", "home_diagnose_step", "home_diagnostic"), + ("home_case_study", "home_case_study_step", "home_case_study"), + ): + jobs[name] = { + "steps": [ + { + "backend": backend, + "agent_wrapper": wrapper, + "memory_limit_mib": config.memory_limit_mib, + **({"concurrency": 4, "output_token_limit": 2500} if name == "home_case_study" else {}), + }, + ], + } app = Application( workspace_dir=str(workspace), plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]], components={ - "home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}}, + "home.evaluation": { + "default": {"sampling_policy": config.sampling_policy.model_dump(mode="json"), **memory_options}, + }, "agent_wrapper": { "home_optimizer": { "validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes, @@ -81,6 +106,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins }, }, }, + jobs=jobs, service={"backend": "cli"}, enable_logo=False, log_to_file=False, diff --git a/benchmark/longmemeval/README.md b/benchmark/longmemeval/README.md index a589b690..d32a07d4 100644 --- a/benchmark/longmemeval/README.md +++ b/benchmark/longmemeval/README.md @@ -145,6 +145,11 @@ The top-level `max_evaluations` sets the optimizer's formal-evaluation budget (positive integer, default 10). A slot is consumed at first sandbox startup, including later failures or cancellations; initial baseline and debug are excluded. +The top-level `memory_limit_mib` sets the host agent sandbox memory limit for +optimization, diagnosis, case studies and evaluation reviews. Set `memory_limit_mib: 0` +in the runner YAML to disable the Linux cgroup limit. Null or omission keeps platform +defaults (4096 MiB on Linux); macOS does not enforce memory limits. + After preparation, run local one-shot optimization with the same configuration: ```bash diff --git a/benchmark/longmemeval/README_ZH.md b/benchmark/longmemeval/README_ZH.md index ffb8e2da..eca0693e 100644 --- a/benchmark/longmemeval/README_ZH.md +++ b/benchmark/longmemeval/README_ZH.md @@ -131,6 +131,10 @@ Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=. 顶层 `max_evaluations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。 首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。 +顶层 `memory_limit_mib` 配置优化、诊断、case study 和评估审查的宿主 agent 沙箱内存限制。 +在 runner YAML 中设置 `memory_limit_mib: 0` 可关闭 Linux cgroup 内存限制; +省略或设为 null 时保留平台默认值(Linux 为 4096 MiB),macOS 不执行内存限制。 + 准备完成后,用同一份配置进行本地一次性优化: ```bash diff --git a/benchmark/longmemeval/config_home.yaml b/benchmark/longmemeval/config_home.yaml index 764773b5..2ebbf1c9 100644 --- a/benchmark/longmemeval/config_home.yaml +++ b/benchmark/longmemeval/config_home.yaml @@ -4,6 +4,8 @@ workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5 # Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug. max_evaluations: 10 +# Host agent sandbox memory limit in MiB; null keeps platform defaults, 0 disables the Linux cgroup limit. +memory_limit_mib: 0 # Policy defaults for Home evaluation; each call still controls sampling=true/false. sampling_policy: max_cases: 5 diff --git a/benchmark/longmemeval/run_home.py b/benchmark/longmemeval/run_home.py index 1ab63cd1..e401cb82 100644 --- a/benchmark/longmemeval/run_home.py +++ b/benchmark/longmemeval/run_home.py @@ -36,6 +36,7 @@ class RunHomeConfig(BaseModel): sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy) validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False) max_evaluations: int = Field(default=10, ge=1) + memory_limit_mib: int | None = Field(default=None, ge=0) def project_path(value: str, project_root: Path) -> Path: @@ -69,11 +70,35 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins """Apply the same runner configuration to each preparation or optimization lifecycle.""" workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir) load_env(PROJECT_ROOT / ".env", override=False) + memory_options = {} + jobs = {} + if config.memory_limit_mib is not None: + memory_options = { + "candidate_gate_memory_limit_mib": config.memory_limit_mib, + "construction_equivalence_memory_limit_mib": config.memory_limit_mib, + } + for name, backend, wrapper in ( + ("home_optimize", "home_optimize_step", "home_optimizer"), + ("home_diagnose", "home_diagnose_step", "home_diagnostic"), + ("home_case_study", "home_case_study_step", "home_case_study"), + ): + jobs[name] = { + "steps": [ + { + "backend": backend, + "agent_wrapper": wrapper, + "memory_limit_mib": config.memory_limit_mib, + **({"concurrency": 4, "output_token_limit": 2500} if name == "home_case_study" else {}), + }, + ], + } app = Application( workspace_dir=str(workspace), plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]], components={ - "home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}}, + "home.evaluation": { + "default": {"sampling_policy": config.sampling_policy.model_dump(mode="json"), **memory_options}, + }, "agent_wrapper": { "home_optimizer": { "validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes, @@ -81,6 +106,7 @@ async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, ins }, }, }, + jobs=jobs, service={"backend": "cli"}, enable_logo=False, log_to_file=False, diff --git a/plugins/home/tests/test_benchmark_runner.py b/plugins/home/tests/test_benchmark_runner.py index 68fc471c..0b6e95e1 100644 --- a/plugins/home/tests/test_benchmark_runner.py +++ b/plugins/home/tests/test_benchmark_runner.py @@ -272,6 +272,52 @@ async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchma assert not observed +@pytest.mark.asyncio +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +@pytest.mark.parametrize("limit", ["omitted", None, 0, 1024, -1, True, "0", 1.5]) +async def test_runner_sandbox_memory_policy(tmp_path, monkeypatch, benchmark, limit): + """Keep zero explicit across every host sandbox and preserve platform defaults when unset.""" + module = import_module(f"benchmark.{benchmark}.run_home") + config = yaml.safe_load(module.DEFAULT_CONFIG.read_text()) + if limit == "omitted": + config.pop("memory_limit_mib", None) + else: + config["memory_limit_mib"] = limit + path = tmp_path / "home.yaml" + path.write_text(yaml.safe_dump(config)) + application = AsyncMock() + application.run_job.return_value = Response(answer={}) + observed = {} + + def factory(**kwargs): + observed.update(kwargs) + return application + + monkeypatch.setattr(module, "Application", factory) + if isinstance(limit, bool) or limit in (-1, "0", 1.5): + with pytest.raises(ValueError): + await module.optimize(path) + assert not observed + return + await module.optimize(path) + evaluation = observed["components"]["home.evaluation"]["default"] + if limit in ("omitted", None): + assert observed["jobs"] == {} + assert "candidate_gate_memory_limit_mib" not in evaluation + assert "construction_equivalence_memory_limit_mib" not in evaluation + else: + assert evaluation["candidate_gate_memory_limit_mib"] == limit + assert evaluation["construction_equivalence_memory_limit_mib"] == limit + defaults = yaml.safe_load((ROOT / "plugins/home/src/reme_home/plugin.yaml").read_text())[ + "application_defaults" + ]["jobs"] + assert set(observed["jobs"]) == {"home_optimize", "home_diagnose", "home_case_study"} + for name, job in observed["jobs"].items(): + assert job["steps"] == [{**defaults[name]["steps"][0], "memory_limit_mib": limit}] + application.run_job.assert_awaited_once_with("home_optimize") + application.close.assert_awaited_once() + + @pytest.mark.parametrize("module", [beam_runner, lme_runner]) def test_runner_rejects_old_budget_name(module): """Reject obsolete config keys instead of using the default budget."""