diff --git a/benchmark/beam/README.md b/benchmark/beam/README.md
index 8834c85f..9045b865 100644
--- a/benchmark/beam/README.md
+++ b/benchmark/beam/README.md
@@ -164,6 +164,27 @@ The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`
`home.evaluation` Component. It still only prepares; it does not run evaluations.
Each evaluation controls `sampling` and `record_history` separately.
+The top-level `validation_cycle_time_budget_minutes` configures the host optimizer's
+advisory time between completed formal evaluations. It defaults to 150 minutes;
+set a positive number to change it or null to disable timed reminders.
+
+The top-level `max_validations` sets the optimizer's formal-evaluation budget
+(positive integer, default 10). A slot is consumed at first sandbox startup,
+including later failures or cancellations; initial baseline and debug are excluded.
+
+After preparation, run local one-shot optimization with the same configuration:
+
+```bash
+python benchmark/beam/run_home.py --optimize
+# Use the same --config and --workspace-dir overrides as preparation, if supplied.
+python benchmark/beam/run_home.py --optimize --instruction "Improve retrieval quality"
+```
+
+This starts a fresh application, applies both budgets and the sampling policy,
+runs `home_optimize`, and closes the application. It does not repeat preparation.
+Python callers can use `await optimize(config_path, workspace_dir=...)` from the
+same module. The default `run()` and CLI behavior remain preparation only.
+
Python callers can use the same lifecycle:
```python
diff --git a/benchmark/beam/README_ZH.md b/benchmark/beam/README_ZH.md
index 44cd7a93..42bd4ea6 100644
--- a/benchmark/beam/README_ZH.md
+++ b/benchmark/beam/README_ZH.md
@@ -154,4 +154,21 @@ Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
和 `record_history`。
+顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的
+建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。
+
+顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
+首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
+
+准备完成后,用同一份配置进行本地一次性优化:
+
+```bash
+python benchmark/beam/run_home.py --optimize
+```
+
+若准备时指定了 `--config` 或 `--workspace-dir`,优化时传入相同参数。
+可用 `--instruction "优化检索质量"` 指定目标。该入口将配置中的两个预算和采样策略
+应用到实际 `home_optimize` 调用,不重复准备;Python 可调用同模块的 `await optimize(...)`。
+不加 `--optimize` 时仍只准备。
+
默认选择 BEAM **100K 全部 case**。
diff --git a/benchmark/beam/config_home.yaml b/benchmark/beam/config_home.yaml
index 5f00d4cc..6011264a 100644
--- a/benchmark/beam/config_home.yaml
+++ b/benchmark/beam/config_home.yaml
@@ -1,5 +1,10 @@
-# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
+# Preparation by default; --optimize applies these settings to the prepared workspace.
+# Paths are relative to the ReMe repository root, not the cwd.
workspace_dir: benchmark/beam/workspaces/home-100K
+# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
+max_validations: 10
+# Target minutes between completed formal optimizer evaluations; null disables timed reminders.
+validation_cycle_time_budget_minutes: 150
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
sampling_policy:
max_cases: 5
@@ -18,8 +23,6 @@ preparation:
source_ref: HEAD # Committed source only; resolved and saved as a full SHA.
config: benchmark.yaml
plugins: [beam]
- candidate:
- paths: [plugins/beam/src]
debug:
enabled: true
sessions_per_case: 3
diff --git a/benchmark/beam/run_home.py b/benchmark/beam/run_home.py
index 49270e72..d951dfcc 100644
--- a/benchmark/beam/run_home.py
+++ b/benchmark/beam/run_home.py
@@ -1,22 +1,145 @@
-"""Prepare the beam Home workspace, dataset and candidate code; do not evaluate."""
+"""Prepare the BEAM Home workspace, or optimize it with --optimize after preparation."""
-from pathlib import Path
+import argparse
+import asyncio
+import json
import sys
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict, Field
+import yaml
# Support direct execution from any working directory as well as module imports.
-_PROJECT_ROOT = Path(__file__).resolve().parents[2]
-if str(_PROJECT_ROOT) not in sys.path:
- sys.path.insert(0, str(_PROJECT_ROOT))
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+ sys.path.insert(0, str(PROJECT_ROOT))
-from benchmark.home_runner import main, prepare # noqa: E402 # pylint: disable=wrong-import-position
+# pylint: disable=wrong-import-position
+from reme_home.preparation.spec import PreparationSpec # noqa: E402
+from reme_home.judge import JUDGE_PLUGINS # noqa: E402
+from reme_home.sampling import SamplingPolicy # noqa: E402
+
+from reme.application import Application # noqa: E402
+
+# pylint: enable=wrong-import-position
DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml")
+class RunHomeConfig(BaseModel):
+ """Runner settings kept separate from Home's public preparation contract."""
+
+ model_config = ConfigDict(extra="forbid", strict=True)
+ workspace_dir: str = Field(min_length=1)
+ preparation: dict
+ sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
+ validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
+ max_validations: int = Field(default=10, ge=1)
+
+
+def project_path(value: str, project_root: Path) -> Path:
+ """Use repository-relative paths consistently regardless of the caller's cwd."""
+ if value.startswith("~"):
+ raise ValueError("Use absolute or project-relative paths, not home-relative paths")
+ return (project_root / value).resolve()
+
+
+def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
+ """Resolve source paths and validate explicit Home preparation parameters."""
+ raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
+ config = RunHomeConfig.model_validate(raw)
+ config.preparation.setdefault("candidate", {"paths": ["plugins/beam/src"]})
+ spec = PreparationSpec.model_validate(config.preparation)
+ spec.benchmark.source = str(project_path(spec.benchmark.source, project_root))
+ if spec.benchmark.evidence_source is not None:
+ spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
+ spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
+ workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
+ return workspace, spec.model_dump(mode="json"), config
+
+
+def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
+ """Return resolved preparation inputs, preserving the preparation-only caller contract."""
+ workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
+ return workspace, spec
+
+
+async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None):
+ """Apply the same runner configuration to each preparation or optimization lifecycle."""
+ workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
+ app = Application(
+ workspace_dir=str(workspace),
+ plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
+ components={
+ "home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}},
+ "agent_wrapper": {
+ "home_optimizer": {
+ "validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
+ "max_validations": config.max_validations,
+ },
+ },
+ },
+ service={"backend": "cli"},
+ enable_logo=False,
+ log_to_file=False,
+ )
+ try:
+ await app.start()
+ if optimize_workspace:
+ arguments = {} if instruction is None else {"instruction": instruction}
+ response = await app.run_job("home_optimize", **arguments)
+ else:
+ response = await app.run_job("home_prepare", spec=spec)
+ if not response.success:
+ raise RuntimeError(response.answer)
+ return {"workspace_dir": str(workspace), **response.answer}
+ finally:
+ await app.close()
+
+
+async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
+ """Prepare once without running evaluations or optimization."""
+ return await _run(config_path, workspace_dir=workspace_dir)
+
+
+async def optimize(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir=None, instruction=None) -> dict:
+ """Optimize an already prepared workspace using this config's actual model budgets."""
+ return await _run(config_path, workspace_dir=workspace_dir, optimize_workspace=True, instruction=instruction)
+
+
async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict:
"""Prepare through the same Home Job as the CLI, then close the application."""
return await prepare(config_path, workspace_dir=workspace_dir)
+def main(default_config: Path = DEFAULT_CONFIG) -> None:
+ """Run preparation or explicit local one-shot optimization with the benchmark config."""
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration")
+ parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)")
+ parser.add_argument("--optimize", action="store_true", help="Optimize an already prepared workspace")
+ parser.add_argument("--instruction", help="Optimization focus; requires --optimize")
+ args = parser.parse_args()
+ if args.instruction is not None and not args.optimize:
+ parser.error("--instruction requires --optimize")
+ if args.optimize:
+ result = asyncio.run(optimize(args.config, workspace_dir=args.workspace_dir, instruction=args.instruction))
+ print(json.dumps(result, ensure_ascii=False, indent=2))
+ return
+ result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir))
+ print(
+ json.dumps(
+ {
+ "workspace_dir": result["workspace_dir"],
+ "case_count": result["dataset"]["case_count"],
+ "query_count": result["dataset"]["query_count"],
+ "source_commit": result["code"]["source_commit"],
+ "initial_commit": result["code"]["initial_commit"],
+ },
+ indent=2,
+ ),
+ )
+
+
if __name__ == "__main__":
- main(DEFAULT_CONFIG)
+ main()
diff --git a/benchmark/home_runner.py b/benchmark/home_runner.py
deleted file mode 100644
index e4c2a23e..00000000
--- a/benchmark/home_runner.py
+++ /dev/null
@@ -1,93 +0,0 @@
-"""Shared preparation-only entry point for benchmark Home experiments."""
-
-import argparse
-import asyncio
-import json
-from pathlib import Path
-
-from pydantic import BaseModel, ConfigDict, Field
-import yaml
-from reme_home.preparation.spec import PreparationSpec
-from reme_home.judge import JUDGE_PLUGINS
-from reme_home.sampling import SamplingPolicy
-
-from reme.application import Application
-
-PROJECT_ROOT = Path(__file__).resolve().parents[1]
-
-
-class RunHomeConfig(BaseModel):
- """Runner settings kept separate from Home's public preparation contract."""
-
- model_config = ConfigDict(extra="forbid", strict=True)
- workspace_dir: str = Field(min_length=1)
- preparation: dict
- sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
-
-
-def project_path(value: str, project_root: Path) -> Path:
- """Use repository-relative paths consistently regardless of the caller's cwd."""
- if value.startswith("~"):
- raise ValueError("Use absolute or project-relative paths, not home-relative paths")
- return (project_root / value).resolve()
-
-
-def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
- """Resolve source paths and validate explicit Home preparation parameters."""
- raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
- config = RunHomeConfig.model_validate(raw)
- spec = PreparationSpec.model_validate(config.preparation)
- spec.benchmark.source = str(project_path(spec.benchmark.source, project_root))
- if spec.benchmark.evidence_source is not None:
- spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
- spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
- workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
- return workspace, spec.model_dump(mode="json"), config.sampling_policy
-
-
-def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
- """Return resolved preparation inputs, preserving the preparation-only caller contract."""
- workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
- return workspace, spec
-
-
-async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
- """Start Home, prepare once, and close all resources without evaluating cases."""
- workspace, spec, policy = _load_config(config_path, workspace_dir=workspace_dir)
- app = Application(
- workspace_dir=str(workspace),
- plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
- components={"home.evaluation": {"default": {"sampling_policy": policy.model_dump(mode="json")}}},
- service={"backend": "cli"},
- enable_logo=False,
- log_to_file=False,
- )
- try:
- await app.start()
- response = await app.run_job("home_prepare", spec=spec)
- if not response.success:
- raise RuntimeError(response.answer)
- return {"workspace_dir": str(workspace), **response.answer}
- finally:
- await app.close()
-
-
-def main(default_config: Path) -> None:
- """Run a benchmark's default preparation from the command line."""
- parser = argparse.ArgumentParser(description=__doc__)
- parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration")
- parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)")
- args = parser.parse_args()
- result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir))
- print(
- json.dumps(
- {
- "workspace_dir": result["workspace_dir"],
- "case_count": result["dataset"]["case_count"],
- "query_count": result["dataset"]["query_count"],
- "source_commit": result["code"]["source_commit"],
- "initial_commit": result["code"]["initial_commit"],
- },
- indent=2,
- ),
- )
diff --git a/benchmark/longmemeval/README.md b/benchmark/longmemeval/README.md
index 61e02b6a..71467a90 100644
--- a/benchmark/longmemeval/README.md
+++ b/benchmark/longmemeval/README.md
@@ -137,6 +137,27 @@ The top-level `sampling_policy` configures Home evaluation defaults: `max_cases`
`home.evaluation` Component. It still only prepares; it does not run evaluations.
Each evaluation controls `sampling` and `record_history` separately.
+The top-level `validation_cycle_time_budget_minutes` configures the host optimizer's
+advisory time between completed formal evaluations. It defaults to 150 minutes;
+set a positive number to change it or null to disable timed reminders.
+
+The top-level `max_validations` sets the optimizer's formal-evaluation budget
+(positive integer, default 10). A slot is consumed at first sandbox startup,
+including later failures or cancellations; initial baseline and debug are excluded.
+
+After preparation, run local one-shot optimization with the same configuration:
+
+```bash
+python benchmark/longmemeval/run_home.py --optimize
+# Use the same --config and --workspace-dir overrides as preparation, if supplied.
+python benchmark/longmemeval/run_home.py --optimize --instruction "Improve retrieval quality"
+```
+
+This starts a fresh application, applies both budgets and the sampling policy,
+runs `home_optimize`, and closes the application. It does not repeat preparation.
+Python callers can use `await optimize(config_path, workspace_dir=...)` from the
+same module. The default `run()` and CLI behavior remain preparation only.
+
Python callers can use the same lifecycle:
```python
diff --git a/benchmark/longmemeval/README_ZH.md b/benchmark/longmemeval/README_ZH.md
index 85e38bf4..7b93b7ee 100644
--- a/benchmark/longmemeval/README_ZH.md
+++ b/benchmark/longmemeval/README_ZH.md
@@ -125,6 +125,23 @@ Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=.
`home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling`
和 `record_history`。
+顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的
+建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。
+
+顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。
+首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。
+
+准备完成后,用同一份配置进行本地一次性优化:
+
+```bash
+python benchmark/longmemeval/run_home.py --optimize
+```
+
+若准备时指定了 `--config` 或 `--workspace-dir`,优化时传入相同参数。
+可用 `--instruction "优化检索质量"` 指定目标。该入口将配置中的两个预算和采样策略
+应用到实际 `home_optimize` 调用,不重复准备;Python 可调用同模块的 `await optimize(...)`。
+不加 `--optimize` 时仍只准备。
+
默认使用 `dev_meta` 已保存的 **group_1–group_5,共 250 个 case**,
ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchmark.case_ids`,
运行时无需读取分组文件,group_6–group_10 保持为留出集。
diff --git a/benchmark/longmemeval/config_home.yaml b/benchmark/longmemeval/config_home.yaml
index aa887faf..22991232 100644
--- a/benchmark/longmemeval/config_home.yaml
+++ b/benchmark/longmemeval/config_home.yaml
@@ -1,6 +1,9 @@
-# Preparation only. Paths are relative to the ReMe repository root, not the cwd.
+# Preparation by default; --optimize applies these settings to the prepared workspace.
+# Paths are relative to the ReMe repository root, not the cwd.
# Preserve dev_meta's fixed training split; groups 6-10 remain held out.
workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5
+# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug.
+max_validations: 10
# Policy defaults for Home evaluation; each call still controls sampling=true/false.
sampling_policy:
max_cases: 5
diff --git a/benchmark/longmemeval/run_home.py b/benchmark/longmemeval/run_home.py
index b959555e..04fea91f 100644
--- a/benchmark/longmemeval/run_home.py
+++ b/benchmark/longmemeval/run_home.py
@@ -1,22 +1,145 @@
-"""Prepare the longmemeval Home workspace, dataset and candidate code; do not evaluate."""
+"""Prepare the LongMemEval Home workspace, or optimize it with --optimize after preparation."""
-from pathlib import Path
+import argparse
+import asyncio
+import json
import sys
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict, Field
+import yaml
# Support direct execution from any working directory as well as module imports.
-_PROJECT_ROOT = Path(__file__).resolve().parents[2]
-if str(_PROJECT_ROOT) not in sys.path:
- sys.path.insert(0, str(_PROJECT_ROOT))
+PROJECT_ROOT = Path(__file__).resolve().parents[2]
+if str(PROJECT_ROOT) not in sys.path:
+ sys.path.insert(0, str(PROJECT_ROOT))
-from benchmark.home_runner import main, prepare # noqa: E402 # pylint: disable=wrong-import-position
+# pylint: disable=wrong-import-position
+from reme_home.preparation.spec import PreparationSpec # noqa: E402
+from reme_home.judge import JUDGE_PLUGINS # noqa: E402
+from reme_home.sampling import SamplingPolicy # noqa: E402
+
+from reme.application import Application # noqa: E402
+
+# pylint: enable=wrong-import-position
DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml")
+class RunHomeConfig(BaseModel):
+ """Runner settings kept separate from Home's public preparation contract."""
+
+ model_config = ConfigDict(extra="forbid", strict=True)
+ workspace_dir: str = Field(min_length=1)
+ preparation: dict
+ sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy)
+ validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False)
+ max_validations: int = Field(default=10, ge=1)
+
+
+def project_path(value: str, project_root: Path) -> Path:
+ """Use repository-relative paths consistently regardless of the caller's cwd."""
+ if value.startswith("~"):
+ raise ValueError("Use absolute or project-relative paths, not home-relative paths")
+ return (project_root / value).resolve()
+
+
+def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
+ """Resolve source paths and validate explicit Home preparation parameters."""
+ raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8"))
+ config = RunHomeConfig.model_validate(raw)
+ config.preparation.setdefault("candidate", {"paths": ["plugins/lme/src"]})
+ spec = PreparationSpec.model_validate(config.preparation)
+ spec.benchmark.source = str(project_path(spec.benchmark.source, project_root))
+ if spec.benchmark.evidence_source is not None:
+ spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root))
+ spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root))
+ workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root)
+ return workspace, spec.model_dump(mode="json"), config
+
+
+def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT):
+ """Return resolved preparation inputs, preserving the preparation-only caller contract."""
+ workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root)
+ return workspace, spec
+
+
+async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None):
+ """Apply the same runner configuration to each preparation or optimization lifecycle."""
+ workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir)
+ app = Application(
+ workspace_dir=str(workspace),
+ plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]],
+ components={
+ "home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}},
+ "agent_wrapper": {
+ "home_optimizer": {
+ "validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes,
+ "max_validations": config.max_validations,
+ },
+ },
+ },
+ service={"backend": "cli"},
+ enable_logo=False,
+ log_to_file=False,
+ )
+ try:
+ await app.start()
+ if optimize_workspace:
+ arguments = {} if instruction is None else {"instruction": instruction}
+ response = await app.run_job("home_optimize", **arguments)
+ else:
+ response = await app.run_job("home_prepare", spec=spec)
+ if not response.success:
+ raise RuntimeError(response.answer)
+ return {"workspace_dir": str(workspace), **response.answer}
+ finally:
+ await app.close()
+
+
+async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict:
+ """Prepare once without running evaluations or optimization."""
+ return await _run(config_path, workspace_dir=workspace_dir)
+
+
+async def optimize(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir=None, instruction=None) -> dict:
+ """Optimize an already prepared workspace using this config's actual model budgets."""
+ return await _run(config_path, workspace_dir=workspace_dir, optimize_workspace=True, instruction=instruction)
+
+
async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict:
"""Prepare through the same Home Job as the CLI, then close the application."""
return await prepare(config_path, workspace_dir=workspace_dir)
+def main(default_config: Path = DEFAULT_CONFIG) -> None:
+ """Run preparation or explicit local one-shot optimization with the benchmark config."""
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration")
+ parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)")
+ parser.add_argument("--optimize", action="store_true", help="Optimize an already prepared workspace")
+ parser.add_argument("--instruction", help="Optimization focus; requires --optimize")
+ args = parser.parse_args()
+ if args.instruction is not None and not args.optimize:
+ parser.error("--instruction requires --optimize")
+ if args.optimize:
+ result = asyncio.run(optimize(args.config, workspace_dir=args.workspace_dir, instruction=args.instruction))
+ print(json.dumps(result, ensure_ascii=False, indent=2))
+ return
+ result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir))
+ print(
+ json.dumps(
+ {
+ "workspace_dir": result["workspace_dir"],
+ "case_count": result["dataset"]["case_count"],
+ "query_count": result["dataset"]["query_count"],
+ "source_commit": result["code"]["source_commit"],
+ "initial_commit": result["code"]["initial_commit"],
+ },
+ indent=2,
+ ),
+ )
+
+
if __name__ == "__main__":
- main(DEFAULT_CONFIG)
+ main()
diff --git a/plugins/home/README.md b/plugins/home/README.md
index 7f84a3f1..2e06cbc1 100644
--- a/plugins/home/README.md
+++ b/plugins/home/README.md
@@ -838,3 +838,150 @@ reme home_case_study evaluation_path=evaluations/candidate/abcdef0/1 'case_ids=[
Calls with `_allowed_paths` restrictions fail closed, as other Home artifact
Jobs do. Structured inspection tools run in the host with their existing
containment checks; only Bash/Python subprocesses run inside the OS sandbox.
+
+## Diagnostic and optimizer agents
+
+`home_diagnose` and `home_optimize` provide the minimal Meta-ReMe workflow using
+Home's existing Jobs. Enable Home with a benchmark judge and prepare the
+workspace first. Start the configured service in one terminal:
+
+```bash
+reme start config=/absolute/path/to/home.yaml
+```
+
+Call from another terminal:
+
+```bash
+reme home_diagnose 'evaluation_paths=["evaluations/init/abcdef0/1"]' 'instruction=Explain the lowest-scoring cases'
+reme home_optimize 'instruction=Improve evidence retrieval while reducing query tokens'
+```
+
+Diagnosis requires explicit, unique result directories, including failed runs
+with saved evidence. Each call creates a fresh diagnostic agent, delegates deep
+case investigation to `home_case_study`, and returns ranked, evidence-supported
+causes without prescribing fixes. An optimizer invocation automatically injects
+its available caller messages and compression summary; the model cannot supply
+or override that trajectory. Standalone calls record an empty trajectory.
+Reports and trajectories are saved under `home/logs/diagnostic-reports/` and
+`home/logs/diagnostic-trajectories/` with their evidence paths and session IDs.
+
+Optimization reuses a completed full initial baseline covering every prepared
+training case and query, with no infrastructure errors. Otherwise it runs that
+baseline first, requiring clean `init` to be checked out. It never switches away
+from an existing candidate to repair a missing baseline. Baseline observations
+feed sampling history. Candidate execution failures retain Home's existing
+failure thresholds; low scores are valid evidence. Infrastructure failures stop
+baseline startup and are distinguished from candidate defects during iteration.
+
+The optimizer diagnoses, edits declared `candidate_paths`, commits through
+managed Git Jobs, runs debug until execution passes, then evaluates using the
+host's adaptive sampling policy. Formal evaluations always rebuild memory.
+The agent cannot change execution settings, selections, sampling mode, or
+construction reuse through its evaluation tool; only `debug` is model-selected.
+Normal `home_evaluate` callers retain their existing interface and replay support.
+Unchanged candidates and infrastructure retries follow the existing repetition
+rules. There is no admission gate, architecture agent,
+construction-equivalence reviewer, or query-only optimizer policy in this workflow.
+
+Both agents share the case-study OS sandbox implementation. All file tools,
+Bash and Python use it, including file metadata and Edit/Write checks. Evidence,
+fixed runtime and Git metadata are read-only. Each call receives a private
+scratch directory under `home/tmp/`; only the optimizer additionally receives
+writes to declared candidate roots. Network is denied. Diagnostic writes are
+limited to scratch. Host Jobs alone write reports, evaluation artifacts and Git
+metadata. Sandbox startup is probed before model calls and never falls back to
+unrestricted execution. Declared write roots must exist and remain real paths.
+Only one optimizer can run per Home workspace; cancellation releases its lock
+after nested tool cleanup. Separate diagnostic calls and case studies may run
+concurrently with fresh state.
+
+The final optimizer response must identify `branch_name`, full `commit_sha`,
+`evaluation_path` and `report`. The host verifies a clean checkout at that exact
+branch/commit and a matching completed, scored, end-to-end evaluation without
+infrastructure failures. Sampled formal evidence qualifies; debug or query-only
+evidence does not. Keeping the initial baseline is valid. Invalid final claims
+fail while preserving changes and the raw report in
+`home/logs/optimization-reports/`. The host verifies provenance rather than
+ranking candidates by a single average score.
+
+Dedicated `as_llm` and `agent_wrapper` components are named `home_optimizer` and
+`home_diagnostic`; override them through ordinary application configuration.
+Both default to `qwen3.8-max`, with context sizes 160,000 and 128,000, respectively.
+The optimizer has 3,000 iterations and a 150-minute advisory validation-cycle timer;
+configure `components.agent_wrapper.home_optimizer.validation_cycle_time_budget_minutes`
+(null disables timed reminders). The previous `time_budget_minutes` setting is a
+fallback when the cycle setting is absent. Diagnosis retains 80 iterations and
+a 60-minute whole-invocation timer. Neither timer enforces a time limit.
+
+The optimizer follows dev_meta b53048e4: before reasoning it checks 50%, 75%, 90%,
+then 100% and each additional 10% of the cycle budget. Reminders include elapsed
+time, budget percentage, completed context compressions and guidance to finish
+editing, commit, pass the smoke test and begin validation. Temporary and durable
+reminders alternate, beginning with temporary. The cycle includes nested diagnosis,
+debug and evaluation time. Only a new counted formal evaluation returning
+`success` or `completed` resets the clock, thresholds, alternation and compression
+count; scores and candidate failures do not control reset. As in dev_meta, a
+completed result can reset even when its case results contain infrastructure errors;
+an incomplete failure, rejected call, cancellation, debug or pre-sandbox failure
+cannot. The initial full baseline precedes the first optimizer cycle and is excluded
+from its formal-validation numbering. A durable hint records the completed
+validation number and commit prefix, expires that cycle's old timer reminders and
+announces the next cycle with the remaining formal-validation count.
+Persisted old reminders are retained as in dev_meta.
+Hints use AgentScope's current reply message so accumulated token usage is retained.
+Evaluation tools queue transitions in request state; the optimizer injects them
+before the next reasoning call, after all tool results have been written. Thus a
+transition never separates an API tool call from its matching result.
+
+`components.agent_wrapper.home_optimizer.max_validations` defaults to 10, matching
+dev_meta. `get_validation_budget` reports consumed and remaining slots, the search
+phase and next-candidate guidance. A non-init formal run consumes one slot when
+its first sandbox initializes successfully, including runs that later fail or are
+cancelled. Initial baselines, debug, rejected calls and pre-start failures are free.
+Accounting reuses persistent started-run evidence and survives optimizer restarts;
+removing the rebuildable sampling cache does not restore the budget. The evaluator
+checks the limit while holding the formal-validation lock, before starting another
+run. At exhaustion it returns `VALIDATION_BUDGET_EXHAUSTED`; the optimizer selects
+from existing evidence. The first `max_validations // 2` starts favor structural
+exploration; later starts favor evidence-driven refinement. This guidance is advisory.
+Ordinary `home_evaluate` calls remain unbounded unless the host supplies
+`max_validations`; all recorded non-init formal starts in that workspace count.
+
+Sessions and model usage follow ReMe's wrapper conventions; each nested call reports its own usage.
+The optimizer follows dev_meta's compression recovery policy. Recoverable model
+or summary-output failures leave context intact below the emergency ratio
+`(trigger_ratio + 1) / 2`. At or above that ratio (90% for an 80% trigger), old
+messages are atomically saved under `logs/context-offloads/`; recent context and
+the existing summary are retained, with a reminder pointing to the readable JSON
+file. This fallback makes no additional model call and counts as one completed
+compression. Cancellation, invalid configuration and uncompressible system prompts
+still propagate during normal operation. Diagnostic and case-study agents retain
+their existing compression behavior. Bounded draft notebook tools are not included.
+
+Host-only execution settings belong in the optimizer Step; model, iteration and
+timer settings belong in its wrapper. For example:
+
+```yaml
+jobs:
+ home_optimize:
+ steps:
+ - backend: home_optimize_step
+ agent_wrapper: home_optimizer
+ evaluation_options:
+ concurrency: 4
+ timeout_seconds: 7200
+ max_construction_failure_rate: 0.03
+ # memory_limit_mib: 0
+components:
+ agent_wrapper:
+ home_optimizer:
+ react_config:
+ max_iters: 3000
+ validation_cycle_time_budget_minutes: 150
+ max_validations: 10
+```
+
+`evaluation_options` also accepts `base_image`, `fail_fast` and
+`max_artifact_bytes`. Omit `fail_fast` to retain Home's debug/scored defaults.
+Restricted `_allowed_paths` calls fail closed. No real model or Docker benchmark
+is required by the isolated workflow tests.
diff --git a/plugins/home/src/reme_home/agent/bounded.py b/plugins/home/src/reme_home/agent/bounded.py
index d89df5a9..7db6a5a9 100644
--- a/plugins/home/src/reme_home/agent/bounded.py
+++ b/plugins/home/src/reme_home/agent/bounded.py
@@ -116,7 +116,8 @@ def _format_minutes(minutes: float) -> str:
"""Format a timer duration compactly."""
if minutes < 1:
- return f"{max(0, round(minutes * 60))} seconds"
+ seconds = max(0, round(minutes * 60))
+ return f"{seconds} {'second' if seconds == 1 else 'seconds'}"
rounded = round(minutes, 1)
value = int(rounded) if rounded.is_integer() else rounded
unit = "minute" if value == 1 else "minutes"
diff --git a/plugins/home/src/reme_home/agent/compression.py b/plugins/home/src/reme_home/agent/compression.py
new file mode 100644
index 00000000..0241611e
--- /dev/null
+++ b/plugins/home/src/reme_home/agent/compression.py
@@ -0,0 +1,126 @@
+"""Meta-ReMe compression recovery adapted to AgentScope's native context pipeline."""
+
+import asyncio
+import logging
+from pathlib import Path
+from uuid import uuid4
+
+from agentscope.exception import ToolJSONDecodeError
+from agentscope.message import TextBlock
+import jsonschema
+import openai
+from pydantic import ValidationError
+
+from ..evaluation.artifacts import contained, write_json
+from .bounded import BoundedReActAgent
+
+logger = logging.getLogger(__name__)
+
+
+class RecoverableCompressionAgent(BoundedReActAgent):
+ """Skip recoverable failures until emergency offload is needed.
+
+ Override the native implementation so compression middleware, successful-change
+ counters, finalization and cancellation remain owned by their existing layers.
+ """
+
+ def __init__(self, *args, fallback_directory=None, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.fallback_directory = Path(fallback_directory) if fallback_directory is not None else None
+
+ async def _compress_context_impl(self, context_config=None, instructions=None):
+ """Keep the old trigger-derived emergency boundary without silent truncation."""
+ config = context_config or self.context_config
+ model_input = await self._prepare_model_input()
+ context_ratio = await self.model.count_tokens(**model_input) / self.model.context_size
+ if context_ratio < config.trigger_ratio:
+ return
+ # The SDK's default truncation would hide the failure and discard old
+ # evidence before Home can decide whether emergency offload is needed.
+ config = config.model_copy(update={"compression_fallback_to_truncation": False})
+ try:
+ await super()._compress_context_impl(context_config=config, instructions=instructions)
+ except Exception as exc:
+ if not self._is_recoverable_compression_error(exc):
+ raise
+ emergency_ratio = (config.trigger_ratio + 1.0) / 2.0
+ if context_ratio < emergency_ratio:
+ logger.warning(
+ "Context compression failed at %.1f%%, below the %.1f%% emergency threshold; skipping: %s: %s",
+ context_ratio * 100,
+ emergency_ratio * 100,
+ type(exc).__name__,
+ exc,
+ )
+ return
+ logger.warning(
+ "Context compression failed at %.1f%%; using local offload at the %.1f%% emergency threshold: %s: %s",
+ context_ratio * 100,
+ emergency_ratio * 100,
+ type(exc).__name__,
+ exc,
+ )
+ await self._fallback_compress_context(exc, context_ratio, config)
+
+ @staticmethod
+ def _is_recoverable_compression_error(exc):
+ """Match dev_meta's model/output failures, excluding uncompressible prompts."""
+ if isinstance(exc, RuntimeError):
+ return not str(exc).startswith("The system prompt")
+ return isinstance(exc, (ToolJSONDecodeError, jsonschema.ValidationError, ValidationError, openai.OpenAIError))
+
+ async def _fallback_compress_context(self, cause, context_ratio, config):
+ """Persist older messages before replacing state; never call a second model."""
+ if self.fallback_directory is None:
+ raise RuntimeError("Optimizer fallback_directory is required for local context compression") from cause
+ model_input = await self._prepare_model_input()
+ old_messages, recent_messages = await self._split_context_for_compression(
+ config.reserve_ratio * self.model.context_size,
+ model_input.get("tools", []),
+ )
+ if not old_messages:
+ raise RuntimeError(
+ "Local fallback compression cannot remove any context while preserving the configured reserve",
+ ) from cause
+
+ async def apply_change():
+ contained(self.fallback_directory.parent, self.fallback_directory, directory=True)
+ path = contained(self.fallback_directory, f"context-{uuid4().hex}.json")
+ write_json(
+ path,
+ {
+ "session_id": self.state.session_id,
+ "context_ratio": context_ratio,
+ "trigger_ratio": config.trigger_ratio,
+ "emergency_ratio": (config.trigger_ratio + 1.0) / 2.0,
+ "failure": {"type": type(cause).__name__, "message": str(cause)},
+ "messages": [message.model_dump(mode="json", exclude_none=False) for message in old_messages],
+ },
+ )
+ notice = (
+ "Model-based context compression failed near the context limit. "
+ f"Older context was offloaded without semantic summarization to '{path}'. "
+ "The recent context is preserved; use Read on that file if older experiments, decisions, "
+ "validation results, or tool outputs are needed."
+ )
+ await self._clear_unreserved_read_cache(recent_messages)
+ # Recent SDK versions also carry current-reply usage through compression.
+ usage = self._get_reply_usage()
+ summary = self.state.summary
+ self.state.summary = (
+ "\n".join(part for part in (summary, notice) if part)
+ if isinstance(summary, str)
+ else [*summary, TextBlock(text=notice)]
+ )
+ self.state.context = recent_messages
+ if usage is not None and self._get_reply_usage() is None:
+ self.state.append_context(self.name, [])
+ self.state.context[-1].usage = usage
+ logger.warning("Local compression offloaded %d message(s) to %s", len(old_messages), path)
+
+ task = asyncio.create_task(apply_change())
+ try:
+ await asyncio.shield(task)
+ except asyncio.CancelledError:
+ await task
+ raise
diff --git a/plugins/home/src/reme_home/agent/diagnostic.py b/plugins/home/src/reme_home/agent/diagnostic.py
new file mode 100644
index 00000000..3c184303
--- /dev/null
+++ b/plugins/home/src/reme_home/agent/diagnostic.py
@@ -0,0 +1,159 @@
+"""Diagnose saved Home evaluations with an isolated evidence-driven agent."""
+
+import asyncio
+import copy
+import json
+
+from agentscope.state import AgentState
+from agentscope.tool import FunctionTool
+from pydantic import Field, field_validator
+
+from reme.steps.base_step import BaseStep, Ref
+
+from ..evaluation.artifacts import contained, read_json, write_json
+from ..evaluation.component import blocking
+from ..preparation.spec import SpecModel
+from ..workspace import HomeWorkspace
+from .workflow_common import required_job, save_record, workflow_reply
+from .wrapper import BoundedAgentWrapper
+
+
+class DiagnosticInput(SpecModel):
+ """Require explicit evaluation evidence; caller state is never model-selected."""
+
+ evaluation_paths: list[str] = Field(min_length=1)
+ instruction: str = Field(default="Identify and rank the evidence-supported root causes.", min_length=1)
+
+ @field_validator("evaluation_paths")
+ @classmethod
+ def validate_paths(cls, values):
+ """Reject empty and duplicate input references before creating an agent."""
+ if any(not value.strip() for value in values) or len(set(values)) != len(values):
+ raise ValueError("evaluation_paths must contain unique nonempty paths")
+ return values
+
+
+def evaluation_paths(home, values):
+ """Require explicit, unique local runs with actual evidence, including failed runs."""
+ if not values:
+ raise ValueError("evaluation_paths must identify at least one run")
+ paths = []
+ root = contained(home, "evaluations", directory=True)
+ for value in values:
+ path = contained(home, value, directory=True)
+ if not path.is_relative_to(root) or path == root:
+ raise ValueError("evaluation_paths must be inside home/evaluations")
+ read_json(contained(path, "manifest.json"))
+ evidence = [path / "summary.json", path / "failure.json"]
+ evidence += list(path.glob("cases/*/memory_construction/result.json"))
+ evidence += list(path.glob("cases/*/queries/result.json"))
+ if not any(contained(path, item).is_file() for item in evidence):
+ raise ValueError(f"Evaluation contains no results: {path}")
+ paths.append(str(path))
+ if len(paths) != len(set(paths)):
+ raise ValueError("evaluation_paths must be unique")
+ return paths
+
+
+def trajectory(state):
+ """Serialize only caller messages and compression summary, never credentials or permissions."""
+ if state is not None and not isinstance(state, AgentState):
+ raise TypeError("Injected caller state must be an AgentState")
+ if state is None:
+ return {"messages": [], "summary": None}
+ summary = state.summary
+ if isinstance(summary, list):
+ summary = [block.model_dump(mode="json") for block in summary]
+ return {
+ "session_id": state.session_id,
+ "messages": [message.model_dump(mode="json") for message in state.context],
+ "summary": summary,
+ }
+
+
+def diagnostic_tool(wrapper):
+ """Inject the optimizer's real trajectory into a fresh diagnostic invocation."""
+ job = required_job(wrapper, "home_diagnose")
+
+ async def diagnose(
+ evaluation_paths: list[str], # pylint: disable=redefined-outer-name
+ instruction: str = "Identify and rank the evidence-supported root causes.",
+ **kwargs,
+ ):
+ if set(kwargs) - {"_agent_state"}:
+ raise ValueError("Caller trajectory is host-owned and cannot be supplied by the model")
+ response = await job(
+ instruction=instruction,
+ evaluation_paths=evaluation_paths,
+ caller_trajectory=trajectory(kwargs.get("_agent_state")),
+ )
+ return {"success": response.success, "answer": response.answer, "metadata": response.metadata}
+
+ tool = FunctionTool(
+ diagnose,
+ name="home_diagnose",
+ description=job.description,
+ is_concurrency_safe=False,
+ is_state_injected=True,
+ )
+ tool.input_schema = copy.deepcopy(job.parameters)
+ return tool
+
+
+class HomeDiagnoseStep(BaseStep):
+ """Aggregate explicit evaluation evidence through a read-only diagnostic agent."""
+
+ workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
+ agent_wrapper: BoundedAgentWrapper = Ref(BoundedAgentWrapper, "agent_wrapper")
+
+ async def execute(self):
+ """Validate evidence, save the host trajectory and persist a provenance-bearing report."""
+ if self.context.get("_allowed_paths") is not None:
+ raise ValueError("Home diagnosis requires access to its managed evidence")
+ if not self.workspace.is_started or not self.agent_wrapper.is_started:
+ raise RuntimeError("Start the Home workspace and diagnostic wrapper first")
+ spec = DiagnosticInput.model_validate(
+ {key: self.context[key] for key in DiagnosticInput.model_fields if key in self.context},
+ )
+ home = self.workspace.home_path
+ paths = await blocking(evaluation_paths, home, spec.evaluation_paths)
+ caller = self.context.get("caller_trajectory", trajectory(None))
+ if not isinstance(caller, dict):
+ raise ValueError("Caller trajectory must be a host-supplied object")
+ source = await blocking(save_record, home, "diagnostic-trajectories", caller)
+ payload = {
+ "status": "running",
+ "instruction": spec.instruction,
+ "evaluation_paths": paths,
+ "trajectory_path": source,
+ }
+ report_path = await blocking(save_record, home, "diagnostic-reports", payload)
+ self.context.response.metadata.update(diagnostic_report_path=report_path, trajectory_path=source)
+ try:
+ result = await workflow_reply(
+ self.agent_wrapper,
+ home,
+ "diagnostic",
+ f"Diagnostic request:\n{spec.instruction}\n\nComplete available caller trajectory: {source}\n"
+ f"Explicit evaluation paths: {json.dumps(paths, ensure_ascii=False)}",
+ job_names=("home_case_study",),
+ memory_limit_mib=self.kwargs.get("memory_limit_mib"),
+ )
+ if not isinstance(result.get("result"), str) or not result["result"].strip():
+ raise ValueError("Diagnostic agent returned no report")
+ payload.update(
+ status="completed",
+ diagnosis=result["result"],
+ session_id=result.get("session_id"),
+ usage=result.get("usage"),
+ )
+ self.context.response.answer = {**payload, "diagnostic_report_path": report_path}
+ self.context.response.metadata.update(
+ {key: result[key] for key in ("session_id", "usage") if key in result},
+ )
+ except BaseException as exc:
+ payload.update(status="cancelled" if isinstance(exc, asyncio.CancelledError) else "failed", error=str(exc))
+ raise
+ finally:
+ await blocking(write_json, contained(home, report_path), payload)
+ return self.context.response
diff --git a/plugins/home/src/reme_home/agent/diagnostic.yaml b/plugins/home/src/reme_home/agent/diagnostic.yaml
new file mode 100644
index 00000000..16b7b143
--- /dev/null
+++ b/plugins/home/src/reme_home/agent/diagnostic.yaml
@@ -0,0 +1,32 @@
+system_prompt: |-
+ You are Home's read-only code diagnostician, migrated from Meta-ReMe.
+ Identify and rank evidence-supported mechanisms that caused evaluation failures or limited candidate quality.
+ The candidate repository is {code_repository}; guides are under {docs}; private scratch is {scratch}.
+ The caller provides explicit evaluation paths and its available complete trajectory, including any compression summary.
+
+ Produce one concise report organized as Priority scope, Failure clusters, Ranked shared root causes, Outliers,
+ and Missing evidence. State which low-scoring cases and question types you prioritized and what you deferred.
+ For each cause, name affected cases and queries, lifecycle stage, causal chain, supporting and contradicting
+ evidence, confidence, and the smallest missing observation. Separate fixed-runtime limitations.
+
+ Read architecture-guide.md and validation-artifact-guide.md when needed. Confirm each run's exact commit,
+ status, score contract and coverage. Compare runs only where contracts and case/query coverage permit it.
+ Prioritize the lowest-scoring cases and question types; you need not explain every failure.
+ Use home_case_study batches early as the default for deep case investigation, even for one case. Pass one
+ evaluation_path, case_ids and a shared focus; do not put case-specific instructions in the shared focus.
+ Use its reports to identify shared causes, then inspect only remaining cross-case or code-level evidence.
+ Direct investigation is appropriate for quick lookups, aggregate comparisons and specific evidence checks.
+
+ Classify infrastructure, candidate-execution, answer-quality and judging failures separately. Trace extraction,
+ persistence, overwrite, indexing, retrieval, ranking, context presentation or reasoning as evidence requires.
+ Treat partial_success construction as requiring repair. Do not replace persisted runtime failure statuses.
+ Original sessions and derived memory are both evidence; absence from summaries does not prove unavailability.
+ Prefer independent evidence_session_ids; inferred evidence is a lead, not proof. Explicitly state uncertainty.
+ Use home_list_validation_results and home_inspect_validation to locate evidence. Pin revision to the evaluated
+ full commit when using home_list_code_tree, home_search_code and home_read_code_file. The current checkout may
+ differ. Read artifacts and runtime docs with Read, Grep, Glob, AutoBash or python_execute, using absolute paths.
+ Keep stderr visible; denied reads are not proof of missing content.
+
+ The OS sandbox allows evidence reads and writes only to private scratch. You may create disposable scripts
+ and analysis files there. Do not modify candidate code, datasets, evaluations, reports or Git metadata.
+ Do not run ReMe, evaluations or network operations. Do not implement or prescribe fixes; report supported causes.
diff --git a/plugins/home/src/reme_home/agent/optimizer.py b/plugins/home/src/reme_home/agent/optimizer.py
new file mode 100644
index 00000000..6d3c9fb5
--- /dev/null
+++ b/plugins/home/src/reme_home/agent/optimizer.py
@@ -0,0 +1,513 @@
+"""Optimize Home candidates and verify their baseline and final evaluation evidence."""
+
+import asyncio
+import json
+import math
+import time
+from pathlib import Path
+from typing import Callable
+
+from agentscope.message import HintBlock
+from agentscope.state import AgentState
+from agentscope.tool import Edit, FunctionTool
+from pydantic import Field
+
+from reme.steps.base_step import BaseStep, Ref
+
+from ..code.repository import validate_repository, require_clean, validate_tree, status
+from ..evaluation.artifacts import contained, read_json, write_json
+from ..evaluation.component import blocking
+from ..evaluation.inputs import EvaluationSpec, load_inputs
+from ..evaluation.history import read_state
+from ..evaluation.budget import validate_max_validations, validation_budget_status
+from ..preparation.spec import SpecModel
+from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
+from .diagnostic import diagnostic_tool, evaluation_paths
+from .bounded import TimeBudgetReminder, _format_minutes, _remove_hint_by_identity
+from .compression import RecoverableCompressionAgent
+from .workflow_common import required_job, save_record, workflow_reply
+from .wrapper import CaseStudyAgentWrapper
+
+MUTATION_JOBS = (
+ "home_create_candidate_branch",
+ "home_switch_candidate_branch",
+ "home_merge_candidate_branch",
+ "home_commit_candidate",
+)
+PENDING_CYCLE_TRANSITIONS = "home.optimizer.pending_cycle_transitions"
+
+
+class ValidationCycleTimer:
+ """Emit escalating reminders as a scored-validation cycle ages."""
+
+ def __init__(self, time_budget_minutes: float, *, clock: Callable[[], float] = time.monotonic) -> None:
+ if (
+ not isinstance(time_budget_minutes, (int, float))
+ or isinstance(time_budget_minutes, bool)
+ or not math.isfinite(time_budget_minutes)
+ or time_budget_minutes <= 0
+ ):
+ raise ValueError("time_budget_minutes must be a positive number")
+ self.time_budget_minutes = float(time_budget_minutes)
+ self._clock = clock
+ self._started_at = self._clock()
+ self._last_reminded_ratio = 0.0
+ self._next_reminder_is_persistent = False
+ self.context_compression_count = 0
+
+ def reset(self) -> None:
+ """Start a fresh cycle after a scored validation finishes."""
+
+ self._started_at = self._clock()
+ self._last_reminded_ratio = 0.0
+ self._next_reminder_is_persistent = False
+ self.context_compression_count = 0
+
+ def record_context_compression(self) -> None:
+ """Record one completed compression since the last scored validation."""
+
+ self.context_compression_count += 1
+
+ def next_reminder(self) -> TimeBudgetReminder | None:
+ """Return one reminder for the latest newly reached deadline."""
+
+ elapsed_minutes = max(0.0, (self._clock() - self._started_at) / 60.0)
+ elapsed_ratio = elapsed_minutes / self.time_budget_minutes
+ reached_ratio: float | None = None
+ if elapsed_ratio >= 1.0:
+ reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10
+ else:
+ for threshold in (0.9, 0.75, 0.5):
+ if elapsed_ratio >= threshold:
+ reached_ratio = threshold
+ break
+ if reached_ratio is None or reached_ratio <= self._last_reminded_ratio:
+ return None
+ self._last_reminded_ratio = reached_ratio
+ persistent = self._next_reminder_is_persistent
+ self._next_reminder_is_persistent = not persistent
+ elapsed = _format_minutes(elapsed_minutes)
+ percentage = round(reached_ratio * 100)
+ timing = (
+ f"The optimizer agent run has used {elapsed} since the last scored validation "
+ f"({percentage}% of its {_format_minutes(self.time_budget_minutes)} soft time budget). "
+ )
+ compression = (
+ f"It has completed {self.context_compression_count} context compressions since the last "
+ "scored validation. "
+ )
+ if elapsed_ratio >= 1.0:
+ return TimeBudgetReminder(
+ content=(
+ f"{timing}{compression}"
+ "Prioritize completing the code changes, committing them, passing the smoke test, and starting "
+ "validation as soon as possible."
+ ),
+ persistent=persistent,
+ )
+ remaining = _format_minutes(self.time_budget_minutes - elapsed_minutes)
+ return TimeBudgetReminder(
+ content=(
+ f"{timing}{compression}"
+ f"Complete the code changes, commit them, pass the smoke test, and start validation within {remaining}."
+ ""
+ ),
+ persistent=persistent,
+ )
+
+
+class OptimizerAgent(RecoverableCompressionAgent):
+ """Add Meta-ReMe's per-validation reminders to Home's bounded agent."""
+
+ def __init__(self, *args, validation_cycle_timer=None, **kwargs):
+ super().__init__(*args, **kwargs)
+ self.validation_cycle_timer = validation_cycle_timer
+
+ async def _reasoning(self, *args, **kwargs):
+ """Deliver hints after tool results without splitting the SDK's reply message."""
+ transitions = self.state.middle_context.pop(PENDING_CYCLE_TRANSITIONS, [])
+ for content in transitions:
+ self.state.append_context(
+ self.name,
+ [HintBlock(source="meta_reme_validation_cycle_transition", hint=content)],
+ )
+ timer = self.validation_cycle_timer
+ reminder = timer.next_reminder() if timer is not None else None
+ message = None
+ if reminder is not None:
+ message = HintBlock(source="meta_reme_validation_timer", hint=reminder.content)
+ self.state.append_context(self.name, [message])
+ try:
+ async for item in super()._reasoning(*args, **kwargs):
+ yield item
+ finally:
+ if reminder is not None and not reminder.persistent:
+ _remove_hint_by_identity(self.state.context, message)
+
+ async def compress_context(self, *args, **kwargs):
+ """Count completed context replacements in the current validation cycle."""
+ summary_before, context_before = self.state.summary, self.state.context
+ await super().compress_context(*args, **kwargs)
+ if self.validation_cycle_timer is not None and (
+ self.state.summary is not summary_before or self.state.context is not context_before
+ ):
+ self.validation_cycle_timer.record_context_compression()
+
+
+class OptimizerAgentWrapper(CaseStudyAgentWrapper):
+ """Keep each optimizer's tools and cycle timer bound to one reply invocation."""
+
+ def create_cycle_timer(self, options=None):
+ """Use the legacy time setting as a fallback; explicit null disables reminders."""
+ options = self._merged_kwargs(options or {})
+ budget = options.get("validation_cycle_time_budget_minutes", options.get("time_budget_minutes"))
+ return None if budget is None else ValidationCycleTimer(budget)
+
+ def validation_limit(self):
+ """Keep the formal count limit host-owned and independent of elapsed time."""
+ return validate_max_validations(self._merged_kwargs({}).get("max_validations", 10))
+
+ def _create_agent(self, *, options, **kwargs):
+ timer = (
+ options["_validation_cycle_timer"]
+ if "_validation_cycle_timer" in options
+ else self.create_cycle_timer(options)
+ )
+ return OptimizerAgent(
+ validation_cycle_timer=timer,
+ fallback_directory=options.get("_context_fallback_directory", self.session_path / "context-offloads"),
+ **kwargs,
+ )
+
+
+def validation_cycle_transition(manifest, validation_number, remaining):
+ """Expire old reminders and announce the next cycle with its remaining budget."""
+ commit = manifest["commit_sha"][:7]
+ return HintBlock(
+ source="meta_reme_validation_cycle_transition",
+ hint=(
+ f"Formal validation {validation_number} completed successfully for commit {commit}. "
+ f"The previous cycle timer has been reset, and all timer reminders for cycle {validation_number} "
+ f"are now expired. Remaining formal validations: {remaining}. Start cycle {validation_number + 1}."
+ ""
+ ),
+ )
+
+
+def completed_validation(home, result, started_before):
+ """Match dev_meta's new-count plus completed-status boundary, independent of scores."""
+ if not isinstance(result, dict) or result.get("status") not in {"success", "completed"}:
+ return None
+ run = contained(home, result["result_dir"], directory=True)
+ key = run.relative_to(contained(home, "evaluations", directory=True)).as_posix()
+ started = read_state(home)["started_validations"]
+ # The initial full baseline precedes optimizer creation in dev_meta and
+ # never consumes its validation budget. Home records it in sampling history.
+ if key in started_before or key not in started or key.startswith("init/"):
+ return None
+ manifest = read_json(contained(run, "manifest.json"))
+ if (
+ manifest.get("status") != "completed"
+ or manifest.get("mode") != "scored"
+ or manifest.get("dataset_kind") != "training-set"
+ or not manifest.get("sandbox_started_at")
+ ):
+ return None
+ return manifest, sum(not name.startswith("init/") for name in started)
+
+
+class OptimizationInput(SpecModel):
+ """The host owns runtime settings; the caller supplies only the optimization focus."""
+
+ instruction: str = Field(default="Improve the candidate using the completed baseline evaluation.", min_length=1)
+
+
+class OptimizationResult(SpecModel):
+ """An agent claim that must be checked against Git and saved formal evidence."""
+
+ branch_name: str = Field(min_length=1)
+ commit_sha: str = Field(pattern="^[0-9a-f]{40}$")
+ evaluation_path: str = Field(min_length=1)
+ report: str = Field(min_length=1)
+
+
+def parse_result(text):
+ """Accept a JSON object, optionally enclosed in one Markdown JSON fence."""
+ text = text.strip()
+ if text.startswith("```") and text.endswith("```"):
+ text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip()
+ return OptimizationResult.model_validate(json.loads(text))
+
+
+EVALUATION_OPTIONS = {
+ "concurrency",
+ "timeout_seconds",
+ "base_image",
+ "fail_fast",
+ "max_construction_failure_rate",
+ "max_artifact_bytes",
+}
+
+
+def evaluation_options(raw):
+ """Validate host-only execution settings without exposing selection or replay."""
+ if not isinstance(raw, dict) or set(raw) - EVALUATION_OPTIONS:
+ raise ValueError("evaluation_options may contain only Home execution settings")
+ return EvaluationSpec.model_validate(raw).model_dump(exclude_unset=True)
+
+
+def verify_evaluation(home, path, commit, *, baseline=False):
+ """Require completed end-to-end evidence with no infrastructure failures."""
+ run = Path(evaluation_paths(home, [str(path)])[0])
+ manifest = read_json(contained(run, "manifest.json"))
+ summary = read_json(contained(run, "summary.json"))
+ required = {
+ "commit_sha": commit,
+ "status": "completed",
+ "success": True,
+ "mode": "scored",
+ "dataset_kind": "training-set",
+ "execution_mode": "full_validation",
+ }
+ if any(manifest.get(key) != value for key, value in required.items()):
+ raise ValueError("Evaluation must match the commit and complete without infrastructure failures")
+ if (
+ not manifest.get("sandbox_started_at")
+ or not summary.get("case_count")
+ or summary.get("scoring_enabled") is not True
+ or any(summary.get(key, {}).get("infra_error") != 0 for key in ("case_status_counts", "query_status_counts"))
+ ):
+ raise ValueError("Evaluation must match the commit and complete without infrastructure failures")
+ _record, dataset, cases = load_inputs(home, EvaluationSpec())
+ if manifest.get("fingerprints", {}).get("dataset") != dataset.normalized_fingerprint:
+ raise ValueError("Evaluation dataset no longer matches the prepared dataset")
+ selected = manifest.get("case_ids", [])
+ queries = manifest.get("query_ids", {})
+ expected = {case.case_id: {q.query_id for q in case.queries} for case in cases}
+ if (
+ not selected
+ or len(set(selected)) != len(selected)
+ or set(selected) - expected.keys()
+ or set(queries) != set(selected)
+ or any(not ids or len(set(ids)) != len(ids) or set(ids) - expected[key] for key, ids in queries.items())
+ ):
+ raise ValueError("Evaluation has an invalid case/query selection")
+ if baseline and (set(selected) != set(expected) or any(set(queries[key]) != ids for key, ids in expected.items())):
+ raise ValueError("Initial baseline must cover every prepared training case and query")
+ rows = summary.get("cases", {})
+ if (
+ set(rows) != set(selected)
+ or summary["case_count"] != len(selected)
+ or summary.get("expected_query_count") != sum(len(ids) for ids in queries.values())
+ ):
+ raise ValueError("Evaluation summary has incomplete case coverage")
+ # Failed construction can legitimately omit answers; the recorded candidate
+ # failure is evidence, not an infrastructure failure or a missing result.
+ for key, row in rows.items():
+ if set(row.get("queries", {})) != set(queries[key]) and row.get("construction", {}).get("status") != "failure":
+ raise ValueError(f"Evaluation summary has incomplete query coverage: {key}")
+ return {"evaluation_path": str(run), "manifest": manifest, "summary": summary}
+
+
+def baseline_state(home):
+ """Inspect a clean repository and locate a usable initial full evaluation."""
+ _validate_directory(home, home / "metadata")
+ with _initialization_lock(home / "metadata/workspace.lock"):
+ repo, record = validate_repository(home)
+ require_clean(repo)
+ validate_tree(repo, record, "HEAD")
+ initial = record["code"]["initial_commit"]
+ current = status(repo, record)
+ for manifest in sorted((home / "evaluations/init").glob("*/*/manifest.json"), reverse=True):
+ try:
+ evidence = verify_evaluation(home, manifest.parent, initial, baseline=True)
+ return current, evidence
+ except (ValueError, OSError, KeyError, TypeError):
+ continue
+ if current["current_branch"] != "init" or current["commit_sha"] != initial:
+ raise ValueError("A complete initial baseline is missing; check out clean init before home_optimize")
+ return current, None
+
+
+def verify_final(home, result):
+ """Check the actual checkout and evidence; never repair or discard agent edits."""
+ _validate_directory(home, home / "metadata")
+ with _initialization_lock(home / "metadata/workspace.lock"):
+ repo, record = validate_repository(home)
+ require_clean(repo)
+ validate_tree(repo, record, "HEAD")
+ current = status(repo, record)
+ if current["current_branch"] != result.branch_name or current["commit_sha"] != result.commit_sha:
+ raise ValueError("Final branch/commit does not match the checked-out HEAD")
+ evidence = verify_evaluation(home, result.evaluation_path, result.commit_sha)
+ if evidence["manifest"]["branch_name"] != result.branch_name:
+ raise ValueError("Final evaluation belongs to a different branch")
+ return evidence["evaluation_path"]
+
+
+def validation_budget_tool(home, max_validations):
+ """Expose dev_meta's read-only count, phase and next-candidate guidance."""
+ validate_max_validations(max_validations)
+
+ async def get_validation_budget():
+ """Return started formal validations, remaining count and optimization guidance."""
+ return await blocking(validation_budget_status, home, max_validations)
+
+ return FunctionTool(
+ get_validation_budget,
+ name="get_validation_budget",
+ description=(
+ "Report how many formal optimizer validations have started at least one sandbox, how many remain, "
+ "and whether the next candidate should emphasize structural code changes or evidence-driven refinement. "
+ "Debug-case smoke validations, the initial baseline and failures before sandbox startup are excluded."
+ ),
+ is_read_only=True,
+ )
+
+
+def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10):
+ """Only debug mode is model-selected; all formal runs rebuild construction."""
+ job = required_job(wrapper, "home_evaluate")
+ validate_max_validations(max_validations)
+
+ async def evaluate(debug: bool = False, **kwargs):
+ """Run debug after committing; after debug passes, run a sampled formal evaluation.
+
+ Debug checks execution, not answer quality. Inspect infrastructure failures
+ separately; retries keep the existing commit and selection recovery rules.
+ """
+ if not isinstance(debug, bool):
+ raise ValueError("debug must be boolean")
+ if set(kwargs) - {"_agent_state"}:
+ raise ValueError("Optimizer state is host-owned and cannot be supplied by the model")
+ state = kwargs.get("_agent_state")
+ if not isinstance(state, AgentState):
+ raise TypeError("Cycle reminders require the host-injected AgentState")
+ started_before = await blocking(lambda: set(read_state(home)["started_validations"])) if not debug else set()
+ response = await job(
+ **options,
+ **({"max_validations": max_validations} if not debug else {}),
+ debug=debug,
+ sampling=not debug,
+ record_history=True,
+ case_ids=None,
+ query_ids=None,
+ reuse_construction_from=None,
+ )
+ if not debug:
+ budget = await blocking(validation_budget_status, home, max_validations)
+ if isinstance(response.answer, dict):
+ response.answer = {**response.answer, "validation_budget": budget}
+ completed = await blocking(completed_validation, home, response.answer, started_before)
+ if completed is not None:
+ manifest, validation_number = completed
+ if timer is not None:
+ timer.reset()
+ transition = validation_cycle_transition(manifest, validation_number, budget["remaining_validations"])
+ state.middle_context.setdefault(PENDING_CYCLE_TRANSITIONS, []).append(transition.hint)
+ return {"success": response.success, "answer": response.answer, "metadata": response.metadata}
+
+ tool = FunctionTool(evaluate, name="home_evaluate", is_concurrency_safe=False, is_state_injected=True)
+ tool.input_schema = {
+ "type": "object",
+ "properties": {"debug": {"type": "boolean", "default": False}},
+ "additionalProperties": False,
+ }
+ return tool
+
+
+class HomeOptimizeStep(BaseStep):
+ """Require a full initial baseline, run the coding agent, and verify its final claim."""
+
+ workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace")
+ agent_wrapper: OptimizerAgentWrapper = Ref(OptimizerAgentWrapper, "agent_wrapper")
+
+ async def execute(self):
+ """One optimizer per workspace; never hold the evaluation/Git lock across model calls."""
+ if self.context.get("_allowed_paths") is not None:
+ raise ValueError("Home optimization requires access to its managed workspace")
+ if not self.workspace.is_started or not self.agent_wrapper.is_started:
+ raise RuntimeError("Start the Home workspace and optimizer wrapper first")
+ spec = OptimizationInput.model_validate(
+ {key: self.context[key] for key in OptimizationInput.model_fields if key in self.context},
+ )
+ options = evaluation_options(self.kwargs.get("evaluation_options", {}))
+ self.agent_wrapper.validation_limit()
+ home = self.workspace.home_path
+ _validate_directory(home, self.workspace.state_path)
+ with _initialization_lock(self.workspace.state_path / "optimization.lock"):
+ _current, baseline = await blocking(baseline_state, home)
+ if baseline is None:
+ response = await required_job(self.agent_wrapper, "home_evaluate")(
+ **options,
+ debug=False,
+ sampling=False,
+ record_history=True,
+ case_ids=None,
+ query_ids=None,
+ reuse_construction_from=None,
+ )
+ if not response.success:
+ raise RuntimeError(f"Initial baseline failed: {response.answer}")
+ current, _ = await blocking(baseline_state, home)
+ baseline = await blocking(
+ verify_evaluation,
+ home,
+ response.answer["result_dir"],
+ current["commit_sha"],
+ baseline=True,
+ )
+ return await self._optimize(home, spec, baseline, options)
+
+ async def _optimize(self, home, spec, baseline, options):
+ payload = {"status": "running", "instruction": spec.instruction, "baseline_path": baseline["evaluation_path"]}
+ report_path = await blocking(save_record, home, "optimization-reports", payload)
+ self.context.response.metadata["optimization_report_path"] = report_path
+ try:
+ timer = self.agent_wrapper.create_cycle_timer()
+ max_validations = self.agent_wrapper.validation_limit()
+ budget = await blocking(validation_budget_status, home, max_validations)
+ reply = await workflow_reply(
+ self.agent_wrapper,
+ home,
+ "optimizer",
+ f"Begin optimization.\n{spec.instruction}\n\n"
+ f"Completed full initial baseline: {baseline['evaluation_path']}\n"
+ f"Formal validation budget: {json.dumps(budget)}\n"
+ "Return exactly one JSON object with branch_name, commit_sha (full SHA), evaluation_path and report. "
+ "Leave that evaluated branch and commit checked out with no uncommitted changes.",
+ tool_factory=lambda backend: [
+ Edit(backend=backend),
+ diagnostic_tool(self.agent_wrapper),
+ validation_budget_tool(home, max_validations),
+ evaluation_tool(
+ self.agent_wrapper,
+ options,
+ home=home,
+ timer=timer,
+ max_validations=max_validations,
+ ),
+ ],
+ job_names=MUTATION_JOBS,
+ agent_options={
+ "_validation_cycle_timer": timer,
+ "_context_fallback_directory": contained(home, "logs/context-offloads", directory=True),
+ },
+ memory_limit_mib=self.kwargs.get("memory_limit_mib"),
+ )
+ payload.update(
+ agent_report=reply.get("result"),
+ session_id=reply.get("session_id"),
+ usage=reply.get("usage"),
+ )
+ result = parse_result(reply["result"])
+ evidence_path = await blocking(verify_final, home, result)
+ payload.update(result.model_dump(), evaluation_path=evidence_path, status="completed")
+ self.context.response.answer = {**payload, "optimization_report_path": report_path}
+ self.context.response.metadata.update({key: reply[key] for key in ("session_id", "usage") if key in reply})
+ except BaseException as exc:
+ payload.update(status="cancelled" if isinstance(exc, asyncio.CancelledError) else "failed", error=str(exc))
+ raise
+ finally:
+ await blocking(write_json, contained(home, report_path), payload)
+ return self.context.response
diff --git a/plugins/home/src/reme_home/agent/optimizer.yaml b/plugins/home/src/reme_home/agent/optimizer.yaml
new file mode 100644
index 00000000..151304df
--- /dev/null
+++ b/plugins/home/src/reme_home/agent/optimizer.yaml
@@ -0,0 +1,60 @@
+system_prompt: |-
+ You are Home's primary optimization agent, migrated from Meta-ReMe. Execute the optimization loop;
+ your output is the best formally evaluated candidate, with its branch and exact commit checked out.
+ ReMe is local-first and file-native. User-owned memory is the source of truth; indexes and caches are rebuildable.
+ Improve general memory mechanisms, robustness, recoverability and transparent behavior. Never introduce
+ benchmark-specific shortcuts, held-out answer leakage or case-specific rules. Fixed runtime is host-owned.
+ Candidate source is {code_repository}; guides are under {docs}; disposable scratch is {scratch}.
+
+ Optimize answer quality by supplying precise evidence at lower query-time token cost. Score improvement is
+ more important than query token reduction. Every proposed change should aim for both. A candidate branch
+ represents one coherent, falsifiable hypothesis. Candidate paths are the only editable source roots.
+
+ Repeat this loop:
+ 1. Start from the completed full baseline supplied by the host or the current candidate's completed formal
+ evaluation. Inventory evaluations and branches, commits and coverage before editing.
+ 2. Diagnose before editing. Strongly prefer home_diagnose with the actual evaluation_paths; it automatically
+ receives your available full trajectory. Prioritize low-scoring cases and question types, identify at least
+ two distinct supported causes when evidence permits, estimate impact, coverage and confidence, and rank
+ directions. Direct investigation is appropriate when evidence is already narrow and clear. Treat partial
+ construction as a defect. Distinguish infrastructure errors from candidate behavior; do not patch candidate
+ code to compensate for infrastructure outages. Defer unrelated, redundant and low-impact investigation.
+ 3. Choose lineage before editing. Continue the current direction on its branch; fork a promising branch for
+ a distinct direction with home_create_candidate_branch. Never edit init. Implement the highest-impact
+ supported direction, or two or three compatible changes supporting one coherent claim. Inspect diffs before
+ merging. Create, switch, merge and commit exclusively through the managed Home Jobs; keep evaluated history.
+ Preserve a branch at each evaluated commit you may want to select finally: before advancing such a branch,
+ fork a new candidate. Managed switching selects existing branch heads, not arbitrary historical commits.
+ 4. Commit before testing, then call home_evaluate(debug=true). Repair concrete execution failures, commit and
+ repeat until debug passes. Debug establishes execution only; it has no answer-quality score.
+ 5. Call home_evaluate(debug=false) on clean committed code after debug passes. Formal evaluation uses host-owned
+ adaptive sampling and always rebuilds memory before answering. Inspect the resulting evidence before the
+ next iteration. Candidate failures and low scores require a code change; infrastructure failures may retry
+ the same commit using the host's frozen previous selection. Empty commits do not bypass repetition checks.
+
+ Call get_validation_budget before planning the next candidate. The host limits formal validations; one run
+ consumes one slot when its first sandbox starts successfully, even if it later fails or is cancelled. Debug,
+ initial baseline and pre-sandbox failures do not consume slots. The budget persists across optimizer invocations.
+ During structural_exploration prefer substantial mechanisms or coherent architecture changes; during
+ evidence_driven_refinement prefer evidence-driven parameters, prompts or scoped changes. When exhausted,
+ select the best existing evaluated branch without another formal validation. There is no candidate admission
+ gate; do not call architecture or construction-equivalence tools. Use the configured iteration cap and advisory time
+ reminders to finish a useful search; time reminders do not terminate execution. The cycle clock includes diagnosis,
+ editing, debug and formal evaluation. Only a newly counted, completed formal evaluation resets it and its context
+ compression count; debug, rejected calls and incomplete evaluations do not. A cycle transition expires earlier
+ timer reminders and announces the next cycle. You may stop once the evidence
+ supports a final selection. Do not keep evaluating without a concrete improvement hypothesis.
+ Compare scores on common cases and queries, weigh coverage, difficulty, regressions and lineage, and do not
+ rank sampled candidates by aggregate score alone. Prefer quality, stability, query cost and simplicity.
+ Do not primarily improve scores by increasing top-k, iteration limits, token budgets or output length.
+ Investigate better extraction, persistence, routing, retrieval, fusion, deduplication, reranking and context selection.
+
+ Read the runtime guides when needed. Tools can inspect memory checkpoints, evidence and commit-pinned code.
+ Use revision explicitly when examining code associated with an evaluation. Shell and Python run within the same
+ OS file sandbox as Read/Edit/Write; only declared candidate roots and scratch are writable. Network is denied.
+ Never write Git metadata, runtime, datasets, evaluation artifacts or reports. Run ReMe only through home_evaluate.
+
+ Finish with a clean checkout of the best evaluated branch and full commit. Choosing the baseline is valid if
+ improvements are unsupported. Return exactly one JSON object with branch_name, commit_sha, evaluation_path,
+ and report. report must explain the chosen evidence, comparisons, regressions, costs and limitations. The host
+ will reject a mismatched checkout or missing, incomplete, debug, query-only or infrastructure-failed evidence.
diff --git a/plugins/home/src/reme_home/agent/workflow_common.py b/plugins/home/src/reme_home/agent/workflow_common.py
new file mode 100644
index 00000000..45ec2f81
--- /dev/null
+++ b/plugins/home/src/reme_home/agent/workflow_common.py
@@ -0,0 +1,230 @@
+"""Shared sandbox, tools and invocation helpers for Home workflow agents."""
+
+import json
+import sys
+from pathlib import Path
+from uuid import uuid4
+
+from agentscope.tool import BackendBase, DirEntry, Glob, Grep, Read, Toolkit, Write
+import yaml
+
+from ..auto_bash import AutoBash
+from ..code.repository import validate_repository
+from ..evaluation.artifacts import contained, write_json
+from ..evaluation.component import blocking
+from .case_study import python_tool
+from .sandbox import create_case_study_sandbox
+
+INSPECTION_JOBS = (
+ "home_list_validation_results",
+ "home_inspect_validation",
+ "home_trace_query_evidence",
+ "home_inspect_memory_history",
+ "home_compare_memory",
+ "home_code_status",
+ "home_list_candidate_branches",
+ "home_code_diff",
+ "home_list_code_tree",
+ "home_read_code_file",
+ "home_search_code",
+)
+
+
+def required_job(wrapper, name):
+ """Do not silently bypass a disabled or unstarted Home Job."""
+ job = wrapper.app_context.jobs[name]
+ if not job.enabled or not job.is_started:
+ raise RuntimeError(f"Workflow requires a started Job: {name}")
+ return job
+
+
+def create_toolkit(wrapper, backend, scope, *, extra_tools=(), job_names=()):
+ """All builtin tools share the OS sandbox; privileged mutations use managed Jobs."""
+ tools = [
+ Read(backend=backend),
+ Glob(backend=backend),
+ Grep(backend=backend),
+ Write(backend=backend),
+ AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=False),
+ python_tool(backend, scope["scratch"]),
+ ]
+ tools[-1].description = (
+ tools[-1]
+ .description.replace(
+ "No host path is writable.",
+ "Only the host-granted scratch and candidate paths are writable.",
+ )
+ .replace("case directory", "scratch directory")
+ )
+ tools[-2].description = (
+ "Run shell commands inside the OS file sandbox; network is denied. Use absolute allowed paths. "
+ "Only private scratch and, for the optimizer, declared candidate paths are writable. "
+ "Git metadata is read-only: commit, switch, create and merge only through Home Jobs. "
+ "Run ReMe evaluations only through home_evaluate, never through shell or Python. "
+ "Keep stderr visible and use set -o pipefail for pipelines. timeout is in milliseconds."
+ )
+ tools.extend(extra_tools)
+ tools.extend(
+ wrapper._make_tool(required_job(wrapper, name)) # pylint: disable=protected-access
+ for name in (*INSPECTION_JOBS, *job_names)
+ )
+ for tool in tools:
+ tool.is_concurrency_safe = False
+ return Toolkit(tools=tools)
+
+
+async def workflow_reply(
+ wrapper,
+ home,
+ role,
+ task,
+ *,
+ tool_factory=None,
+ job_names=(),
+ agent_options=None,
+ memory_limit_mib=None,
+):
+ """Use fresh state, the dedicated configured model and sandboxed tools for every call."""
+ scope = await blocking(prepare_scope, home, role)
+ backend = WorkflowSandbox(scope, memory_limit_mib)
+ probe = await backend.exec_shell(
+ [sys.executable, "-I", "-B", "-c", "print('home workflow sandbox')"],
+ timeout=10,
+ )
+ if not probe.ok():
+ raise RuntimeError("Home workflow sandbox unavailable: " + probe.stderr.decode(errors="replace"))
+ prompt = yaml.safe_load(Path(__file__).with_name(f"{role}.yaml").read_text(encoding="utf-8"))
+ return await wrapper.reply(
+ task,
+ system_prompt=prompt["system_prompt"].format(**scope),
+ toolkit=create_toolkit(
+ wrapper,
+ backend,
+ scope,
+ extra_tools=tool_factory(backend) if tool_factory is not None else (),
+ job_names=job_names,
+ ),
+ builtin_tools=[],
+ job_tools=[],
+ skills=[],
+ resume=None,
+ session_id=None,
+ fork_session=False,
+ output_schema=None,
+ permission_mode="bypass",
+ **(agent_options or {}),
+ )
+
+
+def prepare_scope(home, role):
+ """Grant evidence reads and private scratch writes, plus declared optimizer roots."""
+ repo, record = validate_repository(home)
+ scratch = contained(home, f"tmp/{role}-{uuid4().hex}", directory=True)
+ scratch.mkdir()
+ roots = [
+ contained(home, name, directory=True)
+ for name in ("code", "datasets", "docs", "evaluations", "logs", "sampling")
+ ]
+ roots.append(contained(home, "metadata/preparation.json"))
+ writable = [scratch]
+ if role == "optimizer":
+ for value in record["code"]["candidate_paths"]:
+ path = contained(repo, value)
+ if ".git" in path.relative_to(repo).parts or path == repo:
+ raise ValueError("Optimizer cannot write Git metadata or the repository root")
+ writable.append(path)
+ return {
+ "home": str(home),
+ "code_repository": str(repo),
+ "scratch": str(scratch),
+ "docs": str(home / "docs"),
+ "read_only_paths": [str(path) for path in roots],
+ "writable_paths": [str(path) for path in writable],
+ }
+
+
+def save_record(home, category, payload):
+ """Write a uniquely named host-owned provenance record, outside agent write grants."""
+ path = contained(home, f"logs/{category}/{uuid4().hex}.json")
+ write_json(path, payload)
+ return str(path)
+
+
+_INSPECT = """\
+import json, pathlib, sys
+path = pathlib.Path(sys.argv[2])
+def entry(p):
+ s = p.stat()
+ return dict(name=p.name, is_dir=p.is_dir(), size_bytes=None if p.is_dir() else s.st_size, mtime=s.st_mtime)
+if sys.argv[1] == 'stat':
+ print(json.dumps(entry(path)))
+else:
+ result = []
+ for p in path.iterdir():
+ try:
+ result.append(entry(p))
+ except FileNotFoundError:
+ pass
+ print(json.dumps(result))
+"""
+
+
+class WorkflowSandbox(BackendBase):
+ """Route every builtin I/O primitive through Home's OS sandbox, including metadata.
+
+ Rebuild the backend before each operation: managed branch switches can
+ replace an allowed file's inode. Core validation still checks paths and
+ identities for each subprocess; no operation falls back to local I/O.
+ """
+
+ def __init__(self, scope, memory_limit_mib=None):
+ self.scope = scope
+ self.memory_limit_mib = memory_limit_mib
+
+ def _backend(self):
+ return create_case_study_sandbox(
+ cwd=self.scope["scratch"],
+ read_only_paths=self.scope["read_only_paths"],
+ writable_paths=self.scope["writable_paths"],
+ memory_limit_mib=self.memory_limit_mib,
+ )
+
+ async def getcwd(self):
+ """Use private scratch as the default, never the host process directory."""
+ return self.scope["scratch"]
+
+ async def exec_shell(self, command, *, cwd=None, timeout=None):
+ """Keep command execution, environment and process cleanup inside isolation."""
+ return await self._backend().exec_shell(command, cwd=cwd, timeout=timeout)
+
+ async def read_file(self, path):
+ """Use core sandboxed file reads, not LocalBackend's host reads."""
+ return await self._backend().backend.read_file(path)
+
+ async def write_file(self, path, data):
+ """Use core sandboxed writes, including parent creation and stdin transport."""
+ await self._backend().backend.write_file(path, data)
+
+ async def _inspect(self, operation, path):
+ result = await self.exec_shell([sys.executable, "-I", "-B", "-c", _INSPECT, operation, path], timeout=10)
+ if not result.ok():
+ raise OSError(result.stderr.decode(errors="replace"))
+ return json.loads(result.stdout)
+
+ async def stat(self, path):
+ """Use portable sandboxed metadata on Linux and macOS."""
+ return DirEntry(**await self._inspect("stat", path))
+
+ async def scandir(self, path):
+ """Avoid GNU find assumptions and host filesystem metadata access."""
+ return [DirEntry(**item) for item in await self._inspect("scandir", path)]
+
+ async def stat_mtime(self, path):
+ """Preserve the builtin Edit/Write optimistic modification checks."""
+ return (await self.stat(path)).mtime
+
+ async def list_dir(self, path, *, recursive=False):
+ """Use the portable metadata implementation for immediate children."""
+ if recursive:
+ return await super().list_dir(path, recursive=True)
+ return [entry.name for entry in await self.scandir(path)]
diff --git a/plugins/home/src/reme_home/evaluation/budget.py b/plugins/home/src/reme_home/evaluation/budget.py
new file mode 100644
index 00000000..3b4cc752
--- /dev/null
+++ b/plugins/home/src/reme_home/evaluation/budget.py
@@ -0,0 +1,43 @@
+"""Formal optimizer budgets derived from Home's durable sandbox-start evidence."""
+
+from .history import read_state
+
+
+def validate_max_validations(value):
+ """Accept only a positive host-supplied integer, matching dev_meta."""
+ if not isinstance(value, int) or isinstance(value, bool) or value < 1:
+ raise ValueError("max_validations must be a positive integer")
+ return value
+
+
+def validation_budget_status(home, max_validations):
+ """Report started formal runs, independently of completion, score or sampler success."""
+ validate_max_validations(max_validations)
+ used = sum(not key.startswith("init/") for key in read_state(home)["started_validations"])
+ remaining = max(0, max_validations - used)
+ gated = max_validations // 2
+ if not remaining:
+ phase = "exhausted"
+ guidance = (
+ "The formal validation budget is exhausted; select among existing evidence without another validation."
+ )
+ elif used < gated:
+ phase = "structural_exploration"
+ guidance = (
+ "Prefer a substantial code-level mechanism or coherent plugin architecture change. "
+ "Do not spend an early validation on prompt-only or single-parameter tuning."
+ )
+ else:
+ phase = "evidence_driven_refinement"
+ guidance = (
+ "Prefer evidence-driven parameter tuning, fine-grained prompt refinement, or a small scoped code change. "
+ "Avoid a new broad architecture unless existing validation evidence clearly requires it."
+ )
+ return {
+ "used_validations": used,
+ "max_validations": max_validations,
+ "remaining_validations": remaining,
+ "gated_validation_count": gated,
+ "optimization_phase": phase,
+ "next_validation_guidance": guidance,
+ }
diff --git a/plugins/home/src/reme_home/evaluation/component.py b/plugins/home/src/reme_home/evaluation/component.py
index 606d3ac0..84c9d3c6 100644
--- a/plugins/home/src/reme_home/evaluation/component.py
+++ b/plugins/home/src/reme_home/evaluation/component.py
@@ -29,6 +29,7 @@ from ..judge import HomeJudge
from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory
from ..sampling import SamplingPolicy, create_sampling_plan
from .history import HomeEvaluationHistory
+from .budget import validation_budget_status
from .repetition import previous_candidate_result
@@ -85,7 +86,25 @@ class HomeEvaluation(BaseComponent):
_validate_directory(self.workspace_path.resolve(), self.workspace.state_path)
lock = nullcontext() if spec.debug else _initialization_lock(self.workspace.state_path / "validation.lock")
with lock:
- return await self._evaluate(spec)
+ budgeted = spec.max_validations is not None and not spec.debug
+ if budgeted:
+ budget = await blocking(validation_budget_status, self.workspace.home_path, spec.max_validations)
+ if budget["remaining_validations"] < 1:
+ return {
+ "status": "rejected",
+ "error_code": "VALIDATION_BUDGET_EXHAUSTED",
+ "error": "Optimizer validation budget is exhausted; no sandbox was started.",
+ "validation_budget": budget,
+ "result_dir": None,
+ }
+ result = await self._evaluate(spec)
+ if budgeted:
+ result["validation_budget"] = await blocking(
+ validation_budget_status,
+ self.workspace.home_path,
+ spec.max_validations,
+ )
+ return result
def _prepare(self, spec, environment):
home = self.workspace.home_path
diff --git a/plugins/home/src/reme_home/evaluation/inputs.py b/plugins/home/src/reme_home/evaluation/inputs.py
index fbeea7fc..a319988d 100644
--- a/plugins/home/src/reme_home/evaluation/inputs.py
+++ b/plugins/home/src/reme_home/evaluation/inputs.py
@@ -29,6 +29,7 @@ class EvaluationSpec(SpecModel):
debug: bool = False
sampling: bool = False
record_history: bool = True
+ max_validations: int | None = Field(default=None, ge=1)
sampling_policy: SamplingPolicy | None = None
concurrency: int = Field(default=4, ge=1)
timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False)
@@ -67,6 +68,8 @@ class EvaluationSpec(SpecModel):
@model_validator(mode="after")
def resolve_mode(self):
"""Debug stops on failure by default and never reuses scored evidence."""
+ if self.max_validations is not None and not self.debug and not self.record_history:
+ raise ValueError("Budgeted formal evaluations require record_history=true")
if self.debug and self.reuse_construction_from:
raise ValueError("Construction reuse requires training-set")
if self.sampling and not self.debug and (self.case_ids is not None or self.query_ids is not None):
diff --git a/plugins/home/src/reme_home/evaluation/step.py b/plugins/home/src/reme_home/evaluation/step.py
index b9e24773..d2dcc4d3 100644
--- a/plugins/home/src/reme_home/evaluation/step.py
+++ b/plugins/home/src/reme_home/evaluation/step.py
@@ -39,6 +39,9 @@ class HomeEvaluateStep(BaseStep):
self.context.response.metadata["result_dir"] = str(error.path)
return self.context.response
self.context.response.answer = result
+ if result.get("status") == "rejected":
+ self.context.response.success = False
+ return self.context.response
# Candidate failures are evaluation observations; infrastructure failures
# must remain visible to ordinary Job callers checking Response.success.
self.context.response.success = result["case_status_counts"]["infra_error"] == 0
diff --git a/plugins/home/src/reme_home/plugin.yaml b/plugins/home/src/reme_home/plugin.yaml
index 7b22f3aa..7bddc013 100644
--- a/plugins/home/src/reme_home/plugin.yaml
+++ b/plugins/home/src/reme_home/plugin.yaml
@@ -1,4 +1,8 @@
backends:
+ home_optimizer_agent: reme_home.agent.optimizer:OptimizerAgentWrapper
+ home_diagnose_step: reme_home.agent.diagnostic:HomeDiagnoseStep
+ home_optimize_step: reme_home.agent.optimizer:HomeOptimizeStep
+ home_workflow_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper
home_case_study_step: reme_home.agent.case_study:HomeCaseStudyStep
home_case_study_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper
home_trace_query_evidence_step: reme_home.evaluation.evidence_step:HomeTraceQueryEvidenceStep
@@ -20,6 +24,30 @@ backends:
application_defaults:
components:
as_llm:
+ home_optimizer:
+ backend: ${LLM_BACKEND:-openai}
+ model: qwen3.8-max
+ stream: false
+ context_size: 160000
+ max_retries: 3
+ credential:
+ api_key: ${LLM_API_KEY:-}
+ base_url: ${LLM_BASE_URL:-}
+ parameters:
+ max_tokens: 65536
+ thinking_enable: false
+ home_diagnostic:
+ backend: ${LLM_BACKEND:-openai}
+ model: qwen3.8-max
+ stream: false
+ context_size: 128000
+ max_retries: 3
+ credential:
+ api_key: ${LLM_API_KEY:-}
+ base_url: ${LLM_BASE_URL:-}
+ parameters:
+ max_tokens: 65536
+ thinking_enable: false
home_case_study:
backend: ${LLM_BACKEND:-openai}
model: qwen3.7-max
@@ -33,6 +61,22 @@ application_defaults:
max_tokens: 65536
thinking_enable: false
agent_wrapper:
+ home_optimizer:
+ backend: home_optimizer_agent
+ max_validations: 10
+ as_llm: home_optimizer
+ validation_cycle_time_budget_minutes: 150
+ context_config:
+ trigger_ratio: 0.85
+ reserve_ratio: 0.15
+ react_config:
+ max_iters: 3000
+ home_diagnostic:
+ backend: home_workflow_agent
+ as_llm: home_diagnostic
+ time_budget_minutes: 60
+ react_config:
+ max_iters: 80
home_case_study:
backend: home_case_study_agent
as_llm: home_case_study
@@ -64,6 +108,41 @@ application_defaults:
default:
backend: home_workspace
jobs:
+ home_diagnose:
+ backend: base
+ description: "Diagnose explicit evaluation runs with a fresh read-only agent; aggregate case studies and return evidence-supported root causes without prescribing fixes"
+ parameters:
+ type: object
+ additionalProperties: false
+ required: [evaluation_paths]
+ properties:
+ evaluation_paths:
+ type: array
+ minItems: 1
+ uniqueItems: true
+ items: {type: string, minLength: 1}
+ description: "Explicit evaluation result directories, absolute or relative to home/"
+ instruction:
+ type: string
+ minLength: 1
+ description: "Diagnostic question or suspected regression"
+ steps:
+ - backend: home_diagnose_step
+ agent_wrapper: home_diagnostic
+ home_optimize:
+ backend: base
+ description: "Check or run the full initial baseline, then diagnose, edit, commit, debug and evaluate candidates; verify the final clean checkout against saved formal evidence"
+ parameters:
+ type: object
+ additionalProperties: false
+ properties:
+ instruction:
+ type: string
+ minLength: 1
+ description: "Optimization focus; runtime, selection and sandbox settings are host-owned"
+ steps:
+ - backend: home_optimize_step
+ agent_wrapper: home_optimizer
home_case_study:
backend: base
description: "Investigate up to eight evaluation cases concurrently with a shared focus; each case uses a fresh read-only sandboxed agent and returns its own report or error in input order"
@@ -227,6 +306,10 @@ application_defaults:
description: "Stop on the first failure; defaults to debug (false for scored runs, true for debug)"
max_construction_failure_rate: {type: number, minimum: 0, maximum: 1}
max_artifact_bytes: {type: integer, minimum: 1}
+ max_validations:
+ type: [integer, "null"]
+ minimum: 1
+ description: "Optional host formal-evaluation budget; optimizer injects its configured limit"
steps:
- backend: home_evaluate_step
home_prepare:
diff --git a/plugins/home/tests/test_benchmark_runner.py b/plugins/home/tests/test_benchmark_runner.py
index 797768c6..9a6a92ae 100644
--- a/plugins/home/tests/test_benchmark_runner.py
+++ b/plugins/home/tests/test_benchmark_runner.py
@@ -1,6 +1,8 @@
"""Benchmark Home defaults and lifecycle wiring, without models or dataset writes."""
import hashlib
+import json
+import sys
from importlib import import_module
from pathlib import Path
from unittest.mock import AsyncMock
@@ -10,7 +12,8 @@ import yaml
from reme.schema import Response
-from benchmark import home_runner
+from benchmark.beam import run_home as beam_runner
+from benchmark.longmemeval import run_home as lme_runner
ROOT = Path(__file__).resolve().parents[3]
@@ -21,7 +24,7 @@ def test_preserved_lme_split_and_evidence():
assert hashlib.sha256((directory / "home_evidence_sessions.jsonl").read_bytes()).hexdigest() == (
"35655c51210a96b595910923576fa97b93c81abd82e3cfcda801180007c23a7d"
)
- workspace, spec = home_runner.load_config(directory / "config_home.yaml")
+ workspace, spec = lme_runner.load_config(directory / "config_home.yaml")
selected = spec["benchmark"]["case_ids"]
assert len(selected) == len(set(selected)) == 250
# Hash the original first-five-group sequence to preserve both IDs and order.
@@ -36,7 +39,7 @@ def test_preserved_lme_split_and_evidence():
def test_beam_defaults_use_all_100k_cases_from_any_cwd(tmp_path, monkeypatch):
"""Paths and selection do not depend on where Python was launched."""
monkeypatch.chdir(tmp_path)
- workspace, spec = home_runner.load_config(ROOT / "benchmark/beam/config_home.yaml")
+ workspace, spec = beam_runner.load_config(ROOT / "benchmark/beam/config_home.yaml")
assert spec["benchmark"]["variant"] == "100K"
assert spec["benchmark"]["all_cases"] is True
assert spec["benchmark"]["case_ids"] is None
@@ -47,6 +50,18 @@ def test_beam_defaults_use_all_100k_cases_from_any_cwd(tmp_path, monkeypatch):
assert workspace == ROOT / "benchmark/beam/workspaces/home-100K"
+@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
+def test_runner_preserves_explicit_candidate_paths(tmp_path, benchmark):
+ """Runner defaults preserve custom scopes from existing configurations."""
+ module = import_module(f"benchmark.{benchmark}.run_home")
+ config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
+ config["preparation"]["candidate"] = {"paths": ["reme/steps"]}
+ path = tmp_path / "home.yaml"
+ path.write_text(yaml.safe_dump(config))
+ _, spec = module.load_config(path)
+ assert spec["candidate"]["paths"] == ["reme/steps"]
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, benchmark):
@@ -60,7 +75,7 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
observed.update(kwargs)
return application
- monkeypatch.setattr(home_runner, "Application", factory)
+ monkeypatch.setattr(module, "Application", factory)
result = await module.run(workspace_dir=str(tmp_path / "experiment"))
assert observed["plugins"] == ["home", "beam-judge" if benchmark == "beam" else "lme-judge"]
assert observed["service"] == {"backend": "cli"}
@@ -72,6 +87,10 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
"ewma_alpha": 0.3,
"uniform_exploration": 0.2,
}
+ assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
+ "validation_cycle_time_budget_minutes": 150,
+ "max_validations": 10,
+ }
application.start.assert_awaited_once()
application.close.assert_awaited_once()
assert application.run_job.await_count == 1
@@ -80,9 +99,11 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b
@pytest.mark.asyncio
-async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch):
+@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
+async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch, benchmark):
"""Partial policies merge typed defaults and never leak into preparation inputs."""
- config = yaml.safe_load((ROOT / "benchmark/beam/config_home.yaml").read_text())
+ module = import_module(f"benchmark.{benchmark}.run_home")
+ config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
config["sampling_policy"] = {"max_cases": 2, "max_queries": 7}
path = tmp_path / "home.yaml"
path.write_text(yaml.safe_dump(config))
@@ -94,8 +115,8 @@ async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monk
observed.update(kwargs)
return application
- monkeypatch.setattr(home_runner, "Application", factory)
- await home_runner.prepare(path)
+ monkeypatch.setattr(module, "Application", factory)
+ await module.prepare(path)
policy = observed["components"]["home.evaluation"]["default"]["sampling_policy"]
assert policy["max_cases"] == 2 and policy["max_queries"] == 7 and policy["ewma_alpha"] == 0.3
assert "sampling_policy" not in application.run_job.call_args.kwargs["spec"]
@@ -103,22 +124,193 @@ async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monk
config["sampling_policy"] = invalid
path.write_text(yaml.safe_dump(config))
with pytest.raises(ValueError):
- await home_runner.prepare(path)
+ await module.prepare(path)
assert application.start.await_count == 1
assert application.run_job.await_count == 1
assert application.run_job.call_args.args == ("home_prepare",)
@pytest.mark.asyncio
+@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
+@pytest.mark.parametrize("budget", ["omitted", 45.5, None])
+async def test_runner_passes_cycle_budget_to_optimizer(tmp_path, monkeypatch, benchmark, budget):
+ """Both local runners apply the default, custom duration and explicit disable value."""
+ module = import_module(f"benchmark.{benchmark}.run_home")
+ config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
+ if budget == "omitted":
+ config.pop("validation_cycle_time_budget_minutes", None)
+ else:
+ config["validation_cycle_time_budget_minutes"] = budget
+ path = tmp_path / "home.yaml"
+ path.write_text(yaml.safe_dump(config))
+ application = AsyncMock()
+ application.run_job.return_value = Response(answer={})
+ observed = {}
+
+ def factory(**kwargs):
+ observed.update(kwargs)
+ return application
+
+ monkeypatch.setattr(module, "Application", factory)
+ await module.run(path)
+ expected = 150 if budget == "omitted" else budget
+ assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
+ "validation_cycle_time_budget_minutes": expected,
+ "max_validations": 10,
+ }
+ assert "validation_cycle_time_budget_minutes" not in application.run_job.call_args.kwargs["spec"]
+ application.close.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
+@pytest.mark.parametrize("budget", [0, -1, True, "150", float("nan"), float("inf")])
+async def test_runner_rejects_invalid_cycle_budget_before_start(tmp_path, monkeypatch, benchmark, budget):
+ """Reject nonpositive, nonnumeric and nonfinite durations without creating an application."""
+ module = import_module(f"benchmark.{benchmark}.run_home")
+ config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
+ config["validation_cycle_time_budget_minutes"] = budget
+ path = tmp_path / "home.yaml"
+ path.write_text(yaml.safe_dump(config))
+ created = []
+ monkeypatch.setattr(module, "Application", lambda **kwargs: created.append(kwargs))
+ with pytest.raises(ValueError):
+ await module.run(path)
+ assert not created
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
@pytest.mark.parametrize("failure", ["startup", "preparation"])
-async def test_runner_propagates_failure_and_closes(monkeypatch, failure):
+async def test_runner_propagates_failure_and_closes(monkeypatch, failure, benchmark):
"""A failed preparation is never printed or returned as success."""
application = AsyncMock()
if failure == "startup":
application.start.side_effect = RuntimeError("startup failed")
else:
application.run_job.return_value = Response(success=False, answer="already prepared")
- monkeypatch.setattr(home_runner, "Application", lambda **kwargs: application)
+ module = import_module(f"benchmark.{benchmark}.run_home")
+ monkeypatch.setattr(module, "Application", lambda **kwargs: application)
with pytest.raises(RuntimeError):
- await home_runner.prepare(ROOT / "benchmark/beam/config_home.yaml")
+ await module.prepare(module.DEFAULT_CONFIG)
application.close.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
+@pytest.mark.parametrize("limit", ["omitted", 3, 0, -1, True, "10", 1.5, None])
+async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchmark, limit):
+ """Validate host count budgets and pass them only to the optimizer wrapper."""
+ module = import_module(f"benchmark.{benchmark}.run_home")
+ config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
+ if limit == "omitted":
+ config.pop("max_validations", None)
+ else:
+ config["max_validations"] = limit
+ path = tmp_path / "home.yaml"
+ path.write_text(yaml.safe_dump(config))
+ application = AsyncMock()
+ application.run_job.return_value = Response(answer={})
+ observed = {}
+
+ def factory(**kwargs):
+ observed.update(kwargs)
+ return application
+
+ monkeypatch.setattr(module, "Application", factory)
+ if limit in ("omitted", 3):
+ await module.prepare(path)
+ assert observed["components"]["agent_wrapper"]["home_optimizer"]["max_validations"] == (
+ 10 if limit == "omitted" else limit
+ )
+ assert "max_validations" not in application.run_job.call_args.kwargs["spec"]
+ else:
+ with pytest.raises(ValueError):
+ await module.prepare(path)
+ assert not observed
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
+@pytest.mark.parametrize("minutes", [30, None])
+async def test_optimizer_lifecycle_uses_same_config_and_does_not_prepare(tmp_path, monkeypatch, benchmark, minutes):
+ """A separate optimization launch consumes the same count, timer and sampling settings."""
+ module = import_module(f"benchmark.{benchmark}.run_home")
+ config = yaml.safe_load(module.DEFAULT_CONFIG.read_text())
+ config.update(max_validations=3, validation_cycle_time_budget_minutes=minutes)
+ path = tmp_path / "home.yaml"
+ path.write_text(yaml.safe_dump(config))
+ application = AsyncMock()
+ application.run_job.return_value = Response(answer={"status": "completed"})
+ observed = {}
+
+ def factory(**kwargs):
+ observed.update(kwargs)
+ return application
+
+ monkeypatch.setattr(module, "Application", factory)
+ result = await module.optimize(path, workspace_dir=str(tmp_path / "existing"), instruction="Improve retrieval")
+ assert result["status"] == "completed"
+ assert observed["components"]["agent_wrapper"]["home_optimizer"] == {
+ "max_validations": 3,
+ "validation_cycle_time_budget_minutes": minutes,
+ }
+ application.run_job.assert_awaited_once_with("home_optimize", instruction="Improve retrieval")
+ application.close.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
+async def test_optimizer_failure_closes_application(monkeypatch, benchmark):
+ """Failed optimization retains the normal cleanup and error contract."""
+ module = import_module(f"benchmark.{benchmark}.run_home")
+ application = AsyncMock()
+ application.run_job.return_value = Response(success=False, answer="optimization failed")
+ monkeypatch.setattr(module, "Application", lambda **kwargs: application)
+ with pytest.raises(RuntimeError, match="optimization failed"):
+ await module.optimize()
+ application.run_job.assert_awaited_once_with("home_optimize")
+ application.close.assert_awaited_once()
+
+
+@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
+def test_optimizer_cli_forwards_configuration_and_focus(tmp_path, monkeypatch, capsys, benchmark):
+ """The explicit CLI mode reaches optimization and preserves the preparation default."""
+ module = import_module(f"benchmark.{benchmark}.run_home")
+ config = tmp_path / "home.yaml"
+ optimize = AsyncMock(return_value={"status": "completed"})
+ prepare = AsyncMock()
+ monkeypatch.setattr(module, "optimize", optimize)
+ monkeypatch.setattr(module, "prepare", prepare)
+ monkeypatch.setattr(
+ sys,
+ "argv",
+ [
+ "run_home.py",
+ "--optimize",
+ "--config",
+ str(config),
+ "--workspace-dir",
+ str(tmp_path / "existing"),
+ "--instruction",
+ "Improve retrieval",
+ ],
+ )
+ module.main()
+ optimize.assert_awaited_once_with(
+ config,
+ workspace_dir=str(tmp_path / "existing"),
+ instruction="Improve retrieval",
+ )
+ prepare.assert_not_called()
+ assert json.loads(capsys.readouterr().out) == {"status": "completed"}
+
+
+@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
+def test_cli_rejects_optimization_focus_in_preparation_mode(monkeypatch, benchmark):
+ """Do not silently discard an instruction when the caller forgot --optimize."""
+ module = import_module(f"benchmark.{benchmark}.run_home")
+ monkeypatch.setattr(sys, "argv", ["run_home.py", "--instruction", "Improve retrieval"])
+ with pytest.raises(SystemExit) as error:
+ module.main()
+ assert error.value.code == 2
diff --git a/plugins/home/tests/test_optimizer_compression.py b/plugins/home/tests/test_optimizer_compression.py
new file mode 100644
index 00000000..eae651b5
--- /dev/null
+++ b/plugins/home/tests/test_optimizer_compression.py
@@ -0,0 +1,215 @@
+"""Compression recovery parity with dev_meta, without model or filesystem services."""
+
+# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access
+# pylint: disable=redefined-outer-name,unused-argument
+
+import asyncio
+import json
+
+from agentscope.agent import Agent, ContextConfig
+from agentscope.message import AssistantMsg, TextBlock, UserMsg, Usage
+from agentscope.state import AgentState
+import jsonschema
+import openai
+import pytest
+from reme_home.agent import compression, optimizer
+
+
+class FailingCompressionModel:
+ def __init__(self, ratio=0.9, failure=None):
+ self.context_size = 90 / ratio
+ self.failure = failure or RuntimeError("Failed to generate structured output for model.")
+ self.calls = 0
+
+ async def count_tokens(self, messages, tools=None):
+ return sum(10 for message in messages if (message.get_text_content() or "").startswith("message-"))
+
+ async def generate_structured_output(self, **kwargs):
+ self.calls += 1
+ raise self.failure
+
+
+def make_agent(tmp_path, ratio=0.9, trigger=0.8, failure=None, summary="existing summary"):
+ return optimizer.OptimizerAgent(
+ name="optimizer",
+ system_prompt="system",
+ model=FailingCompressionModel(ratio, failure),
+ state=AgentState(
+ summary=summary,
+ context=[UserMsg(name="user", content=f"message-{index}") for index in range(9)],
+ ),
+ context_config=ContextConfig(trigger_ratio=trigger, reserve_ratio=0.2),
+ fallback_directory=tmp_path / "offloads",
+ validation_cycle_timer=optimizer.ValidationCycleTimer(150),
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("trigger", "ratio", "expected"),
+ [(0.8, 0.79, False), (0.8, 0.85, False), (0.8, 0.9, True), (0.65, 0.82, False), (0.65, 0.825, True)],
+)
+async def test_real_compression_failure_uses_dev_meta_emergency_threshold(tmp_path, trigger, ratio, expected):
+ agent = make_agent(tmp_path, ratio, trigger)
+ context_before, summary_before = agent.state.context, agent.state.summary
+ await agent.compress_context()
+ assert agent.validation_cycle_timer.context_compression_count == int(expected)
+ assert len(list(tmp_path.rglob("context-*.json"))) == int(expected)
+ assert agent.model.calls == int(ratio >= trigger)
+ assert agent.context_config.compression_fallback_to_truncation # The shared config was not mutated.
+ if not expected:
+ assert agent.state.context is context_before
+ assert agent.state.summary is summary_before
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("summary", ["existing summary", [TextBlock(text="existing summary")]])
+@pytest.mark.parametrize("reserve_ratio", [0.2, 0])
+async def test_offload_retains_full_messages_summary_usage_and_recent_context(tmp_path, summary, reserve_ratio):
+ agent = make_agent(tmp_path, summary=summary)
+ agent.context_config.reserve_ratio = reserve_ratio
+ agent.state.context[-1] = AssistantMsg(
+ id=agent.state.reply_id,
+ name=agent.name,
+ content="message-8",
+ usage=Usage(input_tokens=100, output_tokens=10),
+ )
+ original = [msg.model_dump(mode="json", exclude_none=False) for msg in agent.state.context]
+ await agent.compress_context()
+ (path,) = tmp_path.rglob("context-*.json")
+ data = json.loads(path.read_text())
+ assert data["messages"] == original[:7] if reserve_ratio else data["messages"] == original
+ if reserve_ratio:
+ assert [msg.get_text_content() for msg in agent.state.context] == ["message-7", "message-8"]
+ else:
+ assert len(agent.state.context) == 1
+ assert agent.state.context[0].content == [] # SDK usage carrier after full offload.
+ assert data["session_id"] == agent.state.session_id
+ assert data["context_ratio"] == 0.9
+ assert data["emergency_ratio"] == 0.9
+ assert data["failure"]["type"] == "RuntimeError"
+ text = agent.state.summary
+ if isinstance(text, list):
+ text = "\n".join(block.text for block in text)
+ assert "existing summary" in text
+ assert "without semantic summarization" in text
+ assert str(path) in text
+ assert agent._get_reply_usage().input_tokens == 100
+ assert not list(path.parent.glob(".result-*"))
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "failure",
+ [RuntimeError("The system prompt exceeds threshold"), ValueError("bad configuration"), asyncio.CancelledError()],
+)
+async def test_unrecoverable_failures_and_cancellation_propagate(tmp_path, failure):
+ agent = make_agent(tmp_path, failure=failure)
+ original = agent.state.context
+ with pytest.raises(type(failure)):
+ await agent.compress_context()
+ assert agent.state.context is original
+ assert not list(tmp_path.rglob("*.json"))
+ assert agent.validation_cycle_timer.context_compression_count == 0
+
+
+@pytest.mark.parametrize(
+ ("failure", "expected"),
+ [
+ (RuntimeError("provider failed"), True),
+ (RuntimeError("The system prompt exceeds threshold"), False),
+ (jsonschema.ValidationError("invalid summary"), True),
+ (openai.OpenAIError("provider unavailable"), True),
+ (ValueError("programming error"), False),
+ (OSError("disk full"), False),
+ ],
+)
+def test_recoverable_error_classifier(failure, expected):
+ assert compression.RecoverableCompressionAgent._is_recoverable_compression_error(failure) is expected
+
+
+@pytest.mark.asyncio
+async def test_override_config_and_successful_compression_use_native_pipeline(tmp_path, monkeypatch):
+ agent = make_agent(tmp_path, ratio=0.85)
+ config = ContextConfig(trigger_ratio=0.65, reserve_ratio=0.1)
+ calls = []
+
+ async def compress(self, context_config=None, instructions=None):
+ calls.append((context_config, instructions))
+ self.state.summary = "semantic summary"
+ self.state.context = []
+
+ monkeypatch.setattr(Agent, "_compress_context_impl", compress)
+ await agent.compress_context(context_config=config)
+ assert calls[0][0].trigger_ratio == 0.65
+ assert calls[0][0].reserve_ratio == 0.1
+ assert not calls[0][0].compression_fallback_to_truncation
+ assert config.compression_fallback_to_truncation
+ assert agent.state.summary == "semantic summary"
+ assert agent.validation_cycle_timer.context_compression_count == 1
+ assert not list(tmp_path.rglob("*.json"))
+
+
+@pytest.mark.asyncio
+async def test_offload_write_failure_keeps_context_and_does_not_count(tmp_path, monkeypatch):
+ agent = make_agent(tmp_path)
+ original = agent.state.context
+
+ def fail(*args, **kwargs):
+ raise OSError("disk full")
+
+ monkeypatch.setattr(compression, "write_json", fail)
+ with pytest.raises(OSError, match="disk full"):
+ await agent.compress_context()
+ assert agent.state.context is original
+ assert agent.state.summary == "existing summary"
+ assert agent.validation_cycle_timer.context_compression_count == 0
+
+
+@pytest.mark.asyncio
+async def test_cancellation_during_apply_finishes_consistent_offload_then_propagates(tmp_path, monkeypatch):
+ agent = make_agent(tmp_path)
+ started, release = asyncio.Event(), asyncio.Event()
+
+ async def clear_cache(messages):
+ started.set()
+ await release.wait()
+
+ monkeypatch.setattr(agent, "_clear_unreserved_read_cache", clear_cache)
+ task = asyncio.create_task(agent.compress_context())
+ await started.wait()
+ task.cancel()
+ release.set()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert len(list(tmp_path.rglob("context-*.json"))) == 1
+ assert "offloaded" in agent.state.summary
+ assert len(agent.state.context) == 2
+
+
+@pytest.mark.asyncio
+async def test_missing_directory_or_unsplittable_context_fails_explicitly(tmp_path, monkeypatch):
+ agent = make_agent(tmp_path)
+ agent.fallback_directory = None
+ with pytest.raises(RuntimeError, match="fallback_directory"):
+ await agent.compress_context()
+ agent.fallback_directory = tmp_path / "offloads"
+
+ async def no_old_messages(*args):
+ return [], agent.state.context
+
+ monkeypatch.setattr(agent, "_split_context_for_compression", no_old_messages)
+ with pytest.raises(RuntimeError, match="cannot remove any context"):
+ await agent.compress_context()
+ assert not list(tmp_path.rglob("*.json"))
+
+
+def test_wrapper_uses_host_offload_directory(tmp_path):
+ wrapper = optimizer.OptimizerAgentWrapper(name="optimizer")
+ agent = wrapper._create_agent(
+ options={"_context_fallback_directory": tmp_path / "logs/context-offloads"},
+ name="optimizer",
+ system_prompt="system",
+ model=FailingCompressionModel(),
+ )
+ assert agent.fallback_directory == tmp_path / "logs/context-offloads"
diff --git a/plugins/home/tests/test_optimizer_cycle.py b/plugins/home/tests/test_optimizer_cycle.py
new file mode 100644
index 00000000..04cd50f8
--- /dev/null
+++ b/plugins/home/tests/test_optimizer_cycle.py
@@ -0,0 +1,297 @@
+"""Meta-ReMe cycle timing parity with fake clocks and isolated evaluation evidence."""
+
+# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access
+# pylint: disable=redefined-outer-name,unused-argument
+
+import asyncio
+import copy
+from types import SimpleNamespace
+
+import pytest
+from agentscope.agent import ModelConfig, ReActConfig
+from agentscope.message import HintBlock, TextBlock, UserMsg
+from agentscope.model import ChatResponse, ChatUsage
+from agentscope.state import AgentState
+from reme_home.agent import optimizer
+from reme_home.agent.bounded import BoundedReActAgent
+from reme_home.evaluation.artifacts import write_json
+from test_bounded_agent import RecordingModel
+
+
+def make_agent(timer, model=None):
+ return optimizer.OptimizerAgent(
+ name="optimizer",
+ system_prompt="Improve general memory mechanisms.",
+ model=model or RecordingModel(),
+ validation_cycle_timer=timer,
+ model_config=ModelConfig(max_retries=0),
+ react_config=ReActConfig(max_iters=100),
+ )
+
+
+def cycle_hints(messages, source="meta_reme_validation_timer"):
+ return [
+ block
+ for message in messages
+ for block in message.content
+ if isinstance(block, HintBlock) and block.source == source
+ ]
+
+
+@pytest.mark.parametrize("budget", [True, 0, -1, float("nan"), float("inf"), "10"])
+def test_cycle_rejects_invalid_budget(budget):
+ with pytest.raises(ValueError, match="positive"):
+ optimizer.ValidationCycleTimer(budget)
+
+
+@pytest.mark.asyncio
+async def test_cycle_injects_dev_meta_reminders_and_resets():
+ now = [0.0]
+ timer = optimizer.ValidationCycleTimer(100, clock=lambda: now[0])
+
+ class SnapshotModel(RecordingModel):
+ async def __call__(self, **kwargs):
+ return await super().__call__(**copy.deepcopy(kwargs))
+
+ agent = make_agent(timer, SnapshotModel())
+ timer.record_context_compression()
+ timer.record_context_compression()
+ delivered = []
+ for minute in (49, 50, 74, 75, 90, 100, 109, 110, 120):
+ now[0] = minute * 60
+ _ = [item async for item in agent._reasoning()]
+ reminders = [block.hint for block in cycle_hints(agent.model.calls[-1]["messages"])]
+ if minute in (50, 75, 90, 100, 110, 120):
+ delivered.append(reminders[-1])
+ assert timer.next_reminder() is None
+ for percentage, text in zip((50, 75, 90, 100, 110, 120), delivered):
+ assert f"({percentage}% of its 100 minutes soft time budget)" in text
+ assert "2 context compressions since the last scored validation" in text
+ assert "start validation within 50 minutes" in delivered[0]
+ assert "start validation within 10 minutes" in delivered[2]
+ assert "starting validation as soon as possible" in delivered[3]
+ persistent = cycle_hints(agent.state.context)
+ assert len(persistent) == 3
+ for percentage, message in zip((75, 100, 120), persistent):
+ assert f"({percentage}%" in message.hint
+ timer.reset()
+ assert timer.context_compression_count == 0
+ now[0] += 60
+ assert timer.next_reminder() is None
+ # Reset keeps historical persistent messages and restarts alternation.
+ assert cycle_hints(agent.state.context) == persistent
+ now[0] = 170 * 60
+ assert timer.next_reminder().persistent is False
+
+
+def test_cycle_skips_old_thresholds_and_formats_seconds():
+ now = [0.0]
+ timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0])
+ now[0] = 59
+ reminder = timer.next_reminder()
+ assert "(90%" in reminder.content
+ assert "within 1 second." in reminder.content
+ now[0] = 101
+ assert "(160%" in timer.next_reminder().content
+ assert timer.next_reminder() is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("failure", [RuntimeError("model unavailable"), asyncio.CancelledError()])
+async def test_cycle_ephemeral_cleanup_on_error_and_cancellation(monkeypatch, failure):
+ now = [0.0]
+ timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0])
+ agent = make_agent(timer)
+
+ async def fail(*args, **kwargs):
+ raise failure
+ yield # pylint: disable=unreachable
+
+ monkeypatch.setattr(BoundedReActAgent, "_reasoning", fail)
+ now[0] = 30
+ with pytest.raises(type(failure)):
+ _ = [item async for item in agent._reasoning()]
+ assert not cycle_hints(agent.state.context)
+ now[0] = 45
+ with pytest.raises(type(failure)):
+ _ = [item async for item in agent._reasoning()]
+ assert len(cycle_hints(agent.state.context)) == 1 # The second reminder is persistent.
+
+
+@pytest.mark.asyncio
+async def test_cycle_counts_only_completed_context_replacements(monkeypatch):
+ timer = optimizer.ValidationCycleTimer(1)
+ agent = make_agent(timer)
+
+ async def unchanged(self):
+ pass
+
+ monkeypatch.setattr(BoundedReActAgent, "compress_context", unchanged)
+ await agent.compress_context()
+ assert timer.context_compression_count == 0
+
+ async def compressed(self):
+ self.state.summary = "Retained previous evidence."
+ self.state.context = []
+
+ monkeypatch.setattr(BoundedReActAgent, "compress_context", compressed)
+ await agent.compress_context()
+ assert timer.context_compression_count == 1
+
+ async def failed(self):
+ raise RuntimeError("compression unavailable")
+
+ monkeypatch.setattr(BoundedReActAgent, "compress_context", failed)
+ with pytest.raises(RuntimeError, match="unavailable"):
+ await agent.compress_context()
+ assert timer.context_compression_count == 1
+
+
+def write_history(home, started):
+ write_json(
+ home / "sampling/state.json",
+ {
+ "schema_version": 2,
+ "round": len(started),
+ "successful_round": 0,
+ "started_validations": {key: index for index, key in enumerate(started, 1)},
+ "processed_validations": [],
+ "cases": {},
+ "queries": {},
+ },
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("timed", [False, True])
+@pytest.mark.parametrize(
+ "outcome,reset",
+ [
+ ("completed", True),
+ ("success", True),
+ ("candidate_failure", True),
+ ("completed_infra_error", True),
+ ("debug", False),
+ ("rejected", False),
+ ("failed", False),
+ ("partial", False),
+ ("not_started", False),
+ ("preexisting", False),
+ ("cancelled", False),
+ ],
+)
+async def test_evaluation_reset_boundary_matches_dev_meta(tmp_path, outcome, reset, timed):
+ home = tmp_path
+ previous = ["init/initial/1", "candidate/previous/1"]
+ key = "candidate/current/1"
+ if outcome == "preexisting":
+ previous.append(key)
+ write_history(home, previous)
+ now = [0.0]
+ timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0])
+ timer.record_context_compression()
+ now[0] = 45
+ timer.next_reminder() # First reminder is ephemeral.
+ state = AgentState()
+ old = UserMsg(name="meta_reme_validation_timer", content="Previous persistent reminder.")
+ state.context.append(old)
+
+ class EvaluationJob:
+ enabled = is_started = True
+
+ async def __call__(self, **kwargs):
+ assert kwargs["debug"] is (outcome == "debug")
+ # Tool execution time remains part of the previous cycle until completion.
+ now[0] = 90
+ if outcome == "cancelled":
+ raise asyncio.CancelledError()
+ if outcome not in {"debug", "rejected", "not_started", "preexisting"}:
+ write_history(home, [*previous, key])
+ manifest = {
+ "commit_sha": "1b2cdcc" + "0" * 33,
+ "status": "failed" if outcome in {"failed", "partial"} else "completed",
+ "mode": "debug" if outcome == "debug" else "scored",
+ "dataset_kind": "training-set",
+ "sandbox_started_at": None if outcome == "not_started" else "2026-10-31T00:00:00Z",
+ }
+ run = home / "evaluations" / key
+ write_json(run / "manifest.json", manifest)
+ status = outcome if outcome in {"success", "rejected", "failed", "partial"} else "completed"
+ return SimpleNamespace(
+ success=outcome not in {"completed_infra_error", "rejected", "failed", "partial"},
+ answer={"status": status, "result_dir": str(run), "mean_query_score": 0.0},
+ metadata={},
+ )
+
+ wrapper = SimpleNamespace(app_context=SimpleNamespace(jobs={"home_evaluate": EvaluationJob()}))
+ tool = optimizer.evaluation_tool(wrapper, {}, home=home, timer=timer if timed else None)
+ assert set(tool.input_schema["properties"]) == {"debug"}
+ if outcome == "cancelled":
+ with pytest.raises(asyncio.CancelledError):
+ await tool._func(debug=False, _agent_state=state)
+ else:
+ await tool._func(debug=outcome == "debug", _agent_state=state)
+ assert len(state.context) == 1 # The tool cannot insert anything before its result is written.
+ pending = state.middle_context.get(optimizer.PENDING_CYCLE_TRANSITIONS, [])
+ assert len(pending) == int(reset)
+ agent = make_agent(None)
+ agent.state = state
+ _ = [item async for item in agent._reasoning()]
+ transitions = cycle_hints(state.context, "meta_reme_validation_cycle_transition")
+ assert len(transitions) == int(reset)
+ assert state.context[0] is old
+ if reset:
+ assert transitions[0].hint == (
+ "Formal validation 2 completed successfully for commit 1b2cdcc. "
+ "The previous cycle timer has been reset, and all timer reminders for cycle 2 are now expired. "
+ "Remaining formal validations: 8. Start cycle 3."
+ )
+ if reset and timed:
+ assert timer.context_compression_count == 0
+ assert timer.next_reminder() is None
+ now[0] += 30
+ assert timer.next_reminder().persistent is False
+ else:
+ assert timer.context_compression_count == 1
+ assert "(150%" in timer.next_reminder().content
+
+
+def test_wrapper_owns_fresh_cycle_timers_and_supports_disabling():
+ wrapper = optimizer.OptimizerAgentWrapper(name="optimizer", validation_cycle_time_budget_minutes=150)
+ first, second = wrapper.create_cycle_timer(), wrapper.create_cycle_timer()
+ assert first is not second
+ assert first.time_budget_minutes == 150
+ first.record_context_compression()
+ assert second.context_compression_count == 0
+ assert wrapper.create_cycle_timer({"validation_cycle_time_budget_minutes": None}) is None
+ legacy = optimizer.OptimizerAgentWrapper(name="optimizer", time_budget_minutes=75)
+ assert legacy.create_cycle_timer().time_budget_minutes == 75
+ assert legacy.create_cycle_timer({"validation_cycle_time_budget_minutes": 30}).time_budget_minutes == 30
+
+
+@pytest.mark.asyncio
+async def test_temporary_and_persistent_reminders_preserve_total_reply_usage():
+ now = [0.0]
+
+ class AdvancingModel(RecordingModel):
+ async def __call__(self, **kwargs):
+ result = await super().__call__(**kwargs)
+ now[0] = 30 if len(self.calls) == 1 else 45
+ return result
+
+ model = AdvancingModel(
+ final=ChatResponse(
+ content=[TextBlock(text="final report")],
+ is_last=True,
+ usage=ChatUsage(input_tokens=2, output_tokens=3, time=0),
+ ),
+ )
+ agent = make_agent(optimizer.ValidationCycleTimer(1, clock=lambda: now[0]), model)
+ agent.react_config.max_iters = 2
+ reply = await agent.reply(UserMsg(name="user", content="Investigate"))
+ assert len(model.calls) == 3
+ assert reply.usage.input_tokens == 6
+ assert reply.usage.output_tokens == 9
+ retained = cycle_hints(agent.state.context)
+ assert len(retained) == 1
+ assert "(75%" in retained[0].hint
diff --git a/plugins/home/tests/test_validation_budget.py b/plugins/home/tests/test_validation_budget.py
new file mode 100644
index 00000000..ef222cc8
--- /dev/null
+++ b/plugins/home/tests/test_validation_budget.py
@@ -0,0 +1,142 @@
+"""Formal budget accounting through real evaluation Jobs and fake Docker boundaries."""
+
+# pylint: disable=missing-function-docstring,missing-class-docstring,unused-argument,protected-access
+
+import asyncio
+
+import pytest
+from reme_home.agent.optimizer import OptimizerAgentWrapper, validation_budget_tool
+from reme_home.evaluation import component as evaluation
+from reme_home.evaluation.artifacts import write_json
+from reme_home.evaluation.budget import validation_budget_status
+from reme_home.evaluation.inputs import EvaluationSpec
+from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
+ FakeFactory,
+ commit_candidate,
+ home_application,
+ preparation_inputs,
+ prepared_home,
+)
+
+
+def seed_history(home, keys):
+ write_json(
+ home / "sampling/state.json",
+ {
+ "schema_version": 2,
+ "round": len(keys),
+ "successful_round": 0,
+ "cases": {},
+ "queries": {},
+ "started_validations": {key: index for index, key in enumerate(keys, 1)},
+ "processed_validations": [],
+ },
+ )
+
+
+@pytest.mark.parametrize(
+ "used,phase,remaining",
+ [
+ (0, "structural_exploration", 10),
+ (4, "structural_exploration", 6),
+ (5, "evidence_driven_refinement", 5),
+ (9, "evidence_driven_refinement", 1),
+ (10, "exhausted", 0),
+ (11, "exhausted", 0),
+ ],
+)
+def test_dev_meta_phase_boundaries(tmp_path, used, phase, remaining):
+ seed_history(tmp_path, ["init/first/1", *(f"candidate/hash/{index}" for index in range(used))])
+ result = validation_budget_status(tmp_path, 10)
+ assert result["used_validations"] == used
+ assert result["remaining_validations"] == remaining
+ assert result["optimization_phase"] == phase
+ assert result["gated_validation_count"] == 5
+ assert result["next_validation_guidance"]
+
+
+@pytest.mark.parametrize("invalid", [True, False, 0, -1, 1.5, "10", None])
+def test_invalid_optimizer_budgets_rejected(invalid):
+ wrapper = OptimizerAgentWrapper(name="optimizer", max_validations=invalid)
+ with pytest.raises(ValueError, match="positive integer"):
+ wrapper.validation_limit()
+ if invalid is not None:
+ with pytest.raises(ValueError):
+ EvaluationSpec(max_validations=invalid)
+
+
+def test_budgeted_evaluation_cannot_disable_started_history():
+ with pytest.raises(ValueError, match="record_history"):
+ EvaluationSpec(max_validations=10, record_history=False)
+ assert EvaluationSpec(debug=True, max_validations=1, record_history=False).debug
+ assert OptimizerAgentWrapper(name="optimizer").validation_limit() == 10
+
+
+@pytest.mark.asyncio
+async def test_budget_tool_reports_persisted_count_without_mutation(tmp_path):
+ seed_history(tmp_path, ["init/base/1", "candidate/hash/1"])
+ before = (tmp_path / "sampling/state.json").read_bytes()
+ tool = validation_budget_tool(tmp_path, 3)
+ assert tool.name == "get_validation_budget"
+ result = await tool._func()
+ assert result["remaining_validations"] == 2
+ assert result["optimization_phase"] == "evidence_driven_refinement"
+ assert (tmp_path / "sampling/state.json").read_bytes() == before
+
+
+@pytest.mark.asyncio
+async def test_exhaustion_blocks_before_pool_and_survives_cache_rebuild(app, prepared):
+ baseline = await app.run_job("home_evaluate")
+ assert baseline.success
+ assert validation_budget_status(prepared, 1)["used_validations"] == 0
+ commit_candidate(prepared)
+ result = await app.run_job("home_evaluate", max_validations=1)
+ assert result.success, result.answer
+ assert result.answer["validation_budget"]["remaining_validations"] == 0
+ count = len(FakeFactory.instances)
+ # A fresh status lookup and cache rebuild both retain the charge.
+ (prepared / "sampling/state.json").unlink()
+ assert validation_budget_status(prepared, 1)["used_validations"] == 1
+ commit_candidate(prepared)
+ rejected = await app.run_job("home_evaluate", max_validations=1)
+ assert not rejected.success
+ assert rejected.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED"
+ assert rejected.answer["result_dir"] is None
+ assert len(FakeFactory.instances) == count
+ debug = await app.run_job("home_evaluate", debug=True, max_validations=1)
+ assert debug.success, debug.answer
+ assert "validation_budget" not in debug.answer
+ assert validation_budget_status(prepared, 1)["used_validations"] == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("outcome", ["candidate_failure", "infra_failure", "cancelled", "prestart_failure"])
+async def test_charge_depends_on_first_sandbox_start_not_result(app, prepared, monkeypatch, outcome):
+ commit_candidate(prepared)
+ if outcome == "candidate_failure":
+ monkeypatch.setattr(FakeFactory, "fail_build", "1")
+ elif outcome == "infra_failure":
+ monkeypatch.setattr(FakeFactory, "bad_export", True)
+ elif outcome == "cancelled":
+ monkeypatch.setattr(FakeFactory, "block", asyncio.Event())
+ else:
+
+ class PrestartFailureFactory(FakeFactory):
+ async def create_cases(self, ids, **kwargs):
+ raise RuntimeError("sandbox initialization failed")
+
+ monkeypatch.setattr(evaluation, "DockerReMeSandboxFactory", PrestartFailureFactory)
+ task = asyncio.create_task(app.run_job("home_evaluate", max_validations=1))
+ if outcome == "cancelled":
+ while not FakeFactory.instances:
+ await asyncio.sleep(0)
+ await FakeFactory.instances[-1].entered.wait()
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ else:
+ await task
+ expected = int(outcome != "prestart_failure")
+ assert validation_budget_status(prepared, 1)["used_validations"] == expected
+ (prepared / "sampling/state.json").unlink(missing_ok=True)
+ assert validation_budget_status(prepared, 1)["used_validations"] == expected
diff --git a/plugins/home/tests/test_workflows.py b/plugins/home/tests/test_workflows.py
new file mode 100644
index 00000000..2c9dcd82
--- /dev/null
+++ b/plugins/home/tests/test_workflows.py
@@ -0,0 +1,496 @@
+"""Minimal Home agents exercise real Jobs, Git and AS tools with fake model/Docker boundaries."""
+
+# pylint: disable=missing-function-docstring,missing-class-docstring,redefined-outer-name
+# pylint: disable=unused-argument,protected-access
+
+import asyncio
+import json
+import sys
+from importlib import import_module
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import yaml
+from agentscope.formatter import OpenAIChatFormatter
+from agentscope.message import HintBlock, TextBlock, ToolCallBlock
+from agentscope.model import ChatResponse, ChatUsage
+from agentscope.tool import LocalBackend
+from jsonschema import Draft202012Validator
+from reme_home.agent import optimizer as optimizer_module, diagnostic as diagnostic_module, workflow_common, case_study
+from reme_home.evaluation.artifacts import read_json, write_json
+from reme_home.preparation.code import git
+from reme_home.workspace import _initialization_lock
+
+from test_bounded_agent import RecordingModel
+from test_case_study import RecordingSandbox
+from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import
+ FakeFactory,
+ home_application,
+ preparation_inputs,
+ prepared_home,
+)
+
+
+class ScriptedModel(RecordingModel):
+ def __init__(self, calls=(), final="Supported diagnostic report."):
+ super().__init__()
+ self.script = list(calls)
+ self.report = final
+
+ async def __call__(self, **kwargs):
+ index = len(self.calls)
+ self.calls.append(kwargs)
+ if index < len(self.script):
+ name, arguments = self.script[index]
+ arguments = arguments() if callable(arguments) else arguments
+ content = [ToolCallBlock(id=f"call-{index}", name=name, input=json.dumps(arguments))]
+ else:
+ final = self.report() if callable(self.report) else self.report
+ content = [TextBlock(text=final)]
+ return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0))
+
+
+@pytest.fixture
+def sandbox(monkeypatch):
+ policies = []
+
+ def create(**kwargs):
+ # Mock only subprocess isolation. Writes are restricted by the real
+ # sandbox in the dedicated OS tests below, not by this local fake.
+ policies.append(kwargs)
+ local = LocalBackend()
+ return SimpleNamespace(backend=local, exec_shell=local.exec_shell)
+
+ monkeypatch.setattr(workflow_common, "create_case_study_sandbox", create)
+ monkeypatch.setattr(case_study, "create_case_study_sandbox", RecordingSandbox)
+ return policies
+
+
+def set_model(app, name, model):
+ wrapper = app.context.components["agent_wrapper"][name]
+ wrapper.as_llm = SimpleNamespace(model=model)
+ return wrapper
+
+
+async def baseline(app):
+ result = await app.run_job("home_evaluate")
+ assert result.success, result.answer
+ return result.answer["result_dir"]
+
+
+def final_report(home, branch="init", path=None):
+ repo = home / "code/repo"
+ if path is None:
+ path = sorted((home / "evaluations" / branch).glob("*/*/summary.json"))[-1].parent
+ return json.dumps(
+ {
+ "branch_name": branch,
+ "commit_sha": git(repo, "rev-parse", "HEAD"),
+ "evaluation_path": str(path),
+ "report": "Selected using matching formal evidence; sampled coverage limits confidence.",
+ },
+ )
+
+
+@pytest.mark.asyncio
+async def test_real_optimization_diagnoses_edits_debugs_and_evaluates(app, prepared, sandbox, monkeypatch):
+ home = prepared
+ now, timers = [0.0], []
+ timer_class = optimizer_module.ValidationCycleTimer
+
+ def create_timer(budget):
+ timer = timer_class(budget, clock=lambda: now[0])
+ timers.append(timer)
+ return timer
+
+ class TimedModel(ScriptedModel):
+ async def __call__(self, **kwargs):
+ formatted = await OpenAIChatFormatter().format(kwargs["messages"])
+ pending = set()
+ for message in formatted:
+ if pending:
+ assert message["role"] == "tool", formatted
+ pending.remove(message["tool_call_id"])
+ else:
+ assert message["role"] != "tool", formatted
+ pending.update(call["id"] for call in message.get("tool_calls", []))
+ assert not pending
+ response = await super().__call__(**kwargs)
+ now[0] += 60 * 60
+ return response
+
+ monkeypatch.setattr(optimizer_module, "ValidationCycleTimer", create_timer)
+ target = home / "code/repo/plugins/beam/src/reme_beam/__init__.py"
+ baseline_path = await baseline(app)
+ diagnostic = ScriptedModel(
+ [("home_case_study", {"evaluation_path": baseline_path, "case_ids": ["1"], "focus": "Trace wrong answers"})],
+ )
+ set_model(app, "home_diagnostic", diagnostic)
+ set_model(app, "home_case_study", ScriptedModel())
+ optimizer = TimedModel(
+ [
+ ("get_validation_budget", {}),
+ ("home_diagnose", {"evaluation_paths": [baseline_path]}),
+ ("home_create_candidate_branch", {"branch_name": "mechanism"}),
+ ("Read", {"file_path": str(target)}),
+ ("Edit", {"file_path": str(target), "old_string": "value = 1", "new_string": "value = 2"}),
+ ("home_commit_candidate", {"message": "Change mechanism"}),
+ ("home_evaluate", {"debug": True}),
+ ("home_evaluate", {"debug": False}),
+ ],
+ final=lambda: final_report(home, "mechanism"),
+ )
+ set_model(app, "home_optimizer", optimizer)
+ response = await app.run_job("home_optimize", instruction="Improve general retrieval")
+ assert response.success, response.answer
+ assert target.read_text() == "value = 2\n"
+ assert response.answer["branch_name"] == "mechanism"
+ assert response.answer["status"] == "completed"
+ transitions = [
+ block.hint
+ for message in optimizer.calls[-1]["messages"]
+ for block in message.content
+ if isinstance(block, HintBlock) and block.source == "meta_reme_validation_cycle_transition"
+ ]
+ assert len(transitions) == 1
+ assert "Formal validation 1 completed successfully" in transitions[0]
+ assert "Remaining formal validations: 9. Start cycle 2." in transitions[0]
+ assert len(timers) == 1
+ assert timers[0]._started_at == now[0] - 60 * 60 # Reset before the final reasoning call.
+ assert any(
+ isinstance(block, HintBlock) and block.source == "meta_reme_validation_timer"
+ for message in optimizer.calls[-1]["messages"]
+ for block in message.content
+ )
+ assert response.metadata["usage"]["input_tokens"] == 2 * len(optimizer.calls)
+ assert response.metadata["usage"]["output_tokens"] == 3 * len(optimizer.calls)
+ assert "sampled coverage" in response.answer["report"]
+ assert read_json(Path(response.metadata["optimization_report_path"]))["status"] == "completed"
+ assert len(diagnostic.calls) == 2
+ source = next((home / "logs/diagnostic-trajectories").glob("*.json"))
+ trace = read_json(source)
+ assert trace["session_id"] and any("Improve general retrieval" in str(msg) for msg in trace["messages"])
+ assert "permission_context" not in trace
+ schemas = {item["function"]["name"]: item["function"]["parameters"] for item in optimizer.calls[0]["tools"]}
+ assert set(schemas["home_evaluate"]["properties"]) == {"debug"}
+ assert "caller_trajectory" not in schemas["home_diagnose"]["properties"]
+ assert "get_validation_budget" in schemas
+ assert not any("equivalence" in name for name in schemas)
+ for manifest_path in (home / "evaluations").glob("*/*/*/manifest.json"):
+ manifest = read_json(manifest_path)
+ assert manifest["execution_mode"] == "full_validation"
+ assert any(read_json(path)["mode"] == "debug" for path in (home / "evaluations").glob("*/*/*/manifest.json"))
+ diagnostic_schemas = {item["function"]["name"] for item in diagnostic.calls[0]["tools"]}
+ assert "home_case_study" in diagnostic_schemas
+ assert not set(optimizer_module.MUTATION_JOBS) & diagnostic_schemas
+ assert "Edit" not in diagnostic_schemas and "home_evaluate" not in diagnostic_schemas
+ assert "Write" in diagnostic_schemas # Private scratch only.
+ assert any(str(target.parent.parent) in policy["writable_paths"] for policy in sandbox)
+ assert all(str(home / "code/repo") not in policy["writable_paths"] for policy in sandbox)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"])
+async def test_benchmark_optimizer_applies_budgets_to_real_workflow(
+ app,
+ prepared,
+ inputs,
+ sandbox,
+ monkeypatch,
+ tmp_path,
+ benchmark,
+):
+ baseline_path = await baseline(app)
+ await app.close()
+ module = import_module(f"benchmark.{benchmark}.run_home")
+ config = tmp_path / "home.yaml"
+ config.write_text(
+ yaml.safe_dump(
+ {
+ "workspace_dir": str(prepared.parent),
+ "preparation": inputs[0],
+ "max_validations": 3,
+ "validation_cycle_time_budget_minutes": 30,
+ },
+ ),
+ )
+ application_class = type(app)
+ agents = []
+ model = ScriptedModel([("get_validation_budget", {})], final=lambda: final_report(prepared, path=baseline_path))
+
+ def factory(**kwargs):
+ application = application_class(**kwargs, log_to_console=False)
+ original_start = application.start
+
+ async def start():
+ await original_start()
+ wrapper = set_model(application, "home_optimizer", model)
+ original_create = wrapper._create_agent
+
+ def create(**options):
+ agent = original_create(**options)
+ agents.append(agent)
+ return agent
+
+ monkeypatch.setattr(wrapper, "_create_agent", create)
+
+ monkeypatch.setattr(application, "start", start)
+ return application
+
+ monkeypatch.setattr(module, "Application", factory)
+ result = await module.optimize(config, instruction="Keep the best existing candidate")
+ assert result["status"] == "completed"
+ assert result["branch_name"] == "init"
+ assert len(agents) == 1
+ assert agents[0].validation_cycle_timer.time_budget_minutes == 30
+ task = "\n".join(msg.get_text_content() or "" for msg in model.calls[0]["messages"])
+ assert '"max_validations": 3' in task
+ assert '"remaining_validations": 3' in task
+ assert len(FakeFactory.instances) == 1 # Only the earlier baseline ran.
+
+
+@pytest.mark.asyncio
+async def test_optimizer_creates_baseline_once_and_verifies_baseline_fallback(app, prepared, sandbox):
+ model = ScriptedModel(final=lambda: final_report(prepared))
+ set_model(app, "home_optimizer", model)
+ response = await app.run_job("home_optimize")
+ assert response.success, response.answer
+ assert len(FakeFactory.instances) == 1
+ assert response.answer["branch_name"] == "init"
+ run = response.answer["baseline_path"]
+ manifest = read_json(Path(run) / "manifest.json")
+ assert set(manifest["case_ids"]) == {"1", "2"} and not manifest["options"]["sampling"]
+ assert manifest["options"]["record_history"]
+ set_model(app, "home_optimizer", ScriptedModel(final=lambda: final_report(prepared)))
+ second = await app.run_job("home_optimize")
+ assert second.success, second.answer
+ assert len(FakeFactory.instances) == 1
+ assert second.answer["session_id"] != response.answer["session_id"]
+
+
+@pytest.mark.asyncio
+async def test_standalone_diagnosis_fresh_state_and_persisted_provenance(app, prepared, sandbox):
+ path = await baseline(app)
+ model = ScriptedModel()
+ set_model(app, "home_diagnostic", model)
+ first = await app.run_job("home_diagnose", evaluation_paths=[path])
+ second = await app.run_job("home_diagnose", evaluation_paths=[path], instruction="Compare failure clusters")
+ assert first.success and second.success
+ assert first.answer["session_id"] != second.answer["session_id"]
+ assert first.answer["trajectory_path"] != second.answer["trajectory_path"]
+ assert first.answer["evaluation_paths"] == [path]
+ record = read_json(Path(first.answer["diagnostic_report_path"]))
+ assert record["diagnosis"] == "Supported diagnostic report."
+ assert read_json(Path(first.answer["trajectory_path"])) == {"messages": [], "summary": None}
+ assert all(
+ len(policy["writable_paths"]) == 1 and "/tmp/diagnostic-" in policy["writable_paths"][0] for policy in sandbox
+ )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("change", ["infra", "partial", "debug", "replay", "wrong_commit"])
+async def test_bad_baseline_rejected_before_optimizer(app, prepared, sandbox, change):
+ path = Path(await baseline(app))
+ manifest = read_json(path / "manifest.json")
+ summary = read_json(path / "summary.json")
+ if change == "infra":
+ summary["case_status_counts"]["infra_error"] = 1
+ elif change == "partial":
+ manifest["case_ids"] = ["1"]
+ manifest["query_ids"] = {"1": manifest["query_ids"]["1"]}
+ elif change == "debug":
+ manifest["mode"] = "debug"
+ elif change == "replay":
+ manifest["execution_mode"] = "query_only_replay"
+ else:
+ manifest["commit_sha"] = "0" * 40
+ write_json(path / "manifest.json", manifest)
+ write_json(path / "summary.json", summary)
+ # Missing valid baseline must not silently switch away from a candidate.
+ response = await app.run_job("home_create_candidate_branch", branch_name="existing")
+ assert response.success
+ model = ScriptedModel()
+ set_model(app, "home_optimizer", model)
+ response = await app.run_job("home_optimize")
+ assert not response.success and "baseline is missing" in str(response.answer)
+ assert not model.calls and not sandbox
+ assert git(prepared / "code/repo", "branch", "--show-current") == "existing"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("change", ["dirty", "branch", "commit", "path", "infra", "debug", "replay", "invalid_json"])
+async def test_final_claim_rejected_without_discarding_changes(app, prepared, sandbox, change):
+ path = Path(await baseline(app))
+ target = prepared / "code/repo/plugins/beam/src/reme_beam/__init__.py"
+
+ def finish():
+ claim = json.loads(final_report(prepared, path=path))
+ if change == "dirty":
+ target.write_text("preserve unfinished changes\n")
+ elif change == "branch":
+ claim["branch_name"] = "invented"
+ elif change == "commit":
+ claim["commit_sha"] = "0" * 40
+ elif change == "path":
+ claim["evaluation_path"] = str(prepared / "tmp")
+ elif change == "infra":
+ summary = read_json(path / "summary.json")
+ summary["query_status_counts"]["infra_error"] = 1
+ write_json(path / "summary.json", summary)
+ elif change in {"debug", "replay"}:
+ manifest = read_json(path / "manifest.json")
+ manifest["mode" if change == "debug" else "execution_mode"] = (
+ "debug" if change == "debug" else "query_only_replay"
+ )
+ write_json(path / "manifest.json", manifest)
+ else:
+ return "not a structured final claim"
+ return json.dumps(claim)
+
+ set_model(app, "home_optimizer", ScriptedModel(final=finish))
+ response = await app.run_job("home_optimize")
+ assert not response.success
+ record = read_json(Path(response.metadata["optimization_report_path"]))
+ assert record["status"] == "failed" and record["error"]
+ if change == "dirty":
+ assert target.read_text() == "preserve unfinished changes\n"
+
+
+@pytest.mark.asyncio
+async def test_infrastructure_baseline_failure_never_calls_optimizer(app, prepared, sandbox, monkeypatch):
+ async def broken(*args, **kwargs):
+ raise RuntimeError("Docker unavailable")
+
+ monkeypatch.setattr(FakeFactory, "create_cases", broken)
+ model = ScriptedModel()
+ set_model(app, "home_optimizer", model)
+ response = await app.run_job("home_optimize")
+ assert not response.success and not model.calls
+ assert "baseline failed" in str(response.answer)
+
+
+@pytest.mark.asyncio
+async def test_invalid_scope_schema_and_injected_trajectory(app, prepared, sandbox):
+ path = await baseline(app)
+ wrapper = set_model(app, "home_optimizer", ScriptedModel())
+ tool = diagnostic_module.diagnostic_tool(wrapper)
+ with pytest.raises(ValueError, match="host-owned"):
+ await tool._func(evaluation_paths=[path], caller_trajectory={"spoof": True})
+ for values in ([str(prepared / "tmp")], [path, path], [], ["../escape"]):
+ result = await app.run_job("home_diagnose", evaluation_paths=values)
+ assert not result.success
+ result = await app.run_job("home_diagnose", evaluation_paths=[path], _allowed_paths=[path])
+ assert not result.success and not sandbox
+ for name in ("home_optimize", "home_diagnose"):
+ Draft202012Validator.check_schema(app.context.jobs[name].parameters)
+ assert "caller_trajectory" not in app.context.jobs["home_diagnose"].parameters["properties"]
+ with pytest.raises(ValueError, match="execution settings"):
+ optimizer_module.evaluation_options({"reuse_construction_from": path})
+
+
+@pytest.mark.asyncio
+async def test_single_optimizer_lock_and_cancellation_cleanup(app, prepared, sandbox, monkeypatch):
+ await baseline(app)
+ entered, release = asyncio.Event(), asyncio.Event()
+
+ async def blocked(*args, **kwargs):
+ entered.set()
+ await release.wait()
+
+ monkeypatch.setattr(optimizer_module, "workflow_reply", blocked)
+ running = asyncio.create_task(app.run_job("home_optimize"))
+ await asyncio.wait_for(entered.wait(), 5)
+ second = await app.run_job("home_optimize")
+ assert not second.success and "already in progress" in str(second.answer)
+ running.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await running
+ reports = list((prepared / "logs/optimization-reports").glob("*.json"))
+ assert len(reports) == 1 and read_json(reports[0])["status"] == "cancelled"
+ with _initialization_lock(prepared / "metadata/optimization.lock"):
+ pass
+
+
+@pytest.mark.asyncio
+async def test_metadata_link_rejected_before_opening_any_lock(app, prepared, tmp_path):
+ outside = tmp_path / "outside"
+ outside.mkdir()
+ state = prepared / "metadata"
+ saved = prepared / "saved-metadata"
+ state.rename(saved)
+ state.symlink_to(outside, target_is_directory=True)
+ try:
+ result = await app.run_job("home_optimize")
+ assert not result.success and "real directory" in str(result.answer)
+ with pytest.raises(ValueError, match="real directory"):
+ optimizer_module.baseline_state(prepared)
+ with pytest.raises(ValueError, match="real directory"):
+ optimizer_module.verify_final(prepared, None)
+ assert not list(outside.iterdir())
+ finally:
+ state.unlink()
+ saved.rename(state)
+
+
+@pytest.mark.asyncio
+async def test_workflow_sandbox_enforces_read_write_and_git_boundaries(tmp_path):
+ home = tmp_path / "home"
+ repo = home / "repo"
+ candidate = repo / "src"
+ scratch = home / "scratch"
+ evidence = home / "evidence"
+ for directory in (candidate, repo / ".git", scratch, evidence):
+ directory.mkdir(parents=True, exist_ok=True)
+ target = candidate / "module.py"
+ target.write_text("old\n")
+ secret = tmp_path / "outside.txt"
+ secret.write_text("outside")
+ (evidence / "result.json").write_text("evidence")
+ for role in ("optimizer", "diagnostic"):
+ scope = {
+ "scratch": str(scratch),
+ "read_only_paths": [str(repo), str(evidence)],
+ "writable_paths": [str(scratch), *([str(candidate)] if role == "optimizer" else [])],
+ }
+ backend = workflow_common.WorkflowSandbox(scope, memory_limit_mib=0)
+ assert (await backend.read_file(str(target))) in (b"old\n", b"new\n")
+ assert (await backend.stat(str(target))).size_bytes > 0
+ assert "module.py" in await backend.list_dir(str(candidate))
+ await backend.write_file(str(scratch / "analysis.py"), b"print('analysis')\n")
+ assert (await backend.exec_shell([sys.executable, "-I", "-B", str(scratch / "analysis.py")])).ok()
+ for path in (repo / ".git/config", evidence / "result.json", secret):
+ with pytest.raises(OSError):
+ await backend.write_file(str(path), b"forbidden")
+ with pytest.raises(OSError):
+ await backend.read_file(str(secret))
+ if role == "optimizer":
+ await backend.write_file(str(target), b"new\n")
+ # Replaced roots after managed Git operations must not retain stale inodes.
+ replacement = candidate / "replacement"
+ replacement.write_text("new\n")
+ replacement.replace(target)
+ assert await backend.read_file(str(target)) == b"new\n"
+ else:
+ with pytest.raises(OSError):
+ await backend.write_file(str(target), b"forbidden")
+ assert secret.read_text() == "outside" and (evidence / "result.json").read_text() == "evidence"
+ # A single-file candidate root must survive a managed checkout replacing
+ # its inode without accidentally granting writes to its containing directory.
+ file_scope = {
+ "scratch": str(scratch),
+ "read_only_paths": [str(repo)],
+ "writable_paths": [str(scratch), str(target)],
+ }
+ backend = workflow_common.WorkflowSandbox(file_scope, memory_limit_mib=0)
+ await backend.write_file(str(target), b"file root\n")
+ replacement = candidate / "replacement"
+ replacement.write_text("replaced root\n")
+ replacement.replace(target)
+ assert await backend.read_file(str(target)) == b"replaced root\n"
+ with pytest.raises(OSError):
+ await backend.write_file(str(candidate / "sibling.py"), b"forbidden")
+ target.unlink()
+ target.symlink_to(secret)
+ with pytest.raises(ValueError, match="symlinks"):
+ await backend.read_file(str(target))