From 9d80f4b14ee72a8af3da2554664596abdbd02b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=85=B1=E7=89=9B=E8=82=89?= Date: Sat, 31 Oct 2026 16:55:54 +0800 Subject: [PATCH] feat(home): migrate optimization workflows and evaluation budgets --- benchmark/beam/README.md | 21 + benchmark/beam/README_ZH.md | 17 + benchmark/beam/config_home.yaml | 9 +- benchmark/beam/run_home.py | 137 ++++- benchmark/home_runner.py | 93 ---- benchmark/longmemeval/README.md | 21 + benchmark/longmemeval/README_ZH.md | 17 + benchmark/longmemeval/config_home.yaml | 5 +- benchmark/longmemeval/run_home.py | 137 ++++- plugins/home/README.md | 147 +++++ plugins/home/src/reme_home/agent/bounded.py | 3 +- .../home/src/reme_home/agent/compression.py | 126 +++++ .../home/src/reme_home/agent/diagnostic.py | 159 ++++++ .../home/src/reme_home/agent/diagnostic.yaml | 32 ++ plugins/home/src/reme_home/agent/optimizer.py | 513 ++++++++++++++++++ .../home/src/reme_home/agent/optimizer.yaml | 60 ++ .../src/reme_home/agent/workflow_common.py | 230 ++++++++ .../home/src/reme_home/evaluation/budget.py | 43 ++ .../src/reme_home/evaluation/component.py | 21 +- .../home/src/reme_home/evaluation/inputs.py | 3 + plugins/home/src/reme_home/evaluation/step.py | 3 + plugins/home/src/reme_home/plugin.yaml | 83 +++ plugins/home/tests/test_benchmark_runner.py | 216 +++++++- .../home/tests/test_optimizer_compression.py | 215 ++++++++ plugins/home/tests/test_optimizer_cycle.py | 297 ++++++++++ plugins/home/tests/test_validation_budget.py | 142 +++++ plugins/home/tests/test_workflows.py | 496 +++++++++++++++++ 27 files changed, 3121 insertions(+), 125 deletions(-) delete mode 100644 benchmark/home_runner.py create mode 100644 plugins/home/src/reme_home/agent/compression.py create mode 100644 plugins/home/src/reme_home/agent/diagnostic.py create mode 100644 plugins/home/src/reme_home/agent/diagnostic.yaml create mode 100644 plugins/home/src/reme_home/agent/optimizer.py create mode 100644 plugins/home/src/reme_home/agent/optimizer.yaml create mode 100644 plugins/home/src/reme_home/agent/workflow_common.py create mode 100644 plugins/home/src/reme_home/evaluation/budget.py create mode 100644 plugins/home/tests/test_optimizer_compression.py create mode 100644 plugins/home/tests/test_optimizer_cycle.py create mode 100644 plugins/home/tests/test_validation_budget.py create mode 100644 plugins/home/tests/test_workflows.py diff --git a/benchmark/beam/README.md b/benchmark/beam/README.md index 8834c85f..9045b865 100644 --- a/benchmark/beam/README.md +++ b/benchmark/beam/README.md @@ -164,6 +164,27 @@ The top-level `sampling_policy` configures Home evaluation defaults: `max_cases` `home.evaluation` Component. It still only prepares; it does not run evaluations. Each evaluation controls `sampling` and `record_history` separately. +The top-level `validation_cycle_time_budget_minutes` configures the host optimizer's +advisory time between completed formal evaluations. It defaults to 150 minutes; +set a positive number to change it or null to disable timed reminders. + +The top-level `max_validations` sets the optimizer's formal-evaluation budget +(positive integer, default 10). A slot is consumed at first sandbox startup, +including later failures or cancellations; initial baseline and debug are excluded. + +After preparation, run local one-shot optimization with the same configuration: + +```bash +python benchmark/beam/run_home.py --optimize +# Use the same --config and --workspace-dir overrides as preparation, if supplied. +python benchmark/beam/run_home.py --optimize --instruction "Improve retrieval quality" +``` + +This starts a fresh application, applies both budgets and the sampling policy, +runs `home_optimize`, and closes the application. It does not repeat preparation. +Python callers can use `await optimize(config_path, workspace_dir=...)` from the +same module. The default `run()` and CLI behavior remain preparation only. + Python callers can use the same lifecycle: ```python diff --git a/benchmark/beam/README_ZH.md b/benchmark/beam/README_ZH.md index 44cd7a93..42bd4ea6 100644 --- a/benchmark/beam/README_ZH.md +++ b/benchmark/beam/README_ZH.md @@ -154,4 +154,21 @@ Python 中也可调用 `await benchmark.beam.run_home.run(workspace_dir=...)`。 `home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling` 和 `record_history`。 +顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的 +建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。 + +顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。 +首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。 + +准备完成后,用同一份配置进行本地一次性优化: + +```bash +python benchmark/beam/run_home.py --optimize +``` + +若准备时指定了 `--config` 或 `--workspace-dir`,优化时传入相同参数。 +可用 `--instruction "优化检索质量"` 指定目标。该入口将配置中的两个预算和采样策略 +应用到实际 `home_optimize` 调用,不重复准备;Python 可调用同模块的 `await optimize(...)`。 +不加 `--optimize` 时仍只准备。 + 默认选择 BEAM **100K 全部 case**。 diff --git a/benchmark/beam/config_home.yaml b/benchmark/beam/config_home.yaml index 5f00d4cc..6011264a 100644 --- a/benchmark/beam/config_home.yaml +++ b/benchmark/beam/config_home.yaml @@ -1,5 +1,10 @@ -# Preparation only. Paths are relative to the ReMe repository root, not the cwd. +# Preparation by default; --optimize applies these settings to the prepared workspace. +# Paths are relative to the ReMe repository root, not the cwd. workspace_dir: benchmark/beam/workspaces/home-100K +# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug. +max_validations: 10 +# Target minutes between completed formal optimizer evaluations; null disables timed reminders. +validation_cycle_time_budget_minutes: 150 # Policy defaults for Home evaluation; each call still controls sampling=true/false. sampling_policy: max_cases: 5 @@ -18,8 +23,6 @@ preparation: source_ref: HEAD # Committed source only; resolved and saved as a full SHA. config: benchmark.yaml plugins: [beam] - candidate: - paths: [plugins/beam/src] debug: enabled: true sessions_per_case: 3 diff --git a/benchmark/beam/run_home.py b/benchmark/beam/run_home.py index 49270e72..d951dfcc 100644 --- a/benchmark/beam/run_home.py +++ b/benchmark/beam/run_home.py @@ -1,22 +1,145 @@ -"""Prepare the beam Home workspace, dataset and candidate code; do not evaluate.""" +"""Prepare the BEAM Home workspace, or optimize it with --optimize after preparation.""" -from pathlib import Path +import argparse +import asyncio +import json import sys +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field +import yaml # Support direct execution from any working directory as well as module imports. -_PROJECT_ROOT = Path(__file__).resolve().parents[2] -if str(_PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(_PROJECT_ROOT)) +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) -from benchmark.home_runner import main, prepare # noqa: E402 # pylint: disable=wrong-import-position +# pylint: disable=wrong-import-position +from reme_home.preparation.spec import PreparationSpec # noqa: E402 +from reme_home.judge import JUDGE_PLUGINS # noqa: E402 +from reme_home.sampling import SamplingPolicy # noqa: E402 + +from reme.application import Application # noqa: E402 + +# pylint: enable=wrong-import-position DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml") +class RunHomeConfig(BaseModel): + """Runner settings kept separate from Home's public preparation contract.""" + + model_config = ConfigDict(extra="forbid", strict=True) + workspace_dir: str = Field(min_length=1) + preparation: dict + sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy) + validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False) + max_validations: int = Field(default=10, ge=1) + + +def project_path(value: str, project_root: Path) -> Path: + """Use repository-relative paths consistently regardless of the caller's cwd.""" + if value.startswith("~"): + raise ValueError("Use absolute or project-relative paths, not home-relative paths") + return (project_root / value).resolve() + + +def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): + """Resolve source paths and validate explicit Home preparation parameters.""" + raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) + config = RunHomeConfig.model_validate(raw) + config.preparation.setdefault("candidate", {"paths": ["plugins/beam/src"]}) + spec = PreparationSpec.model_validate(config.preparation) + spec.benchmark.source = str(project_path(spec.benchmark.source, project_root)) + if spec.benchmark.evidence_source is not None: + spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root)) + spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root)) + workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root) + return workspace, spec.model_dump(mode="json"), config + + +def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): + """Return resolved preparation inputs, preserving the preparation-only caller contract.""" + workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root) + return workspace, spec + + +async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None): + """Apply the same runner configuration to each preparation or optimization lifecycle.""" + workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir) + app = Application( + workspace_dir=str(workspace), + plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]], + components={ + "home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}}, + "agent_wrapper": { + "home_optimizer": { + "validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes, + "max_validations": config.max_validations, + }, + }, + }, + service={"backend": "cli"}, + enable_logo=False, + log_to_file=False, + ) + try: + await app.start() + if optimize_workspace: + arguments = {} if instruction is None else {"instruction": instruction} + response = await app.run_job("home_optimize", **arguments) + else: + response = await app.run_job("home_prepare", spec=spec) + if not response.success: + raise RuntimeError(response.answer) + return {"workspace_dir": str(workspace), **response.answer} + finally: + await app.close() + + +async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict: + """Prepare once without running evaluations or optimization.""" + return await _run(config_path, workspace_dir=workspace_dir) + + +async def optimize(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir=None, instruction=None) -> dict: + """Optimize an already prepared workspace using this config's actual model budgets.""" + return await _run(config_path, workspace_dir=workspace_dir, optimize_workspace=True, instruction=instruction) + + async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict: """Prepare through the same Home Job as the CLI, then close the application.""" return await prepare(config_path, workspace_dir=workspace_dir) +def main(default_config: Path = DEFAULT_CONFIG) -> None: + """Run preparation or explicit local one-shot optimization with the benchmark config.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration") + parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)") + parser.add_argument("--optimize", action="store_true", help="Optimize an already prepared workspace") + parser.add_argument("--instruction", help="Optimization focus; requires --optimize") + args = parser.parse_args() + if args.instruction is not None and not args.optimize: + parser.error("--instruction requires --optimize") + if args.optimize: + result = asyncio.run(optimize(args.config, workspace_dir=args.workspace_dir, instruction=args.instruction)) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return + result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir)) + print( + json.dumps( + { + "workspace_dir": result["workspace_dir"], + "case_count": result["dataset"]["case_count"], + "query_count": result["dataset"]["query_count"], + "source_commit": result["code"]["source_commit"], + "initial_commit": result["code"]["initial_commit"], + }, + indent=2, + ), + ) + + if __name__ == "__main__": - main(DEFAULT_CONFIG) + main() diff --git a/benchmark/home_runner.py b/benchmark/home_runner.py deleted file mode 100644 index e4c2a23e..00000000 --- a/benchmark/home_runner.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Shared preparation-only entry point for benchmark Home experiments.""" - -import argparse -import asyncio -import json -from pathlib import Path - -from pydantic import BaseModel, ConfigDict, Field -import yaml -from reme_home.preparation.spec import PreparationSpec -from reme_home.judge import JUDGE_PLUGINS -from reme_home.sampling import SamplingPolicy - -from reme.application import Application - -PROJECT_ROOT = Path(__file__).resolve().parents[1] - - -class RunHomeConfig(BaseModel): - """Runner settings kept separate from Home's public preparation contract.""" - - model_config = ConfigDict(extra="forbid", strict=True) - workspace_dir: str = Field(min_length=1) - preparation: dict - sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy) - - -def project_path(value: str, project_root: Path) -> Path: - """Use repository-relative paths consistently regardless of the caller's cwd.""" - if value.startswith("~"): - raise ValueError("Use absolute or project-relative paths, not home-relative paths") - return (project_root / value).resolve() - - -def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): - """Resolve source paths and validate explicit Home preparation parameters.""" - raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) - config = RunHomeConfig.model_validate(raw) - spec = PreparationSpec.model_validate(config.preparation) - spec.benchmark.source = str(project_path(spec.benchmark.source, project_root)) - if spec.benchmark.evidence_source is not None: - spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root)) - spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root)) - workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root) - return workspace, spec.model_dump(mode="json"), config.sampling_policy - - -def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): - """Return resolved preparation inputs, preserving the preparation-only caller contract.""" - workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root) - return workspace, spec - - -async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict: - """Start Home, prepare once, and close all resources without evaluating cases.""" - workspace, spec, policy = _load_config(config_path, workspace_dir=workspace_dir) - app = Application( - workspace_dir=str(workspace), - plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]], - components={"home.evaluation": {"default": {"sampling_policy": policy.model_dump(mode="json")}}}, - service={"backend": "cli"}, - enable_logo=False, - log_to_file=False, - ) - try: - await app.start() - response = await app.run_job("home_prepare", spec=spec) - if not response.success: - raise RuntimeError(response.answer) - return {"workspace_dir": str(workspace), **response.answer} - finally: - await app.close() - - -def main(default_config: Path) -> None: - """Run a benchmark's default preparation from the command line.""" - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration") - parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)") - args = parser.parse_args() - result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir)) - print( - json.dumps( - { - "workspace_dir": result["workspace_dir"], - "case_count": result["dataset"]["case_count"], - "query_count": result["dataset"]["query_count"], - "source_commit": result["code"]["source_commit"], - "initial_commit": result["code"]["initial_commit"], - }, - indent=2, - ), - ) diff --git a/benchmark/longmemeval/README.md b/benchmark/longmemeval/README.md index 61e02b6a..71467a90 100644 --- a/benchmark/longmemeval/README.md +++ b/benchmark/longmemeval/README.md @@ -137,6 +137,27 @@ The top-level `sampling_policy` configures Home evaluation defaults: `max_cases` `home.evaluation` Component. It still only prepares; it does not run evaluations. Each evaluation controls `sampling` and `record_history` separately. +The top-level `validation_cycle_time_budget_minutes` configures the host optimizer's +advisory time between completed formal evaluations. It defaults to 150 minutes; +set a positive number to change it or null to disable timed reminders. + +The top-level `max_validations` sets the optimizer's formal-evaluation budget +(positive integer, default 10). A slot is consumed at first sandbox startup, +including later failures or cancellations; initial baseline and debug are excluded. + +After preparation, run local one-shot optimization with the same configuration: + +```bash +python benchmark/longmemeval/run_home.py --optimize +# Use the same --config and --workspace-dir overrides as preparation, if supplied. +python benchmark/longmemeval/run_home.py --optimize --instruction "Improve retrieval quality" +``` + +This starts a fresh application, applies both budgets and the sampling policy, +runs `home_optimize`, and closes the application. It does not repeat preparation. +Python callers can use `await optimize(config_path, workspace_dir=...)` from the +same module. The default `run()` and CLI behavior remain preparation only. + Python callers can use the same lifecycle: ```python diff --git a/benchmark/longmemeval/README_ZH.md b/benchmark/longmemeval/README_ZH.md index 85e38bf4..7b93b7ee 100644 --- a/benchmark/longmemeval/README_ZH.md +++ b/benchmark/longmemeval/README_ZH.md @@ -125,6 +125,23 @@ Python 中也可调用 `await benchmark.longmemeval.run_home.run(workspace_dir=. `home.evaluation` Component,仍只准备、不执行评测;每次评测分别控制 `sampling` 和 `record_history`。 +顶层 `validation_cycle_time_budget_minutes` 配置宿主 optimizer 两次已完成正式评测之间的 +建议用时,默认 150 分钟;可改为其他正数,或设为 null 关闭时间提醒。 + +顶层 `max_validations` 配置 optimizer 的正式评测次数预算,必须为正整数,默认 10。 +首次成功启动 sandbox 时扣一次,之后失败或取消也不退还;初始 baseline 和 debug 不计入。 + +准备完成后,用同一份配置进行本地一次性优化: + +```bash +python benchmark/longmemeval/run_home.py --optimize +``` + +若准备时指定了 `--config` 或 `--workspace-dir`,优化时传入相同参数。 +可用 `--instruction "优化检索质量"` 指定目标。该入口将配置中的两个预算和采样策略 +应用到实际 `home_optimize` 调用,不重复准备;Python 可调用同模块的 `await optimize(...)`。 +不加 `--optimize` 时仍只准备。 + 默认使用 `dev_meta` 已保存的 **group_1–group_5,共 250 个 case**, ID 已按原顺序直接写入 [config_home.yaml](config_home.yaml) 的 `benchmark.case_ids`, 运行时无需读取分组文件,group_6–group_10 保持为留出集。 diff --git a/benchmark/longmemeval/config_home.yaml b/benchmark/longmemeval/config_home.yaml index aa887faf..22991232 100644 --- a/benchmark/longmemeval/config_home.yaml +++ b/benchmark/longmemeval/config_home.yaml @@ -1,6 +1,9 @@ -# Preparation only. Paths are relative to the ReMe repository root, not the cwd. +# Preparation by default; --optimize applies these settings to the prepared workspace. +# Paths are relative to the ReMe repository root, not the cwd. # Preserve dev_meta's fixed training split; groups 6-10 remain held out. workspace_dir: benchmark/longmemeval/workspaces/home-groups-1-5 +# Formal optimizer evaluations; charged once the first sandbox starts, excluding init/debug. +max_validations: 10 # Policy defaults for Home evaluation; each call still controls sampling=true/false. sampling_policy: max_cases: 5 diff --git a/benchmark/longmemeval/run_home.py b/benchmark/longmemeval/run_home.py index b959555e..04fea91f 100644 --- a/benchmark/longmemeval/run_home.py +++ b/benchmark/longmemeval/run_home.py @@ -1,22 +1,145 @@ -"""Prepare the longmemeval Home workspace, dataset and candidate code; do not evaluate.""" +"""Prepare the LongMemEval Home workspace, or optimize it with --optimize after preparation.""" -from pathlib import Path +import argparse +import asyncio +import json import sys +from pathlib import Path + +from pydantic import BaseModel, ConfigDict, Field +import yaml # Support direct execution from any working directory as well as module imports. -_PROJECT_ROOT = Path(__file__).resolve().parents[2] -if str(_PROJECT_ROOT) not in sys.path: - sys.path.insert(0, str(_PROJECT_ROOT)) +PROJECT_ROOT = Path(__file__).resolve().parents[2] +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) -from benchmark.home_runner import main, prepare # noqa: E402 # pylint: disable=wrong-import-position +# pylint: disable=wrong-import-position +from reme_home.preparation.spec import PreparationSpec # noqa: E402 +from reme_home.judge import JUDGE_PLUGINS # noqa: E402 +from reme_home.sampling import SamplingPolicy # noqa: E402 + +from reme.application import Application # noqa: E402 + +# pylint: enable=wrong-import-position DEFAULT_CONFIG = Path(__file__).with_name("config_home.yaml") +class RunHomeConfig(BaseModel): + """Runner settings kept separate from Home's public preparation contract.""" + + model_config = ConfigDict(extra="forbid", strict=True) + workspace_dir: str = Field(min_length=1) + preparation: dict + sampling_policy: SamplingPolicy = Field(default_factory=SamplingPolicy) + validation_cycle_time_budget_minutes: float | None = Field(default=150, gt=0, allow_inf_nan=False) + max_validations: int = Field(default=10, ge=1) + + +def project_path(value: str, project_root: Path) -> Path: + """Use repository-relative paths consistently regardless of the caller's cwd.""" + if value.startswith("~"): + raise ValueError("Use absolute or project-relative paths, not home-relative paths") + return (project_root / value).resolve() + + +def _load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): + """Resolve source paths and validate explicit Home preparation parameters.""" + raw = yaml.safe_load(Path(config_path).read_text(encoding="utf-8")) + config = RunHomeConfig.model_validate(raw) + config.preparation.setdefault("candidate", {"paths": ["plugins/lme/src"]}) + spec = PreparationSpec.model_validate(config.preparation) + spec.benchmark.source = str(project_path(spec.benchmark.source, project_root)) + if spec.benchmark.evidence_source is not None: + spec.benchmark.evidence_source = str(project_path(spec.benchmark.evidence_source, project_root)) + spec.runtime.source_repo = str(project_path(spec.runtime.source_repo, project_root)) + workspace = project_path(workspace_dir if workspace_dir is not None else config.workspace_dir, project_root) + return workspace, spec.model_dump(mode="json"), config + + +def load_config(config_path: str | Path, *, workspace_dir: str | None = None, project_root: Path = PROJECT_ROOT): + """Return resolved preparation inputs, preserving the preparation-only caller contract.""" + workspace, spec, _ = _load_config(config_path, workspace_dir=workspace_dir, project_root=project_root) + return workspace, spec + + +async def _run(config_path, *, workspace_dir=None, optimize_workspace=False, instruction=None): + """Apply the same runner configuration to each preparation or optimization lifecycle.""" + workspace, spec, config = _load_config(config_path, workspace_dir=workspace_dir) + app = Application( + workspace_dir=str(workspace), + plugins=["home", JUDGE_PLUGINS[spec["benchmark"]["name"]]], + components={ + "home.evaluation": {"default": {"sampling_policy": config.sampling_policy.model_dump(mode="json")}}, + "agent_wrapper": { + "home_optimizer": { + "validation_cycle_time_budget_minutes": config.validation_cycle_time_budget_minutes, + "max_validations": config.max_validations, + }, + }, + }, + service={"backend": "cli"}, + enable_logo=False, + log_to_file=False, + ) + try: + await app.start() + if optimize_workspace: + arguments = {} if instruction is None else {"instruction": instruction} + response = await app.run_job("home_optimize", **arguments) + else: + response = await app.run_job("home_prepare", spec=spec) + if not response.success: + raise RuntimeError(response.answer) + return {"workspace_dir": str(workspace), **response.answer} + finally: + await app.close() + + +async def prepare(config_path: str | Path, *, workspace_dir: str | None = None) -> dict: + """Prepare once without running evaluations or optimization.""" + return await _run(config_path, workspace_dir=workspace_dir) + + +async def optimize(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir=None, instruction=None) -> dict: + """Optimize an already prepared workspace using this config's actual model budgets.""" + return await _run(config_path, workspace_dir=workspace_dir, optimize_workspace=True, instruction=instruction) + + async def run(config_path: str | Path = DEFAULT_CONFIG, *, workspace_dir: str | None = None) -> dict: """Prepare through the same Home Job as the CLI, then close the application.""" return await prepare(config_path, workspace_dir=workspace_dir) +def main(default_config: Path = DEFAULT_CONFIG) -> None: + """Run preparation or explicit local one-shot optimization with the benchmark config.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--config", type=Path, default=default_config, help="Home runner YAML configuration") + parser.add_argument("--workspace-dir", help="Override workspace directory (absolute or project-relative)") + parser.add_argument("--optimize", action="store_true", help="Optimize an already prepared workspace") + parser.add_argument("--instruction", help="Optimization focus; requires --optimize") + args = parser.parse_args() + if args.instruction is not None and not args.optimize: + parser.error("--instruction requires --optimize") + if args.optimize: + result = asyncio.run(optimize(args.config, workspace_dir=args.workspace_dir, instruction=args.instruction)) + print(json.dumps(result, ensure_ascii=False, indent=2)) + return + result = asyncio.run(prepare(args.config, workspace_dir=args.workspace_dir)) + print( + json.dumps( + { + "workspace_dir": result["workspace_dir"], + "case_count": result["dataset"]["case_count"], + "query_count": result["dataset"]["query_count"], + "source_commit": result["code"]["source_commit"], + "initial_commit": result["code"]["initial_commit"], + }, + indent=2, + ), + ) + + if __name__ == "__main__": - main(DEFAULT_CONFIG) + main() diff --git a/plugins/home/README.md b/plugins/home/README.md index 7f84a3f1..2e06cbc1 100644 --- a/plugins/home/README.md +++ b/plugins/home/README.md @@ -838,3 +838,150 @@ reme home_case_study evaluation_path=evaluations/candidate/abcdef0/1 'case_ids=[ Calls with `_allowed_paths` restrictions fail closed, as other Home artifact Jobs do. Structured inspection tools run in the host with their existing containment checks; only Bash/Python subprocesses run inside the OS sandbox. + +## Diagnostic and optimizer agents + +`home_diagnose` and `home_optimize` provide the minimal Meta-ReMe workflow using +Home's existing Jobs. Enable Home with a benchmark judge and prepare the +workspace first. Start the configured service in one terminal: + +```bash +reme start config=/absolute/path/to/home.yaml +``` + +Call from another terminal: + +```bash +reme home_diagnose 'evaluation_paths=["evaluations/init/abcdef0/1"]' 'instruction=Explain the lowest-scoring cases' +reme home_optimize 'instruction=Improve evidence retrieval while reducing query tokens' +``` + +Diagnosis requires explicit, unique result directories, including failed runs +with saved evidence. Each call creates a fresh diagnostic agent, delegates deep +case investigation to `home_case_study`, and returns ranked, evidence-supported +causes without prescribing fixes. An optimizer invocation automatically injects +its available caller messages and compression summary; the model cannot supply +or override that trajectory. Standalone calls record an empty trajectory. +Reports and trajectories are saved under `home/logs/diagnostic-reports/` and +`home/logs/diagnostic-trajectories/` with their evidence paths and session IDs. + +Optimization reuses a completed full initial baseline covering every prepared +training case and query, with no infrastructure errors. Otherwise it runs that +baseline first, requiring clean `init` to be checked out. It never switches away +from an existing candidate to repair a missing baseline. Baseline observations +feed sampling history. Candidate execution failures retain Home's existing +failure thresholds; low scores are valid evidence. Infrastructure failures stop +baseline startup and are distinguished from candidate defects during iteration. + +The optimizer diagnoses, edits declared `candidate_paths`, commits through +managed Git Jobs, runs debug until execution passes, then evaluates using the +host's adaptive sampling policy. Formal evaluations always rebuild memory. +The agent cannot change execution settings, selections, sampling mode, or +construction reuse through its evaluation tool; only `debug` is model-selected. +Normal `home_evaluate` callers retain their existing interface and replay support. +Unchanged candidates and infrastructure retries follow the existing repetition +rules. There is no admission gate, architecture agent, +construction-equivalence reviewer, or query-only optimizer policy in this workflow. + +Both agents share the case-study OS sandbox implementation. All file tools, +Bash and Python use it, including file metadata and Edit/Write checks. Evidence, +fixed runtime and Git metadata are read-only. Each call receives a private +scratch directory under `home/tmp/`; only the optimizer additionally receives +writes to declared candidate roots. Network is denied. Diagnostic writes are +limited to scratch. Host Jobs alone write reports, evaluation artifacts and Git +metadata. Sandbox startup is probed before model calls and never falls back to +unrestricted execution. Declared write roots must exist and remain real paths. +Only one optimizer can run per Home workspace; cancellation releases its lock +after nested tool cleanup. Separate diagnostic calls and case studies may run +concurrently with fresh state. + +The final optimizer response must identify `branch_name`, full `commit_sha`, +`evaluation_path` and `report`. The host verifies a clean checkout at that exact +branch/commit and a matching completed, scored, end-to-end evaluation without +infrastructure failures. Sampled formal evidence qualifies; debug or query-only +evidence does not. Keeping the initial baseline is valid. Invalid final claims +fail while preserving changes and the raw report in +`home/logs/optimization-reports/`. The host verifies provenance rather than +ranking candidates by a single average score. + +Dedicated `as_llm` and `agent_wrapper` components are named `home_optimizer` and +`home_diagnostic`; override them through ordinary application configuration. +Both default to `qwen3.8-max`, with context sizes 160,000 and 128,000, respectively. +The optimizer has 3,000 iterations and a 150-minute advisory validation-cycle timer; +configure `components.agent_wrapper.home_optimizer.validation_cycle_time_budget_minutes` +(null disables timed reminders). The previous `time_budget_minutes` setting is a +fallback when the cycle setting is absent. Diagnosis retains 80 iterations and +a 60-minute whole-invocation timer. Neither timer enforces a time limit. + +The optimizer follows dev_meta b53048e4: before reasoning it checks 50%, 75%, 90%, +then 100% and each additional 10% of the cycle budget. Reminders include elapsed +time, budget percentage, completed context compressions and guidance to finish +editing, commit, pass the smoke test and begin validation. Temporary and durable +reminders alternate, beginning with temporary. The cycle includes nested diagnosis, +debug and evaluation time. Only a new counted formal evaluation returning +`success` or `completed` resets the clock, thresholds, alternation and compression +count; scores and candidate failures do not control reset. As in dev_meta, a +completed result can reset even when its case results contain infrastructure errors; +an incomplete failure, rejected call, cancellation, debug or pre-sandbox failure +cannot. The initial full baseline precedes the first optimizer cycle and is excluded +from its formal-validation numbering. A durable hint records the completed +validation number and commit prefix, expires that cycle's old timer reminders and +announces the next cycle with the remaining formal-validation count. +Persisted old reminders are retained as in dev_meta. +Hints use AgentScope's current reply message so accumulated token usage is retained. +Evaluation tools queue transitions in request state; the optimizer injects them +before the next reasoning call, after all tool results have been written. Thus a +transition never separates an API tool call from its matching result. + +`components.agent_wrapper.home_optimizer.max_validations` defaults to 10, matching +dev_meta. `get_validation_budget` reports consumed and remaining slots, the search +phase and next-candidate guidance. A non-init formal run consumes one slot when +its first sandbox initializes successfully, including runs that later fail or are +cancelled. Initial baselines, debug, rejected calls and pre-start failures are free. +Accounting reuses persistent started-run evidence and survives optimizer restarts; +removing the rebuildable sampling cache does not restore the budget. The evaluator +checks the limit while holding the formal-validation lock, before starting another +run. At exhaustion it returns `VALIDATION_BUDGET_EXHAUSTED`; the optimizer selects +from existing evidence. The first `max_validations // 2` starts favor structural +exploration; later starts favor evidence-driven refinement. This guidance is advisory. +Ordinary `home_evaluate` calls remain unbounded unless the host supplies +`max_validations`; all recorded non-init formal starts in that workspace count. + +Sessions and model usage follow ReMe's wrapper conventions; each nested call reports its own usage. +The optimizer follows dev_meta's compression recovery policy. Recoverable model +or summary-output failures leave context intact below the emergency ratio +`(trigger_ratio + 1) / 2`. At or above that ratio (90% for an 80% trigger), old +messages are atomically saved under `logs/context-offloads/`; recent context and +the existing summary are retained, with a reminder pointing to the readable JSON +file. This fallback makes no additional model call and counts as one completed +compression. Cancellation, invalid configuration and uncompressible system prompts +still propagate during normal operation. Diagnostic and case-study agents retain +their existing compression behavior. Bounded draft notebook tools are not included. + +Host-only execution settings belong in the optimizer Step; model, iteration and +timer settings belong in its wrapper. For example: + +```yaml +jobs: + home_optimize: + steps: + - backend: home_optimize_step + agent_wrapper: home_optimizer + evaluation_options: + concurrency: 4 + timeout_seconds: 7200 + max_construction_failure_rate: 0.03 + # memory_limit_mib: 0 +components: + agent_wrapper: + home_optimizer: + react_config: + max_iters: 3000 + validation_cycle_time_budget_minutes: 150 + max_validations: 10 +``` + +`evaluation_options` also accepts `base_image`, `fail_fast` and +`max_artifact_bytes`. Omit `fail_fast` to retain Home's debug/scored defaults. +Restricted `_allowed_paths` calls fail closed. No real model or Docker benchmark +is required by the isolated workflow tests. diff --git a/plugins/home/src/reme_home/agent/bounded.py b/plugins/home/src/reme_home/agent/bounded.py index d89df5a9..7db6a5a9 100644 --- a/plugins/home/src/reme_home/agent/bounded.py +++ b/plugins/home/src/reme_home/agent/bounded.py @@ -116,7 +116,8 @@ def _format_minutes(minutes: float) -> str: """Format a timer duration compactly.""" if minutes < 1: - return f"{max(0, round(minutes * 60))} seconds" + seconds = max(0, round(minutes * 60)) + return f"{seconds} {'second' if seconds == 1 else 'seconds'}" rounded = round(minutes, 1) value = int(rounded) if rounded.is_integer() else rounded unit = "minute" if value == 1 else "minutes" diff --git a/plugins/home/src/reme_home/agent/compression.py b/plugins/home/src/reme_home/agent/compression.py new file mode 100644 index 00000000..0241611e --- /dev/null +++ b/plugins/home/src/reme_home/agent/compression.py @@ -0,0 +1,126 @@ +"""Meta-ReMe compression recovery adapted to AgentScope's native context pipeline.""" + +import asyncio +import logging +from pathlib import Path +from uuid import uuid4 + +from agentscope.exception import ToolJSONDecodeError +from agentscope.message import TextBlock +import jsonschema +import openai +from pydantic import ValidationError + +from ..evaluation.artifacts import contained, write_json +from .bounded import BoundedReActAgent + +logger = logging.getLogger(__name__) + + +class RecoverableCompressionAgent(BoundedReActAgent): + """Skip recoverable failures until emergency offload is needed. + + Override the native implementation so compression middleware, successful-change + counters, finalization and cancellation remain owned by their existing layers. + """ + + def __init__(self, *args, fallback_directory=None, **kwargs): + super().__init__(*args, **kwargs) + self.fallback_directory = Path(fallback_directory) if fallback_directory is not None else None + + async def _compress_context_impl(self, context_config=None, instructions=None): + """Keep the old trigger-derived emergency boundary without silent truncation.""" + config = context_config or self.context_config + model_input = await self._prepare_model_input() + context_ratio = await self.model.count_tokens(**model_input) / self.model.context_size + if context_ratio < config.trigger_ratio: + return + # The SDK's default truncation would hide the failure and discard old + # evidence before Home can decide whether emergency offload is needed. + config = config.model_copy(update={"compression_fallback_to_truncation": False}) + try: + await super()._compress_context_impl(context_config=config, instructions=instructions) + except Exception as exc: + if not self._is_recoverable_compression_error(exc): + raise + emergency_ratio = (config.trigger_ratio + 1.0) / 2.0 + if context_ratio < emergency_ratio: + logger.warning( + "Context compression failed at %.1f%%, below the %.1f%% emergency threshold; skipping: %s: %s", + context_ratio * 100, + emergency_ratio * 100, + type(exc).__name__, + exc, + ) + return + logger.warning( + "Context compression failed at %.1f%%; using local offload at the %.1f%% emergency threshold: %s: %s", + context_ratio * 100, + emergency_ratio * 100, + type(exc).__name__, + exc, + ) + await self._fallback_compress_context(exc, context_ratio, config) + + @staticmethod + def _is_recoverable_compression_error(exc): + """Match dev_meta's model/output failures, excluding uncompressible prompts.""" + if isinstance(exc, RuntimeError): + return not str(exc).startswith("The system prompt") + return isinstance(exc, (ToolJSONDecodeError, jsonschema.ValidationError, ValidationError, openai.OpenAIError)) + + async def _fallback_compress_context(self, cause, context_ratio, config): + """Persist older messages before replacing state; never call a second model.""" + if self.fallback_directory is None: + raise RuntimeError("Optimizer fallback_directory is required for local context compression") from cause + model_input = await self._prepare_model_input() + old_messages, recent_messages = await self._split_context_for_compression( + config.reserve_ratio * self.model.context_size, + model_input.get("tools", []), + ) + if not old_messages: + raise RuntimeError( + "Local fallback compression cannot remove any context while preserving the configured reserve", + ) from cause + + async def apply_change(): + contained(self.fallback_directory.parent, self.fallback_directory, directory=True) + path = contained(self.fallback_directory, f"context-{uuid4().hex}.json") + write_json( + path, + { + "session_id": self.state.session_id, + "context_ratio": context_ratio, + "trigger_ratio": config.trigger_ratio, + "emergency_ratio": (config.trigger_ratio + 1.0) / 2.0, + "failure": {"type": type(cause).__name__, "message": str(cause)}, + "messages": [message.model_dump(mode="json", exclude_none=False) for message in old_messages], + }, + ) + notice = ( + "Model-based context compression failed near the context limit. " + f"Older context was offloaded without semantic summarization to '{path}'. " + "The recent context is preserved; use Read on that file if older experiments, decisions, " + "validation results, or tool outputs are needed." + ) + await self._clear_unreserved_read_cache(recent_messages) + # Recent SDK versions also carry current-reply usage through compression. + usage = self._get_reply_usage() + summary = self.state.summary + self.state.summary = ( + "\n".join(part for part in (summary, notice) if part) + if isinstance(summary, str) + else [*summary, TextBlock(text=notice)] + ) + self.state.context = recent_messages + if usage is not None and self._get_reply_usage() is None: + self.state.append_context(self.name, []) + self.state.context[-1].usage = usage + logger.warning("Local compression offloaded %d message(s) to %s", len(old_messages), path) + + task = asyncio.create_task(apply_change()) + try: + await asyncio.shield(task) + except asyncio.CancelledError: + await task + raise diff --git a/plugins/home/src/reme_home/agent/diagnostic.py b/plugins/home/src/reme_home/agent/diagnostic.py new file mode 100644 index 00000000..3c184303 --- /dev/null +++ b/plugins/home/src/reme_home/agent/diagnostic.py @@ -0,0 +1,159 @@ +"""Diagnose saved Home evaluations with an isolated evidence-driven agent.""" + +import asyncio +import copy +import json + +from agentscope.state import AgentState +from agentscope.tool import FunctionTool +from pydantic import Field, field_validator + +from reme.steps.base_step import BaseStep, Ref + +from ..evaluation.artifacts import contained, read_json, write_json +from ..evaluation.component import blocking +from ..preparation.spec import SpecModel +from ..workspace import HomeWorkspace +from .workflow_common import required_job, save_record, workflow_reply +from .wrapper import BoundedAgentWrapper + + +class DiagnosticInput(SpecModel): + """Require explicit evaluation evidence; caller state is never model-selected.""" + + evaluation_paths: list[str] = Field(min_length=1) + instruction: str = Field(default="Identify and rank the evidence-supported root causes.", min_length=1) + + @field_validator("evaluation_paths") + @classmethod + def validate_paths(cls, values): + """Reject empty and duplicate input references before creating an agent.""" + if any(not value.strip() for value in values) or len(set(values)) != len(values): + raise ValueError("evaluation_paths must contain unique nonempty paths") + return values + + +def evaluation_paths(home, values): + """Require explicit, unique local runs with actual evidence, including failed runs.""" + if not values: + raise ValueError("evaluation_paths must identify at least one run") + paths = [] + root = contained(home, "evaluations", directory=True) + for value in values: + path = contained(home, value, directory=True) + if not path.is_relative_to(root) or path == root: + raise ValueError("evaluation_paths must be inside home/evaluations") + read_json(contained(path, "manifest.json")) + evidence = [path / "summary.json", path / "failure.json"] + evidence += list(path.glob("cases/*/memory_construction/result.json")) + evidence += list(path.glob("cases/*/queries/result.json")) + if not any(contained(path, item).is_file() for item in evidence): + raise ValueError(f"Evaluation contains no results: {path}") + paths.append(str(path)) + if len(paths) != len(set(paths)): + raise ValueError("evaluation_paths must be unique") + return paths + + +def trajectory(state): + """Serialize only caller messages and compression summary, never credentials or permissions.""" + if state is not None and not isinstance(state, AgentState): + raise TypeError("Injected caller state must be an AgentState") + if state is None: + return {"messages": [], "summary": None} + summary = state.summary + if isinstance(summary, list): + summary = [block.model_dump(mode="json") for block in summary] + return { + "session_id": state.session_id, + "messages": [message.model_dump(mode="json") for message in state.context], + "summary": summary, + } + + +def diagnostic_tool(wrapper): + """Inject the optimizer's real trajectory into a fresh diagnostic invocation.""" + job = required_job(wrapper, "home_diagnose") + + async def diagnose( + evaluation_paths: list[str], # pylint: disable=redefined-outer-name + instruction: str = "Identify and rank the evidence-supported root causes.", + **kwargs, + ): + if set(kwargs) - {"_agent_state"}: + raise ValueError("Caller trajectory is host-owned and cannot be supplied by the model") + response = await job( + instruction=instruction, + evaluation_paths=evaluation_paths, + caller_trajectory=trajectory(kwargs.get("_agent_state")), + ) + return {"success": response.success, "answer": response.answer, "metadata": response.metadata} + + tool = FunctionTool( + diagnose, + name="home_diagnose", + description=job.description, + is_concurrency_safe=False, + is_state_injected=True, + ) + tool.input_schema = copy.deepcopy(job.parameters) + return tool + + +class HomeDiagnoseStep(BaseStep): + """Aggregate explicit evaluation evidence through a read-only diagnostic agent.""" + + workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace") + agent_wrapper: BoundedAgentWrapper = Ref(BoundedAgentWrapper, "agent_wrapper") + + async def execute(self): + """Validate evidence, save the host trajectory and persist a provenance-bearing report.""" + if self.context.get("_allowed_paths") is not None: + raise ValueError("Home diagnosis requires access to its managed evidence") + if not self.workspace.is_started or not self.agent_wrapper.is_started: + raise RuntimeError("Start the Home workspace and diagnostic wrapper first") + spec = DiagnosticInput.model_validate( + {key: self.context[key] for key in DiagnosticInput.model_fields if key in self.context}, + ) + home = self.workspace.home_path + paths = await blocking(evaluation_paths, home, spec.evaluation_paths) + caller = self.context.get("caller_trajectory", trajectory(None)) + if not isinstance(caller, dict): + raise ValueError("Caller trajectory must be a host-supplied object") + source = await blocking(save_record, home, "diagnostic-trajectories", caller) + payload = { + "status": "running", + "instruction": spec.instruction, + "evaluation_paths": paths, + "trajectory_path": source, + } + report_path = await blocking(save_record, home, "diagnostic-reports", payload) + self.context.response.metadata.update(diagnostic_report_path=report_path, trajectory_path=source) + try: + result = await workflow_reply( + self.agent_wrapper, + home, + "diagnostic", + f"Diagnostic request:\n{spec.instruction}\n\nComplete available caller trajectory: {source}\n" + f"Explicit evaluation paths: {json.dumps(paths, ensure_ascii=False)}", + job_names=("home_case_study",), + memory_limit_mib=self.kwargs.get("memory_limit_mib"), + ) + if not isinstance(result.get("result"), str) or not result["result"].strip(): + raise ValueError("Diagnostic agent returned no report") + payload.update( + status="completed", + diagnosis=result["result"], + session_id=result.get("session_id"), + usage=result.get("usage"), + ) + self.context.response.answer = {**payload, "diagnostic_report_path": report_path} + self.context.response.metadata.update( + {key: result[key] for key in ("session_id", "usage") if key in result}, + ) + except BaseException as exc: + payload.update(status="cancelled" if isinstance(exc, asyncio.CancelledError) else "failed", error=str(exc)) + raise + finally: + await blocking(write_json, contained(home, report_path), payload) + return self.context.response diff --git a/plugins/home/src/reme_home/agent/diagnostic.yaml b/plugins/home/src/reme_home/agent/diagnostic.yaml new file mode 100644 index 00000000..16b7b143 --- /dev/null +++ b/plugins/home/src/reme_home/agent/diagnostic.yaml @@ -0,0 +1,32 @@ +system_prompt: |- + You are Home's read-only code diagnostician, migrated from Meta-ReMe. + Identify and rank evidence-supported mechanisms that caused evaluation failures or limited candidate quality. + The candidate repository is {code_repository}; guides are under {docs}; private scratch is {scratch}. + The caller provides explicit evaluation paths and its available complete trajectory, including any compression summary. + + Produce one concise report organized as Priority scope, Failure clusters, Ranked shared root causes, Outliers, + and Missing evidence. State which low-scoring cases and question types you prioritized and what you deferred. + For each cause, name affected cases and queries, lifecycle stage, causal chain, supporting and contradicting + evidence, confidence, and the smallest missing observation. Separate fixed-runtime limitations. + + Read architecture-guide.md and validation-artifact-guide.md when needed. Confirm each run's exact commit, + status, score contract and coverage. Compare runs only where contracts and case/query coverage permit it. + Prioritize the lowest-scoring cases and question types; you need not explain every failure. + Use home_case_study batches early as the default for deep case investigation, even for one case. Pass one + evaluation_path, case_ids and a shared focus; do not put case-specific instructions in the shared focus. + Use its reports to identify shared causes, then inspect only remaining cross-case or code-level evidence. + Direct investigation is appropriate for quick lookups, aggregate comparisons and specific evidence checks. + + Classify infrastructure, candidate-execution, answer-quality and judging failures separately. Trace extraction, + persistence, overwrite, indexing, retrieval, ranking, context presentation or reasoning as evidence requires. + Treat partial_success construction as requiring repair. Do not replace persisted runtime failure statuses. + Original sessions and derived memory are both evidence; absence from summaries does not prove unavailability. + Prefer independent evidence_session_ids; inferred evidence is a lead, not proof. Explicitly state uncertainty. + Use home_list_validation_results and home_inspect_validation to locate evidence. Pin revision to the evaluated + full commit when using home_list_code_tree, home_search_code and home_read_code_file. The current checkout may + differ. Read artifacts and runtime docs with Read, Grep, Glob, AutoBash or python_execute, using absolute paths. + Keep stderr visible; denied reads are not proof of missing content. + + The OS sandbox allows evidence reads and writes only to private scratch. You may create disposable scripts + and analysis files there. Do not modify candidate code, datasets, evaluations, reports or Git metadata. + Do not run ReMe, evaluations or network operations. Do not implement or prescribe fixes; report supported causes. diff --git a/plugins/home/src/reme_home/agent/optimizer.py b/plugins/home/src/reme_home/agent/optimizer.py new file mode 100644 index 00000000..6d3c9fb5 --- /dev/null +++ b/plugins/home/src/reme_home/agent/optimizer.py @@ -0,0 +1,513 @@ +"""Optimize Home candidates and verify their baseline and final evaluation evidence.""" + +import asyncio +import json +import math +import time +from pathlib import Path +from typing import Callable + +from agentscope.message import HintBlock +from agentscope.state import AgentState +from agentscope.tool import Edit, FunctionTool +from pydantic import Field + +from reme.steps.base_step import BaseStep, Ref + +from ..code.repository import validate_repository, require_clean, validate_tree, status +from ..evaluation.artifacts import contained, read_json, write_json +from ..evaluation.component import blocking +from ..evaluation.inputs import EvaluationSpec, load_inputs +from ..evaluation.history import read_state +from ..evaluation.budget import validate_max_validations, validation_budget_status +from ..preparation.spec import SpecModel +from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory +from .diagnostic import diagnostic_tool, evaluation_paths +from .bounded import TimeBudgetReminder, _format_minutes, _remove_hint_by_identity +from .compression import RecoverableCompressionAgent +from .workflow_common import required_job, save_record, workflow_reply +from .wrapper import CaseStudyAgentWrapper + +MUTATION_JOBS = ( + "home_create_candidate_branch", + "home_switch_candidate_branch", + "home_merge_candidate_branch", + "home_commit_candidate", +) +PENDING_CYCLE_TRANSITIONS = "home.optimizer.pending_cycle_transitions" + + +class ValidationCycleTimer: + """Emit escalating reminders as a scored-validation cycle ages.""" + + def __init__(self, time_budget_minutes: float, *, clock: Callable[[], float] = time.monotonic) -> None: + if ( + not isinstance(time_budget_minutes, (int, float)) + or isinstance(time_budget_minutes, bool) + or not math.isfinite(time_budget_minutes) + or time_budget_minutes <= 0 + ): + raise ValueError("time_budget_minutes must be a positive number") + self.time_budget_minutes = float(time_budget_minutes) + self._clock = clock + self._started_at = self._clock() + self._last_reminded_ratio = 0.0 + self._next_reminder_is_persistent = False + self.context_compression_count = 0 + + def reset(self) -> None: + """Start a fresh cycle after a scored validation finishes.""" + + self._started_at = self._clock() + self._last_reminded_ratio = 0.0 + self._next_reminder_is_persistent = False + self.context_compression_count = 0 + + def record_context_compression(self) -> None: + """Record one completed compression since the last scored validation.""" + + self.context_compression_count += 1 + + def next_reminder(self) -> TimeBudgetReminder | None: + """Return one reminder for the latest newly reached deadline.""" + + elapsed_minutes = max(0.0, (self._clock() - self._started_at) / 60.0) + elapsed_ratio = elapsed_minutes / self.time_budget_minutes + reached_ratio: float | None = None + if elapsed_ratio >= 1.0: + reached_ratio = math.floor(elapsed_ratio * 10 + 1e-9) / 10 + else: + for threshold in (0.9, 0.75, 0.5): + if elapsed_ratio >= threshold: + reached_ratio = threshold + break + if reached_ratio is None or reached_ratio <= self._last_reminded_ratio: + return None + self._last_reminded_ratio = reached_ratio + persistent = self._next_reminder_is_persistent + self._next_reminder_is_persistent = not persistent + elapsed = _format_minutes(elapsed_minutes) + percentage = round(reached_ratio * 100) + timing = ( + f"The optimizer agent run has used {elapsed} since the last scored validation " + f"({percentage}% of its {_format_minutes(self.time_budget_minutes)} soft time budget). " + ) + compression = ( + f"It has completed {self.context_compression_count} context compressions since the last " + "scored validation. " + ) + if elapsed_ratio >= 1.0: + return TimeBudgetReminder( + content=( + f"{timing}{compression}" + "Prioritize completing the code changes, committing them, passing the smoke test, and starting " + "validation as soon as possible." + ), + persistent=persistent, + ) + remaining = _format_minutes(self.time_budget_minutes - elapsed_minutes) + return TimeBudgetReminder( + content=( + f"{timing}{compression}" + f"Complete the code changes, commit them, pass the smoke test, and start validation within {remaining}." + "" + ), + persistent=persistent, + ) + + +class OptimizerAgent(RecoverableCompressionAgent): + """Add Meta-ReMe's per-validation reminders to Home's bounded agent.""" + + def __init__(self, *args, validation_cycle_timer=None, **kwargs): + super().__init__(*args, **kwargs) + self.validation_cycle_timer = validation_cycle_timer + + async def _reasoning(self, *args, **kwargs): + """Deliver hints after tool results without splitting the SDK's reply message.""" + transitions = self.state.middle_context.pop(PENDING_CYCLE_TRANSITIONS, []) + for content in transitions: + self.state.append_context( + self.name, + [HintBlock(source="meta_reme_validation_cycle_transition", hint=content)], + ) + timer = self.validation_cycle_timer + reminder = timer.next_reminder() if timer is not None else None + message = None + if reminder is not None: + message = HintBlock(source="meta_reme_validation_timer", hint=reminder.content) + self.state.append_context(self.name, [message]) + try: + async for item in super()._reasoning(*args, **kwargs): + yield item + finally: + if reminder is not None and not reminder.persistent: + _remove_hint_by_identity(self.state.context, message) + + async def compress_context(self, *args, **kwargs): + """Count completed context replacements in the current validation cycle.""" + summary_before, context_before = self.state.summary, self.state.context + await super().compress_context(*args, **kwargs) + if self.validation_cycle_timer is not None and ( + self.state.summary is not summary_before or self.state.context is not context_before + ): + self.validation_cycle_timer.record_context_compression() + + +class OptimizerAgentWrapper(CaseStudyAgentWrapper): + """Keep each optimizer's tools and cycle timer bound to one reply invocation.""" + + def create_cycle_timer(self, options=None): + """Use the legacy time setting as a fallback; explicit null disables reminders.""" + options = self._merged_kwargs(options or {}) + budget = options.get("validation_cycle_time_budget_minutes", options.get("time_budget_minutes")) + return None if budget is None else ValidationCycleTimer(budget) + + def validation_limit(self): + """Keep the formal count limit host-owned and independent of elapsed time.""" + return validate_max_validations(self._merged_kwargs({}).get("max_validations", 10)) + + def _create_agent(self, *, options, **kwargs): + timer = ( + options["_validation_cycle_timer"] + if "_validation_cycle_timer" in options + else self.create_cycle_timer(options) + ) + return OptimizerAgent( + validation_cycle_timer=timer, + fallback_directory=options.get("_context_fallback_directory", self.session_path / "context-offloads"), + **kwargs, + ) + + +def validation_cycle_transition(manifest, validation_number, remaining): + """Expire old reminders and announce the next cycle with its remaining budget.""" + commit = manifest["commit_sha"][:7] + return HintBlock( + source="meta_reme_validation_cycle_transition", + hint=( + f"Formal validation {validation_number} completed successfully for commit {commit}. " + f"The previous cycle timer has been reset, and all timer reminders for cycle {validation_number} " + f"are now expired. Remaining formal validations: {remaining}. Start cycle {validation_number + 1}." + "" + ), + ) + + +def completed_validation(home, result, started_before): + """Match dev_meta's new-count plus completed-status boundary, independent of scores.""" + if not isinstance(result, dict) or result.get("status") not in {"success", "completed"}: + return None + run = contained(home, result["result_dir"], directory=True) + key = run.relative_to(contained(home, "evaluations", directory=True)).as_posix() + started = read_state(home)["started_validations"] + # The initial full baseline precedes optimizer creation in dev_meta and + # never consumes its validation budget. Home records it in sampling history. + if key in started_before or key not in started or key.startswith("init/"): + return None + manifest = read_json(contained(run, "manifest.json")) + if ( + manifest.get("status") != "completed" + or manifest.get("mode") != "scored" + or manifest.get("dataset_kind") != "training-set" + or not manifest.get("sandbox_started_at") + ): + return None + return manifest, sum(not name.startswith("init/") for name in started) + + +class OptimizationInput(SpecModel): + """The host owns runtime settings; the caller supplies only the optimization focus.""" + + instruction: str = Field(default="Improve the candidate using the completed baseline evaluation.", min_length=1) + + +class OptimizationResult(SpecModel): + """An agent claim that must be checked against Git and saved formal evidence.""" + + branch_name: str = Field(min_length=1) + commit_sha: str = Field(pattern="^[0-9a-f]{40}$") + evaluation_path: str = Field(min_length=1) + report: str = Field(min_length=1) + + +def parse_result(text): + """Accept a JSON object, optionally enclosed in one Markdown JSON fence.""" + text = text.strip() + if text.startswith("```") and text.endswith("```"): + text = text.split("\n", 1)[1].rsplit("```", 1)[0].strip() + return OptimizationResult.model_validate(json.loads(text)) + + +EVALUATION_OPTIONS = { + "concurrency", + "timeout_seconds", + "base_image", + "fail_fast", + "max_construction_failure_rate", + "max_artifact_bytes", +} + + +def evaluation_options(raw): + """Validate host-only execution settings without exposing selection or replay.""" + if not isinstance(raw, dict) or set(raw) - EVALUATION_OPTIONS: + raise ValueError("evaluation_options may contain only Home execution settings") + return EvaluationSpec.model_validate(raw).model_dump(exclude_unset=True) + + +def verify_evaluation(home, path, commit, *, baseline=False): + """Require completed end-to-end evidence with no infrastructure failures.""" + run = Path(evaluation_paths(home, [str(path)])[0]) + manifest = read_json(contained(run, "manifest.json")) + summary = read_json(contained(run, "summary.json")) + required = { + "commit_sha": commit, + "status": "completed", + "success": True, + "mode": "scored", + "dataset_kind": "training-set", + "execution_mode": "full_validation", + } + if any(manifest.get(key) != value for key, value in required.items()): + raise ValueError("Evaluation must match the commit and complete without infrastructure failures") + if ( + not manifest.get("sandbox_started_at") + or not summary.get("case_count") + or summary.get("scoring_enabled") is not True + or any(summary.get(key, {}).get("infra_error") != 0 for key in ("case_status_counts", "query_status_counts")) + ): + raise ValueError("Evaluation must match the commit and complete without infrastructure failures") + _record, dataset, cases = load_inputs(home, EvaluationSpec()) + if manifest.get("fingerprints", {}).get("dataset") != dataset.normalized_fingerprint: + raise ValueError("Evaluation dataset no longer matches the prepared dataset") + selected = manifest.get("case_ids", []) + queries = manifest.get("query_ids", {}) + expected = {case.case_id: {q.query_id for q in case.queries} for case in cases} + if ( + not selected + or len(set(selected)) != len(selected) + or set(selected) - expected.keys() + or set(queries) != set(selected) + or any(not ids or len(set(ids)) != len(ids) or set(ids) - expected[key] for key, ids in queries.items()) + ): + raise ValueError("Evaluation has an invalid case/query selection") + if baseline and (set(selected) != set(expected) or any(set(queries[key]) != ids for key, ids in expected.items())): + raise ValueError("Initial baseline must cover every prepared training case and query") + rows = summary.get("cases", {}) + if ( + set(rows) != set(selected) + or summary["case_count"] != len(selected) + or summary.get("expected_query_count") != sum(len(ids) for ids in queries.values()) + ): + raise ValueError("Evaluation summary has incomplete case coverage") + # Failed construction can legitimately omit answers; the recorded candidate + # failure is evidence, not an infrastructure failure or a missing result. + for key, row in rows.items(): + if set(row.get("queries", {})) != set(queries[key]) and row.get("construction", {}).get("status") != "failure": + raise ValueError(f"Evaluation summary has incomplete query coverage: {key}") + return {"evaluation_path": str(run), "manifest": manifest, "summary": summary} + + +def baseline_state(home): + """Inspect a clean repository and locate a usable initial full evaluation.""" + _validate_directory(home, home / "metadata") + with _initialization_lock(home / "metadata/workspace.lock"): + repo, record = validate_repository(home) + require_clean(repo) + validate_tree(repo, record, "HEAD") + initial = record["code"]["initial_commit"] + current = status(repo, record) + for manifest in sorted((home / "evaluations/init").glob("*/*/manifest.json"), reverse=True): + try: + evidence = verify_evaluation(home, manifest.parent, initial, baseline=True) + return current, evidence + except (ValueError, OSError, KeyError, TypeError): + continue + if current["current_branch"] != "init" or current["commit_sha"] != initial: + raise ValueError("A complete initial baseline is missing; check out clean init before home_optimize") + return current, None + + +def verify_final(home, result): + """Check the actual checkout and evidence; never repair or discard agent edits.""" + _validate_directory(home, home / "metadata") + with _initialization_lock(home / "metadata/workspace.lock"): + repo, record = validate_repository(home) + require_clean(repo) + validate_tree(repo, record, "HEAD") + current = status(repo, record) + if current["current_branch"] != result.branch_name or current["commit_sha"] != result.commit_sha: + raise ValueError("Final branch/commit does not match the checked-out HEAD") + evidence = verify_evaluation(home, result.evaluation_path, result.commit_sha) + if evidence["manifest"]["branch_name"] != result.branch_name: + raise ValueError("Final evaluation belongs to a different branch") + return evidence["evaluation_path"] + + +def validation_budget_tool(home, max_validations): + """Expose dev_meta's read-only count, phase and next-candidate guidance.""" + validate_max_validations(max_validations) + + async def get_validation_budget(): + """Return started formal validations, remaining count and optimization guidance.""" + return await blocking(validation_budget_status, home, max_validations) + + return FunctionTool( + get_validation_budget, + name="get_validation_budget", + description=( + "Report how many formal optimizer validations have started at least one sandbox, how many remain, " + "and whether the next candidate should emphasize structural code changes or evidence-driven refinement. " + "Debug-case smoke validations, the initial baseline and failures before sandbox startup are excluded." + ), + is_read_only=True, + ) + + +def evaluation_tool(wrapper, options, *, home, timer=None, max_validations=10): + """Only debug mode is model-selected; all formal runs rebuild construction.""" + job = required_job(wrapper, "home_evaluate") + validate_max_validations(max_validations) + + async def evaluate(debug: bool = False, **kwargs): + """Run debug after committing; after debug passes, run a sampled formal evaluation. + + Debug checks execution, not answer quality. Inspect infrastructure failures + separately; retries keep the existing commit and selection recovery rules. + """ + if not isinstance(debug, bool): + raise ValueError("debug must be boolean") + if set(kwargs) - {"_agent_state"}: + raise ValueError("Optimizer state is host-owned and cannot be supplied by the model") + state = kwargs.get("_agent_state") + if not isinstance(state, AgentState): + raise TypeError("Cycle reminders require the host-injected AgentState") + started_before = await blocking(lambda: set(read_state(home)["started_validations"])) if not debug else set() + response = await job( + **options, + **({"max_validations": max_validations} if not debug else {}), + debug=debug, + sampling=not debug, + record_history=True, + case_ids=None, + query_ids=None, + reuse_construction_from=None, + ) + if not debug: + budget = await blocking(validation_budget_status, home, max_validations) + if isinstance(response.answer, dict): + response.answer = {**response.answer, "validation_budget": budget} + completed = await blocking(completed_validation, home, response.answer, started_before) + if completed is not None: + manifest, validation_number = completed + if timer is not None: + timer.reset() + transition = validation_cycle_transition(manifest, validation_number, budget["remaining_validations"]) + state.middle_context.setdefault(PENDING_CYCLE_TRANSITIONS, []).append(transition.hint) + return {"success": response.success, "answer": response.answer, "metadata": response.metadata} + + tool = FunctionTool(evaluate, name="home_evaluate", is_concurrency_safe=False, is_state_injected=True) + tool.input_schema = { + "type": "object", + "properties": {"debug": {"type": "boolean", "default": False}}, + "additionalProperties": False, + } + return tool + + +class HomeOptimizeStep(BaseStep): + """Require a full initial baseline, run the coding agent, and verify its final claim.""" + + workspace: HomeWorkspace = Ref(HomeWorkspace, "home.workspace") + agent_wrapper: OptimizerAgentWrapper = Ref(OptimizerAgentWrapper, "agent_wrapper") + + async def execute(self): + """One optimizer per workspace; never hold the evaluation/Git lock across model calls.""" + if self.context.get("_allowed_paths") is not None: + raise ValueError("Home optimization requires access to its managed workspace") + if not self.workspace.is_started or not self.agent_wrapper.is_started: + raise RuntimeError("Start the Home workspace and optimizer wrapper first") + spec = OptimizationInput.model_validate( + {key: self.context[key] for key in OptimizationInput.model_fields if key in self.context}, + ) + options = evaluation_options(self.kwargs.get("evaluation_options", {})) + self.agent_wrapper.validation_limit() + home = self.workspace.home_path + _validate_directory(home, self.workspace.state_path) + with _initialization_lock(self.workspace.state_path / "optimization.lock"): + _current, baseline = await blocking(baseline_state, home) + if baseline is None: + response = await required_job(self.agent_wrapper, "home_evaluate")( + **options, + debug=False, + sampling=False, + record_history=True, + case_ids=None, + query_ids=None, + reuse_construction_from=None, + ) + if not response.success: + raise RuntimeError(f"Initial baseline failed: {response.answer}") + current, _ = await blocking(baseline_state, home) + baseline = await blocking( + verify_evaluation, + home, + response.answer["result_dir"], + current["commit_sha"], + baseline=True, + ) + return await self._optimize(home, spec, baseline, options) + + async def _optimize(self, home, spec, baseline, options): + payload = {"status": "running", "instruction": spec.instruction, "baseline_path": baseline["evaluation_path"]} + report_path = await blocking(save_record, home, "optimization-reports", payload) + self.context.response.metadata["optimization_report_path"] = report_path + try: + timer = self.agent_wrapper.create_cycle_timer() + max_validations = self.agent_wrapper.validation_limit() + budget = await blocking(validation_budget_status, home, max_validations) + reply = await workflow_reply( + self.agent_wrapper, + home, + "optimizer", + f"Begin optimization.\n{spec.instruction}\n\n" + f"Completed full initial baseline: {baseline['evaluation_path']}\n" + f"Formal validation budget: {json.dumps(budget)}\n" + "Return exactly one JSON object with branch_name, commit_sha (full SHA), evaluation_path and report. " + "Leave that evaluated branch and commit checked out with no uncommitted changes.", + tool_factory=lambda backend: [ + Edit(backend=backend), + diagnostic_tool(self.agent_wrapper), + validation_budget_tool(home, max_validations), + evaluation_tool( + self.agent_wrapper, + options, + home=home, + timer=timer, + max_validations=max_validations, + ), + ], + job_names=MUTATION_JOBS, + agent_options={ + "_validation_cycle_timer": timer, + "_context_fallback_directory": contained(home, "logs/context-offloads", directory=True), + }, + memory_limit_mib=self.kwargs.get("memory_limit_mib"), + ) + payload.update( + agent_report=reply.get("result"), + session_id=reply.get("session_id"), + usage=reply.get("usage"), + ) + result = parse_result(reply["result"]) + evidence_path = await blocking(verify_final, home, result) + payload.update(result.model_dump(), evaluation_path=evidence_path, status="completed") + self.context.response.answer = {**payload, "optimization_report_path": report_path} + self.context.response.metadata.update({key: reply[key] for key in ("session_id", "usage") if key in reply}) + except BaseException as exc: + payload.update(status="cancelled" if isinstance(exc, asyncio.CancelledError) else "failed", error=str(exc)) + raise + finally: + await blocking(write_json, contained(home, report_path), payload) + return self.context.response diff --git a/plugins/home/src/reme_home/agent/optimizer.yaml b/plugins/home/src/reme_home/agent/optimizer.yaml new file mode 100644 index 00000000..151304df --- /dev/null +++ b/plugins/home/src/reme_home/agent/optimizer.yaml @@ -0,0 +1,60 @@ +system_prompt: |- + You are Home's primary optimization agent, migrated from Meta-ReMe. Execute the optimization loop; + your output is the best formally evaluated candidate, with its branch and exact commit checked out. + ReMe is local-first and file-native. User-owned memory is the source of truth; indexes and caches are rebuildable. + Improve general memory mechanisms, robustness, recoverability and transparent behavior. Never introduce + benchmark-specific shortcuts, held-out answer leakage or case-specific rules. Fixed runtime is host-owned. + Candidate source is {code_repository}; guides are under {docs}; disposable scratch is {scratch}. + + Optimize answer quality by supplying precise evidence at lower query-time token cost. Score improvement is + more important than query token reduction. Every proposed change should aim for both. A candidate branch + represents one coherent, falsifiable hypothesis. Candidate paths are the only editable source roots. + + Repeat this loop: + 1. Start from the completed full baseline supplied by the host or the current candidate's completed formal + evaluation. Inventory evaluations and branches, commits and coverage before editing. + 2. Diagnose before editing. Strongly prefer home_diagnose with the actual evaluation_paths; it automatically + receives your available full trajectory. Prioritize low-scoring cases and question types, identify at least + two distinct supported causes when evidence permits, estimate impact, coverage and confidence, and rank + directions. Direct investigation is appropriate when evidence is already narrow and clear. Treat partial + construction as a defect. Distinguish infrastructure errors from candidate behavior; do not patch candidate + code to compensate for infrastructure outages. Defer unrelated, redundant and low-impact investigation. + 3. Choose lineage before editing. Continue the current direction on its branch; fork a promising branch for + a distinct direction with home_create_candidate_branch. Never edit init. Implement the highest-impact + supported direction, or two or three compatible changes supporting one coherent claim. Inspect diffs before + merging. Create, switch, merge and commit exclusively through the managed Home Jobs; keep evaluated history. + Preserve a branch at each evaluated commit you may want to select finally: before advancing such a branch, + fork a new candidate. Managed switching selects existing branch heads, not arbitrary historical commits. + 4. Commit before testing, then call home_evaluate(debug=true). Repair concrete execution failures, commit and + repeat until debug passes. Debug establishes execution only; it has no answer-quality score. + 5. Call home_evaluate(debug=false) on clean committed code after debug passes. Formal evaluation uses host-owned + adaptive sampling and always rebuilds memory before answering. Inspect the resulting evidence before the + next iteration. Candidate failures and low scores require a code change; infrastructure failures may retry + the same commit using the host's frozen previous selection. Empty commits do not bypass repetition checks. + + Call get_validation_budget before planning the next candidate. The host limits formal validations; one run + consumes one slot when its first sandbox starts successfully, even if it later fails or is cancelled. Debug, + initial baseline and pre-sandbox failures do not consume slots. The budget persists across optimizer invocations. + During structural_exploration prefer substantial mechanisms or coherent architecture changes; during + evidence_driven_refinement prefer evidence-driven parameters, prompts or scoped changes. When exhausted, + select the best existing evaluated branch without another formal validation. There is no candidate admission + gate; do not call architecture or construction-equivalence tools. Use the configured iteration cap and advisory time + reminders to finish a useful search; time reminders do not terminate execution. The cycle clock includes diagnosis, + editing, debug and formal evaluation. Only a newly counted, completed formal evaluation resets it and its context + compression count; debug, rejected calls and incomplete evaluations do not. A cycle transition expires earlier + timer reminders and announces the next cycle. You may stop once the evidence + supports a final selection. Do not keep evaluating without a concrete improvement hypothesis. + Compare scores on common cases and queries, weigh coverage, difficulty, regressions and lineage, and do not + rank sampled candidates by aggregate score alone. Prefer quality, stability, query cost and simplicity. + Do not primarily improve scores by increasing top-k, iteration limits, token budgets or output length. + Investigate better extraction, persistence, routing, retrieval, fusion, deduplication, reranking and context selection. + + Read the runtime guides when needed. Tools can inspect memory checkpoints, evidence and commit-pinned code. + Use revision explicitly when examining code associated with an evaluation. Shell and Python run within the same + OS file sandbox as Read/Edit/Write; only declared candidate roots and scratch are writable. Network is denied. + Never write Git metadata, runtime, datasets, evaluation artifacts or reports. Run ReMe only through home_evaluate. + + Finish with a clean checkout of the best evaluated branch and full commit. Choosing the baseline is valid if + improvements are unsupported. Return exactly one JSON object with branch_name, commit_sha, evaluation_path, + and report. report must explain the chosen evidence, comparisons, regressions, costs and limitations. The host + will reject a mismatched checkout or missing, incomplete, debug, query-only or infrastructure-failed evidence. diff --git a/plugins/home/src/reme_home/agent/workflow_common.py b/plugins/home/src/reme_home/agent/workflow_common.py new file mode 100644 index 00000000..45ec2f81 --- /dev/null +++ b/plugins/home/src/reme_home/agent/workflow_common.py @@ -0,0 +1,230 @@ +"""Shared sandbox, tools and invocation helpers for Home workflow agents.""" + +import json +import sys +from pathlib import Path +from uuid import uuid4 + +from agentscope.tool import BackendBase, DirEntry, Glob, Grep, Read, Toolkit, Write +import yaml + +from ..auto_bash import AutoBash +from ..code.repository import validate_repository +from ..evaluation.artifacts import contained, write_json +from ..evaluation.component import blocking +from .case_study import python_tool +from .sandbox import create_case_study_sandbox + +INSPECTION_JOBS = ( + "home_list_validation_results", + "home_inspect_validation", + "home_trace_query_evidence", + "home_inspect_memory_history", + "home_compare_memory", + "home_code_status", + "home_list_candidate_branches", + "home_code_diff", + "home_list_code_tree", + "home_read_code_file", + "home_search_code", +) + + +def required_job(wrapper, name): + """Do not silently bypass a disabled or unstarted Home Job.""" + job = wrapper.app_context.jobs[name] + if not job.enabled or not job.is_started: + raise RuntimeError(f"Workflow requires a started Job: {name}") + return job + + +def create_toolkit(wrapper, backend, scope, *, extra_tools=(), job_names=()): + """All builtin tools share the OS sandbox; privileged mutations use managed Jobs.""" + tools = [ + Read(backend=backend), + Glob(backend=backend), + Grep(backend=backend), + Write(backend=backend), + AutoBash(cwd=scope["scratch"], backend=backend, is_read_only=False), + python_tool(backend, scope["scratch"]), + ] + tools[-1].description = ( + tools[-1] + .description.replace( + "No host path is writable.", + "Only the host-granted scratch and candidate paths are writable.", + ) + .replace("case directory", "scratch directory") + ) + tools[-2].description = ( + "Run shell commands inside the OS file sandbox; network is denied. Use absolute allowed paths. " + "Only private scratch and, for the optimizer, declared candidate paths are writable. " + "Git metadata is read-only: commit, switch, create and merge only through Home Jobs. " + "Run ReMe evaluations only through home_evaluate, never through shell or Python. " + "Keep stderr visible and use set -o pipefail for pipelines. timeout is in milliseconds." + ) + tools.extend(extra_tools) + tools.extend( + wrapper._make_tool(required_job(wrapper, name)) # pylint: disable=protected-access + for name in (*INSPECTION_JOBS, *job_names) + ) + for tool in tools: + tool.is_concurrency_safe = False + return Toolkit(tools=tools) + + +async def workflow_reply( + wrapper, + home, + role, + task, + *, + tool_factory=None, + job_names=(), + agent_options=None, + memory_limit_mib=None, +): + """Use fresh state, the dedicated configured model and sandboxed tools for every call.""" + scope = await blocking(prepare_scope, home, role) + backend = WorkflowSandbox(scope, memory_limit_mib) + probe = await backend.exec_shell( + [sys.executable, "-I", "-B", "-c", "print('home workflow sandbox')"], + timeout=10, + ) + if not probe.ok(): + raise RuntimeError("Home workflow sandbox unavailable: " + probe.stderr.decode(errors="replace")) + prompt = yaml.safe_load(Path(__file__).with_name(f"{role}.yaml").read_text(encoding="utf-8")) + return await wrapper.reply( + task, + system_prompt=prompt["system_prompt"].format(**scope), + toolkit=create_toolkit( + wrapper, + backend, + scope, + extra_tools=tool_factory(backend) if tool_factory is not None else (), + job_names=job_names, + ), + builtin_tools=[], + job_tools=[], + skills=[], + resume=None, + session_id=None, + fork_session=False, + output_schema=None, + permission_mode="bypass", + **(agent_options or {}), + ) + + +def prepare_scope(home, role): + """Grant evidence reads and private scratch writes, plus declared optimizer roots.""" + repo, record = validate_repository(home) + scratch = contained(home, f"tmp/{role}-{uuid4().hex}", directory=True) + scratch.mkdir() + roots = [ + contained(home, name, directory=True) + for name in ("code", "datasets", "docs", "evaluations", "logs", "sampling") + ] + roots.append(contained(home, "metadata/preparation.json")) + writable = [scratch] + if role == "optimizer": + for value in record["code"]["candidate_paths"]: + path = contained(repo, value) + if ".git" in path.relative_to(repo).parts or path == repo: + raise ValueError("Optimizer cannot write Git metadata or the repository root") + writable.append(path) + return { + "home": str(home), + "code_repository": str(repo), + "scratch": str(scratch), + "docs": str(home / "docs"), + "read_only_paths": [str(path) for path in roots], + "writable_paths": [str(path) for path in writable], + } + + +def save_record(home, category, payload): + """Write a uniquely named host-owned provenance record, outside agent write grants.""" + path = contained(home, f"logs/{category}/{uuid4().hex}.json") + write_json(path, payload) + return str(path) + + +_INSPECT = """\ +import json, pathlib, sys +path = pathlib.Path(sys.argv[2]) +def entry(p): + s = p.stat() + return dict(name=p.name, is_dir=p.is_dir(), size_bytes=None if p.is_dir() else s.st_size, mtime=s.st_mtime) +if sys.argv[1] == 'stat': + print(json.dumps(entry(path))) +else: + result = [] + for p in path.iterdir(): + try: + result.append(entry(p)) + except FileNotFoundError: + pass + print(json.dumps(result)) +""" + + +class WorkflowSandbox(BackendBase): + """Route every builtin I/O primitive through Home's OS sandbox, including metadata. + + Rebuild the backend before each operation: managed branch switches can + replace an allowed file's inode. Core validation still checks paths and + identities for each subprocess; no operation falls back to local I/O. + """ + + def __init__(self, scope, memory_limit_mib=None): + self.scope = scope + self.memory_limit_mib = memory_limit_mib + + def _backend(self): + return create_case_study_sandbox( + cwd=self.scope["scratch"], + read_only_paths=self.scope["read_only_paths"], + writable_paths=self.scope["writable_paths"], + memory_limit_mib=self.memory_limit_mib, + ) + + async def getcwd(self): + """Use private scratch as the default, never the host process directory.""" + return self.scope["scratch"] + + async def exec_shell(self, command, *, cwd=None, timeout=None): + """Keep command execution, environment and process cleanup inside isolation.""" + return await self._backend().exec_shell(command, cwd=cwd, timeout=timeout) + + async def read_file(self, path): + """Use core sandboxed file reads, not LocalBackend's host reads.""" + return await self._backend().backend.read_file(path) + + async def write_file(self, path, data): + """Use core sandboxed writes, including parent creation and stdin transport.""" + await self._backend().backend.write_file(path, data) + + async def _inspect(self, operation, path): + result = await self.exec_shell([sys.executable, "-I", "-B", "-c", _INSPECT, operation, path], timeout=10) + if not result.ok(): + raise OSError(result.stderr.decode(errors="replace")) + return json.loads(result.stdout) + + async def stat(self, path): + """Use portable sandboxed metadata on Linux and macOS.""" + return DirEntry(**await self._inspect("stat", path)) + + async def scandir(self, path): + """Avoid GNU find assumptions and host filesystem metadata access.""" + return [DirEntry(**item) for item in await self._inspect("scandir", path)] + + async def stat_mtime(self, path): + """Preserve the builtin Edit/Write optimistic modification checks.""" + return (await self.stat(path)).mtime + + async def list_dir(self, path, *, recursive=False): + """Use the portable metadata implementation for immediate children.""" + if recursive: + return await super().list_dir(path, recursive=True) + return [entry.name for entry in await self.scandir(path)] diff --git a/plugins/home/src/reme_home/evaluation/budget.py b/plugins/home/src/reme_home/evaluation/budget.py new file mode 100644 index 00000000..3b4cc752 --- /dev/null +++ b/plugins/home/src/reme_home/evaluation/budget.py @@ -0,0 +1,43 @@ +"""Formal optimizer budgets derived from Home's durable sandbox-start evidence.""" + +from .history import read_state + + +def validate_max_validations(value): + """Accept only a positive host-supplied integer, matching dev_meta.""" + if not isinstance(value, int) or isinstance(value, bool) or value < 1: + raise ValueError("max_validations must be a positive integer") + return value + + +def validation_budget_status(home, max_validations): + """Report started formal runs, independently of completion, score or sampler success.""" + validate_max_validations(max_validations) + used = sum(not key.startswith("init/") for key in read_state(home)["started_validations"]) + remaining = max(0, max_validations - used) + gated = max_validations // 2 + if not remaining: + phase = "exhausted" + guidance = ( + "The formal validation budget is exhausted; select among existing evidence without another validation." + ) + elif used < gated: + phase = "structural_exploration" + guidance = ( + "Prefer a substantial code-level mechanism or coherent plugin architecture change. " + "Do not spend an early validation on prompt-only or single-parameter tuning." + ) + else: + phase = "evidence_driven_refinement" + guidance = ( + "Prefer evidence-driven parameter tuning, fine-grained prompt refinement, or a small scoped code change. " + "Avoid a new broad architecture unless existing validation evidence clearly requires it." + ) + return { + "used_validations": used, + "max_validations": max_validations, + "remaining_validations": remaining, + "gated_validation_count": gated, + "optimization_phase": phase, + "next_validation_guidance": guidance, + } diff --git a/plugins/home/src/reme_home/evaluation/component.py b/plugins/home/src/reme_home/evaluation/component.py index 606d3ac0..84c9d3c6 100644 --- a/plugins/home/src/reme_home/evaluation/component.py +++ b/plugins/home/src/reme_home/evaluation/component.py @@ -29,6 +29,7 @@ from ..judge import HomeJudge from ..workspace import HomeWorkspace, _initialization_lock, _validate_directory from ..sampling import SamplingPolicy, create_sampling_plan from .history import HomeEvaluationHistory +from .budget import validation_budget_status from .repetition import previous_candidate_result @@ -85,7 +86,25 @@ class HomeEvaluation(BaseComponent): _validate_directory(self.workspace_path.resolve(), self.workspace.state_path) lock = nullcontext() if spec.debug else _initialization_lock(self.workspace.state_path / "validation.lock") with lock: - return await self._evaluate(spec) + budgeted = spec.max_validations is not None and not spec.debug + if budgeted: + budget = await blocking(validation_budget_status, self.workspace.home_path, spec.max_validations) + if budget["remaining_validations"] < 1: + return { + "status": "rejected", + "error_code": "VALIDATION_BUDGET_EXHAUSTED", + "error": "Optimizer validation budget is exhausted; no sandbox was started.", + "validation_budget": budget, + "result_dir": None, + } + result = await self._evaluate(spec) + if budgeted: + result["validation_budget"] = await blocking( + validation_budget_status, + self.workspace.home_path, + spec.max_validations, + ) + return result def _prepare(self, spec, environment): home = self.workspace.home_path diff --git a/plugins/home/src/reme_home/evaluation/inputs.py b/plugins/home/src/reme_home/evaluation/inputs.py index fbeea7fc..a319988d 100644 --- a/plugins/home/src/reme_home/evaluation/inputs.py +++ b/plugins/home/src/reme_home/evaluation/inputs.py @@ -29,6 +29,7 @@ class EvaluationSpec(SpecModel): debug: bool = False sampling: bool = False record_history: bool = True + max_validations: int | None = Field(default=None, ge=1) sampling_policy: SamplingPolicy | None = None concurrency: int = Field(default=4, ge=1) timeout_seconds: float = Field(default=7200.0, gt=0, allow_inf_nan=False) @@ -67,6 +68,8 @@ class EvaluationSpec(SpecModel): @model_validator(mode="after") def resolve_mode(self): """Debug stops on failure by default and never reuses scored evidence.""" + if self.max_validations is not None and not self.debug and not self.record_history: + raise ValueError("Budgeted formal evaluations require record_history=true") if self.debug and self.reuse_construction_from: raise ValueError("Construction reuse requires training-set") if self.sampling and not self.debug and (self.case_ids is not None or self.query_ids is not None): diff --git a/plugins/home/src/reme_home/evaluation/step.py b/plugins/home/src/reme_home/evaluation/step.py index b9e24773..d2dcc4d3 100644 --- a/plugins/home/src/reme_home/evaluation/step.py +++ b/plugins/home/src/reme_home/evaluation/step.py @@ -39,6 +39,9 @@ class HomeEvaluateStep(BaseStep): self.context.response.metadata["result_dir"] = str(error.path) return self.context.response self.context.response.answer = result + if result.get("status") == "rejected": + self.context.response.success = False + return self.context.response # Candidate failures are evaluation observations; infrastructure failures # must remain visible to ordinary Job callers checking Response.success. self.context.response.success = result["case_status_counts"]["infra_error"] == 0 diff --git a/plugins/home/src/reme_home/plugin.yaml b/plugins/home/src/reme_home/plugin.yaml index 7b22f3aa..7bddc013 100644 --- a/plugins/home/src/reme_home/plugin.yaml +++ b/plugins/home/src/reme_home/plugin.yaml @@ -1,4 +1,8 @@ backends: + home_optimizer_agent: reme_home.agent.optimizer:OptimizerAgentWrapper + home_diagnose_step: reme_home.agent.diagnostic:HomeDiagnoseStep + home_optimize_step: reme_home.agent.optimizer:HomeOptimizeStep + home_workflow_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper home_case_study_step: reme_home.agent.case_study:HomeCaseStudyStep home_case_study_agent: reme_home.agent.wrapper:CaseStudyAgentWrapper home_trace_query_evidence_step: reme_home.evaluation.evidence_step:HomeTraceQueryEvidenceStep @@ -20,6 +24,30 @@ backends: application_defaults: components: as_llm: + home_optimizer: + backend: ${LLM_BACKEND:-openai} + model: qwen3.8-max + stream: false + context_size: 160000 + max_retries: 3 + credential: + api_key: ${LLM_API_KEY:-} + base_url: ${LLM_BASE_URL:-} + parameters: + max_tokens: 65536 + thinking_enable: false + home_diagnostic: + backend: ${LLM_BACKEND:-openai} + model: qwen3.8-max + stream: false + context_size: 128000 + max_retries: 3 + credential: + api_key: ${LLM_API_KEY:-} + base_url: ${LLM_BASE_URL:-} + parameters: + max_tokens: 65536 + thinking_enable: false home_case_study: backend: ${LLM_BACKEND:-openai} model: qwen3.7-max @@ -33,6 +61,22 @@ application_defaults: max_tokens: 65536 thinking_enable: false agent_wrapper: + home_optimizer: + backend: home_optimizer_agent + max_validations: 10 + as_llm: home_optimizer + validation_cycle_time_budget_minutes: 150 + context_config: + trigger_ratio: 0.85 + reserve_ratio: 0.15 + react_config: + max_iters: 3000 + home_diagnostic: + backend: home_workflow_agent + as_llm: home_diagnostic + time_budget_minutes: 60 + react_config: + max_iters: 80 home_case_study: backend: home_case_study_agent as_llm: home_case_study @@ -64,6 +108,41 @@ application_defaults: default: backend: home_workspace jobs: + home_diagnose: + backend: base + description: "Diagnose explicit evaluation runs with a fresh read-only agent; aggregate case studies and return evidence-supported root causes without prescribing fixes" + parameters: + type: object + additionalProperties: false + required: [evaluation_paths] + properties: + evaluation_paths: + type: array + minItems: 1 + uniqueItems: true + items: {type: string, minLength: 1} + description: "Explicit evaluation result directories, absolute or relative to home/" + instruction: + type: string + minLength: 1 + description: "Diagnostic question or suspected regression" + steps: + - backend: home_diagnose_step + agent_wrapper: home_diagnostic + home_optimize: + backend: base + description: "Check or run the full initial baseline, then diagnose, edit, commit, debug and evaluate candidates; verify the final clean checkout against saved formal evidence" + parameters: + type: object + additionalProperties: false + properties: + instruction: + type: string + minLength: 1 + description: "Optimization focus; runtime, selection and sandbox settings are host-owned" + steps: + - backend: home_optimize_step + agent_wrapper: home_optimizer home_case_study: backend: base description: "Investigate up to eight evaluation cases concurrently with a shared focus; each case uses a fresh read-only sandboxed agent and returns its own report or error in input order" @@ -227,6 +306,10 @@ application_defaults: description: "Stop on the first failure; defaults to debug (false for scored runs, true for debug)" max_construction_failure_rate: {type: number, minimum: 0, maximum: 1} max_artifact_bytes: {type: integer, minimum: 1} + max_validations: + type: [integer, "null"] + minimum: 1 + description: "Optional host formal-evaluation budget; optimizer injects its configured limit" steps: - backend: home_evaluate_step home_prepare: diff --git a/plugins/home/tests/test_benchmark_runner.py b/plugins/home/tests/test_benchmark_runner.py index 797768c6..9a6a92ae 100644 --- a/plugins/home/tests/test_benchmark_runner.py +++ b/plugins/home/tests/test_benchmark_runner.py @@ -1,6 +1,8 @@ """Benchmark Home defaults and lifecycle wiring, without models or dataset writes.""" import hashlib +import json +import sys from importlib import import_module from pathlib import Path from unittest.mock import AsyncMock @@ -10,7 +12,8 @@ import yaml from reme.schema import Response -from benchmark import home_runner +from benchmark.beam import run_home as beam_runner +from benchmark.longmemeval import run_home as lme_runner ROOT = Path(__file__).resolve().parents[3] @@ -21,7 +24,7 @@ def test_preserved_lme_split_and_evidence(): assert hashlib.sha256((directory / "home_evidence_sessions.jsonl").read_bytes()).hexdigest() == ( "35655c51210a96b595910923576fa97b93c81abd82e3cfcda801180007c23a7d" ) - workspace, spec = home_runner.load_config(directory / "config_home.yaml") + workspace, spec = lme_runner.load_config(directory / "config_home.yaml") selected = spec["benchmark"]["case_ids"] assert len(selected) == len(set(selected)) == 250 # Hash the original first-five-group sequence to preserve both IDs and order. @@ -36,7 +39,7 @@ def test_preserved_lme_split_and_evidence(): def test_beam_defaults_use_all_100k_cases_from_any_cwd(tmp_path, monkeypatch): """Paths and selection do not depend on where Python was launched.""" monkeypatch.chdir(tmp_path) - workspace, spec = home_runner.load_config(ROOT / "benchmark/beam/config_home.yaml") + workspace, spec = beam_runner.load_config(ROOT / "benchmark/beam/config_home.yaml") assert spec["benchmark"]["variant"] == "100K" assert spec["benchmark"]["all_cases"] is True assert spec["benchmark"]["case_ids"] is None @@ -47,6 +50,18 @@ def test_beam_defaults_use_all_100k_cases_from_any_cwd(tmp_path, monkeypatch): assert workspace == ROOT / "benchmark/beam/workspaces/home-100K" +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +def test_runner_preserves_explicit_candidate_paths(tmp_path, benchmark): + """Runner defaults preserve custom scopes from existing configurations.""" + module = import_module(f"benchmark.{benchmark}.run_home") + config = yaml.safe_load(module.DEFAULT_CONFIG.read_text()) + config["preparation"]["candidate"] = {"paths": ["reme/steps"]} + path = tmp_path / "home.yaml" + path.write_text(yaml.safe_dump(config)) + _, spec = module.load_config(path) + assert spec["candidate"]["paths"] == ["reme/steps"] + + @pytest.mark.asyncio @pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, benchmark): @@ -60,7 +75,7 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b observed.update(kwargs) return application - monkeypatch.setattr(home_runner, "Application", factory) + monkeypatch.setattr(module, "Application", factory) result = await module.run(workspace_dir=str(tmp_path / "experiment")) assert observed["plugins"] == ["home", "beam-judge" if benchmark == "beam" else "lme-judge"] assert observed["service"] == {"backend": "cli"} @@ -72,6 +87,10 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b "ewma_alpha": 0.3, "uniform_exploration": 0.2, } + assert observed["components"]["agent_wrapper"]["home_optimizer"] == { + "validation_cycle_time_budget_minutes": 150, + "max_validations": 10, + } application.start.assert_awaited_once() application.close.assert_awaited_once() assert application.run_job.await_count == 1 @@ -80,9 +99,11 @@ async def test_python_runner_starts_prepares_and_closes(tmp_path, monkeypatch, b @pytest.mark.asyncio -async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch): +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monkeypatch, benchmark): """Partial policies merge typed defaults and never leak into preparation inputs.""" - config = yaml.safe_load((ROOT / "benchmark/beam/config_home.yaml").read_text()) + module = import_module(f"benchmark.{benchmark}.run_home") + config = yaml.safe_load(module.DEFAULT_CONFIG.read_text()) config["sampling_policy"] = {"max_cases": 2, "max_queries": 7} path = tmp_path / "home.yaml" path.write_text(yaml.safe_dump(config)) @@ -94,8 +115,8 @@ async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monk observed.update(kwargs) return application - monkeypatch.setattr(home_runner, "Application", factory) - await home_runner.prepare(path) + monkeypatch.setattr(module, "Application", factory) + await module.prepare(path) policy = observed["components"]["home.evaluation"]["default"]["sampling_policy"] assert policy["max_cases"] == 2 and policy["max_queries"] == 7 and policy["ewma_alpha"] == 0.3 assert "sampling_policy" not in application.run_job.call_args.kwargs["spec"] @@ -103,22 +124,193 @@ async def test_runner_validates_and_passes_custom_sampling_policy(tmp_path, monk config["sampling_policy"] = invalid path.write_text(yaml.safe_dump(config)) with pytest.raises(ValueError): - await home_runner.prepare(path) + await module.prepare(path) assert application.start.await_count == 1 assert application.run_job.await_count == 1 assert application.run_job.call_args.args == ("home_prepare",) @pytest.mark.asyncio +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +@pytest.mark.parametrize("budget", ["omitted", 45.5, None]) +async def test_runner_passes_cycle_budget_to_optimizer(tmp_path, monkeypatch, benchmark, budget): + """Both local runners apply the default, custom duration and explicit disable value.""" + module = import_module(f"benchmark.{benchmark}.run_home") + config = yaml.safe_load(module.DEFAULT_CONFIG.read_text()) + if budget == "omitted": + config.pop("validation_cycle_time_budget_minutes", None) + else: + config["validation_cycle_time_budget_minutes"] = budget + path = tmp_path / "home.yaml" + path.write_text(yaml.safe_dump(config)) + application = AsyncMock() + application.run_job.return_value = Response(answer={}) + observed = {} + + def factory(**kwargs): + observed.update(kwargs) + return application + + monkeypatch.setattr(module, "Application", factory) + await module.run(path) + expected = 150 if budget == "omitted" else budget + assert observed["components"]["agent_wrapper"]["home_optimizer"] == { + "validation_cycle_time_budget_minutes": expected, + "max_validations": 10, + } + assert "validation_cycle_time_budget_minutes" not in application.run_job.call_args.kwargs["spec"] + application.close.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +@pytest.mark.parametrize("budget", [0, -1, True, "150", float("nan"), float("inf")]) +async def test_runner_rejects_invalid_cycle_budget_before_start(tmp_path, monkeypatch, benchmark, budget): + """Reject nonpositive, nonnumeric and nonfinite durations without creating an application.""" + module = import_module(f"benchmark.{benchmark}.run_home") + config = yaml.safe_load(module.DEFAULT_CONFIG.read_text()) + config["validation_cycle_time_budget_minutes"] = budget + path = tmp_path / "home.yaml" + path.write_text(yaml.safe_dump(config)) + created = [] + monkeypatch.setattr(module, "Application", lambda **kwargs: created.append(kwargs)) + with pytest.raises(ValueError): + await module.run(path) + assert not created + + +@pytest.mark.asyncio +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) @pytest.mark.parametrize("failure", ["startup", "preparation"]) -async def test_runner_propagates_failure_and_closes(monkeypatch, failure): +async def test_runner_propagates_failure_and_closes(monkeypatch, failure, benchmark): """A failed preparation is never printed or returned as success.""" application = AsyncMock() if failure == "startup": application.start.side_effect = RuntimeError("startup failed") else: application.run_job.return_value = Response(success=False, answer="already prepared") - monkeypatch.setattr(home_runner, "Application", lambda **kwargs: application) + module = import_module(f"benchmark.{benchmark}.run_home") + monkeypatch.setattr(module, "Application", lambda **kwargs: application) with pytest.raises(RuntimeError): - await home_runner.prepare(ROOT / "benchmark/beam/config_home.yaml") + await module.prepare(module.DEFAULT_CONFIG) application.close.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +@pytest.mark.parametrize("limit", ["omitted", 3, 0, -1, True, "10", 1.5, None]) +async def test_runner_formal_budget_configuration(tmp_path, monkeypatch, benchmark, limit): + """Validate host count budgets and pass them only to the optimizer wrapper.""" + module = import_module(f"benchmark.{benchmark}.run_home") + config = yaml.safe_load(module.DEFAULT_CONFIG.read_text()) + if limit == "omitted": + config.pop("max_validations", None) + else: + config["max_validations"] = limit + path = tmp_path / "home.yaml" + path.write_text(yaml.safe_dump(config)) + application = AsyncMock() + application.run_job.return_value = Response(answer={}) + observed = {} + + def factory(**kwargs): + observed.update(kwargs) + return application + + monkeypatch.setattr(module, "Application", factory) + if limit in ("omitted", 3): + await module.prepare(path) + assert observed["components"]["agent_wrapper"]["home_optimizer"]["max_validations"] == ( + 10 if limit == "omitted" else limit + ) + assert "max_validations" not in application.run_job.call_args.kwargs["spec"] + else: + with pytest.raises(ValueError): + await module.prepare(path) + assert not observed + + +@pytest.mark.asyncio +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +@pytest.mark.parametrize("minutes", [30, None]) +async def test_optimizer_lifecycle_uses_same_config_and_does_not_prepare(tmp_path, monkeypatch, benchmark, minutes): + """A separate optimization launch consumes the same count, timer and sampling settings.""" + module = import_module(f"benchmark.{benchmark}.run_home") + config = yaml.safe_load(module.DEFAULT_CONFIG.read_text()) + config.update(max_validations=3, validation_cycle_time_budget_minutes=minutes) + path = tmp_path / "home.yaml" + path.write_text(yaml.safe_dump(config)) + application = AsyncMock() + application.run_job.return_value = Response(answer={"status": "completed"}) + observed = {} + + def factory(**kwargs): + observed.update(kwargs) + return application + + monkeypatch.setattr(module, "Application", factory) + result = await module.optimize(path, workspace_dir=str(tmp_path / "existing"), instruction="Improve retrieval") + assert result["status"] == "completed" + assert observed["components"]["agent_wrapper"]["home_optimizer"] == { + "max_validations": 3, + "validation_cycle_time_budget_minutes": minutes, + } + application.run_job.assert_awaited_once_with("home_optimize", instruction="Improve retrieval") + application.close.assert_awaited_once() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +async def test_optimizer_failure_closes_application(monkeypatch, benchmark): + """Failed optimization retains the normal cleanup and error contract.""" + module = import_module(f"benchmark.{benchmark}.run_home") + application = AsyncMock() + application.run_job.return_value = Response(success=False, answer="optimization failed") + monkeypatch.setattr(module, "Application", lambda **kwargs: application) + with pytest.raises(RuntimeError, match="optimization failed"): + await module.optimize() + application.run_job.assert_awaited_once_with("home_optimize") + application.close.assert_awaited_once() + + +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +def test_optimizer_cli_forwards_configuration_and_focus(tmp_path, monkeypatch, capsys, benchmark): + """The explicit CLI mode reaches optimization and preserves the preparation default.""" + module = import_module(f"benchmark.{benchmark}.run_home") + config = tmp_path / "home.yaml" + optimize = AsyncMock(return_value={"status": "completed"}) + prepare = AsyncMock() + monkeypatch.setattr(module, "optimize", optimize) + monkeypatch.setattr(module, "prepare", prepare) + monkeypatch.setattr( + sys, + "argv", + [ + "run_home.py", + "--optimize", + "--config", + str(config), + "--workspace-dir", + str(tmp_path / "existing"), + "--instruction", + "Improve retrieval", + ], + ) + module.main() + optimize.assert_awaited_once_with( + config, + workspace_dir=str(tmp_path / "existing"), + instruction="Improve retrieval", + ) + prepare.assert_not_called() + assert json.loads(capsys.readouterr().out) == {"status": "completed"} + + +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +def test_cli_rejects_optimization_focus_in_preparation_mode(monkeypatch, benchmark): + """Do not silently discard an instruction when the caller forgot --optimize.""" + module = import_module(f"benchmark.{benchmark}.run_home") + monkeypatch.setattr(sys, "argv", ["run_home.py", "--instruction", "Improve retrieval"]) + with pytest.raises(SystemExit) as error: + module.main() + assert error.value.code == 2 diff --git a/plugins/home/tests/test_optimizer_compression.py b/plugins/home/tests/test_optimizer_compression.py new file mode 100644 index 00000000..eae651b5 --- /dev/null +++ b/plugins/home/tests/test_optimizer_compression.py @@ -0,0 +1,215 @@ +"""Compression recovery parity with dev_meta, without model or filesystem services.""" + +# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access +# pylint: disable=redefined-outer-name,unused-argument + +import asyncio +import json + +from agentscope.agent import Agent, ContextConfig +from agentscope.message import AssistantMsg, TextBlock, UserMsg, Usage +from agentscope.state import AgentState +import jsonschema +import openai +import pytest +from reme_home.agent import compression, optimizer + + +class FailingCompressionModel: + def __init__(self, ratio=0.9, failure=None): + self.context_size = 90 / ratio + self.failure = failure or RuntimeError("Failed to generate structured output for model.") + self.calls = 0 + + async def count_tokens(self, messages, tools=None): + return sum(10 for message in messages if (message.get_text_content() or "").startswith("message-")) + + async def generate_structured_output(self, **kwargs): + self.calls += 1 + raise self.failure + + +def make_agent(tmp_path, ratio=0.9, trigger=0.8, failure=None, summary="existing summary"): + return optimizer.OptimizerAgent( + name="optimizer", + system_prompt="system", + model=FailingCompressionModel(ratio, failure), + state=AgentState( + summary=summary, + context=[UserMsg(name="user", content=f"message-{index}") for index in range(9)], + ), + context_config=ContextConfig(trigger_ratio=trigger, reserve_ratio=0.2), + fallback_directory=tmp_path / "offloads", + validation_cycle_timer=optimizer.ValidationCycleTimer(150), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("trigger", "ratio", "expected"), + [(0.8, 0.79, False), (0.8, 0.85, False), (0.8, 0.9, True), (0.65, 0.82, False), (0.65, 0.825, True)], +) +async def test_real_compression_failure_uses_dev_meta_emergency_threshold(tmp_path, trigger, ratio, expected): + agent = make_agent(tmp_path, ratio, trigger) + context_before, summary_before = agent.state.context, agent.state.summary + await agent.compress_context() + assert agent.validation_cycle_timer.context_compression_count == int(expected) + assert len(list(tmp_path.rglob("context-*.json"))) == int(expected) + assert agent.model.calls == int(ratio >= trigger) + assert agent.context_config.compression_fallback_to_truncation # The shared config was not mutated. + if not expected: + assert agent.state.context is context_before + assert agent.state.summary is summary_before + + +@pytest.mark.asyncio +@pytest.mark.parametrize("summary", ["existing summary", [TextBlock(text="existing summary")]]) +@pytest.mark.parametrize("reserve_ratio", [0.2, 0]) +async def test_offload_retains_full_messages_summary_usage_and_recent_context(tmp_path, summary, reserve_ratio): + agent = make_agent(tmp_path, summary=summary) + agent.context_config.reserve_ratio = reserve_ratio + agent.state.context[-1] = AssistantMsg( + id=agent.state.reply_id, + name=agent.name, + content="message-8", + usage=Usage(input_tokens=100, output_tokens=10), + ) + original = [msg.model_dump(mode="json", exclude_none=False) for msg in agent.state.context] + await agent.compress_context() + (path,) = tmp_path.rglob("context-*.json") + data = json.loads(path.read_text()) + assert data["messages"] == original[:7] if reserve_ratio else data["messages"] == original + if reserve_ratio: + assert [msg.get_text_content() for msg in agent.state.context] == ["message-7", "message-8"] + else: + assert len(agent.state.context) == 1 + assert agent.state.context[0].content == [] # SDK usage carrier after full offload. + assert data["session_id"] == agent.state.session_id + assert data["context_ratio"] == 0.9 + assert data["emergency_ratio"] == 0.9 + assert data["failure"]["type"] == "RuntimeError" + text = agent.state.summary + if isinstance(text, list): + text = "\n".join(block.text for block in text) + assert "existing summary" in text + assert "without semantic summarization" in text + assert str(path) in text + assert agent._get_reply_usage().input_tokens == 100 + assert not list(path.parent.glob(".result-*")) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "failure", + [RuntimeError("The system prompt exceeds threshold"), ValueError("bad configuration"), asyncio.CancelledError()], +) +async def test_unrecoverable_failures_and_cancellation_propagate(tmp_path, failure): + agent = make_agent(tmp_path, failure=failure) + original = agent.state.context + with pytest.raises(type(failure)): + await agent.compress_context() + assert agent.state.context is original + assert not list(tmp_path.rglob("*.json")) + assert agent.validation_cycle_timer.context_compression_count == 0 + + +@pytest.mark.parametrize( + ("failure", "expected"), + [ + (RuntimeError("provider failed"), True), + (RuntimeError("The system prompt exceeds threshold"), False), + (jsonschema.ValidationError("invalid summary"), True), + (openai.OpenAIError("provider unavailable"), True), + (ValueError("programming error"), False), + (OSError("disk full"), False), + ], +) +def test_recoverable_error_classifier(failure, expected): + assert compression.RecoverableCompressionAgent._is_recoverable_compression_error(failure) is expected + + +@pytest.mark.asyncio +async def test_override_config_and_successful_compression_use_native_pipeline(tmp_path, monkeypatch): + agent = make_agent(tmp_path, ratio=0.85) + config = ContextConfig(trigger_ratio=0.65, reserve_ratio=0.1) + calls = [] + + async def compress(self, context_config=None, instructions=None): + calls.append((context_config, instructions)) + self.state.summary = "semantic summary" + self.state.context = [] + + monkeypatch.setattr(Agent, "_compress_context_impl", compress) + await agent.compress_context(context_config=config) + assert calls[0][0].trigger_ratio == 0.65 + assert calls[0][0].reserve_ratio == 0.1 + assert not calls[0][0].compression_fallback_to_truncation + assert config.compression_fallback_to_truncation + assert agent.state.summary == "semantic summary" + assert agent.validation_cycle_timer.context_compression_count == 1 + assert not list(tmp_path.rglob("*.json")) + + +@pytest.mark.asyncio +async def test_offload_write_failure_keeps_context_and_does_not_count(tmp_path, monkeypatch): + agent = make_agent(tmp_path) + original = agent.state.context + + def fail(*args, **kwargs): + raise OSError("disk full") + + monkeypatch.setattr(compression, "write_json", fail) + with pytest.raises(OSError, match="disk full"): + await agent.compress_context() + assert agent.state.context is original + assert agent.state.summary == "existing summary" + assert agent.validation_cycle_timer.context_compression_count == 0 + + +@pytest.mark.asyncio +async def test_cancellation_during_apply_finishes_consistent_offload_then_propagates(tmp_path, monkeypatch): + agent = make_agent(tmp_path) + started, release = asyncio.Event(), asyncio.Event() + + async def clear_cache(messages): + started.set() + await release.wait() + + monkeypatch.setattr(agent, "_clear_unreserved_read_cache", clear_cache) + task = asyncio.create_task(agent.compress_context()) + await started.wait() + task.cancel() + release.set() + with pytest.raises(asyncio.CancelledError): + await task + assert len(list(tmp_path.rglob("context-*.json"))) == 1 + assert "offloaded" in agent.state.summary + assert len(agent.state.context) == 2 + + +@pytest.mark.asyncio +async def test_missing_directory_or_unsplittable_context_fails_explicitly(tmp_path, monkeypatch): + agent = make_agent(tmp_path) + agent.fallback_directory = None + with pytest.raises(RuntimeError, match="fallback_directory"): + await agent.compress_context() + agent.fallback_directory = tmp_path / "offloads" + + async def no_old_messages(*args): + return [], agent.state.context + + monkeypatch.setattr(agent, "_split_context_for_compression", no_old_messages) + with pytest.raises(RuntimeError, match="cannot remove any context"): + await agent.compress_context() + assert not list(tmp_path.rglob("*.json")) + + +def test_wrapper_uses_host_offload_directory(tmp_path): + wrapper = optimizer.OptimizerAgentWrapper(name="optimizer") + agent = wrapper._create_agent( + options={"_context_fallback_directory": tmp_path / "logs/context-offloads"}, + name="optimizer", + system_prompt="system", + model=FailingCompressionModel(), + ) + assert agent.fallback_directory == tmp_path / "logs/context-offloads" diff --git a/plugins/home/tests/test_optimizer_cycle.py b/plugins/home/tests/test_optimizer_cycle.py new file mode 100644 index 00000000..04cd50f8 --- /dev/null +++ b/plugins/home/tests/test_optimizer_cycle.py @@ -0,0 +1,297 @@ +"""Meta-ReMe cycle timing parity with fake clocks and isolated evaluation evidence.""" + +# pylint: disable=missing-function-docstring,missing-class-docstring,protected-access +# pylint: disable=redefined-outer-name,unused-argument + +import asyncio +import copy +from types import SimpleNamespace + +import pytest +from agentscope.agent import ModelConfig, ReActConfig +from agentscope.message import HintBlock, TextBlock, UserMsg +from agentscope.model import ChatResponse, ChatUsage +from agentscope.state import AgentState +from reme_home.agent import optimizer +from reme_home.agent.bounded import BoundedReActAgent +from reme_home.evaluation.artifacts import write_json +from test_bounded_agent import RecordingModel + + +def make_agent(timer, model=None): + return optimizer.OptimizerAgent( + name="optimizer", + system_prompt="Improve general memory mechanisms.", + model=model or RecordingModel(), + validation_cycle_timer=timer, + model_config=ModelConfig(max_retries=0), + react_config=ReActConfig(max_iters=100), + ) + + +def cycle_hints(messages, source="meta_reme_validation_timer"): + return [ + block + for message in messages + for block in message.content + if isinstance(block, HintBlock) and block.source == source + ] + + +@pytest.mark.parametrize("budget", [True, 0, -1, float("nan"), float("inf"), "10"]) +def test_cycle_rejects_invalid_budget(budget): + with pytest.raises(ValueError, match="positive"): + optimizer.ValidationCycleTimer(budget) + + +@pytest.mark.asyncio +async def test_cycle_injects_dev_meta_reminders_and_resets(): + now = [0.0] + timer = optimizer.ValidationCycleTimer(100, clock=lambda: now[0]) + + class SnapshotModel(RecordingModel): + async def __call__(self, **kwargs): + return await super().__call__(**copy.deepcopy(kwargs)) + + agent = make_agent(timer, SnapshotModel()) + timer.record_context_compression() + timer.record_context_compression() + delivered = [] + for minute in (49, 50, 74, 75, 90, 100, 109, 110, 120): + now[0] = minute * 60 + _ = [item async for item in agent._reasoning()] + reminders = [block.hint for block in cycle_hints(agent.model.calls[-1]["messages"])] + if minute in (50, 75, 90, 100, 110, 120): + delivered.append(reminders[-1]) + assert timer.next_reminder() is None + for percentage, text in zip((50, 75, 90, 100, 110, 120), delivered): + assert f"({percentage}% of its 100 minutes soft time budget)" in text + assert "2 context compressions since the last scored validation" in text + assert "start validation within 50 minutes" in delivered[0] + assert "start validation within 10 minutes" in delivered[2] + assert "starting validation as soon as possible" in delivered[3] + persistent = cycle_hints(agent.state.context) + assert len(persistent) == 3 + for percentage, message in zip((75, 100, 120), persistent): + assert f"({percentage}%" in message.hint + timer.reset() + assert timer.context_compression_count == 0 + now[0] += 60 + assert timer.next_reminder() is None + # Reset keeps historical persistent messages and restarts alternation. + assert cycle_hints(agent.state.context) == persistent + now[0] = 170 * 60 + assert timer.next_reminder().persistent is False + + +def test_cycle_skips_old_thresholds_and_formats_seconds(): + now = [0.0] + timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0]) + now[0] = 59 + reminder = timer.next_reminder() + assert "(90%" in reminder.content + assert "within 1 second." in reminder.content + now[0] = 101 + assert "(160%" in timer.next_reminder().content + assert timer.next_reminder() is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("failure", [RuntimeError("model unavailable"), asyncio.CancelledError()]) +async def test_cycle_ephemeral_cleanup_on_error_and_cancellation(monkeypatch, failure): + now = [0.0] + timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0]) + agent = make_agent(timer) + + async def fail(*args, **kwargs): + raise failure + yield # pylint: disable=unreachable + + monkeypatch.setattr(BoundedReActAgent, "_reasoning", fail) + now[0] = 30 + with pytest.raises(type(failure)): + _ = [item async for item in agent._reasoning()] + assert not cycle_hints(agent.state.context) + now[0] = 45 + with pytest.raises(type(failure)): + _ = [item async for item in agent._reasoning()] + assert len(cycle_hints(agent.state.context)) == 1 # The second reminder is persistent. + + +@pytest.mark.asyncio +async def test_cycle_counts_only_completed_context_replacements(monkeypatch): + timer = optimizer.ValidationCycleTimer(1) + agent = make_agent(timer) + + async def unchanged(self): + pass + + monkeypatch.setattr(BoundedReActAgent, "compress_context", unchanged) + await agent.compress_context() + assert timer.context_compression_count == 0 + + async def compressed(self): + self.state.summary = "Retained previous evidence." + self.state.context = [] + + monkeypatch.setattr(BoundedReActAgent, "compress_context", compressed) + await agent.compress_context() + assert timer.context_compression_count == 1 + + async def failed(self): + raise RuntimeError("compression unavailable") + + monkeypatch.setattr(BoundedReActAgent, "compress_context", failed) + with pytest.raises(RuntimeError, match="unavailable"): + await agent.compress_context() + assert timer.context_compression_count == 1 + + +def write_history(home, started): + write_json( + home / "sampling/state.json", + { + "schema_version": 2, + "round": len(started), + "successful_round": 0, + "started_validations": {key: index for index, key in enumerate(started, 1)}, + "processed_validations": [], + "cases": {}, + "queries": {}, + }, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("timed", [False, True]) +@pytest.mark.parametrize( + "outcome,reset", + [ + ("completed", True), + ("success", True), + ("candidate_failure", True), + ("completed_infra_error", True), + ("debug", False), + ("rejected", False), + ("failed", False), + ("partial", False), + ("not_started", False), + ("preexisting", False), + ("cancelled", False), + ], +) +async def test_evaluation_reset_boundary_matches_dev_meta(tmp_path, outcome, reset, timed): + home = tmp_path + previous = ["init/initial/1", "candidate/previous/1"] + key = "candidate/current/1" + if outcome == "preexisting": + previous.append(key) + write_history(home, previous) + now = [0.0] + timer = optimizer.ValidationCycleTimer(1, clock=lambda: now[0]) + timer.record_context_compression() + now[0] = 45 + timer.next_reminder() # First reminder is ephemeral. + state = AgentState() + old = UserMsg(name="meta_reme_validation_timer", content="Previous persistent reminder.") + state.context.append(old) + + class EvaluationJob: + enabled = is_started = True + + async def __call__(self, **kwargs): + assert kwargs["debug"] is (outcome == "debug") + # Tool execution time remains part of the previous cycle until completion. + now[0] = 90 + if outcome == "cancelled": + raise asyncio.CancelledError() + if outcome not in {"debug", "rejected", "not_started", "preexisting"}: + write_history(home, [*previous, key]) + manifest = { + "commit_sha": "1b2cdcc" + "0" * 33, + "status": "failed" if outcome in {"failed", "partial"} else "completed", + "mode": "debug" if outcome == "debug" else "scored", + "dataset_kind": "training-set", + "sandbox_started_at": None if outcome == "not_started" else "2026-10-31T00:00:00Z", + } + run = home / "evaluations" / key + write_json(run / "manifest.json", manifest) + status = outcome if outcome in {"success", "rejected", "failed", "partial"} else "completed" + return SimpleNamespace( + success=outcome not in {"completed_infra_error", "rejected", "failed", "partial"}, + answer={"status": status, "result_dir": str(run), "mean_query_score": 0.0}, + metadata={}, + ) + + wrapper = SimpleNamespace(app_context=SimpleNamespace(jobs={"home_evaluate": EvaluationJob()})) + tool = optimizer.evaluation_tool(wrapper, {}, home=home, timer=timer if timed else None) + assert set(tool.input_schema["properties"]) == {"debug"} + if outcome == "cancelled": + with pytest.raises(asyncio.CancelledError): + await tool._func(debug=False, _agent_state=state) + else: + await tool._func(debug=outcome == "debug", _agent_state=state) + assert len(state.context) == 1 # The tool cannot insert anything before its result is written. + pending = state.middle_context.get(optimizer.PENDING_CYCLE_TRANSITIONS, []) + assert len(pending) == int(reset) + agent = make_agent(None) + agent.state = state + _ = [item async for item in agent._reasoning()] + transitions = cycle_hints(state.context, "meta_reme_validation_cycle_transition") + assert len(transitions) == int(reset) + assert state.context[0] is old + if reset: + assert transitions[0].hint == ( + "Formal validation 2 completed successfully for commit 1b2cdcc. " + "The previous cycle timer has been reset, and all timer reminders for cycle 2 are now expired. " + "Remaining formal validations: 8. Start cycle 3." + ) + if reset and timed: + assert timer.context_compression_count == 0 + assert timer.next_reminder() is None + now[0] += 30 + assert timer.next_reminder().persistent is False + else: + assert timer.context_compression_count == 1 + assert "(150%" in timer.next_reminder().content + + +def test_wrapper_owns_fresh_cycle_timers_and_supports_disabling(): + wrapper = optimizer.OptimizerAgentWrapper(name="optimizer", validation_cycle_time_budget_minutes=150) + first, second = wrapper.create_cycle_timer(), wrapper.create_cycle_timer() + assert first is not second + assert first.time_budget_minutes == 150 + first.record_context_compression() + assert second.context_compression_count == 0 + assert wrapper.create_cycle_timer({"validation_cycle_time_budget_minutes": None}) is None + legacy = optimizer.OptimizerAgentWrapper(name="optimizer", time_budget_minutes=75) + assert legacy.create_cycle_timer().time_budget_minutes == 75 + assert legacy.create_cycle_timer({"validation_cycle_time_budget_minutes": 30}).time_budget_minutes == 30 + + +@pytest.mark.asyncio +async def test_temporary_and_persistent_reminders_preserve_total_reply_usage(): + now = [0.0] + + class AdvancingModel(RecordingModel): + async def __call__(self, **kwargs): + result = await super().__call__(**kwargs) + now[0] = 30 if len(self.calls) == 1 else 45 + return result + + model = AdvancingModel( + final=ChatResponse( + content=[TextBlock(text="final report")], + is_last=True, + usage=ChatUsage(input_tokens=2, output_tokens=3, time=0), + ), + ) + agent = make_agent(optimizer.ValidationCycleTimer(1, clock=lambda: now[0]), model) + agent.react_config.max_iters = 2 + reply = await agent.reply(UserMsg(name="user", content="Investigate")) + assert len(model.calls) == 3 + assert reply.usage.input_tokens == 6 + assert reply.usage.output_tokens == 9 + retained = cycle_hints(agent.state.context) + assert len(retained) == 1 + assert "(75%" in retained[0].hint diff --git a/plugins/home/tests/test_validation_budget.py b/plugins/home/tests/test_validation_budget.py new file mode 100644 index 00000000..ef222cc8 --- /dev/null +++ b/plugins/home/tests/test_validation_budget.py @@ -0,0 +1,142 @@ +"""Formal budget accounting through real evaluation Jobs and fake Docker boundaries.""" + +# pylint: disable=missing-function-docstring,missing-class-docstring,unused-argument,protected-access + +import asyncio + +import pytest +from reme_home.agent.optimizer import OptimizerAgentWrapper, validation_budget_tool +from reme_home.evaluation import component as evaluation +from reme_home.evaluation.artifacts import write_json +from reme_home.evaluation.budget import validation_budget_status +from reme_home.evaluation.inputs import EvaluationSpec +from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import + FakeFactory, + commit_candidate, + home_application, + preparation_inputs, + prepared_home, +) + + +def seed_history(home, keys): + write_json( + home / "sampling/state.json", + { + "schema_version": 2, + "round": len(keys), + "successful_round": 0, + "cases": {}, + "queries": {}, + "started_validations": {key: index for index, key in enumerate(keys, 1)}, + "processed_validations": [], + }, + ) + + +@pytest.mark.parametrize( + "used,phase,remaining", + [ + (0, "structural_exploration", 10), + (4, "structural_exploration", 6), + (5, "evidence_driven_refinement", 5), + (9, "evidence_driven_refinement", 1), + (10, "exhausted", 0), + (11, "exhausted", 0), + ], +) +def test_dev_meta_phase_boundaries(tmp_path, used, phase, remaining): + seed_history(tmp_path, ["init/first/1", *(f"candidate/hash/{index}" for index in range(used))]) + result = validation_budget_status(tmp_path, 10) + assert result["used_validations"] == used + assert result["remaining_validations"] == remaining + assert result["optimization_phase"] == phase + assert result["gated_validation_count"] == 5 + assert result["next_validation_guidance"] + + +@pytest.mark.parametrize("invalid", [True, False, 0, -1, 1.5, "10", None]) +def test_invalid_optimizer_budgets_rejected(invalid): + wrapper = OptimizerAgentWrapper(name="optimizer", max_validations=invalid) + with pytest.raises(ValueError, match="positive integer"): + wrapper.validation_limit() + if invalid is not None: + with pytest.raises(ValueError): + EvaluationSpec(max_validations=invalid) + + +def test_budgeted_evaluation_cannot_disable_started_history(): + with pytest.raises(ValueError, match="record_history"): + EvaluationSpec(max_validations=10, record_history=False) + assert EvaluationSpec(debug=True, max_validations=1, record_history=False).debug + assert OptimizerAgentWrapper(name="optimizer").validation_limit() == 10 + + +@pytest.mark.asyncio +async def test_budget_tool_reports_persisted_count_without_mutation(tmp_path): + seed_history(tmp_path, ["init/base/1", "candidate/hash/1"]) + before = (tmp_path / "sampling/state.json").read_bytes() + tool = validation_budget_tool(tmp_path, 3) + assert tool.name == "get_validation_budget" + result = await tool._func() + assert result["remaining_validations"] == 2 + assert result["optimization_phase"] == "evidence_driven_refinement" + assert (tmp_path / "sampling/state.json").read_bytes() == before + + +@pytest.mark.asyncio +async def test_exhaustion_blocks_before_pool_and_survives_cache_rebuild(app, prepared): + baseline = await app.run_job("home_evaluate") + assert baseline.success + assert validation_budget_status(prepared, 1)["used_validations"] == 0 + commit_candidate(prepared) + result = await app.run_job("home_evaluate", max_validations=1) + assert result.success, result.answer + assert result.answer["validation_budget"]["remaining_validations"] == 0 + count = len(FakeFactory.instances) + # A fresh status lookup and cache rebuild both retain the charge. + (prepared / "sampling/state.json").unlink() + assert validation_budget_status(prepared, 1)["used_validations"] == 1 + commit_candidate(prepared) + rejected = await app.run_job("home_evaluate", max_validations=1) + assert not rejected.success + assert rejected.answer["error_code"] == "VALIDATION_BUDGET_EXHAUSTED" + assert rejected.answer["result_dir"] is None + assert len(FakeFactory.instances) == count + debug = await app.run_job("home_evaluate", debug=True, max_validations=1) + assert debug.success, debug.answer + assert "validation_budget" not in debug.answer + assert validation_budget_status(prepared, 1)["used_validations"] == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("outcome", ["candidate_failure", "infra_failure", "cancelled", "prestart_failure"]) +async def test_charge_depends_on_first_sandbox_start_not_result(app, prepared, monkeypatch, outcome): + commit_candidate(prepared) + if outcome == "candidate_failure": + monkeypatch.setattr(FakeFactory, "fail_build", "1") + elif outcome == "infra_failure": + monkeypatch.setattr(FakeFactory, "bad_export", True) + elif outcome == "cancelled": + monkeypatch.setattr(FakeFactory, "block", asyncio.Event()) + else: + + class PrestartFailureFactory(FakeFactory): + async def create_cases(self, ids, **kwargs): + raise RuntimeError("sandbox initialization failed") + + monkeypatch.setattr(evaluation, "DockerReMeSandboxFactory", PrestartFailureFactory) + task = asyncio.create_task(app.run_job("home_evaluate", max_validations=1)) + if outcome == "cancelled": + while not FakeFactory.instances: + await asyncio.sleep(0) + await FakeFactory.instances[-1].entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + else: + await task + expected = int(outcome != "prestart_failure") + assert validation_budget_status(prepared, 1)["used_validations"] == expected + (prepared / "sampling/state.json").unlink(missing_ok=True) + assert validation_budget_status(prepared, 1)["used_validations"] == expected diff --git a/plugins/home/tests/test_workflows.py b/plugins/home/tests/test_workflows.py new file mode 100644 index 00000000..2c9dcd82 --- /dev/null +++ b/plugins/home/tests/test_workflows.py @@ -0,0 +1,496 @@ +"""Minimal Home agents exercise real Jobs, Git and AS tools with fake model/Docker boundaries.""" + +# pylint: disable=missing-function-docstring,missing-class-docstring,redefined-outer-name +# pylint: disable=unused-argument,protected-access + +import asyncio +import json +import sys +from importlib import import_module +from pathlib import Path +from types import SimpleNamespace + +import pytest +import yaml +from agentscope.formatter import OpenAIChatFormatter +from agentscope.message import HintBlock, TextBlock, ToolCallBlock +from agentscope.model import ChatResponse, ChatUsage +from agentscope.tool import LocalBackend +from jsonschema import Draft202012Validator +from reme_home.agent import optimizer as optimizer_module, diagnostic as diagnostic_module, workflow_common, case_study +from reme_home.evaluation.artifacts import read_json, write_json +from reme_home.preparation.code import git +from reme_home.workspace import _initialization_lock + +from test_bounded_agent import RecordingModel +from test_case_study import RecordingSandbox +from test_evaluation import ( # noqa: F401 # pylint: disable=unused-import + FakeFactory, + home_application, + preparation_inputs, + prepared_home, +) + + +class ScriptedModel(RecordingModel): + def __init__(self, calls=(), final="Supported diagnostic report."): + super().__init__() + self.script = list(calls) + self.report = final + + async def __call__(self, **kwargs): + index = len(self.calls) + self.calls.append(kwargs) + if index < len(self.script): + name, arguments = self.script[index] + arguments = arguments() if callable(arguments) else arguments + content = [ToolCallBlock(id=f"call-{index}", name=name, input=json.dumps(arguments))] + else: + final = self.report() if callable(self.report) else self.report + content = [TextBlock(text=final)] + return ChatResponse(content=content, is_last=True, usage=ChatUsage(input_tokens=2, output_tokens=3, time=0)) + + +@pytest.fixture +def sandbox(monkeypatch): + policies = [] + + def create(**kwargs): + # Mock only subprocess isolation. Writes are restricted by the real + # sandbox in the dedicated OS tests below, not by this local fake. + policies.append(kwargs) + local = LocalBackend() + return SimpleNamespace(backend=local, exec_shell=local.exec_shell) + + monkeypatch.setattr(workflow_common, "create_case_study_sandbox", create) + monkeypatch.setattr(case_study, "create_case_study_sandbox", RecordingSandbox) + return policies + + +def set_model(app, name, model): + wrapper = app.context.components["agent_wrapper"][name] + wrapper.as_llm = SimpleNamespace(model=model) + return wrapper + + +async def baseline(app): + result = await app.run_job("home_evaluate") + assert result.success, result.answer + return result.answer["result_dir"] + + +def final_report(home, branch="init", path=None): + repo = home / "code/repo" + if path is None: + path = sorted((home / "evaluations" / branch).glob("*/*/summary.json"))[-1].parent + return json.dumps( + { + "branch_name": branch, + "commit_sha": git(repo, "rev-parse", "HEAD"), + "evaluation_path": str(path), + "report": "Selected using matching formal evidence; sampled coverage limits confidence.", + }, + ) + + +@pytest.mark.asyncio +async def test_real_optimization_diagnoses_edits_debugs_and_evaluates(app, prepared, sandbox, monkeypatch): + home = prepared + now, timers = [0.0], [] + timer_class = optimizer_module.ValidationCycleTimer + + def create_timer(budget): + timer = timer_class(budget, clock=lambda: now[0]) + timers.append(timer) + return timer + + class TimedModel(ScriptedModel): + async def __call__(self, **kwargs): + formatted = await OpenAIChatFormatter().format(kwargs["messages"]) + pending = set() + for message in formatted: + if pending: + assert message["role"] == "tool", formatted + pending.remove(message["tool_call_id"]) + else: + assert message["role"] != "tool", formatted + pending.update(call["id"] for call in message.get("tool_calls", [])) + assert not pending + response = await super().__call__(**kwargs) + now[0] += 60 * 60 + return response + + monkeypatch.setattr(optimizer_module, "ValidationCycleTimer", create_timer) + target = home / "code/repo/plugins/beam/src/reme_beam/__init__.py" + baseline_path = await baseline(app) + diagnostic = ScriptedModel( + [("home_case_study", {"evaluation_path": baseline_path, "case_ids": ["1"], "focus": "Trace wrong answers"})], + ) + set_model(app, "home_diagnostic", diagnostic) + set_model(app, "home_case_study", ScriptedModel()) + optimizer = TimedModel( + [ + ("get_validation_budget", {}), + ("home_diagnose", {"evaluation_paths": [baseline_path]}), + ("home_create_candidate_branch", {"branch_name": "mechanism"}), + ("Read", {"file_path": str(target)}), + ("Edit", {"file_path": str(target), "old_string": "value = 1", "new_string": "value = 2"}), + ("home_commit_candidate", {"message": "Change mechanism"}), + ("home_evaluate", {"debug": True}), + ("home_evaluate", {"debug": False}), + ], + final=lambda: final_report(home, "mechanism"), + ) + set_model(app, "home_optimizer", optimizer) + response = await app.run_job("home_optimize", instruction="Improve general retrieval") + assert response.success, response.answer + assert target.read_text() == "value = 2\n" + assert response.answer["branch_name"] == "mechanism" + assert response.answer["status"] == "completed" + transitions = [ + block.hint + for message in optimizer.calls[-1]["messages"] + for block in message.content + if isinstance(block, HintBlock) and block.source == "meta_reme_validation_cycle_transition" + ] + assert len(transitions) == 1 + assert "Formal validation 1 completed successfully" in transitions[0] + assert "Remaining formal validations: 9. Start cycle 2." in transitions[0] + assert len(timers) == 1 + assert timers[0]._started_at == now[0] - 60 * 60 # Reset before the final reasoning call. + assert any( + isinstance(block, HintBlock) and block.source == "meta_reme_validation_timer" + for message in optimizer.calls[-1]["messages"] + for block in message.content + ) + assert response.metadata["usage"]["input_tokens"] == 2 * len(optimizer.calls) + assert response.metadata["usage"]["output_tokens"] == 3 * len(optimizer.calls) + assert "sampled coverage" in response.answer["report"] + assert read_json(Path(response.metadata["optimization_report_path"]))["status"] == "completed" + assert len(diagnostic.calls) == 2 + source = next((home / "logs/diagnostic-trajectories").glob("*.json")) + trace = read_json(source) + assert trace["session_id"] and any("Improve general retrieval" in str(msg) for msg in trace["messages"]) + assert "permission_context" not in trace + schemas = {item["function"]["name"]: item["function"]["parameters"] for item in optimizer.calls[0]["tools"]} + assert set(schemas["home_evaluate"]["properties"]) == {"debug"} + assert "caller_trajectory" not in schemas["home_diagnose"]["properties"] + assert "get_validation_budget" in schemas + assert not any("equivalence" in name for name in schemas) + for manifest_path in (home / "evaluations").glob("*/*/*/manifest.json"): + manifest = read_json(manifest_path) + assert manifest["execution_mode"] == "full_validation" + assert any(read_json(path)["mode"] == "debug" for path in (home / "evaluations").glob("*/*/*/manifest.json")) + diagnostic_schemas = {item["function"]["name"] for item in diagnostic.calls[0]["tools"]} + assert "home_case_study" in diagnostic_schemas + assert not set(optimizer_module.MUTATION_JOBS) & diagnostic_schemas + assert "Edit" not in diagnostic_schemas and "home_evaluate" not in diagnostic_schemas + assert "Write" in diagnostic_schemas # Private scratch only. + assert any(str(target.parent.parent) in policy["writable_paths"] for policy in sandbox) + assert all(str(home / "code/repo") not in policy["writable_paths"] for policy in sandbox) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("benchmark", ["beam", "longmemeval"]) +async def test_benchmark_optimizer_applies_budgets_to_real_workflow( + app, + prepared, + inputs, + sandbox, + monkeypatch, + tmp_path, + benchmark, +): + baseline_path = await baseline(app) + await app.close() + module = import_module(f"benchmark.{benchmark}.run_home") + config = tmp_path / "home.yaml" + config.write_text( + yaml.safe_dump( + { + "workspace_dir": str(prepared.parent), + "preparation": inputs[0], + "max_validations": 3, + "validation_cycle_time_budget_minutes": 30, + }, + ), + ) + application_class = type(app) + agents = [] + model = ScriptedModel([("get_validation_budget", {})], final=lambda: final_report(prepared, path=baseline_path)) + + def factory(**kwargs): + application = application_class(**kwargs, log_to_console=False) + original_start = application.start + + async def start(): + await original_start() + wrapper = set_model(application, "home_optimizer", model) + original_create = wrapper._create_agent + + def create(**options): + agent = original_create(**options) + agents.append(agent) + return agent + + monkeypatch.setattr(wrapper, "_create_agent", create) + + monkeypatch.setattr(application, "start", start) + return application + + monkeypatch.setattr(module, "Application", factory) + result = await module.optimize(config, instruction="Keep the best existing candidate") + assert result["status"] == "completed" + assert result["branch_name"] == "init" + assert len(agents) == 1 + assert agents[0].validation_cycle_timer.time_budget_minutes == 30 + task = "\n".join(msg.get_text_content() or "" for msg in model.calls[0]["messages"]) + assert '"max_validations": 3' in task + assert '"remaining_validations": 3' in task + assert len(FakeFactory.instances) == 1 # Only the earlier baseline ran. + + +@pytest.mark.asyncio +async def test_optimizer_creates_baseline_once_and_verifies_baseline_fallback(app, prepared, sandbox): + model = ScriptedModel(final=lambda: final_report(prepared)) + set_model(app, "home_optimizer", model) + response = await app.run_job("home_optimize") + assert response.success, response.answer + assert len(FakeFactory.instances) == 1 + assert response.answer["branch_name"] == "init" + run = response.answer["baseline_path"] + manifest = read_json(Path(run) / "manifest.json") + assert set(manifest["case_ids"]) == {"1", "2"} and not manifest["options"]["sampling"] + assert manifest["options"]["record_history"] + set_model(app, "home_optimizer", ScriptedModel(final=lambda: final_report(prepared))) + second = await app.run_job("home_optimize") + assert second.success, second.answer + assert len(FakeFactory.instances) == 1 + assert second.answer["session_id"] != response.answer["session_id"] + + +@pytest.mark.asyncio +async def test_standalone_diagnosis_fresh_state_and_persisted_provenance(app, prepared, sandbox): + path = await baseline(app) + model = ScriptedModel() + set_model(app, "home_diagnostic", model) + first = await app.run_job("home_diagnose", evaluation_paths=[path]) + second = await app.run_job("home_diagnose", evaluation_paths=[path], instruction="Compare failure clusters") + assert first.success and second.success + assert first.answer["session_id"] != second.answer["session_id"] + assert first.answer["trajectory_path"] != second.answer["trajectory_path"] + assert first.answer["evaluation_paths"] == [path] + record = read_json(Path(first.answer["diagnostic_report_path"])) + assert record["diagnosis"] == "Supported diagnostic report." + assert read_json(Path(first.answer["trajectory_path"])) == {"messages": [], "summary": None} + assert all( + len(policy["writable_paths"]) == 1 and "/tmp/diagnostic-" in policy["writable_paths"][0] for policy in sandbox + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("change", ["infra", "partial", "debug", "replay", "wrong_commit"]) +async def test_bad_baseline_rejected_before_optimizer(app, prepared, sandbox, change): + path = Path(await baseline(app)) + manifest = read_json(path / "manifest.json") + summary = read_json(path / "summary.json") + if change == "infra": + summary["case_status_counts"]["infra_error"] = 1 + elif change == "partial": + manifest["case_ids"] = ["1"] + manifest["query_ids"] = {"1": manifest["query_ids"]["1"]} + elif change == "debug": + manifest["mode"] = "debug" + elif change == "replay": + manifest["execution_mode"] = "query_only_replay" + else: + manifest["commit_sha"] = "0" * 40 + write_json(path / "manifest.json", manifest) + write_json(path / "summary.json", summary) + # Missing valid baseline must not silently switch away from a candidate. + response = await app.run_job("home_create_candidate_branch", branch_name="existing") + assert response.success + model = ScriptedModel() + set_model(app, "home_optimizer", model) + response = await app.run_job("home_optimize") + assert not response.success and "baseline is missing" in str(response.answer) + assert not model.calls and not sandbox + assert git(prepared / "code/repo", "branch", "--show-current") == "existing" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("change", ["dirty", "branch", "commit", "path", "infra", "debug", "replay", "invalid_json"]) +async def test_final_claim_rejected_without_discarding_changes(app, prepared, sandbox, change): + path = Path(await baseline(app)) + target = prepared / "code/repo/plugins/beam/src/reme_beam/__init__.py" + + def finish(): + claim = json.loads(final_report(prepared, path=path)) + if change == "dirty": + target.write_text("preserve unfinished changes\n") + elif change == "branch": + claim["branch_name"] = "invented" + elif change == "commit": + claim["commit_sha"] = "0" * 40 + elif change == "path": + claim["evaluation_path"] = str(prepared / "tmp") + elif change == "infra": + summary = read_json(path / "summary.json") + summary["query_status_counts"]["infra_error"] = 1 + write_json(path / "summary.json", summary) + elif change in {"debug", "replay"}: + manifest = read_json(path / "manifest.json") + manifest["mode" if change == "debug" else "execution_mode"] = ( + "debug" if change == "debug" else "query_only_replay" + ) + write_json(path / "manifest.json", manifest) + else: + return "not a structured final claim" + return json.dumps(claim) + + set_model(app, "home_optimizer", ScriptedModel(final=finish)) + response = await app.run_job("home_optimize") + assert not response.success + record = read_json(Path(response.metadata["optimization_report_path"])) + assert record["status"] == "failed" and record["error"] + if change == "dirty": + assert target.read_text() == "preserve unfinished changes\n" + + +@pytest.mark.asyncio +async def test_infrastructure_baseline_failure_never_calls_optimizer(app, prepared, sandbox, monkeypatch): + async def broken(*args, **kwargs): + raise RuntimeError("Docker unavailable") + + monkeypatch.setattr(FakeFactory, "create_cases", broken) + model = ScriptedModel() + set_model(app, "home_optimizer", model) + response = await app.run_job("home_optimize") + assert not response.success and not model.calls + assert "baseline failed" in str(response.answer) + + +@pytest.mark.asyncio +async def test_invalid_scope_schema_and_injected_trajectory(app, prepared, sandbox): + path = await baseline(app) + wrapper = set_model(app, "home_optimizer", ScriptedModel()) + tool = diagnostic_module.diagnostic_tool(wrapper) + with pytest.raises(ValueError, match="host-owned"): + await tool._func(evaluation_paths=[path], caller_trajectory={"spoof": True}) + for values in ([str(prepared / "tmp")], [path, path], [], ["../escape"]): + result = await app.run_job("home_diagnose", evaluation_paths=values) + assert not result.success + result = await app.run_job("home_diagnose", evaluation_paths=[path], _allowed_paths=[path]) + assert not result.success and not sandbox + for name in ("home_optimize", "home_diagnose"): + Draft202012Validator.check_schema(app.context.jobs[name].parameters) + assert "caller_trajectory" not in app.context.jobs["home_diagnose"].parameters["properties"] + with pytest.raises(ValueError, match="execution settings"): + optimizer_module.evaluation_options({"reuse_construction_from": path}) + + +@pytest.mark.asyncio +async def test_single_optimizer_lock_and_cancellation_cleanup(app, prepared, sandbox, monkeypatch): + await baseline(app) + entered, release = asyncio.Event(), asyncio.Event() + + async def blocked(*args, **kwargs): + entered.set() + await release.wait() + + monkeypatch.setattr(optimizer_module, "workflow_reply", blocked) + running = asyncio.create_task(app.run_job("home_optimize")) + await asyncio.wait_for(entered.wait(), 5) + second = await app.run_job("home_optimize") + assert not second.success and "already in progress" in str(second.answer) + running.cancel() + with pytest.raises(asyncio.CancelledError): + await running + reports = list((prepared / "logs/optimization-reports").glob("*.json")) + assert len(reports) == 1 and read_json(reports[0])["status"] == "cancelled" + with _initialization_lock(prepared / "metadata/optimization.lock"): + pass + + +@pytest.mark.asyncio +async def test_metadata_link_rejected_before_opening_any_lock(app, prepared, tmp_path): + outside = tmp_path / "outside" + outside.mkdir() + state = prepared / "metadata" + saved = prepared / "saved-metadata" + state.rename(saved) + state.symlink_to(outside, target_is_directory=True) + try: + result = await app.run_job("home_optimize") + assert not result.success and "real directory" in str(result.answer) + with pytest.raises(ValueError, match="real directory"): + optimizer_module.baseline_state(prepared) + with pytest.raises(ValueError, match="real directory"): + optimizer_module.verify_final(prepared, None) + assert not list(outside.iterdir()) + finally: + state.unlink() + saved.rename(state) + + +@pytest.mark.asyncio +async def test_workflow_sandbox_enforces_read_write_and_git_boundaries(tmp_path): + home = tmp_path / "home" + repo = home / "repo" + candidate = repo / "src" + scratch = home / "scratch" + evidence = home / "evidence" + for directory in (candidate, repo / ".git", scratch, evidence): + directory.mkdir(parents=True, exist_ok=True) + target = candidate / "module.py" + target.write_text("old\n") + secret = tmp_path / "outside.txt" + secret.write_text("outside") + (evidence / "result.json").write_text("evidence") + for role in ("optimizer", "diagnostic"): + scope = { + "scratch": str(scratch), + "read_only_paths": [str(repo), str(evidence)], + "writable_paths": [str(scratch), *([str(candidate)] if role == "optimizer" else [])], + } + backend = workflow_common.WorkflowSandbox(scope, memory_limit_mib=0) + assert (await backend.read_file(str(target))) in (b"old\n", b"new\n") + assert (await backend.stat(str(target))).size_bytes > 0 + assert "module.py" in await backend.list_dir(str(candidate)) + await backend.write_file(str(scratch / "analysis.py"), b"print('analysis')\n") + assert (await backend.exec_shell([sys.executable, "-I", "-B", str(scratch / "analysis.py")])).ok() + for path in (repo / ".git/config", evidence / "result.json", secret): + with pytest.raises(OSError): + await backend.write_file(str(path), b"forbidden") + with pytest.raises(OSError): + await backend.read_file(str(secret)) + if role == "optimizer": + await backend.write_file(str(target), b"new\n") + # Replaced roots after managed Git operations must not retain stale inodes. + replacement = candidate / "replacement" + replacement.write_text("new\n") + replacement.replace(target) + assert await backend.read_file(str(target)) == b"new\n" + else: + with pytest.raises(OSError): + await backend.write_file(str(target), b"forbidden") + assert secret.read_text() == "outside" and (evidence / "result.json").read_text() == "evidence" + # A single-file candidate root must survive a managed checkout replacing + # its inode without accidentally granting writes to its containing directory. + file_scope = { + "scratch": str(scratch), + "read_only_paths": [str(repo)], + "writable_paths": [str(scratch), str(target)], + } + backend = workflow_common.WorkflowSandbox(file_scope, memory_limit_mib=0) + await backend.write_file(str(target), b"file root\n") + replacement = candidate / "replacement" + replacement.write_text("replaced root\n") + replacement.replace(target) + assert await backend.read_file(str(target)) == b"replaced root\n" + with pytest.raises(OSError): + await backend.write_file(str(candidate / "sibling.py"), b"forbidden") + target.unlink() + target.symlink_to(secret) + with pytest.raises(ValueError, match="symlinks"): + await backend.read_file(str(target))